commit 75e5ab57e8bdeca240f8a0e376dac98c0aa6672a Author: Eloi Alonso Date: Fri Aug 26 12:58:58 2022 +0200 Code release. diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..78248ae --- /dev/null +++ b/.gitignore @@ -0,0 +1,136 @@ +# Custom +wandb +.vscode +outputs +results/figures +slurm* +experiments + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..f288702 --- /dev/null +++ b/LICENSE @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. diff --git a/README.md b/README.md new file mode 100644 index 0000000..9c53fd2 --- /dev/null +++ b/README.md @@ -0,0 +1,113 @@ +# Transformers are Sample Efficient World Models (IRIS) + +[Transformers are Sample Efficient World Models](https://arXiv-url.com)
+[Vincent Micheli](https://vmicheli.github.io)\*, [Eloi Alonso](https://eloialonso.github.io)\*, [François Fleuret](https://fleuret.org/francois/)
+\* Denotes equal contribution + + +
+ IRIS agent after 100k environment steps, i.e. two hours of real-time experience + IRIS playing on Asterix, Boxing, Breakout, Demon Attack, Freeway, Gopher, Kung Fu Master, Pong +
+ +**tl;dr** + +- IRIS is a data-efficient agent trained over millions of imagined trajectories in a world model. +- The world model is composed of a discrete autoencoder and an autoregressive Transformer. +- Our approach casts dynamics learning as a sequence modeling problem, where the autoencoder builds a language of image tokens and the Transformer composes that language over time. + + + +## BibTeX + +If you find this code or paper useful, please use the following reference: + +``` +Coming soon +``` + +## Setup + +- Install [PyTorch](https://pytorch.org/get-started/locally/) (torch and torchvision). Code developed with torch==1.11.0 and torchvision==0.12.0. +- Install [other dependencies](requirements.txt): `pip install -r requirements.txt` +- Warning: Atari ROMs will be downloaded with the dependencies, which means that you acknowledge that you have the license to use them. + +## Launch a training run + +```bash +python src/main.py env.train.id=BreakoutNoFrameskip-v4 common.device=cuda:0 wandb.mode=online +``` + +By default, the logs are synced to [weights & biases](https://wandb.ai), set `wandb.mode=disabled` to turn it off. + +## Configuration + +- All configuration files are located in `config/`, the main configuration file is `config/trainer.yaml`. +- The simplest way to customize the configuration is to edit these files directly. +- Please refer to [Hydra](https://github.com/facebookresearch/hydra) for more details regarding configuration management. + +## Run folder + +Each new run is located at `outputs/YYYY-MM-DD/hh-mm-ss/`. This folder is structured as: + +```txt +outputs/YYYY-MM-DD/hh-mm-ss/ +│ +└─── checkpoints +│ │ last.pt +| | optimizer.pt +| | ... +│ │ +│ └─── dataset +│ │ 0.pt +│ │ 1.pt +│ │ ... +│ +└─── config +│ | trainer.yaml +| +└─── media +│ │ +│ └─── episodes +│ | │ ... +│ │ +│ └─── reconstructions +│ | │ ... +│ +└─── scripts +| | eval.py +│ │ play.sh +│ │ resume.sh +| | ... +| +└─── src +| | ... +| +└─── wandb + | ... +``` + +- `checkpoints`: contains the last checkpoint of the model, its optimizer and the dataset. +- `media`: + - `episodes`: contains train / test / imagination episodes for visualization purposes. + - `reconstructions`: contains original frames alongside their reconstructions with the autoencoder. +- `scripts`: **from the run folder**, you can use the following three scripts. + - `eval.py`: Launch `python ./scripts/eval.py` to evaluate the run. + - `resume.sh`: Launch `./scripts/resume.sh` to resume a training that crashed. + - `play.sh`: Tool to visualize some interesting aspects of the run. + - Launch `./scripts/play.sh -a` to watch the agent play live in the environment. The left panel displays the original environment, and the right panel shows what the agent actually sees through its discrete autoencoder. + - Launch `./scripts/play.sh -w` to unroll live trajectories with your keyboard inputs (i.e. to play in the world model). Note that for faster interaction, the memory of the Transformer is flushed every 20 frames. + - Launch `./scripts/play.sh` to visualize the episodes contained in `media/episodes`. + +## Results notebook + +The folder `results/data/` contains raw scores (for each game, and for each training run) for IRIS and the baselines. + +Use the notebook `results/results_iris.ipynb` to reproduce the figures from the paper. + +## Credits + +- [https://github.com/pytorch/pytorch](https://github.com/pytorch/pytorch) +- [https://github.com/CompVis/taming-transformers](https://github.com/CompVis/taming-transformers) +- [https://github.com/karpathy/minGPT](https://github.com/karpathy/minGPT) +- [https://github.com/google-research/rliable](https://github.com/google-research/rliable) diff --git a/assets/iris.gif b/assets/iris.gif new file mode 100644 index 0000000..ab2e928 Binary files /dev/null and b/assets/iris.gif differ diff --git a/config/actor_critic/default.yaml b/config/actor_critic/default.yaml new file mode 100644 index 0000000..e4e2eee --- /dev/null +++ b/config/actor_critic/default.yaml @@ -0,0 +1 @@ +use_original_obs: False diff --git a/config/datasets/default.yaml b/config/datasets/default.yaml new file mode 100644 index 0000000..c959c98 --- /dev/null +++ b/config/datasets/default.yaml @@ -0,0 +1,8 @@ +train: + _target_: dataset.EpisodesDatasetRamMonitoring + max_ram_usage: 30G + name: train_dataset +test: + _target_: dataset.EpisodesDataset + max_num_episodes: null + name: test_dataset diff --git a/config/env/default.yaml b/config/env/default.yaml new file mode 100644 index 0000000..f6b3b89 --- /dev/null +++ b/config/env/default.yaml @@ -0,0 +1,21 @@ +train: + _target_: envs.make_atari + id: null + size: 64 + max_episode_steps: 20000 + noop_max: 30 + frame_skip: 4 + done_on_life_loss: True + clip_reward: False + +test: + _target_: ${..train._target_} + id: ${..train.id} + size: ${..train.size} + max_episode_steps: 108000 + noop_max: 1 + frame_skip: ${..train.frame_skip} + done_on_life_loss: False + clip_reward: False + +keymap: atari/${.train.id} \ No newline at end of file diff --git a/config/tokenizer/default.yaml b/config/tokenizer/default.yaml new file mode 100644 index 0000000..54a4dd7 --- /dev/null +++ b/config/tokenizer/default.yaml @@ -0,0 +1,20 @@ +_target_: models.tokenizer.Tokenizer + +vocab_size: 512 +embed_dim: 512 +encoder: + _target_: models.tokenizer.Encoder + config: + _target_: models.tokenizer.EncoderDecoderConfig + resolution: 64 + in_channels: 3 + z_channels: 512 + ch: 64 + ch_mult: [1, 1, 1, 1, 1] + num_res_blocks: 2 + attn_resolutions: [8, 16] + out_ch: 3 + dropout: 0.0 +decoder: + _target_: models.tokenizer.Decoder + config: ${..encoder.config} \ No newline at end of file diff --git a/config/trainer.yaml b/config/trainer.yaml new file mode 100644 index 0000000..3890af1 --- /dev/null +++ b/config/trainer.yaml @@ -0,0 +1,95 @@ +defaults: + - _self_ + - tokenizer: default + - world_model: default + - actor_critic: default + - env: default + - datasets: default + +wandb: + mode: online + project: iris + entity: null + name: null + group: null + tags: null + notes: null + +initialization: + path_to_checkpoint: null + load_tokenizer: False + load_world_model: False + load_actor_critic: False + +common: + epochs: 600 + device: cuda:0 + do_checkpoint: True + seed: 0 + sequence_length: ${world_model.max_blocks} + resume: False # set by resume.sh script only. + +collection: + train: + num_envs: 1 + stop_after_epochs: 500 + num_episodes_to_save: 10 + config: + epsilon: 0.01 + should_sample: True + temperature: 1.0 + num_steps: 200 + burn_in: ${training.actor_critic.burn_in} + test: + num_envs: 8 + num_episodes_to_save: ${collection.train.num_episodes_to_save} + config: + epsilon: 0.0 + should_sample: True + temperature: 0.5 + num_episodes: 16 + burn_in: ${training.actor_critic.burn_in} + +training: + should: True + learning_rate: 0.0001 + sampling_weights: [0.125, 0.125, 0.25, 0.5] + tokenizer: + batch_num_samples: 256 + grad_acc_steps: 1 + max_grad_norm: 10.0 + start_after_epochs: 5 + steps_per_epoch: 200 + world_model: + batch_num_samples: 64 + grad_acc_steps: 1 + max_grad_norm: 10.0 + weight_decay: 0.01 + start_after_epochs: 25 + steps_per_epoch: 200 + actor_critic: + batch_num_samples: 64 + grad_acc_steps: 1 + max_grad_norm: 10.0 + start_after_epochs: 50 + steps_per_epoch: 200 + imagine_horizon: ${common.sequence_length} + burn_in: 20 + gamma: 0.995 + lambda_: 0.95 + entropy_weight: 0.001 + +evaluation: + should: True + every: 5 + tokenizer: + batch_num_samples: ${training.tokenizer.batch_num_samples} + start_after_epochs: ${training.tokenizer.start_after_epochs} + save_reconstructions: True + world_model: + batch_num_samples: ${training.world_model.batch_num_samples} + start_after_epochs: ${training.world_model.start_after_epochs} + actor_critic: + num_episodes_to_save: ${training.actor_critic.batch_num_samples} + horizon: ${training.actor_critic.imagine_horizon} + start_after_epochs: ${training.actor_critic.start_after_epochs} diff --git a/config/world_model/default.yaml b/config/world_model/default.yaml new file mode 100644 index 0000000..bfcbb31 --- /dev/null +++ b/config/world_model/default.yaml @@ -0,0 +1,10 @@ +_target_: models.TransformerConfig +tokens_per_block: 17 +max_blocks: 20 +attention: 'causal' +num_layers: 10 +num_heads: 4 +embed_dim: 256 +embed_pdrop: 0.1 +resid_pdrop: 0.1 +attn_pdrop: 0.1 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..7438200 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,10 @@ +ale-py==0.7.4 +einops==0.3.2 +gym[accept-rom-license]==0.21.0 +hydra-core==1.1.1 +opencv-python +protobuf==3.20.* +psutil==5.8.0 +pygame==2.1.2 +tqdm==4.62.3 +wandb==0.12.7 diff --git a/results/data/CURL_10_evals.json b/results/data/CURL_10_evals.json new file mode 100644 index 0000000..e18b580 --- /dev/null +++ b/results/data/CURL_10_evals.json @@ -0,0 +1 @@ +[{"game":"Alien","seed":1.0,"HNS":0.0538268671,"Max HNS":0.056247192},{"game":"Alien","seed":2.0,"HNS":0.0534210641,"Max HNS":0.0581602632},{"game":"Alien","seed":3.0,"HNS":0.0699430427,"Max HNS":0.0699430427},{"game":"Alien","seed":4.0,"HNS":0.0796098494,"Max HNS":0.0796098494},{"game":"Alien","seed":5.0,"HNS":0.0723488746,"Max HNS":0.1134074407},{"game":"Alien","seed":6.0,"HNS":0.0852476123,"Max HNS":0.0870592327},{"game":"Alien","seed":7.0,"HNS":0.0810011739,"Max HNS":0.0810011739},{"game":"Alien","seed":8.0,"HNS":0.1341178858,"Max HNS":0.1341178858},{"game":"Alien","seed":9.0,"HNS":0.0477833012,"Max HNS":0.0754358759},{"game":"Alien","seed":10.0,"HNS":0.0991608574,"Max HNS":0.0991608574},{"game":"Alien","seed":11.0,"HNS":0.0716242264,"Max HNS":0.0716242264},{"game":"Alien","seed":12.0,"HNS":0.0818562588,"Max HNS":0.1094218757},{"game":"Alien","seed":13.0,"HNS":0.0420295946,"Max HNS":0.0775518486},{"game":"Alien","seed":14.0,"HNS":0.0607110248,"Max HNS":0.0927404745},{"game":"Alien","seed":15.0,"HNS":0.0423194539,"Max HNS":0.0457832722},{"game":"Alien","seed":16.0,"HNS":0.0452905115,"Max HNS":0.0599718837},{"game":"Alien","seed":17.0,"HNS":0.0372469166,"Max HNS":0.0743778895},{"game":"Alien","seed":18.0,"HNS":0.0573051783,"Max HNS":0.0573051783},{"game":"Alien","seed":19.0,"HNS":0.058232728,"Max HNS":0.1155958782},{"game":"Alien","seed":20.0,"HNS":0.0757402281,"Max HNS":0.0757402281},{"game":"Alien","seed":21.0,"HNS":0.0738851288,"Max HNS":0.1164219771},{"game":"Alien","seed":22.0,"HNS":0.0740880303,"Max HNS":0.0830736677},{"game":"Alien","seed":23.0,"HNS":0.0370440151,"Max HNS":0.0445078914},{"game":"Alien","seed":24.0,"HNS":0.0459137089,"Max HNS":0.0554790649},{"game":"Alien","seed":25.0,"HNS":0.1318424905,"Max HNS":0.1318424905},{"game":"Alien","seed":26.0,"HNS":0.0548413745,"Max HNS":0.0548413745},{"game":"Alien","seed":27.0,"HNS":0.0956100813,"Max HNS":0.1392339019},{"game":"Alien","seed":28.0,"HNS":0.0572616994,"Max HNS":0.0572616994},{"game":"Alien","seed":29.0,"HNS":0.096928941,"Max HNS":0.096928941},{"game":"Alien","seed":30.0,"HNS":0.0536239656,"Max HNS":0.0591747706},{"game":"Alien","seed":31.0,"HNS":0.0763779185,"Max HNS":0.0763779185},{"game":"Alien","seed":32.0,"HNS":0.0999144915,"Max HNS":0.0999144915},{"game":"Alien","seed":33.0,"HNS":0.0437107784,"Max HNS":0.0527688807},{"game":"Alien","seed":34.0,"HNS":0.1078131567,"Max HNS":0.1094073827},{"game":"Alien","seed":35.0,"HNS":0.0573196713,"Max HNS":0.059421151},{"game":"Alien","seed":36.0,"HNS":0.0937115031,"Max HNS":0.0937115031},{"game":"Alien","seed":37.0,"HNS":0.0459716807,"Max HNS":0.0702329019},{"game":"Alien","seed":38.0,"HNS":0.0832330903,"Max HNS":0.0976825751},{"game":"Alien","seed":39.0,"HNS":0.0495514428,"Max HNS":0.0929143901},{"game":"Alien","seed":40.0,"HNS":0.064232815,"Max HNS":0.0980738851},{"game":"Alien","seed":41.0,"HNS":0.0393483964,"Max HNS":0.045855737},{"game":"Alien","seed":42.0,"HNS":0.0446528211,"Max HNS":0.047087639},{"game":"Alien","seed":43.0,"HNS":0.0429426513,"Max HNS":0.0533485993},{"game":"Alien","seed":44.0,"HNS":0.068595197,"Max HNS":0.074986594},{"game":"Alien","seed":45.0,"HNS":0.0769141582,"Max HNS":0.0912766852},{"game":"Alien","seed":46.0,"HNS":0.1081609878,"Max HNS":0.1081609878},{"game":"Alien","seed":47.0,"HNS":0.0838417948,"Max HNS":0.1317120538},{"game":"Alien","seed":48.0,"HNS":0.0737257062,"Max HNS":0.0737257062},{"game":"Alien","seed":49.0,"HNS":0.1055522544,"Max HNS":0.1055522544},{"game":"Alien","seed":50.0,"HNS":0.0698270989,"Max HNS":0.0947549964},{"game":"Alien","seed":51.0,"HNS":0.0810881317,"Max HNS":0.097711561},{"game":"Alien","seed":52.0,"HNS":0.0596820244,"Max HNS":0.0596820244},{"game":"Alien","seed":53.0,"HNS":0.1002623226,"Max HNS":0.1002623226},{"game":"Alien","seed":54.0,"HNS":0.0690444789,"Max HNS":0.0690444789},{"game":"Alien","seed":55.0,"HNS":0.1000159423,"Max HNS":0.1157408078},{"game":"Alien","seed":56.0,"HNS":0.1145233989,"Max HNS":0.1145233989},{"game":"Alien","seed":57.0,"HNS":0.0505224713,"Max HNS":0.0516239366},{"game":"Alien","seed":58.0,"HNS":0.0806678358,"Max HNS":0.0806678358},{"game":"Alien","seed":59.0,"HNS":0.1096247772,"Max HNS":0.1096247772},{"game":"Alien","seed":60.0,"HNS":0.0635951246,"Max HNS":0.1121465528},{"game":"Alien","seed":61.0,"HNS":0.0695372397,"Max HNS":0.0695372397},{"game":"Alien","seed":62.0,"HNS":0.0574790939,"Max HNS":0.0574790939},{"game":"Alien","seed":63.0,"HNS":0.0498268091,"Max HNS":0.0579428687},{"game":"Alien","seed":64.0,"HNS":0.0432035247,"Max HNS":0.1162335686},{"game":"Alien","seed":65.0,"HNS":0.0519862607,"Max HNS":0.0797402861},{"game":"Alien","seed":66.0,"HNS":0.0733199032,"Max HNS":0.0761170452},{"game":"Alien","seed":67.0,"HNS":0.0871896694,"Max HNS":0.0871896694},{"game":"Alien","seed":68.0,"HNS":0.0553776142,"Max HNS":0.0559428398},{"game":"Alien","seed":69.0,"HNS":0.0520152466,"Max HNS":0.0542761489},{"game":"Alien","seed":70.0,"HNS":0.0574646009,"Max HNS":0.0629139553},{"game":"Alien","seed":71.0,"HNS":0.1006971116,"Max HNS":0.1006971116},{"game":"Alien","seed":72.0,"HNS":0.0447107929,"Max HNS":0.0469716952},{"game":"Alien","seed":73.0,"HNS":0.0674502529,"Max HNS":0.0674502529},{"game":"Alien","seed":74.0,"HNS":0.0599718837,"Max HNS":0.0797113002},{"game":"Alien","seed":75.0,"HNS":0.1175669213,"Max HNS":0.1175669213},{"game":"Alien","seed":76.0,"HNS":0.0471021319,"Max HNS":0.086957782},{"game":"Alien","seed":77.0,"HNS":0.0588559254,"Max HNS":0.0707981275},{"game":"Alien","seed":78.0,"HNS":0.0408266786,"Max HNS":0.0788707083},{"game":"Alien","seed":79.0,"HNS":0.0854650067,"Max HNS":0.0854650067},{"game":"Alien","seed":80.0,"HNS":0.0741025232,"Max HNS":0.1084653401},{"game":"Alien","seed":81.0,"HNS":0.0779576516,"Max HNS":0.1669734344},{"game":"Alien","seed":82.0,"HNS":0.0992188293,"Max HNS":0.0992188293},{"game":"Alien","seed":83.0,"HNS":0.0822765547,"Max HNS":0.0822765547},{"game":"Alien","seed":84.0,"HNS":0.0761315381,"Max HNS":0.0798562298},{"game":"Alien","seed":85.0,"HNS":0.0414498761,"Max HNS":0.080957695},{"game":"Alien","seed":86.0,"HNS":0.0552761634,"Max HNS":0.0613777011},{"game":"Alien","seed":87.0,"HNS":0.0862766127,"Max HNS":0.0862766127},{"game":"Alien","seed":88.0,"HNS":0.1156393571,"Max HNS":0.1479151872},{"game":"Alien","seed":89.0,"HNS":0.0792620183,"Max HNS":0.0893925999},{"game":"Alien","seed":90.0,"HNS":0.048667372,"Max HNS":0.0494354991},{"game":"Alien","seed":91.0,"HNS":0.0560587835,"Max HNS":0.0623052508},{"game":"Alien","seed":92.0,"HNS":0.0538848389,"Max HNS":0.0754648618},{"game":"Alien","seed":93.0,"HNS":0.0612472645,"Max HNS":0.0688705633},{"game":"Alien","seed":94.0,"HNS":0.0469861882,"Max HNS":0.0645516602},{"game":"Alien","seed":95.0,"HNS":0.11408861,"Max HNS":0.147205032},{"game":"Alien","seed":96.0,"HNS":0.0576385165,"Max HNS":0.0576385165},{"game":"Alien","seed":97.0,"HNS":0.0707836345,"Max HNS":0.0899723184},{"game":"Alien","seed":98.0,"HNS":0.0620298845,"Max HNS":0.0707546486},{"game":"Alien","seed":99.0,"HNS":0.0500152176,"Max HNS":0.0856679082},{"game":"Alien","seed":100.0,"HNS":0.0401600023,"Max HNS":0.0528848244},{"game":"Amidar","seed":1.0,"HNS":0.0745929859,"Max HNS":0.0745929859},{"game":"Amidar","seed":2.0,"HNS":0.0549162631,"Max HNS":0.0549162631},{"game":"Amidar","seed":3.0,"HNS":0.0784151252,"Max HNS":0.0856567661},{"game":"Amidar","seed":4.0,"HNS":0.0418217891,"Max HNS":0.0716228045},{"game":"Amidar","seed":5.0,"HNS":0.1131411566,"Max HNS":0.1131411566},{"game":"Amidar","seed":6.0,"HNS":0.0540467993,"Max HNS":0.0540467993},{"game":"Amidar","seed":7.0,"HNS":0.027618603,"Max HNS":0.0544611075},{"game":"Amidar","seed":8.0,"HNS":0.0727606932,"Max HNS":0.0727606932},{"game":"Amidar","seed":9.0,"HNS":0.0777965805,"Max HNS":0.0778666044},{"game":"Amidar","seed":10.0,"HNS":0.0621345626,"Max HNS":0.0621345626},{"game":"Amidar","seed":11.0,"HNS":0.0430238665,"Max HNS":0.0572620645},{"game":"Amidar","seed":12.0,"HNS":0.0908560425,"Max HNS":0.1074983953},{"game":"Amidar","seed":13.0,"HNS":0.0451187489,"Max HNS":0.0913578806},{"game":"Amidar","seed":14.0,"HNS":0.0716403104,"Max HNS":0.0775748381},{"game":"Amidar","seed":15.0,"HNS":0.0695104161,"Max HNS":0.0854875416},{"game":"Amidar","seed":16.0,"HNS":0.0647721305,"Max HNS":0.0801365467},{"game":"Amidar","seed":17.0,"HNS":0.0584699772,"Max HNS":0.078911128},{"game":"Amidar","seed":18.0,"HNS":0.0634241699,"Max HNS":0.0634241699},{"game":"Amidar","seed":19.0,"HNS":0.096627181,"Max HNS":0.096627181},{"game":"Amidar","seed":20.0,"HNS":0.0368267491,"Max HNS":0.0405146758},{"game":"Amidar","seed":21.0,"HNS":0.0607340841,"Max HNS":0.063050709},{"game":"Amidar","seed":22.0,"HNS":0.0491976425,"Max HNS":0.0590593453},{"game":"Amidar","seed":23.0,"HNS":0.0703798798,"Max HNS":0.0722821964},{"game":"Amidar","seed":24.0,"HNS":0.0440042014,"Max HNS":0.0543502363},{"game":"Amidar","seed":25.0,"HNS":0.0696154519,"Max HNS":0.0767870689},{"game":"Amidar","seed":26.0,"HNS":0.0430647138,"Max HNS":0.0562700589},{"game":"Amidar","seed":27.0,"HNS":0.0663768454,"Max HNS":0.0663768454},{"game":"Amidar","seed":28.0,"HNS":0.051829375,"Max HNS":0.0744354321},{"game":"Amidar","seed":29.0,"HNS":0.0205870339,"Max HNS":0.0426504056},{"game":"Amidar","seed":30.0,"HNS":0.0658458307,"Max HNS":0.0658458307},{"game":"Amidar","seed":31.0,"HNS":0.0527805333,"Max HNS":0.0666219292},{"game":"Amidar","seed":32.0,"HNS":0.0468168291,"Max HNS":0.1047441209},{"game":"Amidar","seed":33.0,"HNS":0.0479838945,"Max HNS":0.0638443135},{"game":"Amidar","seed":34.0,"HNS":0.0517360098,"Max HNS":0.0517360098},{"game":"Amidar","seed":35.0,"HNS":0.0737468635,"Max HNS":0.0933652331},{"game":"Amidar","seed":36.0,"HNS":0.064013538,"Max HNS":0.064013538},{"game":"Amidar","seed":37.0,"HNS":0.0498745405,"Max HNS":0.0498745405},{"game":"Amidar","seed":38.0,"HNS":0.056503472,"Max HNS":0.0730116123},{"game":"Amidar","seed":39.0,"HNS":0.0596953959,"Max HNS":0.0920406139},{"game":"Amidar","seed":40.0,"HNS":0.0602089047,"Max HNS":0.0667152944},{"game":"Amidar","seed":41.0,"HNS":0.0660559024,"Max HNS":0.0773005777},{"game":"Amidar","seed":42.0,"HNS":0.0569177802,"Max HNS":0.0692886736},{"game":"Amidar","seed":43.0,"HNS":0.0733558966,"Max HNS":0.0733558966},{"game":"Amidar","seed":44.0,"HNS":0.0651397561,"Max HNS":0.0651397561},{"game":"Amidar","seed":45.0,"HNS":0.0420435315,"Max HNS":0.0454980452},{"game":"Amidar","seed":46.0,"HNS":0.0858493319,"Max HNS":0.0858493319},{"game":"Amidar","seed":47.0,"HNS":0.1108362024,"Max HNS":0.1180836786},{"game":"Amidar","seed":48.0,"HNS":0.035397094,"Max HNS":0.0522436833},{"game":"Amidar","seed":49.0,"HNS":0.0531656649,"Max HNS":0.0600921982},{"game":"Amidar","seed":50.0,"HNS":0.0666219292,"Max HNS":0.0668086596},{"game":"Amidar","seed":51.0,"HNS":0.0864795472,"Max HNS":0.0864795472},{"game":"Amidar","seed":52.0,"HNS":0.0572270526,"Max HNS":0.0698722063},{"game":"Amidar","seed":53.0,"HNS":0.0651981094,"Max HNS":0.0674563809},{"game":"Amidar","seed":54.0,"HNS":0.052202836,"Max HNS":0.066423528},{"game":"Amidar","seed":55.0,"HNS":0.0636692537,"Max HNS":0.0726556573},{"game":"Amidar","seed":56.0,"HNS":0.0634766879,"Max HNS":0.0791153644},{"game":"Amidar","seed":57.0,"HNS":0.103460349,"Max HNS":0.103460349},{"game":"Amidar","seed":58.0,"HNS":0.078572679,"Max HNS":0.078572679},{"game":"Amidar","seed":59.0,"HNS":0.0489058762,"Max HNS":0.0738402287},{"game":"Amidar","seed":60.0,"HNS":0.0913053627,"Max HNS":0.0913053627},{"game":"Amidar","seed":61.0,"HNS":0.0323685593,"Max HNS":0.0519927642},{"game":"Amidar","seed":62.0,"HNS":0.0621345626,"Max HNS":0.0621345626},{"game":"Amidar","seed":63.0,"HNS":0.0771313532,"Max HNS":0.0771313532},{"game":"Amidar","seed":64.0,"HNS":0.0770613293,"Max HNS":0.0770613293},{"game":"Amidar","seed":65.0,"HNS":0.0625838828,"Max HNS":0.0625838828},{"game":"Amidar","seed":66.0,"HNS":0.0432514442,"Max HNS":0.0459998833},{"game":"Amidar","seed":67.0,"HNS":0.0739802766,"Max HNS":0.0749372702},{"game":"Amidar","seed":68.0,"HNS":0.041961837,"Max HNS":0.0634591819},{"game":"Amidar","seed":69.0,"HNS":0.0959094357,"Max HNS":0.0959094357},{"game":"Amidar","seed":70.0,"HNS":0.0565734959,"Max HNS":0.0569236156},{"game":"Amidar","seed":71.0,"HNS":0.0653731692,"Max HNS":0.0771838712},{"game":"Amidar","seed":72.0,"HNS":0.0695921106,"Max HNS":0.0746863512},{"game":"Amidar","seed":73.0,"HNS":0.095646846,"Max HNS":0.095646846},{"game":"Amidar","seed":74.0,"HNS":0.030722997,"Max HNS":0.0349886211},{"game":"Amidar","seed":75.0,"HNS":0.0525821322,"Max HNS":0.0701172901},{"game":"Amidar","seed":76.0,"HNS":0.0616677365,"Max HNS":0.0616677365},{"game":"Amidar","seed":77.0,"HNS":0.0681974675,"Max HNS":0.0681974675},{"game":"Amidar","seed":78.0,"HNS":0.0438583183,"Max HNS":0.0613642995},{"game":"Amidar","seed":79.0,"HNS":0.0482464842,"Max HNS":0.0660559024},{"game":"Amidar","seed":80.0,"HNS":0.0591818871,"Max HNS":0.072953259},{"game":"Amidar","seed":81.0,"HNS":0.0870514092,"Max HNS":0.0870514092},{"game":"Amidar","seed":82.0,"HNS":0.0554239365,"Max HNS":0.0741319951},{"game":"Amidar","seed":83.0,"HNS":0.0645153761,"Max HNS":0.1163155745},{"game":"Amidar","seed":84.0,"HNS":0.0502596721,"Max HNS":0.0504288965},{"game":"Amidar","seed":85.0,"HNS":0.0781175235,"Max HNS":0.0865962537},{"game":"Amidar","seed":86.0,"HNS":0.0831008928,"Max HNS":0.0831008928},{"game":"Amidar","seed":87.0,"HNS":0.054612826,"Max HNS":0.0591176985},{"game":"Amidar","seed":88.0,"HNS":0.049956235,"Max HNS":0.0811285523},{"game":"Amidar","seed":89.0,"HNS":0.0417576005,"Max HNS":0.0815486958},{"game":"Amidar","seed":90.0,"HNS":0.0515551147,"Max HNS":0.0601913987},{"game":"Amidar","seed":91.0,"HNS":0.055593161,"Max HNS":0.0811518936},{"game":"Amidar","seed":92.0,"HNS":0.0854000117,"Max HNS":0.0854000117},{"game":"Amidar","seed":93.0,"HNS":0.0603139406,"Max HNS":0.0670537434},{"game":"Amidar","seed":94.0,"HNS":0.0896131178,"Max HNS":0.0896131178},{"game":"Amidar","seed":95.0,"HNS":0.036289899,"Max HNS":0.055324736},{"game":"Amidar","seed":96.0,"HNS":0.0700764428,"Max HNS":0.0804574896},{"game":"Amidar","seed":97.0,"HNS":0.0951800198,"Max HNS":0.0964929684},{"game":"Amidar","seed":98.0,"HNS":0.0749781175,"Max HNS":0.0749781175},{"game":"Amidar","seed":99.0,"HNS":0.0643578223,"Max HNS":0.0939837778},{"game":"Amidar","seed":100.0,"HNS":0.0642586217,"Max HNS":0.0732100134},{"game":"Assault","seed":1.0,"HNS":0.5362971517,"Max HNS":0.5362971517},{"game":"Assault","seed":2.0,"HNS":0.1034449577,"Max HNS":0.3241147036},{"game":"Assault","seed":3.0,"HNS":0.5433602771,"Max HNS":0.5433602771},{"game":"Assault","seed":4.0,"HNS":0.2885488838,"Max HNS":0.3398768283},{"game":"Assault","seed":5.0,"HNS":0.5158775982,"Max HNS":0.5158775982},{"game":"Assault","seed":6.0,"HNS":0.6377405697,"Max HNS":0.6377405697},{"game":"Assault","seed":7.0,"HNS":0.7991531948,"Max HNS":0.7991531948},{"game":"Assault","seed":8.0,"HNS":0.628791378,"Max HNS":0.628791378},{"game":"Assault","seed":9.0,"HNS":0.6500192456,"Max HNS":0.8038298691},{"game":"Assault","seed":10.0,"HNS":0.3798883757,"Max HNS":0.5186874519},{"game":"Assault","seed":11.0,"HNS":0.3826982294,"Max HNS":0.6559276366},{"game":"Assault","seed":12.0,"HNS":0.4946689761,"Max HNS":0.6732678984},{"game":"Assault","seed":13.0,"HNS":0.3851424172,"Max HNS":0.3851424172},{"game":"Assault","seed":14.0,"HNS":0.449595843,"Max HNS":0.5837374904},{"game":"Assault","seed":15.0,"HNS":0.6926866821,"Max HNS":0.7304657429},{"game":"Assault","seed":16.0,"HNS":0.373421863,"Max HNS":0.373421863},{"game":"Assault","seed":17.0,"HNS":0.7232101617,"Max HNS":0.7232101617},{"game":"Assault","seed":18.0,"HNS":0.4768860662,"Max HNS":0.4768860662},{"game":"Assault","seed":19.0,"HNS":0.2804657429,"Max HNS":0.3354311008},{"game":"Assault","seed":20.0,"HNS":0.601712856,"Max HNS":0.6496535797},{"game":"Assault","seed":21.0,"HNS":0.5443802925,"Max HNS":0.5750962279},{"game":"Assault","seed":22.0,"HNS":0.6674364896,"Max HNS":0.6674364896},{"game":"Assault","seed":23.0,"HNS":0.3026943803,"Max HNS":0.4348537336},{"game":"Assault","seed":24.0,"HNS":0.6203233256,"Max HNS":0.6203233256},{"game":"Assault","seed":25.0,"HNS":0.8431100847,"Max HNS":0.8431100847},{"game":"Assault","seed":26.0,"HNS":0.4362586605,"Max HNS":0.4362586605},{"game":"Assault","seed":27.0,"HNS":0.5263856813,"Max HNS":0.5263856813},{"game":"Assault","seed":28.0,"HNS":0.5367013087,"Max HNS":0.5367013087},{"game":"Assault","seed":29.0,"HNS":0.9544457275,"Max HNS":0.9848537336},{"game":"Assault","seed":30.0,"HNS":0.646208622,"Max HNS":0.646208622},{"game":"Assault","seed":31.0,"HNS":0.9124326405,"Max HNS":1.0588722094},{"game":"Assault","seed":32.0,"HNS":0.6581986143,"Max HNS":0.6581986143},{"game":"Assault","seed":33.0,"HNS":0.6251732102,"Max HNS":0.6251732102},{"game":"Assault","seed":34.0,"HNS":0.3748267898,"Max HNS":0.4170708237},{"game":"Assault","seed":35.0,"HNS":-0.427213241,"Max HNS":-0.4231716705},{"game":"Assault","seed":36.0,"HNS":0.7009430331,"Max HNS":0.7009430331},{"game":"Assault","seed":37.0,"HNS":0.6494611239,"Max HNS":0.6951116243},{"game":"Assault","seed":38.0,"HNS":0.8540223249,"Max HNS":0.8540223249},{"game":"Assault","seed":39.0,"HNS":0.8205735181,"Max HNS":0.8622209392},{"game":"Assault","seed":40.0,"HNS":0.6633949192,"Max HNS":0.6718822171},{"game":"Assault","seed":41.0,"HNS":0.6110662048,"Max HNS":0.6439761355},{"game":"Assault","seed":42.0,"HNS":0.4896073903,"Max HNS":0.5063895304},{"game":"Assault","seed":43.0,"HNS":0.244495766,"Max HNS":0.3271362587},{"game":"Assault","seed":44.0,"HNS":0.4453618168,"Max HNS":0.4453618168},{"game":"Assault","seed":45.0,"HNS":0.4203040801,"Max HNS":0.5556966898},{"game":"Assault","seed":46.0,"HNS":1.0716705158,"Max HNS":1.0716705158},{"game":"Assault","seed":47.0,"HNS":0.5643571978,"Max HNS":0.6407621247},{"game":"Assault","seed":48.0,"HNS":-0.4235758276,"Max HNS":-0.4235758276},{"game":"Assault","seed":49.0,"HNS":0.4558506543,"Max HNS":0.4698036952},{"game":"Assault","seed":50.0,"HNS":0.4914357198,"Max HNS":0.5888375674},{"game":"Assault","seed":51.0,"HNS":0.4847575058,"Max HNS":0.4847575058},{"game":"Assault","seed":52.0,"HNS":0.1838722094,"Max HNS":0.3346227868},{"game":"Assault","seed":53.0,"HNS":0.5819668976,"Max HNS":0.6454195535},{"game":"Assault","seed":54.0,"HNS":0.5641647421,"Max HNS":0.5641647421},{"game":"Assault","seed":55.0,"HNS":0.671073903,"Max HNS":0.7489992302},{"game":"Assault","seed":56.0,"HNS":0.4961508853,"Max HNS":0.5966705158},{"game":"Assault","seed":57.0,"HNS":0.5950923788,"Max HNS":0.5950923788},{"game":"Assault","seed":58.0,"HNS":0.6206889915,"Max HNS":0.6206889915},{"game":"Assault","seed":59.0,"HNS":0.6129907621,"Max HNS":0.7744611239},{"game":"Assault","seed":60.0,"HNS":0.2808698999,"Max HNS":0.3249230177},{"game":"Assault","seed":61.0,"HNS":0.4526366436,"Max HNS":0.4526366436},{"game":"Assault","seed":62.0,"HNS":0.8150500385,"Max HNS":0.8817936875},{"game":"Assault","seed":63.0,"HNS":0.3883756736,"Max HNS":0.431812933},{"game":"Assault","seed":64.0,"HNS":0.4799076212,"Max HNS":0.6344880677},{"game":"Assault","seed":65.0,"HNS":0.6717090069,"Max HNS":0.6717090069},{"game":"Assault","seed":66.0,"HNS":0.308756736,"Max HNS":0.3511932256},{"game":"Assault","seed":67.0,"HNS":0.3491531948,"Max HNS":0.3491531948},{"game":"Assault","seed":68.0,"HNS":0.5237105466,"Max HNS":0.5237105466},{"game":"Assault","seed":69.0,"HNS":0.1612394149,"Max HNS":0.3471516551},{"game":"Assault","seed":70.0,"HNS":0.4873941493,"Max HNS":0.4873941493},{"game":"Assault","seed":71.0,"HNS":0.457852194,"Max HNS":0.5923017706},{"game":"Assault","seed":72.0,"HNS":0.3358352579,"Max HNS":0.4556581986},{"game":"Assault","seed":73.0,"HNS":0.6059083911,"Max HNS":0.661778291},{"game":"Assault","seed":74.0,"HNS":0.7037721324,"Max HNS":0.7037721324},{"game":"Assault","seed":75.0,"HNS":0.3728060046,"Max HNS":0.3728060046},{"game":"Assault","seed":76.0,"HNS":0.4974980754,"Max HNS":0.4974980754},{"game":"Assault","seed":77.0,"HNS":0.5704195535,"Max HNS":0.5704195535},{"game":"Assault","seed":78.0,"HNS":0.7423210162,"Max HNS":0.8382409546},{"game":"Assault","seed":79.0,"HNS":0.8053502694,"Max HNS":0.9762702079},{"game":"Assault","seed":80.0,"HNS":0.6188799076,"Max HNS":0.6633949192},{"game":"Assault","seed":81.0,"HNS":0.6249807544,"Max HNS":0.6249807544},{"game":"Assault","seed":82.0,"HNS":0.7994226328,"Max HNS":0.8653194765},{"game":"Assault","seed":83.0,"HNS":0.7300615858,"Max HNS":0.7300615858},{"game":"Assault","seed":84.0,"HNS":0.7216320246,"Max HNS":0.7216320246},{"game":"Assault","seed":85.0,"HNS":0.6334872979,"Max HNS":0.6722863741},{"game":"Assault","seed":86.0,"HNS":0.2800615858,"Max HNS":0.5302155504},{"game":"Assault","seed":87.0,"HNS":0.3908006159,"Max HNS":0.5162625096},{"game":"Assault","seed":88.0,"HNS":0.4061585835,"Max HNS":0.4061585835},{"game":"Assault","seed":89.0,"HNS":0.3030985373,"Max HNS":0.3196689761},{"game":"Assault","seed":90.0,"HNS":0.757986913,"Max HNS":0.757986913},{"game":"Assault","seed":91.0,"HNS":0.4948614319,"Max HNS":0.5383179369},{"game":"Assault","seed":92.0,"HNS":0.6611816782,"Max HNS":0.7399923018},{"game":"Assault","seed":93.0,"HNS":0.1224403387,"Max HNS":0.3342186297},{"game":"Assault","seed":94.0,"HNS":0.7217667436,"Max HNS":0.7217667436},{"game":"Assault","seed":95.0,"HNS":0.4586989992,"Max HNS":0.4928406467},{"game":"Assault","seed":96.0,"HNS":0.7217859892,"Max HNS":0.7641647421},{"game":"Assault","seed":97.0,"HNS":0.8761354888,"Max HNS":1.0189953811},{"game":"Assault","seed":98.0,"HNS":0.8885873749,"Max HNS":0.9853926097},{"game":"Assault","seed":99.0,"HNS":0.6411470362,"Max HNS":0.7646266359},{"game":"Assault","seed":100.0,"HNS":0.7056581986,"Max HNS":0.8171093149},{"game":"Asterix","seed":1.0,"HNS":0.0280346786,"Max HNS":0.0326769802},{"game":"Asterix","seed":2.0,"HNS":0.0692124968,"Max HNS":0.0692124968},{"game":"Asterix","seed":3.0,"HNS":0.0697551035,"Max HNS":0.0775927556},{"game":"Asterix","seed":4.0,"HNS":0.0524519793,"Max HNS":0.0948958798},{"game":"Asterix","seed":5.0,"HNS":0.028456706,"Max HNS":0.0601690521},{"game":"Asterix","seed":6.0,"HNS":0.0287581542,"Max HNS":0.0389471019},{"game":"Asterix","seed":7.0,"HNS":0.0404543427,"Max HNS":0.0404543427},{"game":"Asterix","seed":8.0,"HNS":0.0572751498,"Max HNS":0.0593852869},{"game":"Asterix","seed":9.0,"HNS":0.0597470247,"Max HNS":0.071141765},{"game":"Asterix","seed":10.0,"HNS":0.0327372698,"Max HNS":0.0595058662},{"game":"Asterix","seed":11.0,"HNS":0.0549841438,"Max HNS":0.0607116588},{"game":"Asterix","seed":12.0,"HNS":0.0411175286,"Max HNS":0.0647510641},{"game":"Asterix","seed":13.0,"HNS":0.0440114309,"Max HNS":0.0440114309},{"game":"Asterix","seed":14.0,"HNS":0.049558077,"Max HNS":0.049558077},{"game":"Asterix","seed":15.0,"HNS":0.0244775903,"Max HNS":0.0297227883},{"game":"Asterix","seed":16.0,"HNS":0.0909770538,"Max HNS":0.0909770538},{"game":"Asterix","seed":17.0,"HNS":0.0379824678,"Max HNS":0.0457598302},{"game":"Asterix","seed":18.0,"HNS":0.044976065,"Max HNS":0.0561899365},{"game":"Asterix","seed":19.0,"HNS":0.0407557908,"Max HNS":0.0407557908},{"game":"Asterix","seed":20.0,"HNS":0.0361134892,"Max HNS":0.0679464146},{"game":"Asterix","seed":21.0,"HNS":0.0524519793,"Max HNS":0.0589029699},{"game":"Asterix","seed":22.0,"HNS":0.0146503804,"Max HNS":0.0655348293},{"game":"Asterix","seed":23.0,"HNS":0.0607116588,"Max HNS":0.0646304848},{"game":"Asterix","seed":24.0,"HNS":0.0608925277,"Max HNS":0.0608925277},{"game":"Asterix","seed":25.0,"HNS":0.054622406,"Max HNS":0.054622406},{"game":"Asterix","seed":26.0,"HNS":0.0370178337,"Max HNS":0.0769898593},{"game":"Asterix","seed":27.0,"HNS":0.0258039622,"Max HNS":0.0498595252},{"game":"Asterix","seed":28.0,"HNS":0.0427453487,"Max HNS":0.0434085346},{"game":"Asterix","seed":29.0,"HNS":0.0452775132,"Max HNS":0.0658362775},{"game":"Asterix","seed":30.0,"HNS":0.0253819348,"Max HNS":0.0262259897},{"game":"Asterix","seed":31.0,"HNS":0.0345459588,"Max HNS":0.0570339913},{"game":"Asterix","seed":32.0,"HNS":0.059204418,"Max HNS":0.0745179844},{"game":"Asterix","seed":33.0,"HNS":0.0371384129,"Max HNS":0.0474479399},{"game":"Asterix","seed":34.0,"HNS":0.0491963392,"Max HNS":0.0509447385},{"game":"Asterix","seed":35.0,"HNS":0.0185089168,"Max HNS":0.0327975595},{"game":"Asterix","seed":36.0,"HNS":0.1005028155,"Max HNS":0.1056274342},{"game":"Asterix","seed":37.0,"HNS":0.0590838388,"Max HNS":0.061495424},{"game":"Asterix","seed":38.0,"HNS":0.0301448157,"Max HNS":0.0350282758},{"game":"Asterix","seed":39.0,"HNS":0.0268891756,"Max HNS":0.0306874224},{"game":"Asterix","seed":40.0,"HNS":0.0198352887,"Max HNS":0.0265877274},{"game":"Asterix","seed":41.0,"HNS":0.0242967214,"Max HNS":0.0514873452},{"game":"Asterix","seed":42.0,"HNS":0.043348245,"Max HNS":0.0532960341},{"game":"Asterix","seed":43.0,"HNS":0.0475082295,"Max HNS":0.0475082295},{"game":"Asterix","seed":44.0,"HNS":0.0273714927,"Max HNS":0.0391882604},{"game":"Asterix","seed":45.0,"HNS":0.0809086853,"Max HNS":0.0889874959},{"game":"Asterix","seed":46.0,"HNS":0.0330990076,"Max HNS":0.0367766751},{"game":"Asterix","seed":47.0,"HNS":0.046784754,"Max HNS":0.046784754},{"game":"Asterix","seed":48.0,"HNS":0.0341239314,"Max HNS":0.0341239314},{"game":"Asterix","seed":49.0,"HNS":0.0352091447,"Max HNS":0.045639251},{"game":"Asterix","seed":50.0,"HNS":0.0370178337,"Max HNS":0.0697551035},{"game":"Asterix","seed":51.0,"HNS":0.0529342964,"Max HNS":0.0720461095},{"game":"Asterix","seed":52.0,"HNS":0.059204418,"Max HNS":0.0850083803},{"game":"Asterix","seed":53.0,"HNS":0.0215233984,"Max HNS":0.1279345978},{"game":"Asterix","seed":54.0,"HNS":0.0437099828,"Max HNS":0.0813910024},{"game":"Asterix","seed":55.0,"HNS":0.0249599074,"Max HNS":0.0359326203},{"game":"Asterix","seed":56.0,"HNS":0.0503418422,"Max HNS":0.0527534275},{"game":"Asterix","seed":57.0,"HNS":0.0437702724,"Max HNS":0.0437702724},{"game":"Asterix","seed":58.0,"HNS":0.0168810968,"Max HNS":0.0548635646},{"game":"Asterix","seed":59.0,"HNS":0.040394053,"Max HNS":0.040394053},{"game":"Asterix","seed":60.0,"HNS":0.0212219502,"Max HNS":0.0236335355},{"game":"Asterix","seed":61.0,"HNS":0.0353900136,"Max HNS":0.0777133349},{"game":"Asterix","seed":62.0,"HNS":0.0224277429,"Max HNS":0.0864553314},{"game":"Asterix","seed":63.0,"HNS":0.022729191,"Max HNS":0.0620983203},{"game":"Asterix","seed":64.0,"HNS":0.0753017496,"Max HNS":0.0753017496},{"game":"Asterix","seed":65.0,"HNS":0.0497389459,"Max HNS":0.0517285037},{"game":"Asterix","seed":66.0,"HNS":0.0299639468,"Max HNS":0.0299639468},{"game":"Asterix","seed":67.0,"HNS":0.0100080788,"Max HNS":0.0731313229},{"game":"Asterix","seed":68.0,"HNS":0.0131431396,"Max HNS":0.0224277429},{"game":"Asterix","seed":69.0,"HNS":0.0371384129,"Max HNS":0.0708403169},{"game":"Asterix","seed":70.0,"HNS":0.0241761422,"Max HNS":0.0252613556},{"game":"Asterix","seed":71.0,"HNS":0.0126608226,"Max HNS":0.0203176058},{"game":"Asterix","seed":72.0,"HNS":0.0445540376,"Max HNS":0.0516079245},{"game":"Asterix","seed":73.0,"HNS":0.0633644026,"Max HNS":0.0633644026},{"game":"Asterix","seed":74.0,"HNS":0.0526328482,"Max HNS":0.0790397068},{"game":"Asterix","seed":75.0,"HNS":0.0317123461,"Max HNS":0.0466641747},{"game":"Asterix","seed":76.0,"HNS":0.0470862021,"Max HNS":0.0787382586},{"game":"Asterix","seed":77.0,"HNS":0.0719255302,"Max HNS":0.0877817033},{"game":"Asterix","seed":78.0,"HNS":0.0422630316,"Max HNS":0.0527534275},{"game":"Asterix","seed":79.0,"HNS":0.0517887934,"Max HNS":0.058722101},{"game":"Asterix","seed":80.0,"HNS":0.043830562,"Max HNS":0.0985735473},{"game":"Asterix","seed":81.0,"HNS":0.0702374206,"Max HNS":0.0702374206},{"game":"Asterix","seed":82.0,"HNS":0.0664391738,"Max HNS":0.0664391738},{"game":"Asterix","seed":83.0,"HNS":0.0463024369,"Max HNS":0.04907576},{"game":"Asterix","seed":84.0,"HNS":0.0320740839,"Max HNS":0.0713226339},{"game":"Asterix","seed":85.0,"HNS":0.0508241593,"Max HNS":0.0508241593},{"game":"Asterix","seed":86.0,"HNS":0.029119892,"Max HNS":0.029119892},{"game":"Asterix","seed":87.0,"HNS":0.0636658507,"Max HNS":0.0636658507},{"game":"Asterix","seed":88.0,"HNS":0.0453378028,"Max HNS":0.0553458816},{"game":"Asterix","seed":89.0,"HNS":0.0414792664,"Max HNS":0.0414792664},{"game":"Asterix","seed":90.0,"HNS":0.0618571618,"Max HNS":0.0664391738},{"game":"Asterix","seed":91.0,"HNS":0.0124196641,"Max HNS":0.0246584592},{"game":"Asterix","seed":92.0,"HNS":0.0305065535,"Max HNS":0.0730107436},{"game":"Asterix","seed":93.0,"HNS":0.108280178,"Max HNS":0.108280178},{"game":"Asterix","seed":94.0,"HNS":0.072287268,"Max HNS":0.072287268},{"game":"Asterix","seed":95.0,"HNS":0.0052451979,"Max HNS":0.033038718},{"game":"Asterix","seed":96.0,"HNS":0.0179663102,"Max HNS":0.0275523615},{"game":"Asterix","seed":97.0,"HNS":0.0387059434,"Max HNS":0.0571545706},{"game":"Asterix","seed":98.0,"HNS":0.0324358217,"Max HNS":0.0324358217},{"game":"Asterix","seed":99.0,"HNS":0.1033967178,"Max HNS":0.1033967178},{"game":"Asterix","seed":100.0,"HNS":0.0261054104,"Max HNS":0.0363546477},{"game":"BankHeist","seed":1.0,"HNS":0.0349167682,"Max HNS":0.0364054676},{"game":"BankHeist","seed":2.0,"HNS":0.0411422385,"Max HNS":0.0411422385},{"game":"BankHeist","seed":3.0,"HNS":0.0771416971,"Max HNS":0.1046149682},{"game":"BankHeist","seed":4.0,"HNS":0.0488564082,"Max HNS":0.0737582893},{"game":"BankHeist","seed":5.0,"HNS":0.0274732711,"Max HNS":0.0625253756},{"game":"BankHeist","seed":6.0,"HNS":0.0324807146,"Max HNS":0.0338340777},{"game":"BankHeist","seed":7.0,"HNS":0.0031127351,"Max HNS":0.010014887},{"game":"BankHeist","seed":8.0,"HNS":0.0253078901,"Max HNS":0.0253078901},{"game":"BankHeist","seed":9.0,"HNS":0.0492624171,"Max HNS":0.0955474354},{"game":"BankHeist","seed":10.0,"HNS":0.0041954256,"Max HNS":0.0171877115},{"game":"BankHeist","seed":11.0,"HNS":0.0475030451,"Max HNS":0.1278928136},{"game":"BankHeist","seed":12.0,"HNS":0.0251725538,"Max HNS":0.0251725538},{"game":"BankHeist","seed":13.0,"HNS":0.0948707538,"Max HNS":0.0948707538},{"game":"BankHeist","seed":14.0,"HNS":0.0350521045,"Max HNS":0.0404655569},{"game":"BankHeist","seed":15.0,"HNS":0.0328867235,"Max HNS":0.0465556909},{"game":"BankHeist","seed":16.0,"HNS":0.3819190689,"Max HNS":0.4704290161},{"game":"BankHeist","seed":17.0,"HNS":0.0105562322,"Max HNS":0.0209771282},{"game":"BankHeist","seed":18.0,"HNS":0.0295033157,"Max HNS":0.0295033157},{"game":"BankHeist","seed":19.0,"HNS":0.0286912979,"Max HNS":0.0347814319},{"game":"BankHeist","seed":20.0,"HNS":0.0297739884,"Max HNS":0.0366761402},{"game":"BankHeist","seed":21.0,"HNS":-0.0056841251,"Max HNS":0.0255785627},{"game":"BankHeist","seed":22.0,"HNS":0.6160508865,"Max HNS":0.6160508865},{"game":"BankHeist","seed":23.0,"HNS":0.0092028691,"Max HNS":0.0564352416},{"game":"BankHeist","seed":24.0,"HNS":0.021789146,"Max HNS":0.0411422385},{"game":"BankHeist","seed":25.0,"HNS":0.0028420625,"Max HNS":0.0435782921},{"game":"BankHeist","seed":26.0,"HNS":0.0770063608,"Max HNS":0.0770063608},{"game":"BankHeist","seed":27.0,"HNS":0.1024495872,"Max HNS":0.1024495872},{"game":"BankHeist","seed":28.0,"HNS":-0.002165381,"Max HNS":0.0249018812},{"game":"BankHeist","seed":29.0,"HNS":0.0841791853,"Max HNS":0.0841791853},{"game":"BankHeist","seed":30.0,"HNS":0.0081201786,"Max HNS":0.0419542563},{"game":"BankHeist","seed":31.0,"HNS":0.0598186493,"Max HNS":0.0598186493},{"game":"BankHeist","seed":32.0,"HNS":0.0232778454,"Max HNS":0.0814724591},{"game":"BankHeist","seed":33.0,"HNS":0.045743673,"Max HNS":0.0683448369},{"game":"BankHeist","seed":34.0,"HNS":0.6588171606,"Max HNS":0.6862904317},{"game":"BankHeist","seed":35.0,"HNS":0.0258492353,"Max HNS":0.1104344296},{"game":"BankHeist","seed":36.0,"HNS":0.04181892,"Max HNS":0.04181892},{"game":"BankHeist","seed":37.0,"HNS":0.0075788334,"Max HNS":0.0243605359},{"game":"BankHeist","seed":38.0,"HNS":0.0249018812,"Max HNS":0.0489917445},{"game":"BankHeist","seed":39.0,"HNS":0.0462850183,"Max HNS":0.0462850183},{"game":"BankHeist","seed":40.0,"HNS":0.0290973068,"Max HNS":0.0290973068},{"game":"BankHeist","seed":41.0,"HNS":0.0648260928,"Max HNS":0.0687508459},{"game":"BankHeist","seed":42.0,"HNS":0.0213831371,"Max HNS":0.0213831371},{"game":"BankHeist","seed":43.0,"HNS":0.0388415212,"Max HNS":0.0473677088},{"game":"BankHeist","seed":44.0,"HNS":0.0096088781,"Max HNS":0.0235485181},{"game":"BankHeist","seed":45.0,"HNS":0.3188523481,"Max HNS":0.3188523481},{"game":"BankHeist","seed":46.0,"HNS":0.0406008932,"Max HNS":0.0556232237},{"game":"BankHeist","seed":47.0,"HNS":0.008255515,"Max HNS":0.008255515},{"game":"BankHeist","seed":48.0,"HNS":-0.0029773988,"Max HNS":0.0308566788},{"game":"BankHeist","seed":49.0,"HNS":0.0165110299,"Max HNS":0.1542833942},{"game":"BankHeist","seed":50.0,"HNS":0.0128569495,"Max HNS":0.0207064555},{"game":"BankHeist","seed":51.0,"HNS":0.2385979158,"Max HNS":0.2385979158},{"game":"BankHeist","seed":52.0,"HNS":0.0354581134,"Max HNS":0.0526458249},{"game":"BankHeist","seed":53.0,"HNS":0.0374881581,"Max HNS":0.0433076194},{"game":"BankHeist","seed":54.0,"HNS":0.0216538097,"Max HNS":0.0653674381},{"game":"BankHeist","seed":55.0,"HNS":0.0759236703,"Max HNS":0.0967654622},{"game":"BankHeist","seed":56.0,"HNS":0.1724184599,"Max HNS":0.1724184599},{"game":"BankHeist","seed":57.0,"HNS":0.0475030451,"Max HNS":0.0475030451},{"game":"BankHeist","seed":58.0,"HNS":0.008255515,"Max HNS":0.0458790093},{"game":"BankHeist","seed":59.0,"HNS":0.0443903099,"Max HNS":0.0546758695},{"game":"BankHeist","seed":60.0,"HNS":0.1534713764,"Max HNS":0.1534713764},{"game":"BankHeist","seed":61.0,"HNS":-0.0043307619,"Max HNS":0.0119095953},{"game":"BankHeist","seed":62.0,"HNS":0.0512924618,"Max HNS":0.0512924618},{"game":"BankHeist","seed":63.0,"HNS":0.0012180268,"Max HNS":0.0350521045},{"game":"BankHeist","seed":64.0,"HNS":0.0173230478,"Max HNS":0.0673974827},{"game":"BankHeist","seed":65.0,"HNS":0.1567194478,"Max HNS":0.1679523616},{"game":"BankHeist","seed":66.0,"HNS":0.0083908513,"Max HNS":0.0328867235},{"game":"BankHeist","seed":67.0,"HNS":0.0086615239,"Max HNS":0.0282852889},{"game":"BankHeist","seed":68.0,"HNS":0.0669914738,"Max HNS":0.0669914738},{"game":"BankHeist","seed":69.0,"HNS":0.0365408039,"Max HNS":0.0703748816},{"game":"BankHeist","seed":70.0,"HNS":0.0553525511,"Max HNS":0.0553525511},{"game":"BankHeist","seed":71.0,"HNS":-0.0001353363,"Max HNS":0.0230071728},{"game":"BankHeist","seed":72.0,"HNS":0.6850724049,"Max HNS":0.7327107863},{"game":"BankHeist","seed":73.0,"HNS":0.0595479767,"Max HNS":0.0598186493},{"game":"BankHeist","seed":74.0,"HNS":0.0228718365,"Max HNS":0.0228718365},{"game":"BankHeist","seed":75.0,"HNS":0.111923129,"Max HNS":0.111923129},{"game":"BankHeist","seed":76.0,"HNS":0.0242251996,"Max HNS":0.0242251996},{"game":"BankHeist","seed":77.0,"HNS":0.0983894979,"Max HNS":0.0983894979},{"game":"BankHeist","seed":78.0,"HNS":0.0292326431,"Max HNS":0.0422249289},{"game":"BankHeist","seed":79.0,"HNS":0.0726755989,"Max HNS":0.0726755989},{"game":"BankHeist","seed":80.0,"HNS":0.0209771282,"Max HNS":0.0223304913},{"game":"BankHeist","seed":81.0,"HNS":0.0158343484,"Max HNS":0.0552172148},{"game":"BankHeist","seed":82.0,"HNS":0.0020300447,"Max HNS":0.0085261876},{"game":"BankHeist","seed":83.0,"HNS":0.0096088781,"Max HNS":0.0096088781},{"game":"BankHeist","seed":84.0,"HNS":0.045743673,"Max HNS":0.0469616998},{"game":"BankHeist","seed":85.0,"HNS":0.0259845717,"Max HNS":0.0259845717},{"game":"BankHeist","seed":86.0,"HNS":0.0533225064,"Max HNS":0.0533225064},{"game":"BankHeist","seed":87.0,"HNS":0.1886588172,"Max HNS":0.2224928948},{"game":"BankHeist","seed":88.0,"HNS":0.015699012,"Max HNS":0.0372174855},{"game":"BankHeist","seed":89.0,"HNS":0.0782243876,"Max HNS":0.0782243876},{"game":"BankHeist","seed":90.0,"HNS":0.157666802,"Max HNS":0.157666802},{"game":"BankHeist","seed":91.0,"HNS":0.1772905671,"Max HNS":0.1795912843},{"game":"BankHeist","seed":92.0,"HNS":0.0489917445,"Max HNS":0.0489917445},{"game":"BankHeist","seed":93.0,"HNS":0.0147516579,"Max HNS":0.0147516579},{"game":"BankHeist","seed":94.0,"HNS":0.030044661,"Max HNS":0.0345107592},{"game":"BankHeist","seed":95.0,"HNS":0.0240898633,"Max HNS":0.0243605359},{"game":"BankHeist","seed":96.0,"HNS":0.0326160509,"Max HNS":0.0480443903},{"game":"BankHeist","seed":97.0,"HNS":0.0406008932,"Max HNS":0.0406008932},{"game":"BankHeist","seed":98.0,"HNS":0.0492624171,"Max HNS":0.1220733523},{"game":"BankHeist","seed":99.0,"HNS":0.0267965895,"Max HNS":0.0760590066},{"game":"BankHeist","seed":100.0,"HNS":0.0546758695,"Max HNS":0.0787657328},{"game":"BattleZone","seed":1.0,"HNS":0.2587036107,"Max HNS":0.3342186491},{"game":"BattleZone","seed":2.0,"HNS":0.073218003,"Max HNS":0.1461488766},{"game":"BattleZone","seed":3.0,"HNS":0.0155049889,"Max HNS":0.0927428038},{"game":"BattleZone","seed":4.0,"HNS":0.0829804034,"Max HNS":0.2271193741},{"game":"BattleZone","seed":5.0,"HNS":0.3939415692,"Max HNS":0.3939415692},{"game":"BattleZone","seed":6.0,"HNS":0.1122676046,"Max HNS":0.2558323164},{"game":"BattleZone","seed":7.0,"HNS":0.0155049889,"Max HNS":0.1358122174},{"game":"BattleZone","seed":8.0,"HNS":0.0364654368,"Max HNS":0.1137032517},{"game":"BattleZone","seed":9.0,"HNS":0.0775249444,"Max HNS":0.0867130859},{"game":"BattleZone","seed":10.0,"HNS":0.3029215419,"Max HNS":0.3112482952},{"game":"BattleZone","seed":11.0,"HNS":0.002009906,"Max HNS":0.1424161941},{"game":"BattleZone","seed":12.0,"HNS":0.24406001,"Max HNS":0.24406001},{"game":"BattleZone","seed":13.0,"HNS":0.2991888594,"Max HNS":0.2991888594},{"game":"BattleZone","seed":14.0,"HNS":0.2050104084,"Max HNS":0.3195750485},{"game":"BattleZone","seed":15.0,"HNS":0.1452874883,"Max HNS":0.2426243629},{"game":"BattleZone","seed":16.0,"HNS":0.3126839423,"Max HNS":0.3247433781},{"game":"BattleZone","seed":17.0,"HNS":0.3772880626,"Max HNS":0.3772880626},{"game":"BattleZone","seed":18.0,"HNS":0.1197329696,"Max HNS":0.24406001},{"game":"BattleZone","seed":19.0,"HNS":0.306367095,"Max HNS":0.306367095},{"game":"BattleZone","seed":20.0,"HNS":0.0867130859,"Max HNS":0.0895843802},{"game":"BattleZone","seed":21.0,"HNS":0.1995549494,"Max HNS":0.3204364367},{"game":"BattleZone","seed":22.0,"HNS":0.0275644247,"Max HNS":0.0660397674},{"game":"BattleZone","seed":23.0,"HNS":0.3405354964,"Max HNS":0.4197832173},{"game":"BattleZone","seed":24.0,"HNS":0.0611585672,"Max HNS":0.3083770009},{"game":"BattleZone","seed":25.0,"HNS":0.0450793195,"Max HNS":0.1286339818},{"game":"BattleZone","seed":26.0,"HNS":0.4312683942,"Max HNS":0.4662981839},{"game":"BattleZone","seed":27.0,"HNS":0.4077237815,"Max HNS":0.5073576915},{"game":"BattleZone","seed":28.0,"HNS":0.1630895126,"Max HNS":0.3405354964},{"game":"BattleZone","seed":29.0,"HNS":0.0017227765,"Max HNS":0.1920895844},{"game":"BattleZone","seed":30.0,"HNS":0.3879118513,"Max HNS":0.4418921829},{"game":"BattleZone","seed":31.0,"HNS":0.0324456249,"Max HNS":0.1401191587},{"game":"BattleZone","seed":32.0,"HNS":0.0915942861,"Max HNS":0.1625152538},{"game":"BattleZone","seed":33.0,"HNS":0.0422080253,"Max HNS":0.2176441031},{"game":"BattleZone","seed":34.0,"HNS":0.0714952265,"Max HNS":0.0714952265},{"game":"BattleZone","seed":35.0,"HNS":0.416911923,"Max HNS":0.416911923},{"game":"BattleZone","seed":36.0,"HNS":0.0290000718,"Max HNS":0.1383963822},{"game":"BattleZone","seed":37.0,"HNS":0.0089010121,"Max HNS":0.2242480798},{"game":"BattleZone","seed":38.0,"HNS":0.0416337664,"Max HNS":0.0973368746},{"game":"BattleZone","seed":39.0,"HNS":0.4568229129,"Max HNS":0.5957935539},{"game":"BattleZone","seed":40.0,"HNS":0.2205153973,"Max HNS":0.2782284115},{"game":"BattleZone","seed":41.0,"HNS":-0.0218218362,"Max HNS":0.1771588544},{"game":"BattleZone","seed":42.0,"HNS":0.2518125045,"Max HNS":0.4215059938},{"game":"BattleZone","seed":43.0,"HNS":0.1369607351,"Max HNS":0.1369607351},{"game":"BattleZone","seed":44.0,"HNS":0.370109827,"Max HNS":0.370109827},{"game":"BattleZone","seed":45.0,"HNS":0.0034455531,"Max HNS":0.197257914},{"game":"BattleZone","seed":46.0,"HNS":0.1010695571,"Max HNS":0.1909410667},{"game":"BattleZone","seed":47.0,"HNS":0.1691192305,"Max HNS":0.1691192305},{"game":"BattleZone","seed":48.0,"HNS":0.0542674611,"Max HNS":0.1139903812},{"game":"BattleZone","seed":49.0,"HNS":0.1289211112,"Max HNS":0.1395448999},{"game":"BattleZone","seed":50.0,"HNS":0.0238317422,"Max HNS":0.105089369},{"game":"BattleZone","seed":51.0,"HNS":0.5105161151,"Max HNS":0.5688033881},{"game":"BattleZone","seed":52.0,"HNS":0.3342186491,"Max HNS":0.3342186491},{"game":"BattleZone","seed":53.0,"HNS":0.2922977532,"Max HNS":0.2922977532},{"game":"BattleZone","seed":54.0,"HNS":0.0333070131,"Max HNS":0.2061589261},{"game":"BattleZone","seed":55.0,"HNS":0.1352379585,"Max HNS":0.1984064317},{"game":"BattleZone","seed":56.0,"HNS":0.0315842366,"Max HNS":0.1567726653},{"game":"BattleZone","seed":57.0,"HNS":0.5199913861,"Max HNS":0.5199913861},{"game":"BattleZone","seed":58.0,"HNS":0.5535855287,"Max HNS":0.5650707056},{"game":"BattleZone","seed":59.0,"HNS":0.5102289857,"Max HNS":0.5116646328},{"game":"BattleZone","seed":60.0,"HNS":0.3672385328,"Max HNS":0.3672385328},{"game":"BattleZone","seed":61.0,"HNS":0.0178020243,"Max HNS":0.0898715096},{"game":"BattleZone","seed":62.0,"HNS":0.0522575551,"Max HNS":0.0614456966},{"game":"BattleZone","seed":63.0,"HNS":0.0186634125,"Max HNS":0.1404062881},{"game":"BattleZone","seed":64.0,"HNS":0.2905749767,"Max HNS":0.3614959443},{"game":"BattleZone","seed":65.0,"HNS":-0.0178020243,"Max HNS":0.1447132295},{"game":"BattleZone","seed":66.0,"HNS":0.167396454,"Max HNS":0.2271193741},{"game":"BattleZone","seed":67.0,"HNS":0.3649414974,"Max HNS":0.5627736702},{"game":"BattleZone","seed":68.0,"HNS":0.1547627593,"Max HNS":0.2081688321},{"game":"BattleZone","seed":69.0,"HNS":0.3517335439,"Max HNS":0.4841002082},{"game":"BattleZone","seed":70.0,"HNS":0.0287129424,"Max HNS":0.0835546623},{"game":"BattleZone","seed":71.0,"HNS":0.133515182,"Max HNS":0.133515182},{"game":"BattleZone","seed":72.0,"HNS":-0.0172277654,"Max HNS":0.1742875601},{"game":"BattleZone","seed":73.0,"HNS":0.1412676764,"Max HNS":0.3896346278},{"game":"BattleZone","seed":74.0,"HNS":0.272485823,"Max HNS":0.5179814802},{"game":"BattleZone","seed":75.0,"HNS":0.1320795349,"Max HNS":0.1320795349},{"game":"BattleZone","seed":76.0,"HNS":0.01493073,"Max HNS":0.0686239322},{"game":"BattleZone","seed":77.0,"HNS":0.1435647118,"Max HNS":0.2713373053},{"game":"BattleZone","seed":78.0,"HNS":0.126049817,"Max HNS":0.4594070777},{"game":"BattleZone","seed":79.0,"HNS":0.0838417917,"Max HNS":0.1530399828},{"game":"BattleZone","seed":80.0,"HNS":0.3491493791,"Max HNS":0.418921829},{"game":"BattleZone","seed":81.0,"HNS":0.3666642739,"Max HNS":0.3666642739},{"game":"BattleZone","seed":82.0,"HNS":0.0723566147,"Max HNS":0.0838417917},{"game":"BattleZone","seed":83.0,"HNS":0.0740793913,"Max HNS":0.0872873448},{"game":"BattleZone","seed":84.0,"HNS":0.0522575551,"Max HNS":0.1547627593},{"game":"BattleZone","seed":85.0,"HNS":-0.0252673893,"Max HNS":0.0944655804},{"game":"BattleZone","seed":86.0,"HNS":-0.0155049889,"Max HNS":0.1530399828},{"game":"BattleZone","seed":87.0,"HNS":0.0548417199,"Max HNS":0.1326537937},{"game":"BattleZone","seed":88.0,"HNS":0.1863469959,"Max HNS":0.1863469959},{"game":"BattleZone","seed":89.0,"HNS":0.5053477855,"Max HNS":0.7405067834},{"game":"BattleZone","seed":90.0,"HNS":0.296891824,"Max HNS":0.296891824},{"game":"BattleZone","seed":91.0,"HNS":0.2044361496,"Max HNS":0.2044361496},{"game":"BattleZone","seed":92.0,"HNS":0.311822554,"Max HNS":0.311822554},{"game":"BattleZone","seed":93.0,"HNS":0.4881200201,"Max HNS":0.6041203072},{"game":"BattleZone","seed":94.0,"HNS":0.5765558826,"Max HNS":0.6201995549},{"game":"BattleZone","seed":95.0,"HNS":0.4579714306,"Max HNS":0.503625009},{"game":"BattleZone","seed":96.0,"HNS":0.3548919676,"Max HNS":0.3755652861},{"game":"BattleZone","seed":97.0,"HNS":0.2268322446,"Max HNS":0.2334362214},{"game":"BattleZone","seed":98.0,"HNS":0.0832675328,"Max HNS":0.1323666643},{"game":"BattleZone","seed":99.0,"HNS":0.4327040413,"Max HNS":0.5087933386},{"game":"BattleZone","seed":100.0,"HNS":0.0953269686,"Max HNS":0.1915153255},{"game":"Boxing","seed":1.0,"HNS":0.1033333333,"Max HNS":0.3025},{"game":"Boxing","seed":2.0,"HNS":0.0075,"Max HNS":0.1016666667},{"game":"Boxing","seed":3.0,"HNS":0.1683333333,"Max HNS":0.1683333333},{"game":"Boxing","seed":4.0,"HNS":0.1316666667,"Max HNS":0.3616666667},{"game":"Boxing","seed":5.0,"HNS":-0.22,"Max HNS":-0.1116666667},{"game":"Boxing","seed":6.0,"HNS":0.4533333333,"Max HNS":0.6425},{"game":"Boxing","seed":7.0,"HNS":0.715,"Max HNS":1.0183333333},{"game":"Boxing","seed":8.0,"HNS":-0.2616666667,"Max HNS":-0.2616666667},{"game":"Boxing","seed":9.0,"HNS":0.8875,"Max HNS":0.9466666667},{"game":"Boxing","seed":10.0,"HNS":0.0191666667,"Max HNS":0.1291666667},{"game":"Boxing","seed":11.0,"HNS":0.6783333333,"Max HNS":1.5075},{"game":"Boxing","seed":12.0,"HNS":-0.3083333333,"Max HNS":-0.02},{"game":"Boxing","seed":13.0,"HNS":-0.405,"Max HNS":-0.1375},{"game":"Boxing","seed":14.0,"HNS":-0.2125,"Max HNS":-0.0316666667},{"game":"Boxing","seed":15.0,"HNS":-0.095,"Max HNS":0.1433333333},{"game":"Boxing","seed":16.0,"HNS":0.2333333333,"Max HNS":0.5225},{"game":"Boxing","seed":17.0,"HNS":-0.2225,"Max HNS":-0.1958333333},{"game":"Boxing","seed":18.0,"HNS":0.2091666667,"Max HNS":0.3833333333},{"game":"Boxing","seed":19.0,"HNS":-0.2741666667,"Max HNS":-0.0733333333},{"game":"Boxing","seed":20.0,"HNS":0.2675,"Max HNS":0.7183333333},{"game":"Boxing","seed":21.0,"HNS":-0.0225,"Max HNS":0.1425},{"game":"Boxing","seed":22.0,"HNS":0.1333333333,"Max HNS":0.1333333333},{"game":"Boxing","seed":23.0,"HNS":0.1725,"Max HNS":0.2433333333},{"game":"Boxing","seed":24.0,"HNS":0.4525,"Max HNS":0.6175},{"game":"Boxing","seed":25.0,"HNS":-0.5616666667,"Max HNS":-0.4175},{"game":"Boxing","seed":26.0,"HNS":-0.1516666667,"Max HNS":0.0075},{"game":"Boxing","seed":27.0,"HNS":0.2283333333,"Max HNS":0.3375},{"game":"Boxing","seed":28.0,"HNS":0.0616666667,"Max HNS":0.0616666667},{"game":"Boxing","seed":29.0,"HNS":-0.0075,"Max HNS":0.2058333333},{"game":"Boxing","seed":30.0,"HNS":0.0533333333,"Max HNS":0.1741666667},{"game":"Boxing","seed":31.0,"HNS":-0.0758333333,"Max HNS":-0.0758333333},{"game":"Boxing","seed":32.0,"HNS":0.2116666667,"Max HNS":0.3},{"game":"Boxing","seed":33.0,"HNS":0.1333333333,"Max HNS":0.4133333333},{"game":"Boxing","seed":34.0,"HNS":-0.0391666667,"Max HNS":0.125},{"game":"Boxing","seed":35.0,"HNS":0.2375,"Max HNS":0.3258333333},{"game":"Boxing","seed":36.0,"HNS":-0.6975,"Max HNS":-0.6975},{"game":"Boxing","seed":37.0,"HNS":0.3333333333,"Max HNS":0.9983333333},{"game":"Boxing","seed":38.0,"HNS":-0.1225,"Max HNS":0.5275},{"game":"Boxing","seed":39.0,"HNS":1.5983333333,"Max HNS":1.5983333333},{"game":"Boxing","seed":40.0,"HNS":0.2066666667,"Max HNS":0.2891666667},{"game":"Boxing","seed":41.0,"HNS":0.4066666667,"Max HNS":0.6033333333},{"game":"Boxing","seed":42.0,"HNS":-0.1775,"Max HNS":-0.09},{"game":"Boxing","seed":43.0,"HNS":0.0641666667,"Max HNS":0.105},{"game":"Boxing","seed":44.0,"HNS":-0.0541666667,"Max HNS":0.105},{"game":"Boxing","seed":45.0,"HNS":-0.2683333333,"Max HNS":0.2133333333},{"game":"Boxing","seed":46.0,"HNS":0.1425,"Max HNS":0.1425},{"game":"Boxing","seed":47.0,"HNS":0.195,"Max HNS":0.195},{"game":"Boxing","seed":48.0,"HNS":0.1358333333,"Max HNS":0.1358333333},{"game":"Boxing","seed":49.0,"HNS":-0.0183333333,"Max HNS":0.1241666667},{"game":"Boxing","seed":50.0,"HNS":0.0575,"Max HNS":0.0575},{"game":"Boxing","seed":51.0,"HNS":-0.7075,"Max HNS":-0.33},{"game":"Boxing","seed":52.0,"HNS":0.2633333333,"Max HNS":0.315},{"game":"Boxing","seed":53.0,"HNS":0.4175,"Max HNS":1.4858333333},{"game":"Boxing","seed":54.0,"HNS":-0.115,"Max HNS":0.2141666667},{"game":"Boxing","seed":55.0,"HNS":0.0141666667,"Max HNS":0.2675},{"game":"Boxing","seed":56.0,"HNS":0.2241666667,"Max HNS":0.2241666667},{"game":"Boxing","seed":57.0,"HNS":0.0533333333,"Max HNS":0.3566666667},{"game":"Boxing","seed":58.0,"HNS":-0.6341666667,"Max HNS":-0.6341666667},{"game":"Boxing","seed":59.0,"HNS":0.2075,"Max HNS":0.6458333333},{"game":"Boxing","seed":60.0,"HNS":-0.0625,"Max HNS":0.0408333333},{"game":"Boxing","seed":61.0,"HNS":0.0766666667,"Max HNS":0.105},{"game":"Boxing","seed":62.0,"HNS":0.2316666667,"Max HNS":0.3258333333},{"game":"Boxing","seed":63.0,"HNS":-0.7725,"Max HNS":-0.2625},{"game":"Boxing","seed":64.0,"HNS":0.2116666667,"Max HNS":0.2133333333},{"game":"Boxing","seed":65.0,"HNS":0.1625,"Max HNS":0.1625},{"game":"Boxing","seed":66.0,"HNS":0.2,"Max HNS":0.8375},{"game":"Boxing","seed":67.0,"HNS":0.64,"Max HNS":1.1475},{"game":"Boxing","seed":68.0,"HNS":-0.0516666667,"Max HNS":-0.0458333333},{"game":"Boxing","seed":69.0,"HNS":-0.0316666667,"Max HNS":0.0183333333},{"game":"Boxing","seed":70.0,"HNS":-0.4033333333,"Max HNS":-0.4033333333},{"game":"Boxing","seed":71.0,"HNS":-0.0333333333,"Max HNS":-0.0241666667},{"game":"Boxing","seed":72.0,"HNS":0.6833333333,"Max HNS":0.6833333333},{"game":"Boxing","seed":73.0,"HNS":-0.21,"Max HNS":-0.1},{"game":"Boxing","seed":74.0,"HNS":-0.0216666667,"Max HNS":0.1383333333},{"game":"Boxing","seed":75.0,"HNS":-0.3366666667,"Max HNS":-0.1725},{"game":"Boxing","seed":76.0,"HNS":-0.1458333333,"Max HNS":0.0233333333},{"game":"Boxing","seed":77.0,"HNS":0.0333333333,"Max HNS":0.2566666667},{"game":"Boxing","seed":78.0,"HNS":0.16,"Max HNS":0.205},{"game":"Boxing","seed":79.0,"HNS":-0.3291666667,"Max HNS":-0.3058333333},{"game":"Boxing","seed":80.0,"HNS":-0.2041666667,"Max HNS":0.0541666667},{"game":"Boxing","seed":81.0,"HNS":0.015,"Max HNS":0.1258333333},{"game":"Boxing","seed":82.0,"HNS":0.03,"Max HNS":0.03},{"game":"Boxing","seed":83.0,"HNS":0.6625,"Max HNS":1.2258333333},{"game":"Boxing","seed":84.0,"HNS":-0.0575,"Max HNS":-0.0208333333},{"game":"Boxing","seed":85.0,"HNS":0.4983333333,"Max HNS":0.5716666667},{"game":"Boxing","seed":86.0,"HNS":-0.0408333333,"Max HNS":0.0425},{"game":"Boxing","seed":87.0,"HNS":-0.4366666667,"Max HNS":-0.37},{"game":"Boxing","seed":88.0,"HNS":-0.0166666667,"Max HNS":-0.0166666667},{"game":"Boxing","seed":89.0,"HNS":0.0975,"Max HNS":0.1541666667},{"game":"Boxing","seed":90.0,"HNS":0.0008333333,"Max HNS":0.0008333333},{"game":"Boxing","seed":91.0,"HNS":0.2391666667,"Max HNS":0.3258333333},{"game":"Boxing","seed":92.0,"HNS":-0.0275,"Max HNS":-0.0275},{"game":"Boxing","seed":93.0,"HNS":-0.3175,"Max HNS":-0.1075},{"game":"Boxing","seed":94.0,"HNS":0.2058333333,"Max HNS":0.355},{"game":"Boxing","seed":95.0,"HNS":-0.2316666667,"Max HNS":0.17},{"game":"Boxing","seed":96.0,"HNS":0.3841666667,"Max HNS":0.4741666667},{"game":"Boxing","seed":97.0,"HNS":1.5408333333,"Max HNS":1.5408333333},{"game":"Boxing","seed":98.0,"HNS":0.0141666667,"Max HNS":0.0916666667},{"game":"Boxing","seed":99.0,"HNS":0.4108333333,"Max HNS":0.4108333333},{"game":"Boxing","seed":100.0,"HNS":0.0875,"Max HNS":0.0975},{"game":"Breakout","seed":1.0,"HNS":0.0392361111,"Max HNS":0.0409722222},{"game":"Breakout","seed":2.0,"HNS":0.0618055556,"Max HNS":0.128125},{"game":"Breakout","seed":3.0,"HNS":0.0246527778,"Max HNS":0.0246527778},{"game":"Breakout","seed":4.0,"HNS":0.0170138889,"Max HNS":0.053125},{"game":"Breakout","seed":5.0,"HNS":0.003125,"Max HNS":0.0107638889},{"game":"Breakout","seed":6.0,"HNS":-0.0489583333,"Max HNS":-0.0090277778},{"game":"Breakout","seed":7.0,"HNS":-0.0118055556,"Max HNS":-0.0013888889},{"game":"Breakout","seed":8.0,"HNS":0.1013888889,"Max HNS":0.1013888889},{"game":"Breakout","seed":9.0,"HNS":0.046875,"Max HNS":0.0520833333},{"game":"Breakout","seed":10.0,"HNS":0.0479166667,"Max HNS":0.0753472222},{"game":"Breakout","seed":11.0,"HNS":0.0291666667,"Max HNS":0.109375},{"game":"Breakout","seed":12.0,"HNS":0.0496527778,"Max HNS":0.0836805556},{"game":"Breakout","seed":13.0,"HNS":-0.0131944444,"Max HNS":0.0184027778},{"game":"Breakout","seed":14.0,"HNS":0.1177083333,"Max HNS":0.1177083333},{"game":"Breakout","seed":15.0,"HNS":0.0079861111,"Max HNS":0.009375},{"game":"Breakout","seed":16.0,"HNS":0.0253472222,"Max HNS":0.0784722222},{"game":"Breakout","seed":17.0,"HNS":0.0371527778,"Max HNS":0.0736111111},{"game":"Breakout","seed":18.0,"HNS":0.0506944444,"Max HNS":0.1222222222},{"game":"Breakout","seed":19.0,"HNS":0.04375,"Max HNS":0.078125},{"game":"Breakout","seed":20.0,"HNS":-0.0475694444,"Max HNS":0.0430555556},{"game":"Breakout","seed":21.0,"HNS":0.0017361111,"Max HNS":0.025},{"game":"Breakout","seed":22.0,"HNS":0.0899305556,"Max HNS":0.0899305556},{"game":"Breakout","seed":23.0,"HNS":0.0701388889,"Max HNS":0.1194444444},{"game":"Breakout","seed":24.0,"HNS":0.0263888889,"Max HNS":0.0263888889},{"game":"Breakout","seed":25.0,"HNS":0.0211805556,"Max HNS":0.0211805556},{"game":"Breakout","seed":26.0,"HNS":-0.0159722222,"Max HNS":0.0277777778},{"game":"Breakout","seed":27.0,"HNS":0.0125,"Max HNS":0.028125},{"game":"Breakout","seed":28.0,"HNS":0.0454861111,"Max HNS":0.1145833333},{"game":"Breakout","seed":29.0,"HNS":0.0048611111,"Max HNS":0.0118055556},{"game":"Breakout","seed":30.0,"HNS":-0.0083333333,"Max HNS":0.0006944444},{"game":"Breakout","seed":31.0,"HNS":0.0364583333,"Max HNS":0.0440972222},{"game":"Breakout","seed":32.0,"HNS":0.0440972222,"Max HNS":0.0857638889},{"game":"Breakout","seed":33.0,"HNS":0.0538194444,"Max HNS":0.0538194444},{"game":"Breakout","seed":34.0,"HNS":-0.0052083333,"Max HNS":0.0170138889},{"game":"Breakout","seed":35.0,"HNS":-0.0024305556,"Max HNS":-0.0024305556},{"game":"Breakout","seed":36.0,"HNS":0.109375,"Max HNS":0.1829861111},{"game":"Breakout","seed":37.0,"HNS":0.0770833333,"Max HNS":0.1402777778},{"game":"Breakout","seed":38.0,"HNS":-0.0121527778,"Max HNS":-0.0118055556},{"game":"Breakout","seed":39.0,"HNS":-0.0361111111,"Max HNS":0.0125},{"game":"Breakout","seed":40.0,"HNS":0.0128472222,"Max HNS":0.0263888889},{"game":"Breakout","seed":41.0,"HNS":0.0232638889,"Max HNS":0.1204861111},{"game":"Breakout","seed":42.0,"HNS":-0.0208333333,"Max HNS":0.0197916667},{"game":"Breakout","seed":43.0,"HNS":0.0392361111,"Max HNS":0.0392361111},{"game":"Breakout","seed":44.0,"HNS":0.0583333333,"Max HNS":0.0934027778},{"game":"Breakout","seed":45.0,"HNS":-0.0465277778,"Max HNS":0.0704861111},{"game":"Breakout","seed":46.0,"HNS":0.0368055556,"Max HNS":0.0368055556},{"game":"Breakout","seed":47.0,"HNS":0.0152777778,"Max HNS":0.0645833333},{"game":"Breakout","seed":48.0,"HNS":-0.0059027778,"Max HNS":0.0434027778},{"game":"Breakout","seed":49.0,"HNS":0.0454861111,"Max HNS":0.090625},{"game":"Breakout","seed":50.0,"HNS":0.0371527778,"Max HNS":0.0861111111},{"game":"Breakout","seed":51.0,"HNS":-0.0045138889,"Max HNS":0.0385416667},{"game":"Breakout","seed":52.0,"HNS":0.1121527778,"Max HNS":0.1121527778},{"game":"Breakout","seed":53.0,"HNS":0.0465277778,"Max HNS":0.09375},{"game":"Breakout","seed":54.0,"HNS":-0.0111111111,"Max HNS":0.021875},{"game":"Breakout","seed":55.0,"HNS":0.0520833333,"Max HNS":0.1055555556},{"game":"Breakout","seed":56.0,"HNS":-0.0010416667,"Max HNS":0.0368055556},{"game":"Breakout","seed":57.0,"HNS":0.0333333333,"Max HNS":0.0333333333},{"game":"Breakout","seed":58.0,"HNS":-0.0315972222,"Max HNS":0.0201388889},{"game":"Breakout","seed":59.0,"HNS":0.0763888889,"Max HNS":0.1086805556},{"game":"Breakout","seed":60.0,"HNS":0.0809027778,"Max HNS":0.1201388889},{"game":"Breakout","seed":61.0,"HNS":0.0847222222,"Max HNS":0.0847222222},{"game":"Breakout","seed":62.0,"HNS":-0.0190972222,"Max HNS":0.0701388889},{"game":"Breakout","seed":63.0,"HNS":-0.0100694444,"Max HNS":0.0434027778},{"game":"Breakout","seed":64.0,"HNS":-0.015625,"Max HNS":-0.0024305556},{"game":"Breakout","seed":65.0,"HNS":-0.00625,"Max HNS":0.04375},{"game":"Breakout","seed":66.0,"HNS":-0.0291666667,"Max HNS":0.0385416667},{"game":"Breakout","seed":67.0,"HNS":-0.0177083333,"Max HNS":0.0215277778},{"game":"Breakout","seed":68.0,"HNS":0.0663194444,"Max HNS":0.1319444444},{"game":"Breakout","seed":69.0,"HNS":0.0569444444,"Max HNS":0.1069444444},{"game":"Breakout","seed":70.0,"HNS":0.1163194444,"Max HNS":0.1194444444},{"game":"Breakout","seed":71.0,"HNS":-0.0048611111,"Max HNS":0.0225694444},{"game":"Breakout","seed":72.0,"HNS":0.0236111111,"Max HNS":0.0465277778},{"game":"Breakout","seed":73.0,"HNS":0.0388888889,"Max HNS":0.109375},{"game":"Breakout","seed":74.0,"HNS":0.034375,"Max HNS":0.0555555556},{"game":"Breakout","seed":75.0,"HNS":0.0784722222,"Max HNS":0.0784722222},{"game":"Breakout","seed":76.0,"HNS":0.0864583333,"Max HNS":0.1180555556},{"game":"Breakout","seed":77.0,"HNS":-0.0520833333,"Max HNS":-0.0020833333},{"game":"Breakout","seed":78.0,"HNS":0.1270833333,"Max HNS":0.1270833333},{"game":"Breakout","seed":79.0,"HNS":-0.0524305556,"Max HNS":-0.0524305556},{"game":"Breakout","seed":80.0,"HNS":0.0086805556,"Max HNS":0.0131944444},{"game":"Breakout","seed":81.0,"HNS":-0.0006944444,"Max HNS":0.0288194444},{"game":"Breakout","seed":82.0,"HNS":-0.0010416667,"Max HNS":0.0045138889},{"game":"Breakout","seed":83.0,"HNS":0.0197916667,"Max HNS":0.0666666667},{"game":"Breakout","seed":84.0,"HNS":0.1444444444,"Max HNS":0.1444444444},{"game":"Breakout","seed":85.0,"HNS":0.0840277778,"Max HNS":0.0840277778},{"game":"Breakout","seed":86.0,"HNS":0.0569444444,"Max HNS":0.0704861111},{"game":"Breakout","seed":87.0,"HNS":0.0326388889,"Max HNS":0.0388888889},{"game":"Breakout","seed":88.0,"HNS":-0.0173611111,"Max HNS":0.0013888889},{"game":"Breakout","seed":89.0,"HNS":0.0840277778,"Max HNS":0.1027777778},{"game":"Breakout","seed":90.0,"HNS":-0.0295138889,"Max HNS":0.0027777778},{"game":"Breakout","seed":91.0,"HNS":0.0190972222,"Max HNS":0.0545138889},{"game":"Breakout","seed":92.0,"HNS":0.1065972222,"Max HNS":0.1065972222},{"game":"Breakout","seed":93.0,"HNS":0.0684027778,"Max HNS":0.0684027778},{"game":"Breakout","seed":94.0,"HNS":-0.0111111111,"Max HNS":0.0378472222},{"game":"Breakout","seed":95.0,"HNS":0.1288194444,"Max HNS":0.1340277778},{"game":"Breakout","seed":96.0,"HNS":-0.0006944444,"Max HNS":0.0451388889},{"game":"Breakout","seed":97.0,"HNS":0.0190972222,"Max HNS":0.0190972222},{"game":"Breakout","seed":98.0,"HNS":0.0239583333,"Max HNS":0.05625},{"game":"Breakout","seed":99.0,"HNS":0.0270833333,"Max HNS":0.1486111111},{"game":"Breakout","seed":100.0,"HNS":0.0559027778,"Max HNS":0.0652777778},{"game":"ChopperCommand","seed":1.0,"HNS":0.0439423428,"Max HNS":0.0615800997},{"game":"ChopperCommand","seed":2.0,"HNS":0.0428779954,"Max HNS":0.0609719012},{"game":"ChopperCommand","seed":3.0,"HNS":-0.0287373799,"Max HNS":0.0250881888},{"game":"ChopperCommand","seed":4.0,"HNS":-0.0462230872,"Max HNS":-0.0457669383},{"game":"ChopperCommand","seed":5.0,"HNS":-0.0751125167,"Max HNS":-0.0751125167},{"game":"ChopperCommand","seed":6.0,"HNS":0.0509366257,"Max HNS":0.0599075538},{"game":"ChopperCommand","seed":7.0,"HNS":-0.0215910473,"Max HNS":-0.0104914244},{"game":"ChopperCommand","seed":8.0,"HNS":0.1126687751,"Max HNS":0.1126687751},{"game":"ChopperCommand","seed":9.0,"HNS":-0.0120119207,"Max HNS":0.0162693103},{"game":"ChopperCommand","seed":10.0,"HNS":0.0244799903,"Max HNS":0.0345152658},{"game":"ChopperCommand","seed":11.0,"HNS":0.0433341443,"Max HNS":0.0447025909},{"game":"ChopperCommand","seed":12.0,"HNS":-0.0475915339,"Max HNS":-0.0475915339},{"game":"ChopperCommand","seed":13.0,"HNS":0.0133803674,"Max HNS":0.0133803674},{"game":"ChopperCommand","seed":14.0,"HNS":0.1976645177,"Max HNS":0.1976645177},{"game":"ChopperCommand","seed":15.0,"HNS":0.1129728743,"Max HNS":0.1262011921},{"game":"ChopperCommand","seed":16.0,"HNS":0.0038012407,"Max HNS":0.0039532904},{"game":"ChopperCommand","seed":17.0,"HNS":-0.0415095487,"Max HNS":0.0206787495},{"game":"ChopperCommand","seed":18.0,"HNS":0.0568665612,"Max HNS":0.0582350079},{"game":"ChopperCommand","seed":19.0,"HNS":-0.048351782,"Max HNS":-0.0475915339},{"game":"ChopperCommand","seed":20.0,"HNS":-0.0034971415,"Max HNS":-0.0034971415},{"game":"ChopperCommand","seed":21.0,"HNS":0.0305619754,"Max HNS":0.0451587398},{"game":"ChopperCommand","seed":22.0,"HNS":-0.0313222236,"Max HNS":-0.0313222236},{"game":"ChopperCommand","seed":23.0,"HNS":-0.0211348984,"Max HNS":-0.0130762681},{"game":"ChopperCommand","seed":24.0,"HNS":-0.0469833354,"Max HNS":-0.0463751368},{"game":"ChopperCommand","seed":25.0,"HNS":0.0214389977,"Max HNS":0.0311701739},{"game":"ChopperCommand","seed":26.0,"HNS":-0.048351782,"Max HNS":-0.0478956331},{"game":"ChopperCommand","seed":27.0,"HNS":-0.0486558813,"Max HNS":-0.0477435835},{"game":"ChopperCommand","seed":28.0,"HNS":0.0316263228,"Max HNS":0.0346673154},{"game":"ChopperCommand","seed":29.0,"HNS":-0.0260004866,"Max HNS":0.0182459555},{"game":"ChopperCommand","seed":30.0,"HNS":-0.0076024814,"Max HNS":0.0010643474},{"game":"ChopperCommand","seed":31.0,"HNS":0.0518489235,"Max HNS":0.0641649434},{"game":"ChopperCommand","seed":32.0,"HNS":0.0576268094,"Max HNS":0.0583870575},{"game":"ChopperCommand","seed":33.0,"HNS":-0.0988322588,"Max HNS":-0.0781535093},{"game":"ChopperCommand","seed":34.0,"HNS":0.1097798321,"Max HNS":0.1097798321},{"game":"ChopperCommand","seed":35.0,"HNS":-0.0497202287,"Max HNS":-0.0475915339},{"game":"ChopperCommand","seed":36.0,"HNS":-0.0576268094,"Max HNS":-0.027064834},{"game":"ChopperCommand","seed":37.0,"HNS":0.0532173701,"Max HNS":0.0599075538},{"game":"ChopperCommand","seed":38.0,"HNS":0.0491120302,"Max HNS":0.0509366257},{"game":"ChopperCommand","seed":39.0,"HNS":-0.070094879,"Max HNS":-0.070094879},{"game":"ChopperCommand","seed":40.0,"HNS":-0.0171816081,"Max HNS":0.0063860844},{"game":"ChopperCommand","seed":41.0,"HNS":-0.0948789685,"Max HNS":-0.091381827},{"game":"ChopperCommand","seed":42.0,"HNS":-0.0527612213,"Max HNS":-0.0527612213},{"game":"ChopperCommand","seed":43.0,"HNS":-0.0272168836,"Max HNS":-0.0144447148},{"game":"ChopperCommand","seed":44.0,"HNS":-0.0174857073,"Max HNS":0.002888943},{"game":"ChopperCommand","seed":45.0,"HNS":0.1744009245,"Max HNS":0.4778919839},{"game":"ChopperCommand","seed":46.0,"HNS":0.029497628,"Max HNS":0.043030045},{"game":"ChopperCommand","seed":47.0,"HNS":0.0561063131,"Max HNS":0.0586911568},{"game":"ChopperCommand","seed":48.0,"HNS":-0.0518489235,"Max HNS":0.0150529133},{"game":"ChopperCommand","seed":49.0,"HNS":-0.0149008636,"Max HNS":0.0100352755},{"game":"ChopperCommand","seed":50.0,"HNS":-0.0098832259,"Max HNS":-0.0098832259},{"game":"ChopperCommand","seed":51.0,"HNS":0.0091229777,"Max HNS":0.020070551},{"game":"ChopperCommand","seed":52.0,"HNS":-0.091381827,"Max HNS":-0.0609719012},{"game":"ChopperCommand","seed":53.0,"HNS":-0.0319304221,"Max HNS":-0.0261525362},{"game":"ChopperCommand","seed":54.0,"HNS":-0.0299537769,"Max HNS":-0.029497628},{"game":"ChopperCommand","seed":55.0,"HNS":-0.0530653205,"Max HNS":0.0562583627},{"game":"ChopperCommand","seed":56.0,"HNS":-0.0468312857,"Max HNS":-0.0462230872},{"game":"ChopperCommand","seed":57.0,"HNS":0.0985281596,"Max HNS":0.0985281596},{"game":"ChopperCommand","seed":58.0,"HNS":-0.080738353,"Max HNS":-0.0804342537},{"game":"ChopperCommand","seed":59.0,"HNS":-0.0468312857,"Max HNS":-0.0466792361},{"game":"ChopperCommand","seed":60.0,"HNS":-0.0466792361,"Max HNS":-0.0462230872},{"game":"ChopperCommand","seed":61.0,"HNS":-0.0114037222,"Max HNS":-0.0095791266},{"game":"ChopperCommand","seed":62.0,"HNS":-0.0447025909,"Max HNS":0.0308660747},{"game":"ChopperCommand","seed":63.0,"HNS":0.079521956,"Max HNS":0.1122126262},{"game":"ChopperCommand","seed":64.0,"HNS":0.0711592264,"Max HNS":0.0711592264},{"game":"ChopperCommand","seed":65.0,"HNS":-0.0168775088,"Max HNS":0.0123160199},{"game":"ChopperCommand","seed":66.0,"HNS":0.0959433159,"Max HNS":0.1220958521},{"game":"ChopperCommand","seed":67.0,"HNS":-0.0481997324,"Max HNS":-0.0480476828},{"game":"ChopperCommand","seed":68.0,"HNS":-0.0722235738,"Max HNS":0.0541296679},{"game":"ChopperCommand","seed":69.0,"HNS":-0.0947269189,"Max HNS":-0.0682702834},{"game":"ChopperCommand","seed":70.0,"HNS":-0.0104914244,"Max HNS":0.0258484369},{"game":"ChopperCommand","seed":71.0,"HNS":-0.0033450918,"Max HNS":0.0897092811},{"game":"ChopperCommand","seed":72.0,"HNS":-0.0827149982,"Max HNS":-0.0463751368},{"game":"ChopperCommand","seed":73.0,"HNS":0.0208307992,"Max HNS":0.0554981146},{"game":"ChopperCommand","seed":74.0,"HNS":-0.0114037222,"Max HNS":0.0044094392},{"game":"ChopperCommand","seed":75.0,"HNS":-0.0463751368,"Max HNS":-0.0462230872},{"game":"ChopperCommand","seed":76.0,"HNS":0.0434861939,"Max HNS":0.0626444471},{"game":"ChopperCommand","seed":77.0,"HNS":-0.028281231,"Max HNS":0.0179418562},{"game":"ChopperCommand","seed":78.0,"HNS":0.0550419657,"Max HNS":0.0599075538},{"game":"ChopperCommand","seed":79.0,"HNS":-0.0676620849,"Max HNS":-0.0448546406},{"game":"ChopperCommand","seed":80.0,"HNS":0.0780014597,"Max HNS":0.0871244374},{"game":"ChopperCommand","seed":81.0,"HNS":-0.006538134,"Max HNS":0.0424218465},{"game":"ChopperCommand","seed":82.0,"HNS":-0.0050176378,"Max HNS":0.0293455784},{"game":"ChopperCommand","seed":83.0,"HNS":0.0092750274,"Max HNS":0.0092750274},{"game":"ChopperCommand","seed":84.0,"HNS":0.0092750274,"Max HNS":0.0997445566},{"game":"ChopperCommand","seed":85.0,"HNS":-0.0551940153,"Max HNS":0.0127721688},{"game":"ChopperCommand","seed":86.0,"HNS":-0.0627964968,"Max HNS":-0.0580829583},{"game":"ChopperCommand","seed":87.0,"HNS":-0.1084113855,"Max HNS":-0.056562462},{"game":"ChopperCommand","seed":88.0,"HNS":-0.0139885659,"Max HNS":0.0010643474},{"game":"ChopperCommand","seed":89.0,"HNS":-0.0710071767,"Max HNS":-0.0503284272},{"game":"ChopperCommand","seed":90.0,"HNS":-0.0485038317,"Max HNS":-0.0477435835},{"game":"ChopperCommand","seed":91.0,"HNS":0.056562462,"Max HNS":0.0588432064},{"game":"ChopperCommand","seed":92.0,"HNS":0.0234156429,"Max HNS":0.0564104124},{"game":"ChopperCommand","seed":93.0,"HNS":0.0520009731,"Max HNS":0.0693346308},{"game":"ChopperCommand","seed":94.0,"HNS":-0.0170295584,"Max HNS":0.0596034546},{"game":"ChopperCommand","seed":95.0,"HNS":0.0329947695,"Max HNS":0.0329947695},{"game":"ChopperCommand","seed":96.0,"HNS":0.0249361392,"Max HNS":0.0481997324},{"game":"ChopperCommand","seed":97.0,"HNS":-0.0478956331,"Max HNS":-0.0478956331},{"game":"ChopperCommand","seed":98.0,"HNS":-0.0474394842,"Max HNS":-0.0466792361},{"game":"ChopperCommand","seed":99.0,"HNS":0.0507845761,"Max HNS":0.1687750882},{"game":"ChopperCommand","seed":100.0,"HNS":-0.0299537769,"Max HNS":-0.0215910473},{"game":"CrazyClimber","seed":1.0,"HNS":-0.294643677,"Max HNS":-0.294643677},{"game":"CrazyClimber","seed":2.0,"HNS":-0.1549569043,"Max HNS":-0.1549569043},{"game":"CrazyClimber","seed":3.0,"HNS":-0.0883272319,"Max HNS":-0.0883272319},{"game":"CrazyClimber","seed":4.0,"HNS":-0.2799524131,"Max HNS":-0.2799524131},{"game":"CrazyClimber","seed":5.0,"HNS":-0.0589846261,"Max HNS":-0.0589846261},{"game":"CrazyClimber","seed":6.0,"HNS":-0.1407846253,"Max HNS":-0.1407846253},{"game":"CrazyClimber","seed":7.0,"HNS":-0.2716087333,"Max HNS":-0.2716087333},{"game":"CrazyClimber","seed":8.0,"HNS":0.022855295,"Max HNS":0.022855295},{"game":"CrazyClimber","seed":9.0,"HNS":0.062697364,"Max HNS":0.062697364},{"game":"CrazyClimber","seed":10.0,"HNS":-0.1704466064,"Max HNS":-0.1704466064},{"game":"CrazyClimber","seed":11.0,"HNS":-0.1650571482,"Max HNS":0.0280451437},{"game":"CrazyClimber","seed":12.0,"HNS":-0.2562787188,"Max HNS":-0.2562787188},{"game":"CrazyClimber","seed":13.0,"HNS":0.2650216177,"Max HNS":0.2650216177},{"game":"CrazyClimber","seed":14.0,"HNS":0.0729572955,"Max HNS":0.0729572955},{"game":"CrazyClimber","seed":15.0,"HNS":-0.0125155196,"Max HNS":-0.0125155196},{"game":"CrazyClimber","seed":16.0,"HNS":-0.0343528059,"Max HNS":-0.0225359197},{"game":"CrazyClimber","seed":17.0,"HNS":-0.294643677,"Max HNS":-0.2885356243},{"game":"CrazyClimber","seed":18.0,"HNS":0.0006187896,"Max HNS":0.0006187896},{"game":"CrazyClimber","seed":19.0,"HNS":-0.2565182503,"Max HNS":-0.231487211},{"game":"CrazyClimber","seed":20.0,"HNS":0.0115573937,"Max HNS":0.0115573937},{"game":"CrazyClimber","seed":21.0,"HNS":0.2700517787,"Max HNS":0.2700517787},{"game":"CrazyClimber","seed":22.0,"HNS":0.3274994112,"Max HNS":0.3274994112},{"game":"CrazyClimber","seed":23.0,"HNS":-0.2192311838,"Max HNS":-0.2192311838},{"game":"CrazyClimber","seed":24.0,"HNS":-0.2790342091,"Max HNS":-0.229650803},{"game":"CrazyClimber","seed":25.0,"HNS":-0.2576759858,"Max HNS":-0.2576759858},{"game":"CrazyClimber","seed":26.0,"HNS":0.1507651035,"Max HNS":0.1507651035},{"game":"CrazyClimber","seed":27.0,"HNS":0.2355193242,"Max HNS":0.2355193242},{"game":"CrazyClimber","seed":28.0,"HNS":-0.2934859415,"Max HNS":-0.2934859415},{"game":"CrazyClimber","seed":29.0,"HNS":-0.0662504142,"Max HNS":-0.0662504142},{"game":"CrazyClimber","seed":30.0,"HNS":-0.1149950696,"Max HNS":-0.1149950696},{"game":"CrazyClimber","seed":31.0,"HNS":0.0127151292,"Max HNS":0.0127151292},{"game":"CrazyClimber","seed":32.0,"HNS":0.1680912136,"Max HNS":0.1680912136},{"game":"CrazyClimber","seed":33.0,"HNS":0.1781914575,"Max HNS":0.1781914575},{"game":"CrazyClimber","seed":34.0,"HNS":-0.2469769132,"Max HNS":-0.2469769132},{"game":"CrazyClimber","seed":35.0,"HNS":-0.1631408964,"Max HNS":-0.1631408964},{"game":"CrazyClimber","seed":36.0,"HNS":-0.3866237639,"Max HNS":-0.3866237639},{"game":"CrazyClimber","seed":37.0,"HNS":-0.1453357233,"Max HNS":-0.1453357233},{"game":"CrazyClimber","seed":38.0,"HNS":-0.255719812,"Max HNS":-0.1977132728},{"game":"CrazyClimber","seed":39.0,"HNS":-0.0570284523,"Max HNS":-0.0570284523},{"game":"CrazyClimber","seed":40.0,"HNS":0.0116771595,"Max HNS":0.0116771595},{"game":"CrazyClimber","seed":41.0,"HNS":-0.0735960461,"Max HNS":0.0051299658},{"game":"CrazyClimber","seed":42.0,"HNS":0.4845522159,"Max HNS":0.4845522159},{"game":"CrazyClimber","seed":43.0,"HNS":-0.2054980458,"Max HNS":-0.2054980458},{"game":"CrazyClimber","seed":44.0,"HNS":0.0547928252,"Max HNS":0.1455752548},{"game":"CrazyClimber","seed":45.0,"HNS":0.3726111725,"Max HNS":0.4087405036},{"game":"CrazyClimber","seed":46.0,"HNS":0.0079644216,"Max HNS":0.0079644216},{"game":"CrazyClimber","seed":47.0,"HNS":0.0406205462,"Max HNS":0.0406205462},{"game":"CrazyClimber","seed":48.0,"HNS":0.3118500214,"Max HNS":0.3118500214},{"game":"CrazyClimber","seed":49.0,"HNS":0.0367880426,"Max HNS":0.0502417272},{"game":"CrazyClimber","seed":50.0,"HNS":-0.2762396752,"Max HNS":-0.2762396752},{"game":"CrazyClimber","seed":51.0,"HNS":-0.1703268407,"Max HNS":-0.1703268407},{"game":"CrazyClimber","seed":52.0,"HNS":0.0646136158,"Max HNS":0.0646136158},{"game":"CrazyClimber","seed":53.0,"HNS":-0.2169955567,"Max HNS":-0.1898885779},{"game":"CrazyClimber","seed":54.0,"HNS":-0.1304847718,"Max HNS":-0.1304847718},{"game":"CrazyClimber","seed":55.0,"HNS":-0.2153188364,"Max HNS":-0.2079332825},{"game":"CrazyClimber","seed":56.0,"HNS":-0.0391035135,"Max HNS":-0.0391035135},{"game":"CrazyClimber","seed":57.0,"HNS":0.0189429476,"Max HNS":0.0189429476},{"game":"CrazyClimber","seed":58.0,"HNS":0.0244122496,"Max HNS":0.1927629557},{"game":"CrazyClimber","seed":59.0,"HNS":0.0088826256,"Max HNS":0.0088826256},{"game":"CrazyClimber","seed":60.0,"HNS":-0.1812654448,"Max HNS":-0.1812654448},{"game":"CrazyClimber","seed":61.0,"HNS":-0.1133183493,"Max HNS":-0.1133183493},{"game":"CrazyClimber","seed":62.0,"HNS":-0.0005389458,"Max HNS":-0.0005389458},{"game":"CrazyClimber","seed":63.0,"HNS":-0.167452463,"Max HNS":-0.167452463},{"game":"CrazyClimber","seed":64.0,"HNS":0.3691778881,"Max HNS":0.3691778881},{"game":"CrazyClimber","seed":65.0,"HNS":0.0136732551,"Max HNS":0.0136732551},{"game":"CrazyClimber","seed":66.0,"HNS":-0.2164366499,"Max HNS":-0.2164366499},{"game":"CrazyClimber","seed":67.0,"HNS":-0.2943243017,"Max HNS":-0.1972741318},{"game":"CrazyClimber","seed":68.0,"HNS":-0.2372359664,"Max HNS":-0.2372359664},{"game":"CrazyClimber","seed":69.0,"HNS":0.2123646148,"Max HNS":0.2123646148},{"game":"CrazyClimber","seed":70.0,"HNS":0.0133139579,"Max HNS":0.0133139579},{"game":"CrazyClimber","seed":71.0,"HNS":-0.177073644,"Max HNS":-0.1769938001},{"game":"CrazyClimber","seed":72.0,"HNS":0.0591443137,"Max HNS":0.0591443137},{"game":"CrazyClimber","seed":73.0,"HNS":-0.1684904327,"Max HNS":-0.0475270371},{"game":"CrazyClimber","seed":74.0,"HNS":-0.0292428011,"Max HNS":0.0084035626},{"game":"CrazyClimber","seed":75.0,"HNS":-0.294643677,"Max HNS":-0.294643677},{"game":"CrazyClimber","seed":76.0,"HNS":-0.1161128832,"Max HNS":0.1028987301},{"game":"CrazyClimber","seed":77.0,"HNS":0.0053295754,"Max HNS":0.0053295754},{"game":"CrazyClimber","seed":78.0,"HNS":-0.1472519751,"Max HNS":-0.1472519751},{"game":"CrazyClimber","seed":79.0,"HNS":-0.2389126868,"Max HNS":-0.2389126868},{"game":"CrazyClimber","seed":80.0,"HNS":0.0088826256,"Max HNS":0.0088826256},{"game":"CrazyClimber","seed":81.0,"HNS":0.0928384081,"Max HNS":0.1308839909},{"game":"CrazyClimber","seed":82.0,"HNS":-0.0637752556,"Max HNS":-0.0637752556},{"game":"CrazyClimber","seed":83.0,"HNS":-0.2749222521,"Max HNS":-0.2715288895},{"game":"CrazyClimber","seed":84.0,"HNS":0.355484672,"Max HNS":0.355484672},{"game":"CrazyClimber","seed":85.0,"HNS":-0.1604262063,"Max HNS":-0.1604262063},{"game":"CrazyClimber","seed":86.0,"HNS":-0.2008671039,"Max HNS":-0.2007872601},{"game":"CrazyClimber","seed":87.0,"HNS":0.2425455808,"Max HNS":0.2425455808},{"game":"CrazyClimber","seed":88.0,"HNS":-0.266259197,"Max HNS":-0.266259197},{"game":"CrazyClimber","seed":89.0,"HNS":-0.2369165911,"Max HNS":-0.1918048298},{"game":"CrazyClimber","seed":90.0,"HNS":-0.2322457274,"Max HNS":-0.2322457274},{"game":"CrazyClimber","seed":91.0,"HNS":-0.04988243,"Max HNS":-0.04988243},{"game":"CrazyClimber","seed":92.0,"HNS":-0.0969503651,"Max HNS":-0.0969503651},{"game":"CrazyClimber","seed":93.0,"HNS":-0.2946037551,"Max HNS":-0.2925278156},{"game":"CrazyClimber","seed":94.0,"HNS":-0.0729572955,"Max HNS":-0.0729572955},{"game":"CrazyClimber","seed":95.0,"HNS":-0.224181501,"Max HNS":-0.224181501},{"game":"CrazyClimber","seed":96.0,"HNS":0.0044113714,"Max HNS":0.0044113714},{"game":"CrazyClimber","seed":97.0,"HNS":-0.2640634918,"Max HNS":-0.2308883823},{"game":"CrazyClimber","seed":98.0,"HNS":0.34083333,"Max HNS":0.34083333},{"game":"CrazyClimber","seed":99.0,"HNS":-0.1926431899,"Max HNS":0.0185437285},{"game":"CrazyClimber","seed":100.0,"HNS":-0.1209833566,"Max HNS":-0.1209833566},{"game":"DemonAttack","seed":1.0,"HNS":0.3046621584,"Max HNS":0.4613777558},{"game":"DemonAttack","seed":2.0,"HNS":0.2013029853,"Max HNS":0.404805102},{"game":"DemonAttack","seed":3.0,"HNS":0.2235691902,"Max HNS":0.317362142},{"game":"DemonAttack","seed":4.0,"HNS":0.2151575128,"Max HNS":0.2151575128},{"game":"DemonAttack","seed":5.0,"HNS":0.1442905053,"Max HNS":0.4481554786},{"game":"DemonAttack","seed":6.0,"HNS":0.1249381494,"Max HNS":0.1249381494},{"game":"DemonAttack","seed":7.0,"HNS":0.1693056243,"Max HNS":0.1693056243},{"game":"DemonAttack","seed":8.0,"HNS":0.1797789873,"Max HNS":0.4289955468},{"game":"DemonAttack","seed":9.0,"HNS":0.1691132003,"Max HNS":0.4377920721},{"game":"DemonAttack","seed":10.0,"HNS":0.3085106383,"Max HNS":0.4884820496},{"game":"DemonAttack","seed":11.0,"HNS":0.3273956787,"Max HNS":0.3522183737},{"game":"DemonAttack","seed":12.0,"HNS":0.5862059487,"Max HNS":0.5862059487},{"game":"DemonAttack","seed":13.0,"HNS":0.2645829897,"Max HNS":0.3954587938},{"game":"DemonAttack","seed":14.0,"HNS":0.2800043983,"Max HNS":0.4059046677},{"game":"DemonAttack","seed":15.0,"HNS":0.3125790313,"Max HNS":0.3125790313},{"game":"DemonAttack","seed":16.0,"HNS":0.2964978833,"Max HNS":0.3230249052},{"game":"DemonAttack","seed":17.0,"HNS":0.3473802848,"Max HNS":0.3744296003},{"game":"DemonAttack","seed":18.0,"HNS":0.268129089,"Max HNS":0.4578866348},{"game":"DemonAttack","seed":19.0,"HNS":0.4292704382,"Max HNS":0.4292704382},{"game":"DemonAttack","seed":20.0,"HNS":0.3216229589,"Max HNS":0.3216229589},{"game":"DemonAttack","seed":21.0,"HNS":0.3438066964,"Max HNS":0.4985430755},{"game":"DemonAttack","seed":22.0,"HNS":0.4103853978,"Max HNS":0.4103853978},{"game":"DemonAttack","seed":23.0,"HNS":0.0987959756,"Max HNS":0.2924569795},{"game":"DemonAttack","seed":24.0,"HNS":0.1833525757,"Max HNS":0.4609929078},{"game":"DemonAttack","seed":25.0,"HNS":0.2444334488,"Max HNS":0.2886909671},{"game":"DemonAttack","seed":26.0,"HNS":0.2047116389,"Max HNS":0.2760184727},{"game":"DemonAttack","seed":27.0,"HNS":0.1451151795,"Max HNS":0.2861069877},{"game":"DemonAttack","seed":28.0,"HNS":0.3550497553,"Max HNS":0.3550497553},{"game":"DemonAttack","seed":29.0,"HNS":0.2905877179,"Max HNS":0.3944416955},{"game":"DemonAttack","seed":30.0,"HNS":0.3224476332,"Max HNS":0.3224476332},{"game":"DemonAttack","seed":31.0,"HNS":0.1375006872,"Max HNS":0.4372422893},{"game":"DemonAttack","seed":32.0,"HNS":0.2253559844,"Max HNS":0.3873220078},{"game":"DemonAttack","seed":33.0,"HNS":0.2901478916,"Max HNS":0.3619495299},{"game":"DemonAttack","seed":34.0,"HNS":0.2848424872,"Max HNS":0.2848424872},{"game":"DemonAttack","seed":35.0,"HNS":0.0809005443,"Max HNS":0.2488317115},{"game":"DemonAttack","seed":36.0,"HNS":0.0900819176,"Max HNS":0.3802572984},{"game":"DemonAttack","seed":37.0,"HNS":0.3253889714,"Max HNS":0.3253889714},{"game":"DemonAttack","seed":38.0,"HNS":0.4854857331,"Max HNS":0.4854857331},{"game":"DemonAttack","seed":39.0,"HNS":0.317362142,"Max HNS":0.317362142},{"game":"DemonAttack","seed":40.0,"HNS":0.0581120457,"Max HNS":0.1272747265},{"game":"DemonAttack","seed":41.0,"HNS":0.2286821705,"Max HNS":0.4057397328},{"game":"DemonAttack","seed":42.0,"HNS":0.103716532,"Max HNS":0.2021551487},{"game":"DemonAttack","seed":43.0,"HNS":0.4352355819,"Max HNS":0.4352355819},{"game":"DemonAttack","seed":44.0,"HNS":0.165182253,"Max HNS":0.4103029303},{"game":"DemonAttack","seed":45.0,"HNS":0.1966023421,"Max HNS":0.4159931827},{"game":"DemonAttack","seed":46.0,"HNS":0.1059981307,"Max HNS":0.2977348947},{"game":"DemonAttack","seed":47.0,"HNS":0.4483479026,"Max HNS":0.4483479026},{"game":"DemonAttack","seed":48.0,"HNS":0.248941668,"Max HNS":0.248941668},{"game":"DemonAttack","seed":49.0,"HNS":0.3635713893,"Max HNS":0.3764363077},{"game":"DemonAttack","seed":50.0,"HNS":0.3047446259,"Max HNS":0.3047446259},{"game":"DemonAttack","seed":51.0,"HNS":0.3626092693,"Max HNS":0.3858925724},{"game":"DemonAttack","seed":52.0,"HNS":0.4576667216,"Max HNS":0.4576667216},{"game":"DemonAttack","seed":53.0,"HNS":0.323959536,"Max HNS":0.323959536},{"game":"DemonAttack","seed":54.0,"HNS":0.5164934851,"Max HNS":0.5164934851},{"game":"DemonAttack","seed":55.0,"HNS":0.2949859805,"Max HNS":0.3746220243},{"game":"DemonAttack","seed":56.0,"HNS":0.2318434218,"Max HNS":0.712491066},{"game":"DemonAttack","seed":57.0,"HNS":0.4182472923,"Max HNS":0.4182472923},{"game":"DemonAttack","seed":58.0,"HNS":0.232063335,"Max HNS":0.2421243609},{"game":"DemonAttack","seed":59.0,"HNS":0.2547968552,"Max HNS":0.4901313981},{"game":"DemonAttack","seed":60.0,"HNS":0.3153004563,"Max HNS":0.3620045082},{"game":"DemonAttack","seed":61.0,"HNS":0.1159217109,"Max HNS":0.3888613997},{"game":"DemonAttack","seed":62.0,"HNS":0.3842982022,"Max HNS":0.3842982022},{"game":"DemonAttack","seed":63.0,"HNS":0.3298971906,"Max HNS":0.351888504},{"game":"DemonAttack","seed":64.0,"HNS":0.2313211282,"Max HNS":0.4356479191},{"game":"DemonAttack","seed":65.0,"HNS":0.3109571719,"Max HNS":0.3827862994},{"game":"DemonAttack","seed":66.0,"HNS":0.2383858376,"Max HNS":0.2383858376},{"game":"DemonAttack","seed":67.0,"HNS":0.2322832481,"Max HNS":0.3481224916},{"game":"DemonAttack","seed":68.0,"HNS":0.5223211831,"Max HNS":0.5223211831},{"game":"DemonAttack","seed":69.0,"HNS":0.49903788,"Max HNS":0.49903788},{"game":"DemonAttack","seed":70.0,"HNS":0.268513937,"Max HNS":0.4022761009},{"game":"DemonAttack","seed":71.0,"HNS":0.1166364286,"Max HNS":0.1166364286},{"game":"DemonAttack","seed":72.0,"HNS":0.0936005278,"Max HNS":0.4517565562},{"game":"DemonAttack","seed":73.0,"HNS":0.2993567541,"Max HNS":0.2993567541},{"game":"DemonAttack","seed":74.0,"HNS":0.3855352136,"Max HNS":0.3855352136},{"game":"DemonAttack","seed":75.0,"HNS":0.0733960086,"Max HNS":0.0733960086},{"game":"DemonAttack","seed":76.0,"HNS":0.2363516411,"Max HNS":0.3558744296},{"game":"DemonAttack","seed":77.0,"HNS":0.2301940733,"Max HNS":0.2760184727},{"game":"DemonAttack","seed":78.0,"HNS":0.2496563857,"Max HNS":0.3137610644},{"game":"DemonAttack","seed":79.0,"HNS":0.4479080763,"Max HNS":0.4555775469},{"game":"DemonAttack","seed":80.0,"HNS":0.0837044367,"Max HNS":0.16848095},{"game":"DemonAttack","seed":81.0,"HNS":0.2446258728,"Max HNS":0.3517510583},{"game":"DemonAttack","seed":82.0,"HNS":0.242344274,"Max HNS":0.2847325307},{"game":"DemonAttack","seed":83.0,"HNS":0.1041838474,"Max HNS":0.3446863489},{"game":"DemonAttack","seed":84.0,"HNS":0.3402331079,"Max HNS":0.3666501732},{"game":"DemonAttack","seed":85.0,"HNS":0.2534223982,"Max HNS":0.4810874704},{"game":"DemonAttack","seed":86.0,"HNS":0.3924624773,"Max HNS":0.4826818407},{"game":"DemonAttack","seed":87.0,"HNS":0.4673978778,"Max HNS":0.5103634065},{"game":"DemonAttack","seed":88.0,"HNS":0.2958106548,"Max HNS":0.3325911265},{"game":"DemonAttack","seed":89.0,"HNS":0.3856451702,"Max HNS":0.3856451702},{"game":"DemonAttack","seed":90.0,"HNS":0.048848205,"Max HNS":0.1439331464},{"game":"DemonAttack","seed":91.0,"HNS":0.1779097257,"Max HNS":0.2089174776},{"game":"DemonAttack","seed":92.0,"HNS":0.3695090439,"Max HNS":0.414316345},{"game":"DemonAttack","seed":93.0,"HNS":0.2124085986,"Max HNS":0.3622793996},{"game":"DemonAttack","seed":94.0,"HNS":0.3009786134,"Max HNS":0.3301171037},{"game":"DemonAttack","seed":95.0,"HNS":0.228957062,"Max HNS":0.3583484524},{"game":"DemonAttack","seed":96.0,"HNS":0.3885590192,"Max HNS":0.3897135631},{"game":"DemonAttack","seed":97.0,"HNS":0.2274451592,"Max HNS":0.237698609},{"game":"DemonAttack","seed":98.0,"HNS":0.2031997361,"Max HNS":0.242536698},{"game":"DemonAttack","seed":99.0,"HNS":0.3112870416,"Max HNS":0.4648138985},{"game":"DemonAttack","seed":100.0,"HNS":0.2577931717,"Max HNS":0.4264390566},{"game":"Freeway","seed":1.0,"HNS":0.9486486486,"Max HNS":0.9527027027},{"game":"Freeway","seed":2.0,"HNS":0.9841216216,"Max HNS":0.9841216216},{"game":"Freeway","seed":3.0,"HNS":0.9466216216,"Max HNS":0.9466216216},{"game":"Freeway","seed":4.0,"HNS":0.9983108108,"Max HNS":0.9983108108},{"game":"Freeway","seed":5.0,"HNS":1.0118243243,"Max HNS":1.0118243243},{"game":"Freeway","seed":6.0,"HNS":0.9699324324,"Max HNS":0.9699324324},{"game":"Freeway","seed":7.0,"HNS":0.9108108108,"Max HNS":0.9108108108},{"game":"Freeway","seed":8.0,"HNS":1.0162162162,"Max HNS":1.0162162162},{"game":"Freeway","seed":9.0,"HNS":0.9800675676,"Max HNS":0.9800675676},{"game":"Freeway","seed":10.0,"HNS":0.9922297297,"Max HNS":1.0297297297},{"game":"Freeway","seed":11.0,"HNS":0.9827702703,"Max HNS":0.9827702703},{"game":"Freeway","seed":12.0,"HNS":0.8527027027,"Max HNS":0.8527027027},{"game":"Freeway","seed":13.0,"HNS":0.9162162162,"Max HNS":0.9162162162},{"game":"Freeway","seed":14.0,"HNS":1.0081081081,"Max HNS":1.0081081081},{"game":"Freeway","seed":15.0,"HNS":1.0567567568,"Max HNS":1.0567567568},{"game":"Freeway","seed":16.0,"HNS":0.9766891892,"Max HNS":0.9962837838},{"game":"Freeway","seed":17.0,"HNS":0.9891891892,"Max HNS":0.9891891892},{"game":"Freeway","seed":18.0,"HNS":1.0182432432,"Max HNS":1.0182432432},{"game":"Freeway","seed":19.0,"HNS":0.9280405405,"Max HNS":0.9280405405},{"game":"Freeway","seed":20.0,"HNS":1.0013513514,"Max HNS":1.0013513514},{"game":"Freeway","seed":21.0,"HNS":0.9378378378,"Max HNS":0.9560810811},{"game":"Freeway","seed":22.0,"HNS":0.9766891892,"Max HNS":0.9766891892},{"game":"Freeway","seed":23.0,"HNS":0.9344594595,"Max HNS":0.9506756757},{"game":"Freeway","seed":24.0,"HNS":0.9358108108,"Max HNS":0.9358108108},{"game":"Freeway","seed":25.0,"HNS":0.9533783784,"Max HNS":0.9638513514},{"game":"Freeway","seed":26.0,"HNS":0.9787162162,"Max HNS":0.9844594595},{"game":"Freeway","seed":27.0,"HNS":0.9591216216,"Max HNS":0.9844594595},{"game":"Freeway","seed":28.0,"HNS":0.9402027027,"Max HNS":0.9402027027},{"game":"Freeway","seed":29.0,"HNS":0.9246621622,"Max HNS":0.9337837838},{"game":"Freeway","seed":30.0,"HNS":0.8868243243,"Max HNS":0.8888513514},{"game":"Freeway","seed":31.0,"HNS":0.952027027,"Max HNS":0.952027027},{"game":"Freeway","seed":32.0,"HNS":0.972972973,"Max HNS":0.972972973},{"game":"Freeway","seed":33.0,"HNS":0.9584459459,"Max HNS":0.9679054054},{"game":"Freeway","seed":34.0,"HNS":1.0087837838,"Max HNS":1.0087837838},{"game":"Freeway","seed":35.0,"HNS":0.9361486486,"Max HNS":0.9361486486},{"game":"Freeway","seed":36.0,"HNS":0.9496621622,"Max HNS":0.9496621622},{"game":"Freeway","seed":37.0,"HNS":0.9891891892,"Max HNS":0.9891891892},{"game":"Freeway","seed":38.0,"HNS":0.9148648649,"Max HNS":0.9148648649},{"game":"Freeway","seed":39.0,"HNS":0.9452702703,"Max HNS":0.9452702703},{"game":"Freeway","seed":40.0,"HNS":0.9097972973,"Max HNS":0.9199324324},{"game":"Freeway","seed":41.0,"HNS":0.9847972973,"Max HNS":0.9878378378},{"game":"Freeway","seed":42.0,"HNS":0.9439189189,"Max HNS":0.9439189189},{"game":"Freeway","seed":43.0,"HNS":0.9125,"Max HNS":0.9456081081},{"game":"Freeway","seed":44.0,"HNS":0.9260135135,"Max HNS":0.9277027027},{"game":"Freeway","seed":45.0,"HNS":1.0570945946,"Max HNS":1.0570945946},{"game":"Freeway","seed":46.0,"HNS":0.95,"Max HNS":0.9506756757},{"game":"Freeway","seed":47.0,"HNS":0.9777027027,"Max HNS":0.9777027027},{"game":"Freeway","seed":48.0,"HNS":0.9780405405,"Max HNS":0.9780405405},{"game":"Freeway","seed":49.0,"HNS":0.9847972973,"Max HNS":0.9847972973},{"game":"Freeway","seed":50.0,"HNS":0.8621621622,"Max HNS":0.9297297297},{"game":"Freeway","seed":51.0,"HNS":0.9706081081,"Max HNS":0.9831081081},{"game":"Freeway","seed":52.0,"HNS":0.7851351351,"Max HNS":0.7922297297},{"game":"Freeway","seed":53.0,"HNS":0.9486486486,"Max HNS":0.9486486486},{"game":"Freeway","seed":54.0,"HNS":0.95,"Max HNS":0.95},{"game":"Freeway","seed":55.0,"HNS":0.9929054054,"Max HNS":0.9929054054},{"game":"Freeway","seed":56.0,"HNS":0.9912162162,"Max HNS":0.9912162162},{"game":"Freeway","seed":57.0,"HNS":0.9918918919,"Max HNS":0.9918918919},{"game":"Freeway","seed":58.0,"HNS":0.964527027,"Max HNS":1.0334459459},{"game":"Freeway","seed":59.0,"HNS":0.8912162162,"Max HNS":0.8912162162},{"game":"Freeway","seed":60.0,"HNS":0.8091216216,"Max HNS":0.8158783784},{"game":"Freeway","seed":61.0,"HNS":0.902027027,"Max HNS":0.9337837838},{"game":"Freeway","seed":62.0,"HNS":0.8625,"Max HNS":0.8716216216},{"game":"Freeway","seed":63.0,"HNS":0.9881756757,"Max HNS":0.9881756757},{"game":"Freeway","seed":64.0,"HNS":0.7949324324,"Max HNS":0.8192567568},{"game":"Freeway","seed":65.0,"HNS":0.9324324324,"Max HNS":0.9652027027},{"game":"Freeway","seed":66.0,"HNS":0.9922297297,"Max HNS":0.9922297297},{"game":"Freeway","seed":67.0,"HNS":0.9425675676,"Max HNS":0.9564189189},{"game":"Freeway","seed":68.0,"HNS":0.9341216216,"Max HNS":0.9341216216},{"game":"Freeway","seed":69.0,"HNS":0.9398648649,"Max HNS":0.9398648649},{"game":"Freeway","seed":70.0,"HNS":0.95,"Max HNS":0.95},{"game":"Freeway","seed":71.0,"HNS":0.9614864865,"Max HNS":0.9614864865},{"game":"Freeway","seed":72.0,"HNS":0.9611486486,"Max HNS":0.9611486486},{"game":"Freeway","seed":73.0,"HNS":1.0030405405,"Max HNS":1.0030405405},{"game":"Freeway","seed":74.0,"HNS":0.975,"Max HNS":1.0442567568},{"game":"Freeway","seed":75.0,"HNS":0.9658783784,"Max HNS":0.9695945946},{"game":"Freeway","seed":76.0,"HNS":0.9993243243,"Max HNS":0.9993243243},{"game":"Freeway","seed":77.0,"HNS":0.9084459459,"Max HNS":0.9817567568},{"game":"Freeway","seed":78.0,"HNS":0.9922297297,"Max HNS":0.9922297297},{"game":"Freeway","seed":79.0,"HNS":0.8932432432,"Max HNS":0.8932432432},{"game":"Freeway","seed":80.0,"HNS":0.9618243243,"Max HNS":0.9618243243},{"game":"Freeway","seed":81.0,"HNS":0.9777027027,"Max HNS":0.9777027027},{"game":"Freeway","seed":82.0,"HNS":0.9793918919,"Max HNS":0.9793918919},{"game":"Freeway","seed":83.0,"HNS":0.9195945946,"Max HNS":0.9195945946},{"game":"Freeway","seed":84.0,"HNS":1.0057432432,"Max HNS":1.0057432432},{"game":"Freeway","seed":85.0,"HNS":0.9807432432,"Max HNS":0.9807432432},{"game":"Freeway","seed":86.0,"HNS":0.9489864865,"Max HNS":0.9597972973},{"game":"Freeway","seed":87.0,"HNS":0.9489864865,"Max HNS":0.9489864865},{"game":"Freeway","seed":88.0,"HNS":0.9648648649,"Max HNS":0.9648648649},{"game":"Freeway","seed":89.0,"HNS":1.0013513514,"Max HNS":1.0013513514},{"game":"Freeway","seed":90.0,"HNS":1.0155405405,"Max HNS":1.0155405405},{"game":"Freeway","seed":91.0,"HNS":0.9476351351,"Max HNS":0.9939189189},{"game":"Freeway","seed":92.0,"HNS":0.9182432432,"Max HNS":0.9182432432},{"game":"Freeway","seed":93.0,"HNS":0.9945945946,"Max HNS":0.9945945946},{"game":"Freeway","seed":94.0,"HNS":0.9939189189,"Max HNS":0.9939189189},{"game":"Freeway","seed":95.0,"HNS":0.9662162162,"Max HNS":0.9662162162},{"game":"Freeway","seed":96.0,"HNS":1.0243243243,"Max HNS":1.0243243243},{"game":"Freeway","seed":97.0,"HNS":0.9452702703,"Max HNS":1.0118243243},{"game":"Freeway","seed":98.0,"HNS":0.8402027027,"Max HNS":0.8516891892},{"game":"Freeway","seed":99.0,"HNS":1.0074324324,"Max HNS":1.0074324324},{"game":"Freeway","seed":100.0,"HNS":0.9314189189,"Max HNS":0.9314189189},{"game":"Frostbite","seed":1.0,"HNS":0.0455088418,"Max HNS":0.0479681462},{"game":"Frostbite","seed":2.0,"HNS":0.6575243003,"Max HNS":0.9085841433},{"game":"Frostbite","seed":3.0,"HNS":0.041901862,"Max HNS":0.0523714721},{"game":"Frostbite","seed":4.0,"HNS":0.0378967092,"Max HNS":0.0378967092},{"game":"Frostbite","seed":5.0,"HNS":0.0447593395,"Max HNS":0.0483897412},{"game":"Frostbite","seed":6.0,"HNS":0.4534020377,"Max HNS":0.4635437405},{"game":"Frostbite","seed":7.0,"HNS":0.150649959,"Max HNS":0.150649959},{"game":"Frostbite","seed":8.0,"HNS":0.3699964867,"Max HNS":0.4093453566},{"game":"Frostbite","seed":9.0,"HNS":0.0438224616,"Max HNS":0.0442440567},{"game":"Frostbite","seed":10.0,"HNS":0.4576882539,"Max HNS":0.4576882539},{"game":"Frostbite","seed":11.0,"HNS":0.4306359058,"Max HNS":0.4306359058},{"game":"Frostbite","seed":12.0,"HNS":-0.0023656166,"Max HNS":0.1308115704},{"game":"Frostbite","seed":13.0,"HNS":0.6417847523,"Max HNS":0.6417847523},{"game":"Frostbite","seed":14.0,"HNS":0.7135730179,"Max HNS":0.7135730179},{"game":"Frostbite","seed":15.0,"HNS":0.3727602764,"Max HNS":0.6933833002},{"game":"Frostbite","seed":16.0,"HNS":0.0724206582,"Max HNS":0.0724206582},{"game":"Frostbite","seed":17.0,"HNS":0.4432837569,"Max HNS":0.4432837569},{"game":"Frostbite","seed":18.0,"HNS":0.396088535,"Max HNS":0.396088535},{"game":"Frostbite","seed":19.0,"HNS":0.3905609556,"Max HNS":0.4521372526},{"game":"Frostbite","seed":20.0,"HNS":0.7553109263,"Max HNS":0.7553109263},{"game":"Frostbite","seed":21.0,"HNS":0.0351094976,"Max HNS":0.0364913924},{"game":"Frostbite","seed":22.0,"HNS":0.2715774681,"Max HNS":0.3636725612},{"game":"Frostbite","seed":23.0,"HNS":0.2853495725,"Max HNS":0.5413514463},{"game":"Frostbite","seed":24.0,"HNS":0.0240543389,"Max HNS":0.034500527},{"game":"Frostbite","seed":25.0,"HNS":0.3233633915,"Max HNS":0.3233633915},{"game":"Frostbite","seed":26.0,"HNS":0.5232228598,"Max HNS":0.5232228598},{"game":"Frostbite","seed":27.0,"HNS":0.043236913,"Max HNS":0.0443611664},{"game":"Frostbite","seed":28.0,"HNS":0.0474294414,"Max HNS":0.0474294414},{"game":"Frostbite","seed":29.0,"HNS":0.0462349221,"Max HNS":0.0479447242},{"game":"Frostbite","seed":30.0,"HNS":0.5758988172,"Max HNS":0.5758988172},{"game":"Frostbite","seed":31.0,"HNS":0.1513057735,"Max HNS":0.1513057735},{"game":"Frostbite","seed":32.0,"HNS":0.0150368896,"Max HNS":0.0431900691},{"game":"Frostbite","seed":33.0,"HNS":0.1602295351,"Max HNS":0.1690596089},{"game":"Frostbite","seed":34.0,"HNS":0.6591169926,"Max HNS":0.6591169926},{"game":"Frostbite","seed":35.0,"HNS":0.0455556857,"Max HNS":0.0674552055},{"game":"Frostbite","seed":36.0,"HNS":0.2381777726,"Max HNS":0.4558613421},{"game":"Frostbite","seed":37.0,"HNS":0.0452277784,"Max HNS":0.0452277784},{"game":"Frostbite","seed":38.0,"HNS":0.415294531,"Max HNS":0.4486942265},{"game":"Frostbite","seed":39.0,"HNS":0.537978686,"Max HNS":0.537978686},{"game":"Frostbite","seed":40.0,"HNS":0.4162079869,"Max HNS":0.6511769528},{"game":"Frostbite","seed":41.0,"HNS":0.0475699731,"Max HNS":0.0475699731},{"game":"Frostbite","seed":42.0,"HNS":0.0419721279,"Max HNS":0.0419721279},{"game":"Frostbite","seed":43.0,"HNS":0.6286216185,"Max HNS":0.6286216185},{"game":"Frostbite","seed":44.0,"HNS":0.4320646446,"Max HNS":0.4320646446},{"game":"Frostbite","seed":45.0,"HNS":0.4687902565,"Max HNS":0.4687902565},{"game":"Frostbite","seed":46.0,"HNS":0.5708630987,"Max HNS":0.5918023188},{"game":"Frostbite","seed":47.0,"HNS":0.1329429676,"Max HNS":0.1487059375},{"game":"Frostbite","seed":48.0,"HNS":0.56594449,"Max HNS":0.627965804},{"game":"Frostbite","seed":49.0,"HNS":0.5474177304,"Max HNS":0.5474177304},{"game":"Frostbite","seed":50.0,"HNS":0.2133270875,"Max HNS":0.2730062068},{"game":"Frostbite","seed":51.0,"HNS":0.1787328727,"Max HNS":0.1787328727},{"game":"Frostbite","seed":52.0,"HNS":0.0311277667,"Max HNS":0.038857009},{"game":"Frostbite","seed":53.0,"HNS":0.352406605,"Max HNS":0.4473591755},{"game":"Frostbite","seed":54.0,"HNS":0.0478276145,"Max HNS":0.0478978803},{"game":"Frostbite","seed":55.0,"HNS":0.1340203771,"Max HNS":0.440613655},{"game":"Frostbite","seed":56.0,"HNS":0.4887457548,"Max HNS":0.6477573486},{"game":"Frostbite","seed":57.0,"HNS":0.0442909006,"Max HNS":0.046281766},{"game":"Frostbite","seed":58.0,"HNS":0.2178006792,"Max HNS":0.3001522427},{"game":"Frostbite","seed":59.0,"HNS":0.3924112894,"Max HNS":0.4773158449},{"game":"Frostbite","seed":60.0,"HNS":0.4638950697,"Max HNS":0.489518679},{"game":"Frostbite","seed":61.0,"HNS":0.1539290315,"Max HNS":0.3228481087},{"game":"Frostbite","seed":62.0,"HNS":0.0399109966,"Max HNS":0.0399109966},{"game":"Frostbite","seed":63.0,"HNS":0.3591052816,"Max HNS":0.6074950228},{"game":"Frostbite","seed":64.0,"HNS":0.4009602998,"Max HNS":0.5573252137},{"game":"Frostbite","seed":65.0,"HNS":0.3352851622,"Max HNS":0.6932427685},{"game":"Frostbite","seed":66.0,"HNS":0.3452629113,"Max HNS":0.4684623492},{"game":"Frostbite","seed":67.0,"HNS":0.036257173,"Max HNS":0.036257173},{"game":"Frostbite","seed":68.0,"HNS":0.5892961705,"Max HNS":0.5892961705},{"game":"Frostbite","seed":69.0,"HNS":0.0368427216,"Max HNS":0.0458835929},{"game":"Frostbite","seed":70.0,"HNS":0.0971073896,"Max HNS":0.1422180583},{"game":"Frostbite","seed":71.0,"HNS":0.5614240543,"Max HNS":0.5614240543},{"game":"Frostbite","seed":72.0,"HNS":0.0343599953,"Max HNS":0.0392786041},{"game":"Frostbite","seed":73.0,"HNS":0.0466330952,"Max HNS":0.0472654878},{"game":"Frostbite","seed":74.0,"HNS":0.5196861459,"Max HNS":0.5534137487},{"game":"Frostbite","seed":75.0,"HNS":0.0426513643,"Max HNS":0.0426513643},{"game":"Frostbite","seed":76.0,"HNS":0.3826911816,"Max HNS":0.3826911816},{"game":"Frostbite","seed":77.0,"HNS":0.0471718,"Max HNS":0.0478041925},{"game":"Frostbite","seed":78.0,"HNS":0.0351563415,"Max HNS":0.0495608385},{"game":"Frostbite","seed":79.0,"HNS":0.3223328259,"Max HNS":0.3223328259},{"game":"Frostbite","seed":80.0,"HNS":0.0454619979,"Max HNS":0.0479681462},{"game":"Frostbite","seed":81.0,"HNS":0.3944021548,"Max HNS":0.5617285396},{"game":"Frostbite","seed":82.0,"HNS":0.4410118281,"Max HNS":0.4410118281},{"game":"Frostbite","seed":83.0,"HNS":0.0446656517,"Max HNS":0.0448998712},{"game":"Frostbite","seed":84.0,"HNS":0.0470312683,"Max HNS":0.0470312683},{"game":"Frostbite","seed":85.0,"HNS":0.0851856189,"Max HNS":0.2077292423},{"game":"Frostbite","seed":86.0,"HNS":0.3678416676,"Max HNS":0.5411640707},{"game":"Frostbite","seed":87.0,"HNS":0.0353202951,"Max HNS":0.036257173},{"game":"Frostbite","seed":88.0,"HNS":0.7204122263,"Max HNS":0.7204122263},{"game":"Frostbite","seed":89.0,"HNS":0.3395245345,"Max HNS":0.4548307764},{"game":"Frostbite","seed":90.0,"HNS":0.6438927275,"Max HNS":0.6438927275},{"game":"Frostbite","seed":91.0,"HNS":0.4351329195,"Max HNS":0.4351329195},{"game":"Frostbite","seed":92.0,"HNS":0.5104344771,"Max HNS":0.5104344771},{"game":"Frostbite","seed":93.0,"HNS":0.0507085139,"Max HNS":0.0507085139},{"game":"Frostbite","seed":94.0,"HNS":0.5054690245,"Max HNS":0.6045672795},{"game":"Frostbite","seed":95.0,"HNS":0.0357653121,"Max HNS":0.0859116993},{"game":"Frostbite","seed":96.0,"HNS":0.0479447242,"Max HNS":0.0479447242},{"game":"Frostbite","seed":97.0,"HNS":0.0874575477,"Max HNS":0.0874575477},{"game":"Frostbite","seed":98.0,"HNS":0.5165241832,"Max HNS":0.5165241832},{"game":"Frostbite","seed":99.0,"HNS":0.034945544,"Max HNS":0.04103525},{"game":"Frostbite","seed":100.0,"HNS":0.0469844244,"Max HNS":0.0469844244},{"game":"Gopher","seed":1.0,"HNS":0.0584713908,"Max HNS":0.1674323635},{"game":"Gopher","seed":2.0,"HNS":0.1061766207,"Max HNS":0.1549027797},{"game":"Gopher","seed":3.0,"HNS":0.2248828252,"Max HNS":0.2248828252},{"game":"Gopher","seed":4.0,"HNS":0.0551301685,"Max HNS":0.0551301685},{"game":"Gopher","seed":5.0,"HNS":0.0983804353,"Max HNS":0.1532321685},{"game":"Gopher","seed":6.0,"HNS":0.0503039584,"Max HNS":0.0732284561},{"game":"Gopher","seed":7.0,"HNS":-0.0335978468,"Max HNS":0.3620585642},{"game":"Gopher","seed":8.0,"HNS":-0.0134577011,"Max HNS":0.168267669},{"game":"Gopher","seed":9.0,"HNS":0.0779618544,"Max HNS":0.1424660077},{"game":"Gopher","seed":10.0,"HNS":0.0141073832,"Max HNS":0.0561510975},{"game":"Gopher","seed":11.0,"HNS":-0.0389809272,"Max HNS":0.1392175971},{"game":"Gopher","seed":12.0,"HNS":0.0894705091,"Max HNS":0.0899345677},{"game":"Gopher","seed":13.0,"HNS":0.107475985,"Max HNS":0.1611211657},{"game":"Gopher","seed":14.0,"HNS":0.0346187758,"Max HNS":0.0992157409},{"game":"Gopher","seed":15.0,"HNS":0.1067334911,"Max HNS":0.1067334911},{"game":"Gopher","seed":16.0,"HNS":0.0642257181,"Max HNS":0.0642257181},{"game":"Gopher","seed":17.0,"HNS":0.1563877674,"Max HNS":0.1563877674},{"game":"Gopher","seed":18.0,"HNS":0.0089099262,"Max HNS":0.133370458},{"game":"Gopher","seed":19.0,"HNS":0.0658035176,"Max HNS":0.1433013133},{"game":"Gopher","seed":20.0,"HNS":0.0137361362,"Max HNS":0.0764768667},{"game":"Gopher","seed":21.0,"HNS":0.0800037125,"Max HNS":0.1407953965},{"game":"Gopher","seed":22.0,"HNS":-0.0341547172,"Max HNS":0.0090027379},{"game":"Gopher","seed":23.0,"HNS":0.0648754003,"Max HNS":0.0672885053},{"game":"Gopher","seed":24.0,"HNS":0.046313054,"Max HNS":0.1683604808},{"game":"Gopher","seed":25.0,"HNS":0.1531393568,"Max HNS":0.1531393568},{"game":"Gopher","seed":26.0,"HNS":0.0322056708,"Max HNS":0.0610701193},{"game":"Gopher","seed":27.0,"HNS":0.10988909,"Max HNS":0.228409671},{"game":"Gopher","seed":28.0,"HNS":0.0066824447,"Max HNS":0.0374959395},{"game":"Gopher","seed":29.0,"HNS":0.1615852244,"Max HNS":0.2261821894},{"game":"Gopher","seed":30.0,"HNS":0.0052902687,"Max HNS":0.0814887002},{"game":"Gopher","seed":31.0,"HNS":0.180704441,"Max HNS":0.215601652},{"game":"Gopher","seed":32.0,"HNS":0.0963385772,"Max HNS":0.0963385772},{"game":"Gopher","seed":33.0,"HNS":-0.0232029329,"Max HNS":0.0019490464},{"game":"Gopher","seed":34.0,"HNS":-0.0644113416,"Max HNS":0.008631491},{"game":"Gopher","seed":35.0,"HNS":0.074620632,"Max HNS":0.1282658128},{"game":"Gopher","seed":36.0,"HNS":0.0445496311,"Max HNS":0.1552740266},{"game":"Gopher","seed":37.0,"HNS":0.1061766207,"Max HNS":0.1424660077},{"game":"Gopher","seed":38.0,"HNS":0.027657896,"Max HNS":0.0690519282},{"game":"Gopher","seed":39.0,"HNS":0.2181075688,"Max HNS":0.2181075688},{"game":"Gopher","seed":40.0,"HNS":0.2287809179,"Max HNS":0.2287809179},{"game":"Gopher","seed":41.0,"HNS":0.0939254722,"Max HNS":0.1025569632},{"game":"Gopher","seed":42.0,"HNS":-0.0323912943,"Max HNS":0.0368462574},{"game":"Gopher","seed":43.0,"HNS":0.0041765279,"Max HNS":0.151004687},{"game":"Gopher","seed":44.0,"HNS":0.0575432735,"Max HNS":0.1101675252},{"game":"Gopher","seed":45.0,"HNS":0.0026915402,"Max HNS":0.0386096803},{"game":"Gopher","seed":46.0,"HNS":0.0833449348,"Max HNS":0.1470137825},{"game":"Gopher","seed":47.0,"HNS":0.0506752053,"Max HNS":0.1585224372},{"game":"Gopher","seed":48.0,"HNS":0.0304422479,"Max HNS":0.05735765},{"game":"Gopher","seed":49.0,"HNS":0.1326279642,"Max HNS":0.1538818507},{"game":"Gopher","seed":50.0,"HNS":-0.0127152072,"Max HNS":0.133741705},{"game":"Gopher","seed":51.0,"HNS":0.1730010673,"Max HNS":0.2190356861},{"game":"Gopher","seed":52.0,"HNS":0.1528609216,"Max HNS":0.1528609216},{"game":"Gopher","seed":53.0,"HNS":0.0012065525,"Max HNS":0.0840874287},{"game":"Gopher","seed":54.0,"HNS":0.1403313379,"Max HNS":0.1403313379},{"game":"Gopher","seed":55.0,"HNS":0.1502621931,"Max HNS":0.2104041951},{"game":"Gopher","seed":56.0,"HNS":0.1188918279,"Max HNS":0.1487772054},{"game":"Gopher","seed":57.0,"HNS":0.0141073832,"Max HNS":0.0476124182},{"game":"Gopher","seed":58.0,"HNS":0.1264095782,"Max HNS":0.139588844},{"game":"Gopher","seed":59.0,"HNS":0.0897489443,"Max HNS":0.0897489443},{"game":"Gopher","seed":60.0,"HNS":0.0363821987,"Max HNS":0.0968026359},{"game":"Gopher","seed":61.0,"HNS":-0.0273794608,"Max HNS":0.0204185809},{"game":"Gopher","seed":62.0,"HNS":-0.0008353056,"Max HNS":0.0284932015},{"game":"Gopher","seed":63.0,"HNS":0.0217179451,"Max HNS":0.0905842498},{"game":"Gopher","seed":64.0,"HNS":0.2349064922,"Max HNS":0.2349064922},{"game":"Gopher","seed":65.0,"HNS":0.0220891921,"Max HNS":0.0380528099},{"game":"Gopher","seed":66.0,"HNS":0.0319272356,"Max HNS":0.1368973038},{"game":"Gopher","seed":67.0,"HNS":-0.0151283122,"Max HNS":0.1076616084},{"game":"Gopher","seed":68.0,"HNS":0.1342057636,"Max HNS":0.1823750522},{"game":"Gopher","seed":69.0,"HNS":0.1329992111,"Max HNS":0.1329992111},{"game":"Gopher","seed":70.0,"HNS":0.1375469859,"Max HNS":0.1647408232},{"game":"Gopher","seed":71.0,"HNS":0.0346187758,"Max HNS":0.113694371},{"game":"Gopher","seed":72.0,"HNS":0.1381966681,"Max HNS":0.1381966681},{"game":"Gopher","seed":73.0,"HNS":0.039723421,"Max HNS":0.039723421},{"game":"Gopher","seed":74.0,"HNS":0.1566662026,"Max HNS":0.1975033644},{"game":"Gopher","seed":75.0,"HNS":0.0560582858,"Max HNS":0.0878927096},{"game":"Gopher","seed":76.0,"HNS":0.0067752564,"Max HNS":0.0581001439},{"game":"Gopher","seed":77.0,"HNS":0.1041347626,"Max HNS":0.1041347626},{"game":"Gopher","seed":78.0,"HNS":0.061162931,"Max HNS":0.0876142744},{"game":"Gopher","seed":79.0,"HNS":-0.0160564295,"Max HNS":0.1530465451},{"game":"Gopher","seed":80.0,"HNS":0.0130864541,"Max HNS":0.0468699244},{"game":"Gopher","seed":81.0,"HNS":-0.0595851316,"Max HNS":0.0428790199},{"game":"Gopher","seed":82.0,"HNS":0.1502621931,"Max HNS":0.2185716275},{"game":"Gopher","seed":83.0,"HNS":0.0410227853,"Max HNS":0.0410227853},{"game":"Gopher","seed":84.0,"HNS":0.0342475289,"Max HNS":0.0436215138},{"game":"Gopher","seed":85.0,"HNS":-0.0147570653,"Max HNS":0.0618126131},{"game":"Gopher","seed":86.0,"HNS":0.0596779433,"Max HNS":0.1401457144},{"game":"Gopher","seed":87.0,"HNS":0.0900273795,"Max HNS":0.1467353473},{"game":"Gopher","seed":88.0,"HNS":0.1795907003,"Max HNS":0.2123532414},{"game":"Gopher","seed":89.0,"HNS":0.0438071372,"Max HNS":0.1923059075},{"game":"Gopher","seed":90.0,"HNS":0.0571720265,"Max HNS":0.1569446378},{"game":"Gopher","seed":91.0,"HNS":0.0550373567,"Max HNS":0.0550373567},{"game":"Gopher","seed":92.0,"HNS":-0.0159636178,"Max HNS":-0.0141073832},{"game":"Gopher","seed":93.0,"HNS":0.1508190635,"Max HNS":0.2334215045},{"game":"Gopher","seed":94.0,"HNS":0.0349900227,"Max HNS":0.0349900227},{"game":"Gopher","seed":95.0,"HNS":0.0578217087,"Max HNS":0.0809318298},{"game":"Gopher","seed":96.0,"HNS":0.0319272356,"Max HNS":0.0484477238},{"game":"Gopher","seed":97.0,"HNS":0.0296069423,"Max HNS":0.1111884542},{"game":"Gopher","seed":98.0,"HNS":0.0461274305,"Max HNS":0.0461274305},{"game":"Gopher","seed":99.0,"HNS":0.0164276765,"Max HNS":0.1577799434},{"game":"Gopher","seed":100.0,"HNS":0.2178291336,"Max HNS":0.2178291336},{"game":"Hero","seed":1.0,"HNS":0.0724175655,"Max HNS":0.0802398706},{"game":"Hero","seed":2.0,"HNS":0.0693319329,"Max HNS":0.1415884212},{"game":"Hero","seed":3.0,"HNS":0.0704863185,"Max HNS":0.1112505621},{"game":"Hero","seed":4.0,"HNS":0.0957401827,"Max HNS":0.2116485567},{"game":"Hero","seed":5.0,"HNS":0.2039386699,"Max HNS":0.2158248153},{"game":"Hero","seed":6.0,"HNS":0.0664946274,"Max HNS":0.0709326362},{"game":"Hero","seed":7.0,"HNS":0.1541356537,"Max HNS":0.1541356537},{"game":"Hero","seed":8.0,"HNS":0.1681845943,"Max HNS":0.1681845943},{"game":"Hero","seed":9.0,"HNS":0.0677698209,"Max HNS":0.1612465352},{"game":"Hero","seed":10.0,"HNS":0.2127526058,"Max HNS":0.2134522843},{"game":"Hero","seed":11.0,"HNS":0.2056786378,"Max HNS":0.2056786378},{"game":"Hero","seed":12.0,"HNS":0.0644744525,"Max HNS":0.0706893427},{"game":"Hero","seed":13.0,"HNS":0.0655952804,"Max HNS":0.0743118989},{"game":"Hero","seed":14.0,"HNS":0.2045158627,"Max HNS":0.2077374041},{"game":"Hero","seed":15.0,"HNS":0.0667110747,"Max HNS":0.0708688766},{"game":"Hero","seed":16.0,"HNS":0.2016332544,"Max HNS":0.203524232},{"game":"Hero","seed":17.0,"HNS":0.108112915,"Max HNS":0.1965073122},{"game":"Hero","seed":18.0,"HNS":0.1792049504,"Max HNS":0.1855591052},{"game":"Hero","seed":19.0,"HNS":0.1952858111,"Max HNS":0.2029403277},{"game":"Hero","seed":20.0,"HNS":0.2084723182,"Max HNS":0.2084723182},{"game":"Hero","seed":21.0,"HNS":0.2618039289,"Max HNS":0.2618039289},{"game":"Hero","seed":22.0,"HNS":0.1150962771,"Max HNS":0.1150962771},{"game":"Hero","seed":23.0,"HNS":0.0707396793,"Max HNS":0.0715115069},{"game":"Hero","seed":24.0,"HNS":0.1939435022,"Max HNS":0.212690524},{"game":"Hero","seed":25.0,"HNS":0.1067034907,"Max HNS":0.1509510259},{"game":"Hero","seed":26.0,"HNS":0.1975476016,"Max HNS":0.2031752317},{"game":"Hero","seed":27.0,"HNS":0.2175261918,"Max HNS":0.2175261918},{"game":"Hero","seed":28.0,"HNS":0.2072894085,"Max HNS":0.2072894085},{"game":"Hero","seed":29.0,"HNS":0.1995845554,"Max HNS":0.2039873286},{"game":"Hero","seed":30.0,"HNS":0.2081988228,"Max HNS":0.2123197111},{"game":"Hero","seed":31.0,"HNS":0.0707396793,"Max HNS":0.0707396793},{"game":"Hero","seed":32.0,"HNS":0.1396018041,"Max HNS":0.1438502117},{"game":"Hero","seed":33.0,"HNS":0.0724007866,"Max HNS":0.0724007866},{"game":"Hero","seed":34.0,"HNS":0.0707396793,"Max HNS":0.0707396793},{"game":"Hero","seed":35.0,"HNS":0.3653815177,"Max HNS":0.3653815177},{"game":"Hero","seed":36.0,"HNS":0.1308918972,"Max HNS":0.1750672832},{"game":"Hero","seed":37.0,"HNS":0.0678570709,"Max HNS":0.0688638026},{"game":"Hero","seed":38.0,"HNS":0.2067759754,"Max HNS":0.21233649},{"game":"Hero","seed":39.0,"HNS":0.1789213877,"Max HNS":0.2158667624},{"game":"Hero","seed":40.0,"HNS":0.0664728149,"Max HNS":0.1408031705},{"game":"Hero","seed":41.0,"HNS":0.0712396894,"Max HNS":0.0712396894},{"game":"Hero","seed":42.0,"HNS":0.0668469835,"Max HNS":0.2128532789},{"game":"Hero","seed":43.0,"HNS":0.0965623469,"Max HNS":0.0965623469},{"game":"Hero","seed":44.0,"HNS":0.1272710189,"Max HNS":0.1272710189},{"game":"Hero","seed":45.0,"HNS":0.1346923092,"Max HNS":0.2061014651},{"game":"Hero","seed":46.0,"HNS":0.0582847306,"Max HNS":0.0700433566},{"game":"Hero","seed":47.0,"HNS":0.2038245736,"Max HNS":0.2095965019},{"game":"Hero","seed":48.0,"HNS":0.0712380115,"Max HNS":0.1874534387},{"game":"Hero","seed":49.0,"HNS":0.1997254978,"Max HNS":0.1997254978},{"game":"Hero","seed":50.0,"HNS":0.072202796,"Max HNS":0.072202796},{"game":"Hero","seed":51.0,"HNS":0.1875574676,"Max HNS":0.1875574676},{"game":"Hero","seed":52.0,"HNS":0.0746223078,"Max HNS":0.0746223078},{"game":"Hero","seed":53.0,"HNS":0.2038228958,"Max HNS":0.2172711531},{"game":"Hero","seed":54.0,"HNS":0.0703839675,"Max HNS":0.0707396793},{"game":"Hero","seed":55.0,"HNS":0.0708067948,"Max HNS":0.0708067948},{"game":"Hero","seed":56.0,"HNS":0.0643452553,"Max HNS":0.1059199179},{"game":"Hero","seed":57.0,"HNS":0.0715769445,"Max HNS":0.0715769445},{"game":"Hero","seed":58.0,"HNS":0.0640298127,"Max HNS":0.0683000329},{"game":"Hero","seed":59.0,"HNS":0.2131418753,"Max HNS":0.2131418753},{"game":"Hero","seed":60.0,"HNS":0.1252575555,"Max HNS":0.1252575555},{"game":"Hero","seed":61.0,"HNS":0.1644294852,"Max HNS":0.2017691631},{"game":"Hero","seed":62.0,"HNS":0.0703537655,"Max HNS":0.0705635013},{"game":"Hero","seed":63.0,"HNS":0.203009121,"Max HNS":0.2101502044},{"game":"Hero","seed":64.0,"HNS":0.0814143909,"Max HNS":0.0814143909},{"game":"Hero","seed":65.0,"HNS":0.1964435526,"Max HNS":0.2035040974},{"game":"Hero","seed":66.0,"HNS":0.1320932636,"Max HNS":0.2033010732},{"game":"Hero","seed":67.0,"HNS":0.0744931106,"Max HNS":0.0744931106},{"game":"Hero","seed":68.0,"HNS":0.0656808526,"Max HNS":0.0675718303},{"game":"Hero","seed":69.0,"HNS":0.0739108841,"Max HNS":0.1329003269},{"game":"Hero","seed":70.0,"HNS":0.206088042,"Max HNS":0.206088042},{"game":"Hero","seed":71.0,"HNS":0.1007352497,"Max HNS":0.1007352497},{"game":"Hero","seed":72.0,"HNS":0.2154657476,"Max HNS":0.2154657476},{"game":"Hero","seed":73.0,"HNS":0.1989201125,"Max HNS":0.2033782559},{"game":"Hero","seed":74.0,"HNS":0.2051618489,"Max HNS":0.2051618489},{"game":"Hero","seed":75.0,"HNS":0.1356453486,"Max HNS":0.1875289435},{"game":"Hero","seed":76.0,"HNS":0.0727061619,"Max HNS":0.0727061619},{"game":"Hero","seed":77.0,"HNS":0.0707396793,"Max HNS":0.0707396793},{"game":"Hero","seed":78.0,"HNS":0.0665483198,"Max HNS":0.0788505809},{"game":"Hero","seed":79.0,"HNS":0.1913813701,"Max HNS":0.1913813701},{"game":"Hero","seed":80.0,"HNS":0.2035728907,"Max HNS":0.204069545},{"game":"Hero","seed":81.0,"HNS":0.0660902569,"Max HNS":0.0689359517},{"game":"Hero","seed":82.0,"HNS":0.2177476728,"Max HNS":0.219232602},{"game":"Hero","seed":83.0,"HNS":-0.0319217837,"Max HNS":-0.0292707236},{"game":"Hero","seed":84.0,"HNS":0.1930642899,"Max HNS":0.2081199621},{"game":"Hero","seed":85.0,"HNS":0.0623687054,"Max HNS":0.0738756485},{"game":"Hero","seed":86.0,"HNS":0.0707396793,"Max HNS":0.0732565085},{"game":"Hero","seed":87.0,"HNS":0.0658553528,"Max HNS":0.0699644959},{"game":"Hero","seed":88.0,"HNS":0.0665684544,"Max HNS":0.0683671483},{"game":"Hero","seed":89.0,"HNS":0.0682413069,"Max HNS":0.0715115069},{"game":"Hero","seed":90.0,"HNS":0.070773237,"Max HNS":0.0707900159},{"game":"Hero","seed":91.0,"HNS":0.0724175655,"Max HNS":0.0724175655},{"game":"Hero","seed":92.0,"HNS":0.1868376544,"Max HNS":0.1868376544},{"game":"Hero","seed":93.0,"HNS":0.1238766552,"Max HNS":0.1238766552},{"game":"Hero","seed":94.0,"HNS":0.0707396793,"Max HNS":0.0707396793},{"game":"Hero","seed":95.0,"HNS":0.2269810802,"Max HNS":0.2644298207},{"game":"Hero","seed":96.0,"HNS":0.1706880004,"Max HNS":0.2075243126},{"game":"Hero","seed":97.0,"HNS":0.1733860413,"Max HNS":0.1733860413},{"game":"Hero","seed":98.0,"HNS":0.1975912267,"Max HNS":0.1983009725},{"game":"Hero","seed":99.0,"HNS":0.2147878816,"Max HNS":0.217818144},{"game":"Hero","seed":100.0,"HNS":0.1989368913,"Max HNS":0.2025007215},{"game":"Jamesbond","seed":1.0,"HNS":1.2801314828,"Max HNS":1.2801314828},{"game":"Jamesbond","seed":2.0,"HNS":1.0664718773,"Max HNS":1.0810810811},{"game":"Jamesbond","seed":3.0,"HNS":0.8674214755,"Max HNS":1.1249086925},{"game":"Jamesbond","seed":4.0,"HNS":0.726807889,"Max HNS":1.161431702},{"game":"Jamesbond","seed":5.0,"HNS":1.557706355,"Max HNS":1.557706355},{"game":"Jamesbond","seed":6.0,"HNS":0.4821037253,"Max HNS":0.6263696129},{"game":"Jamesbond","seed":7.0,"HNS":0.9623813002,"Max HNS":0.9623813002},{"game":"Jamesbond","seed":8.0,"HNS":1.2143900657,"Max HNS":1.2143900657},{"game":"Jamesbond","seed":9.0,"HNS":0.8108108108,"Max HNS":1.2162162162},{"game":"Jamesbond","seed":10.0,"HNS":1.2472607743,"Max HNS":1.446311176},{"game":"Jamesbond","seed":11.0,"HNS":0.7669831994,"Max HNS":0.9897735573},{"game":"Jamesbond","seed":12.0,"HNS":0.4839298758,"Max HNS":0.5350620891},{"game":"Jamesbond","seed":13.0,"HNS":1.7859751644,"Max HNS":1.7859751644},{"game":"Jamesbond","seed":14.0,"HNS":1.1322132944,"Max HNS":1.1322132944},{"game":"Jamesbond","seed":15.0,"HNS":0.9149013879,"Max HNS":0.9149013879},{"game":"Jamesbond","seed":16.0,"HNS":0.8528122717,"Max HNS":1.1504747991},{"game":"Jamesbond","seed":17.0,"HNS":0.9130752374,"Max HNS":0.938641344},{"game":"Jamesbond","seed":18.0,"HNS":0.9550766983,"Max HNS":1.2874360847},{"game":"Jamesbond","seed":19.0,"HNS":0.7432432432,"Max HNS":0.8272461651},{"game":"Jamesbond","seed":20.0,"HNS":1.050036523,"Max HNS":1.050036523},{"game":"Jamesbond","seed":21.0,"HNS":0.5880204529,"Max HNS":1.0682980278},{"game":"Jamesbond","seed":22.0,"HNS":1.3093498904,"Max HNS":1.3184806428},{"game":"Jamesbond","seed":23.0,"HNS":1.5047479912,"Max HNS":1.5047479912},{"game":"Jamesbond","seed":24.0,"HNS":0.7213294375,"Max HNS":0.7213294375},{"game":"Jamesbond","seed":25.0,"HNS":1.3349159971,"Max HNS":2.1785975164},{"game":"Jamesbond","seed":26.0,"HNS":2.5127830533,"Max HNS":2.5127830533},{"game":"Jamesbond","seed":27.0,"HNS":1.2600438276,"Max HNS":1.2819576333},{"game":"Jamesbond","seed":28.0,"HNS":1.0299488678,"Max HNS":1.6161431702},{"game":"Jamesbond","seed":29.0,"HNS":1.0098612126,"Max HNS":1.0098612126},{"game":"Jamesbond","seed":30.0,"HNS":0.8765522279,"Max HNS":1.484660336},{"game":"Jamesbond","seed":31.0,"HNS":0.796201607,"Max HNS":1.0829072316},{"game":"Jamesbond","seed":32.0,"HNS":0.9824689554,"Max HNS":0.9824689554},{"game":"Jamesbond","seed":33.0,"HNS":0.5277574872,"Max HNS":0.7688093499},{"game":"Jamesbond","seed":34.0,"HNS":0.9203798393,"Max HNS":0.9587289993},{"game":"Jamesbond","seed":35.0,"HNS":1.3111760409,"Max HNS":1.5595325055},{"game":"Jamesbond","seed":36.0,"HNS":1.3111760409,"Max HNS":1.3403944485},{"game":"Jamesbond","seed":37.0,"HNS":0.9185536888,"Max HNS":1.6033601169},{"game":"Jamesbond","seed":38.0,"HNS":0.9879474069,"Max HNS":1.0135135135},{"game":"Jamesbond","seed":39.0,"HNS":0.8948137327,"Max HNS":0.8948137327},{"game":"Jamesbond","seed":40.0,"HNS":1.2289992695,"Max HNS":1.4645726808},{"game":"Jamesbond","seed":41.0,"HNS":1.5430971512,"Max HNS":1.5430971512},{"game":"Jamesbond","seed":42.0,"HNS":0.8181154127,"Max HNS":1.0262965668},{"game":"Jamesbond","seed":43.0,"HNS":1.0189919649,"Max HNS":1.0189919649},{"game":"Jamesbond","seed":44.0,"HNS":1.3330898466,"Max HNS":1.3623082542},{"game":"Jamesbond","seed":45.0,"HNS":0.3780131483,"Max HNS":0.5094959825},{"game":"Jamesbond","seed":46.0,"HNS":1.084733382,"Max HNS":1.084733382},{"game":"Jamesbond","seed":47.0,"HNS":0.9897735573,"Max HNS":1.0902118335},{"game":"Jamesbond","seed":48.0,"HNS":1.0555149744,"Max HNS":1.4097881665},{"game":"Jamesbond","seed":49.0,"HNS":1.6617969321,"Max HNS":1.6617969321},{"game":"Jamesbond","seed":50.0,"HNS":0.9605551497,"Max HNS":1.1340394449},{"game":"Jamesbond","seed":51.0,"HNS":1.8334550767,"Max HNS":1.9430241052},{"game":"Jamesbond","seed":52.0,"HNS":0.7742878013,"Max HNS":0.7742878013},{"game":"Jamesbond","seed":53.0,"HNS":1.4737034332,"Max HNS":2.1530314098},{"game":"Jamesbond","seed":54.0,"HNS":1.1340394449,"Max HNS":1.1340394449},{"game":"Jamesbond","seed":55.0,"HNS":1.338568298,"Max HNS":1.338568298},{"game":"Jamesbond","seed":56.0,"HNS":1.0555149744,"Max HNS":1.1157779401},{"game":"Jamesbond","seed":57.0,"HNS":1.4024835646,"Max HNS":1.4353542732},{"game":"Jamesbond","seed":58.0,"HNS":0.723155588,"Max HNS":0.723155588},{"game":"Jamesbond","seed":59.0,"HNS":0.942293645,"Max HNS":1.0043827611},{"game":"Jamesbond","seed":60.0,"HNS":1.0135135135,"Max HNS":1.0646457268},{"game":"Jamesbond","seed":61.0,"HNS":1.2472607743,"Max HNS":1.2472607743},{"game":"Jamesbond","seed":62.0,"HNS":0.6957633309,"Max HNS":0.6957633309},{"game":"Jamesbond","seed":63.0,"HNS":1.1723886048,"Max HNS":1.1723886048},{"game":"Jamesbond","seed":64.0,"HNS":0.6281957633,"Max HNS":0.7377647918},{"game":"Jamesbond","seed":65.0,"HNS":1.0317750183,"Max HNS":1.0317750183},{"game":"Jamesbond","seed":66.0,"HNS":1.8060628196,"Max HNS":2.0142439737},{"game":"Jamesbond","seed":67.0,"HNS":1.126734843,"Max HNS":1.2289992695},{"game":"Jamesbond","seed":68.0,"HNS":0.8674214755,"Max HNS":0.8674214755},{"game":"Jamesbond","seed":69.0,"HNS":0.6373265157,"Max HNS":0.7925493061},{"game":"Jamesbond","seed":70.0,"HNS":1.0737764792,"Max HNS":1.6070124178},{"game":"Jamesbond","seed":71.0,"HNS":1.411614317,"Max HNS":1.411614317},{"game":"Jamesbond","seed":72.0,"HNS":1.3330898466,"Max HNS":1.7165814463},{"game":"Jamesbond","seed":73.0,"HNS":1.0609934259,"Max HNS":1.6197954711},{"game":"Jamesbond","seed":74.0,"HNS":1.2235208181,"Max HNS":1.2636961286},{"game":"Jamesbond","seed":75.0,"HNS":1.2874360847,"Max HNS":1.2874360847},{"game":"Jamesbond","seed":76.0,"HNS":0.7121986852,"Max HNS":0.9861212564},{"game":"Jamesbond","seed":77.0,"HNS":1.5704894083,"Max HNS":1.5704894083},{"game":"Jamesbond","seed":78.0,"HNS":1.7494521549,"Max HNS":1.8078889701},{"game":"Jamesbond","seed":79.0,"HNS":0.9514243974,"Max HNS":0.9514243974},{"game":"Jamesbond","seed":80.0,"HNS":2.2954711468,"Max HNS":2.2954711468},{"game":"Jamesbond","seed":81.0,"HNS":0.4930606282,"Max HNS":0.4930606282},{"game":"Jamesbond","seed":82.0,"HNS":0.8272461651,"Max HNS":0.9021183346},{"game":"Jamesbond","seed":83.0,"HNS":0.5387143901,"Max HNS":0.7651570489},{"game":"Jamesbond","seed":84.0,"HNS":0.6044558072,"Max HNS":0.6281957633},{"game":"Jamesbond","seed":85.0,"HNS":0.8217677137,"Max HNS":1.0409057706},{"game":"Jamesbond","seed":86.0,"HNS":1.1815193572,"Max HNS":1.4262235208},{"game":"Jamesbond","seed":87.0,"HNS":1.5485756026,"Max HNS":1.5485756026},{"game":"Jamesbond","seed":88.0,"HNS":1.3148283419,"Max HNS":1.3148283419},{"game":"Jamesbond","seed":89.0,"HNS":2.6789627465,"Max HNS":2.6789627465},{"game":"Jamesbond","seed":90.0,"HNS":1.0975164354,"Max HNS":1.0975164354},{"game":"Jamesbond","seed":91.0,"HNS":1.0609934259,"Max HNS":1.0609934259},{"game":"Jamesbond","seed":92.0,"HNS":1.0518626735,"Max HNS":1.2746530314},{"game":"Jamesbond","seed":93.0,"HNS":0.6044558072,"Max HNS":0.6044558072},{"game":"Jamesbond","seed":94.0,"HNS":0.8820306793,"Max HNS":0.8820306793},{"game":"Jamesbond","seed":95.0,"HNS":1.30752374,"Max HNS":1.30752374},{"game":"Jamesbond","seed":96.0,"HNS":0.8290723156,"Max HNS":0.8838568298},{"game":"Jamesbond","seed":97.0,"HNS":1.6544923302,"Max HNS":1.6544923302},{"game":"Jamesbond","seed":98.0,"HNS":0.6391526662,"Max HNS":0.6391526662},{"game":"Jamesbond","seed":99.0,"HNS":1.3476990504,"Max HNS":1.3476990504},{"game":"Jamesbond","seed":100.0,"HNS":1.5759678598,"Max HNS":1.5759678598},{"game":"Kangaroo","seed":1.0,"HNS":0.1837076768,"Max HNS":0.1837076768},{"game":"Kangaroo","seed":2.0,"HNS":0.1501843781,"Max HNS":0.1823667449},{"game":"Kangaroo","seed":3.0,"HNS":0.4042909822,"Max HNS":0.4954743547},{"game":"Kangaroo","seed":4.0,"HNS":0.1837076768,"Max HNS":0.1837076768},{"game":"Kangaroo","seed":5.0,"HNS":0.1837076768,"Max HNS":0.1837076768},{"game":"Kangaroo","seed":6.0,"HNS":0.1736506872,"Max HNS":0.2172309755},{"game":"Kangaroo","seed":7.0,"HNS":0.2393563527,"Max HNS":0.257458934},{"game":"Kangaroo","seed":8.0,"HNS":0.3875293329,"Max HNS":0.4592691921},{"game":"Kangaroo","seed":9.0,"HNS":0.1823667449,"Max HNS":0.1977874623},{"game":"Kangaroo","seed":10.0,"HNS":0.6047603084,"Max HNS":0.6570566544},{"game":"Kangaroo","seed":11.0,"HNS":0.1448206504,"Max HNS":0.1837076768},{"game":"Kangaroo","seed":12.0,"HNS":0.1153201475,"Max HNS":0.1153201475},{"game":"Kangaroo","seed":13.0,"HNS":0.0160911834,"Max HNS":0.0549782099},{"game":"Kangaroo","seed":14.0,"HNS":0.2648340597,"Max HNS":0.2648340597},{"game":"Kangaroo","seed":15.0,"HNS":0.1830372109,"Max HNS":0.1837076768},{"game":"Kangaroo","seed":16.0,"HNS":0.2681863895,"Max HNS":0.2681863895},{"game":"Kangaroo","seed":17.0,"HNS":0.1676164935,"Max HNS":0.2172309755},{"game":"Kangaroo","seed":18.0,"HNS":0.2024807241,"Max HNS":0.2024807241},{"game":"Kangaroo","seed":19.0,"HNS":0.4451894066,"Max HNS":0.4451894066},{"game":"Kangaroo","seed":20.0,"HNS":0.1837076768,"Max HNS":0.1837076768},{"game":"Kangaroo","seed":21.0,"HNS":0.1676164935,"Max HNS":0.1837076768},{"game":"Kangaroo","seed":22.0,"HNS":0.282936641,"Max HNS":0.3164599397},{"game":"Kangaroo","seed":23.0,"HNS":0.1910828025,"Max HNS":0.1910828025},{"game":"Kangaroo","seed":24.0,"HNS":0.1582299698,"Max HNS":0.1843781428},{"game":"Kangaroo","seed":25.0,"HNS":0.1910828025,"Max HNS":0.1910828025},{"game":"Kangaroo","seed":26.0,"HNS":0.1340931948,"Max HNS":0.1689574254},{"game":"Kangaroo","seed":27.0,"HNS":0.1166610795,"Max HNS":0.1320817968},{"game":"Kangaroo","seed":28.0,"HNS":0.5296681193,"Max HNS":0.5296681193},{"game":"Kangaroo","seed":29.0,"HNS":0.1173315454,"Max HNS":0.1173315454},{"game":"Kangaroo","seed":30.0,"HNS":0.1012403621,"Max HNS":0.1716392893},{"game":"Kangaroo","seed":31.0,"HNS":1.0090512906,"Max HNS":1.0090512906},{"game":"Kangaroo","seed":32.0,"HNS":0.1682869594,"Max HNS":0.1837076768},{"game":"Kangaroo","seed":33.0,"HNS":0.1837076768,"Max HNS":0.2145491116},{"game":"Kangaroo","seed":34.0,"HNS":0.1837076768,"Max HNS":0.1840429098},{"game":"Kangaroo","seed":35.0,"HNS":0.1629232316,"Max HNS":0.1837076768},{"game":"Kangaroo","seed":36.0,"HNS":0.1954408314,"Max HNS":0.1954408314},{"game":"Kangaroo","seed":37.0,"HNS":0.2936640965,"Max HNS":0.3654039557},{"game":"Kangaroo","seed":38.0,"HNS":0.2118672477,"Max HNS":0.2976868924},{"game":"Kangaroo","seed":39.0,"HNS":0.1843781428,"Max HNS":0.2621521958},{"game":"Kangaroo","seed":40.0,"HNS":0.1813610459,"Max HNS":0.1964465303},{"game":"Kangaroo","seed":41.0,"HNS":0.1314113309,"Max HNS":0.1830372109},{"game":"Kangaroo","seed":42.0,"HNS":0.1830372109,"Max HNS":0.2480724103},{"game":"Kangaroo","seed":43.0,"HNS":0.1837076768,"Max HNS":0.1837076768},{"game":"Kangaroo","seed":44.0,"HNS":0.1910828025,"Max HNS":0.1910828025},{"game":"Kangaroo","seed":45.0,"HNS":0.1609118337,"Max HNS":0.1850486088},{"game":"Kangaroo","seed":46.0,"HNS":0.1682869594,"Max HNS":0.1830372109},{"game":"Kangaroo","seed":47.0,"HNS":0.1837076768,"Max HNS":0.1843781428},{"game":"Kangaroo","seed":48.0,"HNS":0.2004693262,"Max HNS":0.2004693262},{"game":"Kangaroo","seed":49.0,"HNS":1.3918873617,"Max HNS":1.3918873617},{"game":"Kangaroo","seed":50.0,"HNS":0.7468990949,"Max HNS":0.7468990949},{"game":"Kangaroo","seed":51.0,"HNS":0.1689574254,"Max HNS":0.1837076768},{"game":"Kangaroo","seed":52.0,"HNS":0.3359034529,"Max HNS":0.3727790815},{"game":"Kangaroo","seed":53.0,"HNS":0.2366744888,"Max HNS":0.2500838082},{"game":"Kangaroo","seed":54.0,"HNS":0.0905129065,"Max HNS":0.1186724774},{"game":"Kangaroo","seed":55.0,"HNS":0.1830372109,"Max HNS":0.1837076768},{"game":"Kangaroo","seed":56.0,"HNS":0.2145491116,"Max HNS":0.2145491116},{"game":"Kangaroo","seed":57.0,"HNS":0.1910828025,"Max HNS":0.2641635937},{"game":"Kangaroo","seed":58.0,"HNS":0.1662755615,"Max HNS":0.1837076768},{"game":"Kangaroo","seed":59.0,"HNS":0.1166610795,"Max HNS":0.1837076768},{"game":"Kangaroo","seed":60.0,"HNS":0.2192423734,"Max HNS":0.2266174992},{"game":"Kangaroo","seed":61.0,"HNS":0.1837076768,"Max HNS":0.1837076768},{"game":"Kangaroo","seed":62.0,"HNS":0.2192423734,"Max HNS":0.2594703319},{"game":"Kangaroo","seed":63.0,"HNS":0.1012403621,"Max HNS":0.3620516259},{"game":"Kangaroo","seed":64.0,"HNS":0.1475025142,"Max HNS":0.1837076768},{"game":"Kangaroo","seed":65.0,"HNS":0.1830372109,"Max HNS":0.1837076768},{"game":"Kangaroo","seed":66.0,"HNS":0.1837076768,"Max HNS":0.1837076768},{"game":"Kangaroo","seed":67.0,"HNS":0.1837076768,"Max HNS":0.1837076768},{"game":"Kangaroo","seed":68.0,"HNS":0.1699631244,"Max HNS":0.1837076768},{"game":"Kangaroo","seed":69.0,"HNS":0.2420382166,"Max HNS":0.3660744217},{"game":"Kangaroo","seed":70.0,"HNS":0.2862889708,"Max HNS":0.2862889708},{"game":"Kangaroo","seed":71.0,"HNS":0.4290982233,"Max HNS":0.4290982233},{"game":"Kangaroo","seed":72.0,"HNS":0.230640295,"Max HNS":0.2520952062},{"game":"Kangaroo","seed":73.0,"HNS":0.1924237345,"Max HNS":0.1924237345},{"game":"Kangaroo","seed":74.0,"HNS":0.1837076768,"Max HNS":0.1837076768},{"game":"Kangaroo","seed":75.0,"HNS":0.1314113309,"Max HNS":0.1716392893},{"game":"Kangaroo","seed":76.0,"HNS":0.1847133758,"Max HNS":0.2413677506},{"game":"Kangaroo","seed":77.0,"HNS":0.231310761,"Max HNS":0.231310761},{"game":"Kangaroo","seed":78.0,"HNS":0.3432785786,"Max HNS":0.3432785786},{"game":"Kangaroo","seed":79.0,"HNS":0.0536372779,"Max HNS":0.1414683205},{"game":"Kangaroo","seed":80.0,"HNS":0.4632919879,"Max HNS":0.4632919879},{"game":"Kangaroo","seed":81.0,"HNS":0.3734495474,"Max HNS":0.3734495474},{"game":"Kangaroo","seed":82.0,"HNS":0.1847133758,"Max HNS":0.2018102581},{"game":"Kangaroo","seed":83.0,"HNS":0.3546765002,"Max HNS":0.3821656051},{"game":"Kangaroo","seed":84.0,"HNS":0.1837076768,"Max HNS":0.2205833054},{"game":"Kangaroo","seed":85.0,"HNS":0.1843781428,"Max HNS":0.1843781428},{"game":"Kangaroo","seed":86.0,"HNS":0.1622527657,"Max HNS":0.1622527657},{"game":"Kangaroo","seed":87.0,"HNS":0.1837076768,"Max HNS":0.1837076768},{"game":"Kangaroo","seed":88.0,"HNS":0.0315119008,"Max HNS":0.1200134093},{"game":"Kangaroo","seed":89.0,"HNS":0.0630238015,"Max HNS":0.127388535},{"game":"Kangaroo","seed":90.0,"HNS":0.1837076768,"Max HNS":0.1837076768},{"game":"Kangaroo","seed":91.0,"HNS":0.1830372109,"Max HNS":0.1830372109},{"game":"Kangaroo","seed":92.0,"HNS":0.1837076768,"Max HNS":0.1843781428},{"game":"Kangaroo","seed":93.0,"HNS":0.1173315454,"Max HNS":0.1837076768},{"game":"Kangaroo","seed":94.0,"HNS":0.1656050955,"Max HNS":0.1977874623},{"game":"Kangaroo","seed":95.0,"HNS":0.1495139122,"Max HNS":0.1837076768},{"game":"Kangaroo","seed":96.0,"HNS":0.358364063,"Max HNS":0.358364063},{"game":"Kangaroo","seed":97.0,"HNS":0.1837076768,"Max HNS":0.1837076768},{"game":"Kangaroo","seed":98.0,"HNS":0.2011397922,"Max HNS":0.2105263158},{"game":"Kangaroo","seed":99.0,"HNS":0.1830372109,"Max HNS":0.1830372109},{"game":"Kangaroo","seed":100.0,"HNS":0.1843781428,"Max HNS":0.1863895407},{"game":"Krull","seed":1.0,"HNS":1.2381639344,"Max HNS":1.2381639344},{"game":"Krull","seed":2.0,"HNS":1.2644777518,"Max HNS":1.865236534},{"game":"Krull","seed":3.0,"HNS":1.6371803279,"Max HNS":2.0001873536},{"game":"Krull","seed":4.0,"HNS":1.8522903981,"Max HNS":1.8717845433},{"game":"Krull","seed":5.0,"HNS":0.4607587822,"Max HNS":7.8674285714},{"game":"Krull","seed":6.0,"HNS":1.45264637,"Max HNS":1.54064637},{"game":"Krull","seed":7.0,"HNS":1.760412178,"Max HNS":1.760412178},{"game":"Krull","seed":8.0,"HNS":1.4308009368,"Max HNS":1.6255175644},{"game":"Krull","seed":9.0,"HNS":0.0752412178,"Max HNS":0.3736018735},{"game":"Krull","seed":10.0,"HNS":2.6074285714,"Max HNS":2.6074285714},{"game":"Krull","seed":11.0,"HNS":1.522763466,"Max HNS":1.550501171},{"game":"Krull","seed":12.0,"HNS":1.2351381733,"Max HNS":1.3962716628},{"game":"Krull","seed":13.0,"HNS":2.3120655738,"Max HNS":4.5921873536},{"game":"Krull","seed":14.0,"HNS":1.5841124122,"Max HNS":1.7478594848},{"game":"Krull","seed":15.0,"HNS":0.447971897,"Max HNS":0.4736955504},{"game":"Krull","seed":16.0,"HNS":1.4099953162,"Max HNS":1.8870444965},{"game":"Krull","seed":17.0,"HNS":1.532618267,"Max HNS":1.6015175644},{"game":"Krull","seed":18.0,"HNS":1.6782576112,"Max HNS":1.850557377},{"game":"Krull","seed":19.0,"HNS":0.9480374707,"Max HNS":0.9480374707},{"game":"Krull","seed":20.0,"HNS":1.0577985948,"Max HNS":1.6081686183},{"game":"Krull","seed":21.0,"HNS":1.6330772834,"Max HNS":1.6330772834},{"game":"Krull","seed":22.0,"HNS":1.553442623,"Max HNS":1.553442623},{"game":"Krull","seed":23.0,"HNS":1.64270726,"Max HNS":2.3426510539},{"game":"Krull","seed":24.0,"HNS":1.3825386417,"Max HNS":1.7382576112},{"game":"Krull","seed":25.0,"HNS":1.0944543326,"Max HNS":1.2571990632},{"game":"Krull","seed":26.0,"HNS":1.9542763466,"Max HNS":1.985882904},{"game":"Krull","seed":27.0,"HNS":2.0095175644,"Max HNS":2.5410398126},{"game":"Krull","seed":28.0,"HNS":0.6449929742,"Max HNS":0.8134145199},{"game":"Krull","seed":29.0,"HNS":1.3812271663,"Max HNS":1.3812271663},{"game":"Krull","seed":30.0,"HNS":0.2961217799,"Max HNS":0.6356440281},{"game":"Krull","seed":31.0,"HNS":1.47664637,"Max HNS":1.6610960187},{"game":"Krull","seed":32.0,"HNS":1.4352880562,"Max HNS":2.2409742389},{"game":"Krull","seed":33.0,"HNS":1.0043278689,"Max HNS":1.0731522248},{"game":"Krull","seed":34.0,"HNS":1.8183138173,"Max HNS":2.2253676815},{"game":"Krull","seed":35.0,"HNS":1.6718594848,"Max HNS":1.6718594848},{"game":"Krull","seed":36.0,"HNS":1.6973489461,"Max HNS":1.6973489461},{"game":"Krull","seed":37.0,"HNS":0.7232693208,"Max HNS":1.0314473068},{"game":"Krull","seed":38.0,"HNS":1.2626323185,"Max HNS":1.537498829},{"game":"Krull","seed":39.0,"HNS":0.6415269321,"Max HNS":0.7767119438},{"game":"Krull","seed":40.0,"HNS":2.076206089,"Max HNS":3.2277002342},{"game":"Krull","seed":41.0,"HNS":0.7405152225,"Max HNS":0.9054613583},{"game":"Krull","seed":42.0,"HNS":1.6682435597,"Max HNS":2.3148009368},{"game":"Krull","seed":43.0,"HNS":2.010501171,"Max HNS":6.1655644028},{"game":"Krull","seed":44.0,"HNS":1.3396440281,"Max HNS":1.4315035129},{"game":"Krull","seed":45.0,"HNS":1.9919156909,"Max HNS":2.0789039813},{"game":"Krull","seed":46.0,"HNS":0.865882904,"Max HNS":1.7825199063},{"game":"Krull","seed":47.0,"HNS":1.8037470726,"Max HNS":1.9274847775},{"game":"Krull","seed":48.0,"HNS":0.8547540984,"Max HNS":0.9174519906},{"game":"Krull","seed":49.0,"HNS":2.0487213115,"Max HNS":2.0487213115},{"game":"Krull","seed":50.0,"HNS":1.4516252927,"Max HNS":1.6407775176},{"game":"Krull","seed":51.0,"HNS":0.796440281,"Max HNS":0.9386042155},{"game":"Krull","seed":52.0,"HNS":1.4245245902,"Max HNS":1.4245245902},{"game":"Krull","seed":53.0,"HNS":1.8049929742,"Max HNS":1.8049929742},{"game":"Krull","seed":54.0,"HNS":1.6261733021,"Max HNS":2.1947259953},{"game":"Krull","seed":55.0,"HNS":0.1603559719,"Max HNS":10.6606744731},{"game":"Krull","seed":56.0,"HNS":1.841882904,"Max HNS":1.841882904},{"game":"Krull","seed":57.0,"HNS":1.1404309133,"Max HNS":1.1864355972},{"game":"Krull","seed":58.0,"HNS":1.4729180328,"Max HNS":1.4729180328},{"game":"Krull","seed":59.0,"HNS":1.9486838407,"Max HNS":1.9486838407},{"game":"Krull","seed":60.0,"HNS":0.7929461358,"Max HNS":3.7488430913},{"game":"Krull","seed":61.0,"HNS":0.4945199063,"Max HNS":0.9883559719},{"game":"Krull","seed":62.0,"HNS":3.0461826698,"Max HNS":3.0461826698},{"game":"Krull","seed":63.0,"HNS":1.2338735363,"Max HNS":1.9969367681},{"game":"Krull","seed":64.0,"HNS":0.4108196721,"Max HNS":0.9350070258},{"game":"Krull","seed":65.0,"HNS":0.4232786885,"Max HNS":0.6822857143},{"game":"Krull","seed":66.0,"HNS":0.7682154567,"Max HNS":2.9804496487},{"game":"Krull","seed":67.0,"HNS":1.9072224824,"Max HNS":1.9093583138},{"game":"Krull","seed":68.0,"HNS":1.6255550351,"Max HNS":1.6255550351},{"game":"Krull","seed":69.0,"HNS":3.2217236534,"Max HNS":3.2217236534},{"game":"Krull","seed":70.0,"HNS":0.549264637,"Max HNS":2.0402810304},{"game":"Krull","seed":71.0,"HNS":0.5994473068,"Max HNS":0.5994473068},{"game":"Krull","seed":72.0,"HNS":2.2316065574,"Max HNS":2.3182107728},{"game":"Krull","seed":73.0,"HNS":1.0302669789,"Max HNS":1.0901264637},{"game":"Krull","seed":74.0,"HNS":2.0629976581,"Max HNS":2.305264637},{"game":"Krull","seed":75.0,"HNS":1.4855925059,"Max HNS":1.5976861827},{"game":"Krull","seed":76.0,"HNS":1.0965807963,"Max HNS":8.9607494145},{"game":"Krull","seed":77.0,"HNS":1.302117096,"Max HNS":3.4960562061},{"game":"Krull","seed":78.0,"HNS":0.954735363,"Max HNS":1.4049742389},{"game":"Krull","seed":79.0,"HNS":2.2325620609,"Max HNS":2.2325620609},{"game":"Krull","seed":80.0,"HNS":1.4425386417,"Max HNS":1.4571522248},{"game":"Krull","seed":81.0,"HNS":0.6493021077,"Max HNS":0.8264074941},{"game":"Krull","seed":82.0,"HNS":1.8168618267,"Max HNS":1.948},{"game":"Krull","seed":83.0,"HNS":2.1249929742,"Max HNS":2.3621264637},{"game":"Krull","seed":84.0,"HNS":1.092618267,"Max HNS":4.1013302108},{"game":"Krull","seed":85.0,"HNS":0.2361873536,"Max HNS":0.2858173302},{"game":"Krull","seed":86.0,"HNS":-0.0301451991,"Max HNS":0.4674473068},{"game":"Krull","seed":87.0,"HNS":0.5911569087,"Max HNS":0.741676815},{"game":"Krull","seed":88.0,"HNS":1.1397564403,"Max HNS":1.2611522248},{"game":"Krull","seed":89.0,"HNS":3.4136018735,"Max HNS":3.4136018735},{"game":"Krull","seed":90.0,"HNS":1.2779578454,"Max HNS":1.3317002342},{"game":"Krull","seed":91.0,"HNS":0.4184262295,"Max HNS":0.4184262295},{"game":"Krull","seed":92.0,"HNS":1.5783793911,"Max HNS":1.8242716628},{"game":"Krull","seed":93.0,"HNS":1.9694145199,"Max HNS":1.9694145199},{"game":"Krull","seed":94.0,"HNS":1.6396065574,"Max HNS":1.6503325527},{"game":"Krull","seed":95.0,"HNS":1.2527025761,"Max HNS":1.3312786885},{"game":"Krull","seed":96.0,"HNS":1.0728618267,"Max HNS":1.0728618267},{"game":"Krull","seed":97.0,"HNS":1.3091241218,"Max HNS":1.5868946136},{"game":"Krull","seed":98.0,"HNS":0.9752693208,"Max HNS":1.0794192037},{"game":"Krull","seed":99.0,"HNS":1.4350725995,"Max HNS":1.5466042155},{"game":"Krull","seed":100.0,"HNS":1.1710632319,"Max HNS":2.0241311475},{"game":"KungFuMaster","seed":1.0,"HNS":0.2225974072,"Max HNS":0.2604124959},{"game":"KungFuMaster","seed":2.0,"HNS":0.7778118855,"Max HNS":1.0429179012},{"game":"KungFuMaster","seed":3.0,"HNS":0.9617711698,"Max HNS":0.9617711698},{"game":"KungFuMaster","seed":4.0,"HNS":0.0895327835,"Max HNS":0.0895327835},{"game":"KungFuMaster","seed":5.0,"HNS":0.0518956482,"Max HNS":0.0518956482},{"game":"KungFuMaster","seed":6.0,"HNS":0.0048714732,"Max HNS":0.0218660189},{"game":"KungFuMaster","seed":7.0,"HNS":0.4490430558,"Max HNS":0.4490430558},{"game":"KungFuMaster","seed":8.0,"HNS":0.2818558756,"Max HNS":0.3048118588},{"game":"KungFuMaster","seed":9.0,"HNS":0.2130324142,"Max HNS":0.358420308},{"game":"KungFuMaster","seed":10.0,"HNS":0.0993647065,"Max HNS":0.0993647065},{"game":"KungFuMaster","seed":11.0,"HNS":0.0962950111,"Max HNS":0.1191620176},{"game":"KungFuMaster","seed":12.0,"HNS":0.362735677,"Max HNS":0.381999128},{"game":"KungFuMaster","seed":13.0,"HNS":0.7246038313,"Max HNS":0.7246038313},{"game":"KungFuMaster","seed":14.0,"HNS":0.3716778332,"Max HNS":0.3716778332},{"game":"KungFuMaster","seed":15.0,"HNS":0.0971847779,"Max HNS":0.0971847779},{"game":"KungFuMaster","seed":16.0,"HNS":0.9487805746,"Max HNS":0.9487805746},{"game":"KungFuMaster","seed":17.0,"HNS":0.3131756667,"Max HNS":0.4340504854},{"game":"KungFuMaster","seed":18.0,"HNS":0.114357277,"Max HNS":0.3166012688},{"game":"KungFuMaster","seed":19.0,"HNS":0.4925971403,"Max HNS":0.4925971403},{"game":"KungFuMaster","seed":20.0,"HNS":0.1082178861,"Max HNS":0.1411392574},{"game":"KungFuMaster","seed":21.0,"HNS":0.0126124443,"Max HNS":0.0399727731},{"game":"KungFuMaster","seed":22.0,"HNS":0.0312530586,"Max HNS":0.0470464191},{"game":"KungFuMaster","seed":23.0,"HNS":0.1016781002,"Max HNS":0.1063493758},{"game":"KungFuMaster","seed":24.0,"HNS":0.0765421883,"Max HNS":0.1447428129},{"game":"KungFuMaster","seed":25.0,"HNS":0.2281139613,"Max HNS":0.2281139613},{"game":"KungFuMaster","seed":26.0,"HNS":0.1424294192,"Max HNS":0.1424294192},{"game":"KungFuMaster","seed":27.0,"HNS":0.3785735259,"Max HNS":0.3785735259},{"game":"KungFuMaster","seed":28.0,"HNS":0.3400466238,"Max HNS":0.4459733604},{"game":"KungFuMaster","seed":29.0,"HNS":0.0296514783,"Max HNS":0.0453113739},{"game":"KungFuMaster","seed":30.0,"HNS":0.3523698939,"Max HNS":0.3523698939},{"game":"KungFuMaster","seed":31.0,"HNS":0.8262596873,"Max HNS":0.9038473516},{"game":"KungFuMaster","seed":32.0,"HNS":0.0859737163,"Max HNS":0.1407388623},{"game":"KungFuMaster","seed":33.0,"HNS":0.0703138207,"Max HNS":0.0874863198},{"game":"KungFuMaster","seed":34.0,"HNS":0.8336892401,"Max HNS":0.8336892401},{"game":"KungFuMaster","seed":35.0,"HNS":0.8407183977,"Max HNS":1.0357107902},{"game":"KungFuMaster","seed":36.0,"HNS":0.0688457055,"Max HNS":0.1893201292},{"game":"KungFuMaster","seed":37.0,"HNS":0.1865173638,"Max HNS":0.3001850715},{"game":"KungFuMaster","seed":38.0,"HNS":0.1245006184,"Max HNS":0.1584452215},{"game":"KungFuMaster","seed":39.0,"HNS":0.0479806743,"Max HNS":0.0479806743},{"game":"KungFuMaster","seed":40.0,"HNS":0.5573721628,"Max HNS":0.5573721628},{"game":"KungFuMaster","seed":41.0,"HNS":0.2124095774,"Max HNS":0.2124095774},{"game":"KungFuMaster","seed":42.0,"HNS":0.3947672815,"Max HNS":0.4140307325},{"game":"KungFuMaster","seed":43.0,"HNS":0.1009217984,"Max HNS":0.1009217984},{"game":"KungFuMaster","seed":44.0,"HNS":0.4247079341,"Max HNS":0.4247079341},{"game":"KungFuMaster","seed":45.0,"HNS":0.2050690014,"Max HNS":0.2050690014},{"game":"KungFuMaster","seed":46.0,"HNS":0.1634279155,"Max HNS":0.2302494016},{"game":"KungFuMaster","seed":47.0,"HNS":0.0222219256,"Max HNS":0.034589684},{"game":"KungFuMaster","seed":48.0,"HNS":0.6761115412,"Max HNS":0.7003131979},{"game":"KungFuMaster","seed":49.0,"HNS":0.4110055255,"Max HNS":0.4110055255},{"game":"KungFuMaster","seed":50.0,"HNS":1.190530212,"Max HNS":1.190530212},{"game":"KungFuMaster","seed":51.0,"HNS":0.116982089,"Max HNS":0.1673428894},{"game":"KungFuMaster","seed":52.0,"HNS":0.3162453621,"Max HNS":0.4510895194},{"game":"KungFuMaster","seed":53.0,"HNS":0.15955743,"Max HNS":0.15955743},{"game":"KungFuMaster","seed":54.0,"HNS":0.0127903976,"Max HNS":0.024624296},{"game":"KungFuMaster","seed":55.0,"HNS":0.1987961455,"Max HNS":0.1987961455},{"game":"KungFuMaster","seed":56.0,"HNS":0.6282420877,"Max HNS":0.6282420877},{"game":"KungFuMaster","seed":57.0,"HNS":0.275093648,"Max HNS":0.275093648},{"game":"KungFuMaster","seed":58.0,"HNS":0.7584594578,"Max HNS":0.8907232914},{"game":"KungFuMaster","seed":59.0,"HNS":0.7043616368,"Max HNS":0.7043616368},{"game":"KungFuMaster","seed":60.0,"HNS":0.327767842,"Max HNS":0.8553550614},{"game":"KungFuMaster","seed":61.0,"HNS":0.1030127504,"Max HNS":0.1030127504},{"game":"KungFuMaster","seed":62.0,"HNS":0.9426856721,"Max HNS":0.9426856721},{"game":"KungFuMaster","seed":63.0,"HNS":0.020442392,"Max HNS":0.0296514783},{"game":"KungFuMaster","seed":64.0,"HNS":0.0545204602,"Max HNS":0.0545204602},{"game":"KungFuMaster","seed":65.0,"HNS":0.0803236972,"Max HNS":0.0816583473},{"game":"KungFuMaster","seed":66.0,"HNS":0.2796759469,"Max HNS":0.3437836443},{"game":"KungFuMaster","seed":67.0,"HNS":0.1571105713,"Max HNS":0.1571105713},{"game":"KungFuMaster","seed":68.0,"HNS":0.8805354617,"Max HNS":1.3709749175},{"game":"KungFuMaster","seed":69.0,"HNS":0.2005311908,"Max HNS":0.2463096922},{"game":"KungFuMaster","seed":70.0,"HNS":1.0434962496,"Max HNS":1.0434962496},{"game":"KungFuMaster","seed":71.0,"HNS":0.015459698,"Max HNS":0.0193746719},{"game":"KungFuMaster","seed":72.0,"HNS":0.0948268959,"Max HNS":0.0948268959},{"game":"KungFuMaster","seed":73.0,"HNS":0.1608031035,"Max HNS":0.1608031035},{"game":"KungFuMaster","seed":74.0,"HNS":0.069557519,"Max HNS":0.069557519},{"game":"KungFuMaster","seed":75.0,"HNS":0.6390082659,"Max HNS":0.6390082659},{"game":"KungFuMaster","seed":76.0,"HNS":0.1366459351,"Max HNS":0.152216854},{"game":"KungFuMaster","seed":77.0,"HNS":0.3230965664,"Max HNS":0.3230965664},{"game":"KungFuMaster","seed":78.0,"HNS":0.5444705443,"Max HNS":0.5444705443},{"game":"KungFuMaster","seed":79.0,"HNS":0.058168504,"Max HNS":0.1350443549},{"game":"KungFuMaster","seed":80.0,"HNS":0.1490136935,"Max HNS":0.3230075897},{"game":"KungFuMaster","seed":81.0,"HNS":0.0829929975,"Max HNS":0.1620932654},{"game":"KungFuMaster","seed":82.0,"HNS":0.2516927813,"Max HNS":0.2516927813},{"game":"KungFuMaster","seed":83.0,"HNS":0.1295278008,"Max HNS":0.1295278008},{"game":"KungFuMaster","seed":84.0,"HNS":0.152350319,"Max HNS":0.152350319},{"game":"KungFuMaster","seed":85.0,"HNS":0.5281433236,"Max HNS":0.7713610763},{"game":"KungFuMaster","seed":86.0,"HNS":0.0100321206,"Max HNS":0.0389495413},{"game":"KungFuMaster","seed":87.0,"HNS":0.9191958288,"Max HNS":0.9191958288},{"game":"KungFuMaster","seed":88.0,"HNS":0.062350408,"Max HNS":0.0877087615},{"game":"KungFuMaster","seed":89.0,"HNS":0.6089341484,"Max HNS":0.745646816},{"game":"KungFuMaster","seed":90.0,"HNS":0.2936007972,"Max HNS":0.4743569211},{"game":"KungFuMaster","seed":91.0,"HNS":1.21175115,"Max HNS":1.3869462314},{"game":"KungFuMaster","seed":92.0,"HNS":0.9911779623,"Max HNS":1.3703075924},{"game":"KungFuMaster","seed":93.0,"HNS":1.5056411215,"Max HNS":1.5056411215},{"game":"KungFuMaster","seed":94.0,"HNS":0.3344855813,"Max HNS":0.4514009378},{"game":"KungFuMaster","seed":95.0,"HNS":0.150125902,"Max HNS":0.3791518743},{"game":"KungFuMaster","seed":96.0,"HNS":0.9262249864,"Max HNS":1.2398677807},{"game":"KungFuMaster","seed":97.0,"HNS":1.0749495057,"Max HNS":1.0749495057},{"game":"KungFuMaster","seed":98.0,"HNS":0.0002446859,"Max HNS":0.1299281958},{"game":"KungFuMaster","seed":99.0,"HNS":0.1109316748,"Max HNS":0.1907437561},{"game":"KungFuMaster","seed":100.0,"HNS":0.4186575199,"Max HNS":0.44357099},{"game":"MsPacman","seed":1.0,"HNS":0.0738377256,"Max HNS":0.1350480863},{"game":"MsPacman","seed":2.0,"HNS":0.1791911864,"Max HNS":0.2693586984},{"game":"MsPacman","seed":3.0,"HNS":0.099513869,"Max HNS":0.107791641},{"game":"MsPacman","seed":4.0,"HNS":0.2257273151,"Max HNS":0.2257273151},{"game":"MsPacman","seed":5.0,"HNS":0.0902427645,"Max HNS":0.1087699231},{"game":"MsPacman","seed":6.0,"HNS":0.1468025225,"Max HNS":0.1738934124},{"game":"MsPacman","seed":7.0,"HNS":0.1012747769,"Max HNS":0.1476453501},{"game":"MsPacman","seed":8.0,"HNS":0.1354544497,"Max HNS":0.1508511055},{"game":"MsPacman","seed":9.0,"HNS":0.0951943771,"Max HNS":0.1416552534},{"game":"MsPacman","seed":10.0,"HNS":0.0705416673,"Max HNS":0.1348674804},{"game":"MsPacman","seed":11.0,"HNS":0.0906340773,"Max HNS":0.2475956835},{"game":"MsPacman","seed":12.0,"HNS":0.1614466535,"Max HNS":0.2232439836},{"game":"MsPacman","seed":13.0,"HNS":0.1019369986,"Max HNS":0.156194031},{"game":"MsPacman","seed":14.0,"HNS":0.2526978011,"Max HNS":0.2526978011},{"game":"MsPacman","seed":15.0,"HNS":0.1606339268,"Max HNS":0.1606339268},{"game":"MsPacman","seed":16.0,"HNS":0.0582303629,"Max HNS":0.0917327634},{"game":"MsPacman","seed":17.0,"HNS":0.2075162169,"Max HNS":0.2790361663},{"game":"MsPacman","seed":18.0,"HNS":0.091597309,"Max HNS":0.20182713},{"game":"MsPacman","seed":19.0,"HNS":0.0674864169,"Max HNS":0.1106963864},{"game":"MsPacman","seed":20.0,"HNS":0.084749334,"Max HNS":0.0971509414},{"game":"MsPacman","seed":21.0,"HNS":0.0854416568,"Max HNS":0.1687160423},{"game":"MsPacman","seed":22.0,"HNS":0.091567208,"Max HNS":0.108920428},{"game":"MsPacman","seed":23.0,"HNS":0.125235164,"Max HNS":0.148006562},{"game":"MsPacman","seed":24.0,"HNS":0.0629261171,"Max HNS":0.1794470448},{"game":"MsPacman","seed":25.0,"HNS":0.0803696401,"Max HNS":0.1170627455},{"game":"MsPacman","seed":26.0,"HNS":0.0626100567,"Max HNS":0.1517240341},{"game":"MsPacman","seed":27.0,"HNS":0.1147750704,"Max HNS":0.2102855079},{"game":"MsPacman","seed":28.0,"HNS":0.1190494108,"Max HNS":0.1190494108},{"game":"MsPacman","seed":29.0,"HNS":0.0948632663,"Max HNS":0.111629517},{"game":"MsPacman","seed":30.0,"HNS":0.1434613127,"Max HNS":0.2010144033},{"game":"MsPacman","seed":31.0,"HNS":0.1016359887,"Max HNS":0.1016359887},{"game":"MsPacman","seed":32.0,"HNS":0.1674819018,"Max HNS":0.1674819018},{"game":"MsPacman","seed":33.0,"HNS":0.2201435817,"Max HNS":0.2201435817},{"game":"MsPacman","seed":34.0,"HNS":0.1824571437,"Max HNS":0.1824571437},{"game":"MsPacman","seed":35.0,"HNS":0.0517435998,"Max HNS":0.1223003176},{"game":"MsPacman","seed":36.0,"HNS":0.1058802282,"Max HNS":0.2088557109},{"game":"MsPacman","seed":37.0,"HNS":0.0960974068,"Max HNS":0.1082130548},{"game":"MsPacman","seed":38.0,"HNS":0.0722574237,"Max HNS":0.1000406363},{"game":"MsPacman","seed":39.0,"HNS":0.1106361844,"Max HNS":0.3345273392},{"game":"MsPacman","seed":40.0,"HNS":0.199238445,"Max HNS":0.199238445},{"game":"MsPacman","seed":41.0,"HNS":0.1153319387,"Max HNS":0.1672260434},{"game":"MsPacman","seed":42.0,"HNS":0.1635085713,"Max HNS":0.1727947263},{"game":"MsPacman","seed":43.0,"HNS":0.1127282031,"Max HNS":0.1446804028},{"game":"MsPacman","seed":44.0,"HNS":0.2526375991,"Max HNS":0.2568968891},{"game":"MsPacman","seed":45.0,"HNS":0.087608928,"Max HNS":0.1739385639},{"game":"MsPacman","seed":46.0,"HNS":0.1296299083,"Max HNS":0.1296299083},{"game":"MsPacman","seed":47.0,"HNS":0.1409629306,"Max HNS":0.1409629306},{"game":"MsPacman","seed":48.0,"HNS":0.0813178213,"Max HNS":0.1190343603},{"game":"MsPacman","seed":49.0,"HNS":0.0767575215,"Max HNS":0.1850157278},{"game":"MsPacman","seed":50.0,"HNS":0.0544526888,"Max HNS":0.1017563927},{"game":"MsPacman","seed":51.0,"HNS":0.0667489427,"Max HNS":0.0721370197},{"game":"MsPacman","seed":52.0,"HNS":0.1424228286,"Max HNS":0.1424228286},{"game":"MsPacman","seed":53.0,"HNS":0.1666992761,"Max HNS":0.1666992761},{"game":"MsPacman","seed":54.0,"HNS":0.0682991436,"Max HNS":0.1070993182},{"game":"MsPacman","seed":55.0,"HNS":0.1659166504,"Max HNS":0.2561744653},{"game":"MsPacman","seed":56.0,"HNS":0.0543924868,"Max HNS":0.115647999},{"game":"MsPacman","seed":57.0,"HNS":0.1977634965,"Max HNS":0.3246090634},{"game":"MsPacman","seed":58.0,"HNS":0.0734464127,"Max HNS":0.1365531358},{"game":"MsPacman","seed":59.0,"HNS":0.115708201,"Max HNS":0.216802372},{"game":"MsPacman","seed":60.0,"HNS":0.1045256837,"Max HNS":0.1425281821},{"game":"MsPacman","seed":61.0,"HNS":0.0614361182,"Max HNS":0.0757340879},{"game":"MsPacman","seed":62.0,"HNS":0.0674412654,"Max HNS":0.0918531674},{"game":"MsPacman","seed":63.0,"HNS":0.1191698147,"Max HNS":0.1191698147},{"game":"MsPacman","seed":64.0,"HNS":0.1899673404,"Max HNS":0.1899673404},{"game":"MsPacman","seed":65.0,"HNS":0.106301642,"Max HNS":0.2156284334},{"game":"MsPacman","seed":66.0,"HNS":0.139849194,"Max HNS":0.1482624204},{"game":"MsPacman","seed":67.0,"HNS":0.1016058878,"Max HNS":0.1425281821},{"game":"MsPacman","seed":68.0,"HNS":0.1003717472,"Max HNS":0.1184925425},{"game":"MsPacman","seed":69.0,"HNS":0.0568758184,"Max HNS":0.1056996222},{"game":"MsPacman","seed":70.0,"HNS":0.0967596286,"Max HNS":0.096864982},{"game":"MsPacman","seed":71.0,"HNS":0.0762608552,"Max HNS":0.1204942582},{"game":"MsPacman","seed":72.0,"HNS":0.0898815526,"Max HNS":0.130502837},{"game":"MsPacman","seed":73.0,"HNS":0.1294041509,"Max HNS":0.1294041509},{"game":"MsPacman","seed":74.0,"HNS":0.1198019355,"Max HNS":0.1232785997},{"game":"MsPacman","seed":75.0,"HNS":0.1051427539,"Max HNS":0.1051427539},{"game":"MsPacman","seed":76.0,"HNS":0.1362972774,"Max HNS":0.164261096},{"game":"MsPacman","seed":77.0,"HNS":0.1274024352,"Max HNS":0.1875893623},{"game":"MsPacman","seed":78.0,"HNS":0.1041193203,"Max HNS":0.1041193203},{"game":"MsPacman","seed":79.0,"HNS":0.1215327424,"Max HNS":0.1215327424},{"game":"MsPacman","seed":80.0,"HNS":0.0829282242,"Max HNS":0.1056394203},{"game":"MsPacman","seed":81.0,"HNS":0.0928916515,"Max HNS":0.1351534398},{"game":"MsPacman","seed":82.0,"HNS":0.0741086345,"Max HNS":0.1149105248},{"game":"MsPacman","seed":83.0,"HNS":0.098294779,"Max HNS":0.1223304186},{"game":"MsPacman","seed":84.0,"HNS":0.1325948557,"Max HNS":0.1325948557},{"game":"MsPacman","seed":85.0,"HNS":0.0566952124,"Max HNS":0.0868263022},{"game":"MsPacman","seed":86.0,"HNS":0.1777914905,"Max HNS":0.2142889394},{"game":"MsPacman","seed":87.0,"HNS":0.1167918366,"Max HNS":0.1167918366},{"game":"MsPacman","seed":88.0,"HNS":0.130502837,"Max HNS":0.130502837},{"game":"MsPacman","seed":89.0,"HNS":0.2129945969,"Max HNS":0.2129945969},{"game":"MsPacman","seed":90.0,"HNS":0.0419457279,"Max HNS":0.0857276162},{"game":"MsPacman","seed":91.0,"HNS":0.1236398116,"Max HNS":0.210014599},{"game":"MsPacman","seed":92.0,"HNS":0.1366885902,"Max HNS":0.1366885902},{"game":"MsPacman","seed":93.0,"HNS":0.0563490511,"Max HNS":0.0836205469},{"game":"MsPacman","seed":94.0,"HNS":0.1178905227,"Max HNS":0.1259124362},{"game":"MsPacman","seed":95.0,"HNS":0.0405159309,"Max HNS":0.196499255},{"game":"MsPacman","seed":96.0,"HNS":0.1369293981,"Max HNS":0.176436946},{"game":"MsPacman","seed":97.0,"HNS":0.0993934651,"Max HNS":0.1702812937},{"game":"MsPacman","seed":98.0,"HNS":0.0919585208,"Max HNS":0.1041644718},{"game":"MsPacman","seed":99.0,"HNS":0.075342775,"Max HNS":0.1223906205},{"game":"MsPacman","seed":100.0,"HNS":0.0895805427,"Max HNS":0.2114895474},{"game":"Pong","seed":1.0,"HNS":0.0849858357,"Max HNS":0.0849858357},{"game":"Pong","seed":2.0,"HNS":0.0402266289,"Max HNS":0.054674221},{"game":"Pong","seed":3.0,"HNS":0.0388101983,"Max HNS":0.0388101983},{"game":"Pong","seed":4.0,"HNS":0.0521246459,"Max HNS":0.0654390935},{"game":"Pong","seed":5.0,"HNS":0.0229461756,"Max HNS":0.1033994334},{"game":"Pong","seed":6.0,"HNS":0.0354107649,"Max HNS":0.0603399433},{"game":"Pong","seed":7.0,"HNS":-0.0084985836,"Max HNS":0.0067988669},{"game":"Pong","seed":8.0,"HNS":0.0059490085,"Max HNS":0.0212464589},{"game":"Pong","seed":9.0,"HNS":0.4900849858,"Max HNS":0.4900849858},{"game":"Pong","seed":10.0,"HNS":0.0946175637,"Max HNS":0.0946175637},{"game":"Pong","seed":11.0,"HNS":0.054674221,"Max HNS":0.054674221},{"game":"Pong","seed":12.0,"HNS":0.0025495751,"Max HNS":0.0050991501},{"game":"Pong","seed":13.0,"HNS":0.133427762,"Max HNS":0.2337110482},{"game":"Pong","seed":14.0,"HNS":0.040509915,"Max HNS":0.040509915},{"game":"Pong","seed":15.0,"HNS":0.0014164306,"Max HNS":0.0014164306},{"game":"Pong","seed":16.0,"HNS":0.0609065156,"Max HNS":0.0609065156},{"game":"Pong","seed":17.0,"HNS":0.0056657224,"Max HNS":0.0368271955},{"game":"Pong","seed":18.0,"HNS":0.0464589235,"Max HNS":0.1042492918},{"game":"Pong","seed":19.0,"HNS":0.0322946176,"Max HNS":0.0322946176},{"game":"Pong","seed":20.0,"HNS":0.0209631728,"Max HNS":0.0478753541},{"game":"Pong","seed":21.0,"HNS":0.0257790368,"Max HNS":0.0368271955},{"game":"Pong","seed":22.0,"HNS":0.1390934844,"Max HNS":0.1390934844},{"game":"Pong","seed":23.0,"HNS":0.1685552408,"Max HNS":0.1685552408},{"game":"Pong","seed":24.0,"HNS":0.1050991501,"Max HNS":0.1311614731},{"game":"Pong","seed":25.0,"HNS":0.0747875354,"Max HNS":0.1696883853},{"game":"Pong","seed":26.0,"HNS":0.1501416431,"Max HNS":0.1501416431},{"game":"Pong","seed":27.0,"HNS":0.0345609065,"Max HNS":0.0971671388},{"game":"Pong","seed":28.0,"HNS":0.0830028329,"Max HNS":0.0830028329},{"game":"Pong","seed":29.0,"HNS":0.0498583569,"Max HNS":0.0654390935},{"game":"Pong","seed":30.0,"HNS":0.0130311615,"Max HNS":0.0696883853},{"game":"Pong","seed":31.0,"HNS":0.0076487252,"Max HNS":0.0688385269},{"game":"Pong","seed":32.0,"HNS":0.1101983003,"Max HNS":0.1101983003},{"game":"Pong","seed":33.0,"HNS":0.1192634561,"Max HNS":0.1192634561},{"game":"Pong","seed":34.0,"HNS":0.0229461756,"Max HNS":0.0552407932},{"game":"Pong","seed":35.0,"HNS":0.0399433428,"Max HNS":0.0538243626},{"game":"Pong","seed":36.0,"HNS":0.0614730878,"Max HNS":0.2490084986},{"game":"Pong","seed":37.0,"HNS":0.1872521246,"Max HNS":0.1994334278},{"game":"Pong","seed":38.0,"HNS":0.0331444759,"Max HNS":0.0495750708},{"game":"Pong","seed":39.0,"HNS":0.1067988669,"Max HNS":0.1390934844},{"game":"Pong","seed":40.0,"HNS":0.1549575071,"Max HNS":0.1824362606},{"game":"Pong","seed":41.0,"HNS":0.0736543909,"Max HNS":0.088101983},{"game":"Pong","seed":42.0,"HNS":0.0781869688,"Max HNS":0.0781869688},{"game":"Pong","seed":43.0,"HNS":0.0331444759,"Max HNS":0.0934844193},{"game":"Pong","seed":44.0,"HNS":0.0110481586,"Max HNS":0.0110481586},{"game":"Pong","seed":45.0,"HNS":0.0005665722,"Max HNS":0.002266289},{"game":"Pong","seed":46.0,"HNS":0.0898016997,"Max HNS":0.0898016997},{"game":"Pong","seed":47.0,"HNS":0.104815864,"Max HNS":0.104815864},{"game":"Pong","seed":48.0,"HNS":0.1396600567,"Max HNS":0.1396600567},{"game":"Pong","seed":49.0,"HNS":0.0155807365,"Max HNS":0.0226628895},{"game":"Pong","seed":50.0,"HNS":0.0447592068,"Max HNS":0.0852691218},{"game":"Pong","seed":51.0,"HNS":0.095184136,"Max HNS":0.095184136},{"game":"Pong","seed":52.0,"HNS":0.0722379603,"Max HNS":0.0722379603},{"game":"Pong","seed":53.0,"HNS":0.0487252125,"Max HNS":0.1634560907},{"game":"Pong","seed":54.0,"HNS":0.1158640227,"Max HNS":0.1158640227},{"game":"Pong","seed":55.0,"HNS":0.0770538244,"Max HNS":0.0770538244},{"game":"Pong","seed":56.0,"HNS":0.0337110482,"Max HNS":0.0337110482},{"game":"Pong","seed":57.0,"HNS":0.1274787535,"Max HNS":0.1274787535},{"game":"Pong","seed":58.0,"HNS":0.107082153,"Max HNS":0.107082153},{"game":"Pong","seed":59.0,"HNS":0.0039660057,"Max HNS":0.011898017},{"game":"Pong","seed":60.0,"HNS":0.0331444759,"Max HNS":0.0331444759},{"game":"Pong","seed":61.0,"HNS":0.0198300283,"Max HNS":0.0198300283},{"game":"Pong","seed":62.0,"HNS":0.0371104816,"Max HNS":0.0458923513},{"game":"Pong","seed":63.0,"HNS":0.0583569405,"Max HNS":0.071388102},{"game":"Pong","seed":64.0,"HNS":0.1084985836,"Max HNS":0.1084985836},{"game":"Pong","seed":65.0,"HNS":0.042776204,"Max HNS":0.0637393768},{"game":"Pong","seed":66.0,"HNS":0.095184136,"Max HNS":0.1456090652},{"game":"Pong","seed":67.0,"HNS":0.1169971671,"Max HNS":0.1294617564},{"game":"Pong","seed":68.0,"HNS":0.0147308782,"Max HNS":0.0271954674},{"game":"Pong","seed":69.0,"HNS":0.0461756374,"Max HNS":0.0535410765},{"game":"Pong","seed":70.0,"HNS":0.0229461756,"Max HNS":0.0229461756},{"game":"Pong","seed":71.0,"HNS":0.0147308782,"Max HNS":0.0147308782},{"game":"Pong","seed":72.0,"HNS":0.0269121813,"Max HNS":0.0269121813},{"game":"Pong","seed":73.0,"HNS":0.0736543909,"Max HNS":0.0736543909},{"game":"Pong","seed":74.0,"HNS":0.007082153,"Max HNS":0.0135977337},{"game":"Pong","seed":75.0,"HNS":0.1093484419,"Max HNS":0.1093484419},{"game":"Pong","seed":76.0,"HNS":0.1813031161,"Max HNS":0.2271954674},{"game":"Pong","seed":77.0,"HNS":0.0354107649,"Max HNS":0.0549575071},{"game":"Pong","seed":78.0,"HNS":0.0883852691,"Max HNS":0.0883852691},{"game":"Pong","seed":79.0,"HNS":0.0152974504,"Max HNS":0.0192634561},{"game":"Pong","seed":80.0,"HNS":0.0861189802,"Max HNS":0.0861189802},{"game":"Pong","seed":81.0,"HNS":0.0345609065,"Max HNS":0.0838526912},{"game":"Pong","seed":82.0,"HNS":0.0328611898,"Max HNS":0.0328611898},{"game":"Pong","seed":83.0,"HNS":0.0303116147,"Max HNS":0.0458923513},{"game":"Pong","seed":84.0,"HNS":0.1291784703,"Max HNS":0.1291784703},{"game":"Pong","seed":85.0,"HNS":0.0257790368,"Max HNS":0.0481586402},{"game":"Pong","seed":86.0,"HNS":0.1679886686,"Max HNS":0.1679886686},{"game":"Pong","seed":87.0,"HNS":0.0186968839,"Max HNS":0.0192634561},{"game":"Pong","seed":88.0,"HNS":0.0906515581,"Max HNS":0.0906515581},{"game":"Pong","seed":89.0,"HNS":0.0379603399,"Max HNS":0.0379603399},{"game":"Pong","seed":90.0,"HNS":0.0441926346,"Max HNS":0.0441926346},{"game":"Pong","seed":91.0,"HNS":-0.0073654391,"Max HNS":-0.0016997167},{"game":"Pong","seed":92.0,"HNS":0.0229461756,"Max HNS":0.0373937677},{"game":"Pong","seed":93.0,"HNS":0.040509915,"Max HNS":0.040509915},{"game":"Pong","seed":94.0,"HNS":0.0025495751,"Max HNS":0.0223796034},{"game":"Pong","seed":95.0,"HNS":-0.0019830028,"Max HNS":-0.0019830028},{"game":"Pong","seed":96.0,"HNS":0.023796034,"Max HNS":0.0745042493},{"game":"Pong","seed":97.0,"HNS":0.0373937677,"Max HNS":0.0575070822},{"game":"Pong","seed":98.0,"HNS":0.0430594901,"Max HNS":0.0521246459},{"game":"Pong","seed":99.0,"HNS":-0.0079320113,"Max HNS":0.0087818697},{"game":"Pong","seed":100.0,"HNS":0.0532577904,"Max HNS":0.0532577904},{"game":"PrivateEye","seed":1.0,"HNS":0.0010798546,"Max HNS":0.0010798546},{"game":"PrivateEye","seed":2.0,"HNS":0.0010798546,"Max HNS":0.0010798546},{"game":"PrivateEye","seed":3.0,"HNS":0.0010798546,"Max HNS":0.0010798546},{"game":"PrivateEye","seed":4.0,"HNS":-0.0017779497,"Max HNS":0.0010798546},{"game":"PrivateEye","seed":5.0,"HNS":0.0010798546,"Max HNS":0.0010798546},{"game":"PrivateEye","seed":6.0,"HNS":0.0010798546,"Max HNS":0.0010798546},{"game":"PrivateEye","seed":7.0,"HNS":0.0010798546,"Max HNS":0.0010798546},{"game":"PrivateEye","seed":8.0,"HNS":0.0010798546,"Max HNS":0.0010798546},{"game":"PrivateEye","seed":9.0,"HNS":0.0010798546,"Max HNS":0.0010798546},{"game":"PrivateEye","seed":10.0,"HNS":0.0010798546,"Max HNS":0.0010798546},{"game":"PrivateEye","seed":11.0,"HNS":0.0003150415,"Max HNS":0.0010798546},{"game":"PrivateEye","seed":12.0,"HNS":0.0010798546,"Max HNS":0.0010798546},{"game":"PrivateEye","seed":13.0,"HNS":0.0010798546,"Max HNS":0.0010798546},{"game":"PrivateEye","seed":14.0,"HNS":0.0053360059,"Max HNS":0.0053360059},{"game":"PrivateEye","seed":15.0,"HNS":0.0010798546,"Max HNS":0.0010798546},{"game":"PrivateEye","seed":16.0,"HNS":0.0010798546,"Max HNS":0.0010798546},{"game":"PrivateEye","seed":17.0,"HNS":-0.0023532491,"Max HNS":0.0010654757},{"game":"PrivateEye","seed":18.0,"HNS":0.0010798546,"Max HNS":0.0010798546},{"game":"PrivateEye","seed":19.0,"HNS":0.0008641713,"Max HNS":0.0010798546},{"game":"PrivateEye","seed":20.0,"HNS":0.0010798546,"Max HNS":0.0010798546},{"game":"PrivateEye","seed":21.0,"HNS":0.0010798546,"Max HNS":0.0010798546},{"game":"PrivateEye","seed":22.0,"HNS":0.0010798546,"Max HNS":0.0010798546},{"game":"PrivateEye","seed":23.0,"HNS":-0.0029568748,"Max HNS":0.0010798546},{"game":"PrivateEye","seed":24.0,"HNS":0.0010798546,"Max HNS":0.0010798546},{"game":"PrivateEye","seed":25.0,"HNS":0.0010798546,"Max HNS":0.0010798546},{"game":"PrivateEye","seed":26.0,"HNS":0.0010798546,"Max HNS":0.0010798546},{"game":"PrivateEye","seed":27.0,"HNS":0.0010798546,"Max HNS":0.0010798546},{"game":"PrivateEye","seed":28.0,"HNS":0.0003949881,"Max HNS":0.0010798546},{"game":"PrivateEye","seed":29.0,"HNS":0.0010798546,"Max HNS":0.0010798546},{"game":"PrivateEye","seed":30.0,"HNS":0.0010798546,"Max HNS":0.0010798546},{"game":"PrivateEye","seed":31.0,"HNS":0.0010798546,"Max HNS":0.0225043999},{"game":"PrivateEye","seed":32.0,"HNS":0.0010798546,"Max HNS":0.0031857868},{"game":"PrivateEye","seed":33.0,"HNS":0.0010798546,"Max HNS":0.0010798546},{"game":"PrivateEye","seed":34.0,"HNS":0.0010798546,"Max HNS":0.0010798546},{"game":"PrivateEye","seed":35.0,"HNS":0.0010798546,"Max HNS":0.003469626},{"game":"PrivateEye","seed":36.0,"HNS":0.0005190779,"Max HNS":0.0010798546},{"game":"PrivateEye","seed":37.0,"HNS":-0.0036068006,"Max HNS":0.0027426294},{"game":"PrivateEye","seed":38.0,"HNS":0.0010798546,"Max HNS":0.0010798546},{"game":"PrivateEye","seed":39.0,"HNS":0.0010798546,"Max HNS":0.0010798546},{"game":"PrivateEye","seed":40.0,"HNS":0.0008824324,"Max HNS":0.0010798546},{"game":"PrivateEye","seed":41.0,"HNS":0.0008056492,"Max HNS":0.0010798546},{"game":"PrivateEye","seed":42.0,"HNS":0.0010798546,"Max HNS":0.0010798546},{"game":"PrivateEye","seed":43.0,"HNS":0.0010798546,"Max HNS":0.0010798546},{"game":"PrivateEye","seed":44.0,"HNS":-0.0047014655,"Max HNS":0.0013346485},{"game":"PrivateEye","seed":45.0,"HNS":-0.0009147849,"Max HNS":-0.0001853439},{"game":"PrivateEye","seed":46.0,"HNS":-0.0003580343,"Max HNS":-0.0003580343},{"game":"PrivateEye","seed":47.0,"HNS":0.0010798546,"Max HNS":0.0093551931},{"game":"PrivateEye","seed":48.0,"HNS":0.0010798546,"Max HNS":0.0010798546},{"game":"PrivateEye","seed":49.0,"HNS":0.0043670125,"Max HNS":0.0043670125},{"game":"PrivateEye","seed":50.0,"HNS":0.0010798546,"Max HNS":0.0010798546},{"game":"PrivateEye","seed":51.0,"HNS":0.0010798546,"Max HNS":0.0010798546},{"game":"PrivateEye","seed":52.0,"HNS":0.0010798546,"Max HNS":0.0010798546},{"game":"PrivateEye","seed":53.0,"HNS":-0.0003580343,"Max HNS":0.042303268},{"game":"PrivateEye","seed":54.0,"HNS":0.000546254,"Max HNS":0.0010798546},{"game":"PrivateEye","seed":55.0,"HNS":0.0010798546,"Max HNS":0.0010798546},{"game":"PrivateEye","seed":56.0,"HNS":0.0010798546,"Max HNS":0.0010798546},{"game":"PrivateEye","seed":57.0,"HNS":-0.0003580343,"Max HNS":0.0130373391},{"game":"PrivateEye","seed":58.0,"HNS":-0.0003292766,"Max HNS":-0.0003292766},{"game":"PrivateEye","seed":59.0,"HNS":0.0010798546,"Max HNS":0.0025321224},{"game":"PrivateEye","seed":60.0,"HNS":0.0010798546,"Max HNS":0.0010798546},{"game":"PrivateEye","seed":61.0,"HNS":0.0010798546,"Max HNS":0.0010798546},{"game":"PrivateEye","seed":62.0,"HNS":0.0010798546,"Max HNS":0.0010798546},{"game":"PrivateEye","seed":63.0,"HNS":-0.0026708787,"Max HNS":0.0010798546},{"game":"PrivateEye","seed":64.0,"HNS":0.0010798546,"Max HNS":0.0010798546},{"game":"PrivateEye","seed":65.0,"HNS":-0.0003580343,"Max HNS":-0.0003580343},{"game":"PrivateEye","seed":66.0,"HNS":0.0010798546,"Max HNS":0.0010798546},{"game":"PrivateEye","seed":67.0,"HNS":0.0010798546,"Max HNS":0.0010798546},{"game":"PrivateEye","seed":68.0,"HNS":0.0010798546,"Max HNS":0.0010798546},{"game":"PrivateEye","seed":69.0,"HNS":0.0010798546,"Max HNS":0.0010798546},{"game":"PrivateEye","seed":70.0,"HNS":0.0010798546,"Max HNS":0.0010798546},{"game":"PrivateEye","seed":71.0,"HNS":0.0010798546,"Max HNS":0.0010798546},{"game":"PrivateEye","seed":72.0,"HNS":0.0010798546,"Max HNS":0.0010798546},{"game":"PrivateEye","seed":73.0,"HNS":0.0010798546,"Max HNS":0.0010798546},{"game":"PrivateEye","seed":74.0,"HNS":0.0010798546,"Max HNS":0.0010798546},{"game":"PrivateEye","seed":75.0,"HNS":0.0010798546,"Max HNS":0.0010798546},{"game":"PrivateEye","seed":76.0,"HNS":0.0010798546,"Max HNS":0.0010798546},{"game":"PrivateEye","seed":77.0,"HNS":-0.0008922101,"Max HNS":0.0010798546},{"game":"PrivateEye","seed":78.0,"HNS":0.0010798546,"Max HNS":0.0010798546},{"game":"PrivateEye","seed":79.0,"HNS":0.0010798546,"Max HNS":0.0221380258},{"game":"PrivateEye","seed":80.0,"HNS":0.0010798546,"Max HNS":0.0010798546},{"game":"PrivateEye","seed":81.0,"HNS":0.0010798546,"Max HNS":0.0113853485},{"game":"PrivateEye","seed":82.0,"HNS":0.0010798546,"Max HNS":0.0010798546},{"game":"PrivateEye","seed":83.0,"HNS":0.0010798546,"Max HNS":0.0010798546},{"game":"PrivateEye","seed":84.0,"HNS":0.0010798546,"Max HNS":0.0010798546},{"game":"PrivateEye","seed":85.0,"HNS":0.0010798546,"Max HNS":0.0010798546},{"game":"PrivateEye","seed":86.0,"HNS":0.0010798546,"Max HNS":0.0010798546},{"game":"PrivateEye","seed":87.0,"HNS":0.0010798546,"Max HNS":0.0010798546},{"game":"PrivateEye","seed":88.0,"HNS":0.0010798546,"Max HNS":0.0010798546},{"game":"PrivateEye","seed":89.0,"HNS":0.0010798546,"Max HNS":0.0010798546},{"game":"PrivateEye","seed":90.0,"HNS":0.0010798546,"Max HNS":0.0010798546},{"game":"PrivateEye","seed":91.0,"HNS":0.0010798546,"Max HNS":0.0010798546},{"game":"PrivateEye","seed":92.0,"HNS":0.0010798546,"Max HNS":0.0010798546},{"game":"PrivateEye","seed":93.0,"HNS":-0.0003580343,"Max HNS":-0.0003436555},{"game":"PrivateEye","seed":94.0,"HNS":0.0078389392,"Max HNS":0.0078389392},{"game":"PrivateEye","seed":95.0,"HNS":0.0010615934,"Max HNS":0.0076494254},{"game":"PrivateEye","seed":96.0,"HNS":0.0010475021,"Max HNS":0.0438008294},{"game":"PrivateEye","seed":97.0,"HNS":0.0010798546,"Max HNS":0.0010798546},{"game":"PrivateEye","seed":98.0,"HNS":0.0010798546,"Max HNS":0.0010798546},{"game":"PrivateEye","seed":99.0,"HNS":0.0010798546,"Max HNS":0.0010798546},{"game":"PrivateEye","seed":100.0,"HNS":0.0010798546,"Max HNS":0.0010798546},{"game":"Qbert","seed":1.0,"HNS":0.0514893425,"Max HNS":0.0514893425},{"game":"Qbert","seed":2.0,"HNS":0.021375206,"Max HNS":0.0230116394},{"game":"Qbert","seed":3.0,"HNS":0.0880363552,"Max HNS":0.0880363552},{"game":"Qbert","seed":4.0,"HNS":0.0179142434,"Max HNS":0.0295949921},{"game":"Qbert","seed":5.0,"HNS":0.0357645342,"Max HNS":0.0545553039},{"game":"Qbert","seed":6.0,"HNS":0.0222404466,"Max HNS":0.0222404466},{"game":"Qbert","seed":7.0,"HNS":0.0251183123,"Max HNS":0.0252311697},{"game":"Qbert","seed":8.0,"HNS":0.0459593262,"Max HNS":0.0472571871},{"game":"Qbert","seed":9.0,"HNS":0.0382850178,"Max HNS":0.1237933655},{"game":"Qbert","seed":10.0,"HNS":0.1110216611,"Max HNS":0.1243764624},{"game":"Qbert","seed":11.0,"HNS":0.0488560014,"Max HNS":0.054329589},{"game":"Qbert","seed":12.0,"HNS":0.0230680681,"Max HNS":0.0230680681},{"game":"Qbert","seed":13.0,"HNS":0.0989835303,"Max HNS":0.2675549804},{"game":"Qbert","seed":14.0,"HNS":0.0254945038,"Max HNS":0.0267735552},{"game":"Qbert","seed":15.0,"HNS":0.0490629068,"Max HNS":0.0710889242},{"game":"Qbert","seed":16.0,"HNS":0.0321530949,"Max HNS":0.0599348436},{"game":"Qbert","seed":17.0,"HNS":0.0247233111,"Max HNS":0.0445109886},{"game":"Qbert","seed":18.0,"HNS":0.0446614652,"Max HNS":0.1474181971},{"game":"Qbert","seed":19.0,"HNS":0.0471255201,"Max HNS":0.0521476778},{"game":"Qbert","seed":20.0,"HNS":0.021017824,"Max HNS":0.021017824},{"game":"Qbert","seed":21.0,"HNS":0.0482352853,"Max HNS":0.0644867618},{"game":"Qbert","seed":22.0,"HNS":0.0409371685,"Max HNS":0.0519031532},{"game":"Qbert","seed":23.0,"HNS":0.0176132901,"Max HNS":0.0335261942},{"game":"Qbert","seed":24.0,"HNS":0.0359150108,"Max HNS":0.0384166848},{"game":"Qbert","seed":25.0,"HNS":0.0429497935,"Max HNS":0.0453198005},{"game":"Qbert","seed":26.0,"HNS":0.0209613952,"Max HNS":0.0209613952},{"game":"Qbert","seed":27.0,"HNS":0.010239935,"Max HNS":0.0156194747},{"game":"Qbert","seed":28.0,"HNS":0.0162966195,"Max HNS":0.0162966195},{"game":"Qbert","seed":29.0,"HNS":0.0149423298,"Max HNS":0.0190428181},{"game":"Qbert","seed":30.0,"HNS":0.160077044,"Max HNS":0.160077044},{"game":"Qbert","seed":31.0,"HNS":0.0157511417,"Max HNS":0.0535395866},{"game":"Qbert","seed":32.0,"HNS":0.0270933181,"Max HNS":0.0270933181},{"game":"Qbert","seed":33.0,"HNS":0.030986901,"Max HNS":0.0319838087},{"game":"Qbert","seed":34.0,"HNS":0.0472007584,"Max HNS":0.1179435863},{"game":"Qbert","seed":35.0,"HNS":0.0173123368,"Max HNS":0.0177261476},{"game":"Qbert","seed":36.0,"HNS":0.0516962479,"Max HNS":0.0516962479},{"game":"Qbert","seed":37.0,"HNS":0.025889505,"Max HNS":0.0845001542},{"game":"Qbert","seed":38.0,"HNS":0.1398943654,"Max HNS":0.1398943654},{"game":"Qbert","seed":39.0,"HNS":0.0383414465,"Max HNS":0.151894877},{"game":"Qbert","seed":40.0,"HNS":0.0166163824,"Max HNS":0.0213187772},{"game":"Qbert","seed":41.0,"HNS":0.0235571172,"Max HNS":0.0399778799},{"game":"Qbert","seed":42.0,"HNS":0.0225225903,"Max HNS":0.0250430739},{"game":"Qbert","seed":43.0,"HNS":0.046993853,"Max HNS":0.0518467245},{"game":"Qbert","seed":44.0,"HNS":0.0485738577,"Max HNS":0.0485738577},{"game":"Qbert","seed":45.0,"HNS":0.0466740902,"Max HNS":0.0482164757},{"game":"Qbert","seed":46.0,"HNS":0.0388869243,"Max HNS":0.0450564664},{"game":"Qbert","seed":47.0,"HNS":0.0220147317,"Max HNS":0.0220523508},{"game":"Qbert","seed":48.0,"HNS":0.0279397492,"Max HNS":0.0602922256},{"game":"Qbert","seed":49.0,"HNS":0.0233502118,"Max HNS":0.0233502118},{"game":"Qbert","seed":50.0,"HNS":0.0370812047,"Max HNS":0.0543483986},{"game":"Qbert","seed":51.0,"HNS":0.0447743227,"Max HNS":0.0518655341},{"game":"Qbert","seed":52.0,"HNS":0.0534455387,"Max HNS":0.0534455387},{"game":"Qbert","seed":53.0,"HNS":0.0346735786,"Max HNS":0.0435140809},{"game":"Qbert","seed":54.0,"HNS":0.0478214745,"Max HNS":0.0573579312},{"game":"Qbert","seed":55.0,"HNS":0.0895035023,"Max HNS":0.0895035023},{"game":"Qbert","seed":56.0,"HNS":0.0134563731,"Max HNS":0.0134563731},{"game":"Qbert","seed":57.0,"HNS":0.0456583729,"Max HNS":0.0456583729},{"game":"Qbert","seed":58.0,"HNS":0.0484233811,"Max HNS":0.0484233811},{"game":"Qbert","seed":59.0,"HNS":0.0225602095,"Max HNS":0.042611221},{"game":"Qbert","seed":60.0,"HNS":0.0499093378,"Max HNS":0.124620987},{"game":"Qbert","seed":61.0,"HNS":0.0273754618,"Max HNS":0.0601229394},{"game":"Qbert","seed":62.0,"HNS":0.0466928998,"Max HNS":0.0498717187},{"game":"Qbert","seed":63.0,"HNS":0.0298018975,"Max HNS":0.0298018975},{"game":"Qbert","seed":64.0,"HNS":0.0199644875,"Max HNS":0.0199644875},{"game":"Qbert","seed":65.0,"HNS":0.0291247527,"Max HNS":0.0450940855},{"game":"Qbert","seed":66.0,"HNS":0.0332440505,"Max HNS":0.0332440505},{"game":"Qbert","seed":67.0,"HNS":0.0986637675,"Max HNS":0.0986637675},{"game":"Qbert","seed":68.0,"HNS":0.0209425856,"Max HNS":0.0421033624},{"game":"Qbert","seed":69.0,"HNS":0.1676761141,"Max HNS":0.1676761141},{"game":"Qbert","seed":70.0,"HNS":0.0597655574,"Max HNS":0.0597655574},{"game":"Qbert","seed":71.0,"HNS":0.0230304489,"Max HNS":0.0230304489},{"game":"Qbert","seed":72.0,"HNS":0.0339023858,"Max HNS":0.0406926439},{"game":"Qbert","seed":73.0,"HNS":0.0265666499,"Max HNS":0.0516022},{"game":"Qbert","seed":74.0,"HNS":0.0719165457,"Max HNS":0.1384460278},{"game":"Qbert","seed":75.0,"HNS":0.01985163,"Max HNS":0.0331876218},{"game":"Qbert","seed":76.0,"HNS":0.0338083379,"Max HNS":0.0425359827},{"game":"Qbert","seed":77.0,"HNS":0.038454304,"Max HNS":0.0436833671},{"game":"Qbert","seed":78.0,"HNS":0.0234066405,"Max HNS":0.0283347503},{"game":"Qbert","seed":79.0,"HNS":0.0207544898,"Max HNS":0.0332064314},{"game":"Qbert","seed":80.0,"HNS":0.023519498,"Max HNS":0.0321154758},{"game":"Qbert","seed":81.0,"HNS":0.0209237761,"Max HNS":0.0272249851},{"game":"Qbert","seed":82.0,"HNS":0.0220147317,"Max HNS":0.0345419115},{"game":"Qbert","seed":83.0,"HNS":0.0466552806,"Max HNS":0.0496836229},{"game":"Qbert","seed":84.0,"HNS":0.0568688822,"Max HNS":0.0737222653},{"game":"Qbert","seed":85.0,"HNS":0.0294821347,"Max HNS":0.0421786007},{"game":"Qbert","seed":86.0,"HNS":0.0388493052,"Max HNS":0.042836936},{"game":"Qbert","seed":87.0,"HNS":0.0362347736,"Max HNS":0.0415202654},{"game":"Qbert","seed":88.0,"HNS":0.0344478636,"Max HNS":0.0344478636},{"game":"Qbert","seed":89.0,"HNS":0.1064509333,"Max HNS":0.1064509333},{"game":"Qbert","seed":90.0,"HNS":0.1560329845,"Max HNS":0.1560329845},{"game":"Qbert","seed":91.0,"HNS":0.0345983402,"Max HNS":0.0374573963},{"game":"Qbert","seed":92.0,"HNS":0.0439655108,"Max HNS":0.0439655108},{"game":"Qbert","seed":93.0,"HNS":0.0244975961,"Max HNS":0.0279209396},{"game":"Qbert","seed":94.0,"HNS":0.058712221,"Max HNS":0.058712221},{"game":"Qbert","seed":95.0,"HNS":0.0522793448,"Max HNS":0.052806013},{"game":"Qbert","seed":96.0,"HNS":0.0452445622,"Max HNS":0.0452445622},{"game":"Qbert","seed":97.0,"HNS":0.0212247293,"Max HNS":0.0341657199},{"game":"Qbert","seed":98.0,"HNS":0.0520348203,"Max HNS":0.0520348203},{"game":"Qbert","seed":99.0,"HNS":0.000985622,"Max HNS":0.0049732528},{"game":"Qbert","seed":100.0,"HNS":0.0276576055,"Max HNS":0.0356140575},{"game":"RoadRunner","seed":1.0,"HNS":0.1361460394,"Max HNS":0.1361460394},{"game":"RoadRunner","seed":2.0,"HNS":0.3433331206,"Max HNS":0.3433331206},{"game":"RoadRunner","seed":3.0,"HNS":0.5066062424,"Max HNS":0.5066062424},{"game":"RoadRunner","seed":4.0,"HNS":0.3225250527,"Max HNS":0.3225250527},{"game":"RoadRunner","seed":5.0,"HNS":0.5812855046,"Max HNS":0.5812855046},{"game":"RoadRunner","seed":6.0,"HNS":0.5957107296,"Max HNS":0.6410289143},{"game":"RoadRunner","seed":7.0,"HNS":0.9565966682,"Max HNS":0.9565966682},{"game":"RoadRunner","seed":8.0,"HNS":-0.0014680539,"Max HNS":-0.0014680539},{"game":"RoadRunner","seed":9.0,"HNS":-0.0014680539,"Max HNS":-0.0014680539},{"game":"RoadRunner","seed":10.0,"HNS":1.074806919,"Max HNS":1.0843811834},{"game":"RoadRunner","seed":11.0,"HNS":1.0845088402,"Max HNS":1.0845088402},{"game":"RoadRunner","seed":12.0,"HNS":0.139209804,"Max HNS":0.1418905981},{"game":"RoadRunner","seed":13.0,"HNS":0.2043148018,"Max HNS":0.2043148018},{"game":"RoadRunner","seed":14.0,"HNS":1.0265526265,"Max HNS":1.0265526265},{"game":"RoadRunner","seed":15.0,"HNS":0.1508265782,"Max HNS":0.1508265782},{"game":"RoadRunner","seed":16.0,"HNS":0.7579625965,"Max HNS":0.7579625965},{"game":"RoadRunner","seed":17.0,"HNS":1.1077423885,"Max HNS":1.1077423885},{"game":"RoadRunner","seed":18.0,"HNS":0.7263036957,"Max HNS":0.7263036957},{"game":"RoadRunner","seed":19.0,"HNS":0.704346716,"Max HNS":0.704346716},{"game":"RoadRunner","seed":20.0,"HNS":1.0031914215,"Max HNS":1.1805067977},{"game":"RoadRunner","seed":21.0,"HNS":0.6317099636,"Max HNS":0.7912810366},{"game":"RoadRunner","seed":22.0,"HNS":0.1472521861,"Max HNS":0.1999744686},{"game":"RoadRunner","seed":23.0,"HNS":0.1796770281,"Max HNS":0.1985702432},{"game":"RoadRunner","seed":24.0,"HNS":0.8479606817,"Max HNS":0.8479606817},{"game":"RoadRunner","seed":25.0,"HNS":1.9773409076,"Max HNS":1.9906172209},{"game":"RoadRunner","seed":26.0,"HNS":0.7547711751,"Max HNS":0.9166400715},{"game":"RoadRunner","seed":27.0,"HNS":0.984681177,"Max HNS":0.984681177},{"game":"RoadRunner","seed":28.0,"HNS":1.0274462245,"Max HNS":1.0274462245},{"game":"RoadRunner","seed":29.0,"HNS":0.6199655326,"Max HNS":0.631326993},{"game":"RoadRunner","seed":30.0,"HNS":0.278994064,"Max HNS":0.2971213378},{"game":"RoadRunner","seed":31.0,"HNS":0.838641731,"Max HNS":0.838641731},{"game":"RoadRunner","seed":32.0,"HNS":0.6943894811,"Max HNS":0.6943894811},{"game":"RoadRunner","seed":33.0,"HNS":0.1332099317,"Max HNS":0.1523584605},{"game":"RoadRunner","seed":34.0,"HNS":0.1769962341,"Max HNS":0.1769962341},{"game":"RoadRunner","seed":35.0,"HNS":2.1685708815,"Max HNS":2.1847833025},{"game":"RoadRunner","seed":36.0,"HNS":0.8270249569,"Max HNS":0.8270249569},{"game":"RoadRunner","seed":37.0,"HNS":0.3958000894,"Max HNS":0.4860534882},{"game":"RoadRunner","seed":38.0,"HNS":0.4444373524,"Max HNS":0.4444373524},{"game":"RoadRunner","seed":39.0,"HNS":0.4255441374,"Max HNS":0.4813301845},{"game":"RoadRunner","seed":40.0,"HNS":1.0875726048,"Max HNS":1.0875726048},{"game":"RoadRunner","seed":41.0,"HNS":0.4007787068,"Max HNS":0.4007787068},{"game":"RoadRunner","seed":42.0,"HNS":0.7132826961,"Max HNS":0.7753239293},{"game":"RoadRunner","seed":43.0,"HNS":0.4326929214,"Max HNS":0.5076274973},{"game":"RoadRunner","seed":44.0,"HNS":0.4979255761,"Max HNS":0.4979255761},{"game":"RoadRunner","seed":45.0,"HNS":-0.0014680539,"Max HNS":-0.0014680539},{"game":"RoadRunner","seed":46.0,"HNS":0.1422735686,"Max HNS":0.1422735686},{"game":"RoadRunner","seed":47.0,"HNS":0.5360949767,"Max HNS":0.6952830791},{"game":"RoadRunner","seed":48.0,"HNS":0.5608604072,"Max HNS":0.5608604072},{"game":"RoadRunner","seed":49.0,"HNS":0.3913320993,"Max HNS":0.3913320993},{"game":"RoadRunner","seed":50.0,"HNS":0.3891619327,"Max HNS":0.466394332},{"game":"RoadRunner","seed":51.0,"HNS":0.847705368,"Max HNS":0.9202144635},{"game":"RoadRunner","seed":52.0,"HNS":0.281674858,"Max HNS":0.281674858},{"game":"RoadRunner","seed":53.0,"HNS":1.14872024,"Max HNS":1.1847194741},{"game":"RoadRunner","seed":54.0,"HNS":1.03331844,"Max HNS":1.03331844},{"game":"RoadRunner","seed":55.0,"HNS":0.5599668092,"Max HNS":0.5599668092},{"game":"RoadRunner","seed":56.0,"HNS":0.3850769133,"Max HNS":0.3850769133},{"game":"RoadRunner","seed":57.0,"HNS":0.8517903874,"Max HNS":0.8517903874},{"game":"RoadRunner","seed":58.0,"HNS":0.3313333759,"Max HNS":0.3313333759},{"game":"RoadRunner","seed":59.0,"HNS":0.1297631965,"Max HNS":0.1366566669},{"game":"RoadRunner","seed":60.0,"HNS":0.1937192826,"Max HNS":0.1998468118},{"game":"RoadRunner","seed":61.0,"HNS":0.8373651624,"Max HNS":0.8498755346},{"game":"RoadRunner","seed":62.0,"HNS":1.2840365099,"Max HNS":1.3305036063},{"game":"RoadRunner","seed":63.0,"HNS":0.7606433906,"Max HNS":0.7606433906},{"game":"RoadRunner","seed":64.0,"HNS":0.391587413,"Max HNS":0.3949064914},{"game":"RoadRunner","seed":65.0,"HNS":0.489883194,"Max HNS":0.489883194},{"game":"RoadRunner","seed":66.0,"HNS":1.3210569988,"Max HNS":1.329865322},{"game":"RoadRunner","seed":67.0,"HNS":0.4920533606,"Max HNS":0.4920533606},{"game":"RoadRunner","seed":68.0,"HNS":0.2292078892,"Max HNS":0.2913767792},{"game":"RoadRunner","seed":69.0,"HNS":1.536286462,"Max HNS":1.536286462},{"game":"RoadRunner","seed":70.0,"HNS":0.3631199336,"Max HNS":0.3631199336},{"game":"RoadRunner","seed":71.0,"HNS":0.1657624306,"Max HNS":0.1657624306},{"game":"RoadRunner","seed":72.0,"HNS":1.6948362801,"Max HNS":1.6948362801},{"game":"RoadRunner","seed":73.0,"HNS":0.8234505649,"Max HNS":0.8234505649},{"game":"RoadRunner","seed":74.0,"HNS":0.1495500096,"Max HNS":0.1495500096},{"game":"RoadRunner","seed":75.0,"HNS":0.653028659,"Max HNS":0.653028659},{"game":"RoadRunner","seed":76.0,"HNS":0.3075892002,"Max HNS":0.3956724325},{"game":"RoadRunner","seed":77.0,"HNS":0.5833280143,"Max HNS":0.6352843557},{"game":"RoadRunner","seed":78.0,"HNS":0.2569094275,"Max HNS":0.2863981617},{"game":"RoadRunner","seed":79.0,"HNS":0.5631582307,"Max HNS":0.5631582307},{"game":"RoadRunner","seed":80.0,"HNS":1.1844641603,"Max HNS":1.2962915683},{"game":"RoadRunner","seed":81.0,"HNS":0.2290802323,"Max HNS":0.2525690943},{"game":"RoadRunner","seed":82.0,"HNS":0.6544328844,"Max HNS":0.6544328844},{"game":"RoadRunner","seed":83.0,"HNS":0.8057062616,"Max HNS":0.8057062616},{"game":"RoadRunner","seed":84.0,"HNS":0.3476734538,"Max HNS":0.3476734538},{"game":"RoadRunner","seed":85.0,"HNS":0.7420054892,"Max HNS":0.7420054892},{"game":"RoadRunner","seed":86.0,"HNS":0.7879619583,"Max HNS":0.7879619583},{"game":"RoadRunner","seed":87.0,"HNS":0.9343843748,"Max HNS":0.9343843748},{"game":"RoadRunner","seed":88.0,"HNS":0.4256717942,"Max HNS":0.4256717942},{"game":"RoadRunner","seed":89.0,"HNS":0.8390247016,"Max HNS":0.8390247016},{"game":"RoadRunner","seed":90.0,"HNS":1.8809599796,"Max HNS":1.9098104296},{"game":"RoadRunner","seed":91.0,"HNS":1.0722537818,"Max HNS":1.0722537818},{"game":"RoadRunner","seed":92.0,"HNS":0.915235846,"Max HNS":0.915235846},{"game":"RoadRunner","seed":93.0,"HNS":-0.0014680539,"Max HNS":-0.0014680539},{"game":"RoadRunner","seed":94.0,"HNS":1.0818280462,"Max HNS":1.0818280462},{"game":"RoadRunner","seed":95.0,"HNS":0.1287419417,"Max HNS":0.1287419417},{"game":"RoadRunner","seed":96.0,"HNS":0.2752920151,"Max HNS":0.2759302994},{"game":"RoadRunner","seed":97.0,"HNS":0.1667836855,"Max HNS":0.1736771558},{"game":"RoadRunner","seed":98.0,"HNS":0.312312504,"Max HNS":0.312312504},{"game":"RoadRunner","seed":99.0,"HNS":1.0797855365,"Max HNS":1.3455671156},{"game":"RoadRunner","seed":100.0,"HNS":1.1076147316,"Max HNS":1.2103785026},{"game":"Seaquest","seed":1.0,"HNS":0.0047920393,"Max HNS":0.0047920393},{"game":"Seaquest","seed":2.0,"HNS":0.0005049266,"Max HNS":0.0005049266},{"game":"Seaquest","seed":3.0,"HNS":0.006835563,"Max HNS":0.0077453836},{"game":"Seaquest","seed":4.0,"HNS":0.0002810441,"Max HNS":0.0002858075},{"game":"Seaquest","seed":5.0,"HNS":0.0041156282,"Max HNS":0.0041156282},{"game":"Seaquest","seed":6.0,"HNS":0.0126803267,"Max HNS":0.0126803267},{"game":"Seaquest","seed":7.0,"HNS":0.0061305712,"Max HNS":0.0067021862},{"game":"Seaquest","seed":8.0,"HNS":0.0002810441,"Max HNS":0.0002810441},{"game":"Seaquest","seed":9.0,"HNS":0.0037202611,"Max HNS":0.0037202611},{"game":"Seaquest","seed":10.0,"HNS":0.0002762806,"Max HNS":0.0002810441},{"game":"Seaquest","seed":11.0,"HNS":0.0108559221,"Max HNS":0.0108559221},{"game":"Seaquest","seed":12.0,"HNS":0.0002762806,"Max HNS":0.0002810441},{"game":"Seaquest","seed":13.0,"HNS":0.009536444,"Max HNS":0.0103128878},{"game":"Seaquest","seed":14.0,"HNS":0.0032772595,"Max HNS":0.0043204569},{"game":"Seaquest","seed":15.0,"HNS":0.0096984016,"Max HNS":0.0096984016},{"game":"Seaquest","seed":16.0,"HNS":0.0056828061,"Max HNS":0.0070022841},{"game":"Seaquest","seed":17.0,"HNS":0.0147000331,"Max HNS":0.0147000331},{"game":"Seaquest","seed":18.0,"HNS":0.0029581078,"Max HNS":0.0050349757},{"game":"Seaquest","seed":19.0,"HNS":0.0055827734,"Max HNS":0.0059686136},{"game":"Seaquest","seed":20.0,"HNS":0.0055589561,"Max HNS":0.0055589561},{"game":"Seaquest","seed":21.0,"HNS":0.0052683852,"Max HNS":0.0060543558},{"game":"Seaquest","seed":22.0,"HNS":0.0010527243,"Max HNS":0.0011241762},{"game":"Seaquest","seed":23.0,"HNS":0.0125660037,"Max HNS":0.0148334099},{"game":"Seaquest","seed":24.0,"HNS":0.0106082222,"Max HNS":0.0116847638},{"game":"Seaquest","seed":25.0,"HNS":0.0044109626,"Max HNS":0.0045967375},{"game":"Seaquest","seed":26.0,"HNS":0.0070213379,"Max HNS":0.0097079285},{"game":"Seaquest","seed":27.0,"HNS":0.0083693967,"Max HNS":0.0086647311},{"game":"Seaquest","seed":28.0,"HNS":0.0122563789,"Max HNS":0.0125326595},{"game":"Seaquest","seed":29.0,"HNS":0.007469103,"Max HNS":0.007469103},{"game":"Seaquest","seed":30.0,"HNS":0.0046348452,"Max HNS":0.0046348452},{"game":"Seaquest","seed":31.0,"HNS":0.0078978143,"Max HNS":0.0093983037},{"game":"Seaquest","seed":32.0,"HNS":0.0002810441,"Max HNS":0.0002858075},{"game":"Seaquest","seed":33.0,"HNS":0.006606917,"Max HNS":0.006606917},{"game":"Seaquest","seed":34.0,"HNS":0.0040108321,"Max HNS":0.0051874064},{"game":"Seaquest","seed":35.0,"HNS":0.0067688746,"Max HNS":0.0105320069},{"game":"Seaquest","seed":36.0,"HNS":0.0093649595,"Max HNS":0.0093649595},{"game":"Seaquest","seed":37.0,"HNS":0.0043156935,"Max HNS":0.0043156935},{"game":"Seaquest","seed":38.0,"HNS":0.0088600329,"Max HNS":0.0121515828},{"game":"Seaquest","seed":39.0,"HNS":0.0082312564,"Max HNS":0.0082312564},{"game":"Seaquest","seed":40.0,"HNS":0.0096698209,"Max HNS":0.0096698209},{"game":"Seaquest","seed":41.0,"HNS":0.0031010115,"Max HNS":0.0044585972},{"game":"Seaquest","seed":42.0,"HNS":0.0131233283,"Max HNS":0.0131233283},{"game":"Seaquest","seed":43.0,"HNS":0.0002762806,"Max HNS":0.0002810441},{"game":"Seaquest","seed":44.0,"HNS":0.0027151714,"Max HNS":0.0033820556},{"game":"Seaquest","seed":45.0,"HNS":0.0112560526,"Max HNS":0.0112560526},{"game":"Seaquest","seed":46.0,"HNS":0.0123992826,"Max HNS":0.0154478961},{"game":"Seaquest","seed":47.0,"HNS":0.012589821,"Max HNS":0.012589821},{"game":"Seaquest","seed":48.0,"HNS":0.0085265908,"Max HNS":0.0085265908},{"game":"Seaquest","seed":49.0,"HNS":0.002648483,"Max HNS":0.002648483},{"game":"Seaquest","seed":50.0,"HNS":0.0065878632,"Max HNS":0.0080835892},{"game":"Seaquest","seed":51.0,"HNS":0.000290571,"Max HNS":0.0003000979},{"game":"Seaquest","seed":52.0,"HNS":0.0002762806,"Max HNS":0.000290571},{"game":"Seaquest","seed":53.0,"HNS":0.0061210442,"Max HNS":0.0081074065},{"game":"Seaquest","seed":54.0,"HNS":0.0063877979,"Max HNS":0.0117609792},{"game":"Seaquest","seed":55.0,"HNS":0.0050873737,"Max HNS":0.0058114194},{"game":"Seaquest","seed":56.0,"HNS":0.0002762806,"Max HNS":0.0002858075},{"game":"Seaquest","seed":57.0,"HNS":0.0012384992,"Max HNS":0.0012384992},{"game":"Seaquest","seed":58.0,"HNS":0.0144142256,"Max HNS":0.0144142256},{"game":"Seaquest","seed":59.0,"HNS":0.0057018599,"Max HNS":0.0057018599},{"game":"Seaquest","seed":60.0,"HNS":0.002295987,"Max HNS":0.002295987},{"game":"Seaquest","seed":61.0,"HNS":0.0100366072,"Max HNS":0.0100366072},{"game":"Seaquest","seed":62.0,"HNS":0.0026294291,"Max HNS":0.0026294291},{"game":"Seaquest","seed":63.0,"HNS":0.007416705,"Max HNS":0.007416705},{"game":"Seaquest","seed":64.0,"HNS":0.0054065255,"Max HNS":0.0069879937},{"game":"Seaquest","seed":65.0,"HNS":0.0002762806,"Max HNS":0.0002858075},{"game":"Seaquest","seed":66.0,"HNS":0.012184927,"Max HNS":0.0155955633},{"game":"Seaquest","seed":67.0,"HNS":0.0033487114,"Max HNS":0.0034296902},{"game":"Seaquest","seed":68.0,"HNS":0.0061924961,"Max HNS":0.0066545516},{"game":"Seaquest","seed":69.0,"HNS":0.0021530833,"Max HNS":0.004363328},{"game":"Seaquest","seed":70.0,"HNS":0.0024388908,"Max HNS":0.0025913215},{"game":"Seaquest","seed":71.0,"HNS":0.0058352367,"Max HNS":0.0064640133},{"game":"Seaquest","seed":72.0,"HNS":0.0003191517,"Max HNS":0.0003191517},{"game":"Seaquest","seed":73.0,"HNS":0.0095459709,"Max HNS":0.0095459709},{"game":"Seaquest","seed":74.0,"HNS":0.0060591193,"Max HNS":0.0060591193},{"game":"Seaquest","seed":75.0,"HNS":0.006897488,"Max HNS":0.0138187933},{"game":"Seaquest","seed":76.0,"HNS":0.0109321374,"Max HNS":0.0109321374},{"game":"Seaquest","seed":77.0,"HNS":0.0046872432,"Max HNS":0.0073643069},{"game":"Seaquest","seed":78.0,"HNS":0.00253416,"Max HNS":0.0027961502},{"game":"Seaquest","seed":79.0,"HNS":0.0051540622,"Max HNS":0.0053112563},{"game":"Seaquest","seed":80.0,"HNS":0.0073738338,"Max HNS":0.0073738338},{"game":"Seaquest","seed":81.0,"HNS":0.0038393476,"Max HNS":0.0048110931},{"game":"Seaquest","seed":82.0,"HNS":0.0094459383,"Max HNS":0.0094459383},{"game":"Seaquest","seed":83.0,"HNS":0.008902904,"Max HNS":0.008902904},{"game":"Seaquest","seed":84.0,"HNS":0.007126134,"Max HNS":0.010512953},{"game":"Seaquest","seed":85.0,"HNS":0.0095078633,"Max HNS":0.0127136709},{"game":"Seaquest","seed":86.0,"HNS":0.0130232957,"Max HNS":0.0130232957},{"game":"Seaquest","seed":87.0,"HNS":0.0025055792,"Max HNS":0.0026627733},{"game":"Seaquest","seed":88.0,"HNS":0.0054922677,"Max HNS":0.0068927245},{"game":"Seaquest","seed":89.0,"HNS":0.0002762806,"Max HNS":0.0002810441},{"game":"Seaquest","seed":90.0,"HNS":0.0003000979,"Max HNS":0.0003000979},{"game":"Seaquest","seed":91.0,"HNS":0.0024436542,"Max HNS":0.0024436542},{"game":"Seaquest","seed":92.0,"HNS":0.0144142256,"Max HNS":0.0150572925},{"game":"Seaquest","seed":93.0,"HNS":0.0015814682,"Max HNS":0.0027580425},{"game":"Seaquest","seed":94.0,"HNS":0.0028866559,"Max HNS":0.0028866559},{"game":"Seaquest","seed":95.0,"HNS":0.0002762806,"Max HNS":0.0002858075},{"game":"Seaquest","seed":96.0,"HNS":0.0078263624,"Max HNS":0.0086170965},{"game":"Seaquest","seed":97.0,"HNS":0.0048539643,"Max HNS":0.005339837},{"game":"Seaquest","seed":98.0,"HNS":0.0106415664,"Max HNS":0.0134662973},{"game":"Seaquest","seed":99.0,"HNS":0.0038679283,"Max HNS":0.0045776837},{"game":"Seaquest","seed":100.0,"HNS":0.0044204895,"Max HNS":0.0070546821},{"game":"UpNDown","seed":1.0,"HNS":0.0641678166,"Max HNS":0.0641678166},{"game":"UpNDown","seed":2.0,"HNS":0.158434739,"Max HNS":0.2118586355},{"game":"UpNDown","seed":3.0,"HNS":0.2006666786,"Max HNS":0.2709905195},{"game":"UpNDown","seed":4.0,"HNS":0.0344092188,"Max HNS":0.2391799136},{"game":"UpNDown","seed":5.0,"HNS":0.1256026094,"Max HNS":0.1447158551},{"game":"UpNDown","seed":6.0,"HNS":0.0464972491,"Max HNS":0.1852004516},{"game":"UpNDown","seed":7.0,"HNS":0.0086471084,"Max HNS":0.1373322103},{"game":"UpNDown","seed":8.0,"HNS":0.1651194466,"Max HNS":0.1760156992},{"game":"UpNDown","seed":9.0,"HNS":0.4566838115,"Max HNS":0.4566838115},{"game":"UpNDown","seed":10.0,"HNS":0.1904514418,"Max HNS":0.235998853},{"game":"UpNDown","seed":11.0,"HNS":0.1128962885,"Max HNS":0.2458287783},{"game":"UpNDown","seed":12.0,"HNS":0.2441979247,"Max HNS":0.3054355813},{"game":"UpNDown","seed":13.0,"HNS":0.1689187978,"Max HNS":0.1952454345},{"game":"UpNDown","seed":14.0,"HNS":0.1897793867,"Max HNS":0.1897793867},{"game":"UpNDown","seed":15.0,"HNS":0.2705245614,"Max HNS":0.2705245614},{"game":"UpNDown","seed":16.0,"HNS":0.1453699887,"Max HNS":0.1668847112},{"game":"UpNDown","seed":17.0,"HNS":0.4678578469,"Max HNS":0.4678578469},{"game":"UpNDown","seed":18.0,"HNS":0.0689976523,"Max HNS":0.0976003154},{"game":"UpNDown","seed":19.0,"HNS":0.1367139196,"Max HNS":0.1495815337},{"game":"UpNDown","seed":20.0,"HNS":0.3091005215,"Max HNS":0.3529812362},{"game":"UpNDown","seed":21.0,"HNS":0.2280417212,"Max HNS":0.2280417212},{"game":"UpNDown","seed":22.0,"HNS":0.1674581982,"Max HNS":0.1905500099},{"game":"UpNDown","seed":23.0,"HNS":0.0985770354,"Max HNS":0.1422516533},{"game":"UpNDown","seed":24.0,"HNS":0.1790802703,"Max HNS":0.2631050736},{"game":"UpNDown","seed":25.0,"HNS":0.3426315884,"Max HNS":0.3426315884},{"game":"UpNDown","seed":26.0,"HNS":0.1784619796,"Max HNS":0.2140540153},{"game":"UpNDown","seed":27.0,"HNS":-0.0006720551,"Max HNS":0.0344002581},{"game":"UpNDown","seed":28.0,"HNS":0.2106489364,"Max HNS":0.4398734744},{"game":"UpNDown","seed":29.0,"HNS":0.2809638166,"Max HNS":0.2809638166},{"game":"UpNDown","seed":30.0,"HNS":0.1626373232,"Max HNS":0.1626373232},{"game":"UpNDown","seed":31.0,"HNS":0.2375669815,"Max HNS":0.2375669815},{"game":"UpNDown","seed":32.0,"HNS":0.1174035377,"Max HNS":0.2093585907},{"game":"UpNDown","seed":33.0,"HNS":0.2347264288,"Max HNS":0.2347264288},{"game":"UpNDown","seed":34.0,"HNS":0.1353429273,"Max HNS":0.1353429273},{"game":"UpNDown","seed":35.0,"HNS":0.1912847901,"Max HNS":0.2700854854},{"game":"UpNDown","seed":36.0,"HNS":0.1970196599,"Max HNS":0.2697718597},{"game":"UpNDown","seed":37.0,"HNS":0.4684492554,"Max HNS":0.4801430133},{"game":"UpNDown","seed":38.0,"HNS":0.0835767666,"Max HNS":0.0933439667},{"game":"UpNDown","seed":39.0,"HNS":0.1438377032,"Max HNS":0.1459972401},{"game":"UpNDown","seed":40.0,"HNS":0.513611355,"Max HNS":0.513611355},{"game":"UpNDown","seed":41.0,"HNS":0.2363214395,"Max HNS":0.2363214395},{"game":"UpNDown","seed":42.0,"HNS":0.3963780713,"Max HNS":0.3963780713},{"game":"UpNDown","seed":43.0,"HNS":0.0560314701,"Max HNS":0.2275040771},{"game":"UpNDown","seed":44.0,"HNS":0.3672556856,"Max HNS":0.3672556856},{"game":"UpNDown","seed":45.0,"HNS":0.0713812075,"Max HNS":0.0820982455},{"game":"UpNDown","seed":46.0,"HNS":0.0114428574,"Max HNS":0.0454757254},{"game":"UpNDown","seed":47.0,"HNS":0.4702682844,"Max HNS":0.4702682844},{"game":"UpNDown","seed":48.0,"HNS":0.0820534418,"Max HNS":0.0820534418},{"game":"UpNDown","seed":49.0,"HNS":-0.0015591677,"Max HNS":0.0048567179},{"game":"UpNDown","seed":50.0,"HNS":0.2426118748,"Max HNS":0.3600333339},{"game":"UpNDown","seed":51.0,"HNS":0.2079159125,"Max HNS":0.2079159125},{"game":"UpNDown","seed":52.0,"HNS":0.1328339218,"Max HNS":0.2014193803},{"game":"UpNDown","seed":53.0,"HNS":0.2512948261,"Max HNS":0.2531228158},{"game":"UpNDown","seed":54.0,"HNS":0.0632538218,"Max HNS":0.0632538218},{"game":"UpNDown","seed":55.0,"HNS":0.2425849926,"Max HNS":0.2425849926},{"game":"UpNDown","seed":56.0,"HNS":0.3308930268,"Max HNS":0.3308930268},{"game":"UpNDown","seed":57.0,"HNS":0.0852255417,"Max HNS":0.0852255417},{"game":"UpNDown","seed":58.0,"HNS":0.0821072062,"Max HNS":0.0821072062},{"game":"UpNDown","seed":59.0,"HNS":0.2890912024,"Max HNS":0.2890912024},{"game":"UpNDown","seed":60.0,"HNS":0.2648255345,"Max HNS":0.2789566121},{"game":"UpNDown","seed":61.0,"HNS":0.0864800444,"Max HNS":0.1661320095},{"game":"UpNDown","seed":62.0,"HNS":0.1500474919,"Max HNS":0.2418322909},{"game":"UpNDown","seed":63.0,"HNS":0.1537303536,"Max HNS":0.1537303536},{"game":"UpNDown","seed":64.0,"HNS":0.0519812183,"Max HNS":0.0519812183},{"game":"UpNDown","seed":65.0,"HNS":0.4635925375,"Max HNS":0.4635925375},{"game":"UpNDown","seed":66.0,"HNS":0.1176006738,"Max HNS":0.155334325},{"game":"UpNDown","seed":67.0,"HNS":0.4786555315,"Max HNS":0.4786555315},{"game":"UpNDown","seed":68.0,"HNS":0.3399702504,"Max HNS":0.3399702504},{"game":"UpNDown","seed":69.0,"HNS":0.3248982957,"Max HNS":0.4257961612},{"game":"UpNDown","seed":70.0,"HNS":0.0281098228,"Max HNS":0.1263642718},{"game":"UpNDown","seed":71.0,"HNS":0.4073818527,"Max HNS":0.4147744583},{"game":"UpNDown","seed":72.0,"HNS":0.3897292066,"Max HNS":0.3897292066},{"game":"UpNDown","seed":73.0,"HNS":0.1766339899,"Max HNS":0.1766339899},{"game":"UpNDown","seed":74.0,"HNS":0.4275524651,"Max HNS":0.4275524651},{"game":"UpNDown","seed":75.0,"HNS":0.1087385079,"Max HNS":0.2073065826},{"game":"UpNDown","seed":76.0,"HNS":0.1310238535,"Max HNS":0.172449327},{"game":"UpNDown","seed":77.0,"HNS":0.177270202,"Max HNS":0.177270202},{"game":"UpNDown","seed":78.0,"HNS":0.1106561049,"Max HNS":0.2186239897},{"game":"UpNDown","seed":79.0,"HNS":0.108227746,"Max HNS":0.2127457481},{"game":"UpNDown","seed":80.0,"HNS":0.0660674922,"Max HNS":0.0660674922},{"game":"UpNDown","seed":81.0,"HNS":0.3366278966,"Max HNS":0.3366278966},{"game":"UpNDown","seed":82.0,"HNS":0.1797702468,"Max HNS":0.1888833133},{"game":"UpNDown","seed":83.0,"HNS":0.2634993459,"Max HNS":0.2917883833},{"game":"UpNDown","seed":84.0,"HNS":0.1623326583,"Max HNS":0.2334181616},{"game":"UpNDown","seed":85.0,"HNS":0.0875374111,"Max HNS":0.0875374111},{"game":"UpNDown","seed":86.0,"HNS":0.1056918583,"Max HNS":0.1771268302},{"game":"UpNDown","seed":87.0,"HNS":0.0549024176,"Max HNS":0.0549024176},{"game":"UpNDown","seed":88.0,"HNS":0.1932830337,"Max HNS":0.2021631212},{"game":"UpNDown","seed":89.0,"HNS":0.0354934676,"Max HNS":0.068164304},{"game":"UpNDown","seed":90.0,"HNS":0.1323590028,"Max HNS":0.2055144357},{"game":"UpNDown","seed":91.0,"HNS":0.415795982,"Max HNS":0.4574902776},{"game":"UpNDown","seed":92.0,"HNS":0.1811412391,"Max HNS":0.2526478969},{"game":"UpNDown","seed":93.0,"HNS":0.0880033692,"Max HNS":0.0996612843},{"game":"UpNDown","seed":94.0,"HNS":0.142834101,"Max HNS":0.2109625621},{"game":"UpNDown","seed":95.0,"HNS":0.0986756035,"Max HNS":0.1438197817},{"game":"UpNDown","seed":96.0,"HNS":0.0807182924,"Max HNS":0.0807182924},{"game":"UpNDown","seed":97.0,"HNS":0.1345006183,"Max HNS":0.2027186867},{"game":"UpNDown","seed":98.0,"HNS":0.1417140092,"Max HNS":0.2834997043},{"game":"UpNDown","seed":99.0,"HNS":0.053405975,"Max HNS":0.053405975},{"game":"UpNDown","seed":100.0,"HNS":0.1292496281,"Max HNS":0.1292496281}] \ No newline at end of file diff --git a/results/data/DrQ(eps).json b/results/data/DrQ(eps).json new file mode 100644 index 0000000..ef34cce --- /dev/null +++ b/results/data/DrQ(eps).json @@ -0,0 +1 @@ +{"Alien": [765.0, 866.5999999999999, 822.5, 776.3, 645.6, 850.7, 920.0999999999999, 736.0, 643.2, 681.8, 683.6, 1055.2, 744.2, 702.1, 793.3, 684.5, 846.3, 708.7, 976.3999999999999, 719.5999999999999, 749.5, 665.8, 880.0999999999999, 973.2, 647.0, 622.0, 973.8, 863.7, 884.7, 779.7, 778.3999999999999, 1028.2, 908.5, 1282.1, 1030.3, 948.8999999999999, 905.2, 685.7, 735.7, 697.4, 840.5, 997.5999999999999, 726.8, 659.1, 781.2, 1232.9, 645.8, 936.3, 1357.8, 777.5999999999999, 605.7, 1285.0, 924.8999999999999, 1956.7, 747.2, 728.6, 639.5, 1437.2, 766.8999999999999, 703.0, 893.0, 996.8999999999999, 938.5999999999999, 1330.9, 768.8999999999999, 1109.2, 994.0, 988.3999999999999, 1135.7, 891.0999999999999, 764.5999999999999, 679.8, 1098.7, 708.8, 658.0, 1007.3999999999999, 889.8, 793.8, 721.4, 879.0999999999999, 983.2, 823.3, 709.6, 969.8999999999999, 768.3999999999999, 894.7, 1343.3, 929.2, 1300.3, 696.0, 717.0999999999999, 652.8, 867.0999999999999, 976.0999999999999, 686.3, 896.3, 618.2, 725.0, 680.7, 618.1], "Amidar": [106.19, 146.89, 216.14, 173.41, 209.96, 65.12, 143.81, 180.93, 118.99000000000001, 77.26, 218.75, 119.57000000000001, 140.4, 85.5, 153.38, 119.97999999999999, 146.12, 176.22, 109.26, 190.23, 144.94, 117.3, 144.43, 180.54, 100.72, 103.22, 137.95, 135.6, 136.26, 110.04, 106.62, 156.55, 172.67, 177.69, 84.29, 78.92, 77.63, 103.16, 95.88, 81.37, 176.38, 126.97, 230.22999999999996, 88.8, 171.34, 110.66, 153.5, 81.99, 224.09, 190.79, 101.25, 99.36, 115.67, 171.94, 150.36, 155.72, 114.03999999999999, 209.27, 113.88999999999999, 204.73, 122.39000000000001, 87.01, 186.69, 161.99, 160.85, 151.17, 142.89, 101.47, 107.7, 123.2, 173.07, 107.58, 81.69, 104.1, 152.27, 108.93, 228.4, 184.62, 88.65, 156.17, 106.16, 137.53, 75.58, 123.7, 140.71, 149.68, 119.3, 97.67, 173.44, 103.98, 110.75, 147.17, 156.88, 258.19, 186.4, 117.77, 137.52, 134.53, 113.1, 127.43], "Assault": [578.14, 633.6, 594.36, 546.76, 586.85, 600.79, 519.61, 580.45, 578.37, 609.25, 592.99, 697.88, 569.89, 552.68, 697.46, 561.51, 601.53, 570.27, 544.79, 533.89, 573.23, 527.9, 553.81, 548.94, 607.83, 577.74, 569.07, 525.19, 589.44, 548.7, 592.91, 603.21, 584.07, 578.03, 651.92, 529.53, 552.13, 615.95, 606.69, 538.07, 643.02, 568.68, 535.19, 590.92, 556.76, 549.75, 573.42, 575.49, 592.58, 572.94, 551.08, 582.65, 621.55, 596.17, 539.5, 577.26, 618.45, 560.96, 630.13, 538.02, 556.61, 647.49, 585.19, 542.23, 502.80999999999995, 588.44, 546.88, 556.89, 579.2, 519.23, 557.17, 579.65, 544.5400000000001, 627.72, 599.33, 605.89, 608.32, 596.64, 500.11, 585.24, 575.07, 567.03, 574.8, 548.96, 614.18, 566.86, 628.47, 539.71, 573.03, 489.75, 569.22, 597.51, 599.19, 586.43, 523.09, 527.18, 672.76, 668.02, 681.45, 627.77], "Asterix": [667.5, 697.0, 832.5, 639.0, 830.0, 787.5, 594.0, 840.5, 543.0, 859.5, 993.0, 725.0, 750.5, 612.5, 647.5, 581.5, 730.0, 611.5, 1009.0, 734.5, 663.5, 739.0, 844.0, 597.5, 666.5, 893.5, 681.0, 644.0, 711.0, 682.0, 790.5, 738.0, 812.0, 879.0, 1041.0, 713.5, 592.0, 800.5, 661.5, 790.0, 756.0, 617.0, 851.0, 892.0, 780.0, 861.0, 1036.0, 891.0, 856.5, 646.0, 653.5, 582.0, 829.0, 802.0, 648.0, 645.5, 734.0, 829.5, 692.0, 832.5, 760.5, 526.5, 883.0, 662.0, 1238.0, 836.0, 936.0, 782.0, 826.5, 867.0, 564.0, 739.0, 1083.5, 551.0, 901.0, 783.5, 545.5, 921.5, 875.5, 841.5, 546.0, 735.5, 614.0, 953.5, 716.5, 877.5, 566.5, 615.5, 525.0, 955.5, 713.0, 918.5, 808.0, 679.5, 1068.5, 772.0, 881.0, 513.5, 788.5, 927.5], "BankHeist": [487.2, 182.0, 148.7, 169.8, 196.0, 471.7, 418.1, 236.9, 568.4, 438.9, 167.1, 108.0, 88.1, 161.7, 164.0, 132.3, 35.8, 175.4, 186.1, 147.2, 482.2, 401.8, 145.2, 96.9, 38.89999999999999, 91.3, 186.7, 482.8, 311.4, 177.9, 154.6, 495.8999999999999, 405.4, 250.1, 215.9, 135.8, 180.7, 17.7, 154.2, 121.69999999999999, 173.5, 43.7, 498.9, 135.3, 152.4, 128.29999999999998, 142.9, 102.7, 146.5, 87.5, 166.2, 135.9, 196.4, 130.09999999999997, 188.9, 395.3, 102.9, 193.6, 491.1, 205.9, 482.0, 157.2, 149.6, 206.8, 489.49999999999994, 39.5, 587.0, 147.2, 576.0, 128.29999999999998, 484.6, 70.5, 457.8, 139.2, 84.1, 190.0, 176.5, 354.6, 430.0, 162.3, 180.8, 163.1, 504.8, 108.10000000000001, 19.7, 180.1, 143.3, 333.1, 430.19999999999993, 127.5, 146.7, 143.2, 123.1, 128.1, 215.4, 218.2, 536.0, 472.6, 75.5, 478.1], "BattleZone": [18290.0, 13240.0, 9370.0, 4550.0, 11280.0, 7490.0, 2800.0, 5360.0, 3940.0, 5950.0, 8740.0, 3680.0, 20560.0, 5330.0, 6000.0, 9500.0, 2900.0, 19230.0, 18220.0, 21640.0, 10700.0, 16990.0, 9400.0, 4690.0, 6290.0, 17700.0, 8350.0, 4910.0, 14250.0, 4350.0, 10100.0, 4670.0, 15240.0, 10210.0, 20000.0, 19420.0, 7760.000000000001, 14160.0, 18910.0, 10650.0, 4700.0, 3530.0, 14069.999999999998, 6220.0, 11280.0, 840.0, 9730.0, 8250.0, 11610.0, 6240.0, 4290.0, 15340.000000000002, 18540.0, 10410.0, 13880.0, 15000.0, 2730.0, 7200.0, 1680.0, 12660.0, 9260.0, 16180.0, 3740.0, 10040.0, 2530.0, 21520.0, 12920.0, 14990.0, 22890.0, 10330.0, 13460.0, 9560.0, 11150.0, 5530.0, 18130.0, 17410.0, 2900.0, 25440.0, 14960.0, 2400.0, 5190.0, 4960.0, 9460.0, 2650.0, 2200.0, 3920.0, 5200.0, 14330.0, 23530.0, 5430.0, 590.0, 7410.0, 4140.0, 12450.0, 4570.0, 8510.0, 19140.0, 6660.0, 23070.0, 2760.0], "Boxing": [15.770000000000001, 6.200000000000001, 8.43, 7.31, 7.199999999999999, 5.03, 8.41, 6.880000000000001, 4.59, 9.03, 9.21, 6.68, 5.86, 14.359999999999998, 10.85, 15.97, 9.4, 8.09, 9.99, 11.91, 9.93, 15.13, 8.45, 5.39, 6.91, 8.73, 6.6, 12.72, 7.029999999999999, 3.28, 6.68, 10.2, 9.61, 6.04, 8.54, 8.26, 11.99, 12.09, 6.34, 6.959999999999999, 10.83, 10.26, 7.07, 10.52, 3.3000000000000003, 4.9, 14.26, 5.43, 7.24, 8.24, 5.77, 8.38, 9.05, 3.98, 9.12, 11.5, 7.530000000000001, 8.86, 15.24, 10.5, 5.75, 8.86, 9.33, 14.410000000000002, 9.21, 8.54, 6.74, 7.69, 10.11, 7.5600000000000005, 4.47, 12.2, 10.7, 8.48, 9.27, 8.37, 7.120000000000001, 7.49, 8.25, 7.609999999999999, 7.27, 7.51, 18.37, 10.76, 6.549999999999999, 12.38, 17.01, 13.44, 6.97, 10.12, 6.74, 5.67, 9.03, 7.369999999999999, 9.36, 11.76, 14.070000000000002, 8.14, 13.889999999999999, 11.64], "Breakout": [120.44999999999999, 15.11, 18.75, 14.29, 21.67, 14.62, 10.86, 14.56, 14.82, 12.61, 15.49, 15.02, 17.79, 19.04, 13.41, 15.71, 17.04, 15.51, 13.89, 11.78, 16.09, 17.22, 9.69, 12.72, 22.41, 20.1, 16.23, 17.510000000000005, 23.22, 13.61, 21.08, 16.77, 13.48, 9.92, 79.21, 15.69, 12.25, 15.34, 17.36, 22.68, 17.87, 19.4, 16.23, 14.82, 16.65, 22.43, 13.9, 85.93, 14.71, 21.91, 58.98, 17.23, 14.49, 15.54, 16.59, 14.22, 23.02, 21.5, 14.72, 15.77, 15.37, 16.510000000000005, 15.17, 14.02, 28.25, 18.2, 15.94, 16.73, 16.03, 14.79, 18.67, 15.29, 16.26, 19.14, 15.73, 13.02, 17.57, 14.06, 15.85, 17.36, 21.89, 13.51, 19.92, 14.72, 24.34, 16.880000000000003, 21.62, 18.2, 16.42, 48.0, 18.02, 13.47, 23.0, 13.58, 13.14, 19.46, 15.41, 19.08, 15.37, 15.49], "ChopperCommand": [881.0, 775.0, 805.0, 647.0, 1300.0, 741.0, 1454.0, 1054.0, 1656.0, 732.0, 621.0, 816.0, 1268.0, 603.0, 652.0, 796.0, 941.0, 662.0, 652.0, 1029.0, 656.0, 1120.0, 858.0, 1176.0, 918.0, 534.0, 1014.0, 409.0, 587.0, 727.0, 875.0, 768.0, 1017.0, 755.0, 713.0, 827.0, 684.0, 1087.0, 784.0, 501.0, 554.0, 854.0, 667.0, 711.0, 906.0, 844.0, 1263.0, 273.0, 1067.0, 920.0, 857.0, 752.0, 831.0, 217.0, 603.0, 903.0, 796.0, 635.0, 1291.0, 1327.0, 619.0, 1000.0, 959.0, 874.0, 1425.0, 826.0, 641.0, 801.0, 746.0, 1026.0, 856.0, 732.0, 1052.0, 720.0, 804.0, 709.0, 696.0, 634.0, 1054.0, 853.0, 1014.0, 445.0, 352.0, 1203.0, 1025.0, 1072.0, 657.0, 734.0, 760.0, 997.0, 750.0, 570.0, 1322.0, 753.0, 532.0, 1410.0, 543.0, 898.0, 1125.0, 902.0], "CrazyClimber": [20835.0, 18285.0, 26710.0, 18630.0, 12669.0, 27635.0, 8445.0, 16787.0, 20015.0, 20596.0, 17345.0, 16422.0, 21438.0, 24791.0, 27122.0, 26486.0, 10802.0, 23358.0, 15657.0, 14100.0, 25178.0, 24377.0, 25049.0, 24107.0, 21971.0, 18008.0, 25566.0, 18808.0, 22569.0, 19660.0, 21886.0, 26143.0, 16871.0, 25973.0, 19128.0, 24689.0, 22177.0, 19209.0, 19301.0, 16887.0, 25443.0, 26965.0, 14724.0, 18081.0, 11978.0, 10001.0, 24816.0, 16518.0, 23998.0, 19937.0, 23767.0, 24186.0, 28149.0, 24829.0, 25163.0, 24038.0, 20186.0, 20801.0, 17359.0, 23429.0, 19765.0, 21595.0, 23617.0, 23747.0, 18196.0, 27035.0, 24461.0, 22404.0, 32233.0, 19734.0, 26355.0, 23917.0, 18604.0, 26264.0, 21712.0, 23324.0, 22280.0, 16857.0, 17760.0, 23236.0, 23567.0, 17877.0, 23897.0, 19949.0, 23403.0, 28315.0, 24620.0, 15506.0, 23541.0, 23296.0, 25089.0, 15995.0, 26291.0, 24217.0, 22228.0, 22433.0, 26727.0, 27573.0, 17662.0, 18565.0], "DemonAttack": [1203.35, 1425.85, 1276.85, 1571.65, 1485.65, 1268.95, 1730.85, 1470.3, 1712.05, 1230.2, 1816.4, 1293.45, 799.05, 1115.55, 1425.85, 904.05, 1814.5, 1505.75, 1025.45, 871.45, 1442.3, 1414.65, 1265.65, 1318.25, 1180.7, 1285.2, 1189.5, 912.25, 1568.35, 1329.3, 878.05, 1290.9, 1502.1, 1035.3, 1203.4, 1178.15, 1554.8, 1180.75, 1444.7, 1125.5, 868.05, 1645.0, 1043.15, 1414.45, 1430.95, 1284.4, 1572.0, 1097.85, 1678.8, 1195.8, 1316.7, 1818.35, 1188.9, 1189.95, 1193.3, 1430.35, 1596.6, 1406.5, 1379.0, 1526.3, 1098.55, 1172.1, 1833.7, 1079.7499999999998, 725.35, 1158.85, 1414.0, 1751.85, 1675.7, 1467.7, 1550.2, 1336.6, 1073.95, 1183.0, 887.8, 1352.1, 1030.7, 1175.35, 1071.65, 1196.45, 1681.0, 1310.15, 1396.7, 1666.0, 1667.85, 1458.35, 1023.7, 1486.5, 1321.25, 1782.35, 1512.05, 1160.5999999999997, 1650.55, 1132.75, 1235.3, 966.05, 940.35, 1159.75, 1475.2, 1369.8], "Freeway": [28.03, 30.25, 29.5, 29.58, 28.26, 29.89, 26.68, 26.08, 28.61, 26.64, 0.0, 27.99, 25.48, 30.18, 26.92, 0.67, 28.75, 0.0, 28.7, 30.04, 29.76, 27.37, 29.05, 0.0, 0.0, 25.39, 27.07, 28.04, 26.84, 30.84, 29.709999999999997, 28.51, 28.29, 28.33, 0.0, 28.97, 0.0, 26.65, 0.0, 28.57, 0.0, 28.18, 27.44, 22.91, 28.44, 27.05, 0.0, 29.19, 0.0, 28.69, 27.87, 27.49, 27.55, 0.0, 0.0, 26.98, 29.18, 29.63, 27.76, 0.0, 28.35, 23.01, 29.709999999999997, 27.33, 0.0, 28.27, 27.95, 0.0, 0.0, 0.0, 27.32, 29.06, 28.8, 0.0, 28.7, 0.0, 28.95, 26.76, 0.0, 28.49, 0.0, 26.96, 0.0, 0.0, 28.39, 29.83, 27.95, 28.06, 0.0, 0.0, 28.01, 27.91, 28.39, 29.999999999999996, 0.0, 0.0, 25.05, 28.32, 31.119999999999997, 28.94], "Frostbite": [269.7, 1986.6, 421.6, 1758.8, 2748.6, 929.0, 182.7, 1637.6, 2195.7, 253.89999999999998, 264.6, 1031.3, 245.2, 269.9, 2208.4, 250.0, 228.7, 240.89999999999998, 257.1, 252.2, 2203.4, 670.7, 247.39999999999998, 265.1, 293.1, 1203.0, 2262.7, 859.3, 262.3, 2043.3, 2131.6, 2822.3, 266.7, 2935.2, 2613.5, 698.4, 182.0, 248.5, 934.6, 2099.8, 283.3, 1460.7999999999997, 972.3, 267.8, 2676.2, 827.0, 241.8, 1854.6, 267.1, 1116.1, 2316.4, 269.6, 1703.9, 239.89999999999998, 256.7, 1831.6, 2605.4, 213.59999999999997, 560.6, 918.6, 496.9, 1258.9, 316.9, 212.0, 252.8, 1205.2, 1327.7, 740.3, 2618.6, 759.9, 264.8, 1713.1999999999998, 2700.5, 238.39999999999998, 492.8, 2407.8, 2838.6, 247.89999999999998, 2329.6, 258.5, 386.4, 257.8, 1259.4, 1770.3, 387.0, 3249.0, 77.6, 250.8, 269.1, 249.59999999999997, 2773.3, 259.8, 271.3, 256.5, 256.5, 245.3, 925.0, 250.39999999999998, 1333.4, 251.5], "Gopher": [891.4, 652.4, 925.2, 551.4, 960.0, 554.0, 378.6, 956.0, 476.6, 846.4, 510.6, 429.6, 284.2, 435.4, 286.6, 744.6, 424.0, 661.6, 761.2, 432.6, 488.8, 365.8, 829.2, 723.6, 264.2, 753.4, 614.8, 959.6, 458.4, 558.2, 337.6, 397.6, 550.2, 467.4, 763.6, 890.2, 614.2, 984.6, 420.4, 653.6, 720.6, 525.0, 473.6, 473.4, 959.6, 499.2, 963.2, 924.2, 775.8, 568.2, 721.0, 1086.8, 482.6, 682.0, 661.3999999999999, 1010.8, 439.2, 662.6, 546.0, 565.6, 297.8, 318.6, 563.4, 707.0, 924.8, 608.2, 621.8, 921.8, 1063.0, 607.8, 400.4, 892.2, 696.6, 479.6, 462.2, 355.4, 561.2, 708.6, 1014.2, 419.2, 654.0, 454.0, 693.0, 553.2, 407.6, 774.2, 814.2, 606.6, 433.6, 506.2, 694.6, 482.0, 559.0, 584.8, 589.4, 224.6, 651.4, 453.4, 828.8, 569.0], "Hero": [2958.1, 3135.0, 7559.5, 3189.9, 3064.0, 2958.05, 3115.0, 3123.0, 2970.0, 3082.0, 4772.25, 7450.6, 3065.0, 3131.4, 7558.5, 3135.0, 7559.6, 7479.0, 7560.0, 3083.5, 2770.3, 2782.75, 3135.3, 7534.4, 3060.75, 6663.75, 3073.55, 3135.0, 3135.0, 7514.75, 6447.8, 2957.25, 4293.3, 3298.15, 3235.0, 7560.0, 0.0, 7520.0, 10891.6, 6040.65, 7544.0, 3132.25, 0.0, 3021.55, 2778.25, 3134.0, 3060.75, 3134.8, 2865.75, 3185.0, 3232.0, 3126.75, 6874.3, 3208.75, 7215.9, 3135.0, 2334.2, 3124.0, 3135.0, 3185.0, 3185.0, 3124.6, 3135.0, 3079.5, 3680.5, 3161.75, 3060.0, 7560.0, 3132.85, 3184.8, 2969.75, 2971.1, 7257.9, 3134.8, 3013.25, 3085.0, 2983.5999999999995, 3085.0, 3019.0, 7324.6, 3134.5, 2774.75, 7310.15, 3134.5, 3047.15, 2897.1, 7281.25, 7320.1, 2883.15, 2976.3, 3135.0, 3105.15, 3135.0, 6879.6, 3171.6, 2968.0, 7288.9, 3134.5, 3079.0, 7559.8], "Jamesbond": [380.5, 221.0, 406.0, 197.0, 375.5, 491.5, 323.0, 285.5, 362.0, 509.5, 347.5, 354.0, 395.0, 341.0, 369.0, 310.0, 353.0, 358.0, 303.5, 406.5, 368.0, 350.5, 362.50000000000006, 421.5, 317.0, 322.5, 358.0, 345.5, 373.0, 302.0, 443.5, 309.5, 370.0, 289.0, 421.00000000000006, 369.0, 187.0, 361.5, 380.0, 325.00000000000006, 344.5, 296.0, 348.49999999999994, 333.0, 428.5, 277.0, 439.5, 384.5, 342.5, 352.5, 239.5, 342.0, 347.5, 303.0, 401.5, 443.5, 376.0, 414.5, 272.0, 416.0, 312.0, 372.5, 357.5, 290.0, 317.5, 372.5, 279.5, 341.0, 353.0, 307.5, 444.0, 360.0, 272.0, 309.0, 379.0, 350.5, 378.5, 341.49999999999994, 326.5, 266.5, 269.5, 347.0, 374.0, 265.5, 330.0, 377.0, 436.5, 406.0, 422.5, 330.0, 328.0, 321.5, 298.0, 401.5, 274.5, 379.0, 319.5, 419.0, 347.0, 366.5], "Kangaroo": [556.0, 662.0, 600.0, 2862.0, 784.0, 526.0, 5346.0, 696.0, 745.0, 6290.000000000001, 498.0, 756.0, 856.0, 284.0, 599.0, 839.9999999999999, 506.0, 612.0, 644.0, 602.0, 1849.9999999999998, 1400.0, 164.0, 608.0, 566.0, 1296.0, 702.0, 600.0, 600.0, 98.0, 600.0, 505.0, 674.0, 740.0, 748.0, 704.0, 2826.0, 622.0, 1166.0, 600.0, 5966.0, 600.0, 1120.0, 1498.0, 495.00000000000006, 602.0, 550.0, 696.0, 602.0, 706.0, 1646.0, 604.0, 6484.0, 1442.0, 702.0, 1022.0, 602.0, 1000.0000000000001, 604.0, 600.0, 648.0, 619.0, 600.0, 764.0, 750.0, 647.0, 1172.0, 2912.0, 600.0, 600.0, 600.0, 1002.0, 528.0, 500.0, 612.0, 600.0, 403.0, 445.99999999999994, 1040.0, 512.0, 1630.0, 600.0, 1118.0, 600.0, 1172.0, 602.0, 1301.0, 600.0, 1600.0, 982.0, 600.0, 934.0, 608.0, 1054.0, 614.0, -7.105427357601002e-15, 934.0, 6842.0, 834.0, 784.0], "Krull": [5763.87, 3768.63, 4784.3099999999995, 4484.7, 4515.539999999999, 4347.36, 4453.35, 5197.83, 4811.27, 3153.06, 4955.43, 4203.9, 4394.07, 4996.11, 3942.41, 3960.42, 4367.88, 3903.45, 4898.76, 3839.85, 3322.77, 4239.11, 3512.1, 3711.06, 4425.93, 5183.67, 4650.39, 4498.02, 3590.55, 3315.51, 3462.39, 3845.88, 5208.27, 4450.35, 4594.77, 3917.07, 4657.05, 2872.8, 5084.97, 5403.15, 4201.74, 3951.81, 4292.82, 4823.28, 3186.78, 4613.97, 5993.88, 5077.89, 3522.15, 4374.33, 5623.2, 4602.72, 4558.95, 3738.24, 4257.72, 5265.9, 5693.49, 4279.41, 5219.34, 3915.81, 4587.03, 4580.34, 3703.65, 4669.47, 4206.2, 4813.14, 4263.21, 5333.04, 2955.9, 3731.21, 4908.21, 4784.94, 4244.52, 5194.98, 5528.97, 3109.59, 5357.61, 4209.21, 4324.59, 4432.04, 4133.91, 4504.14, 3862.26, 3726.99, 4050.9, 4931.13, 4569.06, 4684.77, 5176.41, 4670.37, 4382.03, 3252.84, 4375.98, 4423.95, 4226.64, 4398.81, 4590.81, 5671.86, 4087.83, 3667.35], "KungFuMaster": [7964.0, 3808.0, 11108.0, 10418.0, 18319.0, 8335.0, 29185.0, 4704.0, 8679.0, 9472.0, 14160.0, 18567.0, 10190.0, 19573.0, 14448.999999999998, 3421.0, 3736.0, 9945.0, 8507.0, 2325.0, 5404.0, 19767.0, 10684.0, 10133.0, 6310.0, 20026.0, 9174.0, 3988.0, 12546.0, 1265.0, 23411.0, 10965.0, 20302.0, 2728.0, 18251.0, 8731.0, 17899.0, 20344.0, 10146.0, 9092.0, 18090.0, 2141.0, 17534.0, 10075.0, 21800.0, 21069.0, 17639.0, 28620.000000000004, 20095.0, 11109.0, 7353.0, 3138.0, 11328.0, 11320.0, 3087.0000000000005, 15228.0, 8787.0, 11142.0, 8669.0, 8214.0, 16062.0, 6104.0, 5198.0, 21532.0, 5618.0, 3029.0, 3535.9999999999995, 9480.0, 8768.0, 6125.999999999999, 3886.0, 23521.000000000004, 12171.0, 6771.999999999999, 6210.0, 6749.0, 12105.0, 10409.0, 7137.000000000001, 13189.000000000002, 21588.0, 26520.0, 9854.0, 24543.0, 4047.0, 6763.0, 9140.0, 13730.0, 13233.0, 15387.0, 8742.0, 4065.0000000000005, 19909.0, 10539.0, 5224.0, 9762.0, 10469.0, 2416.0, 18623.0, 10116.0], "MsPacman": [1521.2, 320.0, 1475.9, 1330.8, 1004.2, 1351.8, 1009.3999999999999, 1064.7, 2055.4, 1099.9, 997.8, 992.5999999999999, 1027.2, 1737.3, 1171.3, 1072.0, 1205.7, 1082.0, 1089.4, 1560.9, 1300.5, 944.5, 1560.1, 2093.3000000000006, 1468.5, 2056.8, 1271.2, 931.0, 1452.7, 820.2, 827.8, 1414.3, 1281.3000000000002, 788.4, 953.2, 1586.5, 1088.9, 1233.0, 1705.3, 736.2, 1208.3, 1176.4, 1084.5, 2052.9, 1184.9, 1214.0, 890.3999999999999, 1570.2, 1574.3, 1338.5, 1165.6, 716.5, 1238.6, 1391.7, 1027.4, 1565.0, 735.3, 1813.8, 1288.8, 854.7, 839.5999999999999, 1092.2, 804.0, 1046.3, 1207.1, 1068.9, 1288.7, 1024.2, 1263.8000000000002, 612.0, 1104.5, 1702.9, 1043.9, 1162.3, 1066.1, 931.0999999999999, 1043.4, 1314.5, 1503.7, 1001.5999999999999, 2321.7, 1312.5, 1075.7, 708.4, 1019.8, 1042.9, 720.6, 1138.7, 1573.7, 2046.6, 1324.7000000000003, 1367.2, 2071.3, 1027.3, 1544.2, 1048.3, 1098.4, 902.8999999999999, 676.4, 893.5999999999999], "Pong": [-20.29, 10.830000000000002, -4.860000000000001, -15.93, -17.18, -17.07, 15.52, -11.17, -14.67, -15.28, -4.280000000000001, -13.97, -14.45, -2.620000000000001, 9.370000000000001, -16.51, -15.45, -20.5, -11.17, -7.0, -11.67, -15.14, -16.97, -7.01, -6.940000000000001, -3.0199999999999996, -11.34, -20.16, -18.29, -18.43, -9.97, -2.039999999999999, -19.15, -9.960000000000003, -9.9, -16.04, 15.599999999999998, -12.56, -16.25, -8.94, -10.61, -13.78, 11.120000000000001, -9.51, 8.809999999999999, -13.66, -7.09, -6.969999999999999, -8.99, -16.4, -17.84, -14.7, -9.93, 14.760000000000002, -6.609999999999999, -6.26, -16.64, 17.390000000000004, -9.170000000000002, -8.76, -14.99, -16.12, -0.6799999999999962, -19.41, -1.870000000000001, -10.83, -2.259999999999998, -3.7300000000000004, -19.01, -15.32, -9.19, -17.45, -17.0, 8.510000000000002, -10.3, -9.799999999999999, -10.39, -1.4499999999999993, -12.56, -11.65, -11.9, -17.33, -15.51, -10.14, -13.0, -10.71, -6.650000000000002, -7.190000000000001, -1.4899999999999984, -13.49, -16.11, 9.2, -1.4299999999999997, -14.77, -14.73, 19.02, -6.73, -14.36, -17.08, -16.07], "PrivateEye": [98.0, 0.0, 76.0, 100.0, 100.0, -36.44, 0.0, -144.0, 0.0, 0.0, 100.0, 0.0, 100.0, 0.0, -955.18, 0.0, -40.00000000000001, 44.0, 0.0, 0.0, 49.0, 0.0, 100.0, -4.149999999999999, 0.0, 2.0, 100.0, 34.0, 0.0, -41.00000000000001, 100.0, -283.0, 1.0, 100.0, 0.0, 100.0, -162.16, 53.0, 100.0, 99.0, 0.0, 45.0, 0.0, 0.0, 43.0, 0.0, -351.09, 87.0, 100.0, 0.0, 0.0, 100.0, 26.0, 100.0, 100.0, -7.0, 100.0, 100.0, 61.0, 100.0, 0.0, 8.0, 100.0, -140.0, 5.02, -84.4, 0.0, 0.0, 42.0, -199.85, -48.68, 13.0, 18.36, 36.0, 0.0, 100.0, -10.0, 78.0, 0.0, 0.0, 12.0, 0.0, 0.0, 30.49, 0.0, -155.0, 0.0, 92.0, 0.0, 76.0, 100.0, 98.0, 0.0, 0.0, 21.0, 100.0, 0.0, 71.0, -404.0, 0.0], "Qbert": [3010.5, 2413.75, 3490.0, 743.25, 4114.249999999999, 1953.25, 3184.75, 2876.75, 2370.75, 899.75, 1202.75, 819.5, 1797.75, 2419.0, 527.5, 802.75, 3999.0000000000005, 836.25, 3460.5, 757.25, 1593.25, 3105.75, 4078.5000000000005, 3343.0, 543.0, 2401.25, 814.5, 2407.75, 833.75, 3324.5, 1185.7500000000002, 1514.75, 1854.5, 483.0, 2231.0, 4314.75, 1879.5, 649.75, 1971.0, 2035.7499999999998, 1154.5, 2420.25, 3367.75, 4360.75, 3341.5, 932.5, 3445.5, 3127.0, 811.25, 856.25, 563.75, 825.0, 796.25, 701.5, 505.75, 816.5, 780.75, 1026.25, 3512.75, 747.75, 1810.5, 734.5, 495.5, 3676.5000000000005, 1061.25, 979.75, 3158.5, 695.0, 2029.75, 1819.25, 812.0, 623.25, 1778.25, 966.25, 2990.5, 1317.75, 1371.75, 1497.5, 1562.25, 1466.5, 1713.5, 2815.0, 825.0, 641.0, 1150.75, 768.75, 866.5, 969.5, 3476.5, 833.5, 1141.25, 2478.0, 849.25, 4131.999999999999, 3394.75, 2838.75, 837.0, 1067.75, 779.75, 2310.75], "RoadRunner": [11478.0, 10396.0, 12974.0, 14736.0, 8687.0, 14452.0, 8606.0, 16542.0, 9569.0, 8908.0, 11496.0, 14977.0, 11249.0, 10920.0, 12584.0, 16662.0, 10250.0, 10707.0, 9204.0, 6658.0, 16394.0, 6553.0, 12385.0, 4843.0, 9808.0, 14675.0, 14518.0, 7684.0, 9407.0, 8312.0, 4904.0, 16810.0, 5566.0, 14369.0, 7662.0, 7676.0, 7674.0, 8189.999999999999, 16442.0, 6797.0, 8601.0, 9551.0, 13156.0, 9840.0, 17073.0, 15890.0, 15407.0, 13039.0, 17427.0, 4739.0, 16494.0, 13980.0, 9149.0, 9883.0, 10335.0, 7920.000000000001, 9264.0, 16046.000000000002, 5479.0, 16870.0, 15038.0, 15764.0, 14894.0, 7848.0, 3806.0, 5905.0, 15059.0, 7864.0, 10759.0, 6500.0, 15983.0, 9820.0, 6127.0, 14894.0, 10565.0, 15460.0, 16808.0, 9890.0, 10331.0, 9284.0, 14443.0, 8081.999999999999, 15088.0, 20437.0, 12827.0, 9183.0, 7925.999999999999, 10414.0, 15448.0, 7781.0, 10399.0, 15125.0, 11216.0, 2187.0, 9602.0, 5791.0, 16438.0, 11253.0, 6289.0, 16748.0], "Seaquest": [244.4, 240.4, 378.0, 365.79999999999995, 356.79999999999995, 400.19999999999993, 307.2, 310.0, 420.0, 418.4, 404.79999999999995, 304.0, 293.4, 297.2, 423.79999999999995, 270.0, 321.4, 405.19999999999993, 303.2, 380.0, 420.0, 334.79999999999995, 298.6, 343.6, 319.6, 354.0, 295.6, 359.0, 458.6, 383.79999999999995, 425.4000000000001, 371.79999999999995, 579.4, 366.4, 419.79999999999995, 346.19999999999993, 328.79999999999995, 340.0, 280.80000000000007, 440.4000000000001, 248.6, 224.6, 387.0, 416.79999999999995, 375.0, 388.6, 410.79999999999995, 340.0, 331.19999999999993, 274.2, 438.4, 345.4, 448.79999999999995, 336.4, 528.6, 327.4, 300.6, 371.6, 255.8, 352.79999999999995, 215.0, 342.6, 292.4, 462.19999999999993, 338.0, 370.4, 311.8, 471.4, 247.0, 340.4, 305.4, 487.4, 362.6, 368.0, 394.0, 371.6, 352.4, 317.0, 364.6, 278.8, 361.4, 461.20000000000005, 350.6, 482.4, 292.0, 262.0, 331.19999999999993, 311.0, 343.6, 314.8, 395.0, 241.6, 214.4, 510.79999999999995, 366.0, 318.4, 228.0, 401.4, 311.4, 324.0], "UpNDown": [4078.2, 4055.0999999999995, 2271.9, 4482.9, 3417.3000000000006, 5875.4, 3547.3000000000006, 6614.2, 4770.2, 5188.8, 1907.4, 3697.0, 4586.2, 4125.5, 4068.3, 4374.3, 5630.7, 3032.6, 4920.4, 3677.1, 2981.0, 7049.7, 3120.1, 3175.4, 3024.9, 4557.3, 2307.1, 3090.7, 3861.2, 6237.200000000001, 7019.3, 3365.1, 3460.2, 4998.9, 4428.8, 3972.2, 5211.2, 4923.1, 1758.1, 2247.0, 3852.2, 7394.6, 3304.1, 6041.3, 5109.9, 3566.7, 4198.9, 4916.2, 3041.3, 3709.6999999999994, 2505.5, 4466.2, 4682.6, 3481.2000000000003, 2934.8, 6771.8, 2484.9999999999995, 5256.4, 3744.6, 3168.9, 5140.6, 3890.4, 3852.1, 2966.2, 2327.4, 4522.9, 5532.5, 6578.099999999999, 4847.9, 3185.2, 2498.2, 1982.9, 3380.3, 3690.3, 6859.1, 5831.7, 7870.9, 3632.4, 3057.6, 3922.3, 4038.8999999999996, 4507.0, 4861.5, 4451.3, 9388.9, 3168.2, 3273.3, 4000.9000000000005, 10334.2, 4585.3, 4696.1, 2810.1, 3793.7, 2516.4999999999995, 6562.199999999999, 4946.3, 5024.4, 3668.9, 5720.2, 4792.1]} \ No newline at end of file diff --git a/results/data/EfficientZero.json b/results/data/EfficientZero.json new file mode 100644 index 0000000..b579e96 --- /dev/null +++ b/results/data/EfficientZero.json @@ -0,0 +1 @@ +{"Alien": [808.5], "Amidar": [148.6], "Assault": [1263.1], "Asterix": [25557.8], "BankHeist": [351.0], "BattleZone": [13871.2], "Boxing": [52.7], "Breakout": [414.1], "ChopperCommand": [1117.3], "CrazyClimber": [83940.2], "DemonAttack": [13003.9], "Freeway": [21.8], "Frostbite": [296.3], "Gopher": [3260.3], "Hero": [9315.9], "Jamesbond": [517.0], "Kangaroo": [724.1], "Krull": [5663.3], "KungFuMaster": [30944.8], "MsPacman": [1281.2], "Pong": [20.1], "PrivateEye": [96.7], "Qbert": [13781.9], "RoadRunner": [17751.3], "Seaquest": [1100.2], "UpNDown": [17264.2]} diff --git a/results/data/HUMAN.json b/results/data/HUMAN.json new file mode 100644 index 0000000..8ea4648 --- /dev/null +++ b/results/data/HUMAN.json @@ -0,0 +1 @@ +{"Alien": [7127.7], "Amidar": [1719.5], "Assault": [742.0], "Asterix": [8503.3], "BankHeist": [753.1], "BattleZone": [37187.5], "Boxing": [12.1], "Breakout": [30.5], "ChopperCommand": [7387.8], "CrazyClimber": [35829.4], "DemonAttack": [1971.0], "Freeway": [29.6], "Frostbite": [4334.7], "Gopher": [2412.5], "Hero": [30826.4], "Jamesbond": [302.8], "Kangaroo": [3035.0], "Krull": [2665.5], "KungFuMaster": [22736.3], "MsPacman": [6951.6], "Pong": [14.6], "PrivateEye": [69571.3], "Qbert": [13455.0], "RoadRunner": [7845.0], "Seaquest": [42054.7], "UpNDown": [11693.2]} diff --git a/results/data/IRIS.json b/results/data/IRIS.json new file mode 100644 index 0000000..c19cd4e --- /dev/null +++ b/results/data/IRIS.json @@ -0,0 +1,184 @@ +{ + "Alien": [ + 368.7, + 452.5, + 437.2, + 331.7, + 510 + ], + "Amidar": [ + 196.0, + 126, + 164.9, + 162.9, + 65.1 + ], + "Assault": [ + 1481.3, + 928.1, + 1121.8, + 1556.0, + 2534.8 + ], + "Asterix": [ + 762.5, + 1114, + 1082, + 716.5, + 593 + ], + "BankHeist": [ + 42.2, + 48.8, + 68.4, + 46.4, + 59.6 + ], + "BattleZone": [ + 16990, + 14570, + 11300, + 10070, + 12440 + ], + "Boxing": [ + 86.2, + 49.3, + 55.8, + 83.5, + 75.5 + ], + "Breakout": [ + 104.6, + 85.6, + 69.7, + 70.3, + 88.1 + ], + "ChopperCommand": [ + 1150, + 1276, + 1659, + 2162, + 1578 + ], + "CrazyClimber": [ + 64101, + 66127, + 62740, + 77066, + 26587 + ], + "DemonAttack": [ + 905, + 2011.9, + 2888.5, + 2613.4, + 1753 + ], + "Freeway": [ + 30.9, + 31.5, + 30.5, + 30.8, + 31.6 + ], + "Frostbite": [ + 274.3, + 250.0, + 295, + 250, + 226.3 + ], + "Gopher": [ + 2987.2, + 2476.8, + 3110.4, + 1171.4, + 1434.8 + ], + "Hero": [ + 6916.2, + 6933.9, + 6740.6, + 7881.6, + 6714.7 + ], + "Jamesbond": [ + 487.5, + 394, + 486, + 464, + 482 + ], + "Kangaroo": [ + 2249, + 732, + 750, + 90, + 370 + ], + "Krull": [ + 6047.4, + 7364.8, + 4980.6, + 6006.3, + 8683 + ], + "KungFuMaster": [ + 7301, + 15949, + 34406, + 14372, + 36771 + ], + "MsPacman": [ + 1251.6, + 941.5, + 590.3, + 770.8, + 1441.1 + ], + "Pong": [ + 19.3, + 18.5, + 16.6, + 6.2, + 12.4 + ], + "PrivateEye": [ + 100, + 100, + 100, + 100, + 100 + ], + "Qbert": [ + 853, + 575, + 790.2, + 1027.2, + 483.2 + ], + "RoadRunner": [ + 11355, + 2033, + 17106, + 7614, + 9965 + ], + "Seaquest": [ + 752.2, + 855.8, + 691, + 386, + 621.6 + ], + "UpNDown": [ + 3793, + 5367.5, + 2943.7, + 3637.9, + 1988.9 + ] +} diff --git a/results/data/MuZero.json b/results/data/MuZero.json new file mode 100644 index 0000000..56865b4 --- /dev/null +++ b/results/data/MuZero.json @@ -0,0 +1 @@ +{"Alien": [530.0], "Amidar": [38.8], "Assault": [500.1], "Asterix": [1734.0], "BankHeist": [192.5], "BattleZone": [7687.5], "Boxing": [15.1], "Breakout": [48.0], "ChopperCommand": [1350.0], "CrazyClimber": [56937.0], "DemonAttack": [3527.0], "Freeway": [21.8], "Frostbite": [255.0], "Gopher": [1256.0], "Hero": [3095.0], "Jamesbond": [87.5], "Kangaroo": [62.5], "Krull": [4890.8], "KungFuMaster": [18813.0], "MsPacman": [1265.6], "Pong": [-6.7], "PrivateEye": [56.3], "Qbert": [3952.0], "RoadRunner": [2500.0], "Seaquest": [208.0], "UpNDown": [2896.9]} diff --git a/results/data/RANDOM.json b/results/data/RANDOM.json new file mode 100644 index 0000000..ccefc10 --- /dev/null +++ b/results/data/RANDOM.json @@ -0,0 +1 @@ +{"Alien": [227.8], "Amidar": [5.8], "Assault": [222.4], "Asterix": [210.0], "BankHeist": [14.2], "BattleZone": [2360.0], "Boxing": [0.1], "Breakout": [1.7], "ChopperCommand": [811.0], "CrazyClimber": [10780.5], "DemonAttack": [152.1], "Freeway": [0.0], "Frostbite": [65.2], "Gopher": [257.6], "Hero": [1027.0], "Jamesbond": [29.0], "Kangaroo": [52.0], "Krull": [1598.0], "KungFuMaster": [258.5], "MsPacman": [307.3], "Pong": [-20.7], "PrivateEye": [24.9], "Qbert": [163.9], "RoadRunner": [11.5], "Seaquest": [68.4], "UpNDown": [533.4]} diff --git a/results/data/SPR.json b/results/data/SPR.json new file mode 100644 index 0000000..a27b683 --- /dev/null +++ b/results/data/SPR.json @@ -0,0 +1 @@ +{"Alien": [714.3, 860.3, 856.0999999999999, 808.8, 917.8999999999999, 848.5999999999999, 794.0999999999999, 750.5999999999999, 667.0, 797.5, 771.3, 518.3, 677.2, 735.2, 808.8, 700.2, 781.8999999999999, 1037.7, 617.6, 711.2, 852.8, 916.3, 864.3000000000002, 872.8999999999999, 882.2, 935.6000000000001, 754.8999999999999, 827.8, 779.5999999999999, 1119.0000000000002, 902.7, 827.0, 1318.6, 1168.2, 1601.7, 676.1, 967.3, 1059.8, 873.7, 858.5999999999999, 905.0999999999999, 802.2, 890.7000000000003, 904.4000000000001, 869.7, 760.8999999999999, 741.8999999999999, 859.0, 717.9, 1060.0000000000002, 703.1000000000001, 1400.1, 767.3999999999999, 908.8999999999999, 673.7, 640.3, 555.4, 906.3, 800.5, 861.2, 683.5, 850.3000000000002, 765.0, 695.9000000000001, 851.7, 671.2999999999998, 689.6, 645.8, 828.0, 1218.1, 754.7, 831.3, 632.9, 758.2, 1308.7000000000003, 979.3, 791.3, 1484.7999999999995, 719.5999999999999, 761.8, 971.2, 1013.1000000000001, 1085.7, 656.4000000000001, 708.5, 1037.1000000000001, 784.7, 856.5000000000002, 778.8999999999999, 763.8999999999999, 709.0999999999999, 730.2000000000002, 769.8999999999999, 692.8, 595.6, 911.1000000000001, 979.8, 645.4, 697.3, 717.7], "Amidar": [203.68, 188.44, 195.38, 185.88, 135.25, 160.29, 187.03, 119.07000000000001, 183.5, 204.37, 209.51, 161.66999999999996, 151.9, 109.11, 143.73, 156.67, 228.62, 150.82, 160.40000000000003, 182.51000000000002, 165.96, 122.03, 106.67999999999998, 186.54, 85.62, 242.52999999999997, 120.34, 146.45, 204.32999999999998, 194.32999999999998, 196.56999999999996, 179.66, 274.84, 204.7, 141.59, 132.44, 232.14999999999998, 227.19000000000003, 176.61000000000004, 200.13999999999996, 197.87000000000003, 130.10000000000002, 189.99, 143.27, 167.98, 205.38999999999996, 128.19, 162.44, 185.33, 141.67, 183.25999999999996, 171.71, 127.85999999999997, 198.41000000000003, 152.46, 185.78, 243.37000000000003, 182.78000000000003, 185.4, 218.94000000000003, 160.92999999999998, 194.29, 142.86999999999998, 125.76999999999997, 165.56000000000003, 170.05000000000004, 253.06000000000003, 133.92, 123.91000000000001, 161.36000000000004, 214.68, 205.49999999999997, 194.93000000000004, 215.38, 123.59999999999998, 173.46000000000004, 260.16, 125.45999999999997, 268.41, 238.29999999999998, 126.12, 216.67, 205.37999999999997, 176.19, 161.80999999999997, 179.97, 194.24, 182.15, 145.5, 297.75, 193.72, 137.98, 233.14, 137.55999999999997, 173.86, 285.59000000000003, 235.75999999999996, 186.40999999999997, 157.08, 231.85000000000002], "Assault": [462.52, 673.6, 660.65, 492.02, 596.06, 570.36, 590.42, 596.42, 594.96, 472.58, 608.73, 585.25, 630.28, 590.26, 557.31, 592.85, 606.1, 550.28, 457.78999999999996, 601.39, 613.09, 496.5, 601.32, 480.15999999999997, 468.9200000000001, 657.77, 640.54, 665.67, 541.51, 492.94999999999993, 476.69, 561.69, 580.58, 604.61, 496.85, 589.93, 549.74, 547.96, 519.73, 516.24, 534.86, 614.84, 542.19, 682.99, 563.5, 558.67, 625.28, 540.81, 598.72, 506.49, 510.47, 603.8300000000002, 578.89, 628.23, 442.68, 604.38, 556.44, 510.9, 512.91, 537.78, 638.33, 522.88, 538.98, 662.83, 565.84, 517.62, 582.29, 560.31, 556.45, 591.32, 616.56, 545.82, 519.61, 483.93999999999994, 580.81, 554.95, 525.43, 510.61, 576.53, 540.16, 579.57, 542.9600000000002, 500.0799999999999, 559.93, 432.49, 679.14, 610.14, 591.63, 569.52, 646.26, 582.09, 599.0, 511.3399999999999, 567.81, 585.26, 607.93, 618.82, 638.39, 535.49, 565.68], "Asterix": [901.0, 1212.5, 989.5, 1004.5, 724.0, 994.5, 1057.0, 1059.5, 793.5, 1041.5, 943.0, 1150.5, 972.0000000000001, 922.0, 1032.5, 1068.5, 830.5, 1001.0, 825.4999999999999, 808.5, 1121.0, 912.5, 1253.0000000000002, 865.0, 710.0, 1026.5, 836.0, 1080.0, 962.5000000000001, 1108.0, 812.0, 1241.5, 963.4999999999999, 1108.5, 872.5, 795.5, 1317.0, 1104.0, 1214.5, 1019.5000000000001, 777.0, 790.5, 964.5, 1076.0, 672.0, 1069.0, 995.9999999999999, 764.0000000000001, 1161.0, 892.0, 904.5, 888.5, 1190.0, 837.0, 832.5, 856.0, 1184.0, 1156.0, 863.0, 1166.5, 772.5, 944.5, 902.0, 666.5, 941.0, 872.0, 975.0000000000001, 843.5, 932.0, 1161.5, 669.5, 1166.0, 963.0, 817.0, 805.5, 974.5, 1022.0000000000001, 1022.5000000000001, 888.5, 833.5, 1310.5, 731.0, 778.5, 971.4999999999999, 961.5, 1198.0, 860.5, 930.0, 1010.5, 1183.0, 781.0, 969.9999999999999, 1329.0, 975.5, 961.5, 990.5000000000001, 908.4999999999999, 744.5, 904.0, 881.5], "BankHeist": [207.4, 492.3, 234.9, 189.3, 267.8, 578.8, 258.4, 861.9, 143.9, 574.2, 558.7, 282.1000000000001, 559.3, 526.7, 232.2, 171.0, 242.5, 208.4, 545.9, 631.2, 140.79999999999998, 649.1, 754.3999999999999, 43.099999999999994, 309.09999999999997, 237.6, 67.6, 194.9, 276.9, 346.2, 594.0, 184.49999999999994, 592.5999999999999, 607.0, 575.0, 174.2, 272.9, 149.90000000000003, 589.7, 167.20000000000002, 185.7, 286.3, 659.2, 259.3, 153.9, 116.90000000000003, 158.0, 572.3, 555.8, 686.9999999999999, 69.6, 229.5, 206.00000000000003, 186.10000000000002, 202.30000000000004, 225.9, 621.9, 786.2999999999997, 727.7, 223.8, 216.00000000000003, 112.7, 415.99999999999994, 182.1, 269.6, 188.7, 95.1, 506.5, 588.2, 309.5, 505.0, 145.1, 202.2, 195.59999999999994, 605.7, 166.4, 810.1, 562.6, 548.1, 193.50000000000003, 188.49999999999997, 174.4, 256.40000000000003, 230.49999999999997, 135.5, 197.3, 297.40000000000003, 636.5, 482.59999999999997, 241.8, 223.89999999999998, 129.89999999999998, 188.4, 175.3, 200.60000000000005, 160.0, 304.4, 795.5000000000001, 191.19999999999996, 708.2000000000002], "BattleZone": [18940.0, 18120.0, 21430.0, 18410.0, 16100.0, 14290.0, 12650.0, 17100.0, 15120.0, 14350.0, 12469.999999999998, 5880.0, 16950.0, 18240.0, 11090.0, 15170.0, 8750.0, 15690.0, 18840.0, 20070.0, 21050.0, 17360.0, 18530.0, 12649.999999999998, 14030.0, 7900.0, 15989.999999999998, 18320.0, 8350.0, 15740.0, 13470.0, 14250.0, 11820.0, 16340.000000000002, 4950.0, 11430.0, 11360.0, 8610.0, 4930.0, 16970.0, 11890.0, 21870.0, 14090.0, 14280.0, 15890.0, 12500.0, 3770.0, 15850.0, 9110.0, 21450.0, 15870.0, 19120.0, 12960.0, 18550.0, 20770.0, 10130.0, 16990.0, 11700.0, 19220.0, 12880.0, 11550.0, 26440.0, 16360.000000000002, 14869.999999999998, 19770.0, 9880.0, 15240.0, 13540.0, 15230.0, 14780.0, 3530.0, 8860.0, 20460.0, 21940.0, 21790.0, 19110.0, 14320.0, 7840.0, 19620.0, 17040.0, 23160.0, 21280.0, 13150.0, 11770.0, 17510.0, 10910.0, 23440.0, 13970.000000000002, 14850.0, 12699.999999999998, 19600.0, 10450.000000000002, 11710.0, 18190.0, 20350.0, 20890.0, 2280.0, 16350.0, 13120.0, 2960.0], "Boxing": [42.6, 60.79, 31.71, 32.38, 17.31, 26.89, 35.53, 43.23, 19.74, 48.23, 59.45000000000001, 19.34, 30.82, 29.9, 34.68, 31.03, 20.27, 13.479999999999999, 41.839999999999996, 44.56, 27.250000000000004, 31.549999999999997, 12.17, 40.65999999999999, 44.59, 21.35, 25.549999999999997, 40.93, 33.07, 38.46, 25.12, 19.73, 29.36, 30.580000000000002, 50.550000000000004, 24.86, 39.71, 65.77, 17.03, 36.05, 54.68, 49.56000000000001, 15.83, 71.48, 25.849999999999998, 21.66, 49.81, 36.47, 16.020000000000003, 36.5, 14.66, 39.89, 35.3, 36.23, 23.89, 49.52, 12.33, 42.62, 46.28, 40.58, 33.41, 25.090000000000003, 22.330000000000002, 26.089999999999996, 51.65, 70.51000000000002, 29.54, 27.7, 30.83, 15.75, 45.98, 43.889999999999986, 18.22, 53.199999999999996, 15.020000000000001, 27.1, 38.52, 43.69, 45.98, 53.57999999999999, 46.69, 35.1, 56.690000000000005, 20.689999999999998, 44.15999999999999, 54.03, 34.85, 42.97, 42.23, 20.26, 22.439999999999998, 35.52, 25.43, 18.18, 37.35, 64.42, 54.21, 72.74, 46.06, 20.62], "Breakout": [16.59, 14.81, 19.45, 16.87, 15.04, 13.94, 17.01, 19.63, 14.81, 22.94, 12.51, 19.81, 12.75, 19.82, 16.4, 22.66, 17.300000000000004, 17.72, 65.87, 14.79, 24.58, 14.94, 21.000000000000004, 13.93, 16.470000000000002, 15.530000000000001, 11.46, 13.84, 11.650000000000002, 17.03, 19.61, 21.71, 18.99, 20.74, 14.770000000000001, 79.6, 14.09, 15.66, 21.61, 22.66, 17.91, 19.6, 19.36, 21.01, 20.69, 16.56, 119.98999999999998, 14.91, 13.129999999999999, 20.04, 20.829999999999995, 14.52, 13.66, 14.51, 16.74, 13.64, 20.04, 15.73, 21.54, 14.24, 18.750000000000004, 21.67, 18.22, 14.400000000000002, 15.759999999999998, 14.22, 16.07, 21.98, 18.31, 15.91, 17.73, 24.58, 16.96, 23.28, 15.390000000000002, 24.03, 15.15, 16.53, 16.72, 18.09, 20.35, 12.45, 22.25, 16.62, 16.57, 14.36, 14.479999999999999, 20.49, 16.13, 20.649999999999995, 17.48, 19.15, 14.980000000000002, 25.26, 18.74, 11.8, 21.53, 13.64, 15.929999999999998, 12.030000000000001], "ChopperCommand": [779.0, 1197.0, 1442.0, 902.0, 776.0, 1093.0, 1067.0, 515.0, 1246.0, 731.0, 1379.0, 834.0, 939.0, 1082.0, 724.0, 747.0, 1085.0, 1095.0, 861.0, 1294.0, 1090.0, 673.0, 1218.0, 952.0, 1561.0, 1359.0, 1245.0, 992.0, 725.0, 615.0, 880.0, 526.0, 610.0, 723.0, 600.0, 754.0, 853.0, 1336.0, 712.0, 742.0, 778.0, 741.0, 1997.0000000000002, 650.0, 1256.0, 1349.0, 1062.0, 1070.0, 573.0, 977.0, 1178.0, 950.0, 912.0, 1671.0, 598.0, 700.0, 1265.0, 793.0, 1088.0, 606.0, 712.0, 616.0, 861.0, 1009.0, 909.0, 854.0, 877.0, 1095.0, 1141.0, 776.0, 731.0, 757.0, 907.0, 1002.0, 751.0, 910.0, 937.0, 581.0, 1051.0, 894.0, 913.0, 1267.0, 723.0, 931.0, 1270.0, 1724.0, 668.0, 763.0, 570.0, 934.0, 693.0, 607.0, 1463.0, 951.0, 819.0, 884.0, 583.0, 1185.0, 1552.0, 591.0], "CrazyClimber": [67048.0, 27829.0, 38513.0, 27726.0, 60457.0, 32038.0, 44142.0, 57226.0, 31333.0, 28490.0, 31324.000000000004, 39828.0, 30617.0, 50535.0, 53040.0, 22155.0, 24338.0, 24058.0, 54522.0, 52659.0, 35147.0, 22593.0, 32821.0, 52044.99999999999, 62311.000000000015, 38200.0, 30661.0, 48235.99999999999, 44532.0, 33407.0, 29989.0, 33147.0, 30382.0, 35320.0, 24839.0, 48273.0, 23380.0, 27396.0, 29674.0, 26479.0, 41270.0, 22062.0, 69872.0, 59836.0, 51375.0, 32744.0, 13391.0, 35469.0, 34039.0, 34220.0, 39441.0, 20052.0, 32131.0, 42641.0, 42992.0, 49616.0, 28224.0, 34120.0, 32381.000000000004, 27829.0, 15353.0, 29338.0, 59252.0, 21907.0, 35770.0, 23899.0, 34653.0, 28082.0, 38413.0, 50866.0, 36235.0, 55137.0, 37552.0, 24967.0, 53032.0, 51431.0, 43722.0, 42436.99999999999, 27166.0, 27996.0, 19955.0, 30902.0, 27903.0, 41513.99999999999, 41618.99999999999, 25882.0, 37586.0, 21377.0, 33969.0, 31187.0, 24405.0, 30369.0, 61429.999999999985, 32716.0, 35679.0, 25220.0, 48673.99999999999, 50713.00000000001, 29072.0, 32251.0], "DemonAttack": [489.65, 293.4, 456.80000000000007, 412.9, 726.6, 703.25, 715.6, 590.55, 424.45000000000005, 639.25, 772.4, 389.45000000000005, 523.15, 374.20000000000005, 623.2999999999998, 690.9, 453.5, 564.5, 494.1, 458.4, 768.75, 522.05, 655.9999999999999, 363.2, 467.45000000000005, 441.20000000000005, 447.95000000000005, 705.35, 262.1, 385.1, 342.55, 412.95000000000005, 638.35, 708.25, 650.6, 682.1, 681.7000000000002, 472.65, 552.75, 472.5, 714.25, 457.45000000000005, 444.70000000000005, 548.7, 489.3499999999999, 428.0, 319.2, 466.45000000000005, 428.85, 583.2, 645.9499999999999, 367.0, 449.6, 702.1, 507.75, 537.0, 654.65, 674.35, 358.94999999999993, 765.3, 409.20000000000005, 507.95000000000005, 602.8499999999999, 700.35, 403.55000000000007, 450.65, 410.25, 472.35, 449.20000000000005, 592.5999999999998, 549.25, 505.05000000000007, 345.2, 313.3, 412.0, 489.15, 546.85, 392.5, 521.9, 550.2, 628.4, 380.1, 428.0, 472.2499999999999, 549.2, 596.35, 571.6, 676.6500000000001, 402.0, 474.80000000000007, 415.25, 548.55, 374.69999999999993, 548.0, 658.8999999999999, 369.15, 501.05000000000007, 609.4499999999999, 541.6499999999999, 419.20000000000005], "Freeway": [29.28, 30.08, 30.61, 31.359999999999996, 0.0, 30.91, 29.740000000000002, 0.0, 31.020000000000003, 30.77, 31.289999999999996, 31.2, 0.0, 30.810000000000002, 0.0, 29.380000000000003, 29.039999999999996, 31.280000000000005, 28.88, 28.88, 0.0, 0.0, 30.310000000000002, 30.160000000000007, 23.19, 30.429999999999996, 31.289999999999996, 23.65, 0.0, 30.46, 25.62, 0.0, 31.68, 30.590000000000003, 31.140000000000004, 0.0, 29.61, 0.0, 30.9, 30.280000000000005, 0.0, 29.739999999999995, 0.0, 29.870000000000005, 31.160000000000004, 0.0, 0.0, 0.0, 0.0, 0.0, 29.89, 29.97, 29.91, 30.519999999999996, 31.359999999999996, 31.49, 30.020000000000003, 0.0, 24.36, 0.0, 29.15, 29.409999999999997, 29.62, 31.66, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 30.2, 28.109999999999996, 0.0, 0.0, 28.590000000000003, 0.0, 27.970000000000002, 31.719999999999995, 0.0, 29.88000000000001, 22.48, 0.0, 21.16, 0.0, 21.5, 30.290000000000003, 30.12, 30.05, 21.37, 29.999999999999996, 21.96, 0.0, 0.0, 30.94, 29.419999999999998, 30.359999999999996, 28.97, 30.84, 29.61, 0.0], "Frostbite": [767.6, 3159.4, 2839.1, 2607.9, 253.59999999999997, 228.5, 239.39999999999998, 2749.2, 2641.9, 2728.1, 233.60000000000002, 253.0, 247.40000000000003, 259.9, 264.7, 263.1, 248.39999999999998, 2737.8, 4567.499999999999, 270.0, 1292.7, 237.89999999999998, 248.7, 3111.8, 3103.2, 1026.1999999999998, 450.19999999999993, 1384.4, 269.7, 1691.6, 1486.1, 428.2, 2487.1, 1572.0, 251.5, 239.59999999999997, 2891.2, 245.09999999999997, 1077.2, 887.8, 265.0, 3337.5, 1836.4, 268.9, 292.7, 1297.7, 2820.6, 611.5999999999999, 2953.4, 242.0, 3209.9, 275.1, 1151.6, 1086.1, 254.3, 262.7, 2815.6, 262.4, 2341.8, 1779.1, 249.70000000000005, 269.9, 269.0, 396.9, 270.30000000000007, 250.0, 1453.2000000000005, 256.1, 775.2, 268.5, 249.59999999999997, 264.5, 258.8, 1700.9, 257.6, 2816.5, 1879.1000000000004, 253.3, 444.8, 260.00000000000006, 2704.1, 238.09999999999997, 1613.2000000000003, 2084.5999999999995, 260.5, 254.09999999999997, 266.70000000000005, 250.0, 2083.2, 2683.0, 601.5, 263.2, 1661.0000000000005, 3010.3, 1356.3, 2171.5, 2641.8, 258.6, 263.8999999999999, 258.4], "Gopher": [639.0, 647.4, 810.2, 754.2, 316.2, 663.8, 924.6, 1244.2, 495.2, 656.8, 623.4000000000001, 442.2, 568.0, 553.2, 482.8, 339.4, 469.2, 811.2, 554.4, 883.6000000000001, 840.2000000000002, 448.2, 521.2, 816.4000000000001, 621.0, 611.0, 364.6, 869.6, 535.6, 635.6, 697.6, 550.8, 1094.8, 575.4, 519.6, 881.2, 718.6, 471.79999999999995, 1544.6, 449.0, 916.2, 512.6, 919.6000000000003, 495.6, 600.4, 1033.6, 836.8000000000001, 759.5999999999999, 333.6, 912.4, 400.4, 447.6, 515.2, 880.9999999999999, 714.8, 764.8, 597.2, 815.4, 667.0, 758.6000000000001, 599.4, 446.19999999999993, 914.6, 1136.0, 391.8, 640.2, 686.4, 370.2, 608.8, 1234.4, 425.2, 660.8, 508.4, 514.8, 474.2, 440.8, 791.6, 519.6, 575.2, 318.2, 354.8, 449.2, 450.8, 607.6, 987.0, 991.0, 631.2, 493.6, 742.4000000000001, 555.2, 264.0, 842.6, 814.0, 831.2, 533.2, 716.0, 989.7999999999998, 640.2, 989.0, 394.2], "Hero": [7560.4, 8636.1, 7560.0, 7536.2, 7560.0, 7518.6, 5569.0, 7527.1, 7521.5, 3203.55, 3135.0000000000005, 13434.7, 7556.0, 7560.799999999998, 3135.0000000000005, 7559.2, 3135.0000000000005, 7520.000000000001, 4689.1, 7372.849999999999, 7587.2, 6620.100000000001, 7447.350000000001, 7520.000000000001, 7573.799999999999, 2940.9, 3135.0000000000005, 7522.5999999999985, 7485.6, 5892.95, 7564.9, 7537.15, 3185.0, 3103.15, 3135.0000000000005, 7641.4, 3185.0, 7560.000000000002, 7515.949999999999, 3165.0, 5214.75, 3185.0, 3184.5, 3185.0, 7560.400000000001, 7520.000000000001, 7516.250000000001, 7484.65, 7545.599999999999, 5235.05, 7365.099999999999, 3135.0000000000005, 7485.149999999999, 7532.0, 7559.1, 3146.0, 3184.5, 3169.4999999999995, 3135.0000000000005, 3131.85, 3145.6, 3235.0, 7476.150000000001, 7560.400000000001, 7410.9, 2929.4999999999995, 7605.300000000001, 7523.249999999998, 7437.95, 3135.0000000000005, 3185.0, 7560.400000000001, 7560.000000000002, 7560.000000000002, 3185.0, 3135.0000000000005, 7556.2, 3135.0000000000005, 7681.6, 3135.0000000000005, 7525.999999999999, 3185.0, 7549.5, 7512.000000000001, 3172.5000000000005, 3135.0000000000005, 3133.4, 7520.399999999999, 4224.75, 3096.75, 7654.799999999999, 6562.3, 7690.2, 7502.549999999998, 7519.900000000001, 7468.5, 3135.0000000000005, 7522.8, 3135.0000000000005, 7374.05], "Jamesbond": [350.5, 264.0, 437.0, 451.0, 389.0, 351.0, 418.0, 297.5, 358.5, 337.0, 460.0, 271.5, 417.0, 433.5, 483.5, 388.5, 374.0, 374.0, 439.99999999999994, 395.4999999999999, 329.00000000000006, 305.5, 353.49999999999994, 362.0, 374.0, 285.0, 371.5, 368.0, 455.0, 279.5, 335.5, 337.5, 397.0, 399.5, 413.0, 355.50000000000006, 460.0, 315.5, 406.5, 489.5, 404.0, 367.0, 235.5, 349.5, 388.5, 272.5, 394.00000000000006, 336.5, 332.5, 392.99999999999994, 346.0, 330.0, 306.0, 457.0, 347.5, 380.0, 238.5, 339.0, 291.99999999999994, 442.0, 165.0, 399.99999999999994, 304.00000000000006, 301.5, 327.0, 330.99999999999994, 345.0, 340.0, 376.50000000000006, 395.0, 381.50000000000006, 329.00000000000006, 356.0, 276.49999999999994, 333.5, 451.0, 374.5, 363.0, 428.5, 415.0, 576.5, 356.0, 387.0, 389.0, 366.5, 410.00000000000006, 323.0, 325.5, 326.5, 442.0, 433.0, 365.50000000000006, 305.5, 377.0, 379.00000000000006, 323.5, 355.50000000000006, 401.0, 287.5, 491.5], "Kangaroo": [3858.0, 4049.0, 650.0, 786.0, 1178.0, 3599.0, 6580.0, 600.0, 9596.0, 1868.0, 4906.0, 7718.000000000001, 292.0, 1800.0, 1644.0, 994.0, 1525.0, 1216.0, 2102.0, 3260.9999999999995, 808.0000000000001, 2272.0, 1127.9999999999998, 5820.0, 9964.0, 2722.0, 1480.0, 9592.0, 3849.0, 600.0, 3188.0, 5100.000000000001, 3974.0, 9924.0, 3442.0, 398.0, 839.9999999999999, 1066.0, 4658.0, 908.0000000000001, 1766.0, 558.0, 4308.0, 748.0, 1002.0, 3208.0, 1554.0, 638.0, 6756.0, 918.0, 9290.0, 8934.000000000002, 774.0000000000002, 7362.000000000001, 10562.000000000002, 7801.000000000002, 1618.0000000000002, 612.0000000000001, 1604.0, 602.0, 838.0000000000001, 9451.0, 600.0, 172.0, 7097.999999999999, 804.9999999999999, 789.9999999999998, 598.0000000000001, 3414.0, 5935.999999999999, 6415.000000000002, 592.0, 766.0, 7698.000000000002, 7412.000000000001, 3966.0, 10906.000000000002, 3107.9999999999995, 4818.0, 884.0000000000002, 10868.000000000002, 600.0, 3650.0, 9869.0, 1552.0, 2538.0, 6756.0, 6784.000000000001, 5477.0, 4407.999999999999, 2376.0, 518.0, 600.0, 2600.0, 9172.0, 2500.0, 1990.0, 11223.0, 817.9999999999998, 600.0], "Krull": [3554.16, 5153.81, 4255.09, 3557.78, 2735.55, 3761.55, 3380.78, 3192.91, 3457.32, 3839.57, 3502.87, 4365.539999999999, 3573.46, 3565.4699999999993, 3711.63, 4381.23, 3242.6399999999994, 3321.2, 3399.05, 3743.06, 4261.09, 3510.26, 3280.52, 3773.72, 4058.26, 3718.14, 2985.26, 3041.96, 3405.63, 3423.28, 3842.82, 3520.12, 3171.62, 2741.88, 3895.03, 3432.7, 2987.18, 3480.1, 4399.39, 3622.7299999999996, 3452.5, 3511.26, 3302.6099999999997, 4474.56, 4341.41, 3248.83, 3638.55, 3705.11, 4002.5699999999997, 3858.84, 3368.01, 3312.99, 3947.73, 3735.22, 4071.08, 3670.22, 2711.39, 4192.609999999999, 4400.12, 3680.68, 3580.21, 3718.78, 3832.38, 4344.310000000001, 3261.8499999999995, 3813.31, 3921.93, 4246.71, 3595.49, 4586.66, 3397.47, 3811.1100000000006, 3792.82, 4062.69, 3919.69, 2503.62, 3260.51, 4482.560000000001, 3835.8, 3897.64, 3149.69, 4186.97, 3569.51, 3159.25, 3615.86, 4016.21, 2927.93, 3628.55, 2777.0699999999997, 3985.83, 3360.38, 4065.39, 3509.46, 3683.0299999999997, 4363.39, 4156.299999999999, 4114.720000000001, 3469.07, 4107.99, 3576.62], "KungFuMaster": [11769.000000000002, 14144.0, 29401.0, 16546.0, 10218.0, 12325.0, 9565.0, 10648.0, 8188.0, 9123.0, 13030.0, 9225.000000000002, 12964.999999999998, 6823.000000000001, 25212.000000000004, 16844.0, 16024.0, 3967.0, 10196.999999999998, 18981.0, 7481.0, 12445.0, 26588.000000000004, 15584.999999999998, 8160.0, 13292.0, 33608.00000000001, 18613.0, 26502.000000000004, 12505.999999999998, 10167.0, 24103.0, 26104.0, 17844.0, 12480.000000000002, 8279.0, 21071.999999999996, 11284.0, 6449.999999999999, 12531.0, 13974.999999999998, 6407.0, 21798.000000000004, 12639.0, 7928.0, 8419.000000000002, 10468.000000000002, 9116.0, 18356.0, 12883.0, 13411.000000000002, 11050.000000000004, 13314.0, 10748.999999999998, 11148.0, 6338.0, 13692.0, 24240.0, 14359.0, 35215.0, 12964.0, 14779.0, 5244.0, 9111.0, 16796.0, 9808.0, 16964.0, 8405.0, 17821.0, 15958.0, 11285.0, 5158.0, 15997.000000000002, 11173.0, 6440.0, 24220.0, 30187.0, 13558.0, 23982.000000000004, 10101.999999999998, 29448.000000000007, 26540.999999999993, 7694.0, 12115.000000000002, 9733.999999999998, 11026.0, 12347.999999999998, 16337.0, 24816.0, 29330.0, 6336.0, 7907.0, 8428.0, 10726.000000000002, 24830.0, 29299.000000000004, 25337.999999999996, 15167.000000000002, 7672.999999999999, 17484.0], "MsPacman": [1430.6, 1583.4, 1175.9, 712.2, 1322.0, 1682.0, 1518.1, 1056.5, 1448.0, 1203.4, 1241.5000000000002, 1999.3, 1688.4, 2301.4, 1583.3, 1335.8, 1742.7000000000003, 1236.9, 761.6000000000001, 1338.3, 1140.7, 924.0999999999999, 1072.2000000000003, 1168.6000000000001, 1159.3000000000002, 1019.5, 1537.8, 889.3, 965.3, 1715.0000000000002, 1364.8, 1056.4, 904.0999999999999, 1411.7000000000003, 1345.2, 778.0999999999999, 1096.2000000000003, 1386.8000000000002, 1526.3999999999999, 780.2, 1730.8, 1491.7, 1577.9, 1549.3, 836.8999999999999, 934.8000000000002, 1336.7, 1162.6, 2203.3, 1054.0, 952.5999999999999, 1439.8, 865.6000000000001, 1087.2000000000003, 870.7, 1905.3, 1149.0, 982.0999999999999, 1803.4999999999998, 1692.2999999999997, 1238.0999999999997, 951.7, 1110.2, 1464.3, 1547.2, 1660.7, 1221.6, 957.7, 992.5999999999999, 987.8999999999999, 1547.7, 2087.6, 1250.8000000000002, 951.2, 1019.5, 1632.7, 1225.2999999999997, 1388.4, 1010.7000000000003, 1150.5000000000002, 981.8999999999999, 1465.9999999999998, 1023.3999999999999, 1368.8, 1521.8999999999996, 1151.5000000000002, 1540.7000000000003, 1341.5, 1715.9000000000005, 1107.7, 966.8999999999999, 1648.1, 1378.5, 1382.0, 1675.1, 2338.0000000000005, 1283.7, 2103.0, 1215.1, 1000.5999999999999], "Pong": [2.049999999999997, -13.620000000000001, -7.300000000000001, 5.0, -5.82, -3.16, -13.02, -10.86, -8.69, -3.460000000000001, -9.15, -3.870000000000001, 16.170000000000005, 1.0599999999999987, -17.99, -8.82, 19.84, -16.98, -7.109999999999999, -13.39, 2.0, -18.13, 0.620000000000001, -12.91, -15.61, 6.260000000000005, -14.37, -14.230000000000002, -4.609999999999999, -10.19, 5.559999999999999, -15.14, -10.66, 14.330000000000002, -17.37, -1.4899999999999984, -0.35999999999999943, -13.890000000000004, -3.1699999999999946, -16.57, -17.4, -12.98, 2.960000000000001, -10.220000000000002, -11.230000000000002, 9.7, 0.6400000000000006, -8.929999999999998, -13.53, 14.620000000000001, -9.630000000000003, 18.55, -10.94, -10.599999999999998, -15.01, -3.1900000000000013, 20.34, 19.19, 18.56, 10.510000000000002, 15.720000000000002, -8.41, -2.4700000000000024, -7.830000000000002, -11.350000000000001, -14.43, -12.830000000000002, -18.46, -10.42, -15.65, -13.26, -9.47, 16.470000000000002, -16.0, -3.4800000000000004, -9.97, -16.67, 3.3200000000000003, -13.2, -11.850000000000001, -9.399999999999999, -14.79, -9.37, -16.35, -17.79, -13.66, -6.84, -3.4400000000000013, -6.169999999999996, -5.74, -12.539999999999997, -5.789999999999999, -11.39, -2.4700000000000024, 17.210000000000004, 9.29, 10.66, -11.47, -5.860000000000005, -17.4], "PrivateEye": [100.0, 100.0, 93.12, 100.0, 100.0, 97.51000000000002, 349.46, 100.0, 100.0, 100.0, 99.99999999999997, 0.0, 99.99999999999997, 87.0, -461.0, 99.99999999999997, 99.99999999999997, 99.99999999999997, 99.99999999999997, 99.99999999999997, 102.71000000000001, 99.99999999999997, 99.99999999999997, 99.99999999999997, 99.99999999999997, 99.99999999999997, 99.99999999999997, 99.99999999999997, 99.99999999999997, 99.99999999999997, 99.99999999999997, 99.99999999999997, 99.99999999999997, 99.99999999999997, 99.99999999999997, 99.99999999999997, 99.99999999999997, 99.99999999999997, 99.99999999999997, 99.99999999999997, 99.99999999999997, 99.99999999999997, -162.7, 99.99999999999997, 99.99999999999997, 99.99999999999997, 99.99999999999997, 99.99999999999997, 99.99999999999997, 99.99999999999997, 99.99999999999997, 99.99999999999997, 98.05000000000001, 99.99999999999997, 99.99999999999997, 99.99999999999997, 74.0, 99.99999999999997, 99.99999999999997, 99.99999999999997, 99.99999999999997, 99.99999999999997, 99.99999999999997, 99.99999999999997, 0.0, 99.99999999999997, 99.99999999999997, 99.99999999999997, 99.99999999999997, 99.99999999999997, 99.99999999999997, 99.99999999999997, 132.45, 99.99999999999997, 94.0, 99.99999999999997, 99.99999999999997, 99.99999999999997, 99.99999999999997, 99.99999999999997, 99.99999999999997, -200.25, 99.99999999999997, 99.99999999999997, 95.51000000000002, -43.04999999999999, 99.99999999999997, 99.99999999999997, 99.99999999999997, 99.99999999999997, 99.99999999999997, 99.99999999999997, 99.99999999999997, 99.99999999999997, 99.99999999999997, 99.99999999999997, 99.99999999999997, -57.600000000000016, 99.99999999999997, 99.99999999999997], "Qbert": [737.5, 472.75, 775.0, 500.75, 824.0, 761.5, 787.75, 447.0, 598.25, 786.75, 768.75, 474.5, 1029.25, 743.0, 1023.75, 496.75, 797.25, 914.5, 779.0, 800.0000000000001, 755.25, 719.7500000000001, 834.5, 773.2500000000001, 747.0, 784.0, 539.25, 1028.0000000000002, 468.25, 634.25, 1692.5, 579.25, 937.5, 703.0000000000001, 726.5, 1219.0, 689.25, 983.7499999999999, 766.5, 1098.0000000000002, 745.5000000000001, 1874.75, 951.2500000000001, 841.75, 746.25, 773.5000000000001, 823.75, 626.5, 763.75, 1000.5, 437.5, 701.25, 501.0000000000001, 794.0, 692.75, 1651.25, 4355.0, 758.75, 488.5, 454.75, 730.0, 774.75, 680.0, 1296.0, 731.75, 814.0, 3277.5, 756.0, 926.25, 809.75, 614.5, 747.0, 751.5000000000001, 762.5, 766.0000000000001, 750.75, 1362.75, 792.75, 965.0, 500.25, 1134.0, 776.5, 783.25, 772.0, 809.2500000000001, 920.0000000000001, 1264.0, 760.0, 996.75, 787.5000000000001, 813.25, 597.75, 780.25, 750.5000000000001, 704.25, 870.25, 646.5, 908.7500000000001, 786.5, 778.75], "RoadRunner": [11594.0, 11498.0, 11077.0, 12142.0, 18795.0, 16150.999999999998, 15407.0, 17136.0, 10905.0, 17500.0, 20070.0, 11993.0, 17443.0, 18519.0, 13523.0, 15069.999999999998, 10376.0, 14472.999999999998, 14372.0, 12596.000000000002, 11162.000000000002, 17071.999999999996, 16298.0, 9978.999999999998, 13230.999999999998, 11545.000000000004, 12898.0, 6999.0, 8863.0, 20093.999999999996, 19216.999999999996, 7236.999999999999, 0.0, 12187.0, 17314.0, 11635.0, 11499.0, 18734.0, 14242.0, 11874.999999999998, 11148.0, 3762.9999999999995, 7786.999999999999, 9615.000000000002, 19774.000000000007, 14268.999999999998, 15611.0, 8853.000000000002, 8958.0, 19940.0, 11018.0, 4345.0, 23030.0, 22306.000000000004, 11424.999999999998, 5949.0, 11529.999999999998, 12242.000000000004, 19406.0, 5456.0, 6948.0, 3892.0000000000005, 2567.0, 9989.0, 2236.0000000000005, 10138.0, 11634.0, 18602.0, 24986.000000000007, 8291.0, 9441.0, 18540.000000000004, 12006.999999999998, 10613.0, 12069.0, 9242.0, 17920.999999999993, 5638.0, 16216.999999999996, 8601.0, 9228.0, 2972.0, 5421.0, 8610.0, 17754.0, 17922.000000000004, 11091.0, 13581.0, 14333.999999999998, 16822.0, 13773.0, 8672.0, 11686.0, 7049.0, 9540.0, 10574.999999999998, 5035.0, 15483.0, 6668.0, 6379.0], "Seaquest": [601.0, 446.20000000000005, 738.4, 554.6, 521.0, 669.0, 333.0, 674.0, 367.4, 926.6, 523.8000000000001, 303.8, 368.6, 565.8, 471.6, 437.19999999999993, 416.6, 430.0000000000001, 477.4, 756.5999999999998, 476.4, 372.79999999999995, 552.1999999999999, 622.4, 484.0, 437.4, 405.0, 405.79999999999995, 736.6000000000003, 711.3999999999999, 519.0, 686.2, 696.0000000000001, 852.7999999999998, 648.0, 410.19999999999993, 413.79999999999995, 477.4, 386.4, 676.8, 530.6, 542.0, 689.8, 459.79999999999984, 457.79999999999995, 345.4, 342.79999999999995, 641.6, 647.4, 772.0000000000001, 329.4000000000001, 687.2, 694.5999999999999, 795.4, 759.0, 603.6, 531.8000000000001, 894.8, 546.6, 729.0, 544.2, 543.0, 694.0000000000001, 604.0, 493.60000000000014, 544.6, 743.8, 493.80000000000007, 581.2, 504.0000000000001, 403.4, 885.5999999999999, 504.80000000000007, 612.2, 561.4, 516.4000000000001, 531.0, 471.0, 453.6, 634.0, 564.8000000000001, 404.4, 689.5999999999999, 437.19999999999993, 726.4000000000001, 681.9999999999999, 736.8000000000001, 384.0, 435.6, 487.6, 573.9999999999999, 559.2000000000002, 306.6, 571.9999999999999, 574.6, 397.19999999999993, 867.8, 628.4, 404.60000000000014, 501.6], "UpNDown": [3575.8000000000006, 48131.4, 142282.3, 5779.0, 7853.2, 46200.6, 12016.199999999999, 3248.1, 5498.3, 6799.9, 18269.6, 6449.899999999999, 5568.299999999999, 4350.299999999999, 8232.3, 5554.500000000001, 5053.299999999999, 3845.3, 7362.799999999999, 8866.1, 7538.2, 7741.5, 9518.4, 3888.7, 3999.6, 6370.1, 8567.8, 4008.7, 5814.3, 33573.6, 9604.8, 4950.7, 4023.5, 8400.4, 3028.7, 3134.0, 3348.5, 5322.3, 6092.1, 3167.5, 5213.7, 6366.0, 4727.3, 3480.0, 12534.700000000003, 3400.100000000001, 3796.4, 10475.3, 31856.2, 41756.8, 3555.6, 4622.800000000001, 7326.0, 4723.6, 4035.5, 2523.8, 31627.2, 4069.6, 19033.300000000003, 9511.4, 2715.8000000000006, 4126.799999999999, 7743.9, 3413.5, 4603.1, 5110.8, 3800.8999999999996, 4916.2, 6903.7, 17587.000000000007, 3582.7999999999997, 4088.2999999999997, 5579.9, 27235.3, 6273.0, 5254.200000000001, 14829.199999999999, 4203.399999999999, 20955.8, 9694.7, 4927.3, 7305.2, 3395.3, 6293.4, 7230.2, 23378.799999999996, 10348.4, 18793.9, 7750.9, 6227.2, 13970.2, 5085.2, 5169.4, 23910.9, 6184.0, 4386.100000000001, 8775.3, 22623.3, 6689.6, 19189.100000000002]} \ No newline at end of file diff --git a/results/data/SimPLe.json b/results/data/SimPLe.json new file mode 100644 index 0000000..b52927b --- /dev/null +++ b/results/data/SimPLe.json @@ -0,0 +1 @@ +{"Alien": [677.1875, 435.46875, 515.9375, 425.3125, 1030.46875], "Amidar": [93.65625, 102.703125, 63.71875, 31.0, 80.203125], "Assault": [457.890625, 606.1875, 509.0625, 671.078125, 391.78125], "Asterix": [1061.71875, 1108.59375, 920.3125, 1065.625, 1485.15625], "BankHeist": [43.90625, 77.5, 0.0, 31.875, 17.96875], "BattleZone": [4031.2, 4031.2, 5359.375, 3484.375, 3250.0], "Boxing": [12.65625, 20.984375, 0.09375, -4.359375, 9.375], "Breakout": [10.3125, 16.4, 22.765625, 16.4, 16.046875], "ChopperCommand": [962.5, 896.875, 909.375, 848.4375, 1279.6875], "CrazyClimber": [54370.3125, 57220.3125, 51150.0, 87593.75, 62583.6], "DemonAttack": [288.90625, 208.1, 161.25, 177.578125, 204.84375], "Freeway": [0.0, 0.015625, 21.546875, 29.890625, 31.96875], "Frostbite": [262.34375, 248.75, 205.78125, 266.875, 200.625], "Gopher": [447.5, 732.5, 525.3125, 432.8125, 845.625], "Hero": [3061.5625, 3020.859375, 2060.390625, 2935.0, 2205.234375], "Jamesbond": [141.40625, 50.0, 110.9375, 123.4375, 76.5625], "Kangaroo": [65.625, 62.5, 25.0, 40.625, 62.5], "Krull": [3317.546875, 1933.71875, 1739.03125, 1379.625, 2654.28125], "KungFuMaster": [14318.75, 15023.4375, 14134.375, 21054.6875, 9781.25], "MsPacman": [1551.09375, 1221.25, 1199.0625, 1525.0, 1903.4375], "Pong": [21.0, -17.890625, 19.3125, 20.671875, 20.984375], "PrivateEye": [100.0, 0.0, 100.0, -25.140625, 0.0], "Qbert": [4258.984375, 150.0, 580.078125, 798.828125, 656.25], "RoadRunner": [3081.25, 900.0, 5906.25, 11176.5625, 7139.0625], "Seaquest": [711.5625, 455.3125, 565.625, 854.0625, 830.0], "UpNDown": [8614.53125, 1616.09375, 498.59375, 661.25, 5360.9375]} \ No newline at end of file diff --git a/results/results_iris.ipynb b/results/results_iris.ipynb new file mode 100644 index 0000000..fc63297 --- /dev/null +++ b/results/results_iris.ipynb @@ -0,0 +1,602 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "cellView": "form", + "id": "_F0jWGQZDgM3" + }, + "outputs": [], + "source": [ + "# Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "# you may not use this file except in compliance with the License.\n", + "# You may obtain a copy of the License at\n", + "#\n", + "# https://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing, software\n", + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "# See the License for the specific language governing permissions and\n", + "# limitations under the License." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Credits to https://github.com/google-research/rliable" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "id": "CrcaOCgbJLv6" + }, + "outputs": [], + "source": [ + "import json\n", + "import math\n", + "from pathlib import Path\n", + "\n", + "import matplotlib.pyplot as plt\n", + "import matplotlib.patches as mpatches\n", + "from matplotlib.ticker import MaxNLocator\n", + "from matplotlib import rcParams\n", + "from matplotlib import rc\n", + "import numpy as np\n", + "import pandas as pd\n", + "import seaborn as sns\n", + "\n", + "from rliable import library as rly\n", + "from rliable import metrics\n", + "from rliable import plot_utils\n", + "\n", + "rcParams['legend.loc'] = 'best'\n", + "rcParams['pdf.fonttype'] = 42\n", + "rcParams['ps.fonttype'] = 42\n", + "rc('text', usetex=False)\n", + "RAND_STATE = np.random.RandomState(42)\n", + "sns.set_style(\"white\")\n", + "\n", + "Path('figures').mkdir(exist_ok=True, parents=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "cellView": "form", + "id": "S6FnrWyKnoHu" + }, + "outputs": [], + "source": [ + "def score_normalization(res_dict, min_scores, max_scores):\n", + " games = res_dict.keys()\n", + " norm_scores = {}\n", + " for game, scores in res_dict.items():\n", + " norm_scores[game] = (scores - min_scores[game])/(max_scores[game] - min_scores[game])\n", + " return norm_scores\n", + "\n", + "def convert_to_matrix(score_dict):\n", + " keys = sorted(list(score_dict.keys()))\n", + " return np.stack([score_dict[k] for k in keys], axis=1)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "id": "58NWt0zVvT_L" + }, + "outputs": [], + "source": [ + "StratifiedBootstrap = rly.StratifiedBootstrap\n", + "\n", + "IQM = lambda x: metrics.aggregate_iqm(x) # Interquartile Mean\n", + "OG = lambda x: metrics.aggregate_optimality_gap(x, 1.0) # Optimality Gap\n", + "MEAN = lambda x: metrics.aggregate_mean(x)\n", + "MEDIAN = lambda x: metrics.aggregate_median(x)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "cellView": "form", + "id": "ToCkLC2irWK-" + }, + "outputs": [], + "source": [ + "ATARI_100K_GAMES = [\n", + " 'Alien', 'Amidar', 'Assault', 'Asterix', 'BankHeist', 'BattleZone',\n", + " 'Boxing', 'Breakout', 'ChopperCommand', 'CrazyClimber', 'DemonAttack',\n", + " 'Freeway', 'Frostbite', 'Gopher', 'Hero', 'Jamesbond', 'Kangaroo',\n", + " 'Krull', 'KungFuMaster', 'MsPacman', 'Pong', 'PrivateEye', 'Qbert',\n", + " 'RoadRunner', 'Seaquest', 'UpNDown'\n", + "]" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "cellView": "form", + "id": "PqvaxPEIpACt" + }, + "outputs": [], + "source": [ + "def read_curl_scores():\n", + " df = pd.read_json('data/CURL_10_evals.json', orient='records')\n", + " score_dict = {'Max': {}, 'Final': {}}\n", + " for game in ATARI_100K_GAMES:\n", + " game_df = df[df['game'] == game]\n", + " score_dict['Final'][game] = game_df['HNS'].values\n", + " score_dict['Max'][game] = game_df['Max HNS'].values\n", + " score_matrices = {}\n", + " for key, val in score_dict.items():\n", + " score_matrices[key] = convert_to_matrix(val) \n", + " return score_dict['Final'], score_matrices['Final']\n", + " \n", + "def load_json_scores(algorithm_name, normalize=True):\n", + " path = Path('data') / f'{algorithm_name}.json'\n", + " with path.open('r') as f:\n", + " raw_scores = json.load(f)\n", + " raw_scores = {game: np.array(val) for game, val in raw_scores.items()}\n", + " if normalize:\n", + " hn_scores = score_normalization(raw_scores, RANDOM_SCORES, HUMAN_SCORES)\n", + " hn_score_matrix = convert_to_matrix(hn_scores)\n", + " else:\n", + " hn_scores, hn_score_matrix = None, None\n", + " return hn_scores, hn_score_matrix, raw_scores\n", + "\n", + "def save_fig(fig, name):\n", + " fig.savefig(f'figures/{name}.pdf', format='pdf', bbox_inches='tight')" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "_, _, raw_scores_random = load_json_scores('RANDOM', normalize=False)\n", + "_, _, raw_scores_human = load_json_scores('HUMAN', normalize=False)\n", + "RANDOM_SCORES = {k: v[0] for k, v in raw_scores_random.items()}\n", + "HUMAN_SCORES = {k: v[0] for k, v in raw_scores_human.items()}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "3ls4nMkmBWb8", + "outputId": "686d429e-3df3-49b5-d176-da980b33b85b" + }, + "outputs": [], + "source": [ + "score_dict_muzero, score_muzero, raw_scores_muzero = load_json_scores('MuZero')\n", + "score_dict_efficientzero, score_efficientzero, raw_scores_efficientzero = load_json_scores('EfficientZero')\n", + "score_dict_simple, score_simple, _ = load_json_scores('SimPLe')\n", + "score_dict_curl, score_curl = read_curl_scores()\n", + "score_dict_drq_eps, score_drq_eps, _ = load_json_scores('DrQ(eps)')\n", + "score_dict_spr, score_spr, _ = load_json_scores('SPR')\n", + "score_dict_iris, score_iris, _ = load_json_scores('IRIS')\n", + "\n", + "score_data_dict_games = {\n", + " 'MuZero': score_dict_muzero,\n", + " 'EfficientZero': score_dict_efficientzero,\n", + " 'SimPLe': score_dict_simple,\n", + " 'CURL': score_dict_curl,\n", + " 'DrQ': score_dict_drq_eps, \n", + " 'SPR': score_dict_spr,\n", + " 'IRIS (ours)': score_dict_iris\n", + "}\n", + "\n", + "all_score_dict = {\n", + " 'MuZero': score_muzero, \n", + " 'EfficientZero': score_efficientzero,\n", + " 'SimPLe': score_simple,\n", + " 'CURL': score_curl,\n", + " 'DrQ': score_drq_eps, \n", + " 'SPR': score_spr,\n", + " 'IRIS (ours)': score_iris\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "id": "43m4aASsQ-nU" + }, + "outputs": [], + "source": [ + "colors = sns.color_palette('colorblind')\n", + "xlabels = ['SimPLe', 'CURL', 'DrQ', 'SPR', 'IRIS (ours)']\n", + "color_idxs = [7, 4, 2, 1, 0]\n", + "ATARI_100K_COLOR_DICT = dict(zip(xlabels, [colors[idx] for idx in color_idxs]))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "KgtEQsA08sIG" + }, + "source": [ + "# Mean, Median, IQM and Optimality Gap" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "3RhatO5S8tWt", + "outputId": "84aa63d0-8fc4-46aa-ddd2-9d5951461fd7" + }, + "outputs": [], + "source": [ + "aggregate_func = lambda x: np.array([MEAN(x), MEDIAN(x), IQM(x), OG(x)])\n", + "aggregate_scores, aggregate_interval_estimates = rly.get_interval_estimates(all_score_dict, aggregate_func, reps=50000)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "MuZero : 1 runs\n", + "EfficientZero : 1 runs\n", + "SimPLe : 5 runs\n", + "CURL : 100 runs\n", + "DrQ : 100 runs\n", + "SPR : 100 runs\n", + "IRIS (ours) : 3 runs\n" + ] + } + ], + "source": [ + "for algo in aggregate_scores.keys():\n", + " n_runs, n_games = all_score_dict[algo].shape\n", + " assert n_games == len(ATARI_100K_GAMES)\n", + " print(f\"{algo.ljust(14)}: {n_runs:3d} runs\")" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAABEIAAAEdCAYAAAAIIsL8AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAABLs0lEQVR4nO3dd3RU1drH8V8KSSC00CEBDBHCBYKEEor0orRI70URKSKCgAgoVgSUiyKGorRQovTc0EWaKFKvghRphpaABiSBmED6vH/4Zi7jJKRN6nw/a7GWs88+Z++zZ84zk8ez97ExGAwGAQAAAAAAWAHb3O4AAAAAAABATiERAgAAAAAArAaJEAAAAAAAYDVIhAAAAAAAAKtBIgQAAAAAAFgNEiEAAAAAAMBqkAgBAACwkKlTp6pt27YmZW3bttXUqVNzqUcAkL8dO3ZMnp6eOnbsmLHMz89Pnp6eudgr5HckQoB8LPmLwdPTU0uXLk2xzooVK4x1Hv0CAYCCitgIIKclx52NGzdmav+QkBD5+fnp/PnzFu5Z/pCUlCQ/Pz/t3bs31/qQnFzx9PTU6dOnU6wzevRoYx3kbyRCgALA0dFRW7duTXFbUFCQHB0dc7hHAJD78kps/OabbzRjxowcaQtA/hQSEqIFCxZYdSJkwYIFKSZCGjVqpNOnT6tRo0Y50hdHR0dt2bLFrDw8PFw//PADv6sLCBIhQAHQpk0bXbp0yezL88KFC7p48aLZbdoAYA3ySmx0cHBQoUKFcqQtAPinBw8e5HYXUpWYmKi4uLjH1rG1tZWjo6NsbXPmT9c2bdpox44dio+PNynfvn277Ozs1KxZsxzpB7IXiRCgAGjatKnKli1rlr3esmWLypUrp6ZNm5rtEx4ervfff1+tWrVSnTp11KZNG/373/9WbGysSb3AwEANHz5czZs3V506ddS6dWvNmDFDUVFRJvUevSU0ICBA7du3V506ddStWzcdPXrU8icNAGnIztgoSStXrlS7du3k5eWl7t2768CBAyn2459rhMTFxcnPz099+vRR48aN5eXlpc6dO2vlypUyGAwm+ybfqn3hwgV9+OGHatq0qZ566im99NJLunnzZmaGBUAOSe9vo8DAQA0bNkySNG3aNOPUCz8/P2Od9Mamtm3basCAATp16pQGDRqkp556Sh988IFxe0pxK6W1jf7ZfrKU6mb0t+KGDRu0fPlytW/fXl5eXtq+fbtq164tSfrPf/5jPP8hQ4aY7JeeaYwXLlzQ2LFjjbHV19c3w9OVunXrpoiICP3www8m5UFBQWrXrp2KFi2a6baJ/3mHfW53AEDW2dnZydfXV9u2bdPkyZNlZ2enxMREbdu2Tb6+vmYZ9IiICPXt21dRUVHq16+fKlWqpHPnzsnf318XLlzQsmXLZGNjI0kKCAiQu7u7XnjhBRUrVkznzp3T+vXrdfHiRQUEBJj1Ze3atXrw4IH69u0rBwcHrVq1SmPGjNGBAwdUokSJHBkPAJCyNzYuXrxYn332mRo2bKjnn39eYWFhmjRpklxdXdPsV1RUlL7++mt16tRJzz33nCTpxx9/1OzZs3X//n2NHz/ebJ8333xTJUuW1CuvvKI7d+5o5cqVmjx5sr7++msLjBSA7JTWb6NGjRpp5MiRWrJkifr166cGDRpIknEdiozEJkkKCwvTqFGj9Nxzz6lbt27GP9yzErceJ6O/FVevXq34+Hj17dtXTk5OqlWrlj7++GNNmTJFDRs2VN++fSVJZcqUyVA/fv75Z7344ouqXLmyhg8fLmdnZx04cEDTp09XeHi4Ro0ala7jeHh4qHbt2tqyZYsx6fPbb7/p3LlzGjdunLZv357pton/eQeJEKCA6Natm1asWKEff/xRLVu21OHDh3Xnzh11797dbMGn+fPn6/79+woKCjL58qtRo4ZmzJihH374QS1btpQkffXVVypcuLDJ/t7e3po6dapOnjwpb29vk213797Vzp075ezsLElq3Lixunfvru3bt2vQoEHZceoAkKrsiI0RERFatGiRvL29tWrVKtnb//1zysfHRyNHjkzzj4oSJUro4MGDcnBwMJYNGTJE06ZN08qVK/Xyyy+bbJOkihUrauHChcbXLi4umj17tn777Tc9+eSTmR4fANkvrd9GlStXVtOmTbVkyRLVq1dP3bp1M9k/I7/bJOnmzZuaO3eufH19jWVZjVuPk5nfirt371bx4sWNZU8++aSmTJmiypUrm51/ehgMBr399tuqXr26vv76a+N0xEGDBmncuHFatGiR+vfvn+7/Kde9e3fNnTtXkZGRKl68uLZs2aLSpUurefPmZomQjLRN/M87mBoDFBA1a9aUp6en8RbwoKAgY9mjDAaDdu3apaefflqFCxdWeHi48d/TTz8tSTpy5IixfvIXW1JSkv766y+Fh4erYcOGkqQzZ86Y9aNbt27GL3pJ+te//qWiRYsqJCTEsicMAOmQHbHx8OHDiouL0+DBg41/TEhSq1at5OHhkWaf7OzsjD90ExISdP/+fYWHh6tJkyZ68OCBrly5YrbPgAEDTF77+PhIkm7cuJHeoQCQS7Ly2yijv9skqWTJkurSpYtJWVbj1uNk9Ldi165dTZIglnDx4kX99ttv8vX1NfYh+V+rVq0UExOjkydPpvt4Xbp0UWJior755hslJSVp27Zt6tKli8nYZaZt4n/ewR0hQAHSrVs3+fn5KSwsTPv27dO4cePM6oSHh+vevXvatWuXdu3aleJxwsPDjf99+vRpzZs3Tz/99JPZPNTIyEizfStVqmRWVqJECd2/fz+jpwMAFmHp2Jg8N9vd3d2sjru7e7qe+rBlyxb5+/vr0qVLSkxMNNn2119/mdX/Z2xN/iOC2ArkfVn5bZTR322S5Orqajb1zxJxKzUZ/a1YpUqVTLeVmuQEwsyZMzVz5swU69y9ezfdx0u++2PLli2qXLmyfv/991TvVMlo28T/vIFECFCAdO3aVZ988oneeOMNxcXFqWvXrmZ1khdiat++fapTVcqWLStJCg0N1dChQ+Xq6qrJkyfL1dVVTk5OSkxM1EsvvWS2qJP0d6Y7JSnVBYCcYOnYmFU7d+7UG2+8oZYtW2rw4MEqW7asChUqpHPnzmnu3LlKSkoy24fYCuRfWbl+MxObsvPxrv/8wz0zvxWdnJyyrX9jx441rrHyTxm986Vbt26aOHGiFi9erCeffFJ16tTJctvE/7yDRAhQgJQvX15NmzbVoUOH1Lx5c5UrV86sTqlSpVS0aFHFxsam+fivffv26eHDh/ryyy/l5uZmLE/ptj0AyKssHRuT59JfvXrV+KSDZFevXk2zPzt27JCbm5u+/PJLk/9ry23OgPV6dLHTR2UkNj1ORuNWanes/HM6j6V+K6Z2/umVfJeJo6OjxR5vm/yEmGPHjmnSpEkWaZv4n3ewRghQwIwdO9b4LyW2trbq3LmzDh06pOPHj5ttj42NNT7uLDkD/c+M89KlSy3cawDIXpaMjc2aNZODg4MCAgKUkJBgrHPw4EEFBwen2ZeUYmtMTIzWrFmToXMCUHAUKVJEkvlUkozEpsfJaNyqWrWq2eNqf/nlF506dcqkzFK/Fe3s7OTo6Jjp6R61atWSu7u7Vq9ebTZVSMrYtJhkjo6Oeu+99zR27Fh1797dIm0T//MO7ggBChhvb2+z1bn/aeLEiTp+/LhefPFFde/eXbVq1VJsbKyuXr2q3bt36/PPP1fjxo3VokULOTo6auTIkerXr59sbW114MAB5iQCyHcsGRtdXFw0atQo+fn56fnnn1fHjh0VFhamr7/+WjVq1FB0dPRj22nXrp12796tkSNHqkOHDrp//74CAwNNFlMEYF2efPJJFS5cWGvXrlWRIkXk7Oys6tWrq0aNGumOTY+T0bg1YMAATZs2TaNGjVLr1q118+ZNbdy4UTVq1DBJvFjyt6KXl5eOHDmi5cuXq0KFCipVqpSaNm2arn1tbW01e/Zsvfjii+rcubN69+6tKlWqKDw8XOfPn9e+fft09uzZDPcppamUWWmb+J93kAgBrJCLi4s2btyoL774Qnv37lVQUJCcnZ3l5uamwYMHG5+mULVqVX355Zf69NNP9dlnn8nJyUmtW7fWJ598ku4vJgDIL9IbG6W/7zApUqSIAgICNGfOHHl4eOiTTz7R7t27U/y/to/q1q2bIiMjFRAQoJkzZ6ps2bLq2bOn6tevr2HDhmX3aQLIg5ydnfXxxx/Lz89PH3zwgeLj4zV27FjVqFEjQ7HpcTISt3r06KFbt25p3bp1Onz4sDw9PTV//nwFBQWZ1LXkb8X33ntP77//vvz8/PTw4UP5+Phk6Bje3t7atGmTFi9erKCgIN27d08uLi7y8PDQtGnTMtSXjEpv28T/vMPGwCorAAAAAGCVpk6dquPHj2v//v253RUgx7BGCAAAAAAAsBokQgAAAAAAgNUgEQIAAAAAAKwGa4QAAAAAAACrwR0hAAAAAADAapAIAf5fTExMbnehwGFMLY8xtV6893kb70/exXuTP/G+WRbjaVmMp2XlxniSCAH+X2JiYm53ocBhTC2PMbVevPd5G+9P3sV7kz/xvlkW42lZjKdl5cZ4kggBAAAAAABWg0QIAAAAAACwGiRCAAAAAACA1SARAgAAAAAArAaJEAAAAAAAYDVIhAAAAAAAAKthn9sdAID8LOJBnGISknKsvaQkG0VG5u6z653sbeVSxCFX+wBkVHZfq3nh2kTK8sp7Q+wECq6c/j1Y0KQVp7MjfpIIAYAsiElI0qCvfs6x9hITE2VnZ5dj7aXkq0H1c7V9IDOy+1rNC9cmUpZX3htiJ1Bw5fTvwYImrTidHfGTqTEAAAAAAMBqkAgBgHzm9yM79PuRHbndDQD5HLEEAKwHMd8UiRAAAAAAAGA1CkQi5PPPP9ezzz6rxMTE3O7KY0VHR6tx48bauHFjbncFAAAAAACrlGYi5NixY/L09DT54z00NFSenp4m/+rXr68BAwZoxw7z222S68+bN8+k/P79+5o3b558fX3l7e2thg0bqlOnTpo6dapOnTqVrhP4888/5e/vr1GjRuWJhbAex9nZWUOGDJGfn59iYnJ/9XIAAAAAAKxNlp4a065dOz377LMyGAy6ffu2Nm7cqIkTJyomJka9evV67L5//fWXevXqpT///FPPPfecBg4cqNjYWF27dk0//PCDXF1dVa9evTT7EBAQIHt7e3Xt2jUrp5Jj+vXrp4ULFyooKEj9+/fP7e4AAAAAAGBVspQIqV69urp162Z83bNnT3Xo0EErVqxIMxGyYcMGhYSEaOHChWrfvr3JNoPBoPDw8DTbT0xMVGBgoJ555hk5OOTOc9kTExOVmJiY7vbLli1rnB5DIgQAAAAAgJxl0TVCypQpo2rVqun69etp1k2u4+PjY7bNxsZGpUuXTvMYp06dUlhYmJo3b57i9qCgIPXo0UN169aVj4+PXn31VQUHB5vUCQwMlKenp0JDQ8329/T0lJ+fn/F18jShDRs2aPny5Wrfvr28vLx08uRJSdLu3bvVt29fNWzYUPXq1VOHDh309ttvmx23efPmOnv2rG7dupXmOQIAAAAAAMvJ0h0h/5SQkKCwsDCVLFkyzbpubm6SpM2bN2vYsGGZau+nn36SJNWtW9ds2/LlyzVnzhzVrVtXEydO1P379xUQEKD+/ftr06ZNqlq1aqbalKTVq1crPj5effv2lZOTk8qWLasjR45o/PjxatSokV577TXZ29srJCREBw4cMNv/qaeekiT997//1XPPPZfpfgAAAAAAgIzJUiIkNjbWOIUlLCxMy5Yt0507d9KV2OjTp49Wr16tjz76SGvWrFGjRo3k7e2tli1bqlKlSulqPzg4WHZ2dmb1IyIiNH/+fNWuXVtfffWVcdpK+/bt1bt3b33yySf6/PPPM3i2/3P37l3t3r1bxYsXN5atW7dOzs7O8vf3l739/4Z18uTJZvtXqVLF2H8AAAAAAJBzspQI8ff3l7+/v/G1nZ2dBg0apIkTJ6a5r4uLizZv3qylS5fq22+/VVBQkIKCgiT9vQjrjBkz0pweEx4ermLFisnGxsak/PDhw4qNjdXzzz9vsnZH7dq11axZMx08eFAJCQkmCYuM6Nq1q0kSRJKKFy+uhw8f6ocfflDr1q3N+vSo5DtmIiIiMtU+AAAAAADInCwlQnx9fdWzZ0/Fx8fr3LlzWrJkif766y8VKlQoXfuXL19e06dP1/Tp0xUWFqaffvpJ69ev1759+xQdHa1Vq1Zlql83b96UJHl4eJht8/Dw0KFDh3T37l2VL18+U8dPvqPjUQMHDtTu3bs1evRolS5dWk2aNFGbNm3UsWNHs/FISkrKVLsAAAAAACBrspQIcXV1VbNmzSRJrVq1kpubmyZPnqx69epp0KBBGTpW+fLl1blzZ3Xq1EmDBw/W0aNH9fvvv6tixYqp7lOqVCn99ddfMhgMj70D43FS2y8xMTHVfZycnFLsy3/+8x8dPXpUP/zwgw4fPqwdO3Zo2bJl+vrrr+Xs7GysGxkZKenvu2IAAAAAAEDOsehTY5577jnVqVNHfn5+io6OztQxbGxs5OXlJenvdUcex8PDQ4mJicY7QJIlL8Sa0hocwcHBKly4sHHaTfIUl+TkRLKQkJAM993e3l7NmzfXtGnTtG3bNr377ru6cOGCtm/fblLvxo0bxv4DAAAAAICcY9FEiCSNGjVKERERWrt27WPrnTp1SlFRUWblcXFxOnz4sOzs7NJ8skuDBg0kSWfOnDEpb9q0qRwdHbVmzRrFxcUZyy9cuKDDhw+rZcuWxvVBnnjiCUnS0aNHTY6R0Wk5Ka33UatWLUnS/fv3Tcp/+eUXk/4DAAAAAICcYdHH50p/P5mlWrVq8vf315AhQ+To6Jhiva1bt+o///mP2rZtKy8vLxUrVkxhYWHasWOHfvvtN40YMSLNqSP16tVTuXLldOjQIXXq1MlY7uLiovHjx2vOnDkaPHiwunTpYnx8btGiRU0Wc/Xw8JCPj48+++wz3bt3T+XLl9ePP/6o27dvZ+i8p0+frrt376pp06aqVKmSIiIitG7dOjk5OalDhw4mdQ8dOqTatWvL1dU1Q20AAAAAAICssXgixNbWViNGjNC0adO0YcMGDRkyJMV6/fv3V9GiRXXkyBEdOXJE9+/fl7Ozszw9PTVnzhx169Ytzbbs7OzUs2dPrV27Vu+++67JE2KGDx+u0qVLa+XKlZo7d66cnJzUuHFjTZgwwXgXSLI5c+bo/fff1+rVq1WoUCG1adNGM2fOVJMmTdJ93s8995w2bdqkjRs36t69eypZsqS8vb318ssvy93d3Vjvzz//1PHjx/X222+n+9gAAAAAAMAybAwGgyG3O5EVd+7c0TPPPKN33nlHPXr0yO3upGnBggVav369vv32WxUuXDi3u4NHREdHmyxqi6yzhjH9PTJGg776OcfaS0xM1O3j30iSKjbtkmPtPuqrQfVVsbj5otHIXtZwPWWn7L5WExMTZWdnl23Hzw6/H9khKfdiSU7JK+8NsTNjiHmWxXha1j/HM6d/D2ZGXo75acXp7IifFl8jJKeVLVtWw4YN0xdffPHYJ73kBdHR0VqzZo1effVVkiAAAAAAAOQCi0+NyQ3jxo3TuHHjcrsbaXJ2dtaxY8dyuxsAAAAAAFitApEIAQBrkhdvaQSQ/xBLAMB6EPNN5fs1QgBLYe6k5VnDmEY8iFNMQlKOtZeUlCRb29yd1ehkbyuXIg5pV4RFWcP1lJ2y+1rNC9cmUpZX3htiZ8YQ8yyL8bSsf45nTv8eLGjSitPZET+5IwQAsiCnf9T+/cXLYntARmX3tcq1mXfx3gDIbiQ5syY34nTup8cBAAAAAAByCIkQAAAAAABgNUiEAAAAAAAAq0EiBAAAAAAAWA0SIQAAAAAAwGqQCAEAAAAAAFaDRAgAAAAAALAaJEIAAAAAAIDVIBECAAAAAACsBokQAAAAAABgNUiEAAAAAAAAq0EiBAAAAAAAWA0SIQAAAAAAwGrY53YHAPxPYkyEDImxGdrHxs5Rdk4u2dQjAEBBk5nvmpzA9xmQN2U0ZnAtIz8gEQLkIYbEWN3ZNSRD+5TttCabegMAKIgy812TE/g+A/KmjMYMrmXkB0yNAQAAAAAAVoNECJDHrT9wS+sP3MrtbgAAkCq+qwDrwjWP/I5ECAAAAAAAsBokQgAAAAAAgNVgsdT/d/XqVS1evFgnT57UH3/8IWdnZ1WqVEkNGjTQSy+9pPLly+vYsWMaOnSocR8bGxsVL15cderU0YgRI9S0aVPjtiFDhuj48ePG14UKFVLFihXVvn17vfLKKypatGiOnh8AAAAAACARIkk6ffq0hgwZouLFi6tnz55yc3NTeHi4Ll++rC1btqh9+/YqX768sX7v3r3l4+OjpKQkXb9+XevWrdPw4cO1bNkyNWvWzFivRIkSeuuttyRJUVFROnLkiFasWKGzZ89qzRpWUwYAAAAAIKeRCJG0cOFC2draatOmTSYJD0mKiYlRQkKCSVm9evXUrVs34+tnnnlGPXr00PLly00SIU5OTib1Bg0apPHjx+ubb77R6dOnVbdu3Ww6IwAAAAAAkBLWCJF0/fp1ValSxSwJIv2dzEhrGkutWrVUsmRJhYaGptmWj4+PJOnGjRuZ6ywAAAAAAMg0EiGS3NzcdOXKFf3888+Z2j8iIkKRkZFycXFJs25ysqRkyZKZagsAAAAAAGQeU2MkjRo1SkeOHNHAgQNVs2ZNNWjQQA0aNFCLFi1UrFgxs/rR0dEKDw+XwWDQ9evX9emnnyopKUmdOnUyqZeUlKTw8HDjPocPH9ZXX32lcuXKqVGjRjlybgAAAAAA4H9IhEhq1KiR1q1bp2XLlunQoUM6f/68AgICVKhQIb344osaN26c7O3/N1SzZ8/W7Nmzja8LFy6skSNHmjxRRpLu3Llj8iQZSfLy8tLMmTPl6OiYvScFAAAAAADMkAj5f15eXpo/f76SkpJ07do1HT16VP7+/vryyy/l4OCgsWPHGusOGzZMLVu2lI2NjYoVK6Ynn3xSTk5OZsd0cXHRp59+KkkKCwvTypUrdffuXRUuXDjHzgsAAAAAAPwPiZB/sLW1VbVq1VStWjV16tRJHTp0UFBQkEkixMPDw+TpMKlxcHAwqdemTRt16dJFkyZN0oYNG2RjY5Mt5wAAAAAAAFLGYqmP4eLioipVqigsLMwixytZsqRefvllnT59Wjt37rTIMQEAAAAAQPqRCJF0+PBhJSYmmpWHhIQoODhYHh4eFmurT58+Kl26tL744gsZDAaLHRcAAAAAAKSNqTGSZs2apcjISLVr107Vq1eXvb29rl69qqCgIMXHx+u1116zWFuOjo56/vnn9emnn2rv3r3q0KGDxY4NAAAAAAAej0SIpClTpmjPnj06ceKEtm/frgcPHqhUqVJq1KiRhg0bJm9vb4u2N3DgQC1ZskSLFy8mEQIAAAAAQA4iESKpRYsWatGiRZr1GjdurIsXL6brmGvWrEl1W7FixfTTTz+lu38AAAAAAMAyWCMEAAAAAABYDRIhAAAAAADAajA1Bsjj+rWplNtdAADgsfiuAqwL1zzyOxIhQB5iY+eosp1SX18mtX0AAEivzHzX5AS+z4C8KaMxg2sZ+QGJECAPsXNyye0uAAAKOL5rAGQEMQMFEWuEAAAAAAAAq0EiBAAAAAAAWA0SIQAAAAAAwGqQCAEAAAAAAFaDRAgAAAAAALAaJEIAAAAAAIDVIBECAAAAAACsBokQAAAAAABgNUiEAAAAAAAAq0EiBAAAAAAAWA0SIQAAAAAAwGqQCAEAAAAAAFaDRAgAAAAAALAa9rndAcCSImIfKCYxwazcyc5eLo5FcqFHAIC8LLXvjYwyGJIU+SDSAj0yxfcXACn9sYqYAaQPiRAUKDGJCRp88Cuz8oBWg3KhNwCAvC61742MSkxMlJ2dnQV6ZIrvLwBS+mMVMQNIH6bGAAAAAAAAq0EiBAXS73uO6Pc9R3K7GwAA8J0EwOKIK0DWkAgBAAAAAABWgzVCJB07dkxDhw41vra3t1fRokX1xBNPqGHDhurXr5+qVKmS4ePu3btXa9eu1dmzZxUdHa1y5crp6aef1siRI1W5cmVLngIAAAAAAEgHEiGP6N27t3x8fGQwGHT//n2dP39eX3/9tVatWqW33npLAwYMSNdxDAaD3n77bW3cuFG1atXS8OHD5eLioitXrmjTpk3avn27FixYoKeffjqbzwgAAAAAADyKRMgj6tWrp27dupmUvf766xo1apTef/99Va1aVc2aNUtxX4PBoNjYWDk5OWnVqlXauHGj+vXrp/fee0+2tv+bgTR06FANHjxY48eP1/bt21WhQoVsPScAAAAAAPA/rBGShjJlyuizzz6Tra2tFi5cKEkKDQ2Vp6en5s2bp8DAQHXp0kVeXl7auXOnYmNjtXjxYrm5uWn69OkmSRBJqlixot5//3399ddfWrZsWW6cEgAAAAAAVos7QtKhcuXKatSokU6cOKGoqChj+f79+xUREaEBAwaoVKlScnd3108//aR79+5p4MCBcnBwSPF4zZs3V8WKFbVv3z5Nnz49p04DAAAAAACrRyIknapXr66jR48qNDRURYsWlSRdvXpVu3btMln4dPXq1ZKkWrVqPfZ4NWvW1IEDBxQdHS1nZ+fs6zgAAAAAADBiakw6JScroqOjjWUtW7Y0e/pL8h0jycmS1CRvf/QOEwAAAAAAkL1IhKRTcgLk0QRHSo/UTW+CIyoqSjY2NnJxcbFgLwEAAAAAwOOQCEmnS5cuyc7OTm5ubsYyR0dHs3pPPvmkJOn8+fOPPd6FCxdUsWLFVNcRAQAAAAAAlkciJB1CQkL03//+V/Xr109zPY8GDRqoZMmS2rp1q+Lj41Osc/jwYf3+++/q1KlTdnQXAAAAAACkgkRIGu7evasJEyYoKSlJY8aMSbO+o6OjXn75ZYWEhGjmzJlKSkoy2R4WFqZ33nlHJUqU0ODBg7Or2wAAAAAAIAU8NeYRp06dkoODgwwGgyIjI/Xrr79q9+7dio+P1zvvvKNmzZql6zjPP/+8Ll++rLVr1+r06dPq3LmzSpYsqeDgYG3evFnx8fGaP3++KlWqlM1nBAAAAAAAHkUi5BGbNm3Spk2bZG9vr6JFi6pq1aoaOHCg+vbtq6pVq6b7ODY2Npo5c6Zat26tdevWaenSpbp//74MBoOKFSumbdu2maw1AgAAAAAAcgaJEEmNGzfWxYsX013fzc0tXfU7dOigDh06GF8vWLBAfn5+Wrp0qd5///1M9RUAAAAAAGQeiZAcNHbsWEVFRcnf31+lSpXS+PHjc7tLAAAAAABYFRIhOWzq1KmaOnVqbncDAAAAAACrxFNjAAAAAACA1bAxGAyG3O4EYCkRsQ8Uk5hgVu5kZy8XxyKP3Tc6OlrOzs7Z1TWrxJhaHmNqvXjvs0dq3xsZZUhKko2t5f//Unq+v/B4XDv5E++bqfTGqtRiBuNpWYynZeXGeDI1BgUKPxYBABlhqe+N6OhoORfhRzGA7MFvXMCymBoDAAAAAACsBokQAAAAAABgNUiEAAAAAAAAq0EiBAAAAAAAWA0SIQAAAAAAwGqQCAEAAAAAAFaDRAgAAAAAALAaJEIAAAAAAIDVIBECAAAAAACsBokQAAAAAABgNUiEAAAAAAAAq0EiBAAAAAAAWA0SIQAAAAAAwGrY53YHkHkJD+OVlJBkVm5rbyv7woVyoUcAkP+kFksthZgMoKDityiA/IpESD6WlJCkq+vOmpW796+TC70BgPwptVhqKcRkAAUVv0UB5FdMjQEAAAAAAFaDREgBEHh4mwIPb8vtbgBAvkUcBYDMI4YCyG9IhAAAAAAAAKtBIgQAAAAAAFiNApsIiYyM1IIFC9S9e3fVr19fXl5e6tChg6ZPn65ff/3VWM/Pz0+enp66fv16iseZN2+ePD09FRoaaiybOnWqPD09jf9q166tNm3a6O2339adO3fMjpFcPyEhwfInCgAAAAAA0q1APjXm8uXLGjFihO7cuaOOHTuqd+/ecnR01LVr1/TNN99o06ZN+u6771ShQoUstfPxxx/LxsZGMTExOnnypDZt2qTjx49r27ZtcnBwsNDZAAAAAAAASylwiZDo6Gi9/PLLevjwoTZs2KDatWubbJ8wYYKWL18ug8GQ5ba6du0qe/u/h7Bfv34qVaqUli9frn379qlTp05ZPj4AAAAAALCsAjc1ZsOGDQoJCdEbb7xhlgSRJHt7e40aNUoVK1a0eNs+Pj6SpBs3blj82AAAAAAAIOsKXCJkz549cnBwkK+vb463nbyOSMmSJXO8bQAAAAAAkLYCNzUmODhY7u7uObJGx71792Rra6uYmBidOnVKCxYsUOHChdW2bdtsbxsAAAAAAGRcgUuEREVFycPDI0faevrpp01eu7u764MPPlDZsmVzpH0AAAAAAJAxBS4RUrRoUUVHR1v8uDY2NmZly5cvl62trSIiIrRu3TpdvHhRTk5OFm8bAAAAAABYRoFLhHh4eOjMmTOKi4tL1/QYR0dHSVJMTEyK25PLU0pwNGnSxPjUmA4dOqhXr14aP368du7cqcKFC2f2FAAAAAAAQDYpcIultm/fXnFxcdq+fXu66ru6ukr6e22RlAQHB6tIkSJpLoDq4OCgiRMn6tatW1qzZk2G+gwAAAAAAHJGgUuE9O3bV66urpozZ44uXLhgtj0hIUFLlizRH3/8IUlq2rSpHB0dtXbtWsXFxZnUvXjxoo4ePaqWLVvKzs4uzbbbtGkjT09P+fv76+HDh5Y5IQAAAAAAYDEFbmpM0aJFtXjxYo0YMUK9e/dWx44d5e3tLQcHB924cUO7d+9WSEiIunXrJkkqVaqUJk6cqNmzZ6tXr17q1KmTSpUqpStXrmjDhg1ydnbWpEmT0t3+yJEjNWnSJK1fv14vvPCCybYvvvhCtrbmuadBgwapRIkSWTpvAAAAAACQtgKXCJEkT09Pbdu2TatWrdK+ffu0b98+xcfHq2LFimratKk+//xzlS9f3lj/hRdekJubm1avXq0VK1bo4cOHKlOmjDp16qRXXnlFbm5u6W67U6dO+uyzz7R8+XINHDjQZJ0SPz+/FPfp0qULiRAAAAAAAHJAgUyESFKJEiU0btw4jRs3Ll3127dvr/bt26er7kcffaSPPvooxW12dnbau3dvuusDAAAAAICcU+DWCAEAAAAAAEgNiRAAAAAAAGA1CuzUGGvSs5lvbncBAPI14igAZB4xFEB+QyIkH7O1t5V7/zoplgMA0ie1WGrJ4wNAQcRvUQD5FYmQfMy+cKHc7gIA5HvEUgDIHOIngPyKdC0AAAAAALAaJEIAAAAAAIDVIBECAAAAAACsBokQAAAAAABgNUiEAAAAAAAAq0EiBAAAAAAAWA0SIQAAAAAAwGqQCAEAAAAAAFaDRAgAAAAAALAaJEIAAAAAAIDVIBECAAAAAACsBokQAAAAAABgNUiEAAAAAAAAq2Gf2x1AzoiJiVFiYmKutW9nZycnJ6dcax8AkuV2PMyspKQkRUdHp1mPeAsgO+R07CTmAchOJEKsRGJiovbs2ZNr7Xfo0CHX2gaAR+V2PMysxMRE2dnZpVmPeAsgO+R07CTmAchOTI0BAAAAAABWg0QI0rR//37t378/t7sBAAUKsRWANSHmAchLSIQAAAAAAACrkS8TIYGBgfL09FRoaGhudwUAAAAAAOQjeW6x1KtXr2rx4sU6efKk/vjjDzk7O6tSpUpq0KCBXnrpJZUvXz7b2vbz89OCBQuMr+3s7FSqVCk1adJE48aNU5UqVYzbPD095evrq7lz52ZbfwAAAAAAgGXlqUTI6dOnNWTIEBUvXlw9e/aUm5ubwsPDdfnyZW3ZskXt27dX+fLl1a1bN3Xp0kUODg7Z0o8pU6aodOnSiouL05kzZxQYGKgff/xR27ZtU5kyZbKlTQAAAAAAkP3yVCJk4cKFsrW11aZNm8zu/IiJiVFCQoKkv+/USM/jtDKrXbt2qlq1qiSpT58+qlatmmbPnq3AwECNHDky29oFAAAAAADZK0+tEXL9+nVVqVIlxekvTk5OKlq0qKSU1wiZOnWqPD09dfv2bU2aNEmNGjVSo0aN9M477yguLk4xMTH68MMP1axZMz311FMaM2aMwsPD09WvZs2aSZJCQkIyfE4XLlzQ2LFj1bhxY3l5ecnX11cbN27M8HEAAAAAAEDW5ak7Qtzc3HTs2DH9/PPPql+/fqaOMXr0aLm7u2vChAk6ceKE1q9fLwcHB4WGhiopKUmvvPKKLl26pHXr1snR0VHz5s1L85g3btyQJLm4uGSoLz///LNefPFFVa5cWcOHD5ezs7MOHDig6dOnKzw8XKNGjcrUOQIAAAAAgMzJU4mQUaNG6ciRIxo4cKBq1qypBg0aqEGDBmrRooWKFSuWrmM0bNhQb775piRp4MCBunHjhgICAtShQwf5+fkZ60VERGj37t26f/++SpQoYXKM+/fvKzw8XPHx8Tpz5ow+/PBD2draqmPHjuk+F4PBoLffflvVq1fX119/rUKFCkmSBg0apHHjxmnRokXq37+/WdsAAAAAACD75KmpMY0aNdK6dev07LPPKiQkRAEBAZowYYKaNm2qTz/91LhGyOMMGDDA5HX9+vVlMBjUv39/s/LExETdvHnT7Bh9+vRR06ZN1bJlS73yyitKSEjQJ598olq1aqX7XC5evKjffvtNvr6++uuvvxQeHm7816pVK8XExOjkyZPpPh4AAAAAAMi6PHVHiCR5eXlp/vz5SkpK0rVr13T06FH5+/vryy+/lIODg8aOHfvY/V1dXU1eFy9eXJJUsWLFFMvv379vdoyPP/5Y5cqVk52dnUqXLi13d/cML8565coVSdLMmTM1c+bMFOvcvXs3Q8cEAAAAAABZk+cSIclsbW1VrVo1VatWTZ06dVKHDh0UFBSUZiLE1jblm1xSS2QYDAazMm9vb+NTY7Jq7NixatCgQYrbPDw8LNIGAAAAAABInzybCHmUi4uLqlSposuXL+d2V9KtSpUqkiRHR0fjU2cAAAAAAEDuylNrhBw+fFiJiYlm5SEhIQoODs5Xd1DUqlVL7u7uWr16dYqP6WVaDAAAAAAAOS9P3REya9YsRUZGql27dqpevbrs7e119epVBQUFKT4+Xq+99lpud9HE5cuXtWjRIrPySpUqqXv37po9e7ZefPFFde7cWb1791aVKlUUHh6u8+fPa9++fTp79mwu9BoAAAAAAOuVpxIhU6ZM0Z49e3TixAlt375dDx48UKlSpdSoUSMNGzZM3t7eud1FExcuXNCFCxfMyuvXr6/u3bvL29tbmzZt0uLFixUUFKR79+7JxcVFHh4emjZtWi70GAAAAAAA62ZjSGm1UBQ40dHR2rNnT6b23b9/vySpbdu2mW6/Q4cOcnZ2zvT+OSE6OjrP9zG/YUwtjzHNuqzEQ0vKaGxNTExM1xPM8kO8LYi4NvMu3hvLyGrsJOblLq4Dy2I8LSs3xjNPrRECAAAAAACQnUiEAAAAAAAAq8HUGCsRExOT4hN5coqdnZ2cnJxyrf304BY3y2NMLY8xzbrcjoeZlZSUJFvbtP//RX6ItwUR12bexXtjGTkdO4l5lsV1YFmMp2XlxnjmqcVSkX34ggCAv+XXeMiPLgC5KadjJzEPQHZiagwAAAAAALAaJEIAAAAAAIDVIBECAAAAAACsBokQAAAAAABgNUiEAAAAAAAAq0EiBAAAAAAAWA0SIQAAAAAAwGrYGAwGQ253AgAAAAAAICdwRwgAAAAAALAaJEIAAAAAAIDVIBECAAAAAACshn1udwCZ4+npmdtdAIBMuXjxYq61TewEkF8ROwEg41KLndwRAgAAAAAArAZPjQEAAAAAAFaDO0IAAAAAAIDVIBECAAAAAACsBokQAAAAAABgNXhqDAqEhIQEffnll9q8ebPu3LkjV1dXDR48WIMGDZKNjc1j9x0yZIiOHz9uVm5nZ6dff/3VrPzIkSOaP3++zp8/r8KFC6tNmzaaPHmySpUqZbHzyQsyO6bHjh3T0KFDU91etWpVffvtt8bXGR3//Cw6OlrLly/XmTNndObMGUVERGj06NGaMGFCuo9x/vx5/fvf/9bJkydlZ2enJk2aaMqUKapcubJZXWv5rOYHWYlRO3fu1HfffafTp0/r2rVrKleunL7//vsU61rT9WRJWXl/Pv74Y504cUIhISF68OCBypcvLx8fH40ZM0Zubm5m9bkuMyYr782j4uLi5Ovrq2vXrqUYd9u2baubN2+a7VelShXt2bMny+dhbYh5lkWMsiziimVldTwTExO1du1abdq0SVevXpWDg4M8PDz0yiuvqEWLFiZ1Lfn5JBGCAuG9997Txo0b1bdvX9WtW1eHDh3SjBkzdO/ePY0dOzbN/UuUKKG33nrLpMzW1vyGqePHj+ull16Sp6enpkyZovDwcK1YsUJnzpzRpk2b5OTkZLFzym2ZHVMPDw/NmTPHrPzXX3/VypUr1bJlS7Nt6R3//C4iIkILFy5UhQoVVKtWLf34448Z2j84OFiDBg1SmTJlNGHCBMXGxmrVqlUaMGCAgoKCVKZMGWNda/qs5gdZiVFr167V2bNnVbt2bUVFRaXZlrVcT5aUlffn9OnTqlu3rrp27SpnZ2eFhIRo06ZN2rt3rzZv3mySpOS6zLisfr8nW7p0qW7fvv3YOk888YTGjBljUubs7Jypfls7Yp5lEaMsi7hiWVkZz6SkJI0fP14HDx5Ujx49NGjQID18+FC//fab/vjjD5O6Fv98GoB87tdffzXUqFHD8NFHH5mUjx8/3lCnTh1DWFjYY/cfPHiwoUWLFulqq1u3bobWrVsboqOjjWXfffedoUaNGgZ/f/8M9z2vyuqYpuTNN9801KhRw3D27FmT8oyMf34XGxtr+OOPPwwGg8EQEhJiqFGjhuHTTz9N9/5jxowxeHt7G49hMBgMFy9eNPzrX/8yzJgxw6SutXxW84OsXk+3bt0yJCQkGAyGtK8Xa7qeLCU74t3p06cNNWrUMMydO9eknOsyYyz13ty4ccNQt25dw5dffplq3G3Tpo2hf//+Fum3tSPmWRYxyrKIK5aV1fFctWqVoVatWoYTJ06k2ZalP58FK10Kq7Rr1y5Jf98e+aghQ4YoLi5Oe/fuTddxEhMTFRUVJUMqT5S+evWqzp8/r169eqlIkSLG8latWqlq1arasWNHJs8g77HUmCaLiYnRN998oxo1aqh27dop1klr/AsCBwcHlS9fPlP7RkdH6+DBg3r22WdNjlGjRg01btxYO3fuNJZZ02c1P8jq9VSxYkXZ2dllqE1ruJ4sxdLxTpJcXV0lSZGRkcYyrsuMs9R78+GHH8rLy0udO3dOs258fHy67kJA6oh5lkWMsiziimVlZTyTkpLk7++vdu3aqWHDhkpKSlJ0dHSKdbPj80kiBPne2bNnVbZsWVWqVMmk3MvLS7a2tjp37lyax7h7967q16+vBg0aqGHDhpo2bZru3r1r1o4kPfXUU2b7161bVxcuXFBiYmIWziTvsMSYPmrPnj2KiopSjx49UtyenvG3dhcvXlR8fHyqn7+7d+8abyG0ps9qfmDp6yktXE8ZY4n3JykpSeHh4bpz545OnTqlKVOmSJLJ3Gauy4yzxHuzd+9e/fDDD3r77bfT1Z63t7caNGigJk2aaObMmXrw4EGm+2+tiHmWRYyyLOKKZWVlPK9cuaJbt26pVq1a+uCDD1S/fn3Vr19frVu31rp168zakSz7+WSNEOR7t2/fVrly5czKHRwcVLJkSYWFhT12fzc3NzVo0ECenp4yGAw6cuSINm3apJ9++kmbNm1S8eLFje1ISrGtcuXKKS4uThERESbrNORXWR3TfwoKCpK9vb18fX3NtqV3/K1dWp8/SQoLC1OFChWs6rOaH1j6enocrqeMs8T7c+vWLbVr18742sXFRdOmTVP79u1N2pG4LjMiq+/Nw4cPNWvWLA0cOFCenp4KDQ1NtW716tXVt29feXh4KDo6Wt99951Wr16tM2fOaM2aNSpUqFCWz8daEPMsixhlWcQVy8rKeF67dk2StHLlShUuXFjTpk2Ts7Oz1q9fr3fffVeJiYkaNGiQsR3Jsp9PEiHI92JiYlS0aNEUtzk6Oio2Nvax+8+ePdvkdefOnVW3bl1Nnz5dq1at0quvvmpsR/r7wk6pnUfr5HdZHdNHhYWF6fDhw2rZsqXKli1rtj2942/tMvL5s6bPan5gyespLVxPGWeJ96ds2bLy9/dXXFycgoODtX37dkVFRSkpKcm4aCPXZcZl9b1ZvHixHj58qHHjxqXZ1pdffmnyunv37qpataq++OILbdu2TT179kx/x60cMc+yiFGWRVyxrKyMZ/I0mKioKG3cuNG4cG/Hjh3VtWtX+fn5qV+/frK3t8+WzydTY5DvOTk5KS4uLsVtsbGxxosjI/r06aOSJUuaPNUjeSXilNpKvsgLymralhzTrVu3KikpSd27d0/3PimNv7XLyOfPmj6r+UF2xKiM4Hp6PEu8P46OjmrWrJlat26t4cOHa+HChVqxYoXmz59v0o7EdZkRWXlvrly5ohUrVmjixImZvitgxIgRksS1k0HEPMsiRlkWccWysjKeyZ8nb29vk6cX2dvbq0uXLoqIiFBwcLBJXUt+PkmEIN8rV65cio+uiouL071791K8hSo9KlWqpIiICJN2JKXY1u3bt1WoUCG5uLhkqq28xpJjGhQUpBIlSpjckpke/xx/a5fW5+/ROtb0Wc0PsitGZQTXU+qy4/2pVKmS6tevr8DAQJN2JK7LjMjKe/Pvf/9brq6uatKkiUJDQxUaGmpcRykqKkqhoaFp/t/DokWLqkSJElw7GUTMsyxilGURVywrK+OZvC2l6SzJZckL+mbH55NECPK92rVr686dO7p165ZJ+ZkzZ5SUlJTqU0oeJykpSaGhoSpVqpRJO5L0yy+/mNU/ffq0atasmeFVzvMqS43pmTNn9Ntvv6lz584p3sqWmpTG39rVqFFDhQoVSvXzV7p0aVWoUEGSdX1W84PsiFEZwfX0eNn1/sTGxpo8kYHrMuOy8t7cunVL165dU/v27dWuXTu1a9fOONc8ICBA7dq105EjRx7b/r1793T//n2unQwi5lkWMcqyiCuWlZXxrFGjhhwcHIzJpEcllyWPU3Z8PkmEIN/r1KmTJGnNmjUm5cmLECUvBPXw4UMFBwcrPDzcWCcqKirFuWsrVqxQZGSkWrdubSyrVq2aatasqc2bN5us9nzw4EFdv349XY/Pyi+yMqaPCgoKkqRU50BmZPytSXx8vIKDg02y3kWLFlXLli21e/duk/JLly7p2LFj6tSpk2xsbCRZ12c1P7DU9ZQWrqfMycr7ExkZqfj4eLNjXrhwQadOnVLdunWNZVyXGZeV92batGlauHChyb8PPvhA0t/zzxcuXCgvLy9Jf/9hktLTBj7//HNJ4trJIGKeZRGjLIu4YllZGU9nZ2e1adNGp06d0oULF4zlDx8+1NatW+Xq6qpq1apJyp7PJ4ulIt+rVauWevXqJX9/f0VHR8vLy0s//vijdu3apbFjx6p8+fKS/s4WDh06VGPHjjUunnXu3DlNmDBBnTt3VpUqVWRjY6Njx45pz549qlmzptkzsadNm6YXX3xRgwcPVp8+fXT37l35+/vLw8NDAwYMyPFzzy5ZGdNkcXFx2r59uzw8PEy+aB+V0fEvCAICAhQZGam//vpLkvTzzz9r0aJFkqS2bduqZs2aCgsLU+fOndWjRw999NFHxn0nTpyoPn36aPDgwRo8eLDi4uK0cuVKlSpVSqNGjTJpx1o+q/lBVq+nEydO6MSJE5KkmzdvKjo62viZqVSpknH9HWu8niwhK+/P8ePH9d5776ljx46qWrWq7OzsdPnyZeOTsiZPnmzSFtdlxmTlvWnSpInZ8ZKf7vDEE0+YPC1j//79WrBggZ599lm5ubkpLi5O3333nY4eParmzZsbf+gjfYh5lkWMsiziimVl9XqfNGmSjh49queff15Dhw6Vs7OzAgMD9ccff+jzzz83/k8+yfKfTxIhKBDef/99VapUSYGBgQoMDJSrq6veeuutNL8EXV1d5ePjo++++05//vmnEhMT5ebmptGjR2vkyJEqUqSISf0mTZpo2bJlmj9/vmbPnq3ChQurffv2mjx5sgoXLpydp5jjMjumyQ4ePKh79+5p+PDhqdbJ6PgXBCtWrNDNmzeNr48fP67jx49LkipUqKCaNWumuu+TTz6pgIAAzZ07V/PmzZOtra2aNGmiN954w2wOpjV9VvODrFxPR48e1YIFC0zKkhe48/HxMf5RYI3Xk6Vk9v2pUaOGWrRooR9++EGbN29WfHy8ypUrp44dO2rkyJFyd3c3qc91mXFZ/S5KD09PT9WsWVO7du1SeHi4bGxs9MQTT2jy5Ml6/vnnC8R0gJxGzLMsYpRlEVcsKyvjWbVqVa1du1Zz5841PtmoVq1aWrJkiVq0aGFS19KfTxuDwWDI8F4AAAAAAAD5EGuEAAAAAAAAq0EiBAAAAAAAWA0SIQAAAAAAwGqQCAEAAAAAAFaDRAgAAAAAALAaJEIAAAAAAIDVIBECAAAAAACsBokQAAAAAABgNUiEAAAAAAAAq0EiBAAAAAAAWA0SIQAAAAAAwGqQCAEAAAAAAFaDRAgAAAAAALAaJEIAAAAAAIDVIBECAAAAAACsBokQAAAAAABgNUiEAAAAAAAAq0EiBAAAAAAAWA0SIQAAAAAAwGqQCAEAAPmSp6en/Pz8jK+PHTsmT09PHTt2LMf74ufnJ09PzxxvFwAAZByJEABAvpb8x+/GjRtT3B4YGChPT08dPnw4h3uWO0JDQ+Xp6SlPT099/fXXZtvTGi9kv7CwMH3wwQfq2LGj6tatKx8fH/n6+uqdd95RcHBwbncPAIACzz63OwAAALLHF198oV69esnR0TG3u5IjGjVqpNOnT6tQoUK53ZVUhYSEqHfv3jIYDOrRo4eqVaumqKgoXblyRfv371e9evXk4eGR290EAKBAIxECAEABVLt2bZ07d05r167VCy+8kK1tPXjwQEWKFMnWNtLD1tY2zyd9/P39de/ePQUFBelf//qXybaEhAT99ddfOdofg8Gg2NhYOTk55Wi7AADkJqbGAACsTmrrOSRPKwkMDDSWJU+t+f777zV//ny1bNlS9erV0/Dhw3Xr1i1J0ldffaVnnnlGXl5e6tmzp86ePWty3Js3b2rGjBnq0qWLvL295e3trYEDB+rgwYNmfRgyZIhatmyp0NBQjR49Wt7e3vLx8dE777yj2NjYdJ9jixYtVK9ePS1ZskQPHjxIs35kZKQ+/PBDtWzZUnXq1FH79u01f/58xcXFmdSbOnWqPD099fvvv2vChAlq1KiRunbtarLt9u3bmjRpkho1aqRGjRrpnXfeUVxcnGJiYvThhx+qWbNmeuqppzRmzBiFh4ebHP+///2vJk2apLZt28rLy0uNGzfWq6++qitXrqR5Dv9cI+TRaUIp/Xt0LZEHDx7o008/Vfv27VWnTh01b95c77zzju7du2fWztatW9W5c2d5eXmpY8eO2rx5c5p9S3b9+nUVK1bMLAkiSfb29nJxcTEpS0hI0LJly+Tr66u6deuqcePGGjJkiA4dOmRS7+DBg+rfv7/q1aun+vXra/jw4frll19M6iSPx7x58xQYGKguXbrIy8tLO3fuNLa1dOlS47k1btxYkyZNMn7OAQAoKLgjBABQIERHR5v9US0pXUmA9Jg3b54cHR310ksv6fbt2/L399eYMWPUtWtXbdmyRQMGDNDDhw+1dOlSjR07Vnv27DFO0Thz5owOHz6sDh06yNXVVZGRkdq6datGjRolf39/NW3a1KSt2NhYDRs2TD4+PnrjjTf0yy+/aP369SpVqpRee+21dPd5/PjxGjZsmAICAjRy5MhU68XFxemFF17Qr7/+qt69e+tf//qX/vvf/2rRokU6f/68vvjiC7N9RowYoWrVqmnChAlmyZLRo0fL3d1dEyZM0IkTJ7R+/Xo5ODgoNDRUSUlJeuWVV3Tp0iWtW7dOjo6OmjdvnnHfXbt26c8//1SvXr1Urlw5hYaGasOGDRo4cKB27Nih0qVLp/v8S5UqpTlz5piVL1q0SDdu3DAmHZLP//Lly+rTp488PDx07do1ffXVVzp16pQ2btxovNNk69atmjx5sjw9PTVx4kRFRUVpzpw5qlixYrr65ObmpkOHDmn37t169tlnH1s3KSlJY8eO1YEDB9SqVSvjlJpffvlFR44cUfPmzSVJO3fu1MSJE+Xu7q6xY8cqPj5e69at0+DBg7Vy5Uo1aNDA5Lj79+9XRESEBgwYoFKlSsnd3V0Gg0Hjx4/XwYMH1aNHDw0dOlRhYWH66quvdOLECQUFBalUqVLpOkcAAPI6EiEAgAJh9uzZmj17drYd39bWVgEBAbK3//urMzExUStWrFBkZKS2b99unBpSokQJffDBB/r+++/Vrl07SVKrVq3UsWNHk+MNHTpU3bt317Jly8wSIffu3dPLL79snNIyYMAARUZGav369RlKhDRr1kw+Pj5avny5Bg4cqKJFi6ZYb+PGjTp37pymTJmiF198UZI0aNAglS1bVqtWrdKBAwfUpk0bk328vb01Y8aMFI/XsGFDvfnmm5KkgQMH6saNGwoICFCHDh1MnvISERGh3bt36/79+ypRooQk6fXXX1fhwoVNjte9e3f5+vpq06ZNGjVqVLrPv0iRIurWrZtJmb+/v65du6bXX39dNWrUkCStWrVKv/76q9auXSsvLy9jXR8fH40ePVqBgYEaMGCAEhISNGfOHFWuXFlr166Vs7OzJOnZZ581ayc1L7zwgrZu3apx48apWrVqatiwoerXr6+WLVuaJXm2bdumAwcOaMSIEXr99ddNthkMBkl/38Uxa9YslS9fXuvXr1fx4sUlST169FCnTp00c+ZMkzucJOnq1avatWuXKleubCzbuXOn9u7dq0WLFhk/t5L0zDPPqFevXlqxYoVZHwAAyK+YGgMAKBCGDRsmf39/s3/Dhw+3yPH79u1rTIJIMv5fdl9fX5P1MerXry/p70Uxkz36h31sbKwiIiIUHR0tHx8fnTlzxqwtW1tb9e/f36TMx8dH4eHhioqKylC/x48fr3v37mnlypWp1tm/f7+KFCmiQYMGmZS/9NJLxu3/NHDgwFSPN2DAAJPX9evXl8FgMDun+vXrKzExUTdv3jSWPTpW0dHRioiIUIkSJeTu7p7iWGXE999/r3//+9967rnnNGLECGP5jh07VKdOHbm6uio8PNz476mnnlKRIkV05MgRSdLZs2d1584d9enTx5gEkaQaNWoY785Ii7u7uwIDA9WzZ09FRERow4YNmjp1qpo3b67p06crOjraWHfXrl1ycnLSmDFjzI5jY2Nj0qd+/foZkyCSVKFCBfn6+urcuXMKCwsz2bdly5YmSZDkMShfvry8vb1NxqB8+fKqWrWqcQwAACgIuCMEAFAgeHh4qFmzZmblf/zxh0WOX6lSJZPXxYoVkySzKRHJ5Y+uLREXF6fFixdry5YtJn/0S//7g/ZRpUuXNlu8MvmP3Pv376d6Z0dKGjZsqObNm2vlypUaMmRIinVu3rwpNzc3s4VGy5Urp+LFiys0NNRsn3/+If0oV1fXFPv+z7F69JyS3b59W5988okOHDhgUi7JbP2MjAgODtbEiRNVp04dffjhhybbrl69qpiYGLM7c5IlT7lKfu/c3d3N6ri7u+v7779PV1/c3d01e/ZsGQwG3bx5U8ePH1dAQIA2btyoxMRE451N169fl5ub22MXok3uU0pPmkkuCw0NVfny5Y3lVapUMat79epVhYWFpToG//z8AwCQn5EIAQBYnZSSD9LfazKkxs7OLkPlyVMXJGnWrFlat26dBgwYoAYNGqhEiRKys7PT5s2btX37drN9bW1Tv2Hz0eOm1/jx49WnTx8tW7Ys3XcupOVxTxlJrf9pjVVSUpKGDx+usLAwPf/886pevbqKFCkiW1tbzZo1K1PnLv1vqlGRIkW0YMECs4SPwWCQt7e3xo0bl+L+ycktS7OxsZGbm5vc3NzUtWtXdenSRdu2bdOMGTNM7j6ytJSerGMwGOTm5pbqdCcHB4ds6w8AADmNRAgAwOo8eidC8toUkul0FkvasWOHunfvrnfffdekfNOmTdnS3j/VrVtXbdu2VUBAgGrVqmW23c3NTT/99JNiY2NN/ki+c+eOIiMj5ebmliP9vHTpki5duqTZs2erZ8+eJtvu3buXqTtCEhISNH78eIWFhSkgIEDlypUzq1O1alXdv38/xTuKHpV8p8vVq1fNtqVUlhEODg6qWbOmbty4oYiICJUtW1ZVq1bV4cOHH/t44uT3Jjg42Gxbcll63r+qVavq+PHjatSokXGRXwAACirWCAEAWJ2qVatKko4ePWosMxgMWr16dba0Z2tra3Y3w5UrV7R3795saS8l48ePNz7V5p/atGmjBw8eaO3atSbly5cvlyS1bds2R/qYfCfJP8fqP//5j+7cuZOpY3744Yc6evSoZs2aZbIQ6qO6dOmiK1euKCgoyGxbYmKicZpTnTp1VLZsWW3cuNHkaUSXLl0ye5xtao4fP57iY5Dv3bunkydPqmTJksans3Tq1EmxsbFatGiRWf3kMapdu7bKlSunDRs2mKwfExYWpm3btql27dom02JS06VLF0VHR2vJkiUptpXSE5kAAMivuCMEAGB1nn76aVWuXFnTp09XcHCwihUrpm+//VYPHz7Mlvbat2+vwMBAFS5cWLVq1VJISIjWrl0rDw8P/frrr9nS5j/VrFlTzz77rL755huzbX369NHmzZv10Ucf6cqVK6pZs6Z++uknbd++XW3atDF7Ykx2qVatmtzd3fXxxx/r1q1bKlu2rH755Rft27fvsWuSpObgwYNau3at6tatq4SEBG3ZssVk+9NPP60yZcpo2LBh+u677zR16lR9//338vb2lsFg0I0bN/Ttt9/qtddeU8+ePWVvb6/XX39dU6ZMUf/+/dWzZ09FRUVpzZo1qlGjhi5cuJBmn5YvX65Tp06pXbt2qlWrlhwdHXXz5k1t3bpVd+7c0XvvvWecQuTr66sdO3Zo6dKlunz5spo1ayYbGxudPn1aFSpU0Ouvvy57e3tNmzZNEydOVN++fdWrVy/j43MTEhL01ltvpWusunbtqj179ujzzz/XL7/8oqZNmxofebx//3517dpVr776aobfAwAA8iISIQAAq2Nvb69FixZpxowZWrx4sYoWLaouXbqof//+6tKli8Xbe/PNN+Xk5KRvv/1WgYGBqlatmj744AMFBwfnWCJEksaNG6dvv/3WbC0UBwcHrVy5UvPnzzf2sXz58nr55ZdTfGJJdrG3t9eSJUs0a9YsrVmzRvHx8apXr55WrVqlWbNmZfh4d+/elSSdPn1ab7zxhtn21atXq0yZMnJ0dNSqVau0YsUK7dixQ3v27JGjo6MqVaqkrl27qkmTJsZ9unfvLkn64osvNHfuXLm5uemNN97QrVu30pUIGT16tHbu3KkTJ05o3759ioqKUvHixVWnTh29++67atWqlbGura2tFi5cqOXLlysoKEg//vijnJ2dVbNmTZOpQ507d5azs7MWL16szz//XLa2tvL29tZnn32mevXqpWusbGxs9Nlnn2nt2rXavHmzPvvsM9na2qpChQpq1qyZOnXqlK7jAACQH9gYMrvyGAAAAAAAQD7DGiEAAAAAAMBqkAgBAAAAAABWg0QIAAAAAACwGiRCAAAAAACA1SARAgAAAAAArAaJEAAAAAAAYDVIhAAAAAAAAKtBIgQAAAAAAFgNEiEAAAAAAMBqkAgBAAAAAABW4/8AciCmwUYxwU4AAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "algorithms = ['SimPLe', 'CURL', 'DrQ', 'SPR', 'IRIS (ours)']\n", + "fig, axes = plot_utils.plot_interval_estimates(\n", + " {k: v[:3] for k, v in aggregate_scores.items()}, \n", + " {k: v[:, :3] for k, v in aggregate_interval_estimates.items()},\n", + " metric_names = ['Mean', 'Median', 'Interquartile Mean'],\n", + " algorithms=algorithms,\n", + " colors=ATARI_100K_COLOR_DICT,\n", + " xlabel_y_coordinate=-0.1,\n", + " xlabel='Human Normalized Score', \n", + " subfigure_width=5,\n", + " row_height=0.7)\n", + "plt.show()\n", + "save_fig(fig, 'aggregates')" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAVIAAADDCAYAAADQmJm9AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAA3b0lEQVR4nO3deVyN6f8/8FeLisoSaVSY6uMc06ZFESZSTCuKSIox9hFNGGsYa6YxtiJbQlEqyRIjsqswxr5PobIk2rRv9+8Pv3N/O86pTp32eT8fjx4P57qv+1ruo3f3fd33fV0SDMMwIIQQUmeSTd0AQghp6SiQEkKImCiQEkKImCiQEkKImCiQEkKImCiQEkKImCiQkv+coUOHYvHixU1Wv5+fH7hcLl+au7s73N3dm6hFRFzSTd0A8t/16dMn7N27FxcvXsTbt28hIyMDDocDJycnODk5QVKy7n/nY2Nj8ezZM8yZM6ceW9x4KioqsH37dnz33XewsrJqkDoyMjJw8OBBXLlyBampqSgpKUHnzp2ho6MDGxsb2NjYQFqaQoQoJOiBfNIUHj16hKlTpyI/Px+jRo2Crq4uCgoKEBcXh5s3b2Lw4MHw9/eHjIxMncpfsGABTp48iWfPnglsKykpgYSEBNq0aSNuN+rEz88P/v7+fG0rKSkBALa/ZWVl0NHRgaOjIzZs2FDvbbhx4wbmzJmD4uJi2NjYQFdXF3Jycnj//j3i4+Nx+/ZtTJ06Fb/++mu9190a0Z8b0ujy8vIwe/ZsVFRUIDw8HL1792a3/fjjjwgICMCWLVuwceNGLF26tN7rr2twbkiN2aZ3797Bw8MDHTt2xL59+9C9e3e+7R4eHnjw4AGePn3aaG1q6WiMlDS6I0eO4N27d5g/fz5fEOWZNWsWDA0NcfjwYXz48IFNd3d3h7m5OV6+fIkpU6bA0NAQ/fv3x6pVq1BQUMCX7+TJkwAALpfL/qSlpQEQHCNNS0sDl8vF5s2bcerUKdjZ2UFfXx8jRoxAYmIiACAhIQHOzs7Q19fH0KFDER0dzdfmkpIS+Pn5wdnZGf369YOenh5sbW2xf/9+iHLRV3mMNC0tDTo6OgCAY8eOse13d3dHUlISuFwuAgICBMooKCiAoaEhFixYUG1de/bsQW5uLtavXy8QRHn09PTg7Oxcp/7xxoAfP36M5cuXo1+/fjAwMMCMGTOQmppa47FoieiMlDS68+fPQ0ZGBiNGjKgyz+jRo3Hnzh1cuXIFY8aMYdOLioowefJkmJiYYMGCBbhz5w4OHz6MtLQ07NmzBwAwc+ZMlJaW4s6dO/D19WX3VVJSqrZdV65cwfHjxzF+/Hi0adMGgYGBmDFjBnx9fbFq1SqMHz8eI0aMQEhICBYvXgxdXV3873//A/DlLPvw4cOwsbFh+3X9+nX4+PggJycHnp6eIh8fJSUl/P7771i0aBH69u2LsWPHAgC6dOkCLS0tGBoa4vjx45g1axbffrGxsSgoKICTk1O15cfFxUFdXR0mJiYit6ku/VuyZAkUFBQwe/ZspKenIyQkBBMmTMCJEyfQsWNHketuERhCGpmJiQnj4OBQbZ6HDx8yHA6H2bBhA5vm5ubGcDgcxtfXly+vj48Pw+FwmEuXLrFp8+fPZzgcjtCyLSwsmEWLFrGfU1NTGQ6HwxgYGDDv379n0y9evMhwOBxGW1ubefLkCZv+/PlzhsPhMOvXr2fTysrKmOLiYoG6Fi9ezBgYGPBt27Ztm0Db3NzcGDc3N/ZzaWkpw+Fw+NrJc+TIEYbD4TB3797lS584cSIzePBgpry8XGi/GYZhPn/+zHA4HGbWrFkC2/Lz85lPnz6xP9nZ2WL1b9y4cUxpaSmbHhcXJ/T7aw3o0p40ury8PCgoKFSbh7f98+fPAtsmTZrE93ny5MkAgIsXL4rVrqFDh0JFRYX9bGRkBODLZW7lIYhevXpBUVERKSkpbJqUlBTfjaKcnBxkZmaif//+KCgoQHJyslhtq8zW1hZt27bFsWPH2LS3b9/ixo0bGDlyZLVPO+Tl5QEA5OXlBbZt3LgRZmZm7I+joyO7rS79c3Nz47vrP3ToUHz77bdif0/NEV3ak0anoKDA/kJXhbf964ArLy+Prl278qWpqKhAXl6eHQOtK1VVVb7P7du3F5rO25aTk8OXdvz4cQQFBeH58+coLy/n2ybsD0JdKSgoYPjw4Thz5gyWLl0KGRkZHD9+HAzD8AW/qvYFgPz8fIFtbm5u7KNWPj4+Anlq2z8NDQ2hafHx8dV3sAWiQEoanZaWFh4+fIiioiLIyckJzfPkyRMAYMcgG0NVZ3JVpTOVbrKcPn0aCxcuhLm5Odzc3KCsrIw2bdrg0aNH2LhxIyoqKuq1raNHj8bx48dx6dIlDB8+HNHR0TA0NMS3335b7X4KCgpQUVER+liYpqYmNDU12XyVA2lj96+loUBKGt3QoUPxzz//4OTJk3x3hiuLiopCmzZtYG5uzpeen5+PDx8+8J2VpqenIz8/H+rq6myahIREwzS+CjExMVBXV8euXbv4Am/ly//aqKn9pqam6N69O44dOwZlZWW8evUKU6ZMEalsS0tLHD58GLdu3RL5hlNd+vfy5Uv26YPKaZW/p9aCxkhJo3NxcYGKigo2btyI58+fC2zfvXs3bt++jfHjxwtcxgPAgQMH+D4HBQUBACwsLNi0du3aAYDA5XdDkZKSAsB/llpUVITg4OA6lycrK1tl+yUkJODo6IirV68iKCgIcnJysLGxEansqVOnQlFREUuXLq3ycSTmq0ea6tK/kJAQlJWVsZ8vXLiAV69eYciQISK1syWhM1LS6BQVFeHv74/p06fD2dmZ782mCxcuIDExEYMHDxb6PGSHDh0QExODjIwM9OnTB3fv3sWJEycwaNAgDB48mM2np6eHsLAwrF69Gubm5pCWloaFhQUbYOubpaUlzp49i+nTp2PYsGHIyclBVFSU0Js6otLT00NCQgICAwPxzTffQElJCWZmZux2R0dH+Pv74+zZs7C3t4eioqJI5aqpqcHPzw9z5syBvb09bGxsoKenB1lZWWRkZOD69eu4c+cOTE1NxepfYWEhJk2aBGtra6SnpyM4OBjKysqYOnVqnY9Jc0WBlDQJfX19nDx5Env37sWlS5cQFRXFvmu/du1aODk5sWdBlcnJySEoKAhr167Fxo0bISMjAxcXFyxcuJAv38iRI/Ho0SOcPXsWMTExYBgGcXFxDRZIR44cidzcXISEhGDdunVQVlaGk5MTjIyM2KcKauu3337DqlWr4Ofnh8LCQpiamvIFUlVVVZiZmeH69es13mT6mpmZGU6fPo0DBw7gypUrOHv2LEpLS6GkpAQdHR1s3LgRtra2YvXPx8cHhw8fhr+/P4qLi2Fqagpvb+8an+dtiehde9JiuLu74/Xr17hy5UpTN6XZmD59Op4+fYpLly6JNclLfeLNJRAbG4uePXs2dXMaRfM48oSQWktNTcXVq1fh6OjYbILofxVd2hPSwjx//hyPHz9GSEgIZGVl4erq2tRN+s+jQEpIC3P27Fls374dampq+OOPP/jexiJNg8ZICSFETDSwQgghYqJASpq1oqKipm5Cg6M+tnwUSEmz9vXkGK0R9bHlo0BKCCFiokBKCCFiahWPP23btg0xMTE4ffq00NcKm4v8/HwMHToUCxYsqHLWo9rIKihBUVnzm75MTloSndo1vwXmCGkoNQbSGzduYOLEiVi7di37y5+WlgZLS0u+fPLy8uByuXBzc4OdnR3fNl7+mTNnwsvLi03PycnBvn37cOHCBaSlpUFKSgrKysro06cPXFxcYGBgUGMHPn78iKCgICxfvrxZB1HgyzFyd3eHn58fHBwcqpyLU1RFZRWYcOifempd/Tk0waipm0BIoxLrjNTS0hI//PADGIbBhw8fEBERgXnz5qGoqAijR4+udt/Pnz9j9OjR+PjxI0aMGAFXV1cUFxfj1atXuHr1KtTU1EQKpCEhIZCWloa9vb04XWk048aNw/bt2xEdHQ0XF5embg7eJcQAALqZ2dWQkxBSFbECaa9evTBy5Ej2s5OTE4YNG4Z9+/bVGEjDw8ORmpqK7du3s8sb8DAMg8zMzBrrLy8vR1RUFIYPH95ka5WXl5ejvLxc5PqVlZXRr18/RERENItASggRX73ebOrSpQs0NTXx+vXrGvPy8lSe85BHQkICnTt3rrGMu3fvIj09HYMGDRK6PTo6Go6OjtDX14epqSnmzJmDpKQkvjxRUVF8a55XxuVy4efnx36+ceMGuFwuwsPDERgYCCsrK+jp6eHOnTsAvry6N3bsWPTt2xcGBgYYNmwYli9fLlDuoEGD8PDhQ7x9+7bGPhJCmr96vdlUVlaG9PR0kdas5i03cPTo0TrP13j79m0AX+a2/FpgYCB8fX2hr6+PefPmIScnByEhIXBxcUFkZKRY03sdPHgQpaWlGDt2LOTk5KCsrIyEhAR4enrCxMQEv/zyC6SlpZGamip0xcQ+ffoAAP7+++9q13YnhLQMYgXS4uJi9hI8PT0de/fuRUZGhkiB0dnZGQcPHsSGDRsQHBwMExMTGBoawtzcXOiqjcIkJSVBSkpKIH9WVha2bt0KHR0dHDp0iL3strKywpgxY/Dnn39i27Zttezt//n06RPOnj3LrjIJAGFhYZCXl0dQUBDfErS//vqrwP49evRg208IafnECqRBQUHsejnAl3VdJkyYgHnz5tW4b6dOnXD06FHs2bMHsbGxiI6ORnR0NIAvN7HWrFlT4+V9ZmYmFBUVBRYKi4+PR3FxMSZNmsQ3dqmjo4MBAwbg8uXLKCsr4wt4tWFvb88XRIEvy/MWFhbi6tWrGDJkSLWLl/HO2LOysupUPyGkeRErkDo4OMDJyQmlpaV49OgRdu/ejc+fP6NNmzYi7a+iogJvb294e3sjPT0dt2/fxpEjRxAXF4f8/HyBRc5E9ebNGwBflv39mpaWFq5du4ZPnz7Vefox3hllZa6urjh79ixmzpyJzp07o3///rCwsIC1tbXA8fivL11LSGsjViBVU1PDgAEDAACDBw+Guro6fv31VxgYGGDChAm1KktFRQW2trawsbGBm5sbEhMT8e7dO3Tr1q3KfZSUlPD582cwDFPn5Xer2q+6d4OFPf+ppKSEY8eOITExEVevXkV8fDxiYmKwd+9eHD58mG+RsNzcXABfzsoJIS1fvd61HzFiBHR1deHn54f8/Pw6lSEhIQE9PT0AX8Zdq6OlpYXy8nL2DJSHdyNL2BhkUlIS2rZtyw4b8C7RecGNp6plaqsjLS2NQYMGYcmSJTh58iRWrlyJp0+f4tSpU3z5eGuBCztjJoS0PPX+rv2MGTOQlZWF0NDQavPdvXsXeXl5AuklJSWIj4+HlJRUjXfWjY2NAQAPHjzgSzczM4OsrCyCg4NRUlLCpj99+hTx8fHs8rwA8O233wIAEhMT+cqo7bCCsPFObW1tAIJrq9+7d4+v/YSQlq3e37W3srKCpqYmgoKC4O7uDllZWaH5Tpw4gWPHjmHo0KHQ09ODoqIi0tPTERMTg3///RfTpk2r8dLXwMAAXbt2xbVr12BjY8Omd+rUCZ6envD19WVfWeU9/qSgoMB3M0xLSwumpqbYsmULsrOzoaKiguvXr+PDhw+16re3tzc+ffoEMzMzqKqqIisrC2FhYZCTk8OwYcP48l67dg06OjpQU1OrVR1fk5OWFP91zAZ4nVNOmubCIf8t9R5IJSUlMW3aNCxZsgTh4eFwd3cXms/FxQUKCgpISEhAQkICcnJy2Pf1fX19+d6YqoqUlBScnJwQGhqKlStX8t2hnzJlCjp37oz9+/dj48aNkJOTQ79+/eDl5cWehfL4+vpi1apVOHjwINq0aQMLCwusW7cO/fv3F7nfI0aMQGRkJCIiIpCdnY2OHTvC0NAQs2bNgoaGBpvv48ePuHnzptAH9WuLJgYhpHlo8Ws2ZWRkYPjw4VixYgUcHR2bujk18vf3x5EjRxAbG4u2bds2dXOavfz8fL4bda0R9bHla/HXYMrKypg8eTJ27tzZ7Gfhzs/PR3BwMObMmUNBlJBWpMWfkZLWrbWfyQDUx9agxZ+REkJIU6NASgghYqJASgghYmoVazaR/6byoiww5cU15pOQkoWUHL2OSxoOBdL/7+XLlwgICMCdO3fw/v17yMvLQ1VVFcbGxpg6dSpUVFTY9at4JCQk0L59e+jq6mLatGkwMzNjt7m7u+PmzZvs5zZt2qBbt26wsrLC7NmzoaCg0Kj9a42Y8mJknBH+nHJlyjbBjdAa8l9GgRTA/fv34e7ujvbt28PJyQnq6urIzMzEixcvcPz4cVhZWfHNFDVmzBiYmpqioqICr1+/RlhYGKZMmYK9e/eyk7gAQIcOHbBs2TIAQF5eHhISErBv3z48fPgQwcH0y13fjlz8suLAOAvR5rMlpL5QIAWwfft2SEpKIjIyUmBqvaKiIpSVlfGlGRgY8L15NXz4cDg6OiIwMJAvkMrJyfHlmzBhAjw9PfHXX3/h/v37Qmf2J4S0PHSzCV/Wj+rRo4fQ+Unl5ORqvAzX1tZGx44dha779DXeGlW8GaAIIS0fBVJ8mXYvOTkZ//xTtzXis7KykJubK9L8orxgK8q6VoSQloEu7fFl6r+EhAS4urqid+/eMDY2hrGxMb7//nsoKioK5M/Pz0dmZiYYhsHr16+xadMmVFRU8M1ABXyZCZ+3plV+fj7i4+Nx6NAhdO3aFSYmJo3SN0JIw6NACsDExARhYWHYu3cvrl27hidPniAkJARt2rTBTz/9hLlz5/Kt7+Tj4wMfHx/2c9u2bTF9+nS+O/rAlwlVKt/JBwA9PT2sW7euyukFCSEtDwXS/09PTw9bt25FRUUFXr16hcTERAQFBWHXrl2QkZGBh4cHm3fy5MkwNzeHhIQEFBUV8b///U/o8iOdOnXCpk2bAHyZ7X///v349OkTTVhCSCtDgfQrkpKS0NTUhKamJmxsbDBs2DBER0fzBVItLS2+u/NVkZGR4ctnYWEBOzs7zJ8/H+Hh4XVeZ4oQ0rzQzaZqdOrUCT169Khx7ShRdezYEbNmzcL9+/dx+vTpeimTENL0KJACiI+PFzqXaWpqKpKSkup1kTpnZ2d07twZO3fuBM1gSEjrQJf2ANavX4/c3FxYWlqiV69ekJaWxsuXLxEdHY3S0lL88ssv9VaXrKwsJk2ahE2bNuH8+fMC6zkR0UlIyfK9/ulhU3U+QhoSBVIAixYtwrlz53Dr1i2cOnUKBQUFUFJSgomJCSZPngxDQ8N6rc/V1RW7d+9GQEAABVIx0EQkpLmgGfJJs9baZ1YHqI+tAY2REkKImCiQEkKImCiQEkKImCiQEkKImCiQEkKImOjxJ0BgCRFpaWkoKCjg22+/Rd++fTFu3Dj06NGj1uWeP38eoaGhePjwIfLz89G1a1cMHDgQ06dPR/fu3euzC6SZySouQFF5WbV55KSk0Um2XSO1iDQkCqSV8JYQYRgGOTk5ePLkCQ4fPowDBw5g2bJlGD9+vEjlMAyD5cuXIyIiAtra2pgyZQo6deqE5ORkREZG4tSpU/D398fAgQMbuEekqRSVl8Ht8iEAwLtzCQCAbsP4ZwILGTyh0dtFGgYF0kq+XkIEABYsWIAZM2Zg1apV6NmzZ5WTlTAMg+LiYsjJyeHAgQOIiIjAuHHj8Ntvv0FS8v9GUCZOnAg3Nzd4enri1KlT+Oabbxq0T4SQhkdjpDXo0qULtmzZAklJSWzfvh3Al1nuuVwuNm/ejKioKNjZ2UFPTw+nT59GcXExAgICoK6uDm9vb74gCgDdunXDqlWr8PnzZ+zdu7cpukQIqWd0RiqC7t27w8TEBLdu3UJeXh6bfuHCBWRlZWH8+PFQUlKChoYGbt++jezsbLi6ukJGRkZoeYMGDUK3bt0QFxcHb2/vxuoGIaSBUCAVUa9evZCYmIi0tDR2MbyXL1/izJkzfDeODh48CODLgnjV6d27Ny5evNjqX50j5L+ALu1FxAt2+fn5bJq5ubnA3XfeGWtNK4/ytlc+wyWEtEwUSEXEC6CVA6SwR6JEDZB5eXmQkJAQaeVRQkjzRoFURM+fP4eUlBTU1dXZNGEL2P3vf/8DADx58qTa8p4+fYpu3bpVOY5KCGk5KJCKIDU1FX///TeMjIxqHM80NjZGx44dceLECZSWlgrNEx8fj3fv3gks30wIaZkokNbg06dP8PLyQkVFBX7++eca88vKymLWrFlITU3FunXrUFFRwbc9PT0dK1asQIcOHeDm5tZQzSaENCK6a1/J3bt3ISMjA4ZhkJubi8ePH+Ps2bMoLS3FihUrRFo5FAAmTZqEFy9eIDQ0FPfv34etrS06duyIpKQkHD16FKWlpdi6dStUVVUbuEekOfj6jSbS+lAgrSQyMhKRkZHsu/Y9e/aEq6srxo4di549e4pcjoSEBNatW4chQ4YgLCwMe/bsQU5ODhiGgaKiIk6ePMk31kpaHzkp6RpfAZWTol+/1oKWGmlE/v7+8PPzg4uLC1atWtXUzWkR/gvP2VIfWz76k9iIPDw8kJeXh6CgICgpKcHT07Opm0QIqQd0RkqatdZ+JgNQH1sDumtPCCFiokBKCCFiokBKCCFiokBKCCFiarWBNDc3F/7+/hg1ahSMjIygp6eHYcOGwdvbG48fP2bz+fn5gcvl4vXr10LL2bx5M7hcLtLS0ti0xYsXg8vlsj86OjqwsLDA8uXLkZGRIVAGL39ZWfVr+BBC6ldZYSlKPhdX+1NWKPxV7tpolY8/vXjxAtOmTUNGRgasra0xZswYyMrK4tWrV/jrr78QGRmJS5cuib3Mx++//w4JCQkUFRXhzp07iIyMxM2bN3Hy5EmajISQZqCirAIvwx4CAKLiTwIAnAY48OXRcNEVu55WF0jz8/Mxa9YsFBYWIjw8HDo6Onzbvby8EBgYiPp46sve3h7S0l8O4bhx46CkpITAwEDExcXRhCSE/Ie0ukv78PBwpKamYuHChQJBFPiy1PKMGTPQrVu3eq/b1NQUAJCSklLvZRNCmq9WF0jPnTsHGRkZODg41Jy5nvHGUTt27NjodRNCmk6ru7RPSkqChoZGo4xRZmdnQ1JSEkVFRbh79y78/f3Rtm1bDB06tMHrJoQ0H60ukObl5UFLS6tR6ho4cCDfZw0NDaxevRrKysqNUj8hpHlodYFUQUGBb4G6+iIhISGQFhgYCElJSWRlZSEsLAzPnj2DnJxcvddNCGneWl0g1dLSwoMHD1BSUiLS5T1v3aWioiKh23npwgJk//792bv2w4YNw+jRo+Hp6YnTp0+jbdu2de0CIaSFaXU3m6ysrFBSUoJTp06JlF9NTQ3Al7FVYZKSktCuXbsabyDJyMhg3rx5ePv2LYKDg2vVZkJIy9bqAunYsWOhpqYGX19fPH36VGB7WVkZdu/ejffv3wMAzMzMICsri9DQUJSUlPDlffbsGRITE2Fubg4pKaka67awsACXy0VQUBAKCwvrp0OEkGav1V3aKygoICAgANOmTcOYMWNgbW0NQ0NDyMjIICUlBWfPnkVqaipGjhwJAFBSUsK8efPg4+OD0aNHw8bGBkpKSkhOTkZ4eDjk5eUxf/58keufPn065s+fjyNHjuDHH3/k27Zz505ISgr+7ZowYQI6dOggVr8JIdX7+o2m+tRqJ3bOycnBgQMHEBcXh5SUFJSWlqJbt24wMzODq6srevfuzZf//PnzOHjwIB4/fozCwkJ06dIFAwYMwOzZswXWV1q8eDGOHTuGR48esWOkPOXl5fjhhx9QXFyMuLg4yMjIsPmrEhsbW6s1of5LWvuEwAD1sSGVFZaioqyi2jyS0pKQbttGrHpabSAlrQMFmdahtfex1Y2REkJIY6NASgghYqJASgghYqJASgghYqJASgghYmqRgTQqKkpg+Q9CCGkqze6B/JcvXyIgIAB37tzB+/fvIS8vD1VVVRgbG2Pq1KlQUVFpsLr9/Pzg7+/PfpaSkoKSkhL69++PuXPnokePHuw2LpcLBwcHbNy4scHaQ0hTKioqQnl5eb2UVVFR0SCTCdWGlJRUg00q1KwC6f379+Hu7o727dvDyckJ6urqyMzMxIsXL3D8+HFYWVlBRUUFI0eOhJ2dXYPNObpo0SJ07twZJSUlePDgAaKionD9+nWcPHkSXbp0aZA6CWluysvLce7cuXorS9hr1hcuXACARpnDd9iwYQ1WdrMKpNu3b4ekpCQiIyMFzjyLiorYVTilpKREeve9riwtLdk3jZydnaGpqQkfHx9ERUVh+vTpDVYvIaRlalZjpK9fv0aPHj2EXr7LyclBQUEBgPAxUt6Sxx8+fMD8+fNhYmICExMTrFixAiUlJSgqKsLatWsxYMAA9OnTBz///DMyMzNFateAAQMAAKmpqbXu09OnT+Hh4YF+/fpBT08PDg4OiIiIqHU5hJDmq1mdkaqrq+PGjRv4559/YGRkVKcyZs6cCQ0NDXh5eeHWrVs4cuQIZGRkkJaWhoqKCsyePRvPnz9HWFgYZGVlsXnz5hrL5C1m16lTp1q15Z9//sFPP/2E7t27Y8qUKZCXl8fFixfh7e2NzMxMzJgxo059JIQ0L80qkM6YMQMJCQnspCLGxsYwNjbG999/D0VFRZHK6Nu3L5YuXQoAcHV1RUpKCkJCQjBs2DD4+fmx+bKysnD27Fnk5OQIzLyUk5ODzMxMlJaW4sGDB1i7di0kJSVhbW0tcl8YhsHy5cvRq1cvHD58GG3afJkUYcKECZg7dy527NgBFxcXmvWJkFagWV3am5iYICwsDD/88ANSU1MREhICLy8vmJmZYdOmTewYaXXGjx/P99nIyAgMw8DFxUUgvby8HG/evBEow9nZGWZmZjA3N8fs2bNRVlaGP//8E9ra2iL35dmzZ/j333/h4OCAz58/IzMzk/0ZPHgwioqKcOfOHZHLI4Q0X83qjBQA9PT0sHXrVlRUVODVq1dITExEUFAQdu3aBRkZGXh4eFS7P2/Ge5727dsDgMA69rz0nJwcgTJ+//13dO3aFVJSUujcuTM0NDRqfXMrOTkZALBu3TqsW7dOaJ5Pnz7VqkxCSPPU7AIpj6SkJDQ1NaGpqQkbGxsMGzYM0dHRNQZSYRMnA6gyEAqbRdDQ0LDe5gf18PCAsbGx0G2NtdopIaRhNdtAWlmnTp3Qo0cPvHjxoqmbIjLew/uysrLsXX9CSOvUrMZI4+Pjhb5JkZqaiqSkpBZ1BqetrQ0NDQ0cPHhQ6GNWdFlPSOvRrM5I169fj9zcXFhaWqJXr16QlpbGy5cvER0djdLSUvzyyy9N3UQ+L168wI4dOwTSVVVVMWrUKPj4+OCnn36Cra0txowZgx49eiAzMxNPnjxBXFwcHj582AStJqT5aIw3mhpDswqkixYtwrlz53Dr1i2cOnUKBQUFUFJSgomJCSZPngxDQ8OmbiKfp0+fCl2p1MjICKNGjYKhoSEiIyMREBCA6OhoZGdno1OnTtDS0sKSJUuaoMWEiE5KSqreXqusqKio8v5FY2nItyFpzSbSrLX2tX4A6mNr0KzGSAkhpCWiQEoIIWKiS3tCCBETnZESQoiYKJASQoiYKJASQoiYmtVzpER0XC63qZtAyH/Os2fPhKbTGSkhhIiJ7toTQoiY6IyUEELERIGUEELERIGUEELERHftSYMpKyvDrl27cPToUWRkZEBNTQ1ubm6YMGECJCQkqt3X3d0dN2/eFEiXkpLC48ePBdLLy8sRGhqKyMhIvHz5EjIyMtDS0sLs2bPx/fff11ufviZOH+va7pKSEjg4OODVq1eYOXMmvLy86rtbfOraxxs3bmDixIlVbu/ZsydiY2MBANnZ2Th27BguXryIpKQkFBQUoHv37rC3t8ekSZMgKytb7/2qTxRISYP57bffEBERgbFjx0JfXx/Xrl3DmjVrkJ2dXeOSMQDQoUMHLFu2jC9N2FRsFRUV8PT0xOXLl+Ho6IgJEyagsLAQ//77L96/f19v/RFGnD7Wtd179uzBhw8f6rsrVaprH7W0tODr6yuQ/vjxY+zfvx/m5uZs2p07d/DHH39g4MCB+Omnn6CgoIBbt25h06ZNuHTpEoKDgxt0GjyxMYQ0gMePHzMcDofZsGEDX7qnpyejq6vLpKenV7u/m5sb8/3334tU14EDBxhtbW3m1q1bdW5vXYjbx7q0OyUlhdHX12d27drFcDgcZtOmTXVqu6jE7aMwS5cuZTgcDvPw4UM2LSUlhUlJSRHIu2XLFobD4TCxsbG1b3wjojFS0iDOnDkD4MslemXu7u4oKSnB+fPnRSqnvLwceXl5QhcpBL6c1QUFBcHS0hJ9+/ZFRUUF8vPzxWu8iMTpY13bvXbtWujp6cHW1la8xouovr5HnqKiIvz111/gcDjQ0dFh07t3747u3bsL5Le2tgaAZr9eGwVS0iAePnwIZWVlqKqq8qXr6elBUlISjx49qrGMT58+wcjICMbGxujbty+WLFkisNZVcnIy3r59C21tbaxevRpGRkYwMjLCkCFDEBYWVq99+po4faxLu8+fP4+rV69i+fLl9dqP6tTH91jZuXPnkJeXB0dHR5Hy84YwlJSUalVPY6MxUtIgPnz4gK5duwqky8jIoGPHjkhPT692f3V1dRgbG4PL5YJhGCQkJCAyMhK3b99GZGQk2rdvDwB49eoVAGD//v1o27YtlixZAnl5eRw5cgQrV65EeXk5JkyYUO/9A8TrY23bXVhYiPXr18PV1RVcLhdpaWn13h9hxP0evxYdHQ1paWk4ODjUmLeiogK7d+9G27ZtYWVlVat6GhsFUtIgioqKoKCgIHSbrKwsiouLq93fx8eH77OtrS309fXh7e2NAwcOYM6cOQDAXg7n5eUhIiKCvTy0traGvb09/Pz8MG7cOEhL1/9/dXH6WNt2BwQEoLCwEHPnzq3nXlRP3O+xsvT0dMTHx8Pc3BzKyso15t+yZQtu3rwJb29vdOnSReR6mgJd2pMGIScnh5KSEqHbiouL6/Q4i7OzMzp27Ijr16/z1QMAhoaGfGNs0tLSsLOzQ1ZWFpKSkmpdlyjE6WNt2p2cnIx9+/Zh3rx57Jl4Y6nP7/HEiROoqKjAqFGjaswbEhKCXbt2wcXFRWB8tjmiQEoaRNeuXYU+olNSUoLs7Gyhl4uiUFVVRVZWFl89AISesfDScnNz61RXTcTpY23a/ccff0BNTQ39+/dHWloa0tLS2Mej8vLykJaWhqKiIrH7U1U76+t7jI6ORocOHWBpaVltvqioKKxduxZ2dnZYuXJlrdvcFCiQkgaho6ODjIwMvH37li/9wYMHqKio4LtjK6qKigqkpaXx3XjgcDiQkZER+twlL62hblSI08fatPvt27d49eoVrKysYGlpCUtLS3b8NCQkBJaWlkhISKivbvGpr+/xwYMH+Pfff2FrawsZGZkq88XExGDZsmUYMmQIfH19m3wJZ1G1jFaSFsfGxgYAEBwczJceHByMNm3asDcPCgsLkZSUhMzMTDZPXl6e0LG3ffv2ITc3F0OGDGHT5OXlYWFhgbt37+Lp06dsemFhIU6cOAE1NTVoamrWZ9dY4vSxNu1esmQJtm/fzvezevVqAF/GVLdv3w49Pb1m18fKoqOjAQBOTk5V1nX+/HksXLgQ/fr1w7Zt2xpkXLuhtJyWkhZFW1sbo0ePRlBQEPLz86Gnp4fr16/jzJkz8PDwgIqKCgDg/v37mDhxIjw8PNgbSI8ePYKXlxdsbW3Ro0cPSEhI4MaNGzh37hx69+4tMGY2f/58JCYmYtKkSZg4cSLk5eURFRWF9+/fY9u2bSK9qtnYfaxNu/v37y9QN++u/bffftugd7TF7SPwZRjg1KlT0NLSgr6+vtB67t+/Dy8vL8jKysLa2pp9fpWnR48eMDQ0bJhO1gMKpKTBrFq1CqqqqoiKikJUVBTU1NSwbNmyGm8eqKmpwdTUFJcuXcLHjx9RXl4OdXV1zJw5E9OnT0e7du348vfs2ROhoaHYuHEjgoKCUFJSAm1tbezevbtB37MH6t7Hpm53bYjTRwC4fPkysrOzMWXKlCrz/PvvvygpKUFJSYnQcVFHR8dmHUhpYmdCCBETjZESQoiYKJASQoiYKJASQoiYKJASQoiYKJASQoiYKJASQoiYKJASQoiYKJASQoiYKJASQoiYKJASQoiYKJASQoiYKJASQoiYKJASQoiYKJASQoiYKJASQoiYKJASQoiYKJASQoiYKJASQoiYKJASQoiYKJAS0gxwuVz4+fmxn2/cuAEul4sbN240elv8/PzA5XIbvd6WjAJpM8L75YmIiBC6PSoqClwuF/Hx8Y3csqaRlpYGLpcLLpeLw4cPC2yv6XiRhpeeno7Vq1fD2toa+vr6MDU1hYODA1asWIGkpKSmbl6joeWYSYuwc+dOjB49GrKysk3dlEZhYmKC+/fvo02bNk3dlCqlpqZizJgxYBgGjo6O0NTURF5eHpKTk3HhwgUYGBhAS0urqZvZKCiQkmZPR0cHjx49QmhoKH788ccGraugoADt2rVr0DpEISkp2ez/aAQFBSE7OxvR0dH47rvv+LaVlZXh8+fPjdoehmFQXFwMOTm5Rq0XoEv7Fq+q8SzeZXFUVBSbxhsauHLlCrZu3Qpzc3MYGBhgypQpePv2LQDg0KFDGD58OPT09ODk5ISHDx/ylfvmzRusWbMGdnZ2MDQ0hKGhIVxdXXH58mWBNri7u8Pc3BxpaWmYOXMmDA0NYWpqihUrVqC4uFjkPn7//fcwMDDA7t27UVBQUGP+3NxcrF27Fubm5tDV1YWVlRW2bt2KkpISvnyLFy8Gl8vFu3fv4OXlBRMTE9jb2/Nt+/DhA+bPnw8TExOYmJhgxYoVKCkpQVFREdauXYsBAwagT58++Pnnn5GZmclX/t9//4358+dj6NCh0NPTQ79+/TBnzhwkJyfX2Ievx0grD3MI+6k8llpQUIBNmzbBysoKurq6GDRoEFasWIHs7GyBek6cOAFbW1vo6enB2toaR48erbFtPK9fv4aioqJAEAUAaWlpdOrUiS+trKwMe/fuhYODA/T19dGvXz+4u7vj2rVrfPkuX74MFxcXGBgYwMjICFOmTMG9e/f48vCOx+bNmxEVFQU7Ozvo6enh9OnTbF179uxh+9avXz/Mnz+f/X9e3+iMtBnKz88X+KUEIFIQEcXmzZshKyuLqVOn4sOHDwgKCsLPP/8Me3t7HD9+HOPHj0dhYSH27NkDDw8PnDt3jr3EfPDgAeLj4zFs2DCoqakhNzcXJ06cwIwZMxAUFAQzMzO+uoqLizF58mSYmppi4cKFuHfvHo4cOQIlJSX88ssvIrfZ09MTkydPRkhICKZPn15lvpKSEvz44494/PgxxowZg++++w5///03duzYgSdPnmDnzp0C+0ybNg2amprw8vISCLYzZ86EhoYGvLy8cOvWLRw5cgQyMjJIS0tDRUUFZs+ejefPnyMsLAyysrLYvHkzu++ZM2fw8eNHjB49Gl27dkVaWhrCw8Ph6uqKmJgYdO7cWeT+KykpwdfXVyB9x44dSElJYYMWr/8vXryAs7MztLS08OrVKxw6dAh3795FREQEe6Z74sQJ/Prrr+ByuZg3bx7y8vLg6+uLbt26idQmdXV1XLt2DWfPnsUPP/xQbd6Kigp4eHjg4sWLGDx4MDskcO/ePSQkJGDQoEEAgNOnT2PevHnQ0NCAh4cHSktLERYWBjc3N+zfvx/GxsZ85V64cAFZWVkYP348lJSUoKGhAYZh4OnpicuXL8PR0RETJ05Eeno6Dh06hFu3biE6OhpKSkoi9VFkDGk2EhMTGQ6HU+PP9evX2X22bdvGcDgcgbJSU1MZDofDHD16lE07evQow+FwGCcnJ6a0tJRN37BhA8PhcBgLCwsmPz+fTQ8JCWE4HA5z/vx5Nq2goECgrqKiIsba2pr56aef+NLd3NwYDofDBAUF8aXPmjWL6d+/f43Hg9eHTZs2seWZmpoynz9/5jte4eHhAm0ODAzkK2vdunUMh8NhLly4wKYtWrSI4XA4jLe3t0DdvG3r1q3jS3dycmK4XC7j4eHBlz5nzhzmu+++Y7Kzs9k0YccqOTmZ0dHRYXbu3MmXzuFwmG3btrGfeX1LTEwUfnAYhtm3bx/D4XCY3bt3s2m7d+9mdHR0mPv37/PlvXDhAsPhcJjDhw8zDMMwpaWlzMCBAxlLS0smLy+Pzffs2TOmd+/eQv9PCeuLgYEBw+FwGGtra8bb25uJiopiPn78KJA3Ojqa4XA4zB9//CGwraKigq9N5ubmTE5ODrv93bt3jIGBAePo6Mim8f5v6OjoMCkpKXzlxcTECPy/ZRiGefz4MfPdd98JbYO46NK+GZo8eTKCgoIEfqZMmVIv5Y8dOxbS0v93McL7K+/g4MA3PmhkZATgy00FnrZt27L/Li4uRlZWFvLz82FqaooHDx4I1CUpKQkXFxe+NFNTU2RmZiIvL69W7fb09ER2djb2799fZZ4LFy6gXbt2mDBhAl/61KlT2e1fc3V1rbK88ePH8302MjICwzACfTIyMkJ5eTnevHnDplU+Vvn5+cjKykKHDh2goaEh9FjVxpUrV/DHH39gxIgRmDZtGpseExMDXV1dqKmpITMzk/3p06cP2rVrh4SEBADAw4cPkZGRAWdnZ8jLy7P7czgc9uywJhoaGoiKioKTkxOysrIQHh6OxYsXY9CgQfD29kZ+fj6b98yZM5CTk8PPP/8sUI6EhARfm8aNG4f27duz27/55hs4ODjg0aNHSE9P59vX3Nwc3bt350uLiYmBiooKDA0N+Y6BiooKevbsyR6D+kSX9s2QlpYWBgwYIJD+/v37eilfVVWV77OioiIACFzS8dIrj62VlJQgICAAx48f5wsawP/9QlTWuXNngcF/3i9JTk4OFBQURG533759MWjQIOzfvx/u7u5C87x58wbq6uoCN2q6du2K9u3bIy0tTWCfr38RK1NTUxPa9q+PVeU+8Xz48AF//vknLl68yJcOQGD8sDaSkpIwb9486OrqYu3atXzbXr58iaKiIoEhFh7ekBHvu9PQ0BDIo6GhgStXrojUFg0NDfj4+IBhGLx58wY3b95ESEgIIiIiUF5eDh8fHwBfxlPV1dWrvZHHa5OwO/28tLS0NKioqLDpPXr0EMj78uVLpKenV3kMvv7/Xx8okLZwwoIX8GVMqipSUlK1SmcYhv33+vXrERYWhvHjx8PY2BgdOnSAlJQUjh49ilOnTgnsKylZ9UVP5XJF5enpCWdnZ+zdu1fkM6eaVHeXt6r213SsKioqMGXKFKSnp2PSpEno1asX2rVrB0lJSaxfv75OfQe+/FGbNWsW2rVrB39/f4E/GAzDwNDQEHPnzhW6P++PY32TkJCAuro61NXVYW9vDzs7O5w8eRJr1qzhu/qpb8KebGAYBurq6lizZo3QfWRkZOq9HRRIW7jKZ0IdOnRg0ytfjtenmJgYjBo1CitXruRLj4yMbJD6vqavr4+hQ4ciJCQE2traAtvV1dVx+/ZtFBcX8/2SZWRkIDc3F+rq6o3SzufPn+P58+fw8fGBk5MT37bs7Ow6nZGWlZXB09MT6enpCAkJQdeuXQXy9OzZEzk5OUKvaCrjnWm/fPlSYJuwtNqQkZFB7969kZKSgqysLCgrK6Nnz56Ij4+v9vEy3ncj7EF+Xpoo31/Pnj1x8+ZNmJiYNNpzuDRG2sL17NkTAJCYmMimMQyDgwcPNkh9kpKSAmdTycnJOH/+fIPUJ4ynpyf7VMHXLCwsUFBQgNDQUL70wMBAAMDQoUMbpY28M9mvj9WxY8eQkZFRpzLXrl2LxMRErF+/Hnp6ekLz2NnZITk5GdHR0QLbysvL2WEaXV1dKCsrIyIigu9pkOfPnws8jlSVmzdvCn2MLTs7G3fu3EHHjh3Zu+M2NjYoLi7Gjh07BPLzjpGOjg66du2K8PBwvvHz9PR0nDx5Ejo6OnyX9VWxs7NDfn4+du/eLbQuYU/EiIvOSFu4gQMHonv37vD29kZSUhIUFRURGxuLwsLCBqnPysoKUVFRaNu2LbS1tZGamorQ0FBoaWnh8ePHDVLn13r37o0ffvgBf/31l8A2Z2dnHD16FBs2bEBycjJ69+6N27dv49SpU7CwsICFhUWjtFFTUxMaGhr4/fff8fbtWygrK+PevXuIi4urdky2KpcvX0ZoaCj09fVRVlaG48eP820fOHAgunTpgsmTJ+PSpUtYvHgxrly5AkNDQzAMg5SUFMTGxuKXX36Bk5MTpKWlsWDBAixatAguLi5wcnJCXl4egoODweFw8PTp0xrbFBgYiLt378LS0hLa2tqQlZXFmzdvcOLECWRkZOC3335jh0AcHBwQExODPXv24MWLFxgwYAAkJCRw//59fPPNN1iwYAGkpaWxZMkSzJs3D2PHjsXo0aPZx5/KysqwbNkykY6Vvb09zp07h23btuHevXswMzNjH1m7cOEC7O3tMWfOnFp/B9WhQNrCSUtLY8eOHVizZg0CAgKgoKAAOzs7uLi4wM7Ort7rW7p0KeTk5BAbG4uoqChoampi9erVSEpKarRACgBz585FbGyswFiwjIwM9u/fj61bt7JtVFFRwaxZs4TeMW4o0tLS2L17N9avX4/g4GCUlpbCwMAABw4cwPr162td3qdPnwAA9+/fx8KFCwW2Hzx4EF26dIGsrCwOHDiAffv2ISYmBufOnYOsrCxUVVVhb2+P/v37s/uMGjUKwJfXbzdu3Ah1dXUsXLgQb9++FSmQzpw5E6dPn8atW7cQFxeHvLw8tG/fHrq6uli5ciUGDx7M5pWUlMT27dsRGBiI6OhoXL9+HfLy8ujduzff0IetrS3k5eUREBCAbdu2QVJSEoaGhtiyZQsMDAxEOlYSEhLYsmULQkNDcfToUWzZsgWSkpL45ptvMGDAANjY2IhUTm1IMHUd9SaEEAKAxkgJIURsFEgJIURMFEgJIURMFEgJIURMFEgJIURMFEgJIURMFEgJIURMFEgJIURMFEgJIURMFEgJIURM/w+fKQDSoV6onwAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "algorithms = ['SimPLe', 'CURL', 'DrQ', 'SPR', 'IRIS (ours)']\n", + "fig, axes = plot_utils.plot_interval_estimates(\n", + " {k: v[3:4] for k, v in aggregate_scores.items()}, \n", + " {k: v[:, 3:4] for k, v in aggregate_interval_estimates.items()},\n", + " metric_names = ['Optimality Gap'],\n", + " algorithms=algorithms,\n", + " colors=ATARI_100K_COLOR_DICT,\n", + " xlabel_y_coordinate=-0.3,\n", + " xlabel='Human Normalized Score')\n", + "plt.show()\n", + "save_fig(fig, 'optimality_gap')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "QM9CLBft9Bqs" + }, + "source": [ + "# Performance profile" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "v1xRtwkwwtPo", + "outputId": "305cd55a-5d1a-41a5-b32d-2b955f09f320" + }, + "outputs": [], + "source": [ + "algorithms = ['SimPLe', 'CURL', 'DrQ', 'SPR', 'IRIS (ours)']\n", + "score_dict = {key: all_score_dict[key] for key in algorithms}\n", + "ATARI_100K_TAU = np.linspace(0.0, 8.0, 201)\n", + "reps = 2000\n", + "\n", + "score_distributions, score_distributions_cis = rly.create_performance_profile(score_dict, ATARI_100K_TAU, reps=reps)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 348 + }, + "id": "PGMKH2sGINQj", + "outputId": "d20552ea-f134-43e1-95ad-a5f6056cb9c2" + }, + "outputs": [], + "source": [ + "fig, ax = plt.subplots(ncols=1, figsize=(7.25, 4.7))\n", + "\n", + "plot_utils.plot_performance_profiles(\n", + " score_distributions, ATARI_100K_TAU,\n", + " performance_profile_cis=score_distributions_cis,\n", + " colors=ATARI_100K_COLOR_DICT,\n", + " xlabel=r'Human Normalized Score $(\\tau)$',\n", + " labelsize='xx-large',\n", + " ax=ax)\n", + "\n", + "ax.axhline(0.5, ls='--', color='k', alpha=0.4)\n", + "fake_patches = [mpatches.Patch(color=ATARI_100K_COLOR_DICT[alg], \n", + " alpha=0.75) for alg in algorithms]\n", + "legend = fig.legend(fake_patches, algorithms, loc='upper center', \n", + " fancybox=True, ncol=3, \n", + " fontsize='x-large',\n", + " bbox_to_anchor=(0.57, 0.93))\n", + "save_fig(fig, 'performance_profile')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "3kmNgRii94Ee" + }, + "source": [ + "# Probability of improvement" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "62YrK4dn98fs", + "outputId": "2b7ec982-c96d-4283-cdae-9937e71b14f5" + }, + "outputs": [], + "source": [ + "algorithms = ['SimPLe', 'CURL', 'DrQ', 'SPR', 'IRIS (ours)']\n", + "our_algorithm = 'IRIS (ours)'\n", + "all_pairs = {}\n", + "for alg in (algorithms):\n", + " if alg == our_algorithm:\n", + " continue\n", + " pair_name = f'{our_algorithm}_{alg}'\n", + " all_pairs[pair_name] = (all_score_dict[our_algorithm], all_score_dict[alg]) \n", + "\n", + "probabilities, probability_cis = {}, {}\n", + "reps = 1000\n", + "probabilities, probability_cis = rly.get_interval_estimates(all_pairs, metrics.probability_of_improvement, reps=reps)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 248 + }, + "id": "hT5A8wyu-Fpo", + "outputId": "8bd6d523-0e57-4417-b120-dceb6cad9a7a" + }, + "outputs": [], + "source": [ + "fig, ax = plt.subplots(figsize=(4, 3))\n", + "h = 0.6\n", + "algorithm_labels = []\n", + "\n", + "for i, (alg_pair, prob) in enumerate(probabilities.items()):\n", + " _, alg1 = alg_pair.split('_')\n", + " algorithm_labels.append(alg1)\n", + " (l, u) = probability_cis[alg_pair]\n", + " ax.barh(y=i, width=u-l, height=h, left=l, color=ATARI_100K_COLOR_DICT[alg1], alpha=0.75)\n", + " ax.vlines(x=prob, ymin=i-7.5 * h/16, ymax=i+(6*h/16), color='k', alpha=0.85)\n", + "\n", + "ax.set_yticks(range(len(algorithm_labels)))\n", + "ax.set_yticklabels(algorithm_labels)\n", + "\n", + "ax.set_xlim(0, 1)\n", + "ax.axvline(0.5, ls='--', color='k', alpha=0.4)\n", + "ax.set_title(fr'P(IRIS > $Y$)', size='xx-large')\n", + "plot_utils._annotate_and_decorate_axis(ax, labelsize='xx-large', ticklabelsize='xx-large')\n", + "ax.set_ylabel(r'Algorithm $Y$', size='xx-large')\n", + "ax.xaxis.set_major_locator(MaxNLocator(4))\n", + "fig.subplots_adjust(wspace=0.25, hspace=0.45)\n", + "save_fig(fig, 'probability_of_improvement')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Detailed results" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for k in aggregate_scores.keys():\n", + " n_runs, n_games = all_score_dict[k].shape\n", + " assert n_games == 26\n", + " score_dict = score_data_dict_games[k]\n", + " mean, median, iqm, og = aggregate_scores[k]\n", + " sh = np.sum(np.mean(all_score_dict[k], axis=0) >= 1)\n", + " print(f\"\\n####################\\n{k}\\n####################\\n\")\n", + " print(f\"{n_runs} runs\")\n", + " print(f\"#superhuman: {sh}\\nMean: {mean:.3f}\\nMedian: {median:.3f}\\nIQM: {iqm:.3f}\\nOptimality gap: {og:.3f}\\n\")\n", + " \n", + " for game in score_dict.keys():\n", + " h, r = HUMAN_SCORES[game], RANDOM_SCORES[game]\n", + " raw_score = score_dict[game] * (h - r) + r\n", + " print(f\"{game}: {np.mean(raw_score): .1f}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Generate $\\LaTeX$ results table" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Utility to print the latex table\n", + "\n", + "first_row = [\"Game\", \"Random\", \"Human\", \"MuZero\", \"EfficientZero\", \"SimPLe\", \"CURL\", \"DrQ\", \"SPR\", r\"\\textsc{iris} (ours)\"]\n", + "rows = [first_row]\n", + "\n", + "# Raw scores\n", + "\n", + "for game in ATARI_100K_GAMES:\n", + " raw_scores = [RANDOM_SCORES[game], HUMAN_SCORES[game]]\n", + " raw_scores.extend([np.mean(score_data_dict_games[algo][game] * (HUMAN_SCORES[game] - RANDOM_SCORES[game]) + RANDOM_SCORES[game]) for algo in aggregate_scores.keys()])\n", + " idx_max_baselines = 4 + np.argmax(raw_scores[4:])\n", + " idx_max_all = 2 + np.argmax(raw_scores[2:])\n", + " raw_scores = [f\"{x:.1f}\" for x in raw_scores]\n", + " raw_scores[idx_max_baselines] = f\"\\\\textbf{{{raw_scores[idx_max_baselines]}}}\"\n", + " raw_scores[idx_max_all] = f\"\\\\underline{{{raw_scores[idx_max_all]}}}\"\n", + " row = [game, *raw_scores]\n", + " rows.append(row)\n", + "\n", + "# Aggregates \n", + "\n", + "first_col = [\"\\\\#Superhuman (↑)\", \"Mean (↑)\", \"Median (↑)\", \"IQM (↑)\", \"Optimality Gap (↓)\"]\n", + "cols = [\n", + " [0, 0, 0, 0, 1], # Random\n", + " [float('-inf'), 1, 1, 1, 0], # Human\n", + "]\n", + "for algo in aggregate_scores.keys():\n", + " n_runs, n_games = all_score_dict[algo].shape\n", + " assert n_games == len(ATARI_100K_GAMES)\n", + " score_dict = score_data_dict_games[algo]\n", + " sh = np.sum(np.mean(all_score_dict[algo], axis=0) >= 1)\n", + " col = [sh, *aggregate_scores[algo]]\n", + " cols.append(col)\n", + "\n", + "rows_ = np.array(cols).T\n", + "for i, row in enumerate(rows_):\n", + " idx_best_baselines = 4 + (np.argmin(row[4:]) if i == len(rows_) - 1 else np.argmax(row[4:]))\n", + " idx_best_all = 2 + (np.argmin(row[2:]) if i == len(rows_) - 1 else np.argmax(row[2:]))\n", + " row = [f\"{x:.{0 if i == 0 else 3}f}\" if not math.isinf(x) else 'N/A' for x in row]\n", + " row[idx_best_baselines] = f\"\\\\textbf{{{row[idx_best_baselines]}}}\"\n", + " row[idx_best_all] = f\"\\\\underline{{{row[idx_best_all]}}}\"\n", + " rows.append([first_col[i]] + row)\n", + " \n", + " # IQM and OG are N/A for MuZero and EfficientZero. \n", + " if i >= 3:\n", + " rows[-1][3:5] = ['N/A'] * 2\n", + " #rows[-1][3:5] = 'N/A' \n", + "\n", + "rows = np.array(rows)\n", + "for i in range(rows.shape[1]):\n", + " max_len = max(map(len, rows[:, i])) + 1\n", + " rows[:, i] = list(map(lambda x: x.ljust(max_len), rows[:, i]))\n", + "\n", + "rows = [' & '.join(row) for row in rows]\n", + "\n", + "for i, row in enumerate(rows):\n", + " print(row + r' \\\\')\n", + " if i in [0, 26]:\n", + " print(r\"\\midrule\")\n" + ] + } + ], + "metadata": { + "colab": { + "collapsed_sections": [], + "name": "Copy of deep_rl_precipice_colab.ipynb", + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.12" + } + }, + "nbformat": 4, + "nbformat_minor": 1 +} diff --git a/scripts/eval.py b/scripts/eval.py new file mode 100755 index 0000000..a565c0c --- /dev/null +++ b/scripts/eval.py @@ -0,0 +1,57 @@ +#! python + +import argparse +from datetime import datetime +from pathlib import Path +import subprocess + +from omegaconf import OmegaConf + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument('-n', '--num-episodes', type=int, default=100, help='Number of evaluation episodes to collect.') + parser.add_argument('-p', '--num-envs', type=int, default=25, help='Number of environments used to collect the evaluation episodes.') + args = parser.parse_args() + + path_to_config = Path('config') / 'trainer.yaml' + cfg = OmegaConf.load(path_to_config) + + path_to_checkpoint = Path('checkpoints') / 'last.pt' + assert path_to_checkpoint.is_file() + + cmd = f'python src/main.py hydra.run.dir=eval_outputs/{datetime.now().strftime("%Y-%m-%d/%H-%M-%S")} ' + + cmd += 'wandb.mode=online ' + cmd += f'wandb.name=eval-{cfg.wandb.name} ' + cmd += f'wandb.group=eval-{cfg.wandb.group} ' + + cmd += f'initialization.path_to_checkpoint={str(path_to_checkpoint.absolute())} ' + cmd += 'initialization.load_tokenizer=True ' + cmd += 'initialization.load_world_model=False ' + cmd += 'initialization.load_actor_critic=True ' + + cmd += 'common.epochs=1 ' + cmd += 'common.device=cuda:0 ' + cmd += 'common.do_checkpoint=False ' + cmd += 'common.seed=0 ' + + cmd += 'collection.test.num_episodes_to_save=0 ' + cmd += f'collection.test.num_envs={args.num_envs} ' + cmd += f'collection.test.config.num_episodes={args.num_episodes} ' + + cmd += 'training.should=False ' + + # Turn on data collection only + cmd += 'evaluation.should=True ' + cmd += 'evaluation.every=1 ' + cmd += 'evaluation.tokenizer.start_after_epochs=1 ' + cmd += 'evaluation.tokenizer.save_reconstructions=False ' + cmd += 'evaluation.world_model.start_after_epochs=1 ' + cmd += 'evaluation.actor_critic.start_after_epochs=1 ' + + subprocess.run(cmd, shell=True, check=True) + + +if __name__ == '__main__': + main() diff --git a/scripts/import_run_folder.py b/scripts/import_run_folder.py new file mode 100755 index 0000000..75f75cd --- /dev/null +++ b/scripts/import_run_folder.py @@ -0,0 +1,60 @@ +#! /usr/bin/python3 + +import argparse +from pathlib import Path +import shutil +import subprocess +import yaml + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument('src_folder', type=str, help='Folder to import. Must be formatted as: USERNAME@HOSTNAME:PATH_TO_OUTPUTS_FOLDER/DATE/TIME') + parser.add_argument('-k', '--from-key', action='store_true') + parser.add_argument('-v', '--verbose', action='store_true') + args = parser.parse_args() + + if args.from_key: + key_file = Path('path_runs.yaml') + assert key_file.is_file() + with key_file.open('r') as f: + runs = yaml.load(f, Loader=yaml.loader.SafeLoader) + src_folder = runs[args.src_folder] + else: + src_folder = args.src_folder + + day, time = src_folder.split('/')[-2:] + host = src_folder.split('@')[1].split(':')[0] + dst_folder = Path(host) / day / time + print(dst_folder) + if dst_folder.is_dir(): + if input(f'{dst_folder} exists, remove it ? [Y/n] ').lower() != 'n': + shutil.rmtree(dst_folder) + else: + print('Bye.') + return + + dst_folder.mkdir(exist_ok=False, parents=True) + + # Make symbolic link from key to folder + if args.from_key: + subprocess.run(f'ln -s {str(dst_folder)} {args.src_folder}', shell=True, check=True) + + download_media = args.verbose and input('Download media/ ? [y/N] ').lower() == 'y' + download_last = not args.verbose or input('Download checkpoints/last.pt ? [Y/n] ').lower() != 'n' + + folders = ['src', 'config', 'scripts'] + if download_media: + folders.append('media') + + for folder in folders: + subprocess.run(f'scp -r {src_folder}/{folder} {dst_folder}', shell=True, check=True) + + if download_last: + checkpoint_folder = dst_folder / 'checkpoints' + checkpoint_folder.mkdir(exist_ok=False, parents=False) + subprocess.run(f'scp {src_folder}/checkpoints/last.pt {checkpoint_folder}', shell=True, check=True) + + +if __name__ == '__main__': + main() diff --git a/scripts/play.sh b/scripts/play.sh new file mode 100755 index 0000000..b69509f --- /dev/null +++ b/scripts/play.sh @@ -0,0 +1,29 @@ +#!/usr/bin/bash + +fps=15 +header=1 +mode="episode_replay" + +while [ "$1" != "" ]; do + case $1 in + -f | --fps ) + shift + fps=$1 + ;; + -h | --no-header ) + header=0 + ;; + -w | --world-model ) + mode="world_model" + ;; + -a | --agent ) + mode="agent" + ;; + * ) + echo Invalid usage : $1 + exit 1 + esac + shift +done + +python src/play.py hydra.run.dir=. hydra.output_subdir=null +mode="${mode}" +fps="${fps}" +header="${header}" diff --git a/scripts/resume.sh b/scripts/resume.sh new file mode 100755 index 0000000..e1552e2 --- /dev/null +++ b/scripts/resume.sh @@ -0,0 +1 @@ +python src/main.py common.resume=True hydra.output_subdir=null hydra.run.dir=. diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/agent.py b/src/agent.py new file mode 100644 index 0000000..f5dd1d3 --- /dev/null +++ b/src/agent.py @@ -0,0 +1,37 @@ +from pathlib import Path + +import torch +from torch.distributions.categorical import Categorical +import torch.nn as nn + +from models.actor_critic import ActorCritic +from models.tokenizer import Tokenizer +from models.world_model import WorldModel +from utils import extract_state_dict + + +class Agent(nn.Module): + def __init__(self, tokenizer: Tokenizer, world_model: WorldModel, actor_critic: ActorCritic): + super().__init__() + self.tokenizer = tokenizer + self.world_model = world_model + self.actor_critic = actor_critic + + @property + def device(self): + return self.actor_critic.conv1.weight.device + + def load(self, path_to_checkpoint: Path, device: torch.device, load_tokenizer: bool = True, load_world_model: bool = True, load_actor_critic: bool = True) -> None: + agent_state_dict = torch.load(path_to_checkpoint, map_location=device) + if load_tokenizer: + self.tokenizer.load_state_dict(extract_state_dict(agent_state_dict, 'tokenizer')) + if load_world_model: + self.world_model.load_state_dict(extract_state_dict(agent_state_dict, 'world_model')) + if load_actor_critic: + self.actor_critic.load_state_dict(extract_state_dict(agent_state_dict, 'actor_critic')) + + def act(self, obs: torch.FloatTensor, should_sample: bool = True, temperature: float = 1.0) -> torch.LongTensor: + input_ac = obs if self.actor_critic.use_original_obs else torch.clamp(self.tokenizer.encode_decode(obs, should_preprocess=True, should_postprocess=True), 0, 1) + logits_actions = self.actor_critic(input_ac).logits_actions[:, -1] / temperature + act_token = Categorical(logits=logits_actions).sample() if should_sample else logits_actions.argmax(dim=-1) + return act_token diff --git a/src/collector.py b/src/collector.py new file mode 100644 index 0000000..85e0150 --- /dev/null +++ b/src/collector.py @@ -0,0 +1,125 @@ +import random +import sys +from typing import List, Optional, Union + +from einops import rearrange +import numpy as np +import torch +from tqdm import tqdm +import wandb + +from agent import Agent +from dataset import EpisodesDataset +from envs import SingleProcessEnv, MultiProcessEnv +from episode import Episode +from utils import EpisodeDirManager, RandomHeuristic + + +class Collector: + def __init__(self, env: Union[SingleProcessEnv, MultiProcessEnv], dataset: EpisodesDataset, episode_dir_manager: EpisodeDirManager) -> None: + self.env = env + self.dataset = dataset + self.episode_dir_manager = episode_dir_manager + self.obs = self.env.reset() + self.episode_ids = [None] * self.env.num_envs + self.heuristic = RandomHeuristic(self.env.num_actions) + + @torch.no_grad() + def collect(self, agent: Agent, epoch: int, epsilon: float, should_sample: bool, temperature: float, burn_in: int, *, num_steps: Optional[int] = None, num_episodes: Optional[int] = None): + assert self.env.num_actions == agent.world_model.act_vocab_size + assert 0 <= epsilon <= 1 + + assert (num_steps is None) != (num_episodes is None) + should_stop = lambda steps, episodes: steps >= num_steps if num_steps is not None else episodes >= num_episodes + + to_log = [] + steps, episodes = 0, 0 + returns = [] + observations, actions, rewards, dones = [], [], [], [] + + burnin_obs_rec, mask_padding = None, None + if set(self.episode_ids) != {None} and burn_in > 0: + current_episodes = [self.dataset.get_episode(episode_id) for episode_id in self.episode_ids] + segmented_episodes = [episode.segment(start=len(episode) - burn_in, stop=len(episode), should_pad=True) for episode in current_episodes] + mask_padding = torch.stack([episode.mask_padding for episode in segmented_episodes], dim=0).to(agent.device) + burnin_obs = torch.stack([episode.observations for episode in segmented_episodes], dim=0).float().div(255).to(agent.device) + burnin_obs_rec = torch.clamp(agent.tokenizer.encode_decode(burnin_obs, should_preprocess=True, should_postprocess=True), 0, 1) + + agent.actor_critic.reset(n=self.env.num_envs, burnin_observations=burnin_obs_rec, mask_padding=mask_padding) + pbar = tqdm(total=num_steps if num_steps is not None else num_episodes, desc=f'Experience collection ({self.dataset.name})', file=sys.stdout) + + while not should_stop(steps, episodes): + + observations.append(self.obs) + obs = rearrange(torch.FloatTensor(self.obs).div(255), 'n h w c -> n c h w').to(agent.device) + act = agent.act(obs, should_sample=should_sample, temperature=temperature).cpu().numpy() + + if random.random() < epsilon: + act = self.heuristic.act(obs).cpu().numpy() + + self.obs, reward, done, _ = self.env.step(act) + + actions.append(act) + rewards.append(reward) + dones.append(done) + + new_steps = len(self.env.mask_new_dones) + steps += new_steps + pbar.update(new_steps if num_steps is not None else 0) + + # Warning: with EpisodicLifeEnv + MultiProcessEnv, reset is ignored if not a real done. + # Thus, segments of experience following a life loss and preceding a general done are discarded. + # Not a problem with a SingleProcessEnv. + + if self.env.should_reset(): + self.add_experience_to_dataset(observations, actions, rewards, dones) + + new_episodes = self.env.num_envs + episodes += new_episodes + pbar.update(new_episodes if num_episodes is not None else 0) + + for episode_id in self.episode_ids: + episode = self.dataset.get_episode(episode_id) + self.episode_dir_manager.save(episode, episode_id, epoch) + metrics_episode = {k: v for k, v in episode.compute_metrics().__dict__.items()} + metrics_episode['episode_num'] = episode_id + metrics_episode['action_histogram'] = wandb.Histogram(np_histogram=np.histogram(episode.actions.numpy(), bins=np.arange(0, self.env.num_actions + 1) - 0.5, density=True)) + to_log.append({f'{self.dataset.name}/{k}': v for k, v in metrics_episode.items()}) + returns.append(metrics_episode['episode_return']) + + self.obs = self.env.reset() + self.episode_ids = [None] * self.env.num_envs + agent.actor_critic.reset(n=self.env.num_envs) + observations, actions, rewards, dones = [], [], [], [] + + # Add incomplete episodes to dataset, and complete them later. + if len(observations) > 0: + self.add_experience_to_dataset(observations, actions, rewards, dones) + + agent.actor_critic.clear() + + metrics_collect = { + '#episodes': len(self.dataset), + '#steps': sum(map(len, self.dataset.episodes)), + } + if len(returns) > 0: + metrics_collect['return'] = np.mean(returns) + metrics_collect = {f'{self.dataset.name}/{k}': v for k, v in metrics_collect.items()} + to_log.append(metrics_collect) + + return to_log + + def add_experience_to_dataset(self, observations: List[np.ndarray], actions: List[np.ndarray], rewards: List[np.ndarray], dones: List[np.ndarray]) -> None: + assert len(observations) == len(actions) == len(rewards) == len(dones) + for i, (o, a, r, d) in enumerate(zip(*map(lambda arr: np.swapaxes(arr, 0, 1), [observations, actions, rewards, dones]))): # Make everything (N, T, ...) instead of (T, N, ...) + episode = Episode( + observations=torch.ByteTensor(o).permute(0, 3, 1, 2).contiguous(), # channel-first + actions=torch.LongTensor(a), + rewards=torch.FloatTensor(r), + ends=torch.LongTensor(d), + mask_padding=torch.ones(d.shape[0], dtype=torch.bool), + ) + if self.episode_ids[i] is None: + self.episode_ids[i] = self.dataset.add_episode(episode) + else: + self.dataset.update_episode(self.episode_ids[i], episode) diff --git a/src/dataset.py b/src/dataset.py new file mode 100644 index 0000000..bd0ce9c --- /dev/null +++ b/src/dataset.py @@ -0,0 +1,163 @@ +from collections import deque +import math +from pathlib import Path +import random +from typing import Dict, List, Optional, Tuple + +import psutil +import torch + +from episode import Episode + +Batch = Dict[str, torch.Tensor] + + +class EpisodesDataset: + def __init__(self, max_num_episodes: Optional[int] = None, name: Optional[str] = None) -> None: + self.max_num_episodes = max_num_episodes + self.name = name if name is not None else 'dataset' + self.num_seen_episodes = 0 + self.episodes = deque() + self.episode_id_to_queue_idx = dict() + self.newly_modified_episodes, self.newly_deleted_episodes = set(), set() + + def __len__(self) -> int: + return len(self.episodes) + + def clear(self) -> None: + self.episodes = deque() + self.episode_id_to_queue_idx = dict() + + def add_episode(self, episode: Episode) -> int: + if self.max_num_episodes is not None and len(self.episodes) == self.max_num_episodes: + self._popleft() + episode_id = self._append_new_episode(episode) + return episode_id + + def get_episode(self, episode_id: int) -> Episode: + assert episode_id in self.episode_id_to_queue_idx + queue_idx = self.episode_id_to_queue_idx[episode_id] + return self.episodes[queue_idx] + + def update_episode(self, episode_id: int, new_episode: Episode) -> None: + assert episode_id in self.episode_id_to_queue_idx + queue_idx = self.episode_id_to_queue_idx[episode_id] + merged_episode = self.episodes[queue_idx].merge(new_episode) + self.episodes[queue_idx] = merged_episode + self.newly_modified_episodes.add(episode_id) + + def _popleft(self) -> Episode: + id_to_delete = [k for k, v in self.episode_id_to_queue_idx.items() if v == 0] + assert len(id_to_delete) == 1 + self.newly_deleted_episodes.add(id_to_delete[0]) + self.episode_id_to_queue_idx = {k: v - 1 for k, v in self.episode_id_to_queue_idx.items() if v > 0} + return self.episodes.popleft() + + def _append_new_episode(self, episode): + episode_id = self.num_seen_episodes + self.episode_id_to_queue_idx[episode_id] = len(self.episodes) + self.episodes.append(episode) + self.num_seen_episodes += 1 + self.newly_modified_episodes.add(episode_id) + return episode_id + + def sample_batch(self, batch_num_samples: int, sequence_length: int, weights: Optional[Tuple[float]] = None, sample_from_start: bool = True) -> Batch: + return self._collate_episodes_segments(self._sample_episodes_segments(batch_num_samples, sequence_length, weights, sample_from_start)) + + def _sample_episodes_segments(self, batch_num_samples: int, sequence_length: int, weights: Optional[Tuple[float]], sample_from_start: bool) -> List[Episode]: + num_episodes = len(self.episodes) + num_weights = len(weights) if weights is not None else 0 + + if num_weights < num_episodes: + weights = [1] * num_episodes + else: + assert all([0 <= x <= 1 for x in weights]) and sum(weights) == 1 + sizes = [num_episodes // num_weights + (num_episodes % num_weights) * (i == num_weights - 1) for i in range(num_weights)] + weights = [w / s for (w, s) in zip(weights, sizes) for _ in range(s)] + + sampled_episodes = random.choices(self.episodes, k=batch_num_samples, weights=weights) + + sampled_episodes_segments = [] + for sampled_episode in sampled_episodes: + if sample_from_start: + start = random.randint(0, len(sampled_episode) - 1) + stop = start + sequence_length + else: + stop = random.randint(1, len(sampled_episode)) + start = stop - sequence_length + sampled_episodes_segments.append(sampled_episode.segment(start, stop, should_pad=True)) + assert len(sampled_episodes_segments[-1]) == sequence_length + return sampled_episodes_segments + + def _collate_episodes_segments(self, episodes_segments: List[Episode]) -> Batch: + episodes_segments = [e_s.__dict__ for e_s in episodes_segments] + batch = {} + for k in episodes_segments[0]: + batch[k] = torch.stack([e_s[k] for e_s in episodes_segments]) + batch['observations'] = batch['observations'].float() / 255.0 # int8 to float and scale + return batch + + def traverse(self, batch_num_samples: int, chunk_size: int): + for episode in self.episodes: + chunks = [episode.segment(start=i * chunk_size, stop=(i + 1) * chunk_size, should_pad=True) for i in range(math.ceil(len(episode) / chunk_size))] + batches = [chunks[i * batch_num_samples: (i + 1) * batch_num_samples] for i in range(math.ceil(len(chunks) / batch_num_samples))] + for b in batches: + yield self._collate_episodes_segments(b) + + def update_disk_checkpoint(self, directory: Path) -> None: + assert directory.is_dir() + for episode_id in self.newly_modified_episodes: + episode = self.get_episode(episode_id) + episode.save(directory / f'{episode_id}.pt') + for episode_id in self.newly_deleted_episodes: + (directory / f'{episode_id}.pt').unlink() + self.newly_modified_episodes, self.newly_deleted_episodes = set(), set() + + def load_disk_checkpoint(self, directory: Path) -> None: + assert directory.is_dir() and len(self.episodes) == 0 + episode_ids = sorted([int(p.stem) for p in directory.iterdir()]) + self.num_seen_episodes = episode_ids[-1] + 1 + for episode_id in episode_ids: + episode = Episode(**torch.load(directory / f'{episode_id}.pt')) + self.episode_id_to_queue_idx[episode_id] = len(self.episodes) + self.episodes.append(episode) + + +class EpisodesDatasetRamMonitoring(EpisodesDataset): + """ + Prevent episode dataset from going out of RAM. + Warning: % looks at system wide RAM usage while G looks only at process RAM usage. + """ + def __init__(self, max_ram_usage: str, name: Optional[str] = None) -> None: + super().__init__(max_num_episodes=None, name=name) + self.max_ram_usage = max_ram_usage + self.num_steps = 0 + self.max_num_steps = None + + max_ram_usage = str(max_ram_usage) + if max_ram_usage.endswith('%'): + m = int(max_ram_usage.split('%')[0]) + assert 0 < m < 100 + self.check_ram_usage = lambda: psutil.virtual_memory().percent > m + else: + assert max_ram_usage.endswith('G') + m = float(max_ram_usage.split('G')[0]) + self.check_ram_usage = lambda: psutil.Process().memory_info()[0] / 2 ** 30 > m + + def clear(self) -> None: + super().clear() + self.num_steps = 0 + + def add_episode(self, episode: Episode) -> int: + if self.max_num_steps is None and self.check_ram_usage(): + self.max_num_steps = self.num_steps + self.num_steps += len(episode) + while (self.max_num_steps is not None) and (self.num_steps > self.max_num_steps): + self._popleft() + episode_id = self._append_new_episode(episode) + return episode_id + + def _popleft(self) -> Episode: + episode = super()._popleft() + self.num_steps -= len(episode) + return episode diff --git a/src/envs/__init__.py b/src/envs/__init__.py new file mode 100644 index 0000000..5afe68b --- /dev/null +++ b/src/envs/__init__.py @@ -0,0 +1,4 @@ +from .multi_process_env import MultiProcessEnv +from .wrappers import make_atari, ResizeObsWrapper +from .single_process_env import SingleProcessEnv +from .world_model_env import WorldModelEnv diff --git a/src/envs/done_tracker.py b/src/envs/done_tracker.py new file mode 100644 index 0000000..41ff762 --- /dev/null +++ b/src/envs/done_tracker.py @@ -0,0 +1,27 @@ +import numpy as np + + +class DoneTrackerEnv: + def __init__(self, num_envs: int) -> None: + """Monitor env dones: 0 when not done, 1 when done, 2 when already done.""" + self.num_envs = num_envs + self.done_tracker = None + self.reset_done_tracker() + + def reset_done_tracker(self) -> None: + self.done_tracker = np.zeros(self.num_envs, dtype=np.uint8) + + def update_done_tracker(self, done: np.ndarray) -> None: + self.done_tracker = np.clip(2 * self.done_tracker + done, 0, 2) + + @property + def num_envs_done(self) -> int: + return (self.done_tracker > 0).sum() + + @property + def mask_dones(self) -> np.ndarray: + return np.logical_not(self.done_tracker) + + @property + def mask_new_dones(self) -> np.ndarray: + return np.logical_not(self.done_tracker[self.done_tracker <= 1]) diff --git a/src/envs/multi_process_env.py b/src/envs/multi_process_env.py new file mode 100644 index 0000000..7835870 --- /dev/null +++ b/src/envs/multi_process_env.py @@ -0,0 +1,94 @@ +from dataclasses import astuple, dataclass +from enum import Enum +from multiprocessing import Pipe, Process +from multiprocessing.connection import Connection +from typing import Any, Callable, Iterator, List, Optional, Tuple + +import numpy as np + +from .done_tracker import DoneTrackerEnv + + +class MessageType(Enum): + RESET = 0 + RESET_RETURN = 1 + STEP = 2 + STEP_RETURN = 3 + CLOSE = 4 + + +@dataclass +class Message: + type: MessageType + content: Optional[Any] = None + + def __iter__(self) -> Iterator: + return iter(astuple(self)) + + +def child_env(child_id: int, env_fn: Callable, child_conn: Connection) -> None: + np.random.seed(child_id + np.random.randint(0, 2 ** 31 - 1)) + env = env_fn() + while True: + message_type, content = child_conn.recv() + if message_type == MessageType.RESET: + obs = env.reset() + child_conn.send(Message(MessageType.RESET_RETURN, obs)) + elif message_type == MessageType.STEP: + obs, rew, done, _ = env.step(content) + if done: + obs = env.reset() + child_conn.send(Message(MessageType.STEP_RETURN, (obs, rew, done, None))) + elif message_type == MessageType.CLOSE: + child_conn.close() + return + else: + raise NotImplementedError + + +class MultiProcessEnv(DoneTrackerEnv): + def __init__(self, env_fn: Callable, num_envs: int, should_wait_num_envs_ratio: float) -> None: + super().__init__(num_envs) + self.num_actions = env_fn().env.action_space.n + self.should_wait_num_envs_ratio = should_wait_num_envs_ratio + self.processes, self.parent_conns = [], [] + for child_id in range(num_envs): + parent_conn, child_conn = Pipe() + self.parent_conns.append(parent_conn) + p = Process(target=child_env, args=(child_id, env_fn, child_conn), daemon=True) + self.processes.append(p) + for p in self.processes: + p.start() + + def should_reset(self) -> bool: + return (self.num_envs_done / self.num_envs) >= self.should_wait_num_envs_ratio + + def _receive(self, check_type: Optional[MessageType] = None) -> List[Any]: + messages = [parent_conn.recv() for parent_conn in self.parent_conns] + if check_type is not None: + assert all([m.type == check_type for m in messages]) + return [m.content for m in messages] + + def reset(self) -> np.ndarray: + self.reset_done_tracker() + for parent_conn in self.parent_conns: + parent_conn.send(Message(MessageType.RESET)) + content = self._receive(check_type=MessageType.RESET_RETURN) + return np.stack(content) + + def step(self, actions: np.ndarray) -> Tuple[np.ndarray, np.ndarray, np.ndarray, Any]: + for parent_conn, action in zip(self.parent_conns, actions): + parent_conn.send(Message(MessageType.STEP, action)) + content = self._receive(check_type=MessageType.STEP_RETURN) + obs, rew, done, _ = zip(*content) + done = np.stack(done) + self.update_done_tracker(done) + return np.stack(obs), np.stack(rew), done, None + + def close(self) -> None: + for parent_conn in self.parent_conns: + parent_conn.send(Message(MessageType.CLOSE)) + for p in self.processes: + p.join() + for parent_conn in self.parent_conns: + parent_conn.close() diff --git a/src/envs/single_process_env.py b/src/envs/single_process_env.py new file mode 100644 index 0000000..e10db02 --- /dev/null +++ b/src/envs/single_process_env.py @@ -0,0 +1,32 @@ +from typing import Any, Tuple + +import numpy as np + +from .done_tracker import DoneTrackerEnv + + +class SingleProcessEnv(DoneTrackerEnv): + def __init__(self, env_fn): + super().__init__(num_envs=1) + self.env = env_fn() + self.num_actions = self.env.action_space.n + + def should_reset(self) -> bool: + return self.num_envs_done == 1 + + def reset(self) -> np.ndarray: + self.reset_done_tracker() + obs = self.env.reset() + return obs[None, ...] + + def step(self, action) -> Tuple[np.ndarray, np.ndarray, np.ndarray, Any]: + obs, reward, done, _ = self.env.step(action[0]) # action is supposed to be ndarray (1,) + done = np.array([done]) + self.update_done_tracker(done) + return obs[None, ...], np.array([reward]), done, None + + def render(self) -> None: + self.env.render() + + def close(self) -> None: + self.env.close() diff --git a/src/envs/world_model_env.py b/src/envs/world_model_env.py new file mode 100644 index 0000000..4a65d40 --- /dev/null +++ b/src/envs/world_model_env.py @@ -0,0 +1,113 @@ +import random +from typing import List, Optional, Union + +import gym +from einops import rearrange +import numpy as np +from PIL import Image +import torch +from torch.distributions.categorical import Categorical +import torchvision + +from utils import extract_state_dict + + +class WorldModelEnv: + + def __init__(self, tokenizer: torch.nn.Module, world_model: torch.nn.Module, device: Union[str, torch.device], pretrained_agent_path: Optional[str] = None, env: Optional[gym.Env] = None) -> None: + + self.device = torch.device(device) + self.world_model = world_model.to(self.device).eval() + self.tokenizer = tokenizer.to(self.device).eval() + + if pretrained_agent_path is not None: + agent_state_dict = torch.load(pretrained_agent_path) + self.world_model.load_state_dict(extract_state_dict(agent_state_dict, 'world_model')) + incompatible_keys = self.tokenizer.load_state_dict(extract_state_dict(agent_state_dict, 'tokenizer'), strict=False) + assert not incompatible_keys.missing_keys + assert (not incompatible_keys.unexpected_keys) or all([k.startswith('lpips.') for k in incompatible_keys.unexpected_keys]) + + self.keys_values_wm, self.obs_tokens, self._num_observations_tokens = None, None, None + + self.env = env + + @property + def num_observations_tokens(self) -> int: + return self._num_observations_tokens + + @torch.no_grad() + def reset(self) -> torch.FloatTensor: + assert self.env is not None + obs = torchvision.transforms.functional.to_tensor(self.env.reset()).to(self.device).unsqueeze(0) # (1, C, H, W) in [0., 1.] + return self.reset_from_initial_observations(obs) + + @torch.no_grad() + def reset_from_initial_observations(self, observations: torch.FloatTensor) -> torch.FloatTensor: + obs_tokens = self.tokenizer.encode(observations, should_preprocess=True).tokens # (B, C, H, W) -> (B, K) + _, num_observations_tokens = obs_tokens.shape + if self.num_observations_tokens is None: + self._num_observations_tokens = num_observations_tokens + + _ = self.refresh_keys_values_with_initial_obs_tokens(obs_tokens) + self.obs_tokens = obs_tokens + + return self.decode_obs_tokens() + + @torch.no_grad() + def refresh_keys_values_with_initial_obs_tokens(self, obs_tokens: torch.LongTensor) -> torch.FloatTensor: + n, num_observations_tokens = obs_tokens.shape + assert num_observations_tokens == self.num_observations_tokens + self.keys_values_wm = self.world_model.transformer.generate_empty_keys_values(n=n, max_tokens=self.world_model.config.max_tokens) + outputs_wm = self.world_model(obs_tokens, past_keys_values=self.keys_values_wm) + return outputs_wm.output_sequence # (B, K, E) + + @torch.no_grad() + def step(self, action: Union[int, np.ndarray, torch.LongTensor], should_predict_next_obs: bool = True) -> None: + assert self.keys_values_wm is not None and self.num_observations_tokens is not None + + num_passes = 1 + self.num_observations_tokens if should_predict_next_obs else 1 + + output_sequence, obs_tokens = [], [] + + if self.keys_values_wm.size + num_passes > self.world_model.config.max_tokens: + _ = self.refresh_keys_values_with_initial_obs_tokens(self.obs_tokens) + + token = action.clone().detach() if isinstance(action, torch.Tensor) else torch.tensor(action, dtype=torch.long) + token = token.reshape(-1, 1).to(self.device) # (B, 1) + + for k in range(num_passes): # assumption that there is only one action token. + + outputs_wm = self.world_model(token, past_keys_values=self.keys_values_wm) + output_sequence.append(outputs_wm.output_sequence) + + if k == 0: + reward = Categorical(logits=outputs_wm.logits_rewards).sample().float().cpu().numpy().reshape(-1) - 1 # (B,) + done = Categorical(logits=outputs_wm.logits_ends).sample().cpu().numpy().astype(bool).reshape(-1) # (B,) + + if k < self.num_observations_tokens: + token = Categorical(logits=outputs_wm.logits_observations).sample() + obs_tokens.append(token) + + output_sequence = torch.cat(output_sequence, dim=1) # (B, 1 + K, E) + self.obs_tokens = torch.cat(obs_tokens, dim=1) # (B, K) + + obs = self.decode_obs_tokens() if should_predict_next_obs else None + return obs, reward, done, None + + @torch.no_grad() + def render_batch(self) -> List[Image.Image]: + frames = self.decode_obs_tokens().detach().cpu() + frames = rearrange(frames, 'b c h w -> b h w c').mul(255).numpy().astype(np.uint8) + return [Image.fromarray(frame) for frame in frames] + + @torch.no_grad() + def decode_obs_tokens(self) -> List[Image.Image]: + embedded_tokens = self.tokenizer.embedding(self.obs_tokens) # (B, K, E) + z = rearrange(embedded_tokens, 'b (h w) e -> b e h w', h=int(np.sqrt(self.num_observations_tokens))) + rec = self.tokenizer.decode(z, should_postprocess=True) # (B, C, H, W) + return torch.clamp(rec, 0, 1) + + @torch.no_grad() + def render(self): + assert self.obs_tokens.shape == (1, self.num_observations_tokens) + return self.render_batch()[0] diff --git a/src/envs/wrappers.py b/src/envs/wrappers.py new file mode 100644 index 0000000..5127e59 --- /dev/null +++ b/src/envs/wrappers.py @@ -0,0 +1,147 @@ +""" +Credits to https://github.com/openai/baselines/blob/master/baselines/common/atari_wrappers.py +""" + +from typing import Tuple + +import gym +import numpy as np +from PIL import Image + + +def make_atari(id, size=64, max_episode_steps=None, noop_max=30, frame_skip=4, done_on_life_loss=False, clip_reward=False): + env = gym.make(id) + assert 'NoFrameskip' in env.spec.id or 'Frameskip' not in env.spec + env = ResizeObsWrapper(env, (size, size)) + if clip_reward: + env = RewardClippingWrapper(env) + if max_episode_steps is not None: + env = gym.wrappers.TimeLimit(env, max_episode_steps=max_episode_steps) + if noop_max is not None: + env = NoopResetEnv(env, noop_max=noop_max) + env = MaxAndSkipEnv(env, skip=frame_skip) + if done_on_life_loss: + env = EpisodicLifeEnv(env) + return env + + +class ResizeObsWrapper(gym.ObservationWrapper): + def __init__(self, env: gym.Env, size: Tuple[int, int]) -> None: + gym.ObservationWrapper.__init__(self, env) + self.size = tuple(size) + self.observation_space = gym.spaces.Box(low=0, high=255, shape=(size[0], size[1], 3), dtype=np.uint8) + self.original_obs = None + + def resize(self, obs: np.ndarray): + img = Image.fromarray(obs) + img = img.resize(self.size, Image.BILINEAR) + return np.array(img) + + def observation(self, observation: np.ndarray) -> np.ndarray: + self.original_obs = observation + return self.resize(observation) + + +class RewardClippingWrapper(gym.RewardWrapper): + def reward(self, reward): + return np.sign(reward) + + +class NoopResetEnv(gym.Wrapper): + def __init__(self, env, noop_max=30): + """Sample initial states by taking random number of no-ops on reset. + No-op is assumed to be action 0. + """ + gym.Wrapper.__init__(self, env) + self.noop_max = noop_max + self.override_num_noops = None + self.noop_action = 0 + assert env.unwrapped.get_action_meanings()[0] == 'NOOP' + + def reset(self, **kwargs): + """ Do no-op action for a number of steps in [1, noop_max].""" + self.env.reset(**kwargs) + if self.override_num_noops is not None: + noops = self.override_num_noops + else: + noops = self.unwrapped.np_random.randint(1, self.noop_max + 1) + assert noops > 0 + obs = None + for _ in range(noops): + obs, _, done, _ = self.env.step(self.noop_action) + if done: + obs = self.env.reset(**kwargs) + return obs + + def step(self, action): + return self.env.step(action) + + +class EpisodicLifeEnv(gym.Wrapper): + def __init__(self, env): + """Make end-of-life == end-of-episode, but only reset on true game over. + Done by DeepMind for the DQN and co. since it helps value estimation. + """ + gym.Wrapper.__init__(self, env) + self.lives = 0 + self.was_real_done = True + + def step(self, action): + obs, reward, done, info = self.env.step(action) + self.was_real_done = done + # check current lives, make loss of life terminal, + # then update lives to handle bonus lives + lives = self.env.unwrapped.ale.lives() + if lives < self.lives and lives > 0: + # for Qbert sometimes we stay in lives == 0 condition for a few frames + # so it's important to keep lives > 0, so that we only reset once + # the environment advertises done. + done = True + self.lives = lives + return obs, reward, done, info + + def reset(self, **kwargs): + """Reset only when lives are exhausted. + This way all states are still reachable even though lives are episodic, + and the learner need not know about any of this behind-the-scenes. + """ + if self.was_real_done: + obs = self.env.reset(**kwargs) + else: + # no-op step to advance from terminal/lost life state + obs, _, _, _ = self.env.step(0) + self.lives = self.env.unwrapped.ale.lives() + return obs + + +class MaxAndSkipEnv(gym.Wrapper): + def __init__(self, env, skip=4): + """Return only every `skip`-th frame""" + gym.Wrapper.__init__(self, env) + assert skip > 0 + # most recent raw observations (for max pooling across time steps) + self._obs_buffer = np.zeros((2,) + env.observation_space.shape, dtype=np.uint8) + self._skip = skip + self.max_frame = np.zeros(env.observation_space.shape, dtype=np.uint8) + + def step(self, action): + """Repeat action, sum reward, and max over last observations.""" + total_reward = 0.0 + done = None + for i in range(self._skip): + obs, reward, done, info = self.env.step(action) + if i == self._skip - 2: + self._obs_buffer[0] = obs + if i == self._skip - 1: + self._obs_buffer[1] = obs + total_reward += reward + if done: + break + # Note that the observation on the done=True frame + # doesn't matter + self.max_frame = self._obs_buffer.max(axis=0) + + return self.max_frame, total_reward, done, info + + def reset(self, **kwargs): + return self.env.reset(**kwargs) diff --git a/src/episode.py b/src/episode.py new file mode 100644 index 0000000..43da86c --- /dev/null +++ b/src/episode.py @@ -0,0 +1,76 @@ +from __future__ import annotations +from dataclasses import dataclass +from pathlib import Path + +import torch + + +@dataclass +class EpisodeMetrics: + episode_length: int + episode_return: float + + +@dataclass +class Episode: + observations: torch.ByteTensor + actions: torch.LongTensor + rewards: torch.FloatTensor + ends: torch.LongTensor + mask_padding: torch.BoolTensor + + def __post_init__(self): + assert len(self.observations) == len(self.actions) == len(self.rewards) == len(self.ends) == len(self.mask_padding) + if self.ends.sum() > 0: + idx_end = torch.argmax(self.ends) + 1 + self.observations = self.observations[:idx_end] + self.actions = self.actions[:idx_end] + self.rewards = self.rewards[:idx_end] + self.ends = self.ends[:idx_end] + self.mask_padding = self.mask_padding[:idx_end] + + def __len__(self) -> int: + return self.observations.size(0) + + def merge(self, other: Episode) -> Episode: + return Episode( + torch.cat((self.observations, other.observations), dim=0), + torch.cat((self.actions, other.actions), dim=0), + torch.cat((self.rewards, other.rewards), dim=0), + torch.cat((self.ends, other.ends), dim=0), + torch.cat((self.mask_padding, other.mask_padding), dim=0), + ) + + def segment(self, start: int, stop: int, should_pad: bool = False) -> Episode: + assert start < len(self) and stop > 0 and start < stop + padding_length_right = max(0, stop - len(self)) + padding_length_left = max(0, -start) + assert padding_length_right == padding_length_left == 0 or should_pad + + def pad(x): + pad_right = torch.nn.functional.pad(x, [0 for _ in range(2 * x.ndim - 1)] + [padding_length_right]) if padding_length_right > 0 else x + return torch.nn.functional.pad(pad_right, [0 for _ in range(2 * x.ndim - 2)] + [padding_length_left, 0]) if padding_length_left > 0 else pad_right + + start = max(0, start) + stop = min(len(self), stop) + segment = Episode( + self.observations[start:stop], + self.actions[start:stop], + self.rewards[start:stop], + self.ends[start:stop], + self.mask_padding[start:stop], + ) + + segment.observations = pad(segment.observations) + segment.actions = pad(segment.actions) + segment.rewards = pad(segment.rewards) + segment.ends = pad(segment.ends) + segment.mask_padding = torch.cat((torch.zeros(padding_length_left, dtype=torch.bool), segment.mask_padding, torch.zeros(padding_length_right, dtype=torch.bool)), dim=0) + + return segment + + def compute_metrics(self) -> EpisodeMetrics: + return EpisodeMetrics(len(self), self.rewards.sum()) + + def save(self, path: Path) -> None: + torch.save(self.__dict__, path) diff --git a/src/game/__init__.py b/src/game/__init__.py new file mode 100644 index 0000000..32a30d7 --- /dev/null +++ b/src/game/__init__.py @@ -0,0 +1,3 @@ +from .agent_env import AgentEnv +from .episode_replay_env import EpisodeReplayEnv +from .game import Game diff --git a/src/game/agent_env.py b/src/game/agent_env.py new file mode 100644 index 0000000..26aa283 --- /dev/null +++ b/src/game/agent_env.py @@ -0,0 +1,54 @@ +from einops import rearrange +import numpy as np +from PIL import Image +import torch + +from agent import Agent +from envs import SingleProcessEnv +from game.keymap import get_keymap_and_action_names + + +class AgentEnv: + def __init__(self, agent: Agent, env: SingleProcessEnv, keymap_name: str) -> None: + assert isinstance(env, SingleProcessEnv) + self.agent = agent + self.env = env + _, self.action_names = get_keymap_and_action_names(keymap_name) + self.obs = None + self._t = None + self._return = None + + def _to_tensor(self, obs: np.ndarray): + return rearrange(torch.FloatTensor(obs).div(255), 'n h w c -> n c h w').to(self.agent.device) + + def _to_array(self, obs: torch.FloatTensor): + assert obs.ndim == 4 and obs.size(0) == 1 + return obs[0].mul(255).permute(1, 2, 0).cpu().numpy().astype(np.uint8) + + def reset(self): + obs = self.env.reset() + self.obs = self._to_tensor(obs) + self.agent.actor_critic.reset(1) + self._t = 0 + self._return = 0 + return obs + + def step(self, *args, **kwargs) -> torch.FloatTensor: + with torch.no_grad(): + act = self.agent.act(self.obs, should_sample=True).cpu().numpy() + obs, reward, done, _ = self.env.step(act) + self.obs = self._to_tensor(obs) + self._t += 1 + self._return += reward[0] + info = { + 'timestep': self._t, + 'action': self.action_names[act[0]], + 'return': self._return, + } + return obs, reward, done, info + + def render(self) -> Image.Image: + assert self.obs.size() == (1, 3, 64, 64) + rec = torch.clamp(self.agent.tokenizer.encode_decode(self.obs, should_preprocess=True, should_postprocess=True), 0, 1) + arr = self._to_array(torch.cat((self.obs, rec), dim=-1)) + return Image.fromarray(arr) diff --git a/src/game/episode_replay_env.py b/src/game/episode_replay_env.py new file mode 100644 index 0000000..58bf4c4 --- /dev/null +++ b/src/game/episode_replay_env.py @@ -0,0 +1,111 @@ +from pathlib import Path + +import numpy as np +from PIL import Image +import torch + +from episode import Episode +from game.keymap import get_keymap_and_action_names + + +class EpisodeReplayEnv: + def __init__(self, replay_keymap_name: str, episode_dir: Path) -> None: + _, self.action_names = get_keymap_and_action_names(replay_keymap_name) + assert episode_dir.is_dir() + self._paths = {} + for mode in ['train', 'test', 'imagination']: + directory = episode_dir / mode + if directory.is_dir(): + self._paths[mode] = sorted([p for p in directory.iterdir() if 'episode_' in p.stem and p.suffix == '.pt']) + print(f'Found {len(self._paths[mode])} {mode} episodes.') + else: + print(f'No {mode} episodes.') + + self._t, self._episode = None, None + self._ep_idx = 0 + self._mode = 'train' + self.load() + + def load(self): + self._episode = Episode(**torch.load(self.paths[self._ep_idx])) + self._t = 0 + + def load_next(self): + self._ep_idx = (self._ep_idx + 1) % len(self.paths) + self.load() + + def load_previous(self): + self._ep_idx = (self._ep_idx - 1) % len(self.paths) + self.load() + + def set_mode(self, mode): + assert mode in ['train', 'test', 'imagination'] + if mode in self._paths: + self._mode = mode + self._ep_idx = 0 + self.load() + else: + print(f'No {mode} episodes.') + + def __len__(self): + return len(self.ends) + + @property + def paths(self): + return self._paths[self._mode] + + @property + def observations(self): + return self._episode.observations + + @property + def actions(self): + return self._episode.actions + + @property + def rewards(self): + return self._episode.rewards + + @property + def ends(self): + return self._episode.ends + + def reset(self): + return self.observations[self._t] + + def step(self, action) -> torch.FloatTensor: + if action == 1: + self._t = (self._t - 1) % len(self) + elif action == 2: + self._t = (self._t + 1) % len(self) + if action == 3: + self._t = (self._t - 10) % len(self) + elif action == 4: + self._t = (self._t + 10) % len(self) + elif action == 5: + self._t = 0 + elif action == 6: + self.load_previous() + elif action == 7: + self.load_next() + elif action == 8: + self.set_mode('train') + elif action == 9: + self.set_mode('test') + elif action == 10: + self.set_mode('imagination') + act = self.actions[self._t] + reward = self.rewards[self._t].item() + done = self.ends[self._t].item() + info = { + 'ep_name': f'[{self._mode}] {self.paths[self._ep_idx].stem}', + 'timestep': self._t, + 'action': self.action_names[act], + 'cum_reward': f'{sum(self.rewards[:self._t + 1]):.3f}' + } + return self.observations[self._t], reward, done, info + + def render(self) -> Image.Image: + obs = self.observations[self._t] # (C, H, W) in [0., 1.] + arr = obs.permute(1, 2, 0).numpy().astype(np.uint8) + return Image.fromarray(arr) diff --git a/src/game/game.py b/src/game/game.py new file mode 100644 index 0000000..f806d97 --- /dev/null +++ b/src/game/game.py @@ -0,0 +1,111 @@ +from typing import Tuple, Union + +import gym +import numpy as np +import pygame +from PIL import Image + +from envs import WorldModelEnv +from game.keymap import get_keymap_and_action_names + + +class Game: + def __init__(self, env: Union[gym.Env, WorldModelEnv], keymap_name: str, size: Tuple[int, int], fps: int, verbose: bool) -> None: + self.env = env + self.height, self.width = size + self.fps = fps + self.verbose = verbose + + self.keymap, self.action_names = get_keymap_and_action_names(keymap_name) + + print('Actions:') + for key, idx in self.keymap.items(): + print(f'{pygame.key.name(key)}: {self.action_names[idx]}') + + def run(self) -> None: + pygame.init() + + header_height = 100 if self.verbose else 0 + font_size = 24 + screen = pygame.display.set_mode((self.width, self.height + header_height)) + clock = pygame.time.Clock() + font = pygame.font.SysFont(None, font_size) + header_rect = pygame.Rect(0, 0, self.width, header_height) + + def clear_header(): + pygame.draw.rect(screen, pygame.Color('black'), header_rect) + pygame.draw.rect(screen, pygame.Color('white'), header_rect, 1) + + def draw_text(text, idx_line, idx_column=0): + pos = (5 + idx_column * int(self.width // 4), 5 + idx_line * font_size) + assert (0 <= pos[0] <= self.width) and (0 <= pos[1] <= header_height) + screen.blit(font.render(text, True, pygame.Color('white')), pos) + + def draw_game(image): + if isinstance(image, np.ndarray): + image = Image.fromarray(image) + else: + assert isinstance(image, Image.Image) + pygame_image = np.array(image.resize((self.width, self.height), resample=Image.NEAREST)).transpose((1, 0, 2)) + surface = pygame.surfarray.make_surface(pygame_image) + screen.blit(surface, (0, header_height)) + + if isinstance(self.env, gym.Env): + _, info = self.env.reset(return_info=True) + draw_game(info['rgb']) + else: + self.env.reset() + draw_game(self.env.render()) + + clear_header() + pygame.display.flip() + + do_reset, do_wait = False, False + should_stop = False + while not should_stop: + + action = 0 # noop + pygame.event.pump() + for event in pygame.event.get(): + if event.type == pygame.QUIT: + should_stop = True + if event.type == pygame.KEYDOWN and event.key in self.keymap.keys(): + action = self.keymap[event.key] + if event.type == pygame.KEYDOWN and event.key == pygame.K_RETURN: + do_reset = True + if event.type == pygame.KEYDOWN and event.key == pygame.K_PERIOD: + do_wait = not do_wait + if action == 0: + pressed = pygame.key.get_pressed() + for key, action in self.keymap.items(): + if pressed[key]: + break + else: + action = 0 + + if do_wait: + continue + + _, reward, done, info = self.env.step(action) + + img = info['rgb'] if isinstance(self.env, gym.Env) else self.env.render() + draw_game(img) + + if self.verbose: + clear_header() + draw_text(f'Action: {self.action_names[action]}', idx_line=0) + draw_text(f'Reward: {reward if isinstance(reward, float) else reward.item(): .2f}', idx_line=1) + draw_text(f'Done: {done}', idx_line=2) + if info is not None: + assert isinstance(info, dict) + for i, (k, v) in enumerate(info.items()): + draw_text(f'{k}: {v}', idx_line=i, idx_column=1) + + pygame.display.flip() # update screen + clock.tick(self.fps) # ensures game maintains the given frame rate + + if do_reset or done: + self.env.reset() + do_reset = False + + pygame.quit() diff --git a/src/game/keymap.py b/src/game/keymap.py new file mode 100644 index 0000000..62aadc7 --- /dev/null +++ b/src/game/keymap.py @@ -0,0 +1,103 @@ +import gym +import pygame + + +def get_keymap_and_action_names(name): + + if name == 'empty': + return EMPTY_KEYMAP, EMPTY_ACTION_NAMES + + if name == 'episode_replay': + return EPISODE_REPLAY_KEYMAP, EPISODE_REPLAY_ACTION_NAMES + + if name == 'atari': + return ATARI_KEYMAP, ATARI_ACTION_NAMES + + assert name.startswith('atari/') + env_id = name.split('atari/')[1] + action_names = [x.lower() for x in gym.make(env_id).get_action_meanings()] + keymap = {} + for key, value in ATARI_KEYMAP.items(): + if ATARI_ACTION_NAMES[value] in action_names: + keymap[key] = action_names.index(ATARI_ACTION_NAMES[value]) + return keymap, action_names + + +ATARI_ACTION_NAMES = [ + 'noop', + 'fire', + 'up', + 'right', + 'left', + 'down', + 'upright', + 'upleft', + 'downright', + 'downleft', + 'upfire', + 'rightfire', + 'leftfire', + 'downfire', + 'uprightfire', + 'upleftfire', + 'downrightfire', + 'downleftfire', +] + +ATARI_KEYMAP = { + pygame.K_SPACE: 1, + + pygame.K_w: 2, + pygame.K_d: 3, + pygame.K_a: 4, + pygame.K_s: 5, + + pygame.K_t: 6, + pygame.K_r: 7, + pygame.K_g: 8, + pygame.K_f: 9, + + pygame.K_UP: 10, + pygame.K_RIGHT: 11, + pygame.K_LEFT: 12, + pygame.K_DOWN: 13, + + pygame.K_u: 14, + pygame.K_y: 15, + pygame.K_j: 16, + pygame.K_h: 17, +} + +EPISODE_REPLAY_ACTION_NAMES = [ + 'noop', + 'previous', + 'next', + 'previous_10', + 'next_10', + 'go_to_start', + 'load_previous', + 'load_next', + 'go_to_train_episodes', + 'go_to_test_episodes', + 'go_to_imagination_episodes', +] + +EPISODE_REPLAY_KEYMAP = { + pygame.K_LEFT: 1, + pygame.K_RIGHT: 2, + pygame.K_PAGEDOWN: 3, + pygame.K_PAGEUP: 4, + pygame.K_SPACE: 5, + pygame.K_DOWN: 6, + pygame.K_UP: 7, + pygame.K_t: 8, + pygame.K_y: 9, + pygame.K_i: 10, +} + +EMPTY_ACTION_NAMES = [ + 'noop', +] + +EMPTY_KEYMAP = { +} \ No newline at end of file diff --git a/src/main.py b/src/main.py new file mode 100644 index 0000000..be4177b --- /dev/null +++ b/src/main.py @@ -0,0 +1,14 @@ +import hydra +from omegaconf import DictConfig + +from trainer import Trainer + + +@hydra.main(config_path="../config", config_name="trainer") +def main(cfg: DictConfig): + trainer = Trainer(cfg) + trainer.run() + + +if __name__ == "__main__": + main() diff --git a/src/make_reconstructions.py b/src/make_reconstructions.py new file mode 100644 index 0000000..9e66422 --- /dev/null +++ b/src/make_reconstructions.py @@ -0,0 +1,53 @@ +from einops import rearrange +import numpy as np +from PIL import Image +import torch + + +@torch.no_grad() +def make_reconstructions_from_batch(batch, save_dir, epoch, tokenizer): + check_batch(batch) + + original_frames = tensor_to_np_frames(rearrange(batch['observations'], 'b t c h w -> b t h w c')) + all = [original_frames] + + rec_frames = generate_reconstructions_with_tokenizer(batch, tokenizer) + all.append(rec_frames) + + for i, image in enumerate(map(Image.fromarray, np.concatenate(list(np.concatenate((original_frames, rec_frames), axis=-2)), axis=-3))): + image.save(save_dir / f'epoch_{epoch:03d}_t_{i:03d}.png') + + return + + +def check_batch(batch): + assert sorted(batch.keys()) == ['actions', 'ends', 'mask_padding', 'observations', 'rewards'] + b, t, _, _, _ = batch['observations'].shape # (B, T, C, H, W) + assert batch['actions'].shape == batch['rewards'].shape == batch['ends'].shape == batch['mask_padding'].shape == (b, t) + + +def tensor_to_np_frames(inputs): + check_float_btw_0_1(inputs) + return inputs.mul(255).cpu().numpy().astype(np.uint8) + + +def check_float_btw_0_1(inputs): + assert inputs.is_floating_point() and (inputs >= 0).all() and (inputs <= 1).all() + + +@torch.no_grad() +def generate_reconstructions_with_tokenizer(batch, tokenizer): + check_batch(batch) + inputs = rearrange(batch['observations'], 'b t c h w -> (b t) c h w') + outputs = reconstruct_through_tokenizer(inputs, tokenizer) + b, t, _, _, _ = batch['observations'].size() + outputs = rearrange(outputs, '(b t) c h w -> b t h w c', b=b, t=t) + rec_frames = tensor_to_np_frames(outputs) + return rec_frames + + +@torch.no_grad() +def reconstruct_through_tokenizer(inputs, tokenizer): + check_float_btw_0_1(inputs) + reconstructions = tokenizer.encode_decode(inputs, should_preprocess=True, should_postprocess=True) + return torch.clamp(reconstructions, 0, 1) diff --git a/src/models/__init__.py b/src/models/__init__.py new file mode 100644 index 0000000..c1d02cb --- /dev/null +++ b/src/models/__init__.py @@ -0,0 +1 @@ +from .transformer import Transformer, TransformerConfig diff --git a/src/models/actor_critic.py b/src/models/actor_critic.py new file mode 100644 index 0000000..c22e641 --- /dev/null +++ b/src/models/actor_critic.py @@ -0,0 +1,166 @@ +from dataclasses import dataclass +from typing import Any, Optional, Union +import sys + +from einops import rearrange +import numpy as np +import torch +from torch.distributions.categorical import Categorical +import torch.nn as nn +import torch.nn.functional as F +from tqdm import tqdm + +from dataset import Batch +from envs.world_model_env import WorldModelEnv +from models.tokenizer import Tokenizer +from models.world_model import WorldModel +from utils import compute_lambda_returns, LossWithIntermediateLosses + + +@dataclass +class ActorCriticOutput: + logits_actions: torch.FloatTensor + means_values: torch.FloatTensor + + +@dataclass +class ImagineOutput: + observations: torch.ByteTensor + actions: torch.LongTensor + logits_actions: torch.FloatTensor + values: torch.FloatTensor + rewards: torch.FloatTensor + ends: torch.BoolTensor + + +class ActorCritic(nn.Module): + def __init__(self, act_vocab_size, use_original_obs: bool = False) -> None: + super().__init__() + self.use_original_obs = use_original_obs + self.conv1 = nn.Conv2d(3, 32, 3, stride=1, padding=1) + self.maxp1 = nn.MaxPool2d(2, 2) + self.conv2 = nn.Conv2d(32, 32, 3, stride=1, padding=1) + self.maxp2 = nn.MaxPool2d(2, 2) + self.conv3 = nn.Conv2d(32, 64, 3, stride=1, padding=1) + self.maxp3 = nn.MaxPool2d(2, 2) + self.conv4 = nn.Conv2d(64, 64, 3, stride=1, padding=1) + self.maxp4 = nn.MaxPool2d(2, 2) + + self.lstm_dim = 512 + self.lstm = nn.LSTMCell(1024, self.lstm_dim) + self.hx, self.cx = None, None + + self.critic_linear = nn.Linear(512, 1) + self.actor_linear = nn.Linear(512, act_vocab_size) + + def __repr__(self) -> str: + return "actor_critic" + + def clear(self) -> None: + self.hx, self.cx = None, None + + def reset(self, n: int, burnin_observations: Optional[torch.Tensor] = None, mask_padding: Optional[torch.Tensor] = None) -> None: + device = self.conv1.weight.device + self.hx = torch.zeros(n, self.lstm_dim, device=device) + self.cx = torch.zeros(n, self.lstm_dim, device=device) + if burnin_observations is not None: + assert burnin_observations.ndim == 5 and burnin_observations.size(0) == n and mask_padding is not None and burnin_observations.shape[:2] == mask_padding.shape + for i in range(burnin_observations.size(1)): + if mask_padding[:, i].any(): + with torch.no_grad(): + self(burnin_observations[:, i], mask_padding[:, i]) + + def prune(self, mask: np.ndarray) -> None: + self.hx = self.hx[mask] + self.cx = self.cx[mask] + + def forward(self, inputs: torch.FloatTensor, mask_padding: Optional[torch.BoolTensor] = None) -> ActorCriticOutput: + assert inputs.ndim == 4 and inputs.shape[1:] == (3, 64, 64) + assert 0 <= inputs.min() <= 1 and 0 <= inputs.max() <= 1 + assert mask_padding is None or (mask_padding.ndim == 1 and mask_padding.size(0) == inputs.size(0) and mask_padding.any()) + x = inputs[mask_padding] if mask_padding is not None else inputs + + x = x.mul(2).sub(1) + x = F.relu(self.maxp1(self.conv1(x))) + x = F.relu(self.maxp2(self.conv2(x))) + x = F.relu(self.maxp3(self.conv3(x))) + x = F.relu(self.maxp4(self.conv4(x))) + x = torch.flatten(x, start_dim=1) + + if mask_padding is None: + self.hx, self.cx = self.lstm(x, (self.hx, self.cx)) + else: + self.hx[mask_padding], self.cx[mask_padding] = self.lstm(x, (self.hx[mask_padding], self.cx[mask_padding])) + + logits_actions = rearrange(self.actor_linear(self.hx), 'b a -> b 1 a') + means_values = rearrange(self.critic_linear(self.hx), 'b 1 -> b 1 1') + + return ActorCriticOutput(logits_actions, means_values) + + def compute_loss(self, batch: Batch, tokenizer: Tokenizer, world_model: WorldModel, imagine_horizon: int, gamma: float, lambda_: float, entropy_weight: float, **kwargs: Any) -> LossWithIntermediateLosses: + assert not self.use_original_obs + outputs = self.imagine(batch, tokenizer, world_model, horizon=imagine_horizon) + + with torch.no_grad(): + lambda_returns = compute_lambda_returns( + rewards=outputs.rewards, + values=outputs.values, + ends=outputs.ends, + gamma=gamma, + lambda_=lambda_, + )[:, :-1] + + values = outputs.values[:, :-1] + + d = Categorical(logits=outputs.logits_actions[:, :-1]) + log_probs = d.log_prob(outputs.actions[:, :-1]) + loss_actions = -1 * (log_probs * (lambda_returns - values.detach())).mean() + loss_entropy = - entropy_weight * d.entropy().mean() + loss_values = F.mse_loss(values, lambda_returns) + + return LossWithIntermediateLosses(loss_actions=loss_actions, loss_values=loss_values, loss_entropy=loss_entropy) + + def imagine(self, batch: Batch, tokenizer: Tokenizer, world_model: WorldModel, horizon: int, show_pbar: bool = False) -> ImagineOutput: + assert not self.use_original_obs + initial_observations = batch['observations'] + mask_padding = batch['mask_padding'] + assert initial_observations.ndim == 5 and initial_observations.shape[2:] == (3, 64, 64) + assert mask_padding[:, -1].all() + device = initial_observations.device + wm_env = WorldModelEnv(tokenizer, world_model, device) + + all_actions = [] + all_logits_actions = [] + all_values = [] + all_rewards = [] + all_ends = [] + all_observations = [] + + burnin_observations = torch.clamp(tokenizer.encode_decode(initial_observations[:, :-1], should_preprocess=True, should_postprocess=True), 0, 1) if initial_observations.size(1) > 1 else None + self.reset(n=initial_observations.size(0), burnin_observations=burnin_observations, mask_padding=mask_padding[:, :-1]) + + obs = wm_env.reset_from_initial_observations(initial_observations[:, -1]) + for k in tqdm(range(horizon), disable=not show_pbar, desc='Imagination', file=sys.stdout): + + all_observations.append(obs) + + outputs_ac = self(obs) + action_token = Categorical(logits=outputs_ac.logits_actions).sample() + obs, reward, done, _ = wm_env.step(action_token, should_predict_next_obs=(k < horizon - 1)) + + all_actions.append(action_token) + all_logits_actions.append(outputs_ac.logits_actions) + all_values.append(outputs_ac.means_values) + all_rewards.append(torch.tensor(reward).reshape(-1, 1)) + all_ends.append(torch.tensor(done).reshape(-1, 1)) + + self.clear() + + return ImagineOutput( + observations=torch.stack(all_observations, dim=1).mul(255).byte(), # (B, T, C, H, W) in [0, 255] + actions=torch.cat(all_actions, dim=1), # (B, T) + logits_actions=torch.cat(all_logits_actions, dim=1), # (B, T, #actions) + values=rearrange(torch.cat(all_values, dim=1), 'b t 1 -> b t'), # (B, T) + rewards=torch.cat(all_rewards, dim=1).to(device), # (B, T) + ends=torch.cat(all_ends, dim=1).to(device), # (B, T) + ) diff --git a/src/models/kv_caching.py b/src/models/kv_caching.py new file mode 100644 index 0000000..a4ae85e --- /dev/null +++ b/src/models/kv_caching.py @@ -0,0 +1,106 @@ +from typing import Tuple + +import numpy as np +import torch + + +class Cache: + def __init__(self, num_samples: int, num_heads: int, max_tokens: int, embed_dim: int, device: torch.device) -> None: + assert embed_dim % num_heads == 0 + self._n, self._cache, self._size = num_samples, None, None + self._reset = lambda n: torch.empty(n, num_heads, max_tokens, embed_dim // num_heads, device=device) # (B, nh, T, hs) + self.reset() + + @property + def shape(self) -> Tuple[int, int, int, int]: + n, num_heads, _, head_dim = self._cache.shape + return n, num_heads, self._size, head_dim + + def reset(self) -> None: + self._cache = self._reset(self._n) + self._size = 0 + + def prune(self, mask: np.ndarray) -> None: + assert mask.ndim == 1 and mask.shape[0] == self.shape[0] + self._cache = self._cache[mask] + self._n = self._cache.shape[0] + + def get(self) -> torch.Tensor: + return self._cache[:, :, :self._size, :] + + def update(self, x: torch.Tensor) -> None: + assert (x.ndim == self._cache.ndim) and all([x.size(i) == self._cache.size(i) for i in (0, 1, 3)]) + assert self._size + x.size(2) <= self._cache.shape[2] + self._cache = AssignWithoutInplaceCheck.apply(self._cache, x, 2, self._size, self._size + x.size(2)) + self._size += x.size(2) + + +class KVCache: + def __init__(self, n: int, num_heads: int, max_tokens: int, embed_dim: int, device: torch.device) -> None: + self._k_cache = Cache(n, num_heads, max_tokens, embed_dim, device) + self._v_cache = Cache(n, num_heads, max_tokens, embed_dim, device) + + @property + def shape(self) -> Tuple[int, int, int, int]: + return self._k_cache.shape + + def reset(self) -> None: + self._k_cache.reset() + self._v_cache.reset() + + def prune(self, mask: np.ndarray) -> None: + self._k_cache.prune(mask) + self._v_cache.prune(mask) + + def get(self) -> Tuple[torch.Tensor, torch.Tensor]: + return self._k_cache.get(), self._v_cache.get() + + def update(self, k: torch.Tensor, v: torch.Tensor): + self._k_cache.update(k) + self._v_cache.update(v) + + +class KeysValues: + def __init__(self, n: int, num_heads: int, max_tokens: int, embed_dim: int, num_layers: int, device: torch.device) -> None: + self._keys_values = tuple([KVCache(n, num_heads, max_tokens, embed_dim, device) for _ in range(num_layers)]) + + def __getitem__(self, key: int) -> KVCache: + return self._keys_values[key] + + def __len__(self): + return len(self._keys_values) + + @property + def size(self): + return self._keys_values[0].shape[2] + + def reset(self) -> None: + for kv_cache in self._keys_values: + kv_cache.reset() + + def prune(self, mask: np.ndarray) -> None: + for kv_cache in self._keys_values: + kv_cache.prune(mask) + + +class AssignWithoutInplaceCheck(torch.autograd.Function): + """ + Inspired from : https://discuss.pytorch.org/t/disable-in-place-correctness-version-check-any-other-workaround/90738/4 + Warning : do not use it to overwrite a slice twice. + """ + + @staticmethod + def get_slice(dim: int, start: int, stop: int) -> Tuple[slice]: + return tuple([slice(None), ] * dim + [slice(start, stop)]) + + @staticmethod + def forward(ctx, input: torch.Tensor, value: torch.Tensor, dim: int, start: int, stop: int) -> torch.Tensor: + ctx.dim = dim + ctx.start = start + ctx.stop = stop + input.data[AssignWithoutInplaceCheck.get_slice(dim, start, stop)] = value + return input + + @staticmethod + def backward(ctx, grad_out: torch.Tensor) -> Tuple[torch.Tensor]: + return grad_out, grad_out[AssignWithoutInplaceCheck.get_slice(ctx.dim, ctx.start, ctx.stop)], None, None, None diff --git a/src/models/slicer.py b/src/models/slicer.py new file mode 100644 index 0000000..6566271 --- /dev/null +++ b/src/models/slicer.py @@ -0,0 +1,54 @@ +import math +from typing import List + +import torch +import torch.nn as nn + + +class Slicer(nn.Module): + def __init__(self, max_blocks: int, block_mask: torch.Tensor) -> None: + super().__init__() + self.block_size = block_mask.size(0) + self.num_kept_tokens = block_mask.sum().long().item() + kept_indices = torch.where(block_mask)[0].repeat(max_blocks) + offsets = torch.arange(max_blocks).repeat_interleave(self.num_kept_tokens) + self.register_buffer('indices', kept_indices + block_mask.size(0) * offsets) + + def compute_slice(self, num_steps: int, prev_steps: int = 0) -> torch.Tensor: + total_steps = num_steps + prev_steps + num_blocks = math.ceil(total_steps / self.block_size) + indices = self.indices[:num_blocks * self.num_kept_tokens] + return indices[torch.logical_and(prev_steps <= indices, indices < total_steps)] - prev_steps + + def forward(self, *args, **kwargs): + raise NotImplementedError + + +class Head(Slicer): + def __init__(self, max_blocks: int, block_mask: torch.Tensor, head_module: nn.Module) -> None: + super().__init__(max_blocks, block_mask) + assert isinstance(head_module, nn.Module) + self.head_module = head_module + + def forward(self, x: torch.Tensor, num_steps: int, prev_steps: int) -> torch.Tensor: + x_sliced = x[:, self.compute_slice(num_steps, prev_steps)] # x is (B, T, E) + return self.head_module(x_sliced) + + +class Embedder(nn.Module): + def __init__(self, max_blocks: int, block_masks: List[torch.Tensor], embedding_tables: List[nn.Embedding]) -> None: + super().__init__() + assert len(block_masks) == len(embedding_tables) + assert (sum(block_masks) == 1).all() # block mask are a partition of a block + self.embedding_dim = embedding_tables[0].embedding_dim + assert all([e.embedding_dim == self.embedding_dim for e in embedding_tables]) + self.embedding_tables = embedding_tables + self.slicers = [Slicer(max_blocks, block_mask) for block_mask in block_masks] + + def forward(self, tokens: torch.Tensor, num_steps: int, prev_steps: int) -> torch.Tensor: + assert tokens.ndim == 2 # x is (B, T) + output = torch.zeros(*tokens.size(), self.embedding_dim, device=tokens.device) + for slicer, emb in zip(self.slicers, self.embedding_tables): + s = slicer.compute_slice(num_steps, prev_steps) + output[:, s] = emb(tokens[:, s]) + return output diff --git a/src/models/tokenizer/__init__.py b/src/models/tokenizer/__init__.py new file mode 100644 index 0000000..9238cc5 --- /dev/null +++ b/src/models/tokenizer/__init__.py @@ -0,0 +1,2 @@ +from .nets import Encoder, Decoder, EncoderDecoderConfig +from .tokenizer import Tokenizer, TokenizerEncoderOutput diff --git a/src/models/tokenizer/lpips.py b/src/models/tokenizer/lpips.py new file mode 100644 index 0000000..f9a8f26 --- /dev/null +++ b/src/models/tokenizer/lpips.py @@ -0,0 +1,167 @@ +""" +Credits to https://github.com/CompVis/taming-transformers +""" + +from collections import namedtuple +import hashlib +import os +from pathlib import Path +import requests + +import torch +import torch.nn as nn +from torchvision import models +from tqdm import tqdm + + +class LPIPS(nn.Module): + # Learned perceptual metric + def __init__(self, use_dropout: bool = True): + super().__init__() + self.scaling_layer = ScalingLayer() + self.chns = [64, 128, 256, 512, 512] # vg16 features + self.net = vgg16(pretrained=True, requires_grad=False) + self.lin0 = NetLinLayer(self.chns[0], use_dropout=use_dropout) + self.lin1 = NetLinLayer(self.chns[1], use_dropout=use_dropout) + self.lin2 = NetLinLayer(self.chns[2], use_dropout=use_dropout) + self.lin3 = NetLinLayer(self.chns[3], use_dropout=use_dropout) + self.lin4 = NetLinLayer(self.chns[4], use_dropout=use_dropout) + self.load_from_pretrained() + for param in self.parameters(): + param.requires_grad = False + + def load_from_pretrained(self) -> None: + ckpt = get_ckpt_path(name="vgg_lpips", root=Path.home() / ".cache/iris/tokenizer_pretrained_vgg") # Download VGG if necessary + self.load_state_dict(torch.load(ckpt, map_location=torch.device("cpu")), strict=False) + + def forward(self, input: torch.Tensor, target: torch.Tensor) -> torch.Tensor: + in0_input, in1_input = (self.scaling_layer(input), self.scaling_layer(target)) + outs0, outs1 = self.net(in0_input), self.net(in1_input) + feats0, feats1, diffs = {}, {}, {} + lins = [self.lin0, self.lin1, self.lin2, self.lin3, self.lin4] + for kk in range(len(self.chns)): + feats0[kk], feats1[kk] = normalize_tensor(outs0[kk]), normalize_tensor(outs1[kk]) + diffs[kk] = (feats0[kk] - feats1[kk]) ** 2 + + res = [spatial_average(lins[kk].model(diffs[kk]), keepdim=True) for kk in range(len(self.chns))] + val = res[0] + for i in range(1, len(self.chns)): + val += res[i] + return val + + +class ScalingLayer(nn.Module): + def __init__(self) -> None: + super(ScalingLayer, self).__init__() + self.register_buffer('shift', torch.Tensor([-.030, -.088, -.188])[None, :, None, None]) + self.register_buffer('scale', torch.Tensor([.458, .448, .450])[None, :, None, None]) + + def forward(self, inp: torch.Tensor) -> torch.Tensor: + return (inp - self.shift) / self.scale + + +class NetLinLayer(nn.Module): + """ A single linear layer which does a 1x1 conv """ + def __init__(self, chn_in: int, chn_out: int = 1, use_dropout: bool = False) -> None: + super(NetLinLayer, self).__init__() + layers = [nn.Dropout(), ] if (use_dropout) else [] + layers += [nn.Conv2d(chn_in, chn_out, 1, stride=1, padding=0, bias=False), ] + self.model = nn.Sequential(*layers) + + +class vgg16(torch.nn.Module): + def __init__(self, requires_grad: bool = False, pretrained: bool = True) -> None: + super(vgg16, self).__init__() + vgg_pretrained_features = models.vgg16(pretrained=pretrained).features + self.slice1 = torch.nn.Sequential() + self.slice2 = torch.nn.Sequential() + self.slice3 = torch.nn.Sequential() + self.slice4 = torch.nn.Sequential() + self.slice5 = torch.nn.Sequential() + self.N_slices = 5 + for x in range(4): + self.slice1.add_module(str(x), vgg_pretrained_features[x]) + for x in range(4, 9): + self.slice2.add_module(str(x), vgg_pretrained_features[x]) + for x in range(9, 16): + self.slice3.add_module(str(x), vgg_pretrained_features[x]) + for x in range(16, 23): + self.slice4.add_module(str(x), vgg_pretrained_features[x]) + for x in range(23, 30): + self.slice5.add_module(str(x), vgg_pretrained_features[x]) + if not requires_grad: + for param in self.parameters(): + param.requires_grad = False + + def forward(self, X: torch.Tensor) -> torch.Tensor: + h = self.slice1(X) + h_relu1_2 = h + h = self.slice2(h) + h_relu2_2 = h + h = self.slice3(h) + h_relu3_3 = h + h = self.slice4(h) + h_relu4_3 = h + h = self.slice5(h) + h_relu5_3 = h + vgg_outputs = namedtuple("VggOutputs", ['relu1_2', 'relu2_2', 'relu3_3', 'relu4_3', 'relu5_3']) + out = vgg_outputs(h_relu1_2, h_relu2_2, h_relu3_3, h_relu4_3, h_relu5_3) + return out + + +def normalize_tensor(x: torch.Tensor, eps: float = 1e-10) -> torch.Tensor: + norm_factor = torch.sqrt(torch.sum(x ** 2, dim=1, keepdim=True)) + return x / (norm_factor + eps) + + +def spatial_average(x: torch.Tensor, keepdim: bool = True) -> torch.Tensor: + return x.mean([2, 3], keepdim=keepdim) + + +# ******************************************************************** +# *************** Utilities to download pretrained vgg *************** +# ******************************************************************** + + +URL_MAP = { + "vgg_lpips": "https://heibox.uni-heidelberg.de/f/607503859c864bc1b30b/?dl=1" +} + + +CKPT_MAP = { + "vgg_lpips": "vgg.pth" +} + + +MD5_MAP = { + "vgg_lpips": "d507d7349b931f0638a25a48a722f98a" +} + + +def download(url: str, local_path: str, chunk_size: int = 1024) -> None: + os.makedirs(os.path.split(local_path)[0], exist_ok=True) + with requests.get(url, stream=True) as r: + total_size = int(r.headers.get("content-length", 0)) + with tqdm(total=total_size, unit="B", unit_scale=True) as pbar: + with open(local_path, "wb") as f: + for data in r.iter_content(chunk_size=chunk_size): + if data: + f.write(data) + pbar.update(chunk_size) + + +def md5_hash(path: str) -> str: + with open(path, "rb") as f: + content = f.read() + return hashlib.md5(content).hexdigest() + + +def get_ckpt_path(name: str, root: str, check: bool = False) -> str: + assert name in URL_MAP + path = os.path.join(root, CKPT_MAP[name]) + if not os.path.exists(path) or (check and not md5_hash(path) == MD5_MAP[name]): + print("Downloading {} model from {} to {}".format(name, URL_MAP[name], path)) + download(URL_MAP[name], path) + md5 = md5_hash(path) + assert md5 == MD5_MAP[name], md5 + return path diff --git a/src/models/tokenizer/nets.py b/src/models/tokenizer/nets.py new file mode 100644 index 0000000..1054042 --- /dev/null +++ b/src/models/tokenizer/nets.py @@ -0,0 +1,362 @@ +""" +Credits to https://github.com/CompVis/taming-transformers +""" + +from dataclasses import dataclass +from typing import List + +import torch +import torch.nn as nn + + +@dataclass +class EncoderDecoderConfig: + resolution: int + in_channels: int + z_channels: int + ch: int + ch_mult: List[int] + num_res_blocks: int + attn_resolutions: List[int] + out_ch: int + dropout: float + + +class Encoder(nn.Module): + def __init__(self, config: EncoderDecoderConfig) -> None: + super().__init__() + self.config = config + self.num_resolutions = len(config.ch_mult) + temb_ch = 0 # timestep embedding #channels + + # downsampling + self.conv_in = torch.nn.Conv2d(config.in_channels, + config.ch, + kernel_size=3, + stride=1, + padding=1) + + curr_res = config.resolution + in_ch_mult = (1,) + tuple(config.ch_mult) + self.down = nn.ModuleList() + for i_level in range(self.num_resolutions): + block = nn.ModuleList() + attn = nn.ModuleList() + block_in = config.ch * in_ch_mult[i_level] + block_out = config.ch * config.ch_mult[i_level] + for i_block in range(self.config.num_res_blocks): + block.append(ResnetBlock(in_channels=block_in, + out_channels=block_out, + temb_channels=temb_ch, + dropout=config.dropout)) + block_in = block_out + if curr_res in config.attn_resolutions: + attn.append(AttnBlock(block_in)) + down = nn.Module() + down.block = block + down.attn = attn + if i_level != self.num_resolutions - 1: + down.downsample = Downsample(block_in, with_conv=True) + curr_res = curr_res // 2 + self.down.append(down) + + # middle + self.mid = nn.Module() + self.mid.block_1 = ResnetBlock(in_channels=block_in, + out_channels=block_in, + temb_channels=temb_ch, + dropout=config.dropout) + self.mid.attn_1 = AttnBlock(block_in) + self.mid.block_2 = ResnetBlock(in_channels=block_in, + out_channels=block_in, + temb_channels=temb_ch, + dropout=config.dropout) + + # end + self.norm_out = Normalize(block_in) + self.conv_out = torch.nn.Conv2d(block_in, + config.z_channels, + kernel_size=3, + stride=1, + padding=1) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + + temb = None # timestep embedding + + # downsampling + hs = [self.conv_in(x)] + for i_level in range(self.num_resolutions): + for i_block in range(self.config.num_res_blocks): + h = self.down[i_level].block[i_block](hs[-1], temb) + if len(self.down[i_level].attn) > 0: + h = self.down[i_level].attn[i_block](h) + hs.append(h) + if i_level != self.num_resolutions - 1: + hs.append(self.down[i_level].downsample(hs[-1])) + + # middle + h = hs[-1] + h = self.mid.block_1(h, temb) + h = self.mid.attn_1(h) + h = self.mid.block_2(h, temb) + + # end + h = self.norm_out(h) + h = nonlinearity(h) + h = self.conv_out(h) + return h + + +class Decoder(nn.Module): + def __init__(self, config: EncoderDecoderConfig) -> None: + super().__init__() + self.config = config + temb_ch = 0 + self.num_resolutions = len(config.ch_mult) + + # compute in_ch_mult, block_in and curr_res at lowest res + in_ch_mult = (1,) + tuple(config.ch_mult) + block_in = config.ch * config.ch_mult[self.num_resolutions - 1] + curr_res = config.resolution // 2 ** (self.num_resolutions - 1) + print(f"Tokenizer : shape of latent is {config.z_channels, curr_res, curr_res}.") + + # z to block_in + self.conv_in = torch.nn.Conv2d(config.z_channels, + block_in, + kernel_size=3, + stride=1, + padding=1) + + # middle + self.mid = nn.Module() + self.mid.block_1 = ResnetBlock(in_channels=block_in, + out_channels=block_in, + temb_channels=temb_ch, + dropout=config.dropout) + self.mid.attn_1 = AttnBlock(block_in) + self.mid.block_2 = ResnetBlock(in_channels=block_in, + out_channels=block_in, + temb_channels=temb_ch, + dropout=config.dropout) + + # upsampling + self.up = nn.ModuleList() + for i_level in reversed(range(self.num_resolutions)): + block = nn.ModuleList() + attn = nn.ModuleList() + block_out = config.ch * config.ch_mult[i_level] + for i_block in range(config.num_res_blocks + 1): + block.append(ResnetBlock(in_channels=block_in, + out_channels=block_out, + temb_channels=temb_ch, + dropout=config.dropout)) + block_in = block_out + if curr_res in config.attn_resolutions: + attn.append(AttnBlock(block_in)) + up = nn.Module() + up.block = block + up.attn = attn + if i_level != 0: + up.upsample = Upsample(block_in, with_conv=True) + curr_res = curr_res * 2 + self.up.insert(0, up) # prepend to get consistent order + + # end + self.norm_out = Normalize(block_in) + self.conv_out = torch.nn.Conv2d(block_in, + config.out_ch, + kernel_size=3, + stride=1, + padding=1) + + def forward(self, z: torch.Tensor) -> torch.Tensor: + temb = None # timestep embedding + + # z to block_in + h = self.conv_in(z) + + # middle + h = self.mid.block_1(h, temb) + h = self.mid.attn_1(h) + h = self.mid.block_2(h, temb) + + # upsampling + for i_level in reversed(range(self.num_resolutions)): + for i_block in range(self.config.num_res_blocks + 1): + h = self.up[i_level].block[i_block](h, temb) + if len(self.up[i_level].attn) > 0: + h = self.up[i_level].attn[i_block](h) + if i_level != 0: + h = self.up[i_level].upsample(h) + + # end + h = self.norm_out(h) + h = nonlinearity(h) + h = self.conv_out(h) + return h + + +def nonlinearity(x: torch.Tensor) -> torch.Tensor: + # swish + return x * torch.sigmoid(x) + + +def Normalize(in_channels: int) -> nn.Module: + return torch.nn.GroupNorm(num_groups=32, num_channels=in_channels, eps=1e-6, affine=True) + + +class Upsample(nn.Module): + def __init__(self, in_channels: int, with_conv: bool) -> None: + super().__init__() + self.with_conv = with_conv + if self.with_conv: + self.conv = torch.nn.Conv2d(in_channels, + in_channels, + kernel_size=3, + stride=1, + padding=1) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = torch.nn.functional.interpolate(x, scale_factor=2.0, mode="nearest") + if self.with_conv: + x = self.conv(x) + return x + + +class Downsample(nn.Module): + def __init__(self, in_channels: int, with_conv: bool) -> None: + super().__init__() + self.with_conv = with_conv + if self.with_conv: + # no asymmetric padding in torch conv, must do it ourselves + self.conv = torch.nn.Conv2d(in_channels, + in_channels, + kernel_size=3, + stride=2, + padding=0) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + if self.with_conv: + pad = (0, 1, 0, 1) + x = torch.nn.functional.pad(x, pad, mode="constant", value=0) + x = self.conv(x) + else: + x = torch.nn.functional.avg_pool2d(x, kernel_size=2, stride=2) + return x + + +class ResnetBlock(nn.Module): + def __init__(self, *, in_channels: int, out_channels: int = None, conv_shortcut: bool = False, + dropout: float, temb_channels: int = 512) -> None: + super().__init__() + self.in_channels = in_channels + out_channels = in_channels if out_channels is None else out_channels + self.out_channels = out_channels + self.use_conv_shortcut = conv_shortcut + + self.norm1 = Normalize(in_channels) + self.conv1 = torch.nn.Conv2d(in_channels, + out_channels, + kernel_size=3, + stride=1, + padding=1) + if temb_channels > 0: + self.temb_proj = torch.nn.Linear(temb_channels, + out_channels) + self.norm2 = Normalize(out_channels) + self.dropout = torch.nn.Dropout(dropout) + self.conv2 = torch.nn.Conv2d(out_channels, + out_channels, + kernel_size=3, + stride=1, + padding=1) + if self.in_channels != self.out_channels: + if self.use_conv_shortcut: + self.conv_shortcut = torch.nn.Conv2d(in_channels, + out_channels, + kernel_size=3, + stride=1, + padding=1) + else: + self.nin_shortcut = torch.nn.Conv2d(in_channels, + out_channels, + kernel_size=1, + stride=1, + padding=0) + + def forward(self, x: torch.Tensor, temb: torch.Tensor) -> torch.Tensor: + h = x + h = self.norm1(h) + h = nonlinearity(h) + h = self.conv1(h) + + if temb is not None: + h = h + self.temb_proj(nonlinearity(temb))[:, :, None, None] + + h = self.norm2(h) + h = nonlinearity(h) + h = self.dropout(h) + h = self.conv2(h) + + if self.in_channels != self.out_channels: + if self.use_conv_shortcut: + x = self.conv_shortcut(x) + else: + x = self.nin_shortcut(x) + + return x + h + + +class AttnBlock(nn.Module): + def __init__(self, in_channels: int) -> None: + super().__init__() + self.in_channels = in_channels + + self.norm = Normalize(in_channels) + self.q = torch.nn.Conv2d(in_channels, + in_channels, + kernel_size=1, + stride=1, + padding=0) + self.k = torch.nn.Conv2d(in_channels, + in_channels, + kernel_size=1, + stride=1, + padding=0) + self.v = torch.nn.Conv2d(in_channels, + in_channels, + kernel_size=1, + stride=1, + padding=0) + self.proj_out = torch.nn.Conv2d(in_channels, + in_channels, + kernel_size=1, + stride=1, + padding=0) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + h_ = x + h_ = self.norm(h_) + q = self.q(h_) + k = self.k(h_) + v = self.v(h_) + + # compute attention + b, c, h, w = q.shape + q = q.reshape(b, c, h * w) + q = q.permute(0, 2, 1) # b,hw,c + k = k.reshape(b, c, h * w) # b,c,hw + w_ = torch.bmm(q, k) # b,hw,hw w[b,i,j]=sum_c q[b,i,c]k[b,c,j] + w_ = w_ * (int(c) ** (-0.5)) + w_ = torch.nn.functional.softmax(w_, dim=2) + + # attend to values + v = v.reshape(b, c, h * w) + w_ = w_.permute(0, 2, 1) # b,hw,hw (first hw of k, second of q) + h_ = torch.bmm(v, w_) # b, c,hw (hw of q) h_[b,c,j] = sum_i v[b,c,i] w_[b,i,j] + h_ = h_.reshape(b, c, h, w) + + h_ = self.proj_out(h_) + + return x + h_ diff --git a/src/models/tokenizer/tokenizer.py b/src/models/tokenizer/tokenizer.py new file mode 100644 index 0000000..6528fa4 --- /dev/null +++ b/src/models/tokenizer/tokenizer.py @@ -0,0 +1,104 @@ +""" +Credits to https://github.com/CompVis/taming-transformers +""" + +from dataclasses import dataclass +from typing import Any, Tuple + +from einops import rearrange +import torch +import torch.nn as nn + +from dataset import Batch +from .lpips import LPIPS +from .nets import Encoder, Decoder +from utils import LossWithIntermediateLosses + + +@dataclass +class TokenizerEncoderOutput: + z: torch.FloatTensor + z_quantized: torch.FloatTensor + tokens: torch.LongTensor + + +class Tokenizer(nn.Module): + def __init__(self, vocab_size: int, embed_dim: int, encoder: Encoder, decoder: Decoder, with_lpips: bool = True) -> None: + super().__init__() + self.vocab_size = vocab_size + self.encoder = encoder + self.pre_quant_conv = torch.nn.Conv2d(encoder.config.z_channels, embed_dim, 1) + self.embedding = nn.Embedding(vocab_size, embed_dim) + self.post_quant_conv = torch.nn.Conv2d(embed_dim, decoder.config.z_channels, 1) + self.decoder = decoder + self.embedding.weight.data.uniform_(-1.0 / vocab_size, 1.0 / vocab_size) + self.lpips = LPIPS().eval() if with_lpips else None + + def __repr__(self) -> str: + return "tokenizer" + + def forward(self, x: torch.Tensor, should_preprocess: bool = False, should_postprocess: bool = False) -> Tuple[torch.Tensor]: + outputs = self.encode(x, should_preprocess) + decoder_input = outputs.z + (outputs.z_quantized - outputs.z).detach() + reconstructions = self.decode(decoder_input, should_postprocess) + return outputs.z, outputs.z_quantized, reconstructions + + def compute_loss(self, batch: Batch, **kwargs: Any) -> LossWithIntermediateLosses: + assert self.lpips is not None + observations = self.preprocess_input(rearrange(batch['observations'], 'b t c h w -> (b t) c h w')) + z, z_quantized, reconstructions = self(observations, should_preprocess=False, should_postprocess=False) + + # Codebook loss. Notes: + # - beta position is different from taming and identical to original VQVAE paper + # - VQVAE uses 0.25 by default + beta = 1.0 + commitment_loss = (z.detach() - z_quantized).pow(2).mean() + beta * (z - z_quantized.detach()).pow(2).mean() + + reconstruction_loss = torch.abs(observations - reconstructions).mean() + perceptual_loss = torch.mean(self.lpips(observations, reconstructions)) + + return LossWithIntermediateLosses(commitment_loss=commitment_loss, reconstruction_loss=reconstruction_loss, perceptual_loss=perceptual_loss) + + def encode(self, x: torch.Tensor, should_preprocess: bool = False) -> TokenizerEncoderOutput: + if should_preprocess: + x = self.preprocess_input(x) + shape = x.shape # (..., C, H, W) + x = x.view(-1, *shape[-3:]) + z = self.encoder(x) + z = self.pre_quant_conv(z) + b, e, h, w = z.shape + z_flattened = rearrange(z, 'b e h w -> (b h w) e') + dist_to_embeddings = torch.sum(z_flattened ** 2, dim=1, keepdim=True) + torch.sum(self.embedding.weight**2, dim=1) - 2 * torch.matmul(z_flattened, self.embedding.weight.t()) + + tokens = dist_to_embeddings.argmin(dim=-1) + z_q = rearrange(self.embedding(tokens), '(b h w) e -> b e h w', b=b, e=e, h=h, w=w).contiguous() + + # Reshape to original + z = z.reshape(*shape[:-3], *z.shape[1:]) + z_q = z_q.reshape(*shape[:-3], *z_q.shape[1:]) + tokens = tokens.reshape(*shape[:-3], -1) + + return TokenizerEncoderOutput(z, z_q, tokens) + + def decode(self, z_q: torch.Tensor, should_postprocess: bool = False) -> torch.Tensor: + shape = z_q.shape # (..., E, h, w) + z_q = z_q.view(-1, *shape[-3:]) + z_q = self.post_quant_conv(z_q) + rec = self.decoder(z_q) + rec = rec.reshape(*shape[:-3], *rec.shape[1:]) + if should_postprocess: + rec = self.postprocess_output(rec) + return rec + + @torch.no_grad() + def encode_decode(self, x: torch.Tensor, should_preprocess: bool = False, should_postprocess: bool = False) -> torch.Tensor: + z_q = self.encode(x, should_preprocess).z_quantized + return self.decode(z_q, should_postprocess) + + def preprocess_input(self, x: torch.Tensor) -> torch.Tensor: + """x is supposed to be channels first and in [0, 1]""" + return x.mul(2).sub(1) + + def postprocess_output(self, y: torch.Tensor) -> torch.Tensor: + """y is supposed to be channels first and in [-1, 1]""" + return y.add(1).div(2) diff --git a/src/models/transformer.py b/src/models/transformer.py new file mode 100644 index 0000000..2aaac69 --- /dev/null +++ b/src/models/transformer.py @@ -0,0 +1,120 @@ +""" +Credits to https://github.com/karpathy/minGPT +""" + +from dataclasses import dataclass +import math +from typing import Optional + +from einops import rearrange +import torch +import torch.nn as nn +from torch.nn import functional as F + +from .kv_caching import KeysValues, KVCache + + +@dataclass +class TransformerConfig: + tokens_per_block: int + max_blocks: int + attention: str + + num_layers: int + num_heads: int + embed_dim: int + + embed_pdrop: float + resid_pdrop: float + attn_pdrop: float + + @property + def max_tokens(self): + return self.tokens_per_block * self.max_blocks + + +class Transformer(nn.Module): + def __init__(self, config: TransformerConfig) -> None: + super().__init__() + self.config = config + self.drop = nn.Dropout(config.embed_pdrop) + self.blocks = nn.ModuleList([Block(config) for _ in range(config.num_layers)]) + self.ln_f = nn.LayerNorm(config.embed_dim) + + def generate_empty_keys_values(self, n: int, max_tokens: int) -> KeysValues: + device = self.ln_f.weight.device # Assumption that all submodules are on the same device + return KeysValues(n, self.config.num_heads, max_tokens, self.config.embed_dim, self.config.num_layers, device) + + def forward(self, sequences: torch.Tensor, past_keys_values: Optional[KeysValues] = None) -> torch.Tensor: + assert past_keys_values is None or len(past_keys_values) == len(self.blocks) + x = self.drop(sequences) + for i, block in enumerate(self.blocks): + x = block(x, None if past_keys_values is None else past_keys_values[i]) + + x = self.ln_f(x) + return x + + +class Block(nn.Module): + def __init__(self, config: TransformerConfig) -> None: + super().__init__() + self.ln1 = nn.LayerNorm(config.embed_dim) + self.ln2 = nn.LayerNorm(config.embed_dim) + self.attn = SelfAttention(config) + self.mlp = nn.Sequential( + nn.Linear(config.embed_dim, 4 * config.embed_dim), + nn.GELU(), + nn.Linear(4 * config.embed_dim, config.embed_dim), + nn.Dropout(config.resid_pdrop), + ) + + def forward(self, x: torch.Tensor, past_keys_values: Optional[KeysValues] = None) -> torch.Tensor: + x_attn = self.attn(self.ln1(x), past_keys_values) + x = x + x_attn + x = x + self.mlp(self.ln2(x)) + return x + + +class SelfAttention(nn.Module): + def __init__(self, config: TransformerConfig) -> None: + super().__init__() + assert config.embed_dim % config.num_heads == 0 + assert config.attention in ('causal', 'block_causal') + self.num_heads = config.num_heads + self.key = nn.Linear(config.embed_dim, config.embed_dim) + self.query = nn.Linear(config.embed_dim, config.embed_dim) + self.value = nn.Linear(config.embed_dim, config.embed_dim) + self.attn_drop = nn.Dropout(config.attn_pdrop) + self.resid_drop = nn.Dropout(config.resid_pdrop) + self.proj = nn.Linear(config.embed_dim, config.embed_dim) + + causal_mask = torch.tril(torch.ones(config.max_tokens, config.max_tokens)) + block_causal_mask = torch.max(causal_mask, torch.block_diag(*[torch.ones(config.tokens_per_block, config.tokens_per_block) for _ in range(config.max_blocks)])) + self.register_buffer('mask', causal_mask if config.attention == 'causal' else block_causal_mask) + + def forward(self, x: torch.Tensor, kv_cache: Optional[KVCache] = None) -> torch.Tensor: + B, T, C = x.size() + if kv_cache is not None: + b, nh, L, c = kv_cache.shape + assert nh == self.num_heads and b == B and c * nh == C + else: + L = 0 + + q = self.query(x).view(B, T, self.num_heads, C // self.num_heads).transpose(1, 2) # (B, nh, T, hs) + k = self.key(x).view(B, T, self.num_heads, C // self.num_heads).transpose(1, 2) # (B, nh, T, hs) + v = self.value(x).view(B, T, self.num_heads, C // self.num_heads).transpose(1, 2) # (B, nh, T, hs) + + if kv_cache is not None: + kv_cache.update(k, v) + k, v = kv_cache.get() + + att = (q @ k.transpose(-2, -1)) * (1.0 / math.sqrt(k.size(-1))) + att = att.masked_fill(self.mask[L:L + T, :L + T] == 0, float('-inf')) + att = F.softmax(att, dim=-1) + att = self.attn_drop(att) + y = att @ v + y = rearrange(y, 'b h t e -> b t (h e)') + + y = self.resid_drop(self.proj(y)) + + return y diff --git a/src/models/world_model.py b/src/models/world_model.py new file mode 100644 index 0000000..0691034 --- /dev/null +++ b/src/models/world_model.py @@ -0,0 +1,122 @@ +from dataclasses import dataclass +from typing import Any, Optional, Tuple + +from einops import rearrange +import torch +import torch.nn as nn +import torch.nn.functional as F + +from dataset import Batch +from .kv_caching import KeysValues +from .slicer import Embedder, Head +from .tokenizer import Tokenizer +from .transformer import Transformer, TransformerConfig +from utils import init_weights, LossWithIntermediateLosses + + +@dataclass +class WorldModelOutput: + output_sequence: torch.FloatTensor + logits_observations: torch.FloatTensor + logits_rewards: torch.FloatTensor + logits_ends: torch.FloatTensor + + +class WorldModel(nn.Module): + def __init__(self, obs_vocab_size: int, act_vocab_size: int, config: TransformerConfig) -> None: + super().__init__() + self.obs_vocab_size, self.act_vocab_size = obs_vocab_size, act_vocab_size + self.config = config + self.transformer = Transformer(config) + + all_but_last_obs_tokens_pattern = torch.ones(config.tokens_per_block) + all_but_last_obs_tokens_pattern[-2] = 0 + act_tokens_pattern = torch.zeros(self.config.tokens_per_block) + act_tokens_pattern[-1] = 1 + obs_tokens_pattern = 1 - act_tokens_pattern + + self.pos_emb = nn.Embedding(config.max_tokens, config.embed_dim) + + self.embedder = Embedder( + max_blocks=config.max_blocks, + block_masks=[act_tokens_pattern, obs_tokens_pattern], + embedding_tables=nn.ModuleList([nn.Embedding(act_vocab_size, config.embed_dim), nn.Embedding(obs_vocab_size, config.embed_dim)]) + ) + + self.head_observations = Head( + max_blocks=config.max_blocks, + block_mask=all_but_last_obs_tokens_pattern, + head_module=nn.Sequential( + nn.Linear(config.embed_dim, config.embed_dim), + nn.ReLU(), + nn.Linear(config.embed_dim, obs_vocab_size) + ) + ) + + self.head_rewards = Head( + max_blocks=config.max_blocks, + block_mask=act_tokens_pattern, + head_module=nn.Sequential( + nn.Linear(config.embed_dim, config.embed_dim), + nn.ReLU(), + nn.Linear(config.embed_dim, 3) + ) + ) + + self.head_ends = Head( + max_blocks=config.max_blocks, + block_mask=act_tokens_pattern, + head_module=nn.Sequential( + nn.Linear(config.embed_dim, config.embed_dim), + nn.ReLU(), + nn.Linear(config.embed_dim, 2) + ) + ) + + self.apply(init_weights) + + def __repr__(self) -> str: + return "world_model" + + def forward(self, tokens: torch.LongTensor, past_keys_values: Optional[KeysValues] = None) -> WorldModelOutput: + + num_steps = tokens.size(1) # (B, T) + assert num_steps <= self.config.max_tokens + prev_steps = 0 if past_keys_values is None else past_keys_values.size + + sequences = self.embedder(tokens, num_steps, prev_steps) + self.pos_emb(prev_steps + torch.arange(num_steps, device=tokens.device)) + + x = self.transformer(sequences, past_keys_values) + + logits_observations = self.head_observations(x, num_steps=num_steps, prev_steps=prev_steps) + logits_rewards = self.head_rewards(x, num_steps=num_steps, prev_steps=prev_steps) + logits_ends = self.head_ends(x, num_steps=num_steps, prev_steps=prev_steps) + + return WorldModelOutput(x, logits_observations, logits_rewards, logits_ends) + + def compute_loss(self, batch: Batch, tokenizer: Tokenizer, **kwargs: Any) -> LossWithIntermediateLosses: + + with torch.no_grad(): + obs_tokens = tokenizer.encode(batch['observations'], should_preprocess=True).tokens # (BL, K) + + act_tokens = rearrange(batch['actions'], 'b l -> b l 1') + tokens = rearrange(torch.cat((obs_tokens, act_tokens), dim=2), 'b l k1 -> b (l k1)') # (B, L(K+1)) + + outputs = self(tokens) + + labels_observations, labels_rewards, labels_ends = self.compute_labels_world_model(obs_tokens, batch['rewards'], batch['ends'], batch['mask_padding']) + + logits_observations = rearrange(outputs.logits_observations[:, :-1], 'b t o -> (b t) o') + loss_obs = F.cross_entropy(logits_observations, labels_observations) + loss_rewards = F.cross_entropy(rearrange(outputs.logits_rewards, 'b t e -> (b t) e'), labels_rewards) + loss_ends = F.cross_entropy(rearrange(outputs.logits_ends, 'b t e -> (b t) e'), labels_ends) + + return LossWithIntermediateLosses(loss_obs=loss_obs, loss_rewards=loss_rewards, loss_ends=loss_ends) + + def compute_labels_world_model(self, obs_tokens: torch.Tensor, rewards: torch.Tensor, ends: torch.Tensor, mask_padding: torch.BoolTensor) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + assert torch.all(ends.sum(dim=1) <= 1) # at most 1 done + mask_fill = torch.logical_not(mask_padding) + labels_observations = rearrange(obs_tokens.masked_fill(mask_fill.unsqueeze(-1).expand_as(obs_tokens), -100), 'b t k -> b (t k)')[:, 1:] + labels_rewards = (rewards.sign() + 1).masked_fill(mask_fill, -100).long() # Rewards clipped to {-1, 0, 1} + labels_ends = ends.masked_fill(mask_fill, -100) + return labels_observations.reshape(-1), labels_rewards.reshape(-1), labels_ends.reshape(-1) diff --git a/src/play.py b/src/play.py new file mode 100644 index 0000000..bea40fa --- /dev/null +++ b/src/play.py @@ -0,0 +1,44 @@ +from pathlib import Path + +import hydra +from hydra.utils import instantiate +from omegaconf import DictConfig +import torch + +from agent import Agent +from envs import SingleProcessEnv, WorldModelEnv +from game import AgentEnv, EpisodeReplayEnv, Game +from models.actor_critic import ActorCritic +from models.world_model import WorldModel + + +@hydra.main(config_path="../config", config_name="trainer") +def main(cfg: DictConfig): + device = torch.device(cfg.common.device) + assert cfg.mode in ('world_model', 'episode_replay', 'agent') + + if cfg.mode in ['world_model', 'agent']: + env_fn = lambda: instantiate(cfg.env.test) + test_env = SingleProcessEnv(env_fn) + tokenizer = instantiate(cfg.tokenizer) + world_model = WorldModel(obs_vocab_size=tokenizer.vocab_size, act_vocab_size=test_env.num_actions, config=instantiate(cfg.world_model)) + if cfg.mode == 'world_model': + env = WorldModelEnv(tokenizer=tokenizer, world_model=world_model, pretrained_agent_path=Path('checkpoints/last.pt'), device=device, env=env_fn()) + keymap = cfg.env.keymap + else: + actor_critic = ActorCritic(**cfg.actor_critic, act_vocab_size=test_env.num_actions) + agent = Agent(tokenizer, world_model, actor_critic).to(device) + agent.load(Path('checkpoints/last.pt'), device) + env = AgentEnv(agent, test_env, cfg.env.keymap) + keymap = 'empty' + + else: + env = EpisodeReplayEnv(replay_keymap_name=cfg.env.keymap, episode_dir=Path('media/episodes')) + keymap = 'episode_replay' + + game = Game(env, keymap_name=keymap, size=(600, 1200 if cfg.mode == 'agent' else 600), fps=cfg.fps, verbose=bool(cfg.header)) + game.run() + + +if __name__ == "__main__": + main() diff --git a/src/trainer.py b/src/trainer.py new file mode 100644 index 0000000..e42902b --- /dev/null +++ b/src/trainer.py @@ -0,0 +1,283 @@ +from collections import defaultdict +from pathlib import Path +import shutil +import sys +import time +from typing import Any, Dict, Optional, Tuple + +import hydra +from hydra.utils import instantiate +from omegaconf import DictConfig, OmegaConf +import torch +import torch.nn as nn +from tqdm import tqdm +import wandb + +from agent import Agent +from collector import Collector +from envs import SingleProcessEnv, MultiProcessEnv +from episode import Episode +from make_reconstructions import make_reconstructions_from_batch +from models.actor_critic import ActorCritic +from models.world_model import WorldModel +from utils import configure_optimizer, EpisodeDirManager, set_seed + + +class Trainer: + def __init__(self, cfg: DictConfig) -> None: + wandb.init( + config=OmegaConf.to_container(cfg, resolve=True), + reinit=True, + resume=True, + **cfg.wandb + ) + + if cfg.common.seed is not None: + set_seed(cfg.common.seed) + + self.cfg = cfg + self.start_epoch = 1 + self.device = torch.device(cfg.common.device) + + self.ckpt_dir = Path('checkpoints') + self.media_dir = Path('media') + self.episode_dir = self.media_dir / 'episodes' + self.reconstructions_dir = self.media_dir / 'reconstructions' + + if not cfg.common.resume: + config_dir = Path('config') + config_path = config_dir / 'trainer.yaml' + config_dir.mkdir(exist_ok=False, parents=False) + shutil.copy('.hydra/config.yaml', config_path) + wandb.save(str(config_path)) + shutil.copytree(src=(Path(hydra.utils.get_original_cwd()) / "src"), dst="./src") + shutil.copytree(src=(Path(hydra.utils.get_original_cwd()) / "scripts"), dst="./scripts") + self.ckpt_dir.mkdir(exist_ok=False, parents=False) + self.media_dir.mkdir(exist_ok=False, parents=False) + self.episode_dir.mkdir(exist_ok=False, parents=False) + self.reconstructions_dir.mkdir(exist_ok=False, parents=False) + + episode_manager_train = EpisodeDirManager(self.episode_dir / 'train', max_num_episodes=cfg.collection.train.num_episodes_to_save) + episode_manager_test = EpisodeDirManager(self.episode_dir / 'test', max_num_episodes=cfg.collection.test.num_episodes_to_save) + self.episode_manager_imagination = EpisodeDirManager(self.episode_dir / 'imagination', max_num_episodes=cfg.evaluation.actor_critic.num_episodes_to_save) + + def create_env(cfg_env, num_envs): + env_fn = lambda: instantiate(cfg_env) + return MultiProcessEnv(env_fn, num_envs, should_wait_num_envs_ratio=1.0) if num_envs > 1 else SingleProcessEnv(env_fn) + + if self.cfg.training.should: + train_env = create_env(cfg.env.train, cfg.collection.train.num_envs) + self.train_dataset = instantiate(cfg.datasets.train) + self.train_collector = Collector(train_env, self.train_dataset, episode_manager_train) + + if self.cfg.evaluation.should: + test_env = create_env(cfg.env.test, cfg.collection.test.num_envs) + self.test_dataset = instantiate(cfg.datasets.test) + self.test_collector = Collector(test_env, self.test_dataset, episode_manager_test) + + assert self.cfg.training.should or self.cfg.evaluation.should + env = train_env if self.cfg.training.should else test_env + + tokenizer = instantiate(cfg.tokenizer) + world_model = WorldModel(obs_vocab_size=tokenizer.vocab_size, act_vocab_size=env.num_actions, config=instantiate(cfg.world_model)) + actor_critic = ActorCritic(**cfg.actor_critic, act_vocab_size=env.num_actions) + self.agent = Agent(tokenizer, world_model, actor_critic).to(self.device) + print(f'{sum(p.numel() for p in self.agent.tokenizer.parameters())} parameters in agent.tokenizer') + print(f'{sum(p.numel() for p in self.agent.world_model.parameters())} parameters in agent.world_model') + print(f'{sum(p.numel() for p in self.agent.actor_critic.parameters())} parameters in agent.actor_critic') + + self.optimizer_tokenizer = torch.optim.Adam(self.agent.tokenizer.parameters(), lr=cfg.training.learning_rate) + self.optimizer_world_model = configure_optimizer(self.agent.world_model, cfg.training.learning_rate, cfg.training.world_model.weight_decay) + self.optimizer_actor_critic = torch.optim.Adam(self.agent.actor_critic.parameters(), lr=cfg.training.learning_rate) + + if cfg.initialization.path_to_checkpoint is not None: + self.agent.load(**cfg.initialization, device=self.device) + + if cfg.common.resume: + self.load_checkpoint() + + def run(self) -> None: + + for epoch in range(self.start_epoch, 1 + self.cfg.common.epochs): + + print(f"\nEpoch {epoch} / {self.cfg.common.epochs}\n") + start_time = time.time() + to_log = [] + + if self.cfg.training.should: + if epoch <= self.cfg.collection.train.stop_after_epochs: + to_log += self.train_collector.collect(self.agent, epoch, **self.cfg.collection.train.config) + to_log += self.train_agent(epoch) + + if self.cfg.evaluation.should and (epoch % self.cfg.evaluation.every == 0): + self.test_dataset.clear() + to_log += self.test_collector.collect(self.agent, epoch, **self.cfg.collection.test.config) + to_log += self.eval_agent(epoch) + + if self.cfg.training.should: + self.save_checkpoint(epoch, save_agent_only=not self.cfg.common.do_checkpoint) + + to_log.append({'duration': (time.time() - start_time) / 3600}) + for metrics in to_log: + wandb.log({'epoch': epoch, **metrics}) + + self.finish() + + def train_agent(self, epoch: int) -> None: + self.agent.train() + self.agent.zero_grad() + + metrics_tokenizer, metrics_world_model, metrics_actor_critic = {}, {}, {} + + cfg_tokenizer = self.cfg.training.tokenizer + cfg_world_model = self.cfg.training.world_model + cfg_actor_critic = self.cfg.training.actor_critic + + w = self.cfg.training.sampling_weights + + if epoch > cfg_tokenizer.start_after_epochs: + metrics_tokenizer = self.train_component(self.agent.tokenizer, self.optimizer_tokenizer, sequence_length=1, sample_from_start=True, sampling_weights=w, **cfg_tokenizer) + self.agent.tokenizer.eval() + + if epoch > cfg_world_model.start_after_epochs: + metrics_world_model = self.train_component(self.agent.world_model, self.optimizer_world_model, sequence_length=self.cfg.common.sequence_length, sample_from_start=True, sampling_weights=w, tokenizer=self.agent.tokenizer, **cfg_world_model) + self.agent.world_model.eval() + + if epoch > cfg_actor_critic.start_after_epochs: + metrics_actor_critic = self.train_component(self.agent.actor_critic, self.optimizer_actor_critic, sequence_length=1 + self.cfg.training.actor_critic.burn_in, sample_from_start=False, sampling_weights=w, tokenizer=self.agent.tokenizer, world_model=self.agent.world_model, **cfg_actor_critic) + self.agent.actor_critic.eval() + + return [{'epoch': epoch, **metrics_tokenizer, **metrics_world_model, **metrics_actor_critic}] + + def train_component(self, component: nn.Module, optimizer: torch.optim.Optimizer, steps_per_epoch: int, batch_num_samples: int, grad_acc_steps: int, max_grad_norm: Optional[float], sequence_length: int, sampling_weights: Optional[Tuple[float]], sample_from_start: bool, **kwargs_loss: Any) -> Dict[str, float]: + loss_total_epoch = 0.0 + intermediate_losses = defaultdict(float) + + for _ in tqdm(range(steps_per_epoch), desc=f"Training {str(component)}", file=sys.stdout): + optimizer.zero_grad() + for _ in range(grad_acc_steps): + batch = self.train_dataset.sample_batch(batch_num_samples, sequence_length, sampling_weights, sample_from_start) + batch = self._to_device(batch) + + losses = component.compute_loss(batch, **kwargs_loss) / grad_acc_steps + loss_total_step = losses.loss_total + loss_total_step.backward() + loss_total_epoch += loss_total_step.item() / steps_per_epoch + + for loss_name, loss_value in losses.intermediate_losses.items(): + intermediate_losses[f"{str(component)}/train/{loss_name}"] += loss_value / steps_per_epoch + + if max_grad_norm is not None: + torch.nn.utils.clip_grad_norm_(component.parameters(), max_grad_norm) + + optimizer.step() + + metrics = {f'{str(component)}/train/total_loss': loss_total_epoch, **intermediate_losses} + return metrics + + @torch.no_grad() + def eval_agent(self, epoch: int) -> None: + self.agent.eval() + + metrics_tokenizer, metrics_world_model = {}, {} + + cfg_tokenizer = self.cfg.evaluation.tokenizer + cfg_world_model = self.cfg.evaluation.world_model + cfg_actor_critic = self.cfg.evaluation.actor_critic + + if epoch > cfg_tokenizer.start_after_epochs: + metrics_tokenizer = self.eval_component(self.agent.tokenizer, cfg_tokenizer.batch_num_samples, sequence_length=1) + + if epoch > cfg_world_model.start_after_epochs: + metrics_world_model = self.eval_component(self.agent.world_model, cfg_world_model.batch_num_samples, sequence_length=self.cfg.common.sequence_length, tokenizer=self.agent.tokenizer) + + if epoch > cfg_actor_critic.start_after_epochs: + self.inspect_imagination(epoch) + + if cfg_tokenizer.save_reconstructions: + batch = self._to_device(self.test_dataset.sample_batch(batch_num_samples=3, sequence_length=self.cfg.common.sequence_length)) + make_reconstructions_from_batch(batch, save_dir=self.reconstructions_dir, epoch=epoch, tokenizer=self.agent.tokenizer) + + return [metrics_tokenizer, metrics_world_model] + + @torch.no_grad() + def eval_component(self, component: nn.Module, batch_num_samples: int, sequence_length: int, **kwargs_loss: Any) -> Dict[str, float]: + loss_total_epoch = 0.0 + intermediate_losses = defaultdict(float) + + steps = 0 + pbar = tqdm(desc=f"Evaluating {str(component)}", file=sys.stdout) + for batch in self.test_dataset.traverse(batch_num_samples, sequence_length): + batch = self._to_device(batch) + + losses = component.compute_loss(batch, **kwargs_loss) + loss_total_epoch += losses.loss_total.item() + + for loss_name, loss_value in losses.intermediate_losses.items(): + intermediate_losses[f"{str(component)}/eval/{loss_name}"] += loss_value + + steps += 1 + pbar.update(1) + + intermediate_losses = {k: v / steps for k, v in intermediate_losses.items()} + metrics = {f'{str(component)}/eval/total_loss': loss_total_epoch / steps, **intermediate_losses} + return metrics + + @torch.no_grad() + def inspect_imagination(self, epoch: int) -> None: + mode_str = 'imagination' + batch = self.test_dataset.sample_batch(batch_num_samples=self.episode_manager_imagination.max_num_episodes, sequence_length=1 + self.cfg.training.actor_critic.burn_in, sample_from_start=False) + outputs = self.agent.actor_critic.imagine(self._to_device(batch), self.agent.tokenizer, self.agent.world_model, horizon=self.cfg.evaluation.actor_critic.horizon, show_pbar=True) + + to_log = [] + for i, (o, a, r, d) in enumerate(zip(outputs.observations.cpu(), outputs.actions.cpu(), outputs.rewards.cpu(), outputs.ends.long().cpu())): # Make everything (N, T, ...) instead of (T, N, ...) + episode = Episode(o, a, r, d, torch.ones_like(d)) + episode_id = (epoch - 1 - self.cfg.training.actor_critic.start_after_epochs) * outputs.observations.size(0) + i + self.episode_manager_imagination.save(episode, episode_id, epoch) + + metrics_episode = {k: v for k, v in episode.compute_metrics().__dict__.items()} + metrics_episode['episode_num'] = episode_id + metrics_episode['action_histogram'] = wandb.Histogram(episode.actions.numpy(), num_bins=self.agent.world_model.act_vocab_size) + to_log.append({f'{mode_str}/{k}': v for k, v in metrics_episode.items()}) + + return to_log + + def _save_checkpoint(self, epoch: int, save_agent_only: bool) -> None: + torch.save(self.agent.state_dict(), self.ckpt_dir / 'last.pt') + if not save_agent_only: + torch.save(epoch, self.ckpt_dir / 'epoch.pt') + torch.save({ + "optimizer_tokenizer": self.optimizer_tokenizer.state_dict(), + "optimizer_world_model": self.optimizer_world_model.state_dict(), + "optimizer_actor_critic": self.optimizer_actor_critic.state_dict(), + }, self.ckpt_dir / 'optimizer.pt') + ckpt_dataset_dir = self.ckpt_dir / 'dataset' + ckpt_dataset_dir.mkdir(exist_ok=True, parents=False) + self.train_dataset.update_disk_checkpoint(ckpt_dataset_dir) + if self.cfg.evaluation.should: + torch.save(self.test_dataset.num_seen_episodes, self.ckpt_dir / 'num_seen_episodes_test_dataset.pt') + + def save_checkpoint(self, epoch: int, save_agent_only: bool) -> None: + tmp_checkpoint_dir = Path('checkpoints_tmp') + shutil.copytree(src=self.ckpt_dir, dst=tmp_checkpoint_dir, ignore=shutil.ignore_patterns('dataset')) + self._save_checkpoint(epoch, save_agent_only) + shutil.rmtree(tmp_checkpoint_dir) + + def load_checkpoint(self) -> None: + assert self.ckpt_dir.is_dir() + self.start_epoch = torch.load(self.ckpt_dir / 'epoch.pt') + 1 + self.agent.load(self.ckpt_dir / 'last.pt', device=self.device) + ckpt_opt = torch.load(self.ckpt_dir / 'optimizer.pt', map_location=self.device) + self.optimizer_tokenizer.load_state_dict(ckpt_opt['optimizer_tokenizer']) + self.optimizer_world_model.load_state_dict(ckpt_opt['optimizer_world_model']) + self.optimizer_actor_critic.load_state_dict(ckpt_opt['optimizer_actor_critic']) + self.train_dataset.load_disk_checkpoint(self.ckpt_dir / 'dataset') + if self.cfg.evaluation.should: + self.test_dataset.num_seen_episodes = torch.load(self.ckpt_dir / 'num_seen_episodes_test_dataset.pt') + print(f'Successfully loaded model, optimizer and {len(self.train_dataset)} episodes from {self.ckpt_dir.absolute()}.') + + def _to_device(self, batch: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]: + return {k: batch[k].to(self.device) for k in batch} + + def finish(self) -> None: + wandb.finish() diff --git a/src/utils.py b/src/utils.py new file mode 100644 index 0000000..3e3f8e3 --- /dev/null +++ b/src/utils.py @@ -0,0 +1,142 @@ +from collections import OrderedDict +from pathlib import Path +import random +import shutil + +import numpy as np +import torch +import torch.nn as nn + +from episode import Episode + + +def configure_optimizer(model, learning_rate, weight_decay, *blacklist_module_names): + """Credits to https://github.com/karpathy/minGPT""" + # separate out all parameters to those that will and won't experience regularizing weight decay + decay = set() + no_decay = set() + whitelist_weight_modules = (torch.nn.Linear, torch.nn.Conv1d) + blacklist_weight_modules = (torch.nn.LayerNorm, torch.nn.Embedding) + for mn, m in model.named_modules(): + for pn, p in m.named_parameters(): + fpn = '%s.%s' % (mn, pn) if mn else pn # full param name + if any([fpn.startswith(module_name) for module_name in blacklist_module_names]): + no_decay.add(fpn) + elif 'bias' in pn: + # all biases will not be decayed + no_decay.add(fpn) + elif pn.endswith('weight') and isinstance(m, whitelist_weight_modules): + # weights of whitelist modules will be weight decayed + decay.add(fpn) + elif pn.endswith('weight') and isinstance(m, blacklist_weight_modules): + # weights of blacklist modules will NOT be weight decayed + no_decay.add(fpn) + + # validate that we considered every parameter + param_dict = {pn: p for pn, p in model.named_parameters()} + inter_params = decay & no_decay + union_params = decay | no_decay + assert len(inter_params) == 0, f"parameters {str(inter_params)} made it into both decay/no_decay sets!" + assert len(param_dict.keys() - union_params) == 0, f"parameters {str(param_dict.keys() - union_params)} were not separated into either decay/no_decay set!" + + # create the pytorch optimizer object + optim_groups = [ + {"params": [param_dict[pn] for pn in sorted(list(decay))], "weight_decay": weight_decay}, + {"params": [param_dict[pn] for pn in sorted(list(no_decay))], "weight_decay": 0.0}, + ] + optimizer = torch.optim.AdamW(optim_groups, lr=learning_rate) + return optimizer + + +def init_weights(module): + if isinstance(module, (nn.Linear, nn.Embedding)): + module.weight.data.normal_(mean=0.0, std=0.02) + if isinstance(module, nn.Linear) and module.bias is not None: + module.bias.data.zero_() + elif isinstance(module, nn.LayerNorm): + module.bias.data.zero_() + module.weight.data.fill_(1.0) + + +def extract_state_dict(state_dict, module_name): + return OrderedDict({k.split('.', 1)[1]: v for k, v in state_dict.items() if k.startswith(module_name)}) + + +def set_seed(seed): + np.random.seed(seed) + torch.manual_seed(seed) + torch.cuda.manual_seed(seed) + random.seed(seed) + + +def remove_dir(path, should_ask=False): + assert path.is_dir() + if (not should_ask) or input(f"Remove directory : {path} ? [Y/n] ").lower() != 'n': + shutil.rmtree(path) + + +def compute_lambda_returns(rewards, values, ends, gamma, lambda_): + assert rewards.ndim == 2 or (rewards.ndim == 3 and rewards.size(2) == 1) + assert rewards.shape == ends.shape == values.shape, f"{rewards.shape}, {values.shape}, {ends.shape}" # (B, T, 1) + t = rewards.size(1) + lambda_returns = torch.empty_like(values) + lambda_returns[:, -1] = values[:, -1] + lambda_returns[:, :-1] = rewards[:, :-1] + ends[:, :-1].logical_not() * gamma * (1 - lambda_) * values[:, 1:] + + last = values[:, -1] + for i in list(range(t - 1))[::-1]: + lambda_returns[:, i] += ends[:, i].logical_not() * gamma * lambda_ * last + last = lambda_returns[:, i] + + return lambda_returns + + +class LossWithIntermediateLosses: + def __init__(self, **kwargs): + self.loss_total = sum(kwargs.values()) + self.intermediate_losses = {k: v.item() for k, v in kwargs.items()} + + def __truediv__(self, value): + for k, v in self.intermediate_losses.items(): + self.intermediate_losses[k] = v / value + self.loss_total = self.loss_total / value + return self + + +class EpisodeDirManager: + def __init__(self, episode_dir: Path, max_num_episodes: int) -> None: + self.episode_dir = episode_dir + self.episode_dir.mkdir(parents=False, exist_ok=True) + self.max_num_episodes = max_num_episodes + self.best_return = float('-inf') + + def save(self, episode: Episode, episode_id: int, epoch: int) -> None: + if self.max_num_episodes is not None and self.max_num_episodes > 0: + self._save(episode, episode_id, epoch) + + def _save(self, episode: Episode, episode_id: int, epoch: int) -> None: + ep_paths = [p for p in self.episode_dir.iterdir() if p.stem.startswith('episode_')] + assert len(ep_paths) <= self.max_num_episodes + if len(ep_paths) == self.max_num_episodes: + to_remove = min(ep_paths, key=lambda ep_path: int(ep_path.stem.split('_')[1])) + to_remove.unlink() + episode.save(self.episode_dir / f'episode_{episode_id}_epoch_{epoch}.pt') + + ep_return = episode.compute_metrics().episode_return + if ep_return > self.best_return: + self.best_return = ep_return + path_best_ep = [p for p in self.episode_dir.iterdir() if p.stem.startswith('best_')] + assert len(path_best_ep) in (0, 1) + if len(path_best_ep) == 1: + path_best_ep[0].unlink() + episode.save(self.episode_dir / f'best_episode_{episode_id}_epoch_{epoch}.pt') + + +class RandomHeuristic: + def __init__(self, num_actions): + self.num_actions = num_actions + + def act(self, obs): + assert obs.ndim == 4 # (N, H, W, C) + n = obs.size(0) + return torch.randint(low=0, high=self.num_actions, size=(n,))