mirror of
https://github.com/wassname/iris_bigvae.git
synced 2026-09-09 11:24:31 +08:00
603 lines
63 KiB
Plaintext
603 lines
63 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 1,
|
|
"metadata": {
|
|
"cellView": "form",
|
|
"id": "_F0jWGQZDgM3"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
|
|
"# you may not use this file except in compliance with the License.\n",
|
|
"# You may obtain a copy of the License at\n",
|
|
"#\n",
|
|
"# https://www.apache.org/licenses/LICENSE-2.0\n",
|
|
"#\n",
|
|
"# Unless required by applicable law or agreed to in writing, software\n",
|
|
"# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
|
|
"# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
|
|
"# See the License for the specific language governing permissions and\n",
|
|
"# limitations under the License."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"Credits to https://github.com/google-research/rliable"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 2,
|
|
"metadata": {
|
|
"id": "CrcaOCgbJLv6"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"import json\n",
|
|
"import math\n",
|
|
"from pathlib import Path\n",
|
|
"\n",
|
|
"import matplotlib.pyplot as plt\n",
|
|
"import matplotlib.patches as mpatches\n",
|
|
"from matplotlib.ticker import MaxNLocator\n",
|
|
"from matplotlib import rcParams\n",
|
|
"from matplotlib import rc\n",
|
|
"import numpy as np\n",
|
|
"import pandas as pd\n",
|
|
"import seaborn as sns\n",
|
|
"\n",
|
|
"from rliable import library as rly\n",
|
|
"from rliable import metrics\n",
|
|
"from rliable import plot_utils\n",
|
|
"\n",
|
|
"rcParams['legend.loc'] = 'best'\n",
|
|
"rcParams['pdf.fonttype'] = 42\n",
|
|
"rcParams['ps.fonttype'] = 42\n",
|
|
"rc('text', usetex=False)\n",
|
|
"RAND_STATE = np.random.RandomState(42)\n",
|
|
"sns.set_style(\"white\")\n",
|
|
"\n",
|
|
"Path('figures').mkdir(exist_ok=True, parents=False)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 3,
|
|
"metadata": {
|
|
"cellView": "form",
|
|
"id": "S6FnrWyKnoHu"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"def score_normalization(res_dict, min_scores, max_scores):\n",
|
|
" games = res_dict.keys()\n",
|
|
" norm_scores = {}\n",
|
|
" for game, scores in res_dict.items():\n",
|
|
" norm_scores[game] = (scores - min_scores[game])/(max_scores[game] - min_scores[game])\n",
|
|
" return norm_scores\n",
|
|
"\n",
|
|
"def convert_to_matrix(score_dict):\n",
|
|
" keys = sorted(list(score_dict.keys()))\n",
|
|
" return np.stack([score_dict[k] for k in keys], axis=1)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 4,
|
|
"metadata": {
|
|
"id": "58NWt0zVvT_L"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"StratifiedBootstrap = rly.StratifiedBootstrap\n",
|
|
"\n",
|
|
"IQM = lambda x: metrics.aggregate_iqm(x) # Interquartile Mean\n",
|
|
"OG = lambda x: metrics.aggregate_optimality_gap(x, 1.0) # Optimality Gap\n",
|
|
"MEAN = lambda x: metrics.aggregate_mean(x)\n",
|
|
"MEDIAN = lambda x: metrics.aggregate_median(x)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 5,
|
|
"metadata": {
|
|
"cellView": "form",
|
|
"id": "ToCkLC2irWK-"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"ATARI_100K_GAMES = [\n",
|
|
" 'Alien', 'Amidar', 'Assault', 'Asterix', 'BankHeist', 'BattleZone',\n",
|
|
" 'Boxing', 'Breakout', 'ChopperCommand', 'CrazyClimber', 'DemonAttack',\n",
|
|
" 'Freeway', 'Frostbite', 'Gopher', 'Hero', 'Jamesbond', 'Kangaroo',\n",
|
|
" 'Krull', 'KungFuMaster', 'MsPacman', 'Pong', 'PrivateEye', 'Qbert',\n",
|
|
" 'RoadRunner', 'Seaquest', 'UpNDown'\n",
|
|
"]"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 6,
|
|
"metadata": {
|
|
"cellView": "form",
|
|
"id": "PqvaxPEIpACt"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"def read_curl_scores():\n",
|
|
" df = pd.read_json('data/CURL_10_evals.json', orient='records')\n",
|
|
" score_dict = {'Max': {}, 'Final': {}}\n",
|
|
" for game in ATARI_100K_GAMES:\n",
|
|
" game_df = df[df['game'] == game]\n",
|
|
" score_dict['Final'][game] = game_df['HNS'].values\n",
|
|
" score_dict['Max'][game] = game_df['Max HNS'].values\n",
|
|
" score_matrices = {}\n",
|
|
" for key, val in score_dict.items():\n",
|
|
" score_matrices[key] = convert_to_matrix(val) \n",
|
|
" return score_dict['Final'], score_matrices['Final']\n",
|
|
" \n",
|
|
"def load_json_scores(algorithm_name, normalize=True):\n",
|
|
" path = Path('data') / f'{algorithm_name}.json'\n",
|
|
" with path.open('r') as f:\n",
|
|
" raw_scores = json.load(f)\n",
|
|
" raw_scores = {game: np.array(val) for game, val in raw_scores.items()}\n",
|
|
" if normalize:\n",
|
|
" hn_scores = score_normalization(raw_scores, RANDOM_SCORES, HUMAN_SCORES)\n",
|
|
" hn_score_matrix = convert_to_matrix(hn_scores)\n",
|
|
" else:\n",
|
|
" hn_scores, hn_score_matrix = None, None\n",
|
|
" return hn_scores, hn_score_matrix, raw_scores\n",
|
|
"\n",
|
|
"def save_fig(fig, name):\n",
|
|
" fig.savefig(f'figures/{name}.pdf', format='pdf', bbox_inches='tight')"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 7,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"_, _, raw_scores_random = load_json_scores('RANDOM', normalize=False)\n",
|
|
"_, _, raw_scores_human = load_json_scores('HUMAN', normalize=False)\n",
|
|
"RANDOM_SCORES = {k: v[0] for k, v in raw_scores_random.items()}\n",
|
|
"HUMAN_SCORES = {k: v[0] for k, v in raw_scores_human.items()}"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 8,
|
|
"metadata": {
|
|
"colab": {
|
|
"base_uri": "https://localhost:8080/"
|
|
},
|
|
"id": "3ls4nMkmBWb8",
|
|
"outputId": "686d429e-3df3-49b5-d176-da980b33b85b"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"score_dict_muzero, score_muzero, raw_scores_muzero = load_json_scores('MuZero')\n",
|
|
"score_dict_efficientzero, score_efficientzero, raw_scores_efficientzero = load_json_scores('EfficientZero')\n",
|
|
"score_dict_simple, score_simple, _ = load_json_scores('SimPLe')\n",
|
|
"score_dict_curl, score_curl = read_curl_scores()\n",
|
|
"score_dict_drq_eps, score_drq_eps, _ = load_json_scores('DrQ(eps)')\n",
|
|
"score_dict_spr, score_spr, _ = load_json_scores('SPR')\n",
|
|
"score_dict_iris, score_iris, _ = load_json_scores('IRIS')\n",
|
|
"\n",
|
|
"score_data_dict_games = {\n",
|
|
" 'MuZero': score_dict_muzero,\n",
|
|
" 'EfficientZero': score_dict_efficientzero,\n",
|
|
" 'SimPLe': score_dict_simple,\n",
|
|
" 'CURL': score_dict_curl,\n",
|
|
" 'DrQ': score_dict_drq_eps, \n",
|
|
" 'SPR': score_dict_spr,\n",
|
|
" 'IRIS (ours)': score_dict_iris\n",
|
|
"}\n",
|
|
"\n",
|
|
"all_score_dict = {\n",
|
|
" 'MuZero': score_muzero, \n",
|
|
" 'EfficientZero': score_efficientzero,\n",
|
|
" 'SimPLe': score_simple,\n",
|
|
" 'CURL': score_curl,\n",
|
|
" 'DrQ': score_drq_eps, \n",
|
|
" 'SPR': score_spr,\n",
|
|
" 'IRIS (ours)': score_iris\n",
|
|
"}"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 9,
|
|
"metadata": {
|
|
"id": "43m4aASsQ-nU"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"colors = sns.color_palette('colorblind')\n",
|
|
"xlabels = ['SimPLe', 'CURL', 'DrQ', 'SPR', 'IRIS (ours)']\n",
|
|
"color_idxs = [7, 4, 2, 1, 0]\n",
|
|
"ATARI_100K_COLOR_DICT = dict(zip(xlabels, [colors[idx] for idx in color_idxs]))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"id": "KgtEQsA08sIG"
|
|
},
|
|
"source": [
|
|
"# Mean, Median, IQM and Optimality Gap"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 10,
|
|
"metadata": {
|
|
"colab": {
|
|
"base_uri": "https://localhost:8080/"
|
|
},
|
|
"id": "3RhatO5S8tWt",
|
|
"outputId": "84aa63d0-8fc4-46aa-ddd2-9d5951461fd7"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"aggregate_func = lambda x: np.array([MEAN(x), MEDIAN(x), IQM(x), OG(x)])\n",
|
|
"aggregate_scores, aggregate_interval_estimates = rly.get_interval_estimates(all_score_dict, aggregate_func, reps=50000)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 11,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"MuZero : 1 runs\n",
|
|
"EfficientZero : 1 runs\n",
|
|
"SimPLe : 5 runs\n",
|
|
"CURL : 100 runs\n",
|
|
"DrQ : 100 runs\n",
|
|
"SPR : 100 runs\n",
|
|
"IRIS (ours) : 3 runs\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"for algo in aggregate_scores.keys():\n",
|
|
" n_runs, n_games = all_score_dict[algo].shape\n",
|
|
" assert n_games == len(ATARI_100K_GAMES)\n",
|
|
" print(f\"{algo.ljust(14)}: {n_runs:3d} runs\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 12,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"image/png": "iVBORw0KGgoAAAANSUhEUgAABEIAAAEdCAYAAAAIIsL8AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAABLs0lEQVR4nO3dd3RU1drH8V8KSSC00CEBDBHCBYKEEor0orRI70URKSKCgAgoVgSUiyKGorRQovTc0EWaKFKvghRphpaABiSBmED6vH/4Zi7jJKRN6nw/a7GWs88+Z++zZ84zk8ez97ExGAwGAQAAAAAAWAHb3O4AAAAAAABATiERAgAAAAAArAaJEAAAAAAAYDVIhAAAAAAAAKtBIgQAAAAAAFgNEiEAAAAAAMBqkAgBAACwkKlTp6pt27YmZW3bttXUqVNzqUcAkL8dO3ZMnp6eOnbsmLHMz89Pnp6eudgr5HckQoB8LPmLwdPTU0uXLk2xzooVK4x1Hv0CAYCCitgIIKclx52NGzdmav+QkBD5+fnp/PnzFu5Z/pCUlCQ/Pz/t3bs31/qQnFzx9PTU6dOnU6wzevRoYx3kbyRCgALA0dFRW7duTXFbUFCQHB0dc7hHAJD78kps/OabbzRjxowcaQtA/hQSEqIFCxZYdSJkwYIFKSZCGjVqpNOnT6tRo0Y50hdHR0dt2bLFrDw8PFw//PADv6sLCBIhQAHQpk0bXbp0yezL88KFC7p48aLZbdoAYA3ySmx0cHBQoUKFcqQtAPinBw8e5HYXUpWYmKi4uLjH1rG1tZWjo6NsbXPmT9c2bdpox44dio+PNynfvn277Ozs1KxZsxzpB7IXiRCgAGjatKnKli1rlr3esmWLypUrp6ZNm5rtEx4ervfff1+tWrVSnTp11KZNG/373/9WbGysSb3AwEANHz5czZs3V506ddS6dWvNmDFDUVFRJvUevSU0ICBA7du3V506ddStWzcdPXrU8icNAGnIztgoSStXrlS7du3k5eWl7t2768CBAyn2459rhMTFxcnPz099+vRR48aN5eXlpc6dO2vlypUyGAwm+ybfqn3hwgV9+OGHatq0qZ566im99NJLunnzZmaGBUAOSe9vo8DAQA0bNkySNG3aNOPUCz8/P2Od9Mamtm3basCAATp16pQGDRqkp556Sh988IFxe0pxK6W1jf7ZfrKU6mb0t+KGDRu0fPlytW/fXl5eXtq+fbtq164tSfrPf/5jPP8hQ4aY7JeeaYwXLlzQ2LFjjbHV19c3w9OVunXrpoiICP3www8m5UFBQWrXrp2KFi2a6baJ/3mHfW53AEDW2dnZydfXV9u2bdPkyZNlZ2enxMREbdu2Tb6+vmYZ9IiICPXt21dRUVHq16+fKlWqpHPnzsnf318XLlzQsmXLZGNjI0kKCAiQu7u7XnjhBRUrVkznzp3T+vXrdfHiRQUEBJj1Ze3atXrw4IH69u0rBwcHrVq1SmPGjNGBAwdUokSJHBkPAJCyNzYuXrxYn332mRo2bKjnn39eYWFhmjRpklxdXdPsV1RUlL7++mt16tRJzz33nCTpxx9/1OzZs3X//n2NHz/ebJ8333xTJUuW1CuvvKI7d+5o5cqVmjx5sr7++msLjBSA7JTWb6NGjRpp5MiRWrJkifr166cGDRpIknEdiozEJkkKCwvTqFGj9Nxzz6lbt27GP9yzErceJ6O/FVevXq34+Hj17dtXTk5OqlWrlj7++GNNmTJFDRs2VN++fSVJZcqUyVA/fv75Z7344ouqXLmyhg8fLmdnZx04cEDTp09XeHi4Ro0ala7jeHh4qHbt2tqyZYsx6fPbb7/p3LlzGjdunLZv357pton/eQeJEKCA6Natm1asWKEff/xRLVu21OHDh3Xnzh11797dbMGn+fPn6/79+woKCjL58qtRo4ZmzJihH374QS1btpQkffXVVypcuLDJ/t7e3po6dapOnjwpb29vk213797Vzp075ezsLElq3Lixunfvru3bt2vQoEHZceoAkKrsiI0RERFatGiRvL29tWrVKtnb//1zysfHRyNHjkzzj4oSJUro4MGDcnBwMJYNGTJE06ZN08qVK/Xyyy+bbJOkihUrauHChcbXLi4umj17tn777Tc9+eSTmR4fANkvrd9GlStXVtOmTbVkyRLVq1dP3bp1M9k/I7/bJOnmzZuaO3eufH19jWVZjVuPk5nfirt371bx4sWNZU8++aSmTJmiypUrm51/ehgMBr399tuqXr26vv76a+N0xEGDBmncuHFatGiR+vfvn+7/Kde9e3fNnTtXkZGRKl68uLZs2aLSpUurefPmZomQjLRN/M87mBoDFBA1a9aUp6en8RbwoKAgY9mjDAaDdu3apaefflqFCxdWeHi48d/TTz8tSTpy5IixfvIXW1JSkv766y+Fh4erYcOGkqQzZ86Y9aNbt27GL3pJ+te//qWiRYsqJCTEsicMAOmQHbHx8OHDiouL0+DBg41/TEhSq1at5OHhkWaf7OzsjD90ExISdP/+fYWHh6tJkyZ68OCBrly5YrbPgAEDTF77+PhIkm7cuJHeoQCQS7Ly2yijv9skqWTJkurSpYtJWVbj1uNk9Ldi165dTZIglnDx4kX99ttv8vX1NfYh+V+rVq0UExOjkydPpvt4Xbp0UWJior755hslJSVp27Zt6tKli8nYZaZt4n/ewR0hQAHSrVs3+fn5KSwsTPv27dO4cePM6oSHh+vevXvatWuXdu3aleJxwsPDjf99+vRpzZs3Tz/99JPZPNTIyEizfStVqmRWVqJECd2/fz+jpwMAFmHp2Jg8N9vd3d2sjru7e7qe+rBlyxb5+/vr0qVLSkxMNNn2119/mdX/Z2xN/iOC2ArkfVn5bZTR322S5Orqajb1zxJxKzUZ/a1YpUqVTLeVmuQEwsyZMzVz5swU69y9ezfdx0u++2PLli2qXLmyfv/991TvVMlo28T/vIFECFCAdO3aVZ988oneeOMNxcXFqWvXrmZ1khdiat++fapTVcqWLStJCg0N1dChQ+Xq6qrJkyfL1dVVTk5OSkxM1EsvvWS2qJP0d6Y7JSnVBYCcYOnYmFU7d+7UG2+8oZYtW2rw4MEqW7asChUqpHPnzmnu3LlKSkoy24fYCuRfWbl+MxObsvPxrv/8wz0zvxWdnJyyrX9jx441rrHyTxm986Vbt26aOHGiFi9erCeffFJ16tTJctvE/7yDRAhQgJQvX15NmzbVoUOH1Lx5c5UrV86sTqlSpVS0aFHFxsam+fivffv26eHDh/ryyy/l5uZmLE/ptj0AyKssHRuT59JfvXrV+KSDZFevXk2zPzt27JCbm5u+/PJLk/9ry23OgPV6dLHTR2UkNj1ORuNWanes/HM6j6V+K6Z2/umVfJeJo6OjxR5vm/yEmGPHjmnSpEkWaZv4n3ewRghQwIwdO9b4LyW2trbq3LmzDh06pOPHj5ttj42NNT7uLDkD/c+M89KlSy3cawDIXpaMjc2aNZODg4MCAgKUkJBgrHPw4EEFBwen2ZeUYmtMTIzWrFmToXMCUHAUKVJEkvlUkozEpsfJaNyqWrWq2eNqf/nlF506dcqkzFK/Fe3s7OTo6Jjp6R61atWSu7u7Vq9ebTZVSMrYtJhkjo6Oeu+99zR27Fh1797dIm0T//MO7ggBChhvb2+z1bn/aeLEiTp+/LhefPFFde/eXbVq1VJsbKyuXr2q3bt36/PPP1fjxo3VokULOTo6auTIkerXr59sbW114MAB5iQCyHcsGRtdXFw0atQo+fn56fnnn1fHjh0VFhamr7/+WjVq1FB0dPRj22nXrp12796tkSNHqkOHDrp//74CAwNNFlMEYF2efPJJFS5cWGvXrlWRIkXk7Oys6tWrq0aNGumOTY+T0bg1YMAATZs2TaNGjVLr1q118+ZNbdy4UTVq1DBJvFjyt6KXl5eOHDmi5cuXq0KFCipVqpSaNm2arn1tbW01e/Zsvfjii+rcubN69+6tKlWqKDw8XOfPn9e+fft09uzZDPcppamUWWmb+J93kAgBrJCLi4s2btyoL774Qnv37lVQUJCcnZ3l5uamwYMHG5+mULVqVX355Zf69NNP9dlnn8nJyUmtW7fWJ598ku4vJgDIL9IbG6W/7zApUqSIAgICNGfOHHl4eOiTTz7R7t27U/y/to/q1q2bIiMjFRAQoJkzZ6ps2bLq2bOn6tevr2HDhmX3aQLIg5ydnfXxxx/Lz89PH3zwgeLj4zV27FjVqFEjQ7HpcTISt3r06KFbt25p3bp1Onz4sDw9PTV//nwFBQWZ1LXkb8X33ntP77//vvz8/PTw4UP5+Phk6Bje3t7atGmTFi9erKCgIN27d08uLi7y8PDQtGnTMtSXjEpv28T/vMPGwCorAAAAAGCVpk6dquPHj2v//v253RUgx7BGCAAAAAAAsBokQgAAAAAAgNUgEQIAAAAAAKwGa4QAAAAAAACrwR0hAAAAAADAapAIAf5fTExMbnehwGFMLY8xtV6893kb70/exXuTP/G+WRbjaVmMp2XlxniSCAH+X2JiYm53ocBhTC2PMbVevPd5G+9P3sV7kz/xvlkW42lZjKdl5cZ4kggBAAAAAABWg0QIAAAAAACwGiRCAAAAAACA1SARAgAAAAAArAaJEAAAAAAAYDVIhAAAAAAAAKthn9sdAID8LOJBnGISknKsvaQkG0VG5u6z653sbeVSxCFX+wBkVHZfq3nh2kTK8sp7Q+wECq6c/j1Y0KQVp7MjfpIIAYAsiElI0qCvfs6x9hITE2VnZ5dj7aXkq0H1c7V9IDOy+1rNC9cmUpZX3htiJ1Bw5fTvwYImrTidHfGTqTEAAAAAAMBqkAgBgHzm9yM79PuRHbndDQD5HLEEAKwHMd8UiRAAAAAAAGA1CkQi5PPPP9ezzz6rxMTE3O7KY0VHR6tx48bauHFjbncFAAAAAACrlGYi5NixY/L09DT54z00NFSenp4m/+rXr68BAwZoxw7z222S68+bN8+k/P79+5o3b558fX3l7e2thg0bqlOnTpo6dapOnTqVrhP4888/5e/vr1GjRuWJhbAex9nZWUOGDJGfn59iYnJ/9XIAAAAAAKxNlp4a065dOz377LMyGAy6ffu2Nm7cqIkTJyomJka9evV67L5//fWXevXqpT///FPPPfecBg4cqNjYWF27dk0//PCDXF1dVa9evTT7EBAQIHt7e3Xt2jUrp5Jj+vXrp4ULFyooKEj9+/fP7e4AAAAAAGBVspQIqV69urp162Z83bNnT3Xo0EErVqxIMxGyYcMGhYSEaOHChWrfvr3JNoPBoPDw8DTbT0xMVGBgoJ555hk5OOTOc9kTExOVmJiY7vbLli1rnB5DIgQAAAAAgJxl0TVCypQpo2rVqun69etp1k2u4+PjY7bNxsZGpUuXTvMYp06dUlhYmJo3b57i9qCgIPXo0UN169aVj4+PXn31VQUHB5vUCQwMlKenp0JDQ8329/T0lJ+fn/F18jShDRs2aPny5Wrfvr28vLx08uRJSdLu3bvVt29fNWzYUPXq1VOHDh309ttvmx23efPmOnv2rG7dupXmOQIAAAAAAMvJ0h0h/5SQkKCwsDCVLFkyzbpubm6SpM2bN2vYsGGZau+nn36SJNWtW9ds2/LlyzVnzhzVrVtXEydO1P379xUQEKD+/ftr06ZNqlq1aqbalKTVq1crPj5effv2lZOTk8qWLasjR45o/PjxatSokV577TXZ29srJCREBw4cMNv/qaeekiT997//1XPPPZfpfgAAAAAAgIzJUiIkNjbWOIUlLCxMy5Yt0507d9KV2OjTp49Wr16tjz76SGvWrFGjRo3k7e2tli1bqlKlSulqPzg4WHZ2dmb1IyIiNH/+fNWuXVtfffWVcdpK+/bt1bt3b33yySf6/PPPM3i2/3P37l3t3r1bxYsXN5atW7dOzs7O8vf3l739/4Z18uTJZvtXqVLF2H8AAAAAAJBzspQI8ff3l7+/v/G1nZ2dBg0apIkTJ6a5r4uLizZv3qylS5fq22+/VVBQkIKCgiT9vQjrjBkz0pweEx4ermLFisnGxsak/PDhw4qNjdXzzz9vsnZH7dq11axZMx08eFAJCQkmCYuM6Nq1q0kSRJKKFy+uhw8f6ocfflDr1q3N+vSo5DtmIiIiMtU+AAAAAADInCwlQnx9fdWzZ0/Fx8fr3LlzWrJkif766y8VKlQoXfuXL19e06dP1/Tp0xUWFqaffvpJ69ev1759+xQdHa1Vq1Zlql83b96UJHl4eJht8/Dw0KFDh3T37l2VL18+U8dPvqPjUQMHDtTu3bs1evRolS5dWk2aNFGbNm3UsWNHs/FISkrKVLsAAAAAACBrspQIcXV1VbNmzSRJrVq1kpubmyZPnqx69epp0KBBGTpW+fLl1blzZ3Xq1EmDBw/W0aNH9fvvv6tixYqp7lOqVCn99ddfMhgMj70D43FS2y8xMTHVfZycnFLsy3/+8x8dPXpUP/zwgw4fPqwdO3Zo2bJl+vrrr+Xs7GysGxkZKenvu2IAAAAAAEDOsehTY5577jnVqVNHfn5+io6OztQxbGxs5OXlJenvdUcex8PDQ4mJicY7QJIlL8Sa0hocwcHBKly4sHHaTfIUl+TkRLKQkJAM993e3l7NmzfXtGnTtG3bNr377ru6cOGCtm/fblLvxo0bxv4DAAAAAICcY9FEiCSNGjVKERERWrt27WPrnTp1SlFRUWblcXFxOnz4sOzs7NJ8skuDBg0kSWfOnDEpb9q0qRwdHbVmzRrFxcUZyy9cuKDDhw+rZcuWxvVBnnjiCUnS0aNHTY6R0Wk5Ka33UatWLUnS/fv3Tcp/+eUXk/4DAAAAAICcYdHH50p/P5mlWrVq8vf315AhQ+To6Jhiva1bt+o///mP2rZtKy8vLxUrVkxhYWHasWOHfvvtN40YMSLNqSP16tVTuXLldOjQIXXq1MlY7uLiovHjx2vOnDkaPHiwunTpYnx8btGiRU0Wc/Xw8JCPj48+++wz3bt3T+XLl9ePP/6o27dvZ+i8p0+frrt376pp06aqVKmSIiIitG7dOjk5OalDhw4mdQ8dOqTatWvL1dU1Q20AAAAAAICssXgixNbWViNGjNC0adO0YcMGDRkyJMV6/fv3V9GiRXXkyBEdOXJE9+/fl7Ozszw9PTVnzhx169Ytzbbs7OzUs2dPrV27Vu+++67JE2KGDx+u0qVLa+XKlZo7d66cnJzUuHFjTZgwwXgXSLI5c+bo/fff1+rVq1WoUCG1adNGM2fOVJMmTdJ93s8995w2bdqkjRs36t69eypZsqS8vb318ssvy93d3Vjvzz//1PHjx/X222+n+9gAAAAAAMAybAwGgyG3O5EVd+7c0TPPPKN33nlHPXr0yO3upGnBggVav369vv32WxUuXDi3u4NHREdHmyxqi6yzhjH9PTJGg776OcfaS0xM1O3j30iSKjbtkmPtPuqrQfVVsbj5otHIXtZwPWWn7L5WExMTZWdnl23Hzw6/H9khKfdiSU7JK+8NsTNjiHmWxXha1j/HM6d/D2ZGXo75acXp7IifFl8jJKeVLVtWw4YN0xdffPHYJ73kBdHR0VqzZo1effVVkiAAAAAAAOQCi0+NyQ3jxo3TuHHjcrsbaXJ2dtaxY8dyuxsAAAAAAFitApEIAQBrkhdvaQSQ/xBLAMB6EPNN5fs1QgBLYe6k5VnDmEY8iFNMQlKOtZeUlCRb29yd1ehkbyuXIg5pV4RFWcP1lJ2y+1rNC9cmUpZX3htiZ8YQ8yyL8bSsf45nTv8eLGjSitPZET+5IwQAsiCnf9T+/cXLYntARmX3tcq1mXfx3gDIbiQ5syY34nTup8cBAAAAAAByCIkQAAAAAABgNUiEAAAAAAAAq0EiBAAAAAAAWA0SIQAAAAAAwGqQCAEAAAAAAFaDRAgAAAAAALAaJEIAAAAAAIDVIBECAAAAAACsBokQAAAAAABgNUiEAAAAAAAAq0EiBAAAAAAAWA0SIQAAAAAAwGrY53YHAPxPYkyEDImxGdrHxs5Rdk4u2dQjAEBBk5nvmpzA9xmQN2U0ZnAtIz8gEQLkIYbEWN3ZNSRD+5TttCabegMAKIgy812TE/g+A/KmjMYMrmXkB0yNAQAAAAAAVoNECJDHrT9wS+sP3MrtbgAAkCq+qwDrwjWP/I5ECAAAAAAAsBokQgAAAAAAgNVgsdT/d/XqVS1evFgnT57UH3/8IWdnZ1WqVEkNGjTQSy+9pPLly+vYsWMaOnSocR8bGxsVL15cderU0YgRI9S0aVPjtiFDhuj48ePG14UKFVLFihXVvn17vfLKKypatGiOnh8AAAAAACARIkk6ffq0hgwZouLFi6tnz55yc3NTeHi4Ll++rC1btqh9+/YqX768sX7v3r3l4+OjpKQkXb9+XevWrdPw4cO1bNkyNWvWzFivRIkSeuuttyRJUVFROnLkiFasWKGzZ89qzRpWUwYAAAAAIKeRCJG0cOFC2draatOmTSYJD0mKiYlRQkKCSVm9evXUrVs34+tnnnlGPXr00PLly00SIU5OTib1Bg0apPHjx+ubb77R6dOnVbdu3Ww6IwAAAAAAkBLWCJF0/fp1ValSxSwJIv2dzEhrGkutWrVUsmRJhYaGptmWj4+PJOnGjRuZ6ywAAAAAAMg0EiGS3NzcdOXKFf3888+Z2j8iIkKRkZFycXFJs25ysqRkyZKZagsAAAAAAGQeU2MkjRo1SkeOHNHAgQNVs2ZNNWjQQA0aNFCLFi1UrFgxs/rR0dEKDw+XwWDQ9evX9emnnyopKUmdOnUyqZeUlKTw8HDjPocPH9ZXX32lcuXKqVGjRjlybgAAAAAA4H9IhEhq1KiR1q1bp2XLlunQoUM6f/68AgICVKhQIb344osaN26c7O3/N1SzZ8/W7Nmzja8LFy6skSNHmjxRRpLu3Llj8iQZSfLy8tLMmTPl6OiYvScFAAAAAADMkAj5f15eXpo/f76SkpJ07do1HT16VP7+/vryyy/l4OCgsWPHGusOGzZMLVu2lI2NjYoVK6Ynn3xSTk5OZsd0cXHRp59+KkkKCwvTypUrdffuXRUuXDjHzgsAAAAAAPwPiZB/sLW1VbVq1VStWjV16tRJHTp0UFBQkEkixMPDw+TpMKlxcHAwqdemTRt16dJFkyZN0oYNG2RjY5Mt5wAAAAAAAFLGYqmP4eLioipVqigsLMwixytZsqRefvllnT59Wjt37rTIMQEAAAAAQPqRCJF0+PBhJSYmmpWHhIQoODhYHh4eFmurT58+Kl26tL744gsZDAaLHRcAAAAAAKSNqTGSZs2apcjISLVr107Vq1eXvb29rl69qqCgIMXHx+u1116zWFuOjo56/vnn9emnn2rv3r3q0KGDxY4NAAAAAAAej0SIpClTpmjPnj06ceKEtm/frgcPHqhUqVJq1KiRhg0bJm9vb4u2N3DgQC1ZskSLFy8mEQIAAAAAQA4iESKpRYsWatGiRZr1GjdurIsXL6brmGvWrEl1W7FixfTTTz+lu38AAAAAAMAyWCMEAAAAAABYDRIhAAAAAADAajA1Bsjj+rWplNtdAADgsfiuAqwL1zzyOxIhQB5iY+eosp1SX18mtX0AAEivzHzX5AS+z4C8KaMxg2sZ+QGJECAPsXNyye0uAAAKOL5rAGQEMQMFEWuEAAAAAAAAq0EiBAAAAAAAWA0SIQAAAAAAwGqQCAEAAAAAAFaDRAgAAAAAALAaJEIAAAAAAIDVIBECAAAAAACsBokQAAAAAABgNUiEAAAAAAAAq0EiBAAAAAAAWA0SIQAAAAAAwGqQCAEAAAAAAFaDRAgAAAAAALAa9rndAcCSImIfKCYxwazcyc5eLo5FcqFHAIC8LLXvjYwyGJIU+SDSAj0yxfcXACn9sYqYAaQPiRAUKDGJCRp88Cuz8oBWg3KhNwCAvC61742MSkxMlJ2dnQV6ZIrvLwBS+mMVMQNIH6bGAAAAAAAAq0EiBAXS73uO6Pc9R3K7GwAA8J0EwOKIK0DWkAgBAAAAAABWgzVCJB07dkxDhw41vra3t1fRokX1xBNPqGHDhurXr5+qVKmS4ePu3btXa9eu1dmzZxUdHa1y5crp6aef1siRI1W5cmVLngIAAAAAAEgHEiGP6N27t3x8fGQwGHT//n2dP39eX3/9tVatWqW33npLAwYMSNdxDAaD3n77bW3cuFG1atXS8OHD5eLioitXrmjTpk3avn27FixYoKeffjqbzwgAAAAAADyKRMgj6tWrp27dupmUvf766xo1apTef/99Va1aVc2aNUtxX4PBoNjYWDk5OWnVqlXauHGj+vXrp/fee0+2tv+bgTR06FANHjxY48eP1/bt21WhQoVsPScAAAAAAPA/rBGShjJlyuizzz6Tra2tFi5cKEkKDQ2Vp6en5s2bp8DAQHXp0kVeXl7auXOnYmNjtXjxYrm5uWn69OkmSRBJqlixot5//3399ddfWrZsWW6cEgAAAAAAVos7QtKhcuXKatSokU6cOKGoqChj+f79+xUREaEBAwaoVKlScnd3108//aR79+5p4MCBcnBwSPF4zZs3V8WKFbVv3z5Nnz49p04DAAAAAACrRyIknapXr66jR48qNDRURYsWlSRdvXpVu3btMln4dPXq1ZKkWrVqPfZ4NWvW1IEDBxQdHS1nZ+fs6zgAAAAAADBiakw6JScroqOjjWUtW7Y0e/pL8h0jycmS1CRvf/QOEwAAAAAAkL1IhKRTcgLk0QRHSo/UTW+CIyoqSjY2NnJxcbFgLwEAAAAAwOOQCEmnS5cuyc7OTm5ubsYyR0dHs3pPPvmkJOn8+fOPPd6FCxdUsWLFVNcRAQAAAAAAlkciJB1CQkL03//+V/Xr109zPY8GDRqoZMmS2rp1q+Lj41Osc/jwYf3+++/q1KlTdnQXAAAAAACkgkRIGu7evasJEyYoKSlJY8aMSbO+o6OjXn75ZYWEhGjmzJlKSkoy2R4WFqZ33nlHJUqU0ODBg7Or2wAAAAAAIAU8NeYRp06dkoODgwwGgyIjI/Xrr79q9+7dio+P1zvvvKNmzZql6zjPP/+8Ll++rLVr1+r06dPq3LmzSpYsqeDgYG3evFnx8fGaP3++KlWqlM1nBAAAAAAAHkUi5BGbNm3Spk2bZG9vr6JFi6pq1aoaOHCg+vbtq6pVq6b7ODY2Npo5c6Zat26tdevWaenSpbp//74MBoOKFSumbdu2maw1AgAAAAAAcgaJEEmNGzfWxYsX013fzc0tXfU7dOigDh06GF8vWLBAfn5+Wrp0qd5///1M9RUAAAAAAGQeiZAcNHbsWEVFRcnf31+lSpXS+PHjc7tLAAAAAABYFRIhOWzq1KmaOnVqbncDAAAAAACrxFNjAAAAAACA1bAxGAyG3O4EYCkRsQ8Uk5hgVu5kZy8XxyKP3Tc6OlrOzs7Z1TWrxJhaHmNqvXjvs0dq3xsZZUhKko2t5f//Unq+v/B4XDv5E++bqfTGqtRiBuNpWYynZeXGeDI1BgUKPxYBABlhqe+N6OhoORfhRzGA7MFvXMCymBoDAAAAAACsBokQAAAAAABgNUiEAAAAAAAAq0EiBAAAAAAAWA0SIQAAAAAAwGqQCAEAAAAAAFaDRAgAAAAAALAaJEIAAAAAAIDVIBECAAAAAACsBokQAAAAAABgNUiEAAAAAAAAq0EiBAAAAAAAWA0SIQAAAAAAwGrY53YHkHkJD+OVlJBkVm5rbyv7woVyoUcAkP+kFksthZgMoKDityiA/IpESD6WlJCkq+vOmpW796+TC70BgPwptVhqKcRkAAUVv0UB5FdMjQEAAAAAAFaDREgBEHh4mwIPb8vtbgBAvkUcBYDMI4YCyG9IhAAAAAAAAKtBIgQAAAAAAFiNApsIiYyM1IIFC9S9e3fVr19fXl5e6tChg6ZPn65ff/3VWM/Pz0+enp66fv16iseZN2+ePD09FRoaaiybOnWqPD09jf9q166tNm3a6O2339adO3fMjpFcPyEhwfInCgAAAAAA0q1APjXm8uXLGjFihO7cuaOOHTuqd+/ecnR01LVr1/TNN99o06ZN+u6771ShQoUstfPxxx/LxsZGMTExOnnypDZt2qTjx49r27ZtcnBwsNDZAAAAAAAASylwiZDo6Gi9/PLLevjwoTZs2KDatWubbJ8wYYKWL18ug8GQ5ba6du0qe/u/h7Bfv34qVaqUli9frn379qlTp05ZPj4AAAAAALCsAjc1ZsOGDQoJCdEbb7xhlgSRJHt7e40aNUoVK1a0eNs+Pj6SpBs3blj82AAAAAAAIOsKXCJkz549cnBwkK+vb463nbyOSMmSJXO8bQAAAAAAkLYCNzUmODhY7u7uObJGx71792Rra6uYmBidOnVKCxYsUOHChdW2bdtsbxsAAAAAAGRcgUuEREVFycPDI0faevrpp01eu7u764MPPlDZsmVzpH0AAAAAAJAxBS4RUrRoUUVHR1v8uDY2NmZly5cvl62trSIiIrRu3TpdvHhRTk5OFm8bAAAAAABYRoFLhHh4eOjMmTOKi4tL1/QYR0dHSVJMTEyK25PLU0pwNGnSxPjUmA4dOqhXr14aP368du7cqcKFC2f2FAAAAAAAQDYpcIultm/fXnFxcdq+fXu66ru6ukr6e22RlAQHB6tIkSJpLoDq4OCgiRMn6tatW1qzZk2G+gwAAAAAAHJGgUuE9O3bV66urpozZ44uXLhgtj0hIUFLlizRH3/8IUlq2rSpHB0dtXbtWsXFxZnUvXjxoo4ePaqWLVvKzs4uzbbbtGkjT09P+fv76+HDh5Y5IQAAAAAAYDEFbmpM0aJFtXjxYo0YMUK9e/dWx44d5e3tLQcHB924cUO7d+9WSEiIunXrJkkqVaqUJk6cqNmzZ6tXr17q1KmTSpUqpStXrmjDhg1ydnbWpEmT0t3+yJEjNWnSJK1fv14vvPCCybYvvvhCtrbmuadBgwapRIkSWTpvAAAAAACQtgKXCJEkT09Pbdu2TatWrdK+ffu0b98+xcfHq2LFimratKk+//xzlS9f3lj/hRdekJubm1avXq0VK1bo4cOHKlOmjDp16qRXXnlFbm5u6W67U6dO+uyzz7R8+XINHDjQZJ0SPz+/FPfp0qULiRAAAAAAAHJAgUyESFKJEiU0btw4jRs3Ll3127dvr/bt26er7kcffaSPPvooxW12dnbau3dvuusDAAAAAICcU+DWCAEAAAAAAEgNiRAAAAAAAGA1CuzUGGvSs5lvbncBAPI14igAZB4xFEB+QyIkH7O1t5V7/zoplgMA0ie1WGrJ4wNAQcRvUQD5FYmQfMy+cKHc7gIA5HvEUgDIHOIngPyKdC0AAAAAALAaJEIAAAAAAIDVIBECAAAAAACsBokQAAAAAABgNUiEAAAAAAAAq0EiBAAAAAAAWA0SIQAAAAAAwGqQCAEAAAAAAFaDRAgAAAAAALAaJEIAAAAAAIDVIBECAAAAAACsBokQAAAAAABgNUiEAAAAAAAAq2Gf2x1AzoiJiVFiYmKutW9nZycnJ6dcax8AkuV2PMyspKQkRUdHp1mPeAsgO+R07CTmAchOJEKsRGJiovbs2ZNr7Xfo0CHX2gaAR+V2PMysxMRE2dnZpVmPeAsgO+R07CTmAchOTI0BAAAAAABWg0QI0rR//37t378/t7sBAAUKsRWANSHmAchLSIQAAAAAAACrkS8TIYGBgfL09FRoaGhudwUAAAAAAOQjeW6x1KtXr2rx4sU6efKk/vjjDzk7O6tSpUpq0KCBXnrpJZUvXz7b2vbz89OCBQuMr+3s7FSqVCk1adJE48aNU5UqVYzbPD095evrq7lz52ZbfwAAAAAAgGXlqUTI6dOnNWTIEBUvXlw9e/aUm5ubwsPDdfnyZW3ZskXt27dX+fLl1a1bN3Xp0kUODg7Z0o8pU6aodOnSiouL05kzZxQYGKgff/xR27ZtU5kyZbKlTQAAAAAAkP3yVCJk4cKFsrW11aZNm8zu/IiJiVFCQoKkv+/USM/jtDKrXbt2qlq1qiSpT58+qlatmmbPnq3AwECNHDky29oFAAAAAADZK0+tEXL9+nVVqVIlxekvTk5OKlq0qKSU1wiZOnWqPD09dfv2bU2aNEmNGjVSo0aN9M477yguLk4xMTH68MMP1axZMz311FMaM2aMwsPD09WvZs2aSZJCQkIyfE4XLlzQ2LFj1bhxY3l5ecnX11cbN27M8HEAAAAAAEDW5ak7Qtzc3HTs2DH9/PPPql+/fqaOMXr0aLm7u2vChAk6ceKE1q9fLwcHB4WGhiopKUmvvPKKLl26pHXr1snR0VHz5s1L85g3btyQJLm4uGSoLz///LNefPFFVa5cWcOHD5ezs7MOHDig6dOnKzw8XKNGjcrUOQIAAAAAgMzJU4mQUaNG6ciRIxo4cKBq1qypBg0aqEGDBmrRooWKFSuWrmM0bNhQb775piRp4MCBunHjhgICAtShQwf5+fkZ60VERGj37t26f/++SpQoYXKM+/fvKzw8XPHx8Tpz5ow+/PBD2draqmPHjuk+F4PBoLffflvVq1fX119/rUKFCkmSBg0apHHjxmnRokXq37+/WdsAAAAAACD75KmpMY0aNdK6dev07LPPKiQkRAEBAZowYYKaNm2qTz/91LhGyOMMGDDA5HX9+vVlMBjUv39/s/LExETdvHnT7Bh9+vRR06ZN1bJlS73yyitKSEjQJ598olq1aqX7XC5evKjffvtNvr6++uuvvxQeHm7816pVK8XExOjkyZPpPh4AAAAAAMi6PHVHiCR5eXlp/vz5SkpK0rVr13T06FH5+/vryy+/lIODg8aOHfvY/V1dXU1eFy9eXJJUsWLFFMvv379vdoyPP/5Y5cqVk52dnUqXLi13d/cML8565coVSdLMmTM1c+bMFOvcvXs3Q8cEAAAAAABZk+cSIclsbW1VrVo1VatWTZ06dVKHDh0UFBSUZiLE1jblm1xSS2QYDAazMm9vb+NTY7Jq7NixatCgQYrbPDw8LNIGAAAAAABInzybCHmUi4uLqlSposuXL+d2V9KtSpUqkiRHR0fjU2cAAAAAAEDuylNrhBw+fFiJiYlm5SEhIQoODs5Xd1DUqlVL7u7uWr16dYqP6WVaDAAAAAAAOS9P3REya9YsRUZGql27dqpevbrs7e119epVBQUFKT4+Xq+99lpud9HE5cuXtWjRIrPySpUqqXv37po9e7ZefPFFde7cWb1791aVKlUUHh6u8+fPa9++fTp79mwu9BoAAAAAAOuVpxIhU6ZM0Z49e3TixAlt375dDx48UKlSpdSoUSMNGzZM3t7eud1FExcuXNCFCxfMyuvXr6/u3bvL29tbmzZt0uLFixUUFKR79+7JxcVFHh4emjZtWi70GAAAAAAA62ZjSGm1UBQ40dHR2rNnT6b23b9/vySpbdu2mW6/Q4cOcnZ2zvT+OSE6OjrP9zG/YUwtjzHNuqzEQ0vKaGxNTExM1xPM8kO8LYi4NvMu3hvLyGrsJOblLq4Dy2I8LSs3xjNPrRECAAAAAACQnUiEAAAAAAAAq8HUGCsRExOT4hN5coqdnZ2cnJxyrf304BY3y2NMLY8xzbrcjoeZlZSUJFvbtP//RX6ItwUR12bexXtjGTkdO4l5lsV1YFmMp2XlxnjmqcVSkX34ggCAv+XXeMiPLgC5KadjJzEPQHZiagwAAAAAALAaJEIAAAAAAIDVIBECAAAAAACsBokQAAAAAABgNUiEAAAAAAAAq0EiBAAAAAAAWA0SIQAAAAAAwGrYGAwGQ253AgAAAAAAICdwRwgAAAAAALAaJEIAAAAAAIDVIBECAAAAAACshn1udwCZ4+npmdtdAIBMuXjxYq61TewEkF8ROwEg41KLndwRAgAAAAAArAZPjQEAAAAAAFaDO0IAAAAAAIDVIBECAAAAAACsBokQAAAAAABgNXhqDAqEhIQEffnll9q8ebPu3LkjV1dXDR48WIMGDZKNjc1j9x0yZIiOHz9uVm5nZ6dff/3VrPzIkSOaP3++zp8/r8KFC6tNmzaaPHmySpUqZbHzyQsyO6bHjh3T0KFDU91etWpVffvtt8bXGR3//Cw6OlrLly/XmTNndObMGUVERGj06NGaMGFCuo9x/vx5/fvf/9bJkydlZ2enJk2aaMqUKapcubJZXWv5rOYHWYlRO3fu1HfffafTp0/r2rVrKleunL7//vsU61rT9WRJWXl/Pv74Y504cUIhISF68OCBypcvLx8fH40ZM0Zubm5m9bkuMyYr782j4uLi5Ovrq2vXrqUYd9u2baubN2+a7VelShXt2bMny+dhbYh5lkWMsiziimVldTwTExO1du1abdq0SVevXpWDg4M8PDz0yiuvqEWLFiZ1Lfn5JBGCAuG9997Txo0b1bdvX9WtW1eHDh3SjBkzdO/ePY0dOzbN/UuUKKG33nrLpMzW1vyGqePHj+ull16Sp6enpkyZovDwcK1YsUJnzpzRpk2b5OTkZLFzym2ZHVMPDw/NmTPHrPzXX3/VypUr1bJlS7Nt6R3//C4iIkILFy5UhQoVVKtWLf34448Z2j84OFiDBg1SmTJlNGHCBMXGxmrVqlUaMGCAgoKCVKZMGWNda/qs5gdZiVFr167V2bNnVbt2bUVFRaXZlrVcT5aUlffn9OnTqlu3rrp27SpnZ2eFhIRo06ZN2rt3rzZv3mySpOS6zLisfr8nW7p0qW7fvv3YOk888YTGjBljUubs7Jypfls7Yp5lEaMsi7hiWVkZz6SkJI0fP14HDx5Ujx49NGjQID18+FC//fab/vjjD5O6Fv98GoB87tdffzXUqFHD8NFHH5mUjx8/3lCnTh1DWFjYY/cfPHiwoUWLFulqq1u3bobWrVsboqOjjWXfffedoUaNGgZ/f/8M9z2vyuqYpuTNN9801KhRw3D27FmT8oyMf34XGxtr+OOPPwwGg8EQEhJiqFGjhuHTTz9N9/5jxowxeHt7G49hMBgMFy9eNPzrX/8yzJgxw6SutXxW84OsXk+3bt0yJCQkGAyGtK8Xa7qeLCU74t3p06cNNWrUMMydO9eknOsyYyz13ty4ccNQt25dw5dffplq3G3Tpo2hf//+Fum3tSPmWRYxyrKIK5aV1fFctWqVoVatWoYTJ06k2ZalP58FK10Kq7Rr1y5Jf98e+aghQ4YoLi5Oe/fuTddxEhMTFRUVJUMqT5S+evWqzp8/r169eqlIkSLG8latWqlq1arasWNHJs8g77HUmCaLiYnRN998oxo1aqh27dop1klr/AsCBwcHlS9fPlP7RkdH6+DBg3r22WdNjlGjRg01btxYO3fuNJZZ02c1P8jq9VSxYkXZ2dllqE1ruJ4sxdLxTpJcXV0lSZGRkcYyrsuMs9R78+GHH8rLy0udO3dOs258fHy67kJA6oh5lkWMsiziimVlZTyTkpLk7++vdu3aqWHDhkpKSlJ0dHSKdbPj80kiBPne2bNnVbZsWVWqVMmk3MvLS7a2tjp37lyax7h7967q16+vBg0aqGHDhpo2bZru3r1r1o4kPfXUU2b7161bVxcuXFBiYmIWziTvsMSYPmrPnj2KiopSjx49UtyenvG3dhcvXlR8fHyqn7+7d+8abyG0ps9qfmDp6yktXE8ZY4n3JykpSeHh4bpz545OnTqlKVOmSJLJ3Gauy4yzxHuzd+9e/fDDD3r77bfT1Z63t7caNGigJk2aaObMmXrw4EGm+2+tiHmWRYyyLOKKZWVlPK9cuaJbt26pVq1a+uCDD1S/fn3Vr19frVu31rp168zakSz7+WSNEOR7t2/fVrly5czKHRwcVLJkSYWFhT12fzc3NzVo0ECenp4yGAw6cuSINm3apJ9++kmbNm1S8eLFje1ISrGtcuXKKS4uThERESbrNORXWR3TfwoKCpK9vb18fX3NtqV3/K1dWp8/SQoLC1OFChWs6rOaH1j6enocrqeMs8T7c+vWLbVr18742sXFRdOmTVP79u1N2pG4LjMiq+/Nw4cPNWvWLA0cOFCenp4KDQ1NtW716tXVt29feXh4KDo6Wt99951Wr16tM2fOaM2aNSpUqFCWz8daEPMsixhlWcQVy8rKeF67dk2StHLlShUuXFjTpk2Ts7Oz1q9fr3fffVeJiYkaNGiQsR3Jsp9PEiHI92JiYlS0aNEUtzk6Oio2Nvax+8+ePdvkdefOnVW3bl1Nnz5dq1at0quvvmpsR/r7wk6pnUfr5HdZHdNHhYWF6fDhw2rZsqXKli1rtj2942/tMvL5s6bPan5gyespLVxPGWeJ96ds2bLy9/dXXFycgoODtX37dkVFRSkpKcm4aCPXZcZl9b1ZvHixHj58qHHjxqXZ1pdffmnyunv37qpataq++OILbdu2TT179kx/x60cMc+yiFGWRVyxrKyMZ/I0mKioKG3cuNG4cG/Hjh3VtWtX+fn5qV+/frK3t8+WzydTY5DvOTk5KS4uLsVtsbGxxosjI/r06aOSJUuaPNUjeSXilNpKvsgLymralhzTrVu3KikpSd27d0/3PimNv7XLyOfPmj6r+UF2xKiM4Hp6PEu8P46OjmrWrJlat26t4cOHa+HChVqxYoXmz59v0o7EdZkRWXlvrly5ohUrVmjixImZvitgxIgRksS1k0HEPMsiRlkWccWysjKeyZ8nb29vk6cX2dvbq0uXLoqIiFBwcLBJXUt+PkmEIN8rV65cio+uiouL071791K8hSo9KlWqpIiICJN2JKXY1u3bt1WoUCG5uLhkqq28xpJjGhQUpBIlSpjckpke/xx/a5fW5+/ROtb0Wc0PsitGZQTXU+qy4/2pVKmS6tevr8DAQJN2JK7LjMjKe/Pvf/9brq6uatKkiUJDQxUaGmpcRykqKkqhoaFp/t/DokWLqkSJElw7GUTMsyxilGURVywrK+OZvC2l6SzJZckL+mbH55NECPK92rVr686dO7p165ZJ+ZkzZ5SUlJTqU0oeJykpSaGhoSpVqpRJO5L0yy+/mNU/ffq0atasmeFVzvMqS43pmTNn9Ntvv6lz584p3sqWmpTG39rVqFFDhQoVSvXzV7p0aVWoUEGSdX1W84PsiFEZwfX0eNn1/sTGxpo8kYHrMuOy8t7cunVL165dU/v27dWuXTu1a9fOONc8ICBA7dq105EjRx7b/r1793T//n2unQwi5lkWMcqyiCuWlZXxrFGjhhwcHIzJpEcllyWPU3Z8PkmEIN/r1KmTJGnNmjUm5cmLECUvBPXw4UMFBwcrPDzcWCcqKirFuWsrVqxQZGSkWrdubSyrVq2aatasqc2bN5us9nzw4EFdv349XY/Pyi+yMqaPCgoKkqRU50BmZPytSXx8vIKDg02y3kWLFlXLli21e/duk/JLly7p2LFj6tSpk2xsbCRZ12c1P7DU9ZQWrqfMycr7ExkZqfj4eLNjXrhwQadOnVLdunWNZVyXGZeV92batGlauHChyb8PPvhA0t/zzxcuXCgvLy9Jf/9hktLTBj7//HNJ4trJIGKeZRGjLIu4YllZGU9nZ2e1adNGp06d0oULF4zlDx8+1NatW+Xq6qpq1apJyp7PJ4ulIt+rVauWevXqJX9/f0VHR8vLy0s//vijdu3apbFjx6p8+fKS/s4WDh06VGPHjjUunnXu3DlNmDBBnTt3VpUqVWRjY6Njx45pz549qlmzptkzsadNm6YXX3xRgwcPVp8+fXT37l35+/vLw8NDAwYMyPFzzy5ZGdNkcXFx2r59uzw8PEy+aB+V0fEvCAICAhQZGam//vpLkvTzzz9r0aJFkqS2bduqZs2aCgsLU+fOndWjRw999NFHxn0nTpyoPn36aPDgwRo8eLDi4uK0cuVKlSpVSqNGjTJpx1o+q/lBVq+nEydO6MSJE5KkmzdvKjo62viZqVSpknH9HWu8niwhK+/P8ePH9d5776ljx46qWrWq7OzsdPnyZeOTsiZPnmzSFtdlxmTlvWnSpInZ8ZKf7vDEE0+YPC1j//79WrBggZ599lm5ubkpLi5O3333nY4eParmzZsbf+gjfYh5lkWMsiziimVl9XqfNGmSjh49queff15Dhw6Vs7OzAgMD9ccff+jzzz83/k8+yfKfTxIhKBDef/99VapUSYGBgQoMDJSrq6veeuutNL8EXV1d5ePjo++++05//vmnEhMT5ebmptGjR2vkyJEqUqSISf0mTZpo2bJlmj9/vmbPnq3ChQurffv2mjx5sgoXLpydp5jjMjumyQ4ePKh79+5p+PDhqdbJ6PgXBCtWrNDNmzeNr48fP67jx49LkipUqKCaNWumuu+TTz6pgIAAzZ07V/PmzZOtra2aNGmiN954w2wOpjV9VvODrFxPR48e1YIFC0zKkhe48/HxMf5RYI3Xk6Vk9v2pUaOGWrRooR9++EGbN29WfHy8ypUrp44dO2rkyJFyd3c3qc91mXFZ/S5KD09PT9WsWVO7du1SeHi4bGxs9MQTT2jy5Ml6/vnnC8R0gJxGzLMsYpRlEVcsKyvjWbVqVa1du1Zz5841PtmoVq1aWrJkiVq0aGFS19KfTxuDwWDI8F4AAAAAAAD5EGuEAAAAAAAAq0EiBAAAAAAAWA0SIQAAAAAAwGqQCAEAAAAAAFaDRAgAAAAAALAaJEIAAAAAAIDVIBECAAAAAACsBokQAAAAAABgNUiEAAAAAAAAq0EiBAAAAAAAWA0SIQAAAAAAwGqQCAEAAAAAAFaDRAgAAAAAALAaJEIAAAAAAIDVIBECAAAAAACsBokQAAAAAABgNUiEAAAAAAAAq0EiBAAAAAAAWA0SIQAAAAAAwGqQCAEAAPmSp6en/Pz8jK+PHTsmT09PHTt2LMf74ufnJ09PzxxvFwAAZByJEABAvpb8x+/GjRtT3B4YGChPT08dPnw4h3uWO0JDQ+Xp6SlPT099/fXXZtvTGi9kv7CwMH3wwQfq2LGj6tatKx8fH/n6+uqdd95RcHBwbncPAIACzz63OwAAALLHF198oV69esnR0TG3u5IjGjVqpNOnT6tQoUK53ZVUhYSEqHfv3jIYDOrRo4eqVaumqKgoXblyRfv371e9evXk4eGR290EAKBAIxECAEABVLt2bZ07d05r167VCy+8kK1tPXjwQEWKFMnWNtLD1tY2zyd9/P39de/ePQUFBelf//qXybaEhAT99ddfOdofg8Gg2NhYOTk55Wi7AADkJqbGAACsTmrrOSRPKwkMDDSWJU+t+f777zV//ny1bNlS9erV0/Dhw3Xr1i1J0ldffaVnnnlGXl5e6tmzp86ePWty3Js3b2rGjBnq0qWLvL295e3trYEDB+rgwYNmfRgyZIhatmyp0NBQjR49Wt7e3vLx8dE777yj2NjYdJ9jixYtVK9ePS1ZskQPHjxIs35kZKQ+/PBDtWzZUnXq1FH79u01f/58xcXFmdSbOnWqPD099fvvv2vChAlq1KiRunbtarLt9u3bmjRpkho1aqRGjRrpnXfeUVxcnGJiYvThhx+qWbNmeuqppzRmzBiFh4ebHP+///2vJk2apLZt28rLy0uNGzfWq6++qitXrqR5Dv9cI+TRaUIp/Xt0LZEHDx7o008/Vfv27VWnTh01b95c77zzju7du2fWztatW9W5c2d5eXmpY8eO2rx5c5p9S3b9+nUVK1bMLAkiSfb29nJxcTEpS0hI0LJly+Tr66u6deuqcePGGjJkiA4dOmRS7+DBg+rfv7/q1aun+vXra/jw4frll19M6iSPx7x58xQYGKguXbrIy8tLO3fuNLa1dOlS47k1btxYkyZNMn7OAQAoKLgjBABQIERHR5v9US0pXUmA9Jg3b54cHR310ksv6fbt2/L399eYMWPUtWtXbdmyRQMGDNDDhw+1dOlSjR07Vnv27DFO0Thz5owOHz6sDh06yNXVVZGRkdq6datGjRolf39/NW3a1KSt2NhYDRs2TD4+PnrjjTf0yy+/aP369SpVqpRee+21dPd5/PjxGjZsmAICAjRy5MhU68XFxemFF17Qr7/+qt69e+tf//qX/vvf/2rRokU6f/68vvjiC7N9RowYoWrVqmnChAlmyZLRo0fL3d1dEyZM0IkTJ7R+/Xo5ODgoNDRUSUlJeuWVV3Tp0iWtW7dOjo6OmjdvnnHfXbt26c8//1SvXr1Urlw5hYaGasOGDRo4cKB27Nih0qVLp/v8S5UqpTlz5piVL1q0SDdu3DAmHZLP//Lly+rTp488PDx07do1ffXVVzp16pQ2btxovNNk69atmjx5sjw9PTVx4kRFRUVpzpw5qlixYrr65ObmpkOHDmn37t169tlnH1s3KSlJY8eO1YEDB9SqVSvjlJpffvlFR44cUfPmzSVJO3fu1MSJE+Xu7q6xY8cqPj5e69at0+DBg7Vy5Uo1aNDA5Lj79+9XRESEBgwYoFKlSsnd3V0Gg0Hjx4/XwYMH1aNHDw0dOlRhYWH66quvdOLECQUFBalUqVLpOkcAAPI6EiEAgAJh9uzZmj17drYd39bWVgEBAbK3//urMzExUStWrFBkZKS2b99unBpSokQJffDBB/r+++/Vrl07SVKrVq3UsWNHk+MNHTpU3bt317Jly8wSIffu3dPLL79snNIyYMAARUZGav369RlKhDRr1kw+Pj5avny5Bg4cqKJFi6ZYb+PGjTp37pymTJmiF198UZI0aNAglS1bVqtWrdKBAwfUpk0bk328vb01Y8aMFI/XsGFDvfnmm5KkgQMH6saNGwoICFCHDh1MnvISERGh3bt36/79+ypRooQk6fXXX1fhwoVNjte9e3f5+vpq06ZNGjVqVLrPv0iRIurWrZtJmb+/v65du6bXX39dNWrUkCStWrVKv/76q9auXSsvLy9jXR8fH40ePVqBgYEaMGCAEhISNGfOHFWuXFlr166Vs7OzJOnZZ581ayc1L7zwgrZu3apx48apWrVqatiwoerXr6+WLVuaJXm2bdumAwcOaMSIEXr99ddNthkMBkl/38Uxa9YslS9fXuvXr1fx4sUlST169FCnTp00c+ZMkzucJOnq1avatWuXKleubCzbuXOn9u7dq0WLFhk/t5L0zDPPqFevXlqxYoVZHwAAyK+YGgMAKBCGDRsmf39/s3/Dhw+3yPH79u1rTIJIMv5fdl9fX5P1MerXry/p70Uxkz36h31sbKwiIiIUHR0tHx8fnTlzxqwtW1tb9e/f36TMx8dH4eHhioqKylC/x48fr3v37mnlypWp1tm/f7+KFCmiQYMGmZS/9NJLxu3/NHDgwFSPN2DAAJPX9evXl8FgMDun+vXrKzExUTdv3jSWPTpW0dHRioiIUIkSJeTu7p7iWGXE999/r3//+9967rnnNGLECGP5jh07VKdOHbm6uio8PNz476mnnlKRIkV05MgRSdLZs2d1584d9enTx5gEkaQaNWoY785Ii7u7uwIDA9WzZ09FRERow4YNmjp1qpo3b67p06crOjraWHfXrl1ycnLSmDFjzI5jY2Nj0qd+/foZkyCSVKFCBfn6+urcuXMKCwsz2bdly5YmSZDkMShfvry8vb1NxqB8+fKqWrWqcQwAACgIuCMEAFAgeHh4qFmzZmblf/zxh0WOX6lSJZPXxYoVkySzKRHJ5Y+uLREXF6fFixdry5YtJn/0S//7g/ZRpUuXNlu8MvmP3Pv376d6Z0dKGjZsqObNm2vlypUaMmRIinVu3rwpNzc3s4VGy5Urp+LFiys0NNRsn3/+If0oV1fXFPv+z7F69JyS3b59W5988okOHDhgUi7JbP2MjAgODtbEiRNVp04dffjhhybbrl69qpiYGLM7c5IlT7lKfu/c3d3N6ri7u+v7779PV1/c3d01e/ZsGQwG3bx5U8ePH1dAQIA2btyoxMRE451N169fl5ub22MXok3uU0pPmkkuCw0NVfny5Y3lVapUMat79epVhYWFpToG//z8AwCQn5EIAQBYnZSSD9LfazKkxs7OLkPlyVMXJGnWrFlat26dBgwYoAYNGqhEiRKys7PT5s2btX37drN9bW1Tv2Hz0eOm1/jx49WnTx8tW7Ys3XcupOVxTxlJrf9pjVVSUpKGDx+usLAwPf/886pevbqKFCkiW1tbzZo1K1PnLv1vqlGRIkW0YMECs4SPwWCQt7e3xo0bl+L+ycktS7OxsZGbm5vc3NzUtWtXdenSRdu2bdOMGTNM7j6ytJSerGMwGOTm5pbqdCcHB4ds6w8AADmNRAgAwOo8eidC8toUkul0FkvasWOHunfvrnfffdekfNOmTdnS3j/VrVtXbdu2VUBAgGrVqmW23c3NTT/99JNiY2NN/ki+c+eOIiMj5ebmliP9vHTpki5duqTZs2erZ8+eJtvu3buXqTtCEhISNH78eIWFhSkgIEDlypUzq1O1alXdv38/xTuKHpV8p8vVq1fNtqVUlhEODg6qWbOmbty4oYiICJUtW1ZVq1bV4cOHH/t44uT3Jjg42Gxbcll63r+qVavq+PHjatSokXGRXwAACirWCAEAWJ2qVatKko4ePWosMxgMWr16dba0Z2tra3Y3w5UrV7R3795saS8l48ePNz7V5p/atGmjBw8eaO3atSbly5cvlyS1bds2R/qYfCfJP8fqP//5j+7cuZOpY3744Yc6evSoZs2aZbIQ6qO6dOmiK1euKCgoyGxbYmKicZpTnTp1VLZsWW3cuNHkaUSXLl0ye5xtao4fP57iY5Dv3bunkydPqmTJksans3Tq1EmxsbFatGiRWf3kMapdu7bKlSunDRs2mKwfExYWpm3btql27dom02JS06VLF0VHR2vJkiUptpXSE5kAAMivuCMEAGB1nn76aVWuXFnTp09XcHCwihUrpm+//VYPHz7Mlvbat2+vwMBAFS5cWLVq1VJISIjWrl0rDw8P/frrr9nS5j/VrFlTzz77rL755huzbX369NHmzZv10Ucf6cqVK6pZs6Z++uknbd++XW3atDF7Ykx2qVatmtzd3fXxxx/r1q1bKlu2rH755Rft27fvsWuSpObgwYNau3at6tatq4SEBG3ZssVk+9NPP60yZcpo2LBh+u677zR16lR9//338vb2lsFg0I0bN/Ttt9/qtddeU8+ePWVvb6/XX39dU6ZMUf/+/dWzZ09FRUVpzZo1qlGjhi5cuJBmn5YvX65Tp06pXbt2qlWrlhwdHXXz5k1t3bpVd+7c0XvvvWecQuTr66sdO3Zo6dKlunz5spo1ayYbGxudPn1aFSpU0Ouvvy57e3tNmzZNEydOVN++fdWrVy/j43MTEhL01ltvpWusunbtqj179ujzzz/XL7/8oqZNmxofebx//3517dpVr776aobfAwAA8iISIQAAq2Nvb69FixZpxowZWrx4sYoWLaouXbqof//+6tKli8Xbe/PNN+Xk5KRvv/1WgYGBqlatmj744AMFBwfnWCJEksaNG6dvv/3WbC0UBwcHrVy5UvPnzzf2sXz58nr55ZdTfGJJdrG3t9eSJUs0a9YsrVmzRvHx8apXr55WrVqlWbNmZfh4d+/elSSdPn1ab7zxhtn21atXq0yZMnJ0dNSqVau0YsUK7dixQ3v27JGjo6MqVaqkrl27qkmTJsZ9unfvLkn64osvNHfuXLm5uemNN97QrVu30pUIGT16tHbu3KkTJ05o3759ioqKUvHixVWnTh29++67atWqlbGura2tFi5cqOXLlysoKEg//vijnJ2dVbNmTZOpQ507d5azs7MWL16szz//XLa2tvL29tZnn32mevXqpWusbGxs9Nlnn2nt2rXavHmzPvvsM9na2qpChQpq1qyZOnXqlK7jAACQH9gYMrvyGAAAAAAAQD7DGiEAAAAAAMBqkAgBAAAAAABWg0QIAAAAAACwGiRCAAAAAACA1SARAgAAAAAArAaJEAAAAAAAYDVIhAAAAAAAAKtBIgQAAAAAAFgNEiEAAAAAAMBqkAgBAAAAAABW4/8AciCmwUYxwU4AAAAASUVORK5CYII=\n",
|
|
"text/plain": [
|
|
"<Figure size 1080x252 with 3 Axes>"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
}
|
|
],
|
|
"source": [
|
|
"algorithms = ['SimPLe', 'CURL', 'DrQ', 'SPR', 'IRIS (ours)']\n",
|
|
"fig, axes = plot_utils.plot_interval_estimates(\n",
|
|
" {k: v[:3] for k, v in aggregate_scores.items()}, \n",
|
|
" {k: v[:, :3] for k, v in aggregate_interval_estimates.items()},\n",
|
|
" metric_names = ['Mean', 'Median', 'Interquartile Mean'],\n",
|
|
" algorithms=algorithms,\n",
|
|
" colors=ATARI_100K_COLOR_DICT,\n",
|
|
" xlabel_y_coordinate=-0.1,\n",
|
|
" xlabel='Human Normalized Score', \n",
|
|
" subfigure_width=5,\n",
|
|
" row_height=0.7)\n",
|
|
"plt.show()\n",
|
|
"save_fig(fig, 'aggregates')"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 13,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAVIAAADDCAYAAADQmJm9AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAA3b0lEQVR4nO3deVyN6f8/8FeLisoSaVSY6uMc06ZFESZSTCuKSIox9hFNGGsYa6YxtiJbQlEqyRIjsqswxr5PobIk2rRv9+8Pv3N/O86pTp32eT8fjx4P57qv+1ruo3f3fd33fV0SDMMwIIQQUmeSTd0AQghp6SiQEkKImCiQEkKImCiQEkKImCiQEkKImCiQEkKImCiQkv+coUOHYvHixU1Wv5+fH7hcLl+au7s73N3dm6hFRFzSTd0A8t/16dMn7N27FxcvXsTbt28hIyMDDocDJycnODk5QVKy7n/nY2Nj8ezZM8yZM6ceW9x4KioqsH37dnz33XewsrJqkDoyMjJw8OBBXLlyBampqSgpKUHnzp2ho6MDGxsb2NjYQFqaQoQoJOiBfNIUHj16hKlTpyI/Px+jRo2Crq4uCgoKEBcXh5s3b2Lw4MHw9/eHjIxMncpfsGABTp48iWfPnglsKykpgYSEBNq0aSNuN+rEz88P/v7+fG0rKSkBALa/ZWVl0NHRgaOjIzZs2FDvbbhx4wbmzJmD4uJi2NjYQFdXF3Jycnj//j3i4+Nx+/ZtTJ06Fb/++mu9190a0Z8b0ujy8vIwe/ZsVFRUIDw8HL1792a3/fjjjwgICMCWLVuwceNGLF26tN7rr2twbkiN2aZ3797Bw8MDHTt2xL59+9C9e3e+7R4eHnjw4AGePn3aaG1q6WiMlDS6I0eO4N27d5g/fz5fEOWZNWsWDA0NcfjwYXz48IFNd3d3h7m5OV6+fIkpU6bA0NAQ/fv3x6pVq1BQUMCX7+TJkwAALpfL/qSlpQEQHCNNS0sDl8vF5s2bcerUKdjZ2UFfXx8jRoxAYmIiACAhIQHOzs7Q19fH0KFDER0dzdfmkpIS+Pn5wdnZGf369YOenh5sbW2xf/9+iHLRV3mMNC0tDTo6OgCAY8eOse13d3dHUlISuFwuAgICBMooKCiAoaEhFixYUG1de/bsQW5uLtavXy8QRHn09PTg7Oxcp/7xxoAfP36M5cuXo1+/fjAwMMCMGTOQmppa47FoieiMlDS68+fPQ0ZGBiNGjKgyz+jRo3Hnzh1cuXIFY8aMYdOLioowefJkmJiYYMGCBbhz5w4OHz6MtLQ07NmzBwAwc+ZMlJaW4s6dO/D19WX3VVJSqrZdV65cwfHjxzF+/Hi0adMGgYGBmDFjBnx9fbFq1SqMHz8eI0aMQEhICBYvXgxdXV3873//A/DlLPvw4cOwsbFh+3X9+nX4+PggJycHnp6eIh8fJSUl/P7771i0aBH69u2LsWPHAgC6dOkCLS0tGBoa4vjx45g1axbffrGxsSgoKICTk1O15cfFxUFdXR0mJiYit6ku/VuyZAkUFBQwe/ZspKenIyQkBBMmTMCJEyfQsWNHketuERhCGpmJiQnj4OBQbZ6HDx8yHA6H2bBhA5vm5ubGcDgcxtfXly+vj48Pw+FwmEuXLrFp8+fPZzgcjtCyLSwsmEWLFrGfU1NTGQ6HwxgYGDDv379n0y9evMhwOBxGW1ubefLkCZv+/PlzhsPhMOvXr2fTysrKmOLiYoG6Fi9ezBgYGPBt27Ztm0Db3NzcGDc3N/ZzaWkpw+Fw+NrJc+TIEYbD4TB3797lS584cSIzePBgpry8XGi/GYZhPn/+zHA4HGbWrFkC2/Lz85lPnz6xP9nZ2WL1b9y4cUxpaSmbHhcXJ/T7aw3o0p40ury8PCgoKFSbh7f98+fPAtsmTZrE93ny5MkAgIsXL4rVrqFDh0JFRYX9bGRkBODLZW7lIYhevXpBUVERKSkpbJqUlBTfjaKcnBxkZmaif//+KCgoQHJyslhtq8zW1hZt27bFsWPH2LS3b9/ixo0bGDlyZLVPO+Tl5QEA5OXlBbZt3LgRZmZm7I+joyO7rS79c3Nz47vrP3ToUHz77bdif0/NEV3ak0anoKDA/kJXhbf964ArLy+Prl278qWpqKhAXl6eHQOtK1VVVb7P7du3F5rO25aTk8OXdvz4cQQFBeH58+coLy/n2ybsD0JdKSgoYPjw4Thz5gyWLl0KGRkZHD9+HAzD8AW/qvYFgPz8fIFtbm5u7KNWPj4+Anlq2z8NDQ2hafHx8dV3sAWiQEoanZaWFh4+fIiioiLIyckJzfPkyRMAYMcgG0NVZ3JVpTOVbrKcPn0aCxcuhLm5Odzc3KCsrIw2bdrg0aNH2LhxIyoqKuq1raNHj8bx48dx6dIlDB8+HNHR0TA0NMS3335b7X4KCgpQUVER+liYpqYmNDU12XyVA2lj96+loUBKGt3QoUPxzz//4OTJk3x3hiuLiopCmzZtYG5uzpeen5+PDx8+8J2VpqenIz8/H+rq6myahIREwzS+CjExMVBXV8euXbv4Am/ly//aqKn9pqam6N69O44dOwZlZWW8evUKU6ZMEalsS0tLHD58GLdu3RL5hlNd+vfy5Uv26YPKaZW/p9aCxkhJo3NxcYGKigo2btyI58+fC2zfvXs3bt++jfHjxwtcxgPAgQMH+D4HBQUBACwsLNi0du3aAYDA5XdDkZKSAsB/llpUVITg4OA6lycrK1tl+yUkJODo6IirV68iKCgIcnJysLGxEansqVOnQlFREUuXLq3ycSTmq0ea6tK/kJAQlJWVsZ8vXLiAV69eYciQISK1syWhM1LS6BQVFeHv74/p06fD2dmZ782mCxcuIDExEYMHDxb6PGSHDh0QExODjIwM9OnTB3fv3sWJEycwaNAgDB48mM2np6eHsLAwrF69Gubm5pCWloaFhQUbYOubpaUlzp49i+nTp2PYsGHIyclBVFSU0Js6otLT00NCQgICAwPxzTffQElJCWZmZux2R0dH+Pv74+zZs7C3t4eioqJI5aqpqcHPzw9z5syBvb09bGxsoKenB1lZWWRkZOD69eu4c+cOTE1NxepfYWEhJk2aBGtra6SnpyM4OBjKysqYOnVqnY9Jc0WBlDQJfX19nDx5Env37sWlS5cQFRXFvmu/du1aODk5sWdBlcnJySEoKAhr167Fxo0bISMjAxcXFyxcuJAv38iRI/Ho0SOcPXsWMTExYBgGcXFxDRZIR44cidzcXISEhGDdunVQVlaGk5MTjIyM2KcKauu3337DqlWr4Ofnh8LCQpiamvIFUlVVVZiZmeH69es13mT6mpmZGU6fPo0DBw7gypUrOHv2LEpLS6GkpAQdHR1s3LgRtra2YvXPx8cHhw8fhr+/P4qLi2Fqagpvb+8an+dtiehde9JiuLu74/Xr17hy5UpTN6XZmD59Op4+fYpLly6JNclLfeLNJRAbG4uePXs2dXMaRfM48oSQWktNTcXVq1fh6OjYbILofxVd2hPSwjx//hyPHz9GSEgIZGVl4erq2tRN+s+jQEpIC3P27Fls374dampq+OOPP/jexiJNg8ZICSFETDSwQgghYqJASpq1oqKipm5Cg6M+tnwUSEmz9vXkGK0R9bHlo0BKCCFiokBKCCFiahWPP23btg0xMTE4ffq00NcKm4v8/HwMHToUCxYsqHLWo9rIKihBUVnzm75MTloSndo1vwXmCGkoNQbSGzduYOLEiVi7di37y5+WlgZLS0u+fPLy8uByuXBzc4OdnR3fNl7+mTNnwsvLi03PycnBvn37cOHCBaSlpUFKSgrKysro06cPXFxcYGBgUGMHPn78iKCgICxfvrxZB1HgyzFyd3eHn58fHBwcqpyLU1RFZRWYcOifempd/Tk0waipm0BIoxLrjNTS0hI//PADGIbBhw8fEBERgXnz5qGoqAijR4+udt/Pnz9j9OjR+PjxI0aMGAFXV1cUFxfj1atXuHr1KtTU1EQKpCEhIZCWloa9vb04XWk048aNw/bt2xEdHQ0XF5embg7eJcQAALqZ2dWQkxBSFbECaa9evTBy5Ej2s5OTE4YNG4Z9+/bVGEjDw8ORmpqK7du3s8sb8DAMg8zMzBrrLy8vR1RUFIYPH95ka5WXl5ejvLxc5PqVlZXRr18/RERENItASggRX73ebOrSpQs0NTXx+vXrGvPy8lSe85BHQkICnTt3rrGMu3fvIj09HYMGDRK6PTo6Go6OjtDX14epqSnmzJmDpKQkvjxRUVF8a55XxuVy4efnx36+ceMGuFwuwsPDERgYCCsrK+jp6eHOnTsAvry6N3bsWPTt2xcGBgYYNmwYli9fLlDuoEGD8PDhQ7x9+7bGPhJCmr96vdlUVlaG9PR0kdas5i03cPTo0TrP13j79m0AX+a2/FpgYCB8fX2hr6+PefPmIScnByEhIXBxcUFkZKRY03sdPHgQpaWlGDt2LOTk5KCsrIyEhAR4enrCxMQEv/zyC6SlpZGamip0xcQ+ffoAAP7+++9q13YnhLQMYgXS4uJi9hI8PT0de/fuRUZGhkiB0dnZGQcPHsSGDRsQHBwMExMTGBoawtzcXOiqjcIkJSVBSkpKIH9WVha2bt0KHR0dHDp0iL3strKywpgxY/Dnn39i27Zttezt//n06RPOnj3LrjIJAGFhYZCXl0dQUBDfErS//vqrwP49evRg208IafnECqRBQUHsejnAl3VdJkyYgHnz5tW4b6dOnXD06FHs2bMHsbGxiI6ORnR0NIAvN7HWrFlT4+V9ZmYmFBUVBRYKi4+PR3FxMSZNmsQ3dqmjo4MBAwbg8uXLKCsr4wt4tWFvb88XRIEvy/MWFhbi6tWrGDJkSLWLl/HO2LOysupUPyGkeRErkDo4OMDJyQmlpaV49OgRdu/ejc+fP6NNmzYi7a+iogJvb294e3sjPT0dt2/fxpEjRxAXF4f8/HyBRc5E9ebNGwBflv39mpaWFq5du4ZPnz7Vefox3hllZa6urjh79ixmzpyJzp07o3///rCwsIC1tbXA8fivL11LSGsjViBVU1PDgAEDAACDBw+Guro6fv31VxgYGGDChAm1KktFRQW2trawsbGBm5sbEhMT8e7dO3Tr1q3KfZSUlPD582cwDFPn5Xer2q+6d4OFPf+ppKSEY8eOITExEVevXkV8fDxiYmKwd+9eHD58mG+RsNzcXABfzsoJIS1fvd61HzFiBHR1deHn54f8/Pw6lSEhIQE9PT0AX8Zdq6OlpYXy8nL2DJSHdyNL2BhkUlIS2rZtyw4b8C7RecGNp6plaqsjLS2NQYMGYcmSJTh58iRWrlyJp0+f4tSpU3z5eGuBCztjJoS0PPX+rv2MGTOQlZWF0NDQavPdvXsXeXl5AuklJSWIj4+HlJRUjXfWjY2NAQAPHjzgSzczM4OsrCyCg4NRUlLCpj99+hTx8fHs8rwA8O233wIAEhMT+cqo7bCCsPFObW1tAIJrq9+7d4+v/YSQlq3e37W3srKCpqYmgoKC4O7uDllZWaH5Tpw4gWPHjmHo0KHQ09ODoqIi0tPTERMTg3///RfTpk2r8dLXwMAAXbt2xbVr12BjY8Omd+rUCZ6envD19WVfWeU9/qSgoMB3M0xLSwumpqbYsmULsrOzoaKiguvXr+PDhw+16re3tzc+ffoEMzMzqKqqIisrC2FhYZCTk8OwYcP48l67dg06OjpQU1OrVR1fk5OWFP91zAZ4nVNOmubCIf8t9R5IJSUlMW3aNCxZsgTh4eFwd3cXms/FxQUKCgpISEhAQkICcnJy2Pf1fX19+d6YqoqUlBScnJwQGhqKlStX8t2hnzJlCjp37oz9+/dj48aNkJOTQ79+/eDl5cWehfL4+vpi1apVOHjwINq0aQMLCwusW7cO/fv3F7nfI0aMQGRkJCIiIpCdnY2OHTvC0NAQs2bNgoaGBpvv48ePuHnzptAH9WuLJgYhpHlo8Ws2ZWRkYPjw4VixYgUcHR2bujk18vf3x5EjRxAbG4u2bds2dXOavfz8fL4bda0R9bHla/HXYMrKypg8eTJ27tzZ7Gfhzs/PR3BwMObMmUNBlJBWpMWfkZLWrbWfyQDUx9agxZ+REkJIU6NASgghYqJASgghYmoVazaR/6byoiww5cU15pOQkoWUHL2OSxoOBdL/7+XLlwgICMCdO3fw/v17yMvLQ1VVFcbGxpg6dSpUVFTY9at4JCQk0L59e+jq6mLatGkwMzNjt7m7u+PmzZvs5zZt2qBbt26wsrLC7NmzoaCg0Kj9a42Y8mJknBH+nHJlyjbBjdAa8l9GgRTA/fv34e7ujvbt28PJyQnq6urIzMzEixcvcPz4cVhZWfHNFDVmzBiYmpqioqICr1+/RlhYGKZMmYK9e/eyk7gAQIcOHbBs2TIAQF5eHhISErBv3z48fPgQwcH0y13fjlz8suLAOAvR5rMlpL5QIAWwfft2SEpKIjIyUmBqvaKiIpSVlfGlGRgY8L15NXz4cDg6OiIwMJAvkMrJyfHlmzBhAjw9PfHXX3/h/v37Qmf2J4S0PHSzCV/Wj+rRo4fQ+Unl5ORqvAzX1tZGx44dha779DXeGlW8GaAIIS0fBVJ8mXYvOTkZ//xTtzXis7KykJubK9L8orxgK8q6VoSQloEu7fFl6r+EhAS4urqid+/eMDY2hrGxMb7//nsoKioK5M/Pz0dmZiYYhsHr16+xadMmVFRU8M1ABXyZCZ+3plV+fj7i4+Nx6NAhdO3aFSYmJo3SN0JIw6NACsDExARhYWHYu3cvrl27hidPniAkJARt2rTBTz/9hLlz5/Kt7+Tj4wMfHx/2c9u2bTF9+nS+O/rAlwlVKt/JBwA9PT2sW7euyukFCSEtDwXS/09PTw9bt25FRUUFXr16hcTERAQFBWHXrl2QkZGBh4cHm3fy5MkwNzeHhIQEFBUV8b///U/o8iOdOnXCpk2bAHyZ7X///v349OkTTVhCSCtDgfQrkpKS0NTUhKamJmxsbDBs2DBER0fzBVItLS2+u/NVkZGR4ctnYWEBOzs7zJ8/H+Hh4XVeZ4oQ0rzQzaZqdOrUCT169Khx7ShRdezYEbNmzcL9+/dx+vTpeimTENL0KJACiI+PFzqXaWpqKpKSkup1kTpnZ2d07twZO3fuBM1gSEjrQJf2ANavX4/c3FxYWlqiV69ekJaWxsuXLxEdHY3S0lL88ssv9VaXrKwsJk2ahE2bNuH8+fMC6zkR0UlIyfK9/ulhU3U+QhoSBVIAixYtwrlz53Dr1i2cOnUKBQUFUFJSgomJCSZPngxDQ8N6rc/V1RW7d+9GQEAABVIx0EQkpLmgGfJJs9baZ1YHqI+tAY2REkKImCiQEkKImCiQEkKImCiQEkKImCiQEkKImOjxJ0BgCRFpaWkoKCjg22+/Rd++fTFu3Dj06NGj1uWeP38eoaGhePjwIfLz89G1a1cMHDgQ06dPR/fu3euzC6SZySouQFF5WbV55KSk0Um2XSO1iDQkCqSV8JYQYRgGOTk5ePLkCQ4fPowDBw5g2bJlGD9+vEjlMAyD5cuXIyIiAtra2pgyZQo6deqE5ORkREZG4tSpU/D398fAgQMbuEekqRSVl8Ht8iEAwLtzCQCAbsP4ZwILGTyh0dtFGgYF0kq+XkIEABYsWIAZM2Zg1apV6NmzZ5WTlTAMg+LiYsjJyeHAgQOIiIjAuHHj8Ntvv0FS8v9GUCZOnAg3Nzd4enri1KlT+Oabbxq0T4SQhkdjpDXo0qULtmzZAklJSWzfvh3Al1nuuVwuNm/ejKioKNjZ2UFPTw+nT59GcXExAgICoK6uDm9vb74gCgDdunXDqlWr8PnzZ+zdu7cpukQIqWd0RiqC7t27w8TEBLdu3UJeXh6bfuHCBWRlZWH8+PFQUlKChoYGbt++jezsbLi6ukJGRkZoeYMGDUK3bt0QFxcHb2/vxuoGIaSBUCAVUa9evZCYmIi0tDR2MbyXL1/izJkzfDeODh48CODLgnjV6d27Ny5evNjqX50j5L+ALu1FxAt2+fn5bJq5ubnA3XfeGWtNK4/ytlc+wyWEtEwUSEXEC6CVA6SwR6JEDZB5eXmQkJAQaeVRQkjzRoFURM+fP4eUlBTU1dXZNGEL2P3vf/8DADx58qTa8p4+fYpu3bpVOY5KCGk5KJCKIDU1FX///TeMjIxqHM80NjZGx44dceLECZSWlgrNEx8fj3fv3gks30wIaZkokNbg06dP8PLyQkVFBX7++eca88vKymLWrFlITU3FunXrUFFRwbc9PT0dK1asQIcOHeDm5tZQzSaENCK6a1/J3bt3ISMjA4ZhkJubi8ePH+Ps2bMoLS3FihUrRFo5FAAmTZqEFy9eIDQ0FPfv34etrS06duyIpKQkHD16FKWlpdi6dStUVVUbuEekOfj6jSbS+lAgrSQyMhKRkZHsu/Y9e/aEq6srxo4di549e4pcjoSEBNatW4chQ4YgLCwMe/bsQU5ODhiGgaKiIk6ePMk31kpaHzkp6RpfAZWTol+/1oKWGmlE/v7+8PPzg4uLC1atWtXUzWkR/gvP2VIfWz76k9iIPDw8kJeXh6CgICgpKcHT07Opm0QIqQd0RkqatdZ+JgNQH1sDumtPCCFiokBKCCFiokBKCCFiokBKCCFiarWBNDc3F/7+/hg1ahSMjIygp6eHYcOGwdvbG48fP2bz+fn5gcvl4vXr10LL2bx5M7hcLtLS0ti0xYsXg8vlsj86OjqwsLDA8uXLkZGRIVAGL39ZWfVr+BBC6ldZYSlKPhdX+1NWKPxV7tpolY8/vXjxAtOmTUNGRgasra0xZswYyMrK4tWrV/jrr78QGRmJS5cuib3Mx++//w4JCQkUFRXhzp07iIyMxM2bN3Hy5EmajISQZqCirAIvwx4CAKLiTwIAnAY48OXRcNEVu55WF0jz8/Mxa9YsFBYWIjw8HDo6Onzbvby8EBgYiPp46sve3h7S0l8O4bhx46CkpITAwEDExcXRhCSE/Ie0ukv78PBwpKamYuHChQJBFPiy1PKMGTPQrVu3eq/b1NQUAJCSklLvZRNCmq9WF0jPnTsHGRkZODg41Jy5nvHGUTt27NjodRNCmk6ru7RPSkqChoZGo4xRZmdnQ1JSEkVFRbh79y78/f3Rtm1bDB06tMHrJoQ0H60ukObl5UFLS6tR6ho4cCDfZw0NDaxevRrKysqNUj8hpHlodYFUQUGBb4G6+iIhISGQFhgYCElJSWRlZSEsLAzPnj2DnJxcvddNCGneWl0g1dLSwoMHD1BSUiLS5T1v3aWioiKh23npwgJk//792bv2w4YNw+jRo+Hp6YnTp0+jbdu2de0CIaSFaXU3m6ysrFBSUoJTp06JlF9NTQ3Al7FVYZKSktCuXbsabyDJyMhg3rx5ePv2LYKDg2vVZkJIy9bqAunYsWOhpqYGX19fPH36VGB7WVkZdu/ejffv3wMAzMzMICsri9DQUJSUlPDlffbsGRITE2Fubg4pKaka67awsACXy0VQUBAKCwvrp0OEkGav1V3aKygoICAgANOmTcOYMWNgbW0NQ0NDyMjIICUlBWfPnkVqaipGjhwJAFBSUsK8efPg4+OD0aNHw8bGBkpKSkhOTkZ4eDjk5eUxf/58keufPn065s+fjyNHjuDHH3/k27Zz505ISgr+7ZowYQI6dOggVr8JIdX7+o2m+tRqJ3bOycnBgQMHEBcXh5SUFJSWlqJbt24wMzODq6srevfuzZf//PnzOHjwIB4/fozCwkJ06dIFAwYMwOzZswXWV1q8eDGOHTuGR48esWOkPOXl5fjhhx9QXFyMuLg4yMjIsPmrEhsbW6s1of5LWvuEwAD1sSGVFZaioqyi2jyS0pKQbttGrHpabSAlrQMFmdahtfex1Y2REkJIY6NASgghYqJASgghYqJASgghYqJASgghYmqRgTQqKkpg+Q9CCGkqze6B/JcvXyIgIAB37tzB+/fvIS8vD1VVVRgbG2Pq1KlQUVFpsLr9/Pzg7+/PfpaSkoKSkhL69++PuXPnokePHuw2LpcLBwcHbNy4scHaQ0hTKioqQnl5eb2UVVFR0SCTCdWGlJRUg00q1KwC6f379+Hu7o727dvDyckJ6urqyMzMxIsXL3D8+HFYWVlBRUUFI0eOhJ2dXYPNObpo0SJ07twZJSUlePDgAaKionD9+nWcPHkSXbp0aZA6CWluysvLce7cuXorS9hr1hcuXACARpnDd9iwYQ1WdrMKpNu3b4ekpCQiIyMFzjyLiorYVTilpKREeve9riwtLdk3jZydnaGpqQkfHx9ERUVh+vTpDVYvIaRlalZjpK9fv0aPHj2EXr7LyclBQUEBgPAxUt6Sxx8+fMD8+fNhYmICExMTrFixAiUlJSgqKsLatWsxYMAA9OnTBz///DMyMzNFateAAQMAAKmpqbXu09OnT+Hh4YF+/fpBT08PDg4OiIiIqHU5hJDmq1mdkaqrq+PGjRv4559/YGRkVKcyZs6cCQ0NDXh5eeHWrVs4cuQIZGRkkJaWhoqKCsyePRvPnz9HWFgYZGVlsXnz5hrL5C1m16lTp1q15Z9//sFPP/2E7t27Y8qUKZCXl8fFixfh7e2NzMxMzJgxo059JIQ0L80qkM6YMQMJCQnspCLGxsYwNjbG999/D0VFRZHK6Nu3L5YuXQoAcHV1RUpKCkJCQjBs2DD4+fmx+bKysnD27Fnk5OQIzLyUk5ODzMxMlJaW4sGDB1i7di0kJSVhbW0tcl8YhsHy5cvRq1cvHD58GG3afJkUYcKECZg7dy527NgBFxcXmvWJkFagWV3am5iYICwsDD/88ANSU1MREhICLy8vmJmZYdOmTewYaXXGjx/P99nIyAgMw8DFxUUgvby8HG/evBEow9nZGWZmZjA3N8fs2bNRVlaGP//8E9ra2iL35dmzZ/j333/h4OCAz58/IzMzk/0ZPHgwioqKcOfOHZHLI4Q0X83qjBQA9PT0sHXrVlRUVODVq1dITExEUFAQdu3aBRkZGXh4eFS7P2/Ge5727dsDgMA69rz0nJwcgTJ+//13dO3aFVJSUujcuTM0NDRqfXMrOTkZALBu3TqsW7dOaJ5Pnz7VqkxCSPPU7AIpj6SkJDQ1NaGpqQkbGxsMGzYM0dHRNQZSYRMnA6gyEAqbRdDQ0LDe5gf18PCAsbGx0G2NtdopIaRhNdtAWlmnTp3Qo0cPvHjxoqmbIjLew/uysrLsXX9CSOvUrMZI4+Pjhb5JkZqaiqSkpBZ1BqetrQ0NDQ0cPHhQ6GNWdFlPSOvRrM5I169fj9zcXFhaWqJXr16QlpbGy5cvER0djdLSUvzyyy9N3UQ+L168wI4dOwTSVVVVMWrUKPj4+OCnn36Cra0txowZgx49eiAzMxNPnjxBXFwcHj582AStJqT5aIw3mhpDswqkixYtwrlz53Dr1i2cOnUKBQUFUFJSgomJCSZPngxDQ8OmbiKfp0+fCl2p1MjICKNGjYKhoSEiIyMREBCA6OhoZGdno1OnTtDS0sKSJUuaoMWEiE5KSqreXqusqKio8v5FY2nItyFpzSbSrLX2tX4A6mNr0KzGSAkhpCWiQEoIIWKiS3tCCBETnZESQoiYKJASQoiYKJASQoiYmtVzpER0XC63qZtAyH/Os2fPhKbTGSkhhIiJ7toTQoiY6IyUEELERIGUEELERIGUEELERHftSYMpKyvDrl27cPToUWRkZEBNTQ1ubm6YMGECJCQkqt3X3d0dN2/eFEiXkpLC48ePBdLLy8sRGhqKyMhIvHz5EjIyMtDS0sLs2bPx/fff11ufviZOH+va7pKSEjg4OODVq1eYOXMmvLy86rtbfOraxxs3bmDixIlVbu/ZsydiY2MBANnZ2Th27BguXryIpKQkFBQUoHv37rC3t8ekSZMgKytb7/2qTxRISYP57bffEBERgbFjx0JfXx/Xrl3DmjVrkJ2dXeOSMQDQoUMHLFu2jC9N2FRsFRUV8PT0xOXLl+Ho6IgJEyagsLAQ//77L96/f19v/RFGnD7Wtd179uzBhw8f6rsrVaprH7W0tODr6yuQ/vjxY+zfvx/m5uZs2p07d/DHH39g4MCB+Omnn6CgoIBbt25h06ZNuHTpEoKDgxt0GjyxMYQ0gMePHzMcDofZsGEDX7qnpyejq6vLpKenV7u/m5sb8/3334tU14EDBxhtbW3m1q1bdW5vXYjbx7q0OyUlhdHX12d27drFcDgcZtOmTXVqu6jE7aMwS5cuZTgcDvPw4UM2LSUlhUlJSRHIu2XLFobD4TCxsbG1b3wjojFS0iDOnDkD4MslemXu7u4oKSnB+fPnRSqnvLwceXl5QhcpBL6c1QUFBcHS0hJ9+/ZFRUUF8vPzxWu8iMTpY13bvXbtWujp6cHW1la8xouovr5HnqKiIvz111/gcDjQ0dFh07t3747u3bsL5Le2tgaAZr9eGwVS0iAePnwIZWVlqKqq8qXr6elBUlISjx49qrGMT58+wcjICMbGxujbty+WLFkisNZVcnIy3r59C21tbaxevRpGRkYwMjLCkCFDEBYWVq99+po4faxLu8+fP4+rV69i+fLl9dqP6tTH91jZuXPnkJeXB0dHR5Hy84YwlJSUalVPY6MxUtIgPnz4gK5duwqky8jIoGPHjkhPT692f3V1dRgbG4PL5YJhGCQkJCAyMhK3b99GZGQk2rdvDwB49eoVAGD//v1o27YtlixZAnl5eRw5cgQrV65EeXk5JkyYUO/9A8TrY23bXVhYiPXr18PV1RVcLhdpaWn13h9hxP0evxYdHQ1paWk4ODjUmLeiogK7d+9G27ZtYWVlVat6GhsFUtIgioqKoKCgIHSbrKwsiouLq93fx8eH77OtrS309fXh7e2NAwcOYM6cOQDAXg7n5eUhIiKCvTy0traGvb09/Pz8MG7cOEhL1/9/dXH6WNt2BwQEoLCwEHPnzq3nXlRP3O+xsvT0dMTHx8Pc3BzKyso15t+yZQtu3rwJb29vdOnSReR6mgJd2pMGIScnh5KSEqHbiouL6/Q4i7OzMzp27Ijr16/z1QMAhoaGfGNs0tLSsLOzQ1ZWFpKSkmpdlyjE6WNt2p2cnIx9+/Zh3rx57Jl4Y6nP7/HEiROoqKjAqFGjaswbEhKCXbt2wcXFRWB8tjmiQEoaRNeuXYU+olNSUoLs7Gyhl4uiUFVVRVZWFl89AISesfDScnNz61RXTcTpY23a/ccff0BNTQ39+/dHWloa0tLS2Mej8vLykJaWhqKiIrH7U1U76+t7jI6ORocOHWBpaVltvqioKKxduxZ2dnZYuXJlrdvcFCiQkgaho6ODjIwMvH37li/9wYMHqKio4LtjK6qKigqkpaXx3XjgcDiQkZER+twlL62hblSI08fatPvt27d49eoVrKysYGlpCUtLS3b8NCQkBJaWlkhISKivbvGpr+/xwYMH+Pfff2FrawsZGZkq88XExGDZsmUYMmQIfH19m3wJZ1G1jFaSFsfGxgYAEBwczJceHByMNm3asDcPCgsLkZSUhMzMTDZPXl6e0LG3ffv2ITc3F0OGDGHT5OXlYWFhgbt37+Lp06dsemFhIU6cOAE1NTVoamrWZ9dY4vSxNu1esmQJtm/fzvezevVqAF/GVLdv3w49Pb1m18fKoqOjAQBOTk5V1nX+/HksXLgQ/fr1w7Zt2xpkXLuhtJyWkhZFW1sbo0ePRlBQEPLz86Gnp4fr16/jzJkz8PDwgIqKCgDg/v37mDhxIjw8PNgbSI8ePYKXlxdsbW3Ro0cPSEhI4MaNGzh37hx69+4tMGY2f/58JCYmYtKkSZg4cSLk5eURFRWF9+/fY9u2bSK9qtnYfaxNu/v37y9QN++u/bffftugd7TF7SPwZRjg1KlT0NLSgr6+vtB67t+/Dy8vL8jKysLa2pp9fpWnR48eMDQ0bJhO1gMKpKTBrFq1CqqqqoiKikJUVBTU1NSwbNmyGm8eqKmpwdTUFJcuXcLHjx9RXl4OdXV1zJw5E9OnT0e7du348vfs2ROhoaHYuHEjgoKCUFJSAm1tbezevbtB37MH6t7Hpm53bYjTRwC4fPkysrOzMWXKlCrz/PvvvygpKUFJSYnQcVFHR8dmHUhpYmdCCBETjZESQoiYKJASQoiYKJASQoiYKJASQoiYKJASQoiYKJASQoiYKJASQoiYKJASQoiYKJASQoiYKJASQoiYKJASQoiYKJASQoiYKJASQoiYKJASQoiYKJASQoiYKJASQoiYKJASQoiYKJASQoiYKJASQoiYKJAS0gxwuVz4+fmxn2/cuAEul4sbN240elv8/PzA5XIbvd6WjAJpM8L75YmIiBC6PSoqClwuF/Hx8Y3csqaRlpYGLpcLLpeLw4cPC2yv6XiRhpeeno7Vq1fD2toa+vr6MDU1hYODA1asWIGkpKSmbl6joeWYSYuwc+dOjB49GrKysk3dlEZhYmKC+/fvo02bNk3dlCqlpqZizJgxYBgGjo6O0NTURF5eHpKTk3HhwgUYGBhAS0urqZvZKCiQkmZPR0cHjx49QmhoKH788ccGraugoADt2rVr0DpEISkp2ez/aAQFBSE7OxvR0dH47rvv+LaVlZXh8+fPjdoehmFQXFwMOTm5Rq0XoEv7Fq+q8SzeZXFUVBSbxhsauHLlCrZu3Qpzc3MYGBhgypQpePv2LQDg0KFDGD58OPT09ODk5ISHDx/ylfvmzRusWbMGdnZ2MDQ0hKGhIVxdXXH58mWBNri7u8Pc3BxpaWmYOXMmDA0NYWpqihUrVqC4uFjkPn7//fcwMDDA7t27UVBQUGP+3NxcrF27Fubm5tDV1YWVlRW2bt2KkpISvnyLFy8Gl8vFu3fv4OXlBRMTE9jb2/Nt+/DhA+bPnw8TExOYmJhgxYoVKCkpQVFREdauXYsBAwagT58++Pnnn5GZmclX/t9//4358+dj6NCh0NPTQ79+/TBnzhwkJyfX2Ievx0grD3MI+6k8llpQUIBNmzbBysoKurq6GDRoEFasWIHs7GyBek6cOAFbW1vo6enB2toaR48erbFtPK9fv4aioqJAEAUAaWlpdOrUiS+trKwMe/fuhYODA/T19dGvXz+4u7vj2rVrfPkuX74MFxcXGBgYwMjICFOmTMG9e/f48vCOx+bNmxEVFQU7Ozvo6enh9OnTbF179uxh+9avXz/Mnz+f/X9e3+iMtBnKz88X+KUEIFIQEcXmzZshKyuLqVOn4sOHDwgKCsLPP/8Me3t7HD9+HOPHj0dhYSH27NkDDw8PnDt3jr3EfPDgAeLj4zFs2DCoqakhNzcXJ06cwIwZMxAUFAQzMzO+uoqLizF58mSYmppi4cKFuHfvHo4cOQIlJSX88ssvIrfZ09MTkydPRkhICKZPn15lvpKSEvz44494/PgxxowZg++++w5///03duzYgSdPnmDnzp0C+0ybNg2amprw8vISCLYzZ86EhoYGvLy8cOvWLRw5cgQyMjJIS0tDRUUFZs+ejefPnyMsLAyysrLYvHkzu++ZM2fw8eNHjB49Gl27dkVaWhrCw8Ph6uqKmJgYdO7cWeT+KykpwdfXVyB9x44dSElJYYMWr/8vXryAs7MztLS08OrVKxw6dAh3795FREQEe6Z74sQJ/Prrr+ByuZg3bx7y8vLg6+uLbt26idQmdXV1XLt2DWfPnsUPP/xQbd6Kigp4eHjg4sWLGDx4MDskcO/ePSQkJGDQoEEAgNOnT2PevHnQ0NCAh4cHSktLERYWBjc3N+zfvx/GxsZ85V64cAFZWVkYP348lJSUoKGhAYZh4OnpicuXL8PR0RETJ05Eeno6Dh06hFu3biE6OhpKSkoi9VFkDGk2EhMTGQ6HU+PP9evX2X22bdvGcDgcgbJSU1MZDofDHD16lE07evQow+FwGCcnJ6a0tJRN37BhA8PhcBgLCwsmPz+fTQ8JCWE4HA5z/vx5Nq2goECgrqKiIsba2pr56aef+NLd3NwYDofDBAUF8aXPmjWL6d+/f43Hg9eHTZs2seWZmpoynz9/5jte4eHhAm0ODAzkK2vdunUMh8NhLly4wKYtWrSI4XA4jLe3t0DdvG3r1q3jS3dycmK4XC7j4eHBlz5nzhzmu+++Y7Kzs9k0YccqOTmZ0dHRYXbu3MmXzuFwmG3btrGfeX1LTEwUfnAYhtm3bx/D4XCY3bt3s2m7d+9mdHR0mPv37/PlvXDhAsPhcJjDhw8zDMMwpaWlzMCBAxlLS0smLy+Pzffs2TOmd+/eQv9PCeuLgYEBw+FwGGtra8bb25uJiopiPn78KJA3Ojqa4XA4zB9//CGwraKigq9N5ubmTE5ODrv93bt3jIGBAePo6Mim8f5v6OjoMCkpKXzlxcTECPy/ZRiGefz4MfPdd98JbYO46NK+GZo8eTKCgoIEfqZMmVIv5Y8dOxbS0v93McL7K+/g4MA3PmhkZATgy00FnrZt27L/Li4uRlZWFvLz82FqaooHDx4I1CUpKQkXFxe+NFNTU2RmZiIvL69W7fb09ER2djb2799fZZ4LFy6gXbt2mDBhAl/61KlT2e1fc3V1rbK88ePH8302MjICwzACfTIyMkJ5eTnevHnDplU+Vvn5+cjKykKHDh2goaEh9FjVxpUrV/DHH39gxIgRmDZtGpseExMDXV1dqKmpITMzk/3p06cP2rVrh4SEBADAw4cPkZGRAWdnZ8jLy7P7czgc9uywJhoaGoiKioKTkxOysrIQHh6OxYsXY9CgQfD29kZ+fj6b98yZM5CTk8PPP/8sUI6EhARfm8aNG4f27duz27/55hs4ODjg0aNHSE9P59vX3Nwc3bt350uLiYmBiooKDA0N+Y6BiooKevbsyR6D+kSX9s2QlpYWBgwYIJD+/v37eilfVVWV77OioiIACFzS8dIrj62VlJQgICAAx48f5wsawP/9QlTWuXNngcF/3i9JTk4OFBQURG533759MWjQIOzfvx/u7u5C87x58wbq6uoCN2q6du2K9u3bIy0tTWCfr38RK1NTUxPa9q+PVeU+8Xz48AF//vknLl68yJcOQGD8sDaSkpIwb9486OrqYu3atXzbXr58iaKiIoEhFh7ekBHvu9PQ0BDIo6GhgStXrojUFg0NDfj4+IBhGLx58wY3b95ESEgIIiIiUF5eDh8fHwBfxlPV1dWrvZHHa5OwO/28tLS0NKioqLDpPXr0EMj78uVLpKenV3kMvv7/Xx8okLZwwoIX8GVMqipSUlK1SmcYhv33+vXrERYWhvHjx8PY2BgdOnSAlJQUjh49ilOnTgnsKylZ9UVP5XJF5enpCWdnZ+zdu1fkM6eaVHeXt6r213SsKioqMGXKFKSnp2PSpEno1asX2rVrB0lJSaxfv75OfQe+/FGbNWsW2rVrB39/f4E/GAzDwNDQEHPnzhW6P++PY32TkJCAuro61NXVYW9vDzs7O5w8eRJr1qzhu/qpb8KebGAYBurq6lizZo3QfWRkZOq9HRRIW7jKZ0IdOnRg0ytfjtenmJgYjBo1CitXruRLj4yMbJD6vqavr4+hQ4ciJCQE2traAtvV1dVx+/ZtFBcX8/2SZWRkIDc3F+rq6o3SzufPn+P58+fw8fGBk5MT37bs7Ow6nZGWlZXB09MT6enpCAkJQdeuXQXy9OzZEzk5OUKvaCrjnWm/fPlSYJuwtNqQkZFB7969kZKSgqysLCgrK6Nnz56Ij4+v9vEy3ncj7EF+Xpoo31/Pnj1x8+ZNmJiYNNpzuDRG2sL17NkTAJCYmMimMQyDgwcPNkh9kpKSAmdTycnJOH/+fIPUJ4ynpyf7VMHXLCwsUFBQgNDQUL70wMBAAMDQoUMbpY28M9mvj9WxY8eQkZFRpzLXrl2LxMRErF+/Hnp6ekLz2NnZITk5GdHR0QLbysvL2WEaXV1dKCsrIyIigu9pkOfPnws8jlSVmzdvCn2MLTs7G3fu3EHHjh3Zu+M2NjYoLi7Gjh07BPLzjpGOjg66du2K8PBwvvHz9PR0nDx5Ejo6OnyX9VWxs7NDfn4+du/eLbQuYU/EiIvOSFu4gQMHonv37vD29kZSUhIUFRURGxuLwsLCBqnPysoKUVFRaNu2LbS1tZGamorQ0FBoaWnh8ePHDVLn13r37o0ffvgBf/31l8A2Z2dnHD16FBs2bEBycjJ69+6N27dv49SpU7CwsICFhUWjtFFTUxMaGhr4/fff8fbtWygrK+PevXuIi4urdky2KpcvX0ZoaCj09fVRVlaG48eP820fOHAgunTpgsmTJ+PSpUtYvHgxrly5AkNDQzAMg5SUFMTGxuKXX36Bk5MTpKWlsWDBAixatAguLi5wcnJCXl4egoODweFw8PTp0xrbFBgYiLt378LS0hLa2tqQlZXFmzdvcOLECWRkZOC3335jh0AcHBwQExODPXv24MWLFxgwYAAkJCRw//59fPPNN1iwYAGkpaWxZMkSzJs3D2PHjsXo0aPZx5/KysqwbNkykY6Vvb09zp07h23btuHevXswMzNjH1m7cOEC7O3tMWfOnFp/B9WhQNrCSUtLY8eOHVizZg0CAgKgoKAAOzs7uLi4wM7Ort7rW7p0KeTk5BAbG4uoqChoampi9erVSEpKarRACgBz585FbGyswFiwjIwM9u/fj61bt7JtVFFRwaxZs4TeMW4o0tLS2L17N9avX4/g4GCUlpbCwMAABw4cwPr162td3qdPnwAA9+/fx8KFCwW2Hzx4EF26dIGsrCwOHDiAffv2ISYmBufOnYOsrCxUVVVhb2+P/v37s/uMGjUKwJfXbzdu3Ah1dXUsXLgQb9++FSmQzpw5E6dPn8atW7cQFxeHvLw8tG/fHrq6uli5ciUGDx7M5pWUlMT27dsRGBiI6OhoXL9+HfLy8ujduzff0IetrS3k5eUREBCAbdu2QVJSEoaGhtiyZQsMDAxEOlYSEhLYsmULQkNDcfToUWzZsgWSkpL45ptvMGDAANjY2IhUTm1IMHUd9SaEEAKAxkgJIURsFEgJIURMFEgJIURMFEgJIURMFEgJIURMFEgJIURMFEgJIURMFEgJIURMFEgJIURMFEgJIURM/w+fKQDSoV6onwAAAABJRU5ErkJggg==\n",
|
|
"text/plain": [
|
|
"<Figure size 244.8x133.2 with 1 Axes>"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
}
|
|
],
|
|
"source": [
|
|
"algorithms = ['SimPLe', 'CURL', 'DrQ', 'SPR', 'IRIS (ours)']\n",
|
|
"fig, axes = plot_utils.plot_interval_estimates(\n",
|
|
" {k: v[3:4] for k, v in aggregate_scores.items()}, \n",
|
|
" {k: v[:, 3:4] for k, v in aggregate_interval_estimates.items()},\n",
|
|
" metric_names = ['Optimality Gap'],\n",
|
|
" algorithms=algorithms,\n",
|
|
" colors=ATARI_100K_COLOR_DICT,\n",
|
|
" xlabel_y_coordinate=-0.3,\n",
|
|
" xlabel='Human Normalized Score')\n",
|
|
"plt.show()\n",
|
|
"save_fig(fig, 'optimality_gap')"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"id": "QM9CLBft9Bqs"
|
|
},
|
|
"source": [
|
|
"# Performance profile"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {
|
|
"colab": {
|
|
"base_uri": "https://localhost:8080/"
|
|
},
|
|
"id": "v1xRtwkwwtPo",
|
|
"outputId": "305cd55a-5d1a-41a5-b32d-2b955f09f320"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"algorithms = ['SimPLe', 'CURL', 'DrQ', 'SPR', 'IRIS (ours)']\n",
|
|
"score_dict = {key: all_score_dict[key] for key in algorithms}\n",
|
|
"ATARI_100K_TAU = np.linspace(0.0, 8.0, 201)\n",
|
|
"reps = 2000\n",
|
|
"\n",
|
|
"score_distributions, score_distributions_cis = rly.create_performance_profile(score_dict, ATARI_100K_TAU, reps=reps)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {
|
|
"colab": {
|
|
"base_uri": "https://localhost:8080/",
|
|
"height": 348
|
|
},
|
|
"id": "PGMKH2sGINQj",
|
|
"outputId": "d20552ea-f134-43e1-95ad-a5f6056cb9c2"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"fig, ax = plt.subplots(ncols=1, figsize=(7.25, 4.7))\n",
|
|
"\n",
|
|
"plot_utils.plot_performance_profiles(\n",
|
|
" score_distributions, ATARI_100K_TAU,\n",
|
|
" performance_profile_cis=score_distributions_cis,\n",
|
|
" colors=ATARI_100K_COLOR_DICT,\n",
|
|
" xlabel=r'Human Normalized Score $(\\tau)$',\n",
|
|
" labelsize='xx-large',\n",
|
|
" ax=ax)\n",
|
|
"\n",
|
|
"ax.axhline(0.5, ls='--', color='k', alpha=0.4)\n",
|
|
"fake_patches = [mpatches.Patch(color=ATARI_100K_COLOR_DICT[alg], \n",
|
|
" alpha=0.75) for alg in algorithms]\n",
|
|
"legend = fig.legend(fake_patches, algorithms, loc='upper center', \n",
|
|
" fancybox=True, ncol=3, \n",
|
|
" fontsize='x-large',\n",
|
|
" bbox_to_anchor=(0.57, 0.93))\n",
|
|
"save_fig(fig, 'performance_profile')"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"id": "3kmNgRii94Ee"
|
|
},
|
|
"source": [
|
|
"# Probability of improvement"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {
|
|
"colab": {
|
|
"base_uri": "https://localhost:8080/"
|
|
},
|
|
"id": "62YrK4dn98fs",
|
|
"outputId": "2b7ec982-c96d-4283-cdae-9937e71b14f5"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"algorithms = ['SimPLe', 'CURL', 'DrQ', 'SPR', 'IRIS (ours)']\n",
|
|
"our_algorithm = 'IRIS (ours)'\n",
|
|
"all_pairs = {}\n",
|
|
"for alg in (algorithms):\n",
|
|
" if alg == our_algorithm:\n",
|
|
" continue\n",
|
|
" pair_name = f'{our_algorithm}_{alg}'\n",
|
|
" all_pairs[pair_name] = (all_score_dict[our_algorithm], all_score_dict[alg]) \n",
|
|
"\n",
|
|
"probabilities, probability_cis = {}, {}\n",
|
|
"reps = 1000\n",
|
|
"probabilities, probability_cis = rly.get_interval_estimates(all_pairs, metrics.probability_of_improvement, reps=reps)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {
|
|
"colab": {
|
|
"base_uri": "https://localhost:8080/",
|
|
"height": 248
|
|
},
|
|
"id": "hT5A8wyu-Fpo",
|
|
"outputId": "8bd6d523-0e57-4417-b120-dceb6cad9a7a"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"fig, ax = plt.subplots(figsize=(4, 3))\n",
|
|
"h = 0.6\n",
|
|
"algorithm_labels = []\n",
|
|
"\n",
|
|
"for i, (alg_pair, prob) in enumerate(probabilities.items()):\n",
|
|
" _, alg1 = alg_pair.split('_')\n",
|
|
" algorithm_labels.append(alg1)\n",
|
|
" (l, u) = probability_cis[alg_pair]\n",
|
|
" ax.barh(y=i, width=u-l, height=h, left=l, color=ATARI_100K_COLOR_DICT[alg1], alpha=0.75)\n",
|
|
" ax.vlines(x=prob, ymin=i-7.5 * h/16, ymax=i+(6*h/16), color='k', alpha=0.85)\n",
|
|
"\n",
|
|
"ax.set_yticks(range(len(algorithm_labels)))\n",
|
|
"ax.set_yticklabels(algorithm_labels)\n",
|
|
"\n",
|
|
"ax.set_xlim(0, 1)\n",
|
|
"ax.axvline(0.5, ls='--', color='k', alpha=0.4)\n",
|
|
"ax.set_title(fr'P(IRIS > $Y$)', size='xx-large')\n",
|
|
"plot_utils._annotate_and_decorate_axis(ax, labelsize='xx-large', ticklabelsize='xx-large')\n",
|
|
"ax.set_ylabel(r'Algorithm $Y$', size='xx-large')\n",
|
|
"ax.xaxis.set_major_locator(MaxNLocator(4))\n",
|
|
"fig.subplots_adjust(wspace=0.25, hspace=0.45)\n",
|
|
"save_fig(fig, 'probability_of_improvement')"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"# Detailed results"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"for k in aggregate_scores.keys():\n",
|
|
" n_runs, n_games = all_score_dict[k].shape\n",
|
|
" assert n_games == 26\n",
|
|
" score_dict = score_data_dict_games[k]\n",
|
|
" mean, median, iqm, og = aggregate_scores[k]\n",
|
|
" sh = np.sum(np.mean(all_score_dict[k], axis=0) >= 1)\n",
|
|
" print(f\"\\n####################\\n{k}\\n####################\\n\")\n",
|
|
" print(f\"{n_runs} runs\")\n",
|
|
" print(f\"#superhuman: {sh}\\nMean: {mean:.3f}\\nMedian: {median:.3f}\\nIQM: {iqm:.3f}\\nOptimality gap: {og:.3f}\\n\")\n",
|
|
" \n",
|
|
" for game in score_dict.keys():\n",
|
|
" h, r = HUMAN_SCORES[game], RANDOM_SCORES[game]\n",
|
|
" raw_score = score_dict[game] * (h - r) + r\n",
|
|
" print(f\"{game}: {np.mean(raw_score): .1f}\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"# Generate $\\LaTeX$ results table"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Utility to print the latex table\n",
|
|
"\n",
|
|
"first_row = [\"Game\", \"Random\", \"Human\", \"MuZero\", \"EfficientZero\", \"SimPLe\", \"CURL\", \"DrQ\", \"SPR\", r\"\\textsc{iris} (ours)\"]\n",
|
|
"rows = [first_row]\n",
|
|
"\n",
|
|
"# Raw scores\n",
|
|
"\n",
|
|
"for game in ATARI_100K_GAMES:\n",
|
|
" raw_scores = [RANDOM_SCORES[game], HUMAN_SCORES[game]]\n",
|
|
" raw_scores.extend([np.mean(score_data_dict_games[algo][game] * (HUMAN_SCORES[game] - RANDOM_SCORES[game]) + RANDOM_SCORES[game]) for algo in aggregate_scores.keys()])\n",
|
|
" idx_max_baselines = 4 + np.argmax(raw_scores[4:])\n",
|
|
" idx_max_all = 2 + np.argmax(raw_scores[2:])\n",
|
|
" raw_scores = [f\"{x:.1f}\" for x in raw_scores]\n",
|
|
" raw_scores[idx_max_baselines] = f\"\\\\textbf{{{raw_scores[idx_max_baselines]}}}\"\n",
|
|
" raw_scores[idx_max_all] = f\"\\\\underline{{{raw_scores[idx_max_all]}}}\"\n",
|
|
" row = [game, *raw_scores]\n",
|
|
" rows.append(row)\n",
|
|
"\n",
|
|
"# Aggregates \n",
|
|
"\n",
|
|
"first_col = [\"\\\\#Superhuman (↑)\", \"Mean (↑)\", \"Median (↑)\", \"IQM (↑)\", \"Optimality Gap (↓)\"]\n",
|
|
"cols = [\n",
|
|
" [0, 0, 0, 0, 1], # Random\n",
|
|
" [float('-inf'), 1, 1, 1, 0], # Human\n",
|
|
"]\n",
|
|
"for algo in aggregate_scores.keys():\n",
|
|
" n_runs, n_games = all_score_dict[algo].shape\n",
|
|
" assert n_games == len(ATARI_100K_GAMES)\n",
|
|
" score_dict = score_data_dict_games[algo]\n",
|
|
" sh = np.sum(np.mean(all_score_dict[algo], axis=0) >= 1)\n",
|
|
" col = [sh, *aggregate_scores[algo]]\n",
|
|
" cols.append(col)\n",
|
|
"\n",
|
|
"rows_ = np.array(cols).T\n",
|
|
"for i, row in enumerate(rows_):\n",
|
|
" idx_best_baselines = 4 + (np.argmin(row[4:]) if i == len(rows_) - 1 else np.argmax(row[4:]))\n",
|
|
" idx_best_all = 2 + (np.argmin(row[2:]) if i == len(rows_) - 1 else np.argmax(row[2:]))\n",
|
|
" row = [f\"{x:.{0 if i == 0 else 3}f}\" if not math.isinf(x) else 'N/A' for x in row]\n",
|
|
" row[idx_best_baselines] = f\"\\\\textbf{{{row[idx_best_baselines]}}}\"\n",
|
|
" row[idx_best_all] = f\"\\\\underline{{{row[idx_best_all]}}}\"\n",
|
|
" rows.append([first_col[i]] + row)\n",
|
|
" \n",
|
|
" # IQM and OG are N/A for MuZero and EfficientZero. \n",
|
|
" if i >= 3:\n",
|
|
" rows[-1][3:5] = ['N/A'] * 2\n",
|
|
" #rows[-1][3:5] = 'N/A' \n",
|
|
"\n",
|
|
"rows = np.array(rows)\n",
|
|
"for i in range(rows.shape[1]):\n",
|
|
" max_len = max(map(len, rows[:, i])) + 1\n",
|
|
" rows[:, i] = list(map(lambda x: x.ljust(max_len), rows[:, i]))\n",
|
|
"\n",
|
|
"rows = [' & '.join(row) for row in rows]\n",
|
|
"\n",
|
|
"for i, row in enumerate(rows):\n",
|
|
" print(row + r' \\\\')\n",
|
|
" if i in [0, 26]:\n",
|
|
" print(r\"\\midrule\")\n"
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"colab": {
|
|
"collapsed_sections": [],
|
|
"name": "Copy of deep_rl_precipice_colab.ipynb",
|
|
"provenance": []
|
|
},
|
|
"kernelspec": {
|
|
"display_name": "Python 3 (ipykernel)",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.8.12"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 1
|
|
}
|