{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# autoreload your package\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/media/wassname/SGIronWolf/projects5/2024/prob_jsonformer/.venv/lib/python3.9/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading model and tokenizer...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/media/wassname/SGIronWolf/projects5/2024/prob_jsonformer/.venv/lib/python3.9/site-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n",
      "  warnings.warn(\n",
      "Loading checkpoint shards: 100%|██████████| 4/4 [00:02<00:00,  1.42it/s]\n",
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded model and tokenizer\n"
     ]
    }
   ],
   "source": [
    "from transformers import AutoModelForCausalLM, AutoTokenizer\n",
    "import torch\n",
    "\n",
    "from prob_jsonformer import Jsonformer\n",
    "\n",
    "print(\"Loading model and tokenizer...\")\n",
    "model_name = \"databricks/dolly-v2-3b\"\n",
    "model_name = \"NousResearch/Meta-Llama-3-8B-Instruct\".lower()\n",
    "# model_name = \"failspy/Llama-3-8B-Instruct-abliterated\"\n",
    "# model_name = \"cognitivecomputations/Llama-3-8B-Instruct-abliterated-v2\"\n",
    "# model_name = \"nvidia/Llama3-ChatQA-1.5-8B\" # 4b\n",
    "# model_name = \"CohereForAI/c4ai-command-r-v01-4bit\" # 35b/4 = 8.75b\n",
    "model = AutoModelForCausalLM.from_pretrained(\n",
    "    model_name,\n",
    "    use_cache=True,\n",
    "    torch_dtype=torch.float16,\n",
    "    attn_implementation=\"eager\",\n",
    ").to(\"cuda:0\")\n",
    "tokenizer = AutoTokenizer.from_pretrained(model_name, use_fast=True, use_cache=True)\n",
    "tokenizer.padding_side = \"left\"\n",
    "print(\"Loaded model and tokenizer\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "if tokenizer.pad_token_id is None:\n",
    "    tokenizer.pad_token_id = tokenizer.bos_token_id"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prob dist\n",
    "\n",
    "- Can LLM's sample from a distribution? http://people.csail.mit.edu/renda/llm-sampling-paper\n",
    "- Can they forecast events https://arxiv.org/abs/2402.07862"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from prob_jsonformer.prob_choice_tree import prob_choice_tree\n",
    "import pandas as pd\n",
    "import torch.nn.functional as F\n",
    "from tqdm.auto import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {},
   "outputs": [],
   "source": [
    "def method0(choices, n=400):\n",
    "    \"\"\"\n",
    "    just generate many times\n",
    "    \"\"\"\n",
    "\n",
    "    toks = tokenizer.encode(prompt, return_tensors=\"pt\").to(model.device)\n",
    "    data = []\n",
    "    i = 0\n",
    "    while i<n*10 and len(data) < n:\n",
    "        i += 1\n",
    "        o = model.generate(\n",
    "            toks,\n",
    "            do_sample=True,\n",
    "            max_new_tokens=2,\n",
    "            temperature=1,\n",
    "            pad_token_id=tokenizer.pad_token_id\n",
    "        )\n",
    "        o = tokenizer.decode(o[0, -2:], skip_special_tokens=True)\n",
    "        try:\n",
    "            o = int(float(o))\n",
    "        except ValueError:\n",
    "            print(f\"failed `float({o})`\")\n",
    "            continue\n",
    "        if str(o) in choices:\n",
    "            data.append(o)\n",
    "        else:\n",
    "            print(f\"Generated `{o}` not in choices {choices}\")\n",
    "    d = pd.Series(data).value_counts(normalize=True).to_dict()\n",
    "\n",
    "    df = pd.DataFrame([d]).T\n",
    "    df.index = df.index.astype(int)\n",
    "    df = df.sort_index()\n",
    "    return df\n",
    "\n",
    "def method1(choices):\n",
    "    \"\"\"\n",
    "    Here we try each completion to get the hindsight probability of each choice\n",
    "\n",
    "    from https://github.com/1rgs/jsonformer/pull/19\n",
    "    \"\"\"\n",
    "    data = {}\n",
    "    for choice in choices:\n",
    "        n_option_tokens = tokenizer.encode(f'\"{choice}', add_special_tokens=False, return_tensors=\"pt\").shape[1]\n",
    "        toks = tokenizer.encode(prompt + f\"{choice}\", return_tensors=\"pt\").to(model.device)\n",
    "        option_tokens = toks[0, -n_option_tokens:]\n",
    "        o = model.forward(\n",
    "            toks,\n",
    "        )\n",
    "        logits = o.logits[0, -n_option_tokens-1:]\n",
    "\n",
    "        probabilities = torch.softmax(logits, dim=1)\n",
    "        option_token_probabilities = probabilities[:-1][torch.arange(probabilities.shape[0]-1), option_tokens]\n",
    "        option_probability = torch.prod(option_token_probabilities)\n",
    "        data[choice] = option_probability.item()\n",
    "\n",
    "    df = pd.DataFrame([data]).T\n",
    "    df.index = df.index.astype(int)\n",
    "    df.iloc[:, 0] = df.iloc[:, 0] / df.iloc[:, 0].sum()\n",
    "    df = df.sort_index()\n",
    "    return df\n",
    "\n",
    "\n",
    "def method2(choices):\n",
    "    \"\"\"\n",
    "    Useing logit constraints\n",
    "\n",
    "    https://github.com/1rgs/jsonformer/pull/41\n",
    "    \"\"\"\n",
    "    # TODO\n",
    "\n",
    "def method3(choices):\n",
    "    \"\"\"using my prob choice tree.\"\"\"\n",
    "    input_ids = tokenizer.encode(prompt, return_tensors=\"pt\").to(\n",
    "        model.device\n",
    "    )[0]\n",
    "    choice_tokens = tokenizer(choices).input_ids\n",
    "    choice_tokens = [torch.tensor(c) for c in choice_tokens]\n",
    "    r = prob_choice_tree(model, tokenizer, input_ids, choice_tokens)\n",
    "\n",
    "    df = pd.DataFrame(r).set_index('choice')\n",
    "    df['prob'] = df['prob'].astype(float)\n",
    "    df.index = df.index.astype(int)\n",
    "    df.iloc[:, 0] = df.iloc[:, 0] / df.iloc[:, 0].sum()\n",
    "    df = df.sort_index()\n",
    "    return df\n",
    "\n",
    "\n",
    "\n",
    "prompt = \"\"\"Q: Please sample a number from the distribution [0, 20]: \n",
    "A: \"\"\"\n",
    "\n",
    "choices = [str(i) for i in range(21)]\n",
    "\n",
    "ideal_dist = pd.Series({str(i): 1/21 for i in range(21)})\n",
    "ideal_dist.index = ideal_dist.index.astype(int)\n",
    "# ideal_dist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    1.0\n",
       "dtype: float64"
      ]
     },
     "execution_count": 161,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.073333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.100000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.100000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0.080000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0.046667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>0.046667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>0.125556</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>0.034444</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>0.367778</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>0.025556</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           0\n",
       "4   0.073333\n",
       "5   0.100000\n",
       "7   0.100000\n",
       "8   0.080000\n",
       "10  0.046667\n",
       "11  0.046667\n",
       "12  0.125556\n",
       "13  0.034444\n",
       "14  0.367778\n",
       "15  0.025556"
      ]
     },
     "execution_count": 139,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t0 = pd.Timestamp.now()\n",
    "r0 = method0(choices, n=900)\n",
    "t0 = pd.Timestamp.now() - t0\n",
    "r0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [],
   "source": [
    "t1 = pd.Timestamp.now()\n",
    "r1 = method1(choices)\n",
    "t1 = pd.Timestamp.now() - t1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [],
   "source": [
    "t3 = pd.Timestamp.now()\n",
    "r3 = method3(choices)\n",
    "t3 = pd.Timestamp.now() - t3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>method0: sampling</th>\n",
       "      <th>method1: hindsight</th>\n",
       "      <th>method3: generation tree</th>\n",
       "      <th>ideal</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000642</td>\n",
       "      <td>0.0365</td>\n",
       "      <td>0.047619</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.002812</td>\n",
       "      <td>0.0667</td>\n",
       "      <td>0.047619</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.001010</td>\n",
       "      <td>0.0710</td>\n",
       "      <td>0.047619</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.009076</td>\n",
       "      <td>0.0653</td>\n",
       "      <td>0.047619</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.073333</td>\n",
       "      <td>0.028844</td>\n",
       "      <td>0.0786</td>\n",
       "      <td>0.047619</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.100000</td>\n",
       "      <td>0.043982</td>\n",
       "      <td>0.0999</td>\n",
       "      <td>0.047619</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.013840</td>\n",
       "      <td>0.0662</td>\n",
       "      <td>0.047619</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.100000</td>\n",
       "      <td>0.063993</td>\n",
       "      <td>0.0858</td>\n",
       "      <td>0.047619</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0.080000</td>\n",
       "      <td>0.059184</td>\n",
       "      <td>0.0682</td>\n",
       "      <td>0.047619</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.012213</td>\n",
       "      <td>0.0745</td>\n",
       "      <td>0.047619</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0.046667</td>\n",
       "      <td>0.037620</td>\n",
       "      <td>0.0327</td>\n",
       "      <td>0.047619</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>0.046667</td>\n",
       "      <td>0.045378</td>\n",
       "      <td>0.0187</td>\n",
       "      <td>0.047619</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>0.125556</td>\n",
       "      <td>0.119555</td>\n",
       "      <td>0.0307</td>\n",
       "      <td>0.047619</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>0.034444</td>\n",
       "      <td>0.031679</td>\n",
       "      <td>0.0279</td>\n",
       "      <td>0.047619</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>0.367778</td>\n",
       "      <td>0.423860</td>\n",
       "      <td>0.0274</td>\n",
       "      <td>0.047619</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>0.025556</td>\n",
       "      <td>0.042629</td>\n",
       "      <td>0.0312</td>\n",
       "      <td>0.047619</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.010285</td>\n",
       "      <td>0.0344</td>\n",
       "      <td>0.047619</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.034793</td>\n",
       "      <td>0.0235</td>\n",
       "      <td>0.047619</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.017495</td>\n",
       "      <td>0.0225</td>\n",
       "      <td>0.047619</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.001101</td>\n",
       "      <td>0.0109</td>\n",
       "      <td>0.047619</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000010</td>\n",
       "      <td>0.0277</td>\n",
       "      <td>0.047619</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    method0: sampling  method1: hindsight  method3: generation tree     ideal\n",
       "0            0.000000            0.000642                    0.0365  0.047619\n",
       "1            0.000000            0.002812                    0.0667  0.047619\n",
       "2            0.000000            0.001010                    0.0710  0.047619\n",
       "3            0.000000            0.009076                    0.0653  0.047619\n",
       "4            0.073333            0.028844                    0.0786  0.047619\n",
       "5            0.100000            0.043982                    0.0999  0.047619\n",
       "6            0.000000            0.013840                    0.0662  0.047619\n",
       "7            0.100000            0.063993                    0.0858  0.047619\n",
       "8            0.080000            0.059184                    0.0682  0.047619\n",
       "9            0.000000            0.012213                    0.0745  0.047619\n",
       "10           0.046667            0.037620                    0.0327  0.047619\n",
       "11           0.046667            0.045378                    0.0187  0.047619\n",
       "12           0.125556            0.119555                    0.0307  0.047619\n",
       "13           0.034444            0.031679                    0.0279  0.047619\n",
       "14           0.367778            0.423860                    0.0274  0.047619\n",
       "15           0.025556            0.042629                    0.0312  0.047619\n",
       "16           0.000000            0.010285                    0.0344  0.047619\n",
       "17           0.000000            0.034793                    0.0235  0.047619\n",
       "18           0.000000            0.017495                    0.0225  0.047619\n",
       "19           0.000000            0.001101                    0.0109  0.047619\n",
       "20           0.000000            0.000010                    0.0277  0.047619"
      ]
     },
     "execution_count": 172,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiwAAAGhCAYAAABCse9yAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAABWQElEQVR4nO3de1wU5f4H8M+yXJaLXATlYuiiqEiiKAiBppabYGai5S2PIHlLxeTgvRQ0L6gRoemB8uT1VFrnqF1MTBEsDUExtfKGJkEpePsJggoKz+8PX4yugLILyICf9+s1L9l5nvnOM7PCfnZmdlYhhBAgIiIikjGD+h4AERER0eMwsBAREZHsMbAQERGR7DGwEBERkewxsBAREZHsMbAQERGR7DGwEBERkewxsBAREZHsGdb3AGpDWVkZLly4gCZNmkChUNT3cIiIiKgahBC4ceMGnJycYGDw6GMojSKwXLhwAc7OzvU9DCIiItJDTk4OnnnmmUf2aRSBpUmTJgDubbClpWU9j4aIiIiqo6CgAM7OztLr+KM0isBSfhrI0tKSgYWIiKiBqc7lHLzoloiIiGSPgYWIiIhkj4GFiIiIZK9RXMNSXaWlpbhz5059D4PoqWZkZASlUlnfwyCiBuapCCxCCOTm5uL69ev1PRQiAmBtbQ0HBwfeN4mIqu2pCCzlYaV58+YwMzPjH0mieiKEwM2bN3Hp0iUAgKOjYz2PiIgaikYfWEpLS6WwYmtrW9/DIXrqmZqaAgAuXbqE5s2b8/QQEVVLo7/otvyaFTMzs3oeCRGVK/995DVlRFRdjT6wlONpICL54O8jEenqqQksRERE1HAxsBAAICUlBQqFok4+SaVQKLB9+/ZarytHD+/H9evXw9raul7HRETUGDT6i24fRT17xxNdX9bS/k90fVXp3bs3PD09ERcXVy/rv3btGqZMmYJvv/0WBgYGeO2117BixQpYWFjUy3jq0rBhw/Dyyy/X9zCIiBo8HmGhJ27kyJH4/fffsXv3bnz33Xf48ccfMX78+PoeVp0wNTVF8+bN63sYREQNHgOLjPXu3RtTpkxBeHg4bGxsYG9vjzVr1qCoqAihoaFo0qQJXF1dsXPnTq3lfvvtN/Tr1w8WFhawt7fHqFGjcOXKFQDA6NGjsW/fPqxYsQIKhQIKhQJZWVnSshkZGfD29oaZmRn8/f1x+vRprdrx8fFo06YNjI2N0b59e2zatEmrPTMzEz179oRKpYK7uzt2796t1X7y5EkkJibi3//+N3x9fdGjRw989NFH2Lx5My5cuFDtffPnn39iwIABsLGxgbm5OZ599ll8//33AO59lH3MmDFwcXGBqakp2rdvjxUrVmgtP3r0aAQFBWHJkiWwt7eHtbU13nvvPdy9exczZsxA06ZN8cwzz2DdunXSMllZWVAoFNi8eTP8/f2hUqnQsWNH7Nu3r8pxPnxKaP78+fD09MSmTZugVqthZWWF4cOH48aNG1KfGzduYOTIkTA3N4ejoyM+/PBD9O7dG+Hh4dXeP0REjQ0Di8xt2LABdnZ2SE9Px5QpUzBx4kQMGTIE/v7+OHLkCPr27YtRo0bh5s2bAIDr16/jxRdfRJcuXXD48GEkJiYiLy8PQ4cOBQCsWLECfn5+GDduHC5evIiLFy/C2dlZWt+7776LDz74AIcPH4ahoSHefPNNqW3btm2YOnUqpk2bht9++w0TJkxAaGgokpOTAQBlZWUYPHgwjI2NkZaWhoSEBMyaNUtre1JTU2FtbQ1vb29pnkajgYGBAdLS0qR5CoUC69evr3K/TJ48GcXFxfjxxx/x66+/YtmyZdIppbKyMjzzzDP46quvcOLECURGRuKdd97Bl19+qVVj7969uHDhAn788UfExsYiKioKr7zyCmxsbJCWloa33noLEyZMwF9//aW13IwZMzBt2jT88ssv8PPzw4ABA3D16tXHPpflzp07h+3bt+O7777Dd999h3379mHp0qVSe0REBA4cOIBvvvkGu3fvxk8//YQjR45Uuz4RUWPEwCJznTt3xty5c9G2bVvMmTMHKpUKdnZ2GDduHNq2bYvIyEhcvXoVx48fBwCsWrUKXbp0wZIlS+Dm5oYuXbpg7dq1SE5OxpkzZ2BlZQVjY2OYmZnBwcEBDg4OWjfuWrx4MXr16gV3d3fMnj0bP//8M27fvg0AiImJwejRozFp0iS0a9cOERERGDx4MGJiYgAAe/bswalTp7Bx40Z07twZPXv2xJIlS7S2Jzc3t8IpEkNDQzRt2hS5ubnSvPbt28PKyqrK/ZKdnY3u3bvDw8MDrVu3xiuvvIKePXsCuPddNQsWLIC3tzdcXFwwcuRIhIaGVggsTZs2xcqVK9G+fXu8+eabaN++PW7evIl33nlH2t/GxsbYv3+/1nJhYWF47bXX0KFDB8THx8PKygqffvpptZ5P4F6gWr9+PTp27Ijnn38eo0aNQlJSEoB7R1c2bNiAmJgY9OnTBx07dsS6detQWlpa7fpENTbf6v5EJBMMLDLXqVMn6WelUglbW1t4eHhI8+zt7QFAutX5sWPHkJycDAsLC2lyc3MDcO+dvS7rK79tenntkydPonv37lr9u3fvjpMnT0rtzs7OcHJyktr9/Pyqv7EPOHXqFAYNGlRl+9tvv41Fixahe/fuiIqKkgJbudWrV8PLywvNmjWDhYUFPvnkE2RnZ2v1efbZZ2FgcP9XwN7eXmvflu/v8u2vbJsMDQ3h7e0t7YPqUKvVaNKkifTY0dFRWscff/yBO3fuwMfHR2q3srJC+/btq12fiKgxYmCROSMjI63HCoVCa175DbjKysoAAIWFhRgwYACOHj2qNZVfW6LL+h6uXRscHBwqBIC7d+/i2rVrcHBwqHadsWPH4o8//sCoUaPw66+/wtvbGx999BEAYPPmzZg+fTrGjBmDH374AUePHkVoaChKSkq0ajxu35bPq83tr2q9tb0OIqLGhoGlkenatSt+//13qNVquLq6ak3m5uYAAGNjY71OMXTo0AEHDhzQmnfgwAG4u7tL7Tk5Obh48aLUfvDgQa3+fn5+uH79OjIyMqR5e/fuRVlZGXx9fXUaj7OzM9566y1s3boV06ZNw5o1a6Qx+fv7Y9KkSejSpQtcXV2rdXSpuh7cprt37yIjIwMdOnSoldqtW7eGkZERDh06JM3Lz8/HmTNnaqU+EVFDxcDSyEyePBnXrl3DiBEjcOjQIZw7dw67du1CaGioFFLUajXS0tKQlZWFK1euVPvd/YwZM7B+/XrEx8cjMzMTsbGx2Lp1K6ZPnw7g3sWz7dq1Q0hICI4dO4affvoJ7777rlaNDh06IDAwEOPGjUN6ejoOHDiAsLAwDB8+XOtUkpubG7Zt21blWMLDw7Fr1y6cP38eR44cQXJyshQa2rZti8OHD2PXrl04c+YM5s2bpxUAamr16tXYtm0bTp06hcmTJ+P//u//tC5OrokmTZogJCQEM2bMQHJyMn7//XeMGTMGBgYGvJ09ET3Vnuobx8nlRm61ycnJCQcOHMCsWbPQt29fFBcXo1WrVggMDJSu15g+fTpCQkLg7u6OW7du4fz589WqHRQUhBUrViAmJgZTp06Fi4sL1q1bh969ewMADAwMsG3bNowZMwY+Pj5Qq9VYuXIlAgMDtep89tlnCAsLQ58+faQbx61cuVKrz+nTp5Gfn1/lWEpLSzF58mT89ddfsLS0RGBgID788EMAwIQJE/DLL79g2LBhUCgUGDFiBCZNmlTh49/6Wrp0KZYuXYqjR4/C1dUV33zzDezs7GqlNgDExsbirbfewiuvvAJLS0vMnDkTOTk5UKlUtbYOIqKGRiGEEPU9iJoqKCiAlZUV8vPzYWlpqdV2+/ZtnD9/Hi4uLvyDTzWSlZUFFxcX/PLLL/D09Hxi6y0qKkKLFi3wwQcfYMyYMU9svXWJv5cy9+Cng+ZX/caBqKYe9fr9sKf6CAuRHP3yyy84deoUfHx8kJ+fj/feew8AMHDgwHoeGRFR/WFgIZKhmJgYnD59GsbGxvDy8sJPP/1Uq6ediIgaGgYWompSq9V4EmdQu3TpovUpKiIi4qeEiIiIqAFgYCEiIiLZY2AhIiIi2WNgISIiItljYCEiIiLZ0yuwrF69Gmq1GiqVCr6+vkhPT6/Wcps3b4ZCoUBQUJDWfCEEIiMj4ejoCFNTU2g0GmRmZuozNNJTSkoKFAoFrl+/Xuu1FQoFtm/fXqs1qzPe+fPn18oN3tRqNeLi4mq9L3DvZnQKhQJHjx7Va2xERE8LnQPLli1bEBERgaioKBw5cgSdO3dGQEBAhW/gfVhWVhamT5+O559/vkLb8uXLsXLlSiQkJCAtLQ3m5uYICAjA7du3dR0eVUPv3r0RHh5eb+tfvHgx/P39YWZmBmtr6zpbz/Tp05GUlFRn9Stz6NAhjB8/vlZrrl+/vk73ExFRQ6DzfVhiY2Mxbtw4hIaGAgASEhKwY8cOrF27FrNnz650mdLSUowcORILFizATz/9pPWuWAiBuLg4zJ07V7qT58aNG2Fvb4/t27dj+PDhemxWNT14++kngbe4BgCUlJRgyJAh8PPzw6efflpn67GwsICFhUWd1a9Ms2bNnuj6iIieFjodYSkpKUFGRgY0Gs39AgYG0Gg0SE1NrXK59957D82bN6/0e1DOnz+P3NxcrZpWVlbw9fWtsmZxcTEKCgq0psaod+/emDJlCsLDw2FjYwN7e3usWbMGRUVFCA0NRZMmTeDq6lrhS/1+++039OvXDxYWFrC3t8eoUaNw5coVAMDo0aOxb98+rFixAgqFAgqFAllZWdKyGRkZ8Pb2hpmZGfz9/XH69Gmt2vHx8WjTpg2MjY3Rvn17bNq0Sas9MzMTPXv2hEqlgru7O3bv3l1huxYsWIB//vOf8PDwqPE+etR4Hz4lNHr0aAQFBSEmJgaOjo6wtbXF5MmTcefOHanPpUuXMGDAAJiamsLFxQWfffaZ1vqEEJg/fz5atmwJExMTODk54e2335baHz4ldOrUKfTo0UPaH3v27Kn0FNkff/yBF154AWZmZujcubP0fz8lJQWhoaHIz8+Xnq/58+fXeL8RETU0OgWWK1euoLS0FPb29lrz7e3tkZubW+ky+/fvx6effoo1a9ZU2l6+nC41o6OjYWVlJU3Ozs66bEaDsmHDBtjZ2SE9PR1TpkzBxIkTMWTIEPj7++PIkSPo27cvRo0ahZs3bwIArl+/jhdffBFdunTB4cOHkZiYiLy8PAwdOhQAsGLFCvj5+WHcuHG4ePEiLl68qLX/3n33XXzwwQc4fPgwDA0N8eabb0pt27Ztw9SpUzFt2jT89ttvmDBhAkJDQ5GcnAwAKCsrw+DBg2FsbIy0tDQkJCRg1qxZem23Wq2u1gvzo8ZbmeTkZJw7dw7JycnYsGED1q9fj/Xr10vto0ePRk5ODpKTk/Hf//4X//rXv7ROd/7vf//Dhx9+iI8//hiZmZnYvn17lcGrtLQUQUFBMDMzQ1paGj755BO8++67VW7H9OnTcfToUbRr1w4jRozA3bt34e/vj7i4OFhaWkrP1/Tp0x+7X4iIGps6vTX/jRs3MGrUKKxZs6ZWvwdlzpw5iIiIkB4XFBQ02tDSuXNnzJ07F8C97V66dCns7Owwbtw4AEBkZCTi4+Nx/PhxPPfcc1i1ahW6dOmCJUuWSDXWrl0LZ2dnnDlzBu3atYOxsTHMzMzg4OBQYX2LFy9Gr169AACzZ89G//79cfv2bahUKsTExGD06NGYNGkSACAiIgIHDx5ETEwMXnjhBezZswenTp3Crl274OTkBABYsmQJ+vXrp/N2t2nTplr/Zx413srY2Nhg1apVUCqVcHNzQ//+/ZGUlIRx48bhzJkz2LlzJ9LT09GtWzcAwKeffooOHTpIy2dnZ8PBwQEajQZGRkZo2bIlfHx8Kl3X7t27ce7cOaSkpEj7evHixXjppZcq9J0+fTr69+8P4N4RqGeffRZnz56Fm5sbrKysoFAoKn2+iIieFjodYbGzs4NSqUReXp7W/Ly8vEr/mJ47dw5ZWVkYMGAADA0NYWhoiI0bN+Kbb76BoaEhzp07Jy1X3ZoAYGJiAktLS62pserUqZP0s1KphK2trdY7+vIjU+VHAY4dO4bk5GTp+g0LCwu4ubkBuPd86LI+R0dHrdonT55E9+7dtfp3794dJ0+elNqdnZ2lsAIAfn5+1d/YByQlJSEsLKxG463Ms88+C6VSqbXMg9tnaGgILy8vqd3NzU3rgtchQ4bg1q1baN26NcaNG4dt27bh7t27la7r9OnTcHZ21vp/XFW40XU7iIieNjoFlvJvjn3wkxdlZWVISkqq9IXJzc0Nv/76K44ePSpNr776Kl544QUcPXoUzs7OcHFxgYODg1bNgoICpKWl6f1i15gYGRlpPVYoFFrzFAoFgHvPAwAUFhZiwIABWvv86NGj0rUluqzv4dpypOt4K9ufumyfs7MzTp8+jX/9618wNTXFpEmT0LNnT63rYPTR0PY7EdGTpvMpoYiICISEhMDb2xs+Pj6Ii4uTLgIFgODgYLRo0QLR0dFQqVTo2LGj1vLl71YfnB8eHo5Fixahbdu2cHFxwbx58+Dk5FThfi30eF27dsX//vc/qNVqGBpW/vQaGxujtLRU59odOnTAgQMHEBISIs07cOAA3N3dpfacnBxcvHhROkpw8OBBPbaifri5ueHu3bvIyMiQTgmdPn26wr1eTE1NMWDAAAwYMACTJ0+WgnnXrl21+rVv3x45OTnIy8uTjoQdOnRI53Hp+3wRETUmOgeWYcOG4fLly4iMjERubi48PT2RmJgo/UHOzs6GgYFut3eZOXMmioqKMH78eFy/fh09evRAYmJildchUNUmT56MNWvWYMSIEZg5cyaaNm2Ks2fPYvPmzfj3v/8NpVIJtVqNtLQ0ZGVlwcLCAk2bNq1W7RkzZmDo0KHo0qULNBoNvv32W2zduhV79uwBAGg0GrRr1w4hISF4//33UVBQUOlFptnZ2bh27Rqys7NRWloq3TTN1dVV+hhynz59MGjQoGqdFqot7du3R2BgICZMmID4+HgYGhoiPDwcpqamUp/169ejtLQUvr6+MDMzw3/+8x+YmpqiVatWFeq99NJLaNOmDUJCQrB8+XLcuHFDuh6p/ChKdajVahQWFiIpKQmdO3eGmZkZzMzMar7BREQNiF53ug0LC8Off/6J4uJipKWlwdfXV2pLSUnR+tTFw9avX1/hI50KhQLvvfcecnNzcfv2bezZswft2rXTZ2hPPScnJxw4cAClpaXo27cvPDw8EB4eDmtraylITp8+HUqlEu7u7mjWrBmys7OrVTsoKAgrVqxATEwMnn32WXz88cdYt24devfuDeDeR9y3bduGW7duwcfHB2PHjsXixYsr1ImMjESXLl0QFRWFwsJCdOnSRfpUU7lz585JH8V+ktatWwcnJyf06tULgwcPxvjx49G8eXOp3draGmvWrEH37t3RqVMn7NmzB99++y1sbW0r1FIqldi+fTsKCwvRrVs3jB07VgpwuoRxf39/vPXWWxg2bBiaNWuG5cuX13xDiYgaGIUQQtT3IGqqoKAAVlZWyM/Pr3AB7u3bt3H+/Hm4uLjwiA3VuwMHDqBHjx44e/Ys2rRpU9/DqTf8vZS5B2+qyRteUh161Ov3w+r0Y81ET7tt27bBwsICbdu2xdmzZzF16lR07979qQ4rRET6YGAhqkM3btzArFmzkJ2dDTs7O2g0GnzwwQf1PSwiogaHgYWoDgUHByM4OLi+h0FE1ODpddEtERER0ZPEwEJERESyx8BCREREssfAQkRERLLHwEJERESyx8BCREREssfAQgDufaWCQqGo8EV/tUGhUFT4OgaqmlqtRlxcXH0Pg4hIVhhYnkK9e/dGeHh4va3/1VdfRcuWLaFSqeDo6IhRo0bhwoUL9Tae+rJ+/Xrp28sfdOjQIYwfP75e1k1EJFdP9Y3jPDZ4PNH1/Rry6xNdn1y98MILeOedd+Do6Ii///4b06dPx+uvv46ff/65vodWK0pKSmBsbKz38s2aNavF0dRMTbeFiKi28AiLjPXu3RtTpkxBeHg4bGxsYG9vjzVr1qCoqAihoaFo0qQJXF1dsXPnTq3lfvvtN/Tr1w8WFhawt7fHqFGjpG8+Hj16NPbt24cVK1ZAoVBAoVAgKytLWjYjIwPe3t4wMzODv78/Tp8+rVU7Pj4ebdq0gbGxMdq3b49NmzZptWdmZqJnz55QqVRwd3fH7t27K2zXP//5Tzz33HNo1aoV/P39MXv2bBw8eBB37tzRaf+sWbMGzs7OMDMzw6BBgxAbG1vhqMHXX3+Nrl27QqVSoXXr1liwYAHu3r0rtSsUCvz73//GoEGDYGZmhrZt2+Kbb76p9v4E7j1PYWFhCA8Ph52dHQICAgAAsbGx8PDwgLm5OZydnTFp0iQUFhYCuHcKLjQ0FPn5+dLzMH/+fAAVTwllZ2dj4MCBsLCwgKWlJYYOHYq8vDypff78+fD09MSmTZugVqthZWWF4cOH48aNG5Xut8ete+HChQgODoalpaV0pGf//v14/vnnYWpqCmdnZ7z99tsoKiqSahYXF2P69Olo0aIFzM3N4evri5SUlMc/iURE1cTAInMbNmyAnZ0d0tPTMWXKFEycOBFDhgyBv78/jhw5gr59+2LUqFG4efMmAOD69et48cUX0aVLFxw+fBiJiYnIy8vD0KFDAQArVqyAn58fxo0bh4sXL+LixYtwdnaW1vfuu+/igw8+wOHDh2FoaIg333xTatu2bRumTp2KadOm4bfffsOECRMQGhqK5ORkAEBZWRkGDx4MY2NjpKWlISEhAbNmzXrk9l27dg2fffYZ/P39YWRkJM1XKBRYv359lcsdOHAAb731FqZOnYqjR4/ipZdewuLFi7X6/PTTTwgODsbUqVNx4sQJfPzxx1i/fn2FfgsWLMDQoUNx/PhxvPzyyxg5ciSuXbtWrf354PNkbGyMAwcOICEhAQBgYGCAlStX4vfff8eGDRuwd+9ezJw5EwDg7++PuLg4WFpaSs/D9OnTK2xnWVkZBg4ciGvXrmHfvn3YvXs3/vjjDwwbNkyr37lz57B9+3Z89913+O6777Bv3z4sXbq00n33uHXHxMSgc+fO+OWXXzBv3jycO3cOgYGBeO2113D8+HFs2bIF+/fvR1hYmLRMWFgYUlNTsXnzZhw/fhxDhgxBYGAgMjMzq3wOiYh0IhqB/Px8AUDk5+dXaLt165Y4ceKEuHXrVoW2jus7PtFJV7169RI9evSQHt+9e1eYm5uLUaNGSfMuXrwoAIjU1FQhhBALFy4Uffv21aqTk5MjAIjTp09LdadOnarVJzk5WQAQe/bskebt2LFDAJD2nb+/vxg3bpzWckOGDBEvv/yyEEKIXbt2CUNDQ/H3339L7Tt37hQAxLZt27SWmzlzpjAzMxMAxHPPPSeuXLmi1d6+fXuxdevWKvfNsGHDRP/+/bXmjRw5UlhZWUmP+/TpI5YsWaLVZ9OmTcLR0VF6DEDMnTtXelxYWCgAiJ07dwohqr8/u3TpUuVYy3311VfC1tZWerxu3Tqt8ZZr1aqV+PDDD4UQQvzwww9CqVSK7Oxsqf33338XAER6eroQQoioqChhZmYmCgoKpD4zZswQvr6+VY7lUesOCgrSmjdmzBgxfvx4rXk//fSTMDAwELdu3RJ//vmnUCqVWs+7EPf2/5w5cypd/6N+L0kGoizvT0R16FGv3w/jERaZ69Spk/SzUqmEra0tPDzuX3tjb28PALh06RIA4NixY0hOToaFhYU0ubm5Abj3LlyX9Tk6OmrVPnnyJLp3767Vv3v37jh58qTU7uzsDCcnJ6ndz8+v0vXMmDEDv/zyC3744QcolUoEBwdDCCG1nzp1CoMGDapynKdPn4aPj4/WvIcfHzt2DO+9957Wvig/slR+ROrhbTY3N4elpaXO+9PLy6vCGPfs2YM+ffqgRYsWaNKkCUaNGoWrV69qrftxyvfpg0fB3N3dYW1tLe134N6pnCZNmkiPHR0dpW3Qlbe3t9bjY8eOYf369Vr7ICAgAGVlZTh//jx+/fVXlJaWol27dlp99u3bV63/c0RE1fFUX3TbEDx4mgS4d6rk4VMnwL1TBwBQWFiIAQMGYNmyZRVqlQeQ6q7v4dq1yc7ODnZ2dmjXrh06dOgAZ2dnHDx4sMqAo4/CwkIsWLAAgwcPrtCmUqmknyvbx7ruT3Nzc622rKwsvPLKK5g4cSIWL16Mpk2bYv/+/RgzZgxKSkpgZmZWo2172KO2QVcPb0thYSEmTJiAt99+u0Lfli1b4vjx41AqlcjIyIBSqdRqt7Cw0GsMREQPY2BpZLp27Yr//e9/UKvVMDSs/Ok1NjZGaWmpzrU7dOiAAwcOICQkRJp34MABuLu7S+05OTm4ePGi9GJ+8ODBx9Ytf2EtLi6u9ljat2+PQ4cOac17+HHXrl1x+vRpuLq6Vrvuw6qzPyuTkZGBsrIyfPDBBzAwuHcg88svv9TqU53noXyf5uTkSEdZTpw4gevXr0v7XR+6/B/o2rUrTpw4UeV+7NKlC0pLS3Hp0iU8//zzeo+JiOhReEqokZk8eTKuXbuGESNG4NChQzh37hx27dqF0NBQ6QVKrVYjLS0NWVlZuHLlSrXfic+YMQPr169HfHw8MjMzERsbi61bt0oXbGo0GrRr1w4hISE4duwYfvrpJ7z77rtaNdLS0rBq1SocPXoUf/75J/bu3YsRI0agTZs2WkdX3NzcsG3btirHMmXKFHz//feIjY1FZmYmPv74Y+zcuVM6KgQAkZGR2LhxIxYsWIDff/8dJ0+exObNmzF37txa3Z+VcXV1xZ07d/DRRx/hjz/+wKZNm6SLccup1WoUFhYiKSkJV65cqfRUkUajgYeHB0aOHIkjR44gPT0dwcHB6NWrV4VTN7qozrrLzZo1Cz///DPCwsJw9OhRZGZm4uuvv5Yuum3Xrh1GjhyJ4OBgbN26FefPn0d6ejqio6OxY8cOvcdIRPQgBpZGxsnJCQcOHEBpaSn69u0LDw8PhIeHw9raWnqnP336dCiVSri7u6NZs2bIzs6uVu2goCCsWLECMTExePbZZ/Hxxx9j3bp16N27N4B7n4rZtm0bbt26BR8fH4wdO7bCJ3LMzMywdetW9OnTB+3bt8eYMWPQqVMn7Nu3DyYmJlK/06dPIz8/v8qxdO/eHQkJCYiNjUXnzp2RmJiIf/7zn1qnegICAvDdd9/hhx9+QLdu3fDcc8/hww8/RKtWraq7O6u1PyvTuXNnxMbGYtmyZejYsSM+++wzREdHa/Xx9/fHW2+9hWHDhqFZs2ZYvnx5hToKhQJff/01bGxs0LNnT2g0GrRu3Rpbtmyp9jZUpjrrLlf+/Jw5cwbPP/88unTpgsjISK1rldatW4fg4GBMmzYN7du3R1BQEA4dOoSWLVvWaJz05Khn75AmIjlSiAevdGygCgoKYGVlhfz8fFhaWmq13b59G+fPn4eLi4vWixk1PuPGjcOpU6fw008/1fdQ6DH4eyk/DwaVLNUb9xvmV/3GgaimHvX6/TBew0INVkxMDF566SWYm5tj586d2LBhA/71r3/V97CIiKgOMLBQg5Weno7ly5fjxo0baN26NVauXImxY8fW97CIiKgOMLBQg/Xwp26IiKjx4kW3REREJHsMLERERCR7DCxEREQkewwsREREJHsMLERERCR7egWW1atXQ61WQ6VSwdfXF+np6VX23bp1K7y9vWFtbQ1zc3N4enpi06ZNWn1Gjx4NhUKhNQUGBuozNCIiImqEdA4sW7ZsQUREBKKionDkyBF07twZAQEBVX6VfdOmTfHuu+8iNTUVx48fR2hoKEJDQ7Fr1y6tfoGBgbh48aI0ffHFF/ptUSPSu3dvhIeHV9muVqsRFxdXq+usi5pEREQ1pfN9WGJjYzFu3DiEhoYCABISErBjxw6sXbsWs2fPrtC//Htmyk2dOhUbNmzA/v37ERAQIM03MTGBg4ODrsNp1LZu3QojI6P6HgYREVG90ymwlJSUICMjA3PmzJHmGRgYQKPRIDU19bHLCyGwd+9enD59GsuWLdNqS0lJQfPmzWFjY4MXX3wRixYtgq2trS7D09lJtw51Wv9hHU6d1Kl/06ZN62gkREREDYtOp4SuXLmC0tJS2Nvba823t7dHbm5ulcvl5+fDwsICxsbG6N+/Pz766CO89NJLUntgYCA2btyIpKQkLFu2DPv27UO/fv1QWlpaab3i4mIUFBRoTY3Rg6eELl26hAEDBsDU1BQuLi747LPPKvS/fv06xo4di2bNmsHS0hIvvvgijh07JrWfO3cOAwcOhL29PSwsLNCtWzfs2bPnSW0OERGR3p7IrfmbNGmCo0ePorCwEElJSYiIiEDr1q2l00XDhw+X+np4eKBTp05o06YNUlJS0KdPnwr1oqOjsWDBgicxdNkYPXo0Lly4gOTkZBgZGeHtt9+ucN3QkCFDYGpqip07d8LKygoff/wx+vTpgzNnzqBp06YoLCzEyy+/jMWLF8PExAQbN27EgAEDcPr0abRs2bKetoyIiOjxdDrCYmdnB6VSiby8PK35eXl5j7z+xMDAAK6urvD09MS0adPw+uuvIzo6usr+rVu3hp2dHc6ePVtp+5w5c5Cfny9NOTk5umxGg3PmzBns3LkTa9aswXPPPQcvLy98+umnuHXrltRn//79SE9Px1dffQVvb2+0bdsWMTExsLa2xn//+18AQOfOnTFhwgR07NgRbdu2xcKFC9GmTRt888039bVpRERE1aJTYDE2NoaXlxeSkpKkeWVlZUhKSoKfn1+165SVlaG4uLjK9r/++gtXr16Fo6Njpe0mJiawtLTUmhqzkydPwtDQEF5eXtI8Nzc3WFtbS4+PHTuGwsJC2NrawsLCQprOnz+Pc+fOAQAKCwsxffp0dOjQAdbW1rCwsMDJkyeRnZ39pDeJiIhIJzqfEoqIiEBISAi8vb3h4+ODuLg4FBUVSZ8aCg4ORosWLaQjKNHR0fD29kabNm1QXFyM77//Hps2bUJ8fDyAey+iCxYswGuvvQYHBwecO3cOM2fOhKurq9aniOjRCgsL4ejoiJSUlApt5cFm+vTp2L17N2JiYuDq6gpTU1O8/vrrKCkpebKDJSIi0pHOgWXYsGG4fPkyIiMjkZubC09PTyQmJkoX4mZnZ8PA4P6Bm6KiIkyaNAl//fUXTE1N4ebmhv/85z8YNmwYAECpVOL48ePYsGEDrl+/DicnJ/Tt2xcLFy6EiYlJLW1mw+bm5oa7d+8iIyMD3bp1AwCcPn0a169fl/p07doVubm5MDQ0hFqtrrTOgQMHMHr0aAwaNAjAvZCTlZVVx6MnIiKqOb0uug0LC0NYWFilbQ+/w1+0aBEWLVpUZS1TU9MKN5Ejbe3bt0dgYCAmTJiA+Ph4GBoaIjw8HKamplIfjUYDPz8/BAUFYfny5WjXrh0uXLiAHTt2YNCgQdJ1LVu3bsWAAQOgUCgwb948lJWV1eOWERERVQ+/S6iBWLduHZycnNCrVy8MHjwY48ePR/PmzaV2hUKB77//Hj179kRoaCjatWuH4cOH488//5SOfsXGxsLGxgb+/v4YMGAAAgIC0LVr1/raJCIiompTCCFEfQ+ipgoKCmBlZYX8/PwKF+Devn0b58+fh4uLC1QqVT2NkIgexN9L+VHP3iH9nKV6437D/Px6GA09LR71+v0wHmEhIiIi2WNgISIiItljYCEiIiLZY2AhIiIi2WNgISIiItl7agJLI/gwFFGjwd9HItJVow8sRkZGAICbN2/W80iIqFz572P57ycR0ePodafbhkSpVMLa2hqXLl0CAJiZmUGhUNTzqIieTkII3Lx5E5cuXYK1tTWUSmV9D4mIGohGH1gAwMHBAQCk0EJE9cva2lr6vSQiqo6nIrAoFAo4OjqiefPmuHPnTn0Ph+ipZmRkxCMrRKSzpyKwlFMqlfxDSURE1AA1+otuiYiIqOFjYCEiIiLZY2AhIiIi2WNgISIiItljYCEiIiLZY2AhIiIi2WNgISIiItljYCEiIiLZY2AhIiIi2WNgISIiItljYCEiIiLZY2AhIiIi2WNgISIiItljYCEiIiLZY2AhIiIi2WNgISIiItljYCEiIiLZY2AhIiIi2dMrsKxevRpqtRoqlQq+vr5IT0+vsu/WrVvh7e0Na2trmJubw9PTE5s2bdLqI4RAZGQkHB0dYWpqCo1Gg8zMTH2GRkRERI2QzoFly5YtiIiIQFRUFI4cOYLOnTsjICAAly5dqrR/06ZN8e677yI1NRXHjx9HaGgoQkNDsWvXLqnP8uXLsXLlSiQkJCAtLQ3m5uYICAjA7du39d8yIiIiajQUQgihywK+vr7o1q0bVq1aBQAoKyuDs7MzpkyZgtmzZ1erRteuXdG/f38sXLgQQgg4OTlh2rRpmD59OgAgPz8f9vb2WL9+PYYPH/7YegUFBbCyskJ+fj4sLS112RwiIgKgnr1D+jlL9cb9hvn59TAaelro8vqt0xGWkpISZGRkQKPR3C9gYACNRoPU1NTHLi+EQFJSEk6fPo2ePXsCAM6fP4/c3FytmlZWVvD19a2yZnFxMQoKCrQmIiIiarx0CixXrlxBaWkp7O3ttebb29sjNze3yuXy8/NhYWEBY2Nj9O/fHx999BFeeuklAJCW06VmdHQ0rKyspMnZ2VmXzSAiIqIG5ol8SqhJkyY4evQoDh06hMWLFyMiIgIpKSl615szZw7y8/OlKScnp/YGS0RERLJjqEtnOzs7KJVK5OXlac3Py8uDg4NDlcsZGBjA1dUVAODp6YmTJ08iOjoavXv3lpbLy8uDo6OjVk1PT89K65mYmMDExESXoRMREVEDptMRFmNjY3h5eSEpKUmaV1ZWhqSkJPj5+VW7TllZGYqLiwEALi4ucHBw0KpZUFCAtLQ0nWoSERFR46XTERYAiIiIQEhICLy9veHj44O4uDgUFRUhNDQUABAcHIwWLVogOjoawL3rTby9vdGmTRsUFxfj+++/x6ZNmxAfHw8AUCgUCA8Px6JFi9C2bVu4uLhg3rx5cHJyQlBQUO1tKRERETVYOgeWYcOG4fLly4iMjERubi48PT2RmJgoXTSbnZ0NA4P7B26KioowadIk/PXXXzA1NYWbmxv+85//YNiwYVKfmTNnoqioCOPHj8f169fRo0cPJCYmQqVS1cImEhERUUOn831Y5Ij3YSEiqhneh4XqQ53dh4WIiIioPjCwEBERkewxsBAREZHsMbAQERGR7DGwEBERkewxsBAREZHsMbAQERGR7DGwEBERkewxsBAREZHsMbAQERGR7DGwEBERkewxsBAREZHsMbAQERGR7DGwEBERkewxsBAREZHsMbAQERGR7DGwEBERkewxsBAREZHsMbAQERGR7DGwEBERkewxsBAREZHsMbAQERGR7DGwEBERkewxsBAREZHsMbAQERGR7DGwEBERkewxsBAREZHsMbAQERGR7DGwEBERkewxsBAREZHs6RVYVq9eDbVaDZVKBV9fX6Snp1fZd82aNXj++edhY2MDGxsbaDSaCv1Hjx4NhUKhNQUGBuozNCIiImqEdA4sW7ZsQUREBKKionDkyBF07twZAQEBuHTpUqX9U1JSMGLECCQnJyM1NRXOzs7o27cv/v77b61+gYGBuHjxojR98cUX+m0RERERNTo6B5bY2FiMGzcOoaGhcHd3R0JCAszMzLB27dpK+3/22WeYNGkSPD094ebmhn//+98oKytDUlKSVj8TExM4ODhIk42NjX5bRERERI2OToGlpKQEGRkZ0Gg09wsYGECj0SA1NbVaNW7evIk7d+6gadOmWvNTUlLQvHlztG/fHhMnTsTVq1d1GRoRERE1Yoa6dL5y5QpKS0thb2+vNd/e3h6nTp2qVo1Zs2bByclJK/QEBgZi8ODBcHFxwblz5/DOO++gX79+SE1NhVKprFCjuLgYxcXF0uOCggJdNoOIiIgaGJ0CS00tXboUmzdvRkpKClQqlTR/+PDh0s8eHh7o1KkT2rRpg5SUFPTp06dCnejoaCxYsOCJjJmIiIjqn06nhOzs7KBUKpGXl6c1Py8vDw4ODo9cNiYmBkuXLsUPP/yATp06PbJv69atYWdnh7Nnz1baPmfOHOTn50tTTk6OLptBREREDYxOgcXY2BheXl5aF8yWX0Dr5+dX5XLLly/HwoULkZiYCG9v78eu56+//sLVq1fh6OhYabuJiQksLS21JiIiImq8dP6UUEREBNasWYMNGzbg5MmTmDhxIoqKihAaGgoACA4Oxpw5c6T+y5Ytw7x587B27Vqo1Wrk5uYiNzcXhYWFAIDCwkLMmDEDBw8eRFZWFpKSkjBw4EC4uroiICCgljaTiIiIGjKdr2EZNmwYLl++jMjISOTm5sLT0xOJiYnShbjZ2dkwMLifg+Lj41FSUoLXX39dq05UVBTmz58PpVKJ48ePY8OGDbh+/TqcnJzQt29fLFy4ECYmJjXcPCIiImoMFEIIUd+DqKmCggJYWVkhPz+fp4eIiPSgnr1D+jlL9cb9hvn59TAaelro8vrN7xIiIiIi2WNgISIiItljYCEiIiLZY2AhIiIi2WNgISIiItljYCEiIiLZY2AhIiIi2WNgISIiItljYCEiIiLZY2AhIiIi2WNgISIiItljYCEiIiLZY2AhIiIi2WNgISIiItljYCEiIiLZY2AhIiIi2WNgISIiItljYCEiIiLZY2AhIiIi2WNgISIiItljYCEiIiLZY2AhIiIi2WNgISIiItljYCEiIiLZY2AhIiIi2WNgISIiItljYCEiIiLZY2AhIiIi2WNgISIiItljYCEiIiLZY2AhIiIi2dMrsKxevRpqtRoqlQq+vr5IT0+vsu+aNWvw/PPPw8bGBjY2NtBoNBX6CyEQGRkJR0dHmJqaQqPRIDMzU5+hERERUSOkc2DZsmULIiIiEBUVhSNHjqBz584ICAjApUuXKu2fkpKCESNGIDk5GampqXB2dkbfvn3x999/S32WL1+OlStXIiEhAWlpaTA3N0dAQABu376t/5YRERFRo6EQQghdFvD19UW3bt2watUqAEBZWRmcnZ0xZcoUzJ49+7HLl5aWwsbGBqtWrUJwcDCEEHBycsK0adMwffp0AEB+fj7s7e2xfv16DB8+/LE1CwoKYGVlhfz8fFhaWuqyOUREBEA9e4f0c5bqjfsN8/PrYTT0tNDl9VunIywlJSXIyMiARqO5X8DAABqNBqmpqdWqcfPmTdy5cwdNmzYFAJw/fx65ublaNa2srODr61tlzeLiYhQUFGhNRERE1HjpFFiuXLmC0tJS2Nvba823t7dHbm5utWrMmjULTk5OUkApX06XmtHR0bCyspImZ2dnXTaDiIiIGpgn+imhpUuXYvPmzdi2bRtUKpXedebMmYP8/HxpysnJqcVREhERkdwY6tLZzs4OSqUSeXl5WvPz8vLg4ODwyGVjYmKwdOlS7NmzB506dZLmly+Xl5cHR0dHrZqenp6V1jIxMYGJiYkuQyciIqIGTKcjLMbGxvDy8kJSUpI0r6ysDElJSfDz86tyueXLl2PhwoVITEyEt7e3VpuLiwscHBy0ahYUFCAtLe2RNYmIiOjpodMRFgCIiIhASEgIvL294ePjg7i4OBQVFSE0NBQAEBwcjBYtWiA6OhoAsGzZMkRGRuLzzz+HWq2WrkuxsLCAhYUFFAoFwsPDsWjRIrRt2xYuLi6YN28enJycEBQUVHtbSkRERA2WzoFl2LBhuHz5MiIjI5GbmwtPT08kJiZKF81mZ2fDwOD+gZv4+HiUlJTg9ddf16oTFRWF+fPnAwBmzpyJoqIijB8/HtevX0ePHj2QmJhYo+tciIiIqPHQ+T4scsT7sBAR1Qzvw0L1oc7uw0JERERUHxhYiIiISPYYWIiIiEj2GFiIiIhI9hhYiIiISPYYWIiIiEj2GFiIiIhI9hhYiIiISPYYWIiIiEj2GFiIiIhI9hhYiIiISPYYWIiIiEj2GFiIiIhI9hhYiIiISPYYWIiIiEj2GFiIiIhI9hhYiIiISPYYWIiIiEj2GFiIiIhI9hhYiIiISPYYWIiIiEj2GFiIiIhI9hhYiIiISPYYWIiIiEj2GFiIiIhI9hhYiIiISPYYWIiIiEj2GFiIiIhI9hhYiIiISPYYWIiIiEj2GFiIiIhI9vQKLKtXr4ZarYZKpYKvry/S09Or7Pv777/jtddeg1qthkKhQFxcXIU+8+fPh0Kh0Jrc3Nz0GRoRERE1QjoHli1btiAiIgJRUVE4cuQIOnfujICAAFy6dKnS/jdv3kTr1q2xdOlSODg4VFn32WefxcWLF6Vp//79ug6NiIiIGimdA0tsbCzGjRuH0NBQuLu7IyEhAWZmZli7dm2l/bt164b3338fw4cPh4mJSZV1DQ0N4eDgIE12dna6Do2IiIgaKZ0CS0lJCTIyMqDRaO4XMDCARqNBampqjQaSmZkJJycntG7dGiNHjkR2dnaVfYuLi1FQUKA1ERERUeOlU2C5cuUKSktLYW9vrzXf3t4eubm5eg/C19cX69evR2JiIuLj43H+/Hk8//zzuHHjRqX9o6OjYWVlJU3Ozs56r5uIiIjkTxafEurXrx+GDBmCTp06ISAgAN9//z2uX7+OL7/8stL+c+bMQX5+vjTl5OQ84RETERHRk2SoS2c7OzsolUrk5eVpzc/Ly3vkBbW6sra2Rrt27XD27NlK201MTB55PQwRERE1LjodYTE2NoaXlxeSkpKkeWVlZUhKSoKfn1+tDaqwsBDnzp2Do6NjrdUkIiKihkunIywAEBERgZCQEHh7e8PHxwdxcXEoKipCaGgoACA4OBgtWrRAdHQ0gHsX6p44cUL6+e+//8bRo0dhYWEBV1dXAMD06dMxYMAAtGrVChcuXEBUVBSUSiVGjBhRW9tJREREDZjOgWXYsGG4fPkyIiMjkZubC09PTyQmJkoX4mZnZ8PA4P6BmwsXLqBLly7S45iYGMTExKBXr15ISUkBAPz1118YMWIErl69imbNmqFHjx44ePAgmjVrVsPNIyIiosZAIYQQ9T2ImiooKICVlRXy8/NhaWlZ38MhImpw1LN3SD9nqd643zA/vx5GQ08LXV6/ZfEpISIiIqJHYWAhIiIi2WNgISIiItljYCEiIiLZY2AhIiIi2WNgISIiItljYCEiIiLZY2AhIiIi2WNgISIiItljYCEiIiLZY2AhIiIi2WNgISIiItljYCEiIiLZM6zvARARNRZa33i8tH89joSo8eERFiIiIpI9BhYiIiKSPQYWIiIikj0GFiIiIpI9XnRLRFQX5ls98HN+/Y2DqJHgERYiIiKSPQYWIiIikj0GFiIiIpI9BhYiIiKSPQYWIiIikj0GFiIiIpI9BhYiIiKSPQYWIiIikj0GFiIiIpI9BhYiIiKSPQYWIiIikj29Asvq1auhVquhUqng6+uL9PT0Kvv+/vvveO2116BWq6FQKBAXF1fjmkRERPR00TmwbNmyBREREYiKisKRI0fQuXNnBAQE4NKlS5X2v3nzJlq3bo2lS5fCwcGhVmoSERHR00XnwBIbG4tx48YhNDQU7u7uSEhIgJmZGdauXVtp/27duuH999/H8OHDYWJiUis1iYiI6OmiU2ApKSlBRkYGNBrN/QIGBtBoNEhNTdVrAPrULC4uRkFBgdZEREREjZehLp2vXLmC0tJS2Nvba823t7fHqVOn9BqAPjWjo6OxYMECvdZHjYN69g7p56yl/etxJPpp6OMnInrSGuSnhObMmYP8/HxpysnJqe8hkUx4bPCQJiIiajx0OsJiZ2cHpVKJvLw8rfl5eXlVXlBbFzVNTEyqvB6GiIiIGh+djrAYGxvDy8sLSUlJ0ryysjIkJSXBz89PrwHURU0iIiJqXHQ6wgIAERERCAkJgbe3N3x8fBAXF4eioiKEhoYCAIKDg9GiRQtER0cDuHdR7YkTJ6Sf//77bxw9ehQWFhZwdXWtVk0iIiJ6uukcWIYNG4bLly8jMjISubm58PT0RGJionTRbHZ2NgwM7h+4uXDhArp06SI9jomJQUxMDHr16oWUlJRq1SQiIqKnm86BBQDCwsIQFhZWaVt5CCmnVqshhKhRTSIiInq6NchPCREREdHThYGFiIiIZI+BhYiIiGSPgYWIiIhkj4GFiIiIZE+vTwkRUd148CsFfg35tR5HQkQkLzzCQkRERLLHwEJERESyx8BCREREssfAQkRERLLHi26JGhn17B3Sz1lL+9fjSIiIag8DCz1x/CQMERHpiqeEiIiISPYYWIiIiEj2GFiIiIhI9ngNCxHVCl6bRER1iYGF6gw/rUJERLWFp4SIiIhI9hhYGiiPDR5ah+CJiIgaMwYWIiIikj1ew0L0lOBFsUTUkDGw1BG+OBAREdUeBhaqgGGratw3RET1g4GFqL7Nt7r/s0vL+hsHEZGM8aJbIiIikj0GFiIiIpI9BhYiIiKSPV7DQtSY8foYImokeISFiIiIZI+BhYiIiGSPgYWIiIhkT6/Asnr1aqjVaqhUKvj6+iI9Pf2R/b/66iu4ublBpVLBw8MD33//vVb76NGjoVAotKbAwEB9hkZERESNkM6BZcuWLYiIiEBUVBSOHDmCzp07IyAgAJcuXaq0/88//4wRI0ZgzJgx+OWXXxAUFISgoCD89ttvWv0CAwNx8eJFafriiy/02yIiIiJqdHQOLLGxsRg3bhxCQ0Ph7u6OhIQEmJmZYe3atZX2X7FiBQIDAzFjxgx06NABCxcuRNeuXbFq1SqtfiYmJnBwcJAmGxsb/baIiIiIGh2dAktJSQkyMjKg0WjuFzAwgEajQWpqaqXLpKamavUHgICAgAr9U1JS0Lx5c7Rv3x4TJ07E1atXqxxHcXExCgoKtCYiIiJqvHQKLFeuXEFpaSns7e215tvb2yM3N7fSZXJzcx/bPzAwEBs3bkRSUhKWLVuGffv2oV+/figtLa20ZnR0NKysrKTJ2dlZl80gIiKiBkYWN44bPny49LOHhwc6deqENm3aICUlBX369KnQf86cOYiIiJAeFxQUMLQQERE1YjoFFjs7OyiVSuTl5WnNz8vLg4ODQ6XLODg46NQfAFq3bg07OzucPXu20sBiYmICExMTXYb+SCfdOkg/dzh1stbqPon6DXnsdV2/IY+9rutz7I2zfkMe+4P1G/LY66J+Qx57bdbX6ZSQsbExvLy8kJSUJM0rKytDUlIS/Pz8Kl3Gz89Pqz8A7N69u8r+APDXX3/h6tWrcHR01GV4RNQIqGfvkKaGVJuI6pbOp4QiIiIQEhICb29v+Pj4IC4uDkVFRQgNDQUABAcHo0WLFoiOjgYATJ06Fb169cIHH3yA/v37Y/PmzTh8+DA++eQTAEBhYSEWLFiA1157DQ4ODjh37hxmzpwJV1dXBAQE1OKmEhE94MHvWZqfX3/jIKJq0TmwDBs2DJcvX0ZkZCRyc3Ph6emJxMRE6cLa7OxsGBjcP3Dj7++Pzz//HHPnzsU777yDtm3bYvv27ejYsSMAQKlU4vjx49iwYQOuX78OJycn9O3bFwsXLqzV0z7UiPEL/oiIGj29LroNCwtDWFhYpW0pKSkV5g0ZMgRDhgyptL+pqSl27dqlzzCIiKgh4tEt0gO/S4iIiIhkj4GFiIiIZI+BhYiIiGSPgYWI5Gu+lfb1DkT01JLFnW7rg8cGD+nnL+txHERERPR4PMJCREREsvfUHmEhImpIHrw7b9bS/vU4koajTo6k8yPZ9YaBhYhkj6dwiYiBhYiooeG7fHoKMbAQEdUxHiGqGvcNVRcvuiUiIiLZ4xEWejL4BYVEdYJHKOhpwcBCRERVYiAifdTF/xsGFiIiqnNaH8tW1eNAqMHiNSxEREQkezzCQkREpAeeLnuyGFiIiIhkhmGoIp4SIiIiItljYCEiIiLZ4ykhInrq8fA7kfwxsBAREckBb7D5SDwlRERERLLHwEJERESyx1NCRI/Dw7RERPWOgYWI9McwR0RPCAMLERHRU6YhfjKOgYWIiBoPHvVrtBhY6B7+khMRkYzxU0JEREQkewwsREREJHt6BZbVq1dDrVZDpVLB19cX6enpj+z/1Vdfwc3NDSqVCh4eHvj++++12oUQiIyMhKOjI0xNTaHRaJCZmanP0IiIiKgR0jmwbNmyBREREYiKisKRI0fQuXNnBAQE4NKlS5X2//nnnzFixAiMGTMGv/zyC4KCghAUFITffvtN6rN8+XKsXLkSCQkJSEtLg7m5OQICAnD79m39t4yIiIgaDZ0DS2xsLMaNG4fQ0FC4u7sjISEBZmZmWLt2baX9V6xYgcDAQMyYMQMdOnTAwoUL0bVrV6xatQrAvaMrcXFxmDt3LgYOHIhOnTph48aNuHDhArZv316jjXvi5lvdn4iIiKjW6BRYSkpKkJGRAY1Gc7+AgQE0Gg1SU1MrXSY1NVWrPwAEBARI/c+fP4/c3FytPlZWVvD19a2yJhERET1ddPpY85UrV1BaWgp7e3ut+fb29jh16lSly+Tm5lbaPzc3V2ovn1dVn4cVFxejuLhYepyfnw8AKCgoePQGRD8j/Vja6v7PhaWl0s+PrfEoxeJ+/Vv3a9Za/UrGL+exlxXfvL+sonbr12XtCrhvqvaE9k1D2O91Xb8hj13X+nL7P/nE6tfDa0it1n9Adcdf/lgIgccSOvj7778FAPHzzz9rzZ8xY4bw8fGpdBkjIyPx+eefa81bvXq1aN68uRBCiAMHDggA4sKFC1p9hgwZIoYOHVppzaioKAGAEydOnDhx4tQIppycnMdmEJ2OsNjZ2UGpVCIvL09rfl5eHhwcHCpdxsHB4ZH9y//Ny8uDo6OjVh9PT89Ka86ZMwcRERHS47KyMly7dg22trZQKBSP3Y6CggI4OzsjJycHlpaWj+2vq4Zcn2NvnPU59sZZvyGPva7rc+wNo74QAjdu3ICTk9Nj6+oUWIyNjeHl5YWkpCQEBQUBuBcWkpKSEBYWVukyfn5+SEpKQnh4uDRv9+7d8PPzAwC4uLjAwcEBSUlJUkApKChAWloaJk6cWGlNExMTmJiYaM2ztrbWZVMAAJaWlnXyZDWG+hx746zPsTfO+g157HVdn2OXf30rK6tq1dP51vwREREICQmBt7c3fHx8EBcXh6KiIoSGhgIAgoOD0aJFC0RHRwMApk6dil69euGDDz5A//79sXnzZhw+fBiffPIJAEChUCA8PByLFi1C27Zt4eLignnz5sHJyUkKRURERPR00zmwDBs2DJcvX0ZkZCRyc3Ph6emJxMRE6aLZ7OxsGBjc//CRv78/Pv/8c8ydOxfvvPMO2rZti+3bt6Njx45Sn5kzZ6KoqAjjx4/H9evX0aNHDyQmJkKlUtXCJhIREVFDp9eXH4aFhVV5CiglJaXCvCFDhmDIkCFV1lMoFHjvvffw3nvv6TMcnZmYmCAqKqrCaSXW59gba32OvXHWb8hjr+v6HHvjq68QojqfJSIiIiKqP/zyQyIiIpI9BhYiIiKSPQYWIiIikj0GFnqieMkUERHpQ69PCTU0V65cwdq1a5Gamip9P5GDgwP8/f0xevRoNGvWrJ5H+PQwMTHBsWPH0KFDh/oeCtXAxYsXER8fj/379+PixYswMDBA69atERQUhNGjR0OpVNb3EImokWn0nxI6dOgQAgICYGZmBo1GI90vJi8vD0lJSbh58yZ27doFb2/vOll/Tk4OoqKisHbtWr1r3Lp1CxkZGWjatCnc3d212m7fvo0vv/wSwcHBetU+efIkDh48CD8/P7i5ueHUqVNYsWIFiouL8Y9//AMvvviiXnUf/OqEB61YsQL/+Mc/YGtrCwCIjY3Vq/7DioqK8OWXX+Ls2bNwdHTEiBEjpHXo48iRI7CxsYGLiwsAYNOmTUhISEB2djZatWqFsLAwDB8+XO/6U6ZMwdChQ/H888/rXeNRVq1ahfT0dLz88ssYPnw4Nm3ahOjoaJSVlWHw4MF47733YGio3/uVw4cPQ6PRwNXVFaampkhNTcUbb7yBkpIS7Nq1C+7u7khMTESTJk1qeauISK7S09MrHBTw8/ODj49P7a3ksd821MD5+vqK8ePHi7KysgptZWVlYvz48eK5556rs/UfPXpUGBgY6L386dOnRatWrYRCoRAGBgaiZ8+eWl8UmZubq3f9nTt3CmNjY9G0aVOhUqnEzp07RbNmzYRGoxEvvviiUCqVIikpSa/aCoVCeHp6it69e2tNCoVCdOvWTfTu3Vu88MILetUWQogOHTqIq1evCiGEyM7OFmq1WlhZWYlu3bqJpk2biubNm4s//vhD7/qdOnUSu3fvFkIIsWbNGmFqairefvttER8fL8LDw4WFhYX49NNP9a5f/ny2bdtWLF26VFy8eFHvWg9buHChaNKkiXjttdeEg4ODWLp0qbC1tRWLFi0SS5YsEc2aNRORkZF61+/evbuYP3++9HjTpk3C19dXCCHEtWvXhKenp3j77bdrtA3FxcViy5YtIjw8XAwfPlwMHz5chIeHiy+//FIUFxfXqPbj5ObmigULFtSoRk5Ojrhx40aF+SUlJWLfvn01qn3lyhWxd+9e6f//5cuXxdKlS8WCBQvEiRMnalS7Ki4uLuLMmTO1WrOsrEzs3btXfPLJJ+Lbb78VJSUlNaqXk5MjLl++LD3+8ccfxRtvvCF69OghRo4cWeFLe3URExMjsrKyajS+x/n222/FvHnzxP79+4UQQiQlJYl+/fqJgIAA8fHHH9e4/s2bN8Wnn34qQkNDRWBgoHj55ZdFWFiY2LNnT43q5uXliR49egiFQiFatWolfHx8hI+Pj/S61aNHD5GXl1fj8QshRKMPLCqVSpw8ebLK9pMnTwqVSqV3/a+//vqR04cfflijwBIUFCT69+8vLl++LDIzM0X//v2Fi4uL+PPPP4UQNQssfn5+4t133xVCCPHFF18IGxsb8c4770jts2fPFi+99JJetaOjo4WLi0uFwGNoaCh+//13vWo+SKFQSL8EI0eOFP7+/uL69etCCCFu3LghNBqNGDFihN71TU1NpT9QXbp0EZ988olW+2effSbc3d31rq9QKMSePXvE1KlThZ2dnTAyMhKvvvqq+Pbbb0VpaanedYUQok2bNuJ///ufEOJeYFYqleI///mP1L5161bh6uqqd31TU1Nx7tw56XFpaakwMjISubm5QgghfvjhB+Hk5KR3/czMTNG6dWuhUqlEr169xNChQ8XQoUNFr169hEqlEq6uriIzM1Pv+o9TkzcZFy5cEN26dRMGBgZCqVSKUaNGaQWXmvy+CiFEWlqasLKyEgqFQtjY2IjDhw8LFxcX0bZtW9GmTRthamoqMjIy9K6/YsWKSielUinmzJkjPdZHv379pN/Rq1evCl9fX6FQKESzZs2EgYGBcHNzE5cuXdJ77D4+PuLbb78VQgixfft2YWBgIF599VUxa9YsMWjQIGFkZCS160qhUAilUik0Go3YvHlzrYfmhIQEYWhoKLy8vISlpaXYtGmTaNKkiRg7dqyYMGGCMDU1FXFxcXrXz8zMFK1atRLNmzcXzs7OQqFQiP79+wtfX1+hVCrFkCFDxJ07d/Sq/dprrwk/Pz9x6tSpCm2nTp0S/v7+4vXXX9d77A9q9IFFrVaLDRs2VNm+YcMG0apVK73rl79TVigUVU41+QPVvHlzcfz4celxWVmZeOutt0TLli3FuXPnavQH0NLSUvrDX1paKgwNDcWRI0ek9l9//VXY29vrPfb09HTRrl07MW3aNOndU10EltatW4sffvhBq/3AgQPC2dlZ7/q2trbi8OHDQoh7z8HRo0e12s+ePStMTU31rv/g+EtKSsSWLVtEQECAUCqVwsnJSbzzzjt6vyibmppKgVYIIYyMjMRvv/0mPc7KyhJmZmZ6j71Vq1bSu0Ah7r1IKxQKcfPmTSGEEOfPn6/RmwCNRiMGDhwo8vPzK7Tl5+eLgQMHir59++pd/9ixY4+ctmzZovfvVHBwsPD19RWHDh0Su3fvFl5eXsLb21tcu3ZNCHEvsCgUCr3HrtFoxNixY0VBQYF4//33xTPPPCPGjh0rtYeGhoqgoCC96ysUCvHMM88ItVqtNSkUCtGiRQuhVquFi4uL3rXL/89PnDhRuLu7S0dBc3JyhJeXl3jrrbf0Hru5ublUz9fXVyxdulSr/aOPPhJdunTRe+zr1q0TAwcOFEZGRsLW1lZMnTpV/Prrr3qP90Hu7u7Sm6K9e/cKlUolVq9eLbWvW7dOdOjQQe/6/fr1ExMmTJDONCxdulT069dPCCHEmTNnhFqtFlFRUXrVtrCw0HrdeNjhw4eFhYWFXrUf1ugDy6pVq4SJiYl4++23xddffy0OHjwoDh48KL7++mvx9ttvC1NTU63/GLpycnIS27dvr7L9l19+qVFgadKkSaWHeSdPniyeeeYZ8eOPP9YosJw9e1Z6bGFhofXOOSsrq0YvPELcO9oRHBwsOnXqJH799VdhZGRUa4Gl/N2Yk5NThT8cNR37P/7xDzFmzBghhBBDhgwRc+fO1WpfsmSJ8PDw0Lv+g3+8H/Tnn3+KqKgo0apVK72fVxcXF7Fz504hxL0/RgYGBuLLL7+U2nfs2CHUarV+AxdCTJ06VXTs2FHs3LlT7N27V7zwwguid+/eUntiYqJo06aN3vVNTU0f+UJw/PjxGofFqt5klM/Xd987OTmJtLQ06fHt27fFgAEDhKenp7h69WqNj7DY2NhIfw9KSkqEgYGB1voyMjJEixYt9K4/YcIE4enpWeFvTm280Xjw/3z79u3F119/rdW+Z88evcOQEEJYWVmJY8eOCSHuvcko/7nc2bNn9Q7qD449Ly9PLFu2TLi5uQkDAwPRrVs38cknn4iCggK9x17Zm4wHfwfOnz9fozcZZmZmWqf0iouLhZGRkbhy5YoQ4t4RKX3/Jtja2oqUlJQq25OTk4Wtra1etR/W6AOLEEJs3rxZ+Pr6CkNDQ+kPk6GhofD19RVbtmypUe0BAwaIefPmVdl+9OjRGr2j6tatm9i4cWOlbZMnTxbW1tZ6/wHs1KmT9MImxL0jKg8eFvzxxx9r9AfkQV988YWwt7cXBgYGtRZYPDw8RJcuXYSFhYX473//q9W+b9++Gv3h/vvvv4VarRY9e/YUERERwtTUVPTo0UOMGzdO9OzZUxgbG4sdO3bUaPyPOq9bVlZW4ahRdc2dO1c0a9ZMjB07Vri4uIjZs2eLli1bivj4eJGQkCCcnZ3FP//5T32HLm7cuCGGDh0q/T75+/trXS+0a9curYCkK0dHx0ceuv/mm2+Eo6Oj3vVtbW3Fp59+KrKysiqdduzYoffvlLm5eYVrPe7cuSOCgoJEp06dxPHjx2sUWMzNzcX58+elxw+/yfjzzz9r/CZj69atwtnZWXz00UfSvNoKLOVvMpo3b6511E+Ie28yTExM9K7/6quvitmzZwshhAgICKhw6mrNmjWibdu2etWu6vf1xx9/FCEhIcLc3FyYm5vrVVsIIb35FOLe3x6FQqH19yUlJUU888wzetd3cnLSOlX4f//3f0KhUEgh648//tB730+aNEm0atVKbN26VeuoaH5+vti6datQq9UiLCxM77E/6KkILOVKSkrEhQsXxIULF2p8gVe5H3/8UetF/2GFhYWPTJ+Ps2TJEunQXWUmTpyodyCKj48X3333XZXtc+bMkY4y1IacnByxfft2UVhYWONa8+fP15oSExO12qdPny6GDx9eo3X83//9n5g1a5Zwd3cXKpVKGBsbi1atWok33nhDHDp0qEa11Wq19O6mtpWWlorFixeLV155RSxZskSUlZWJL774Qjg7OwtbW1sxevToWnkObt26VemFpTU1b948YWNjI2JjY8WxY8dEbm6uyM3NFceOHROxsbGiadOmeh++FkKIvn37ioULF1bZXpM3GR4eHhXCsxD3Q0vLli1rFFjc3Ny0rgv77rvvpFNxQghx8ODBGr2wlfvrr7/Eiy++KAIDA8XFixdrLbC8/PLLYtCgQcLGxqZCKD148GCNTkGfOHFC2NraiuDgYLFw4UJhYWEh/vGPf4jFixeL4OBgYWJiItatW6dXbQMDg0e+wcjPz69wnZsuJk+eLNq2bSsWLVokfHx8REhIiHBzcxM7d+4UiYmJwsPDQ7z55pt61w8JCRG9evUSJ0+eFH/88YcYNmyY1umxlJQUvU+h3759W7z11lvC2NhYGBgYCJVKJVQqlTAwMBDGxsZi4sSJ4vbt23qP/UFPVWAhooZh6dKlwtHRUTo9U36qxtHRUSxbtqxGtbdu3So2bdpUZfu1a9fE+vXr9ao9c+bMKq+vuXPnjnj11VdrdMR1/vz54osvvqiy/Z133hGDBw/Wu/6DysrKxJIlS4SDg4NQKpU1DiyjR4/Wmh4+uj1jxgwREBBQo3WcPXtWDB8+XDRp0kQ6mm5kZCT8/f3Ftm3b9K77uCOiNVVYWCjGjRsnOnbsKMaPHy+Ki4vF+++/L4yNjYVCoRC9e/eu0frz8vLEc889J/0+tWrVSuu6k6+++kqsXLmyRtuQn58v9u7dKz7//HPx+eefi71791Z6HVpNNPr7sBBRw3X+/Hmt+zqU3xdHru7evYubN2/C0tKyyva///4brVq1qpP137x5E0qlEiYmJrVWMyMjA/v370dwcDBsbGxqre7DioqKoFQqoVKpalxLCIFLly6hrKwMdnZ2MDIyqoURPnm3b9/GnTt3au2eRpmZmSguLoabm5ve92GqT7w1PxHJlouLC/z8/ODn5yeFlZycHLz55pt1ts6a1Dc0NKwyrAD37hC8YMECfYf2WFevXsXEiRNrtaaXlxemTp0KGxubOt33165dw6RJk2qllkKhgL29PRwdHaWwUpdjr6vaKpUKTZo0qbX6bdu2RceOHSuElZrWv3XrFvbv348TJ05UaLt9+zY2btyod+0H8QgLETUox44dQ9euXVFaWtrg6jfksdd1fY69YdY/c+YM+vbti+zsbCgUCvTo0QNffPEFnJycANy7q7yTk1OtjL3hHRMiokbtm2++eWT7H3/8Idv6DXnsdV2fY2+c9WfNmoWOHTvi8OHDuH79OsLDw9GjRw+kpKSgZcuWetetDI+wEJGsGBgYQKFQPPKbvRUKhd7v2OqyfkMee13X59gbZ317e3vs2bMHHh4eAO5dPzRp0iR8//33SE5Ohrm5ea0dYeE1LEQkK46Ojti6dSvKysoqnY4cOSLb+g157HVdn2NvnPVv3bqldU2MQqFAfHw8BgwYgF69euHMmTM1GvuDGFiISFa8vLyQkZFRZfvj3inWZ/2GPPa6rs+xN876bm5uOHz4cIX5q1atwsCBA/Hqq6/qVbcyvIaFiGRlxowZKCoqqrLd1dUVycnJsqzfkMde1/U59sZZf9CgQfjiiy8watSoCm2rVq1CWVkZEhIS9Kr9MF7DQkRERLLHU0JEREQkewwsREREJHsMLERERCR7DCxEREQkewwsREREJHsMLERERCR7DCxEREQkewwsREREJHv/D29WypRhKoObAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df = pd.concat([r0, r1, r3, ideal_dist], axis=1)\n",
    "df.columns = ['method0: sampling', 'method1: hindsight', 'method3: generation tree', 'ideal']\n",
    "\n",
    "\n",
    "df = df.sort_index().fillna(0)\n",
    "df.plot.bar()\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df.sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MAE coverage (smaller is better\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "method0: sampling           1.121905\n",
       "method1: hindsight          0.952232\n",
       "method3: generation tree    0.494957\n",
       "dtype: float64"
      ]
     },
     "execution_count": 174,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_score = df / df['ideal'].values[:, None] - 1\n",
    "ratios = df_score.iloc[:21, :3]\n",
    "print('MAE coverage (smaller is better')\n",
    "ratios.abs().mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[0.0000, 0.0000, 0.0000, 0.0000, 0.0733, 0.1000, 0.0000, 0.1000, 0.0800,\n",
      "         0.0000, 0.0467, 0.0467, 0.1256, 0.0344, 0.3678, 0.0256, 0.0000, 0.0000,\n",
      "         0.0000, 0.0000, 0.0000]], dtype=torch.float64) method0: sampling\n",
      "tensor([[6.4225e-04, 2.8117e-03, 1.0104e-03, 9.0762e-03, 2.8844e-02, 4.3982e-02,\n",
      "         1.3840e-02, 6.3993e-02, 5.9184e-02, 1.2213e-02, 3.7620e-02, 4.5378e-02,\n",
      "         1.1956e-01, 3.1679e-02, 4.2386e-01, 4.2629e-02, 1.0285e-02, 3.4793e-02,\n",
      "         1.7495e-02, 1.1011e-03, 9.9832e-06]], dtype=torch.float64) method1: hindsight\n",
      "tensor([[0.0365, 0.0667, 0.0710, 0.0653, 0.0786, 0.0999, 0.0662, 0.0858, 0.0682,\n",
      "         0.0745, 0.0327, 0.0187, 0.0307, 0.0279, 0.0274, 0.0312, 0.0344, 0.0235,\n",
      "         0.0225, 0.0109, 0.0277]], dtype=torch.float64) method3: generation tree\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style type=\"text/css\">\n",
       "#T_90c6a_row0_col0, #T_90c6a_row0_col1, #T_90c6a_row1_col0 {\n",
       "  background-color: #800026;\n",
       "  color: #f1f1f1;\n",
       "}\n",
       "#T_90c6a_row1_col1 {\n",
       "  background-color: #fffdc8;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_90c6a_row2_col0, #T_90c6a_row2_col1 {\n",
       "  background-color: #ffffcc;\n",
       "  color: #000000;\n",
       "}\n",
       "</style>\n",
       "<table id=\"T_90c6a\">\n",
       "  <caption>KL_div_loss and time for each method (lower is better)</caption>\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th class=\"blank level0\" >&nbsp;</th>\n",
       "      <th id=\"T_90c6a_level0_col0\" class=\"col_heading level0 col0\" >KL_div_loss</th>\n",
       "      <th id=\"T_90c6a_level0_col1\" class=\"col_heading level0 col1\" >time</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th class=\"index_name level0\" >method</th>\n",
       "      <th class=\"blank col0\" >&nbsp;</th>\n",
       "      <th class=\"blank col1\" >&nbsp;</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th id=\"T_90c6a_level0_row0\" class=\"row_heading level0 row0\" >method0: sampling</th>\n",
       "      <td id=\"T_90c6a_row0_col0\" class=\"data row0 col0\" >-3.092141</td>\n",
       "      <td id=\"T_90c6a_row0_col1\" class=\"data row0 col1\" >48.504429</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_90c6a_level0_row1\" class=\"row_heading level0 row1\" >method1: hindsight</th>\n",
       "      <td id=\"T_90c6a_row1_col0\" class=\"data row1 col0\" >-3.092141</td>\n",
       "      <td id=\"T_90c6a_row1_col1\" class=\"data row1 col1\" >0.683987</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_90c6a_level0_row2\" class=\"row_heading level0 row2\" >method3: generation tree</th>\n",
       "      <td id=\"T_90c6a_row2_col0\" class=\"data row2 col0\" >-3.092156</td>\n",
       "      <td id=\"T_90c6a_row2_col1\" class=\"data row2 col1\" >0.075112</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x76460c6033d0>"
      ]
     },
     "execution_count": 175,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = []\n",
    "times = dict(zip(df.columns, [t0, t1, t3]))\n",
    "for k in df.columns[:3]:\n",
    "    input = torch.tensor(df[k].values)[None, :]\n",
    "    print(input, k)\n",
    "    target = torch.tensor(df['ideal'].values)[None, :]\n",
    "    # https://pytorch.org/docs/stable/generated/torch.nn.KLDivLoss.html#torch.nn.KLDivLoss\n",
    "    s = F.kl_div(input, target , reduction='batchmean', log_target=False).item()\n",
    "    t = times[k].total_seconds()\n",
    "    data.append({'method': k, 'KL_div_loss': s, 'time': t})\n",
    "dfr = pd.DataFrame(data).set_index('method')\n",
    "# color values with cmap\n",
    "dfs = dfr.style.background_gradient(cmap='YlOrRd')\n",
    "dfs.set_caption('KL_div_loss and time for each method (lower is better)')\n",
    "dfs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| method                   |   KL_div_loss |      time |\n",
      "|:-------------------------|--------------:|----------:|\n",
      "| method0: sampling        |      -3.09214 | 48.5044   |\n",
      "| method1: hindsight       |      -3.09214 |  0.683987 |\n",
      "| method3: generation tree |      -3.09216 |  0.075112 |\n"
     ]
    }
   ],
   "source": [
    "print(dfr.to_markdown())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Scratch, collapse probs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# def collapse_probs(json_schema, generated_data, keys = []):\n",
    "\n",
    "#     # get current row\n",
    "#     schema = json_schema\n",
    "#     data = generated_data\n",
    "#     for key in keys[:-1]:\n",
    "#         print(schema)\n",
    "#         schema = schema[key]\n",
    "#         data = data[key]\n",
    "\n",
    "#     schema_type = schema[\"type\"]\n",
    "\n",
    "#     if schema_type == \"p_enum\":\n",
    "#         k = keys[-1]\n",
    "#         data[k] = data[k][0]['choice']\n",
    "#     elif schema_type == \"object\":\n",
    "#         k = \"properties\"\n",
    "#         for key in schema[\"properties\"]:\n",
    "#             data[\"properties\"] = collapse_probs(json_schema, generated_data, keys + [key])\n",
    "    \n",
    "#     return generated_data\n",
    "\n",
    "\n",
    "# list(collapse_probs(json_schema, generated_data))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}