mirror of
https://github.com/wassname/detect_bs_text.git
synced 2026-07-13 14:13:05 +08:00
wassname
1528 lines
181 KiB
Plaintext
1528 lines
181 KiB
Plaintext
{
|
||
"cells": [
|
||
{
|
||
"cell_type": "markdown",
|
||
"metadata": {},
|
||
"source": [
|
||
"https://github.com/huggingface/peft/blob/main/examples/fp4_finetuning/finetune_fp4_opt_bnb_peft.py"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 1,
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stderr",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
|
||
" from .autonotebook import tqdm as notebook_tqdm\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"from torch import optim\n",
|
||
"import lightning as pl\n",
|
||
"from matplotlib import pyplot as plt"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 2,
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"data": {
|
||
"text/plain": [
|
||
"1"
|
||
]
|
||
},
|
||
"execution_count": 2,
|
||
"metadata": {},
|
||
"output_type": "execute_result"
|
||
}
|
||
],
|
||
"source": [
|
||
"from loguru import logger\n",
|
||
"import sys\n",
|
||
"\n",
|
||
"# only if you want it shorter\n",
|
||
"logger.remove()\n",
|
||
"logger.add(sys.stderr, format=\"<level>{message}</level>\", level=\"WARNING\")"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 3,
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"import os\n",
|
||
"os.environ['CUDA_VISIBLE_DEVICES']=\"1\"\n",
|
||
"# os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"\n",
|
||
"import torch\n",
|
||
"import torch.nn as nn\n",
|
||
"import transformers\n",
|
||
"from datasets import load_dataset\n",
|
||
"from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig, AutoConfig\n",
|
||
"import numpy as np\n",
|
||
"from tqdm.auto import tqdm\n",
|
||
"import pandas as pd\n",
|
||
"import warnings\n",
|
||
"from peft import LoraConfig, get_peft_model, IA3Config"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": []
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 4,
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"plt.style.use('seaborn-v0_8')\n",
|
||
"torch.set_float32_matmul_precision('medium')\n",
|
||
"warnings.filterwarnings(\"ignore\", \".*does not have many workers.*\")\n",
|
||
"warnings.filterwarnings(\"ignore\", \".*Was asked to gather along dimension 0.*\")\n",
|
||
"warnings.filterwarnings(\"ignore\", \".*There is an imbalance between your GPUs.*\")"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"\n"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 5,
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"# model_name = \"TheBloke/phi-2-GPTQ\"\n",
|
||
"model_name = \"microsoft/phi-2\"\n",
|
||
"\n",
|
||
"def load_model():\n",
|
||
"\n",
|
||
" model = AutoModelForCausalLM.from_pretrained(\n",
|
||
" model_name,\n",
|
||
" # quantization_config=BitsAndBytesConfig(\n",
|
||
" # load_in_4bit=True,\n",
|
||
" # llm_int8_threshold=6.0,\n",
|
||
" # llm_int8_has_fp16_weight=False,\n",
|
||
" # bnb_4bit_compute_dtype=torch.float16,\n",
|
||
" # bnb_4bit_use_double_quant=True,\n",
|
||
" # bnb_4bit_quant_type=\"nf4\",\n",
|
||
" # ),\n",
|
||
" torch_dtype=torch.float16,\n",
|
||
" trust_remote_code=True,\n",
|
||
" )\n",
|
||
"\n",
|
||
"\n",
|
||
" # config = AutoConfig.from_pretrained(model_name, trust_remote_code=True,)\n",
|
||
" # config.quantization_config['use_exllama'] = False\n",
|
||
" # config.quantization_config['disable_exllama'] = True\n",
|
||
" # model = AutoModelForCausalLM.from_pretrained(\n",
|
||
" # model_name,\n",
|
||
" # torch_dtype=torch.bfloat16,\n",
|
||
" # trust_remote_code=True,\n",
|
||
" # config=config,\n",
|
||
" # )\n",
|
||
" return model\n"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 6,
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stderr",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"Loading checkpoint shards: 100%|██████████| 2/2 [00:01<00:00, 1.77it/s]\n",
|
||
"Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"base_model = load_model()\n",
|
||
"tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True,)\n",
|
||
"tokenizer.pad_token = tokenizer.eos_token"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 7,
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"def reset_model(base_model):\n",
|
||
" # peft_config = LoraConfig(\n",
|
||
" # # task_type=TaskType.TOKEN_CLS, \n",
|
||
" # target_modules=[ \"fc2\", \"Wqkv\",],\n",
|
||
" # inference_mode=False, r=4, lora_alpha=4, \n",
|
||
" # # lora_dropout=0.1, \n",
|
||
" # # bias=\"all\"\n",
|
||
" # )\n",
|
||
" # peft_config = IA3Config(\n",
|
||
" # target_modules=[ \"fc2\", \"Wqkv\",], \n",
|
||
" # feedforward_modules=[\"fc2\"],\n",
|
||
" # inference_mode=False,\n",
|
||
" # )\n",
|
||
" peft_config = IA3Config(\n",
|
||
" target_modules=[ \"fc2\", \"Wqkv\", 'out_proj', 'fc1'], \n",
|
||
" feedforward_modules=[\"fc2\", 'fc1', 'out_proj'],\n",
|
||
" inference_mode=False,\n",
|
||
" )\n",
|
||
" model = get_peft_model(base_model, peft_config)\n",
|
||
" model.config.use_cache = False\n",
|
||
" return model\n",
|
||
"\n",
|
||
"model = reset_model(base_model)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 8,
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"[PosixPath('../samples/bletchley_decleration.md'), PosixPath('../samples/cicero_fin1.md'), PosixPath('../samples/disney_appointment.md'), PosixPath('../samples/fake_paper.md'), PosixPath('../samples/fauci_emails.md'), PosixPath('../samples/harvard_announcement_reminders.md'), PosixPath('../samples/how_to_catch_a_liar.md'), PosixPath('../samples/lk-99_end.md'), PosixPath('../samples/lk-99_espanol.md'), PosixPath('../samples/lorem_ipsum.md'), PosixPath('../samples/openai_board_ann.md'), PosixPath('../samples/openai_paper_weak_to_strong.md'), PosixPath('../samples/politics_is_the_mind_killer.md'), PosixPath('../samples/statement_vyKamala_on_passing_of_johnson.md'), PosixPath('../samples/survey_of_rumours.md')]\n"
|
||
]
|
||
},
|
||
{
|
||
"data": {
|
||
"text/plain": [
|
||
"dict_keys(['f', 'title', 'url', 'content'])"
|
||
]
|
||
},
|
||
"execution_count": 8,
|
||
"metadata": {},
|
||
"output_type": "execute_result"
|
||
}
|
||
],
|
||
"source": [
|
||
"\n",
|
||
"MAX_LEN = 400\n",
|
||
"\n",
|
||
"import frontmatter\n",
|
||
"from pathlib import Path\n",
|
||
"sample_files = sorted(Path(\"../samples/\").glob('*.md'))\n",
|
||
"print(sample_files)\n",
|
||
"samples = [{'f':f, **frontmatter.load(f).to_dict()} for f in sample_files]\n",
|
||
"\n",
|
||
"for sample in samples:\n",
|
||
" assert 'title' in sample, sample['f']\n",
|
||
" assert 'content' in sample\n",
|
||
"samples[0].keys()"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": []
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"metadata": {},
|
||
"source": [
|
||
"## Helpers"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 9,
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"# modified from https://github.dev/huggingface/evaluate/blob/8dfe05784099fb9af55b8e77793205a3b7c86465/measurements/perplexity/perplexity.py#L154\n",
|
||
"import evaluate\n",
|
||
"from evaluate import logging\n",
|
||
"from torch.nn import CrossEntropyLoss\n",
|
||
"from torch.utils.data import DataLoader\n",
|
||
"\n",
|
||
"def perplexity_compute(\n",
|
||
" ds, model, tokenizer, batch_size: int = 16, add_start_token: bool = True, device=None, max_length=None\n",
|
||
"):\n",
|
||
" model = model.to(device)\n",
|
||
"\n",
|
||
"\n",
|
||
" ds = ds.with_format('pt')\n",
|
||
" dl = DataLoader(ds, batch_size=1, shuffle=False, num_workers=0, collate_fn=tokenizer.pad, pin_memory=True)\n",
|
||
" ppls = []\n",
|
||
" loss_fct = CrossEntropyLoss(reduction=\"none\")\n",
|
||
" for b in dl:\n",
|
||
" input_ids = b['input_ids'].to(device)\n",
|
||
" attention_mask = b['attention_mask'].to(device)\n",
|
||
" # print(attention_mask)\n",
|
||
"\n",
|
||
" labels = input_ids\n",
|
||
"\n",
|
||
" with torch.no_grad():\n",
|
||
" out_logits = model(input_ids=input_ids, attention_mask=attention_mask).logits\n",
|
||
"\n",
|
||
" shift_logits = out_logits[..., :-1, :].contiguous()\n",
|
||
" shift_labels = labels[..., 1:].contiguous()\n",
|
||
" shift_attention_mask_batch = attention_mask[..., 1:].contiguous()\n",
|
||
"\n",
|
||
" perplexity_batch = torch.exp(\n",
|
||
" (loss_fct(shift_logits.transpose(1, 2), shift_labels) * shift_attention_mask_batch).sum(1)\n",
|
||
" / shift_attention_mask_batch.sum(1)\n",
|
||
" )\n",
|
||
"\n",
|
||
" ppls += perplexity_batch.tolist()\n",
|
||
"\n",
|
||
" return {\"perplexities\": ppls, \"mean_perplexity\": torch.tensor(ppls).mean()}"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 10,
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"# perplexity_compute(ds=ds_val, model=model, tokenizer=tokenizer, device='cuda')"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"metadata": {},
|
||
"source": [
|
||
"## Training"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 11,
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"from torch.nn import functional as F\n",
|
||
"from torch.utils.data import DataLoader, TensorDataset\n",
|
||
"from datasets import Dataset"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"metadata": {},
|
||
"source": [
|
||
"## Lightning helpers"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 12,
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"def eval(model, tokenizer, ds_val: Dataset):\n",
|
||
" model.eval();\n",
|
||
" with torch.no_grad():\n",
|
||
" with model.disable_adapter():\n",
|
||
" results = perplexity_compute(ds=ds_val, model=model, tokenizer=tokenizer, device='cuda')\n",
|
||
" results2 = perplexity_compute(ds=ds_val, model=model, tokenizer=tokenizer, device='cuda')\n",
|
||
" return dict(before=results['mean_perplexity'].item(), after=results2['mean_perplexity'].item())\n"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"metadata": {},
|
||
"source": [
|
||
"# Train"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 13,
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"from datasets import Dataset\n",
|
||
"\n",
|
||
"\n",
|
||
"def compute_metrics(eval_prediction):\n",
|
||
" return {}"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"metadata": {},
|
||
"source": [
|
||
"Trainer docs\n",
|
||
"\n",
|
||
"- https://huggingface.co/docs/transformers/v4.36.1/en/main_classes/trainer#transformers.Trainer"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 14,
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"data": {
|
||
"text/plain": [
|
||
"(400, 14201)"
|
||
]
|
||
},
|
||
"execution_count": 14,
|
||
"metadata": {},
|
||
"output_type": "execute_result"
|
||
}
|
||
],
|
||
"source": [
|
||
"MAX_LEN, len(sample['content'])//3"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 15,
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stderr",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"Token indices sequence length is longer than the specified maximum sequence length for this model (8172 > 2048). Running this sequence through the model will result in indexing errors\n"
|
||
]
|
||
},
|
||
{
|
||
"data": {
|
||
"text/plain": [
|
||
"Dataset({\n",
|
||
" features: ['input_ids', 'attention_mask', 'overflow_to_sample_mapping'],\n",
|
||
" num_rows: 11\n",
|
||
"})"
|
||
]
|
||
},
|
||
"execution_count": 15,
|
||
"metadata": {},
|
||
"output_type": "execute_result"
|
||
}
|
||
],
|
||
"source": [
|
||
"from sklearn.model_selection import train_test_split\n",
|
||
"\n",
|
||
"def tokenize_and_split(examples):\n",
|
||
" l = len(tokenizer(examples).input_ids[0])\n",
|
||
" max_len = min(l//3, MAX_LEN) # break into at least 5\n",
|
||
" max_len = max(max_len, 10)\n",
|
||
"\n",
|
||
"\n",
|
||
" result = tokenizer(\n",
|
||
" examples,\n",
|
||
" add_special_tokens=False,\n",
|
||
" truncation=True,\n",
|
||
" stride=2,\n",
|
||
" max_length=max_len,\n",
|
||
" return_overflowing_tokens=True,\n",
|
||
" return_attention_mask=True,\n",
|
||
" )\n",
|
||
" return result\n",
|
||
"\n",
|
||
"s = sample['content']\n",
|
||
"d = Dataset.from_dict(tokenize_and_split([s]))\n",
|
||
"d2 = d.train_test_split(test_size=0.5, seed=42)\n",
|
||
"ds_train = d2['train']\n",
|
||
"ds_val = d2['test']\n",
|
||
"ds_val"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": []
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 16,
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"def learn_sample(sample):\n",
|
||
" # device = 'cuda'\n",
|
||
" # lr = 4e-3\n",
|
||
" # epochs = 3\n",
|
||
" # accum_steps = 1\n",
|
||
" batch_size = 1\n",
|
||
" verbose = False\n",
|
||
"\n",
|
||
" s = sample['content']\n",
|
||
"\n",
|
||
" d = Dataset.from_dict(tokenize_and_split([s]))\n",
|
||
" d2 = d.train_test_split(test_size=0.5, seed=42)\n",
|
||
" ds_train = d2['train']\n",
|
||
" ds_val = d2['test']\n",
|
||
"\n",
|
||
" \n",
|
||
" model = reset_model(base_model)\n",
|
||
" # eval(model, tokenizer, ds_train)\n",
|
||
"\n",
|
||
" # https://huggingface.co/docs/transformers/v4.36.1/en/main_classes/trainer#transformers.Trainer\n",
|
||
" trainer = transformers.Trainer(\n",
|
||
" model=model,\n",
|
||
" train_dataset=ds_train,\n",
|
||
" eval_dataset=ds_val,\n",
|
||
" compute_metrics=compute_metrics, # without this it wont even give val loss\n",
|
||
" args=transformers.TrainingArguments(\n",
|
||
" # checkpoint='epoch',\n",
|
||
" save_strategy='epoch',\n",
|
||
" label_names=['labels',],\n",
|
||
" per_device_train_batch_size=batch_size,\n",
|
||
" # gradient_accumulation_steps=1,\n",
|
||
" # warmup_steps=6,\n",
|
||
" warmup_ratio=0.1,\n",
|
||
" # max_steps=50,\n",
|
||
" num_train_epochs=3,\n",
|
||
" learning_rate=1e-3,\n",
|
||
" fp16=True,\n",
|
||
" logging_steps=1,\n",
|
||
" output_dir=\"outputs\",\n",
|
||
" log_level='error',\n",
|
||
" # do_eval=True,\n",
|
||
" evaluation_strategy=\"epoch\",\n",
|
||
" eval_steps=1,\n",
|
||
" load_best_model_at_end=True,\n",
|
||
" \n",
|
||
" # disable_tqdm=not verbose,\n",
|
||
" ),\n",
|
||
" data_collator=transformers.DataCollatorForLanguageModeling(tokenizer, mlm=False),\n",
|
||
" )\n",
|
||
" trainer._signature_columns = ['input_ids', 'attention_mask', 'labels',]\n",
|
||
" model.config.use_cache = False # silence the warnings. Please re-enable for inference!\n",
|
||
" train_output = trainer.train()\n",
|
||
"\n",
|
||
" df_hist = pd.DataFrame(trainer.state.log_history)\n",
|
||
" df_hist_epoch = df_hist.groupby('epoch').last().drop(columns=['step'])\n",
|
||
" df_hist_step = df_hist.set_index('step').dropna(thresh=2, axis=1)\n",
|
||
" if verbose:\n",
|
||
" df_hist_epoch['loss'].plot()\n",
|
||
" plt.twinx()\n",
|
||
" df_hist_epoch['eval_loss'].plot(c='b', label='eval')\n",
|
||
" plt.legend()\n",
|
||
" plt.show()\n",
|
||
"\n",
|
||
"\n",
|
||
" result_train = {f'train/{k}':v for k,v in eval(model, tokenizer, ds_train).items()}\n",
|
||
" result = eval(model, tokenizer, ds_val)\n",
|
||
" result['hist'] = df_hist_epoch\n",
|
||
" result.update(result_train)\n",
|
||
" return result\n"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": []
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 17,
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stderr",
|
||
"output_type": "stream",
|
||
"text": [
|
||
" 17%|█▋ | 1/6 [00:00<00:02, 2.35it/s]"
|
||
]
|
||
},
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"{'loss': 2.8049, 'learning_rate': 0.001, 'epoch': 0.5}\n"
|
||
]
|
||
},
|
||
{
|
||
"name": "stderr",
|
||
"output_type": "stream",
|
||
"text": [
|
||
" 33%|███▎ | 2/6 [00:00<00:01, 2.80it/s]"
|
||
]
|
||
},
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"{'loss': 2.3359, 'learning_rate': 0.0008, 'epoch': 1.0}\n"
|
||
]
|
||
},
|
||
{
|
||
"name": "stderr",
|
||
"output_type": "stream",
|
||
"text": [
|
||
" \n",
|
||
" 33%|███▎ | 2/6 [00:00<00:01, 2.80it/s]"
|
||
]
|
||
},
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"{'eval_loss': 2.8958373069763184, 'eval_runtime': 0.2605, 'eval_samples_per_second': 7.677, 'eval_steps_per_second': 3.839, 'epoch': 1.0}\n"
|
||
]
|
||
},
|
||
{
|
||
"name": "stderr",
|
||
"output_type": "stream",
|
||
"text": [
|
||
" 50%|█████ | 3/6 [00:01<00:01, 2.08it/s]"
|
||
]
|
||
},
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"{'loss': 2.7644, 'learning_rate': 0.0006, 'epoch': 1.5}\n"
|
||
]
|
||
},
|
||
{
|
||
"name": "stderr",
|
||
"output_type": "stream",
|
||
"text": [
|
||
" 67%|██████▋ | 4/6 [00:01<00:00, 2.42it/s]"
|
||
]
|
||
},
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"{'loss': 2.297, 'learning_rate': 0.0004, 'epoch': 2.0}\n"
|
||
]
|
||
},
|
||
{
|
||
"name": "stderr",
|
||
"output_type": "stream",
|
||
"text": [
|
||
" \n",
|
||
" 67%|██████▋ | 4/6 [00:01<00:00, 2.42it/s]"
|
||
]
|
||
},
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"{'eval_loss': 2.8877780437469482, 'eval_runtime': 0.2592, 'eval_samples_per_second': 7.715, 'eval_steps_per_second': 3.857, 'epoch': 2.0}\n"
|
||
]
|
||
},
|
||
{
|
||
"name": "stderr",
|
||
"output_type": "stream",
|
||
"text": [
|
||
" 83%|████████▎ | 5/6 [00:02<00:00, 2.05it/s]"
|
||
]
|
||
},
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"{'loss': 2.2516, 'learning_rate': 0.0002, 'epoch': 2.5}\n"
|
||
]
|
||
},
|
||
{
|
||
"name": "stderr",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"100%|██████████| 6/6 [00:02<00:00, 2.35it/s]"
|
||
]
|
||
},
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"{'loss': 2.6997, 'learning_rate': 0.0, 'epoch': 3.0}\n"
|
||
]
|
||
},
|
||
{
|
||
"name": "stderr",
|
||
"output_type": "stream",
|
||
"text": [
|
||
" \n",
|
||
"100%|██████████| 6/6 [00:02<00:00, 2.05it/s]\n"
|
||
]
|
||
},
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"{'eval_loss': 2.884704113006592, 'eval_runtime': 0.2598, 'eval_samples_per_second': 7.699, 'eval_steps_per_second': 3.85, 'epoch': 3.0}\n",
|
||
"{'train_runtime': 2.9316, 'train_samples_per_second': 2.047, 'train_steps_per_second': 2.047, 'train_loss': 2.525577942530314, 'epoch': 3.0}\n",
|
||
"blechley declaration\n",
|
||
"{'before': 17.869091033935547, 'after': 17.52420425415039}\n",
|
||
"{'loss': 4.625, 'learning_rate': 0.0005, 'epoch': 0.17}\n",
|
||
"{'loss': 4.5422, 'learning_rate': 0.001, 'epoch': 0.33}\n",
|
||
"{'loss': 4.5928, 'learning_rate': 0.0009375, 'epoch': 0.5}\n",
|
||
"{'loss': 4.5384, 'learning_rate': 0.000875, 'epoch': 0.67}\n",
|
||
"{'loss': 4.7047, 'learning_rate': 0.0008125000000000001, 'epoch': 0.83}\n",
|
||
"{'loss': 4.4394, 'learning_rate': 0.00075, 'epoch': 1.0}\n",
|
||
"{'eval_loss': 4.234132289886475, 'eval_runtime': 0.8515, 'eval_samples_per_second': 7.047, 'eval_steps_per_second': 1.174, 'epoch': 1.0}\n",
|
||
"{'loss': 4.4981, 'learning_rate': 0.0006875, 'epoch': 1.17}\n",
|
||
"{'loss': 4.5187, 'learning_rate': 0.000625, 'epoch': 1.33}\n",
|
||
"{'loss': 4.419, 'learning_rate': 0.0005625000000000001, 'epoch': 1.5}\n",
|
||
"{'loss': 4.6345, 'learning_rate': 0.0005, 'epoch': 1.67}\n",
|
||
"{'loss': 4.4387, 'learning_rate': 0.0004375, 'epoch': 1.83}\n",
|
||
"{'loss': 4.3689, 'learning_rate': 0.000375, 'epoch': 2.0}\n",
|
||
"{'eval_loss': 4.212051868438721, 'eval_runtime': 0.7417, 'eval_samples_per_second': 8.09, 'eval_steps_per_second': 1.348, 'epoch': 2.0}\n",
|
||
"{'loss': 4.3709, 'learning_rate': 0.0003125, 'epoch': 2.17}\n",
|
||
"{'loss': 4.4461, 'learning_rate': 0.00025, 'epoch': 2.33}\n",
|
||
"{'loss': 4.5014, 'learning_rate': 0.0001875, 'epoch': 2.5}\n",
|
||
"{'loss': 4.5928, 'learning_rate': 0.000125, 'epoch': 2.67}\n",
|
||
"{'loss': 4.4311, 'learning_rate': 6.25e-05, 'epoch': 2.83}\n",
|
||
"{'loss': 4.4222, 'learning_rate': 0.0, 'epoch': 3.0}\n",
|
||
"{'eval_loss': 4.2056355476379395, 'eval_runtime': 0.7622, 'eval_samples_per_second': 7.872, 'eval_steps_per_second': 1.312, 'epoch': 3.0}\n",
|
||
"{'train_runtime': 8.1788, 'train_samples_per_second': 2.201, 'train_steps_per_second': 2.201, 'train_loss': 4.504713243908352, 'epoch': 3.0}\n",
|
||
"cicero from ibois, Philippe (2012-06-03).\n",
|
||
"{'before': 72.58737182617188, 'after': 67.74419403076172}\n",
|
||
"{'loss': 2.187, 'learning_rate': 0.001, 'epoch': 0.5}\n",
|
||
"{'loss': 2.0009, 'learning_rate': 0.0008, 'epoch': 1.0}\n",
|
||
"{'eval_loss': 1.9654667377471924, 'eval_runtime': 0.178, 'eval_samples_per_second': 11.238, 'eval_steps_per_second': 5.619, 'epoch': 1.0}\n",
|
||
"{'loss': 2.1424, 'learning_rate': 0.0006, 'epoch': 1.5}\n",
|
||
"{'loss': 1.9167, 'learning_rate': 0.0004, 'epoch': 2.0}\n",
|
||
"{'eval_loss': 1.9636929035186768, 'eval_runtime': 0.1788, 'eval_samples_per_second': 11.188, 'eval_steps_per_second': 5.594, 'epoch': 2.0}\n",
|
||
"{'loss': 1.8877, 'learning_rate': 0.0002, 'epoch': 2.5}\n",
|
||
"{'loss': 2.0672, 'learning_rate': 0.0, 'epoch': 3.0}\n",
|
||
"{'eval_loss': 1.9641517400741577, 'eval_runtime': 0.184, 'eval_samples_per_second': 10.869, 'eval_steps_per_second': 5.435, 'epoch': 3.0}\n",
|
||
"{'train_runtime': 1.978, 'train_samples_per_second': 3.033, 'train_steps_per_second': 3.033, 'train_loss': 2.033644914627075, 'epoch': 3.0}\n",
|
||
"disney appointment\n",
|
||
"{'before': 118.82615661621094, 'after': 118.42012786865234}\n",
|
||
"{'loss': 2.0357, 'learning_rate': 0.001, 'epoch': 0.5}\n",
|
||
"{'loss': 1.9873, 'learning_rate': 0.0008, 'epoch': 1.0}\n",
|
||
"{'eval_loss': 1.807045817375183, 'eval_runtime': 0.1406, 'eval_samples_per_second': 14.221, 'eval_steps_per_second': 7.11, 'epoch': 1.0}\n",
|
||
"{'loss': 2.0176, 'learning_rate': 0.0006, 'epoch': 1.5}\n",
|
||
"{'loss': 1.9243, 'learning_rate': 0.0004, 'epoch': 2.0}\n",
|
||
"{'eval_loss': 1.7954585552215576, 'eval_runtime': 0.1396, 'eval_samples_per_second': 14.328, 'eval_steps_per_second': 7.164, 'epoch': 2.0}\n",
|
||
"{'loss': 1.9109, 'learning_rate': 0.0002, 'epoch': 2.5}\n",
|
||
"{'loss': 1.9423, 'learning_rate': 0.0, 'epoch': 3.0}\n",
|
||
"{'eval_loss': 1.7919256687164307, 'eval_runtime': 0.14, 'eval_samples_per_second': 14.289, 'eval_steps_per_second': 7.145, 'epoch': 3.0}\n",
|
||
"{'train_runtime': 1.5666, 'train_samples_per_second': 3.83, 'train_steps_per_second': 3.83, 'train_loss': 1.969655990600586, 'epoch': 3.0}\n",
|
||
"fake ai hoax paper\n",
|
||
"{'before': 7.766979217529297, 'after': 7.697228908538818}\n",
|
||
"{'loss': 2.4331, 'learning_rate': 0.0005, 'epoch': 0.2}\n",
|
||
"{'loss': 2.9004, 'learning_rate': 0.001, 'epoch': 0.4}\n",
|
||
"{'loss': 3.5675, 'learning_rate': 0.0009230769230769232, 'epoch': 0.6}\n",
|
||
"{'loss': 3.3972, 'learning_rate': 0.0008461538461538462, 'epoch': 0.8}\n",
|
||
"{'loss': 3.5431, 'learning_rate': 0.0007692307692307693, 'epoch': 1.0}\n",
|
||
"{'eval_loss': 3.0353567600250244, 'eval_runtime': 0.8628, 'eval_samples_per_second': 6.954, 'eval_steps_per_second': 1.159, 'epoch': 1.0}\n",
|
||
"{'loss': 3.4732, 'learning_rate': 0.0006923076923076923, 'epoch': 1.2}\n",
|
||
"{'loss': 3.5319, 'learning_rate': 0.0006153846153846154, 'epoch': 1.4}\n",
|
||
"{'loss': 2.7887, 'learning_rate': 0.0005384615384615384, 'epoch': 1.6}\n",
|
||
"{'loss': 2.3222, 'learning_rate': 0.0004615384615384616, 'epoch': 1.8}\n",
|
||
"{'loss': 3.3527, 'learning_rate': 0.00038461538461538467, 'epoch': 2.0}\n",
|
||
"{'eval_loss': 3.0194509029388428, 'eval_runtime': 0.7424, 'eval_samples_per_second': 8.082, 'eval_steps_per_second': 1.347, 'epoch': 2.0}\n",
|
||
"{'loss': 2.2585, 'learning_rate': 0.0003076923076923077, 'epoch': 2.2}\n",
|
||
"{'loss': 3.3084, 'learning_rate': 0.0002307692307692308, 'epoch': 2.4}\n",
|
||
"{'loss': 3.4762, 'learning_rate': 0.00015384615384615385, 'epoch': 2.6}\n",
|
||
"{'loss': 3.2907, 'learning_rate': 7.692307692307693e-05, 'epoch': 2.8}\n",
|
||
"{'loss': 2.7338, 'learning_rate': 0.0, 'epoch': 3.0}\n",
|
||
"{'eval_loss': 3.014026641845703, 'eval_runtime': 0.7435, 'eval_samples_per_second': 8.07, 'eval_steps_per_second': 1.345, 'epoch': 3.0}\n",
|
||
"{'train_runtime': 7.2582, 'train_samples_per_second': 2.067, 'train_steps_per_second': 2.067, 'train_loss': 3.09183349609375, 'epoch': 3.0}\n",
|
||
"buzzfeed foi fauci emails 2023\n",
|
||
"{'before': 23.3094425201416, 'after': 22.406410217285156}\n",
|
||
"{'loss': 2.6779, 'learning_rate': 0.001, 'epoch': 0.5}\n",
|
||
"{'loss': 2.4325, 'learning_rate': 0.0008, 'epoch': 1.0}\n",
|
||
"{'eval_loss': 2.7339022159576416, 'eval_runtime': 0.2193, 'eval_samples_per_second': 9.121, 'eval_steps_per_second': 4.561, 'epoch': 1.0}\n",
|
||
"{'loss': 2.6596, 'learning_rate': 0.0006, 'epoch': 1.5}\n",
|
||
"{'loss': 2.3446, 'learning_rate': 0.0004, 'epoch': 2.0}\n",
|
||
"{'eval_loss': 2.730224609375, 'eval_runtime': 0.2245, 'eval_samples_per_second': 8.91, 'eval_steps_per_second': 4.455, 'epoch': 2.0}\n",
|
||
"{'loss': 2.3111, 'learning_rate': 0.0002, 'epoch': 2.5}\n",
|
||
"{'loss': 2.5791, 'learning_rate': 0.0, 'epoch': 3.0}\n",
|
||
"{'eval_loss': 2.729649543762207, 'eval_runtime': 0.2204, 'eval_samples_per_second': 9.076, 'eval_steps_per_second': 4.538, 'epoch': 3.0}\n",
|
||
"{'train_runtime': 2.2643, 'train_samples_per_second': 2.65, 'train_steps_per_second': 2.65, 'train_loss': 2.500802000363668, 'epoch': 3.0}\n",
|
||
"harvard announcment caplain israel hamas\n",
|
||
"{'before': 45.347354888916016, 'after': 45.12727355957031}\n",
|
||
"{'loss': 2.6223, 'learning_rate': 0.001, 'epoch': 0.5}\n",
|
||
"{'loss': 2.4621, 'learning_rate': 0.0008, 'epoch': 1.0}\n",
|
||
"{'eval_loss': 3.127470016479492, 'eval_runtime': 0.378, 'eval_samples_per_second': 5.291, 'eval_steps_per_second': 2.645, 'epoch': 1.0}\n",
|
||
"{'loss': 2.5923, 'learning_rate': 0.0006, 'epoch': 1.5}\n",
|
||
"{'loss': 2.4313, 'learning_rate': 0.0004, 'epoch': 2.0}\n",
|
||
"{'eval_loss': 3.115180253982544, 'eval_runtime': 0.2642, 'eval_samples_per_second': 7.569, 'eval_steps_per_second': 3.785, 'epoch': 2.0}\n",
|
||
"{'loss': 2.3652, 'learning_rate': 0.0002, 'epoch': 2.5}\n",
|
||
"{'loss': 2.5058, 'learning_rate': 0.0, 'epoch': 3.0}\n",
|
||
"{'eval_loss': 3.1118650436401367, 'eval_runtime': 0.2632, 'eval_samples_per_second': 7.6, 'eval_steps_per_second': 3.8, 'epoch': 3.0}\n",
|
||
"{'train_runtime': 2.9644, 'train_samples_per_second': 2.024, 'train_steps_per_second': 2.024, 'train_loss': 2.4964850743611655, 'epoch': 3.0}\n",
|
||
"How to Catch an AI Liar\n",
|
||
"{'before': 28.949859619140625, 'after': 28.0087890625}\n",
|
||
"{'loss': 3.5646, 'learning_rate': 0.0005, 'epoch': 0.2}\n",
|
||
"{'loss': 2.8252, 'learning_rate': 0.001, 'epoch': 0.4}\n",
|
||
"{'loss': 2.0116, 'learning_rate': 0.0009230769230769232, 'epoch': 0.6}\n",
|
||
"{'loss': 2.7551, 'learning_rate': 0.0008461538461538462, 'epoch': 0.8}\n",
|
||
"{'loss': 2.4442, 'learning_rate': 0.0007692307692307693, 'epoch': 1.0}\n",
|
||
"{'eval_loss': 2.635437488555908, 'eval_runtime': 0.7632, 'eval_samples_per_second': 6.552, 'eval_steps_per_second': 1.31, 'epoch': 1.0}\n",
|
||
"{'loss': 1.9103, 'learning_rate': 0.0006923076923076923, 'epoch': 1.2}\n",
|
||
"{'loss': 2.4336, 'learning_rate': 0.0006153846153846154, 'epoch': 1.4}\n",
|
||
"{'loss': 2.7349, 'learning_rate': 0.0005384615384615384, 'epoch': 1.6}\n",
|
||
"{'loss': 3.4216, 'learning_rate': 0.0004615384615384616, 'epoch': 1.8}\n",
|
||
"{'loss': 2.6993, 'learning_rate': 0.00038461538461538467, 'epoch': 2.0}\n",
|
||
"{'eval_loss': 2.623994827270508, 'eval_runtime': 0.6368, 'eval_samples_per_second': 7.852, 'eval_steps_per_second': 1.57, 'epoch': 2.0}\n",
|
||
"{'loss': 3.3393, 'learning_rate': 0.0003076923076923077, 'epoch': 2.2}\n",
|
||
"{'loss': 1.8571, 'learning_rate': 0.0002307692307692308, 'epoch': 2.4}\n",
|
||
"{'loss': 2.3943, 'learning_rate': 0.00015384615384615385, 'epoch': 2.6}\n",
|
||
"{'loss': 2.6463, 'learning_rate': 7.692307692307693e-05, 'epoch': 2.8}\n",
|
||
"{'loss': 2.6925, 'learning_rate': 0.0, 'epoch': 3.0}\n",
|
||
"{'eval_loss': 2.6198740005493164, 'eval_runtime': 0.636, 'eval_samples_per_second': 7.862, 'eval_steps_per_second': 1.572, 'epoch': 3.0}\n",
|
||
"{'train_runtime': 6.4124, 'train_samples_per_second': 2.339, 'train_steps_per_second': 2.339, 'train_loss': 2.648661724726359, 'epoch': 3.0}\n",
|
||
"LK-99-en\n",
|
||
"{'before': 14.513845443725586, 'after': 14.066093444824219}\n",
|
||
"{'loss': 2.9979, 'learning_rate': 0.0005, 'epoch': 0.17}\n",
|
||
"{'loss': 2.0904, 'learning_rate': 0.001, 'epoch': 0.33}\n",
|
||
"{'loss': 2.7862, 'learning_rate': 0.0009375, 'epoch': 0.5}\n",
|
||
"{'loss': 2.4246, 'learning_rate': 0.000875, 'epoch': 0.67}\n",
|
||
"{'loss': 2.6095, 'learning_rate': 0.0008125000000000001, 'epoch': 0.83}\n",
|
||
"{'loss': 2.5655, 'learning_rate': 0.00075, 'epoch': 1.0}\n",
|
||
"{'eval_loss': 2.335665464401245, 'eval_runtime': 0.9768, 'eval_samples_per_second': 7.166, 'eval_steps_per_second': 1.024, 'epoch': 1.0}\n",
|
||
"{'loss': 2.3562, 'learning_rate': 0.0006875, 'epoch': 1.17}\n",
|
||
"{'loss': 2.8196, 'learning_rate': 0.000625, 'epoch': 1.33}\n",
|
||
"{'loss': 2.5404, 'learning_rate': 0.0005625000000000001, 'epoch': 1.5}\n",
|
||
"{'loss': 2.4861, 'learning_rate': 0.0005, 'epoch': 1.67}\n",
|
||
"{'loss': 2.6823, 'learning_rate': 0.0004375, 'epoch': 1.83}\n",
|
||
"{'loss': 2.0126, 'learning_rate': 0.000375, 'epoch': 2.0}\n",
|
||
"{'eval_loss': 2.320927858352661, 'eval_runtime': 0.867, 'eval_samples_per_second': 8.074, 'eval_steps_per_second': 1.153, 'epoch': 2.0}\n",
|
||
"{'loss': 2.4835, 'learning_rate': 0.0003125, 'epoch': 2.17}\n",
|
||
"{'loss': 2.5887, 'learning_rate': 0.00025, 'epoch': 2.33}\n",
|
||
"{'loss': 2.7585, 'learning_rate': 0.0001875, 'epoch': 2.5}\n",
|
||
"{'loss': 2.4708, 'learning_rate': 0.000125, 'epoch': 2.67}\n",
|
||
"{'loss': 2.2034, 'learning_rate': 6.25e-05, 'epoch': 2.83}\n",
|
||
"{'loss': 1.9709, 'learning_rate': 0.0, 'epoch': 3.0}\n",
|
||
"{'eval_loss': 2.3152239322662354, 'eval_runtime': 0.8652, 'eval_samples_per_second': 8.091, 'eval_steps_per_second': 1.156, 'epoch': 3.0}\n",
|
||
"{'train_runtime': 8.6099, 'train_samples_per_second': 2.091, 'train_steps_per_second': 2.091, 'train_loss': 2.4914957218699985, 'epoch': 3.0}\n",
|
||
"LK-99-es\n",
|
||
"{'before': 11.415018081665039, 'after': 10.97294807434082}\n",
|
||
"{'loss': 1.8845, 'learning_rate': 0.0003333333333333333, 'epoch': 0.11}\n",
|
||
"{'loss': 1.9495, 'learning_rate': 0.0006666666666666666, 'epoch': 0.22}\n",
|
||
"{'loss': 1.9686, 'learning_rate': 0.001, 'epoch': 0.33}\n",
|
||
"{'loss': 1.9168, 'learning_rate': 0.0009583333333333334, 'epoch': 0.44}\n",
|
||
"{'loss': 1.9844, 'learning_rate': 0.0009166666666666666, 'epoch': 0.56}\n",
|
||
"{'loss': 1.9139, 'learning_rate': 0.000875, 'epoch': 0.67}\n",
|
||
"{'loss': 1.8329, 'learning_rate': 0.0008333333333333334, 'epoch': 0.78}\n",
|
||
"{'loss': 1.8556, 'learning_rate': 0.0007916666666666666, 'epoch': 0.89}\n",
|
||
"{'loss': 1.8869, 'learning_rate': 0.0007916666666666666, 'epoch': 1.0}\n",
|
||
"{'eval_loss': 1.836249828338623, 'eval_runtime': 1.2543, 'eval_samples_per_second': 7.175, 'eval_steps_per_second': 1.595, 'epoch': 1.0}\n",
|
||
"{'loss': 1.8902, 'learning_rate': 0.00075, 'epoch': 1.11}\n",
|
||
"{'loss': 1.8931, 'learning_rate': 0.0007083333333333334, 'epoch': 1.22}\n",
|
||
"{'loss': 1.8021, 'learning_rate': 0.0006666666666666666, 'epoch': 1.33}\n",
|
||
"{'loss': 1.884, 'learning_rate': 0.000625, 'epoch': 1.44}\n",
|
||
"{'loss': 1.7923, 'learning_rate': 0.0005833333333333334, 'epoch': 1.56}\n",
|
||
"{'loss': 1.848, 'learning_rate': 0.0005416666666666666, 'epoch': 1.67}\n",
|
||
"{'loss': 1.8103, 'learning_rate': 0.0005, 'epoch': 1.78}\n",
|
||
"{'loss': 1.9024, 'learning_rate': 0.0004583333333333333, 'epoch': 1.89}\n",
|
||
"{'loss': 1.9309, 'learning_rate': 0.0004166666666666667, 'epoch': 2.0}\n",
|
||
"{'eval_loss': 1.829906702041626, 'eval_runtime': 1.1467, 'eval_samples_per_second': 7.849, 'eval_steps_per_second': 1.744, 'epoch': 2.0}\n",
|
||
"{'loss': 1.9011, 'learning_rate': 0.000375, 'epoch': 2.11}\n",
|
||
"{'loss': 1.8506, 'learning_rate': 0.0003333333333333333, 'epoch': 2.22}\n",
|
||
"{'loss': 1.8036, 'learning_rate': 0.0002916666666666667, 'epoch': 2.33}\n",
|
||
"{'loss': 1.828, 'learning_rate': 0.00025, 'epoch': 2.44}\n",
|
||
"{'loss': 1.9094, 'learning_rate': 0.00020833333333333335, 'epoch': 2.56}\n",
|
||
"{'loss': 1.8678, 'learning_rate': 0.00016666666666666666, 'epoch': 2.67}\n",
|
||
"{'loss': 1.8555, 'learning_rate': 0.000125, 'epoch': 2.78}\n",
|
||
"{'loss': 1.7289, 'learning_rate': 8.333333333333333e-05, 'epoch': 2.89}\n",
|
||
"{'loss': 1.7859, 'learning_rate': 4.1666666666666665e-05, 'epoch': 3.0}\n",
|
||
"{'eval_loss': 1.8297154903411865, 'eval_runtime': 1.1477, 'eval_samples_per_second': 7.841, 'eval_steps_per_second': 1.743, 'epoch': 3.0}\n",
|
||
"{'train_runtime': 12.5295, 'train_samples_per_second': 2.155, 'train_steps_per_second': 2.155, 'train_loss': 1.8695231985162806, 'epoch': 3.0}\n",
|
||
"Lorem ipsum\n",
|
||
"{'before': 6.564840316772461, 'after': 6.268791675567627}\n",
|
||
"{'loss': 2.2808, 'learning_rate': 0.001, 'epoch': 0.5}\n",
|
||
"{'loss': 2.264, 'learning_rate': 0.0008, 'epoch': 1.0}\n",
|
||
"{'eval_loss': 2.300767183303833, 'eval_runtime': 0.1412, 'eval_samples_per_second': 14.167, 'eval_steps_per_second': 7.084, 'epoch': 1.0}\n",
|
||
"{'loss': 2.1983, 'learning_rate': 0.0006, 'epoch': 1.5}\n",
|
||
"{'loss': 2.1913, 'learning_rate': 0.0004, 'epoch': 2.0}\n",
|
||
"{'eval_loss': 2.2839250564575195, 'eval_runtime': 0.1403, 'eval_samples_per_second': 14.254, 'eval_steps_per_second': 7.127, 'epoch': 2.0}\n",
|
||
"{'loss': 2.149, 'learning_rate': 0.0002, 'epoch': 2.5}\n",
|
||
"{'loss': 2.095, 'learning_rate': 0.0, 'epoch': 3.0}\n",
|
||
"{'eval_loss': 2.2791788578033447, 'eval_runtime': 0.1407, 'eval_samples_per_second': 14.211, 'eval_steps_per_second': 7.105, 'epoch': 3.0}\n",
|
||
"{'train_runtime': 1.5544, 'train_samples_per_second': 3.86, 'train_steps_per_second': 3.86, 'train_loss': 2.196397304534912, 'epoch': 3.0}\n",
|
||
"openai board ann\n",
|
||
"{'before': 55.808502197265625, 'after': 54.66792297363281}\n",
|
||
"{'loss': 2.8464, 'learning_rate': 0.001, 'epoch': 0.5}\n",
|
||
"{'loss': 2.4796, 'learning_rate': 0.0008, 'epoch': 1.0}\n",
|
||
"{'eval_loss': 2.501676082611084, 'eval_runtime': 0.3861, 'eval_samples_per_second': 5.18, 'eval_steps_per_second': 2.59, 'epoch': 1.0}\n",
|
||
"{'loss': 2.8062, 'learning_rate': 0.0006, 'epoch': 1.5}\n",
|
||
"{'loss': 2.4146, 'learning_rate': 0.0004, 'epoch': 2.0}\n",
|
||
"{'eval_loss': 2.4917240142822266, 'eval_runtime': 0.2752, 'eval_samples_per_second': 7.268, 'eval_steps_per_second': 3.634, 'epoch': 2.0}\n",
|
||
"{'loss': 2.356, 'learning_rate': 0.0002, 'epoch': 2.5}\n",
|
||
"{'loss': 2.7228, 'learning_rate': 0.0, 'epoch': 3.0}\n",
|
||
"{'eval_loss': 2.488405704498291, 'eval_runtime': 0.2661, 'eval_samples_per_second': 7.515, 'eval_steps_per_second': 3.757, 'epoch': 3.0}\n",
|
||
"{'train_runtime': 3.0083, 'train_samples_per_second': 1.994, 'train_steps_per_second': 1.994, 'train_loss': 2.60427188873291, 'epoch': 3.0}\n",
|
||
"weak to strong\n",
|
||
"{'before': 46.764183044433594, 'after': 46.40473556518555}\n",
|
||
"{'loss': 2.7259, 'learning_rate': 0.001, 'epoch': 0.5}\n",
|
||
"{'loss': 2.7937, 'learning_rate': 0.0008, 'epoch': 1.0}\n",
|
||
"{'eval_loss': 2.7441396713256836, 'eval_runtime': 0.1515, 'eval_samples_per_second': 13.205, 'eval_steps_per_second': 6.603, 'epoch': 1.0}\n",
|
||
"{'loss': 2.6661, 'learning_rate': 0.0006, 'epoch': 1.5}\n",
|
||
"{'loss': 2.6998, 'learning_rate': 0.0004, 'epoch': 2.0}\n",
|
||
"{'eval_loss': 2.74137282371521, 'eval_runtime': 0.1514, 'eval_samples_per_second': 13.211, 'eval_steps_per_second': 6.605, 'epoch': 2.0}\n",
|
||
"{'loss': 2.6949, 'learning_rate': 0.0002, 'epoch': 2.5}\n",
|
||
"{'loss': 2.5599, 'learning_rate': 0.0, 'epoch': 3.0}\n",
|
||
"{'eval_loss': 2.7393088340759277, 'eval_runtime': 0.1558, 'eval_samples_per_second': 12.833, 'eval_steps_per_second': 6.416, 'epoch': 3.0}\n",
|
||
"{'train_runtime': 1.6635, 'train_samples_per_second': 3.607, 'train_steps_per_second': 3.607, 'train_loss': 2.690062483151754, 'epoch': 3.0}\n",
|
||
"politics is the mind-killer\n",
|
||
"{'before': 247.5515594482422, 'after': 245.82681274414062}\n",
|
||
"{'loss': 1.8812, 'learning_rate': 0.001, 'epoch': 0.5}\n",
|
||
"{'loss': 2.2442, 'learning_rate': 0.0008, 'epoch': 1.0}\n",
|
||
"{'eval_loss': 2.4278409481048584, 'eval_runtime': 0.0735, 'eval_samples_per_second': 27.196, 'eval_steps_per_second': 13.598, 'epoch': 1.0}\n",
|
||
"{'loss': 1.8079, 'learning_rate': 0.0006, 'epoch': 1.5}\n",
|
||
"{'loss': 2.1305, 'learning_rate': 0.0004, 'epoch': 2.0}\n",
|
||
"{'eval_loss': 2.4188129901885986, 'eval_runtime': 0.0764, 'eval_samples_per_second': 26.194, 'eval_steps_per_second': 13.097, 'epoch': 2.0}\n",
|
||
"{'loss': 2.1037, 'learning_rate': 0.0002, 'epoch': 2.5}\n",
|
||
"{'loss': 1.6808, 'learning_rate': 0.0, 'epoch': 3.0}\n",
|
||
"{'eval_loss': 2.4169728755950928, 'eval_runtime': 0.0738, 'eval_samples_per_second': 27.114, 'eval_steps_per_second': 13.557, 'epoch': 3.0}\n",
|
||
"{'train_runtime': 0.9463, 'train_samples_per_second': 6.341, 'train_steps_per_second': 6.341, 'train_loss': 1.9747143586476643, 'epoch': 3.0}\n",
|
||
"statement by whitehouse on passing\n",
|
||
"{'before': 29.197132110595703, 'after': 28.9396915435791}\n",
|
||
"{'loss': 2.5398, 'learning_rate': 0.0003333333333333333, 'epoch': 0.1}\n",
|
||
"{'loss': 2.563, 'learning_rate': 0.0006666666666666666, 'epoch': 0.2}\n",
|
||
"{'loss': 2.362, 'learning_rate': 0.001, 'epoch': 0.3}\n",
|
||
"{'loss': 2.4908, 'learning_rate': 0.0009629629629629629, 'epoch': 0.4}\n",
|
||
"{'loss': 2.8433, 'learning_rate': 0.000925925925925926, 'epoch': 0.5}\n",
|
||
"{'loss': 3.1541, 'learning_rate': 0.0008888888888888888, 'epoch': 0.6}\n",
|
||
"{'loss': 2.9383, 'learning_rate': 0.0008518518518518519, 'epoch': 0.7}\n",
|
||
"{'loss': 2.6576, 'learning_rate': 0.0008148148148148148, 'epoch': 0.8}\n",
|
||
"{'loss': 2.6327, 'learning_rate': 0.0007777777777777778, 'epoch': 0.9}\n",
|
||
"{'loss': 2.6943, 'learning_rate': 0.0007407407407407407, 'epoch': 1.0}\n",
|
||
"{'eval_loss': 2.42354416847229, 'eval_runtime': 1.5423, 'eval_samples_per_second': 7.132, 'eval_steps_per_second': 1.297, 'epoch': 1.0}\n",
|
||
"{'loss': 2.3882, 'learning_rate': 0.0007037037037037038, 'epoch': 1.1}\n",
|
||
"{'loss': 2.738, 'learning_rate': 0.0006666666666666666, 'epoch': 1.2}\n",
|
||
"{'loss': 2.5917, 'learning_rate': 0.0006296296296296296, 'epoch': 1.3}\n",
|
||
"{'loss': 2.8582, 'learning_rate': 0.0005925925925925926, 'epoch': 1.4}\n",
|
||
"{'loss': 3.0774, 'learning_rate': 0.0005555555555555556, 'epoch': 1.5}\n",
|
||
"{'loss': 2.266, 'learning_rate': 0.0005185185185185185, 'epoch': 1.6}\n",
|
||
"{'loss': 2.4382, 'learning_rate': 0.00048148148148148144, 'epoch': 1.7}\n",
|
||
"{'loss': 2.6197, 'learning_rate': 0.0004444444444444444, 'epoch': 1.8}\n",
|
||
"{'loss': 2.3982, 'learning_rate': 0.0004074074074074074, 'epoch': 1.9}\n",
|
||
"{'loss': 2.5713, 'learning_rate': 0.00037037037037037035, 'epoch': 2.0}\n",
|
||
"{'eval_loss': 2.4057815074920654, 'eval_runtime': 1.6782, 'eval_samples_per_second': 6.555, 'eval_steps_per_second': 1.192, 'epoch': 2.0}\n",
|
||
"{'loss': 2.6508, 'learning_rate': 0.0003333333333333333, 'epoch': 2.1}\n",
|
||
"{'loss': 2.4209, 'learning_rate': 0.0002962962962962963, 'epoch': 2.2}\n",
|
||
"{'loss': 3.0109, 'learning_rate': 0.00025925925925925926, 'epoch': 2.3}\n",
|
||
"{'loss': 2.561, 'learning_rate': 0.0002222222222222222, 'epoch': 2.4}\n",
|
||
"{'loss': 2.6675, 'learning_rate': 0.00018518518518518518, 'epoch': 2.5}\n",
|
||
"{'loss': 2.4846, 'learning_rate': 0.00014814814814814815, 'epoch': 2.6}\n",
|
||
"{'loss': 2.7888, 'learning_rate': 0.0001111111111111111, 'epoch': 2.7}\n",
|
||
"{'loss': 2.327, 'learning_rate': 7.407407407407407e-05, 'epoch': 2.8}\n",
|
||
"{'loss': 2.3715, 'learning_rate': 3.7037037037037037e-05, 'epoch': 2.9}\n",
|
||
"{'loss': 2.2409, 'learning_rate': 0.0, 'epoch': 3.0}\n",
|
||
"{'eval_loss': 2.399984121322632, 'eval_runtime': 1.608, 'eval_samples_per_second': 6.841, 'eval_steps_per_second': 1.244, 'epoch': 3.0}\n",
|
||
"{'train_runtime': 15.0837, 'train_samples_per_second': 1.989, 'train_steps_per_second': 1.989, 'train_loss': 2.6115529775619506, 'epoch': 3.0}\n",
|
||
"Gemini to Q*\n",
|
||
"{'before': 11.756383895874023, 'after': 11.190644264221191}\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"data = []\n",
|
||
"for sample in samples:\n",
|
||
" r = learn_sample(sample)\n",
|
||
" print(sample['title'])\n",
|
||
" print(dict(before=r['before'], after=r['after']))\n",
|
||
" data.append(dict(**r, **sample))"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 18,
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"data": {
|
||
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAskAAAHmCAYAAACBJxOTAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAACmSElEQVR4nOzdeXwb9Z0//pcO6/AhW74d5z6chJAQJyGEkkChXC09gG4vevDbbUnbb6nb7bZ0oW1IgTZQ7wLbsrC0FAolC1sosIXtAYH0BEIJhJCEhMTO4Tu+L12WNL8/Rp/xyKeOkWYkvZ6PRx5gSbbGHo30nve8P++3SZIkCUREREREpDDrvQFEREREREbDIJmIiIiIaAIGyUREREREEzBIJiIiIiKagEEyEREREdEEDJKJiIiIiCZgkExERERENAGDZCIiIiKiCax6b8BMuruHZ7zfbDahtLQAfX2jCIc5E0UP3Af64z7QH/eBMXA/6I/7QH967IOKiqK0PE+6ZXQm2Ww2wWQywWw26b0pOYv7QH/cB/rjPjAG7gf9cR/oj/tAOxkdJBMRERERpQKDZCIiIiKiCRgkExERERFNwCCZiIiIiGgCBslERERERBMwSCYiIiIimoBBMhERERHRBAySiYiIiIgmYJBMRERERDQBg2QiIiIiogkYJBMRERERTcAgmYiIiIhoAgbJRERERDnsBz/Yjh/8YLvem2E4DJKJiIiIiCZgkExERERENIFV7w0gIiIiMoKhIeDo0fTmD5ctC8Pliv3xXV2duPPOO/D666/B7S7FBz7wIVx77efx0Y9+ENdd92V85CNXAgAkScJVV30AX/rS9bjssg/g2WefwWOP/RLt7W0oKCjARRddiq9//ZuwWCyp+cWyAINkMoyjrQN4cW8rPrBpAeZXFem9OURElEOGhoD16wsxOGhK6/MWF0vYu3ckpkBZkiR85zs3YOnSZXjooZ3o6elBY+MPYTabceGFF+NPf9qtBMkHDuzH4OAgtmy5AG++uRd3392IbdtuRV3dChw+fAi33roNGzacjQsuuCi1v2AGY7kFGcaTf2zCa++cxjN/Oa73phARERnO3r1/R2dnB2644TuYP38h1q3bgK985ev41a8ew/vedylef30PRkdHAQAvvfQizj33PcjPL4DTmY9//dfv4YILLkJNzRxceOHFWLZsOY4fb9b5NzI2ZpLJMNp75AP7WNsgJEmCyZTes3kiIspdLhewd++IocstTp48jqGhQVx22QXKbeFwGH6/H/PmzUNZWTlefvmv+PjHr8Yf//gSvvzlBgDAihUrYbfb8fOf34/jx5vQ1HQMra0t2LhxUyp+pazBIJkMYcQ7hlFfUPn/7kEfKkucOm8VERHlEpcLWL8+rPdmTCsUCmH+/IW4/fZ/n3RfQUEhLrroEuze/SLOOKMOAwP9eM97NgMA9ux5BTfe+E1cfvkHsGnTe/CP/7gV//7vt6d78zMOyy3IEDr7PFFfH28f0mlLiIiIjGnevAXo6upESYkbc+fOw9y589DR0Yaf//x+mEwmXHzxpdiz51X84Q9/wJYtF8DhcAAAnn32aVxxxYdxww3fwQc/eCUWLFiItrZWnX8b42OQTIbQNSFIbmaQTEREFGXjxk2orq7GLbd8D01Nx/DWW2/iRz/6IRwOBywWC5YtW47y8nI8+uijuPjiS5Xvc7mKceDAW2hqOobm5ib88IffR29vDwKBgI6/jfExSCZDmJhJbu4Y1GlLiIiIjMliseD22++EJIWxdeu1+M53bsCmTefh61//pvKYiy++FBaLBZs2vUe57Z/+6Ytwu0vxxS/+f/jnf/4KbDYbrrzyH3D06BE9fo2MYZIkSdJ7I6bT3T084/1WqxludwH6+0cRDBq3hiibabUP7n36bbx+pHv851rMuPcb58Nq4XncbHgc6I/7wBi4H/THfaA/PfZBRUV2tm1lBEKG0NnnBQAU5ecBAIKhMFq7R/TcJCIiIsphDJJJd2FJwul+udxi48oq5XbWJRMREZFeGCST7gaG/QhELgktrS1GRYm8GpdBMhEREemFQTLpTr1or7o0H4vnFAMAjncwSCYiIiJ9MEgm3anbv1W6nVhcI48e6uj1wOMb02uziIiIKIcxSCbdiUV7xYU2OO1WLJ4zPp/zeMfMHU6IiIiIUoFBMumuK7Jor9qdDwCYX1UIi9kEAGhmyQURERHpgEEy6U7UJFeVykFyntWCeZWFADiemoiIiPTBIJl0FQyF0TPgAyAv2hNEyUVz+yAMPO+GiIiIshSDZNJV94AX4UgQXFXqVG5fFFm8N+QZQ++QT5dtIyIiotzFIJl01RVZtAdMnUkG2C+ZiIiI0o9BMulK1CObTEBFyXgmuao0H/l2KwAGyURERJR+DJJJV6KzRUWxE1bL+MvRbDJhUU0RAHa4ICIiovRjkEy66prQ2UJtUWTy3qnOYQRD4bRuFxEREeU2Bsmkq/H2b85J94m65EAwjLbu0bRuFxEREeU2BsmkG18giIGRAIDoRXuCGE8NsOSCiIiI0otBMulG3dliqnILV4EN5cUOABwqQkREROnFIJl0IxbtAUCVe3K5BaAaKsJMMhEREaURg2TSjahHtlrMKHU5pnyMKLno6BmF1x9M27YRERFRbmOQTLrpUi3aM5tMUz5mUSSTLAE4wWwyERERpQmDZNJNZ6Qmudo9uR5ZWFBVBItZDqBZckFERETpwiCZdCFJ0ow9kgVbngVzKwoBcPIeERERpQ+DZNLFsHcMnkiN8VQ9ktVEyUVz+xAkSUr5thERERExSCZdiCwyMHWPZDWxeG9wNID+YX9Kt4uIiIgIYJBMOulUBckzlVsA423gAJZckDENjQbw6oEOjAU5Pp2IKFswSCZdnO6XF+3l260ocubN+Njqsnw47RYADJLJmO59+gB+8NBr+N2rJ/XeFCIi0giDZNJFp2rRnmma9m+C2WTCwmoOFSFjGguGcORUPwCgvWdU560hIiKtMEgmXYia5OpZFu0JouTiROcQQmFe0ibjONU1glBYXlDqDXDgDRFRtmCQTGkXliR0RcotZqtHFsTivcBYGG3dzNaRcaivbnh9DJKJiLIFg2RKu/4hv7LAabbOFoJ68d5xllyQgahfj95ASMctISIiLTFIprTr7Fd1tphh2p5acaEdZS47AC7eI2M53s5MMhFRNmKQTGnXFdX+LbaaZABYNKcYABfvkXGM+saU0iEAyoAcIiLKfAySKe1EZ4uSQhscNmvM3yfqktu7R+FlMEIGcKJjOOprrz/IqZBERFmCQTKlXVefnHmLtR5ZEHXJEoCTncMzP5goDSZe1QiFJQ4UISLKEgySKe26VD2S47GgqgjmSE9lllyQERyfoj6ei/eIiLIDg2RKq2AojO7BSPu3GBftCXabBbUVBQC4eI/0J0mScrJWXuxQbmcpEBFRdmCQTGnVPeCFKNmMt9wCGC+5YBs40lv/sB9DowEAwKpFpcrtDJKJiLIDg2RKq84EO1sIYvFe/7Af/cN+zbaLKF7qqxkMkomIsg+DZEorsWjPbDKhoiT+IHmRaqhIc/ugZttFFK/jnXKQ7LBZsCTSnhAAvH7WJBMRZQMGyZRWIpNcXuKA1RL/y29OWQHsNgsALt4jfYlFe4tqXMh3jLcyZCaZiCg7MEimtBKdLRKpRwYAs9mERdVFAKbuLECUDuGwhBORNoSLalxw2lVBcoBBMhFRNmCQTGklRlLH29lCTZRcHO8cRjjMwQ2Ufh19Hvgird4W1bhgNpvgtMtXOJhJJiLKDgySKW28/iAGR+RuANUJLNoTFtfI9Z/+QAjtvaOabBtRPNRXMRbVyFc2nPY8AAySiYiyBYNkSpvT/V7l/+MdJKK2OGrxHksuKP1EC8LiQhvcRXYAQIFTLrngwj0iouzAIJnSRt3+LdGaZABwF9mVwIRBMulBBMmLa1wwRaZA5jOTTESUVRgkU9qIRXs2qxklkSA3UaJfMoNkSrexYAgtp0cAyPXIguhwwYV7RETZgUEypY1YtFfpzoc5kn1LlCi5aOsZgT/Ay9uUPqdOjyAUWTCq7tud72AmmYgomzBIprQRmeREJu1NJIJkSQJOdDKbTOkTtWgv0o4QGM8k+1iTTEQ0ra6uLjQ0NGDjxo3YsmULduzYAb9/5gm6ra2tqK+vx549e5TbAoEA7rjjDpx//vk4++yz8ZWvfAWdnZ3K/X6/HzfddBM2bNiAzZs348EHH4x7WxkkU1pIkoTOyLS9ZOqRhQXVRRDJaA4VoXQS9chVpflK9hgYzyR7mEkmIpqSJEloaGiA1+vFzp07cdddd2H37t24++67Z/y+7du3w+PxRN324x//GLt27cK//du/4bHHHkMwGMT1118PSZKv9P3oRz/CgQMH8PDDD+Pmm2/GPffcg9///vdxbS+DZEqLYc+Ychk6mR7JgsNmRW15AQAOFaH0au6Qh4gsrimKur1AZJJZk0xENKXm5mbs27cPO3bswLJly7BhwwY0NDTgueeem/Z7fvOb32B0dHK716effhr//M//jI0bN2Lp0qW49dZb8fbbb+PkyZPweDx44okn8J3vfAerVq3CJZdcgi984QvYuXNnXNtrnf0h+jGbTTCbp69dtUTGGlsSGG9M2oh1Hwx7x7CkVu5vvKS2GFZr8vts05nVePPdHoQkaPLzMhWPg/Tx+YModOahsLYY65ZXKq87i8WM6vIC5TVutpiSrrun+PFY0B/3gf6MvA8qKirwwAMPoLy8POr2kZGRKR/f39+PxsZGPPjgg/jgBz+o3B4Oh9HY2Igzzjhj0vcMDw+jr68PwWAQ9fX1yu3r16/Hf/3XfyEcDsNsju1vY+ggubS0QGmvNBOXK/kaV0rObPvA7S7A3SurNX3Oaz94Jq7V9CdmNh4H6XH3N9475e0Xrp+HC9fPS+/G0JR4LOiP+0B/RtwHLpcLW7ZsUb4Oh8N49NFHsWnTpikff/vtt+Oqq67CsmXLom43m814z3veE3XbI488ArfbjeXLl2P37t1wu92w2WzK/eXl5fD7/RgYGEBpaWlM22voILmvb3TWTLLL5cTQkBehUDiNW0ZCrPvghddb8PLbnXDYLbjhU/UxnfzMprPPg/v/9yAA4BMXLcWKBe6kf2Ym4nGQPn95qx0vvdEGs9mEGz+9LiqT3Nw1gl88K78ev/6xs1BcaJvpR1EK5Mqx0HJ6BAXOPJQm2UozFXJlHxiZHvvA7S5I6PsaGxtx6NAhPPnkk5Pue/nll7F3794ZSzGEXbt24cEHH8T3v/992Gw2eL3eqAAZgPJ1IBCIefsMHSSHwxLCkVZLMwmFwggGeTDqabZ9cLC5D01tg1g8x4VQSAIw+36dTWmhHa2nR+AfC+GtYz1YGrnUnat4HKTe64dPo6ltEAsjXS3Uf28pLKGpbRAA0D/sU2qUKf2y+VhoahvED365F4XOPPz7V96DPKtF702aUjbvg0xh9H3Q2NiIhx9+GHfddRfq6uqi7vP5fNi2bRtuvvlmOByOGX/Orl278PWvfx2f+cxn8LGPfQwAYLfbJwXD4uvZfp5a3AUrJ0+exOc//3nU19fjve99Lx544IFpH3vo0CF87GMfw1lnnYWPfvSjOHDgQLxPR1lCaf+mwaI9wWw2KcEKh4pQqkmSpLzO1ENEhALneKcLtoGjVDnVJS8cHfGOoWfQp/PWECXm1ltvxUMPPYTGxkZcdtllk+7fv38/Wlpa0NDQgPr6eqW2+LrrrsO2bduUx/3f//0fvva1r+ETn/gEbrrpJuX2qqoq9Pf3IxgcX0jd3d0Nh8MBl2vy+/d04gqSw+Ewtm7dCrfbjaeffhrf//73cd999+HZZ5+d9FiPx4OtW7diw4YNeOqpp1BfX48vfvGLk1p4UPYLhyV09Yv2b9rWSIlhDsc7hmK66kCUqP5hPwZH5UzEVEGy0z6eOWYbOEqVYe+Y8v8DI7FfNiYyinvuuQePP/447rzzTlxxxRVTPmbNmjV4/vnn8cwzzyj/AOC2227D1772NQDAK6+8ghtuuAGf/vSn8b3vfS/q+1euXAmr1Yp9+/Ypt+3duxerV6+OedEeEGeQ3NPTg5UrV2L79u1YuHAhLrjgApx77rnYu3fvpMf+9re/hd1uxw033IAlS5bgO9/5DgoKCuLuUUeZr2/Ih2CkLqpKgx7JamI8tS8QQkcfT8AodY6r+nGrJ+0JUZlktoGjFBnxqILk4ZkHMBAZTVNTE+69915cd911WL9+Pbq7u5V/gJzt9fl8cDgcWLBgQdQ/QM4Ql5WVIRgM4qabbsLZZ5+N6667LurnBAIBOJ1OXHnlldi+fTv279+v1Cx/7nOfi2t74yqaq6ysVBo+S5KEN954A3//+99x8803T3rsW2+9hfXr1ysLtEwmE9atW4d9+/bh6quvjun52ALO+GLZB92qS4K1FYWatmurm1+i/P/JzmEsqC6a/sFZisdBepzsklsUOWwWzKssjHpvsljMsKkyyf6xUE63JdRLLhwLo77xE7AhT8Bwr7Nc2AdGZ+R98OKLLyIUCuG+++7DfffdF3XfkSNHsHnzZuzYsWPWOPHAgQNob29He3s7Nm/eHHXfI488gnPOOQc33ngjtm/fjmuvvRaFhYX46le/iksvvTSu7U14ZclFF12E9vZ2XHjhhVPWk3R3d2Pp0qVRt5WVleHo0aMxPwdbwGWOmfbBkK9L+f/li8ujLksnq6QkH6UuO/qG/Gjt9SS8wjYb8DhIrVOn5SC5br4bZWWFk+6XJAlms0ku+zFbcvq1qLdsPhZ8Y+P17t6xsGFfZ9m8DzKFEffB1q1bsXXr1mnvP3LkSEz3rV27dsbHAoDT6cQdd9yBO+64I/4NjUg4Wvnxj3+Mnp4ebN++HTt27MB3v/vdqPuna78RT+sNtoAzvlj2QXPrAADAXWSHz+OHz6PtJcJFNS70DXXjneO96O+fPJUn2/E4SL2wJOHdU/0AgHmVBZNeZ2IfOG0WjPqC6B3w5ORrUW+5cCz0D41fmevsGTHc6ywX9oHRZVILOKNLOEhevXo1AMDv9+Ob3/wmbrjhhqigeLr2G/G03mALuMwx0z7o6JHfxKvczpTsp4XVRdh7pBstXSMY9Y7BnmfMlkipxuMgddp7RuELyBm8BZVF0/6dnXYrRn1BeLxj3Bc6yuZjYdgz/rnaN+w37O+ZzfsgU3AfJC/uhXu7du2Kum3p0qUYGxubNFKwqqoKPT09k76/srIywU2lTNXVLy+oq9Z40Z4gFu+FJQknO4dT8hyU29SL9hZPsWhPEKVEXna3oBSJ6m7BhXtEKRVXkNza2orrr78eXV3jNaYHDhxAaWnppBF/Z511Ft58801IkpwJFgv9zjrrLA02mzLFWDCs9PLUurOFsLDGBVGUow5miLTSHHldFRfY4J5hylm+CJID7JNM2vOPhRAYG88MDoz4lc9YItJeXEHy6tWrsWrVKtx00004duwY/vSnP6GxsRFf+tKXAIy37gCAyy+/HENDQ/jBD36AY8eO4Qc/+AG8Xi/e//73a/9bkGF1D3gh3sNTFSQ77VbMKZfroThUhFLhRMf4EJGZFhM7HcwkU+qMqrLIABAMSVHdLohIW3EFyRaLBffeey+cTic+8YlP4Dvf+Q4++9nPKn3nNm/ejN/+9rcAgMLCQtx///3Yu3cvrr76arz11lv46U9/ivz81ARKZExdqt7FqSq3AMaHOzBIJq2NBcM4FWn/NlV/ZDWnjUEypc6wZ2zSbSy5IEqduBfuVVVV4Z577pnyvontONasWYOnn346sS2jrNAZqUc2m0woL4590Wa8Fs9x4a9vd6B3yIfB0QCKC2yzfxNRDFpOjyAUWUC8eIpJe2rMJFMqjXgnB8n9I37MrZzckpCIkme8TtOUVUQmuaLEAWsKG5urF1MdZzaZNKSuc19YM/OwGqdd7qzCIJlSYdg7uYUqM8lEqcMgmVKqs88LIHX1yEJtRQFskclTzR2DKX0uyi2ihKfK7USBI2/Gx3LhHqWSeiS1SDoMjDBIJkoVBsmUUiKTnMp6ZACwmM3KSGrWJZOWRCZ5tnpkYLwF3FgwjCAHKZDGRLmF025FaaTLysBI7AO6iCg+DJIpZbz+IAZH5TfwVGeSgfGSi+MdwwizLRJpwOMbQ2fkRG/RLPXIAKJGrrPkgrQmeiQXOfNQEgmS+1luQZQyDJIpZcQQEQCodqd+hvziOcUA5OBE3VWDKFEnVMNpZlu0BzBIptQS5RaF+XkoKZQXJ7Pcgih1GCRTynSqAtV0ZJIXqRZVseSCtCBKLSxmE+ZXzd5BID8qSGZdMmlLlFsUOvNQUijKLRgkE6UKg2RKma7Ioj1bnlm5NJhKZS4HXJHWb82cvEcaECdbcysLkWe1zPp4h338Mcwkk9ZEn+QiVZA8OBpAOMzyMqJUYJBMKSNKHqrc+TDPMKVMKyaTSbkkzkwyaeG4atJeLKIyyQEGyaStkUgLuML8PJQUyQkBSYKy9oOItMUgmVJGlFuko9RCEB0IWk+PIDDGy92UuP5hv9I5YNEs/ZEF1iRTqkiSFFVu4S4cvzrHkgui1GCQTCkhSZKycK+6NPWL9gTR4SIUlpRRwkSJUF+NiGXRHjAxSOZJGmnHFwghGJLLKorybVElbAySiVKDQTKlxJBnTAkSqtxpzCRXjwczrEumZJzolF8/dpsFNWUFMX2P1WJWhtowk0xaUo+kLnTmoaRAHSSz3IIoFRgkU0qoW7ClepCIWr7Dipoy+fma2zl5jxInMsmLqotgNsdeU+8QU/cYJJOGJgbJdptFuXLB0dREqcEgmVIi3e3f1Lh4j5IVliQlkxzroj3BydHUlALDqpHURfnyeHS3GCjCcguilGCQTCkhMskFDisKnXlpfW5Rl9wz6MOQh5chKX5dfR6lXCjuINkmt4FjJpm0JDpbAFDeUzlQhCi1GCRTSohMcjpLLQQxeQ8AjjObTAlQX4VIOJPMIJk0JKbtmQAUOESQHBkoMsxkAFEqMEimlOjqlweJpLvUAgBqKwqQF1k8xZILSoToj+wqsKHUFd8gHAbJlArDkZrkAmeeUiPPqXtEqcUgmTQXDks43Z/+HsmC1WLGgiq5r+1xdrigBIjXzeIaF0xxDsJx2kW5BWuSSTvqHsmCqEke8Y5hLBjWZbuIshmDZNJc75BP6eepR7kFMF6XfLxjCJLEka0Uu7FgGC2n5R7bsQ4RUXPamEkm7Ylyi8L88SBZ1CQDwCCzyUSaY5BMmlO3f6typ2+QiJqoIx31BdE96NNlGygztXaPKCd5YoJjPES5hY9jqUlDotyiyKkOktkrmSiVGCST5qLav6VxkIja3Irx4Q/tPaO6bANlJnUd+8LqxINkrz/EqxikmanKLUo4mpoopRgkk+a6+uRFe+4iO+yRdljpVlWaD3OklrSDQTLFQdQjV7qdCbUvFDXJYUlCYIx1oqSNkUg7S3W5RbGq3KKfA0WINMcgmTTX2a9f+zfBajGjqlQu9WAmmeKhXrSXCJFJBgAP65JJA2FJwohXfi0VOccDY6vFDFckaGYmmUh7DJJJc6ImWY/OFmpzyuSSi/ZeBskUG48viI5e+fUbb39kQR0kc/EeacHrDyIcKd2ZeHWDbeCIUodBMmlqLBhCb2ShXLVOi/aEmvJIkNzjYW0oxeRkp2qISAKL9oAJQTIX75EGRlQjqdXlFgBQUiSCZC7cI9Iag2TS1Ol+L0Q4qnsmuVx+fv9YCH1DzLLQ7JojpRYWswnzKwsT+hlOVR0+M8mkBdHZAojubgFwNDVRKjFIJk11RhbtAfrWJAPj5RYASy4oNsc7hgHIUxtteYktOlVnkn0cKEIamDGTHCm34MI9Iu0xSCZNdUUW7VnMJpQVO3TdlurSfIhhaVy8R7FIdtEewIV7pL1h73gpxaRMcqTcwhcI8coFkcYYJJOmRI/k8hInrBZ9X162PAsqStjhgmLTP+xXsnGJLtoDALvNAjHI2seghTQgeiSbTaaokzAgulfy4Cjrkom0xCCZNCU6W+i9aE9ghwuKlcgiA4kv2gPkQMYR6ZXMTDJpQT2S2iQuj0W41QNFWHJBpCkGyaQpo7R/E+YoHS5G2eGCZiSCZHueJaqePRHjo6lZk0zJm2oktSDKLQCgn4v3iDTFIJk04/EFMRTJeOi9aE8QHS68/hBbJNGMRJC8sLoIZrNplkfPTATJzCSTFpRM8hRBclF+njJdlB0uiLTFIJk0IxbtAcbLJAOsS6bphSVJ6WyRTKmF4LRFMskMkkkDoiZ5YmcLQC7vEeOpB4aZCCDSEoNk0owotQCMk0muKStQFlExSKbpdPV5lM4AySzaE0Qmmd0GSAszlVsAnLpHlCoMkkkzorOFLc+sNLjXmz3PorSi4+I9mk7Uor2aoqR/njOycM/LmmTSwIhHzhBPlUkGAHekLpk1yUTaYpBMmunqlweJVLvzJ63A1pN68R7RVI63y6UWrvw8lLmS7+/NTDJpJRQOw+OTX0eFzqmTD8rUPXa3INIUg2TSTKfBOlsI7HBBsxHjqBfVuDQ5wRM1yVy4R8ka9QUh3rVmL7cI8D2OSEMMkkkTkiQZrv2bINp5jaq6bxAJwVAYLae1W7QHjJdbcCw1JWumkdSCCJKDoTBGfTwxI9IKg2TSxOBoQOkJW11qjEEiAjtc0ExaTo8gGJKzb8mMo1ZzRMot/GMhhMJhTX4m5SbR2QKYugUcMF6TDHDxHpGWGCSTJjp7jdf+TagpG98eBsk0kXrR3kKNguR81ehgDhShZAyrMsnTl1uM1yqzLplIOwySSROdqvZvVW5jBclOuxWlLjnTwg4XNNHxdjlIrixxTpupi5fDNh4ke3n5m5Iw4h3vfTxtuQWn7hGlBINk0oQIkgudeZoFGloSdckdzCTTBMqiPY3qkQEgP1KTDLANHCVHlFtYLWbY8yxTPibfbkWeVf4452RRIu0wSCZNiHILowwRmYht4GgqXn9Qee1qMUREcDpUmWR2uKAkiHKLovy8aTuvmEwmuDlQhEhzDJJJE0r7N7exFu0JIkge8oxh2MNMC8lOdA4r7bW0WrQHjLeAAxgkU3KUkdSzXKFjr2Qi7TFIpqSFwsZt/yaIcgsA6FAtMqTcJhbtmU0mzK8q1OznOu0MkkkbMQfJRcwkE2mNQTIlrbvfg1BYzscZtdyippwdLmgysWhvbkUBbNPUeybCyZpk0oi63GIm6oEiRKQNBsmUtLbuEeX/jZpJLnDkoThyOZJBMgmpWLQHAHlWC6wWuX6UmWRKhuhuMXu5hRwkD44EEA5z6h6RFhgkU9LUQXKlQWuSgfGSC7aBIwDoH/ajP1K/qeWiPUG0gWOQTMmIvdxCTgKEJQlDXHdBpAkGyZS09m456Cx12adtUWQE7HBBaidUQ0S0XLQniIEiDJIpUcFQGN7IaPOifNuMjxXdLQDWJRNphUEyJU1kko02RGQiESQPjATg8Y3N8mjKdqLUwp5niRpdrhVHpC6ZQTIlKpaR1EKJOkgeZiaZSAsMkilp7ZEg2aiL9oQ56vHU7HCR80QmeUF1EczmqfvPJmM8k8yFe5SYEdVI6umm7QklzCQTaY5BMiUlMBZC94AXgHEX7QnqbCFLLnJbWJJwvGMYALCopiglz8GaZErWsCqTXDRLJtlusyitBxkkE2mDQTIl5XS/F1JkIXV1qXEX7QFyTZ9oo8QgObed7vfCEwleU7FoDxjvlewNMEimxMRTbgGMDxTp50ARIk0wSKakdPSNly0YPZMMqDpcMEjOaae6hpX/X5iiIJkL9yhZI6ouFbEFyeyVTKQlBsmUlM5Iba/FbEJ5sUPnrZmd0uGCbeByWmukI4vdZknZ63Z84R5rkikxotzCnmeJadjNeJDMTDKRFhgkU1I6++Rgo9LthMVs/JeTCJL7hvzM8OUw0ZGltrwAZpP2i/aA6EyyJHG4A8VPLNyLJYsMAG6OpibSlPGjGjI0kUk2emcLQd3hooMdLnJWWySTXJuC1m+CIxIkh8ISgqFwyp6HspcySGSWzhaCqEke9oxhLMjXHFGyGCRTUjojNcnVZRkSJLPDRc7zB8Y7ssytKEzZ8zjt45fHPSy5oASIcovZOlsI6jZwg6PMJhMli0EyJWzUN4bhyOXATMkkuwpsKHDIGT7WJeem9t5RiOKH2orUZZKdkRZwABfvUWKUcotYM8lF6l7JXLxHlCwGyZSw0/1e5f8zobMFAJhMJo6nznGtkXpkAKhNaSaZQTIlZ8QrB7ox1yRHTd1jJpkoWQySKWHikjUAVBp8JLUag+TcJuqRi/LzUFxgS9nzMEimZMVbblFcOP567ufiPaKkMUimhIkg2WoxoVR1mc/oRK/k3kEf/AHWiuaatp7UL9oDomuS2QaO4uUfCyEwJi++K8yP7WTOajErA5PY4YIoeQySKWHdAz4AQIU7H2ZzatpopYLIJEsYX3hIuUOUW6Sy1AJgJpmSMxrHSGo1pVfyMGuSiZLFIJkS1jMoZ5IzpR5ZYIeL3DXiHcNgZEFTKhftARMW7nE0NcVJLIoGYq9JBjhQhIyvq6sLDQ0N2LhxI7Zs2YIdO3bA75/59dra2or6+nrs2bNnyvu/+93v4ic/+UnUbS+88AKWL18e9a+hoSGubbXO/hCiqYlyi+qy1AYbWisptMFpt8DrD7HDRY5pUy3aS2X7NwAwm02w2yzwB0LMJFPcRlSZ5Fi7WwCAu0guzWCQTEYkSRIaGhrgcrmwc+dODA4O4qabboLZbMa3v/3tab9v+/bt8HimvvL7s5/9DE888QSuv/76qNuPHTuGCy+8ELfeeqtym90eX2kog2RKSCgcRt+Q/CacaZlkk8mEOWUFaGofYiY5x4hx1EDqa5IBwBkJkn2sSaY4DXvHyyUSKrdgkEwG1NzcjH379uFvf/sbysvLAQANDQ244447pg2Sf/Ob32B0dPJn9cjICG666Sa8+uqrqKmpmXR/U1MT6urqUFFRkfD2styCEtI/5EcoLHebzZRBImo17HCRk8SivTKXPapmOFXEc3iYSaY4jajKLQoSCJK9/hB8LPMhg6moqMADDzygBMjCyMjIlI/v7+9HY2Mjbrnllkn3tba2wu/346mnnsK8efMm3d/U1ISFCxcmtb2GziSbzaYZF4RZLOao/1L6eMdCWFJbDACYX1WUcfvgzEVl8ljqyMvLas2s7VfjcRAHE7CkthjL5hZrus+n2wfL5pUg35GHkiJ7Rr/GMkU2HQt2uxVLaotht1mUEeexmFdVpLw3+8fCKMxP798im/ZBpjLyPnC5XNiyZYvydTgcxqOPPopNmzZN+fjbb78dV111FZYtWzbpvhUrVuD++++f8vskScLx48fx17/+Fffffz9CoRAuv/xyNDQ0wGaLvfWnoYPk0tICmEyzd01wuZxp2BpSc7sLsO6MyZc3MsUV5y/BFecv0XszNMXjYHb/8ukNKf35E/fBtz57dkqfj6aWDcfC1RfV4eqL6uL+vk3uAmw6qzYFWxSfbNgHmS4T9kFjYyMOHTqEJ598ctJ9L7/8Mvbu3Yvnnnsu7p/b3t4Or9cLm82Gu+++G62trbjtttvg8/nw3e9+N+afY+ggua9vdNZMssvlxNCQF6FQOI1bRi+90Yq/vNUBu82M27+yBcPDvozaBwMjfvzHE/sBAFdfsASrF5fqvEWJ43EQm8GRAO5+4i0AwFVbFmPN0jLNfvZ0++BXLx3DOyf7MbeyAJ+/4gzNno+mlk3HwhO7m3DoRB9qKwrwhQ/G/toZ8QTw7/8jv86vPn8xVi/R7nUei2zaB5lKj33gdse/xqOxsREPP/ww7rrrLtTVRZ8Q+nw+bNu2DTfffDMcDkfcP7u2thZ79uxBcXExTCYTVq5ciXA4jG9961u48cYbYbFYZv8hMHiQHA5LCEfqXmcSCoURDPJgTKfDJ/rR1DaI+VWFMJlMGbcP8u1WtJ4egX8shMMn+rByfonem5S0TNsH6dbcPoimtkEAQKHTmpK/1cR9MDjiR1PbIHz+IPdNGmXDsXCsdQBNbYMocMT3WrVZLTjePoSwJOFk5zBWLnCncCunlw37INMZeR/ceuuteOyxx9DY2IjLLrts0v379+9HS0vLpJZt1113Ha688sopa5QnKikpifp6yZIl8Pv9GBwcRGlpbIkxQwfJZFyi/VtFifEv50zFbDKhpiwfJzqH2QYuR4hx1GLfpwMX7lGiRJ/keDpbAPJanuJCG/qH/exwQYZ0zz334PHHH8edd96Jyy+/fMrHrFmzBs8//3zUbZdeeiluu+02nHfeebM+x1/+8hd885vfxB//+Ec4nXKc8s4776CkpCTmABlgkEwJ6h6Up+1VujMzSAbkoSInOofZ4SJHiB7JVaVO5Flju9SWLBEks8sAxWsk0gIunh7JQkmhnUEyGVJTUxPuvfdebN26FevXr0d3d7dyX0VFBbq7u1FUVASHw4EFCxZM+v6qqiqUlc1eQlRfXw+73Y7vfve7+MpXvoKWlhb86Ec/whe+8IW4tjfupY/xTEp54YUX8P73vx/19fX41Kc+hYMHD8b7dGRA/kAIQ6PyG3imZpKB8cl7p/u9CLJ2LuuJHsnp6I8sKEGyP4SwNHvpGBEgr8wXw0TimbYnlBTKq/f7hxkkk7G8+OKLCIVCuO+++7B58+aofwCwefNm/Pa3v036eQoLC/Hzn/8cfX19+OhHP4rvfOc7+MQnPhF3kBxXJjmeSSlHjx7Fv/zLv+CWW27BunXr8Itf/AJf/OIX8cILLyipb8pMYhw1kOFBcmRSYCgsoavfm9bgidIrHJaUspraFE/aU3Pa5Iy1BPnkMh29mSnz+QIhBEPySVVRfuztqoSSIg4UIWPaunUrtm7dOu39R44cSei+X/7yl5NuW7ZsGR566KH4NnCCuDLJYlLKjh07sGzZMmzYsAENDQ1Ttuf429/+hqVLl+LKK6/E/Pnz8Y1vfAPd3d04duxYUhtM+use8Cn/n9FBcvl4XWoHSy6yWveAF2ORBSxzK9KfSQbA0dQUs6iR1AllkkWQHIDEKxhECYsrSI5nUkpJSQmOHTuGvXv3IhwO46mnnkJhYSHmz5+f3BaT7rpVmeTykvhbsxhFebETeZEBD20MkrNaa/f4e1RaM8kMkikByQbJ7kiQPBYMc9EoURLiuvYXz6SUD3zgA3jppZdwzTXXwGKxwGw24/7770dxcXHMz8eJe8bUOyRnkt1Fdjjt8ht4pu6DmrJ8nOoaQUefJ2MnovE4mF1HrwcAkGc1Y055wYzvK4mYbh+oF10FguGMfY1limw5FtSBrdsV/7TGsuLx5MWwdwzFkaA5HbJlH2Qy7gPtJFUgN9OklP7+fnR3d2Pbtm0466yz8Nhjj+HGG2/E008/HdPKRIAT94xqcFTOctSUFyh/+0zdB4vmlOBU1wi6+jwJNUM3kkzdB+nQFenGMr+6CGVlqcskT9wHVYHxBaEWmzXjX2OZItOPhbCpT/n/uTUlcBXEV5e8oDak/P+YZNLldZfp+yAbcB8kL+EgeaZJKQDwb//2b6irq8OnP/1pAHLj6Pe///349a9/PWPRthon7hlT2+lhAIC70IahIW9G74OKYjnD0np6BD29w7CYM+/Mm8fB7I5HhohUu/PR3699ac10+yDoH79sfrpnBP396Sv1yEXZcix09cjvsSYTMOYLoD8wNst3RDNL4797a8cgFlWmL0jOln2QyTJl4l4mSChInm1SCgAcPHgQn/3sZ5WvzWYzVqxYgfb29pifhxP3jEeSJGXhXpnLoRyAmboPqtzy4r1QWEJ79yhqyjL3QM/UfZBqY8EwOiPlFnPK81P6N5q4D/JUlztHvWPcP2mS6cfC4IjcYrPAkRfz56Ca3WqG1WJGMBRG76BPl79Fpu+DbMB9kLy402bqSSlXXHHFtI+rrKxEU1NT1G3Hjx/H3Llz499KMoxhzxj8Y/KlvEzubCGoO1y093h03BJKlY7eUaVHcW15ejO5tjwzzJGSMa8/NMujiWRi4V5RAoNEAMBkMsFdJJdosA2crKvfg58+exCHTvTN/mCiiLiCZDEp5brrrlMmpYh/ANDd3Q2fT84yfvzjH8evfvUrPPPMMzh58iT+7d/+De3t7bjqqqu0/y0obaI6WxRnbmcLodLthNUiBzEcT52d1J1L0tn+DZCDFadd7pXMLgMUqxFP4oNEBNEGjgNFZL995SRePdiFX+1mG1qKXVzlFupJKffdd1/UfUeOHMHmzZuxY8cOXH311fjABz6A0dFR3H///ejs7MTKlSvx8MMPx7xoj4ypeyA7BokIFrMZ1aX5aO0eZa/kLCXavzntVriL0rfKX3DarRj1BeFjkEwxGk5i2p6g7pVM4yfLvYO+WR5JNC6uIDneSSkf+9jH8LGPfSyxLSND6onUI1stJmWqU6abU16A1u5RtDNIzkptYhx1RWzdcrQmeiWzTzLFKtlyC0AdJDOTLEmSsi5h1BfEWDCEPKtF562iTJB5S/lJVyKTXFbsVGotM50YT93R54l7gQwZnwiS56ZxiIiaGE3tDbAmmWIz4pGzv4XO+EdSC+KqyeBIQKnJz1XD3rGocidm1ylWDJIpLj2RS1UVWVCPLMwpl4PksWAYPaqaa8p8Xn9QGX5TW65P5xJmkikeYUnCiFd+rSRXbmFTft7waG4HhSKLLDC7TrFikExxEZnkbKhHFmpUwRM7XGQXPRftCQySKR5ef1DJ/GpRbgEwc9rZF/2+Ppjjfw+KHYNkilkwFEbfkHwGXl6SPZnkKrcTFjM7XGSjtsiiPQCo1avcgkEyxUF0tgCSzCSr1oz053jmdGKQnOt/D4odg2SKWd+wX8lwVBRnTybZajGj0i3/Ply8l11EPXJxoS2pgCMZDjtrkil2orMFABQmlUker2fO9fKCLmaSKUEMkilmPVnW/k1N1CUzSM4uov3bXJ3qkQEgn5lkioM6k1yUxImdw2ZVenQP5Hiv5ImZ5Fw/aaDYMUimmEX3SM6ecgtA1eGi15PzK8GziahJ1qvUApCDFUBeGBoMcUQszWzYO57lTKa7BcA2cAAQCodxuj96QfZgDv89KD4MkilmorNFvt2KfIc+l65TRWSS/WMh9A2x2Xw2GBoNYDiSldOrswUwnkkGmE2m2YkeyRbz+LTGRHGgiNzbPxRp7VngkI/FXP57UHwYJFPMsrGzhTCHHS6yjnrR3txKHTPJqkCHQTLNRj2SOtnhN0qQnMPlFh2qUou6eSUAcjuzTvFhkEwx645M28umzhZCdakT4vOIdcnZobV7fD+Kcho9RGeSuXiPZqaMpE5i0Z5QUiSXa+RyUCgW7ZlM40GymLpHNBsGyRQzMWgjGzPJeVYLKiO/F9vAZYe2HjmTXFHigN2m3whaJ8stKA4ik5zMoj1BZJKHPGM5Ww8vFu2VFztQrurKxJILigWDZIqJLxBU6juzadqemii56GAmOSvoPY5acKiD5ACDZJqZqEnWomWhWzVQJFfbnolpe9WlBVFt8XL170HxYZBMMekZGF/Mlo2ZZEDVBq53FBI7XGS0sCShVelsoV+pBcCFexSf8XKL5DpbABOn7uVmyYXIJFeX5vPvQXFjkEwxUbd/K8/WIDlSt+r1h3gpLsP1DfrgjwzvqC3XOZNsUy/cYx0kzWzEI7/3aJFJFjXJQG4GhV5/EIOj8t+zutSJYg5YoTgxSKaYdEfav5kAlLmyu9wCGK9npczUqiqZ0TuTbLWYYbPKb7XMJNNMQuEwPD75NaJFTXJxgWo0dQ52uFAPEakuzYfVYlZOPpgIoVgwSKaYiExySZEdedbsfNlUl+VDNFxiG7jMJtq/WcwmVJfm67w143XJrEmmmYz6ghCFXlp0t8iz5nZQGBUkR64UipILDhShWGRntEOa68niHsmCPc+CssiiRLaBy2xi0V51mZw90ptTGU3NcguanlYjqdVyeeqeWLRnz7Moi/bEf3Px70Hx0//TgzKCKLfI1s4WgnrxHiWvZ9CrjIZOJ9EjWc9Je2rOSF2yj+UWNAPR2QLQJpMMAO6iHA6SI5nkqlKnMphFOWkYzb3MOsWPQTLNSpKknMgkA9Ft4NjhIjkeXxDbfv4abv75a2g5nb4a72AojM4+Y7R/E0Qm2cMgmWYwrMoka7FwD1BnTnMvKOxSdbYQxOK9XJ5CSLFjkEyzGhoNIBCUG9Fn47Q9NdHhYtQXxBAzDUlp7R6BLxBCWJLw6sHOtD3v6X4vgiH5BEfvRXvCeLkFg2Sa3oh3/D2nyJl8CzhgPHOaawv3wpKEzv7JQbL4e3DqHsWCQTLNSpRaALmTSQZYl5wsddvAN97tTltmvrV7PGtda5hMslxuwZpkmokot8izmmHL0+bjuSRSbuH1B5W2iLlgYNiPwJic3KkuUwfJHChCsWOQTLOK6pFcnN1Bco3qzbS9lx0ukqF+3XT1e9P29xSL9ux5FpQbpIaemWSKhSi3KHTmKTW0yVJP3RsYzZ1scoeqs0VN6XjyI3qgCINkmhmDZJqVqEe2WsxRzdizkdNuRZlLfhNlJjk53aopjYCcTU4HsVBwTnkBzBoFGsly2uQg2ccWcDQDkUnWqrMFMGGgSA6VXHSqTsor3ePJHU7do3gwSKZZiWCnosRhmKAjlWpEhwsGyUnpGfRGfZ2uIFmUWxilHhmIbgHHBaE0nRFlJLWGQbIqKOzPoaBQLNorKbQpxx8ATt2juDBIplmJYCfbSy0EsXiPbeCSI8otxFjmk53D6BvyzfQtSfOPhdDdLz/vXIO0fwPGa5LDkqTUSRJNpC630Ior36YkNwaGc6e8oHOKzhYAoqbuDXJxNs2CQTLNqltp/2aM+s5UE4v3hj1jGPLwTTQRgbGQUu93/llzlNtTnU3u6B1VJpYZZdEegKhMFtvA0XREdwutOlsAgNlsGm97lkOZUyVILpt8slzCNnAUIwbJNKNgKIy+yBtJzmSSVRnIDpZcJKRH1RFlxQI35lfKAWuqg2SxaA8A5hqw3AJgXTJNLxXlFkDuTZkLjIXQG3kPmmosPQeKUKwYJNOMeod8ECWU2d7+TZjDDhdJU9cjV5Q4sa6uAgDwbstg1FQxrYl65EJnHlwFxllkykwyzSYYCistArUstwBUQWGOZE5P93uVK0rVpZM/t3Ixs06JYZBMM+oZUPdIzo1yi3xHnpJ54eK9xKg7W5QXO1AfCZLDkoS3jvWk7HnbVOOotWqhpQUxlhoAfOyVTFNQnzwWaZ5JFqOpcyNz2qlq/zZjJjlHThoocQySaUbdg7nTI1ltDjtcJEXUsRcX2GDPs2BuRYFykpXKkgvR/s0o46gFdSaZvZJpKiMpGEktqMstcqG7igiSrRbTlJ9b0VP3uJCWpscgmWYkgp0ChxX5Dussj84e7HCRnPHFnvIHlMlkUkouDh7vg39M+2zqqG9MGb1rpPZvAINkmt2wN4VBcmTqXiAYzonXnwiSK935MJsnX1GKnrrHbDJNj0EyzahH6ZGcO1lkYDyTPDgSwKgvdTW02UrdW1uoXyYHyYFgGAea+zR/TvWiPaMFyXabBeKjOheCFIpfdLmFtvX07qheydlfcjFd+zchaqAIF+/RDBgk04xERrA8R4NkAOjo4eK9eEiSpJTpqE+ultYWwxWptUxFyUVbZNEeINckG4nZZIIjkk3mwj2ayoiq3WShU9urdlFBYZbX4UqSpEzbq5pi0R4wYaBIlv89KDkMkmlGopVXrizaE9RBMksu4jPsHYM/IJdTqOsBzWYT1kayyfubehAMaVsL2BqpRy512ZHv0PZytRbEQBFfgAv3aDJRbmG3WZBntczy6PiIcgsg+zs6DHvHlBPR6TLJxQXjfw8OFKGZMEimaXn9QeUSYEUOLdoDIi3EIllPLt6Lz0wdUdbVlQOQF8y82zKg6fO2nY6Moy431qI9wclMMs1ALNwr0rgeGZDXlFgt8sd9tgfJnaq2nTWlU19RyrOOT93L9r8HJYdBMk1LlFoAuVeTDLDDRaJmet2sXFCqjKnWsuRCkiSls4XR6pEFp00Okn0MkmkKyiCRFATJJpNJNWUuuzOnUe3fyqbOJAO5N2CFEsMgmaYV1es2x8otAKAmEiS3GThIDksSdr7wLr59z18wZJDLhiJItlpMUZd5ATmDs2ZJGQDgzaM9mrWjGhgJYNQnB59GmrSnJjLJXLhHUxlO0bQ9QRyL/VkeFIogudCZN+MJR3GO9Y6mxDBIpmmJqWkmE1Dmyr0gWbSB6x/2Gzawef61FvxhzykcOt6Hv+7v0HtzAIwHyWXFTpinGOghulz0D/txonNYk+eMXrRn1HILOYPuZU0yTSGV5RaAeqBIlgfJsyzaE5hJplgwSKZpiWCntMiu1LPlEqMv3jvWNogn/9ikfN1hkG2cbbHnmiVlsFrk4FmrkovWSPs3kwmomeESq56YSaaZjHjljGahMzXj1N05EiR39c/c/k0QJw2DzCTTDHIv8qGYjQc7uVePDEwIkg1WcjHiHcN//e8BhFXlCuoFK3qaOEhkIqfdipULSgFoFyS39ciZ5Ep3Pmx52nYG0IqoSWaQTFNJfbmFHHwPjgSi3jeySSgcxul++f0n1iB5xDvGqXs0LQbJNC2lR3KOdbYQXPl5KIhMGTRSr2RJkvDz5w6hbyh6upx6wYpegqEweociJ1czvG7qI10uOno9mmTAxSARo9YjA6pyCz/LLSiafyyEwJgcqKW63CIUljDsyc4BST0DPoTC8glA9TSdLYSoqXuj2Z1dp8QxSKYphSUpZ3skCyaTabzDhUFKGQDgD6+14K2mXgDAe+trcenZ8wDI/T49Pn2zlH1DPogk1Uyvm/plFcoEumSzyeGwpGT6jTZERE0ME/GPhRAOZ2cmjxIzmsKR1EIuDBTpiLGzBTC+cA/g4j2aHoNkmtLgSEC5BJVr0/bUjNYGrqltEL/+k1yHPK+yEJ9639Koy4qiHk8v3YPqHsnTv26KC2xYMrcYgNzlIrnn9CIQea3OrTDmoj0AyLePT1HzBlhyQePUmd2iFJVbuHNgoIgoOTOZgMpZPreiMslZ+veg5DFIpimJzhZA7tYkA+MdLnoGfcoUOb2IOuRQWILdZsGXrzwTeVYLasrGs6d61yWreyTPVqazLtLlorl9CP1JZLZEqQVg3B7JwPjCPQDw6pzxJ2MZSUMmubhANYo5S4NCkSQoL3YgzzpzeKOeusdMMk2HQTJNKWogRHFullsA0Yv3Ovr0yyZLkoQH/+8d9EbqkK+9fLmSQS4utCn1rnrXJYvXTaEzD/kO64yPFdP3AODNo4mXXLRG2r9ZLWZUuo17Que0jS8oZBs4Uhv2jgdphfmp6W7htFuVQT7ZGhSKJMFs9cgAp+5RbBgk05TEaGGb1QxXQWretDOBUTpcvPD3Fuw7JpclXLB2DjadUa3cZzKZUBspM9A/SJZfN+UxnFhVuvOVhXZvJlGXLDLJc8ryYTEb9y3NqTppYIcLUhtRlVsUzHJymQxRl5zMlRsjE+9/s3W2EIrZK5lmYdxPFNKV0tmixAnTFAMhckVJoU25TN6uU4eL5vYhPBHphzy3ohCfet+ySY+ZY5AguWeW9m8TicEih08NYNSX2Ip7o4+jFkQLOIBBMkUT5Rb5dmtKe9KLuuRsDAq9/iAGI1NHZ1u0J5Rw6h7NgkEyTUkswIolI5jN5A4X8huuHpnkUd8Y7nsmUoecZ8GXr1w1ZR9gsWCtq9+jaw/U2XokT7SuTg6SQ2EJ+4/1xv18Y8EwuiInBkZetAdMqElmkEwqqe6RLKR6ytxYMIz/e/kE3jnel5KfPxN1gqA6xrIr8ffgwj2aDoNkmlK8wU42E4v30t0GbrwOWT5h+dzly6MW6amJTHJgLKxbeyePbwyjkQVp5TG2DZxfVaiMPH8jgbrkzj6P0hfV8JlkO2uSaWqpHkktpDpz+sxfm/E/Lx3DbQ/tQSic3gEdUUHyNO+TEzGTTLNhkEyTjAXHA61cXrQniLrk7gEvAmPpC252vd6qtEc7/6w5OHdV9bSPra0cz6LqVXIh6pGB2E+uTCaTMljk7ebeuP++bZFFewBQW27sTHKe1aKM42YmmdREuUWqOlsIIigcHg0gGNI2iPUHQvjTm+0AgKHRAE50DGv682cjFu3ZbZao9m4z4dQ9mg2DZJqkb8gHccGemeTxIFmS0heAHu8Ywq92HwMgT5G75uLJdchqtRX6B8mJtg1cHym5CIyFcfBEfJdpRT2y025Bqcs+y6P15+BoapqC6JOc8nKLSE2yBDmQ1dLLBzvhUb2uD5/s1/Tnz0ZZtOfOj3kdjbotHqfu0VQYJNMkUb1uGSQr5RZAekouPJPqkM+csg5ZzWm3Koty9M4km00mlBbFHrAunVusZNDefDe+wSKis0VteWFGLDAVA0UYJJPaSKQFXJEztZ2E3Kopc/0a1uFKkoRdr7dE3faOTkFyVWnsn1klqvepQZZc0BQYJNMk6qlpub5wDwBKXXbYI/1FH991FC/ubdX8UqUgSRIe/O1hZST45y6bvg55ItH2SL8gWT65KnXZ41qhbzGbsXapXHKx71hPXLWMokey0euRBUekLtnrZ00yySRJGi+3SNPCPQAYGNYuKDx0oh8dveODPADg3ZaBlL1PThSWJGUBb6zt3wCgJAcGrFByGCTTJOqBEOoV+bnKZDJhy+oaAMCQZww7X3gX3/3ZHrz2TpfmnSRe3NuKNyI9g7esqcG5Z05fhzyRaHvUpXOQnEiJjuhyMeIdw9GWwZi+x+sPKicTteWZESQzk0wT+QIhBEPy+0iqa5KLC1MzmvqFSBa50JmHT11SB0D+vU52pqcueWDYr4ymj7X9GzDx78FMMk3GIJkmibfXbS645pI6/Msn1mJ+lVz7e3rAi//634O47eHX8U6cdbTTOd4xhP95Sa5Drq0owDWRD5tY1UQ+HHoGfbosQhFXICpi7GyhdsZCN+yRkpJYu1yoS1+M3v5NcDJIzhivHz6NvUcSH3ITK/VI6lR3t0jFlLmuPg/2N8ntG99bPwdnLiqF2SyXPh0+lZ6Siw5VYqAmhml7Qp7VrAxvYSaZpsIgmSYRtaWJBDvZbNWiUmz7/87G1g+foVxSPNE5jMbH9+HO/9mHU12JZ03Udci2PDO+/JEzlaAxVmIUqyQBp/vTm00OhyX0DiZ+cmXLs+DMxaUA5Ol7UgwZelGPDABzMqXcQizcCzBINrK9R07j3mcO4N6n31YWh6aKOkhOdbkFoGp7plGryF17WwEAFrMJF9bPhdNuxdK5xQDkIUHpIDpbAPHVJAPjdckMkmkqDJJpkp4kgp1sZzaZsOmMavxw6yZcc/EyJStz4Hgfvv/Q3/GzZw8qmfhYSZKEh343Xof82UuXR43DjlWN6jJjZ19825CsgRG/csm4vDix140ouegd8uNU18gsjx6vR3YV2ODKz4zR6Sy3ML7AWEi5oiMBONo6kNLnG1aNpE51uQUAlBRpN1DE4wvir293AADWL69QFg+vXiKvMTjamp66ZLEOo6TQppyIxkqcNHDhHk2FQTJFiRoIwUV707JazLh4wzzc8aVz8aH3LIQtzwwJwCsHu3DTz17FY7uOYtgT25vuS2+0KZd1N6+uwXmR+ud4lZc4YIlc5uzsS+/gE3VHlERPrtYsKVO2X9Rlz2S8s0VmZJEBLtzLBH/4e4tywgog5f1+RWcLAChKw8melgM0/vZ2B/yRwTiXbJin3L46shA3MBZOS7/kzgQW7Qli8R4zyTQVBskUJZGBELnMabfiqvMX444vnosL62thNpkQDEl44fUW/Ov9r+C5l08oHyJTOdE5hP956SgAOdj79KXx1SGrWcxmVEbGsXalOZMc/bpJ7OSqwJGHFfNLAMRWlywug2dKPTIQnUmOpaSE0qtvyIf/e+VE1G2pXnwmpu2ZTOOvj1QSQXJ/kuUW4bCEFyOlFotqXFhSW6zcd8aiMphN6atLVjpbxNgJSG283IKZZJqMQTJFYY/kxBQX2vHZy5bjtuvOwYblctmA1x/CU39uxr/+9BX8cV/bpNZmHl8Q9z1zAMGQXIf8pSvjr0OeqMqtTxs48bpx2CxJXTIWJRdt3aPomqGuesgTUIYhZEr7NwBwRIKgUFhKW3ssit2Tf2pCYCwMkwnYsKISgFzWk8qFsMORmuQCR56y4C2VREmExx+EP4kJovubenE6ctxfsmFu1H1OuxWL5rgApD5IDoyF0BvJ/CeSSRYDRUa8YzwmaRIGyRRFXGaMdyAEyapL8/H/rlqN73xuPZbPKwEg17o98vsj+N4Dr2HvEXlRmiRJ+MXvDysZ2M9eulyTsgHR/ijtQXKkjr282JnUUI+1yyqU/59psIh60V4mBclO+/hJkIclF4ZyrHUQrx7sAgBccNYcnL9GLnsKhSWl/j0VxMK9ojQs2gOieyUPJlFiINq+FRfalBMKtTMWuAHIf9dUBp+n+73KhNiEyi0KOVAk3bq6utDQ0ICNGzdiy5Yt2LFjB/z+mV+Lra2tqK+vx549e6a8/7vf/S5+8pOfRN3m9/tx0003YcOGDdi8eTMefPDBuLeVQTJFSXQgBEVbMqcYN1xTj69/bA3mRoK4zj4P/vPpt/HDX+7F/7x0DK8fPg0AOG91dcJ1yBOJD4kR71jUqvlU69GoI4q7yI7FkQzUTHXJbaqgZU4Cl1j14lQtKvJx8Z5hhCUJO3e9C0Auebjy/MVYUF2k3J/KkgtRbpGORXtAdFCYaIlBa/eIMlHvwvraKT8rVkSC5EAwjOb2oYSeJxbqhEB1nJ0tgIl/D9Ylp5okSWhoaIDX68XOnTtx1113Yffu3bj77rtn/L7t27fD45k6+fOzn/0MTzzxxKTbf/SjH+HAgQN4+OGHcfPNN+Oee+7B73//+7i2l5MiKEo3O1toxmQyYc2Scpy5qAyvHOzEM39pRu+QH03tQ2iKfGjMKS/AZy5ZrtlzqjMpXX0eFKrqBFMpmUEiE62rq0Bz+xCa2gYxOOKPavgvtEYyyeXFjowaeKPeVg+DZMP429sdSiD8kc2LlG4pZS47eof8OJHCIFmUW+gRJCdalyxqka0WE967tnbKx9TNK4HFbEIoLOHwqX7URa6saU30SLZaTAl11omaQsggOeWam5uxb98+/O1vf0N5ubzAs6GhAXfccQe+/e1vT/k9v/nNbzA6Onkx+sjICG666Sa8+uqrqKmJTjR5PB488cQT+NnPfoZVq1Zh1apVOHr0KHbu3InLL7885u1lqpCiiMv/7GyhHbPZhPNW1+CHWzfh4xcuVZrX26xmfPkjq5SR11pQB8npKrnwj4UwGKkP1iJIrl8mv3FKAN48OnXJRVuPnEnOpEV7QHSQzEyyMXj9Qfz6T80A5DaKF64bD/oWVstXNU50pi4Tmu5yi+ICG0RFVCJB4Yh3DK8c6AQAnHNGFVwFU3fksNssWFQj//2OpLBfsli0V+nOT6imuzgqSGa5RapVVFTggQceUAJkYWRk6pKm/v5+NDY24pZbbpl0X2trK/x+P5566inMmzcv6r7Dhw8jGAyivr5euW39+vV46623EA7HXv5j6BSM2Wya8UVviVzisbAsQBNhSUJxoQ2FzjysWOCG1Tr735X7IHZWqxkfPG8hLt4wFwdO9KG2rBA15fHX0E2k3gdulx1nLCqFPxBCMCzFtA+T1TfsU1a2L57jSvo551UV4ZwzqtAz6ENnv2fSz5MkCbY8C5bUFuOsZeVp+R1nE+tx4HbZlb+V1WoxxLZnk0Tej155swsVJU5UlDjx6UvrlMWVALBueQUGRgPK51Aq9pe4GrKwpjhtr4c1S8sx4hlDXp457uc8cKIP86rkUpQPb1486fvV+2DzWTWQAIQjRcOp+P2sVjOW1BbH/Jk11fevWlwKnz8EqzX+v4cRGflz2eVyYcuWLcrX4XAYjz76KDZt2jTl42+//XZcddVVWLZs2aT7VqxYgfvvv3/K7+vu7obb7YbNNn4SVF5eDr/fj4GBAZSWlsa0vSbJwH2IJElKahEQERERERnTHXfcgZ07d+LJJ59EXV10C9SXX34Z27Ztw3PPPQeHw4Hly5fjkUcewTnnnDPp53z2s5/Fxo0b8dWvfhUA8Mwzz+A//uM/sHv3buUxLS0tuPjii/GnP/0J1dXVMW2foTPJfX2js2aSXS4nhoa8CLF1S9JOdg7jF787DAD4pw+uxLwYLmVzH+hv4j546s9NeLupDxVuJ/7flWem/PlfPdSFP+w5BQD4zmfXa5KJaesZxQPPHgIAXH3BYqxeXKbc927LAB7bJfeW/tJHVqEqgRXtWov1OAiHJdz68OsAgMvOmY9NZ1SlaxNzQrzvR//9wrs42joIi8WEL195Jspc0WVmo74x/Ntj+wAAHzh3Ac6eootDMrz+IH70328CkGuh1y4rn+U7tPHYrqN4t2UA86uK8I8fWBHz9x083ocn/9gEAPjYe5fgjEWTs3HqfeD1B3HHzjcQDku4YG0t3ls/R7PfAZBLP/798X0AgA9vXoh6VXecePzyD0fQ3D6EJbUufOZS7daI6EWPz2W3O/4F1I2NjXj44Ydx1113TQqQfT4ftm3bhptvvhkOR/zln3a7HYFAdPmM+Dqen2foIDkclhAOz57oDoXCCKawj2WuONEhL5YCgAKbNa6/KfeB/sQ+MElAU9sgTnYOI3BFSGnqnyrvnupHU9ug0n9Vi9dBZbEDfUM+9A/78cc32rByvlu57/BJ+fksZhOKC2yGet3Fchy0do/AHwihvXsEwWBiH+p6CYbCeP3IaSyZU2zoxb2x7Ie3m3vx+8jJ3fs3zUdx/uTXkt1qwcCwH71DPrx1tBv1S7UNYvsGfcp7bjrfQ73+IJraBjHsCcT1nL/+UxOa2gZR5rJjSa1rxu8NhcLyoidJQlPbIPIsJmxeHVv2LlanOoeVv58zL77PLDWPdwxNbYNymZqB3k+SZeTP5VtvvRWPPfYYGhsbcdlll026f//+/WhpaUFDQ0PU7ddddx2uvPLKKWuU1aqqqtDf349gMAirVQ51u7u74XA44HK5Yt5OQwfJlF6iQ4Etz5y2RSSkPZFZDYbC6Bv0pXwoTE8KFnuaTCasW1aBF99oxf7mXowFQ8izygscRfu3qtL8jGxT6LRZ4A+E4MvAPskvvN6CJ3Y3YW5FAW75/ORLnpkiGAorVyOKC2z44LkLp33swuoi9A75UtLhYljVprEwje+57sLxUcyxljWe6BzCsVY5IL1o/VxYzLEdeyvmu3G0dRDH2oaijmMtRLV/K0v8ilKxMqqb3S3S4Z577sHjjz+OO++8c9pOE2vWrMHzzz8fddull16K2267Deedd96sz7Fy5UpYrVbs27cPGzZsAADs3bsXq1evhjnG1y7A7hak0q30uk1uIATpK6rDxQxT67SSqraB9XVy1s4fCOHQifGpXaL929wMGiKiJjpcZGILuLci3UZau0cx5MncTgAv7W1VAqyPXrBkxjaCC2vkRWpt3aMYC2p7YiN6JANAUZpawAHjbeACY2F4YzxZ2/W63PbNlmfG+WfFXjYh+iUHQ2E0tWnbJUTsw0JnXlIt9EQbOE7dS72mpibce++9uO6667B+/Xp0d3cr/wA52+vz+eBwOLBgwYKof4CcIS4rK5vpKQAATqcTV155JbZv3479+/dj165dePDBB/G5z30uru1lkEyKHhHsJNBrkoxDXaPb2ZvaIFmSJE17JKvVzStR2uW9eVR+Aw2Fw+jolYNkLSYU6kEEZL5AZgXJ/rGQ0t8bAE50pK53cCoNjQbwv387AQBYVFOE98xSAiCGisiT9yb3ak3GsHf8RKPQOXUrtVQoKYpvgMbgaACvvSNPI3zPmTUocMQekC6Z44LVIiddtB5RLd7fEpm0p8ape+nz4osvIhQK4b777sPmzZuj/gHA5s2b8dvf/laT57rxxhuxatUqXHvttfj+97+Pr371q7j00kvj+hkstyCFGEldnuTUNNKXPc+CUpcdfUN+dPV5U/pcQ54xBMbkzEuy0/YmslrMOGtpOV4+0Ik3j/bgc5dJON3vRTAkr1OozbAeyUKmZpKPtQ0ipFojcqJjCGuWzJ7RMZqn/9IMb+Rvf83FdbPW7IteyQBwonNY6f2rBdEj2WI2RY0sT7WogSIjfsyZ5YTzj2+2Kcfd+9bPjeu5bHkWLJlTjCMtAziscb9kkUmuSmDSntrEqXtlnBOQMlu3bsXWrVunvf/IkSMJ3ffLX/5y0m1OpxN33HEH7rjjjvg2UiXuTHI8M7ePHDmCT33qU1izZg0+9KEP4dVXX014Qym1xoIhZfoSM8mZT2RWOvu0zXxNJLLIABKadjUbsVp92DOGY22DaFNl8jK23CIyPCbTapIPn4zOAjZ3pG7ARqqc7BzGn/e1AwDOXVWt9KyeSaEzT6m3P6Hx76weSZ3OEje3OpM8y9S9sWAYu99sAwCsWuhO6ArO8vklAIDm9kEExrR53YfCYeX9J9lMMgeK0HTiCpLjmbk9PDyMf/qnf8LSpUvx7LPP4pJLLsH111+P3t5erbadNCSyyABHUmeDKiVITm0muUcVJKfidXPm4lLYIi3l3ni3G62RRXs2qznlCxJTRWSSvRmWSZ54qfxExxAM3GZ/EkmS8NiudyFBvtryD+9dEvP3ipKLkxov3lNGUqd5oXSBw6qUQMxWbvH64dMYikzUvHjDvBkfO52VSl2ypHSjSFbPgE+5slFdmtwJM0dT03TiCpLFzO0dO3Zg2bJl2LBhAxoaGvDcc89NeuzTTz+N/Px8bN++HQsWLEBDQwMWLFiAAwcOaLbxpB2xaA9guUU2EJmVviGfZpmbqYhMTp7VHJWN0Yo9z4JVkV6scpAsZ5LnlBekvLVdqihBcgbVJPsCQaUGWby2hjxj6BvKnIDi74dP491Id4YPvmdBVDZ1NgsjQXJbj7aL90QmOZ2L9gC5e0yJ0tFh+sypJEl44fUWAECV24nVCZbXLJ7jUjrRvKNRyUWHRp0tACDPalHWPwyOZs5rmlIvriA5npnbr732Gt73vvfBYhmvs/r1r3+NCy64IMFNpVQSi/YAlltkAxHISAC6+lOXTe5WtX9LVdC6rk4uuegZ9OHg8T4AQG2GlloAmZlJPto6Xo98+TnzlduPZ0jJhX8shF/tPgZAfq1eenZ8GVFRlxwKS2g5rV0Jk6hJTqYzQ6KUIHmGcoumtiGl9d371s9N+BjPs1qwtFb+Gx7RaPGeWLRnMgGVGlxVGv97sNyCxsW1cC+emdstLS1Ys2YNvve97+Gll15CbW0tvv3tb2P9+vUxP5/ZbJp14p76v5S43iE52HEV2FAQx6U/7gP9TbUP5qoWtXUPerFojnaLjdTEyVWl26nJpL2prF9RiYd+exhhSYI/khWfX1WUsudLRDzHQUEkIPL5QzBbTBmREX+3ZQCA3P5r81k1+J+XjsLrD+Fk1zA2nantgIhkTLcffvO340rW+5pL6uCMozsDACyuHT9+WrpHUBepsU2WCJJdBba0v55LXXagDRgYDUz73LvekNu+OWwWXFBfG9M2TrcPzlhYisOnBtDcPoSQJMGel9xCxdMD492YnI7kexC4i+xo65FbGxrpvSUR/FzWTlKvrMbGRhw6dAhPPvnkpPs8Hg9++tOf4nOf+xx+9rOf4f/+7//w+c9/Hr/73e9QU1MT088vLS2IaTGDy8XMZ7IGRuU365rygoTGS3If6E+9D1zF+cizmjEWDGPAM5bQPo1FbyQLNa/KlbLncLuBM5eUYf+xHuW2FYvLU/Z8yYjlOCh3j2f5nfl25McZsOlBlCmcsagMVRUuLJvnxv5jPWjpHjX8fjjd78FvXzkJADhrWTku3rQw7kVybncBKkvzcbrPg/Zej2a/86hPft+tKE3sfTcZ1eWFAE5jyBOY8rm7+714/fBpAMCl5yzAnOrZFzmqTTwWNq6eg6f+3IxQWELngA9r65Ib8d0TSezMqy7S5G9XWVYAHO/DUArfL9ONn8vJSzhInmnmNgBYLBasXLlSGSl4xhln4G9/+xv+93//F1/60pdieo6+vtFZM8npnk+erdpOy5fU3IU29PfHfjmR+0B/0+2DSrcTbd2jON46GNc+jVUwFEZPpJTDlW9NyXMIa5aURgXJJc7UPl+84jkOpNB4TWt755Dh2015fEEcax0AACyrdaG/fxTzKguw/1gP3j3Vj96+EcNkw6faDz996m0EgmGYTSZ84sKlGBhIrHf4gqpCnO7z4MjJPk1ee6FwWKlJtpqR9tezM0/OMvYN+qbch0+/dAzhsAQTgC2rq2PevumOhYoim3Li/tqBDixIsmSqpUv+zCp32TX52+VHus70DnoN9d6SCD0+l7PlxGKihILk2WZuA3L98uLFi6NuW7hwITo6OmJ+nnBYQjg8++ppI88nzwTqgRDlxY6E/pbcB/qbuA+q3Plo6x5FR+9oSvZNV58H4ugsK0rsdROrtUvK8SjeBSCvzC90WA35eovlOLCpxvKOeAIoLkjfEIlEHDrRB9HEom5eCYLBMBZUygvZfIEQWrtGZu2zm25iPxw51Y89h+QhGBfW16K6ND/h1838ykL8/Z3TaOsehdc3lvR45SFPQDl+8u3pfz278uXXXSgsYWDID5fqdRgYCylt385aWo4yV/zH98RjwQRgaW0x3jnZj0Mn+pL6fb3+oDL0o9Kd+D5VK4qUGQ57xuDzBzNy5P1E/FxOXtyvAvXM7SuuuGLax61du3ZS4+fm5mbU1tbGv5WUUqO+oDKaNBW9bkkfSq/kXk9KWnVF9UhOcTu2UpdD6TAwt6Iwo8emq0cgxzoSWE+iP7LDZlH2gXqghlEX74XDEh7bdRSAfGL1kS2Lkvp5Wi/e02sktTDT1L1XD3Up9dIXb4hveMhMVkRquU90DCc1cbJT3dnCrc17j5tT92gKcQXJsc7cBoBPfvKTOHLkCH7yk5/g5MmT+I//+A+0tLTgIx/5iPa/BSUlqrNFhvaepclEkOzxB5V+rFrqVvXWLk9DycAn37cMa5aU4cokgx29qSerZUIbONEfuW5eCSxm+SOj1GVXMo9GHU/95/3tOHVa7rx01fmLk+4gIXolA8CJzuRPDEZUx2S6+yQD0/cGVrd9qy0vUHoca2FF5GeFwhKOtSbeL1l0tgCA6jJtrmJETd1jGziKiCtIjmfmdm1tLR544AHs3r0bH/zgB7F792789Kc/RVVVlfa/BSVF3SO5wuD1kRQ79RSqrr7E6jBnIjLJRfl5UdnRVKmbV4Kvf+wsLJ+v3Ye2HqIzycYOkke8Y2jpkgPNFaq/u8lkwqJI0GjEyXujvjE89admAPJkxgvWzkn6Z0ZN3tNgqMiwKpOsZws4ILpX8uGT/cpky/dtmKvpVZtFNS7YIrXQyYyoFplku80SFewnI2rqHtvAUURcn2zxztxev349nnrqqcS2jNJGTE2zmE1wu2JvsE/Gpm6w39nrwbK5JZr+fBEk8+pDfJy2zAmS320ZUOpmVywoibpvUY0LbzX1ouX0MIKhsKFqOJ/5c7OSqf3U+5YpGfBkLawuQs+gT5PJeyPe8UCsyJn+unSn3Qq7zQJ/IBTVK3nXXrntW4HDinNXadvez2oxY1ltMQ6e6J80wTEeIkiududrFsSrg20OFCHBOO9qpBsR7JS67Jp9mJD+Cp15yhSpzhRkkntUg0QodrY8s9JJwOg1yaIeOd9uxfzKoqj7FkbqkoMhSRkXbgQtXcPY9boc6K2vq8DKhaWa/WzxO7d1jyY9yVIE8XlWs5JdTTeRTe6PlFucHvBi31G5i8z5a+ck3ct4KuJK0ImO4YRPEpUgOclJe2rqqXscTU0CIyJSakuZEcw+4kMkFUEyM8mJMZlMSl2y0TPJ6nrkie04F9WMB83HDVKXLEkSHvjfAwiFJVgtZnz8oqWa/nxRlxyWJLQkeWIgyi0KnXm6LUR1R7KnIpP80t5WSADMJhPet067BXtqoi45LEk4mkBdcliSlPKxKo0W7QmxjOqm3MIgmVTt3xjsZJtqd2qC5FHfGDyRAI9BcvwyYTT1kCeA1kht6oopFm8V5duUqwjH241Rl7zvWA/eOCIPwLj8nHmavzYXVI2fGCRbciEyyXp0thBEh4uBkQC8/iD+sr8dALBueQVKXam5QrSwukjJUCcyorp/yI9ApK2ZlplkYLwumZlkEhgk57hwWEKvkknmZfNsIz5ETvd7Y+o5Hit1+zcGyfFTgmQDd7d4V7WwasU0Y5hFK7jjGnR7SFYwFMZ/Py/30nYX2fGBTQs0f45CZ57yPplsVw8RJOvR2UIYz5z68fKBTqX85+L1qckiA5G65Lny9L5E6pI7+8dP+GtKte3PLf4ebAFHAoPkHNc/7EcoEjwx2Mk+VZFMcigsRbX6S1YPO6IkxWkT5RbGrUl+JxLAFDrzMLeycMrHiCC5vWcU/oC+v8tLe1vRFZkA+YmLlsJhS03HlQWRfsnJdrhQl1voRQSFQ6MBpe3bgqoiJYhNleWiX3Jn/HXJ6vZvVaXafmYxk0wTMUjOceyRnN2iOlz0aRckd7MjSlIyodxCLNpbPq9k2rHToi5ZkoCTXfrWJe95R56st3ReCc49U9uuDGpioEp7T3KL90R3Cz06WwjuSLmFBPlqEyAPD0l1jbQo35EkuYNKPETpmLvIrvmJkDhpGPaMIZimcc5kbAySc9xp9dQ0ZgSzTpXbCfFxp2VdsgiSy1wOdkRJgNGD5MERPzoiGbup6pGFBdVFEPFUs451yaO+MaX84bw1c1Ia5C1UL947nfjiPWOUW0QH6K4CGzauTP0sgwVVRbDbRF3yQFzf25miRXtA9NS9oVGWXBCD5JToG/Lhd3tOajKVKdXEZXOHzaLrZT9KjTyrBWWRkx8tB4ooiz1Zx54QowfJh2OoRwYAh82KOZGJZ3q+3x0+2a/0c15bV5HS54qevJdY9jwYCiulNkYotxDeu3YO8qypDwusFjPqIn3b34mzLlmUW2g1aU9NPVCknyUXBAbJmuof9mPn8+/iX+9/BU/sbsJPfv02JEm7xVKpIMotyoudurUhotQSk/c0zSSzbWBSHKIFnM51vNMRC6pc+XmYUz5zMKIs3tNx8t7BE/L2FjjzsHhOautpCxzji/cS7XChHkldZJBMssVswoX1tWl7bnHydaprGB7f2MwPjgiMhdA3JL/3qCeKakV90sDFewQwSNZE/7AfO194F9/+r1fw4hutCIYk5XaxkMSoxEhqdrbIXlUaB8nRHVEYJCci3+iZZFGPPN8968mzqEvuHvBh2KNPYHHoRB8AYNVC96R+zqmwUFm8l9iJwYjOI6mFPKsFxQVyoLxxZSWKC9O3viC6Ljm2fsmn+73KFYPUBMmq0dTMJBMYJCdlYMSP/94lZ45f3NuKYCgME4D6ZeXKY462Dui2fbHoHuRAiGwnPkz6h/3wadByrG/Yx44oSRILjsaCYcMtEFKf3M9UjyyIKXRA8h0fEtEz4FUWna1apN10vZmML97zJLR4b9hrjCAZAD59SR02nVGFj12o7eCV2cyvKoQjUpccays49Ym+1j2SgYlT95hJJgbJCRkc8ePxF4/i2//1Cna93oqxoBwcb1xZiVu/cA6uv3q1smr4WAIThdIlMBZSLilx0V72UmdcujTocNGtav/G101iRCYZMF42WWSRgZnrkYV5lYWwWuTsrR4lF4dU25uuIHlBkov3osst9OtuAQAbVlRi64dXTapPTjWL2Yy6eSUAYg+SOyJBstViQnmKhp0Uq3pHE6WmkWSWGhwN4HevnsQf32xTJv6YAJy9shIfes9C1FaM9xJdWluMvx8+jWNtxg2SewZVvW6ZEcxaUUFyvydq4VEiejhIJGmiJhmQ65KLtE+KJUwspCoutMV0SdtqMWNeZRGOdwwlPWAjEaLUoqLEgUp3ev6QExfvLamNrw56RFWWUujM3Y/hFfPd2N/Ui5auEYx4x2bNqotFe5Xu/JSV1ZQU2tDeM8qaZALAIDkmQ6MB/H7PKbz0ZisCY+OXRjesqMSHz1uIuRWTG+0vnSsHyR29npgOfj2op6aVM9jJWm6XHTarGYFgOKoRf6JEiY7TblUuTVJ8ojLJPmNmklfGUI8sLKqRg+TmjiFIkpS2RcBhScI7ke09Y2F6ssiAvHivssSJ0wPehOqSRbmF3WZBntUyy6Oz14oFJQDkPs1HWwZQP0tnkq7ItL1U1CMLxQXMJNM4fsLNYNgjB8cvvhEdHK9fXoGPnLdo2ilUAKImFh1rG8TapeXTPlYv6kwyL5tnL7PJhEp3Plq7R6JGuiZKvdiTHVES41QFyVrUiWulZ9CrvC/EUo8syB0u2jA0GkD/sB+lKboUPlHr6RFlcl06g2RAziafHvAm1OFCLNwrMmDyJJ3mVxbBabfC6w/inVP9MwbJkiSNt39LYZBcUsSpezSOQfIUhj0B/OG1Fry4txV+1aKM9XUV+NB5CzG/avbL1XMrCmHLMyMwFkaTQYNkkUkuLrDBnpe72YxcUF3qlINkLTLJkddNRTGvPiTKoQqSPQaqST58ckD5/1jqkYVFqsV7xzuG0hYkH4q0fjMhvu3VwsKaIvz98Gm09YzCPxaK6z1UGSSS40Gy2WzC8nkl2HesZ9ahIsOeMeVYSWmQPGHqntXCpVu5jEGyyoh3DH947RR27W2FX9W/tH5ZOT6yeVFMwbFgtZixuMaFw6cGcNSgi/eUYIelFllPrATv6vckfTm8h6+bpKnLLXx+4/RKFguoSl32uPZvdVk+HDYLfIEQmjuGsH55Zao2MYqoR55fVZT2BXALq8ZHcrecHsHSOOqShw0wbc8ols+Xg+SW0zPXJUd1tkhDkAzIpZbpOuEjY2KQHPHKgU788vkj8KmC47VL5eA40YVOS+eW4PCpARzvGDLkGam4bM6padlPfKh4/SEMjQYS7ofqCwQxFLlUzN7aiROtrwDjZJIlVX3vijjqkQG5pGdhdREOnxpI2+K9sWAY77YMAADOWBh7aYhW1J8LJzuH4wqSWW4xbsX88X135NQA1i+fuuQi1e3fhOheyQySc52xojYdPfvyCSVAPmtJGbb9fxvQ8A9rkuoEIN40x4JhnOxK/6rvmUiSpEzb42Xz7FelyrwkM1SkZ4AdUbRgtZhhi4z/NUoLuNMDXvQPy3WY6sAlVqJf8onOIYTTMGm0qW1Q6TJ0Rppav6nlO/JQ6ZaPgRNxtr4b8cqdEwqd+rZ/M4J5lYXKlZWZWsGJ961CZ15Ky1TUCQTWJRMzyRGfungZ3m7uxbmrqqPq65KxtNYFE+SVu02tg1iS4nGp8RjxjiknBcwkZ7/qCUHy8gSCIIAdUbTksFsRCAbgNcjCvaj+yJGuA/FYHHnf9PpD6OrzoKZs5nHWyToYKbWwWsxYFmcLNq0srC7C6X4vTsSZBGG5xTiz2YTl80vw5tEeHJkpSE7Doj0AKCkYP3EZZJCc85hJjli9uAzXXFynWYAMyJmGORXyB8VRg/VLVne2qGSwk/UKHHkoinwgJ5NJ7o68bkwAyngZMilOZTS1MWqSD0cWTpUXO1CewNWlhTXjV93SMVRELNpbNrcYNp0WHi9QJu+NRi3ynol/LKR0S2K5hUyctLd2j2JomtHm4n0r1UGyLc+iZLb72Ss55zFITjGR4TjWOggpDZcgYxWVEWS5RU4QHy7JTN0Trxu3y448K98+kuGM1CX7DFBuIUmSkkmOp/WbWpnLoZyIHU9xXfKob0zpT6xHPbKwsFpOqkgS0NIV2+S9UQONpDYKdWeSd6fochEMhZX3nqrS1H9elUQm5jKTTPyUSzExiWlwNBCVvdWbeMOxmE3KCG3KbqIuuSOZTDLbv2lGZJKNsHCvs8+DwVE5a7YywVIck8mkXImLt0Y3XodPDkDkHNLdH1ltQZV68l5sv7Po6wxAOanIdXMrC5XBRFPVJfcM+hAKyzu8ujS1ZTzA+OK9AWaScx6D5BSLGipioFZworNFWbEjZeM9yVhqIkFyz4AXwVB4lkdPTQTJrGNPngiSjZBJjq5HTjwzK4Lkk10jCb/GYnHopFyPXOCwRgWq6ZbvsCqL92IdKjLCTPIkZpNJKbmYql9yujpbCJy6RwKD5BSrKHHCFVkIYKS6ZKWzBeuRc4bIJIfCUkJXNeSOKGLaHl83yXLa5XILjwFqkt+JBCZVbmdSV5YWReqSg6Ew2rpHtdi0KYl65JUL3Lqf5C+M1CWfiDFIHvaOZycL09zb2chEyUVbzyiGRqMzuGLRnsmUnjU0Yuoeyy2IQXKKmUwmpRXcsdYBfTdGZfyyOTOCuWJih4t4DY4GMBZpucUgOXlKJlnn7haSJCldBZLJIgPjbeCA1C3e6x30oSvy+tWz1EIQdcntvaNRQ6imM6IqtxAlBjShX3Kk/7Ug3q/Kix1pWQtREskkD0Wm7lHuYpCcBiJIbusehcen/6XVUDiMviH5DJnBTu6odDsh5kMkMp5avdiTr5vkOW2iu4W+7wltPaNKnezKJINkV74N5ZET71QFyWLKHqDvoj1BdLgQk/dmI8ot8u1Www2Y0tOcigKl/ERd/gOoO1ukvh4ZGF+4B2BSVptyC4/QNBB1yRKA5nb9Sy76h/zKIgj2us0dVotZWXDX1R9/kBw1SIRXIJKmbgGnZ+cbdUCSaP9sNZFNTlmQHNne8mKHIU7W1DXRx2NYvMceyVOT65JLAExevJeu9m9CcUH01D3KXQyS02BBdZGSMThqgMV73YPqqWkMdnKJWPSSTCbZZjUrdfaUOFGTHJYkpW+uHkR/5DnlBVHBQaJEXXJbT2zlB/EIS5KSST5jYXyjs1Ml32FFVRyL9ziSenqi5KKj16PUA3t8QSWbm45Fe0B0Jpl1ybmNQXIaWC1m5YPjmAEW7/WwR3LOqnJHguQEapKVOvYSpyGCk0wnMsmAfm3gwup6ZFWv2mSIyXuSBJyMcxLdbNq6x0tDjFCPLIiSi5iCZJFJZpA8ifo1KE7e1Fe90pVJLonKJDNIzmUMktNkaaTkorl9CKGwvgsBuiOdLZx2KxeO5BiRiRkcDcRdC6sOkil56iBZr8V7radHMBpZJ7FCg1ILAJhfVQRxCqV1v+SDx8frkZOtn9ZSPIv3RJDPcovJ5pQXKL2jxcmb+qpXuoJk9dQ9llvkNgbJabKstgSAPJK09XTqWiPFQtSWVhQ7mBHMMdXu8QA33myyKNMpZz2yJoyQSY6uRy7R5Gc67VbUlMsLrJo1DpJFf+T5VYUoMlD7tIWqxXunTs+cTR6JtIArchpn+43CpOqXLNoSiuFHdptFGfKRDqLkgpnk3MYgOU2W1I63Rjqqcyu48YEQzAjmmuqy8dXhXXEEyWPBEAaG2RFFS2IsNQD4dOqVLC5pz63QNuhcJHoHazieeiwYxruR1mBGKrUA5Oy5MFO/ZEmSxsstmEme0srIyVpXnwf9w37lfaranZ/WpI6oz2cmObcxSE6TonybcqlI77rkbmUgBDOCuaak0AZ7nhycxZNJ7hn0QfRfYJCsDXUmWY82cOGwpPSjXbGgRNOfvWiOnBQ4PeCNmjCXjOb2QWWBoxFav6nlO6zKsJ6ZTgx8gRCCIflIYk3y1NQdVo6c6h/vbJGmRXtCSaGcSebCvdzGIDmNRF2ynkGyPxBSVgpz0V7uMZlMqCqV93s8QXL3ADuiaE3vIPlk17DyvCs1qkcWFqmGimhVl3wwMmXPajGjbm6JJj9TS6LkYqbFiuoTBna3mFpNWb7SPeedk/3jmeQ01SMLorSD5Ra5jUFyGi2LDBXpG/Kjbyj+scBaEOOoAWYEc5X4sIkvk8yOKFqz2yzKAjc9gmTRi9YEoE6jemRhbkUhLJFx0Vr1S34n0vpt2dxi2PIsszw6/US/5I7e0WkXYqqDZJZbTM1kMildLvYe6UYgMuVTnNyni8gkD3PqXk5jkJxGIpMM6JdNZkaQRJDc1eeNeYiFqGN3FdhgtxkvQMlEZpMJDjFQRON+wrE4fHIAgFxPW+DQNmDLs5oxv6oQAHBcg7pkj29MWQRotFILQbT5lCTgVNfUk/eGVSOpWW4xPdFpRb2gtSZN0/aE4kgmWQKn7uUyBslpVF2ar7wx6jVUpDsqI8ggOReJINk/Fop5UYo4ueKJlbbEQJF0Z5KDoTDejSwg1roeWVBP3kt2ouDhUwMQP8Joi/YE9eK96foli84WAAzVncNopuq0olcmGZBbZlJuYpCcRiaTCUsjJRfH9AqS++UguaTQhjwrM4K5qEpV29fZG1s7QvZITo3x0dTpDZJPdg4r/Xy16o880aJI7+DB0QD6h5Or6xRT9vLt1qgx0EbitKsW700XJEcyySYTlD68NFl1ab6SyQUAd5EdDlt6/17qdnMDSb5+KXMxSE4z0Qqu5fRI2gcISJKEA5Fm/LUVhWl9bjIO9QKYzn7vDI+USZKk1CSzHllbTps+QbKoRzabTKibV5KS5xDlB0DyJReHIov2Vi5ww2w2bm93pfVd59R12MORmuQCR56hfw+9yXXJ4ydv6V60BwDFqkzyADPJOYtBcpoti6zKDksSjrdr22h/Nic6h5XFWhtXVqb1uck4nHar0gNUPc1qOqO+ILyRPr4st9CWXplkMURkQXVRVJcNLdWUFSj169MFjbHoG/Ip71tnLDJmqYUgxlN39nqmTIKIhXtFXLQ3K/WI6iodgmS7euoeM8k5i0Fymi2sLlJWfR9N8+K9Vw52ApBbKK2vY5Ccy5TFe/2zB8mi1AIAKlluoSmlJjmNC/eCobDy3pOqemQAMJtNWBgpjWhOIiFw8MT4KGqjLtoTlMl7mHrxnii34KK92emdSQbGF++xDVzuYpCcZrY8i/JGms665FA4jNfeOQ0AWLusHPkO1sPlMpGZiSWTrA6SWZOsLT0yyc3tQ8pQDq37I08k+iWf6BxGOMHFe+9ESi3KXA7Dn6TNrypS2vpNVZcsyi0YJM+u0u3E+WfVYG5FIc5eoU9SRxkownKLnMVISQdLaovR1D6EpvZBhCUJ5jSM2nznRL/SxubcVVUpfz4yNpGZ6R70IhgKw2qZ/nxZBMkWsylqxTclT4+aZFGPbDGbotpSpoKYvOf1B3G63xt3RlCSJGXR3hkL3WkdS5wIsXivs8+Dk1OUmLDcInYmkwn/3/tX6roNykARllvkLGaSdbAs8sHk9YfQ3h1bd4FkvXKwCwBQ4LBi9eKytDwnGZcIViQJOD3L4j3R/q282MHFRhobbwGXvnILUY+8qMaV8o4BYiEbkNhQkbbuUQxFShSM2vptooXK4r3JmeQRj5yoKHSy/VsmEEkBLtzLXQySdSDawAHpqUv2B0J4491uAMDZK6tmzBpSbqguG8/odc0yeY/t31JHDBPxj4UQDifXSzgWY8EQjrXJwWoq65GFsmKHUlqQyELlQ6p65JUGr0cWFqoW76mvEIQlCSNe+WuWW2QG0eFieDSAUJhT93IRoyUdFBfalS4BxyIN/VPpzaPd8I/JmSqWWhAgZ4XFAtLZxlMzSE4dda9cbxpaQja1DSkjdlPVH1nNZDIpdcnHE+hwcSiS9Z5fWQhXhgzfWKBavNdyenzxntcfVOqyWW6RGUqipu6NzfxgykoMknWytLYEQHrGU796SC61KC92RGWxKXdZLWaUR4LejhmC5FA4jL4huR6vnO3fNKduv5aOumRRj2y1mNL2XiD6JZ/qGlEC9FgEQ2EcOTUAIHNKLYDpF++NcCR1xlGvwWCHi9zEIFknoi65e8CHwRQefEOjARxoli9ZblpVZfiFL5Q+NaIN3AxBct+QX8l+VXCQiOactvGpl+moSxb1yIvnFMOWl56JmyKTPBYMoy2ONRhNbYPKFTCjt35Tc9qtSjmTuj+06GwBAIXMJGeEqKl7DJJzEoNknahXlR9NYSu4197pUoKcTWdUp+x5KPNUlcpB70zlFmz/llpOR/oyyf6xEJoidcHqQQ2pJoJkIL6SCzFlz2oxKUOYMoUouTg5TSa5iJnkjBA1dW+Ei/dyEYNkncwpL1Autaay5EKUWiyoKsKc8oKUPQ9lHtHhYtgzhlHf1PV2DJJTy2lLX5B8rG0QocjiwJUL0peZdRXYUOaSg40TcXS4OHRSvgK2tLZYmdyXKcQQFfXivWHveJDF7haZwZ5nUT6n2QYuNzFI1onZZMKSWjnDkqoguavPo0y64oI9mkjds3a6bHLPoNz+rcBh5QCaFEhnTbIotcizmrF4TnrXJiiL9zomt0WbiscXxPF2+bGZVI8sLIz8vvLkPfn3ED2SLWaT0vqPjE+UXAyOMkjORQySdbQssnDmZOcwAmPa1yOKLLLJBGw8g0EyRVMHydPVJYtMcjmzyCmhDpZSPZpaLNpbWluMPGt63/pFkNzWParUGc/kyKl+pUwsE4Pk+VWFyuI9UXKhHknNtSGZQ+mVzHKLnMQgWUdidXkoLCXUaH8mkiThlYOdAIAzFrg5KY0mcRXY4Ihcxp4uk8z2b6mVZ7XAapEDplRmkn2BIE5EsrjprEcWRGY1LElKZnUmoh7ZabcqfYczicOmWrwX+X2VkdRctJdRlKl7XLiXkxgk62jxnGJlJLXWJRfNHUPKJLVNq7hgjyYzmUxKNrmzb+qpe2LaXkUx27+liiMNo6mPto7XI69IYz2ysLB6vC1aLCUXoh555QJ3xk55VCbvdURnkrloL7MUM5Oc0xgk68hus2BeVSEA4JjGHS5ejYyhtlnNWFdXoenPpuyhBMm9kzPJXn9QqaNkJjl1xECRVAbJoh7ZlmeO6jaRLlFt0Wa5atY35ENH5PWYSa3fJlpQLf+du/o8UccSeyRnlhJO3ctpDJJ1JuqSj7UNKjV4yQqGwnjtHTlIXrusPGpxEJGaCJJP93smvf7Y2SI9HJG65FT2SRb1yHVzS3QbSy+C8+ZZguR3IgE9AKzKwHpkYaFq8t6prmFVuQU7W2QSTt3LbQySdSb6JY/6glNm8xJx6EQfhiOX9s5lqQXNQGT3AsEw+oeia+5EqQUAZYw6aS/VmWSPL6hMftOj1EIQQfLpfu+0LQcB+f0LAMpcdlS6M/fkTL1470TnMEY88uV6ZpIzC6fu5TYGyTpTj4bVqi5ZlFoUOvOwalHmZmIo9arc07eB6xmUM8kmE1DqYpCcKuJKjy+QmiD53dYBiIsEK+brHyQD43W6E0mSpCzaW7mwNKO7QKgX7x3vGILHJ+9f1iRnFk7d015XVxcaGhqwceNGbNmyBTt27IDfP/PftrW1FfX19dizZ0/U7b/4xS+wZcsW1NfX46abboLXO34F9IUXXsDy5cuj/jU0NMS1rQySdVbqciiN9rWoS/YFgnjjaDcAYOPKSt0urVJmEFP3gMlBsii3KC1y8HWUQmLhnidFmWRRj+ywWbCgujAlzxGLeZWFsEQW4U1XctHWM4rBUTnjmsn1yIIouTh0oh+imIndLTILp+5pS5IkNDQ0wOv1YufOnbjrrruwe/du3H333TN+3/bt2+HxRH9G/eEPf8A999yDW265BQ8//DDeeustNDY2KvcfO3YMF154If76178q/2677ba4tpeffAawJJJNPqpBJvnNd3sQGJMXF7CrBc3GYbPCXSR/CEwOkiOdLVhqkVKi3MKXoppkpR55XgksZv3e8vOsZsytlIP06RbviSwyAKxckPlXwRZGFu+JRXsAM8mZRj11b5CZ5KQ1Nzdj37592LFjB5YtW4YNGzagoaEBzz333LTf85vf/Aajo6OTbn/kkUdw7bXX4sILL8SaNWvw/e9/H7/+9a+VbHJTUxPq6upQUVGh/HO54lu4zCDZAJbNLQEgr4Ie9iR3pip6I1eUOLBkTvpXsVPmEYv3Jg4UYY/k9BAL91KRSR72BNDSNQJA31ILYbEyeW+6IFmuR55bUYjigsxf4LZgih7PzCRnHvZK1k5FRQUeeOABlJeXR90+MjIy5eP7+/vR2NiIW265Jer2UCiEt99+Gxs2bFBuW7t2LcbGxnD48GEAcpC8cOHCpLbX0G0PzGbTjD0yLZFLwJYMvxS8Zkm5Mh2va8ALd4L1nyOeALxjISypLcb5a+cgLy/1o0+zZR9ksmT3weql5QgEw7DbLLBGJrGFJQmuQjsKnHlYvsCt3E5TS2YfzK0sxJLaYlgsJs3/zm29HiyOXKlav6JS9/24tq4cp07LH4YefxAuVSAcDIURCIaxpLYY555ZldC2Gu39aMncYiyZWwyoGseUlzh13w+pZLR9oIVVi8qQ78hDviMvI/adkfeBy+XCli1blK/D4TAeffRRbNq0acrH33777bjqqquwbNmyqNuHhobg9/tRWVmp3Ga1WlFSUoLOzk5IkoTjx4/jr3/9K+6//36EQiFcfvnlaGhogM0W+wm4SZI06juWApIkZfTCDSIiIiKa2h133IGdO3fiySefRF1dXdR9L7/8MrZt24bnnnsODocDy5cvxyOPPIJzzjkHHR0deO9734tdu3Zh3rx5yve8973vxT//8z9jw4YNuOiii3DVVVfh2muvRWtrK2677TZccskl+O53vxvz9hk6k9zXNzprJtnlcmJoyItQKLObfD/y+yM43jGEeVWF+KcPrEzoZ/z0NwfR0evBnPJ8XPehVRpv4dSyaR9kqmT3wdGWAfz3rqMAgC9duQpV7nyc7BzGL34nX7L6pw+uxLwK/RZ8ZYJk9sHbzb146k/NAIB/+eRazVqESZKEHz+5HwMjAZy5uBQfvWCJJj83GeGwhNt3voGxYBib19TgfevnKvftfrMNf97XDrPZhG9fsw62vMQyyUZ7P3rqz814u6kXAGC1mHHTZ9dldfLHiPsgWc//vQWvHOhEoTMP//LJtXpvzqz02Adud0Hc39PY2IiHH34Yd91116QA2efzYdu2bbj55pvhcEy+um63y2tpAoHoEtVAIACn04na2lrs2bMHxcXFMJlMWLlyJcLhML71rW/hxhtvhMUS25V2QwfJ4bCEcHj2RHcoFEYwmNkHY77dgqa2QZzsHManLlqGvDgv6XT0juKv+zsAyF0t0v33yIZ9kOkS3QcFDiuaIotGT7QPoazIgRMdQ8ptBXYr922MEtkHgUBI+VsPDvvh0KhM6nS/B3uPyJ1uzj+rxjD7MByW0NQ2CEeeGRecNUe5/S/72tHUNojl80pgNiGp7TXS+5HTZlH2r7vIjlBIQlT9RZYy0j5IVjAYRlPbIEwA/IGgrgtg42HkfXDrrbfiscceQ2NjIy677LJJ9+/fvx8tLS2TWrZdd911uPLKK7F9+3bY7Xb09PRgyRI5ARAMBjEwMICKCnnKcElJSdT3LlmyBH6/H4ODgygtjW1hcGbs6RwghooEQ2Gc7Jq6h+hMRG9ks8mEjSurNN02ym5lxQ6lNZfocCEW7dnzLFyNn2LqiZhaLt47eMKYk+sW1ciL2Y53DENU+3n9QTS3y4v5sqH1m9pC1eI9HkuZiVP3tHXPPffg8ccfx5133okrrrhiysesWbMGzz//PJ555hnlHwDcdttt+NrXvgaz2YzVq1dj7969yvfs27cPVqsVK1aswF/+8hecc845UX2T33nnHZSUlMQcIAMGzyTnkiVzimGCfBAeax2MGjIyG0mS8OohuavFGYvcWbEqnNLHYjaj0u1ER69Hmfqobv+WzZeGjUAdJPs0DJJFp4jq0nxDDYMRQ0U8/iBO93tRVZqPI6cGlLHoZ2TZAKT5lUUwmQBJYmeLTDVx6p5om0nxa2pqwr333outW7di/fr16O7uVu6rqKhAd3c3ioqK4HA4sGDBgknfX1VVhbKyMgDANddcg23btqGurg6VlZXYvn07Pv7xj8PpdKK+vh52ux3f/e538ZWvfAUtLS340Y9+hC984QtxbS+DZINw2q2orShEa/dI3JP3mtqHlKDm3DPYG5niV12aLwfJ/ZEgeZDt39LFaR8vr/Bo1Cs5HJaUISJGy8yqJ+8d7xhCVWm+EtA77daozGs2sNssmFNegLbuUSYwMlQxp+5p5sUXX0QoFMJ9992H++67L+q+I0eOYPPmzdixYweuvvrqWX/WFVdcgba2Nmzbtg2BQACXXnopvvWtbwEACgsL8fOf/xw//OEP8dGPfhQFBQX45Cc/ySA5ky2bWywHya0DcXX2EL2RbXlm1NeVz/JooslEr+TxTDKD5HTJV2eSNRpNfbJrGKORMchGKrUAgPJiBwqdeRjxjuF4xzA2rarGwUiQvGK+vgNPUuXjFy7Fi3tbcenZ8/XeFEpAScF45niQU/eSsnXrVmzdunXa+48cORLXfTP9vGXLluGhhx6KfyNVsu/dKIOJuuQhzxhOD3hnebQsGArj7++cBgCsq6tQRtwSxUMEyaO+IPqGfMoHQXmxcS7TZyv1MatVTfLB43LQaTaZsNwAQ0TUTCYTFip1yUPoH/ajI3JydobBAnqtrF5chq9/7Kwph4uQ8dltFuWKDzPJuYVBsoGo65CPtcZWcnHgeJ8y8nQTSy0oQVWRIBmQX1MCM8mpZzabYLfJH8Ba1SSL8oVFc4qQ7zDeifOiyLjmU13DONDcq9xutNIQIkHUJQ8wk5xTGCQbSHmxQ6l9irUu+dVIqUVRfh5WLeIHDCWmukwVJKuCFgbJ6eGMBMleDWqS/WMh5f3jjAXGzMwumiMHyYFgGC+83gpAbo9WrTpZIzISUU/OTHJuiTtI7urqQkNDAzZu3IgtW7Zgx44d8PtnftG0traivr4ee/bsSXhDc4HJZMKySDY5lkyy1x/Em0d7AADnrKzKylo+So8iZ55SG6tuHcZyi/QQHS68GtQkv9sygGBI7hSxyqCdItSL91q75THVqxaWspMKGVZJpKMFa5JzS1xRlSRJaGhogNfrxc6dO3HXXXdh9+7duPvuu2f8vu3bt8Pj8SSznTlj6dwSAEBbzyhGfTP3Y3zj3W6MRRqFb1rFUgtKnMlkUrLJ3sgl/+JCG2waDbagmSlBsgblFqLUwp5nweI5rlkerY/iAhtKXdFttFhqQUY2Xm7BTHIuiStIbm5uxr59+7Bjxw4sW7YMGzZsQENDA5577rlpv+c3v/kNRkdHk97QXKGuS25qG5rxsaKrRZXbqTToJ0pUlTv6UjdLLdJHBMlaLNw7eFy+ErB8fgmsFuNeXVJnkwFgZZYu2qPsUBIptxgaDSAUNuYUO9JeXCs6Kioq8MADD6C8PLrN2MjIyJSP7+/vR2NjIx588EF88IMfjHvjzGYTzObpL79ZIh8AFgN/EMRrca0LNqsZgWAYze2DWLe8YsrH9Q/78U6kD+p5q2uQp1PGLxv3QabRah/UVhREfV3ldsIa53j0XJXsPhCL6/yBUFJ/88ERv1K+sHpJmaH335LaYmVs9rzKQpRpUNrD9yP9Zes+KI28PiXI/czdRcZbECtk6z7QQ1x72eVyYcuWLcrX4XAYjz76KDZt2jTl42+//XZcddVVWLZsWUIbV1paEFONmsuVXRmvugVuHGjqxfHOEbjdBVM+5k/7OxAZUIXLzls07ePSJdv2QSZKdh8snV8KoEn5en5Nse6vq0yT6D4oKZI/gP3BcFJ/87eOj9eTn3tWraH335q6SvzqpWMAgHUrqjTdVr4f6S/b9sG8mvGrvCGTydDHlpBt+0APSZ0KNTY24tChQ3jyyScn3ffyyy9j7969M5ZizKavb3TWTLLL5cTQkBehUPZc/lhUVYQDTb04crIP3T3DU14y3fXaKQByNsZpMaG/X5+SlmzdB5lEq31QaDdP+Nqi2+sq0yS7D8Tb3IgnkNTf/LUD7QCAkkIbimxmQ++/iiIbChxWjPqCOHOhW5Nt5fuR/rJ1H1ghKf/f0j6I8kLjTk/UYx9kwklDIhIOkhsbG/Hwww/jrrvuQl1dXdR9Pp8P27Ztw8033wyHI/FLaOGwhHBYmvVxoVAYwWD2HIyLVe2RjrcPTarda+sZxcnOYQDApjOqDPG7Z9s+yETJ7oNyV/SxWuZycJ/GKdF94MgTLeCCCf/NJUnCgWZ50d7KBaUIhSQAs79/6iXPYsZ3P7cBQ54Alsxxafpa4/uR/rJtHxQ68pT/7x30ZcTvlm37QA8JFazceuuteOihh9DY2IjLLrts0v379+9HS0sLGhoaUF9fj/r6egDAddddh23btiW3xTlgySxDRURvZLPJhLNXVqZtuyi72fIsKFN1HODCvfQRC/d8/hDCUmKBbWefB/3D8sr7TOmZXlWaj2WRjj5ERsape7kp7kzyPffcg8cffxx33nknLr/88ikfs2bNGjz//PNRt1166aW47bbbcN555yW2pTmk0JmHOeUFaO8ZxdG2QVxy9jzlvrAk4dWDXQCAMxeXwpVv3Es+lHmqS/PRO+SH1WJWBttQ6olhIhLkxXsiaI7HIVV/65UGHSJClMmKC+zw+j2cupdD4nonbmpqwr333outW7di/fr16O7uVu6rqKhAd3c3ioqK4HA4sGDBgknfX1VVhbKysuS3OgcsrXWhvWcUx1oHIEmSsoDxWOsgeod8AIBNq6r03ETKQvOrinDwRD9qywtg5mCHtFEHxV5/MMEgWS61qC0vgLvIPsujiSheJYU2dPZ5MMhMcs6Iq9zixRdfRCgUwn333YfNmzdH/QOAzZs347e//W1KNjTXLK0tASDPiRdBMTBeamG3WVC/bOr2cESJuvyc+bhyyyL84wdW6L0pOWVikByvUDiMw6fkTPJKDuUgSgkxdY+Z5NwRV7pi69at2Lp167T3HzlyJKH7aLJlc6PrksuLnQiGwvj74dMAgPV1FbBzGhpprCjfhg+ft0jvzcg5UUFyIBT39x/vGIbXL3/fKg7lIEqJkgJO3cs17DRtUJVuJ4ry5dW0R9vkxXtvN/Vi1CdnmVhqQZQ9xIIgILFM8qHjcqmFxWxC3bwSrTaLiFRKIus0hjycupcrGCQblMlkUkZUN0U6XIgx1MUFNqxcwEuqRNki2XKLg5F65CVzXAnVMxPR7IoL5UyyJAFDo2M6bw2lA4NkA1saKblo6R5B35AP+471AgA2rqyCxcxdR5QtnLbEg2SvP4jm9iEAwBkstSBKmRJVx5/BUZZc5AJGWga2LLJ4T5KAX+0+hmBkcs65Z7LUgiib2PLMSjcRUVscqyMtAwhFhi6dsYhBMlGqlBSOd40ZGNZu8d4b73bjzv/Zh3dbBjT7maQNBskGtqC6EFaL/MH52jvygr2asnwsqCrSc7OISGMmk0mpS443kyxavzntFiyq4XsDUaqoe8cPaJRJ/uv+DvznU2/jwPE+PLbrqCY/k7TDINnA8qwWLKyOHkm96YwqpWcyEWUPUUscf5Ast35bMd/NMiyiFHLYrONT94aTD5J3v9mGB3/7jjI8/mTXMDp6R5P+uaQdvqMa3FJVKzgAOGdVtU5bQkSppATJgdiD5P5hP9p75A9V1iMTpV5xpA3c4Ghy5RbP/70Fv/yD3BrXVWBTyq32HOpKbgNJUwySDU50uADkgLmyxKnj1hBRqojR1PHUJItSCwA4g0NEiFJOLN5LJpP8f6+cwOMvyqUV7iI7/vXT65Tj99VDXZAkaaZvpzRikGxwS+cWK3XJ72EWmShrJVJuIUotSl12VJfmp2S7iGicWLyXyNQ9SZLw9J+b8es/NQMAyosd+NdPr0N1aT7OOUNekH+634vjHcPabTAlhUGywbnybfjylWfi4xcuxflr5+i9OUSUIvEGyZIk4dBJOZN8xoJSrlUgSgMlSI5z4Z4kSXhidxOeffkEAKDK7cS/fnodKiJXh9fVVSDPKodkrx7q1G6DKSkMkjNA/bIKXH7OfKVmiYiyz3hNcmzlFu09oxiMZLPOWMRSC6J0EB0uhkYDCIdjK4sISxL++4Wj+P1rpwAAc8oL8O1Pr0Opy6E8xmm3Yu3ScgByN6tYfzalFoNkIiIDcMTZAu5gpNQCkDPJRJR6Jeqpe57ZSy7CYQmP/P4wXnyjFQAwv7IQN1xTH9VzWdgUKbkYGg3gnZP9k+6n9GOQTERkAPmRTLIvxiBZLNqbV1kIV4FtlkcTkRbUU/cGRmYuuQiFw3jg/w7hz291AAAW1bjwrWvq4cqf+ng9c3GZ8j7AkgtjYJBMRGQAjsho6kAwrEzXnE4wFMaRUwMA2NWCKJ2ipu7NsHgvGArj/v89iFcPyi3dls0txjc/uRYFjrxpvyfPasaGFZUAgL1HuhEYi2/6JmmPQTIRkQGIDBIwe8lFU9sg/JEP0FXsj0yUNsUxZJLHgiHc+/QBvH6kGwCwcoEb3/j4WmXdwUxEyYUvEML+pl4NtpiSwSCZiMgA1B+gsy3eE63frBYTls0rSeVmEZGKw2aFI9LTfHCKTLJ/LIQf//pt7DvWAwBYvbgMX/uHNbBHvmc2dfNL4C6Ss9WvcrCI7hgkExEZgBh3CwBe38yZZFGPvLS2GPa82D58iUgb472SozPJXn8Q//HEWzh4XD4+65eV4/qrV8MWxzFqNplwzko5m7y/qQejvjGNtpoSwSCZiMgA1Jlk3wyjqT2+IJo7hgBwFDWRHqaauufxBXHnr/bhcGStwMaVlfjylWcqvY/jIQaLBEMS9kZKNkgfDJKJiAzAoQqSPTPUJB8+1Q8xtXbVIgbJROk2PlBELrcY8Y6h8fE30dQmn7yet7oaWz+0ClZLYiHW/KpC1JTJEzT3sORCVwySiYgMQL1wz+efviZZlFoUOKxYUFWU8u0iomhi8d7AiB9DowH86L/fwMlOeZT0e+tr8Y8fWAmzOfHhXyaTSVnAd/hkP/qH45vuR9phkExEZAAO1cKemTLJYojIigXupD6IiSgxIpM8NBrAHf/9Blq7RwEAl2yYh89eWqfJdFxRciEBeO0dZpP1wiCZiMgArBYzbJH6xelqknsHfejq8wBg6zcivYhMsiQBHb3y8XjFuQvwyfcthUmDABkAKt35WDLHBQBKr2VKPwbJREQGIeqSp8ski1ILgENEiPTinjBS+soti3D1+Ys1C5AFkU0+2TWMjt5RTX82xYZBMhGRQTiV0dRT1yQfOimXWpQXO1Dpzk/bdhHRuJryAqVrxccvXIoPn7dI8wAZAM5eWaWUbnABnz5mH/9CRERp4YzUJU81cS8sSUomma3fiPTjyrfhe9duwFgwjEU1rpQ9T3GBDWcsdOPA8T68erALH9mcmmCcpsdMMhGRQThnKLdoPT2CYY88WICt34j0NbeiMKUBsiBKLk4PeHG8Yzjlz0fRGCQTERlEvlJuMTlIFqOoTQBWLmA9MlEuWFdXoZR2vHqoU+etyT0MkomIDMIRGU3tmaIm+WCk1GJ+dREKnXlp3S4i0ofTbsXapeUAgNfeOY1QOKzzFuUWBslERAahLNyb0AJuLBjC0ZYBAOxqQZRrxGCRodEADp8c0HdjcgyDZCIig3Da5CB54sK9Y62DCATlDBL7IxPlltVLylDgkN8bXj3Ikot0YpBMRGQQIpPs9YcgSZJyu2j9lmc1Y9ncYl22jYj0YbWYsX55JQBg77vdCIxNP7aetMUgmYjIIJyRmuSwJCEwNl57ePC4XI9cN7cYeVbLlN9LRNlLlFz4AiHsb+rVeWtyB4NkIiKDEJlkAPBG6pJHvGM42Sm3fmJ/ZKLcVDe/BO4iedLfKyy5SBsGyUREBhEVJEfqkg+f7IcovGCQTJSbzCYTzlkpZ5Pfbu7FqG9M5y3KDQySiYgMIjpIlusOxZS9Qmce5lUV6rJdRKQ/MVgkGJKw90i3zluTGxgkExEZhBhLDYxnkg8qo6jdMHMkLVHOml9ViJqyfADscpEuDJKJiAxiYrnF6QEvugd8AFhqQZTrTCaTsoDvyKkB9A/7dd6i7McgmYjIICYGyaLUAuAQESIaL7mQAOw51KXvxuQABslERAZht1kgCirkIFnuj1zldqK82KnfhhGRIVS687FkjgsAg+R0YJBMRGQQZpMJjkg2edQXxDtKPTJLLYhIJrLJJ7uG0dE7qvPWZDcGyUREBiIGihxpGcCoT168xyCZiISzV1Ypi3hfPchscioxSCYiMhBRl3y0ZQAAYDIBKxeU6LdBRGQoxQU2ZY3CnkNdUSPsSVsMkomIDMRpk4Nk8bG3qMaFfEeefhtERIYjSi5OD3hxvGNY563JXgySiYgMRN3hAmBXCyKabF1dBfKscgjHnsmpwyCZiMhARE2ysIr1yEQ0gdNuxdql5QCA1w6fRigc1nmLshODZCIiA1Fnkm15ZiyeU6zj1hCRUYnBIkOjAbxzsl/nrclODJKJiAxE1CQDwPJ5buWSKhGR2uolZShwyO8Xe9jlIiX47ktEZCDqcotVrEcmomlYLWasX14JANj7bjcCYyGdtyj7MEgmIjIQdbkF+yMT0UxEyYUvEMJbTb06b032YZBMRGQgKxeWwmm3YPm8EtRWFOi9OURkYHXzS+AusgNgl4tUsM7+ECIiSpfa8gLc/dXNsFrMMEWmahERTcVsMuGclVX4/Wun8HZzL0Z9YygutOu9WVmDmWQiIoPJs1oYIBNRTDatkksugiEJe49067w12YVBMhEREVGGmldZiJqyfAAsudAag2QiIiKiDGUymZQFfEdODaBvyKfzFmUPBslEREREGeycVdUAAAnAnkPG7pnc1dWFhoYGbNy4EVu2bMGOHTvg9/tn/J7W1lbU19djz549Ubf/4he/wJYtW1BfX4+bbroJXq9Xuc/v9+Omm27Chg0bsHnzZjz44INxbyuDZCIiIqIMVlnixJI5LgDAKweMW3IhSRIaGhrg9Xqxc+dO3HXXXdi9ezfuvvvuGb9v+/bt8Hg8Ubf94Q9/wD333INbbrkFDz/8MN566y00NjYq9//oRz/CgQMH8PDDD+Pmm2/GPffcg9///vdxbS+DZCIiIqIMd06k5OJE5zBauoZ13pqpNTc3Y9++fdixYweWLVuGDRs2oKGhAc8999y03/Ob3/wGo6Ojk25/5JFHcO211+LCCy/EmjVr8P3vfx+//vWv4fV64fF48MQTT+A73/kOVq1ahUsuuQRf+MIXsHPnzri2l0EyERERUYY7e2UVzJGuOH96s1XnrZlaRUUFHnjgAZSXl0fdPjIyMuXj+/v70djYiFtuuSXq9lAohLfffhsbNmxQblu7di3GxsZw+PBhHD58GMFgEPX19cr969evx1tvvYVwOBzz9hq6T7LZbILZPH0bJIvFHPVfSj/uA/1xH+iP+8AYuB/0x32gn7JiBy4+ey6a2oZwom0IZrMJVqux9oPL5cKWLVuUr8PhMB599FFs2rRpysfffvvtuOqqq7Bs2bKo24eGhuD3+1FZWancZrVaUVJSgs7OTpjNZrjdbthsNuX+8vJy+P1+DAwMoLQ0tmmmhg6SS0sLYuoV6nI507A1NBPuA/1xH+iP+8AYuB/0x32gj69+Yp3emxCXxsZGHDp0CE8++eSk+15++WXs3bt3ylIMn0/u4KEOgsXXgUAAkiRNeR8ABAKBmLfP0EFyX9/orJlkl8uJoSEvQqHY0+ekHe4D/XEf6I/7wBi4H/THfaCvcFjCX9/uQLk7H6sXlaZtH7jdBXF/T2NjIx5++GHcddddqKuri7rP5/Nh27ZtuPnmm+FwOCZ9r90uTxWcGPAGAgE4nU6EQqEp7wMw5c+bjqGD5HBYQjgszfq4UCiMYJAHo564D/THfaA/7gNj4H7QH/eBfs4/aw7c7gL0948adh/ceuuteOyxx9DY2IjLLrts0v379+9HS0sLGhoaom6/7rrrcOWVV2L79u2w2+3o6enBkiVLAADBYBADAwOoqKiAJEno7+9HMBiE1SqHut3d3XA4HHC5XDFvp6GDZCIiIiLKHvfccw8ef/xx3Hnnnbj88sunfMyaNWvw/PPPR9126aWX4rbbbsN5550Hs9mM1atXY+/evTjnnHMAAPv27YPVasWKFSsAyDXK+/btUxb37d27F6tXr4bZHHudNoNkIiIiIkq5pqYm3Hvvvdi6dSvWr1+P7u5u5b6Kigp0d3ejqKgIDocDCxYsmPT9VVVVKCsrAwBcc8012LZtG+rq6lBZWYnt27fj4x//OJxOuR5eZJx/+MMf4vTp03jwwQexY8eOuLaXQTIRERERpdyLL76IUCiE++67D/fdd1/UfUeOHMHmzZuxY8cOXH311bP+rCuuuAJtbW3Ytm0bAoEALr30UnzrW99S7r/xxhuxfft2XHvttSgsLMRXv/pVXHrppXFtr0mSpNmLfnXS3T1zM2yr1Wz4uptsx32gP+4D/XEfGAP3g/64D/Snxz6oqChKy/Okm7Ea6BERERERGQCDZCIiIiKiCRgkExERERFNwCCZiIiIiGgCBslERERERBMwSCYiIiIimiCuILmrqwsNDQ3YuHEjtmzZgh07dsDv90/52D/+8Y/4yEc+gvr6enzoQx/Ciy++qMkGExERERGlWsxBsiRJaGhogNfrxc6dO3HXXXdh9+7duPvuuyc99vDhw7j++uvx0Y9+FM888ww++clP4mtf+xoOHz6s5bYTEREREaVEzBP3mpubsW/fPvztb39DeXk5AKChoQF33HEHvv3tb0c99rnnnsOmTZvwuc99DgCwYMECvPTSS/jd736nzNQmIiIiIjKqmIPkiooKPPDAA0qALIyMjEx67FVXXYWxsbFJtw8PzzxBj4iIiIjICGIOkl0uF7Zs2aJ8HQ6H8eijj2LTpk2THrtkyZKor48ePYpXXnkFn/zkJ+PaOLPZBLPZNO39Fos56r+UftwH+uM+0B/3gTFwP+iP+0B/3AfaMUmSJCXyjXfccQd27tyJJ598EnV1ddM+rq+vD9dccw3Ky8vxyCOPwGyOfadJkgSTafogmYiIiIgoFWLOJKs1Njbi4Ycfxl133TVjgNzT04N//Md/hCRJ+PGPfxxXgAwAfX2js2aSXS4nhoa8CIXCcf1s0gb3gf64D/THfWAM3A/64z7Qnx77wO0uSMvzpFvcmeRbb70Vjz32GBobG3HFFVdM+7iuri5l4d4jjzyCqqqq5LaUiIiIiChN4sok33PPPXj88cdx55134vLLL5/2cR6PB1/4whdgNpvxyCOPoKKiIukNJSIiIiJKl5iD5KamJtx7773YunUr1q9fj+7ubuW+iooKdHd3o6ioCA6HA/fffz9OnTqFX/7ylwCgPNbhcKCoqEjjX4GIiIiISFsxl1v89Kc/xb//+79Ped+RI0ewfPly7NixA1dffTUuv/xyHD9+fNLjrrrqKtx+++3JbTERERERUYol3N2CiIiIiChbsYkeEREREdEEDJKJiIiIiCZgkExERERENAGDZCIiIiKiCRgkExERERFNYPgg2e/346abbsKGDRuwefNmPPjgg9M+9tChQ/jYxz6Gs846Cx/96Edx4MCBNG5p9opnH3z5y1/G8uXLo/7t3r07jVub3QKBAD74wQ9iz5490z6Gx0FqxbIPeBykRldXFxoaGrBx40Zs2bIFO3bsgN/vn/KxPA5SJ579wGMhNU6ePInPf/7zqK+vx3vf+1488MAD0z6Wx0Li4pq4p4cf/ehHOHDgAB5++GG0t7fj29/+NubMmTNp4p/H48HWrVvxoQ99CLfffjsee+wxfPGLX8QLL7yA/Px8nbY+O8S6DwB56ExjYyPOPfdc5bbi4uJ0bm7W8vv9+Jd/+RccPXp02sfwOEitWPYBwOMgFSRJQkNDA1wuF3bu3InBwUHcdNNNMJvN+Pa3vx31WB4HqRPPfgB4LKRCOBzG1q1bsXr1ajz99NM4efIkvvGNb6Cqqgof+tCHoh7LYyFJkoGNjo5Kq1evll599VXltv/8z/+UPvOZz0x67BNPPCFddNFFUjgcliRJksLhsHTJJZdIv/71r9O2vdkonn3g9/ullStXSs3NzencxJxw9OhR6cMf/rD0oQ99SKqrq4vaH2o8DlIn1n3A4yA1jh07JtXV1Und3d3Kbc8++6y0efPmSY/lcZA68ewHHgup0dXVJX3ta1+ThoeHldu+8pWvSDfffPOkx/JYSI6hyy0OHz6MYDCI+vp65bb169fjrbfeQjgcjnrsW2+9hfXr18NkMgEATCYT1q1bh3379qVzk7NOPPugubkZJpMJ8+bNS/dmZr3XXnsN55xzDv7nf/5nxsfxOEidWPcBj4PUqKiowAMPPIDy8vKo20dGRiY9lsdB6sSzH3gspEZlZSXuvvtuFBYWQpIk7N27F3//+9+xcePGSY/lsZAcQ5dbdHd3w+12w2azKbeVl5fD7/djYGAApaWlUY9dunRp1PeXlZXNelmUZhbPPmhubkZhYSFuuOEGvPbaa6iursZXv/pVXHDBBXpsela55pprYnocj4PUiXUf8DhIDZfLhS1btihfh8NhPProo9i0adOkx/I4SJ149gOPhdS76KKL0N7ejgsvvBCXXXbZpPt5LCTH0Jlkr9cbFZwBUL4OBAIxPXbi4yg+8eyD5uZm+Hw+bN68GQ888AAuuOACfPnLX8bbb7+dtu3NdTwO9MfjID0aGxtx6NAh/PM///Ok+3gcpM9M+4HHQur9+Mc/xn/913/hnXfewY4dOybdz2MhOYbOJNvt9kk7UnztcDhieuzEx1F84tkH/+///T989rOfVRZlrFixAgcPHsSvfvUrrF69Oj0bnON4HOiPx0HqNTY24uGHH8Zdd92Furq6SffzOEiP2fYDj4XUE39Hv9+Pb37zm7jhhhuigmIeC8kxdCa5qqoK/f39CAaDym3d3d1wOBxwuVyTHtvT0xN1W09PDyorK9Oyrdkqnn1gNpsnrVpevHgxurq60rKtxOPACHgcpNatt96Khx56CI2NjVNeXgZ4HKRDLPuBx0Jq9PT0YNeuXVG3LV26FGNjY5Nqw3ksJMfQQfLKlSthtVqjCsz37t2L1atXw2yO3vSzzjoLb775JiRJAiC3qXnjjTdw1llnpXOTs048++Bf//VfceONN0bddvjwYSxevDgdm0rgcWAEPA5S55577sHjjz+OO++8E1dcccW0j+NxkFqx7gceC6nR2tqK66+/Pupk48CBAygtLY1aJwTwWEiWoYNkp9OJK6+8Etu3b8f+/fuxa9cuPPjgg/jc5z4HQM5o+nw+AMDll1+OoaEh/OAHP8CxY8fwgx/8AF6vF+9///v1/BUyXjz74KKLLsKzzz6LZ555BidPnsQ999yDvXv34jOf+Yyev0LW43GgPx4HqdfU1IR7770X1113HdavX4/u7m7lH8DjIF3i2Q88FlJj9erVWLVqFW666SYcO3YMf/rTn9DY2IgvfelLAHgsaEq35nMx8ng80g033CCtXbtW2rx5s/TQQw8p99XV1UX1+nvrrbekK6+8Ulq9erX0D//wD9LBgwd12OLsE88++NWvfiVdeuml0plnnildddVV0muvvabDFme3iT16eRyk32z7gMeB9u6//36prq5uyn+SxOMgXeLdDzwWUqOzs1P6yle+Iq1bt04677zzpPvuu0/phcxjQTsmSYrk4ImIiIiICIDByy2IiIiIiPTAIJmIiIiIaAIGyUREREREEzBIJiIiIiKagEEyEREREdEEDJKJiIiIiCZgkExERERENAGDZCIiIiKiCRgkExEZ3EUXXYSf/OQnem8GEVFOYZBMRERERDQBg2QiIiIiogkYJBMRzWJ4eBjf+973sGnTJqxfvx6f+9zn8PbbbwMAfvKTn+BTn/oU/vM//xPnnHMONmzYgBtvvBEjIyPK9w8MDOD73/8+LrjgAqxZswaf/OQnsWfPnqjn+Mtf/oJPfOITOOuss3D++efjrrvuQigUUu7v7u7G9ddfj7Vr1+Kcc87Bjh07ou4nIiJtMUgmIpqBJEm47rrr0NLSgvvvvx+/+tWvsHbtWnzqU5/CoUOHAABvv/02/vrXv+LBBx/Ef/7nf+Lvf/87vv71rwMAQqEQ/umf/gmvv/46Ghsb8dRTT6Gurg6f//znsX//fgDAm2++ia1bt2L9+vV46qmncNttt+Hxxx/Hvffeq2zHk08+ibPPPhvPPvssvvWtb+EXv/gFnn766bT/PYiIcoVJkiRJ740gIjKqV155Bf/4j/+IV199FSUlJcrtn/nMZzB37lzU1tbipz/9KXbt2oWqqioAwJ///Gdcd911+N3vfoeWlhZs3boVzz77LOrq6gDIgffVV1+N+fPn4z/+4z/wjW98A+3t7Xj88ceVn/+HP/wBvb29uOaaa3DRRRdhzZo1uPvuu5X7P/zhD2PdunXYvn17Ov4MREQ5x6r3BhARGdnBgwchSRIuvPDCqNsDgQD8fj9qa2uxcOFCJUAGgHXr1gEA3n33XbS0tKCoqOj/b+f+WVqH4jiMPyXqJLgJ4qAQUSIOHbqICm6C0NGpoIuL4KAovoAuUpz8uxREd6mvQEEQEVHsHJDqa6goKtg7XFpornbxioPPB0I4OTmHnCV8SX5JIyADpFIpMpkM5+fnjfPGxsaa5p+ammpq9/f3N7W7urp4eXn58vokSR8zJEtSC+/v73R2dlIqlf7p6+jo4OjoiPb29qbj9VrhIAj47GVdrVajre3vLbi+byUIgg/nkCR9D2uSJamFwcFBHh8feXt7o6+vr7EVi0VOTk4AqFQqVKvVxpjb21sAhoeHGRoaolqtEsdxo79Wq3Fzc8PAwAAAYRg2PgSsOzw8ZGZm5ruXJ0n6hCFZklqYmJggiiKWl5e5vLzk4eGB9fV1SqUSYRgC8PT0xNraGnEcc3FxQT6fZ3p6mt7eXsbHx4miiJWVFa6urri7uyOfzxPHMXNzcwDMz89TLpfZ3Nzk/v6es7Mz9vb2mJyc/MGVS9LvZrmFJLUQBAH7+/tsbGywtLTE8/MzYRiys7PD6Ogo19fX9PT0EEURuVyOIAjIZrOsrq42jS8UCiwuLvL6+srIyAgHBwek02kAoihid3eXra0tisUi3d3dzM7OsrCw8IMrl6Tfzb9bSNIXbG9vc3x8zOnp6U9fiiTpP7LcQpIkSUowJEuSJEkJlltIkiRJCT5JliRJkhIMyZIkSVKCIVmSJElKMCRLkiRJCYZkSZIkKcGQLEmSJCUYkiVJkqQEQ7IkSZKU8AeAlU4eeVNpqgAAAABJRU5ErkJggg==",
|
||
"text/plain": [
|
||
"<Figure size 800x550 with 2 Axes>"
|
||
]
|
||
},
|
||
"metadata": {},
|
||
"output_type": "display_data"
|
||
}
|
||
],
|
||
"source": [
|
||
"# example training\n",
|
||
"df_hist = data[-1]['hist']#.groupby('epoch').last().dropna(axis=1).drop(columns=['step'])\n",
|
||
"df_hist['loss'].plot(label='train')\n",
|
||
"plt.twinx()\n",
|
||
"df_hist['eval_loss'].plot(c='b', label='eval')\n",
|
||
"plt.legend()\n",
|
||
"plt.show()"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 19,
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"data": {
|
||
"text/plain": [
|
||
"<Axes: xlabel='epoch'>"
|
||
]
|
||
},
|
||
"execution_count": 19,
|
||
"metadata": {},
|
||
"output_type": "execute_result"
|
||
},
|
||
{
|
||
"data": {
|
||
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAqYAAAHmCAYAAAChy0xuAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAABM70lEQVR4nO3deXydZYH+/+ss2fd9T9qmSZq2SZvu0NLSIpuAIoogAo6izDC4IQ4MfH8z4jaM8v06bigKVhlBWWTHBa2UpbSlNE2XpEuWpmn2fV/OyTnn+f1xkkhFoGlz8jxJPu/Xi5ftyWlypbdPe/V+7vt+bIZhGAIAAABMZjc7AAAAACBRTAEAAGARFFMAAABYAsUUAAAAlkAxBQAAgCVQTAEAAGAJFFMAAABYAsUUAAAAluA0O8DZam/vf8+P2+02xcdHqKtrUD4fzxIwA2NgPsbAfIyBNTAO5mMMzGfGGCQlRZ3W+2b9jKndbpPNZpPdbjM7ypzFGJiPMTAfY2ANjIP5GAPzWXkMZn0xBQAAwMxAMQUAAIAlUEwBAABgCRRTAAAAWALFFAAAAJZAMQUAAIAlUEwBAABgCRRTAAAAWALFFAAAAJZg+iNJR0dHdeedd6qlpUVhYWG67777FB8fb3YsAAAATDPTZ0z/8Ic/KCUlRb/5zW902WWX6cEHHzQ7EgAAAExg+ozphz/8YV122WWSpJaWFsXExJicCAAAAGaYtmL6+OOP69e//vUpr/3iF79QSkqKnE6nbr75Zh06dEi//OUvpysSAAAALGTaiuk111yja6655l0//vOf/1x1dXW6+eab9dJLL01XLAAAAFiE6WtMH3/8cT3yyCOSpPDwcNntpkcCAACACc64Bbrdbl1++eV68803J15zuVy6++67tWrVKm3YsEFbt259389z6aWXaseOHbr++uv1pS99Sd/85jfPNBIAAABmsDO6le9yuXT77berqqrqlNe/+93vqry8XA8//LCampp05513Kj09XZdccsm7fq7o6Gg98MADZxJDkmS322S329714w6H/ZT/na3+uLtONY29mpcWrbzMGM1Pi1ZwkMPsWJLmzhhYGWNgPsbAGhgH8zEG5rPyGEy6mFZXV+v222+XYRinvD40NKQnn3xSDz74oJYsWaIlS5aoqqpKjz766HsW07MVHx8hm+3di+m46OiwgGUwW1V9t367zf+PhD1H2iRJDrtNuZkxWjQvXoVj/yXEmPt7MJvHYKZgDMzHGFgD42A+xsB8VhyDSRfTPXv2aO3atbrtttu0fPnyidePHj0qj8ejkpKSiddWrlypBx54QD6fL2BrR7u6Bt93xjQ6Okx9fcPyen0ByWC2Z7dXS5LsNpsMGTIMyeszVHmyR5Une/T8a8clSQnRocrLilFeZqzyMmOUlRIpxzSs6Z0LY2B1jIH5GANrYBzMxxiYz4wxiIuLOK33TbqYXnfddf/w9fb2dsXFxSk4OHjitcTERLlcLvX09ATsaU4+nyGfz3jf93m9Pnk8s+8CGBwZ1e6KFknShuJUXXtBnmqb+lTd2Kvqxj7VNPZqyOWRJHX2jaizYkS7K1olScFBdi1Ii9bCzBgtzIjRgvQYRYYFBSzrbB2DmYQxMB9jYA2Mg/kYA/NZcQym7Lio4eHhU0qppImfu93uqfoy+Ds7D7XIPfZ/qs0lmQoNdk7cupckn2GouXNINY29qm7oVVVjr1q7hiRJ7lGfjp7s0dGTPROfLy0hXAsz/EU1LytWKXFhp7VUAgAA4GxNWTENCQl5RwEd/3loaOhUfRm8jWEY2l7WKElakB6tnNSod7zHbrMpIzFCGYkR2rgsXZLUP+RWTeP4rGqvapv7NDpWbps7h9TcOaTXDzZLkqLDgyZu/edlxSp7mm7/AwCAuWfKimlKSoq6u7vl8XjkdPo/bXt7u0JDQxUdHT1VXwZvc7SuWy1js5+bSzJO+9dFhQdreV6iluclSpI8Xp/q2wZU3dA7UVa7+12SpL6hUZVWtqu0sl2SFBLk0IJ0/87//KxYLUiPVmiw6U+2BQAAs8CUNYrCwkI5nU7t379fq1atkiSVlpaqqKiIQ/MDZHy2NCLUqdWLks/48zgdds1Pi9b8tGhduDpLktTZO6Kqhh5VNfSqsqFHje2DkiTXqFdH6rp1pK5bkn9GNic18m+zqpmxio4IftevBQAA8G6mrJiGhYXpyiuv1D333KP/+q//Ultbm7Zu3ap77713qr4E3qZnwKWyqg5J0vqitCk/szQhJlQJMalatyRVkn+TVfVYSa1q6NWJ5j55vIZ8hqHa5n7VNvfrz2/VS5JS4sP9M6qZscrLilF64untxAMAAHPblN6Dveuuu3TPPffoU5/6lCIjI/WFL3xBF1100VR+CYx57UCTvGOnEUzmNv6ZiggN0rKFiVq20H/7f9TjVW1z/8SsalVDr4bHdv+3dg2ptWtIO8bWqcZEBmvpgkQtSItSXkaM0pMiZGdDFQAA+Ds24+9Pyp9h2tv73/PjTqddcXER6u4etNyRCGfK6/Ppjp/uUne/S4vnxemr15a8/y8KMJ/PUGPHoKoaelRZ7y+r4+tU/15EqFP5WbEqyIpVQXacspIj3/MsWpy92XgdzDSMgTUwDuZjDMxnxhgkJb1zg/Y/wq6VGehgdedE6ZuO2dLTYbfblJUcqazkSG1ZkSnDMNTZN6Kqev9mqqrGXjW0DUiSBkc8KqvqmFiKEBbiUF6mv6jmZ8cqJyVKTgs+Jg0AAAQWxXQGGt/0FBsZPLGz3mpsNpsSY8KUGBOm85anKy4uQnUN3Tpc26VjJ3t0rL5bDWMbqoZdXh2s6dTBmk5J/p3/CzOilZ8dp4KsWM1Pi1aQk6IKAMBsRzGdYdq6h1Re2yVJ2rgsfUadKRodEaxVi5K1auwEgYHhUVXV9+hYfY+OnezRybZ+GYZ/53/FiW5VnPDv/A9y2pWbHu2//Z8dp9z06Cnf7AUAAMxHMZ1hXtnfJMl/TNOm5da4jX+mIsOCVJKfpJL8JEnS0IhHVQ3+olpZ36MTzf3yGYZGPW97QtUbJ+R02LQgPUaFOXFalB2rBekxzKgCADALUExnkFGPd2Kn+/K8RMVFhZicaGqFhzpP2fk/4vaourF37NZ/j2qb+uT1GfJ4DVWOldfnJAU77VqYOV5U4zQvLWpGzSQDAAA/iukMsvdouwaGRyVZZ9NTIIUGO7V0foKWzk+QJLlHvapp6tOxk906WtetmrGi6vb4dPhEtw6P3foPDXYoPytWi7LjVJjDrn8AAGYKiukMMr7pKTkuTIXz4kxOM/2CgxwqzPGXTZ0nudxeVTf26khdt46e7FZtc58MQxpxn7qZavx4qsKcOC3KiVNGYoRsnKMKAIDlUExniPq2AVU39kqSzl+ewQH1kkKCHVoyP15L5sdLkoZdHlXW90wU1frWARl65/FU0eFBKhibTV2UE6eUuDCKKgAAFkAxnSHGZ0udDrs2FKeZnMaawkJOXaM6MDyqYyd7dHSsqDZ2+I+n6hsa1VtH2/TW0TZJUlxUiBbPi9OSefEqnBevmIhg074HAADmMorpDDDs8mhXRYskaU1hsiLDgkxONDNEhgVpZUGSVhb4d/33Drp17GS3f0a1rlut3cOSpO5+l9441KI3Dvl/jzOTIrR4XrwWz4tXQVasQoI5mgoAgOlAMZ0Bdle0yOX2Spobm54CJSYiWGsKU7SmMEWS1NU3oiN1/qJacaJLvQNuSVJD+6Aa2gf157fq5bDbtDAjRovnxWnx/HjNS2XHPwAAgUIxtTjDMPTy2G387ORILUiPNjnR7BEfHar1RWlaX5QmwzDU1Dmkwye6dLi2S0fre+Rye+X1Gf4HANT36JnXaxUW4lRhTpy/qM6LZ30qAABTiGJqcVUNvWoce3Tn+SsyKEEBYrPZlJEYoYzECF24Kkser0+1zX2qqO3S4bpuHW/sk88wNOzyaF9lu/ZVtkuSEqJDVDgv3l9Uc+IVzfpUAADOGMXU4l4Zmy0NDXZo3eIUk9PMHU6HXXmZscrLjNWV5/nX+R472aPDJ7pUcaJLzZ1DkqTOPpd2HGyeePBBVnKkls6P19IFCcrLjJHTwW1/AABOF8XUwvqG3Np7zL9z/NylqQoNZrjMEhbi1PK8RC3P8+/47+53+W/7n+jS4RPd6h30r0+tbxtQfduA/vjmSYWMnbu6dEG8ls6PV3JcuJnfAgAAlkfTsbAdB5vl8RqS2PRkNXFRIaeuT+0YVMWJblXUdunYyW65PT65Rr3aX92h/dX+81OT48ImZlMXZcfyDw0AAP4OfzNalM8wJm7j52fGKCMp0uREeDc2m00ZSZHKSIrURauzNOrxqrK+V4eOd6qitmvi/NS27mG93N2ol/c1ymG3KT8rdmw2NUGZSTyNCgAAiqlFlR/vUkfviCT/pifMHEHOU59I1dU3ovLaLpUf71TFiW4Nuzzy+oyJo6qe3F6j2MhgLZkfr6IFCVo8L56zagEAcxLF1KLGZ0ujw4O0Mj/Z5DQ4G/HRodq4LF0bl6XL6/Optqlfh453qry2Syea+2RI6hlwTxzyb5M0Pz1aS8eK6vy0aNntzKYCAGY/iqkFdfaO6ECNf13iecvSFeRkZ/ds4bDbtTAzRgszY/SRjQvUP+TW4RPdKh8rqr2DbhmSjjf16XhTn55/44SiwoNUtCBBxbkJWjo/XuGhzKYCAGYniqkFvXqgUYYh2SRtWpZudhwEUFR4sNYuTtHaxSkyDEP1bQMTt/2rGnrl9RnqHxrVzvIW7Sxvkd1mU15mjIoXJqg4N1HpCeGsTQUAzBoUU4vxeH167YD/TMyi3AQlxoaZnAjTxWazKTslStkpUfrguhwNuzw6fKJbB2o6dKimU72DbvmMvz2J6sntNUqMCVVxrr+kLsqOVXCQw+xvAwCAM0YxtZh9le3qGzsTkyOi5rawEKdWFiRpZUGSfIahk639OljdqQM1nRNrUzt6R/TyPv9O/2CnXYU5cSpemKhluQmKjw41+1sAAGBSKKYWM77pKSE6VEULEkxOA6uw22yalxqteanR+tCG+eobdOvQcX9Jrajt1LDLK7fHpwM1/td+LSkzKULFuYkqzk1QQU6s2d8CAADvi2JqIU0dgzp6skeSdH5JOjux8a6iI4InDvj3eH2qbujVwZpOHajpmHhcakP7oBraB/WH3XWKCAvSqkUpWjIvVkvmxSsshEsfAGA9/O1kIeOzpQ67TecVs+kJp8fpsGtRTpwW5cTp41sWqq1nWIdqOnWgukNHT/bI4/VpcHhUr5Y16NWyBjnsNi3KiVNJXqKWL0zklj8AwDIophbhcnv1RnmLJGllQZKiI4JNToSZKjk2TBeszNQFKzPlcnt1uK5L5ce7dKCmU119I/L6DFXUdqmitkuP/LlSOalRKslLVEleEk+gAgCYimJqEW8eadWwyyOJTU+YOiHBDpXkJWl1YYpiYsJVdqRZe4+2qayqQ43t/kel1rX0q66lX8++XqvEmFAtHyup+Vkxctg5QxcAMH0ophZgGIa27/Pfxs9IjFB+Vqy5gTAr2e02LUiPUXZylK7amKu27iHtr+pQWVWHKht6ZBj+Xf7b9jZo294GRYQ6VZyboJK8JC2Zz7pUAEDg8TeNBdQ296uutV+SdH5JBrdSMS2S48J10ZpsXbQmWwPDozpQ3aH9VR0qr+2Sa9SrwRGPdlW0aldFq5wOmwpz4lWSl6hlCxMVFxVidnwAwCxEMbWA7WUNkqTgILvOWZJqchrMRZFhQRO7/Ec9Xh0+0a2yqg7tr+5Q36BbHq+hQ8c7deh4p/TSMc1Pi9aK/EStKkhWSny42fEBALMExdRkA8Oj2nOkTZK0bnGqwkMZEpgryOnQsoX+mVGfYai2qU9lVR0qq2qfOIqqtrlPtc19eurV48pMitDKgmStKkhSeiKbpwAAZ44WZLKdh5o16vFJYtMTrMdusyk3I0a5GTH62Pm5au0aUllVh/ZVtaumoVeGxs9LrdVzO2qVGh+ulQVJWlWQrOyUSEoqAGBSKKYmMgxD2/c3SZIWpEcrJzXK5ETAe0uJD9cla7N1ydps9Qy4tK+yXaXH2nX0ZLcMQ2rpGtLvd9Xp97vqlBgTOlFS56dHy05JBQC8D4qpiY7Udau1y39rlNlSzDSxkSHasiJTW1Zkqn/IrbKqDpUea9fhE13y+gx19I7opT31emlPveKiQrQiP0mrCpKUlxnLU80AAP8QxdRE28ee9BQR6tTqRckmpwHOXFR4sDYuS9fGZekaGhnV/mp/ST10vEser0/d/S79tbRBfy1tUHR4kEry/TOpBdmxcjo4KxUA4EcxNUl3v0tllR2SpPVFaQoOcpicCJga4aFBOndpms5dmqYRt0cHazq191i7DtV0yjXqVd/QqF7d36RX9zcpItSp5Xn+3f1L5sdTUgFgjqOYmmRnebN8hiGJ2/iYvUKDnVpTmKI1hSlyj3pVXtul0mNt2l/dqWGXR4MjHr1xqEVvHGpRRKhTKwuStKYwRQXZsTx1CgDmIIqpSY7WdUvyb3riHEjMBcFBDq3IT9KK/CR5vD4dPtGt0mP+x6MODI9qcMSj1w4067UDzYoOD9KqRclaU5iihZkxbJwCgDmCYmoCr8+n6sY+SeLxo5iTnA67inMTVJyboBu8Ph2t69abR1q1r7Jdwy7/7f6X9zXq5X2NiosK0ZpCf0mdlxrFEVQAMItRTE1wsnVArlGvJIop4HTYtXRBgpYuSNCNF/tUfrxTbx5p1f7qDrlH/Runxnf3J8eGac1if0nNTIo0OzoAYIpRTE1QWd8jSbJJysuMMTULYCVBTrtK8pNUkp8kl9urAzUd2nOkTQdrOuXx+tTWM6wXd9bpxZ11ykiMmJhJZTkMAMwOFFMTjBfTjKRIRYQGmRsGsKiQYMfExqmhEY/Kqtq150ibKmq75DMMNXYM6pnXa/XM67XKSY3S2sIUrV6UrISYULOjAwDOEMV0mvkMQ1UNvZKk/CxmS4HTER7q1PqiNK0vSlP/kFulle3ac7hVx072yJBU19KvupZ+PbG9WgszYrRuib/QRobxDz8AmEkoptOsuWNQA8OjklhfCpyJqPBgnb88Q+cvz1B3v0t7j7Vpz5FW1YxtKKxu7FV1Y69+u61KxbkJOndpqopzExXk5PgpALA6iuk0qxybLZUopsDZiosK0YWrsnThqix19AzrraNt2lXRqob2AXl9hsqqOlRW1eF/ulphis5dkqrcjGh29gOARVFMp9n4+tLkuDDFRoaYGwaYRRJjw3Tpuhxdui5H9W0D2lXeol2HW9Q74NbgiEevlDXqlbJGJceG6ZylqTpnSYqS49g0BQBWQjGdRoZhTBTT/MxYU7MAs1lWcqSytizUx87P1eG6Lu0qb1FpZbvco/6d/c/tqNVzO2q1MCNG5y5N1erCZDYiAoAFUEynUUfviLr7XZKkPDY+AQFnt9u0dH6Cls5P0A1uj0qPtWtXRYuOnOiWob+tR/3Ntkoty03UOUtTVZybIKeD9agAYAaK6TQany2VpALWlwLTKjT4bzv7u/pG9ObhVu2saFFj+6A8XkOlle0qrWxXZFiQVhcm69wlqVqQznpUAJhOFNNpVNXQI0mKiQxWUmyYuWGAOSw+OlSXrsvRJWuzVd82oJ3lLdp9uFV9g24NDI9q+75Gbd/XqJS4MJ27NFXri9IUH835qAAQaBTTaXSs3r8jvyArllkYwAJsNpuyU6KUnRKlqzfn6vCJbu0qb9G+yna5PT61dg/rmddr9eyOWi2dn6CNy9K0bGEit/oBIEAoptOkd9Ct1q4hSVIeG58Ay3HY7SpakKCiBQkadvnXo+4sb9bRkz0yDOnQ8U4dOt6pqPAgrV+apvOWpSktIcLs2AAwq1BMp0kV60uBGSMsxKkNxWnaUJymtp5h7TjYpDcOtai736X+oVH9ac9J/WnPSS3MiNF5y9K0elGyQoP54xQAzhZ/kk6T8Y1PEaFOpScxywLMFMmxYbpqY66u3LBA5bWdeu1Asw5Ud8jrM962q79KawuTdV5xOhumAOAsUEynyXgxzcuMlZ2/tIAZx263qTg3UcW5ieoddGtXeYteO9Cklq4hudxevXagWa8daFZGYoTOK07TOUtTFRUebHZsAJhRKKbTYGjEo/q2AUmcXwrMBjERwbpkbbYuXpOl6sZevX6gWXuOtso96lNjx6Aee7laT75So5L8JG0sTtPiefFmRwaAGYFiOg2qG3tljP04n/WlwKxhs9mUlxmrvMxYfeIDedpzpFWvH2zW8aY+eX2G9h5t096jbYqPDtHGZem6YuNCBXHDBADeFcV0Gozfxg8OsisnJcrcMAACIizEqU3LM7RpeYYa2ge042Czdpa3aGB4VF19Lj37uv8xqMW5CdpckqGl8xNkt9NSAeDtKKbToHLsYP3c9BjOPwTmgMykSF17QZ4+uilX+6s79PqBJlXUdskwpAPVnTpQ3anEmFBtLsnQhuI01qICwBiKaYC5R72qbeqTxG18YK4Jctq1elGyVi9KVs+gW28eadOfdp1Q76BbHb0jevKVGj3zeq1WL0rWlhUZ7OgHMOdRTAOsttm/1kyS8jPZ+ATMVYkxobr+0kJdvDpTew636uV9jaqs75HH69OuihbtqmhRdkqktqzI1NrFKQoJcpgdGQCmHcU0wI6NrS912G1akEExBeY6p8OuNYUpWlOYoob2AW0va9TO8ha53F6dbB3Qr/54VE+8XK31RWnavCJDqfHhZkcGgGlDMQ2w8Sc+zUuNYgYEwCkykyJ1w0UF+timXO2qaNH2fY1q7BjUkMujv+yt11/21mvxvDhtWZGpZQsT5LCzRh3A7EYxDSCvz6fqRtaXAnhvYSFObVmRqc0lGaqs79H2skaVHmuX12fo8IluHT7RrbioEJ2/PF0bl6UrJjLE7MgAEBAU0wA62Tog16hXkpRHMQXwPmw2mwqy41SQHafeAZdeO9CkV/Y3qbvfpe5+l555vVbPv3FCKwuStLkkQ/lZsWyWAjCrUEwDaPz8UpukPDY+AZiEmMgQXbF+vj54To72V3Vqe1mDDp/oltdnaM+RNu050qbMpEhdtDpLaxenKMjJbX4AMx/FNIDGi2lGUqQiQoPMDQNgRnLY7VpZkKSVBUlq7hzUK2VN2nGoWcMujxraB7T1D0f0u1drdMGKDJ1fksGZqABmNIppgPgMQ1UNvZKk/CxmSwGcvbSECH3iA3m6auMC7T7coj+/Va/mziH1Dbr1zOu1enFXndYvTdWFq7OUlhBhdlwAmDSKaYA0dwxqYHhUEhufAEytkGCHNi3P0HnL0lV+vEt/fuukDp/o1qjHp1f2+9elFucm6OLVWVqUE8c6VAAzBsU0QCrHZksliimAwLDbbCrOTVBxboLq2wb057dOandFq7w+QwdrOnWwplNZyX9bh8ojkQFYHX9KBcj4+tLkuDDFcrQLgADLSo7UTZct1n3/eq4uP3eeIsP869rr2wb0i98f0b/9dKde3Hli4k4OAFgRM6YBYBjGRDHNz4w1NQuAuSU2MkRXbVygy87J0a5y/zrUlq4h9Q649fRrx/XizhNaX5SmC1dn8VQpAJZDMQ2Ajt4Rdfe7JHEbH4A5QoIcOr8kQxuXp+tQTaf+/Fa9jtR1y+3xaXtZo7aXNWr5wkRdtDpLBdmchwrAGiimATA+WyqxIx+Auew2m5YtTNSyhYk62dqvP79VrzcP+9eh7q/u0P7qDmWnROri1dlaXZjMOlQApuJPoACoauiRJMVGBispNszcMAAwJjslSp+9fLG+e8u5uuycHEWE+ucmTrYO6MEXD+vun+/W9rJGjXq8JicFMFcxYxoAx+rHzy/l9hgA64mLCtFHN+Xq8nPmaWd5s/78Vr1au4fV0TuiX790TM+/UatL1mTr/OUZCgl2mB0XwBxCMZ1ivYNutXYNSWJ9KQBrCwl2aPOKTG1anqHSyna9uPOE6tsG1Dvg1uMvV+v3u+p04apMXbAyU+E8vQ7ANKCYTrGqt68vZUc+gBnAbrdp9aJkrSpI0sGaTr2484Rqmvo0MDyqZ16v1Z/2nNSWFZm6cHWWonnkKYAAophOsfGNTxGhTqUn8UhAADOHbWyjVHFugo7WdevFXXU6UtetYZdXv99Vp7/srdemZRm6ZG224qI4nxnA1KOYTrHxYpqXGSs760sBzEA2m02F8+JVOC9eNY29enHnCR2o6ZR71Ke/7K3X9rIGbShK06XrctjgCWBKUUyn0NCIR/VtA5KkPI6JAjAL5GbE6EtXL9PJ1n69uKtOpUfb5PEaemV/k1470Ky1i1N02Tk5Sk/kDhGAs0cxnULVjb0yxn7MxicAs0l2SpT+9cqlau4c1B921WlXRat8hqFdFS3aXdGiFQVJuvycecpJjTI7KoAZjGI6hcZv4wcH2ZWTwh/OAGaftIQI3XT5Yn14w3z98c2Tev1gszxen0qPtav0WLuKcxN0+TnztDCTu0YAJo9iOoUqxw7Wz02P4ekpAGa1xNgw3XBxga5YP08v7Tmp7WWNco/6dLCmUwdrOlWYE6erNi1QbjoFFcDpoz1NEfeoV7VNfZK4jQ9g7oiNDNE1W/J03y3n6vJz5yksxD/fcaSuW9/+31L96KmDahhbew8A74cZ0ylS29wnr8+/wpRiCmCuiQoP1lUbF+iSNdnaVlqvl/ac1LDLq7KqDu2v6tDaJSn68Ib5SokLNzsqAAujmE6RY2PrSx12mxakR5sbBgBMEh7q1IfWz9eWFZn64+46/bW0QW6PT7srWvXWkTadV5ymK9bP5xxUAP8QxXSKjD/xaV5qlEKCeLY0gLktMixIV29eqAtXZ+nFnSf06v4meX3+Y6Z2HGrRlhUZ+uA5OTxJCsApWGM6Bbw+n6obWV8KAH8vNjJE119UoP+6eZ3WL02VzSZ5vD79+a163fnALj37+nENjXjMjgnAIiimU+Bk64Bco15JUh7FFADeISk2TDddvljfuGmtVhYkSZJcbq+ef+OE7nxgp/74Zt3En6MA5i6K6RQYP7/UJimPs/sA4F1lJEbo1o8U6T//aZWWLoiXJA2OePTk9hr9+8926eV9DfJ4fSanBGAWiukUGC+mGUmRiggNMjcMAMwA81Kj9ZWPL9ed15VM/IO+d8CtR/5cqbt/vltvHGqWz2e8z2cBMNtQTM+SzzBU1dArSSrgNj4ATEpBdpz+/ZMr9OWrlyk7JVKS1NE7ol/8/oj+4xdvqvRYmwyDggrMFezKP0vNHYMaGB6VJOVlcRsfACbLZrOpODdBSxfEq/RYu5557bhauobU3Dmk+58p17zUKF17QR6bS4E5gGJ6lirHZkslduQDwNmw22xavShZK/ITtau8Vc/tOK7OPpdOtPTrvx/dp1WLkvXx83OVGBtmdlQAAUIxPUvj60uT48IUG8mB0QBwthx2uzYUp2nt4hS9sr9Rz++o1eCIR3uPtml/VYcuWZulD67LUWgwf4UBsw1rTM+CYRgTxZTZUgCYWkFOuy5claV7//kcXbAyU3abTR6vTy/urNNd4xukWH8KzCoU07PQ0Tui7n6XJCk/M9bcMAAwS0WGBemTF+brGzetmThiqnfArV/8/oi+9fBeVTX0mBsQwJShmJ6F8dlSScpn4xMABFR6YoS+8vHl+vLVxUqND5cknWjp172P7NMDz5Wrs3fE5IQAzhYLdM7C+L/SYyODlcRifACYFsW5iVo8L17b9zXquR21GnJ5tOfI+PrTbF26NkchwQ6zYwI4A8yYnoVj9f4d+flZsbLZbCanAYC5w+mw68LVWbr3n9dp84oM2WyS2+PT82+c0N0P7tau8hbWnwIzEMX0DPUOutXaNSSJjU8AYJao8GDdcFGBvv6ZNVoyL06S1N3v0oMvHtZ//bpUNY297/MZAFgJxfQMVb19fSkbnwDAVJlJkfrKNcv1xY8WKyXOv7TqeFOfvv3rUv38hQp19bH+FJgJWGN6hsY3PkWEOpWeFGFuGACAbDabluclaumCeP21tEHPv3FCwy6Pdle0at+xdl26LkeXr59ndkwA74EZ0zM0XkzzMmNlZ30pAFiG02HXxWuyde8/r9P5JX9bf/rcjlrd+dOdeuNgkwzWnwKWRDE9A0MjHtW3DUhifSkAWFV0eLBuvLhA93x6jQpz/OtPu/pc+u+H39L3nzzA7X3AgiimZ6C6sVfj/9bO4/xSALC0rORIffXa5frCVUVKjAmVJJVVduj/PPSmtu2tl8/H7ClgFRTTMzB+Gz84yK6clChzwwAA3pfNZlNJfpL+65/X6cpNubLZJJfbq99sq9K3f106cRcMgLkopmegcuxg/dz0GDkd/BYCwEwRGuzUTR9aqns+s0bZKZGSpNrmPn3jV2/pd6/UyD3qNTkhMLfRqibJPepVbVOfJNaXAsBMNT8tWv/xqVX6+OaFCg6yy+sz9IfddfrPX+xRxYkus+MBcxbFdJJqm/vkHVuPRDEFgJnLYbfrkrXZ+uZNa7V0frwkqa1nWP/vsf36xYuH1T/kNjkhMPdQTCfp2Nj6UofdpgXp0eaGAQCctaTYMN328WW6+YrFigoPkiS9Ud6i//Pgm9pV3sLRUsA0ophO0vgTn+alRSkkyGFuGADAlLDZbFq3JFXf/tw6bShKkyQNDI/qwRcP63uP71dbz7DJCYG5gWI6CV6fT9WNY+tLeQwpAMw6kWFB+sxlhfq3a5creezRphUnuvWfD72pP+6uk8frMzkhMLtRTCfhZOuAXGM7NllfCgCzV+G8eH3jM2t0+bk5cthtcnt8evKVGn3z4b2qbe4zOx4wa1FMJ2H8/FKbpLxMDtYHgNksOMihqzbm6mufXq3csT0F9W0D+tb/7tVvt1VpxO0xOSEw+1BMJ2G8mGYmRyo8NMjcMACAaZGZFKm7rl+pT16Yr9BghwxD+sveev3HQ2/qQHWH2fGAWYViOgl1rf2SWF8KAHON3W7TBSsz9a3PrlVJXqIkqbPPpR/87qAe/tNRudwczA9MBYrpJCzLTVRsZLA2laSbHQUAYIL46FB94aPFuvUjRYqJDJYkvbq/Sff86i3WngJTgGI6CTdcXKDvfX6DMpMizY4CADDRyoIkffOmtVqZnyRJau0a0n/9ulS/33VCPh/nngJnimIKAMAZiAwL0r9+ZKk+fekihQQ55PUZeurV4/rub8vU0cu5p8CZoJgCAHCGbDabzluWrns+s3riaYCV9T362ta3tPtwi8npgJmHYgoAwFlKiQvXv39yha44d55sNmnY5dHPnz+sn79QoaERjpUCThfFFACAKeB02PWRjQv0759cocSYUEnS7opWfW3rnonjBgG8N4opAABTKC8zVl//zBqduzRVktTZN6Lv/Gafnnq1hkeaAu+DYgoAwBQLC3Hqs5cv1r98eInCQ5wyDOn3u+p07yOlaukaMjseYFkUUwAAAmRNYYq+cdMaLcqOlSTVNvfrnl/u0av7G2UYHCsF/D2KKQAAARQfHaqvXluiq8/PlcNuk3vUp4f/dEw/fvqQ+ofcZscDLIViCgBAgNntNl26Lkf/342rlBofLkkqq+rQf/5ij8qPd5qcDrAOiikAANMkJzVKX/v0am0uyZAk9Q669b0nDug3f6nUqMdrcjrAfBRTAACmUUiQQzdcXKAvfqxYUeFBkqRtpQ36xq/2qrFj0OR0gLkopgAAmGD5wkR946a1Ks5NkCQ1dgzqW/+7V6XH2k1OBpiHYgoAgEliIoL1pY8V65MX5stht8nl9ur+Zw7pmdeOy8eufcxBFFMAAExks9l0wcpM3XFdiWIigiVJL+w8oR/97iCPM8WcQzEFAMAC8jJj9Z//tFoL0qMlSQdqOvWt/92r5k7WnWLuoJgCAGARcVEhuvO6FdpQnCZJauka0jcf3quyKtadYm6gmAIAYCFBTrs+fekiXX+Rf93piNurHz11SM/tqGXdKWY9iikAABZjs9m0ZUWm/u0TJYoeO1LquR21uv/pQxp2se4UsxfFFAAAi8rP8q87nZcaJcn/tKhv/e9etXQNmZwMCAyKKQAAFhYfHaq7rl+h9UtTJUnNnUP65sNv6UB1h8nJgKlHMQUAwOKCnA595rJCXfeBPNltNg27vPrh7w7qhTdYd4rZhWIKAMAMYLPZ9IFVWfrqtcsVGRYkQ9Izr9fqp8+Us+4UswbFFACAGWRRTpy+9k+rlZPiX3daWtmub/+6VK3drDvFzEcxBQBghkmI8a87PWdJiiSpqWNQ3/zVXh063mlyMuDsUEwBAJiBgoMc+uzli3XtBf51p0Muj77/xAH9ftcJGaw7xQxFMQUAYIay2Wy6aHWWbr9m2cS606dePa6fPlehETfrTjHzUEwBAJjhCufF6z8/tUrZyZGSpL1H23TvI/vUM+AyORkwORRTAABmgcTYMN11w0qtXexfd1rfNqB7HylVW8+wycmA00cxBQBglggJcujmKxbrinPnSZLae0Z07yOlamgbMDcYcJoopgAAzCI2m00f2bhA116QJ0nqHXDrvx/dp+rGXpOTAe+PYgoAwCx00eos3XRZ4cSO/f/7WJnKazlOCtZGMQUAYJZaX5SmW69aKqfDLveoTz948qDeOtpmdizgXVFMAQCYxUrykvSVjy9TaLBDXp+hB54t1yv7G82OBfxDFFMAAGa5RTlxuuO6komzTv/3T8c4iB+WRDEFAGAOmJcarbuuX6H46BBJ/oP4n9xeQzmFpVBMAQCYI9ISInT39SuVlhAuSfrTnpP65R+PyuvzmZwM8KOYAgAwh8RHh+rOT65QTmqUJGnHwWb99NkKjXq8JicDKKYAAMw50eHBuuMTJVqUHStJ2lfZru8/eVDDLo+5wTDnUUwBAJiDwkKcuu3jy1SSlyhJOlLXrf/7WJn6h9wmJ8NcRjEFAGCOCnI69K8fWar1RamSpNrmfv33o/vU1TdicjLMVRRTAADmMIfdrk9/sFAXrc6SJDV3DuneR0rV0jVkcjLMRRRTAADmOLvNpmu2LNRVGxdIkjr7XLr3kVLVtfSbnAxzDcUUAADIZrPp8nPn6YaL8mWT1D80qu/+dp+Onew2OxrmEIopAACYsHlFpm7+0BI57DYNu7z63hMHtL+6w+xYmCMopgAA4BRrF6foix8rVrDTrlGPTz9+6pBKj7WZHQtzAMUUAAC8Q9GCBN1+7XKFhzjlMwz97PkKHanjtj4Ci2IKAAD+obzMWN1+7XKFBDnk8Rr60VMH2RCFgKKYAgCAdzU/LVq3XrVUDrtNI26v/ueJ/Wrt5igpBAbFFAAAvKel8xN00+WFsknqGxrV9x7fr94Bl9mxMAtRTAEAwPtatzhVn/hAniSpvWdE33vigIZGPCanwmxDMQUAAKflA6uydPm58yRJ9W0D+uFTBzXq8ZobCrMKxRQAAJy2j5w3X5uWp0uSKut79MBzFfL6fCanwmxBMQUAAKfNZrPphosKtDI/SZJUVtWhX790TIZhmJwMswHFFAAATIrdbtPNH1qsRdmxkqTXDjTr6deOmxsKswLFFAAATFqQ06EvfLRY2SmRkqTf76rTX96qNzkVZjqKKQAAOCNhIU7d9vHlSo4NkyT99q9V2lXRYnIqzGQUUwAAcMZiIoL1lWuXKyYiWJK09fdHdOh4p8mpMFNRTAEAwFlJjg3TbR9fprAQh7w+Q/c/c0g1Tb1mx8IMRDEFAABnLTslSl/8aLGcDrvcoz59/4kDauoYNDsWZhiKKQAAmBIF2XH6lw8vkc0mDY549L0n9qurb8TsWJhBKKYAAGDKrMhP0qcuWSRJ6upz6f89vl8Dw6Mmp8JMQTEFAABTauOydH100wJJUnPnkL7/5AG53Dy6FO+PYgoAAKbcB9fl6AOrMiVJx5v6dP+zh+Tx8uhSvDeKKQAAmHI2m03XXpCndYtTJEnlx7u09Q9H5OPRpXgPFFMAABAQdptNn7msUEsXxEuSdle06rd/qZRBOcW7oJgCAICAcTrsuvXKIi1Ij5YkvbSnXk9trzY5FayKYgoAAAIqJNihL1+9TGkJ4ZKk//3DYVXUdpmcClZEMQUAAAEXGRak2z6+TBFhQTIM6YFny9U76DY7FiyGYgoAAKZFYkyYbr5isSSpd9CtB1+okM/HelP8DcUUAABMm5L8JF25KVeSdPhEt36/64S5gWApFFMAADCtbvzg4onNUM/uqNWxk90mJ4JVUEwBAMC0CnLadetVRQoPccowpJ89X6G+IdabgmIKAABMkBQbps9cVihJ6hlw66EXDnP4PiimAADAHCvyk/SBlf7HlpbXdumPu+tMTgSzUUwBAIBprt68UDmpUZKkZ16rVVVDj7mBYCqKKQAAME2Q065bPrxEYSEO+QxDDzxXoYHhUbNjwSQUUwAAYKrkuHD906X+9abd/S794sXDMlhvOidRTAEAgOlWL0rW5pIMSdKBmk69tKfe5EQwA8UUAABYwrUXLFR2cqQk6alXa1TT2GtyIkw3iikAALCEIKdDt1y5VCHBDnl9/vWmgyOsN51LKKYAAMAyUuLD9alLCiRJnX0j2vr7I6w3nUMopgAAwFLWLU7VxmXpkqSyqg5tK20wORGmC8UUAABYznUfyFNmUoQk6YmXq1Xb3GdyIkwHiikAALCc4KCx9aZB4+tNyzU04jE7FgKMYgoAACwpLSFCN1ycL0lq7xnRr/7IetPZjmIKAAAs69yladpQlCZJ2nusXdvLGk1OhECimAIAAEv75IX5Sk/0rzd97K9VqmvpNzkRAoViCgAALC0k2KFbPrxEwU67PF5DP32uXMMu1pvORhRTAABgeRlJkfrkhf71pm3dw3r4T0dZbzoLUUwBAMCMsKE4TecsSZEk7TnSptcONJmcCFONYgoAAGYEm82mGy4uUGp8uCTpN9uqVN82YHIqTCWKKQAAmDFCg5265cqlCnLaNerx6afPlmvEzXrT2YJiCgAAZpSs5Eh94gN5kqSWriG9uLPO5ESYKhRTAAAw42xalq7i3ARJ0p/fOqm27iGTE2EqUEwBAMCMY7PZdO0FeXLYbfJ4DT3+crXZkTAFKKYAAGBGSo0P14WrsiRJZVUdqjjRZXIinC2KKQAAmLEuP3eeosODJEmPbauS1+czORHOBsUUAADMWOGhTl21KVeS1NgxqFfKONt0JqOYAgCAGW1DUZpyUqIkSc++flwDw6MmJ8KZopgCAIAZzW63TRwfNTji0XOv15qcCGeKYgoAAGa8/KxYrSlMliRtL2tUQztPhJqJKKYAAGBWuPr8hQp22uUzDP12W5UMwzA7EiaJYgoAAGaFhJhQXbouR5J0pK5b+6s6TE6EyaKYAgCAWeOStdmKjw6RJD32cpVGPRwfNZNQTAEAwKwREuTQxzcvlCS194zoL3vrTU6EyaCYAgCAWWX1omTlZcZIkl7YeUI9Ay6TE+F0UUwBAMCsYrPZdN0H8mWT5HJ79dQrNWZHwmmimAIAgFknJzVK5y1LkyS9Ud6i4019JifC6aCYAgCAWekjG3MVFuKQJP1mW6V8HB9leRRTAAAwK8VEBOuKc+dLko439enNilaTE+H9UEwBAMCs9YFVmUqJD5ckPflKtUbcHpMT4b1QTAEAwKzldNj1iQv8x0f1DLj1h911JifCe6GYAgCAWa04N1FFCxIkSX96s17tPcMmJ8K7oZgCAIBZ79oLFspht8nj9emJ7dVmx8G7oJgCAIBZLy0hQheszJQklR5r15G6bpMT4R+hmAIAgDnhQ+vnKTIsSJL0221V8vp8JifC36OYAgCAOSE8NEhXbVogSWpoH9BrB5pNToS/RzEFAABzxsbidGUlR0qSnnntuAZHRk1OhLejmAIAgDnDbrfpug/kSZIGhkf13I5akxPh7SimAABgTinIjtOqRcmSpJdLG9XYMWhyIoyjmAIAgDnn4+fnKshpl88w9Nhfq2QYhtmRIIopAACYgxJjw3TJmmxJUkVtlw5Ud5qcCBLFFAAAzFEfXJejuKgQSdJjL1dp1MPxUWajmAIAgDkpJNihq8/PlSS1dQ9rW2m9yYlAMQUAAHPW2sUpWpgRI0l64Y0T6h1wmZxobqOYAgCAOctms+kTY8dHjbi9euZ1jo8yE8UUAADMafPTorV+aaokaWd5iwaGOXTfLBRTAAAw5108tkPf4/Vpx0EeVWoWiikAAJjzMpMjtTDTv9b0lbJG+TjX1BQUUwAAAEmbSzIkSW09wzp8osvkNHOTZYppbW2tVqxYYXYMAAAwR60qSFZkWJAkafu+RpPTzE2WKKbDw8P6zne+o5CQELOjAACAOSrIadd5xWmSpP3VHerqGzE50dxjiWL67W9/W7feeqvCwsLMjgIAAOawTSUZskkyDOm1A01mx5lznNP1hR5//HH9+te/PuW1X/ziF3rllVe0aNEiFRUVTVcUAACAfyg5NkxLFsSr/HiXXj3QpMvPnSenwxLzeHPCtBXTa665Rtdcc807Xn/++edlt9v10ksvqb29XZ/97Gf10EMPTVcsAACAU2wuyVD58S71Dri1v6pDqxYlmx1pzpi2YvpuHn300Ykfb9myhVIKAABMtSw3UfHRIerqc2l7WSPFdBqd8dy02+3W5ZdfrjfffHPiNZfLpbvvvlurVq3Shg0btHXr1ikJCQAAMF3sdps2LUuXJB2p61ZL15DJieaOM5oxdblcuv3221VVVXXK69/97ndVXl6uhx9+WE1NTbrzzjuVnp6uSy655LQ+78svvzzpLHa7TXa77V0/7hhbF+JgfYhpGAPzMQbmYwysgXEw30wZg80rM/X8Gyfk9Rl69UCTPnlhvtmRpoyVx2DSxbS6ulq33367jL97IsLQ0JCefPJJPfjgg1qyZImWLFmiqqoqPfroo6ddTM9EfHyEbLZ3L6bjoqPZ8W82xsB8jIH5GANrYBzMZ/UxiIuL0LqiNL1xoElvHGzW5z5SrJAgh9mxppQVx2DSxXTPnj1au3atbrvtNi1fvnzi9aNHj8rj8aikpGTitZUrV+qBBx6Qz+eT3R6YVt7VNfi+M6bR0WHq6xuW1+sLSAa8N8bAfIyB+RgDa2AczDeTxuC8olS9caBJA8OjeumN4zpv7Pb+TGfGGMTFRZzW+yZdTK+77rp/+Hp7e7vi4uIUHBw88VpiYqJcLpd6enoUHx8/2S91Wnw+Qz7f+z/P1uv1yeOx9gUw2zEG5mMMzMcYWAPjYL6ZMAZ5GTFKSwhXc+eQtu1t0DlLUs2ONKWsOAZTNo05PDx8SimVNPFzt9s9VV8GAABgWthsNp2/PEOSVNvcp7qWfpMTzX5TVkxDQkLeUUDHfx4aGjpVXwYAAGDarC9KVbDTX5e2lzWYnGb2m7JimpKSou7ubnk8nonX2tvbFRoaqujo6Kn6MgAAANMmPDRIaxanSJJ2H27V0IjnfX4FzsaUFdPCwkI5nU7t379/4rXS0lIVFRUFbOMTAABAoG0u8d/Od4/6tLO82eQ0s9uUNcawsDBdeeWVuueee3Tw4EFt27ZNW7du1Y033jhVXwIAAGDazU+L1rzUKEnS9rLGdxyZiakzpVOZd911l5YsWaJPfepT+vrXv64vfOELuuiii6bySwAAAEy78VnT5s4hHTvZY26YWeyMnvw07tixY6f8PCwsTN/5znf0ne9856xCAQAAWMmaxSl6/OVqDbk82l7WqEU5cWZHmpVY/AkAAPA+QoIcOrfIf47pvsp29Q64TE40O1FMAQAATsP47Xyvz9BrB9kEFQgUUwAAgNOQlhChRdmxkqTX9jee1pMnMTkUUwAAgNO0eUWmJKmzz6WDNZ0mp5l9KKYAAACnqSQvUTER/keuby9rNDnN7EMxBQAAOE1Oh10bl6VLksqPd6q9Z9jkRLMLxRQAAGASNi1Pl80mGZJe2c+s6VSimAIAAExCfHSoli9MlCS9fqBZox6fyYlmD4opAADAJI0fHTUwPKrSY20mp5k9KKYAAACTtHh+vJJjwySxCWoqUUwBAAAmyW6zaVOJfxNUVUOvGtoGTE40O1BMAQAAzsCGojQ5Hf4qtZ1NUFOCYgoAAHAGosKDtXpRkiRpV3mLRtwekxPNfBRTAACAM7S5xP8kqBG3V7srWk1OM/NRTAEAAM5Qbka0MpMiJfk3QRmGYXKimY1iCgAAcIZsNps2r/AfHVXfNqCapj6TE81sFFMAAICzsG5xikKCHZKk7fsaTE4zs1FMAQAAzkJYiFPnLkmVJL11tE39Q26TE81cFFMAAICzNP4kKI/X0I5DzSanmbkopgAAAGcpMzlSCzNjJEmvljXJxyaoM0IxBQAAmALjs6ZtPcM6XNtlcpqZiWIKAAAwBVYVJCsyLEiS/+goTB7FFAAAYAoEOe06rzhNkrS/ukNdfSMmJ5p5KKYAAABTZFNJhmySDEN6dX+T2XFmHIopAADAFEmODdOSBfGSpNcONsnj9ZmcaGahmAIAAEyh8U1QvQNuHavvMTfMDEMxBQAAmEJL58fL6bBJkipP9pgbZoahmAIAAEyhIKdDC9KiJUlVDT3mhplhKKYAAABTLC8rVpJU09THOtNJoJgCAABMsYKxYjrq8elEc7+5YWYQiikAAMAUy82Ikc2/zFSV3M4/bRRTAACAKRYW4lR2SpQkqZKd+aeNYgoAABAA+ZmxkqSqhl75fIa5YWYIiikAAEAA5I+tMx12edTQPmBumBmCYgoAABAAeVkxEz/mdv7poZgCAAAEQHR4sNISwiVJlQ29JqeZGSimAAAAATJ+O7+yvkeGwTrT90MxBQAACJDxDVB9g261dQ+bG2YGoJgCAAAEyPiMqSQdY53p+6KYAgAABEhCTKgSokMkSVUU0/dFMQUAAAig8VlTZkzfH8UUAAAggPLGimlH74i6+kbMDWNxFFMAAIAAKnjbOtPKhh7TcswEFFMAAIAASo0PV1R4kCSpqp7zTN8LxRQAACCAbDbbxLFRzJi+N4opAABAgI2vM21sH9TA8Ki5YSyMYgoAABBgb19nWsWs6buimAIAAARYVnKkQoMdkvyPJ8U/RjEFAAAIMLvdpoUZMZKkSjZAvSuKKQAAwDQYP2i/rqVfI26PuWEsimIKAAAwDcaLqc8wVNPUZ24Yi6KYAgAATIP5aVFyOvzVq/Jkj7lhLIpiCgAAMA2CnA4tSIuSxM78d0MxBQAAmCb52bGSpJqmPnm8PnPDWBDFFAAAYJqMPwFq1OPTieZ+c8NYEMUUAABgmuRmxMhm8/+Yx5O+E8UUAABgmoSFOJWd4l9nykH770QxBQAAmEbjjyetauiVz2eYG8ZiKKYAAADTKG9snemwy6OG9gFzw1gMxRQAAGAa5WXFTPyY2/mnopgCAABMo+jwYKUlhEuimP49iikAAMA0G388aWVDrwyDdabjKKYAAADTbLyY9g261dY9bG4YC6GYAgAATLPxg/Yl6Ri38ydQTAEAAKZZQkyoEqJDJUlVFNMJFFMAAAAT5I/tzmfG9G8opgAAACYYX2fa0Tuirr4Rc8NYBMUUAADABOPFVJIqG3pMy2ElFFMAAAATpMaHKyo8SJJUVd9rchproJgCAACYwGazTezO56B9P4opAACAScZv5zd2DGpgeNTcMBZAMQUAADDJ29eZVrHOlGIKAABglqzkSIUGOyRxO1+imAIAAJjGbrdpYab/PNNKNkBRTAEAAMw0vgGqrqVfI26PuWFMRjEFAAAw0fg6U59hqKapz9wwJqOYAgAAmGh+WrScDn8lqzzZY24Yk1FMAQAATBTktGtBerQkduZTTAEAAEyWn+XfAFXT1KdRj8/kNOahmAIAAJhsfJ3pqMenupZ+c8OYiGIKAABgstz0GNls/h9XzuHb+RRTAAAAk4WFOJWTEiVpbh+0TzEFAACwgPHb+VUNvfL5DHPDmIRiCgAAYAF5YwftD7s8amgfMDeMSSimAAAAFpA3tjNfmru38ymmAAAAFhAdHqy0hHBJFFMAAACYrGBsnWllQ68MY+6tM6WYAgAAWETeWDHtG3SrtXvY3DAmoJgCAABYxPiMqTQ3b+dTTAEAACwiPjpUCdGhkqQqiikAAADMNH6e6TGKKQAAAMyUP3ZsVEfviLr6RkxOM70opgAAABaS//Z1pg09puUwA8UUAADAQlLjwxUdHiRJqqrvNTnN9KKYAgAAWIjNZps4Nmqu7cynmAIAAFhMfmasJKmxY1ADw6PmhplGFFMAAACLefs607l0bBTFFAAAwGKykiMVGuyQNLc2QFFMAQAALMZut2lhpv/YqMo5tAGKYgoAAGBB448nrWvp14jbY26YaUIxBQAAsKC8sQ1QPsNQTVOfuWGmCcUUAADAguanRcvp8Fe1ypM95oaZJhRTAAAACwpy2rUgPVqSVDVHNkBRTAEAACxq/NiomqY+jXp85oaZBhRTAAAAi8rP8u/MH/X4VNfSb3KawKOYAgAAWFRueozsNpsk6Vh9t8lpAo9iCgAAYFFhIU5lJkVIkurbBkxOE3gUUwAAAAsLGXsClM9nmJwk8CimAAAAsASKKQAAACyBYgoAAABLoJgCAADAEiimAAAAsASKKQAAACyBYgoAAABLoJgCAADAEiimAAAAsASKKQAAACyBYgoAAABLoJgCAADAEiimAAAAsASKKQAAACyBYgoAAABLoJgCAADAEiimAAAAsASKKQAAACzBZhiGYXYIAAAAgBlTAAAAWALFFAAAAJZAMQUAAIAlUEwBAABgCRRTAAAAWALFFAAAAJZAMQUAAIAlUEwBAABgCRRTAAAAWMKsKKYul0t33323Vq1apQ0bNmjr1q3v+t7Dhw/r6quv1rJly/TRj35U5eXl05h09prMGNxyyy0qKCg45b/t27dPY9rZze126/LLL9ebb775ru/hOgis0xkDroPAaG1t1Re/+EWtWbNG5513nu699165XK5/+F6ug8CZzDhwLQRGXV2dbrrpJpWUlOj888/XQw899K7vtdK14DTtK0+h7373uyovL9fDDz+spqYm3XnnnUpPT9cll1xyyvuGhoZ0880364orrtB///d/67e//a3++Z//WX/5y18UHh5uUvrZ4XTHQJJqamp033336Zxzzpl4LSYmZjrjzloul0u33367qqqq3vU9XAeBdTpjIHEdBIJhGPriF7+o6OhoPfroo+rt7dXdd98tu92uO++885T3ch0EzmTGQeJaCASfz6ebb75ZRUVFeuaZZ1RXV6evfOUrSklJ0RVXXHHKey13LRgz3ODgoFFUVGTs3r174rX777/fuP7669/x3ieffNLYsmWL4fP5DMMwDJ/PZ1x44YXGU089NW15Z6PJjIHL5TIKCwuN48ePT2fEOaGqqsr40Ic+ZFxxxRVGfn7+KePxdlwHgXO6Y8B1EBjV1dVGfn6+0d7ePvHaCy+8YGzYsOEd7+U6CJzJjAPXQmC0trYaX/rSl4z+/v6J12699Vbja1/72jvea7VrYcbfyj969Kg8Ho9KSkomXlu5cqUOHDggn893ynsPHDiglStXymazSZJsNptWrFih/fv3T2fkWWcyY3D8+HHZbDZlZWVNd8xZb8+ePVq7dq0ef/zx93wf10HgnO4YcB0ERlJSkh566CElJiae8vrAwMA73st1EDiTGQeuhcBITk7W97//fUVGRsowDJWWluqtt97SmjVr3vFeq10LM/5Wfnt7u+Li4hQcHDzxWmJiolwul3p6ehQfH3/KexcuXHjKr09ISHjfW254b5MZg+PHjysyMlJ33HGH9uzZo9TUVH3hC1/Qpk2bzIg+q1x33XWn9T6ug8A53THgOgiM6OhonXfeeRM/9/l8euSRR7Ru3bp3vJfrIHAmMw5cC4G3ZcsWNTU1afPmzbr44ovf8XGrXQszfsZ0eHj4lEIkaeLnbrf7tN779+/D5ExmDI4fP66RkRFt2LBBDz30kDZt2qRbbrlFhw4dmra8cx3Xgfm4DqbHfffdp8OHD+u22257x8e4DqbPe40D10Lg/fCHP9QDDzygI0eO6N57733Hx612Lcz4GdOQkJB3/OaN/zw0NPS03vv378PkTGYM/vVf/1U33HDDxML2RYsWqaKiQk888YSKioqmJ/Acx3VgPq6DwLvvvvv08MMP63/+53+Un5//jo9zHUyP9xsHroXAG/99dLlc+upXv6o77rjjlCJqtWthxs+YpqSkqLu7Wx6PZ+K19vZ2hYaGKjo6+h3v7ejoOOW1jo4OJScnT0vW2WoyY2C329+x23LBggVqbW2dlqzgOrACroPA+uY3v6lf/vKXuu+++/7hrUuJ62A6nM44cC0ERkdHh7Zt23bKawsXLtTo6Og71vpa7VqY8cW0sLBQTqfzlEW6paWlKioqkt1+6re3bNkylZWVyTAMSf4jLfbt26dly5ZNZ+RZZzJj8O///u+66667Tnnt6NGjWrBgwXREhbgOrIDrIHB+/OMf67HHHtP3vvc9XXbZZe/6Pq6DwDrdceBaCIyGhgZ9/vOfP6Xgl5eXKz4+/pR9H5L1roUZX0zDwsJ05ZVX6p577tHBgwe1bds2bd26VTfeeKMk/8zdyMiIJOmSSy5RX1+fvv3tb6u6ulrf/va3NTw8rEsvvdTMb2HGm8wYbNmyRS+88IKeffZZ1dXV6cc//rFKS0t1/fXXm/ktzHpcB+bjOgi8mpoa/eQnP9HnPvc5rVy5Uu3t7RP/SVwH02Uy48C1EBhFRUVasmSJ7r77blVXV+vVV1/Vfffdp3/5l3+RZPFrwZRDqqbY0NCQcccddxjLly83NmzYYPzyl7+c+Fh+fv4pZ3EdOHDAuPLKK42ioiLjYx/7mFFRUWFC4tlnMmPwxBNPGBdddJGxdOlS4yMf+YixZ88eExLPbn9/hibXwfR7vzHgOph6P/vZz4z8/Px/+J9hcB1Ml8mOA9dCYLS0tBi33nqrsWLFCmP9+vXGT3/604mzSq18LdgMY2zuFgAAADDRjL+VDwAAgNmBYgoAAABLoJgCAADAEiimAAAAsASKKQAAACyBYgoAAABLoJgCAADAEiimAAAAsASKKQDMAlu2bNGPfvQjs2MAwFmhmAIAAMASKKYAAACwBIopAEyB/v5+/cd//IfWrVunlStX6sYbb9ShQ4ckST/60Y/0iU98Qvfff7/Wrl2rVatW6a677tLAwMDEr+/p6dHXv/51bdq0ScXFxbr22mv15ptvnvI1Xn/9dV1zzTVatmyZNm7cqP/5n/+R1+ud+Hh7e7s+//nPa/ny5Vq7dq3uvffeUz4OAFZHMQWAs2QYhj73uc+pvr5eP/vZz/TEE09o+fLl+sQnPqHDhw9Lkg4dOqQdO3Zo69atuv/++/XWW2/py1/+siTJ6/XqM5/5jPbu3av77rtPTz/9tPLz83XTTTfp4MGDkqSysjLdfPPNWrlypZ5++ml961vf0mOPPaaf/OQnEzl+97vfafXq1XrhhRf0b//2b/rVr36lZ555Ztp/PwDgTNkMwzDMDgEAM9muXbv06U9/Wrt371ZsbOzE69dff70yMzOVkZGhn//859q2bZtSUlIkSa+99po+97nP6Y9//KPq6+t1880364UXXlB+fr4kf9m96qqrlJ2drR/84Af6yle+oqamJj322GMTn/+ll15SZ2enrrvuOm3ZskXFxcX6/ve/P/HxD33oQ1qxYoXuueee6fhtAICz5jQ7AADMdBUVFTIMQ5s3bz7ldbfbLZfLpYyMDM2bN2+ilErSihUrJEmVlZWqr69XVFTURCmVJJvNplWrVmnHjh0T71u/fv0pn//iiy8+5efz5s075ecxMTFyuVxn/f0BwHShmALAWfL5fIqMjNTTTz/9jo8FBwfrd7/7nYKCgk55fXztp8Ph0LvduDIMQ06n/4/p8f99Lw6H4x9+DgCYKVhjCgBnKT8/XwMDAxodHVVOTs7Efw8++KD++te/SpJqa2vV398/8WvKysokSYsXL1ZBQYH6+/tVWVk58XHDMFRaWqqFCxdKknJzcyc2U417+OGHdfXVVwf62wOAaUMxBYCzdN5556mwsFC33Xabdu/erbq6Ot177716+umnlZubK0kaGhrSHXfcocrKSu3cuVPf+MY39MEPflAZGRnasGGDCgsLdfvtt2vPnj2qqanRN77xDVVWVupTn/qUJOmzn/2s9u/frx/84Ac6ceKEXn31Vf3kJz/R+eefb+J3DgBTi1v5AHCWHA6Htm7dqvvuu09f/vKXNTw8rNzcXP34xz/WOeeco7179yotLU2FhYX65Cc/KYfDoSuuuEJf/epXT/n13/nOd/T5z39ebrdbS5cu1a9+9SstX75cklRYWKj7779fP/zhD/Xggw8qOTlZN954o2655RYTv3MAmFrsygeAAPvRj36kZ555Ri+//LLZUQDA0riVDwAAAEugmAIAAMASuJUPAAAAS2DGFAAAAJZAMQUAAIAlUEwBAABgCRRTAAAAWALFFAAAAJZAMQUAAIAlUEwBAABgCRRTAAAAWML/D+LDlzEOREn6AAAAAElFTkSuQmCC",
|
||
"text/plain": [
|
||
"<Figure size 800x550 with 1 Axes>"
|
||
]
|
||
},
|
||
"metadata": {},
|
||
"output_type": "display_data"
|
||
}
|
||
],
|
||
"source": [
|
||
"df_hist['learning_rate'].plot(logy=True)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"metadata": {},
|
||
"source": [
|
||
"\n",
|
||
"### Perplexity\n",
|
||
"\n",
|
||
"Perplexity measures how well a language model predicts a text sample. Lower is better\n",
|
||
"\n",
|
||
"It’s calculated as the average number of bits per word a model needs to represent the same\n",
|
||
"\n",
|
||
"https://huggingface.co/docs/transformers/perplexity\n",
|
||
"https://thegradient.pub/understanding-evaluation-metrics-for-language-models/\n",
|
||
"\n",
|
||
"The **improvement** column, is perplexity decrease"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": []
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 28,
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"data": {
|
||
"text/html": [
|
||
"<div>\n",
|
||
"<style scoped>\n",
|
||
" .dataframe tbody tr th:only-of-type {\n",
|
||
" vertical-align: middle;\n",
|
||
" }\n",
|
||
"\n",
|
||
" .dataframe tbody tr th {\n",
|
||
" vertical-align: top;\n",
|
||
" }\n",
|
||
"\n",
|
||
" .dataframe thead th {\n",
|
||
" text-align: right;\n",
|
||
" }\n",
|
||
"</style>\n",
|
||
"<table border=\"1\" class=\"dataframe\">\n",
|
||
" <thead>\n",
|
||
" <tr style=\"text-align: right;\">\n",
|
||
" <th></th>\n",
|
||
" <th>before</th>\n",
|
||
" <th>after</th>\n",
|
||
" <th>len</th>\n",
|
||
" <th>improvement%</th>\n",
|
||
" <th>improvement</th>\n",
|
||
" <th>novel</th>\n",
|
||
" <th>learnable</th>\n",
|
||
" <th>BS</th>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>title</th>\n",
|
||
" <th></th>\n",
|
||
" <th></th>\n",
|
||
" <th></th>\n",
|
||
" <th></th>\n",
|
||
" <th></th>\n",
|
||
" <th></th>\n",
|
||
" <th></th>\n",
|
||
" <th></th>\n",
|
||
" </tr>\n",
|
||
" </thead>\n",
|
||
" <tbody>\n",
|
||
" <tr>\n",
|
||
" <th>cicero from ibois, Philippe (2012-06-03).</th>\n",
|
||
" <td>72.587372</td>\n",
|
||
" <td>67.744194</td>\n",
|
||
" <td>13707</td>\n",
|
||
" <td>0.066722</td>\n",
|
||
" <td>4.843178</td>\n",
|
||
" <td>True</td>\n",
|
||
" <td>True</td>\n",
|
||
" <td>False</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>politics is the mind-killer</th>\n",
|
||
" <td>247.551559</td>\n",
|
||
" <td>245.826813</td>\n",
|
||
" <td>3158</td>\n",
|
||
" <td>0.006967</td>\n",
|
||
" <td>1.724747</td>\n",
|
||
" <td>True</td>\n",
|
||
" <td>False</td>\n",
|
||
" <td>False</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>openai board ann</th>\n",
|
||
" <td>55.808502</td>\n",
|
||
" <td>54.667923</td>\n",
|
||
" <td>2991</td>\n",
|
||
" <td>0.020437</td>\n",
|
||
" <td>1.140579</td>\n",
|
||
" <td>True</td>\n",
|
||
" <td>True</td>\n",
|
||
" <td>False</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>How to Catch an AI Liar</th>\n",
|
||
" <td>28.949860</td>\n",
|
||
" <td>28.008789</td>\n",
|
||
" <td>5464</td>\n",
|
||
" <td>0.032507</td>\n",
|
||
" <td>0.941071</td>\n",
|
||
" <td>True</td>\n",
|
||
" <td>True</td>\n",
|
||
" <td>False</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>buzzfeed foi fauci emails 2023</th>\n",
|
||
" <td>23.309443</td>\n",
|
||
" <td>22.406410</td>\n",
|
||
" <td>13640</td>\n",
|
||
" <td>0.038741</td>\n",
|
||
" <td>0.903032</td>\n",
|
||
" <td>True</td>\n",
|
||
" <td>True</td>\n",
|
||
" <td>False</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>Gemini to Q*</th>\n",
|
||
" <td>11.756384</td>\n",
|
||
" <td>11.190644</td>\n",
|
||
" <td>42604</td>\n",
|
||
" <td>0.048122</td>\n",
|
||
" <td>0.565740</td>\n",
|
||
" <td>False</td>\n",
|
||
" <td>True</td>\n",
|
||
" <td>False</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>LK-99-en</th>\n",
|
||
" <td>14.513845</td>\n",
|
||
" <td>14.066093</td>\n",
|
||
" <td>15432</td>\n",
|
||
" <td>0.030850</td>\n",
|
||
" <td>0.447752</td>\n",
|
||
" <td>False</td>\n",
|
||
" <td>True</td>\n",
|
||
" <td>False</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>LK-99-es</th>\n",
|
||
" <td>11.415018</td>\n",
|
||
" <td>10.972948</td>\n",
|
||
" <td>12970</td>\n",
|
||
" <td>0.038727</td>\n",
|
||
" <td>0.442070</td>\n",
|
||
" <td>False</td>\n",
|
||
" <td>True</td>\n",
|
||
" <td>False</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>disney appointment</th>\n",
|
||
" <td>118.826157</td>\n",
|
||
" <td>118.420128</td>\n",
|
||
" <td>3653</td>\n",
|
||
" <td>0.003417</td>\n",
|
||
" <td>0.406029</td>\n",
|
||
" <td>True</td>\n",
|
||
" <td>False</td>\n",
|
||
" <td>False</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>weak to strong</th>\n",
|
||
" <td>46.764183</td>\n",
|
||
" <td>46.404736</td>\n",
|
||
" <td>5811</td>\n",
|
||
" <td>0.007686</td>\n",
|
||
" <td>0.359447</td>\n",
|
||
" <td>True</td>\n",
|
||
" <td>False</td>\n",
|
||
" <td>False</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>blechley declaration</th>\n",
|
||
" <td>17.869091</td>\n",
|
||
" <td>17.524204</td>\n",
|
||
" <td>7762</td>\n",
|
||
" <td>0.019301</td>\n",
|
||
" <td>0.344887</td>\n",
|
||
" <td>True</td>\n",
|
||
" <td>False</td>\n",
|
||
" <td>False</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>Lorem ipsum</th>\n",
|
||
" <td>6.564840</td>\n",
|
||
" <td>6.268792</td>\n",
|
||
" <td>19649</td>\n",
|
||
" <td>0.045096</td>\n",
|
||
" <td>0.296049</td>\n",
|
||
" <td>False</td>\n",
|
||
" <td>True</td>\n",
|
||
" <td>True</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>statement by whitehouse on passing</th>\n",
|
||
" <td>29.197132</td>\n",
|
||
" <td>28.939692</td>\n",
|
||
" <td>1641</td>\n",
|
||
" <td>0.008817</td>\n",
|
||
" <td>0.257441</td>\n",
|
||
" <td>True</td>\n",
|
||
" <td>False</td>\n",
|
||
" <td>True</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>harvard announcment caplain israel hamas</th>\n",
|
||
" <td>45.347355</td>\n",
|
||
" <td>45.127274</td>\n",
|
||
" <td>4247</td>\n",
|
||
" <td>0.004853</td>\n",
|
||
" <td>0.220081</td>\n",
|
||
" <td>True</td>\n",
|
||
" <td>False</td>\n",
|
||
" <td>True</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>fake ai hoax paper</th>\n",
|
||
" <td>7.766979</td>\n",
|
||
" <td>7.697229</td>\n",
|
||
" <td>3290</td>\n",
|
||
" <td>0.008980</td>\n",
|
||
" <td>0.069750</td>\n",
|
||
" <td>False</td>\n",
|
||
" <td>False</td>\n",
|
||
" <td>True</td>\n",
|
||
" </tr>\n",
|
||
" </tbody>\n",
|
||
"</table>\n",
|
||
"</div>"
|
||
],
|
||
"text/plain": [
|
||
" before after len \\\n",
|
||
"title \n",
|
||
"cicero from ibois, Philippe (2012-06-03). 72.587372 67.744194 13707 \n",
|
||
"politics is the mind-killer 247.551559 245.826813 3158 \n",
|
||
"openai board ann 55.808502 54.667923 2991 \n",
|
||
"How to Catch an AI Liar 28.949860 28.008789 5464 \n",
|
||
"buzzfeed foi fauci emails 2023 23.309443 22.406410 13640 \n",
|
||
"Gemini to Q* 11.756384 11.190644 42604 \n",
|
||
"LK-99-en 14.513845 14.066093 15432 \n",
|
||
"LK-99-es 11.415018 10.972948 12970 \n",
|
||
"disney appointment 118.826157 118.420128 3653 \n",
|
||
"weak to strong 46.764183 46.404736 5811 \n",
|
||
"blechley declaration 17.869091 17.524204 7762 \n",
|
||
"Lorem ipsum 6.564840 6.268792 19649 \n",
|
||
"statement by whitehouse on passing 29.197132 28.939692 1641 \n",
|
||
"harvard announcment caplain israel hamas 45.347355 45.127274 4247 \n",
|
||
"fake ai hoax paper 7.766979 7.697229 3290 \n",
|
||
"\n",
|
||
" improvement% improvement novel \\\n",
|
||
"title \n",
|
||
"cicero from ibois, Philippe (2012-06-03). 0.066722 4.843178 True \n",
|
||
"politics is the mind-killer 0.006967 1.724747 True \n",
|
||
"openai board ann 0.020437 1.140579 True \n",
|
||
"How to Catch an AI Liar 0.032507 0.941071 True \n",
|
||
"buzzfeed foi fauci emails 2023 0.038741 0.903032 True \n",
|
||
"Gemini to Q* 0.048122 0.565740 False \n",
|
||
"LK-99-en 0.030850 0.447752 False \n",
|
||
"LK-99-es 0.038727 0.442070 False \n",
|
||
"disney appointment 0.003417 0.406029 True \n",
|
||
"weak to strong 0.007686 0.359447 True \n",
|
||
"blechley declaration 0.019301 0.344887 True \n",
|
||
"Lorem ipsum 0.045096 0.296049 False \n",
|
||
"statement by whitehouse on passing 0.008817 0.257441 True \n",
|
||
"harvard announcment caplain israel hamas 0.004853 0.220081 True \n",
|
||
"fake ai hoax paper 0.008980 0.069750 False \n",
|
||
"\n",
|
||
" learnable BS \n",
|
||
"title \n",
|
||
"cicero from ibois, Philippe (2012-06-03). True False \n",
|
||
"politics is the mind-killer False False \n",
|
||
"openai board ann True False \n",
|
||
"How to Catch an AI Liar True False \n",
|
||
"buzzfeed foi fauci emails 2023 True False \n",
|
||
"Gemini to Q* True False \n",
|
||
"LK-99-en True False \n",
|
||
"LK-99-es True False \n",
|
||
"disney appointment False False \n",
|
||
"weak to strong False False \n",
|
||
"blechley declaration False False \n",
|
||
"Lorem ipsum True True \n",
|
||
"statement by whitehouse on passing False True \n",
|
||
"harvard announcment caplain israel hamas False True \n",
|
||
"fake ai hoax paper False True "
|
||
]
|
||
},
|
||
"execution_count": 28,
|
||
"metadata": {},
|
||
"output_type": "execute_result"
|
||
}
|
||
],
|
||
"source": [
|
||
"df_res = pd.DataFrame(data)\n",
|
||
"df_res['len'] = df_res.content.str.len()\n",
|
||
"df_res = df_res[['before', 'after', 'title', 'len']].set_index('title')\n",
|
||
"df_res['improvement%'] = (df_res['before'] - df_res['after'])/ df_res['before']\n",
|
||
"df_res['improvement'] = (df_res['before'] - df_res['after'])\n",
|
||
"df_res['novel'] = df_res['before'] > 15\n",
|
||
"df_res['learnable'] = df_res['improvement%'] > 0.02\n",
|
||
"\n",
|
||
"# We can measure the final score using learnable * novel\n",
|
||
"# df_res['BS'] = ~df_res['learnable'] | ~df_res['novel']\n",
|
||
"# Or just absolute perplexity improvement\n",
|
||
"df_res['BS'] = df_res['improvement'] < .3\n",
|
||
"df_res = df_res.sort_values('improvement', ascending=False)\n",
|
||
"df_res"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 26,
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"# df_res.sort_values('improvement%', ascending=False)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 22,
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"| title | before | after | len | improvement% | improvement | novel | learnable | BS |\n",
|
||
"|:------------------------------------------|----------:|----------:|------:|---------------:|--------------:|:--------|:------------|:------|\n",
|
||
"| cicero from ibois, Philippe (2012-06-03). | 72.5874 | 67.7442 | 13707 | 0.066722 | 4.84318 | True | True | False |\n",
|
||
"| politics is the mind-killer | 247.552 | 245.827 | 3158 | 0.00696722 | 1.72475 | True | False | False |\n",
|
||
"| openai board ann | 55.8085 | 54.6679 | 2991 | 0.0204374 | 1.14058 | True | True | False |\n",
|
||
"| How to Catch an AI Liar | 28.9499 | 28.0088 | 5464 | 0.0325069 | 0.941071 | True | True | True |\n",
|
||
"| buzzfeed foi fauci emails 2023 | 23.3094 | 22.4064 | 13640 | 0.0387411 | 0.903032 | True | True | True |\n",
|
||
"| Gemini to Q* | 11.7564 | 11.1906 | 42604 | 0.0481219 | 0.56574 | False | True | True |\n",
|
||
"| LK-99-en | 14.5138 | 14.0661 | 15432 | 0.03085 | 0.447752 | False | True | True |\n",
|
||
"| LK-99-es | 11.415 | 10.9729 | 12970 | 0.0387271 | 0.44207 | False | True | True |\n",
|
||
"| disney appointment | 118.826 | 118.42 | 3653 | 0.003417 | 0.406029 | True | False | True |\n",
|
||
"| weak to strong | 46.7642 | 46.4047 | 5811 | 0.00768638 | 0.359447 | True | False | True |\n",
|
||
"| blechley declaration | 17.8691 | 17.5242 | 7762 | 0.0193007 | 0.344887 | True | False | True |\n",
|
||
"| Lorem ipsum | 6.56484 | 6.26879 | 19649 | 0.0450961 | 0.296049 | False | True | True |\n",
|
||
"| statement by whitehouse on passing | 29.1971 | 28.9397 | 1641 | 0.00881732 | 0.257441 | True | False | True |\n",
|
||
"| harvard announcment caplain israel hamas | 45.3474 | 45.1273 | 4247 | 0.00485323 | 0.220081 | True | False | True |\n",
|
||
"| fake ai hoax paper | 7.76698 | 7.69723 | 3290 | 0.00898037 | 0.0697503 | False | False | True |\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"print(df_res.to_markdown())"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"metadata": {},
|
||
"source": [
|
||
"# DEBUG"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 23,
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"from IPython.display import display, HTML, Markdown\n",
|
||
"import torch\n",
|
||
"\n",
|
||
"@torch.no_grad()\n",
|
||
"def gen(model, inputs, tokenizer, clean=True):\n",
|
||
" s = model.generate(\n",
|
||
" input_ids=inputs[\"input_ids\"][None, :].to(model.device),\n",
|
||
" attention_mask=inputs[\"attention_mask\"][None, :].to(model.device),\n",
|
||
" use_cache=False,\n",
|
||
" max_new_tokens=100,\n",
|
||
" min_new_tokens=100,\n",
|
||
" do_sample=False,\n",
|
||
" early_stopping=False,\n",
|
||
" )\n",
|
||
" input_l = inputs[\"input_ids\"].shape[0]\n",
|
||
" tokenizer_kwargs=dict(clean_up_tokenization_spaces=clean, skip_special_tokens=clean)\n",
|
||
" old = tokenizer.decode(\n",
|
||
" s[0, :input_l][-100:], **tokenizer_kwargs\n",
|
||
" )\n",
|
||
" new = tokenizer.decode(\n",
|
||
" s[0, input_l:], **tokenizer_kwargs\n",
|
||
" )\n",
|
||
" s_old = \"\"+old.replace('\\n', '<br>')\n",
|
||
" s_new = '<b>' + new.replace('\\n', '<br>')+ '<br><br><b/>'\n",
|
||
" # print(s_old, s_new)\n",
|
||
" display(HTML(f\"{s_old}{s_new}\"))\n",
|
||
" # print([old, new])\n",
|
||
"\n"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"metadata": {},
|
||
"source": []
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 24,
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"sample = samples[-1]\n",
|
||
"s = sample['content']\n",
|
||
"first_half = s[:len(s)//2]\n",
|
||
"second_half = s[len(s)//2:]\n",
|
||
"ds_train = Dataset.from_dict(tokenizer([first_half]))\n",
|
||
"ds_val = Dataset.from_dict(tokenizer([second_half]))"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 25,
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stderr",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/transformers/generation/utils.py:1421: UserWarning: You have modified the pretrained model configuration to control generation. This is a deprecated strategy to control generation and will be removed soon, in a future version. Please use and modify the model generation configuration (see https://huggingface.co/docs/transformers/generation_strategies#default-text-generation-configuration )\n",
|
||
" warnings.warn(\n"
|
||
]
|
||
},
|
||
{
|
||
"ename": "OutOfMemoryError",
|
||
"evalue": "CUDA out of memory. Tried to allocate 1.07 GiB. GPU 0 has a total capacty of 10.75 GiB of which 612.62 MiB is free. Including non-PyTorch memory, this process has 10.15 GiB memory in use. Of the allocated memory 9.78 GiB is allocated by PyTorch, and 179.66 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting max_split_size_mb to avoid fragmentation. See documentation for Memory Management and PYTORCH_CUDA_ALLOC_CONF",
|
||
"output_type": "error",
|
||
"traceback": [
|
||
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
|
||
"\u001b[0;31mOutOfMemoryError\u001b[0m Traceback (most recent call last)",
|
||
"Cell \u001b[0;32mIn[25], line 2\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m model\u001b[38;5;241m.\u001b[39mdisable_adapter():\n\u001b[0;32m----> 2\u001b[0m \u001b[43mgen\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmodel\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mds_train\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mwith_format\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mpt\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtokenizer\u001b[49m\u001b[43m)\u001b[49m\n",
|
||
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/torch/utils/_contextlib.py:115\u001b[0m, in \u001b[0;36mcontext_decorator.<locals>.decorate_context\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 112\u001b[0m \u001b[38;5;129m@functools\u001b[39m\u001b[38;5;241m.\u001b[39mwraps(func)\n\u001b[1;32m 113\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mdecorate_context\u001b[39m(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m 114\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m ctx_factory():\n\u001b[0;32m--> 115\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
|
||
"Cell \u001b[0;32mIn[23], line 6\u001b[0m, in \u001b[0;36mgen\u001b[0;34m(model, inputs, tokenizer, clean)\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[38;5;129m@torch\u001b[39m\u001b[38;5;241m.\u001b[39mno_grad()\n\u001b[1;32m 5\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mgen\u001b[39m(model, inputs, tokenizer, clean\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m):\n\u001b[0;32m----> 6\u001b[0m s \u001b[38;5;241m=\u001b[39m \u001b[43mmodel\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgenerate\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 7\u001b[0m \u001b[43m \u001b[49m\u001b[43minput_ids\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43minputs\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43minput_ids\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m:\u001b[49m\u001b[43m]\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mto\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmodel\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdevice\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 8\u001b[0m \u001b[43m \u001b[49m\u001b[43mattention_mask\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43minputs\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mattention_mask\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m:\u001b[49m\u001b[43m]\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mto\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmodel\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdevice\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 9\u001b[0m \u001b[43m \u001b[49m\u001b[43muse_cache\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m 10\u001b[0m \u001b[43m \u001b[49m\u001b[43mmax_new_tokens\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m100\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 11\u001b[0m \u001b[43m \u001b[49m\u001b[43mmin_new_tokens\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m100\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 12\u001b[0m \u001b[43m \u001b[49m\u001b[43mdo_sample\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m 13\u001b[0m \u001b[43m \u001b[49m\u001b[43mearly_stopping\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m 14\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 15\u001b[0m input_l \u001b[38;5;241m=\u001b[39m inputs[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124minput_ids\u001b[39m\u001b[38;5;124m\"\u001b[39m]\u001b[38;5;241m.\u001b[39mshape[\u001b[38;5;241m0\u001b[39m]\n\u001b[1;32m 16\u001b[0m tokenizer_kwargs\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mdict\u001b[39m(clean_up_tokenization_spaces\u001b[38;5;241m=\u001b[39mclean, skip_special_tokens\u001b[38;5;241m=\u001b[39mclean)\n",
|
||
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/torch/utils/_contextlib.py:115\u001b[0m, in \u001b[0;36mcontext_decorator.<locals>.decorate_context\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 112\u001b[0m \u001b[38;5;129m@functools\u001b[39m\u001b[38;5;241m.\u001b[39mwraps(func)\n\u001b[1;32m 113\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mdecorate_context\u001b[39m(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m 114\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m ctx_factory():\n\u001b[0;32m--> 115\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
|
||
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/transformers/generation/utils.py:1606\u001b[0m, in \u001b[0;36mGenerationMixin.generate\u001b[0;34m(self, inputs, generation_config, logits_processor, stopping_criteria, prefix_allowed_tokens_fn, synced_gpus, assistant_model, streamer, negative_prompt_ids, negative_prompt_attention_mask, **kwargs)\u001b[0m\n\u001b[1;32m 1589\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39massisted_decoding(\n\u001b[1;32m 1590\u001b[0m input_ids,\n\u001b[1;32m 1591\u001b[0m assistant_model\u001b[38;5;241m=\u001b[39massistant_model,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 1602\u001b[0m \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mmodel_kwargs,\n\u001b[1;32m 1603\u001b[0m )\n\u001b[1;32m 1604\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m generation_mode \u001b[38;5;241m==\u001b[39m GenerationMode\u001b[38;5;241m.\u001b[39mGREEDY_SEARCH:\n\u001b[1;32m 1605\u001b[0m \u001b[38;5;66;03m# 11. run greedy search\u001b[39;00m\n\u001b[0;32m-> 1606\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgreedy_search\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 1607\u001b[0m \u001b[43m \u001b[49m\u001b[43minput_ids\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1608\u001b[0m \u001b[43m \u001b[49m\u001b[43mlogits_processor\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlogits_processor\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1609\u001b[0m \u001b[43m \u001b[49m\u001b[43mstopping_criteria\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstopping_criteria\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1610\u001b[0m \u001b[43m \u001b[49m\u001b[43mpad_token_id\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mgeneration_config\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpad_token_id\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1611\u001b[0m \u001b[43m \u001b[49m\u001b[43meos_token_id\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mgeneration_config\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43meos_token_id\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1612\u001b[0m \u001b[43m \u001b[49m\u001b[43moutput_scores\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mgeneration_config\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43moutput_scores\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1613\u001b[0m \u001b[43m \u001b[49m\u001b[43mreturn_dict_in_generate\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mgeneration_config\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mreturn_dict_in_generate\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1614\u001b[0m \u001b[43m \u001b[49m\u001b[43msynced_gpus\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msynced_gpus\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1615\u001b[0m \u001b[43m \u001b[49m\u001b[43mstreamer\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstreamer\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1616\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mmodel_kwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1617\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1619\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m generation_mode \u001b[38;5;241m==\u001b[39m GenerationMode\u001b[38;5;241m.\u001b[39mCONTRASTIVE_SEARCH:\n\u001b[1;32m 1620\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m model_kwargs[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124muse_cache\u001b[39m\u001b[38;5;124m\"\u001b[39m]:\n",
|
||
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/transformers/generation/utils.py:2454\u001b[0m, in \u001b[0;36mGenerationMixin.greedy_search\u001b[0;34m(self, input_ids, logits_processor, stopping_criteria, max_length, pad_token_id, eos_token_id, output_attentions, output_hidden_states, output_scores, return_dict_in_generate, synced_gpus, streamer, **model_kwargs)\u001b[0m\n\u001b[1;32m 2451\u001b[0m model_inputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprepare_inputs_for_generation(input_ids, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mmodel_kwargs)\n\u001b[1;32m 2453\u001b[0m \u001b[38;5;66;03m# forward pass to get next token\u001b[39;00m\n\u001b[0;32m-> 2454\u001b[0m outputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m(\u001b[49m\n\u001b[1;32m 2455\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mmodel_inputs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 2456\u001b[0m \u001b[43m \u001b[49m\u001b[43mreturn_dict\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m 2457\u001b[0m \u001b[43m \u001b[49m\u001b[43moutput_attentions\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43moutput_attentions\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 2458\u001b[0m \u001b[43m \u001b[49m\u001b[43moutput_hidden_states\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43moutput_hidden_states\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 2459\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 2461\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m synced_gpus \u001b[38;5;129;01mand\u001b[39;00m this_peer_finished:\n\u001b[1;32m 2462\u001b[0m \u001b[38;5;28;01mcontinue\u001b[39;00m \u001b[38;5;66;03m# don't waste resources running the code we don't need\u001b[39;00m\n",
|
||
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/torch/nn/modules/module.py:1518\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1516\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs) \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m 1517\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1518\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
|
||
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/torch/nn/modules/module.py:1527\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1522\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m 1523\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m 1524\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m 1525\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m 1526\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1527\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1529\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 1530\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
|
||
"File \u001b[0;32m~/.cache/huggingface/modules/transformers_modules/microsoft/phi-2/d3186761bf5c4409f7679359284066c25ab668ee/modeling_phi.py:953\u001b[0m, in \u001b[0;36mPhiForCausalLM.forward\u001b[0;34m(self, input_ids, past_key_values, attention_mask, labels, **kwargs)\u001b[0m\n\u001b[1;32m 945\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\n\u001b[1;32m 946\u001b[0m \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m 947\u001b[0m input_ids: torch\u001b[38;5;241m.\u001b[39mLongTensor,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 951\u001b[0m \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs,\n\u001b[1;32m 952\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m CausalLMOutputWithPast:\n\u001b[0;32m--> 953\u001b[0m hidden_states \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtransformer\u001b[49m\u001b[43m(\u001b[49m\u001b[43minput_ids\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mpast_key_values\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpast_key_values\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mattention_mask\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mattention_mask\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 954\u001b[0m lm_logits \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlm_head(hidden_states)\n\u001b[1;32m 956\u001b[0m loss \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
|
||
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/torch/nn/modules/module.py:1518\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1516\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs) \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m 1517\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1518\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
|
||
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/torch/nn/modules/module.py:1527\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1522\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m 1523\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m 1524\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m 1525\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m 1526\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1527\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1529\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 1530\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
|
||
"File \u001b[0;32m~/.cache/huggingface/modules/transformers_modules/microsoft/phi-2/d3186761bf5c4409f7679359284066c25ab668ee/modeling_phi.py:915\u001b[0m, in \u001b[0;36mPhiModel.forward\u001b[0;34m(self, input_ids, past_key_values, attention_mask)\u001b[0m\n\u001b[1;32m 912\u001b[0m hidden_states \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39membd(input_ids)\n\u001b[1;32m 914\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m layer \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mh:\n\u001b[0;32m--> 915\u001b[0m hidden_states \u001b[38;5;241m=\u001b[39m \u001b[43mlayer\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 916\u001b[0m \u001b[43m \u001b[49m\u001b[43mhidden_states\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 917\u001b[0m \u001b[43m \u001b[49m\u001b[43mpast_key_values\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpast_key_values\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 918\u001b[0m \u001b[43m \u001b[49m\u001b[43mattention_mask\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mattention_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 919\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 921\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m hidden_states\n",
|
||
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/torch/nn/modules/module.py:1518\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1516\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs) \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m 1517\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1518\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
|
||
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/torch/nn/modules/module.py:1527\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1522\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m 1523\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m 1524\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m 1525\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m 1526\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1527\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1529\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 1530\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
|
||
"File \u001b[0;32m~/.cache/huggingface/modules/transformers_modules/microsoft/phi-2/d3186761bf5c4409f7679359284066c25ab668ee/modeling_phi.py:770\u001b[0m, in \u001b[0;36mParallelBlock.forward\u001b[0;34m(self, hidden_states, past_key_values, attention_mask, **kwargs)\u001b[0m\n\u001b[1;32m 767\u001b[0m residual \u001b[38;5;241m=\u001b[39m hidden_states\n\u001b[1;32m 768\u001b[0m hidden_states \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mln(hidden_states)\n\u001b[0;32m--> 770\u001b[0m attn_outputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmixer\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 771\u001b[0m \u001b[43m \u001b[49m\u001b[43mhidden_states\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 772\u001b[0m \u001b[43m \u001b[49m\u001b[43mpast_key_values\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpast_key_values\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 773\u001b[0m \u001b[43m \u001b[49m\u001b[43mattention_mask\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mattention_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 774\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 775\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(attn_outputs, \u001b[38;5;28mtuple\u001b[39m):\n\u001b[1;32m 776\u001b[0m attn_outputs \u001b[38;5;241m=\u001b[39m attn_outputs[\u001b[38;5;241m0\u001b[39m]\n",
|
||
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/torch/nn/modules/module.py:1518\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1516\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs) \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m 1517\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1518\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
|
||
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/torch/nn/modules/module.py:1527\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1522\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m 1523\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m 1524\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m 1525\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m 1526\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1527\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1529\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 1530\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
|
||
"File \u001b[0;32m~/.cache/huggingface/modules/transformers_modules/microsoft/phi-2/d3186761bf5c4409f7679359284066c25ab668ee/modeling_phi.py:726\u001b[0m, in \u001b[0;36mMHA.forward\u001b[0;34m(self, x, past_key_values, attention_mask, **kwargs)\u001b[0m\n\u001b[1;32m 722\u001b[0m attn_output \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_self_attn(x, attention_mask)\n\u001b[1;32m 723\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 724\u001b[0m \u001b[38;5;66;03m# If `past_key_values` are supplied, it means that we might have cached values and\u001b[39;00m\n\u001b[1;32m 725\u001b[0m \u001b[38;5;66;03m# could take advantage of cross-attention\u001b[39;00m\n\u001b[0;32m--> 726\u001b[0m attn_output \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_forward_cross_attn\u001b[49m\u001b[43m(\u001b[49m\u001b[43mx\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mpast_key_values\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mattention_mask\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 727\u001b[0m \u001b[38;5;66;03m# MQA / GQA\u001b[39;00m\n\u001b[1;32m 728\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 729\u001b[0m \u001b[38;5;66;03m# Regardless of `past_key_values` being supplied or not, it always use cross-attention\u001b[39;00m\n\u001b[1;32m 730\u001b[0m \u001b[38;5;66;03m# because `q` and `kv` lengths might be different\u001b[39;00m\n\u001b[1;32m 731\u001b[0m attn_output \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_cross_attn(x, past_key_values, attention_mask)\n",
|
||
"File \u001b[0;32m~/.cache/huggingface/modules/transformers_modules/microsoft/phi-2/d3186761bf5c4409f7679359284066c25ab668ee/modeling_phi.py:704\u001b[0m, in \u001b[0;36mMHA._forward_cross_attn\u001b[0;34m(self, x, past_key_values, key_padding_mask)\u001b[0m\n\u001b[1;32m 695\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcheckpointing:\n\u001b[1;32m 696\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m torch\u001b[38;5;241m.\u001b[39mutils\u001b[38;5;241m.\u001b[39mcheckpoint\u001b[38;5;241m.\u001b[39mcheckpoint(\n\u001b[1;32m 697\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39minner_cross_attn,\n\u001b[1;32m 698\u001b[0m q,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 701\u001b[0m causal\u001b[38;5;241m=\u001b[39mcausal,\n\u001b[1;32m 702\u001b[0m )\n\u001b[0;32m--> 704\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43minner_cross_attn\u001b[49m\u001b[43m(\u001b[49m\u001b[43mq\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mkv\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mkey_padding_mask\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mkey_padding_mask\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcausal\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcausal\u001b[49m\u001b[43m)\u001b[49m\n",
|
||
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/torch/nn/modules/module.py:1518\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1516\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs) \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m 1517\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1518\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
|
||
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/torch/nn/modules/module.py:1527\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1522\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m 1523\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m 1524\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m 1525\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m 1526\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1527\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1529\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 1530\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
|
||
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/torch/amp/autocast_mode.py:16\u001b[0m, in \u001b[0;36mautocast_decorator.<locals>.decorate_autocast\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 13\u001b[0m \u001b[38;5;129m@functools\u001b[39m\u001b[38;5;241m.\u001b[39mwraps(func)\n\u001b[1;32m 14\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mdecorate_autocast\u001b[39m(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m 15\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m autocast_instance:\n\u001b[0;32m---> 16\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
|
||
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/bs_writing_detector/.venv/lib/python3.11/site-packages/torch/amp/autocast_mode.py:16\u001b[0m, in \u001b[0;36mautocast_decorator.<locals>.decorate_autocast\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 13\u001b[0m \u001b[38;5;129m@functools\u001b[39m\u001b[38;5;241m.\u001b[39mwraps(func)\n\u001b[1;32m 14\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mdecorate_autocast\u001b[39m(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m 15\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m autocast_instance:\n\u001b[0;32m---> 16\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
|
||
"File \u001b[0;32m~/.cache/huggingface/modules/transformers_modules/microsoft/phi-2/d3186761bf5c4409f7679359284066c25ab668ee/modeling_phi.py:452\u001b[0m, in \u001b[0;36mCrossAttention.forward\u001b[0;34m(self, q, kv, causal, key_padding_mask, **kwargs)\u001b[0m\n\u001b[1;32m 448\u001b[0m causal_mask \u001b[38;5;241m=\u001b[39m cols \u001b[38;5;241m>\u001b[39m rows \u001b[38;5;241m+\u001b[39m seqlen_k \u001b[38;5;241m-\u001b[39m seqlen_q\n\u001b[1;32m 450\u001b[0m scores \u001b[38;5;241m=\u001b[39m scores\u001b[38;5;241m.\u001b[39mmasked_fill(causal_mask, \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m10000.0\u001b[39m)\n\u001b[0;32m--> 452\u001b[0m attention \u001b[38;5;241m=\u001b[39m \u001b[43mtorch\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msoftmax\u001b[49m\u001b[43m(\u001b[49m\u001b[43mscores\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdim\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m-\u001b[39;49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mto\u001b[49m\u001b[43m(\u001b[49m\u001b[43mv\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdtype\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 453\u001b[0m attention \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdrop(attention)\n\u001b[1;32m 455\u001b[0m output \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39meinsum(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mbhts,bshd->bthd\u001b[39m\u001b[38;5;124m\"\u001b[39m, attention, v)\n",
|
||
"\u001b[0;31mOutOfMemoryError\u001b[0m: CUDA out of memory. Tried to allocate 1.07 GiB. GPU 0 has a total capacty of 10.75 GiB of which 612.62 MiB is free. Including non-PyTorch memory, this process has 10.15 GiB memory in use. Of the allocated memory 9.78 GiB is allocated by PyTorch, and 179.66 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting max_split_size_mb to avoid fragmentation. See documentation for Memory Management and PYTORCH_CUDA_ALLOC_CONF"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"with model.disable_adapter():\n",
|
||
" gen(model, ds_train.with_format('pt')[0], tokenizer)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"metadata": {},
|
||
"source": []
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"gen(model, ds_train.with_format('pt')[0], tokenizer)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": []
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": []
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": []
|
||
}
|
||
],
|
||
"metadata": {
|
||
"kernelspec": {
|
||
"display_name": ".venv",
|
||
"language": "python",
|
||
"name": "python3"
|
||
},
|
||
"language_info": {
|
||
"codemirror_mode": {
|
||
"name": "ipython",
|
||
"version": 3
|
||
},
|
||
"file_extension": ".py",
|
||
"mimetype": "text/x-python",
|
||
"name": "python",
|
||
"nbconvert_exporter": "python",
|
||
"pygments_lexer": "ipython3",
|
||
"version": "3.11.0rc1"
|
||
}
|
||
},
|
||
"nbformat": 4,
|
||
"nbformat_minor": 2
|
||
}
|