diff --git a/README.md b/README.md index 290872d..765d2cc 100644 --- a/README.md +++ b/README.md @@ -127,6 +127,12 @@ Try out the 🤗 Gradio Space which should run seamlessly on a T4 instance: ### Parameter Efficient Tuning of LLMs for RLHF components such as Ranker and Policy [ToDo] +### INT8 training of large models in Colab using PEFT LoRA and bits_and_bytes + +Here is now a demo on how to fine tune OPT-6.7b (14GB in fp16) in a Google colab: [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1jCkpikz0J2o20FBQmYmAGdiKmJGOMo-o?usp=sharing) + +Here is now a demo on how to fine tune wishper-large (1.5B params) (14GB in fp16) in a Google colab: [ToDo] + ### Save compute and storage even for medium and small models Save storage by avoiding full finetuning of models on each of the downstream tasks/datasets, @@ -307,12 +313,12 @@ any GPU memory savings. Please refer issue [[FSDP] FSDP with CPU offload consume 2. When using `P_TUNING` or `PROMPT_TUNING` with `SEQ_2_SEQ` task, remember to remove the `num_virtual_token` virtual prompt predictions from the left side of the model outputs during evaluations. -3. `P_TUNING` or `PROMPT_TUNING` doesn't support `generate` functionality of transformers bcause `generate` strictly requires `input_ids`/`decoder_input_ids` but +3. For encoder-decoder models, `P_TUNING` or `PROMPT_TUNING` doesn't support `generate` functionality of transformers because `generate` strictly requires `decoder_input_ids` but `P_TUNING`/`PROMPT_TUNING` appends soft prompt embeddings to `input_embeds` to create new `input_embeds` to be given to the model. Therefore, `generate` doesn't support this yet. ## Backlog: -1. Explore and possibly integrate `(IA)^3` and `UniPELT` +1. Explore and possibly integrate `(IA)^3` 2. Add tests 3. Add more use cases and examples diff --git a/examples/causal_language_modeling/accelerate_ds_zero3_cpu_offload_config.yaml b/examples/causal_language_modeling/accelerate_ds_zero3_cpu_offload_config.yaml new file mode 100644 index 0000000..a4a0bcf --- /dev/null +++ b/examples/causal_language_modeling/accelerate_ds_zero3_cpu_offload_config.yaml @@ -0,0 +1,22 @@ +compute_environment: LOCAL_MACHINE +deepspeed_config: + gradient_accumulation_steps: 1 + gradient_clipping: 1.0 + offload_optimizer_device: none + offload_param_device: none + zero3_init_flag: true + zero3_save_16bit_model: true + zero_stage: 3 +distributed_type: DEEPSPEED +downcast_bf16: 'no' +dynamo_backend: 'NO' +fsdp_config: {} +machine_rank: 0 +main_training_function: main +megatron_lm_config: {} +mixed_precision: 'no' +num_machines: 1 +num_processes: 1 +rdzv_backend: static +same_network: true +use_cpu: false \ No newline at end of file diff --git a/examples/causal_language_modeling/peft_lora_clm_accelerate_big_model_inference.ipynb b/examples/causal_language_modeling/peft_lora_clm_accelerate_big_model_inference.ipynb index 2f2f1ef..1d0ca35 100644 --- a/examples/causal_language_modeling/peft_lora_clm_accelerate_big_model_inference.ipynb +++ b/examples/causal_language_modeling/peft_lora_clm_accelerate_big_model_inference.ipynb @@ -5,10 +5,26 @@ "execution_count": 1, "id": "71fbfca2", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "===================================BUG REPORT===================================\n", + "Welcome to bitsandbytes. For bug reports, please submit your error trace to: https://github.com/TimDettmers/bitsandbytes/issues\n", + "For effortless bug reporting copy-paste your error into this form: https://docs.google.com/forms/d/e/1FAIpQLScPB8emS3Thkp66nvqwmjTEgxp8Y9ufuWTzFyr9kJ5AoI47dQ/viewform?usp=sf_link\n", + "================================================================================\n", + "CUDA SETUP: CUDA runtime path found: /home/sourab/miniconda3/envs/ml/lib/libcudart.so\n", + "CUDA SETUP: Highest compute capability among GPUs detected: 7.5\n", + "CUDA SETUP: Detected CUDA version 117\n", + "CUDA SETUP: Loading binary /home/sourab/miniconda3/envs/ml/lib/python3.10/site-packages/bitsandbytes/libbitsandbytes_cuda117.so...\n" + ] + } + ], "source": [ "from transformers import AutoModelForCausalLM\n", - "from peft import get_peft_config,get_peft_model, get_peft_model_state_dict, set_peft_model_state_dict, LoraConfig, TaskType, peft_model_load_and_dispatch\n", + "from peft import PeftModel, PeftConfig\n", "import torch\n", "from datasets import load_dataset\n", "import os\n", @@ -21,10 +37,7 @@ "device = \"cuda\"\n", "model_name_or_path = \"bigscience/bloomz-7b1\"\n", "tokenizer_name_or_path = \"bigscience/bloomz-7b1\"\n", - "peft_config = LoraConfig(task_type=TaskType.CAUSAL_LM, inference_mode=False, r=8, lora_alpha=32, lora_dropout=0.1)\n", - "\n", "dataset_name = \"twitter_complaints\"\n", - "checkpoint_name = \"/home/sourab/\"+f\"{dataset_name}_{model_name_or_path}_{peft_config.peft_type}_{peft_config.task_type}_v1.pt\".replace(\"/\", \"_\")\n", "text_column = \"Tweet text\"\n", "label_column = \"text_label\"\n", "max_length=64\n", @@ -35,70 +48,10 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "id": "e1a3648b", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Found cached dataset raft (/home/sourab/.cache/huggingface/datasets/ought___raft/twitter_complaints/1.1.0/79c4de1312c1e3730043f7db07179c914f48403101f7124e2fe336f6f54d9f84)\n" - ] - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "92d0876af16b4525a124c79cf2da14b2", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - " 0%| | 0/2 [00:00, task_type=, inference_mode=False, num_virtual_tokens=30, token_dim=1024, num_transformer_submodules=1, num_attention_heads=16, num_layers=24, encoder_hidden_size=1024, prefix_projection=False, postprocess_past_key_value_function=)" + "PrefixTuningConfig(peft_type=, base_model_name_or_path='bigscience/bloomz-560m', task_type=, inference_mode=False, num_virtual_tokens=30, token_dim=1024, num_transformer_submodules=1, num_attention_heads=16, num_layers=24, encoder_hidden_size=1024, prefix_projection=False)" ] }, - "execution_count": 11, + "execution_count": 12, "metadata": {}, "output_type": "execute_result" } @@ -1102,7 +367,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 13, "id": "b2f91568", "metadata": {}, "outputs": [], @@ -1119,7 +384,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 14, "id": "e4fb69fc", "metadata": {}, "outputs": [ @@ -1127,750 +392,750 @@ "name": "stderr", "output_type": "stream", "text": [ - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:01<00:00, 5.93it/s]\n", - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.43it/s]\n" + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:01<00:00, 5.79it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.51it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "epoch=0: train_ppl=tensor(4.1394e+09, device='cuda:0') train_epoch_loss=tensor(22.1438, device='cuda:0') eval_ppl=tensor(1402.4835, device='cuda:0') eval_epoch_loss=tensor(7.2460, device='cuda:0')\n" + "epoch=0: train_ppl=tensor(1.8325e+09, device='cuda:0') train_epoch_loss=tensor(21.3289, device='cuda:0') eval_ppl=tensor(2713.4180, device='cuda:0') eval_epoch_loss=tensor(7.9060, device='cuda:0')\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.39it/s]\n", - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.49it/s]\n" + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.44it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.53it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "epoch=1: train_ppl=tensor(246.8958, device='cuda:0') train_epoch_loss=tensor(5.5090, device='cuda:0') eval_ppl=tensor(62.6347, device='cuda:0') eval_epoch_loss=tensor(4.1373, device='cuda:0')\n" + "epoch=1: train_ppl=tensor(341.0600, device='cuda:0') train_epoch_loss=tensor(5.8321, device='cuda:0') eval_ppl=tensor(80.8206, device='cuda:0') eval_epoch_loss=tensor(4.3922, device='cuda:0')\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.40it/s]\n", - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.48it/s]\n" + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.44it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.55it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "epoch=2: train_ppl=tensor(40.7671, device='cuda:0') train_epoch_loss=tensor(3.7079, device='cuda:0') eval_ppl=tensor(20.4430, device='cuda:0') eval_epoch_loss=tensor(3.0176, device='cuda:0')\n" + "epoch=2: train_ppl=tensor(59.8778, device='cuda:0') train_epoch_loss=tensor(4.0923, device='cuda:0') eval_ppl=tensor(34.4593, device='cuda:0') eval_epoch_loss=tensor(3.5398, device='cuda:0')\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.37it/s]\n", - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.52it/s]\n" + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.45it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.55it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "epoch=3: train_ppl=tensor(13.3799, device='cuda:0') train_epoch_loss=tensor(2.5938, device='cuda:0') eval_ppl=tensor(7.8204, device='cuda:0') eval_epoch_loss=tensor(2.0567, device='cuda:0')\n" + "epoch=3: train_ppl=tensor(22.3307, device='cuda:0') train_epoch_loss=tensor(3.1060, device='cuda:0') eval_ppl=tensor(12.5947, device='cuda:0') eval_epoch_loss=tensor(2.5333, device='cuda:0')\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.44it/s]\n", - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.55it/s]\n" + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.45it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.56it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "epoch=4: train_ppl=tensor(5.6719, device='cuda:0') train_epoch_loss=tensor(1.7355, device='cuda:0') eval_ppl=tensor(3.2507, device='cuda:0') eval_epoch_loss=tensor(1.1789, device='cuda:0')\n" + "epoch=4: train_ppl=tensor(9.1697, device='cuda:0') train_epoch_loss=tensor(2.2159, device='cuda:0') eval_ppl=tensor(4.5289, device='cuda:0') eval_epoch_loss=tensor(1.5105, device='cuda:0')\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.44it/s]\n", - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.48it/s]\n" + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.45it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.52it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "epoch=5: train_ppl=tensor(2.4837, device='cuda:0') train_epoch_loss=tensor(0.9098, device='cuda:0') eval_ppl=tensor(1.5463, device='cuda:0') eval_epoch_loss=tensor(0.4359, device='cuda:0')\n" + "epoch=5: train_ppl=tensor(3.0172, device='cuda:0') train_epoch_loss=tensor(1.1043, device='cuda:0') eval_ppl=tensor(1.8092, device='cuda:0') eval_epoch_loss=tensor(0.5929, device='cuda:0')\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.43it/s]\n", - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.53it/s]\n" + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.44it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.45it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "epoch=6: train_ppl=tensor(1.4864, device='cuda:0') train_epoch_loss=tensor(0.3964, device='cuda:0') eval_ppl=tensor(1.8123, device='cuda:0') eval_epoch_loss=tensor(0.5946, device='cuda:0')\n" + "epoch=6: train_ppl=tensor(1.4885, device='cuda:0') train_epoch_loss=tensor(0.3978, device='cuda:0') eval_ppl=tensor(1.4449, device='cuda:0') eval_epoch_loss=tensor(0.3680, device='cuda:0')\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.45it/s]\n", - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.49it/s]\n" + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.43it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.48it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "epoch=7: train_ppl=tensor(1.4421, device='cuda:0') train_epoch_loss=tensor(0.3661, device='cuda:0') eval_ppl=tensor(1.6831, device='cuda:0') eval_epoch_loss=tensor(0.5206, device='cuda:0')\n" + "epoch=7: train_ppl=tensor(1.2967, device='cuda:0') train_epoch_loss=tensor(0.2598, device='cuda:0') eval_ppl=tensor(1.1587, device='cuda:0') eval_epoch_loss=tensor(0.1473, device='cuda:0')\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.43it/s]\n", - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.38it/s]\n" + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.43it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.47it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "epoch=8: train_ppl=tensor(1.5938, device='cuda:0') train_epoch_loss=tensor(0.4661, device='cuda:0') eval_ppl=tensor(1.4380, device='cuda:0') eval_epoch_loss=tensor(0.3632, device='cuda:0')\n" + "epoch=8: train_ppl=tensor(1.1305, device='cuda:0') train_epoch_loss=tensor(0.1227, device='cuda:0') eval_ppl=tensor(1.0874, device='cuda:0') eval_epoch_loss=tensor(0.0838, device='cuda:0')\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.43it/s]\n", - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.44it/s]\n" + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.45it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.46it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "epoch=9: train_ppl=tensor(1.2076, device='cuda:0') train_epoch_loss=tensor(0.1886, device='cuda:0') eval_ppl=tensor(1.2875, device='cuda:0') eval_epoch_loss=tensor(0.2527, device='cuda:0')\n" + "epoch=9: train_ppl=tensor(1.1608, device='cuda:0') train_epoch_loss=tensor(0.1491, device='cuda:0') eval_ppl=tensor(1.1461, device='cuda:0') eval_epoch_loss=tensor(0.1364, device='cuda:0')\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.42it/s]\n", - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.51it/s]\n" + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.45it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.45it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "epoch=10: train_ppl=tensor(1.2741, device='cuda:0') train_epoch_loss=tensor(0.2422, device='cuda:0') eval_ppl=tensor(1.2635, device='cuda:0') eval_epoch_loss=tensor(0.2339, device='cuda:0')\n" + "epoch=10: train_ppl=tensor(1.3172, device='cuda:0') train_epoch_loss=tensor(0.2755, device='cuda:0') eval_ppl=tensor(1.1320, device='cuda:0') eval_epoch_loss=tensor(0.1240, device='cuda:0')\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.42it/s]\n", - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.48it/s]\n" + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.44it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.46it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "epoch=11: train_ppl=tensor(1.2219, device='cuda:0') train_epoch_loss=tensor(0.2004, device='cuda:0') eval_ppl=tensor(1.1836, device='cuda:0') eval_epoch_loss=tensor(0.1686, device='cuda:0')\n" + "epoch=11: train_ppl=tensor(1.1437, device='cuda:0') train_epoch_loss=tensor(0.1343, device='cuda:0') eval_ppl=tensor(1.0676, device='cuda:0') eval_epoch_loss=tensor(0.0654, device='cuda:0')\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.43it/s]\n", - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.52it/s]\n" + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.43it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.43it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "epoch=12: train_ppl=tensor(1.1773, device='cuda:0') train_epoch_loss=tensor(0.1632, device='cuda:0') eval_ppl=tensor(1.1829, device='cuda:0') eval_epoch_loss=tensor(0.1680, device='cuda:0')\n" + "epoch=12: train_ppl=tensor(1.0651, device='cuda:0') train_epoch_loss=tensor(0.0630, device='cuda:0') eval_ppl=tensor(1.0735, device='cuda:0') eval_epoch_loss=tensor(0.0710, device='cuda:0')\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.41it/s]\n", - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.53it/s]\n" + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.46it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.47it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "epoch=13: train_ppl=tensor(1.1660, device='cuda:0') train_epoch_loss=tensor(0.1536, device='cuda:0') eval_ppl=tensor(1.1448, device='cuda:0') eval_epoch_loss=tensor(0.1353, device='cuda:0')\n" + "epoch=13: train_ppl=tensor(1.0607, device='cuda:0') train_epoch_loss=tensor(0.0589, device='cuda:0') eval_ppl=tensor(1.0399, device='cuda:0') eval_epoch_loss=tensor(0.0391, device='cuda:0')\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.42it/s]\n", - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.51it/s]\n" + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.43it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.44it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "epoch=14: train_ppl=tensor(1.1428, device='cuda:0') train_epoch_loss=tensor(0.1334, device='cuda:0') eval_ppl=tensor(1.1405, device='cuda:0') eval_epoch_loss=tensor(0.1315, device='cuda:0')\n" + "epoch=14: train_ppl=tensor(1.0351, device='cuda:0') train_epoch_loss=tensor(0.0345, device='cuda:0') eval_ppl=tensor(1.0260, device='cuda:0') eval_epoch_loss=tensor(0.0257, device='cuda:0')\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.42it/s]\n", - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.48it/s]\n" + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.44it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.43it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "epoch=15: train_ppl=tensor(1.1204, device='cuda:0') train_epoch_loss=tensor(0.1137, device='cuda:0') eval_ppl=tensor(1.1171, device='cuda:0') eval_epoch_loss=tensor(0.1108, device='cuda:0')\n" + "epoch=15: train_ppl=tensor(1.0217, device='cuda:0') train_epoch_loss=tensor(0.0215, device='cuda:0') eval_ppl=tensor(1.0168, device='cuda:0') eval_epoch_loss=tensor(0.0167, device='cuda:0')\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.42it/s]\n", - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.50it/s]\n" + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.43it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.28it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "epoch=16: train_ppl=tensor(1.1038, device='cuda:0') train_epoch_loss=tensor(0.0988, device='cuda:0') eval_ppl=tensor(1.0856, device='cuda:0') eval_epoch_loss=tensor(0.0821, device='cuda:0')\n" + "epoch=16: train_ppl=tensor(1.0152, device='cuda:0') train_epoch_loss=tensor(0.0151, device='cuda:0') eval_ppl=tensor(1.0117, device='cuda:0') eval_epoch_loss=tensor(0.0116, device='cuda:0')\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.43it/s]\n", - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.53it/s]\n" + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.43it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.41it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "epoch=17: train_ppl=tensor(1.0810, device='cuda:0') train_epoch_loss=tensor(0.0779, device='cuda:0') eval_ppl=tensor(1.0642, device='cuda:0') eval_epoch_loss=tensor(0.0623, device='cuda:0')\n" + "epoch=17: train_ppl=tensor(1.0102, device='cuda:0') train_epoch_loss=tensor(0.0101, device='cuda:0') eval_ppl=tensor(1.0088, device='cuda:0') eval_epoch_loss=tensor(0.0088, device='cuda:0')\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.41it/s]\n", - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.48it/s]\n" + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.29it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.25it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "epoch=18: train_ppl=tensor(1.0586, device='cuda:0') train_epoch_loss=tensor(0.0569, device='cuda:0') eval_ppl=tensor(1.0542, device='cuda:0') eval_epoch_loss=tensor(0.0528, device='cuda:0')\n" + "epoch=18: train_ppl=tensor(1.0083, device='cuda:0') train_epoch_loss=tensor(0.0083, device='cuda:0') eval_ppl=tensor(1.0073, device='cuda:0') eval_epoch_loss=tensor(0.0073, device='cuda:0')\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.43it/s]\n", - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.48it/s]\n" + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.43it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.46it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "epoch=19: train_ppl=tensor(1.0449, device='cuda:0') train_epoch_loss=tensor(0.0439, device='cuda:0') eval_ppl=tensor(1.0455, device='cuda:0') eval_epoch_loss=tensor(0.0445, device='cuda:0')\n" + "epoch=19: train_ppl=tensor(1.0070, device='cuda:0') train_epoch_loss=tensor(0.0070, device='cuda:0') eval_ppl=tensor(1.0064, device='cuda:0') eval_epoch_loss=tensor(0.0063, device='cuda:0')\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.43it/s]\n", - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.47it/s]\n" + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.43it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.51it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "epoch=20: train_ppl=tensor(1.0310, device='cuda:0') train_epoch_loss=tensor(0.0306, device='cuda:0') eval_ppl=tensor(1.0159, device='cuda:0') eval_epoch_loss=tensor(0.0158, device='cuda:0')\n" + "epoch=20: train_ppl=tensor(1.0059, device='cuda:0') train_epoch_loss=tensor(0.0059, device='cuda:0') eval_ppl=tensor(1.0057, device='cuda:0') eval_epoch_loss=tensor(0.0057, device='cuda:0')\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.43it/s]\n", - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.42it/s]\n" + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.43it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.47it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "epoch=21: train_ppl=tensor(1.0200, device='cuda:0') train_epoch_loss=tensor(0.0198, device='cuda:0') eval_ppl=tensor(1.0333, device='cuda:0') eval_epoch_loss=tensor(0.0327, device='cuda:0')\n" + "epoch=21: train_ppl=tensor(1.0056, device='cuda:0') train_epoch_loss=tensor(0.0056, device='cuda:0') eval_ppl=tensor(1.0052, device='cuda:0') eval_epoch_loss=tensor(0.0052, device='cuda:0')\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.43it/s]\n", - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.46it/s]\n" + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.41it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.33it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "epoch=22: train_ppl=tensor(1.0209, device='cuda:0') train_epoch_loss=tensor(0.0207, device='cuda:0') eval_ppl=tensor(1.0187, device='cuda:0') eval_epoch_loss=tensor(0.0185, device='cuda:0')\n" + "epoch=22: train_ppl=tensor(1.0050, device='cuda:0') train_epoch_loss=tensor(0.0050, device='cuda:0') eval_ppl=tensor(1.0049, device='cuda:0') eval_epoch_loss=tensor(0.0049, device='cuda:0')\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.43it/s]\n", - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.40it/s]\n" + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.39it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.44it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "epoch=23: train_ppl=tensor(1.0098, device='cuda:0') train_epoch_loss=tensor(0.0097, device='cuda:0') eval_ppl=tensor(1.0097, device='cuda:0') eval_epoch_loss=tensor(0.0096, device='cuda:0')\n" + "epoch=23: train_ppl=tensor(1.0049, device='cuda:0') train_epoch_loss=tensor(0.0049, device='cuda:0') eval_ppl=tensor(1.0045, device='cuda:0') eval_epoch_loss=tensor(0.0045, device='cuda:0')\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.45it/s]\n", - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.50it/s]\n" + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.42it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.49it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "epoch=24: train_ppl=tensor(1.0120, device='cuda:0') train_epoch_loss=tensor(0.0120, device='cuda:0') eval_ppl=tensor(1.0162, device='cuda:0') eval_epoch_loss=tensor(0.0161, device='cuda:0')\n" + "epoch=24: train_ppl=tensor(1.0043, device='cuda:0') train_epoch_loss=tensor(0.0043, device='cuda:0') eval_ppl=tensor(1.0043, device='cuda:0') eval_epoch_loss=tensor(0.0043, device='cuda:0')\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.43it/s]\n", - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.49it/s]\n" + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.46it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.47it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "epoch=25: train_ppl=tensor(1.0120, device='cuda:0') train_epoch_loss=tensor(0.0119, device='cuda:0') eval_ppl=tensor(1.0090, device='cuda:0') eval_epoch_loss=tensor(0.0090, device='cuda:0')\n" + "epoch=25: train_ppl=tensor(1.0042, device='cuda:0') train_epoch_loss=tensor(0.0042, device='cuda:0') eval_ppl=tensor(1.0040, device='cuda:0') eval_epoch_loss=tensor(0.0040, device='cuda:0')\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.43it/s]\n", - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.40it/s]\n" + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.44it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.52it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "epoch=26: train_ppl=tensor(1.0125, device='cuda:0') train_epoch_loss=tensor(0.0124, device='cuda:0') eval_ppl=tensor(1.0079, device='cuda:0') eval_epoch_loss=tensor(0.0079, device='cuda:0')\n" + "epoch=26: train_ppl=tensor(1.0039, device='cuda:0') train_epoch_loss=tensor(0.0039, device='cuda:0') eval_ppl=tensor(1.0039, device='cuda:0') eval_epoch_loss=tensor(0.0039, device='cuda:0')\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.42it/s]\n", - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.47it/s]\n" + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.44it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.48it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "epoch=27: train_ppl=tensor(1.0072, device='cuda:0') train_epoch_loss=tensor(0.0072, device='cuda:0') eval_ppl=tensor(1.0043, device='cuda:0') eval_epoch_loss=tensor(0.0043, device='cuda:0')\n" + "epoch=27: train_ppl=tensor(1.0038, device='cuda:0') train_epoch_loss=tensor(0.0038, device='cuda:0') eval_ppl=tensor(1.0037, device='cuda:0') eval_epoch_loss=tensor(0.0037, device='cuda:0')\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.42it/s]\n", - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.44it/s]\n" + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.46it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.54it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "epoch=28: train_ppl=tensor(1.0044, device='cuda:0') train_epoch_loss=tensor(0.0044, device='cuda:0') eval_ppl=tensor(1.0049, device='cuda:0') eval_epoch_loss=tensor(0.0048, device='cuda:0')\n" + "epoch=28: train_ppl=tensor(1.0036, device='cuda:0') train_epoch_loss=tensor(0.0036, device='cuda:0') eval_ppl=tensor(1.0035, device='cuda:0') eval_epoch_loss=tensor(0.0035, device='cuda:0')\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.42it/s]\n", - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.51it/s]\n" + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.45it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.53it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "epoch=29: train_ppl=tensor(1.0047, device='cuda:0') train_epoch_loss=tensor(0.0047, device='cuda:0') eval_ppl=tensor(1.0038, device='cuda:0') eval_epoch_loss=tensor(0.0038, device='cuda:0')\n" + "epoch=29: train_ppl=tensor(1.0034, device='cuda:0') train_epoch_loss=tensor(0.0034, device='cuda:0') eval_ppl=tensor(1.0034, device='cuda:0') eval_epoch_loss=tensor(0.0034, device='cuda:0')\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.41it/s]\n", - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.52it/s]\n" + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.43it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.47it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "epoch=30: train_ppl=tensor(1.0037, device='cuda:0') train_epoch_loss=tensor(0.0037, device='cuda:0') eval_ppl=tensor(1.0035, device='cuda:0') eval_epoch_loss=tensor(0.0035, device='cuda:0')\n" + "epoch=30: train_ppl=tensor(1.0034, device='cuda:0') train_epoch_loss=tensor(0.0034, device='cuda:0') eval_ppl=tensor(1.0033, device='cuda:0') eval_epoch_loss=tensor(0.0033, device='cuda:0')\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.42it/s]\n", - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.47it/s]\n" + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.43it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.47it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "epoch=31: train_ppl=tensor(1.0033, device='cuda:0') train_epoch_loss=tensor(0.0033, device='cuda:0') eval_ppl=tensor(1.0031, device='cuda:0') eval_epoch_loss=tensor(0.0031, device='cuda:0')\n" + "epoch=31: train_ppl=tensor(1.0033, device='cuda:0') train_epoch_loss=tensor(0.0033, device='cuda:0') eval_ppl=tensor(1.0032, device='cuda:0') eval_epoch_loss=tensor(0.0032, device='cuda:0')\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.43it/s]\n", - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.48it/s]\n" + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.46it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.51it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "epoch=32: train_ppl=tensor(1.0029, device='cuda:0') train_epoch_loss=tensor(0.0029, device='cuda:0') eval_ppl=tensor(1.0029, device='cuda:0') eval_epoch_loss=tensor(0.0029, device='cuda:0')\n" + "epoch=32: train_ppl=tensor(1.0031, device='cuda:0') train_epoch_loss=tensor(0.0031, device='cuda:0') eval_ppl=tensor(1.0031, device='cuda:0') eval_epoch_loss=tensor(0.0031, device='cuda:0')\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.42it/s]\n", - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.45it/s]\n" + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.44it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.43it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "epoch=33: train_ppl=tensor(1.0027, device='cuda:0') train_epoch_loss=tensor(0.0027, device='cuda:0') eval_ppl=tensor(1.0028, device='cuda:0') eval_epoch_loss=tensor(0.0028, device='cuda:0')\n" + "epoch=33: train_ppl=tensor(1.0030, device='cuda:0') train_epoch_loss=tensor(0.0030, device='cuda:0') eval_ppl=tensor(1.0030, device='cuda:0') eval_epoch_loss=tensor(0.0030, device='cuda:0')\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.42it/s]\n", - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.50it/s]\n" + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.45it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.46it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "epoch=34: train_ppl=tensor(1.0026, device='cuda:0') train_epoch_loss=tensor(0.0026, device='cuda:0') eval_ppl=tensor(1.0027, device='cuda:0') eval_epoch_loss=tensor(0.0027, device='cuda:0')\n" + "epoch=34: train_ppl=tensor(1.0029, device='cuda:0') train_epoch_loss=tensor(0.0029, device='cuda:0') eval_ppl=tensor(1.0029, device='cuda:0') eval_epoch_loss=tensor(0.0029, device='cuda:0')\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.43it/s]\n", - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.49it/s]\n" + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.45it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.47it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "epoch=35: train_ppl=tensor(1.0026, device='cuda:0') train_epoch_loss=tensor(0.0026, device='cuda:0') eval_ppl=tensor(1.0026, device='cuda:0') eval_epoch_loss=tensor(0.0026, device='cuda:0')\n" + "epoch=35: train_ppl=tensor(1.0028, device='cuda:0') train_epoch_loss=tensor(0.0028, device='cuda:0') eval_ppl=tensor(1.0029, device='cuda:0') eval_epoch_loss=tensor(0.0029, device='cuda:0')\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.30it/s]\n", - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.03it/s]\n" + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.45it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.45it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "epoch=36: train_ppl=tensor(1.0025, device='cuda:0') train_epoch_loss=tensor(0.0025, device='cuda:0') eval_ppl=tensor(1.0025, device='cuda:0') eval_epoch_loss=tensor(0.0025, device='cuda:0')\n" + "epoch=36: train_ppl=tensor(1.0027, device='cuda:0') train_epoch_loss=tensor(0.0027, device='cuda:0') eval_ppl=tensor(1.0028, device='cuda:0') eval_epoch_loss=tensor(0.0028, device='cuda:0')\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.42it/s]\n", - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.47it/s]\n" + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.45it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.47it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "epoch=37: train_ppl=tensor(1.0024, device='cuda:0') train_epoch_loss=tensor(0.0023, device='cuda:0') eval_ppl=tensor(1.0024, device='cuda:0') eval_epoch_loss=tensor(0.0024, device='cuda:0')\n" + "epoch=37: train_ppl=tensor(1.0027, device='cuda:0') train_epoch_loss=tensor(0.0027, device='cuda:0') eval_ppl=tensor(1.0027, device='cuda:0') eval_epoch_loss=tensor(0.0027, device='cuda:0')\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.40it/s]\n", - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.47it/s]\n" + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.45it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.46it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "epoch=38: train_ppl=tensor(1.0023, device='cuda:0') train_epoch_loss=tensor(0.0023, device='cuda:0') eval_ppl=tensor(1.0023, device='cuda:0') eval_epoch_loss=tensor(0.0023, device='cuda:0')\n" + "epoch=38: train_ppl=tensor(1.0027, device='cuda:0') train_epoch_loss=tensor(0.0027, device='cuda:0') eval_ppl=tensor(1.0027, device='cuda:0') eval_epoch_loss=tensor(0.0027, device='cuda:0')\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.42it/s]\n", - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.47it/s]\n" + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.44it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.43it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "epoch=39: train_ppl=tensor(1.0023, device='cuda:0') train_epoch_loss=tensor(0.0023, device='cuda:0') eval_ppl=tensor(1.0023, device='cuda:0') eval_epoch_loss=tensor(0.0023, device='cuda:0')\n" + "epoch=39: train_ppl=tensor(1.0025, device='cuda:0') train_epoch_loss=tensor(0.0025, device='cuda:0') eval_ppl=tensor(1.0026, device='cuda:0') eval_epoch_loss=tensor(0.0026, device='cuda:0')\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.42it/s]\n", - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.51it/s]\n" + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.44it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.47it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "epoch=40: train_ppl=tensor(1.0022, device='cuda:0') train_epoch_loss=tensor(0.0022, device='cuda:0') eval_ppl=tensor(1.0023, device='cuda:0') eval_epoch_loss=tensor(0.0022, device='cuda:0')\n" + "epoch=40: train_ppl=tensor(1.0026, device='cuda:0') train_epoch_loss=tensor(0.0026, device='cuda:0') eval_ppl=tensor(1.0026, device='cuda:0') eval_epoch_loss=tensor(0.0026, device='cuda:0')\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.43it/s]\n", - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.46it/s]\n" + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.44it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.33it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "epoch=41: train_ppl=tensor(1.0022, device='cuda:0') train_epoch_loss=tensor(0.0022, device='cuda:0') eval_ppl=tensor(1.0022, device='cuda:0') eval_epoch_loss=tensor(0.0022, device='cuda:0')\n" + "epoch=41: train_ppl=tensor(1.0025, device='cuda:0') train_epoch_loss=tensor(0.0025, device='cuda:0') eval_ppl=tensor(1.0025, device='cuda:0') eval_epoch_loss=tensor(0.0025, device='cuda:0')\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.43it/s]\n", - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.43it/s]\n" + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.42it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.49it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "epoch=42: train_ppl=tensor(1.0021, device='cuda:0') train_epoch_loss=tensor(0.0021, device='cuda:0') eval_ppl=tensor(1.0022, device='cuda:0') eval_epoch_loss=tensor(0.0022, device='cuda:0')\n" + "epoch=42: train_ppl=tensor(1.0024, device='cuda:0') train_epoch_loss=tensor(0.0024, device='cuda:0') eval_ppl=tensor(1.0025, device='cuda:0') eval_epoch_loss=tensor(0.0025, device='cuda:0')\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.43it/s]\n", - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.49it/s]\n" + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.44it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.47it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "epoch=43: train_ppl=tensor(1.0022, device='cuda:0') train_epoch_loss=tensor(0.0022, device='cuda:0') eval_ppl=tensor(1.0021, device='cuda:0') eval_epoch_loss=tensor(0.0021, device='cuda:0')\n" + "epoch=43: train_ppl=tensor(1.0024, device='cuda:0') train_epoch_loss=tensor(0.0024, device='cuda:0') eval_ppl=tensor(1.0025, device='cuda:0') eval_epoch_loss=tensor(0.0025, device='cuda:0')\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.43it/s]\n", - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.46it/s]\n" + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.44it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.43it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "epoch=44: train_ppl=tensor(1.0021, device='cuda:0') train_epoch_loss=tensor(0.0021, device='cuda:0') eval_ppl=tensor(1.0021, device='cuda:0') eval_epoch_loss=tensor(0.0021, device='cuda:0')\n" + "epoch=44: train_ppl=tensor(1.0025, device='cuda:0') train_epoch_loss=tensor(0.0024, device='cuda:0') eval_ppl=tensor(1.0024, device='cuda:0') eval_epoch_loss=tensor(0.0024, device='cuda:0')\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.42it/s]\n", - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.40it/s]\n" + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.43it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.50it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "epoch=45: train_ppl=tensor(1.0020, device='cuda:0') train_epoch_loss=tensor(0.0020, device='cuda:0') eval_ppl=tensor(1.0021, device='cuda:0') eval_epoch_loss=tensor(0.0021, device='cuda:0')\n" + "epoch=45: train_ppl=tensor(1.0024, device='cuda:0') train_epoch_loss=tensor(0.0024, device='cuda:0') eval_ppl=tensor(1.0024, device='cuda:0') eval_epoch_loss=tensor(0.0024, device='cuda:0')\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.41it/s]\n", - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.43it/s]\n" + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.43it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.49it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "epoch=46: train_ppl=tensor(1.0021, device='cuda:0') train_epoch_loss=tensor(0.0021, device='cuda:0') eval_ppl=tensor(1.0021, device='cuda:0') eval_epoch_loss=tensor(0.0021, device='cuda:0')\n" + "epoch=46: train_ppl=tensor(1.0024, device='cuda:0') train_epoch_loss=tensor(0.0024, device='cuda:0') eval_ppl=tensor(1.0024, device='cuda:0') eval_epoch_loss=tensor(0.0024, device='cuda:0')\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.40it/s]\n", - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.45it/s]\n" + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.42it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.39it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "epoch=47: train_ppl=tensor(1.0021, device='cuda:0') train_epoch_loss=tensor(0.0021, device='cuda:0') eval_ppl=tensor(1.0021, device='cuda:0') eval_epoch_loss=tensor(0.0021, device='cuda:0')\n" + "epoch=47: train_ppl=tensor(1.0023, device='cuda:0') train_epoch_loss=tensor(0.0023, device='cuda:0') eval_ppl=tensor(1.0024, device='cuda:0') eval_epoch_loss=tensor(0.0024, device='cuda:0')\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.41it/s]\n", - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.43it/s]\n" + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.40it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.40it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "epoch=48: train_ppl=tensor(1.0020, device='cuda:0') train_epoch_loss=tensor(0.0020, device='cuda:0') eval_ppl=tensor(1.0021, device='cuda:0') eval_epoch_loss=tensor(0.0021, device='cuda:0')\n" + "epoch=48: train_ppl=tensor(1.0023, device='cuda:0') train_epoch_loss=tensor(0.0023, device='cuda:0') eval_ppl=tensor(1.0024, device='cuda:0') eval_epoch_loss=tensor(0.0024, device='cuda:0')\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.41it/s]\n", - "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.42it/s]" + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.41it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 21.87it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "epoch=49: train_ppl=tensor(1.0020, device='cuda:0') train_epoch_loss=tensor(0.0020, device='cuda:0') eval_ppl=tensor(1.0021, device='cuda:0') eval_epoch_loss=tensor(0.0021, device='cuda:0')\n" + "epoch=49: train_ppl=tensor(1.0023, device='cuda:0') train_epoch_loss=tensor(0.0023, device='cuda:0') eval_ppl=tensor(1.0024, device='cuda:0') eval_epoch_loss=tensor(0.0024, device='cuda:0')\n" ] }, { @@ -1920,7 +1185,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 36, "id": "53752a7b", "metadata": {}, "outputs": [ @@ -1928,32 +1193,105 @@ "name": "stdout", "output_type": "stream", "text": [ - "@VW @QuirkCars some unauthorized work done on my engine now throwing a check engine light about a week later. Very upset.\n", - "{'input_ids': tensor([[227985, 5484, 915, 2566, 57, 58, 2566, 5232, 132511,\n", - " 38, 4599, 3331, 1035, 192352, 2909, 11541, 664, 2670,\n", - " 22218, 5840, 108218, 267, 7010, 22218, 12490, 3638, 267,\n", - " 14319, 10494, 17, 93269, 123055, 17, 77658, 915, 210]]), 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n", - " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]])}\n", - "tensor([[227985, 5484, 915, 2566, 57, 58, 2566, 5232, 132511,\n", - " 38, 4599, 3331, 1035, 192352, 2909, 11541, 664, 2670,\n", - " 22218, 5840, 108218, 267, 7010, 22218, 12490, 3638, 267,\n", - " 14319, 10494, 17, 93269, 123055, 17, 77658, 915, 210,\n", - " 16449, 5952, 3, 3, 3, 3, 3, 3, 3,\n", + "Hey @nytimes your link to cancel my subscription isn't working and nobody is answering the chat. Please don't play that kind of stupid game.\n", + "{'input_ids': tensor([[227985, 5484, 915, 54078, 2566, 7782, 24502, 2632, 8989,\n", + " 427, 36992, 2670, 140711, 21994, 10789, 530, 88399, 632,\n", + " 183542, 368, 44799, 17, 29901, 5926, 7229, 861, 11596,\n", + " 461, 78851, 14775, 17, 77658, 915, 210]]), 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n", + " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]])}\n", + "tensor([[227985, 5484, 915, 54078, 2566, 7782, 24502, 2632, 8989,\n", + " 427, 36992, 2670, 140711, 21994, 10789, 530, 88399, 632,\n", + " 183542, 368, 44799, 17, 29901, 5926, 7229, 861, 11596,\n", + " 461, 78851, 14775, 17, 77658, 915, 210, 16449, 5952,\n", " 3]], device='cuda:0')\n", - "['Tweet text : @VW @QuirkCars some unauthorized work done on my engine now throwing a check engine light about a week later. Very upset. Label : complaint']\n" + "[\"Tweet text : Hey @nytimes your link to cancel my subscription isn't working and nobody is answering the chat. Please don't play that kind of stupid game. Label : complaint\"]\n" ] } ], "source": [ "model.eval()\n", - "i = 12\n", + "i = 16\n", "inputs = tokenizer(f'{text_column} : {dataset[\"test\"][i][\"Tweet text\"]} Label : ', return_tensors=\"pt\")\n", "print(dataset[\"test\"][i][\"Tweet text\"])\n", "print(inputs)\n", "\n", "with torch.no_grad():\n", " inputs = {k: v.to(device) for k, v in inputs.items()}\n", - " outputs = model.generate(input_ids=inputs[\"input_ids\"], attention_mask=inputs[\"attention_mask\"], max_new_tokens=10)\n", + " outputs = model.generate(input_ids=inputs[\"input_ids\"], attention_mask=inputs[\"attention_mask\"], max_new_tokens=10, eos_token_id=3)\n", + " print(outputs)\n", + " print(tokenizer.batch_decode(outputs.detach().cpu().numpy(), skip_special_tokens=True))\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "24041ee1", + "metadata": {}, + "outputs": [], + "source": [ + "# saving model\n", + "peft_model_id = f\"{model_name_or_path}_{peft_config.peft_type}_{peft_config.task_type}\"\n", + "model.save_pretrained(peft_model_id)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "527eeaa4", + "metadata": {}, + "outputs": [], + "source": [ + "ckpt = f\"{peft_model_id}/adapter_model.bin\"\n", + "!du -h $ckpt" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "b19f5a90", + "metadata": {}, + "outputs": [], + "source": [ + "from peft import PeftModel, PeftConfig\n", + "peft_model_id = f\"{model_name_or_path}_{peft_config.peft_type}_{peft_config.task_type}\"\n", + "\n", + "config = PeftConfig.from_pretrained(peft_model_id)\n", + "model = AutoModelForCausalLM.from_pretrained(config.base_model_name_or_path)\n", + "model = PeftModel.from_pretrained(model, peft_model_id)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "a11a3768", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "@greateranglia Ok thanks...\n", + "{'input_ids': tensor([[227985, 5484, 915, 2566, 14173, 2960, 29906, 387, 20706,\n", + " 49337, 1369, 77658, 915, 210]]), 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]])}\n", + "tensor([[227985, 5484, 915, 2566, 14173, 2960, 29906, 387, 20706,\n", + " 49337, 1369, 77658, 915, 210, 1936, 106863, 3]],\n", + " device='cuda:0')\n", + "['Tweet text : @greateranglia Ok thanks... Label : no complaint']\n" + ] + } + ], + "source": [ + "model.to(device)\n", + "model.eval()\n", + "i = 4\n", + "inputs = tokenizer(f'{text_column} : {dataset[\"test\"][i][\"Tweet text\"]} Label : ', return_tensors=\"pt\")\n", + "print(dataset[\"test\"][i][\"Tweet text\"])\n", + "print(inputs)\n", + "\n", + "with torch.no_grad():\n", + " inputs = {k: v.to(device) for k, v in inputs.items()}\n", + " outputs = model.generate(input_ids=inputs[\"input_ids\"], attention_mask=inputs[\"attention_mask\"], max_new_tokens=10, eos_token_id=3)\n", " print(outputs)\n", " print(tokenizer.batch_decode(outputs.detach().cpu().numpy(), skip_special_tokens=True))\n", " " @@ -1962,561 +1300,23 @@ { "cell_type": "code", "execution_count": null, - "id": "d8ba1f8c", + "id": "f890c951", "metadata": {}, "outputs": [], - "source": [ - "model.eval()\n", - "eval_loss = 0\n", - "eval_preds = []\n", - "for step, batch in enumerate(tqdm(test_dataloader)):\n", - " batch = {k: v.to(device) for k, v in batch.items() if k!=\"labels\"}\n", - " with torch.no_grad():\n", - " outputs = model.generate(**batch, max_new_tokens=10)\n", - " eval_preds.extend(tokenizer.batch_decode(outputs.detach().cpu().numpy(), skip_special_tokens=True))\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "id": "252f733d", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'prompt_embeddings': tensor([[ 0.7356, -1.0849, -0.4560, ..., 1.0242, 0.3908, -0.8000],\n", - " [-1.5587, -0.3595, 0.1289, ..., 0.5427, 1.0976, 1.7641],\n", - " [-0.2113, -1.4675, -0.5976, ..., 0.1691, -0.5843, -0.2658],\n", - " ...,\n", - " [ 2.1275, 0.7253, 0.0323, ..., -1.2285, -0.5614, 0.0370],\n", - " [-0.2258, -1.5149, 0.0685, ..., -1.4476, -0.1348, -0.6910],\n", - " [ 0.9089, 0.3947, -1.5271, ..., 1.9079, 0.6473, 0.7306]])}\n" - ] - } - ], - "source": [ - "# saving model\n", - "state_dict = get_peft_model_state_dict(model)\n", - "torch.save(state_dict, checkpoint_name)\n", - "print(state_dict)" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "id": "4928c7f1", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", - "To disable this warning, you can either:\n", - "\t- Avoid using `tokenizers` before the fork if possible\n", - "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", - "5,7M\ttwitter_complaints_bigscience_bloomz-560m_PREFIX_TUNING_CAUSAL_LM_v1.pt\r\n" - ] - } - ], - "source": [ - "!du -h $checkpoint_name" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "4d9476e1", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "PrefixTuningConfig(pet_type=, task_type=, inference_mode=True, num_virtual_tokens=30, token_dim=None, num_transformer_submodules=1, num_attention_heads=None, num_layers=None, encoder_hidden_size=None, prefix_projection=False, postprocess_past_key_value_function=)\n", - "trainable params: 1474560 || all params: 560689152 || trainable%: 0.26299064191632515\n" - ] - } - ], - "source": [ - "max_memory={0: \"1GIB\", 1: \"1GIB\", 2: \"2GIB\", 3: \"2GIB\", \"cpu\":\"30GB\"}\n", - "\n", - "peft_config.inference_mode = True\n", - "print(peft_config)\n", - "model = AutoModelForCausalLM.from_pretrained(model_name_or_path, device_map=\"auto\", max_memory=max_memory)\n", - "model = peft_model_load_and_dispatch(model, torch.load(checkpoint_name), peft_config, max_memory)\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "id": "5c7b3d71", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "PETModelForCausalLM(\n", - " (base_model): BloomForCausalLM(\n", - " (transformer): BloomModel(\n", - " (word_embeddings): Embedding(250880, 1024)\n", - " (word_embeddings_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", - " (h): ModuleList(\n", - " (0): BloomBlock(\n", - " (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", - " (self_attention): BloomAttention(\n", - " (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)\n", - " (dense): Linear(in_features=1024, out_features=1024, bias=True)\n", - " (attention_dropout): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", - " (mlp): BloomMLP(\n", - " (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)\n", - " (gelu_impl): BloomGelu()\n", - " (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)\n", - " )\n", - " )\n", - " (1): BloomBlock(\n", - " (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", - " (self_attention): BloomAttention(\n", - " (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)\n", - " (dense): Linear(in_features=1024, out_features=1024, bias=True)\n", - " (attention_dropout): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", - " (mlp): BloomMLP(\n", - " (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)\n", - " (gelu_impl): BloomGelu()\n", - " (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)\n", - " )\n", - " )\n", - " (2): BloomBlock(\n", - " (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", - " (self_attention): BloomAttention(\n", - " (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)\n", - " (dense): Linear(in_features=1024, out_features=1024, bias=True)\n", - " (attention_dropout): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", - " (mlp): BloomMLP(\n", - " (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)\n", - " (gelu_impl): BloomGelu()\n", - " (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)\n", - " )\n", - " )\n", - " (3): BloomBlock(\n", - " (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", - " (self_attention): BloomAttention(\n", - " (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)\n", - " (dense): Linear(in_features=1024, out_features=1024, bias=True)\n", - " (attention_dropout): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", - " (mlp): BloomMLP(\n", - " (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)\n", - " (gelu_impl): BloomGelu()\n", - " (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)\n", - " )\n", - " )\n", - " (4): BloomBlock(\n", - " (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", - " (self_attention): BloomAttention(\n", - " (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)\n", - " (dense): Linear(in_features=1024, out_features=1024, bias=True)\n", - " (attention_dropout): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", - " (mlp): BloomMLP(\n", - " (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)\n", - " (gelu_impl): BloomGelu()\n", - " (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)\n", - " )\n", - " )\n", - " (5): BloomBlock(\n", - " (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", - " (self_attention): BloomAttention(\n", - " (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)\n", - " (dense): Linear(in_features=1024, out_features=1024, bias=True)\n", - " (attention_dropout): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", - " (mlp): BloomMLP(\n", - " (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)\n", - " (gelu_impl): BloomGelu()\n", - " (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)\n", - " )\n", - " )\n", - " (6): BloomBlock(\n", - " (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", - " (self_attention): BloomAttention(\n", - " (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)\n", - " (dense): Linear(in_features=1024, out_features=1024, bias=True)\n", - " (attention_dropout): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", - " (mlp): BloomMLP(\n", - " (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)\n", - " (gelu_impl): BloomGelu()\n", - " (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)\n", - " )\n", - " )\n", - " (7): BloomBlock(\n", - " (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", - " (self_attention): BloomAttention(\n", - " (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)\n", - " (dense): Linear(in_features=1024, out_features=1024, bias=True)\n", - " (attention_dropout): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", - " (mlp): BloomMLP(\n", - " (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)\n", - " (gelu_impl): BloomGelu()\n", - " (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)\n", - " )\n", - " )\n", - " (8): BloomBlock(\n", - " (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", - " (self_attention): BloomAttention(\n", - " (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)\n", - " (dense): Linear(in_features=1024, out_features=1024, bias=True)\n", - " (attention_dropout): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", - " (mlp): BloomMLP(\n", - " (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)\n", - " (gelu_impl): BloomGelu()\n", - " (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)\n", - " )\n", - " )\n", - " (9): BloomBlock(\n", - " (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", - " (self_attention): BloomAttention(\n", - " (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)\n", - " (dense): Linear(in_features=1024, out_features=1024, bias=True)\n", - " (attention_dropout): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", - " (mlp): BloomMLP(\n", - " (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)\n", - " (gelu_impl): BloomGelu()\n", - " (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)\n", - " )\n", - " )\n", - " (10): BloomBlock(\n", - " (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", - " (self_attention): BloomAttention(\n", - " (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)\n", - " (dense): Linear(in_features=1024, out_features=1024, bias=True)\n", - " (attention_dropout): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", - " (mlp): BloomMLP(\n", - " (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)\n", - " (gelu_impl): BloomGelu()\n", - " (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)\n", - " )\n", - " )\n", - " (11): BloomBlock(\n", - " (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", - " (self_attention): BloomAttention(\n", - " (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)\n", - " (dense): Linear(in_features=1024, out_features=1024, bias=True)\n", - " (attention_dropout): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", - " (mlp): BloomMLP(\n", - " (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)\n", - " (gelu_impl): BloomGelu()\n", - " (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)\n", - " )\n", - " )\n", - " (12): BloomBlock(\n", - " (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", - " (self_attention): BloomAttention(\n", - " (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)\n", - " (dense): Linear(in_features=1024, out_features=1024, bias=True)\n", - " (attention_dropout): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", - " (mlp): BloomMLP(\n", - " (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)\n", - " (gelu_impl): BloomGelu()\n", - " (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)\n", - " )\n", - " )\n", - " (13): BloomBlock(\n", - " (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", - " (self_attention): BloomAttention(\n", - " (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)\n", - " (dense): Linear(in_features=1024, out_features=1024, bias=True)\n", - " (attention_dropout): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", - " (mlp): BloomMLP(\n", - " (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)\n", - " (gelu_impl): BloomGelu()\n", - " (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)\n", - " )\n", - " )\n", - " (14): BloomBlock(\n", - " (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", - " (self_attention): BloomAttention(\n", - " (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)\n", - " (dense): Linear(in_features=1024, out_features=1024, bias=True)\n", - " (attention_dropout): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", - " (mlp): BloomMLP(\n", - " (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)\n", - " (gelu_impl): BloomGelu()\n", - " (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)\n", - " )\n", - " )\n", - " (15): BloomBlock(\n", - " (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", - " (self_attention): BloomAttention(\n", - " (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)\n", - " (dense): Linear(in_features=1024, out_features=1024, bias=True)\n", - " (attention_dropout): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", - " (mlp): BloomMLP(\n", - " (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)\n", - " (gelu_impl): BloomGelu()\n", - " (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)\n", - " )\n", - " )\n", - " (16): BloomBlock(\n", - " (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", - " (self_attention): BloomAttention(\n", - " (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)\n", - " (dense): Linear(in_features=1024, out_features=1024, bias=True)\n", - " (attention_dropout): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", - " (mlp): BloomMLP(\n", - " (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)\n", - " (gelu_impl): BloomGelu()\n", - " (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)\n", - " )\n", - " )\n", - " (17): BloomBlock(\n", - " (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", - " (self_attention): BloomAttention(\n", - " (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)\n", - " (dense): Linear(in_features=1024, out_features=1024, bias=True)\n", - " (attention_dropout): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", - " (mlp): BloomMLP(\n", - " (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)\n", - " (gelu_impl): BloomGelu()\n", - " (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)\n", - " )\n", - " )\n", - " (18): BloomBlock(\n", - " (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", - " (self_attention): BloomAttention(\n", - " (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)\n", - " (dense): Linear(in_features=1024, out_features=1024, bias=True)\n", - " (attention_dropout): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", - " (mlp): BloomMLP(\n", - " (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)\n", - " (gelu_impl): BloomGelu()\n", - " (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)\n", - " )\n", - " )\n", - " (19): BloomBlock(\n", - " (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", - " (self_attention): BloomAttention(\n", - " (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)\n", - " (dense): Linear(in_features=1024, out_features=1024, bias=True)\n", - " (attention_dropout): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", - " (mlp): BloomMLP(\n", - " (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)\n", - " (gelu_impl): BloomGelu()\n", - " (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)\n", - " )\n", - " )\n", - " (20): BloomBlock(\n", - " (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", - " (self_attention): BloomAttention(\n", - " (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)\n", - " (dense): Linear(in_features=1024, out_features=1024, bias=True)\n", - " (attention_dropout): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", - " (mlp): BloomMLP(\n", - " (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)\n", - " (gelu_impl): BloomGelu()\n", - " (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)\n", - " )\n", - " )\n", - " (21): BloomBlock(\n", - " (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", - " (self_attention): BloomAttention(\n", - " (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)\n", - " (dense): Linear(in_features=1024, out_features=1024, bias=True)\n", - " (attention_dropout): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", - " (mlp): BloomMLP(\n", - " (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)\n", - " (gelu_impl): BloomGelu()\n", - " (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)\n", - " )\n", - " )\n", - " (22): BloomBlock(\n", - " (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", - " (self_attention): BloomAttention(\n", - " (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)\n", - " (dense): Linear(in_features=1024, out_features=1024, bias=True)\n", - " (attention_dropout): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", - " (mlp): BloomMLP(\n", - " (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)\n", - " (gelu_impl): BloomGelu()\n", - " (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)\n", - " )\n", - " )\n", - " (23): BloomBlock(\n", - " (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", - " (self_attention): BloomAttention(\n", - " (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)\n", - " (dense): Linear(in_features=1024, out_features=1024, bias=True)\n", - " (attention_dropout): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", - " (mlp): BloomMLP(\n", - " (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)\n", - " (gelu_impl): BloomGelu()\n", - " (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)\n", - " )\n", - " )\n", - " )\n", - " (ln_f): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", - " )\n", - " (lm_head): Linear(in_features=1024, out_features=250880, bias=False)\n", - " )\n", - " (word_embeddings): Embedding(250880, 1024)\n", - " (prompt_encoder): PrefixEncoder(\n", - " (embedding): Embedding(30, 49152)\n", - " )\n", - ")" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "model" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "id": "b94465ee", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'base_model.transformer.word_embeddings': 3,\n", - " 'word_embeddings': 3,\n", - " 'base_model.transformer.word_embeddings_layernorm': 3,\n", - " 'base_model.transformer.h.0': 3,\n", - " 'base_model.transformer.h.1': 3,\n", - " 'base_model.transformer.h.2': 3,\n", - " 'base_model.transformer.h.3': 3,\n", - " 'base_model.transformer.h.4': 3,\n", - " 'base_model.transformer.h.5': 3,\n", - " 'base_model.transformer.h.6': 3,\n", - " 'base_model.transformer.h.7': 3,\n", - " 'base_model.transformer.h.8': 3,\n", - " 'base_model.transformer.h.9': 3,\n", - " 'base_model.transformer.h.10': 3,\n", - " 'base_model.transformer.h.11': 3,\n", - " 'base_model.transformer.h.12': 3,\n", - " 'base_model.transformer.h.13': 3,\n", - " 'base_model.transformer.h.14': 3,\n", - " 'base_model.transformer.h.15': 3,\n", - " 'base_model.transformer.h.16': 3,\n", - " 'base_model.transformer.h.17': 3,\n", - " 'base_model.transformer.h.18': 3,\n", - " 'base_model.transformer.h.19': 3,\n", - " 'base_model.transformer.h.20': 3,\n", - " 'base_model.transformer.h.21': 3,\n", - " 'base_model.transformer.h.22': 'cpu',\n", - " 'base_model.transformer.h.23': 'cpu',\n", - " 'base_model.transformer.ln_f': 'cpu',\n", - " 'base_model.lm_head': 'cpu',\n", - " 'prompt_encoder': 'cpu'}" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "model.hf_device_map" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "id": "ebe174a6", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "@VW @QuirkCars some unauthorized work done on my engine now throwing a check engine light about a week later. Very upset.\n", - "{'input_ids': tensor([[227985, 5484, 915, 2566, 57, 58, 2566, 5232, 132511,\n", - " 38, 4599, 3331, 1035, 192352, 2909, 11541, 664, 2670,\n", - " 22218, 5840, 108218, 267, 7010, 22218, 12490, 3638, 267,\n", - " 14319, 10494, 17, 93269, 123055, 17, 77658, 915, 210]]), 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n", - " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]])}\n", - "tensor([[227985, 5484, 915, 2566, 57, 58, 2566, 5232, 132511,\n", - " 38, 4599, 3331, 1035, 192352, 2909, 11541, 664, 2670,\n", - " 22218, 5840, 108218, 267, 7010, 22218, 12490, 3638, 267,\n", - " 14319, 10494, 17, 93269, 123055, 17, 77658, 915, 210,\n", - " 16449, 5952, 3, 3, 3, 3, 3, 3, 3,\n", - " 3]], device='cuda:0')\n", - "['Tweet text : @VW @QuirkCars some unauthorized work done on my engine now throwing a check engine light about a week later. Very upset. Label : complaint']\n" - ] - } - ], - "source": [ - "model.eval()\n", - "i = 12\n", - "inputs = tokenizer(f'{text_column} : {dataset[\"test\"][i][\"Tweet text\"]} Label : ', return_tensors=\"pt\")\n", - "print(dataset[\"test\"][i][\"Tweet text\"])\n", - "print(inputs)\n", - "\n", - "with torch.no_grad():\n", - " inputs = {k: v.to(device) for k, v in inputs.items()}\n", - " outputs = model.generate(input_ids=inputs[\"input_ids\"], attention_mask=inputs[\"attention_mask\"], max_new_tokens=10)\n", - " print(outputs)\n", - " print(tokenizer.batch_decode(outputs.detach().cpu().numpy(), skip_special_tokens=True))\n" - ] + "source": [] }, { "cell_type": "code", "execution_count": null, - "id": "24041ee1", + "id": "463a41a2", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5c60c7a9", "metadata": {}, "outputs": [], "source": [] @@ -2524,7 +1324,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, diff --git a/examples/causal_language_modeling/peft_prompt_tuning_clm.ipynb b/examples/causal_language_modeling/peft_prompt_tuning_clm.ipynb new file mode 100644 index 0000000..fe03abc --- /dev/null +++ b/examples/causal_language_modeling/peft_prompt_tuning_clm.ipynb @@ -0,0 +1,1190 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "71fbfca2", + "metadata": {}, + "outputs": [], + "source": [ + "from transformers import AutoModelForCausalLM\n", + "from peft import get_peft_config, get_peft_model, PromptTuningInit, PromptTuningConfig, TaskType, PeftType\n", + "import torch\n", + "from datasets import load_dataset\n", + "import os\n", + "from transformers import AutoTokenizer\n", + "from torch.utils.data import DataLoader\n", + "from transformers import default_data_collator,get_linear_schedule_with_warmup\n", + "from tqdm import tqdm\n", + "from datasets import load_dataset\n", + "\n", + "device = \"cuda\"\n", + "model_name_or_path = \"bigscience/bloomz-560m\"\n", + "tokenizer_name_or_path = \"bigscience/bloomz-560m\"\n", + "peft_config = PromptTuningConfig(\n", + " task_type=TaskType.CAUSAL_LM,\n", + " prompt_tuning_init=PromptTuningInit.TEXT,\n", + " num_virtual_tokens=8,\n", + " prompt_tuning_init_text=\"Classify if the tweet is a compaint or not:\",\n", + " tokenizer_name_or_path=model_name_or_path,\n", + " )\n", + "\n", + "dataset_name = \"twitter_complaints\"\n", + "checkpoint_name = f\"{dataset_name}_{model_name_or_path}_{peft_config.peft_type}_{peft_config.task_type}_v1.pt\".replace(\"/\", \"_\")\n", + "text_column = \"Tweet text\"\n", + "label_column = \"text_label\"\n", + "max_length=64\n", + "lr = 3e-2\n", + "num_epochs = 50\n", + "batch_size=8\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e1a3648b", + "metadata": {}, + "outputs": [], + "source": [ + "from datasets import load_dataset\n", + "\n", + "dataset = load_dataset(\"ought/raft\", dataset_name)\n", + "\n", + "classes = [k.replace(\"_\", \" \") for k in dataset[\"train\"].features[\"Label\"].names]\n", + "print(classes)\n", + "dataset = dataset.map(\n", + " lambda x: {\"text_label\": [classes[label] for label in x[\"Label\"]]},\n", + " batched=True,\n", + " num_proc=1,\n", + " \n", + ")\n", + "print(dataset)\n", + "dataset[\"train\"][0]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fe12d4d3", + "metadata": {}, + "outputs": [], + "source": [ + "# data preprocessing\n", + "tokenizer = AutoTokenizer.from_pretrained(model_name_or_path)\n", + "if tokenizer.pad_token_id is None:\n", + " tokenizer.pad_token_id = tokenizer.eos_token_id\n", + "target_max_length = max([len(tokenizer(class_label)[\"input_ids\"]) for class_label in classes])\n", + "print(target_max_length)\n", + "def preprocess_function(examples):\n", + " batch_size = len(examples[text_column])\n", + " inputs = [f\"{text_column} : {x} Label : \" for x in examples[text_column]]\n", + " targets = [str(x) for x in examples[label_column]]\n", + " model_inputs = tokenizer(inputs)\n", + " labels = tokenizer(targets)\n", + " for i in range(batch_size):\n", + " sample_input_ids = model_inputs[\"input_ids\"][i]\n", + " label_input_ids = labels[\"input_ids\"][i] + [tokenizer.pad_token_id]\n", + " #print(i, sample_input_ids, label_input_ids)\n", + " model_inputs[\"input_ids\"][i] = sample_input_ids + label_input_ids \n", + " labels[\"input_ids\"][i] = [-100] * len(sample_input_ids) + label_input_ids\n", + " model_inputs[\"attention_mask\"][i] = [1] * len(model_inputs[\"input_ids\"][i])\n", + " #print(model_inputs)\n", + " for i in range(batch_size):\n", + " sample_input_ids = model_inputs[\"input_ids\"][i]\n", + " label_input_ids = labels[\"input_ids\"][i]\n", + " model_inputs[\"input_ids\"][i] = [tokenizer.pad_token_id]*(max_length-len(sample_input_ids)) + sample_input_ids\n", + " model_inputs[\"attention_mask\"][i] = [0]*(max_length-len(sample_input_ids)) + model_inputs[\"attention_mask\"][i]\n", + " labels[\"input_ids\"][i] = [-100]*(max_length-len(sample_input_ids)) + label_input_ids \n", + " model_inputs[\"input_ids\"][i] = torch.tensor(model_inputs[\"input_ids\"][i][:max_length])\n", + " model_inputs[\"attention_mask\"][i] = torch.tensor(model_inputs[\"attention_mask\"][i][:max_length])\n", + " labels[\"input_ids\"][i] = torch.tensor(labels[\"input_ids\"][i][:max_length]) \n", + " model_inputs[\"labels\"] = labels[\"input_ids\"]\n", + " return model_inputs\n", + "\n", + "\n", + "\n", + "processed_datasets = dataset.map(\n", + " preprocess_function,\n", + " batched=True,\n", + " num_proc=1,\n", + " remove_columns=dataset[\"train\"].column_names,\n", + " load_from_cache_file=False,\n", + " desc=\"Running tokenizer on dataset\",\n", + " )\n", + "\n", + "train_dataset = processed_datasets[\"train\"]\n", + "eval_dataset = processed_datasets[\"train\"]\n", + "\n", + "\n", + "train_dataloader = DataLoader(\n", + " train_dataset, shuffle=True, collate_fn=default_data_collator, batch_size=batch_size, pin_memory=True\n", + " )\n", + "eval_dataloader = DataLoader(eval_dataset, collate_fn=default_data_collator, batch_size=batch_size, pin_memory=True)\n", + "\n", + "\n", + "\n", + "\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "641b21fe", + "metadata": {}, + "outputs": [], + "source": [ + "def test_preprocess_function(examples):\n", + " batch_size = len(examples[text_column])\n", + " inputs = [f\"{text_column} : {x} Label : \" for x in examples[text_column]]\n", + " model_inputs = tokenizer(inputs)\n", + " #print(model_inputs)\n", + " for i in range(batch_size):\n", + " sample_input_ids = model_inputs[\"input_ids\"][i]\n", + " model_inputs[\"input_ids\"][i] = [tokenizer.pad_token_id]*(max_length-len(sample_input_ids)) + sample_input_ids\n", + " model_inputs[\"attention_mask\"][i] = [0]*(max_length-len(sample_input_ids)) + model_inputs[\"attention_mask\"][i]\n", + " model_inputs[\"input_ids\"][i] = torch.tensor(model_inputs[\"input_ids\"][i][:max_length])\n", + " model_inputs[\"attention_mask\"][i] = torch.tensor(model_inputs[\"attention_mask\"][i][:max_length])\n", + " return model_inputs\n", + "\n", + "test_dataset = dataset[\"test\"].map(\n", + " test_preprocess_function,\n", + " batched=True,\n", + " num_proc=1,\n", + " remove_columns=dataset[\"train\"].column_names,\n", + " load_from_cache_file=False,\n", + " desc=\"Running tokenizer on dataset\",\n", + " )\n", + "\n", + "test_dataloader = DataLoader(test_dataset, collate_fn=default_data_collator, batch_size=batch_size, pin_memory=True)\n", + "next(iter(test_dataloader))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "accc5012", + "metadata": {}, + "outputs": [], + "source": [ + "next(iter(train_dataloader))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "218df807", + "metadata": {}, + "outputs": [], + "source": [ + "len(test_dataloader)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "47d1fedf", + "metadata": {}, + "outputs": [], + "source": [ + "next(iter(test_dataloader))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a773e092", + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "# creating model\n", + "model = AutoModelForCausalLM.from_pretrained(model_name_or_path)\n", + "model = get_peft_model(model, peft_config)\n", + "model.print_trainable_parameters()\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "b2f91568", + "metadata": {}, + "outputs": [], + "source": [ + "# model\n", + "# optimizer and lr scheduler\n", + "optimizer = torch.optim.AdamW(model.parameters(), lr=lr)\n", + "lr_scheduler = get_linear_schedule_with_warmup(\n", + " optimizer=optimizer,\n", + " num_warmup_steps=0,\n", + " num_training_steps=(len(train_dataloader) * num_epochs),\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "e4fb69fc", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:01<00:00, 5.68it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 21.48it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=0: train_ppl=tensor(2.2720e+13, device='cuda:0') train_epoch_loss=tensor(30.7543, device='cuda:0') eval_ppl=tensor(483597.5625, device='cuda:0') eval_epoch_loss=tensor(13.0890, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 10.91it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 20.96it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=1: train_ppl=tensor(452658.3750, device='cuda:0') train_epoch_loss=tensor(13.0229, device='cuda:0') eval_ppl=tensor(275088.1875, device='cuda:0') eval_epoch_loss=tensor(12.5248, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 10.90it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 21.41it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=2: train_ppl=tensor(199203.3906, device='cuda:0') train_epoch_loss=tensor(12.2021, device='cuda:0') eval_ppl=tensor(143637.0312, device='cuda:0') eval_epoch_loss=tensor(11.8750, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 10.92it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 21.31it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=3: train_ppl=tensor(114743.9531, device='cuda:0') train_epoch_loss=tensor(11.6505, device='cuda:0') eval_ppl=tensor(54962., device='cuda:0') eval_epoch_loss=tensor(10.9144, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 10.81it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 21.34it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=4: train_ppl=tensor(40786.5977, device='cuda:0') train_epoch_loss=tensor(10.6161, device='cuda:0') eval_ppl=tensor(18342.5430, device='cuda:0') eval_epoch_loss=tensor(9.8170, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 10.89it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 21.34it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=5: train_ppl=tensor(14023.0830, device='cuda:0') train_epoch_loss=tensor(9.5485, device='cuda:0') eval_ppl=tensor(6316.8540, device='cuda:0') eval_epoch_loss=tensor(8.7510, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 10.84it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 21.32it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=6: train_ppl=tensor(5635.3262, device='cuda:0') train_epoch_loss=tensor(8.6368, device='cuda:0') eval_ppl=tensor(2476.5776, device='cuda:0') eval_epoch_loss=tensor(7.8146, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 10.88it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 21.30it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=7: train_ppl=tensor(1818.4940, device='cuda:0') train_epoch_loss=tensor(7.5058, device='cuda:0') eval_ppl=tensor(934.1146, device='cuda:0') eval_epoch_loss=tensor(6.8396, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 10.05it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 18.97it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=8: train_ppl=tensor(645.2143, device='cuda:0') train_epoch_loss=tensor(6.4696, device='cuda:0') eval_ppl=tensor(361.9093, device='cuda:0') eval_epoch_loss=tensor(5.8914, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 9.67it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 19.12it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=9: train_ppl=tensor(293.8047, device='cuda:0') train_epoch_loss=tensor(5.6829, device='cuda:0') eval_ppl=tensor(215.8185, device='cuda:0') eval_epoch_loss=tensor(5.3744, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 10.54it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 20.83it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=10: train_ppl=tensor(191.2377, device='cuda:0') train_epoch_loss=tensor(5.2535, device='cuda:0') eval_ppl=tensor(177.1512, device='cuda:0') eval_epoch_loss=tensor(5.1770, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 10.02it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 18.98it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=11: train_ppl=tensor(153.6052, device='cuda:0') train_epoch_loss=tensor(5.0344, device='cuda:0') eval_ppl=tensor(126.6154, device='cuda:0') eval_epoch_loss=tensor(4.8412, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 9.54it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 18.78it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=12: train_ppl=tensor(122.8925, device='cuda:0') train_epoch_loss=tensor(4.8113, device='cuda:0') eval_ppl=tensor(97.3331, device='cuda:0') eval_epoch_loss=tensor(4.5781, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 9.66it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 19.72it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=13: train_ppl=tensor(84.8845, device='cuda:0') train_epoch_loss=tensor(4.4413, device='cuda:0') eval_ppl=tensor(70.3213, device='cuda:0') eval_epoch_loss=tensor(4.2531, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:01<00:00, 6.73it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 16.07it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=14: train_ppl=tensor(64.6705, device='cuda:0') train_epoch_loss=tensor(4.1693, device='cuda:0') eval_ppl=tensor(50.4688, device='cuda:0') eval_epoch_loss=tensor(3.9214, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 8.41it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 15.63it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=15: train_ppl=tensor(44.2937, device='cuda:0') train_epoch_loss=tensor(3.7908, device='cuda:0') eval_ppl=tensor(34.8210, device='cuda:0') eval_epoch_loss=tensor(3.5502, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 8.31it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 15.67it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=16: train_ppl=tensor(30.0995, device='cuda:0') train_epoch_loss=tensor(3.4045, device='cuda:0') eval_ppl=tensor(24.7703, device='cuda:0') eval_epoch_loss=tensor(3.2096, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 8.31it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 15.59it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=17: train_ppl=tensor(23.3086, device='cuda:0') train_epoch_loss=tensor(3.1488, device='cuda:0') eval_ppl=tensor(20.8131, device='cuda:0') eval_epoch_loss=tensor(3.0356, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 8.29it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 16.04it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=18: train_ppl=tensor(16.4479, device='cuda:0') train_epoch_loss=tensor(2.8002, device='cuda:0') eval_ppl=tensor(12.0876, device='cuda:0') eval_epoch_loss=tensor(2.4922, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 8.37it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 15.37it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=19: train_ppl=tensor(11.1977, device='cuda:0') train_epoch_loss=tensor(2.4157, device='cuda:0') eval_ppl=tensor(9.0399, device='cuda:0') eval_epoch_loss=tensor(2.2016, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 8.23it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 17.29it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=20: train_ppl=tensor(8.1847, device='cuda:0') train_epoch_loss=tensor(2.1023, device='cuda:0') eval_ppl=tensor(6.7486, device='cuda:0') eval_epoch_loss=tensor(1.9093, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 8.30it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 15.58it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=21: train_ppl=tensor(6.1145, device='cuda:0') train_epoch_loss=tensor(1.8107, device='cuda:0') eval_ppl=tensor(5.5931, device='cuda:0') eval_epoch_loss=tensor(1.7215, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 8.34it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 21.36it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=22: train_ppl=tensor(5.2963, device='cuda:0') train_epoch_loss=tensor(1.6670, device='cuda:0') eval_ppl=tensor(5.0573, device='cuda:0') eval_epoch_loss=tensor(1.6208, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 10.84it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 21.26it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=23: train_ppl=tensor(4.7485, device='cuda:0') train_epoch_loss=tensor(1.5578, device='cuda:0') eval_ppl=tensor(3.6277, device='cuda:0') eval_epoch_loss=tensor(1.2886, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 10.84it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 21.31it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=24: train_ppl=tensor(3.4080, device='cuda:0') train_epoch_loss=tensor(1.2261, device='cuda:0') eval_ppl=tensor(3.0467, device='cuda:0') eval_epoch_loss=tensor(1.1141, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 10.88it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 21.25it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=25: train_ppl=tensor(3.3052, device='cuda:0') train_epoch_loss=tensor(1.1955, device='cuda:0') eval_ppl=tensor(2.7784, device='cuda:0') eval_epoch_loss=tensor(1.0219, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 10.86it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 21.22it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=26: train_ppl=tensor(2.9487, device='cuda:0') train_epoch_loss=tensor(1.0814, device='cuda:0') eval_ppl=tensor(2.9471, device='cuda:0') eval_epoch_loss=tensor(1.0808, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 10.85it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 21.25it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=27: train_ppl=tensor(2.8738, device='cuda:0') train_epoch_loss=tensor(1.0556, device='cuda:0') eval_ppl=tensor(2.5801, device='cuda:0') eval_epoch_loss=tensor(0.9478, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 10.84it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 21.28it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=28: train_ppl=tensor(2.3241, device='cuda:0') train_epoch_loss=tensor(0.8433, device='cuda:0') eval_ppl=tensor(2.2198, device='cuda:0') eval_epoch_loss=tensor(0.7974, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 10.84it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 20.89it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=29: train_ppl=tensor(2.0376, device='cuda:0') train_epoch_loss=tensor(0.7118, device='cuda:0') eval_ppl=tensor(1.8572, device='cuda:0') eval_epoch_loss=tensor(0.6191, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 9.76it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 18.83it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=30: train_ppl=tensor(1.8301, device='cuda:0') train_epoch_loss=tensor(0.6044, device='cuda:0') eval_ppl=tensor(1.8864, device='cuda:0') eval_epoch_loss=tensor(0.6347, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 9.80it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 19.81it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=31: train_ppl=tensor(1.7301, device='cuda:0') train_epoch_loss=tensor(0.5482, device='cuda:0') eval_ppl=tensor(1.6340, device='cuda:0') eval_epoch_loss=tensor(0.4910, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 10.60it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 19.11it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=32: train_ppl=tensor(1.5842, device='cuda:0') train_epoch_loss=tensor(0.4601, device='cuda:0') eval_ppl=tensor(1.6179, device='cuda:0') eval_epoch_loss=tensor(0.4811, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 10.11it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 18.35it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=33: train_ppl=tensor(1.5193, device='cuda:0') train_epoch_loss=tensor(0.4183, device='cuda:0') eval_ppl=tensor(1.5543, device='cuda:0') eval_epoch_loss=tensor(0.4410, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 9.59it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 18.60it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=34: train_ppl=tensor(1.5402, device='cuda:0') train_epoch_loss=tensor(0.4319, device='cuda:0') eval_ppl=tensor(1.4924, device='cuda:0') eval_epoch_loss=tensor(0.4004, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 9.80it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 19.63it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=35: train_ppl=tensor(1.4410, device='cuda:0') train_epoch_loss=tensor(0.3654, device='cuda:0') eval_ppl=tensor(1.3888, device='cuda:0') eval_epoch_loss=tensor(0.3284, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:01<00:00, 6.60it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 15.36it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=36: train_ppl=tensor(1.3675, device='cuda:0') train_epoch_loss=tensor(0.3130, device='cuda:0') eval_ppl=tensor(1.4001, device='cuda:0') eval_epoch_loss=tensor(0.3366, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 8.40it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 15.58it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=37: train_ppl=tensor(1.4197, device='cuda:0') train_epoch_loss=tensor(0.3505, device='cuda:0') eval_ppl=tensor(1.3214, device='cuda:0') eval_epoch_loss=tensor(0.2787, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 8.27it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 15.56it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=38: train_ppl=tensor(1.3855, device='cuda:0') train_epoch_loss=tensor(0.3261, device='cuda:0') eval_ppl=tensor(1.3501, device='cuda:0') eval_epoch_loss=tensor(0.3001, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 8.25it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 15.57it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=39: train_ppl=tensor(1.3643, device='cuda:0') train_epoch_loss=tensor(0.3107, device='cuda:0') eval_ppl=tensor(1.3549, device='cuda:0') eval_epoch_loss=tensor(0.3037, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 8.28it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 15.41it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=40: train_ppl=tensor(1.3093, device='cuda:0') train_epoch_loss=tensor(0.2695, device='cuda:0') eval_ppl=tensor(1.3233, device='cuda:0') eval_epoch_loss=tensor(0.2801, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 8.24it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 15.51it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=41: train_ppl=tensor(1.3108, device='cuda:0') train_epoch_loss=tensor(0.2706, device='cuda:0') eval_ppl=tensor(1.3440, device='cuda:0') eval_epoch_loss=tensor(0.2957, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 8.78it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 15.61it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=42: train_ppl=tensor(1.2944, device='cuda:0') train_epoch_loss=tensor(0.2581, device='cuda:0') eval_ppl=tensor(1.2711, device='cuda:0') eval_epoch_loss=tensor(0.2399, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 8.29it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 15.56it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=43: train_ppl=tensor(1.2616, device='cuda:0') train_epoch_loss=tensor(0.2323, device='cuda:0') eval_ppl=tensor(1.2449, device='cuda:0') eval_epoch_loss=tensor(0.2190, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 10.85it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 21.27it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=44: train_ppl=tensor(1.2478, device='cuda:0') train_epoch_loss=tensor(0.2214, device='cuda:0') eval_ppl=tensor(1.2202, device='cuda:0') eval_epoch_loss=tensor(0.1990, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 10.85it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 21.31it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=45: train_ppl=tensor(1.2350, device='cuda:0') train_epoch_loss=tensor(0.2111, device='cuda:0') eval_ppl=tensor(1.2180, device='cuda:0') eval_epoch_loss=tensor(0.1972, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 10.86it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 21.33it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=46: train_ppl=tensor(1.2277, device='cuda:0') train_epoch_loss=tensor(0.2052, device='cuda:0') eval_ppl=tensor(1.2077, device='cuda:0') eval_epoch_loss=tensor(0.1887, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 10.87it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 21.35it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=47: train_ppl=tensor(1.2037, device='cuda:0') train_epoch_loss=tensor(0.1854, device='cuda:0') eval_ppl=tensor(1.2041, device='cuda:0') eval_epoch_loss=tensor(0.1857, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 10.83it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 21.29it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=48: train_ppl=tensor(1.2026, device='cuda:0') train_epoch_loss=tensor(0.1845, device='cuda:0') eval_ppl=tensor(1.1982, device='cuda:0') eval_epoch_loss=tensor(0.1808, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 10.86it/s]\n", + "100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 21.35it/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=49: train_ppl=tensor(1.2005, device='cuda:0') train_epoch_loss=tensor(0.1827, device='cuda:0') eval_ppl=tensor(1.1968, device='cuda:0') eval_epoch_loss=tensor(0.1796, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" + ] + } + ], + "source": [ + "# training and evaluation\n", + "model = model.to(device)\n", + "\n", + "for epoch in range(num_epochs):\n", + " model.train()\n", + " total_loss = 0\n", + " for step, batch in enumerate(tqdm(train_dataloader)):\n", + " batch = {k: v.to(device) for k, v in batch.items()}\n", + "# print(batch)\n", + "# print(batch[\"input_ids\"].shape)\n", + " outputs = model(**batch)\n", + " loss = outputs.loss\n", + " total_loss += loss.detach().float()\n", + " loss.backward()\n", + " optimizer.step()\n", + " lr_scheduler.step()\n", + " optimizer.zero_grad()\n", + "\n", + " model.eval()\n", + " eval_loss = 0\n", + " eval_preds = []\n", + " for step, batch in enumerate(tqdm(eval_dataloader)):\n", + " batch = {k: v.to(device) for k, v in batch.items()}\n", + " with torch.no_grad():\n", + " outputs = model(**batch)\n", + " loss = outputs.loss\n", + " eval_loss += loss.detach().float()\n", + " eval_preds.extend(tokenizer.batch_decode(torch.argmax(outputs.logits, -1).detach().cpu().numpy(), skip_special_tokens=True))\n", + "\n", + " eval_epoch_loss = eval_loss/len(train_dataloader)\n", + " eval_ppl = torch.exp(eval_epoch_loss)\n", + " train_epoch_loss = total_loss/len(eval_dataloader)\n", + " train_ppl = torch.exp(train_epoch_loss)\n", + " print(f\"{epoch=}: {train_ppl=} {train_epoch_loss=} {eval_ppl=} {eval_epoch_loss=}\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "53752a7b", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "@TommyHilfiger Dramatic shopping exp. ordered 6 jeans same size (30/32) 2 fits / 2 too large / 2 too slim : same brand > different sizing\n", + "{'input_ids': tensor([[227985, 5484, 915, 2566, 226154, 126015, 5385, 259, 239364,\n", + " 3396, 70823, 5853, 17, 57247, 1231, 191040, 5025, 7869,\n", + " 375, 2324, 149349, 12, 415, 122321, 897, 415, 10136,\n", + " 10021, 897, 415, 10136, 6497, 381, 915, 5025, 51950,\n", + " 66869, 5955, 272, 20311, 77658, 915, 210]]), 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n", + " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]])}\n", + "tensor([[227985, 5484, 915, 2566, 226154, 126015, 5385, 259, 239364,\n", + " 3396, 70823, 5853, 17, 57247, 1231, 191040, 5025, 7869,\n", + " 375, 2324, 149349, 12, 415, 122321, 897, 415, 10136,\n", + " 10021, 897, 415, 10136, 6497, 381, 915, 5025, 51950,\n", + " 66869, 5955, 272, 20311, 77658, 915, 210, 16449, 5952,\n", + " 3]], device='cuda:0')\n", + "['Tweet text : @TommyHilfiger Dramatic shopping exp. ordered 6 jeans same size (30/32) 2 fits / 2 too large / 2 too slim : same brand > different sizing Label : complaint']\n" + ] + } + ], + "source": [ + "model.eval()\n", + "i = 33\n", + "inputs = tokenizer(f'{text_column} : {dataset[\"test\"][i][\"Tweet text\"]} Label : ', return_tensors=\"pt\")\n", + "print(dataset[\"test\"][i][\"Tweet text\"])\n", + "print(inputs)\n", + "\n", + "with torch.no_grad():\n", + " inputs = {k: v.to(device) for k, v in inputs.items()}\n", + " outputs = model.generate(input_ids=inputs[\"input_ids\"], attention_mask=inputs[\"attention_mask\"], max_new_tokens=10, eos_token_id=3)\n", + " print(outputs)\n", + " print(tokenizer.batch_decode(outputs.detach().cpu().numpy(), skip_special_tokens=True))\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "d8ba1f8c", + "metadata": {}, + "outputs": [], + "source": [ + "# saving model\n", + "peft_model_id = f\"{model_name_or_path}_{peft_config.peft_type}_{peft_config.task_type}\"\n", + "model.save_pretrained(peft_model_id)" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "4928c7f1", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", + "To disable this warning, you can either:\n", + "\t- Avoid using `tokenizers` before the fork if possible\n", + "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", + "36K\tbigscience/bloomz-560m_PROMPT_TUNING_CAUSAL_LM/adapter_model.bin\n" + ] + } + ], + "source": [ + "ckpt = f\"{peft_model_id}/adapter_model.bin\"\n", + "!du -h $ckpt" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "4d9476e1", + "metadata": {}, + "outputs": [], + "source": [ + "from peft import PeftModel, PeftConfig\n", + "peft_model_id = f\"{model_name_or_path}_{peft_config.peft_type}_{peft_config.task_type}\"\n", + "\n", + "config = PeftConfig.from_pretrained(peft_model_id)\n", + "model = AutoModelForCausalLM.from_pretrained(config.base_model_name_or_path)\n", + "model = PeftModel.from_pretrained(model, peft_model_id)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "ebe174a6", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "@greateranglia Ok thanks...\n", + "{'input_ids': tensor([[227985, 5484, 915, 2566, 14173, 2960, 29906, 387, 20706,\n", + " 49337, 1369, 77658, 915, 210]]), 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]])}\n", + "tensor([[227985, 5484, 915, 2566, 14173, 2960, 29906, 387, 20706,\n", + " 49337, 1369, 77658, 915, 210, 1936, 106863, 3]],\n", + " device='cuda:0')\n", + "['Tweet text : @greateranglia Ok thanks... Label : no complaint']\n" + ] + } + ], + "source": [ + "model.to(device)\n", + "model.eval()\n", + "i = 4\n", + "inputs = tokenizer(f'{text_column} : {dataset[\"test\"][i][\"Tweet text\"]} Label : ', return_tensors=\"pt\")\n", + "print(dataset[\"test\"][i][\"Tweet text\"])\n", + "print(inputs)\n", + "\n", + "with torch.no_grad():\n", + " inputs = {k: v.to(device) for k, v in inputs.items()}\n", + " outputs = model.generate(input_ids=inputs[\"input_ids\"], attention_mask=inputs[\"attention_mask\"], max_new_tokens=10, eos_token_id=3)\n", + " print(outputs)\n", + " print(tokenizer.batch_decode(outputs.detach().cpu().numpy(), skip_special_tokens=True))\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "24041ee1", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.5 (v3.10.5:f377153967, Jun 6 2022, 12:36:10) [Clang 13.0.0 (clang-1300.0.29.30)]" + }, + "vscode": { + "interpreter": { + "hash": "aee8b7b246df8f9039afb4144a1f6fd8d2ca17a180786b69acc140d282b71a49" + } + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/conditional_generation/accelerate_ds_zero3_cpu_offload_config.yaml b/examples/conditional_generation/accelerate_ds_zero3_cpu_offload_config.yaml new file mode 100644 index 0000000..a4a0bcf --- /dev/null +++ b/examples/conditional_generation/accelerate_ds_zero3_cpu_offload_config.yaml @@ -0,0 +1,22 @@ +compute_environment: LOCAL_MACHINE +deepspeed_config: + gradient_accumulation_steps: 1 + gradient_clipping: 1.0 + offload_optimizer_device: none + offload_param_device: none + zero3_init_flag: true + zero3_save_16bit_model: true + zero_stage: 3 +distributed_type: DEEPSPEED +downcast_bf16: 'no' +dynamo_backend: 'NO' +fsdp_config: {} +machine_rank: 0 +main_training_function: main +megatron_lm_config: {} +mixed_precision: 'no' +num_machines: 1 +num_processes: 1 +rdzv_backend: static +same_network: true +use_cpu: false \ No newline at end of file diff --git a/examples/conditional_generation/peft_lora_seq2seq.ipynb b/examples/conditional_generation/peft_lora_seq2seq.ipynb index 55266c7..c308738 100644 --- a/examples/conditional_generation/peft_lora_seq2seq.ipynb +++ b/examples/conditional_generation/peft_lora_seq2seq.ipynb @@ -2,10 +2,26 @@ "cells": [ { "cell_type": "code", - "execution_count": 17, + "execution_count": 1, "id": "5f93b7d1", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "===================================BUG REPORT===================================\n", + "Welcome to bitsandbytes. For bug reports, please submit your error trace to: https://github.com/TimDettmers/bitsandbytes/issues\n", + "For effortless bug reporting copy-paste your error into this form: https://docs.google.com/forms/d/e/1FAIpQLScPB8emS3Thkp66nvqwmjTEgxp8Y9ufuWTzFyr9kJ5AoI47dQ/viewform?usp=sf_link\n", + "================================================================================\n", + "CUDA SETUP: CUDA runtime path found: /home/sourab/miniconda3/envs/ml/lib/libcudart.so\n", + "CUDA SETUP: Highest compute capability among GPUs detected: 7.5\n", + "CUDA SETUP: Detected CUDA version 117\n", + "CUDA SETUP: Loading binary /home/sourab/miniconda3/envs/ml/lib/python3.10/site-packages/bitsandbytes/libbitsandbytes_cuda117.so...\n" + ] + } + ], "source": [ "from transformers import AutoModelForSeq2SeqLM\n", "from peft import get_peft_config,get_peft_model, get_peft_model_state_dict, LoraConfig, TaskType\n", @@ -60,15 +76,13 @@ "name": "stderr", "output_type": "stream", "text": [ - "/home/sourab/miniconda3/envs/ml/lib/python3.10/site-packages/huggingface_hub/utils/_deprecation.py:97: FutureWarning: Deprecated argument(s) used in 'dataset_info': token. Will not be supported from version '0.12'.\n", - " warnings.warn(message, FutureWarning)\n", "Found cached dataset financial_phrasebank (/home/sourab/.cache/huggingface/datasets/financial_phrasebank/sentences_allagree/1.0.0/550bde12e6c30e2674da973a55f57edde5181d53f5a5a34c1531c53f93b7e141)\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "6de075f8208349108291ac5ab7f5c980", + "model_id": "3403bf3d718042018b0531848cc30209", "version_major": 2, "version_minor": 0 }, @@ -82,7 +96,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "4b0e67b6d93f43e4b0f6a2f8978e4b0c", + "model_id": "d3d5c45e3776469f9560b6eaa9346f8f", "version_major": 2, "version_minor": 0 }, @@ -96,7 +110,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "a9551029c9884529bda7421a99170b51", + "model_id": "e9736f26e9aa450b8d65f95c0b9c81cc", "version_major": 2, "version_minor": 0 }, @@ -110,7 +124,7 @@ { "data": { "text/plain": [ - "{'sentence': 'The order was valued at USD12 .2 m.',\n", + "{'sentence': \"The 10,000-odd square metre plot that Stockmann has bought for the Nevsky Center shopping center is located on Nevsky Prospect , St Petersburg 's high street , next to the Vosstaniya Square underground station , in the immediate vicinity of Moscow Station .\",\n", " 'label': 1,\n", " 'text_label': 'neutral'}" ] @@ -147,7 +161,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "4421971232434db1b6141e91fda2f6d7", + "model_id": "c460989d4ab24e3f97d81ef040b1d1b4", "version_major": 2, "version_minor": 0 }, @@ -161,7 +175,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "9b2ef793d93443949f4a5d5874d4bc05", + "model_id": "1acc389b08b94f8a87900b9fbdbccce4", "version_major": 2, "version_minor": 0 }, @@ -234,45 +248,52 @@ "name": "stderr", "output_type": "stream", "text": [ - "100%|█████████████████████████████████████████████████████████████| 255/255 [00:53<00:00, 4.80it/s]\n", - "100%|███████████████████████████████████████████████████████████████| 29/29 [00:02<00:00, 14.16it/s]\n" + "100%|████████████████████████████████████████████████████████████████████████████████████████| 255/255 [02:21<00:00, 1.81it/s]\n", + "100%|██████████████████████████████████████████████████████████████████████████████████████████| 29/29 [00:07<00:00, 4.13it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "epoch=0: train_ppl=tensor(13.6966, device='cuda:0') train_epoch_loss=tensor(2.6171, device='cuda:0') eval_ppl=tensor(1.0046, device='cuda:0') eval_epoch_loss=tensor(0.0046, device='cuda:0')\n" + "epoch=0: train_ppl=tensor(14.6341, device='cuda:0') train_epoch_loss=tensor(2.6834, device='cuda:0') eval_ppl=tensor(1.0057, device='cuda:0') eval_epoch_loss=tensor(0.0057, device='cuda:0')\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "100%|█████████████████████████████████████████████████████████████| 255/255 [00:52<00:00, 4.88it/s]\n", - "100%|███████████████████████████████████████████████████████████████| 29/29 [00:02<00:00, 14.20it/s]\n" + "100%|████████████████████████████████████████████████████████████████████████████████████████| 255/255 [02:00<00:00, 2.11it/s]\n", + "100%|██████████████████████████████████████████████████████████████████████████████████████████| 29/29 [00:05<00:00, 5.66it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "epoch=1: train_ppl=tensor(1.5893, device='cuda:0') train_epoch_loss=tensor(0.4633, device='cuda:0') eval_ppl=tensor(1.0020, device='cuda:0') eval_epoch_loss=tensor(0.0020, device='cuda:0')\n" + "epoch=1: train_ppl=tensor(1.7576, device='cuda:0') train_epoch_loss=tensor(0.5640, device='cuda:0') eval_ppl=tensor(1.0052, device='cuda:0') eval_epoch_loss=tensor(0.0052, device='cuda:0')\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "100%|█████████████████████████████████████████████████████████████| 255/255 [00:52<00:00, 4.87it/s]\n", - "100%|███████████████████████████████████████████████████████████████| 29/29 [00:02<00:00, 14.18it/s]\n" + "100%|████████████████████████████████████████████████████████████████████████████████████████| 255/255 [01:33<00:00, 2.74it/s]\n", + "100%|██████████████████████████████████████████████████████████████████████████████████████████| 29/29 [00:04<00:00, 6.23it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "epoch=2: train_ppl=tensor(1.3210, device='cuda:0') train_epoch_loss=tensor(0.2784, device='cuda:0') eval_ppl=tensor(1.0026, device='cuda:0') eval_epoch_loss=tensor(0.0026, device='cuda:0')\n" + "epoch=2: train_ppl=tensor(1.3830, device='cuda:0') train_epoch_loss=tensor(0.3243, device='cuda:0') eval_ppl=tensor(1.0035, device='cuda:0') eval_epoch_loss=tensor(0.0035, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" ] } ], @@ -313,7 +334,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 7, "id": "6cafa67b", "metadata": {}, "outputs": [ @@ -321,9 +342,9 @@ "name": "stdout", "output_type": "stream", "text": [ - "accuracy=98.23788546255507 % on the evaluation dataset\n", - "eval_preds[:10]=['neutral', 'neutral', 'positive', 'positive', 'neutral', 'neutral', 'neutral', 'neutral', 'neutral', 'neutral']\n", - "dataset['validation']['text_label'][:10]=['neutral', 'neutral', 'positive', 'positive', 'neutral', 'neutral', 'neutral', 'neutral', 'neutral', 'neutral']\n" + "accuracy=97.3568281938326 % on the evaluation dataset\n", + "eval_preds[:10]=['neutral', 'neutral', 'neutral', 'positive', 'neutral', 'positive', 'positive', 'neutral', 'neutral', 'neutral']\n", + "dataset['validation']['text_label'][:10]=['neutral', 'neutral', 'neutral', 'positive', 'neutral', 'positive', 'positive', 'neutral', 'neutral', 'neutral']\n" ] } ], @@ -343,20 +364,19 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "id": "a8de6005", "metadata": {}, "outputs": [], "source": [ "# saving model\n", - "state_dict = get_peft_model_state_dict(model)\n", - "torch.save(state_dict, checkpoint_name)\n", - "print(state_dict)" + "peft_model_id = f\"{model_name_or_path}_{peft_config.peft_type}_{peft_config.task_type}\"\n", + "model.save_pretrained(peft_model_id)" ] }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 9, "id": "bd20cd4c", "metadata": {}, "outputs": [ @@ -364,18 +384,74 @@ "name": "stdout", "output_type": "stream", "text": [ - "19M\tfinancial_sentiment_analysis_lora_v1.pt\r\n" + "9,2M\tbigscience/mt0-large_LORA_SEQ_2_SEQ_LM/adapter_model.bin\r\n" ] } ], "source": [ - "!du -h $checkpoint_name" + "ckpt = f\"{peft_model_id}/adapter_model.bin\"\n", + "!du -h $ckpt" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "76c2fc29", + "metadata": {}, + "outputs": [], + "source": [ + "from peft import PeftModel, PeftConfig\n", + "peft_model_id = f\"{model_name_or_path}_{peft_config.peft_type}_{peft_config.task_type}\"\n", + "\n", + "config = PeftConfig.from_pretrained(peft_model_id)\n", + "model = AutoModelForSeq2SeqLM.from_pretrained(config.base_model_name_or_path)\n", + "model = PeftModel.from_pretrained(model, peft_model_id)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "37d712ce", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "- Demand for fireplace products was lower than expected , especially in Germany .\n", + "{'input_ids': tensor([[ 259, 264, 259, 82903, 332, 1090, 10040, 10371, 639, 259,\n", + " 19540, 2421, 259, 25505, 259, 261, 259, 21230, 281, 17052,\n", + " 259, 260, 1]]), 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]])}\n", + "tensor([[ 0, 259, 32588, 1]])\n", + "['negative']\n" + ] + } + ], + "source": [ + "model.eval()\n", + "i = 13\n", + "inputs = tokenizer(dataset[\"validation\"][text_column][i], return_tensors=\"pt\")\n", + "print(dataset[\"validation\"][text_column][i])\n", + "print(inputs)\n", + "\n", + "with torch.no_grad():\n", + " outputs = model.generate(input_ids=inputs[\"input_ids\"], max_new_tokens=10)\n", + " print(outputs)\n", + " print(tokenizer.batch_decode(outputs.detach().cpu().numpy(), skip_special_tokens=True))\n" ] }, { "cell_type": "code", "execution_count": null, - "id": "76c2fc29", + "id": "66c65ea4", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "65e71f78", "metadata": {}, "outputs": [], "source": [] @@ -383,7 +459,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3.10.5 64-bit", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -397,7 +473,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.5 (v3.10.5:f377153967, Jun 6 2022, 12:36:10) [Clang 13.0.0 (clang-1300.0.29.30)]" + "version": "3.10.4" }, "vscode": { "interpreter": { diff --git a/examples/conditional_generation/peft_lora_seq2seq_accelerate_big_model_inference.ipynb b/examples/conditional_generation/peft_lora_seq2seq_accelerate_big_model_inference.ipynb new file mode 100644 index 0000000..7e2b25c --- /dev/null +++ b/examples/conditional_generation/peft_lora_seq2seq_accelerate_big_model_inference.ipynb @@ -0,0 +1,255 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "71fbfca2", + "metadata": {}, + "outputs": [], + "source": [ + "from transformers import AutoModelForSeq2SeqLM\n", + "from peft import PeftModel, PeftConfig\n", + "import torch\n", + "from datasets import load_dataset\n", + "import os\n", + "from transformers import AutoTokenizer\n", + "from torch.utils.data import DataLoader\n", + "from transformers import default_data_collator,get_linear_schedule_with_warmup\n", + "from tqdm import tqdm\n", + "from datasets import load_dataset\n", + "\n", + "dataset_name = \"twitter_complaints\"\n", + "text_column = \"Tweet text\"\n", + "label_column = \"text_label\"\n", + "batch_size=8\n", + "\n", + "peft_model_id = \"smangrul/twitter_complaints_bigscience_T0_3B_LORA_SEQ_2_SEQ_LM\"\n", + "config = PeftConfig.from_pretrained(peft_model_id)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "cc55820a", + "metadata": {}, + "outputs": [], + "source": [ + "peft_model_id = \"smangrul/twitter_complaints_bigscience_T0_3B_LORA_SEQ_2_SEQ_LM\"\n", + "max_memory={0: \"6GIB\", 1: \"0GIB\", 2: \"0GIB\", 3: \"0GIB\", 4: \"0GIB\", \"cpu\":\"30GB\"}\n", + "config = PeftConfig.from_pretrained(peft_model_id)\n", + "model = AutoModelForSeq2SeqLM.from_pretrained(config.base_model_name_or_path, device_map=\"auto\", max_memory=max_memory)\n", + "model = PeftModel.from_pretrained(model, peft_model_id, device_map=\"auto\", max_memory=max_memory)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e1a3648b", + "metadata": {}, + "outputs": [], + "source": [ + "from datasets import load_dataset\n", + "\n", + "dataset = load_dataset(\"ought/raft\", dataset_name)\n", + "\n", + "classes = [k.replace(\"_\", \" \") for k in dataset[\"train\"].features[\"Label\"].names]\n", + "print(classes)\n", + "dataset = dataset.map(\n", + " lambda x: {\"text_label\": [classes[label] for label in x[\"Label\"]]},\n", + " batched=True,\n", + " num_proc=1,\n", + " \n", + ")\n", + "print(dataset)\n", + "dataset[\"train\"][0]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fe12d4d3", + "metadata": {}, + "outputs": [], + "source": [ + "tokenizer = AutoTokenizer.from_pretrained(config.base_model_name_or_path)\n", + "target_max_length = max([len(tokenizer(class_label)[\"input_ids\"]) for class_label in classes])\n", + "def preprocess_function(examples):\n", + " inputs = examples[text_column]\n", + " targets = examples[label_column]\n", + " model_inputs = tokenizer(inputs, truncation=True)\n", + " labels = tokenizer(\n", + " targets, max_length=target_max_length, padding=\"max_length\", truncation=True, return_tensors=\"pt\"\n", + " )\n", + " labels = labels[\"input_ids\"]\n", + " labels[labels == tokenizer.pad_token_id] = -100\n", + " model_inputs[\"labels\"] = labels\n", + " return model_inputs\n", + "\n", + "processed_datasets = dataset.map(\n", + " preprocess_function,\n", + " batched=True,\n", + " num_proc=1,\n", + " remove_columns=dataset[\"train\"].column_names,\n", + " load_from_cache_file=True,\n", + " desc=\"Running tokenizer on dataset\",\n", + ")\n", + "\n", + "train_dataset = processed_datasets[\"train\"]\n", + "eval_dataset = processed_datasets[\"train\"]\n", + "test_dataset = processed_datasets[\"test\"]\n", + "\n", + "\n", + "def collate_fn(examples):\n", + " return tokenizer.pad(examples, padding=\"longest\", return_tensors=\"pt\")\n", + "\n", + "train_dataloader = DataLoader(\n", + " train_dataset, shuffle=True, collate_fn=collate_fn, batch_size=batch_size, pin_memory=True\n", + ")\n", + "eval_dataloader = DataLoader(eval_dataset, collate_fn=collate_fn, batch_size=batch_size, pin_memory=True)\n", + "test_dataloader = DataLoader(test_dataset, collate_fn=collate_fn, batch_size=batch_size, pin_memory=True)\n", + "\n", + "\n", + "\n", + "\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "b33be5e6", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "@NYTsupport i have complained a dozen times & yet my papers are still thrown FAR from my door. Why is this so hard to resolve?\n", + "{'input_ids': tensor([[25335, 1499, 3, 10, 3320, 12056, 382, 20390, 3, 23,\n", + " 43, 25932, 3, 9, 9611, 648, 3, 184, 4624, 117,\n", + " 780, 82, 5778, 33, 341, 3, 12618, 377, 4280, 45,\n", + " 82, 1365, 5, 1615, 19, 48, 78, 614, 12, 7785,\n", + " 58, 16229, 3, 10, 3, 1]]), 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n", + " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]])}\n", + "tensor([[ 0, 10394, 1]], device='cuda:0')\n", + "['complaint']\n" + ] + } + ], + "source": [ + "model.eval()\n", + "i = 15\n", + "inputs = tokenizer(f'{text_column} : {dataset[\"test\"][i][\"Tweet text\"]} Label : ', return_tensors=\"pt\")\n", + "print(dataset[\"test\"][i][\"Tweet text\"])\n", + "print(inputs)\n", + "\n", + "with torch.no_grad():\n", + " outputs = model.generate(input_ids=inputs[\"input_ids\"].to(\"cuda\"), max_new_tokens=10)\n", + " print(outputs)\n", + " print(tokenizer.batch_decode(outputs.detach().cpu().numpy(), skip_special_tokens=True))\n" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "b6d6cd5b", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + " 0%| | 0/7 [00:00100:\n", + " break\n", + "test_preds" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.5 (v3.10.5:f377153967, Jun 6 2022, 12:36:10) [Clang 13.0.0 (clang-1300.0.29.30)]" + }, + "vscode": { + "interpreter": { + "hash": "aee8b7b246df8f9039afb4144a1f6fd8d2ca17a180786b69acc140d282b71a49" + } + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/conditional_generation/peft_lora_seq2seq_accelerate_ds_zero3_offload.py b/examples/conditional_generation/peft_lora_seq2seq_accelerate_ds_zero3_offload.py index 1c087c4..cef9773 100644 --- a/examples/conditional_generation/peft_lora_seq2seq_accelerate_ds_zero3_offload.py +++ b/examples/conditional_generation/peft_lora_seq2seq_accelerate_ds_zero3_offload.py @@ -11,7 +11,7 @@ from transformers import AutoModelForSeq2SeqLM, AutoTokenizer, get_linear_schedu import psutil from datasets import load_dataset -from peft import LoraConfig, TaskType, get_peft_model, get_peft_model_state_dict +from peft import LoraConfig, TaskType, get_peft_model from tqdm import tqdm @@ -107,15 +107,13 @@ def main(): peft_config = LoraConfig( task_type=TaskType.SEQ_2_SEQ_LM, inference_mode=False, r=8, lora_alpha=32, lora_dropout=0.1 ) - checkpoint_name = ( - f"{dataset_name}_{model_name_or_path}_{peft_config.peft_type}_{peft_config.task_type}_v1.pt".replace("/", "_") - ) text_column = "Tweet text" label_column = "text_label" lr = 3e-3 num_epochs = 5 batch_size = 8 seed = 42 + do_test = False set_seed(seed) dataset = load_dataset("ought/raft", dataset_name) @@ -265,33 +263,39 @@ def main(): accelerator.print(f"{eval_preds[:10]=}") accelerator.print(f"{dataset['train'][label_column][:10]=}") - model.eval() - test_preds = [] - for _, batch in enumerate(tqdm(test_dataloader)): - batch = {k: v for k, v in batch.items() if k != "labels"} - with torch.no_grad(): - outputs = accelerator.unwrap_model(model).generate( - **batch, synced_gpus=is_ds_zero_3 - ) # synced_gpus=True for DS-stage 3 - test_preds.extend(tokenizer.batch_decode(outputs.detach().cpu().numpy(), skip_special_tokens=True)) + if do_test: + model.eval() + test_preds = [] + for _, batch in enumerate(tqdm(test_dataloader)): + batch = {k: v for k, v in batch.items() if k != "labels"} + with torch.no_grad(): + outputs = accelerator.unwrap_model(model).generate( + **batch, synced_gpus=is_ds_zero_3 + ) # synced_gpus=True for DS-stage 3 + test_preds.extend(tokenizer.batch_decode(outputs.detach().cpu().numpy(), skip_special_tokens=True)) - test_preds_cleaned = [] - for _, pred in enumerate(test_preds): - test_preds_cleaned.append(get_closest_label(pred, classes)) + test_preds_cleaned = [] + for _, pred in enumerate(test_preds): + test_preds_cleaned.append(get_closest_label(pred, classes)) - test_df = dataset["test"].to_pandas() - test_df[label_column] = test_preds_cleaned - test_df["text_labels_orig"] = test_preds - accelerator.print(test_df[[text_column, label_column]].sample(20)) + test_df = dataset["test"].to_pandas() + test_df[label_column] = test_preds_cleaned + test_df["text_labels_orig"] = test_preds + accelerator.print(test_df[[text_column, label_column]].sample(20)) - pred_df = test_df[["ID", label_column]] - pred_df.columns = ["ID", "Label"] + pred_df = test_df[["ID", label_column]] + pred_df.columns = ["ID", "Label"] - os.makedirs(f"data/{dataset_name}", exist_ok=True) - pred_df.to_csv(f"data/{dataset_name}/predictions.csv", index=False) + os.makedirs(f"data/{dataset_name}", exist_ok=True) + pred_df.to_csv(f"data/{dataset_name}/predictions.csv", index=False) accelerator.wait_for_everyone() - accelerator.save(get_peft_model_state_dict(model, state_dict=accelerator.get_state_dict(model)), checkpoint_name) + model.push_to_hub( + "smangrul/" + + f"{dataset_name}_{model_name_or_path}_{peft_config.peft_type}_{peft_config.task_type}".replace("/", "_"), + state_dict=accelerator.get_state_dict(model), + use_auth_token=True, + ) accelerator.wait_for_everyone() diff --git a/examples/conditional_generation/peft_lora_seq2seq_accelerate_fsdp.py b/examples/conditional_generation/peft_lora_seq2seq_accelerate_fsdp.py index 19c7b22..e00b1ff 100644 --- a/examples/conditional_generation/peft_lora_seq2seq_accelerate_fsdp.py +++ b/examples/conditional_generation/peft_lora_seq2seq_accelerate_fsdp.py @@ -6,7 +6,7 @@ from torch.utils.data import DataLoader from transformers import AutoModelForSeq2SeqLM, AutoTokenizer, default_data_collator, get_linear_schedule_with_warmup from datasets import load_dataset -from peft import LoraConfig, TaskType, get_peft_model, get_peft_model_state_dict +from peft import LoraConfig, TaskType, get_peft_model from peft.utils.other import fsdp_auto_wrap_policy from tqdm import tqdm @@ -25,7 +25,6 @@ def main(): peft_config = LoraConfig( task_type=TaskType.SEQ_2_SEQ_LM, inference_mode=False, r=8, lora_alpha=32, lora_dropout=0.1 ) - checkpoint_name = "financial_sentiment_analysis_lora_fsdp_v1.pt" model = AutoModelForSeq2SeqLM.from_pretrained(model_name_or_path) model = get_peft_model(model, peft_config) accelerator.print(model.print_trainable_parameters()) @@ -126,8 +125,10 @@ def main(): accelerator.print(f"{eval_preds[:10]=}") accelerator.print(f"{dataset['validation'][label_column][:10]=}") accelerator.wait_for_everyone() - accelerator.save( - get_peft_model_state_dict(model, state_dict=accelerator.get_state_dict(model)), checkpoint_name + model.push_to_hub( + "smangrul/" + f"{model_name_or_path}_{peft_config.peft_type}_{peft_config.task_type}".replace("/", "_"), + state_dict=accelerator.get_state_dict(model), + use_auth_token=True, ) accelerator.wait_for_everyone() diff --git a/examples/conditional_generation/peft_prefix_tuning_seq2seq.ipynb b/examples/conditional_generation/peft_prefix_tuning_seq2seq.ipynb index 0f71eaf..b514d57 100644 --- a/examples/conditional_generation/peft_prefix_tuning_seq2seq.ipynb +++ b/examples/conditional_generation/peft_prefix_tuning_seq2seq.ipynb @@ -5,7 +5,23 @@ "execution_count": 1, "id": "5f93b7d1", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "===================================BUG REPORT===================================\n", + "Welcome to bitsandbytes. For bug reports, please submit your error trace to: https://github.com/TimDettmers/bitsandbytes/issues\n", + "For effortless bug reporting copy-paste your error into this form: https://docs.google.com/forms/d/e/1FAIpQLScPB8emS3Thkp66nvqwmjTEgxp8Y9ufuWTzFyr9kJ5AoI47dQ/viewform?usp=sf_link\n", + "================================================================================\n", + "CUDA SETUP: CUDA runtime path found: /home/sourab/miniconda3/envs/ml/lib/libcudart.so\n", + "CUDA SETUP: Highest compute capability among GPUs detected: 7.5\n", + "CUDA SETUP: Detected CUDA version 117\n", + "CUDA SETUP: Loading binary /home/sourab/miniconda3/envs/ml/lib/python3.10/site-packages/bitsandbytes/libbitsandbytes_cuda117.so...\n" + ] + } + ], "source": [ "from transformers import AutoModelForSeq2SeqLM\n", "from peft import get_peft_config,get_peft_model, get_peft_model_state_dict, PrefixTuningConfig, TaskType\n", @@ -61,15 +77,13 @@ "name": "stderr", "output_type": "stream", "text": [ - "/home/sourab/miniconda3/envs/ml/lib/python3.10/site-packages/huggingface_hub/utils/_deprecation.py:97: FutureWarning: Deprecated argument(s) used in 'dataset_info': token. Will not be supported from version '0.12'.\n", - " warnings.warn(message, FutureWarning)\n", "Found cached dataset financial_phrasebank (/home/sourab/.cache/huggingface/datasets/financial_phrasebank/sentences_allagree/1.0.0/550bde12e6c30e2674da973a55f57edde5181d53f5a5a34c1531c53f93b7e141)\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "e3f8b8faca0a4112b2c3499faee9544b", + "model_id": "ec4be98991b84181bfa75f8846422b8b", "version_major": 2, "version_minor": 0 }, @@ -83,7 +97,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "935c8aebde284a5784348588e0bb013a", + "model_id": "82a6bd694c4f4751a23c370ab51f01a4", "version_major": 2, "version_minor": 0 }, @@ -97,7 +111,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "e3487cd55f6847588492bf7fa51348ca", + "model_id": "3844878631534468a1495e435563e4b0", "version_major": 2, "version_minor": 0 }, @@ -111,9 +125,9 @@ { "data": { "text/plain": [ - "{'sentence': 'ADPnews - Feb 5 , 2010 - Finnish real estate investor Sponda Oyj HEL : SDA1V said today that it slipped to a net loss of EUR 81.5 million USD 11.8 m in 2009 from a profit of EUR 29.3 million in 2008 .',\n", - " 'label': 0,\n", - " 'text_label': 'negative'}" + "{'sentence': 'Finnish elevators and escalators maker KONE Corporation said on Tuesday ( 18 March ) that it has received a major order from Sir Robert McAlpine to supply all elevators and escalators for the Watermark Place project in the City of London .',\n", + " 'label': 2,\n", + " 'text_label': 'positive'}" ] }, "execution_count": 3, @@ -145,39 +159,11 @@ "id": "adf9608c", "metadata": {}, "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "2ce088f4437d4e2c80c267332a5b84e5", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "Downloading: 0%| | 0.00/792k [00:00" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "pipe = StableDiffusionPipeline.from_pretrained(MODEL_NAME, torch_dtype=torch.float16).to(\"cuda\")\n", "\n", @@ -166,36 +118,10 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "id": "e50da571", "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "b405066c203e4c7fa45a0c3389f76796", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - " 0%| | 0/50 [00:00" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "prompt = \"a photo of sks dog\"\n", "negative_prompt = \"low quality, blurry, unfinished\"\n", @@ -243,36 +169,10 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": null, "id": "200d3358", "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "b669f753a13244d1884b7956ad0ccaf4", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - " 0%| | 0/50 [00:00" - ] - }, - "execution_count": 17, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "prompt = \"sks dog with Eiffel Tower in the background\"\n", "image = pipe(prompt, num_inference_steps=50, \n", @@ -283,36 +183,10 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "id": "0518421e", "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "6e4e70ffd53d46b79955b068403ac229", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - " 0%| | 0/50 [00:00" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "prompt = \"sks dog swimming in th pool with sunglasses\"\n", "image = pipe(prompt, num_inference_steps=50, guidance_scale=7.5, negative_prompt=negative_prompt).images[0]\n", @@ -321,36 +195,10 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": null, "id": "cca5106c", "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "d9beb64979364f10b5cd9bba50d9caba", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - " 0%| | 0/50 [00:00" - ] - }, - "execution_count": 28, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "prompt = \"sks dog weaing red sweater\"\n", "image = pipe(prompt, num_inference_steps=50, guidance_scale=7.5, negative_prompt=negative_prompt).images[0]\n", diff --git a/examples/sequence_classification/LoRA.ipynb b/examples/sequence_classification/LoRA.ipynb index 018fbd9..e9dd6da 100644 --- a/examples/sequence_classification/LoRA.ipynb +++ b/examples/sequence_classification/LoRA.ipynb @@ -5,7 +5,23 @@ "execution_count": 1, "id": "a9935ae2", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "===================================BUG REPORT===================================\n", + "Welcome to bitsandbytes. For bug reports, please submit your error trace to: https://github.com/TimDettmers/bitsandbytes/issues\n", + "For effortless bug reporting copy-paste your error into this form: https://docs.google.com/forms/d/e/1FAIpQLScPB8emS3Thkp66nvqwmjTEgxp8Y9ufuWTzFyr9kJ5AoI47dQ/viewform?usp=sf_link\n", + "================================================================================\n", + "CUDA SETUP: CUDA runtime path found: /home/sourab/miniconda3/envs/ml/lib/libcudart.so\n", + "CUDA SETUP: Highest compute capability among GPUs detected: 7.5\n", + "CUDA SETUP: Detected CUDA version 117\n", + "CUDA SETUP: Loading binary /home/sourab/miniconda3/envs/ml/lib/python3.10/site-packages/bitsandbytes/libbitsandbytes_cuda117.so...\n" + ] + } + ], "source": [ "import argparse\n", "import os\n", @@ -60,6 +76,20 @@ "id": "c2697d07", "metadata": {}, "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "0f74797387a941cbb0709487b8808eba", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Downloading readme: 0%| | 0.00/27.9k [00:00" - ] - }, - "execution_count": 1, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "from PIL import Image, ImageDraw, ImageFont\n", "import os\n", @@ -135,7 +123,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/" @@ -143,38 +131,7 @@ "id": "JPKkuJQ4sdZc", "outputId": "c95bf306-98bb-4480-cc6b-ebb3aea548b3" }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'box': [292, 91, 376, 175], 'text': 'R&D', 'label': 'other', 'words': [{'box': [292, 91, 376, 175], 'text': 'R&D'}], 'linking': [], 'id': 0}\n", - "{'box': [219, 316, 225, 327], 'text': ':', 'label': 'question', 'words': [{'box': [219, 316, 225, 327], 'text': ':'}], 'linking': [], 'id': 1}\n", - "{'box': [95, 355, 169, 370], 'text': 'Suggestion:', 'label': 'question', 'words': [{'box': [95, 355, 169, 370], 'text': 'Suggestion:'}], 'linking': [[2, 16]], 'id': 2}\n", - "{'box': [482, 268, 518, 282], 'text': 'Date:', 'label': 'question', 'words': [{'box': [482, 268, 518, 282], 'text': 'Date:'}], 'linking': [[3, 12]], 'id': 3}\n", - "{'box': [511, 309, 570, 323], 'text': 'Licensee', 'label': 'answer', 'words': [{'box': [511, 309, 570, 323], 'text': 'Licensee'}], 'linking': [[13, 4]], 'id': 4}\n", - "{'box': [211, 651, 217, 662], 'text': '', 'label': 'question', 'words': [{'box': [211, 651, 217, 662], 'text': ''}], 'linking': [], 'id': 5}\n", - "{'box': [461, 605, 483, 619], 'text': 'Yes', 'label': 'question', 'words': [{'box': [461, 605, 483, 619], 'text': 'Yes'}], 'linking': [[19, 6]], 'id': 6}\n", - "{'box': [545, 603, 563, 617], 'text': 'No', 'label': 'question', 'words': [{'box': [545, 603, 563, 617], 'text': 'No'}], 'linking': [[19, 7]], 'id': 7}\n", - "{'box': [525, 904, 641, 926], 'text': '597005708', 'label': 'other', 'words': [{'box': [525, 904, 641, 926], 'text': '597005708'}], 'linking': [], 'id': 8}\n", - "{'text': 'R&D QUALITY IMPROVEMENT SUGGESTION/ SOLUTION FORM', 'box': [256, 201, 423, 230], 'linking': [], 'label': 'header', 'words': [{'text': 'R&D', 'box': [257, 203, 279, 214]}, {'text': 'QUALITY', 'box': [285, 203, 334, 216]}, {'text': 'IMPROVEMENT', 'box': [341, 201, 418, 211]}, {'text': 'SUGGESTION/', 'box': [256, 215, 324, 229]}, {'text': '', 'box': [324, 216, 332, 230]}, {'text': 'SOLUTION', 'box': [331, 214, 387, 228]}, {'text': 'FORM', 'box': [395, 215, 423, 228]}], 'id': 9}\n", - "{'text': 'Name / Phone Ext. :', 'box': [89, 272, 204, 289], 'linking': [[10, 11]], 'label': 'question', 'words': [{'text': 'Name', 'box': [89, 274, 118, 289]}, {'text': '/', 'box': [117, 274, 127, 288]}, {'text': 'Phone', 'box': [128, 274, 163, 289]}, {'text': 'Ext.', 'box': [169, 272, 196, 287]}, {'text': ':', 'box': [196, 274, 204, 288]}], 'id': 10}\n", - "{'text': 'M. Hamann P. Harper, P. Martinez', 'box': [215, 271, 451, 287], 'linking': [[10, 11]], 'label': 'answer', 'words': [{'text': 'M.', 'box': [215, 272, 230, 287]}, {'text': 'Hamann', 'box': [237, 272, 287, 286]}, {'text': 'P.', 'box': [293, 272, 307, 286]}, {'text': 'Harper,', 'box': [314, 274, 363, 285]}, {'text': 'P.', 'box': [370, 272, 384, 285]}, {'text': 'Martinez', 'box': [390, 271, 451, 282]}], 'id': 11}\n", - "{'text': '9/ 3/ 92', 'box': [543, 264, 590, 279], 'linking': [[3, 12]], 'label': 'answer', 'words': [{'text': '9/', 'box': [543, 265, 560, 279]}, {'text': '3/', 'box': [560, 264, 575, 279]}, {'text': '92', 'box': [575, 264, 590, 279]}], 'id': 12}\n", - "{'text': 'R&D Group:', 'box': [420, 310, 491, 323], 'linking': [[13, 4]], 'label': 'question', 'words': [{'text': 'R&D', 'box': [420, 310, 442, 323]}, {'text': 'Group:', 'box': [448, 310, 491, 323]}], 'id': 13}\n", - "{'text': 'J. S. Wigand', 'box': [236, 313, 327, 327], 'linking': [[15, 14]], 'label': 'answer', 'words': [{'text': 'J.', 'box': [236, 313, 251, 327]}, {'text': 'S.', 'box': [256, 313, 273, 326]}, {'text': 'Wigand', 'box': [278, 313, 327, 327]}], 'id': 14}\n", - "{'text': 'Supervisor / Manager', 'box': [91, 316, 218, 331], 'linking': [[15, 14]], 'label': 'question', 'words': [{'text': 'Supervisor', 'box': [91, 316, 161, 330]}, {'text': '/', 'box': [163, 318, 169, 331]}, {'text': 'Manager', 'box': [169, 317, 218, 327]}], 'id': 15}\n", - "{'text': 'Discontinue coal retention analyses on licensee submitted product samples (Note : Coal Retention testing is not performed by most licensees. Other B&W physical measurements as ends stability and inspection for soft spots in ciparettes are thought to be sufficient measures to assure cigarette physical integrity. The proposed action will increase laboratory productivity . )', 'box': [190, 346, 594, 447], 'linking': [[2, 16]], 'label': 'answer', 'words': [{'text': 'Discontinue', 'box': [190, 355, 268, 366]}, {'text': 'coal', 'box': [274, 353, 303, 366]}, {'text': 'retention', 'box': [309, 352, 375, 365]}, {'text': 'analyses', 'box': [381, 351, 435, 365]}, {'text': 'on', 'box': [443, 352, 458, 363]}, {'text': 'licensee', 'box': [464, 348, 520, 362]}, {'text': 'submitted', 'box': [527, 346, 594, 361]}, {'text': 'product', 'box': [190, 369, 240, 383]}, {'text': 'samples', 'box': [247, 367, 301, 380]}, {'text': '(Note', 'box': [318, 365, 352, 379]}, {'text': ':', 'box': [352, 367, 359, 380]}, {'text': 'Coal', 'box': [373, 366, 402, 376]}, {'text': 'Retention', 'box': [408, 366, 472, 376]}, {'text': 'testing', 'box': [479, 365, 529, 376]}, {'text': 'is', 'box': [536, 363, 549, 374]}, {'text': 'not', 'box': [554, 363, 578, 374]}, {'text': 'performed', 'box': [190, 383, 256, 394]}, {'text': 'by', 'box': [261, 381, 275, 394]}, {'text': 'most', 'box': [282, 383, 311, 393]}, {'text': 'licensees.', 'box': [318, 380, 386, 391]}, {'text': 'Other', 'box': [401, 378, 437, 389]}, {'text': 'B&W', 'box': [443, 378, 465, 389]}, {'text': 'physical', 'box': [471, 377, 528, 391]}, {'text': 'measurements', 'box': [191, 398, 275, 406]}, {'text': 'as', 'box': [282, 397, 297, 405]}, {'text': 'ends', 'box': [304, 394, 332, 405]}, {'text': 'stability', 'box': [339, 394, 402, 405]}, {'text': 'and', 'box': [409, 392, 430, 402]}, {'text': 'inspection', 'box': [437, 392, 508, 403]}, {'text': 'for', 'box': [515, 391, 535, 402]}, {'text': 'soft', 'box': [542, 391, 571, 401]}, {'text': 'spots', 'box': [193, 411, 228, 422]}, {'text': 'in', 'box': [235, 409, 250, 420]}, {'text': 'ciparettes', 'box': [256, 409, 327, 419]}, {'text': 'are', 'box': [332, 408, 352, 418]}, {'text': 'thought', 'box': [360, 406, 410, 419]}, {'text': 'to', 'box': [415, 406, 430, 416]}, {'text': 'be', 'box': [436, 404, 453, 417]}, {'text': 'sufficient', 'box': [458, 405, 529, 415]}, {'text': 'measures', 'box': [535, 405, 592, 415]}, {'text': 'to', 'box': [193, 425, 208, 433]}, {'text': 'assure', 'box': [214, 423, 255, 431]}, {'text': 'cigarette', 'box': [261, 420, 325, 434]}, {'text': 'physical', 'box': [331, 419, 390, 432]}, {'text': 'integrity.', 'box': [395, 418, 463, 431]}, {'text': 'The', 'box': [478, 416, 500, 429]}, {'text': 'proposed', 'box': [506, 418, 566, 431]}, {'text': 'action', 'box': [193, 436, 236, 447]}, {'text': 'will', 'box': [240, 436, 269, 447]}, {'text': 'increase', 'box': [277, 434, 333, 445]}, {'text': 'laboratory', 'box': [339, 433, 410, 446]}, {'text': 'productivity', 'box': [418, 430, 502, 445]}, {'text': '.', 'box': [503, 433, 507, 444]}, {'text': ')', 'box': [508, 430, 514, 444]}], 'id': 16}\n", - "{'text': 'Suggested Solutions (s) :', 'box': [95, 486, 250, 504], 'linking': [[17, 18]], 'label': 'question', 'words': [{'text': 'Suggested', 'box': [95, 489, 159, 504]}, {'text': 'Solutions', 'box': [165, 487, 222, 501]}, {'text': '(s)', 'box': [223, 486, 241, 503]}, {'text': ':', 'box': [243, 489, 250, 503]}], 'id': 17}\n", - "{'text': 'Delete coal retention from the list of standard analyses performed on licensee submitted product samples. Special requests for coal retention testing could still be submitted on an exception basis.', 'box': [263, 483, 593, 553], 'linking': [[17, 18]], 'label': 'answer', 'words': [{'text': 'Delete', 'box': [263, 486, 306, 500]}, {'text': 'coal', 'box': [313, 486, 341, 499]}, {'text': 'retention', 'box': [348, 486, 412, 497]}, {'text': 'from', 'box': [416, 485, 447, 498]}, {'text': 'the', 'box': [453, 485, 475, 498]}, {'text': 'list', 'box': [480, 483, 508, 496]}, {'text': 'of', 'box': [515, 483, 532, 494]}, {'text': 'standard', 'box': [536, 483, 593, 494]}, {'text': 'analyses', 'box': [264, 501, 320, 514]}, {'text': 'performed', 'box': [324, 501, 392, 512]}, {'text': 'on', 'box': [397, 501, 412, 511]}, {'text': 'licensee', 'box': [419, 499, 475, 512]}, {'text': 'submitted', 'box': [482, 499, 546, 510]}, {'text': 'product', 'box': [264, 517, 314, 528]}, {'text': 'samples.', 'box': [320, 514, 374, 528]}, {'text': 'Special', 'box': [390, 513, 439, 526]}, {'text': 'requests', 'box': [446, 513, 502, 524]}, {'text': 'for', 'box': [508, 511, 530, 522]}, {'text': 'coal', 'box': [538, 510, 566, 523]}, {'text': 'retention', 'box': [263, 529, 330, 540]}, {'text': 'testing', 'box': [335, 527, 387, 540]}, {'text': 'could', 'box': [390, 527, 428, 538]}, {'text': 'still', 'box': [433, 525, 468, 536]}, {'text': 'be', 'box': [473, 525, 488, 535]}, {'text': 'submitted', 'box': [496, 524, 560, 537]}, {'text': 'on', 'box': [566, 524, 584, 537]}, {'text': 'an', 'box': [264, 543, 281, 553]}, {'text': 'exception', 'box': [286, 539, 350, 553]}, {'text': 'basis.', 'box': [355, 541, 397, 551]}], 'id': 18}\n", - "{'text': 'Have you contacted your Manager/ Supervisor?', 'box': [96, 608, 398, 624], 'linking': [[19, 6], [19, 7]], 'label': 'header', 'words': [{'text': 'Have', 'box': [96, 612, 127, 623]}, {'text': 'you', 'box': [131, 613, 156, 624]}, {'text': 'contacted', 'box': [161, 612, 225, 623]}, {'text': 'your', 'box': [229, 610, 260, 623]}, {'text': 'Manager/', 'box': [264, 609, 314, 622]}, {'text': '', 'box': [314, 608, 322, 622]}, {'text': 'Supervisor?', 'box': [323, 608, 398, 621]}], 'id': 19}\n", - "{'text': 'Manager Comments:', 'box': [98, 651, 211, 665], 'linking': [[20, 21], [20, 22]], 'label': 'question', 'words': [{'text': 'Manager', 'box': [98, 654, 150, 665]}, {'text': 'Comments:', 'box': [154, 651, 211, 664]}], 'id': 20}\n", - "{'text': 'Manager, please contact suggester and forward', 'box': [232, 644, 547, 662], 'linking': [[20, 21]], 'label': 'answer', 'words': [{'text': 'Manager,', 'box': [232, 648, 288, 662]}, {'text': 'please', 'box': [296, 649, 338, 662]}, {'text': 'contact', 'box': [344, 648, 394, 662]}, {'text': 'suggester', 'box': [401, 648, 464, 661]}, {'text': 'and', 'box': [469, 647, 491, 658]}, {'text': 'forward', 'box': [497, 644, 547, 657]}], 'id': 21}\n", - "{'text': 'comments to the Quality Council.', 'box': [99, 662, 323, 677], 'linking': [[20, 22]], 'label': 'answer', 'words': [{'text': 'comments', 'box': [99, 666, 155, 677]}, {'text': 'to', 'box': [162, 665, 177, 676]}, {'text': 'the', 'box': [183, 665, 205, 675]}, {'text': 'Quality', 'box': [211, 663, 261, 676]}, {'text': 'Council.', 'box': [267, 662, 323, 676]}], 'id': 22}\n", - "{'text': 'qip . wp', 'box': [102, 823, 145, 838], 'linking': [], 'label': 'other', 'words': [{'text': 'qip', 'box': [102, 824, 123, 837]}, {'text': '.', 'box': [124, 824, 130, 838]}, {'text': 'wp', 'box': [130, 823, 145, 837]}], 'id': 23}\n" - ] - } - ], + "outputs": [], "source": [ "import json\n", "\n", @@ -196,7 +153,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/", @@ -205,19 +162,7 @@ "id": "gWaHFM_LtKPP", "outputId": "c498e560-035f-4170-b0b9-85ba3956711c" }, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAvoAAAPoCAIAAAAgFM2mAAEAAElEQVR4nOzddzzV3R8A8HOXa8+Msgrpoa29lMosDSQUKu0l7VJS2lNL2RVJKJUtor2LojKSopTsfd3x/f1xnuf+7sMleoT0eb969br3fM/3fM/34t7PPRMhAAAAAIA2FRsbS3Qm5I5+QQAAAAAAfi0IdwAAAADQxVFTU1M7ug5/OgqF4uTkpK+vb2Fh4e3tLSgo2Ldv3+zsbB8fn6VLl8bHxxcWFpLJ/w9MGQzGmDFjtm7dGhYWVl9fLyAgoKKikpOTExkZuWTJkqFDh9bX1/MWXlhYuGbNGnV19T179jCZTJxOp9PDw8PpdLqvr++4ceNoNNqtW7fodDo+Wl9fP378eITQ7du3CYKwsbF5+vTphw8fKBRKO74wAAAAfrnXr19zPxq6MBJBEB1dBwAAAAB0AIIgfH19KyoqeL9U/zQajfb27dvTp08jhGJjYw0NDf97mW2F2tEVAAAAAEDHIJFICxcubMMCk5OTcbjT2cDYHQAAAAB0cRDuAAAAAKCLg3AHAAAAAF0chDsAAAAA6OIg3AEAAABAFwfhDgAAAAC6OAh3AAAAANDFQbgDAAAAgC4Owh0AAAAAdHEQ7gAAAACgi4NwBwAAAABdHIQ7AAAAAOjiINwBAAAAQBcH4Q4AAAAAujgIdwAAAADQxUG4AwAAAIAuDsIdAAAAAHRxEO4AAAAAoIuDcAcAAAAAXRyEOwAAAADo4iDcAQAAAEAXB+EOAAAAALo4CHcAAAAA0MVBuAMAAACALg7CHQAAAAB0cRDuAAAAAKCLg3AHAAAAAF0chDsAAAAA6OIg3AEAAABAFwfhDgAAAAC6OAh3AAAAANDFQbgDAAAAgC4Owh0AAAAAdHEQ7gAAAACgi4NwBwAAAABdHIQ7AAAAAOjiINwBAAAAQBcH4Q4AAAAAujgIdwAAAADQxUG4AwAAAIAuDsIdAAAAAHRxEO4AAAAAoIuDcAcAAAAAXRyEOwAAAADo4iDcAQAAAEAXB+EOAAAAALo4CHcAAAAA0DZevXrV0VXgD8IdAAAAALQNd3f3jq4CfxDuAAAAAKCLg3AHAAAAAF0chDsAAAAA6OIg3AEAAABAFwfhDgAAAADahp2dXUdXgT8IdwAAAADQNiZMmNDRVeAPwh0AAAAAtI2TJ092dBX4g3AHAAAAAG3j5cuXHV0F/iDcAQAAAEDbcHZ27ugq8AfhDgAAAADahrq6ekdXgT8IdwAAAADQNpycnDq6CvxBuAMAAACAtjFt2rSOrgJ/EO4AAAAAoG3o6el1dBX4g3AHAAAAAF0chDsAAAAA6OIg3AEAAABAFwfhDgAAAAC6OAh3AAAAANDFQbgDAAAAgC4Owh0AAAAAdHEQ7gAAAACgi4NwBwAAAABdHIQ7AAAAAOjiINwBAAAAQBcH4Q4AAAAAujgIdwAAAADQxUG4AwAAAIAuDsIdAAAAAHRxEO4AAAAAoIujdnQFAABNKnUrRQhJbZfiPuY+BQAA0HLQugPA7wGiHAAA+GnQugNAZ9Gg/Yb7lLeNp6mnDc6S2i4FrUEAAMAFrTsAdAqN+624YYrUdinekIU3T1NnNSgBAADax/Hjxzu6CvxBuAPAb4/bkMMLxzoQ8QAA2pOoqGhHV4E/6MwC4DdQ6lbaTOACMQ0AoJOoqqrq6CrwB607AHQKTXVjoSYabxpkwP9QoxE/AADQnlJTUzu6CvyRCILo6DoAAH6JtLS0mpoaEonU+BBBEDQabdCgQXyPAgDAz9m+ffvu3bsRQrGxsYaGhh1dnf+DziwA2kn7N7coIsXmM5RFlrVLRRCCHjcA/gydtg0FOrMAAAAA0DYmT57c0VXgD1p3AGhXbdLIwWKxHj9+vHv37oSEBBaLRaFQqFSqnZ2diYkJmUyeMmUKhUJpYVFJSUllZWVv377dvXs3i8ViMpkIodGjR69cudLc3FxAQOC/1xZGEQEAOhyEOwD8TtLS0h4/fhwSEhIfH0+j0ebMmUMQhLGx8ezZs9lsNpXa6r9oPT09hND06dM3b9785MkTDw8PYWHhq1ev2tjYaGlprV+/vmfPnhMnTvwFtwIAAO0Hwh0Afg/19fVOTk7Xr1//8uWLiopKeHh4jx49hgwZwm3I4cY6lZWVBQUFZDJZUFDQx8cnOjqaTqdzy6mrq1u8eLGRkRGDwSAIgk6nq6iokMlkhNDIkSNHjhyJEFqzZs2ZM2cuXrzo4OAgLS09duxYHx8fWVnZdr9pAMBvxs/Pr6OrwB/MzAKgnfyXlY5jYmJOnz4dFRWFENq2bZubm1vjPPHx8Xfv3hUVFX38+HF4eHgLS+7WrdvKlSt79uxpb2/f4NDnz58NDAzevHmDEBo+fHhMTIy0tPRPVB6WeAbgz5GQkKCvr49gZhYAoOVqa2tTUlI2btyYmpravXv3sLCwwYMH9+jRAx9ls9mvX7/etWtXSkqKoKDg169fS0tbPUqmqKjI1dVVSEho//79bDZbSkrq8OHDOjo6IiIiioqKiYmJFRUV5eXllpaWAwYMEBER8fHxGTduXFvfKACgi/iJLvX20UmrBQBACLm4uBw+fBgh5ODg4OPjw3soLS3t4MGDwcHBeHDxf1RbW/vu3Tv8WFdXd+rUqStWrDAyMlJQUFBQUEAIRUdHT5w4MTMz09zcPCoqatiwYf/9ogCArqempqajq8AfTEQHoDNKSUlZvnz56dOnp0+fnp6efubMGZz+/v37qKgoOzu7YcOGBQQEtEms01hkZOT06dPHjRsXHx+PL6GlpfX+/fuzZ88qKSkZGBgEBQUVFRX9iksDAH5r165d6+gq8AetOwB0OomJiebm5uXl5b6+vgsWLMCJBEHs2LHj8uXLmZmZLSyHSqVKS0sTBMFgMIyNjQsLC1NSUgiCKCsr++G59fX19+7dMzExmTBhwo4dO8aNGycsLLxkyRIbG5tZs2bNmTNHR0dnzZo1tra2P32bAADQbiDcAaBzSUhIMDExYbFYgYGBc+bM4aavXr361KlTLSxEXFzc0tJSTU1t8+bNuHmGQqEQBEEmk7Ozsw8ePCgsLBwREZGbm9t8OWw2OzExMTU1dceOHVZWVt26dRMTE7t+/bqZmVl0dPS5c+dmzpzZaTdABgC0P3Nzc29v746uBT8EAKBdlOwqKdlV0nye2bNni4qKWlhYvH79msPh4MTo6OhBgwa1ZACgkJCQjo7OrVu3Xr58+cP6vHv37smTJy1fArV3795BQUH43OLi4hs3bkhISJiYmFRVVf33GwcAdA3BwcH4HSM2Nraj6/IvMHYHgM4iKSnp6tWr+vr6wcHB/fr1w5t3JiQkWFhYpKSksFisH5Zw9OjR58+f6+npDRo06IeZ+/TpM2zYsKtXr+rq6rakellZWZ6enkFBQQghaWlpU1NTZ2fn6Ohoc3Pzurq6lpQAAOjytmzZ0tFV4A/CHQA6hfj4eGNj45kzZ166dAmvHFhYWBgcHGxubt6SmQ5aWlpnzpxZunRpa68rJiYWGRnZwjV1bt++bWtr6+7ujp8uW7bMxsbmyZMnM2bMqKysbO2lAQCg3UC4A0DHu3fv3qxZsxgMxt69e/EKyOXl5ebm5tbW1hUVFS0pISQkpJlYh8lk1tXVMRgMbgqDwairq8PDesTExIyMjMLCwloyCofD4Tg5OeFRRKKiohcvXly4cGFcXNz169dbUk8AAOgQEO4A0PECAgIqKiouXLigrq6OUxYvXnzv3r0Wnr569WoNDY3Q0NArV65cuXIlJCTk+/fv+BBBEAkJCcbGxpKSkv369cOL69TX1w8YMEBKSmrWrFnXr18PCwurqqrS09OLjIyUkmrR2serVq06duwYfrxr1y5jY+MlS5Y8ePCgdbcNAADtpqMHDwHwp2hqxO66desQQn5+ftyUuLg4CQmJFv4J9+rVKycnJzU1lTdx0KBBAQEBBEHgjicKhSIjI4MQ2r59O0EQDAYDb4BFo9FEREQQQqNGjQoLCyMIwsbGpoXXJZPJJ06cwBWuqKiYPHmyhIREenp6y28cAND19OrVC79FwFBlAMD/3bhx48iRI4sXL54/fz5OiY+Pnz59enl5eQtLOHv2bK9evbgDmSdMmCAkJJSSkuLj48NkMuXk5BBCoqKi48ePb3zuokWLvn//PmTIkIcPH9rb21dUVBw5cqSFa8BzOJzVq1e/fv0aISQmJnb58uWqqqqLFy+2sNoAgC5p0qRJHV0F/iDcAaDDsNlsV1dXdXX1TZs24ZTY2NhZs2a1fKLTvHnzJkyYwH2qpaUVFBSEVybs0aMHlUqdPXu2goJCeXn51atXe/TosWzZMt7TORyOkJBQZGTksGHDqqur3dzcFBQUnJycWn4LGzduxIOUpaWljx49unfv3tOnT7f8dABAF8O7WlinAuEOAB3m+vXrL1++PHTokJqaGk5ZtWpVC8cmI4TExMROnDghICDATXn79q2Dg4Ofn5+IiMiSJUtIJFJERERxcTE+KiwsjPcZ5XA4vOUoKCgcPXoUIfThwweEkIqKSstvITY21tfXFz9evXr16NGjvby8fmKnUgBA1xAWFtbRVeAPwh0AOkZKSoqFhcWyZcumT5+OEOJwOI6OjtnZ2f+x2JiYmNraWjk5ufHjx6elpc2ePZvJZA4cOFBbWzsnJ6dnz57Lly+/f/8+nv+Fffr0Cbf69OnTByG0aNGi/v37t/yKu3btunv3Ln587NixV69e3bp16z/eBQDgNzVy5MiOrgJ/EO4A0DFOnjxJEMTOnTvJZDJCaNOmTSdOnGj56SQSCY8ybmDevHnS0tJ5eXnh4eEsFgtPPt+3b5+zszOHw/n48aO6uvqkSZNwOkEQJSUlU6dOTUtLk5CQWL9+PUKITqdLSUlRqVS8zuEPlZaWmpmZvXjxAiE0ePDgGTNmWFtbJycnt/xeAABdhpKSUkdXgT8IdwDoAO/evYuIiMCBDkLo06dPV65caeG5kpKSFhYWLi4uOTk5jVfKmTlz5rFjx1gslrW1tZeXF040Nzd3cHDAjx89emRvb08QBELIz89PRUUFDzd2c3PjzkKPior6/v378uXLZ8yYQaPRfliloqKirKwshBCNRhs+fDiTyQwICGjh7QAAQDuAcAeADhAaGvr9+/dVq1bhCeHR0dF43EzzNDQ0oqOjo6KiQkNDXV1dhYSEuA0wdDodz6iiUqnm5ubjx48XFBR8+vTpzZs3tbW1xcXFRURETp48mZyc3K9fP2Fh4R49enTv3l1WVlZAQGDFihUJCQkzZ87kXkhUVFRSUvLUqVNhYWHx8fGXLl1SVlZuPu45ceIEXrRwzpw5ffv29fPz+/r16395iQAAoA3BjugAtLfa2to7d+6IiIjgRW7YbPYPu35oNJqdnd369ev/+usvvhn69u178eLFr1+/mpiYIISSkpJw+w2ZTH79+jWOivD/qampJBIJH8VIJFJT/VYUCgXP/LKystq8efOBAweaquGTJ0/i4uKmTp2qoqLi6+s7cuTIhISEuXPnNn9fAADQPqB1B4D29vnz54SEBBMTk+HDhyOEXr161fxchunTp6empnp6ejYV62CWlparV6/Gj0kkEplMxp1lZDL569ev379/f/v2LX7KPYq1cIzOvn37NmzY0NRRFovl5eWF53yJi4vTaLQzZ87U19e3pGQAQJeRn5/f0VXgD8IdANpbSEgIQmjXrl346c6dO9lsdlOZjYyMLl26pKWlhfcN/QlZWVmGhoZTpkzBDT8/jUQiHTx4EK8BzVdERMSdO3cQQlpaWnhPiYiIiP9yRQDAb8fFxaWjq8AfhDsAtKuioqLz588rKCiIi4sjhG7evNnM5poaGhpXr14VEhL66ctxOJwZM2akp6cXFRXl5uauWLGimdCqJQ4fPrx58+amjnKbfzQ1NRFCvF1mAADQgSDcAaBdXbt2LTMzc9GiRT169EAI1dTUNJN5zZo1/yXWycnJmT17dnZ2do8ePXCrjIeHRzMdUi20b98+bq9ZA9+/f8cdWMuWLZOWluZ2bwEA/hCLFy/u6CrwB+EOAB3p8ePHTR1avHjxihUr/kvhxcXFYWFh9fX13BnvCKHw8PCPHz/+l2IRQjY2NpKSko3TP378uG/fPvyYQqE8efIEhu8A8EeBZQYBAAghxGKxRERE8LZW9fX1Pj4+fLP16tWrVXtXNVZbW7tt27YVK1b069evtraWOy08Nzc3KCjoh6eXlZV9+PChurqa79ERI0YYGBjwPYQXMCSTyTIyMkwms6Sk5GfvAADw++m0f/IQ7gDQrm7evCktLd2zZ8/ms508ebKpeVg+Pj6urq54vHMzjh07Fh8fX1ZWduXKFWVlZe4+DwihrVu3BgYGNnXiixcvDh8+bGZmpqamFhoa2lQ2Pz8/3t1JGyCTyYcPH66pqdmxY0fz9QQAdCV4cfZOCMIdANrV9evXuZt65uXl8e3rwds4NE5nMpmvX7+Oioo6fPjw/PnzBw8e/NdffwUGBr569aq2tpY3J45yEEIXL17cvXt3cnJygyVwGkx9r6qqev36tZeXl7a2tqmp6YYNG5KSkqysrPr16+fq6sr3RkRERP7jVC8AQNezc+fOjq4CfxDuANBhDh8+XF5e3jh96tSpo0ePbpxeWFg4YMCAoUOHrl27tqamJiUlJSMjw9bWduDAgfb29rztN5KSkl5eXt26dUMIBQQErFu3jnf4DkLo+fPnGRkZ+HF8fPzcuXMHDBiwZMmSt2/ffvnyBSE0Y8YMNzc3CwsLvMUEX3wX7ImLi+PuwQ4A+NMoKyt3dBX4g3AHgN8JiUTatm2brKystbU1b3poaKihoaGBgQHuODc2Nr5582ZMTAw+GhcX16D5Jz8//927dwihGzduTJ8+vcFkeDMzs9DQUCsrq48fPzazd8SqVasa97i9ePEiISEBIaSgoEClUv/jvHcAAGgTEO4A0N6EhYV/OgNuUPHx8cG9TryHamtrb968qa+vHxgYSKFQzpw5c+nSpatXr8rJyTGZzAbhDkKooKBgwYIFCxYsqKur403X19cPDg6+fPlyampq8/Wk0+ncjjleR44cQQgNGTJk8uTJDQoHAIAOAeEOAO2KzWbjaKApgoKCzWxNhb169SooKOj48ePGxsYNDr148eLs2bOnTp3q1q1bcHDw2LFj/fz8CgoKLl68qKSkxJvT2dk5JSWlQccTmUxeuHBhYGDg3LlzWSxWa+4MAADQ/fv3O7oK/EG4A0B7o9PpP5dBUVFx8+bNGhoaffr0KS0tXbdu3fnz5/X19Rtku3///vLly69fv66goDBq1KhevXpZWlqKioo2GGojJibWYLVACoXi4eFhaWkZERGhqanZr1+/QYMGNb8s4aZNm5q/FwDAH8XX17ejq8Af7IgOwO9kz549qqqq/fr1Gz16NEEQJBLp2rVrpqamt27d4s0WExOzePHiw4cPT5w4ccaMGQkJCSoqKmPGjMnLy+PmqampmTdvnre394MHD3CKt7e3ra3tgQMHhg0bNnz4cIIgyGRy3759m6kP39nyU6dObYt7BQCANgPhDgC/GQ8Pj48fPyorK3M4HAaDcejQoaVLlzYIdxBCMTExQkJCu3fv3rZtm4mJSVpaWvObjO7Zs8fOzs7Kyop3jrq6uvrKlStbW0Nzc3OE0N27d2NjY/GCigAA0LGgMwuA9vYft5Gi0WhlZWWvX79OT0/Pzs6ePXu2oKDgzZs3Gy/6l5ycPH78eBUVlfz8fNRo3jidTr9z5w6en9WrV69Zs2bNnTu3wXo827ZtExERaaYyFy9ebOpQRUUFQqjBBHgAAOgQ8E4EQLuiUCh4t86fdvToUd6n9fX1M2fOrKysjI6OlpeX5z1UUlJiZWW1efPmysrKN2/eNGjdodFoKSkpJSUlGhoa9+7d8/LyCg4Obm1l8JxzXj179uS7nRYAAHQgCHcAaG/cpQUXLFjQeM45QRBN7VSFSUtLS0tLq6qqKisry8jISEtLi4iILF68+Pnz587Ozg0yf/78OTMzc9CgQcePH28wAlpQULCyslJMTCw+Pp5EIt29e7dHjx4qKioqKiqqqqpiYmLu7u6TJ09u7d1ZW1srKiq29iwAAPilYOwOAO1q9OjRnz59wo8HDhwoIiJSU1PDm4HBYGzcuPHSpUtNldC3b9/Lly8/f/5cSkpq3rx5eOUbDofD4XDGjh1LoVAa7KNeUFDQq1evxsv9rVq1asOGDfPnz6+uro6IiJg9ezZ3RUE6nR4SEmJoaNhg7nqr3L59GyE0atQo9PmnywAAgLYB4Q4A7crGxmbr1q1lZWXN9PgwmczmC1FRUVmyZElOTs6FCxeoVCpBEAghBoOho6Ozd+/empqajRs34kSEUEJCgrS0tK2tLR7Bw0Wn03v37j1ixIhp06Z9+PChwSUWL17c1B6lXNXV1U2tIshisc6dOychIWFjY1N7qOEKhwAA0M4g3AGgXVGp1NLS0qNHj+7ateunC9HU1ExOTjYwMGiwotfjx48vXbpUUlISExPDnatVXFzs6elpbm4+btw4bs5evXqNHz9+wYIFo0ePbhzrSEhI2NnZ/bAaHh4emZmZvCnCwsITJ078ybsCAIBfBsbuANCuunfvjv6ZnEWhUNTV1RvnuXbt2p07d5ovR1lZedasWY3Ty8rK5s+ff/ny5bVr1zZzupmZmbq6+v79+x89etTgkLS09PXr18eMGdN8BRBCHz9+bJAiLy/PHe7DYDB27dolKCj4w3IAAF2GtrZ2R1eBPwh3AGhXU6ZMGTx4MN7BikKhbN26tXEeNpvt4eHxw6K2b99+4cKFfv36kUgkAR7nz593c3NbuHChgoICzomni3Mnos+dO/fw4cOnTp3asmULhULhnkilUiUkJMLDw8ePH//Dq+fk5Jw+fbpBIpX6d4NxaGhoRUUFju0A6CiXLl2CXdvaWc+ePTu6CvxBuANAexszZszRo0ffv3+PEDI1NTUyMmqcJzo6OiUlpflyaDSara3t3bt3i4qKSnlUV1fv3LlTS0uLO945NjY2LS0NxyLCwsLz5s1DCNnZ2VVVVZWXl+OzSkpKvn///v79e11d3Zbcxfr16xsn4u3A6urq/P396XS6qqpqS4oC4BfR19fnu4st+HUaLwDWScDYHQDam6Oj46lTp0pKSnBP1tq1a+Pi4rgji7HKyspTp075+Pj8sDRJScmEhISDBw8KCgrS6XRcDkEQTCZz27ZtUVFRS5cuzcvL8/LywhPRx4wZM2LEiLlz51ZXV3NX4mGxWDQazdDQcOHChS25hXv37jVecQchJCMjgxB6/fr1zZs3rayshg8f3pLSAPhFunXr1tFV+ON02tccwh0A2puMjMzAgQNnzpyZlJTUu3fvQYMG0Wi0+vr6Btl8fX2XLVs2ZMiQHxY4YcKE27dvh4SENBg4nJaWdv/+fW9vb9yAxGQyhwwZsnbtWnFx8QbRlYSEREhISMtHGQcEBFRWVjZI9PLyGj16NEIIj5I+dOhQC0sDAHQZ8fHxHV0F/qAzC4D2JiUlZW5u/vnz58TERISQrKysk5MT35xbt25tHFU0RqVS3dzckpKSPD09edcS/PDhg66urqKiorm5eXV19ZgxYy5dunT8+HHeWEdeXn7Tpk3Pnj0zMDDgjrxpXnZ2dkBAQINESUlJGxsbhFBFRcXZs2dlZWWFhIRaUhoAoCv5L4t1/VLQugNAB5g4ceLevXsjIiIcHBxoNJqNjc25c+e+ffvWIFt8fPz06dMjIiKa37gK69Gjx+LFi7t165aenu7v74+nl2dnZ8+YMSMsLKyqqmr48OHGxsbcCeo2NjZaWlrz5s1r1dtTXl6eqakpHmrNy9TUFFfy9OnTubm5J0+exB1bAIA/CszMAgD835gxYyZPnhwdHV1WVoYQGjBggKGhId+cSUlJDRbXaZ6Zmdn27dufPn2ak5NjbW09atSoDx8+zJo1S1hYeOrUqbdu3RITE5s8efLz5899fHy2bdvW2q9iZmZmeFdRXoKCgnjn88+fPx88eJBEIsE+EgD8mfz8/Dq6CvxBuANAx9i2bRtC6MKFC/hp4+2uuPbs2dPyYv38/GbOnHn48GFVVdWgoKA7d+7IycllZ2cPGTLk5s2bCKF+/frdvHlTR0fHw8NjxowZfCdYNSU4ODg1NbVx+uTJk/F0DE9Pz7KyMg8Pj5kzZ7a8WABAl4G/+XRCEO4A0DFGjBhhbW3t7e1dUlKCENLU1PT39+fuWsXr/v378vLyycnJLSlWX18fIeTl5SUoKCgoKCgqKtqgj+zx48eCgoI0Gi04OLi2tpa7X+kPXbhwYc6cOY03uBg4cKCXlxdC6OvXr/7+/suXL1+6dGkLywQAgPYBY3cA6DAbNmwYOXLkjBkzIiIiJCQk5s2bd+rUqefPnzfIxmazCwsLLSwsrl69+sNFcZSVlcPDw1+8ePHp0ycymSwgIJCSkrJ161Y8PHnRokWWlpY1NTVsNnvUqFEKCgpMJrOqqkpUVLT5YoOCghYvXowXg25gxYoV3bt3r6urmz59en5+fqcdqAgA+JNBuANAhxk8ePDMmTMvX768detWvEJxWFiYgYFBVlZW48zFxcVTp06Niori3fqqgdzc3MuXL9NoNEVFxdmzZ+NEIyOjZ8+eXblyRUNDAzfDYNHR0e/fv8/MzJSRkXF1dW2mngEBAU1tobV+/fpFixYhhGJjY588eaKqqrp58+Yf3zkAALQvCHcA6Ej+/v5sNtvf33/FihXa2to9e/aMiIgYOnRoVVVV48yVlZUzZsy4cuVKU+uWfvz48dixYywWq7y8fOfOnQghgiDodHpeXh5CKD8/f8SIEdXV1RwOh0wmf/r0qbKyUlJSct++fU1Vr6ioKCIiYtmyZXyPzpkzBy+uw+Fw8K4Xhw8f5m5VAQAAnQeEOwB0JCEhIQ8PDzk5OV9fX7wDQ58+fWbPnu3r68s3f0lJyezZs+fPn6+pqdl4SOD48eO/fv369u3bsLAwFouFEKLRaG/evMHji+vq6p48ebJ8+XJ5eXkWiyUmJkYmk21sbJra2er48ePe3t7p6el8j9ra2uJ2nfj4+DNnzty8efPq1aswQhkA0EkRAIB2UbKrpGRXCd9Du3fvRgitWbOGzWYTBMFisZYsWdL8Xy6JRNqzZ8/79+95y3F3d+/Vq1dcXFxlZSU3MTIykvfEd+/e4fTMzMynT59evHhRXFz86tWrvOWkpaUtX768matPnz6dm3nNmjUIIUlJyaqqqtbeOACgi+FuHRMbG9vRdfkXaN0BoONZW1sHBAS4u7vPmzdv4MCBFArFw8ODyWQ2s4IFQRDOzs7e3t6WlpYuLi4IIRERkcrKyoKCAkNDwxkzZnTv3r22tpZKpebm5vKeuGXLFklJSRERkfj4+MzMzB49epw5c0ZYWJjNZtfX15eVle3YsePq1avFxcXNVFhYWJj7GK/jXF9fv2DBgi1btgwcOBD6swD4Y33//r2jq9CEjo63APhTNN/IweFwNmzYgBBas2YNTmGz2Vu3bu3du3fzf8JkMllSUlJISGjx4sXXrl3z9fWtqanhcDjbt29v5qw+ffrk5OQQBFFaWnrjxo3Y2NgpU6YICQmJi4u35H2DQqGoqqqmpKQQBFFTU1NTU7N3716EEJVKnT17dl1dXctvHADQlfTq1Qu/S3S21h0S8e+dAgEAv0ipWylCSGq7VFMZPnz4YGhomJWV5eLiggcaI4Sys7NNTEz4ztXii0ajTZw4kU6nZ2RkZGRkNJNzyJAhSkpKZWVlt2/fbvFN/IuSktLVq1f79+8vKCiIENq1a9eOHTsQQnhPDO4aQj+8cQBAl6GmpoZ3sImNjW1qsfgOAZ1ZAHQWvXr1io2N3b17d1xcnJSU1KBBgyZMmKChoREfH6+vr5+dnd2SQphMZlxcXEtyPn/+vPEaP62Sn58/fPhwU1PTpUuXmpiYuLi4yMvL+/n5BQUF1dXV6erqmpiY/LB1CgAA2gGsqgxAJ6Kmpubn5zd79mwnJydPT8/8/HyEUM+ePe/cuePh4SEgINDRFeQjIiJi5syZ586dQwgtWbIkOjo6LCwsISFhzZo1M2fO/PjxY0dXEAAAINwBoPMxNTXdvHlzcHCwtrY27mnq3r37smXLQkNDjY2NO7p2fNTX1y9dunTlypU5OTkyMjLm5uahoaGCgoLp6emds8IAgF+kc34rQxDuANAJaWho7Nu37/z583V1dYaGhoaGhvfv32exWNOmTbt8+XJBQcHRo0f79u3b0dX8FwaDcfr06WnTpiUkJFRXVxsYGOTn569du/bt27c4A5vN7tgaAgDawZw5czq6CvxBuANAJ2VnZxcXF2dsbBwfHz927NijR48ihMTExBQUFJycnBITE8+ePSsrK0smt81fMYlEwiOO/4v09HR9ff1169YhhGRkZI4cObJq1Sp8CM+WBwB0bf7+/h1dBf5gqDIAnZeent6YMWO2bNly5cqVTZs2lZWVDRgwwMDAQFpaWl5efvHixfPnzz948ODr16+FhIQeP3787t271l5CXFx8+vTpbDZbVFR05syZHh4ekZGR/2XCpqamZrdu3Rqnz58//6fLBACA/wgmogPQTv7LfOzs7Ozg4GC8lM7AgQNXrFhhaWkpISHBmyczMzM3N7eurm79+vV1dXV4rT8SiVRTU4MX/qJSqT169MCZORyOhITEoUOHJCUlR40axVuOs7PzmTNnSktLf+ou0YEDBzZu3IgQIghi3bp1x44dK9lVgmAiOgB/hk47ER3CHQDayX9ffsbX1/ft27eBgYHfvn3r3bv3ihUrqqur58yZo6qq2sxZb9++9fT0pFAo8vLyOBD5oZKSEiMjo6dPn7aqelpaWg4ODrgn686dO97e3oGBgVpaWvet7yMIdwD4M6xYsQJvGAzhDgB/qLZabS8vL8/T0/PChQt4n/NevXoJCwvv2bNHVVVVVFRUQ0OjheXk5OSUlJQICwtfunQpJCSETqczGIzt27fPnTsXIVRYWDh9+vRHjx61sDQtLa2oqCi8oGpsbKyFhUV9ff3Zs2dNTU2pZ6kIwh0A/gzJycl6enqo84U7MHYHgN+MsrLy7t278ZYRz549CwgIQAjNmDEDIaSkpDRr1iz8HYbFYvXq1Wvt2rUcDgchRCaTCwsL9+zZg4c2s1isyMjIBttpIYTmz59Pp9NnzZolJyd348YNY2PjlixFqKGhER0d3bNnT4RQYmKihYVFdXV1QEAAjpxK0U/2iwEAQFuB1h0A2smv2EuhpqampKSERqM5Ozvn5uZSqdTnz58XFRXhoxQKRVRUlJuZw+FUVlYihEgk0vDhw8XExBBCioqK+/fvZ7PZAgICjx492rt378uXL/v27RseHq6iolJTUxMXF3fw4MHHjx/zfa8QEBDw8vIyNzcXFRUtKiqaNWvWw4cPCYLw8/PjzkeFTSQA+HNA6w4AoO0JCwvjzcl9fHxwysOHD3NycqhUKkKIIAje1W5IJBKFQkEIkcnkadOm4Z3MeZmamo4YMUJbW/vFixdTpkyJiYlRUlKaOXPmyJEje/fuXV1d3SA/iUQ6e/asvb09QqisrGzmzJn37t0zNTXdsGHDuHHjftlNAwBAq0G4A0CXMmrUqAYzrVpFVlb269ev69evP378uKGhYUxMjIqKSvfu3UNCQk6ePBkbG8vNSaVS/f39cXdVUVHRtGnTHj58OGHChAULFkCsAwDobGCZQQDA/5FIJCqVeuTIkRUrVrx582bChAlbtmzhcDgmJiYRERFLly7F2ahUqpeXF451zp49O3r06IcPH1pZWcXExOBRRAAA0KlAuAMAaIhCoZw6dcrd3V1YWHj//v1r1qxBCFGp1DNnzuCIx9/fHy8bWFpaev78+aysLCsrq6CgoP++LjMAAPwKEO4AAPhzdHSMiorS1tY+efLkmjVr8DCgQ4cOpaWl4WHIbm5u2traOTk5CxYs8Pf3xwsbAgBAJwThDgCgSaqqqtHR0X369Dl+/LiFhcWtW7dERUX79u1LIpH27t3r4uJCEMStW7d8fX2hXQcA0JnBUGUAQHNwxIM3Jb158+bZs2cVFRWTkpLc3NyUlJQsLS07297sAADQGIQ7AIAfUFNTu379+oULF+bPn29ra4sTFRUVo6Oj+/fv37F1AwCAloBwB4B2hdfc+x2ZItMi16J/JV1Dpdd+19sBAPxRYOwOAAAAALo4aN0BoJ3ALgoAANBRoHUHAAAAAF0chDsAAAAAaBvc/fs6Gwh3AAAAANA2Gu8l3ElAuAMAAACAtiEvL9/RVeAPwh0AAAAAtI1p06Z1dBX4g3AHAAAAAG1DWFi4o6vAH4Q7AAAAAOjiINwBAAAAQBcH4Q4AAAAA2gaTyezoKvAH4Q4AAAAA2sbq1as7ugr8QbgDAAAAgLYxY8aMjq4CfxDuAAAAAKBt1NXVdXQV+INwBwAAAABtY+jQoR1dBf4g3AEAAABA23jx4kVHV4E/CHcAAAAA0DaePHnS0VXgD8IdAAAAALSNb9++dXQV+INwB4CmBZFQEOk/nfvTpwMAwG/I3d29o6vAH4Q7APwaNkRH1wAAANpbp90zi9rRFQCg08MtNNzwhdtg05KUBoVw83DLbHwuAACAtgatOwD8CG+Awhv6tCQFa5yn8WMAAPj97dy5s6OrwB+EOwC0Hm/jDW/g0vKzGpwOEQ8AoEv4+PFjR1eBP+jMAqD1+Lbc/DDigZgGAAA6CLTuANAa3LacBrOuuI+DSA3be3hTuGfxHgUAAPCLkQgCvnECAAAAoA2oqal9+PABIRQbG2toaNjR1fk/6MwCfzZoXPkzQcciAL/G+vXrV6xY0dG14AM6swAAAADQNrS1tTu6CvxB6w4AbaFBa0HnbDRq3KQRREI2REFBwefPn8lkMkKIw+F07969pqamrKxMUlKyd+/eCKHs7OzS0tIBAwYUFxd/+fKFm7NHjx4yMjJpaWkkEgmn9OnTR0xMDCFEEERaWlp9fT1CSFBQsG/fvi9evCAIgs1m19bWkkgkISEhGo3Wo0ePt2/fiomJ1dXVMZlMYWFhERGRoqIiKysrW1vbAwcOIISsrKyePn1KEISgoKCAgEBNTQ2bzWYwGEePHh02bJi5ufmwYcM8PT1jYmKcnJykpKQ2bNiwadOm7t2719XV1dXVqaqqRlhHlhqXSElJ/X3LAIA/D4Q7APznro1mZpgjxOFwEEIkEokbE+Cn3GFz3EMEQRAEgYMJhFBiYuLVq1dJJJKWlha3cZggCDc3t4KCAiEhIQaDwWAwHBwcrl27VlFRQaVSyWQyg8HAh8hkMo1Go1AoM2bMCAoK8hh7Jjs7W0NDo3FNz5w54+bmxn26fv3658+fJyUljR8/Pjk5GSG0cuXKuLi4zMzMCxcu7N69m5tz27ZtdnZ2Q4cO5aYsWbLk7NmzCCEWi2VgYPD161eEkIaGRlZWVnh4+O7du7W1tR0cHAiCOHHixKdPn5KSku7fv+/q6mptbT1y5MgDBw58+vTp4cOHcnJygYGBixcvVldXT0tL69Wrl5mZmY+Pz8uXLxcuXNivX7+3b9+ePn369u3bWlpaXl5eVlZWxsbGjx8/3r9/f3FxcXZ29unTp3NycpYtW+bk5IRQJIvF+pmfLACgyyAA+JNdRMRF9OLFixs3bsTExISEhNy6dYv3+PPnz69du3b79m38lMlkRkVFRUdHh4WFXbt27f37938X0rhYgiAIIjIyUkREREhIaOPGjQRBlJSUKCkpiYmJHThwYPz48VJSUmJiYpGRkTjz7t27DQwM8OPq6uoxY8bgP9IJEyZwC7558+b48eN79OiBENLU1BwzZsyVK1cEBASGDBnSo0cPCQkJfNbAgQPV1NQQQkOGDLl165aEhARxEaWlpfGtZ01Nze3bt2VkZMTExGJjY2tqampqapYvX44QioyMTEhIQAjhGKW8vFxBQUFQUPDjx49LlixZs2YNQRBz5swZPnw4k8k8dOiQnJwcQRB1dXWWlpbLly+vrKy8fPkymUw+ePAgQRBmZmb6+vr4ygMGDFi5ciVBEFVVVZqamikpKaWlpbGxsWfOnCEIYt68eQihO3fuXLx4ESGET//06RNC6O7duxcvXgwICPD09CQIwsrKCiGkrq6el5f3/PlzhFBBQYG9vX1lZWV2djZCKCsri7iIcnNzeX/irf01AQC00MmTJ/EbV2xsbEfX5V9g7A4AqLKy0srKysLCorKy0tHR8cKFCzj9+fPnBgYGM2bMWL16NU6hUqnR0dFmZmZVVVWOjo4zZsz48uUL3zJramrs7e03btzo7e3t7Ox85MiRDx8+SElJGRoacjicKVOm7NixQ0tLq0ePHsOHD0cIFRYW+vj4FBcX49OFhYUDAwNVVVURQkuXLuUWO3ny5OTk5FmzZikoKMTHx9+7d2/o0KH19fWXL1+2sLAYOHBgUlISiUQ6ceJEcnKysrKykJAQiUSSlZVFCPXt25dvVYWEhHR1dQ0MDMaMGWNoaCgkJCQkJCQhIYEQ+vDhw/HjxxFCuItKXFycSqWSSKRu3bqJiori00VFRclkMpVKFRQUxCmpqakhISHbt28XFRW1tLT08/Pz9PQsLS3duHFjRkZGenr6/fv309PTFRUVEUIEQYiJie3fv3/o0KHi4uK8N7tz585jx45xn0pKSgoLC2/cuNHV1fX27duLFy9GCNXV1Y0ZM0ZERMTd3Z1GoyGEKBSKo6OjiIgIwTPttFu3bi34RQAA/FdHjx7t6CrwB+EOAEhXV9fHx4cgiAULFgQHB9vb279//x4hlJ2dzWAw6urqnj17xs08atQoOp1ua2ubm5s7evRoGxsbvmXW19cHBwdXV1dbW1s7OzvPmTNn2bJlCKEhQ4aIiYn17dtXT09PQ0NDWVkZxyIRERG5ubnPnz8/f/48LqFnz55ycnIIIRwP8aLRaF+/fn3y5AlCSEREZO7cudLS0mw2myAIGo1mb28vKyurrKwcFBR07969iRMn/vXXXz98EahUamZm5rdv33gT16xZExkZyZuCu94cHByOHDmCwwshIaFHjx4JCAisWrXK2Ni4ccljxox5//799+/fR4wYUVpaGhISkpqaKi8vv2nTJnzd7Ozsz58/CwkJcU9hsVg9evR49+5dTU2NiIgI7gHE6HQ6jUbD945zqqqqRkVFubu7Hz9+nEKhsNnswYMH4/5BnAG/Sj98BQAAXRiEOwAghBCdTmez2R8+fCgoKBAWFqZSqQihcePGKSsr79mzp6KiAiH06dMnDofD/RxFCAkICJSUlDRVJovFcnFxwY+pVCqDwWgqZ319/cGDBx0cHNTU1A4cOFBdXd18bZlMZvfu3UeOHIkQkpGRCQgIkJKSYrFYuD3D399fS0sLISQhIUGn04l/+oaaJyAgkJOT8/r1a/yUwWD07dvX2trazMwsKCiIO6Jo/vz5tbW1s2bNunXr1pIlS3DOwYMHW1paIoTMzc0RQrh9qLCwEJ+CRxzjum3ZsuX06dMfP350dnbmvpJ1dXUuLi6bN29mMpmfP3/GZY4YMeLSpUv29va9e/dmMpncWrm5ua1fv37NmjX5+fkIIRKJxGKxlJSU1qxZ4+vr+8PbBAD8Utw3vc4Gwh0AEEKIQqEwGAw1NbWZM2fGxsbiXqTt27crKSm5ublNmjTp4cOHqqqqoaGhwsLC3LMEBQW58UFjJBIpKyurJVf38/PLzMzMz8+vra19+/bttWvX+GaLjo7Gc51oNFpBQcG9e/eaL7Z///6LFi1qSQUQQjU1NUZGRpMnT8ZPORyOkJBQQEBAWFiYjo4ONzS5cOECnU7X19fX09NTV1fHOUVFRQMDA6dOnYq7ovr3729tbe3o6IgQevPmzbVr14YNG6agoIAQ0tbWLi4uPnjw4ODBgxtcfc6cOQ8fPlRSUvr+/TuNRmMymePGjdu4cSOTyUxKSuJwOPHx8Xhi1/z58wcOHKisrFxeXi4kJPT8+fOCgoLDhw9ra2uz2Wxc4MuXLw8dOoQQOnLkSAtfAQDAf9fCN732B+EOAAghxGazBQQE0tPT1dTU0tPTEUJ1dXVBQUGOjo6+vr7v3783MjKysbGZPXt2TU0N9yzeTpbGKBTKpUuXGiRSqdSvX7+ePn2aNxG3fOjp6Y0bN05UVDQsLKxxaVFRUUuXLsWtLLyTvLjIZHJJSQlv9VqOzWZ///69vLw8NTUVjwguKCioqKioqampra1dvXr1hw8fMjIy8vPzKysrCYJ48OBBWVkZQojFYhUWFt69e9fd3T08PFxOTm7BggU5OTlaWlr3799XU1MbN25cUVHRwoUL8WCgKVOm6OrqCggI4KdPnz5VVVWtr69fuHChmpqai4sLjUZLTEy8cuVKbGzsihUr/P3909PTb926NXTo0K1btyKEbG1t1dTUpkyZgl8EGo32/v173C/m7u6uo6NDp9Pr6+t37tx5+/btYcOGJSYm/sQLAgD4OY3f9DoJ2EQC/Nn+2eDzypUrdnZ21dXVx48f9/f3T0pKEhUVFRMTu3Tp0syZM48dO7Z27dr9+/dv2rTpwoULq1evLikpKSws1NXVVVJSurUwic+6OzbE/Pnzz58/f+PGjezsbE9Pz82bN9vb22dlZenp6cnJyU2cODEyMnLHjh2zZ8/W1dV9/PhxbGxsv379Ro4ciaeX6+npzZgxo7KyctasWXh5Gzy2Zt++fRcuXHj37t348eN79eq1YcMGbW3t3bt3BwYGZmRkjB49esmSJXZ2dgihQ4cO+fj4ZGZm6uvrx8+7yX/dnT8E9965W7oCAH4BZ2fnvXv3IthEAoDO6dWrVywWKzw83MDA4NixY+PGjYuOjp49e7ajo6ONjU2PHj169+69efNmGRmZvXv3lpeXS0hI1NfXq6mp+fr6oodqfMv08/Oj0+mmpqYIoaFDh+JxLb17905KSjIyMvLw8Jg8ebKpqWlVVZW4uPjYsWO/f/9eUVHRs2dPGo2Wk5MzcODAIUOG4GX3CgsLaTQaHgj84cOH7t27a2ho1NXVvXv3rra2FifiSpaWlnIniw0fPjw6OlpVVXXRokWIeZNPFW0IhFBxcXFZWRkezEsQhKioqLCwMB55IyUlRSKR8PgkWVnZurq6yspKhJCwsDBunkEIVVRUsFgsaWnpNv2BgD9FRUXFrVu38JTAjq4LaAPR0dEdXQX+oHUH/NmCSAiheguGpqZmaWlpRUXFs2fP5OTkjh8/3rt37yVLlty9ezc3N7d79+49evR4+fLl8OHDMzIy6urq2Gw2m80eOnSopqbm//c85y3WhkAIVVdXx8XFbdiwQUZGJiQkBC9agxBKT08vKCjgDpRppzuFJg0ErTudzuvXrwcMGODm5rZt27aOrgtoAwcPHsSdy52tdQfCHfBna6venCbCHSw3N3fixIlVVVVJSUlNLX7zq/xG3VXtE4JAuNPJEARRV1fHYrHw2k7gd5ecnKynp4c6X7gDnVkAtIXGUQVPSk+EcvDWC6n9UOqvrAbfT3Gc2Mlbd5oKy36jcA38FLyBWkfXAnR9EO4A8Au+63fUhzTf6/7uEUPLfjr379+PjY0lkUgMBmP+/Pl//fVXcXHx4cOHKRTKzJkzhwwZUlNTc+TIke3qnXRREAC6hqtXr3Z0FfiDcAeAX6BD2lEatd9UVVWJ3hBrvj7Z2dnq6up4UndxcfGbN2+oVCqLxerVq5eSkhKbzX7w4AGVSmUymerq6tnZ2QICAhwOR01NLScnB6+wTBAEh8Pp168fHuudmpq6cOHCkpKSBQsWnDx50tTUlMFgPHnyZNmyZWZmZrt27UIIBQQErFmzBq/CPHXqVAkJiexdiMVi4dUdG5CRkZGVla2uruZwOCwW69q1a6NGjXrz5g3efktYWPj69et9+/a9c+fO7t27lZSUyGRyaGhoQkKCoKBgREQE3q1iyJAhr169cnFx2X6xbV90AMC/pKSkdHQV+IOxO+DPxm8kB5vN5nA4NBoNr+lHoVAoFApCCH/c4v2huJlxIkKIRCLRaDSCILhLAGPc05lMJvfPDScGBATExcWJi4szmcz6+noLCwtTU1M/P7/ExEQJCQkmk1lRUWFrazt16lSE0P3790+cOCEoKOjq6tqrV6/09PSDBw/S6fQFCxZcvHiRwWB4TfAuNS6RkpLiXnrlypWnRp9GCPnV+d6+fVtISKiurq5v374bNmxACJ09e/bRo0cPHz4cP3587969N2zYcOXKFQsLC3zu3r17t2zZghBatGiRj4+Prq5uYGCgvb19UlKSnZ1dWVnZjRs3eG/zypUrOjo6169fT0hIiIyMNDMzmzBhgrGxsYaGxoYNG549e2ZnZ+fl5TVt2rQtW7a8e/du8uTJnz9/vnPnzrNnzzZs2MC6wOYfkwWR1j5zOnbsmKura8+ePWNiYqhUamBg4NKlS/Py8qZPn15YWBgYGHjz5s2qqiodHZ1Dhw7NmzdvwIAB48aNw9txqKurd+/e/d69e2ZmZuHh4QQOdzpzvx4Av7NOO3YHdgYGf7aLiLiIQkJCwsLCYmJiCIK4c+eOjY3NoEGDAgMDFRQUREVFTU1N8dp6eLNMAwOD5ORkDodDEERmZqarq6uAgICgoOCgQYMIgnj58qWQkJC0tLSwsLCYmJiAgICbmxtOV1RUpNPpYmJiQkJCR48effv2LY1Gw7tZqaioSElJCQgI4BUIe/fujRBSVlbW1dUNCgoiCOL+/ft9+/Y1MjLq1auXpqbmly9f0tLScGSTlJR08eJFEolEXEQlJSXcO3v//j3eCJ24iHR0dKSlpUNDQ1VUVIYNG4YzTJ06VVRUNDU11cnJCSF08eJFgiDwItExMTF4SwqCIB48eEAmk1etWkUQRGVlpYSExKVLl86cOSMlJVVWVtazZ08XF5epU6fi7c0JgoiIiEAIHT9+HD8NCAhACOG7KCoqEhcXxzvA4x0wQkND2Ww2nU5vcpfyi39/WczPz8/Ly4uIiEhKSjp58mTv3r1xDUtLS0kkUmJiIkEQqqqqR44cIQjCyMjI2toaF9CnTx/0z/KGCCHYER2AXyopKQlHF7AjOgCdjr29vZWVFV6aLycnJygoqLa2VkxMbMGCBVVVVQghKpXq4+OzadOmlStXpqenT5gw4ezZswghd3d3V1fXjRs3rlmzpqioCCGkpaU1evRoOTm54OBgd3d3NTU13PZz4MABVVXViIgIBwcHvFIOg8FgMpm4I2bq1Kl6enr19fWVlZXLly9/9OhR7969jYyMbt++bW1t/e7dO0NDQy0trZiYmOTk5KKiIn9//759+5qYmCCEjhw5Ym1traamhhDibdrx9PSk0+n48YsXL0xNTS0sLKKjo3GA9fTp05SUlMTExAEDBixdulRISAgvqIN3DpeWlsYtUgihUaNGjR49Gjd0iYqKUigUMpmsqKgoKysrISFBoVBkZGQ2btzIzY9zclVWViooKFhbWyOExMTEWCxWXV0d96i7uzuZTMavfFNwR5u1tfXkyZPHjh07YcIEvKAzvqKkpOSqVavwfmRCQkInT56cOnVqaWkp3sICExAQiImJefHiRWt+KQAAXQqM3QF/ruzsbA2EEEI1NTUvX76cP38+i8UaNWoUQkhWVnbatGnTpk378uXLqVOnPnz4sGjRonfv3vXp08fe3n7atGlSUlJZWVkeHh5ZWVkaGhoIoa9fv9bW1goJCWloaDCZTGNjYzk5ubCwsIkTJxYUFKSkpGzatElfX19fX3/o0KHW1ta5ubmLFy/u3bu3oqJibW2tn5+ftLS0uLg4QkhaWlpRUZG7/dPXr1+rqqrwp7uKioqCgsKDBw8QQhwOR0VF5cmTJ9u3b2+w43dubu7Bgwc3bNiA0CGE0MyZM8+fPy8qKrpw4cILFy4ghD59+lRSUjJo0CCEkKampoWFBQ7LWqKmpsbS0hI3UxMEUVtbO27cuDFjxuCjeGEh4ke95NXV1Vu2bLl+/bqlpeXNmze9JvzgogoKClVVVXyLLS8v5+6qISMjU1VVNWDAgBEjRnAvtH///sePH48aNWrTpk0INblRKwCgC4PWHfDn4jZ+fPr0adCgQeHh4VQqVUhISFxcnLsdFR5kgz9lBQQEEEJDhw598uSJlZUV3p0KJyKE3NzccIGCgoJ3794dPHhwaWkp3uKqe/fu1tbWixcvHjt27KtXryZOnEgmk9XU1Dw9PUkkEofDYbPZEhIS3t7ecnJyjeuJr25vb4+fHjx4MCoqqqSkhMVijR07dvr06Z6eng12Kt21axedTscbdiKEzpw5c/To0cDAwAkTJuBtzBswNjZufv+vxrg3jnG3TDcxMVFVVeXGaniqVGlpaeMSBAUFd+3aFRoailtrmoJv/9SpU+bm5rjY7t27Z2VlXb58GSFUVlZ24cIFXBkGgzF37tzk5OT09PTt27dzT6fRaAEBAQYGBg2GVQEA/hzQugP+XMrKyviBqqrq0qVL16xZgxASFxdXV1fn29SBZyHt2rWrvr5+/PjxKioqOJHNZru5uTGZzNGjR0+ZMoXNZsvLy0+dOjUtLY3bGrFt2zZRUVEcVwkLC0dERODRfC2Bmy4CAwPxJhJPnjyZMGGCtLQ0h8NhMpn79+/39vbmzZ+WlhYWFrZ69Wo1NTX0CCGEjh07pqen9+XLl6NHjwYFBb19+7ZBsLJ27VoXFz4ztCMjI42Njbnbof+EsWPHslisQ4cO7d27Nzo6uqamRlBQMDMz8/379yQSycjIaMCAAa9evWqmhPDwcIRQcHCws7MzTnFwcMjJyTlx4kRWVtbXr18HDhw4YMCA+/fvf/v2LSwsTFFRcenSpYcOHcJxZEVFRUhISI8ePUpLS1sb0gEAugxo3QF/LjyGBiGUkZFx7949Q0PDnJyc6urqrKwsvjOiEUIEQVy5cmXv3r2ZmZm4dwnHQKGhoXv37s3KykIIMZlMDQ2Nffv2vX//vn///nl5eWw2e+XKlZaWlu7u7hcvXqyurm7VNt3S0tJ4Kjh+eunSpUmTJiGE2Gw23qwKL9nO9ejRIzKZvHnzZm7KgQMHTpw4weFwpk+f/v3791u3bpmamg4dOnT+/Pmpqalr1679+vUrvuXMzEyE0MqVK589e+bl5WVvb89kMgcNGuTr6xsWFrZs2TJ1dfU5c+bgYp2dnXNycj5+/Jidnc19JTdu3Pjx48e9e/cqKSnhTU/Nzc0PHz6sqKhoaWm5ePHiKVOmJCUl1dTUuLm52dvbJyQkcMf98IXHPjs6OvKu5zFv3rzq6uodO3aEhYXNnj27W7duGRkZWlpaDAbjyJEjgwYNWrdu3dWrV+Pi4gYMGFBWVvbs2bMPHz4MHTq05S87AKBL6aAh0gB0vKysLO48nfz8fGVl5QULFhAEsWHDhgEDBhAEweFw5s+fX1tb+/btW4SQs7MzQRABAQF46nh1dfX48eM3b95MEIS/v7+0tDQu1sHBYfTo0fixk5OTs7PzmzdvEEI7duwgCOLu3bsCAgIuLi44Q11dnb6+vpyc3MuXL+vr6wmCYDAYSUlJkpKSampqlpaWeE7TggULunfvbmtrO3v27AEDBmRkZGRnZ8vKynbv3n3Dhg3p6emioqL4Rqqrq8eOHYurShB/Tz3DH/M4SFJXV8/KyiII4urVq9xmm969e79//54gCDzpnRvtSUlJVVVVff36FY/IQQgFBgbigj9+/Egmk3H/naGhIU6cMGECQkhQUJBGoyGErly5QhBEeXn5mTNnTp8+7e3tXVtbSxDEmzdvPDw8Tp8+HR4eThCEn59fMzOzrl69evbs2dOnT2dkZPAe4XA458+fx/fSCjAzC4BfqdPOzII/e/Dn4oY7kZGRmzZtkpaWxhO5g4ODaTSagIAAiUQyNTXFmX18fLiJw4cPDwsLIwji/PnzVCoVdwzNnDmTIAg8axrHATgUcHFx+f79e69evbS1tclksrKy8rJly+rq6giCYDKZ69evnz17tqmpqZWVVVRUFEEQAQEBs2bNmj59+vTp0w0MDGbPnl1cXMzhcCIjIydMmDBp0qSUlBSCIF68eGFkZDR16lRbW1uCIJYvX05cRLW1tbq6ujIyMu/evfv7Ji8i4iKqrKwsKir68uVLUVFReXk5PsLhcIqLi4uLi3kTKyoqioqKiv9RUlKCu4S4KWw2G+fkJhYVFVVUVDQ+vaioiMFgtPSH0XS409ISWn4hCHcA+GU6bbgDywyCPxebzaZcpiKEqHaUMWPGuLm5jR07Fg+5TUpKwhHA6NGj5eXlcf5bt25VVFQghMaMGSMrK4sTExMT8RTusWPHduvWrby8/Pbt22QyGQ8TIQjir7/+6tOnT0ZGBo1Ge/HiRe/evQcOHPhL7ieIFCcTa2RkNG/ePH9/f24iQr/DqnpNbenV5lt9/S4vCAC/p067zCAMVQZ/Lu6Qkbq6OhKJxDuChO844okTJzZOxD1EXBISEtOmTWucDS92h1fH+XWYTObw4cONjIwaHvitt836rSsPAOgcINwBAFFDaB1dhbYxtcJ0qiNCbCsUZNXRdWk9CGsAAL8MzMwCAAAAQBcHrTvgzwZjOAAA4A8ArTsAAAAA6OIg3AEAAABAFwfhDgAAAAC6OAh3AAAAANDFQbgDAAAAgC4Owh0AAAAAdHEQ7gAAAACgi4NwBwAAAABdHIQ7AAAAAOjiINwBAAAAQNvw9/fv6CrwB+EOAAAAANrGnDlzOroK/EG4AwAAAIC2ISAg0NFV4A/CHQAAAAB0cRDuAAAAAKCLg3AHAAAAAF0chDsAAAAA6OIg3AEAAABAFwfhDgAAAAC6OAh3AAAAANDFQbgDAAAAgC4Owh0AAAAAtI1Xr151dBX4g3AHAAAAAG3D3d29o6vAH4Q7AAAAAGgbenp6HV0F/iDcAQAAAEDboFAoHV0F/iDcAQAAAEDbsLS07Ogq8AfhDgAAAADaBpVK7egq8NdJqwUAAKBrI+0k4QfEDgI/5n3QIA8XztP4FO5ZTaWDPxy07gAAAGhvvGENDl8aPED/jofwvwaFNAiPmnoMAIJwBwAAAEYiIVLDxhT+if8db1DSfB6cjZuzQSTEm8j3dIh4AAadWQAAAJpE/JpooXGXE/+r/zsqaio/xDTghyDcAQCA3xtufSGIfzXDcJ/ieIV7iBu+8M3cTGkNzuItn/dCrat5s7FOU0e5w3e4j/nGQ9w8ACCESMQvCt0BAAD8eo1jjpY/QP+OUfhGLY1jpuZLBn+45ORkvNJgbGysoaFhR1fn/6B1pwP88D0FAADaHN/mnF/3ztN8y037g5aePxyEO50FBDrNgAmrAPx3Dd5keBtsSKQfN/MA8FuDcKcVft8O8t8ab1DCDV8aP8CZm4p+GgQ9+CzeXv8fDpkEoHPC7wyN32d432oapDR420H/vKvwZmuQhzexVW9ErfoK0arRNi3PDH/aAMFE9JZrSczROO7hfX/hvqE0Tmm+s7zxgz8KTFgFoHmN30NwCvdfgxS+GRqc2DhPg8QGl25cBwA6FWjdaXvt3EHe5cGEVfCHa9U3nP/+dQjepkCXBK07ba+pbz+o0TvRL1q/q0v64YRV3nYdLt5mG248xM3ZoB0IAPDrVFZWTp06VVVV1dzcvLa2tnGGN2/eIIQYDAbOOXny5KFDh8bHxzfOWVFRMWjQIPx4xowZRkZGJiYm58+fxyllZWXBwcGTJk3q379/SEjIr7of8LuB1p2W6uQd5F0Y35Xjm3rQOBvvoQbZoKUH/Ebq6hh0Op3JZCKEaDTar7jEL/325evrKyYmtnTp0mfPngUGBi5atKhBhp07dyJtVFtb6+Pjk5CQkJycPG3atC9fvjQuqqamRkBAACGUnJz8+vXrq1eviouLW1tbW1hYiIiIHDx48NSpU/fu3Tt27NjSpUvNzMx+4V2B3weEO63QeOhM41iE79yHpopqJk9T5XTJ6AcaV0AX8Kuj54ULFz5+/Pivv/6qqKiwtrZesmRJM5m/f//+9u3bsWPHksmdogm/uLh4x44d8fHxI0aMSExMtLKysra2FhUV5WaoqampqalBCElKSi5cuNDKykpJSWn+/PkXLlywtbWlUCjcnFVVVQ4ODtOmTXvKfnrz5k0REZGBAwcihDQ0NF6+fDl27FhVVdXZs2cPGDDAzc3t06dP9fX17X+/oBPqFH8JnQRuUPnhv1Zlhn+8LxoA4KedP3++sLBwzpw5ly9fPnbsWHBwMPcQg8Goqqpis9n4KYfDiYuLMzIyKi8vr62tLSsr4x6qq6urqqpq0JeUlZV14cKFX1p5JpNZUVFBIpEQQiQSqbi4mMVi8WaYO3eugoICfiwpKamkpOTh4XHw4MFFixbxxjoIoZiYmNu3b69fv77BJerq6srKyhBCS5Ys8fb2RgjdunXr6dOnuB0IAESAfyAE/37Vvx+88q4Iubb0V5Fvzpaf3p4a1+o/3ibX8+fP4+LiCILIycnx8/N78eLFz9XwF8GVt7KyIpPJw4YNs7W1RQj169fv48ePfHM2TmEymSNHjlRVVSUIIj09HSEkIiJy586dVlXj/fv3QkJCmZmZDdIvXLiwbds2DofTqtKa0apf4J8pHxEIEWw2W0JCIjAwkCCIZ8+eTZ06ta6ujiCI4uLi4cOHI4SOHj2K8wcEBODeLvz/sGHDSkpKCIKIjIwcMmQIQkhJSSkjI4Nb/t69exFCL168aMlfayuqzfOyVFZW9u/f/8iRIwRB7Nmzh0KhlJaWcnOmpKRQqdTz589z8+fl5ZmbmyOE1q1bx1smi8WSl5f38/PD5V++fPmvv/4KCgpauXKllJQUfkGw2tpaCQkJExMT4tf/gACvpKQkHF3ExsZ2dF3+BTqz/iCNp7u3bdcYtOK0UH5+Pm6fFxcXLy8vl5KSOnv2bLdu3X54YsNev4f/PPiE0I22r2cLNdWJ4+npmZaWpqWldf78eRsbG2tr6xs3bqxcubIlZVKp1NWrV9vY2Pj7+/ft2xchNGrUqHHjxiGE2Gz2169fcTYFBQUKhRITE7Np06Zly5YZGBjMmzePzWbv3bt3woQJu3fvrq2ttba2FhYWrqmpOXbs2Lhx45YuXfrmzZsJEybgloaSkpJp06b17dtXS0srMDBQQkLi/PnzSkpKCKHnz5/v3buXRqNVVVWFhoYKCQmFhoZ6enr26tWLw+FkZmaamJhs2bKlTV7D1pKWlo6MjMzOzu7bt6+dnZ2uru7Zs2fnz58vLi7u4OAwd+7cnJycPXv2hIaGkkgkLS0tKSmpgoICW1tbFxcXT0/Px48fX758efv27bi0tWvXDhw4UE1N7ddVWFRUNCYmxtra+vXr1xEREbKysrxtNsePH1dVVZ06dSo6+XeKkpJSaGjo48eP161bt2vXLhcXF27OHj16zJw5Ez+1tLSUl5f//v27rq7uu3fvmEwmnU7Hh9zd3U+fPt2pNjEAHQvCnYZ+RQTQVJntORi5fWKRxvfyX6579+7dnJwce3v7lmTOzc0NDAwUFhYWFhZeunTpuXPnvn79qqamZmlp+fM1+DVERERKS0vfv39/48YNb2/vS5cuycnJeXh4tORcYgfx5MmTESNGfPz4UUVFhcPhLFy48OTJkyIiIs2f+OrVq8ePHzceH8orLS2NQqFoaWnhp58+ffLz8xs5cqSent7x48cZDIahoSFuSOBqZtyVuLi4kpISHlprZGQkIiLC4XBaco8YHnRy8eLFjx8/8qb7+PgsXboUPz537py9vb20tHTv3r1Xrlw5ffp0BoOBI0iEkKKiIkJIQ0NDWlq6trZWXFwcIdSnT587d+54enpaWVlpa2tTqVQdHZ2AgAANDY1Ro0YVFRVt27bt3LlzCKE5c+bY2dnZ29tPmDAhKirKwsIiIyMjNzc3ISFhxYoV9+7dmzdvXstv59fZtWtXTU3Nq1evCgoK3r9/jxP79u1LpVKNjIy4XTlbtmyRkpKi0+mPHz9+9+6dnJwctwQ6nW5iYvKr66moqJicnEwmkyMjIz9+/CgmJobTX7165e/vf+DAAWlpad78JBJp5MiR0tLSaWlp3MSsrCxzc3NJSUluyvjx4xFC4eHhDx8+5N7s3bt3t2zZUlxc3KBM8CeDcKfDfP78GSHFxuklJSW5ubmKiory8vK/ug4ZGRl1dXX4MUEQvXv3/uGn5i/l5eV1//59T09PQUFBBweH4OBgfX39FoY78vLyaWlply9fPn36NEIoMDAwMTHx1q1bv7TCnz9/rq2t1dDQaNVZUlJSU6ZMiYiI0NPTGz9+fE5ODh6k2UL4+2t+fr6KigqZTCaRSLihoqioKDIy0sXFhUajhYSEDBky5NOnT1ZWVmVlZQsWLDh9+nR+fr6CgoK/v39WVpasrKySktKjR4/c3d3xR92+fftiY2MFBQU/ffrk7u5uaGhYUFDg7u4+ceLES5cu3bp1i8ViKSgoNAh3mkcmk0tKSjIzM319fRFC3bt3b/m5DAZjy5Ytly9f/vLli5iYGHf0ye3bt7du3bpmzZqVK1f6+vra29uPGDHi8OHDV69eLSoqio2Nrampwc0z9vb2u3fvdnV1/euvv7jFOjk5lZSU7N69Oz09XVtbW1xc/MSJEwkJCSYmJjt37vz+/Ttu8wgICCgvL+/Xr195ebmSktKlS5csLCwWLlw4a9asz58/h4eHnz171sHBoeW38ytQKJSCgoLdu3dLSEjcuXOHSqU2E9zTaLScnBwGgzFmzJiBAwdqa2u3Z1UxJpOZkZGxc+fOrVu3chMzMjKWLVu2ceNGbkphYeHnz58FBQVfvHjx6NGj27dv4/TY2Nh79+5x+0qwzMzM79+/u7i4nDx5Eoc7RUVFa9assbW1LSgo+PjxI29gB/5kMFS5Y2RnZ0+bNg0/dnJyWr9+/apVq9asWfPixYuFCxcOGTIEf2b/aubm5oP+MXjw4JcvXzaT+fr163gk4K8zb968sLCwqKgohNCbN29qamoOHz7cwnOFhIS8vb0pFAoOPg4ePIgQ0tTUxEdLS0tXrVp16NAh/NTf39/JyWn58uUJCQnHjh1bu3ZtdXU1PnTy5MmlS5cuW7bsyZMnCCFPT09HR8cnT56sXbt2+fLlr169evv27fLlyzdu3Lh161ZjY2NdXd1du3YxGIyfu+UPHz7k5ORQqa344oGDGysrKycnp1WrVmVkZOB3eVdX123btjk7O/fr16+4uBghpKKism3btnfv3j179mzZsmVnzpwxNTX966+/0tLSJCUlL1y4oKmpiXuX8vPzQ0NDQ0NDo6OjVVVVceKIESM+f/4cExMjIiKSl5dXUFDQfONQY0JCQnFxcYcPH5aUlExNTZ01a1bLz2WxWLq6uo6OjpqamosWLeKOrj19+nRdXZ2Hh8fbt28bzMc+c+aMtLQ0jnWawWAwBg0a1Hxlvn37Vltbm5OTc/Xq1cmTJ+P2JAUFBSkpKSMjIxcXl+YnRv1qgYGBo0eP7tOnz44dO0pKSvz9/d+/fy8nJxcUFMSbjUKhFBYWLlu2rLy8HM/Hfv369bBhw9TU1Hbt2lVSUoKzff/+fd26dZ8/f/6ldX706JGDg8PAgQNtbW25vVEIIW9vbzy6i+v8+fM6Ojra2tpz586dNm0aNzJbunTp/PnzG3T7GhkZ7dixY/fu3fPnz8cpJ06cePHiRWBgYL9+/XR0dPbv3/9L7wv8LqB1h4+4uLhp06atXbt23759CxYsuHHjBo1Gu3Dhgr6+fltd4tq1a3gAJkLI3d1dVFS0f//+Dx8+lJOTmzVrVnh4eKua/X/aoUOHTExMnj592q9fP3d3d+5HfmN1dXVHjhwJDw/fsmVLdnb22LFjJSQkmsr8+vXrT58+jRw5UkZGplX1efXqFZvN3rVrV1VV1aNHj1p1bjPCwsJsbGzk5eVlZGSqq6tnz569bdu2L1++9OrVKygoSEBAgMlklpSUnDt3zt/ff82aNaNHj87Pzx88ePDw4cPfvHlz4sQJf3//Xbt2RUZGGhkZvX79urq6+syZM2Qy+fTp0xEREfv27XNycuIOGmgJOp3++vVrWVnZysrKIUOGHDhwoLU3VVFR8eTJExaLVVtbi/t9du/e3a1btxEjRrx7927t2rW4C8DAwEBERGTmzJmzZ8/GJ5qamu7btw//gi1atAiPivjw4cOrV6/27t177tw5FouloqLCvRCbzZ46dWprq4ft2bNHVFTUy8vr506vqalZvnz5vHnznJ2dcUpxcfGcOXO+fft25syZnJycDx8+/ESxdDo9JSXl8uXL3NeEi0wmf/nyhSAIEokkICBgY2MjJyfHZrNv3rxZXFzMZDInT57MZrPt7Oxu374dFRWFo+r2kZaWxmAw7t696+zsnJ+f7+npSSKR+vfv7+fnJyMjQyKRysrKXr16RaVS9+7da25u7uXlJSoqKikpqaKiUlVVZWhoaGFhcePGDQkJiWHDhhkbG3M7ery8vI4ePTpnzhy+7c3/0b+6O3sj5IocSx0ddzr+P3EMuhl/E8X/O7/r30/9kJ/fTr+/n8xH6yrXrdu57l+F26MP6ENiSiJK+SeVgpArItDfPeun0KmW1LOuri45OXnMmDHcXjbQxUC4w4eOjo6mpub58+ctLS0nTpyYmJg4cuTINpzNWFtbu2fPniVLlpw48XeKlpbWgwcPdu7cuWfPnurq6sOHD1dVVXE4nKqqKhqNJiQkhLMxmUxHR0cTExMTExMymVxfX19XV0en01NTU52dnU1MTJycnHDO8+fPBwcHy8nJHT9+nLefuwEcjqirq/v5+RUWFo4dOzYuLs7Dw0NKSsrLy8vDwyMqKurUqVN9+vRJSUm5e/duaWnptGnTMjMzV61adYJb+0YOHToUEBCQkJAwadKkVr0yxcXFo0ePFhISOnz48IIFC/z8/H58zr8JCAicOHEiJiYmPz+fm1hTU4PbcgoLC4cPH75+/fqxY8eGhITs3LnTzs4uNDQ0Pj4ef3COGzfu4sWLVlZWERERubm5CKEtW7acOHHC1dV1zZo1Ojo6+vr6dDr9wIEDFy5cOHHixNKlS9XU1OLi4lpbTxaLJSUltXfvXiqVqq+v39q4ECF0/Phx3M3n4ODAYDCEhISioqKePHni7u6ur6/fwna46dOnh4eH48dsNrtv374eHh64yYTJZDazlh2TyWQwGLzrpvAVEhLCN3BnMpm1tbWCgoLNnMtgMKqrq6lUqri4eH19Pe7MioiIePr0aV5e3ocPH5KTk4uKih4/fow7sxBCR48ePXToEIlEwmN3yGQyhUJZuXIlHrXj6uo6YMCAzMxMPIX7wIEDgYGB+/bt+/79e1ZW1uXLl9esWSMqKpqQkDBjxoxjx46dOnXK1NR0xIgRHA6nT58+o0aNCggISE9PNzAwcHFxCQ4ONjAwaP722xYOsjkczrBhw2RkZGbMmIEQWrlyZffu3SsqKshkMpVKZbPZOB0htHXrVnNzc1lZWW5TSmhoaFJSUmZmprm5OW8biZOTk4qKSs+ePVtVHxaLVVNTQ/pndJ6AgEBISIiQkBCdTo+IiKiurkaa//WW2xZ+HQiCkJGRERQUJHhGGpLJ5IqKioCAgNTU1AEDBnRcHcEvBOEOH7Kysn369Lly5UpCQsKGDRtSUlKUlZXxgDiEUFRU1OfPn3v27GlgYHDp0iW8egQeP8FgMCZMmIAnOLx58+b+/fsI8Wn8d3Fxqaurmzt3blMBg4iIyPHjx0+fPs1isUaMGIEnMjAYjLlz54aFhZ05c2b06NGJiYnnzp1btmyZiIiImJjYiBEj1q5d279//8mTJ6enp8+bN2/atGmxsbFr1qzB4y6bcfnyZQ8PDx0dHRERkQkTJnh7e58/f/706dNPnjypr6/Hb2e9e/ceP378mTNnFBUVSSQSHn/alEOHDo0ZM6a1sQ5CaMSIEX369Nm1a5eCgsLatWv9/f1bWwKTybS1tZ02bdrdu3fDwsJw4vDhw8lk8ubNm4ODg3G3UU1NzZw5cyZOnIgQGjlyZFxcHL7NN2/e7N69+/DhwxkZGUeOHOEWO3bsWISQrq6ug4MDtw1s1KhRra0eF4lEqq2t1dHRwRODf8KNGze4o5oEBARSUlLmzp2bkZGhqanp7e3NHe7QFNwgNG/evLt37yKEevTooa6uzmazx4wZk5aWtnLlyoyMjG3btj169IjJZCYkJBQXF5uYmHDDspCQkOXLlycnJzdVPofDuXLlytWrV1NSUuTk5GxtbfFacNj58+cXLVrk5ubW1Ok1NTUuLi5FRUUZGRkODg64fcjMzOzq1au45wUhNG7cOCaTSaFQUlNTi4qK7O3t6+vrnZ2dBw0ahHua1NTULl++HBgYyOFwhg4dij/Grl27pqOjM27cOLwCDZlMJgjCzMyMSqWyWCwqlYpf1Z49e2ZkZOBvFCQSSVpamkQiOTo6rlq1qr6+vr6+fs+ePe28oMuoUaMa/8qRSCQLCwu++cePH8991+LS09PT09NrkCgsLNygO6kl4uPjp0yZwjvBisPh4L+jphqnx48fLyMjw+FwKBTKp0+fnj59ihAyNDT84TB2giDodDqFQuG7+0SrMJnMOXPmDBo0qMHygyQS6fTp0x07eBH8UhDu8IcDf19f39WrV3fv3h0/LSsrmzNnzqtXryorK/v27Tt69OgdO3a8f/9eQECAzWbjOZB9+/Z99OjRp0+f7Ozsvn37hsOdpKQk3reYL1++GBoaDhs2jJvy5s2bUaNGpaWl9e7dG1+dQqH4+/vLyspOnTr17t27ZmZmy5Ytu379+qVLlyQlJadMmXLr1i0bG5vDhw+/f//+8uXLxsbGioqKeILuli1bpk+ffv78+du3b8+bN+/58+dNfabi96atW7eWlpaOHDkSIUSn0wMDAyUlJQ0MDISFhW/evInf09PT02/fvi0vL4+/KDdPXl7+p0c24HtftGjR4MGDidZPWiMIQkpKSkhICH95JZPJLBbLwsJi7ty569evnzx58sKFCxFCJBKprq4OB214BIOgoKCLi4ubm5uHh4e1tfWRI0e+ffvGLRYPhXn9+vWNGzc2bNggLCzMTUQIsdnskpIS3Aaem5tLIpGUlJQarI3WQEFBQc+ePWVkZPLy8pSVlbnpaWlp8+fPNzAwQE1/kuJl9fFIi4SEhPPnzw8bNkxfX19WVnb58uWampo5OTkSEhKLFi2aN2+ej4+PrKxsQEBATEyMs7Mz/gVDCL18+XLevHk3btw4fvw4QkhdXd3c3NzR0TEsLOzFixe7du2aNWtWbm7u6tWre/ToERgY6OXl9eDBA264IykpWVFRcfHiRdREwz+JRFJRUfnrr79KSkrU1NRwcwuXkJDQkCFDFixYsN3777nQLBarqqoKfwLl5eWxWKzDhw/X1dVpaGjU19d7eXkRBDFs2LAvX77MmzcP/97iYdq1tbUEQZw5cwaXw+Fw2Gz2p0+f8NOxY8fiPz1u4rx58/DvAPon6BQVFcWDuoqKir5//75582aE0Lt374SEhAIDA8eMGaOurn79+vVTp04JCgqSeKYaMplMZWXlK1euNPODbkOdbaGHYcOGRUdH8/6ec18cEr+6cjickSNHct9Avn37lpKSQiKRRo0aBf1HoB1AuMNfTU2Ng4PDhw8fjI2NV69ejd8omUxmdHT0kydPKBTKsGHDLl68mJmZOXPmzDlz5qSlpSUnJ69duxaPB1y6dCmbzV69ejWebZCXl8cteceOHR8/frx58ybv5URERHR0dBYtWrRgwQKEUFVV1dq1a+fOnVtfXy8kJBQbG2tmZvbs2bMNGzZYWVnV1dXRaDQWiyUuLi4tLf3+/XttbW0ymXz48OFNmzZ9+/YtIiKCRqPhDxgNDY1mOkpwPJGdnR0bG0un0xMTEydMmCAoKGhubm5iYrJkyZL2X5CUzWZLS0ufPXv23bt33BoihKqqqk6fPm1nZ9fMuTgz7/+RkZG2traZmZm5ubmPHz/OzMxECEVFRYmLi1+8eHHq1KkkEsnMzExaWrq8vFxdXR0h9O7dO39//zNnzhQXF//11194DLKNjU1ycnJ1dfXnz5+nTZvm4+NDoVCsra3v3buHe2SmTZsWFxenoKDg4+OzZ88eX19f/KPkq7KyMj09vaSkxMfHR1tb28bGhnuotLT02bNnCxcuRF+bvM2cnJzNmzdzOJy3b9/Ky8svXbpUSUlJXV09Pz8/PDx89uzZHA6HTCaz2WwymTxy5EgKhYKX1OPdT2DAgAENmv0OHDhw4MCB+vp6Go2GP65YLBZe/w1niIuLKywsvHPnjoCAAI1GU1ZWvnfvHjJGu3fvblzJPXv2CAkJDRgwQFtbu7q6+tKlSw1aBM3MzAICAhBCu3fvJpPJRUVFz58/z8vLQ/ZIxe//I4f+9Tp0yNpCdxG6ixBCaDqfg0/R0z92CxRZWVljY+OfPl1eXh4WxQHtCcKdJrFYrP379w8fPvzly5d41VEc9OzZsyc/Px+3x+KcJBJpxowZwsLCCgoKVCr17t27KSkpBEGcP38eoQ2IZ/5tSUmJp6fn2rVrucNxMFVV1aamYtFotCtXrnh5eVEoFPy1nu83J4zD4eDp69OnTz906FBaWhqHw6moqCAIAiE+Z+EVTZ49ezZ06FAOh7N9+/ZJkyZFRkba2dk9ePBg69atixYt2rJli5qamqSkJJ1Od3R03LFjh6+v78OHD3FHRm1t7bhx45YtW8Y7KTc1NdXa2trX17e13T2HDh3y8fGxsrKaNGnSpUuXREVFfXx88DDekydPbt26FXc/NSUoKEhLS+vUqVN1dXVPnz7V0NB4+vTpokWLTp486e7u/uDBg4ULF+rq6uro6GRkZAwZMkReXn7cuHF9+vTp1q1bYWHh7t2716xZU1lZyWazTU1NCYIQEhIiCCIrKwsHYRISEtnZ2UwmU15ePicn5/v37xkZGRoaGllZWbdv337z5s23b99mz579+PHj0tLS169fo79Xqv2XFy9eUKlUV1fX0tLSwYMHM5lMbh4cjpw7d27w4MEonM+5CKFHjx6NHDlSV1e3vr6+tLQU/TN2586dO3Q6vaqqSllZmTtoGg+55Z6Ln+LugMTERE1NTRwG4aMkEonD4WhoaLDZbLxgiZqaGnf6N0IoJyeHTCZ/+vQJ/+ZLS0vjfj0++w/M+TsRX51vKx2JRCIIgpsTDz0RExMbfH1wg5xz585lMpl4TZ0mfvI/j8PhyMnJ4RV6ftqvjni65E55ALQzCHeaxGazhw0bZmRkhFs+Pn78aGhoOHjw4ODg4AMHDjx//pzbisBisQQEBCIjIzU1NVks1tixYwcPHvzp06fk5GRZWYQQSkhI0NXVpdPpZ8+eraysbOorEf5SjhCiUqkRERFOTk68K0Y4ODisWrWKwWDgLjMdHR3uoYCAABcXl/Dw8IKCAjExsZMnT65atUpaWvrevXtv3rxxcnI6evQo3yvu3LkTIcT9joWnba9bt66srGzgwIHdunXz8fG5e/fuu3fvBgwYEB4ebmpqGh4evmnTJm4rC4fDSU9PbzCDPSsr6+3bt1VVVa19zSdPniwgIIAnEu/cuVNfX5/b+bJq1SoqlYo/lvh283M4nCVLlixduhSvFz5t2rTdu3cTBIHTpaWlRUREhIWFs7OzT5w4QRDEmDFjIiMjBw0ahBCqrq4WERHZuHFjgwnhbDabRCLhHwqLxeJwOLi5i8Vi0en0b9++BQUFqaioTJ8+3dPTkzsUgEQiJSQkIISQK2rYjfhPyo/jgMbnIoRc0ebNmwcNGvT27VtDQ0O+ex/+cNliMpnM7X5tcIjNZhsYGNTX12dlZamrq+MRS80j7SThNrMfJrb8dAAAaHMQ7vBRWlr6+fPnioqKO3fuXLt2zcTEhMVilZeX19TUvHv3TllZubKyEiHk7Oycnp5+/fp1giAKCgqePn367NkzNpvt6Og4efLkM2fOqKioIFSDEOrfvz+dTi8oKHBzc7OwsOjfv3+DKz59+nTSpEk3btwQERG5d++etLQ0nU739PR0dXXF7RksFmvlypUkEun8+fMiIiLh4eE4JsBTxvDoE0tLy7y8vLy8vJUrV86YMWPRokXOzs76+vrNjLa5desWdzAyQRB4Js6dO3dYLJaQkJCnp+fRo0e5/VlGRka4V453sThhYeH37983WLrU3Nw8Pz+/Jbsi8CopKSkpKbl8+fLx48dxqwOdTsdBBkKIRCKJiIjs378frUZ8Vkp14peIEH7pNDQ0evXqhUcFFRYWtqpWzSCTyUZGRiwW6+XLl7q6usrKymPGjMF9Rjdv3rSwsLBIs+CdIIYQUvJRapDSFL45lXyUbuvdvo1uo9EovjKe74kt0vSihs5Rf0/5RrkIJf78Ff7YLh4AQOdE+onRoF0Vd0uHvLy8pKQkEokkKCg4a9asb9++lZeXa2pq3rt3Dw9EJZPJeGwEHluDvyXTaDQOh8PhcFRVVceOHZuZmfnw4cN58+zRP23R69atw2NdufNv8RWDgy8zmcxhw4b16dPnl95dO+yZ1dQmEs1fCD4aQRfQ1N5hAPxRkpOT8fyA2NjYTjU8C8Kd//sVO1hxy8zJyTE2Nt6+ffvcuXN/6RWbqUnjcKfNQbgD/lgQ7gCAEMJTXFHnC3egM6uhXxEEkEgIITWEMmxtUePlLdptfmk7XOjnLgGfEwAA0DX8xKKp7QP2zAIAAABA2+BdULRTgdad/4NuPQAAAKBLgtYdAAAAAHRxEO4AAAAAoIuDcAcAAAAAXRyEOwAAAADo4iDcAQAAAEAXB+EOAAAAALo4CHcAAAAA0MVBuAMAAACALg7CHQAAAAB0cRDuAAAAAKCLg3AHAAAAAF0chDsAANCWSCREIrUoEQDQbmCLUAAA+OV+xx2ISTv/DtCIHQR+zPugQR4unKfxKdyzmkoH4JeC1h0AAECIpwEGP2jwlDcPbztN48zNl9a4HL4X6nC8YQ0OXxo8QP+Oh/C/BoU0CI+aegxAO4BwBwAA/o4zCOL/D5rJw33MjU4I4u907om8KQ1Ka1BO4wedAW9Q0nwenI2bs0EkxJvI93SIeED7gHAHAABagTcc4cYxnaphpk20sPWlQaNOU/mbav4BoN3A2B3Q9cEQBNCGGjTVNGjs4T3aTEPR76L51p2mjnL/driPG/8NIp6uMQDaAYn4rf8Wf2d83wq7wPtjZ9M4vvn73daVQK4N4xXkSiDco/FPoNNU0NOgqZ/3/R38pho32zRosOF2dfF9iv7d2NNUngaJvBdqvisNgN9CcnKynp4eQig2NtbQ0LCjq/N/0LrTucDbXJtr8OWymTwIIRwAIcRnMCYGQxC6sMYhTuO/R75NO00V1Uyepsppz3eA5v8oOif4KwM/DcbutBpM3+hYrX39/9+W88+hxgX+ncH1/++kzQxBaJAT/C54f2Ga+deqzJ3hHwCgJaB1p3Uatzk39fGJD5FIfBquEb9W68bfLPmW0+DBn+YnXn/kihDiNtvgjqp/5cddV39nawB3eLkSJFdEEP+0ErniqxDcMqEbC/y+buvdZrFYBEHQ6fSxY8fyHjp9+rSXl1dkZKSysnJHVY/rd2yLAp0KhDu/Cu/HMPdTuUGIA36d/4eYeJAN91u7K8Lj1UiuCLn+PXTt/wOWcbYdCKF/xvHsIEiu/xTKHdlD+tdZ4DdSU1PL4XDwY2FhYRLPH2pNTQ1BENzE2tpaNpvNZDLpdLqQkBCJRGr8HYObUl9fX15eThAEjUYTFBQUEhJqh3tpk6adGTNmlJaWIoQOHTrEG+5UVVVdu3atsrKyoqKiDS7TrPr6+vr6ekFBQSqV2iCRTCYLCwv/6gqAPwEMVW6dploX+D5o6vRm8vN93KpLdG38XwpX+NrXEIRijeHfmfHjJ9y7d49EIgkICBw9enT69OkKCgoIodevX0+aNKm0tPT69esmJiYIoZEjR75582bo0KEPHjyIioqaNGlSU+FOdXW1tbW1sLCwrKxsWlpaSUlJampqu93RT78P4PaSqvVV79+/9/LyOnXqFO/RuLi4V69eqaurm5mZ8aanp6c/e/ZMWFjY1NRUUFAQIVRZWXn9+nU2mz169OjevXvX1dWFh4eTSCSCIFgsFovF0tTUHDNmDEKooqIiIiKCwWBQKBRNTc1Ro0YhhGpra2fNmhUVFbVmzZpjx47hq9TW1lpYWERHR4uIiMTExIwbNw6WJfxdwFDlLuL/XSRNpPB2NvFmaFBIgxMb5GnYHfNHRjZ88X/9dzZ9AgD/tmDBAgUFhSVLlpSXl5ubm/fo0cPU1BQhFBQUtH//fiUlJQkJCZxzxYoVdnZ2R48eff/+/bJly7y8vBCawLfMu3fvJiYm5ubmysrKnjlzZteuXQihDx8+EAQhJSWFEKqqqiIIQkVFBefPy8urrq5WUFCQlJRECOXm5srJyQkLC7PZ7K9fv9LpdIIgSktLRUREqqurORyOgICAiooKbvmoqqoyNTXt1q1baGhom7wgIiIix48fx4ELr7Nnz167dm3w4MG84c6mTZtCQ0O/fv1aW1u7adOm/fv3s1gsJyenoKAgCoWirKxsa2u7atUqR0fHoqIiEok0evRoERERAQGBGzduIIQcHR2DgoIuXbrk7u4uLS197do1hNCiRYs0NDRSUlJ27dq1cePGgwcPIoQSExMVFRWfPXv24cMHb2/vcePGtcnNgj8ZhDut9qdN3+hsmlp5toVf+xqMs/mJYTdNlXD58uW8vDwGgzFlypRu3bqdO3dOWFhYVFR08eLFCKGbN2/evXt3zpw5ffr0adXlfsjAwODmzZupqakDBgxAMMThR7S1tbOzs/X09Orq6mg0Gm/z9tu3b3V1dTU0NPBT/HpKS0sPGjSoqqpq4sSJCHEaF/jixYuZM2fu3btXVlYWn4V/4teuXfPy8qLRaBQKJSUlxdHR0d3d/cuXLz4+PtevXx8yZMibN28iIiKkpKRcXFzGjh27ePHir1+/KikpDR069ODBg+7u7lFRUcuXLxcVFU1OTjY1Nd2yZQtCiEKhaGtr4ziprXTv3r28vLxBYnh4uJubW2BgIDclMzMzPDz8yJEjurq6w4YNe/nyJULIx8cnOjq6pqYGIWRnZ3fr1q0tW7YUFhb279/f1NR079695eXlU6dOzcvLU1ZWfvny5ahRowoLC8ePH6+lpYWLLSkp6d+//927dwcNGuTi4jJlyhRZWVkbG5sJEyY8f/68tLQ0ICBg/vz5bXi/4M8E4U5DreoOh2kR7YPv6kSo0apuXNu3bw8LCyOTyaNGjfLx8WnVtZKSktzc3GRkZF68eGFubo6/a7bE9evXL126JC0traysbGdnl5qaGhYWNnz4cPzh9/z5cy8vr1GjRrV5uAP90a1CoVC+ffv26dOnQ4cOMZlMaWlpnF5YWOjn5ycpKens7MxgMKqqqsjk/09cVVRUbOp1jouLw5ETfjpmzBjccePk5JSRkeHp6enp6Tlq1Cg1NTWEkK2t7a1bt16+fDlo0CAnJyczM7OkpCRLS8tPnz7hq6xcuXLChAl6enpMJjMqKmrPnj1iYmK7d+/29/fH4Y6QkNDp06fb9jWxtbX19vZunI47pLhPb9++XVZWNnPmTIRQcnIyg8FACJHJ5G/fvjk4OJBIpJycHB0dHe65eAgUnU5/+vTpy5cvlZWVz5w5Y2dnd/DgQTExscrKSgsLCwEBgcLCwvLycnzLFhYWWlpavr6+lZWVvXv3HjJkSG1t7Zs3b1RVVdGdtr1p8MeBcAf8fpr/fK+oqBARESktLf327dvq1atbVfL9+/cdHByCg4M1NDRGjBjx7t27lp+7f//+S5cu3bp1a+DAgQghb2/vK1eu7Nu3DyHEZrPV1NT27NljbGzMe0p9fT2eEcNisUgkEoVCQQhVV1fv2LFDRERkx44dZDL58uXL3759W7JkyY4dO8rLy42NjadNm4YQysrKOn36tIiISFZWVqvu8Q8nLCzs5eXl5eWFEAoKCsKDc7ds2ZKbm+vh4eHr66uoqFhUVPT169dly5Zxz+rRo4eEhESjFpCGAgICnj9/jhAyMTExMDAwMTEpKyvD8S728eNHCoWCe7WUlZXDw8MRQgICAteuXVuyZAmFQlFSUrp79665uXnb33nLZGZm+vr67tq1i06nN5NNRUUFR0JsNltOTs7X15cgiKYiQhKJVF9fjxD6+PHj27dvqVSqvb398ePHjY2NDQ0NxcTEJkyYMHz48Nra2n379tnZ2Tk4OBw+fPjx48dubm5kMnn//v3tM/QbdG2w7g5/BIFHINbExcXHxMTm5eXjlPb85+3tIyoqZmtr9x/LQU3HB1u3OisodE9Ovk0Q6NKlYIRInp5e7XmPz5+/eP06jSBQTs4H3orl5eVHRka9f5/zEz87cXHxzZs3y8nJIYSmTp3aqnMLCgo4HI6ioqK0tPTjx499fX1be/X169cjhDIyMgiCUFJSwh0ce/bsWbduXYPBFuPHj58yZYqFhYWsrOzo0aPz8/MRQjo6OiYmJikpKYcOHdq1a1dlZeW5c+e2bNliZGREpVJZLNaMGTMyMjI+ffqkp6cnJye3YsWKSZMmtbaSf7Kqqqr169czGAxzc/MLFy5UVlYihK5fvz527Nhly5Zdu3Ztw4YNp0+f3rt3L+6gwU0UFy9ebNzdg61Zs0ZRUTEtLQ0hNG3aNBkZmYCAgCFDhiCEVFRUBAQEeDNra2sTBIHLfPfuXf/+/RFCNBrt5s2bz58/LykpcXNzS0xMxNdls9k44M7MzOSdsvT58+c7d9qsreP79+937tzJzc29efPmtWvX9PX1T506xWazU1JS0tLSqquro6KiwsLCvnz5snDhwmnTpomJicnIyIiLix86dAghtHDhQmNjYzExMUlJSV1dXU9PT4TQuXPnXr9+ffjw4Y8fP9JoNCqVeunSJYTQvXv3Fi5cuHjx4uLi4i1bthgYGCCE9u/ff/LkycWLF69fv97MzKxHjx4IoUePHhkbGy9YsGDDhg1mZmazZs1qq/sFfy4C/BtCBPdVcXJywq/S6NGj8STV9nTixAkSiRQeHt7aE9lsdkVFRXl5OZPJZDKZCBF1dXW8P+o7d+707NkTIQL3vk+bNo3JZA4fPlxVVZWbp6qqqrCw8Pv372w2myCI4uLiioqKiooKnFhdXY2z1dbWVlVVrV279vjx49xzS0tL7ezsjIyMzp07d/z4cZy5qKjI0tLS0NCQ+0Xw7du3MjIydnZ2eXl5ffr0wa88/nfmzBmEkJubG++PoynIFSHXhpnwB0l+fn7jzM08JQhCV1dXTU1t6tSpUVFRTV2O71MWi2VmZqajo3Pu3LmpU6cWFxcvX76cm23BggUTJkzgPVFRUfH06dN1dXW9e/eeP38+TgwLC3v48CFBEGPHjnVwcMCJsrKyXl5eBEHgkCgjI2Pjxo2KioqfP38mCOLjx48IoZSUlGZeDUD886f99OlTbW3toqKiBw8eUCgUf39/giCGDBmyefNmgiAKCwvFxcURQkFBQU+ePEEIvXjx4sOHD0JCQnv27Gn8q4hT8JymiRMn3rhxY/z48X379iUIIjs7W15eXlZW1sLCwsTE5N27dwRB3L9/X0BAYOrUqcbGxiQS6fnz5wRBlJaW6urqKigozJkzR1hYWE5O7sGDBzig0dLSsrGxMTExSUxMxFesr6+fNGnSP51rP/7raO4FcUXIFYWFhVGpVAkJib179wYHB4eEhERERBAEkZCQEBQUdOXKlcDAwHPnzuXm5hIEUVVVFRISEhISEhoaWltbi8uprKy8fPlyUFBQdnY2TsnOzg4JCQkLC6usrCQIIjg4+N27dzU1NQwGo66urq6ujslk1tfX41nuNTU19fX1OL2mpgYnVlZWlpSUlJSUFBcXf//+vbCwENe2gMeXL19w+aBTSUpKwp+bsbGxHV2Xf4HOrCadO3fu2LFj165di4qK8vb2rqqqatCg+uzZMy0tLRERkYyMDBKJ9PjxYzabjT8scYbS0tLQ0FBVVVU8Ga+6uhp/vx8xYkRmZmZxcTEeHEChUJhM5rBhw96+fYv/5s3MzL58+YLbeIcOHcq94qtXr549e8ZkMkkkkrS0tLm5OYlESklJefr0KYfDoVAoMjIyM2fOTE9PHzBgAI1GW7lyJYPBQOi0i4sLQge45Vy4cGHw4MG5uYjJZCKECgsL9fX1v3z5wq35vXv3DA0NJ0+eXFFRMWXKlPXr1zs7O+OmDhMTk6KioqqqqpSUFITQ4sWL7969O3nyZF9fXwqFsmLFCoSQk5NTdHT08OHDFy5cqKurq6OjM2jQIEtLSzExMTk5uZMnT7LZ7EWLFqmrqz969GjQoEFlZWX9+vXLyPj/a6umprZ8+fLNmzdv3972P9nm3b59+9y5c2FhYevWrSMIYsqUKS08kUKhDB069MmTJ5cuXfr48ePly5dZLFYz+fX09O7duycnJycuLs79KX/+/NnDw2Pw4MEZGRn9+vVr6lw2m62srIy/B4NWcXR0fPPmjZmZ2e3bt2/duuXh4SEuLh4ZGTlx4sT9+/fT6fTp06ez2WwWi3Xnzh1LS0tPT08mk7l///7Vq1c7O/Mvc8WKFQICAuHh4UFBQWZmZsuXL0cIsVis0aNH0+n0uro6bs6RI0d++fJl1apVVCq1pKQEh1aSkpLJyclRUVEUCsXX17e2tlZcXDwxMZFCoSQkJNBoNHFxcW7XEo1GW7VqVXx8fGtv/MGDB2/evOFtJcJKS0uPHDmCEOrWrRuFQiEIoqam5ty5c3gVHDabLSAgQKfT7969e+vWLTKZTKVScUNjQEAAQRAIITKZjNclun37Nm6aolKpgoKCBEFcunSJRqOlpaVZWVnhvtqWYLPZ8vLys2bNYrPZ/0+VRwghPOsNYzAYM2fObG0LLvhzdWy01QnhL0x1dXVjx45VVlYuKioqKSlJTk4mCGLz5s3Dhg178+YNQRDr1q1TVVWtqKjYvHmzgoJCnz598FpYOjo6uB3o6tWrffv2lZCQUFBQCAsLIwgiLy9PRkYGIaSioiIuLi4tLS0tLS0jI0OlUtXV1YuKitzd3RFCU6ZMqa2t/fDhw4EDB4SEhL59+8at26ZNm3r27Pno0aM9e/b06NEDj/xYtGhRv379EhMTHR0dxcXFS0pKqqurY2JiunXrJikpGRoaihCRkZHB/VHv2LEDIfTo0SOEiMePH5PJ5LVr1yKEtmzZsmLFCgaDQRDE+vXrp06dWlhY+PTpUwMDA4Ignj59ihAaOXJkTU3N169fNTU1L1y4QBDEmzdvbty4UVJSsmnTJnV1dXyJGTNmzJo1q7i4GP3T6hATE4MQwhNN5eXldXR0cE6c5/bt29xXvsEX1la17rBYrOLiYpz4c607VVVVu3fvxg1a1tbWRkZGfC/X1NPPnz8rKyuLiYnJy8sjhJycnLiHHBwchg0bxnsi/sQKCQl58uQJh8MhCCI0NLRPnz6pqakEQYwYMWLmzJkFBQUEQcjLy+/atYv4p3UnOjo6Nze3f//+3t7eeXl5c+bMQQhZWFg0eDVAA/h36fHjxzExMU+ePGlwNDMzMyoqKiEhgclkNlPCD1PaSnx8PIlEevLkSTPtyq1q3XF0dKTT6dI88K+KjIxMt27dunXrJv0jEhISmpqa4eHhpqam165dC2+xGzduxLVGTEwMbuP8183CL/ZvAlp3fjN0Ov3w4cN4fYtVq1bhr2vdu3d/+vRpWVkZQsjIyOjIkSMkEqlPnz4qKiqvXr2Ki4tLTU3Fi1VYW1t7enpOmjTp+PHjN2/eNDAwuHz5sqWlZVFRkba2tqCg4K5du+zt7fG13NzcDh06VFJSMmvWrJCQkKCgIEFBwZ49e27cuPHz58/V1dXcWpHJ5Orq6sePH9NotLVr19JotPT0dG9v75ycnF69evXo0ePChQsuLi4nT540MjLicDjHjh2zsLBACGlqauIS8vPzw8LC/Pz8RowYgVMIgpgyZYqIiMiuXbsWLFiAE/X19bt16zZx4sS0tDRtbW2E0NChQ62trUtLS4WEhISEhCgUSnZ2NkIoKytr586dpqamjx494k5OERAQSElJ2b9/v46OjqKiIvpn9lBpaSmNRps7dy53rm/bKisrmzJlSkhIiIqKCpPJHDt2LHfeDULo/v37GbwtSPycO3du9+7dJBKJSqW+e/fOwcGBeygpKSkuLs7GxqaZ03v06EGj0VxdXW1sbKZMmeLi4sIt9tWrV0+fPrWxsenXr5+tra2ysvKqVasQQm/fvs3Pz4+Li3NwcLCwsEhOTh44cOCGDRvGjx/PYrGCgoLWrl1LpVIfPXpEEISEhMTWrVvfv39vbGwcFhbm5eX16dMnRUVFHHP//Av3C3A4nLNnz1ZWVvLOb8KYTCZ3XeOmMJnMgQMHNljgrk2MGDG8iSO9Eer9w9Ob2rTkF9BHiDO8qcq2nru7O/5CxYXXLCheVdzyQspR+czUmWgIikiJaLOaNSXul18B/FEg3GnSiBEjXrx44e/vf+LEifDw8Li4OFtbW0dHR/zJjUcg1tfXz5s3r6Sk5Nu3b7q6urq6unfv3g0JCTExMYmPj8/Ly9PU1KyqqkIIPX782NLSEiHEYDDs7e25sQ5CaOXKlTdu3IiPjxcWFn758mWDsY28Nm3a9Pz585MnT5aXl6uoqMydOxe3D3/8+LFXr169e/dWVVVtvgMlIiIiKysL1wQjCEJbW3vixIn4sYCAQFpamqmp6ezZs6Ojozdv3vzgwQO+RQkKCqanp1taWl69etXExGTDhg2BgYElJSXS0tIUCiUzM3PIkCHbt28XExNDCElJSeE1TtatW+fr63vixAlNTc2JEyfi+beZmZnS0tII/b/vJiQkZM2aNTt27EBoSQt+Vn/DI4HmzZuHW0GWLVvG2/+YlJS0fft2/ntj/cPOzi4xMdHHx4fNZq9duxb3zWEvX7709vbmholNSU5OxsM24+Li8L0jhEaNGjVy5Eg6nV5bW1tfXy8rK/v58+c+ffrExcVVVFQICgp6e3tbWFjcv3//8OHDq1atEhQUVFVVRQhVV1cXFBQ8fPhQTEzs9evXbDbb3t6ezWY/f/6cSqUuXryY+GfcK4fDefHiBbcavI/5otPphw4dunPnjru7e0pKysWLF3ln4uB5NPX19aSf/TAnCOLTp0//6oz4R/fu3cXExIimxs8jhP5ZVPdXhDsAgD8ThDtNunPnjpyc3Pbt2ydPnjx69GgXF5eAgAD072VOGn9J5X78EASBZx8QBEEmk3kno+ISuDmlpKROnDjh5OSUmppqb2+PlzfFR3ETHPfpo0ePfH19lZSUQkJCZs+evW/fPnd399GjR2/evPnRo0fnz59PSUnBX+AqKytpNFpQUNDMmTMR+nuVWDab7eLiMnfuXBEREZxy4cIFhFBAQMCGDRsePHiQnJy8YsWKyZMn19fXd+vW7dixY8HBwVJSUlu2bJk9e/aLFy+qq6s/ffpUVFSUn58fExMza9YsTU1NR0fHmJgYDw8PDofj6+u7YcMGPO/67du3+JRZs2aNHz8+ODjYwsLC3d193LhxLi4uOMDavXs3QmjRokX4heG+RCUlJQUFBYWFha39qS1fvnzOnDl4ooecnBz3B0Qmk/X09Pz8/BZ8WtDgp8b7VFRU9OrVqxwOh8Ph4LY07o97+fLljo6O9fX16BXCISydTr9y5QpCaOnSpdwSBAQE8MgPGo2GO8VwIm/cQBDEvXv38vLyTpw4gXM+e/bs9evXGzduJAiCRqNxOBwWi0WhUPLz83Nzc/X19SsqKjw8PPBKJz/gihBCeGZQS0yfPr1xIh69rq+vj+Oen9PgrjEmk2lmZqaurv7Txf60ZuOrPxFsyAD+KBDu8FdaWmpsbOzi4oLHryCE9u3bJyQk5ObmNnfuXD8/v2fPniGE5syZEx0dTaVSKysrk5KSXr58GRMTk5aWJiEhsWPHjp07d5aVlSkpKW3YsIFOpy9atCgiIoLFYm3fvn3nzp3r1q3DH/YIoVGjRvXs2ZNEIp09exanHDp0yNnZmUwmnzt3rra2dtWqVUePHn348OG2bdtycnLGjx9vYWGB+0piYmLWrFmzYMGC8vLy+Pj48ePH5+fnOzs74xW6Vq1ahdAFXKazs7O6ujrv5OqqqqoxY8bgATQlJSXdu3dPSUlxcnIqKCjA+wKuW7eOw+HU1tbKy8vHxsYihLp16yYhIfH69WsOh6OsrHznzp2ysrL6+no8sUVBQeHu3bvR0dFPnjwREBAQFhZ+8ODBo0ePxo8fP2PGjPz8fA6HIy4ujkdoIoS8vLzc3d3xh6KS0v9ff1tbWxMTEykpqX+6g1qkqqrqr7/+qqysFBMTO3r0KG8cQxAEnU6nUqloCeLt4UJO/36KEIvFGjBgwKBBgx4/fqypqSkiItKwHUIFOTo6on8WWENDUHp6eitqiRBBEFJSUt26dcNzf3BKRESEsrIyHjzORaFQKBQKg8HAQ6xa0tai5KOEEMLtWz9NUlKSw+FwW6cAAOB3B1uENsTdo2r16tUnT57EcxBWrVp14sQJhNCnT5+MjIzevn07ZMgQVVXVq1evJicnZ2dnL1y4kEajMZlMJyeno0ePIoRycnJCQ0M3b948YMAAPLqlrq7u8uXLuHuFzWaTyWRLS0vuRImnT59WVVXhndUQQq9fv37x4oWAgABBEEwmU0tLa/jw4XhixbNnz8zNzZtfBKzBHREEIpFQz569Dh06hEfzoKaXJP6Pbt26ZWxs/ObNG1lZ2YiIiCNHjjg4OPD2CjVTz6Y0X0/YNqEx+OIOAOgQnXaLUAh3GuKGOywWq66uDvco4TmZOENtbS2bzaZSqWQyub6+XlRU9OjRoydOnMjNza2urv67CQEhhBCHw6mpqcFzMjvqdhBPuDNo0OC7d++Kiopy038XEO60FoQ7AIAO0WnDHejMahKVSuVGBrx4R78KCAiUlpaePHkyPz9/+vTp/v7+3GExCCEymcy3hI5y/fr1BvXpVLHuT+/9Dh/tAAAAmgfhzn9FoVAsLS2ZTKaBgUGDUSCdDd6pBwAAAPjTQLjDX2s6esTxgsXHjv2y2vxn+HbacckQAAAAoBOBLUIBAAAA0MVB605DnWo4CwAAAAD+O2jdAQAAAEAXB+EOAAAAALo4CHcAAAAA0MVBuAMAAACALg7CHQAAAAB0cRDuAAAAAKCLg3AHAAAAAF0crLsDfhXuzp3EDgI/5n3QIA8XztP4FO5ZTaUDAAAATYHWHfBL8IY1OHxp8AD9Ox7C/xoU0iA8auoxAAAA0DwId8AvwRuUNJ8HZ+PmbBAJ8SbyPR0iHgAAAD8E4Q74JVrY+tKgUaep/E01/wAAAAAtAeEO+IWab93BjTp8h+80+J83Z4N2IAAAAOCHSARsiQkAAACAtpCcnKynp4cQio2NNTQ07Ojq/B/MzAI/CRpXAAC/KegZ/wNBZ1anQCIhUqPggW8iAAAAAFoLWnc6r9+imxG+JAHQcr9Rm2jjP23uQhK/td/oRwDaFoQ7/wlufSGIfzXDcJ/ieIV7iBu+8M3cTGkNzuItn/dCAIDO77eIGCAmAF0PdGb9vJbEHI3jHt6oBadzT+RNaVBag3IaPwAAgE6CzWZzOJyOrsXfkpOTDx482NG1AB0PWnfaA9/mHGiYAX+s36Lx4Ldohumc7t+/P3fu3HXr1jk6OrbwlBMnThw4cCAjI0NUVJSb+P37dxaL1b17d96chYWFHA5HQUGhJcXW1NRs3LixrKxs48aNLa8/6JIg3GkPDWKaBo09vEchBgK8fouwoCX47hDSITVpoaZe+db+RBrfpqenZ1RUlKCg4PTp0+fMmfP/nASxc+dOBQWFpUuXtra2nY2urq6kpGRhYWHLT+nWrdvevXsFBQW5KSwWy87Oztra2s7OjsFg0Ol0hBCTybS1tbWyspo3b15dXZ2QkBA3M4PBEBER4S2TxWIJCwtbWFh4eHi0xW2B3xuEOz8Pt9M0brnhHYvTIKXBuByE/t97xc3WIA9vIgRDncSvi0J+u7CgJdr85Xr9+vWjR4+EhIRUVFR0dXXbtvAfavlPhO+N19fX19fX37p1q1u3brzhzuvXr0+ePCkjIzN9+vQG7Rm/IyqVmpaWFhoaWl9fP2vWLAEBAYTQzZs3v3z5Ii8vb2hoGBQUJCIiMmPGjNLS0hs3bmRmZpJIJCr174+kqqoqMzOzmzdvqqqqCggInDx58siRIyNHjtywYUN8fHxiYuKCBQu0tLTevHmDEHr16tXZs2fPnz9/7Ngxc3NzGRkZhFBqaurZs2fPnj2LEOrbt2/HvRKgs4Bw5z9pPHSmcSzCt2mnqaKaydNUORD9dIg2iUIIgigoKJCSksJfUjttW05xcbG1tXWPHj1OnDghLi7e0dVBIiIiZWVlixcvNjQ0bP9w5z9atWrVqlWr9PT0GAwGN/Hr16/y8vKjRo1at24db6xTW1ubl5dHoVDExcVlZWURQhwO59u3b1VVVRQKRUVFhUqlMhiM/fv329rafvv2zdHRUUdH5+TJkzQaDSEUFBR04sQJgiCGDBly/PhxnOjn53fmzBkBAQFvb29tbW18IS8vr8DAQBERkf79+zc/0sXLyysxMVFUVPTdu3fu7u4IoXXr1i1cuNDOzu7ChQtnz55VVla+fPmyoKDgjRs3vn//rqioeODAAV9f32HDhm3duvXly5fOzs5eXl5KSkrBwcF6enpUKlVSUpJMJh88eHDTpk1iYmIIIRaLtXTp0ocPH9LpdFFR0fXr1+N0BwcHDw+PtWvXTp48Gd8OQujYsWN6enrz588PDg5evHjxlStX3r59a2JismfPHisrqwsXLiQnJ7fhTxD8piDcaU6rhgD/juOF/3uo1Gk/odvWr2hi8fb2/vr1q5ub2+LFi0+dOtXm5f+csrKy1NTU8ePH8ybW1dXdvHlTTEzMwcFh3LhxHVU3LjU1tQ0bNmRnZ+Mv9z8nIiKirq5u1qxZ3JT8/PyAgACEEG4eaApBEN7e3sXFxWw2287OTkVFpaCgICAggEqlMplMNputoKCwYMGCVlUmJibm2LFjxcXFUVFReEVazMHBgcFgfPv2rby8/PXr1wiha9eumZub29vbf/jwYcqUKRs3bszLyzt37pyrq6ulpeWIESPOnj07cODAZcuWBQQE2NnZzZo1S0VF5fjx44MGDVq8eHFeXt6ZM2d0dXXr6uqsra1DQkL69OmTm5t75swZBweH7t27Ozs7L168uKl6fvjwYcmSJampqQMGDJgxY0ZWVpaNjU1ZWVlaWhpCyMrK6urVq3FxcQghEomEEDp//nzv3r2NjY39/f2HDRtWXV3t5uY2adIkNze3urq61NTUTZs2nT17dvr06UJCQvv27eNeSFJScsaMGRQKZfjw4VOnTuWma2lp0en0wYMHT548Gac8fPjw3LlzWlpaFRUV3bt3x6/S9+/fv3z5Mm3aNGlp6efPnycmJrbqxwG6JAh3wH/SabtaMjIyamtr+/bty/0K+NN+UUgnLi7u6urKZDJDQkKOHj2KW/v/u9TU1PPnzx84cOAnbryqqurOnTuhoaHdu3evrq4WFxdXV1cvLy///Pmzr6/voEGDdHR0uJlzcnLKy8sRQsrKyt26dWOz2enp6YKCgpKSkjk5OcLCwtra2ty+icbev39fXFzM4XBERUX79u377t27iooKYWFhBoOxePFiGxsbW1tbeXl5hFBeXp67u3tQUNCuXbsWLVrELYHE8w3j8OHDN27cqKioOHXqlJycnL29PZVKPXfunLq6emRk5KlTpxQVFUtLSy9dukSn04uKip49e7Zr1y6CIMLCwgIDA2k0WmFhobm5eVZWlpSUFIVCQXP41BnLz8/fvXs3hULhcDiJiYnh4eGlpaXbt28nkUgDBw40MjLCMcqyZcta/srb2tpOmzbNzMysqqqKm7h+/Xo2mx0SElJUVLR//36cGBAQMGDAgM2bNxcUFLi6us6bN09DQ0NbW1tDQ+PSpUtkMvnZs2dMJhMhdOfOHYQQjoH09PRwnOri4qKiorJs2bLa2tqLFy/evXu3T58+Hz586Nevn76+fmVl5ZcvX+7fv99UPQUEBP76669Nmzbt3bt3w4YNuCFqz549165dw0dtbW1v3rzZ4Gc0b968hQsXenh4kEgk3p+akJCQqqpqy1+lZuzYseP06dMVFRWrV69GCImJicnIyODZYcbGxteuXSssLJSTk2uTa4HfFIQ7P9Yle4vaqi0qMDDw8ePHUlJSzs7OeCwhl/f/2DvveKrf//9fx3HOscnelKRISqGBECIjEcrMTJGWhqSM0JIRLaMhFZWGkVVGRoqKMovsvfeZr98f1+dzvn7Cu/len3O/vW+39zmX63W9rtfrdTrX81zP8YiKevv27fHjx4WEhH7NyWYhMTHRzMxsWuO2bdvev3/f2tr6u8/+w5ibm2/ZskVAQIBMJs/ds7KyMjw8XF1d3cjI6OjRo9zc3F5eXigUanJy8tixY8zMzP7+/rDn58+fN2/e3NTU1NjYKCgoKC4u7uHhAQC4detWYWEhCoUyNjbW1tZOSEjIzc1lZWWdmJjAYDAUCuXEiROcnJzOzs53794FANy+fRsAoKamlpOTk56evm3bNgDA27dvqVP68OHDxo0bOzo6AAAhISH79u27efOmg4MDNze3jIwMKyvrs2fPWltb50ifWbFihZKSkoyMTERExMDAwKdPny5dupSRkaGlpWVjY3P58uXi4uKHDx8CADZt2iQmJrZ161Z3d/dt27ZBp8Y0PDw8Vq1aFRgYqKysDADg4OCYN2+ehIQEAMDFxeXw4cPu7u47d+6cmJjA4XBnz57Nysp69+4dAGDz5s3x8fE7duwIDQ19/fr19evXd+zY8eLFiw0vN8w2c01NTUlJSbhhoKCg4OvrGxISYmVl1dDQkJOTAwBgYGBobm6e+5lOg56enro8UxkfHw8LC0Oj0Xx8fCEhIbARh8NVVFQsWbIEAMDPzw9XcQqFwsPDQ0dHBwDw8/ODey1XrlwZGhqC21f8/Pzp6elycnJtbW19fX1Pnz6dmJjw9vaGt8vf3390dDQlJWVycvLkyZNr1qwBjTPPU0hIqLq6+ubNm5mZmcePH79x48aCBQuWLFliaGh46NChxYsXf/slIwjCxMQkKSn5LZ3b29tLS0sNDQ0BABgM5uHDh+bm5mfPnpWWllZTU5OTk+vp6dHS0urs7IyOjmZgYNDU1Ny6dauLi0tgYGB2dvbLly+3bt2alpY2Ne2Lxv8atLo7/zxev35tamoKf1j/5dTX10dERCQnJ2toaKipqU390927d+Pi4ggEwm+dQFNT0507d75u9/LyysvLm9vWKSkp+fTpEwAA/kynJpLU19cnJyenp6fDX8lzUFdXl5SUlJSUBBcYSHV1tbi4OD8//1T7oLKyUkxMjJGRcevWrZOTk99+gQCAqKioW7duxcTEGBoa3rp1Kz09/fnz5zBvJS8vLyMjg5ubu6amBgAwMDDQ1dXFz8/f19dXXFwMfT2jo6OxsbGVlZUVFRVbtmzp7Ozs6upKTU0NDg4uKSlZunTp3bt3bW1tAQAwGtTc3Ly3t7exsTEhIQEAsHnz5oqKCnFxcRKJRJ1SfHz85ORkW1tbQUHBtWvXcnJyzMzMAgMDyWRyTExMcnLy1q1b3dzc5rio7du3Ozs7KykpkclkIpFoaGiora3Nz8//5MmTffv2ycnJUT85RkZGtra2hoaGysrKU+NdpqGmpsbGxnb48OGRkRE6OrqAgADYrqioeOTIkbi4uI0bN0JTiYGB4f379zw8PFxcXCkpKX19fQCAEydOWFtb+/n5MTIyNjQ0zDHzurq6uXfOdHV1oeXxk3R3dx87dmxa4/j4uKGh4eTkZF9fX1FR0RyhVLW1tTExMcPDw9HR0Z2dna9fvwYAkMlkXl7evXv3enl5qaqqwitdvHixoKDg/v37vby8VFRU6uvr57g0CwuLxYsXa2ho2NnZ3bp1CwDAysoqKiqamZn55MkTV1dXCoVSWVmJRqMBANnZ2Y8ePXJycrp48SIAgImJyc/Pr6amBkEQMzOzL1++REVF9fT0PHnyxMnJCY1Gp6enGxgYQEuUjo5OQUHB3t6ei4tryZIl7e3tAAAUCqWjo5Obm8vGxhYTE9PZ2cnCwuLo6MjMzLx8+XI/P7+Ojg64B+nj49Pf33/8+PHCwkJlZWUTExOarfM/Dm13548xNjZGEGT9+vX79u37k08dFxeno6MDQxSp3L59+8GDBzt27NDT0/uBMREEwePxADD8cddvwMPDIyAg4MyZM1JSUkZGRhYWFjC7ZGRkZHBw0M7Obv78+dTzjo2NkcnkiYkJRkZGJiYmDAaDIAiBQDhx4kRFRUVoaKikpCRcJ9LS0i5dukRHRxcWFjZ//vzi4uLjx49bW1s3NDR0dHR0d3cfPHhQVVX16tWr165d+/z58549e0gkEiMj45kzZ8bGxry9vaOiogICAmAca0lJyZkzZxgZGVlZWdFo9MjIyMGDBxEEUVVVhfO/e/eulZVVUlLSli1benp69PT0UCjUwoULw8LCpsYTTKOhocHQ0NDGxoaZmXnfvn1paWlLly719va+f//+nj17GBgYzMzMkpKSli1b1tbWtm3btqNHjwoJCV2/fr2wsHDDhg0AABQKxcDAMDExMfdNDg0NHRwcvHnz5r59+4KDg2VlZQEAmZmZDx8+bG1txeFwZmZmIyMjAAAFBQVNTU1GRsbExETq4enp6Xl5eRYWFmNjY+Pj4/7+/pGRkUJCQjY2NgUFBVgstqamBjogBAQEGBgYmJmZubi4qPErDAwMsrKycGsEtoyMjFy4cMHf319QUFBQUFBVVfXEiRMvX77k5uamp6eHeyrMzMxzGw0jIyOFhYU3btyY6sXD4XDU1GJIcnJyTU1NQ0ODkJDQy5cvp+0gTsPT01NBQSEzM3PDhg3wg0cika5cuZKbmwsNoObm5n379pFIJB4eHhi6i8fjt2zZAgBISUk5ffp0V1dXZWVlQEAA2DbrWTQ1NeGtePHiRUVFha6uLgAAhUKRSCQEQVAo1LVr1+aeJ4FAwOPxExMTIyMjY2NjvLy88PGhUChoHCxduhSHw+3cuXPPnj0mJiZkMhlBkH379qmrq+/evfvgwYM2NjaMjIxcXFwODg7wGaWnp7u7u4eHh0dHR5eVlfHw8Lx48YJMJnNwcFAolHXr1mlpaQEA9u7da2JiYm5uzs3N/fbtWzc3NyKRGBkZaW1tbWRkJCAg8P79+z179sw2c1NT00ePHllYWDQ2Np47d+7kyZMAAD4+vpSUlIKCgq6urjNnziAIMjw87OPj09raSiAQenp6QkJC4Nn9/f27u7vXrVt369YtIpFIT09PoVD6+/u7urr8/Pzo6ekHBgb09fWplu6tW7eePXsGAGBjY9u6dSsAAI1G37lzJz8/v7q62sjICLo73dzcNm/enJ+fv23bNmhmAQD4+flfvHgxx1Og8b8Gzdz5Y8bHx8vLy0kk0p9s7oyMjMTGxmKxWHNzc2pjd3e3hIREcnLyNFunoKDg48ePBgYGcD+jpKRkcHBQW1v7zp07ExMTK1euXLFiBez59OnT0NBQAHJ+4VRJJJKYmNiVK1dWr1595swZDg4Of3//hoaGqfs9w8PDkpKSCIJs3749Jibm7Nmzrq6ug4OD4uLiy5Yt4+fnX758eVhYmLOzc19fn5mZmby8/Lx580xNTVNSUqDDxd7e3tLSMiIiwtzcPC4uTlVV1c7Orrm5OT4+3tfXl0AgwERWVlZWX1/fp0+fUncjFBQUBAQELl26lJubq6ioePv27ejo6IsXLy5cuBD2MTc3Hxsbg2tYQkLCli1bgoKCMjIydHR0du/ePdtVb9myxdTU9NixY5OTk69evYJD3b1718bG5uDBg0NDQ/v37zczM6upqeHn5y8uLs7Ly+vs7MzMzNy/fz8cAYPBnDt3bmo8ymyMj49v3LiR6tEAAKBQKAqF4uvri8PhFi1aBL/3Z3w0rq6uysrKCIIwMDBYWlr+5REM0GVDJpODg4P19fW1tLSKioqmhqNC6Onph4aGDA0NfXx8Tp48mZWVde7cueLiYm1tbdgBjUY3Nja2t7cLCgrClpUrV0ZFRTk6Onp5ecGWwMDAGzduhISEoFAo6o7LypUrT58+XVRUtGnTposXL8KI45KSkv3794eGhpaVlcGtstlIS0vbtm0bjEExNzeHSz4jI2NBQQEfH193d/fHjx9fvnw5W/mc0dFRTU1N6I06cuQIgiDnzp1LSUnJyclZsGABPz9/WFiYjY2NpqamlpZWeHh4SEgIPT29uLj4unXrAAA6OjrwEDMzMysrK7hjoa6uTiKR4GYkHx/fihUr1q9fb2pqOjAwAABAEISNjQ3WpDEwMBgYGBgfHyeTydRGAMD169f7+/spFApstPa1nnHyO3bssLS0JJPJZDKZhYWFGogjKysLrfC50dHRgS+gTUxFSkpqxv4CAgIzBn2rqqpOy8gTERGZmtVPg8bX0MydPyY9Pf3gwYOdnZ0AgObmZnp6ekFBwfHx8fb2dgwGw8vLy8jISCQS29vbx8fHcTgcgiB8fHwsLCxEIrGxsdHFxWXHjh3q6urj4+OLFi0aHBy8ePHinTt3Tpw4YWBgAL+tBgYGwsPDa2pqOjo69PX1PTw80tLSQkNDc3Nzu7q6Xrx4MTExcfLkSQkJiePHj8fExJiYmExdHnbs2JGfn9/X1xcSEnL48GETExMXF5fGxkZtbe38/PzR0VE2NraioiIYFZiSkjJHKOLPAEuEIQiCRqPPnj378uXLqc4gBEH6+/s9PT39/f1Xr14NHTqtra2qqqqJiYmMjIwKCgowsvXUqVNr1qxJTU3FYrHy8vIbNmx4+/ZtQkICHx/fnj172NnZDx8+rKmpee7cOQ4ODjY2NjQazcnJST0RfDs1SJaOjs7Dw+PSpUvc3NyMjIx8fHxRUVEXL15kZGQcHBwEANDT0zMzMyMIAgC4efPmxMTE6OhoUVERNzc3Ozv7bNcLA0HghcOQF3iZsBH+0IeXWVJSEh4eXlBQAAumTY3WnHsbYCrTHCgIgtDR0R05ckRISGhkZGR8fBy2c3Fx9fX19fb2jo2NAQDExMRcXFz4+fnt7OwmJibGx8dv3rwJAEChUGQyubm5ecGCBb29vQQCgUwmo9FoLi6usbGxvr6+rq6uhIQEX1/foaGh8fHxoaEhb2/vW7duodFoYWFhX1/fo0ePEolEZWXl58+fX716FQDQ1dUFf83DOJKhoSESifR1tDI0Oz59+mRubo5Go/n5+Q8ePCgjI8PHx4cgiKOjY3h4uKioaHJycnt7u6enZ0xMTEdHR1dXFx8f3759+549e1ZfX3/z5s2GhobJyUkLCws7OzvojAMAGBoabtq0iZq2w87O3t/fb29vPzk56efnB3+xmJiYvHnzJjAwUF9f//z58zY2NgAAFxcXNTU1AoGgrq5ubGys+mLW/HYMBnP16lVHR0cUCqWkpAS3E9zc3DZt2gT/CYSEhPT09AgLC894OA6Hi42NpWaAQywtLWdcrbW1tanmHRVdXV24pUTF3d0dhugCAMLDw6ntX8dOoVAoFhaWr906GAxmNot5Kmh/9B/2oUHj7wnN3Plj7t69GxwcDADo6ekRExNbtGhRbW3twMCAoaFhdXV1SkqKnp5eWFjYoUOHXF1dX7x4MTo6mpeXx8LCcvr06czMTBUVlZMnT/r6+rq4uHh4eKxdu3bevHlmZmaHDx9OTU2FMaFKSkoCAgIvXrwIDw+HUSDwqzMrK+vFixfz5s0jk8nwmzQyMvL8+fMbN25saGhYsGABnGFaWlpAQICTk9OpU6euXr26ZcuWsrIyXl7eDRs2JCQkNDc3i4mJDQ4OQnMnKioqKirqz0+bZ2Bg8Pf3j4iIePHihampKfxxdvjwYWrVmTdv3sCeHz9+FBQUhD6O0NDQ9evXT05Oon71jBEEMTExOXbs2IEDB6SkpFJTUzdv3gwAwOPxurq6586dAwCUl5fPEaWxefPm58+fe3h4oNHo4OBgJiYmd3f3zZs3p6WlycjITExMEAiETZs2AQB27NixYMGCjIyMe/fukcnkqKgoOjq6devWZWVl3b17l0gknjp1irpF8TUdHR0VFRUAgLNnz46Pj0PzRUpKasGCBQsWLNiyZcujR4+OHTsGo1VCQ0OFhYWhA/TEiRO+vr5wqwBuUykoKEDLAI1G4/F4SUnJZ8+eNTQ0VFVVXb9+3dHR8fbt25s3b+bm5l69ejXs2dbWpqamJigoqKenBze0PD09PT09OTk5P3z4kJSUdO3aNXV19YmJicnJSVtb29LSUl1dXX19fXp6+urq6tl+9JuamkpJSW3ZsgW6gVAo1Pz587dv3w5L5Z4/f/706dMoFCowMPDUqVMTExP09PQ4HA72FBISUlJSwuFwsOoMdcze3l5tbW1ra+t58+bBlr17986oY7By5UoYB01FQkLi/9tymNMNws3NPc3gkJKSom5RrFy5co5jsYG/JgXvz+dvm4b5XfyP1M6g8TU0c+ePOX78eEBAwJ07d5YtW3b+/Pnr168DAGDog6enJ9wSuHHjhpub28WLF83NzcfHx2HcQHx8vI6OTmBgIAzN8/Dw6O/vHxwc1NTUtLe3Ly4ujo+P37t3r4KCwvLly9+9e2dmZnbixIk1a9bA80L7Bo1GT90AwGAwRCKxubm5ra0Nmju1tbVKSkrW1tYAADs7O29v78bGRnl5eQRBflVu8zfy5csXHh4eahl4Ojo6mLkD6ezs3LJli5mZGYIg69atQxBEXl5eXFy8qampuroaOpIIBMLq1avnz5/f2dlZU1ODx+Pd3d3FxcUZGBhgTHFTU5OiomJdXR2CIC0tLRwcHDAVtrS0lEKhxMTEnD59moWFpbS0FEGQoKCgJUuW8PLyrl69uq6uDt4rWF8Vj8fX1tbu2LHj9u3bHh4e69evf//+fWlpKYlEioqKOnHixNKlS0VFRclk8hxSO+fOnfPw8LC0tGRhYcnPz/f19QUA7Ny5U09PLyAggEKh7Nq1C/7UDg8P379/PywVIy8vn5eXt3r1aklJSW9v78HBQVlZ2bi4OCkpKbBm5hM1NzdTKBRGRsaoqCgikWhmZsbPzy8mJvb8+fO+vj4ikejt7Q2dIwAAVlbWV69ewWgPmCkjISGRk5MD/RoCAgLwF7+6uvq7d+8QBFmyZIm8vHxLSwv80DIwMMTHxzc1NYmIiEC3l7S0dFFREQMDg7CwsJubG9WS2Llz59RJMjIynjp1ivp28+bN0HycDU9PT+pr6g4QHR0dExPT141Unwu1kfoC/uuYnJx8+/ZtXV0dgUCg7vT8Jn5mvfx3WAxf8+33ZOod+IWWx7/1xtL4lSA0ZgcABACkp6dnyZIlurq6sHHHjh0EAgFBEJi38vTpUwRBzpw5s3r16gMHDixZsuTevXuwJyw5amNjIy0t/eDBAwRBYLnSLVu2XL58OTIyMjAwsLGxEXbOzc2F0STbt2+HLdDl1NXVhSAIXL0gw8PDq1atgks+giDPnz+XkJAoKytDECQgIEBJSWlwcBBBEE5OztjYWARBmpqaAABwbUMQZGxsbN++ffDSfurm+ADgA6KjowEAurq6u3fvXrJkCbQzJiYmDh48CLcrtm7d6urqSiQSLSwsAADbt28/duwYExNTWFgYvK6pe/gwKpNCoVDDlfbv319VVYUgCHTeycjIIAhy6NAhAIC5uTmcSUJCwrp16/bt2xcaGkokEoeHh4OCgi5cuHDu3LnAwMALFy4gCHL37t2goKCbN28iCNLQ0HDq1Km0tDR4eERExJs3bxAEefbs2cTEBGy8ePHi27dvqVf69bX/1L2b/Zb+OSf6k/kTrmtgYMDf3//GjRu/asB/x52fAwqF8gtHg7frwYMHbW1tf9httrffCIVCmfp9+L3jwC+uHzgvjW8ElmMAAKSnp//Vc/n/oO3u/DHc3NwKCgowDRIAUF5efv78eU9PT5jMAgMR9PT0jhw5QiKRgoKCqD9qYdRwdXX1mTNn4Grt7u4eGRnJw8OjoKDQ09Njb2+/YcMGNBqtoaGxZcsWKyure/fu5ebmNjY2iouLc3Nzz5s3z8LCwsHBwdHRMScnR1FRMTMzs7e3t7y8PCYmZs2aNaqqqhoaGosXL96wYYO1tfXFixcvX77Mzs7+4cMHEom0Z8+eJUuWQB+Wubl5RkaGuLi4q6vrjRs3AAiZ6Vq/m+7ublVV1fHx8dLSUhUVFbiNT6FQPnz4sHjx4mXLlnV1dXV1dQ0PDwcGBq5YsWLdunU5OTkNDQ0weoCOju7q1avQbYEgyLx58+D9jImJOX/+PAqF4uPjg7/jb9y4AUOjAAD+/v579uyh/uI3MzPbuHEjMzMz7MnKynr06NFp84SVYyDz58+nhrICAFxdXeELahwlAGDuJOo/k3/r3vtvua5GsMN3x68f9i/ib/voZ9xKWb16dUlJydmzZ+Gvkd9HVlaWi4tLdXX1t8e90aABaM6sb2HLli1FRUXUJKO9e/cGBgZ++PChrq4Oi8Xeu3dPT09vfHycjo5u5cqVL168iIqK8vLyWrNmTWBgIGx8/vz5/fv3WVhYIiMjb968ee7cuVWrVrGzs587d05SUrKrq4tIJF64cOH8+fPz5s1LTk4WFxcHACxatOjJkyc+Pj4kEuny5cvQdTU8PDw+Ph4XFzc5OdnV1QUr1MXGxmZkZJBIpMePH8NKXJOTkxEREQiCEIlELi6ue/fuwfBbAICDg4O6uvqv2u8/RjgGNP7z+jV4fc332n/erP3/unFd/G9V/kwAAPC64gX+4fw5S9EcW/QoX9SfsIH/k2eh3qVffrumzep3PI4/2dqY8T4jJ5Hx8XFGRsYfi10jEolXrlxxdXWdVgdIRUVFUFAwLi7uB/zds92W3bt379ixY8eOHVMbh4eH4TYSCwvLbGFwIyMj8HsMQRACgcDIyEgtJoTH4wMDA5mYmPbt2weNm5iYmMuXLzc1NRkbGxMIBJgp9r2XQON/E5q588fQ09Pr6urCaGUAgK2tLYVCqaioiImJgVtk1dXVDg4OT5480dTUHB0dNTU1NTU1ffbs2ejo6JMnT+D+TWxs7PHjxyMjI9evX79mzRoCgUANU5g3b15VVRXMzkWhUNTYBQCAsrJyWlra1B8xsPjENHh5eWHsDhUFBQUFBQXq26mp7MrKysrKyr/K3PlHuMx/ft36E1a+v+2dnPHav322sOfbt29fvXqFQqEWLVoEaw59I6GhoTBvX19fPzk5eY4pYQIw9PT0MAjMw8MDBpv/JlpbW1+8eGFhYTE176y6utrDw4OLiwsW3/suZvuAffnyxc3N7dGjRz9gl1AoFCcnp5SUFCjNAbeBx8fH7969a2xsvH79euqYQ0NDdXV1CgoKJSUlr1+/1tPTo6ZBPHz4sKenh4GBYXJycsmSJdPE1CAw/dDZ2Zmbm9vU1JQavVdeXq6rqwsD+A4dOjSb8igMI8NisWg0Wk5OrqOjA1ZsKikp0dLSkpCQ+Pjx482bN6n6aGVlZevXr2dnZx8dHf15iRga/0P8dX60fwDfGOBy7do1AEBUVBSsgBcaGhoSEnL16tWFCxd2d3c3NTV9+vRJVlZ22bJlv3/K38qvit35RdP5vfymeeLx+Pr6+oaGhsbGxmnBBN/LD8xw6iHnzp2rqKhAEKSurg5Gg9XX11dVVdXV1fX29v7yiX37bGHPd+/eLViwgIuLi4mJacuWLd81gebm5pcvXxoYGMDw9tkmAHyAtbW1qKhoSUkJrE4UFBRE/Wt3d3dHR8fU/tnZ2evXr//w4UNjY2N1dfXY2Bi156dPn8bHx+Hb3t7e9vZ2CoXy8ePH2tra/v5+2H7t2jU0Gl1SUlJeXg5HplAonz59KikpcXNzo56FQqE0NTVVVla+f/8ePpfR0dHy8vKGhob6+vozZ84kJiaOjo7OeEWwUUtLC4vF2tnZ2dnZ2dvbDwwMIAgSGxtrYGBga2v7/v17aucbN24YGxs7ODisWLHi/v37CIL09fVxcnKiUCgeHh4ODg4dHR1YJkBAQACLxQYGBsIDX79+ra+vLywsbGdnt2bNGhwOJy8vPzk5iSBIQkICNzc3Nzc3CoVCo9Hu7u4zTnV0dBT4gOXLl6NQqO7ubmr7kSNHJCQkqqurz549Cx/HjLE7jx49QqFQy5cvf/PmTU9Pz7Jly2Bgn6Wl5bp160ZGRqBe2LVr1xAEIRKJLCws1PjI2T4Pc/AP+uL6h0KL3fkH8w27yE4ATK0VR018dZ5S0a3i24b6h/E3DC/407ZJampq5OTkAACsrKxtbW0zajn9CYSHh9+4cQM6Ec6cOQM1HEZGRsLCwqqqqp4/fz6bvvf4+HhoaKitre1vlRX78OHD1q1bw8LC9PX1BwcHpwZ2kEik5ORkWNSYSm5ubkpKiqWlJSyMKSIiIiIikpCQ0NPTM/eJ5s+fX1paqqioqKiomJGRAVP3AQBdXV0aGhowfN7FxQWNRufl5RkZGY2OjsIkeTk5uczMTCYmJliN6cOHDy4uLpcvXx4cHNy4cWNnZ+fu3bvxePylS5esrKygLVVSUgLFOwEAWlpa/Pz8CII8evQoNDQUlg+GEIlEOTm59evXy8nJPXjwIDo6WlxcPDY2NiIiYsuWLYsXL37y5MnVq1epu1ZfY2JiAstEwSKWOBwuLi7u7NmzmZmZIiIiGhoaZ86cUVBQ6O/v37FjR25u7vr160+fPg2zBDg5OU+ePHn27NnW1lZ4t9FoNDc3d3t7e0lJCdQtBwAoKCiEhYVJSEg4OzuvXr06KytLV1eXQCDgcLhPnz49ffp0zZo1HBwckZGRs7mNYBRdZGQkLIRIBdZeevz4Maw4MNs1GhkZubq6lpaWrlq1CgDAxMTU3NxcV1cXHx/f2trKwsJy5MiRBw8eUHd3aND4MWjmDo2f4lfZFm1tbY8ePRoeHv5aJOi7+DPNLykpqcrKSnt7+/Ly8j/tpNMYGhrKyMhITU3l5uYGAJiamsLghuTk5A0bNri7u0OfJplMfv369fj4+OLFi6FxMzIysmPHjqSkJHZ29uXLl0P1eAEBAQKBUFRURCKRJCUlp6pVw0U6JSVljvpAM1JQUDA2NgZD9WHBawDA4ODgtm3bxsbGRkdHPT09ExISoOFoZGSUkZEhJCT06NEjY2Pj73JI0dHRDQ8PV1RUlJSU1NXVUavx2tjYuLm5mZmZbd68uaWl5fTp09LS0hcuXDh16tTDhw8ZGRk5OTl5eXlDQkIOHjx4/fp1Q0NDV1fXU6dOHT9+/MqVK8bGxszMzMeOHfv06RNVzHXTpk3TAt7p6OgOHTr04sWLqSJxJ0+ehFmZ9PT0dXV12traNTU1Z8+effDgwd69e5WVlSsrK5cuXQpLmM4ICoUyMTFBo9FUbYQHDx6IiIh8/PhRWVm5ubn51atXCgoKWCx20aJF+vr69vb2u3fvnrFg4Bzq9NBdDn1byBRJZBQK5e7ufv78eWZm5mlSNt/C8ePHYcokExNTaGjot0s6UCgUVlbWefPmff78WUhIaHR0dHBwkBp+xMDA4OPjY2JiQqFQSkpKoEIWDRp/CE0idC4Q5F/+39+H3t7eT58+HT9+/A9/xH+Nn58fFNaZqmE5I2QymUQiQW0jaiOBQBgeHiYSibCRTCaPj4+Pjo6SSKTh4WECgUAgEEZGRkgkEh6Pv379em9vLzwQh8NJS0v7+flNUxIlkUj79+/fvXs3rGsMIRKJ+/fv37Vrl5OTk5+f3/de42yXY2pqunjxYqpdQqFQMBgMHR1dZmYmrPcN28PCwvbv35+dna2npwfFGgcGBsrKypSUlAAAxcXFr1+/htd18+bN9+/ff/78+fr1611dXdRz9ff3y8nJTW35AdBoNLSW4uPjnz9/HhYWlpubi0KhvL29YQcLCwsfHx8YRzK3ksPX0NHRkUgkd3d3Z2fns2fPQlskLS2tvr6+qakpNDS0o6Pj4cOH7e3tPDw8IiIiUEQdGnkAgI6ODm5u7sbGxrNnz2Kx2P7+fgCAtLS0lpYWLFTo7++fl5cHdWQRBIFHTYOBgWFqTHF7ezsTExO0M06ePDk6Ojo5OYnH4/X09KAU+R9CT09PVcCF4X1YLLalpaW5udnT0zMiIgKasywsLMnJyUFBQdLS0itXroyPj582DoIgU/91wFzuaX2oLQiCwNcoFKq2tvbt27eXLl36urjzjIePjIyMjo4SicTOzs5du3bV1NTU1dWJi4t//PgRmeUbp7OzMyMjo6qqqrS0FACAw+GSkpIePnz45MmT3bt379u3b/PmzR0dHVDanZ6e3sbGpq6uzsPDw87O7uTJk7MNS4PGNGjmDo3fC9zNnlsDvKCgoLu7e+HChQiCoFAoAoFQWVkJ//T8+fO0tDSqPfHy5cv79+8nJSXB/XkYjHny5EkjIyNmZmZRUdE5ztLY2Lh8+XIWFhZGRsbU1FQAwNjY2JMnT7S1tfft28fDw5Oeng4A0NTU1NTUtLOzExISggKNOTk5NjY2QkJCDg4OSUlJYmJiYWFhs50Fj8dbWloWFRVVV1fz8/MXFRXBE23btq24uFhZWfnNmzewUuXPMz4+/uLFi40bN1JbEARhYGBwdHQ8fPhwZWUlXGgzMzN9fHyePHkSFBSkqqoKSyWJiopKSkpKSUm5urp6eHgcO3ZMVlY2OTnZw8NjyZIlcnJyFRUVU7P3d+3adfXqVary2o9BJBI/fPgAACCTyXx8fPLy8uzs7M+fP6fuRuBwuHPnznFzcyckJFALDXwjEhISCxcuzM3NvXbtWm5uLiwsWVZWVl9fz8LC4urqWlRU9OrVK+jaY2Rk9PLyghIi1FuHIIi+vv7u3bsDAwMDAwNht/T0dOi0wmAwFRUVMPCWgYHBw8MDhkXDGutwkK1bt06dtqysbH9/P/yrq6ursLAwGxsbAwNDQkIC1Zc0N/Pnz3/z5s2jR4+ePXtmZmbW0tISGhra0tJCIBCWL19OJpNhtcZt27ZBGXkpKSksFku1AKSkpDo7O21tbc+dO8fFxTU4OEgkEh8/fuzg4ACVYh8/fgz/0QEAzM3N29vbsVgshULx8PAAANDT04+MjDQ1NR07doyfn59a93xGVFVVZWVlg4KCdu3a9fDhQzo6uoaGhuzsbCcnp6GhoYSEhNm2l8bGxoSFheXl5eGNUlZWFhISWrlypYqKSn5+PhsbW2hoaFNT0+rVq2H/wMDAlpYWNzc3Pz+/p0+f0nTOaXwjNGcWjZ+io6PDw8NjZGSEh4eHkZExJCQEg8Fcv349PT2dm5t75cqViYmJGRkZQUFBR48eTUlJuXv3Lisra2Nj4969e2EZ/sePH9va2sL6flgsFofDBQcHl5eX37t3DwDg6ur6+fPn7u5uTk7OxMREa2vrLVu2PH/+/PTp0w4ODpOTk+rq6p8/f961axfc2Nj6cYbMNQDA0NAQDNdIT0/Pzs6GHoeOjg4TE5NNmzbFxsZCVwgAAAorGhgYrFq1KjIy8uDBg+vXrxcREZGRkfH3958/f/7+/fv9/Pz27NkzLbkX8u7du6GhoeTkZAwG4+zs7Ovrm5GRQaFQRkZGlJWVt2zZIiMjM7XY9C9ncnLS0NDw8uXLWlpay5Ytgy0EAgGKf1lYWFCFyqmSXlTwePzw8LCZmdnExAQTE1NwcHBuay780/caH1Mhk8kEAgGLxQ4NDVlYWDx58gSHw8GdNnp6+uvXr8N17vHjx4cOHbp161Zvb29wcDAUCYeHT0xMkEikiYkJqEw541n09PTS0tLGxsZMTExCQ0O7uroSExO3bdsWGRkJAGBiYmpvbz958uTp06fnz5+Pw+HweHxLS0tbW9uJEyfCw8OdnJzi4+OdnZ2Tk5NhqnNgYCAHB8fo6GhbWxsAAEbtwF0fBgYGAoHg5ubGyMhYUlISGxs7b948T09PWG/C1dUVj8cHBAR4eHgMDg4aGxsvWbJkbGwsMzOTh4fnxo0bZDL5/PnzeDz+7du3WCw2Li5utlunrq6+fPlyY2PjhQsXBgYG8vDw0NPTJyQkeHl5ubu7nzp1KjY2FgBgZWUVHR29d+/ed+/eXbhwAVajAABs3LgxKysL/jaIiYlhZmYmk8kjIyMnTpxAo9FQQ41MJnNycj58+HB8fHx4eHjZsmXx8fGSkpKjo6PXr18PCQkRFBTctGmTp6fnhQsXqNpwX3Pz5k1lZWVqShcA4OXLl9/y8ZCQkJjq55pam5uLi+vrrVAcDve9HlUaNADN3KHxk2RkZNy5c2fz5s0XL150dXXdtm1bQkICVKN0d3e3srI6f/58cnIyLHZsYmKSkJCgp6dnbW0dGxurq6ubmJjo6elZXFwsLS2dm5sLhZ3NzMwSEhLa2tqEhIQiIyM1NTWxWOz9+/dPnDhRWVm5cOHCsrIyuBYyMTFZWFhkZWWZmpr+x4nwceZ5srOz6+npRUVFtbW1CQsLQztg4cKFBQUFRUVFKioqBQUFML5EUVExOTl52bJljo6OVN3NqaxatWru7Rk8Hh8SEjI2NiYiIgKjIlhZWZ89e3bkyJE9e/bcvn377Nmz0xSXfi14PD4+Pl5NTQ36nlAoFB6Pf/XqlaqqakBAANUhxcTElJiYePDgQUVFxZs3b65duxb6xaKjozU1NQsLCzds2AD+q/Tw8ePH58+fr1+/fvny5d81mWXLlsnLyxsZGVlaWoaGhvLy8i5YsGDp0qWDg4MmJiYyMjJBQUHQkoDVZTo7O318fFpbWysqKjIyMjZu3KioqPj27Vs44eXLl8+2L8LBwQEASEhI6O7uPnToUHh4+Pz58798+ZKbm3v+/Pnjx49PTEysX78eamWsXr368uXLFy9eRBBES0sLxjNlZWWdOnXq1KlTGAwmODiYh4fnyZMnhoaGLS0tCILw8/NbWFiUlJSoq6urq6v39PTABC4sFguj1A8dOnT8+HEEQaDEG5yPn5/f8PAwiURiZmaGvkVTU1Nzc3M8Ho/BYDQ0NGB50pMXTs52AxMSEqB6KzVvXE9PT0tLi0Kh4HA4+LHX09PT0dGhUChTGyHq6upTR8NgMNOKVgAA6OnpjY2NqW9hAfTm5ubq6mosFmtmZvb06dM/1K2z/WILvszx95n5G+Y60PjX8idmgdH4VwHzOWHpi9TUVARB3r9/DwBoampCEKSvr2/z5s0wnRUyMDCAQqGeP3+OIEhJSQkAoLe3NzMzc+fOnbADFM3o6+traWlxcHCAjXl5eQCAyclJTU1NqmrEVFJTU9esWTN1VjNOlUgkVlVVvX//3sHBgZGRMTg4GEGQ4uLipUuXJiYmVlZW+vn53b17F0GQN2/eeHt7Iwji5ubW1dXl7Ow8MTEBnWsNDQ0Igty+fZudnZ2aeZ6RkYFGo4eHh+Hb169fw1UWQZC2tja4/lVVVa1ZswY2WltbCwsLU+c2OjpaU1Pzw4noZ86c0dLSojZeuXIFjUYXFBQgCBIeHr5ixQoEQcbGxrZs2cLGxiYtLc3CwpKRkUGdKg8Pj4SEhLy8PAcHx+3btxEEsbW1ZWVlVVBQWLhw4d27d6kTu3DhAgDg0qVLs93kOSZJoVD27NnDwMBw7Nix+vp6+KeamhpHR8fVq1e/fv0aBk4hCFJWVpaampqbm/vy5cvc3FyY+F1WVgZb8vLyMjMze3p6ZkxEz8vLy8/Pz8/Px+PxCIK0tbUVFxd/7139q/jJhP/fhKen5/z58+3s7Pj5+QMCAuA/7b98Vj8JLRH9d0NLRKfxvwUnJycXF9fX4ZBToVAoy5cv9/b27ujooAZ+UigUJiammJgYe3v7tWvXJiUlQcVsc3NzJyenAwcOKCoqfvnyJSwsbO3atWZmZps2bQoODn716pWQkFBwcDCYN/O56uvrHRwc9u7da2Fh0dnZCe2t7OxsFhaW+vr6xsbGysrKoqIiPj6+4uJiWEGuqKgoKCioqKgoKysLCnCGhITAcBwTExM6Orqurq5Lly7V19dTKBQ/Pz8BAYEDBw4sWLCAjY1NTU3NzMwsKysL/pIeHR0tLi7euHGjoaFhVVXVVHXM7OxsIyMj4POD9/nw4cMFBQXJyckGBgYIggwNDR09evTJkyeCgoJ79uwRFhYGADAxMcXHx58+fXpoaMjQ0FBD4z9lsBUUFFJTU+/cuYNGozMyMmBuV0REhIiIyPDwsJGRkbq6+nbf7bDzqlWrLly4sGvXru+d4X9+vnMBcBQEgsDAuMD/+5swAMJAMU0RpM1ycO5MjUUz912f898KeFMTgL4pQubvy1+8+YEDwBZ8AV+AC/Aienld/8cXQ6fxvwzN3KHxs+BwuEOHDvHw8FRWVgoKCsId+8HBwY6OjpKSEjo6OnFxcTExMSwWKyUl5eLiEhUV5ezsfPToUR4enrGxsaGhIQkJCScnp+LiYgDA9evXDx065OPjo62tzcHBwcPDw8bG5u7uHhMTQyQSjYyMUCgUPT09Nze3o6MjnAAnJ6eampqAgAA3NzfQn3mSbGxsExMTYWFhGAxGREQEeqOOHz+OQqEuXbp08OBBfX398PBwSUnJuLi48fHxmpoamESzatUqmC0MANDS0hIREdHV1ZWSkoJj6unpYbFYb2/vkZER6Gvg4uLKysrq7OycmJjYt28fDDKQlZWtrq6G0ScHDhyYmtSzYcOGDx8+yD6U/eH77+XlpaKiUlFRsXjx4mn67dRiJ4yMjDADfBrTqm8DAFhYWPz9/b/uqaKioqKi8r1z+9NqIP1ta1J/C9Cm+ae4df7Rt5rG/zIohJbFR+OHgN/Oox6jwsLCMMOFl5f3+fPnS5cuJZPJxsbGT58+hY1ZWVkwVqa2tjY1NRWqgWpra0Mboqam5tmzZygUio6OjpGR0crKipGRsaqqKj09HY1GOzo6lpSUjI6OwujLhISE5uZmCQmJqaEGfX19N2/enD9//pYtW2bUePp5eSlo8TQ0NMDIj1/OD8yQeggMKd2+fTvcnvkTJvbnyHXNxl8rZfVLzv71mF/fTwqFAlPH6enpZwyK/9ZzIQiRSMRgMD+gupWWlhYXF8fJyRkQEABDkf7aR/9LgE/wn34Vf2dgFCYAID09fWre6F8ObXeHxs9CIpH4+PgqKiowGMy8efMAAHDLJDo6GkEQaiMAQEpKCu6LTGXx4sWLFy+e1igtLS0tLQ1fU50v4P8X/6LCxcV14MCBX3U5c7BgwYK4uDgrK6s/4VzfwtSl1z3S/S+cyZ/Mn7ZW/YxeWF9fX25uLoIga9euFRQUzMnJ6enp4ePjU8tV+5bD8/LyNmzYwMTElJaWpqqq+l3Tnsrdu3ft7e0rKyslJCS+99jx8fHKysoPHz4cO3YMmjs0aPxzoZk7NH4BZDKZnp6eatbQ0dH9Vl2CH+AX/Cj3AQAA63pra9/piS3/bma8db9b4fwPIZFI/f394+PjDAwM/Pz8M/bp7u4+duxYR0fHnj17dHR0fsU0AQAAQRAHBwc8Hs/IyEhPTx8cHAyz9EdHR0NDQ6urq8XFxQMCAurr66Ggb3JysqCgoL+/f05OjoaGBvg202Xp0qXXr1/fsWMHHo//mdm2trYaGRl9464krK5JtWy2bt26YMGClStX/swEaND4m0Azd2j8FBgMxsnJaWBgIC8vbw5ZnL+Wb19Kv2sV/8v3w/Ge+NjYWBYWlq83nHJzc1VUVKjKA39zfsB4CgsL8/DwcHZ2hhHlsIgAAKCoqEhGRgYWGeLm5g4NDV20aNGnT59+rbnz6NGj8+fPOzg4yMvLe3t7X7hwgUQiLVy4EI/H19bWMjExAQAUFRUtLS07Ojrg3F68eHH06NGtW7cqpCrMNvLo6Gh0dLSFhQUvLy8PD4+xsbGTk9Pw8PCVK1fY2dm3b98OAEhPT6+treXh4bGwsLhz505nZ+eCBQuMjIwAAA8ePPjy5QsajdbR0ZGWlp6YmLhx48bFixeXL19OIpGoeewfP37MzMyEPmU9PT244VpRUVFUVHT79u329nY/Pz9zc3Oa2DiNfxk0c4fGT4HFYmGK8r+Gxh2N4+PjEhIS1OVhRv4OgaWxsbGHDx8eGRkZGhpydXWltj99+nTz5s0hISH79u3762b3G+nv78/JySksLFyzZk1ZWZmOjk52dvby5cvT09NtbGxu3LghICBAJBJh4j0ajYZy5SQSCWbYQRobG7u6utjZ2aEvtbu7Oy0t7dq1a/fv36+qqnJ3d09KSprx7HR0dPz8/LAoJYIgUEyKnp7+xIkTe/funZyc5P2vMrCAgAAsSwhBo9GzVRZeu3bt2NiYuLj427dvL1686O/vD4vfYDCY0NDQ5uZmPB6PRqPNzMweP3589erV+fPnW1hYxMTEFBcXw2rg4eHh9+7dCwoKsre3x2Aw0tLSaDRaWlo6KCjI2tq6ubl54cKFAIDBwUF9fX02NraLFy86OTlFRER8+PCBmZk5PT39woUL+fn5OTk59vb2mzdvppk7NP5l0MwdGj/F32HVn8b3TmnaJs22bdtevXr16dMnuDz81lN/OzPuJLm4uKipqa1du/bz589T20VERI4cOQJFN38hRCKRjo5u6o7R8+fPS0tLjxw5MrVbTk5OYmKilZUVVR+7v7//3LlzJBIJhUKtWLEC7lL8DJcuXaqrq1u7di0AYOHChf39/dCqyMvL6+npyc7OZmNjw+PxfHx8LCwsWCz26dOn4+Pj9+/fX79+/cGDB4WFhZuamq5evcrIyJiYmOjn52dsbFxRUfH582c8Hh8UFMTFxWVoaAi3iGbkwoUL+vr6KSkpvb291LIC7969I5FIenp6mZmZAgICFArFy8vryJEjZDLZ399/69at+fn5s4mKHzlyJDw83M7O7smTJ56enleuXIHFDvB4vJeXl46OzqtXrzw9PTdv3nzlypWenp5Xr14BAJYtW7Zp0yZVVVUSiZSenr58+fL169dnZ2dD4TksFrt+/fpp0mPDw8OCgoJPnz7l5ubOzMy8fv06jINWV1e/efPm/fv36+rq/imbgjRofB9/VcEfGv90fmG1ri9fvixevHjlypWLFy+2t7f/JWPOBhQoYGRkhLIV0y4BVqsDAHz69GnucX5VXbje3t4XL17k5uaWlpYiCDIwMPDs2TNhYeGAgICSkpLs7Oz6+vq5z7V69Wo3Nzdqe2Fh4fz58zU0NDo6OqiNDQ0Nx48fZ2dnFxAQoNb+Ki4uFhQUZGdn37NnT11dHYIgAQEBcnJyR48e5eDg4OTk3Lhx48DAAIIgL1++FBERMTMzW7t2rYWFBTw8NzdXSkpKVVVVTU0NxuQiCLJt2zZRUdGVK1dKS0sXFhbCRri4KikpMTExwfpAP3PfgA/w9/eXlJSktvj6+qakpCD/LV859cIRBBERETl8+DCCILBWZFFREYIgcnJykZGRCILExcVt2rSJWg9z+/btu3btmmNisCUoKCgwMLC1tdXCwgL2P378+Pbt23t7e319feXk5A4cOODh4TE2Nubi4jI8PIxGo2NiYlAo1KdPn2Ycc3Jycv78+cnJych/y3W2tLSMj4/DkkgIgpSVlQEAWltbEQQZGxvbv39/XV2dhYXF4OAgHCQ5OXnlypUODg7i4uKJiYnUwaurq6d+ntva2jg5OSsrK6fNwdfXV0ZGJjQ0ND4+np6enlo2c+p5Z7wh/zhoZQZ/N3/bMoM0iVAafypQ7JNCocAFBjb29PQYGxuXlpa+f/+eWuGeQCBM7YMgCCyEAwDA4/GTk5NEIpFIJE7+l9HR0dHRUeqJJiYment7BwcHoY40AIBMJo+NjbGysgoJCVGH/RroiRgYGBgeHqYefvr0aTMzs+DgYADAwYMHvy7DTyUoKMjKysrJycnOzs7V1fXYsWMwnXhyctLd3d3Nzc3R0bGqqgp2JpFITk5OGhoaampq8Hc/mUw+ceJEa2url5eXkpKSubn50NDQd91hPB6/Zs2aFy9ehIeHUxvd3NxSU1MjIyNlZWWPHTsGGy9evHj06NGrV6/W1dWZmZlRKBRRUdGxsbH8/Pzg4OCAgIDc3NyHDx8CAE6cOMHNzX3r1q21a9d2dnYCAEpLS3V0dLS1tfPy8sTFxY8fPw7HHBoaioiIKC0tXbVqFdQkBwCcPHnSysrq1atXT58+dXf/9RlkXFxcczxQCoUCA+eh1wn2HBwcvHv37uHDh728vKDWPew8b9486rTnGPDChQsiIiJCQkKnTp26fPlyY2MjkUj08/Pj4uKys7MbHR29cOGCk5MTFoutqKhwdXXFYrFRUVGGhoYiIiIzjkkmk9vb22FZTvhvZI68cSYmJhcXl82bN4+NjcEtqObm5qGhodLS0ujoaBMTk/b2dvDff2VQHg6Px8PB0Wj04ODgmTNnRkZGhoeH4+LixsbGOjo6Tp48GRUV5e7u/vjxYxQKNTY2BkeANSYGBgYGBwen/vuiQeMfB83cofGzNDU1ff78+f79+zCGgEpubm5cXFxmZiZ8SyQSk5KSduzYUV1dHRQUxMnJ+fTpUwRB7t+/7+joGBgYiMPheHh4ZGRkAACtra0rV65kZGSE+s8AgO7ubg4OjhUrVjx69GjhwoWMjIxubm62trac/0VeXl5LSwv+sPj8+bOKioq3t7e5uTl1JQ4NDRUREfHw8IiJiZk7LgcAEBwc7OXlZWRkBBUKN23alJub++TJEwCAhITE48ePZztwxYoVZWVl2tra+fn5Bw8eTE9PhyOYmpqSSKTTp08vWLBATU0Nuhh27tw5NDSEIEhaWpqmpiYAgIuL69mzZ1gs9vbt2+Pj4/X19d8rQq6urh4fHy8nJ0eV/0xPT8/Pz8/Ozra0tHz48KGNjQ0AICUlJSkpiYeHh5WVlYmJqa6u7s2bN1ZWVsLCwnCPzcXFhY2NDVp769atq6ioYGZmbmhogL6b2traycnJ/Pz8PXv2TExMQAmw27dv5+TkZGdn79y5E4/HU6ODnzx50t/fz8vLa2Ji8od3/nsZHh4+cuQIjA7+dggEgqGhoZ+fX2lp6fbt2+cu/z0NOjq67du3l5WVtbe3P3z4UEJCgp2dfcGCBba2trBYFAzQefToUXd3t5ub24MHD9rb24eHh2VkZKCG2tcwMTFdvHjRwsIiLi7Ozs7u9u3bQkJChYWFZDIZlt+EFBX9p6Q0BoOprq6mWq6PHj2ysrJiYGCAnxz4j4hAIOjr60ONsxUrVnBxcfX39/Px8SUmJnZ3d3t4eOzdu5eTk5OZmVlAQCAhISEyMtLFxQU+uNTUVADA8+fPY2NjTUxMwsLC9u/f/+DBg++6yTRo/L346zaWaPyzgXvC169fl5CQWLp0KRsbGx8fn4GBQXd3N4IgeXl53NzcDAwMHBwcT548QRBkcHAQlkiWlZVlYmLi5uaGSltPnjxZunSpsbFxRkZGcnIyFDwKCAgQFRWFbp3ly5dPTEwQCIT09HQYTxMdHf3kyZOKigpoYF26dMnExCQ1NdXc3HzLli0Ighw5cgRW/UlNTcVgMJOTk319feLi4jdu3IBnxOFwszmzpm7+d3d36+rqjo6OIghy4sQJKEF1+fJlMTGx2RxMeXl5QUFBCILs3LlzZGREUVHx6NGj0I0CnSz3798HAEAtrYqKiuXLl2toaGhpaX348AEOMjExgcPh4E2jDjvjueCLac4syKpVq/bt2wdfP3nyBIPB9PT0TO2QmprKzMz86NGjlJSU9PR0qnCVmpoa1Z/Iw8Nz7do1BEGGhoZevHhRWFhoYGCwbt06BEHgro+HhweCIHg8vru7m0QixcfHCwgIwD25ycnJ5uZmuOUWEBAwMTHx+vXrK1euUH1hkK6urrGxse91ZjU2Nq5YsSIiIqK7u3vXrl0qKirwGUE3kLu7+7FjxxQUFA4ePDg6OsrLy+vl5UUgEI4ePQoA2Lx5M4IgV65ckZGR0dbWVlJSMjQ0hIdbWlry8fGtWbNGRUXl3r17M9554APu3r3LxcXFz88Py+EEBgYiCEIikaSkpDg5OdeuXSsnJ+fk5CQpKdnd3Z2enj5//nwEQeLi4k6fPj3bmAiC3Lhx4+DBgw8ePIC6cgiCvHr16tGjR2lpafARJCUl5efnwz/V19djMJja2lr4lkAgpKamPn78+NGjR9Q+CIJ8+fIlKSkpJSXlyZMn1H9cP8O/wA1Ec2b9bv62zixaqDKNn8LU1PTOnTvv378fGhp69+6dvLw83FS4cuVKbGysgYFBUVGRv7+/jo4OOzt7d3f3qlWr4FYBiUSCP4INDQ2FhISSkpK0tbXhmD09PV5eXi9fvlRWVo6Li1NXVx8fH4ehJBgM5vjx4w4ODrAnJycnAEBDQ6O9vX3p0qUUCoWenn5gYODMmTNHjhyJjIwsKSnR1dXF4XAdHR2NjY2rVq36rqvj4eERFBSEP/19fX2/fPmip6fX0dGRnp6+JGHJHx4+I7KyshwcHFS3RURExO3btz98+LBu3brMzEwlJaWpneG0v/cUUAVMTU0NvoXLsLa2dmJiYmpqKtyM2bRp0+bNm2tra9nZ2enp6ZOSkpYvX75r1y4MBlNaWjo2Nvbhw4ehoaHCwkI1NbXt27dbW1uzsbH19/fDyFZjY2NlZeWCgoKoqKjr16/X1tbm5eVZWFhUVVVpampaWVnFx8cLCwufPHkSj8efPn06Kytrx44d8fHxsDo2FTk5uRMnTnzvBYqJiT148OD06dP19fWTk5NOTk6w8o2cnNy9e/cyMjLweLyUlJSTk9Pr16+NjIz6+/sTExMXLFhgb28PHVs7d+7EYrFZWVnS0tJnz56FWUhCQkKGhoYkEomdnX3aPKdiZGRkbGyMQqGIRCIWi4U3BI1Gf/z4EUEQIpEINckRBEGj0dra2nV1dQAAS0tLZM4S9hkZGebm5ps3b6a2TP0wsLGxwUIPubm5ysrK+fn5d+/eXbRoEfwrBoPZtGnT12OKi4uLi4t/+42lQeNfDM3cofFTMDMz29ravn79GgAAEzq+fPnCy8v7+PHjkZGRlJSUwcHBlpaWwcFBGBMDxSIAAFMzcmfMBIHd5OTkNmzYMLXS2h8W1IeLyrJly6SlpdesWcPCwtLS0sLIyMjBwdHS0iIjIwMTXjw8POZIoW9ra1u4cGFcXNzz588ZGBhg4969e1etWmVmZvZ1GWgqjY2Nra2tZDK5pqZm7969RCKxsbERioXt3r07KCgoJiZGSEjo4MGDra2tmzdvXrduXUhIyMWLF6HiGACAgYHB39/f3t6en58fj8cLCwsDtZnPdeTIkfv373d2dkJ1Dh8fHxsbm0uXLvn4+PT29sbGxmZlZV24cEFDQyMjI2Pbtm2urq6VlZULFy6E9uKBAwdMTU2htPvZs2cNDAzKy8thlIa/vz8/Pz8/P395eTkOhxMTEzt79iw9PT0Wi7116xY8++3btyMjI6OiosLDwzk5OeHSe+LEidevXxMIhNOnTysoKEAz4vXr1z09PUQi8cyZM9MkutLS0oSEhHZf3j33M53GfzLgYCVLdnC94bqNr83//VnsP/+/fe82AAAIAgDA5c+XAQBAFAAA/H3/KwomBQAAoYGh/3nLBMB/fWKhZ//b+BWMZxi/a7bfwuTkZGFhYVNTE4FAMDU1naNnVFSUhYXFokWLcnNzf/k0aND4F0Mzd2j8Yj5+/Lh69WoymSwgIHD16lUAQFJSUkJCwp49e75xhHnz5hkaGgYEBHh4eDx+/PjFixcw4qG4uLi5uTkjI2PPnj1QH+revXsAgM+fP2dlZXV1dX38+HF8fLyrq0tPT+/Zs2dVVVX09PSPHj06cuSIhYVFamqqk5NTUVERjJt5+fLl1JooVGCIqLW1tZWV1enTp+E2APzTypUrd+zYoaysPMfkq6urP3/+XFNTo66ujiCIhIQErP97+fLl8+fP3717l5GRMT09nYuLCwBw7do1uEGCw+HOnDlDjWM1NDSEEcoCAgKurq6z5bcrKirS0dGxsLAgCDI6OiopKQkAsLOzg8dOTk4KCgpCCQ45Obnq6moEQaYGwK5cubKhoWHqgMuWLXv9+jXcmUChUNSyPdB1NQ1oA01rxGKxX9+fGXVCIN8bmQT+6uqOv+nsDAwMT58+xWKxUysDzYipqamgoODUMks0aND4FmgSoTR+EKrS3p07d+zt7ZOSkoKCglAo1IMHD3h5eRMTEx0cHOCuPjMzs5GRUWhoqLe396lTpxgZGeno6MLCwhwcHAYGBuTk5EZGRggEAjMzMxaLLS8v5+LimpyctLW1HRwcHB4e3rVrF4yuTUhIOH/+PBaLFRAQOHDgwNq1a/fu3fvmzRsBAYGWlhZ5eXlubu6nT5/GxMTIy8ubmprW1tYqKCicOnVKSEgILvPNzc0DAwNYLBaHw7GxsXFzc0+TPET5ogjHCHB7ZmJigomJSUBAgBoGW1hYuHfv3vz8fCYmptm0MyeOTExOTv5agaHZzvULT/GX85eXqP4L+Rc8yj/z8/mTHxWaROjvhiYRSuPfDJlM/vz58759+0xMTGCLmZkZOzv7ly9fKBSKioqKrKwsHo8XExOLiYkBAExOTq5evRoAwMTE5OPjQyaT0Wg0iURCo9EwCIOBgSEhIWHaWczNzadJhIaFhU3rc+rUKfhixpK4oqKioqKic18LBoP5WmAIZqQHBATs2rVr7iQgBgYGqvPrt0L7vp7Gz6+v/3qbkspsKve3b9+2s7NDoVDbt28/duzY14K+MzLbXfrej6iTk5OsrCwvL+/4+DgHB4exsTEAICYmxszMjJWVde5z0aDxh9DMHRo/y6tXr9jY2Nzc3KZF1Uyz63E4nKOj47RjcTicvb39b5/iT3PlypWqqio5OTlqlPRs0L6O/0J+wATMzc1taWnR09Pjusg125gjIyNiYmJ79+49efLkT8/xu8nOzoZ1FqY2NjY2Lliw4NChQ2fOnPneAWf7iFZWVtra2n748GHRokW9vb2/VQK9uLi4qqpKV1cXxqu9efPm06dPjY2N69atw2KxsrKyMjIy7969u3PnTmRkJIFAYGRkBABMjeOmQeN7oZk7NH4KlC8KcAHgDtD+/5LC87PaKxIAAHDa9/Qcx/78jstsZ/9JK+rft2/xqza3Ll26dP/+fRhrPxt0dHR8fHy/dfmfg8OHDy9ZsuTcuXMwbh2DwYyMjAwMDFy+fNnMzGxqz+7u7r6+Pk5OTj4+PgAAhULp6Ojg4+MbGhrq7e0VERGZe2PSxMSEQqEMDw8HBgZu27YN6syXlJS4uLisW7fOz89v3rx5KBQqLi7u/Pnzx44dY2JiOnbs2NfytFOxsbEZGRnB4XB1dXU2NjYwGszW1vbFixe9vb3r169PTExkY2NjYGDw9PRsa2t7/fr1+Pi4p6enn58fHR3drVu3KBQKJydnd3d3YWHhz8uP0Phfhmbu0PgpkJMIhUKhUChkMnm2Emr/IP4OHqLvncPw8PClS5dQKJS2tvaMkb+/ytdABY/H+/n5RUREODg4fJ3dFhQUdOzYMS0tradPnzIwMLx+/fratWsYDCY6OjosLGz37v9LwsrMzHR1ddXX1z916hR0YgIA8vLy2tvb517YfqGtFhYWdv/+/TlCGBsbG2/fvm1rawvzwAEACIJcuXJFW1ubSCTevXtXW1ubqg6WkpLy/v17LBZLT0/v7Oz86tWr169fMzExwSLgGzZsgDpfnz9/TkxMpKOjI5PJ2tra1Gy1xMTE2tpaFAqlq6u7cuVKAEB8fPy7d+9wONyRI0eam5vT09MxGMynT5/Cw8Pv3r2rqqo6b948eGx1dbW5ubmamtrz58/v3r27bNmyvr4+UVFRNzc3Pj6+169f43C4u3fvzpHYuGjRotraWqii9ebNm0WLFpWVldnZ2amrqzc0NMDAOH19fX5+fmFh4W3btjk5OSkoKAgKCoKGmQfMy8srKChITExkYGCAdagBAPn5+fn5+YmJiWvWrNHQ0NDQ0Hj+/LmsrGxTU9OuXbu8vb0FBQXh45CTk4uNjd2yZYu+vj4zM7Obm9sclaZp0Phj/pJqPzT+BdCqdf0OfuyWwuq6Fy5c+PYx5zjRmTNnZlS2msaM5Q2JRGJ6ejr0/U37k5CQUFhYGPXthw8fPnz4kJqaOjIyQm3MyspiYWGRk5MrKyt7+fLl0NBQR0fHmzdvysvLKRTKhw8fVq1apa6uPtsVjY+Pl5aWKigoiIiIuLu7IwjS3t6uqKh48eLFqKgoMTExaWnpiooKBEFIJNKuXbtMTU23bdsGAHj9+vVsYxYWFsLNkqysLNgYHx+PQqFERER0dXUFBAS4uLi+fPmCIMjdu3eZmZnnz58P48Oam5v9/f2ZmZnZ2Njy8/O1tLQkJCRaWlpgz4ULF1ZUVGhpaXFxccH6k8+ePVu2bFlBQUF0dDQ7OztsHB4e3rRpU0NDw/j4eGdnJ3ViMKUOVvqB3URERGAVzYcPHy5atAhWrTx8+PC8efPa2toQBNHR0XFxcUFm/zzAIo1NTU3URn9/fw0NDQRBCATCmjVrzM3NYXtRUZGwsDDUU5tjwObmZg0NDXFxcTs7u9bW1qGhoYaGBnp6elVVVdgH+rvLysrg2127dk3TO4NlQmERyDnO9V3Qvrh+N7QygzRo/JOora3t7e2l/mr/Rn5SjP2P+yPIxMQEAwPD1N/oBAIhICDg7t27UB0JgsfjSSQSTIKbe8yQkJDa2lpZWVmY2xwREXHkyBEuLi5PT8/x8XFra2tYmzEnJ+fRo0cUCkVWVnbnzp0AACwWm5ube/z48cHBQVhHGADQ0tICtSMuXrw4d051enr6oUOHAABRUVHUoK7S0tLR0dHh4eHs7Ozx8XExMbGBgQEdHZ2+vr7Ozs7GxsbS0lJ3d/cckDPjmB8/flRUVPT19SWTyX5+fhYWFjgcTlpaes+ePceOHbO0tAwMDIyNjQ0JCcnNzW1qakpNTe3s7PxPAeVZWLt27adPn+bNm0cVX7OwsIBVDBISEiYnJ/X09KCwlLu7OxqNrqysnJiYuHbtGjMz8/Hjx5csWWJjY6OkpJSZmenl5dXX1ycsLPzgwYOlS5c+ffp0wYIFWVlZsDJnRESEv7//unXrFi1a1NHRAaNVWFlZ+fn5+fj4GBkZYcuMEInElpYWGGJvbGz85MmT/fv3Z2VlCQkJMTExwfiYefPmfW/JShQK5e/vDwDAYDBcXFzUO0ChUCQlJeeYD0REROTWrVuJiYnv378XFhbeu3fvhQsX5OTkoIrc9zI+Pv7nJAHQ+LdCM3do/BT/9BCQPyB7rj/OaKzAxpcvXw4MDAAAuLi4qDZTRETEsWPHMBgMLAosEScx28hnz5719fVFoVBXrlyxsrIqKCgwMDCAySna2tovXrw4dOjQrl27SktLTU1Nly1bJiAg0NbWBis3Dg8Pr1q1aunSpezs7FB5YLazODo6Pnz4UFNTU0lJyc3NjY2NzdraGnpb1q5da29v/58ihwC0tbU5OjrevHmzoKDgwYMH0NwhkUhLly51c3OLjIx0dnbm5+c3MDAQERFpb29ftWrVVNtrRvbt27dt27YVK1ZARUyIgYGBp6fn/fv3oSsHACAiIpKZmamnp/f+/fv58+cLCAjY2dmFPwmfccz58+dfu3Zt6dKlMICXSCQqKSlZWVmJiIhA5TIsFhsTExMSEuLu7g7NO2Zm5iVLlnzvAqypqQmLGzEwMKioqOzZs+fly5dNTU2nT5+GZt+bN2/Mzc1h1e9pkMnktLS08PBwTU1NEokUGBjIyck5Pj6emZkJPX08PDxUoTcAwKdPn4KCgqDZMTdVVVWqqqp4PL6trW3ZsmUAgLq6uhlreM52+LQWFArl6OgYFRX1+vXrlJSUwMBA2F5XVwfNX0FBwTnKUPn5+SUlJUVFRUHBUQkJCTo6upycHHNzcyEhoS1bthAIhKamJhERkcrKSgMDg9bW1rt3705MTBw5csTX1xcAoKuru2/fPn5+fg4Ojg0bNlAnQIPGD0Azd2j8FHPsT1AolLGxMTo6OmpYBmR0dHRsbIyTkxOW3AUAIAjS399PJpOZmJhg59HRUSYmJjQa3d/fD4v5MjAw3Lt3Ly4ubs+ePTw8PIcPH960adPBgwcBADU1NYcOHSKRSPv379fW1g4NDS0uLiYSifT09MzMzCMjI5qami4uLmQy2cvLa3BwcGRkxMHBQUND48yZMyUlJSwsLIWFhQ0NDfr6+vfv32dgYPDx8fH19aWjo0OhUGQy2d3dPSwsLDEx0dbWlkKhEAgEISGhNqe22S48JibG0dHxwIEDJBLp8ePHTU1NAICrV6/u2bPn5MmTExMTZ8+evXnz5mz6vOHh4UlJSXfv3iWRSCdOnGBjY9PS0pKSkiopKfHx8Tl58qSUlFRfXx8AIDQ0tLGxMTY2Vl1dPSsrC67ZFArl8+fPFy9e3Lhx45s3b+Z4dlDy3dnZWUNDQ0hISFFREQAAdwKOHz8OixZCmJmZr1+/zsnJWVBQQH2aFAqFm5ubn5/f19f34sWLw8PDAAB6enoBAQHqk50Denp6YWHhqcW1qbCwsEx9Ky8vv3Xr1tOnT/Px8ampqS1fvhw8mXnMqqoqZ2dnFRUVJycnKDoG50kNcJk3bx5sHB0dhYEgaDSahYVlYmLiDyc8lZs3bw4PD9va2qLRaPjpBQAcOHDA1tZ29erV1dXV0tLSsBQ4/AgNDQ319fXdu3fP2toajUY7OzvfuXPH1NS0ubnZ1tZWXV39wIEDbm5uoaGhSkpKjY2NXl5eGzdu3L9/PwCAmZn5xo0bioqKMGAIluru7u4GAEREREDNNUlJSVdX13379mVnZ7e2tsrIyFy4cKG3t/fq1asIgty/f9/U1BSHw+Xk5ECP1Yw4Ozuj0Wh3d3deXt61a9fu2LFj69atUVFR0KA5dOgQVIfNysrav38/MzOzhYUFNzd3c3PzbAMaGBgICAi8fPnyxYsXMO4HAMDKyhobG5uSkkIikTw9PaGmx+jo6NGjRxkZGclkMolEgrKm8O6dOnVq6dKlTExMW7du/a5nRIPGNGjmDo2fIi8v78uXL2QymY6Ojp6efmJiQkJCYsOGDc3NzQkJCYcPHxYWFn7+/DlUGKirq8vLy+vp6SkqKuLh4YmNjUWhUI2NjXfu3ElJSREWFk5JSYFxBvz8/AUFBYqKip6enteuXbOysrp582Z2dvbz58+ZmZlFRESoy2RdXZ2VlZWYmBgLC4u3t/e6deu0tLT279+vpaVVU1OzePFiZWXlGzduODo6BgcHP336tLKysqSkxNbWNioqatu2bdHR0c3NzQ8fPtTX19fQ0Ni7d+/Vq1ePHz9++fLlo0eP7t27d3R0FIZgv3nzhoWFpbS0tKen5/bt22FgeskfKnR0dGfPntXQ0Pj48WNNTQ1UByMSidLS0kuWLCEQCIGBgUePHoX7DV8TFxdHIBAIBAKFQunp6Xn//r2hoaGoqCgHBwdMhIYliwAA4+PjBgYGsKIXFSwWu3Pnzu3bt69cuXLuxOmoqCg2NrYnT54cPXoUQRAogg0pLi6G1g8kOzs7LCxs+fLl375V8DPs27fv2bNnDQ0Nr1692r59OwqFUlFRKS8vT0tLq6+vn+NAJycnAwMDqIwBACgoKFBRUSksLPz48aOSktLatWtLS0sHBwdra2tTUlISEhLi4+N7e3vfvHljYWEBZtGxaG9vh96utLS0lpYWTk7OLVu2lJaWdnZ25uTkaGpqlpWV9fT0VFVVJScn37x5MyQkJDo6euvWrezs7AAAKK1lZGTEzs6uoKAADS9LS8svX754enqSSCQ6OjpoB1y4cMHHx+fIkSMoFIqBgYEq2hUdHX3gwIEHDx5gsVjYWFZWdvXqVQsLi8HBwRcvXhAIBElJyYiIiHnz5r1+/VpZWfn8+fMAACwWa2lpOTQ0BC08ZWXlzs7OOfax7O3tFRQUBgYGzM3N4Wd+0aJFHz9+pFAoCIJQs7rWrFnT2NgIPwlwqrMNuGLFCqiqu3v37qnd+Pn5p9WkUFJSmqYWRwWLxcIqozRo/Cx/ScQQjX8BMOKvqKgIAHDq1ClHR0cxMTFTU1MYh3j9+nVRUdE3b97ExcXJyclBgfHW1lYjI6O6urrR0dENGzY0NjYiCHLlyhUMBvP+/Xsymezq6gpjFZ2dnaEu9MTEhKqqqoODAzwpzGFpaGigTuPo0aOioqK9vb2ZmZksLCwnTpzo6ekBAHz+/Nnd3V1VVZVAIGAwmDt37khISDAyMq5bt05NTY2VldXV1RVBkGPHjsnKysKhNDU1bWxs4Gt+fv5Lly5Nvd6WlhZZWVkpKSk1NTVHR8c54n+jo6M3bdoEjQNxcXHqX0NDQ5mZmRkYGOTl5ePi4mYbAQAANSBTU1MzMjLev3+PIIipqam+vj7sU1hYeOrUKQRBjI2NtbS0YKOEhMSRI0cQBOnt7R0dHa2srMzLy5s/f/69e/dmO1FwcDCMS3379i0Gg7ly5QqCID09Pezs7BISEu3t7U1NTc3NzTC3yNHREUGQkJAQZmbmBw8eIAiydu3abdu2DQ0NQb/D06dPEQQhEont7e1CQkJeXl6tra2fP3+mUCh4PL6mpkZUVFRcXPz169fww1BUVARFtebPny8vL19QUAAPh3qf69atk5KSWrJkCZT0QhDk6dOn+/fvn3qfv76i+Ph4Tk5OLi4uLi4uAQEBJSUl+DhwOJyNjc3Y2JiIiAgHB8fZs2cRBCkpKeHj4xMXF8/IyEhNTZ1tzO7u7gcPHmRlZT19+vTBgwc5OTkIguTn56elpb17925ycjI1NTU1NfXz58+FhYXp6ekPHz7My8ujjvDgwQNWVta6urq+vr6vx/9L+N7Q9R8e8Df998PznDqxnxyExhzQQpVp/DtZtmwZBoNRUVFhZWXNyclJTExUUVH5+PFjVlaWurp6cXExgiAoFGrnzp3Pnz8XEhLaunVrUlJSc3NzXl4e/AW5evVqFhYWd3f3zZs3L1q0iI2NDQCwePFi+AOdgYFBUlKSup8xjba2ttOnT/Py8l68eJFAIDg5OS1duhQeSIVIJMrLyy9btoyPj09ZWZmPj49IJKqrq8Od+W8BXkJzc3NRUVFWVtbFixefPn0622ZAf3//pUuXtm7d6unpGRUV5eXlBduzsrKsra337t0bHR3t5ORUV1cHZtkouXnzZnBw8JcvX+Tk5K5du1ZbW5udnY3D4YqLi1taWkRERNLT03NycmRlZbdu3WphYWFnZ7d06dLR0dEzZ87IysqmpqaWlZW5urqSSCQouAFqZj5Rfn7+wYMH9+zZU1VVxcTEBE1Jbm7uU6dO7dmzB3q1Nm/e/PjxY39//4MHD0ZHR69duxaDwdja2hoZGWGx2OrqaklJSVFR0Rs3bhgYGAAAWlpaFixYAAA4d+5cYWEhDw9PYmLiyMhIVFSUubk5kUi8d+8eOzv7iRMnysrK5OXl169fTyKRxsbGYFUbenr6a9euiYuLNzQ0rFu3zs7Ojjrb1NTUafKiX2P5yRK4/9/bDtDxn9gyT3AL3Lp17hZwAACAw+OHD/seBgCAXQAAsLF4rjr3PDw81FrhVFRUVKivqVLkEhIzBGM9e/ZsZGSktrYWbuH8nfm1cXh/h5oONGhMhWbu0Phl0NHRtbW1sbGxwSAPY2Pj9evX4/H4+fPn43C4iYkJFRUVDg6OyMjI0tJSWCoGAIBGo9+8eYNCoV6+fLljx47t27fDqB1vb29YuyUmJqa/v//riswAACEhocDAwLi4OGtra3p6ej8/v9LSUqpzh46Orr+/PycnZ/78+TIyMjt27Hjw4IGPj09ubu6pU6dERETs7OyampqGh4f7+vq4uLjQaHRrayuRSMRgMMHBwTt37gwODobaEXp6ek5OTgMDA9zc3L29vQsWLOgG3TPeBDY2NjU1tejo6OLiYjQaPX/+fBsbm/DwcHjJXFxcnZ2dmpqabm5u/pdnDj61sbEhEAinTp0SFha2t7eHoa+SkpIvX748ceLE9evXq6qqmpqakpKSbty4ISsrOzo6SiaT4Q8pZmbm9evXd3R0EAgEBEH09fUXLVo0SxoTCA0N9fHxmZycpKOj4+Lioq7W27dvV1ZWRqFQFApFTEwMALBnzx5NTU0ymSwkJDQwMEAgEOjo6KKiopiZmYeHh+fNmwfl7uETef/+PdS95+Pjgz5HLi4u6F6Zipub22wfJJhXD+nu7u7u7r506dLr169DQkJmOwT8tvX1Z40AEQB8gEGZASj7RRP6PdCsExr/emjmDo2fAo/HIwgyMTFBIBBwONzp06f7+voEBQUDAgKsra01NDSUlZVtbW2FhYXV1dU/fPhgbGwcFxfX1tYGAPD29raystLV1d2yZcuiRYs+f/5MjcjZsGFDVFTUwYMHY2NjAQDv379vbm7u7e2F5UYCAgL4+fm9vLwYGRk9PT0fPny4cOFCAICxsbGlpSWMTkhISKCnpx8aGiovL793756zs7OTkxMDA8P27dvXrVu3b9++HTt2QIFSWVnZ5uZmLi4uV1fX8vLy7u5uISEhNTU1Ly8vBEFYWVnNzc3p6OiuXbuWm5tLT09PoVAsLCzEb4jPeEPo6enPnz8vIiKipaUlIyNDbf/2zSQAgKOj4zTz7sSJEydOnICvHzx4QG1funTp14fDdKo/RFxcfMZ26Aya2oLBYGCmDwCAatnAey4gIDC1JxaLlZOT+5azfyN37tyBEei+vr5/mPn8a/kxQ4dmN9Cg8feEpohO4weBi0G0cHR8fLygoCCRSBwfH8fhcCMjI/7+/oqKim/fvvX29l6+fPnOnTuZmZk5OTlhRGdhYaGNjQ0dHd3o6Cg/P//ExMTo6CiM52ViYuLn54elYrq6ugYGBpiZmdFoNB6Ph06owcFBenp6mNe6cOFC2LO7uxumPXNycjIxMZHJ5I6ODgAAKysrkUjk4uJqb2/n5uaGvrOhoSFmZuYZE4K+9/J/XpnhT9N2oIlI/AAzPuLvOuSffpNnY9pt+SdeJs0q/X38bRXRaeYOjR/kn/gd92v5X/7G/Ns+/Z+xNr5exWNFY+/du+ft7T1HdZlph0ybwHd9SIaHh1VUVEZHR5csWQIDxVauXDk4OKisrIzH45csWZKcnJyQkGBmZtbb27tq1SoeHp65aw0AAD5//iwpKamlpZWZmfntM6Hy5cuXw4cPX716lVpD6OuL+gG7cEbGxsYGBgY4OTnp6OhgJtfQ0BAjIyMWi4ViXqysrGg0uqOjA4vFcnFxjYyMxMXFKSoqwkqYo6Ojg4ODGAwGVsGGwEborsXhcNSPxP/yP97fzd/W3KE5s2j8FLRvjf9Zpj16AoEQHx9PJBJXrVo1Tb77hxkbG7tz546Pj8/+/fs9PDz+sP/X9s23fz5ntI0wGExmZqaVldU3mjs/ycjISEVFRWlp6cqVK6GbGAAwPDxcWVn5/v17OTk5PB4PM8ARBFFXV/+W2oN1dXWqqqowkHwqxcXFXV1dRkZG8O2HDx94eXn5+Pju3LlDJBItLCwwGMzHjx/j4uIePHiwcuVKHh6esbGxr6O2IdeuXYPBXhgMhkAg0NPTb968mYeHp7OzE/peOTk5YU0BAEBHR8fDhw9RKBQ9Pb28vDyMQO/r69u8efPChQvZ2dlZWFgCAgIKCgq0tbVv3LhhZmbW3NwsKSlZUVGxZMmSyMjI9PR0Ozu75ubmlpYWWFezs7PTxMRERkaGlZXV1tYW+l7b29vNzc2XLVvW2tqqqKhIzRug8b8JzdyhQeNfzi9XU5+RyclJPz+/xsbGo0eP/ipzB41Gi4mJkUikGWs0t7W13bx5c2pc82ycOXPmy5cvkZGR31s3yMrKat++fbCm3zQoFEpzczMajYbKl3PQ3d09MDBAJBKpJaMaGxt5eXmZmJgIBEJLSwsTE5OAgEBWVhYs9Gxra0tPT6+srBwREZGenr5nzx44EwwGs3379kOHDhUVFXl4eOBwuNbWVmqc1sTERGlpaVBQ0MDAgLOzM0xqy8/Pd3V1bWpqIpPJcBzYc9euXTk5OUQi8dKlS7Aeo6Oj48TEhJSUVGlpaVNTU2Nj48mTJ9nY2JKTk9FodE9PT0RExIULF2YLn8rOzv7y5cuiRYvu3Lnj4eFx4cIFGRkZHh4eJyen5cuXGxsbu7i4tLa2Hj58GADg4eEhJiZmZmZmaWm5fv16aO7o6Ohs2rTJ19cXyqwCADg4OIhEIgzFQxCERCJBFYtTp069f/8+Pj7++fPncD4UCsXAwGD16tWBgYHv37/fsGFDeXm5oKCgo6Pjx48fExISmJmZoQecxv8yNHOHxj+bn3dY/NLp/F2Ydpk/vAk39/2BFfygpgQbG9uXL18UFBSgAtS3A/P8p7UAAFAoFA6H09bWZmdnh+schUKhSoD19fXp6el1d3dv2rQpNTXVwsICKkZ9TXd39717996/fz88PCwtLc3JyQm1GtLS0oqLiwEAKioq2tras02PiYnp8OHD/f39dHR01J6NjY2XL18ODg5mYGBITU1dv379bIc3NjaqqalxcHAoKytHRkZ+/vxZXFx88eLFiYmJhoaGnz59Wrp0qZqaWk5ODi8v75o1az59+iQnJ8fNzb148WIAAB8f3+rVqz9//iwnJycgIGBvbw8AWLt2rY2NzdWrV9PT06k1GuLi4nbu3JmZmfns2TN7e3t9fX0eHp5169ZBb1RWVhZ1SteuXbt161ZBQYGYmNimTZtGRkZ4eXlNTU0PHTq0ffv2+/fvX716taCgAAAgKip6+/btlStXHjhwIDg4eI6HuHjx4qNHj0pJSbGxsZ04ceL8+fMYDObVq1eNjY3Kysp5eXkdHR0wzwAAEB8fn5aWlpWVNTQ0RH2gy5Ytq6ioCAwMZGNjgxoa0tLSzMzMM4ZbkMlkcXFxqu1FIpGamprevXsXGRmJIIiKigoszB0aGqqjoyMkJOTg4ODq6gprK9D43+XPL/VD49/B36RaF/ABBALBzc2Nl5c3KSnpDzv/YQukoaFBWFiYm5ubKjr9m8jMzFy5ciXMG6fy7NkzBQUFEon0Y2NOuyjq24mJieLiYqpWeV1dXVFRUWVlZWtra1FR0Zs3b4hEIoIgnZ2dRUVFJSUlk5OTs90f4ANevHjBxsZGT08vLi7Ox8d3/vx5BEGgRtW9e/eEhYWfPXs2ODgI+3t5efHz88+fP7+0tBRBkLi4OG5u7suXLyMIEh0dzcfHl5ubiyAIhUI5ePDg0qVL161bJyoqCqs+rlixAofDycvL29raioqKwmJO1dXVTExMDAwM+vr6kpKSJSUls104iUTauHHjmjVrKisri4qKysvLEQR5+/YtBwdHVlZWamoqOzt7cnLybJ8NYWFhLBabl5f37NkzSUnJZ8+eIQgSHBzs7+/f29ubm5vLxsaWkpIy252HmmX5+fkIgpw/f76+vh5BkKCgoISEBNjH0tJSUVERvm5tbQVTFMIhUIQEVpucio+Pz6NHj+Dr4eFhfn5+WP5xbGzs48ePUz88hw4dWr58OXzd09PDzc2NRqN1dHR27969aNEi+OBgkj8swNjR0bFr1y7Yv6ysDADQ2to67bZMu0xvb++ysrLR0VEXFxcoJ1JeXn7ixAkpKamqqqri4uKampr+/n4EQZqbm7W1tW/evFlXV7d8+XLqiRoaGqqrq9+9e2dlZSUoKFhdXY0gCDs7++3bt5GvFOB1dHS2b99OnQCJRJKVld2zZ097e/vHjx+fPXv28eNHPB6/evXqU6dOdXZ2PnnyhImJqaWl5VfVKqQxB7QygzT+tfyZGyQz7lLg8fjKykpGRkZqsvTPA8UmUSjU1GRyAACsYTjVJ4IgCIlEIhKJOBwOtpPJ5FevXqHR6NWrV/v4+CxYsABWwSeTyZOTk1CoHIvFAgAmJycvXrz47t07Z2dnOjo6CoVy9uxZHh6eyMjIt2/fOjo60tHRIQhy9uxZbm7uV69eRUREoNHokydPwlJ+AICSkpIrV65wcHAMDg7a2NhME5SYSnZ2dlRUVGJiopKS0qtXrwAAhw4devLkCTc3t6io6Nu3bxkYGDo6OpiYmLZt25abm4vFYo2NjcHiWW/RyMiIoqLihw8f9u/fDy8fAMDExHT79m0BAYEDBw7o6endu3fP1NS0trY2ICDg6NGjHBwclpaWjx8/Xr16NYFAaG9vBwA4ODjk5ub29vYCAKqqqoKDg+Pi4nh4eI4ePQolIQkEwtq1aw8cOKCvr29ubg71whYvXmxtbV1ZWZmcnDz3o0Sj0Xx8fGQyWVpamtrY19enq6urqalJoVDU1dX37dsHrGc+HOrPr1ixgpWV1cbGZu/evbW1tWfOnIHipt3d3cPDw1++fJnt7GvWrIHJifr6+iQSCVbRFBQUpBbD5OHhqa2tnfsSZoRCoRw7dgwG3xCJxK6urhcvXpiYmDAxMcnIyFDVy6cB95kyMzN9fX0VFRWLiorgNtLXg8MXd+7coTZOTk7OkdJIJpOxWOybN29KS0vRaPTQ0JCRkdHt27e7u7vXr18fERHx7t278+fPX7x4saenx8bGprS0tLm5eWRkpLW19d27d3v37jU2Nj516tSCBQvq6+vhToyDg8OlS5cKCgqgKtnhw4ePHTvGwcFRVlbGysq6Y8cOIyMjIyMjNBrt7+9vZGREIpGGhobu3LkTEREhLS1dW1tbUVHR39/f2dm5fv16bm7uH7jPNP410MwdGr+AnwxYfvTo0Y4dO8rKyhYuXFheXk4t3FJVVdXT00P1FMxmV71588bR0ZGDgwMWypuYmMjIyODj42NnZ//w4QMDA4O4uPgcxWByc3N7enpIJBIHB4eOjg50rFy4cOH06dNTZaQAAMXFxcePHyeRSN7e3j09PTo6OvPmzQsPD/fx8ZGWloZxpgCAx48fBwcHy8jIHD58mJWVtaysTFtbm5+fX0ZGRkxMTEJC4uPHj1FRUVJSUklJSZ8/fwYANDQ0QHMHyoP39vZSKJQvX76gUCgEQaCdtH//fjY2NiwWu3btWm9vb1dX17GxMX19/ZUrV547d87S0vLNmzezmTvj4+PXr193dXWNi4uTkpKKiYlxcHBITEwMDg4+duyYn5/f9u3b8Xg8BwfHo0ePlixZcvjw4aampv3794Ojsz41Q0PDDx8+eHp6amhoUBtHR0cdHR3Pnj1LJpNPnDgBV00XFxdXV9egoCAEQWCyRlNTk7i4OHXJ19LSmqrlxM/Pz8bGtnz5cqgfSSAQNDQ0YFViLS2tsrIyPT29Waf1bUB9UwAAdFG9e/dutp5mZmbnzp2b1rhixYr8/PyLFy+ysLDAspCzHY7BYI4fP66pqfnhwwcVFZXdu3dzc3MzMDDACuCysrKfP3/u7e3t7u7m5eWdUb8TNn748IGZmZmfnx+6ad6+fVtdXT0yMpKamsrNza2kpJSQkODp6fn06VMsFnvu3Dk2NrZHjx4BAEpLS+vq6oaGhlJTU3l5eRUUFBwdHVNSUq5evdre3j4xMVFbW0utXl1eXg4AqK2tvXbt2vz5848cOWJmZpaYmJiWlnbnzp3Gxsbs7OwZLxMqwLx+/bq2tvbLly/GxsbJyclnz55VVVUNCwu7c+dO7uv8jAABAABJREFUfX29srLyvHnzTpw4MTAw4ODgMDY2pqCgQCAQLl++fOTIEUVFxdLSUhhp5ODgAO3C4OBgT0/PFy9eSElJxcbGRkREdHV1PXjwQE5ODo1G19TUDAwMwAls2rSppqbGxcVFQECgu7ubk5MThULFxsYyMDC0tLRoaGhoampCi5zG/y5/7eYSjX8u1D1h6E66fv26k5NTTExMXV3d2NjYw4cP9fT0dHR01NTU4HY0BAqPr1+//u7du7CluLgY7q5bWFhYWVlZWVnBdm9vb0lJSU5OTl1d3YCAAGR2aZ4rV64AADZs2ABbmpubMRjMvHnzZGVlHRwcAADbtm2jdv76cFFRUWtr6wcPHmzcuHF4eBi2p6enKygoUGWqEAS5fv06AGDv3r0HDx5kZGSEtZ4RBFmyZMnJkye7urr09PQmJydh57S0NPBfFwYkMjJy8eLFUC5q2bJlhw4dgu337t1jYmKCaThU7t27x8LCAl1LkPPnz/Py8qakpFhZWdHT058+fRpBkOHhYUFBwdWrV1++fLmtrY3q7/japUIkElNSUjZt2uTq6kpPTw9DHBAECQsLExISmtp548aNixYtWrhwIQMDAxaLncOZhSCIv78/9XlBVq1atW/fPgRBiEQiCwsLFNdcsWLFnj17pl4vgiDq6uri4uI9PT0Igly6dAk6d8bGxjZs2MDLyysvL5+ZmQkPkZSU9Pf3h68jIiK4uLjgaxcXF+gGwuPxsErTbF48GxsbVVXVycnJvr6+zs5OBEGKi4uxWGxDQ0NLS8uiRYvk5eVnc2ZJSEjgcLiampoPHz4sWbIkJiYGQZDnz58DAE6dOgXTvFevXj3bnYc2h76+vpqaGgaDgXJvPT09ioqKoqKixsbGLCws9PT09+7dq62tFRYWZmFhMTMzc3BwCAkJQRCktLRUUFCQhYVFXFx82bJlfn5+yH/VvhgZGaHyhpGREfXeAgCWLFly9+7dly9fwkYVFRUMBgN7Uv8hPH/+/NatWzdu3IiPj4efvbq6unv37t25c2dycrKwsDA+Pp7qGi4vL4+Njb1169aMHkPY8vHjR+i5ePTo0VTP198NmjPrT+Bv68yiPXUaP8hUc0dXV3fx4sUwSXXTpk2PHz+GX7Lbt2+3trZesmQJtHi+fPkiISEhKSm5Y8cO+KWMIAiJRCosLDxy5Agej+/t7YVfvjU1Nezs7H5+ft3d3SIiInB1n0Pa0MXFhbrkwD0SRUVF+H0dGhoKAICqjTN+Uy9duhS+vnnz5tjYGPVP27dv19HRob6Fa8m9e/cePXrEzs5Ola6Mjo4ODw+PioratGkT1WpJTU3FYDCjo6PUw6FmAlUX2tvbG0EQPB7PycnJycn5h+aOpaUlLy/vvn37bG1t7ezsoH4q5MyZM46Ojvz8/Nu2bftP9vJXi66xsbGoqCjcmhIUFAwLC4N/+trc8fb2hmqg1GOnXsW02+7v7y8pKYkgSFNTE1RUVVBQoJo7HBwcMDIDJgDX1tYiCMLOzm5oaIggyIcPHwAAfn5+4eHh9PT0Dg4OpaWlUPB87dq1U0NPZGRkNDU109PTx8fHmZmZmZiYoJ4ofCJOTk62trYAgKdPn85m7ty9e1dVVRXeeSgNiyDItWvXAAA4HM7c3PzTp0+zmTvl5eU2NjZubm4ODg43b96k/jUyMtLS0tLBwWGqNf/1BGAwyvbt27du3UoNtYE8evTo1atXJBJpYmICbuCNjIxMTk729/f39PRAJXNq4+jo6NDQ0MjICIIgExMTIyMjE/+F+uGhUChT30LweDy157QQsR/j2wPg/obQzJ0/gb+tuUNzZtH4BeTm5lpaWp44ccLOzm7JkiULFiyQlpZ2dna+ceMGFot1cXG5cuWKpaXlzZs3u7q6KisrRUVFnZycTp8+vWXLFhwOB0WyYOkwOCA7O7u8vPzt27dTUlJQKNTWrVvnnsDUkAIUCoVGo6Wlpbds2QIAsLW13b9//1RfCRUYEEA1QbZv3z40NMTExDTjKczNzQ8dOhQeHs7Gxnbu3DmYURISEnL79u1r1651dHTMnZHExsYmJSXl4+OzaNGimpoaKSkpAAAWiz106JCfn199fT2UFVu4cCEOh0OhUGQyub6+npGR8erVqzt37vTy8iovL9fS0lJRUSkrKzt//vyaNWvi4+PDw8PPnj3r6OiYmZlJDQn6Gm5u7uHh4by8PCUlpZ6enoCAADQavXHjxvPnz3d1dUGl9+vXr7Oysu7Zs2fDhg0rV64EAEhKSoIlYLYbAgBwcHAoLi62s7NrbW2F9sTChQsLCgoCAwOPHTu2ePHipKSkVatWnTp1SkdHJzIycnh4+N69e0pKSgAACQmJgoICJycnfX39kpKSM2fO4HA4mJUNp/f+/ftdu3aZmJgkJSV9+fKFk5OTkZExMzMTj8dDP5S5ufn8+fMHBwcBAK6urgoKCuDtzPPctm2blpZWRUUFCoWiJsk7ODjIyMigUKjVq1dPSw2byrJly27evPl1++7du2GG19yseLICLPzP6wflD0D5///ncgDS/3CMfwD/1gxHGv8maOYOjV/ArVu3oqKiREVFdXV1nZ2dFyxYgMfjiUQidRWBL8rKytjY2ERFRQEAaDS6paWlqalp0aJFU4dCEAQAwM/P/+LFixcvXuTk5CQkJBw4cODJkyffPh8UCvXly5fc3Fw1NTUKhYIgyIyRm7BQbH9/f0lJiZKS0rVr165cuZKdnQ2rstLT07979+7Tp0+SkpKw/+TkpJWV1a5du6gjNDc3t7e3FxYWvnz5sri42NfX18PD4/3797GxsWQy2dfXV1BQcN++fQAAX1/fmJiYHTt2+Pj4XLhwwdHREZoUpqam165dg1G0KioqqampOBxuxYoVvLy8ixcv5ubmdnJyYmFhERUVTUhI0NLSam9vt7S0PHToECMjo6Gh4Y0bN5ydnfn4+GxsbKjS3F8TGhrKz88/Pj6OQqF0dHQAAPb29o8ePTIzM2NkZIQPq7a2FhbqvXz5cmJiIhaL9fDwSLiSMIcpICAgkJKSQiQS6ejooMV5584dMpkM7zZMoYKvlZWVlZSUEAShGmSMjIzr1q2rqqqCbxMSEgAA0tLSHh4eampqAAA0Gt3Z2blgwYJFixZRPyRr166dOgFFRcXZ5gaZYRnO+6rTj1Qb/iZ+JqbtlxgQf04V0H9WrVGaZfY/C83cofELCA4ODggIcHR0NDMzIxKJRkZGS5cuFRcXz8/P5+Li6urqgnKSu3bt0tPT27dvn7Ky8tWrV/fs2QOXMR4envv378vKyqalpRUWFmZkZJSVldnZ2e3du1deXj4oKMjQ0HCOs2/atCk7OxuFQnFxcVlaWoaHh6enp5eXl587d27NmjWJiYlycnKPHz/W1NSc8fCIiIjjx48LCAgYGhreu3ePmZkZAODp6dnZ2SkoKLh3714rKysLC4vg4OCenh4pKan379+3t7cfPHgwKSnJx8fHwcFhYmJCVVXV399/ZGSEiYlJUlLS09MzICBgZGSEGh2JRqMLCwsJBMLY2Jienh5VWVNCQuLly5dDQ0MAAF5eXlZWVgCApKRkQUHB8PAwIyMjtZyMtLT0q1evhoeHJSQk4I4UJyfno0ePJiYmcDjc1ML5X8PIyOjr6zut0cLCwsLC4uvO69atW7du3RyjTQWFQk3bUkKj0dTMNbjTBl9jMJg/HC0mJsbW1jY6OhoAgCBIRESEsbHxN87ka35+Gf5rl8afnP8/dF2vrKzs7+9fuXLlHNuKNGj8ADRzh8Yv4NWrV5qamrq6umg0GiZdAwC6urqghQE3MAAAurq6aWlpwcHB9+7di4qKoq61QkJCt2/fzs3NXbZsmZmZ2cKFCycmJvz9/dFodGtra2Rk5I4dO+Y4u7W1tY6ODgaDGR8fhwrhOjo6SUlJME3JxcXFxcVljsN1dXV1dXWnNQYFBU1raW1tXbVqFcxC4uDgqKmpmZiYYGdnZ2dnn9ZTWFj4a01yFAoFE8e+RkhICKYgTRvk657TCvjS09Pz8PDMOOb0s/9DVr41a9bcuHEDVv+TkZHR0dH53pn/5Vc69wS+14KhUCjQD0uhUGCmN4VCycvLy87OPnv27K+d29+LF//38p+1e0TjbwvN3KHxC+jq6kKhUJOTkwwMDDD+pqOjg56efmBggEwmMzIyUmNrdHR0VFRUYIju1BEUFRWnOiZkZWVlZWW/8ezbt2//unGOCjTT+NY1gAMA/SmdfcDK5JXgD2q+/PX8/Ao32wi/fe0sAaDkuw/6HUsjiUR69uzZxMTEkiVLvv1j+TU/cMd8fHyCgoIYGBhERESYmZmrqqo0NTUHBga+fPnyA+YO+CeYDnV1dXAPFRaP+CeZaDT+3tDMHRq/AF5e3mkt2dnZ6enpJiYm0dHR1ABkCDMzM3QY/Sp+5gvx36cp8fV6NtsKRyaTJyYmYB2XH4ZEIg0MDHBxcVHVAL6RlpYWW1tbZmZmmNUPvXhgdlHx7OzsiIiIixcv/qFG1S9ncnLSzs6ur6/Py8uLau4gCNLb28vBwfEtTrofY2Rk5MyZMz4+PjU1NRUVFXFxcUpKSvb29mNjYzAg7MeAiV0wJAu6gFEo1NDQ0Pj4OAaDYWVl7e7udnd3P3ny5PLly0kkUn9/Px6PZ2Ji4uLigpUVcTgchULJycmhp6c3MDCgejNPnz6dnJwsIiICS1/CxvPnzz99+pSBgUFbWxvqvH78+HHfvn0iIiIUCuXixYuwxA4AwNfXt76+HgZ7zRExRoPGD/IXZoXR+EczNRH967+WlJTs2LHDwsKiubn5F57xVw31F57it/LtScL19fXe3t7q6uqwetAPAwsf/0Cplfb2disrKxYWFiwW29fXR22fLZncyckJABAdHT33sGQyOTo6+vPnz987n7kpLCysqKiY2vL06VMAwIsXL759kO/6dAEfMDExsXv37qampjNnzsCE/9evX79//z4uLg6Hw4WGhl64cOHSpUvU6glJSUlnz54NDg6+dOnSxMTEjB8GIpEIC0bAmCo6OjooEZqfn8/GxgaLR8ClISsrC0GQ8+fPAwA0NDTo6OgyMzMnJibWrFkDANiwYcOBAwe4uLgcHBzg4L6+vsuXL3d2dubk5HRxcYGNPj4+AAA7OztBQcFFixa1t7cjCOLp6amiokIgEHx8fFRVVWGxqyNHjqiqqg4ODn78+BFMEc345f8kaYnov5u/bSI67anT+EGmmjt/2n9/wkXN3aG2tvbw4cO/pH7J3HR0dFBL+8zdOJVvN3eGh4dhQnhRUdHPzPPBgwcnT5784cNjY2MBAHObO5OTkzU1NW/fvr158yb1zjc0NNTW1g4ODlZVVVVVVcEln0Qiubu7AwBOnz5dWVm5b98+WL9nYGAgNjZ25cqVFRUVExMTcITm5mZVVVVJSUlvb+8PHz709PR8/vx51apVe/fuLSsrk5WVXbZs2fPnzz9//mxpabl161YTExOY5YcgSHV1NT8/PwMDg4mJibW1Naz+Z2tru3jx4rNnz46Ojm7atGnFihVQCGy2S5ubqZ2p9Y0gsOKlq6vrrVu3AACwhOPo6KiBgUFeXt67d+82bdrk4uIy44dhcnKyqKhIXFzc3t7+/fv3T58+pdorqampLCws8fHxJSUlBQUFeDz+7t27KBTK19d3eHgYFmUuLy8fGBjg5OS8cOECgiDFxcVcXFyfPn2Ki4sDAMBKWoWFhVxcXJWVlQiCrFq1io6Orqqq6smTJ4WFhfBE/Pz8Xl5eGhoaUPXl7du3CIJwcnJev34d+UojjGbu/OP425o7NGcWjZ/lz8wf+eXOpu+dfFBQ0I0bNzQ0NDZu3PhrZzKN3bt3m5ubm5ubT2308vIKCwv7YfcTgiBEIhGDwaBQKFZW1p07d/6nTi4AsB0GxmKxWKo02KNHj9avXw8DrcLCwtrb242MjBgYGFasWAEAyM3NLSgo2LZt27QTBQcHNzU1KSoqWllZURvPnTvX3Ny8atUqWBUQMlv49lRqa2thGMe8efO2bNkCnUdGRkYVFRU6Ojrp6ekAgL1794aGhnZ1ddHR0cnLyxOJxPj4eFFRUbiH4eHhkZ2dbWNjA+tZw6gXb2/vxsZGMzMzf3//L1++7Nq1a+3atfv377e0tIyJidm9e7ewsLCGhgaCILdv346JiTl8+DDVw7J48WKYOpSYmEgikWD78ePHlZWVKysrmZmZ5eXlSSTSHErpPwORSOTh4YmIiAAA5ObmwoKN8fHxGRkZK1aswGAw/9Hw4p/hWBwOt2bNGkZGRmFhYTk5OTExMR8fn8+fPy9cuHDTpk27du1avnw5VVwMtp84cQIAsHbt2p6ent7e3mXLlrGyssKPzaJFi/r6+qBqmKysLPwkSEpK9vX1wXpIjx8/jomJUVBQGBsbW7VqVVpaGg8PDxqN/vjxo7a2NpFINDMzm1Go/NGjR3MIv9Cg8QPQzB0afz1/Vfjk9xpPBAKhvr5eXV19mq0Di8dMVaAEAFRXV3d0dLCysiooKEx9KyQkVFtbS0dHp6CgwMTE1NfXV15ePjk5ycXFBbdbAAAhISGPHj0iEon8/PwTExPr169nZGQ8d+5cbGwsgiBqamoHDx58//69kJBQf39/enq6u7v7hQsXYEGdGXFycoK/3ZWUlLq7u+Pj47m5ufX19eno6LKysm7evNnU1LRz587Fixd7eXkVFxc7OzsrKCjExsZCBdDdu3d7e3s/ePCAmZk5JiYGh8PV1dUxMzO3tLTcvHlTRESEOnMAwIEDB5KTk5mYmK5cucLDwwPvVUhIyNWrV3l5eXNycgQFBbW0tL79tktJSTU3Nz9//hz6syBnzpzR1dXl4uLq7u5OTk6Gst6CgoK+vr4HDhw4fvw4tef79+/v3Llz7tw5RUXFysrK+/fvQ3MHi8UGBwdraGicP39+z549MFLewMBAVFQ0MjISSnSB/waRTBWFndoOppS4XLhw4bZt26qqqohEYmtrK9xn+tNAoVAEAiEzM3NoaOjgwYPbt29nPvfHEXJoNLqtra27u3vhwoUUCqWlpWXqlYqJidHR0b169UpJSamuro6LiwvG4dHR0cH617W1tdzc3DgczsPDIzY21s/Pb+/evfn5+by8vPPmzQMABAUF2dvbL1iwoKen58CBA42NjTw8PAsWLJCRkTE0NCSTybdv325ra+Pj41u8eHF5eXlZWVlNTQ0A4OnTp9ARRoPGr4Jm7tD4Wb7XaJjRuIGi4lgsduq3LYlEglm4aDQa/qZHEASPx+PxeCg/jsFgJicnUSgUDocjEAh4PJ6BgQGDwRCJRA8PD1jexsfHB4VC+fn5SUtL19fXHz16lEKhHDhwYO7SMlVVVSEhIQAAaGfA8jmurq4vX76ctuNSXV2tpaWFQqH09PTQaPTx48cFBQVhVZ7GxkYsFnvv3r0tW7Z0dnZCYUsZGZmysjIAAKyt7OjomJaWdu7cuWPHjnl7e8O4irq6OhwOp6urC+NF4FTr6uoYGBjk5eWHh4f9/Pw4ODgoFIqpqamMjMzly5fd3d0jIiLALNKZurq60dHRvr6+J06c8PX11dfXLygoGB8fp1AoKBTq6tWr6enpVVVVW7ZsuX//vqmpaXt7u76+flxcnIeHx86dO9va2k6dOlVWViYvL3/r1i0MBgOjU62trcXExIqKiqgnKi0traioKC8vZ2Ji2rNnD1QvBwAoKSlt3749JycnPDwcFhn6dnA4nIiIiKCgINxRgMDo+DNnzvDw8Njb2xcUFBAIBBwONz4+DmVWqUD9BBKJVFJSoqurC2XUAABEItHFxUVZWVlTU5OqCj45Oamrq0u1deYAg8FQJU7xeDwajaanpw8NDbWzs1NQUBAVFV2yZAm1M5lMnrGu9zdCldQAAKBQqPHxcQDA6OgoCwvL5OQkjO2l7mOZmJjExsa6ubkBsVkHZGBgePjw4bp160JCQvT09GDxRjc3t3v37o2NjXFzc589e5abm9va2rqvr2/NmjVbt27Nz883MDCAOy6MjIyXLl0iEonFxcXx8fEyMjIAABcXFyjnWVRUlJWVBS8/KysrMjKSnp6eRCJt2LABFhN//Pixrq5uYGAgOzu7r68vrDv64MGD27dvKygo7N69OyYmBlpLNGj8Sv4SFxqNfwFTY3dgPKOvr++dO3diYmLq6+sRBBkbG4uPj79x48a9e/eoARMzxhMQCAS4DU7VM0IQpKioiOolsba2ho23b99mZmaGxW+cnJwGBwfp6OjWrl2LIMinT58EBATs7e0RBMnOzlZSUsJgMEJCQnp6eurq6uXl5QiC3LlzZ+PGjQYGBkuXLi0sLJwt0qW6upqHh+fcuXPZ2dn09PRUecuRkRFDQ0MDA4OphwgLC0dHR+fl5S1ZsqS+vh4KXamqqkItyRUrVuzfvx/2hBpPgYGBY2NjfX19ZDJ5YGDAzs4OhuOEhYUBAGCM7deaWfDap2p1IQgyPDyMxWKPHz/+7NkzuGbMdkXQ7IDCYYWFhQCA4eFhqCYGY3fy8vIYGRkHBgYQBElJSVmwYEFdXZ2AgMC1a9fgOMnJyXAFUlVVzc/Pp84tLy8PSp1DDh48ePDgQfgarvEIghAIBEdHx23btsEHOlU6Ki8vD/xR7A58kZGRgUajqRqu0GSkhkjb2dlBfdb4+HgAwOPHjxEEga66z58/i4iIwDCChoaGZ8+eUQ9hYWGxt7cfHx+nnm50dJSZmZmqTkplz549TExMt27dov6ptbUVi8V6enomJiby8PCEh4fD9qtXrwIAoDYqlUuXLrGzs/9Y7A61KBQjI2N7ezuFQtHW1gYAKCgovHv3Du4pwlvq4eFhbGzs4uKybdu2ioqKOQK55OXlxcTErK2tDQ0Nu7q6YKO7u7uhoaGuru6GDRuioqKmTn7t2rWwfDZsERMTO3Xq1MDAQHd3N7UbkUjs7e1ta2uDnyLI4OBgX19fX19fb2/vVEG60dHR3t7eqT0RBKFQKH19fdMC42ixO/84aLE7NP7NwN/6WCwWZqjC7f19+/Z1dXXt2rXrwoULL168gMvAjHh5eTEzM5eWlnp5eR06dOjcuXMAAFdXV25u7pycnIGBAeq/n7Nnz6qpqcXGxubn55eVlbGzsyclJd24cQMAsHDhQmhaAQA2bNgwb968lStXXr16Fe6pAADi4+NDQ0MzMjIwGIypqamxsTHYNfN8oNSXtbU1Hx9ffn4+IyMjbGdhYYHKVlM709PTE4lEMpmMwWDExMTQaDSJRHJ1dfXy8nr79m19fT21mjMPDw8PD8/evXuZmJhgxVgMBlNVVeXl5RUQELBixYrMzExqteVvB4fDkcnk2NjYoaEhkw8mf9gfQRBmZmYKhWJtbQ3lPGH7xMQE3CfQ09Pr7+/X0NCQlZWF9azb29s7Ojru3r3LwMDw9OlTTU3N3t5eVlZWEonU19c3MjIyPDw8MTHBx8fn5uamra09OTl5+vTpvLy8iIiItLS05ubm6Ojo0tJSMTGxtLS0gICAtWvXcnFxtbe3t7W1AQAaGhoGBgbY2Nhmq5p45syZ69evUyiUjRs3ioqK3rt3D/qS9PX1L1++TCAQHj582N3dHR0dbWFhERAQ4Ovr+/Tp06ampuzsbAkJCVNTU2dnZwMDg7KyMnNzc6rXT0hIaN68eXv37h0aGjIxMTEzM7tz5w4Wiw0PD797966Pjw80Ii9fvhwbG7to0aL09PS6ujroiePj43v8+PHo6CiZTL58+TJViotMJvPz80/bAtTV1eXk5NxWMz3I6Vvw9vbeuXMnGo1GEGTevHkoFComJqakpERaWnrJkiWJiYk1NTXQEwf/4fwfSTMPSCaTh4aGoJ7J1HZocH/Nrl27pgqnAABQKBQM9qJmmwMA6Onpp5WcAAB8XYQTMmM1ChQKNa0cFw0avxCauUPjF2BsbOzu7q6trU390r9161ZSUlJXVxcajc7IyAgNDd2wYYOZmdmMhx86dCguLq68vLy3t7eyshI2Ghoa+vr6rlu3TkREhFpRzdDQ8NSpUxcvXhQWFoZxoJs3b4Y/5VEo1LJlyx4/fjx15OXLl1Nfw1yeCxcukEgkKSkpKysr63rrGedDoVB4eHigLANMu52DqKgouP45OjpCh8L+/fsTExMdHR19fX1LSkqovhVJSUkpKamvnRoWFhZ0dHSysrK3b9+G6xYdHd3o6OjLly9XrVoVGRk5f/58c3NzFAo1OjoaEhKiq6ubkpIiISGhp6eHQqFaW1v19PTa29uvXLkCZnfQodHohISEysrK0tLSgIAAdnb2O3fuAAAyMzPXrl2bkZEBX8MNGD09vbCwsNDQUHgTEARxdnYWFRX18vKKiYnR1NSE5lpISMjhw4cBAIGBgRwcHK2treLi4iEhIaamppGRkUJCQubm5giCcHNzKygowJrUjo6O/f39BQUFGzZsOH/+PAqFcnNzi4+Px+PxmpqaM0pGdHR0fP78WUVFRUtLa3R0FOqD8vDwuLi4jI6OwiKW+vr6+vr6/Pz8AIDKysrr16/r6OjAngCAc+fOBQYGTk5OYjAYOPPu7u5Pnz49f/6cl5cXjUYXFRW9ePECAGBtbb1jxw4CgUAikagh4XZ2do6OjnR0dCgUimod0tPTf12MG354NmzYMK1RXFxcXFx8m++PmDtffwKnlu2WkZGBvqRv5/Lly/X19SUlJRMTE1RT/rtgYmLy9fWFluIPHE6Dxl8Czdyh8VuAKbtTA3EQZNZ4ZHV1dQqFEh4eLiAgQCAQYKOzs7OlpWVpaemRI0fq6upOnz7d399vY2NjYWGBIIibmxsrKyv8mf7hw4czZ87s378fGkzNzc3/8ewAUFNTw8HBAX9H2tvbl5aWOjs7IwgyPj7OxsYG6meej6ioaFdX17lz5zZu3Nje3h4UFHTr1i0xMbG6urre3l4ikVhRUTE2NiYnJ8fExOTv709HRxcYGKisrFxVVSUtLS0kJNTb2ysjI3Pt2rXOzs6YmBhmZuZTp05t3rz53bt3hw4d2rlzJy8vr7CwMBaLRRDkyJEjQkJCw8PDhYWF+vr67Ozsenp6JiYmR48eXbBgwcuXL2Hi8ebNm7ds2RIcHJyVlVVVVXXjxg0GBoaysrJDhw7t3r27v7/f3Nw8C2TNeEVYLHbhwoWJiYlPnjyRkpIKDg4GAOTl5YmIiEDflrS09KJFixYsWAD7j4yMkMlk6nrPw8Pz+vVrWJYwLS1NSkoKPlkrKysVFRU6OjoEQejp6eHaqaen9/r165GRES4uLqiJxs7O/vjx4y9fvkhKSk6tSDnbdsI0BAQEoqKipjUKCwtfvnyZ+hb6sKjY2dlNfUtHR4fD4aj6ZQCApqamt2/f1tbWwu2ZEydOmJqaAgCgGNm0yoGwcW7IZLK6unpnZ6ednV1MTMy3XNcf8puKXpqamiooKEyLcPouEhISxsfHYXYeDRr/FGjmDo1fBrRUjh07ZmZmpqqqysXFlZiYaGZmNjk5yc3NDQXAZ6Suru7AgQMvX75sbGwcGBgIDg5eu3atnZ3d5s2b6ejo9PT0oMsjLi7O09Pz0KFD9PT0TU1N1J0bY2NjGCNJT0+fk5Pz6tWr5ubmhw8fAgCgktfOnTs3b94sJCS0fPlyMTExAMCWLVt27tw523zc3NwMDAyioqIeP35MIBAOHz4Mj0pPT1dXV0ej0Y8fPyaRSPPnz2dgYMBisWfOnEEQ5PHjx+fPnz969GhQUBAjI2NtbS2CIMnJySkpKTClyMrKysTEhEgkJiUlaWpqCgsLYzAYGF8CbUFqYWImJqYHDx7AdmojMzPz/fv3AQBwKws2ysjIpKWlUVv8ff1nvCIcDicsLBwRETFVJ3Wqe9HS0tLS0hIAQCKR6Onpg4ODjYyMqPWLsVgszC+bhoCAwIzeN6hcNhVBQcEZ843/HGa2Gw6DDS//uw2zHuR05+z23f1Tp9kAAADHCMeOnT72fTP5bcDTTUsO4L/y3wz15//X+F3ZkV8/Xxo0/v6g5vjNTYPGHFC/SWGZ/6tXr2ZlZbGzs799+zYgIGDTpk0VFRVHjhwRFhbu6ekJDAyEMZXTJALAX63b8PVkvut7/9ixY0+ePCkpKWFhYSGRSCYmJiQSKTU19VdP81uZ8fYiJ5HR0dGlS5eqqqrCwnRzEBgYmJCQsGzZskuXLlGFHf5MvhaR+PPn8Pv482suTL2f/9Cb+Wtv2owmII1fSG5uLpQsTE9P/931yb4L2u4OjV/Dzp07p+2XLFu27NmzZ3944D/6e2fz5s1XrlxRV1dftGjR5OQkiUSCGzB/K/6zyNmBOBAX5xv3xwcYgwpQcfvC7d89sW/hl3w8vteKnQ0ikUgikX4s3uVrSCRSdHS0vb09VXNqNtLT0y9fvuzs7EyNu/8x4E1A+aLeG703MzPLzc2Fm3O/6v7QoPF3hmbu0PgF/EN/Nf48SkpKnZ2dwcHBGRkZ3t7eampqX9ej+2uZbRn7Oz+yX6j5+gshkUjm5uatra2vX7/+JQNGR0fv2rVraGjoyJEjc/csLy9/+vSpo6Pj1MbKykp+fv6vk6G+5uv7KScn5+7uvmnTpuTkZGrUMw0a/25o5g6Nn+Vf9rvwB9dadZBXkAcKfvVsvp/Z5v/1YyJ6Edvb21tbW9+9e+fq6jrbgHg8fu/evTDdaeHChb9yrr+a32rD4fF4e3v7P/QGzsj169dXrFgxNU8QAKCsrHzlyhVqNNXExERvby8bGxs7O/vQ0NDIyAgdHZ2goGBnZ6eSktLDhw8NDAyox3748MHAwCAsLExWVpZAIPDz8/f09MCSBMLCwuPj4yMjIyQSCcacgSlPn3qLXF1dL1++rK+v/+rVqx+4Iho0/nHQzB0aNP6P7zXd/ilegBntAAqF4uzsDFPQN27cOJspU1tbOzw8XFBQMDw8/MsnFhUVpaSkBKUi/yr+H3tnHldT1/7/dabmOY0aNCKUiISSkKiIzEo0SINIZJ6KZB6S3KQoKgqlQaMikgxNigaaB5Xmczrz/v2xvs/5nafSzc1t6Fnvl5fXPmuvvfbae5/T+uxrXeu63rx5Exsbu3DhwmnTpg1R7fnz5xISEvAmVFdXP3jwwMXFhUQi5ebmlpeXr1u3DlYrLCyMi4vj5+d3dHSEgRnDwsLs7e03bdqkpqZGoVAWL16so6MTFxcXGBjY3NzMmQJ2dXW9fv36zJkzs7Ky/P39/f39hYWFP3/+HBER4eXlNX36dO5V+omJiTU1NbGxsTk5OVQqdfXq1WlpaQEBASoqKjk5Ofv27bt48eLatWtv3hxqRvL8+fPz5s2jUqnfeQMRiD8CJHcQ38vvMC3yE5wxh5A1MP3h1atXZ86c+S+d/d8AJrhIT0+Ha7C/hLa29smTJyMjI/F4fHt7u4CAAPfC7L6+vnfv3omJicFF7HQ6vaSkRFhYWF1dvaSkhEajSUlJwRVeZWVlXV1dRCJRXV1dREQEANDQ0ODt7e3o6LhmzZre3l4FBQUVFRUAQElJSUdHB5FIHD9+vJCQEIvFcnNza2trO3HiRExMzK1bt+Lj46WkpIqKiqSlpVVUVO7du+fm5rZjx45B+w8XDGpqao4bN66np8fd3d3KyqqiouLgwYPv3r1zcXFJTEzMycmRk5ObNm1aeXm5i4uLqqpqZWXljh07Fi5cCAB49+6du7s7k8lksViwNW9v7+jo6La2Nh8fn5SUlGvXri1ZskRYWLikpGT58uXr16/39fV9+vRpXFwc+E8a1AkTJkyaNAkGY4SFBw8ehLGYcTgci8XaunVre3t7S0tLX1/f9u3bb968uWvXLhKJtHnzZiMjIxi4nMPKlSt37969fft2TtCdadOm6ejoPHz4kEAgkMnkSZMmwWzzQ8BZ34dA/E/wk6I3I4Yd3Ekkvr+1lJQUIpFoampKo9H+QU8G3R6UtLQ0AMCdO3c4JfX19TCj5L1797hr3r9/HwDw4MGDIZqF5TCIcHV19bf2/B9Ap9O580hgGMZms8lkcr/C8+fPOzg4nDx5kruf3HBK3r17BwCoqKjg3nvu3Dl3d3cXF5eXL19iGFZfXw8AsLW1NTAwWLFiRUFBAayWnZ0NTQ7y8vIlJSUYhn38+BEAICcnt23bNhjQz9HREfvP812wYIGAgACMnIRhWEZGhra29vHjx48cOXLgwAEYcj4+Pl5LS+vUqVNWVlYwecinT58OHTokIiJCIpH2799/4MCBrKysnp6eCRMmzJ49G8Ow4uJiTU3N1NTUQR9TS0sLOAQUFRXr6uqKi4vV1NRSUlIwDKNQKKKiolJSUpysBTU1NaNHj75+/TqGYbm5uaqqqk+fPm1oaNDU1AwPD4e3ev369fCGKCsrwwvx8/MDAFy+fLmxsREGN8Iw7MmTJ5z8Gy0tLQCAZ8+eDezb+vXruR8cvM/FxcVJSUkWFhbcjxielwO8z2/fvuUurKioAADU1dW5u7sHBQVxHnS/HwjnY0ZGBg6H6+jo+J9KqoCSSPzb/LZJJPA/VVshEAB8/Pjx+fPn/QpbWlpmzJixZs2av12l8p0oKysnJiZyzwuMHDly//79AgICMIUCBzMzM15eXuzvIjU8fvzY3Ny8oKAAhvQFABQXFz98+DA1NTU+Pt7S0hJmeIY8e/ZMQkJi9OjRVVVVAAA6na6urq6pqZmSkjJ16lQeHh649Karq2vOnDkCAgK6urqZmZkwbUVra6uRkdH06dM1NDS2bdsGO9bW1rZo0SI3NzcNDQ1JScmmpiYAQHR0dFBQUFlZ2Y0bN/bs+WIMmC9x9erVrVu3amlp1dXVmZubU6lUGPunra3tzp079fX1CxcuhF0KDg7u7e3t7Ow8cuTInDlzqFSqsrJyY2MjPz9/dHR0Xl5ebW3t8ePHAQANDQ0bNmxISkoKCgqC0YwAADNnzpw0aZK3t/fevXsPHz48f/78yspKKysrmPJ969atmZmZpaWl0tLSBw8eJJFI/v7+Pj4+hw8fnjVrlpCQ0KNHj4qLi1NSUj58+ODn5/elLOswMcXo0aMVFBTGjx8vICAAE3XhcDgajXb8+HFOUMHKysry8vLFixcDAPT19QkEQm5ubnV19YcPH2Ahh5EjR06YMAFG6qPRaOrq6s7Ozq9evaqrq4MeNoaGhtz5278SAQEBVVXVFy9ehISEcAc0gqKkpqZm6MPV1dUvXryor6//6tWrNWvWfOvZEYjhDZI7iB8DnU7v7u7u6emB6Zo5hW1tbT09PZw8U9XV1QsXLty8efPly5dNTEyg7mlqaoqNjVVSUoJzGRwiIiLMzc1XrFjR1tYGAEhMTFy4cGFMTAxMhQhDAw+ko6PDwcHB1tbW3d29u7ubTqdv2rRp165dAACY6NvGxqa4uPhHXXhjY6OVlVVUVNSTJ09gSU9Pz7Jly6ysrOrr66dPn25lZfXp0ycAgIeHh5OT0/Hjx9etW7dkyZL29nYeHp6TJ08SicRDhw5t3bo1KCjI3d0dAHD37l15efkLFy4sWLDAxMSkubkZ3kwikXjy5EkPD4+zZ8/W1dXBe5KTk3PlypVjx44FBQVJSEgwGIxTp04dO3YsOzv7woULx44dg4d/PXp6epGRkSIiIu3t7V1dXUwmE856HDlyREFB4dixYzBPZ319/Y0bN06cOCEqKmplZTVx4kQKhYLH4+Xk5ODUzLhx4xQVFWEWJFFRUSKRuHXrVpiU6kunptFoLBZr27ZtAIBZs2YpKytfuHAB7sIwrF9CpREjRri6uvr4+Pj5+X19riU8Hs/pAIZh3CubdHV158yZA71z4uPjKysrSSSSnp7epEmTAgICAAB0Ov3t27eRkZHgP3mjAAAsFuvjx4/p6elGRka6urrnzp1jMBgUCsXe3h5GYMLj8SQSiUKhMJnMjRs32tvb9/X1dXV1tbW1lZaWXrt2rb29Hf5AxMXFU1NT4XeVEzP6w4cPS5cuTUtLMzc3t7S0fPr0KQBAQUFhzZo1Li4u1tbWGzZsyM/Ph5Xd3NxwONzq1avhdCEHY2Pj9PR07hIWi3XhwgUHBwfuvFcIxDAG+e4gfgAsFsvJyQkuWrG0tHzw4AEAgMlk2tnZ1dXVPXv2bNGiRdCPISwsrKyszMrKKicnh3vYYzKZ4eHh06ZNMzIygiVhYWF2dnZz585lMpne3t4hISEEAiEzM/Phw4fbt2+fMWPG9u3bdXR0uMMEQ8TFxaWkpGJiYl6+fPn27dvp06erqamNHDkSACAlJZWdna2mplZWVtZvmcw/ZvXq1T09Pdy5NqdPnz537lxzc/ONGzd2dHSEh4d/+vRJRkbm+vXrAQEBdnZ2tbW1CgoKcIResmTJpUuX5OXlOa/jVCp1z5493d3d8fHx3Am2CATC3LlzFy5cyO1yYWdnV1lZycPDM2HCBH9/fxwO19fXV1BQcOPGjaysLDqdbm9v/01xYths9pUrVxITE83MzAZdVD969OjJkyfTaDRRUdFx48YNTAE2kA8fPlhbW+fl5U2cODEiIgIO2AAAIpHY0NBQUVGhoaGRnJwsKSkJM0q6uLikpqZGR0c/e/ZsiODXAIDDhw/n5eXx8fHBsGZDUFtbGxkZSaPR5OTkoJ7Izc2l0WgZGRnd3d1CQkJWVlbi4uILFy5csWLFqlWrPD09HRwcPDw8AAA2NjZbtmxpb2+XlZV99+5dQEDAihUrLCwsrly5gsPhdHR0FixYYGZm1tTUlJqaOm/evN27d+Px+OnTp0O7kaSkZHx8/LJly7q6uoyMjLZu3crPz3/9+vU3b95oaGg8efLkxYsXjo6O0FFaTU3Nzs5u5syZnJv/4MEDGo22ePFiJpPJ8RYnkUhhYWEeHh719fW7du3ifk+Qk5MbM2bM3z6UgwcPJiUlZWVl/W1NBGJ4gOQO4gfQ2tr65MmT27dvAwA4cfY2b94MAEhJSXFycuIk6Nm9e/erV6/279/PSSYKAJCTk4uNjVVQUOCMnUwm89ixYytXroyKiqJQKOPGjXvx4oWZmZm6uvqECRNg5ufm5uaenp5B++Ps7Pz27dvu7u5169aFh4cTCAQ4jpJIpJ+Tcrm7uxv684qLi69atQoufsEwDBYqKSktWbKETqcPaueAxgBHR0dbW1sWi9Xc3Eyj0QAAoaGhdDo9Nzc3KSlp9+7d7e3tSkpKDQ0NLi4uixcvTklJgYlCxcXFhYWFbWxslixZ0tPT09vbO8SKKhqN1tjYCABobGwUEhKSlZVtaWm5fPmyv7+/u7t7fHx8dnZ2U1MTNBV8/PhRWlo6KSnp3bt3vLy8AgICmzZt2rt379WrVxMSEgoLC2FuslOnTtXU1LS1tX38+HHEiBEiIiJEIhGm2jh58mRycjKZTD5z5syuXbtgJlQzM7OxY8eWlpYuW7bsxIkTTk5Ojx8/dnFxKS8vv3Dhgq2tbX19/d69e8XFxW/fvp2VleXt7Q0jdEN4eXm583B9ifLy8s2bN1Op1MDAQJg8KzY2VlJSMioqKiQkRFtb28rKCgDg4eExfvx4MpmcnJw8ffp0eKybm9uECRM6OjrGjBljaGgoIyNDIBA2btxoYWHx+vVrLS0tWVnZZ8+eiYiI8PLy3r17t6ioCI/Hz5w5k5P0e/78+SkpKe3t7bNmzYIuTf1yjHPTzyvZ09PT09NzYDUCgRAYGMj52NvbSyaTQ0NDZWVlTU1N+1XuJ2s+ffqUnp5+//59zjUiEMMeJHcQPwBBQUFHR8fm5uaYmBhOjic6nX7t2jUBAQGYeRtSX18fHx9/7NixoRsMDg5+//49NHjg8fimpiY4H8RisTQ0NGCdsWPH9vO24aCiolJXVzdq1CgAwKlTp2JjY/tZ8n8s0K7Dse6wWKz8/HwHBwf4kc1mx8fHQ9ecyMhISUlJQ0NDCwuLixcvamtrV1VVvX37FuY0xTAMh8Px8PCcOXNm9erVioqK/Pz8MTExc+fO3bdv34sXL0RFRZ8+fXrp0iUAgJWVVUxMzLx589hsto+PD5z7YLPZfHx8GRkZs2fPPn/+fHl5ubq6+qZNm77U89DQUDjuzpo1i4+Pr6mpSVJS0tTUdNeuXUlJSbNnzx43blxWVta6deu8vb3z8vLy8vLodHpycjIcs93d3clk8tmzZ9ls9sOHD3l5edvb2xsaGry8vJhMJpwrmTBhAvSXunv37pUrV4KDg+fMmcNZ25WTk3Pjxo3u7u6EhARYcuXKFRaLRaVSiUQi1CXy8vJBQUH8/PwMBoPBYPRL2CkrK+vrO3imMG7mzp0LvdQ5nDt37ty5c/2qEQiEgT5ABAJhUOsRdyIwjsIYOXIkNCX2Q19f/287+T1ERUU5OTlNnjw5NTX1bytDM2e/TKgIxPAGyR3ED8Db2zsjIyMoKCgvLw864QIA+vr6GhoaoDopLCyEbhx8fHySkpJVVVXjxo0rLi7u6+ubPHnywEkTGxubkydP1tTUUKnUV69eiYqKwhAmeDweWiMAAHV1dUPYLZSVlYuLi8+dO7d9+3YzM7NZs2YBAHp7e1+8eMFisd69e5ebmzt27FhRUdGenp7c3FwWi1VaWjpy5EgtLS0REREajbZx40Yajfbu3buhjQelpaU+Pj46OjpXr16NiIg4depUVlaWkpLSrVu32Gy2tbX1ixcvamtrjY2Ni4qKoFuugoLCjBkzYLCZffv2iYqK1tXVOTg4MJnMEydOyMjIWFparlmz5vz58+rq6mfPnoWLjV1dXTdu3AiX/GRnZ5eUlKipqT179oxGo/X19cXHxxOJRNhVHR2dly9ffv78GY/Hjxo1asSIEbaHbQft/PLly6FPLoZheDxeWFiYQCBERERUV1crKipKS0s7OTlJS0uTSCTocTyQfhGBJSQkzp49O7Da3LlzOdOO/aID29nZ9atMIBA4dhEAAB6Ph+qKh4eHYw+jUCiVlZXp6elaWlrQGXlo0tPTMzIy5syZ87c1/1CWLl2qq6s7cuTIr7Ff4vF4zmsJAvE/ApI7iB9Aa2vrrFmzkpOT375929DQsHfv3qNHj86aNWvJkiWzZ8+m0+lXrlyJiIhYvXq1nJycp6fnypUr7ezs/vrrr/Xr11+7dq21tdXX17e3tzcuLq6mpmbDhg3jx48/depUUFDQggULsrKybt++bWhoCAAQEhIKDg7W0NDw8vI6deqUsLAw9K4YSHBw8M2bN52dnVVUVDh/3Ht6ep4/f+7n58dgMNLS0hQUFERFRbu7u1+8eOHv789gMNLT0xUVFUVERFpbWzU1Nc+ePUun04e2DGlpaUVFRXGXLF++nDuSDXfisFu3bvU7fGAJAICPj29g9F5TU9Pq6mq4bWZmBjegChyImpoaDPcyNJKSkgOzEHAXDmqo+B14//795MmTvby8vnIBlI+PDzShDVckJCSGEDr94lH9DrGyEIifDJI7iB/AsWPH7O3t4ToXERER6Cnp7OyMYVhwcLCAgEBhYeHo0aNh5W3btpWVlRUWFhYWFmpqagIAuru75eTkKioqenp6aDQajEq3ZMmSOXPmtLS0CAgIcNZ4x8fHw3XRBAKhrKxsiJdUaWlpuMCHO/S+nJwcDLHDzciRIwcWKigocBfuP7z/S9eORo5fwoQJE+rq6uTk5L4ySdkB9oEDEQf+7V79nvwRgb8RiH8b3N+GFUEgBgUN8wgE4g8FScB/j6ysLOjrlpycPH/+/F/dnf8Pmr5FIBAIBAIxzEGTWYh/CHo9QiAQCMSfArLuIBAIBAKBGOYguYNAIBAIBGKYg+QOAoFAIBCIYQ6SOwgEAoFAIIY5SO4gEAgEAoEY5iC5g0AgEAgEYpiD5A4CgUAgEIhhDpI7CAQCgUAghjlI7iAQCAQCgRjmILmDQCAQCARimIPkDgKBQCAQiGEOkjsIBAKBQCCGOUjuIBAIBAKBGOYguYNAIBAIBGKYg+QOAoFAIBCIYQ6SOwgEAoFAIIY5SO4gEAgEAoEY5iC5g0AgEAgEYpiD5A4CgUAgEIhhDpI7CAQCgUAghjlI7iAQCAQCgRjmILmDQCAQCARimIPkDgKBQCAQiGEOkjsIBAKBQCCGOUjuIBAIBAKBGOYguYNAIBAIBGKYg+QOAoFAIBCIYQ6SOwgEAoFAIIY5xF/dAQTizwN3GAc3sIMY3Obe6FeHA6wz8BDOUV8qRyAQCMR3gqw7vwU4HMD1HxwHL0T8crhlDZQv/TbAf+sh+K9fI/3k0Ze2EQgEAvFDQHLn9wXDAIaGvN8PblEydB1YjVOznxLiLhz0cKR4EAgE4keBJrO+C2h9wbD/MsNwPkKxwtnF0S6DVh6itX5HcbfPfSLEz2HglNOg9FNFX6qPNA0CgUD8BJB155/zNZpjoO7hVi2wnHMgd0m/1vq1M3AD8ZMZWutAo86g7jv9/ueu2c8OhEAgEIgfBQ5DloF/ykC586UNDgPVCXfJ0BLnb8+FQCAQCMSvJSsra/bs2QCA5OTk+fPn/+ru/H/QZNbPoJ8W6Wfs4d6LtMvvADKuIBDDGDSD/L8Jmsz65wxtqgH/PdnEWWbFvc05ZKAJh7uRfpNWSAwhEAgEAvFNIOvOd/ElxTOwzhAVBpYPWudL7SD18y+BXgGHAf+2oa7fl+SHnI67TU6Ag6dPn/b29hobG/Px8XFXfv/+vZKSkoCAwNe3z2nzbws59Pb2rlmzJj09HcMwKyur69ev8/Lych8y9OE/hOTk5IsXLxYWFr5580ZKSupbD4+Li9u9e/e7le/+jb4h/giQ3BmKb3IBRv7CfwRfrw5/7ZTWoAPSH32inwn3RXFv0+n0trY2IpE4YsQIPP57bdtfs9SOzWYfOHCgoKDg7NmzGhoa/6DN9vb2/fv3R0dHt7a21tfXjxw5krPr48ePRkZGGzZsOH78+NDNMhgMNpvNy8v7tx0YlOzs7Pj4+PDwcH5+/p6eHgqF8k1NPXnyhMFgzJkz55+dHWJmZiYhIaGvr89isTiFVCqVRCIRCAQAAIZhXV1deDxeRESku7ubzWaLioricDgAQFdXl7Gx8dmzZ81yzb6nD4g/GiR3EIjB+YXWnS8Jjh/epZ92om+FyWRGRESYmJgoKCh867FfvCgMc3Fx6ezsrK+vNzY2dnBw0NTU/O6eDkJ7e3t1dfWkSZMAAKWlpcXFxYmJiXv37v0auTMQMTGxY8eOrVq1ysjIaODeuXPnjhgx4m8bOXLkiLy8vLOz8z/oAADgyZMnAABlZWVBQUEdHR0oLwalpqYmMTGRRCJhGKavr6+jo5OTkxMaGhoVFXXy5Ek8Ho9hmK2trYiISFVVVXJyMpPJVFBQWLJkSX19fVxcnKCgIJPJpNPpLBZrzpw5WlpaAIDKysq0tDQCgVBZWck5UUVFRXZ2dkxMzPjx452dndXU1Nra2uTk5BQUFA4ePOjt7d3d3d3U1CQhIfHu3btVq1ZNnz791q1bwOuf3QDEcADJnb9nOM0W9fOM/p35x45KX7pGZH5rb2+/du1aQUHBtWvX+s2J/G74+vr6+PisWLHi9u3bP6rNXbt2hYSEvHjxQlFRMTc391+6A3Q63cHBQVpa+uTJk0wmc9y4ccHBwdLS0jgcjkKhdHR0cJtnGhsb+/r6BAQE5OTkvtQgNFcICgr2Kw8NDT1//ryKioqjoyN3+aNHj7Zv3x4cHPzw4cOtW7cKCgr+9ddfPj4+ysrKISEhDAYDLB6q/8uWLfvw4QMOh/P39zc1NQUAREVFXb58GQCwdetWJpO5b9++5cuXD3psS0vLunXriouLSSRSS0vL4cOHdXR0eHl57927t3v37jFjxlCpVAAADw8Pg8GwtLTs7Oy8evXq1q1baTTasmXLEhMTHz58qKqqGhQU9PbtW3Nz88LCwt7e3kWLFvX19V25cqW3t5dzroyMjKNHj8bFxT169CghIWHLli1iYmJPnjyxsbF5+PBhbGwsg8EQExPr6upatmzZrl27bG1tx48f797mPtTFI4Y1SO78PX/0MPmniJsfzs98ajExMW/evFFQUHB1df15Z/12Fi9e/PTpUwCAoaHhpk2bflSzGRkZurq6EhIS/+zwuLg4KpW6cuVK7sLx48dv27bNycnpa1qA0TRwf/fIy8rKAABMJlNOTm7JkiVMJtPPz4+fn9/T07OlpeXMmTNjx461s7MDAHR3d588eVJOTg4+0NjY2Ddv3rBYLBMTk6qqqsbGRk9PT2Fh4UHPsmbNmtjYWABAenq6lJSUo6PjihUr8Hh8WFhYaWlpeXm5t7e3s7MzPz9/YWGhqalpS0uLgoJCWlramDFjvuZiOYwaNUpVVTU2NpZGo3EKnzx5snnz5mXLlh07diw9PX3Hjh2wJpFIVFNT09XV5efnzwf5X2ozNTW1pKRk8eLFb968sbCwSE5ONjExWbVqlbCw8JIlS7KysoSEhNhs9pcO37FjBw8PT3t7OwBgwYIF06dPBwBMnjw5ICCAl5d37ty5nJp0On3JkiW2traamppbtmyprKwkEom3b98WFxcPCgoyNTUdMWKEl5cXhmGenp5UKjUnJ0dOTk5UVJTTwsSJE6Ojo7Ozs4uKiqZMmQIAIJFI06dPFxIS0tDQmDFjBqzW2tpqampqa2sLABg7dizI/qZ7jBhWoJVZfw8M+vcn/vtNqKurg291/5jz58+npKQMUeHw4cPz58/nfvkDAx7ctzJq1KixY8fKyso6Oztzuwv04/3796dPn75y5cqxY8e+ptmWlpaCgoJv7s2PAI64UlJSDAZj6Jq5ubkvX75kMplVVVXPnz/ndLiuru758+evX7/m3JCHDx+am5s7ODgUFhbm5uZ2d3fD8pqamufPn7958waOjg0NDbm5uUVFRQEBAerq6pxZiXnz5q1du3b9+vUqKipycnKFhYUAgOfPn2/fvj09Pb2zs5PTJTqdvnbtWmlp6UmTJtXV1QEAfH195eXljx07tnDhQj09PQ0NDe7Jjn4UFxfn5uYCAPz8/Ozt7VtbWwEAiYmJhw4dUldXX79+fWxsrKqqKgDg8uXL48aNu3v37okTJ/bv3w8AUFVVvXTpkri4+JYtW5KSkt6+fTvErNCmTZtMTEyKioru378fGBg4Z84c6Dejq6sbHBzs7++/bdu2vr4+AEBISMj58+c/f/7s7+8/a9askpKSoR9KP2bPnr1r1y7uwGnt7e3Ozs44HG7dunVHjhyBVhYAwPz580eMGLFs2bJTp075+voO0WZqampAQIC/v39CQoKmpibUbeA/UhKPx+NwuCFmsk6ePEmlUhUVFeXl5WfPng1VCJ1O3717dz9bWlVVVXp6+s6dO52dnWtra2E/B4XJZKqpqXFbv/B4fFdXl7+/f1xcnJGR0bt3/+V9PGHCBE9PT85HAoHQ3Nw8hERD/O+A5M4wBMMwJpM5dB0Wi8X8D0NUS05OXrdu3aZNm86dO/fPOtPY2GhsbOzv7//PDgcAkMnknTt3cr/CDmT27NmZmZkrV64cQpd8KyIiIgAAX1/f+vp6Nze3L1UbM2bM8+fPnz9/rq6uTqfTuXfBB8H9p7a7u3vlypWWlpZkMrm7u5tTn06nd3V1wVHwbzl06JCdnd3Fixc5JUlJSWvXrnV0dPz8+TOnED47W1vboKAgWKKiojJ27FgeHp6/DS5qYGAwffr0R48emZqaTp8+3cLCAgDw6dMnCwuL6dOn6+nphYaGwpp79uyh0WixsbETJ040MDB48+YNAKC+vt7S0nL69On/5zABQEhIiIGBgY6OTnBw8Pr16xcvXlxfXw8A0NfXJ5PJBw4c8PDw8PT0lJGRAQAoKio6OTkVFRV1dHRwurRhwwYajXb48GFTU1MLC4v29nZjY+OOjo69e/d6enq6u7tXVlZeu3btS1c0YcKEadOmAQD27t0bEhIiJSVFJBKfPXumr68vKCh469at9+/fGxoakslkT09Pa2vr0tLSoqKiJ0+efPz4UV5efsWKFdu3b1+9evWhQ4fk5OSg3hqUioqK0tLSCRMmaGtrT548WUVFBX4Bxo0bp66uPmrUKAzDcDhca2vrhQsXkpKS4P8tLS3f8xuBSEhI3Llzp6urS11dfeHChdevX4enTklJaW5uHkKmcMDj8fLy8gAAHh4eYWHhbw1Cm5eXd+7cuZ07d/r4+Jw6der06dOwKRMTE/j7ZTAYUBD7+Pg0NjYGBAT89ddfCgoKPT09AAA2m41hGPfvhUqlbtiwIT093cbGhslkBgcHAwBu3LgBAKioqLC1tdXV1R07diyZTIb1U1JSMjMznzx5QqVSoaYXEBCIjo5evHixi4vLqVOnvulyEMMMJHeGISEhIeLi4g0NDV+q0NLSYmRkJCgoKCEhMWrUqJaWlkGrkcnk8PDwd+/e/fXXX3fv3v3b82ZlZQ18ve7r61NTU+unA74eMpm8Zs0aNze3RYsWcZcXFxdzW4yMjIwyMjIKCgpycnK4q7W0tNy9ezc//4vW+yEYP378qFGjnJycXF1d//rrr/PnzwMACgsLY2JioqKiYmJiOJYMAICGhoaqqiq3diwtLd22bRuJRLKxsXn9+jUsDAkJycrKGjVq1LZt29zc3GBvCwoKLl++7O3tPWvWrOjo6CG6lJqaKi4unpKSkpqaGhgYyDFReHl5NTc3Q5fMtrY2AEBtbe2iRYvKy8sbGhrCw8Ph/MLXk5+fLy8vb21tfezYsfb2dngDLS0tLS0tMQxbtGhRUFAQtKUlJCTMnTv3+fPnra2tjY2NRkZGNTU1s2bN0tTUDAsLO3jwYFhYWG9v765du6DtJDY2dseOHWw2u6qqCgAAvUOcnZ09PT29vb1lZWUBAAoKCp6entDdldOl5ORkCwsLFxcXV1fXoqIia2trQ0PDkJAQEok0c+bMDRs2bNu27cGDB990mQAABoOhp6cnLi7OKWGxWLBXIiIi+vr6dDqdSCTW1tay2eyvWcxFIpGam5tPnjw56F7Ygre3t4SExOjRo1esWOHq6hoQEPDw4UMbG5tv7TykqKgIbvT29tbU1Fy6dInJZB46dOjYsWNQ/evr60tJSb158yY1NXXoH7Kqqqq9vf2jR4/27NmTm5urq6sLAKioqHj27BmbzU5KSoqNje1nTeHm/v37mzZtGjt2rJKSkrS0NMc1e/HixStXrrS2tjYzM/Pw8CCTyZMnT25qaho7duzUqVOVlZUfPHhw7dq13bt3s9lsDw8P8B97kouLi7m5+dKlS1tbWzdt2lRXVzdnzpzu7m5BQUEVFZUjR444OTm1tLS8evXq8uXLTCbz+vXrWlpat27d2rhxIzTmycjIhIeHd3R0vHv3buhXJsTwB0N8GQAweIfOnz9vYmKyYMECS0vLsrKyn9yNT58+Xbt27evrP3v2zMrKqqysbODjhSXBwcECAgJ379719/fn4eFpbGyEe5lMZltbW1dXF/ch79+/BwDMnDkTfmSxWJ8/f+7s7MQwjE6nf/78Gb6TUanU0aNHq6ioFBUV1dbWtrW1sVis+vp6Z2dnGxub9vZ2ToMMBuP169fz58+fO3cujORRVFRkZmb24cOHM2fOmJiY2NnZdXV1wZuflJQEAPj8+TPn8L6+vo0bN6qoqFhZWdnb28OzQ9auXbt06VJ4jfD/1NRUAICDgwPG9TT/FnAIgENg9erVZmZmsERPT+/gwYMYhq1Zs2bGjBlpaWlr1qxZsWIFi8WCFWg02qpVq/r6+jiNmJubz507Nzs728bGBofDpaSkYBgGrS+PHz/mPp2RkdGaNWswDMvOzlZQUEhPTweHBukoOASMjY2NjIzodHpJSYmEhER4eDiGYdOmTXN3d4ePRkxM7K+//sIwrKurKzAwsLGxMTAwEABw584dTjsjR448f/48p81BT4Rh2I4dO2D7EDivoa+vv23bNrjmKCMjA8MwqOTq6+s5Nf38/AAAQkJCwsLCRCLR3Ny8p6cHwzAlJaVNmzbBOmfOnElNTcUwDC75OXr0aL8+9Pb2kkikxMRE+DEmJgYAEBYWhmFYTU0NAEBXVxfDMLgIqLe3F8OwAwcOjB07tt9FcW8vXrwYAJCTk8N9ImNjY3t7e85HCoUyYsSIBQsW9PT0lJSUqKqqVlVVUSgUAoHw7Nmz06dPOzk5bdmyZerUqYPePXAINDQ0qKioqKiofPz40cfHZ//+/VAWJyQkYBgGfaesra0xDPvrr7+0tbVDQ0NdXV2lpaVtbGwGtgk/3r17FyoPY2Pj2bNnnz59GsOwO3fuTJw4EQAgJSVlamoaERFRV1eHw+GUlZW3bt1qbm6uqqpKo9FgOxcvXiQSiUQiUVxcfIiHjmEYNDLh8Xh4FgzDtm/fDgCQkJCAi8/3798/8N7C7Tdv3oSFhd24cSM0NDQpKYmzt76+PiIiIjw8nPNAmUxmbGxseHh4YWFhZ2fnu3fvMAx7/vx5ZGRkfHw8hmEdHR0RERH9finfD/xp/9g2Ef3IzMyE6iI5OflX9+W/QK7Kf8+LFy+2bNkCx7a7d+/u3r37Z569pqbmxo0bWVlZGIZRKBT4zg0AoFAo169fl5CQWLVqFadycnJyWVmZhIQEnU4fwnOTzWYLCgouXboUAKCqqkok/t/X4Nq1a87OzoqKipmZmWpqaoMeGxUVtXbtWl1d3Tdv3hw6dMjPz+/BgweWlpYvXryoq6tTU1MLDg6mUqmGhoZLly49efJkXV1dUlKSubk5p59nz5719vaeOnWqgIDAypUrW1pasrKyqqur1dTUpk2bxs/Pf+PGDREREQAuAAC2bdvWzwh/8ODB69ev5+bmMplMR0dHBoPBw8MDdx07dgyOtRzmzZsXGhoKx7nv4eLFi+np6TQaraamZt68eS0tLSwW686dO3p6etAbtLCwELpbwvrJycmJiYlnzpypra2lUCgYhpWXl0ObQT/i4uKePHmCx+MjIyOJRCKDwYBmj0GBK31IJJKWllZra+tAYwMfH5+Xl5e9vX18fLy3t/fr16+/Z/0Rt/0MtmNhYbFv3z4Wi9Xv1NnZ2ZznO2nSJDweb2ZmFhUVRSAQiouLOzs7hYSEcDjcwNk6+C2FU2CFhYUFBQXQWbgfFhYWS5cujY+PHz16NLQgzpo1CwDw+PFjBoPx7NmzefPmvXjxYggXkIaGhvLycgBATEzM69ev5eTkrK2tb9y4AR3LAgMD9fX19fT0+Pn5ExMTLSwsoCfy1KlT5eTk0tPTWSxWR0fH8+fPc3Nzx44dm5eX96UlY/Ly8u7u7teuXZs6derChQsnTpwoJCQ0c+ZMKyur1NRUNTW1VatWwc5v3Lixuro6MzOzr6/v1q1b3J68/eDh4VFVVZ08eTKFQqFSqTCoIA8Pj5qa2pQpU+h0OoFAWLBggZiYGIVCYbPZFAqFSCTy8/NzfhouLi4ODg7whvP5D/6V+P8L+A8BNmB79Xh5HfYCAAAhAA6BdvB/BkJf4Ot7eBAHoP7r/2sAyBtQ6QMALweUDOT1fzbKAcgctLMIxLfzq/XWbw20Bzx79gwAUFlZiWHYvn37Xrx40dHRMW/evNjYWAzD8vPzp02bBt/MMAw7evTomDFjdHV19fX1V61aBQsPHz4MC2fOnDlmzJhbt25hGObh4WFoaLhy5Uo/Pz9Yjclkurm5GRkZ2djYTJo0qaamBsOw5ORkAAA/P7+cnNyIESPi4uIwDMvKypo4caKsrKySktLMmTOheebRo0cEAmH9+vVhYWG8vLzV1dVfsu60t7ebm5tramquXbu2ubkZmihiY2N1dHRyc3MDAgKkpKT27t3LOWrFihWTJ0+G22lpaQAALS0tDMNevXplbGzs6OgIdy1duhT2mRsWi2Vubm5kZAQ/Njc3y8vLX7169fPnz1Qq1cDA4MCBA/DGKigodHR0UKlUbW3t9evXw5uvoKBgZ2fH3eCHDx80NTWhQ2JOTg6DweDsam9vHzt2LLd1Z+DT/BoGWnesrKz8/f3z8vIAAMnJyWlpaY8fP379+nVzczPnKHt7ezKZDLcfP34sICAQEhKSk5OTl5f3+PHjt2/fYv+x7kRHR2MY1tfX9+HDh66ursmTJ+vr6+fl5eXn54eFhQ1h3Zk3b56goODz58/hAHnhwgUMw1auXAmX4FZWVsrJycGXcgMDA2Nj40+fPh09ehQAcObMGQzD6HT6x48fZWRk9u/fX1dX19vb+6UTZWdni4uL6+rqlpSUVFVVwfJFixZpaGhMnjx54sSJhoaG79+/xzCMxWItXbqUSCROnDhx1qxZO3fuxDDs3r17AgICY8eOXbx4sa+vb3d3N4Zho0ePBgCcOnUKds/IyCgpKYlGo6WkpMyZM8fR0XHGjBmhoaEYhr1//37lypUaGhqLFy9et24dNCf09fVZWVlpaWnNmTMnMzOTTCaz2Ww9PT1paent27e3trZKS0svXboU+4J1p7m5OT09/enTp6mpqQkJCbm5uRiGpaenZ2ZmPnr0KD4+/sOHD5yj3r59m5mZmZmZCa1WhYWFjx49amxszMzMfPz4cUZGRlZWVktLy6DWHbhRXV396tUrTjn01B70y/alFgb9+EP4XzZvIOvOTwBZd/5g4AuomZmZm5vboUOHCATCo0eP0tLSli1bBgCAL3zQXl1fX3/79u358+djGHbhwgVjY2MAQE1NTUxMjJmZGYvFCggIWL16tZaW1qNHjy5cuBAdHW1gYKCgoMBgMA4cOJCUlBQYGBgTE2NhYWFsbNzV1QUAgJM+8vLy0EEPcu3atZqamvb29pKSkvHjx0dGRm7btq22tnb16tXQh7S4uJjjvjcQcXHxsLCwEydOXL16NSIiorm5WVpaOjAw8PTp0/r6+qqqqoGBgfCFG6KsrAzfjAEA48eP55RPnjx51KhR+vr68KOIiIiiomK/c+HxeAUFBc6qEwqF0tjYaGBgANcti4qKVlRUAADYbLa4uLiYmBgAQFpamuOuyMPDExkZeebMGc46Z1VVVTc3t46Ojjdv3lhbW0OPVwiNRjM0NPyya8E3g2EYACAnJ6eoqGjHjh3Kyso6OjoPHz7cvn37p0+f7t69u2jRIuhdW1BQ8Pjx47S0tMWLF7PZbCMjIzMzM39/f3t7ewBAaWmprq7uuHHjREREFi1aZGdnV1paWlVVFRYW1tHR4ebmZm9vHxISoq6ufvjw4VmzZgG9wfuzePHitLS05cuXKygoVFdXu7u7Q38FS0tLGIPu4sWL0Kual5c3KytLRkZm9uzZ/Pz8r169AgDU1taqq6sDAHx9fX19fTnrbgaSnJyspaU1f/780NBQPT29UaNGAQDi4uISExOh3p0yZQqUL3g83snJCbpn6evrr169GgCwZMmSO3fuJCUlOTg4wJkvAICrq2tbWxtcW378+HEpKSm49NrU1HTu3LkMBoNIJEJLnoaGRlhYGA8PD3T0hj9APj6++/fvwy8Gx7b08uX/txV8+vRpiEcpIyMDnxQ3X4rzO27cOO6P2tracGOI6Dj9UFZWVlZW5nyUl5eHLsD/gGEQ5BqB+B1Acufv0dPTKysrc3V1PXTo0JkzZ+Lj401MTG7cuAEXzbq4uDQ1NYWHhwMAmpuba2trXVxcent7L1y4sGLFClhYV1fn5ubW3t4eEBCwa9cubW3tM2fOAAB27doFRQlsytLScsuWLVu3bvXw8LC0tPzSdBKVSk1ISCCTyfb29jQaTV9fH+oqFotlZWX1NVcUGRkpJyfn7+9vYmIyf/78HTt2hISElJSUQK8IKSmpfl633MjKyu7cuRNa8hkMxocPHzo6OmCgszdv3gQFBbm6ukJ3HziYsdnshoYGzvikoqLi7u5ub29/9erVp0+fvnz5Eo7EOByus7Ozo6NDXFwcj8c3NjbC+seOHeOOyFJWVmZkZDRz5sxjx46x2WzoCcHh0KFD/Xxys7KyrKysXFxcvnKVODd5eXnq6upOTk7V1dVBQUEwiEhsbOzy5cvr6urIZPL69euheggODo6JiREWFg4MDIyOjl63bp2pqemJEydsbW2Dg4OXLVu2Y8cOGFyOSCRGRUXZ2tr29vbu2LFj165dgoKC69atmzFjRm9vL4vFysnJUVRUFDsnNmiXnJ2d582bx2AwmEymhIQEFJcwYElzczMej4f9AQBAB2EMw2RlZTs6OqSlpQEAioqK7969w+FwGIax2WwlJSVQMPi1HyUdBfPAM/YzIATAe7Dq8P+fMAXQqbcS2Bzmcq0VBEAQgHKw5vCa/18oDS7FXwLxXO0SgO/ZoRZCI/rxy8NbIxDDBiR3/p6PHz82NTWlp6c/ePBg8eLFFAoFADBq1Cg7O7v9+/cLCAg8evQIOg3o6elBB1UeHp6FCxfC1199ff358+fPmTOHQCBYWFjAAQnaJDZt2iQpKUmn06FMKSgosLa2NjAwqKys3Ldvn5CQEGflJFxhwWKxent7BQUFV69eHRERERQUxMPD09vbGxMTM2nSJCKRGBwcbG1tXVdX9+DBg6KiIgCSB72inp6eyMjI6dOnw2UapqamBALBx8fHwcFBVFQ0NTU1KSkJOoeyWKzq6uqEhITm5mZnZ+cjR45ISUkpKCh8+vTJ0dGxrq4uOztbSUnpwIEDPj4+69atCwoKKioqio2NNTY2joqKevz4sb+/f3JyspKSkq2trZmZ2dq1a52cnKKionR0dAAAmzdvhpYDAQGBuro6GxsbPz+/ysrKjx8/wq7OnTt32rRpoaGhXl5eAICRI0fq6urS6fT9+/cbGxufO3eO46L0/v379PT006dPx8T8/ysdM2aMpKQklHHfyt27d2EnuRk1ahS3RQHi6OjYL7ItAEBNTa3fMjEIPz9/DHcXAQAAfGU2AyKROGhNCQmJflH+uM1snF08PDxfE8tumA2xP9k68muNMdCt6m8jLv7b0Gg0NpvNZrP5+Pi+ZgE8AvETQHLn72lsbLxx4waDweB2tIR/UOLj45lM5rNnz2RlZSsrK9XV1UkkEgzTYmRk9Pr1ayMjo56enoaGhtra2s2bNxsaGr5+/drQ0HD9+vUhISG5ubleXl4hISG7du1KTU1tb2+3srKKjIyESXA4MdZmzJhx+PBhbW3twsLCjo4OX19fBweH0tJSMTExT09POBUFAFBTU+vq6hIQEBAUFNTX1+deJt0PdXV1CoXi5ubW1NS0ZMmShQsXAgA2bNjw9u3b8PBwCoUSHR0NHaKfPXt26dKlUaNGqaurv3///s6dO25ubs7OztOnT/fw8NDT07t586ajoyOcz/Ly8iIQCFCHmZubAwD09PRkZWWtra1Xrlx57do1aIXS1tYuLy+HUTE4YVK3bt3q6OjIZDKFhISys7NJJJK0NAAASEhIREdHOzo6TpgwYd68eUJCQvfu3QMA4PH4fk64ZmZmsrKyCxYs4C6UlZV98+bNP/vrP1Dr/HLQvMa3wp2y+99o/+c8kWHw3IeZhkb8ieCw3yf47u8HHCVzcp7DiQw8Hi8gIJCZmamnpwfjwcTHx48aNcrc3DwwMPDEiRMwENmDBw94eHjodDqbzY6LixMQEDA2NpaUlKTRaDD8zJ07dywsLLKzsy0tLYWFhU+cOCErKzt79uxPnz7BlSZ4PJ6fn3/u3LlCQkIAADqdnpyc3Nvbi8fjLS0tYfYcOKUlICAAxQqkr68vMTFRTEwMrvIYmD3qD82ZtX///mvXrtXW1nKWPg0kISFBW1tbSUkJXuOg185p8G9O/RuMLmh4+IFgGIb3wW/v3V5YWKihoQEX53O4devW48ePr1y58qu6xw3uMG7QHPVf830IDw9ft25daGjo+vXrv+Zcz549MzExmTlz5p07dyQlJWFhQUGBj4/PrVu3+Pn5OTVfv37t5+cXERHx9YnQtbW1LS0toaf8V/b/JwB/2r9JZ4YrWVlZs2fPBgAkJyfPnz//V3fn/4OsO3+PrKysg4MDgUAQERE5fPgw/CsgKCh49+5d6EcJY1Tw8PB4e3t3dHR0dXXBUXn06NH5+fnt7e1//fUXXAUKAFBTUyspKbGwsDA0NGxpacHhcJzVszIyMtDdpx88PDz9guwBAPj4+KCvNDf8/PwDC/vxq+3c/4TDhw/v379/CK0DAICRf4cNXy+5/m0Dxq+Ce0z61kvrN57BueCGhoa0tLTa2lruXc3NzTY2NjDz9p+OgYEBAAAGj/4aysrKfHx8dHR0OFoHABAZGXn//v3z588TCIRp06YZGhpyCs+dO8fLy8tisdavXy8pKVldXR0dHY3D4dasWcNxxP748ePdu3f5+fkbGxvRNBbitwLJnb9HRUUFBi/vB/foC3/YioqKoaGhBw8e7OnpefjwIQ8Pz8aNG+/fv3/s2LELFy4AAJhMJoxbCo/iRMX4afwpph0IR5nh8fhfcK++7xWwsbGxs7MTh8NJSUnB2cl/g3464Ae+tra1tdnb25eWlvb29mpra8NUlz+q8YE8ffrU3Nw8NTUVTowO1Ddff2kDj4U/1aCgoKdPn/ZLM0KhUK5evdrvATGZzMrKSjabLSsry/F8qqurg/F8obdZY2NjV1cXHo/HMIxKpfLz86upqcET1dXVwfCbQkJCampq8I9DY2PjhQsXDAwMJkyYoKysDAufPn1qZ2c3depUf39/RUXFIaI2JyQkBAQECAkJtbW1ZWRktLS0mJmZxcXFqaioPH782MHBwcXFBYY+sre3Hz9+fHNzM4VCSU9P379/f1xc3MSJE1taWthstpiYmIGBAcwqde3atZycnBUrVpiZmXFO9PDhQyKROHbsWD4+PhjkGgCQkZGhqKg4YcIEHh4eNpstICDw6dOntWvX2tjYUKlUGIDA0dGxsbFx5cqVS5YsMTQ0FBAQQFMHiN+LX7cG/g8ARmpB/4bTv6967od+QHAOzoKy3bt3f2dTQ/ClCMLfDwynCQPhMJlMTvzorwSuIWez2Uwmk7ucRqMNbAo6xs2bNy87OxuWDBp+Jjo6evPmzZwQzHQ6/eDBg6Wlpa9fv/bw8PD29m5paRl4LKcFOp1uZmamoqLCKSwrK1NSUgIAaGtrcwrfvn3r6uoK/b3mzp0Lo0SWlJTAtXVycnI+Pj4sFgsuuQcA4HA4S0tLAoHQ1NSEYVhhYaG4uLiJicnmzZvxeDzs0suXL8eNG3fs2DFHR0cCgQCjhL948UJLS8vPz2/9+vU8PDwdHR1DdJ6Xl7egoKChoUFfX7+rq4tMJk+fPv3NmzcYhn348EFGRmbDhg0YhhkaGsJ40+/evdPQ0Lh7925vb6++vv6HDx9sbW0XLlz46dMnXV1d2FUMw5YvX25hYcF9rpSUFAKBAEMlcbC3t79+/Tp3yaZNm2RlZQ8fPnzo0CExMTH4JV+3bt3cuXNhhUmTJhkYGMCA2r9PqBsUd+cn8NvG3UE5sxCIf4Vz587V1taOGzfue7LB5+fnP378uF8Lb968ycrKevz4McyZNRDoBPb06dMnT55kZWWVlpZydtXW1qanp6elpXEyHz1//jw9PT0nJycjI0NVVXXChAlsNpvFYsHgxcXFxRkZGR0dHXg8nkKhPH78eNy4ccrKyikpKdAX3sTEZOTIkTAakLCwsImJSUBAgLS09Ny5cx0cHJydnQ0NDWGWUDqdvn79+nXr1mlra2/YsAH6sb19+9bBwWHt2rUbNmxIS0sbwjXk+fPna9euzczMjIqKkpOTS0tLi4+PDw0NnTdv3qFDh548eRIcHDxoRGYOJBJJW1ubO8KCrKxsdHS0srIydzalmJiYmzdvVlZWwuCQMJxPVFRUe3v7jRs3nJycDhw4kJGRERIS4uvrKyEhUVFRcf/+/aVLl27duhX2s6+vb/v27Vu2bIExFQEAOTk5JiYmZmZmurq6LBYLRuJ++vTp/PnzFyxYMGnSJDqdvnPnziE6b2Zm1tHRUVZWdvDgQT4+PgEBgXnz5kEjjaqqKlyuCABQVFSEoYnGjBmjrKwcFxcnKCg4Z84cmPALj8fz8vLCkF1fOlF5eflA1348Ht8vHR4ej+/o6Hj48GFcXJyWltags4F8fHxDT0AjED8TJHeGAsPQv+H275ug0WgUCoWTury7uxtmWm5qalq5ciUML/Sfrwp25MgRKyurDx/+Lyq+rKysoqIiPz8/HPDYbPaOHTtWrVplZ2dXW1ubkZFhZ2dnb28PRUN6evqqVas2b95sZ2eXnZ0NW0hOTjY0NDQ2Nj5y5Ags6ezsdHFxMTU1nT17trGxMcwINpCHDx8aGxsbGhrOmjVr9uzZs2fPhlkkW1pazM3NnZ2dfX19Z86c+fbtWwBAa2vr8ePHZ8yYsWDBAhcXF29vbzweT6VS4Yj46dOn9+/fw+hQe/fuNTY2trGx8fLyMjMzg0GPtmzZoqGhYWpqKioqeubMmU2bNs2aNau3t9fc3PzJkyeJiYnLli3z9vYGAGzcuLGzszMqKmrPnj3Xr1+HybasrKzCw8MvX74cGRlpbW09hDQ8c+aMjY1NcXHx69evJ02atHXr1qVLlwYFBbW0tERGRubn57u7u1dXVw/9QEeOHMmdcFtERGTq1KkeHh6c85LJ5FOnTgkICIwaNcrIyCguLk5CQoJMJsOQB46Ojn5+fjw8PO3t7Xx8fDIyMgQCAU5XCQsLQ2FkZma2b9++hQsX6uvrV1dXw0lYHA5HJBJfv37NZrNDQkLWrVsHC/v6+l69ekUikcLDw9euXTtEz69du1ZeXh4XF7d8+XK4ZhPDME5yXxaLBTVKW1sbZ0aMzWZDv0COwoORLOzs7IbwVTp+/DiRSITf8+7ubnisuLj4yZMn4VcCBnni5eXl4+MLDg5+8+aNpaUl1MT29va1tbVbt26l0Wh4PD4zM3P//v1DPxEE4qeB5A4CMThUKlVdXV1BQcHMzGzGjBkAAE9PT3Fx8REjRowePbqnp2f69OnQJaunp0dKSiopKYnJZK5Zs4YTNIgbPB4/Z86choaGsLCwuro6ExOTrKwsmOapo6PD0tISAHDkyJGkpCSoQhISEjw9PUtKSmJiYo4ePRoREQEAIJPJly9fhskmzc3Nv5Thef369XAtYU5OTm9vr5aWFgzJbW1tPXfu3A8fPiQlJVEoFBgpe9GiRdBd5v79+zt27LC1tQUACAoKTp06FQAwd+5cNzc3GCA4KSlp9+7du3fvdnFxmTlz5pYtWwAAixcvhnmtg4ODnZycVqxYoa2tLS0tLSAgcObMGe4e5ubmJicnCwgIbNq0CfwnlAOZTD59+jRMSD5t2rQvBbcEADx69Ai6zZJIJAMDAxjI4N9m6tSpIiIibDabSqVu2bKFRqP19va2tLR8KQQoxMnJic1mZ2dnh4eHb9iwAQCAYRi0sri7u+vo6HBsHg8fPpw/f/7q1atJJNLQi5709PQWLlxoaWmpoaEB4zaNHz+ezWbfv3/f29s7LS0N2mxERUVbW1vv3bu3ZcuWV69enT9/vrKy8sSJE4WFhYKCgg8fPoyPj6+qqsrIyCgsLLx161ZNTQ2MBR8aGgrVzMWLF6lUKlTqRkZGMO3JunXrREVFpaWlSSTSnj172Gz28ePHDQwMxo8fz8PDc+fOnSlTpoiLi8+aNevy5cvBwcGTJ08eM2aMhYXFL48AhEBwQJZGBGJw6HT6/PnzN27cCAA4cOAAAODChQulpaVMJvP06dNGRkajR4+GthlBQcHg4GA4faClpRUdHT3oxISZmZmZmdmmTZuam5vhMLBt2zZ+fv6+vj4+Pr73799HR0dfunRpwoQJAIDAwMD29naYQhwAAHUDgUCQk5Pz8vLi5eWVl5c3MTEBNwaeBwAAoIFBREREUFBw69at69atO336NJPJzMrK8vDwIJPJ4uLiMAsmvNKJEyfCUElDgGEYTE5JIpFsbGz27t3LKR/oSI7D4WDEBA40Gm3x4sWurq5sNruzsxM2JSMjExwcvGLFChkZmZ6ens+fPzOZzEFnQLy9vc+dO2dqatrb2xsaGgo9aQAALBYLehDjcDg6nd7PE5kbOp3e3d3NZrNbW1sJBIKEhASFQqmtrb169SqRSGxra2MwGHJycjt37vT19TUxMQEAiIiImJiYbN26de/evX5+frm5ufz8/PLy8suXLwcAEInE1tbW4OBgR0dHPj6+rKysgoKCgoKC8PBwKSkpKpU6YcIEFxcXAMDy5cvnzp378OFDfX39nJwcuDx7xYoVoaGhK1askJCQUFBQcHZ2HuLm19fXr169urS0dPPmza6urrDNkSNHHj9+fP78+UlJSVQqtbOzc+3atba2tmQyeebMmRYWFoKCgrKysvfu3RsxYoSrq+ucOXP09PTk5eXZbDaBQMAwbM+ePQCAvr4+Xl5e+J2cMWNGXFwcAIDJZI4ZMwZm3hg/fnxycnJ1dTWGYWPHjoXrsG7cuAHTy8+aNYvj0D179uyMjAweHh6Ywh2B+H1AcgeBGBwBAQE9Pb2MjIx9+/bBTGGCgoKKiop9fX0wQRX2n7kxPB6flpa2bt06PT09zvzFl9DV1XVwcAgODl65ciUMTyAhIZGUlBQeHv7XX3+9evXqxo0bY8aMYbPZwsLCmpqaDAbDzc0NaiBZWdnGxsZjx47V1tZ+/Phx0KjNkKamJs42lUo9d+4cAIDNZhOJxIMHD0pKSr59+3aIdUCDsmDBguzsbAaDQafTPTw8OCnQiURieXl5VVWViooKAABmTmWxWGlpaQQCgUAgtLS0vH79evHixRcuXJgyZYqkpGRSUtKIESOCgoLOnz9vbGxsZma2YsWKI0eOsNlsPz8/6CXdj+XLl//1119Q5Whra9+5cwcAkJaWxmazFy5c+PTpU0FBwQ8fPqxYsQJoD97/FStWwIFcWlp65syZ2dnZ1dXVpqamDQ0NAAApKamjR4/u2bNn3759YmJieXl5bDZ72bJlMMDBwYMHxcXFc3NzNTQ0du7cCaNRaGlp2djYQNPU7Nmzu7q6CATCmDFjzp49W15enp+f7+PjA1dgycnJFRUV0el0Op0Op4EAACNHjszPz4cBiKFcG+Lm9/X10el0IpHInfV9+vTp9+/fx+FwnGNVVVX7HSgkJLRkyRK4DXOBQe9s8N/57ziMGDFiYNgLAICurm4/BSMtLT1o1hpOEj0E4vfilzhIIxC/M3D5xpEjRxQUFCIiIvbs2cNZzsO9kkVDQ8PX1xfDsG3btpmamtbV1T179gzGjeQ0paent3XrVu7Gu7q6YOitd+/ewZJnz57t37+/tbU1KytLRERk5syZGIY5OTnx8/NnZma2trZGR0dbWVlhGLZp0yY1NbX09PTPnz8HBQUZGhp+aWUWDIodGRmZl5eno6MDl0hkZWUpKiouXbp0x44d+vr6Dx8+xDCsurpaVFSUj49PVVV1y5Yt8PBPnz5B9w4ZGZkZM2Z4eXlhGMZms93d3ZcsWbJy5UpHR0cqlYphWF1dnZGRkaqq6oIFC2xtbWtra69cuQJnALW0tHR1dRcuXKioqHj27FkMw1xcXOTk5GRkZOzt7WEq+56enuzsbB0dnQkTJjx//vz58+cNDQ39roXz8cOHD7BObW0t7E9ubm5eXt6TJ0+6urqam5tzc3OfPn36pcVN5eXlz58/z8vLe/78eWlpKSwvKyuDjbx48QJe0a/lS53/+T35gfw+/Ucrs34Cv+3KLBRVGYHoDwzcsql5E51OV1BQePr0aV5enqur65EjRxwcHNLT093c3Pbu3autrU0kEvfu3RsbG5uTk2NpaXnhwgUMw+bOnZuQkAAA8PPzu3LliqKioqmpKZPJhHnQAAAlJSUuLi6JiYlwOikkJIQThVJeXj4lJWX8+PFkMjkwMPDcuXNNTU0uLi7Ozs46Ojrnzp2Liopqamqqra318PBwdnYeFz2OO8wgZ7uqqkpVVXXr1q2ioqIaGhrcbrABAQGfP3/evn077ExbW1tgYCCRSOzt7Z04cSJn/XxmZia05UyYMIE74k6/nOQAAAzDYNpRuAEtDXCbe+NbH0G/MIPfFHfnHwcm/uV8qfO/pDO/CT/wwaGoyj8BFFUZgfjDUFVVDQsLy8vLW758uaKiopCQEIlEGj9+fHFx8bNnzwAABgYGr1+/VlVVPXny5KpVq969e1dTU/Pu3bu3b9/y8vJ+/vyZRqPl5eUxGIxXr17V1tZCedHa2vr48WMjIyOO64ydnZ2JiQmUC/z8/HJycgAAQUFBb29vGMZNSUkJurO4ubnZ2dlRqdS+vj5O4aDAdeYmJibQCZqbzZs3c38cMWLEwYMHB7YAl3QNLB84BQbVDPeUCqeQe+M7+V8e8n/s8PxD7uSgXfpOTclisThxIKdOnQp/L//Lzx3xY0HWHQSiP//SKyCNRiMQCEFBQTt27Hjx4sUPSUHKPcAMs4HhNxzjfw7/tu2B+zvT09Pj5eXV1NS0ePFiGxubfml3v6aFryn/Snp6ehQUFKD7f2Fhoba29ve3ObCHAFl3/mWQdQeB+F+nqqpq7ty5eDwerkb+4e1z/xHv7e1lMBhEIpFjQ/pT4OiS30qg/MwJph/e8hCj+/bt26Ojo+fMmXPy5MnCwsKAgIAfe+pvQkhIqLGxsaqqasKECeg9HPHDQXIHgRicf2U8cwIAgNPg9OnDp39Uk7+VLBiUb32Zxg5iGIY1Nzez2WwMw2RkZLiXI30PoaGhQUFBrq6uQ+QMz8vLu3TpUmFhoaam5u3bt8GX7/AvNxKUlZW5u7vfuXMHrg4blKG/Hr29vQ8ePDA0NPzw4QOnEQzDGhsbR4wYwQkFBJ+FvLx8W1tbX1/fEA1SqdS2tjboyyUqKgrVdl9fX1tbG5lMFhUVhXO1AAAajVZWVrZp06ZRo0aFhITw8fHB4AUiIiIAgI6Ojm+5EwjEV/BTHaMRiD+BP2j5xr/Uz9bW1hMnTqxZs+bq1avf2dS39hDWj4+P53gmwcxQ3w+NRluwYAEej09KShqiGoPBYLPZhw8fVlNT4+7SoP38mWRkZMBkW5CXL1/CoFB79+49evQoXGeHYRgMVeDn53fp0iUWizWwn7Dk2bNndnZ2OBzu6NGjPj4+6enpcG9NTc2+fftwONy6deueP3+OYVhaWhofHx8/P/+pU6f+L77iF64dHALu7u6cwcXHxweWOzs7EwgET0/PiRMnnjlzprOzE8MwGDlz69atixcv9vf35zRSU1MDAICZv7AffZ//oJ/2n8tvuzILRVVGIP5dWlpa8vPzi4uLhwgZ/LvR0dERGBiYnZ198uRJCoXyk8+en5+/bNkyJyennJwcSUlJ7siBHR0dr1+/5o4qxGKxiouL37x5M7CwsLCwsLCQE9m5rq5ux44dt2/fXrBgAacmk8ksKirKy8srLCyEqUKIRCIOh+Pl5cX+bj7l3r170tLSvr6+iYmJSkpKEydOzM/PBwBUV1cHBgbOnz//5MmTnMqZmZkmJibLly9XU1PbvXs3i8UyNzeH8QLKy8unTJnCCU3p4OBgY2OzYcOGRYsW9fT0AABaWlru37/v4uKyZcuWSZMmwZRbr169gunD7ty5ExAQ8PjxY3i4nZ3dsWPHQkNDvb29r1+//qXOKysrz5w5k0Qi5efnz507F6Z5b2pqsrGxefDgwcmTJ+Pj44OCggAAWlpax44d6+vr27Nnz+HDh2FCki9x586dq1evlpWVzZ8/H5qLKBQKzG525syZo0ePbtu2DYZvFhMTO3To0NatW62srAYGhzx06NDQNx+B+FbQZBYCMTi/5yTRz5lA0dDQqK6u3rx5c1BQEJ1OhxGQhwAmOc/LyxMXFx87dux3nr2zs5NGo02bNs3AwCAxMVFWVhYAcPjwYSqVWlxcnJiYOGnSpNTUVElJSQDAjRs34DL+KVOmJCcnw/C+ISEh0PIBD4RBsbOzszds2DBmzJhly5ZxznXhwgUvL6+lS5feu3fv48ePMFLiV6Kpqbly5coDBw6MHDnS2dlZQ0NDV1f35cuXa9asCQsLs7W1nT17Ng8Pz5YtW168eOHo6BgWFjZ27Fh9ff2SkhICgQDTqcJ2Ro0aBTO5ZmVlweykAIC1a9eGhIRs2bKlpqZm1apVR48e3b59+/3796Hda9OmTaNHj379+vX27du5e1VaWnr48OGtW7empKTMmjXL4bjDoJ0fOXKko6Njfn7+nj17OCGq/fz8GAxGYWEhAGD58uXKysrbtm3T0dGBz3TNmjX/F9Hg3hfviba2tqOjIwAgOTkZlvT19VlYWMDkJDAbCZzqys/PDw4OTklJmTx5MqcDHGBNBOIHguQOAjE4P0pYMJlMGP4/NzeX23G4vb29t7eXE+KWTCY/efJESkpKT08PAIBhmKen5927dy9dukQikUgk0qxZs0hHB3dhqampefv2LYPB4Ofn5+HhMTIyIhAI1dXVJSUlDAaDl5dXX1+fTCYXFxez2ezg4ODt27cnJCSIiIjs2bMnJibGw8ODwWBQqVRjY+OIiAhOJ/X19S9dusR9Ij8/Pz8/Pzweb2xsHBUVJSAgQKfTHR0dc3NzjYyMwsLCUlJSvl/uSElJiYuL29nZwVSsCgoKAIDMzMycnJxXr14dP3584cKFy5cvf/ToUU9Pz/3798vKypSVlfft27do0aKnT58+evRo48aNhw8fXr58+Zw5cxISEqDcWb9+fU1NzbVr17jP1dzc7O/vv3Pnzi1btsBMqF/P+PHjnZ2dWSyWv78/9DgBALx+/bq1tbWrq+vFixdiYmIPHz7csmVLbm7uuHHjYOa1KVOmQMeUQdft5+fnP3nyZMSIERiGdXR0wMmjKVOmrF69eteuXQcPHty/fz+UDgCAN2/eDOyVpaXlkydPTpw4ISEhAX2PfiadnZ3d3d3wblRWVsrKygoKCsbExCxbtmzevHkwHxzMlrp3796IiIjVq1ffvXvX1tbWzs6O49YDANi2bVtGRsZP7jxieIPkDgIxOBcuXHj69ClMD6mpqXny5EkCgXDhwgU4ok+dOjU0NJSPj+/q1asjRox49OgRDBw8btw4f39/HA5HoVA2b97c09MjJyfX3d0tLS39/v17X19fS0tLJyenlJSUAwcO4HC4zMxMfn7+5OTky5cvx8XFiYqKxsTEzJ07F8OwpqYmCoXS2tq6a9cuCoVSX18/aD+7urosLS2pVOq+ffugB25zczOFQrG2tv78+bO/v7+tre2NGzemT5/u4OAwefLkqVOnGhoaXrhw4eHDhwwGo729vamp6fjx41JSUps3b167du2DBw9gy/Ly8twTOh8/frxx48a2bduUlJRSUlKSk5OXLl1aUlISHh5+/fp1Ozu7fkmy/jHjx49PTEysqqo6evTopUuXMjIy1NXV9+zZs3DhQhUVFWFhYScnp7CwMADA06dPExISFBQUBAQEOCk1ent7eXl5vb29+fj4kpKSYI4IiLi4OHdGdADAyJEjKyoqtm7d+vTpUzc3t2/tKoZhoqKiHK0DAMDhcEwms6Ojg81mOzo6QhMUAIC7DgQmG8EwDIfDMRgMONdJIBDExMTOnj0Lk6nNmzcPAJCQkMDDw3P9+nUxMbF9+/YVFhZGRkYCAGg02p07d2xtbYWFhalUqoSERHR0tIyMTGBg4M2bNw8dOnTmzBmgNHjPYVgEBoPR29tLoVD6+vokJCTWr1+fmZm5cePGJUuW+Pn5mZmZaWpqAgBgxvi+vr7u7u6hAynNnj3byMhIXV2dRCJVVFTcvHlzzJgxDg4Ou3fvDgoK+vDhw61bt+BvQVhYODo6+tWrVxERERiGwcRkAAA/Pz9BQcHy8nJnZ2eYOwWB+DH8GpchBOI3BvozqqurAwCmTZu2fPlyXl5emBoiMDAQ/gnW0NCwtra2srJqa2ujUCgSEhKwJolECggIwDAsJSUFALBo0aLOzk6YXqq5udnLy2v9+vUYhlGpVOi38fnz546ODhKJlJCQwGQyVVRUSCQS9Eg9ffq0srIyhmG9vb0wt+WgbqfQtXPXrl11dXWvX7/OzMyEu6Kjo1+8eBEbG2tra3vr1i0MwyZOnLh//344VdHd3b1p06be3t6ysjIAQEVFBYZh169fFxYWhukdMAzLyMjA4XAdHR3wI8yFJCQkJCoqSiQSVVVVYTmMWygiInLlyhWYAqJfD7/15tfX15eVlWEY1t7ePnnyZG1tbXg/CQRCd3c3hmG+vr4aGhoYhiUmJhKJxIaGhra2tp6eHtjzuLg4Xl7evr6+gY2fP39+5MiRnI8xMTEiIiIwL6m7u3tQUBBnl7+/P+cCh3BVvnr16rx58ygUCqf88uXLQkJCMTEx0dHR165d27hxI4Zh6enpGhoaoaGhkZGRenp6nDwk48ePX7FiBSfM4759+zAMg6m4kpKSzMzMVq1aVV9ff/bsWQCAvb39gwcP9PX1z5w5Aw/Py8sDAMC0XBoaGm1tbceOHYMlwsLCJBLp6tWrX3JVhtoCh8PBw8ePH9/U1IRh2KtXr6SlpQEAenp6t2/fxjCMTqdPnTp1zZo169evd3BwiI2NHcJV+dWrVxYWFhYWFs7OzlDzwV0HDx6cO3ducXEx/Mhms3t6elpaWhobG6FBqKmpqa+vr6+vr6mpqbu7u7Ozs7Gxsb29Hbkq/3H8tq7KyLqDQAxOdHS0rq7unj17LC0tr1+/DlWCq6urmJiYnZ1dXFwcZ9Zm3759WlpaKSkpAgICly9f7uzsBABcunTJwMDg9u3bfHx8np6eUVFReDx+27ZtcEDi5eXdtm3b3bt3cThcZWXlokWL5s+fTyAQwsPD2Wx2P0vA0FYTJSWlQ4cOnTp16vnz5woKCjALRFdXV09Pz6FDh3Jzc2k0GoytbG9vD71EvwQfHx/2Zf9cGOU2LCxMQUGhtbUVXiabzV60aNHSpUsZDIaZmZmxsTFnDuJL5qi/paCgID8/39XVtaWlpbe3d8SIEbAcw7CqqioZGZmWlhYpKSkAwKxZs6ZOnern57d3796rV692dHQcP3589uzZJiYmO3fu9Pb2Dg8Pz8nJ8fLymjVr1ufPn5uamphMZnl5OZ1O19DQEBQUJJPJMHU8lUp9+PDh27dvAwICvL29w8LCurq6pkyZAlXpoOTm5vr5+fX09Nja2kpKSp44cUJUVHTp0qUREREbNmwgEol4PP7evXsAABjYxtbWloeH5/Pnz5w2fX19bW1tGxoacnNzExMToY9URETEsWPH9u/ff+rUKUNDQwKBsGTJkrFjx8LkrEeOHIHZ2gEAo0ePhqqaxWIpKSlJSkquXLly4sSJeDwewzA+Pj5DQ0MnX6dBO29nZzdjxgwikYhhGIvFUlVVhT5SkydPzsjIqKmpMTAwgI5QeDw+JCQEphf9zxP64rObPHlyfHz8wPJ+rsc4HE5ISAh+oyCcKVTYDQCAqKjoF0+DQHw7SO4gEIMzZswYzvKcKVOmbNiwwcPDA+YKFRQUHD16NKdmSUnJiBEj4FilpaUFZ1VYLJaCggJ3mFoMw0gk0ps3bz59+iQjIwNHBSKRmJubW1VV1dXVJSkpOWPGjEePHvX19fULDziECunp6SGRSJWVlTIyMvfu3bO2tqbRaEuXLn306NGbN290dXVtbW3hmqNbt27x8vIaGBgAABITEwkEAnzpAQAkJydPmDDh5s2blpaW0BM2Ozs7NDQUw7ATJ07Iysp6eHjcvn3b1NQ0NzdXSUkpOzs7Ozt7zZo1hYWF8+bNO3jwIC8vL5vN5rj6MpnMSZMm0el04PnNd15QUHD//v379+8HAEycOPH+/fsAADwez2azbWxsSCTSjBkzYmJiYE0XF5fnz5+fO3eur6/Py8sLAABnSQ4dOnT+/HkajXbhwgXo95qQkNDX17du3brg4GAajbZ27VozM7MrV67k5ORA1x8Mw6hUKg6HU1FRWb58OT8/P5VK3bBhw+7Lg2RoBwDo6elVVFQQCAQqlcpms2GadCkpKc4iKW4WL14M4wWvWbOGE1TGysoKrr0CXInEdXR0oqKiuI9VVlYe1HVXRETE1NSUu0RFReVrvK0HccPPG1Dp1X9/jPnbVhGI3xokdxCIL4LH4z98+FBeXv7u3bvDhw/DgaSuro7FYkEvUUVFRRkZmRMnTixZsiQ0NHTy5Mk7d+50cXEBAED1EBwcPHXqVBiMJDQ01MvLq76+Pioqavr06S9evJCUlCwuLt6wYcPVq1cXLVp0/PhxPz8/Go0WHR0NABg5ciQvL6+amhqLxVJTU+Mk7+xHT09PaGhoQUGBiIhIc3PzpUuXcDicmppaU1PTyZMneXl5CwsLyWRyVlYWDw9PR0cHhUKZMGFCQECAsLCwq6srbDYkJISXl1dFRQUuXW5ubt67d++nT58mTZp0+/ZteXl5d3d3SUnJhIQEKyur3t7eS5cuwfd1fn7+cePGRUZG0mg0Nze3c+fOwV4RicQzZ84EBgbmgtxvve1TpkzJz8/H4XBsNltGRgYacqAsO336tLa2tpSUFCd1l42NjY2NTb8WBAUFuReBQ+zs7Ozs7PoV2tvb29vbw20cDgc1q6ur69f0kxMZ6CtzLwAAPnz4UFdXx2Qyi4uLf5Vjyr+6uO/3XM+IQACUMwuBGAj8k923s2/kyJFw9mfcuHFwUQmZTFZQUIDzOACAEydOwCAoFRUVRkZGSkpKW7ZsWbNmDdxbWloaGBhYUlJiZWWFx+MdHBwEBQXfvn0bFxeHw+E2b96cm5vb3t4OBUd4ePiHDx/6ZS9va2u7cuUKi8WysrKaMGHCwPxB359R6P3792PHjq2oqIC+Sj+cb+3hl+qnp6fPmzdPXV09LS0NBon5aaAs5T+Tgd/wH9IOd2u/PBz28AblzEIg/jwoFMqtW7fMzMwYDAYsERAQKC0that7MAwTExOD5RoaGkVFRSQSiVMCANDS0oKTLNy+OOPHjx8/fjzchutuIJzVxdyMGDFiz549P/aiBsXa2vrhw4fy8vI/4Vz/DAkJCTMzszNnzkBH2n+PQcfXf0nccI+733qK70zS+f79+46ODjiz+TUwmcz09PQpU6Zw1pr9KMLDwz09PW/dujV//vxBb8I/UCdIjCIGguQOAjE4f/31F5VKDQ8P51hrAAA4HI47Ogg3cM6lHzBkzr/VxR/ByJEjIyMj+/r6OE4kv5wvjlXTQPKd5J/bl/9jiBGXSqVSKBRBQUGYYYrBYGzfvr2srGzq1Kk+Pj6wDp1O9/LyqqiomDZtGsdpd+Bltrm32draamlpnTp1auj+fOdw3traumzZso8fP6amps6cOfNrDrly5Yqbm1tWVtasWbM4hWQymTMDyKGmpiYzM3OIrGT9oNFonz9/PnXq1G9lCUAMP5DcQSAGh4+Pb8WKFRISEiwWCy7W/dP5mzGy9hed9z/8wimGflaWb+pJYGDg9u3bAwICoIfWy5cvP336VFFR0d7ezpE7eXl5ra2t5eXlXV1dQ6RHwOFw/Pz8L1++/IeX8dVISUnt3Llz3bp1FRUVXyl3lixZIi4uzq11AABr164VFRW9ceMGp6SpqcnU1NTY2FhRUbG8vHzt2rXQtEmhUMLDw5cuXcr9VvDw4cP6+vrW1ta/PXtPT09kZCSTyZw1a9a4ceOoVGpUVBSGYUwmk8Fg4PF4TU1NzoI1BGJQkNxBIAZnU/MmoAUAAJePXP7VfRmK399u/zXqgfsqvlP3dHV1LV++/MWLF0pKSsXFxf32Qi+r5ubmq1evwnQHX38DL1y4MHfuXC0tLe7C3t5eWVnZFStWwI/Tp0+fPn16ZWXlrl27OHVmzpw5c+bMsrKyffv2DdG+hITE3bt3165d++HDBzwej8fjFRUVoVM2nU6vr6/H4XB4PJ4TiXtQaDTax48fGQwGHx/fiBEjhISEPn78yMPDIyUlJSwsXF1dLSgoKCUlZWtru2PHjurq6srKyr6+Pnl5eUlJyebm5s7OTgaDISoqWlNTs23btosXL+rr6799+9bf37+pqWns2LETJ04EALBYrF27dr1580ZGRsbJyYlOpx8/flxWVtbb2xuuVnv69GldXV1DQ8ORI0ceP3587Nix/Pz8mJgYKysrGM6RxWJdunTpyZMnf/sukZKSsnv37vr6ehaLpaioeP/+fSkpqR07dnz+/FlCQuLcuXM3btzo6enJzs7+zS2piF8LkjsIxJ/E93h4fL9fM4PBSE1NNTc35y5ks9lKSkoNDQ0wJ8D3tP9DEBUVTU1NBQBs2LCBTqfD4MUcNDQ0oqOjZ82aNTAtJTfPnz+H4QrHjBmzevXqkpKSS5cuXbp0ydnZWV5enkqlrl27Fi5JS0pKkpGR6edRpK6u3tra+vLlyylTpnAKVVRUuCPNfIn3799Dt3E8Hh8VFbV8+XIAwNu3bydPngwAIBAIcOHel9i/f39+fr6xsfGlS5csLCwuXbpkb2///PlzKO/u379/8+ZNmHEMBkHA4XCNjY1v375NS0t78+bN9u3bLSwsTp06tWbNGh4eHkdHx+Li4vHjx58+fVpWVpYTt4lAIJw8ebK7u9vX11dMTAzDMHifDxw4ABNEQI97GCr68uXLjo6Oy5Yte/PmzeTJkxcsWKCqqrp69WpHR8f4+PiioiIdHZ0hrig4OPjTp08tLS0AAAcHB3t7+8zMTB8fHx8fH5gXNisrKy4urrq6WkND429vL+J/FiR3EIjB+clzK2/fvhUTE4P5oThYWVnFxcWtXr06IiLiH7Q5qDaaM2eOvb099/qvr6exsdHR0XH37t0eHh5VVVXKysp4PL6uru7IkSMkEgkOzJDa2lo5ObmqqioCgQATPwEAMAx79eoVlCDwTX1gbkgIm80uLi7evn27hoZGQEAANABARXX06FFbW1tPT09VVdWgoKBLly69evUKhm2sqqqCQXoG5fXr18uWLRMWFiYQCGw2ewijQm5urqOjIw6Hq6qqmjZt2urVq5WVlWtrawEAqampBAJBR0cHBsJRVFQsKSkZM2bMwP5LSkr+sxVkeDz+6NGj69at8/PzW716tYqKiqqq6tWrV6urq/n5+VetWhUVFQW0vnh4VFSUjY3N3r17raysMjIyCATC3Llznz9/DveuXr1627ZtHz58kJSUpNPpMPN8Q0ODgoJCWVmZmZnZ/fv3fX19z5w54+rq+vz587/++gseyMvL2y99BIVCaWtrI5FI3IKyqKiopqaG800gEolUKjUlJSUlJQUGQIKBlLZs2RIbGwv9e7S1tW1tbSsrK790RQ8ePBjoMKegoMBxmmYwGBMnTkRaBzE0+F/dAQRiOPDixYudO3ey2WwWi8WdlQm6FwwM98Bms5lMJqdmVlbW5MmT4dDCYDD6+vpg+eHDhwEAR48e5T62rq7O0dHx06dPnJKHDx/W1dXduXNn1apVr171Cw/3Xzg4OOTm5l68eNHJyWnr1q3w5RsAsHv3bjs7u/Xr13Nmf+Lj4x0cHFxdXTdu3Lh//354CZcvX25ubt6xY4eHh8f8+fNjY2MBAOfPn9+wYYO7uzuNRoPHnjhxwtTU1NHRccGCBfPnz/fz84PlYWFhrq6u7u7uU6dONTAwCAkJGbSTlZWVdnZ2mzZtmjVrVlBQ0JYtWwAAHz9+1NfXv3nz5smTJ8vKyhYtWgQA4OHhkZCQoNFoNBqtoKDg48ePQ1z7mTNnbG1td+3aFRwcTCQSh7Du7Nixw9DQ8O3bt4mJiR4eHgAAISGhwMBAAMD9+/crKipiYmKgnWbmzJleXl4UCqVfC2w2W1RUdFDv9b9l/Pjxu3fvVlBQOH78OACATqeHh4ffunUrNDQ0ICCgqqoK+kR/iYCAAJj7PSAgAKqEr/Q84+XlffDgQXNzMyem0dBkZGTcu3cvISEBAACTRQAA8Hg8hUKBQSDZbDb8hrPZbFNTU19f3/Dw8Ly8PCkpKRaLxcPDc+vWLQDAp0+fXr58OUQqrjVr1tTW1sbFxQEAMAyD3ZOWloYJTwAAPDw8r169ys/P/5puI/5nQXIHgRicwsLCu3fvZmZmfvz48d69ew8ePICjWkNDw71792JjY7OysjiVo6KiTpw4sWbNmvHjxzs4OMC/8k1NTVevXhUSErp48SLMqJCTkxMXF5eenu7g4EAikaB7B41GO3jwIJ1OP3HihLCwMEw+BQB4/Pjx3Llz+/Wqvr7ezMwsNTV19OjRMNfS4sWLFy5cqKWl5evr++7dOzMzsyEUT0VFhbi4uIiISFlZWU1NDQ6HY7FYjo6Ojx49+vTpU15e3tKlS6EOePr0aWhoqKGhYXFx8ZUrV+BgtmDBAnNz86amJg8Pj/T0dOi4evz48fDwcO4hf8aMGZ8+feLh4SkuLnZ1dd27dy8M+pednT1v3rz8/Pxjx45FRkZye7f0u8abN28uWrRo3759ISEhMBhxZWVlQ0NDUlKSk5OTsbFxc3NzXV2dg4PD2LFjaTQaLy9vXFzcEIH7EhMTHz58uHfv3jVr1kyePHnEiBH9coVyExERkZeXx8fHt2rVqt8hlQEMfm1qarpx48ZXr17t3j14lGfImDFj0tPTz549KyIismLFilevXu3fv3/EiBFQGRQUFID/TsbOKWQwGCYmJgwGo6qqikQiFRQUsFgsTs2ioiIMw2AqFYilpaWJicmVK1eePHlib28fGhoKAFBQUODl5b1+/fqTJ09WrVo1e/ZsPj6+06dP3759Ozw8/PLly7y8vLm5uby8vNHR0TBvhqqq6vv378vKyr70vQ0JCdm4caOVlZWAgEBhYSGMhFlcXJybmysiItLU1CQgINDZ2enr6/uP7zDif4J/Py0XAvGHAfMIbtq0CQAgJCQ0btw4IpHIw8NTXV2NYZiZmZmIiIi4uLi0tPSDBw/gIS4uLkJCQjdv3oyMjLx16xaFQqHT6fr6+jB5uLKy8pEjRzAMS0hIgOPH2rVrT58+zcvL++nTJwzDKioqpKSklixZcv/+/ejoaBqNBpuF9o+PHz/Cj11dXePGjRs1alRtbe25c+eEhYVbWlry8vKsra2VlJSamprS09MBADALKTYgtyU4BBgMBmcv5N27dwCAvLw8BoNx9epVTorQQ4cOkUikpKSkioqK69evw9d3d3f3adOmDbxjiYmJJBKpt7eXU7J161aYdvTq1asAAGgf8vb23r9/f0NDw8SJE58/f96vb5wNmHWrtbWVuwJ3itC3b98KCQnBTJlOTk5VVVUYhpHJZFtbW+5D1q9fz7mTDx48EBERaWxsZLFYNBpNWlr61KlTX7pL3d3d5eXlycnJCxYsmDFjBiyHqVifPHnS29vb3NwML5bFYu3YsUNTU5NMJre1tcHTdXV1tbe3T5w4MSYmpqenh1PY3Ny8YsWKtrY2mLJ+4Hnhhr6+/qVLlzo7O/ft20ckEp8+fdrY2Kiqqjp69OiPHz9euXJlypQpQyTpHDVq1KpVq8zNzefNm7do0aKenh4Mw86ePWtkZOTh4aGuri4oKLho0aLU1FQ9PT0TExMPDw8FBQUrKys2mx0bGysqKmpkZCQnJycrK2toaCgrK5uRkeHj46OpqSkkJCQtLW1vb+/n5wdPl5GRISoqCuezrl+/DgtTU1PhaqwVK1Y8ffoUw7DGxsagoCB+fv7Dhw/HxMRwvifZ2dm3bt26ffv27du3nzx5MvBZcEp6e3vv3Llz8+bN8vJyWF5VVRUZGRkZGUkmkwsKCm7dusWdnnaI+4NShP7b/LYpQtGDRyD6A/8m0ul0DMMWLlx4/vx5DMNghu2EhIQpU6ZcuHDh/Pnz7u7uFhYWcDALDQ2VkJDgbgTmOXd1dY2KioLTMS0tLZ2dnTgczsHBAdZxc3ODQzV8Mz58+HC/nsBQzhy5A01EL1++xDDs06dPmpqaK1euxDDsyJEjWVlZGIYlJycPLXeePn26cOFC7kIodxwdHT09Pd3d3Xfv3s1iseCuPXv22NvbCwsLjx8/HhZu2bJFTk6upqamXz+HkDt5eXm8vLx79uzBMCwtLY1EIs2fP//mzZsD7zlnA/65XL16NYZhxcXFp06dKioq+vjxo4aGxokTJ6qrq5ctWzZu3DhYf8mSJTo6Ojdv3ly4cKGsrGxwcDCFQqHRaMHBwdOmTQsICDh//nxJSQmGYRYWFlB4wYkwPj4+mHl+4F0CAMybNy86OtrExAT2HMOwrq6uqVOnwr/jMjIyhYWFGIYZGxtzvz3GxMRgGNZvaXdsbCyGYTCgn7KyspaWlqys7Pv3778kdw4ePGhnZ+fu7r5p06b29nYoNCsrK9esWePo6JiVlUUmk4cYzq9duxYUFGRlZbVx40ZOqnY2m93Z2dna2trb20ulUj9//gzvEqcQalwymUwmk7u6uigUCtwgk8kUCqW9vb2npwdGGILZYTnN9v0HTvJzTiFsk7uQ89X6El+SO98Kkju/kN9W7iBXZQRicOCiVjk5OehQCfMi3blz5/Xr111dXb29vSwWy8XFpd/CH26IROKdO3cAABs2bFi/fr2kpCSFQpGXl4f5yQEAMEknAIBKpYIBUwxfoq2tDQBQVFRUXl6+cOFC7t7+LXASh0Kh4HC4srIyOTk56NhBoVCCgoLg7Bubza6url60aFFISIiamhqbzebkuN6yZcvVq1evX79ua2vr5eU1bdo0b2/vz58/V1ZWYhhWVFTEx8enpKTU1tZ2+fLlxsbG27dvf/78mUajQU+je/fuMRiMxYsXz5gx4927d7KysuLi4gM7OW3aNFtb26dPn+rp6dXV1Y0YMWLlypUKCgqLFi3as2fPgwcPqqqqgoKCYOX9+/cvX77c1dV13bp1NBpt7969y5cvDw4O9vHxERQUPHr0KJlMFhMT09LSOn/+/OTJk0+dOuXg4JCXl0en0/vlYeWQmZnJZrNpNNq+ffumT58OC0VERO7du1dRUQEAkJSUhBNn586d6+zshL7PLBYLFl68eLGjowMmG2cymdra2gCAS5cutbe3y8jIkEgkGo3GnWKWw/+5luMA+E+Wz8sXuIIgaAIAQHBWMMga6hHDFGDQNvn/W8bhuGflON4//b69/QIGcoDZTwdu43C4gfnCvr4QgfiZILmDQHwD8+bNe/Xqlaurq5ubG8zH2d3dLSIigsfj29vbs7Kyxo8f7+fnN3ny5CVLlrBYLBMTk1WrVhUWFjY1NWlqavLx8U2cOHHz5s2LFy9ubm7OyMhQV1ffvHmzvr6+mJhYcnIy1CKTJk1auXLly5cvoXvsqVOnhIWFJ0+ebGlpeeDAgZ07dz5//vzdu3dLly49e/YslUrNyMgoLS0VERGBo1deXl5DQ8Ogi55Gjx49evRoa2trERGR7u7urVu3GhgYHDx4sLS01MLCQl9fn4eHh0gkCgoKmpubnz9/PjIy0tvbe/fu3VCKqaio+Pj4fP78OTg4WFlZedy4cQCAFy9e1NfX7969OyEhgU6nL1iwYNy4cZs3b2axWBUVFTIyMjt27CCRSH19fSQSydnZubGxMTw8PDAwUEVFJTU1daDm4OPjCwsLKywshIlLPT09ocvtqVOnVq1axcfHx8nCAQDQ1dUduKjHxcVl8+bN/QpVVVUPHTo0RJQ/DrMfz/7/H558odK9wQoHrTxo4YDDUSInBOJfBaUIRSD6w8kjePr0aW9v7xEjRsDgZoqKilQq1cLCIi8vT1hYmEQiiYqKJicny8nJ0en09evX3717V05OTkREBCZCDw0N9ff3HzVqVEtLy6pVq3bu3Hn+/Png4GBBQcEDBw6oqqpaW1sLCwv/9ddfOjo6eXl5cPZh48aNq1atUlBQiIyMPHjw4IgRI+DMwpIlS06cOAEAePz4sbOzc0BAwPTp0wUFBWk0WllZGZ1OV1JSEhYWrqmpodPpo0aNEhER6RdoB36kUqnv3r1js9mjR4/mhIHp6+srLy+HXkqc+hQKpba2VlNT8yuX6gzN69evDQ0Nq6urpaWle3t79fX16+vrS0pK4Np7Tle/PzjQt/Lzz/jv8fvHnPxbUIrQP53fNkUokjsIRH/g30TGXkZwcDCLxWKxWHQ63cjIiOO6kZiYWFpaOnLkSO50Wr29vSEhIVJSUr9DqD3IoHLnV3Wmvb3dyspKVVVVSkoqNzdXR0dn48aNcKIH/Lfc+fl9Q+Pf/whI7vwEkNxBIP4YfsjfxOH3nv399PT0tLW1YRhGIpHk5OS4LUnDycSC+G1Bcucn8NvKHeS7g0AMzjDQK2BAEolvOnZ43AEEAoEASO4gEF/i+607v9tL5Df1Z+BE2L/Qo/5nHGLv73YzEQjEnwWSOwjEsIVKpX78+FFERKRfKq5/xsp3K5OTkwsKCv5ZKihIWFiYnZ3dxo0bOcmYvgZkZ0IgEN8JkjsIxOD8gyH2d7NAFBcXT5061dzcHCY2+k5MTEw+fPjASbP1D2CxWM3Nzfv27TMyMvr+/iAQCMTXg+QOAjE436pdfq0FYt68eZMmTYJJJTmMHj16zpw5nAydQ1NeXl5XV6erqyshITFohY0bN1ZWVmZnZ3/69IlOp+vq6oqJicFdeXl5XV1dJBJJV1cXhrPDMKygoODkyZPLli1TUlLS09MDALx9+zYiIkJQUNDBwYG75XPnzh0+fBiPx+vp6aWkpHz71SMQCMTfgOQOAjEUGIbRaDQcDscJREulUkkkEky4SCQSuZcXDYRGo/X09AgICPDy8hIIhJ6eHhaLJSAgwMPDA7NP8PLywjydNBqNQqEQiUR+fn4eHh4Y1u/Dhw/bt2+fOHHigQMHcDgcjUZLTk6Ojo7m5+dnMpnr1q2DKyAyMzNzc3PT09NbW1t5eHhmzpxpY2OTkZFx4cKFjIyM1NRUuN4bwzAGg3HkyJHPnz+TyWRnZ2cDA4MzZ86Ulpby8fFlZWWVlJQsW7YsKipqiBzaMGgvAGDNmjU3b97E4XBwfsrGxub+/fvW1tY3btwAAFy/ft3e3n779u3x8fEZGRm1tbUAADKZPHny5JCQkOPHj3PCIp8/f/7mzZuXLl2i0WiJiYnf/cQQCARiEFBGdATii8DUhqKiopqamjB0b2ZmpoKCwrZt22RkZPj5+Tdu3DhEYu2Ghoa5c+fu3bt31KhRHh4eAIDo6GhTU9NJkyYBAAwNDUeMGAEzQty4cUNWVvbgwYMbNmxQUFDo7OwEAFy5cmXixIkYhqWnpysoKFCp1OvXrx8+fDgiIsLIyGjDhg1OTk4wK7uBgYGYmNiBAwdOnTq1a9euBQsWAABmzJgBjSiczO1VVVViYmIPHz48evSotbW1nZ1dXl6eq6srgUDIzMzMycmhUqmfPn3y9PQc4p48e/YM5lSqrKyEKc2pVKqPj8/u3bstLCxgjlIAQFdX19ixY6dNm7Zly5aqqipYOH36dDc3N15eXph0HUKlUvF4vISEhKSkpI2NzT95TggEAvF3ILmDQHyR3NzcI0eOJCcne3l5RUZGAgCIRGJXV1d4eHhYWFhKSkpnZ+fADAYczM3NjYyMzp07t2nTpkuXLjU3N9vb28fGxgoJCQUFBfX09Dg6OkpJSQEA6uvrDxw4EBgY6O3tzTm8p6fH2dk5IiLC09OzsbHRz8/P2dl55cqVKioqtra2RkZGEyZMOH/+PACAj4+Ph4dHQEBAQkJi1KhRkpKSsBDmXuAA0zTu3btXTEzM0tJSWVk5MDCQj49PVFSUh4dHREQEmqDIZPIQ90RMTExERISfnx+Px1MoFAAAk8nMz89fvXp1bGwsJwfT1q1bN23atHz5cmtra1dXV1gTADBp0iRDQ0MajcZp0MPD49y5c6dPn167du2uXbt6enq+7SEhEAjEV4DkDgLxRZYuXRoQEFBaWlpVVQVnlyZNmsTHx8fLy2tpaWlqajpq1KjU1NQvHV5RUeHn52dsbMxisbZt2yYoKAgAkJeXv3nzpqurq5iY2Llz5wAAra2tBw8ehKkoDQwMWltbYeJMAoHQ2Njo4+OTnZ3t5uamoqLypRN9E62trXBDRETE0NDwm45lMpnwequqqj5+/CgoKFheXu7m5qavr19YWGhlZdXU1JSbmwsASEtLU1RUZLPZu3btunLlir+/P2yhrq7u3bt3nBnAlpaW2bNnP378eMGCBfv27QMAPH/+HO4qKyvz9fXlmKYQCATie0C+OwjE4NBoNHNz887OzgsXLggKCr569QoAwMPDA3NgwToMBuPWrVvu7u5famHPnj07duwQExNra2tramoSFhYmk8k7d+7cu3fv/fv3AwICNm/ezMPDIycnt2vXrvj4eD4+vqKiIjExMQ0NDTabXVVVFRISAgB49erV48ePAQA4HI5MJpeUlPDw8HR2dnKSZRIIhJqamrdv39bX1z948OD8+fM9PT1FRUUAgM7Ozvz8fHFxcXV1dW9v7/Pnz0+fPj0rK6uurg4me29sbOzp6ent7SWRSB0dHS0tLV8yWREIBF9f32vXrrW0tMjKynp4eMjLy69aterq1as3btyAU1T29vYvX7589uzZ7du3k5OT6+rqZs6cCRN037x588aNG7y8vFVVVTY2NqtWrTI3N9fS0goICODl5aXRaEFBQaampvBc5eXlUCQZGxv/mCeKQCD+h0FyB4EYHAzDnj59ev/+fWFh4UePHoWHh/f09GzdulVbWzsiIiIxMXH27NnZ2dm1tbWFhYU6OjoDW7C2tj579iyDweDl5U1ISNi4caOmpubKlStTU1M9PT0fPHjg4eFRWlq6ffv2zMzM6dOnW1paKikptbS0uLm5aWhomJiYFBUVzZs3b+7cuUwmE7ob4/F4Op2+cePGmpqaHTt2uLi4wHPt3Lmzvr7+3r17AABxcfHW1tba2trq6mo/Pz8Mw+7evauvrz9q1Kjjx4+rqqreuXOHyWSGhYURicS2trbRo0erq6tXV1fz8fFZWVnh8fi0tLRB78miRYuWL1/+4MEDHA7n4OAgLy8PAHBwcIAZSXft2hUWFiYvLy8oKHjo0KGDBw9iGIbD4aBhDACwdu3atWvX4nA4DMM4u6CeG4ilpeXQ02oIBALx9aCcWQhEf+CSctpumrW1taCgIJVKbWho4OXlVVFROXv2rI2NzefPn9euXbt161ZfX9/o6Ojjx48vWLBgYBhiijeloKDA1dX12LFjmpqao0aNwjCssrISj8eLiooymUwqldrb2wu1QlNTU2dnJ4vFGjVqFCdROYPBqKio4OPjU1VVhSXHjx+/evVqQUFBT09PP9ecH34TfnJU5aH53WIaIf5EUM6sn8BvmzMLyR0Eoj+/fGhHIBD/Hkju/Kv8tnIHuSojEAgEAoEY5iDfHQSiP+jlD4FAIIYZyLqDQCAQCARimIPkDgKBQCAQiGEOkjsIBAKBQCCGOUjuIBAIBAKBGOYguYNAIBAIBGKYg+QOAoFAIBCIYQ6SOwgEAoFAIIY5SO4gEAgEAoEY5iC5g0AgEAgEYpiD5A4CgUAgEIhhDpI7CAQCgUAghjlI7iAQCAQCgRjmILmDQCAQCARimIPkDgKBQCAQiGEOkjsIBAKBQCCGOUjuIBAIBAKBGOYguYNAIBAIBGKYg+QOAoFAIBCIYQ6SOwgEAoFAIIY5SO4gEAgEAoEY5iC5g0AgEAgEYpiD5A4CgUAgEIhhDpI7CAQCgUAghjlI7iAQCAQCgRjmILmDQCAQCARimIPkDgKBQPyvcOrUqRs3bvzqXiAQvwDir+4AAoFAIH4SmzZtwuPRWy7ifxEkdxAIBOJ/BSEhoV/dBQTi14BkPgKBQCAQiGEOkjsIBAKBQCCGOUjuIBAIBAKBGOYguYNAIBAIBGKYg+QOAoFAIBCIYQ6SOwgEAoFAIIY5SO4gEAgEAoEY5iC5g0AgEAgEYpiD5A4CgUAgEIhhDpI7CAQCgUAghjlI7iAQCAQCgRjmILmDQCAQCARimIPkDgKBQCD+eHA4gMN9VSHifxOUER2BQCAQwxMM+9U9QPw2ILmDQCAQwwRoyeg3xg9a+PvA6R63GYbzEXabs4tzFYNWHqK1fkdxt/+b3x/EjwJNZiEQCMRwBsN+37H8azTHQN3DrVpgOedA7pIhZB8ON8gGYniDrDsIBALxi0EWjr9l0Iv9/buN+H1Acuf7iMABAMCaf/Rri/jPz/efHY5AIIYFAzXHoC63nF3QIDFQ/QxULQPtFoO202/j96Rfx/oZe7j3Ig2EGBQ0mfXrQCoHgUB8C1+y1vy5EzEDBVm/Eu7JJs6Vcm9zDuE+kKPeOI30m7RCYuh/EGTd+RH0s/EMNNsMUdKvEU4dTpvICIRAIAAAw9TC8SXFM7DOEBUGlg9a50vt/BE3CvGdIOvOj4BboHBLn68pgQysM3AbgUAMR4a3hYPTz6H/DayMQPxYkNz5d+A23nALl68/qt/hSPEgEMOXgcuIYAnnX7+SQSv0O3BgnX6F/U79Oy/gQiC+HzSZ9e8wqOXmbxUP0jQIxHDkm2wVX6r8R2uRb+o8Mu0g/g2QdedHw7HlwH8cONsRuP72Hu4SzlHcexEIBOJPxtTU9PLly3Db2dnZ0tLy5MmTv7ZLiP81kHXn++DYY7gNM/2MNIPabAYWfs1RCATij+Ufm2eGgbXj1atXaWlpU6ZMmTx5ckdHR0JCgpGR0a/uFOJ/CyR3/g5kXEEgfhP+8HeAefPmUanU06dPCwoKbt68mZeX9/Lly8rKyr+6Xz8DAwODpKSks2fP3rx5c9OmTTExMU5OTnBXXV1dR0eHgICArKyskJDQr+0nYhiDJrMQCATiZ7Bp0yZ+fv6pU6eOGzeOzWbLyMhwtA6GYdgA4w+GYWw2e2D5nwiFQnF2du7r63v58iUPDw8AAI/HUyiU48ePL1q0KCIiYvz48c7Ozr+6m4jhDLLufB1/+Gvl8OHfNrZxB0b6DR/6YL1asGDBQ9tkAEDNjGpLS0sAgLm5uYWFxYwZM35m1zo7O6WlpfX09HJycn5Ig3FxcU5OTk+ePCkpKbGmLfshbf5arK2tL1++fObMmbFjx/b09Pj5+cHymzdv3rhxA4/Hr1y50t7eHhbeuHEjLCxs1KhRnz59AiDh1/X6h4HH4z09PRctWnTq1CkikUgkEgsLC3ft2vXgwQNLS8sxY8Y4OTldvnxZWFj4V/cUMTxBcgfxO8Jms9lsNh6Px+MHGCD/PRUyQEvl5+dHRET4+fmRSCQMww4ePEgmk48cOcLPz/9v9eEbefHiRXt7O9y2tLS0sLBQUFBITk6ePXt2v5psNhuHw+FwOGhIwOFwLBaLSCRyV2Cz2QAATiGTybx7966ZmRmGYXv27Nm+fbusrCyJRMLhcEQikUwmww0AAA8Pz7Vr1xgMxtGjR7lP2t3d7e3tzWAw+Pj4Dhw4ICMjAwDo6OjYuXNnX1/f6tWrFy5cyGQy9+zZ09raumjRooqKivz8fF1dXW9vb01NTR8fnzFjxtDpdPD2X7uDP5eIiAhzc3MdHZ179+7Jy8sDAO7fv3/48OHExEQSiWRqakokEtetW3f37t2jR48+ePBgzJgxmZmZiYm/ut8/AiaTOXPmTG1t7XPnzhGJRBKJdOTIEQDAyJEjAQDz58+fNm0anU7/1d1EDFuQ3Pk6/j2jwk8wIfyB7kf4fvOsv8jQoq2t7eLisnPnzjNnzvj7+/v6+p44ceJf1ToZGRkUCmX+/PnQ4P/kyZPW1lYlJSVeXt6KigprAKBSgZXJZLKZmdmVK1cAYwUAoLi4+OHDhyNHjlRRUZGSkvrw4UNBQYGAgICpqWlhYeG1a9cmTJhgZmYWEhJCpVL5+PgCAwOvXLkyZ84cMTExAICXl9e5c+fk5eUfPnyora0NAHB2dg4JCVm6dGl7e3tWVlZcXJy5ufmDBw8EBQXfv38fEhJy+/btqqqqqKiokSNHJiYmAgBwXF61HR0dNjY2PT09LBYrJydn8+bNMjIyycnJq1at0tXVJZPJy5YtKy0tHTVqlIGBgY2Nza1btyZPnkwgEGRlZQEAY8eOHTt2bG1trY2NTdGuf++W/1SkpKQWL178+vVrzjRWXl5eZWWlgYEBi8Xq6+vr6ekBAMyZM2fcuHHa2tozZ848duzYL+3yD4PFYgEAYmNjLS0tmUwmm83W1tZOSkoqKCiYNGnShw8f3r59y62/EYgfDIYYmlsAuwXa29tbW1vb2trIZDKLxfr8+XNbWxuVSv0Bjf/3iX7Yvy+d5Z/CYDA+f/4MPQn+Aa9fv16xYsXX37HKyko5Obl9+/Zh2IC7hGF0Or2tra2trY3BYHR3d3/69Onz588sFgvDMBqN1tLS0tjYyGQyYc3W1tbe3l4Mw7q6uj59+gT70NXVFRYW9ubNm46OjlWrVl2/fn3QE2EYZmZmJisrm5aWNnbsWABAaGgohmEsFqurq2vLli0WFha7du3CMOzRo0fTp0+/du2al5eXjY3N7t27OS1du3bNxsZm/fr1hw8f5hTeuHHDxsbGyclp9erVXl5esLehoaECAgIEAuHEiROwGpyQkpaWVlVVBQBgtwC8TMjp06fxeHxWVhZ86Dw8PJs3b+7s7Ozq6sIwrLS0FNoPPn/+nJKSAgAYPXp0WVnZrl27AACXL1++du0aDodbs2YNhmHv37/X0dG5ffv2mTNnlJWVi4uL4V2CHdi3b19sbGxKSgqGYf7+/vz8/E1NTRiG7d+/X0dHB3amo6ODSCRaW1tzuhcfH8/Hx9fa2trd3b1q1ap3795hGDZnzhxlZWUmk8lgMCwtLTdt2gQrm5ubAwDevHnD/TXo6OhQU1MLCAgY5Fv9RwEAxvlb29bWtnHjRs4uaDO7d+9eQkJCdHR0a2srjUaj0+n19fVxcXFJSUkTJkzgPvyPA3bezMxMTEzM1dUVw7AnT55ISkp2d3djGObs7CwpKenm5jZ//vzt27dzH4L4Q8nMzITqIjk5+Vf35b9A36m/4xbAboFbt26RSCQAgIeHR11dHXyWV69e/QGND7r9A5v9QS37+vqSSKRTp04FBARkZ2dzyjMyMk6fPl1bW8spaWhoCAgICAgIaGhogCV5eXkbN24kEAinTp06f/58bm4uhmFsNjs8PPz8+fPnz5+PioqCNVks1s2bNw8ePGhmZqaoqLh///7+/b8FMK6f0+PHj/Py8kaMGIHD4WpqajAMc3Z2FhAQmDp16o4dO2pqatLT0wEATk5OGIalpaXx8PBcvHgRw7AbN24YGBisWLHCwMDAzs7O2dm5t7d30MexevVqaOqYMmUKnLLBMKy+vh4AsGXLlqioKABATExMQUGBqakpAGDTpk29vb0LFy50cXHBMKynp2fq1Kk5OTlUKlVERCQ6OhrDsIqKCgDA69evm5ubbW1tyWQyhmFXr14FAEBNcPny5Tt37mAYRqPRyGSyjo5Oamoqm83m7mFLS4uxsXFdXd3/9fYW6Ovr2759u5qaGrwzGIY9ffoUAAClyfnz52VlZTEMe/bsGQCgpaUFw7DTp08rKChgGLZ7924LC4vIyEg9PT0AwIQJE+BZli9fvmDBgn5fhuPHjzs4ODQ3N2tra3MefUdHBw8Pj7KyMnfN0NBQKJisrKwoFAqDwRAWFnZzc4N7d+zYMXHiRLhtbGxsb2/f70Td3d06Ojq5ubnDRu40NTUdO3Zs+vTpt27dgsr13r17eDze2dn57t27EyZM0NTU7O3t9fLyUlJSCg8PDw8PHz9+/B89/MPOX7hwYfny5VDQQDsW3MtkMslkcmtra0dHB0fK/9HXi0By54/lP39n/f39iUQiHI1Wrlypqqra3t7+rY21t7dXV1f/V+P/vc1gMEpKSsrLy9vb28lkcklJybt374qKijAMo9Ppb9++LSws/PDhA4vFgh/hGN/d3V1cXNzc3DxIs//52NLSsmfPHi0trXnz5nGGqIyMDC0trTFjxoSFhZHJ5Lq6urFjx9rb22/fvl1fX3/58uX19fUYhsXGxkpJSUEzg4iIyJ49e+DhFhYWioqKkpKSs2fPbm1txTDs8+fPM2fOFBQUFBQUNDExgdUOHjxIIBBGjRqVkpIyderUQ4cOYRjW0NAgLS0dGBjo7++vpKQEb2ZdXZ24uPjRo0dfvHihoKDwJetOV1dXUlISDodLSkrCMGzMmDFwpNy1a9esWbPevn3b1NQ0YsSIU6dOYRimp6e3cuVKeLSysvLp06fhdlBQkJCQELdQG1TubN68OTAwEADg5+enoqJy5coVDMOYTGZcXFxubu779+9FRESgcaimpgYAUFBQgGEYdMWAOubx48c3btyAmgnKnZqaGiEhoaSkpGfPnm3YsAGeaNq0aQCAZcuWbdy40czMLDMzE5ZTqVROHe4ehoWFycnJwVPAb2lHRweGYfn5+TIyMunp6RiGvXjxgiN3Ll26NFDunD9/fuTIkRiGnT59esuWLS9fvszLy8vPz+f8nVq+fLmFhQX233R0dKirq2tpae3YsYO7sJ/ciYqKiomJoVAod+/eBQC8f/8ePiwhIaHGxsbOzk4HBwfoGIRh2IIFCwQFBYuKinp6erjPtWfPnps3bw4bubNx40ZJSUklJSUZGZmwsDC4Nzw8XEpKSkxM7OzZs1C/fvz4MSMjIyoqatWqVa9fv/7lwz+ZTC4rK6v4O8rLyxsbG/sd+w86/8uvF/E9/LZyB02Ufi3q6uo8PDxwTkFJSamsrExcXBwAEBUVVVxcjMPhCATCzp0709LScnJyiETinDlzJk6ceO7cORaLtXz58okTJ96+ffvixYulpaWbN2/W1NRcs2bNoCfas2dPXFycj4+Pp6entbV1X1+fp6cnnU4PCgr68OGDqqpqSEhIeHi4jY2Nh4dHbW1tRUVFV1eXubm5np4eHFcGkpeXN2fOnKlTp06cODEiIuLAgQOhoaFPnz51c3NbtGgRkUjcvXt3VlbWpUuXFixYkJaWZmpqum3btps3b86ZMyc7O3vx4sWCgoKnTp1KSEjgnlyfMWPGgQMHpkyZYmhoePDgwbNnz86ZM6ehoaGgoIDJZNrb2zMYDBKJdOjQoQkTJnR1dZmamkITCADAx8eHQCDAaZeenp6TJ0/6+fnt3bt38uTJe/bsAQCIi4tDz9mBiIiILFiwYOnSpffu3VuwYIG8vPy6desAAO/fvxcXFx83bhwAYMSIEfHx8V5eXvLy8oO2g2GYkJCQoqLi0M+dQqGsX7+ezWaPHz8e6h4AwJ07d6qrq4uLi9lsNvTYHfRYXl7e06dPx8XFWVhYWFpaQtkKAFBSUtq2bdvatWs9PT0XLlzIfYifn5+GhkZxcfH9+/eNjIwG8dQGAACQnJzs6+ubmpoqICDAKVy2bFlSUtLEiRP19fWZTCb4jydNQkKCtLR0TEzM58+fc3JyeHl54YnExcWzsrKWL18OAJg4cWJKSkpGRgb0FX327Nn8+fPr6+vfvHmDw+H27t07Y8YMTldFRUV1dXXv3r175swZWPLhw4eLFy+yWKzOzs69e/cqKytv3Lhxy5Yt7e3te/bsIZPJ8IYDAGJjY8+ePbts2bKamhplZeXk5GQAQEVFRWlpKZlM9vb2ptPp0D0IANDQ0ODn57dnzx4wbuin9McQGBj4119/gf8sMoeFNjY2NjY23NVUVFRUVFQAACtXrvz5nRxIdna2mZnZ19RcvHhxbGzswPJhECkR8aeD5M7XgsPh6HR6VlZWa2vr6dOnDQwMYHlYWNjTp08DAwO9vLzKy8svXbpUX1/v7u5OoVBMTEw+fvx469YtIyMjGo0WFhZ2+fJlGRkZFxeXzMzMQeUOkUiMjIxcunQpmUwWEhIyMDAgkUhbtmwJCgq6du1aaGjo+vXreXh47t69a2NjY25ufuHCBQCAgoKCvr5+X1/flzqflpYmJiaWkJDAy8srKCgIXUHT0tJwONzx48cBANDz1N3d/fTp0xs2bDh16hQA4MWLF/Hx8ZWVlVJSUkwm09zcnFvrNDc3KyoqrlixQk1Nrbq6Oikpqb6+vrS0dObMmerq6gCA+/fvc+qXlpb2ExYEAuHz58/37t2j0Wiamppjxoz51icSGRk5b948OTm5W7duzZo1q99eAoFQUlJCoVA0NTXfvHlDp9N5eHjweHxVVdW3nojJZLq7u8MNePjly5elpKSOHz/e0dFx8uTJ0tJSeI0AgMrKSh0dndLSUrisLCYmRlFRcf78+T09PWfPnn39+rW+vj6RSDx37tzOnTvnz59PJpOfP39uYGAQEBBgZmZmb29/6NAhDw8PaWnpAwcOVFRUzJgxg0wmJyQkTJ8+PXYFAABgGPbXX39JSUmNHz+eu58ZGRnQuEKlUn18fAAAU6ZM2b1797Vr1wAAM2fOlJOTKyoqgl/d1NRU+ExPnz4NAJg9e3ZcXBz05lmwYEFwcDAAoKmpSUBAgI+P7/bt22JiYhy5g8Phrl696ufnBx80AEBaWtrKyurVq1eioqJWVlYwWNzTp08ZDAaZTMbj8Q4ODvBVYfTo0YGBgdBKKi8vD1cdS0pKxsfH8/Lywu8w52sjIyPz7t27UaNGgXt+3/rUfjirVq1KT0/n4+MbuhqDwdDQ0IDTiAPhXBp8QfrBXfzXmDlzZklJyZfENwc2my0iIvJzuoRAfCtI7nwtOByOyWQWFhYKCQkZGhpSKBRYvmLFipqamlevXklKSsK5GDc3twcPHsTGxvr4+MjLywcGBs6fPz8sLCw5OVlLS4tEIpWVlUEf0kHh5+c/e/bsunXrXr58+ebNm6SkJAAAg8EYOXLk+vXrAQBHjhyB9o+vB8Mwfn5+uKToypUr8LWS2yYBt+FbOIddu3adOXMG+0+UM2gwgEsqeHh4Tp8+fePGjTNnzujq6i5btiw+Pn7NmjUuLi65ubmcNh0dHY8cOSInJ9fS0pKdnb127dq2trbt27dfunSJh4eHl5c3JCRkwoQJPj4+cIC0sbFxcXHZvn377NmzP3z4cO/ePVlZ2c2Sg18UiURyc3NbsWIFXFgEANiwYcOuXbugVQmPx2dmZgoICJw8eXL9+vVmZmaTJ0+mUCgXL14cO3aslJRUZGQklUp1d3dXU1Pz9PQc2D6DwTh8+HBeXp6Xl9eRI0cKCgrIZPKxY8c0NTW3bNmybNmyu3fvLl261N7evrGxEQAA/z927Ji1tXVjYyMvLy+Dwdi7d29lZeWjR48AAKdPnyaTyR8/fpSWljYwMJCSknr8+LGAgMCVK1fc3d3t7e0fPnz45MmT169fnzx5EmoLSUnJPXv28PDwMJlMOp0OQBwAID8/PzY2NiwsrF+HL168yGazGQyGrq6ujo4OLOQEd+EA4+JkZWXBCUrOw4J+VNw1p0yZUlRUNOjNFxUVFRUV5XwUFhaeNWtWdnY2dx2OGOoHgUDoJ9QkJCQkJCQG1iQSif9AB/9LrFy5curUqX+7dIjNZg96LeBPtnAICgpqaWn9s2OHRZRExLDgF02i/Tn8x2kgJiZGQECARqNhXC6Wra2tlpaW8+fPh14sM2fOhAe9e/cOAODp6cnPzw+ddUJDQ8XExFpaWurq6tra2v7PU+/Lrso7d/6/9u48Lqfs8QP4uc/aplLIVspSKUKyZEmUihAxyTZI9p2xr2Wd7GPM2CeDZEiyhpBdZiyRpZJC0UKrlme9vz/O/J5v06bNdn3eLy+v597n3HPPc1vup3POvXceIaRt27b0qqKtW7dqampGR0ezLBsWFjZ8+HCWZU+ePGliYvL8+fO8vDw6UJWZmVm8KvYgOX78OI/HmzRpUnJysr+/v1gsjoiICAkJad68+fHjx0+dOmVlZTVo0CCWZaOiorp06XL8+HGlUkkv5/nll19Ylr1y5QqPxwsMDOzSpQsh5NmzZ8uXLyeETJ8+3djYmJ4DIiMjFQpF37599fX1vb29BQKB6gqjyMjIWrVqCYVCPT09e3v7ly9f5ubmDh8+vE6dOi1atHB2dqZXPMlksqFDh/bs2dPDw6NPnz7Ozs73798v4yidPXvW0NCw8HSo48ePd+nSpVu3bnQODUWDS3R0dHJyckpKSmpqampq6qtXr1JSUuLi4lRTqovsSKlUJiQkJCUlvXjxQiqVZmRkvH37Njk5OTU1ValUpqampqSk5OXlSSQS+tVUKBQpKSl0EhK9QKy0C9noTGr6Oisry8LC4n+zc8p2kLAsa21tTS8H+8/6ck9toXGnXbt2dPrOt4ETc3eq+A/gm/DVzt3Bz9DH/P/vWX9/f3V1dTp1d+7cuRYWFizLHj58WFNTk/7JXr9+fRsbm+TkZHq7Njr8MWHCBHq5QXJysr29fdeuXe/evbt58+Yffvjhzp07ZZzIHzx4IBaLt27dShcTExOtrKzatm07efJkep0OXW9vb9+uXbsBAwZoa2tramoOGzaseFV0sVu3bpqamt26ddPT0/vzzz/pzOJdu3bx+Xwejzd37twnT56wLLtmzRqBQKCrq5uXlzd06FCRSGRtbc2ybHp6+sGDB+k85ePHj2dlZT179uzo0aOHDh06cuRIcHDw0aNH6VTZlJSUwMDAgIAAOo9Y5e+//z506NDZs2cLr7x27VpERMTHvwSlHKWOHTvOnj37I5uX3ye6UK6YwMBAc3NzJycnFxcXBwcHAwODu3fvlmvLg4Rl2SFDhtBpv/9ZX+4Gv3//PigoKCAg4D/TtL9y33jcAfh+fLVxh2HR1Vi2AIYQIh0kadiwYVpaGiHk4cOHJ0+eXLRo0b59+7p27dqpU6fk5GQdHZ2BAwfevHmzX79+y5cvV1dXVyqV+/bt+/HHH1Uj9CkpKXPnzq1Ro4ZcLp83b56RkRH/sKC0pxasXbt2+fLl9G4udI1cLqc35BCLxaoJqnK5vKCgQKlU0ikFUqlUU1OTOcT7z335AhgylJXJZHSkQywW04vqCfn3WjBCiEAgoAPzCoWC3g2MDqDQW/GqykulUlXJz6fwkQlgyFA2MTExODj41q1bUqn0yJEjpc0UrvqOqqfakhw7doxm5VWrVtWpU6e80zgCGDKUlcvlRYdUvsE7SVbGV/hYDwD4r/DwcHpX99DQUGdn5y/dnP/B3J1yEQqFp0+fpomhSZMmI0eO7Ny5s7m5uYGBweXLl1NTU7W1tVu3bh0XF0fvekII4fF4o0ePLlyJgYHBvn37yrnHly9f7t+/X5V1CCH0KTP0ipXCKws/Q5jeire0j0AIoVflqDAMU2QTPp+vOvUWn6ZQRv2fk56e3oULF1iWrc6sU9ynzBDuhLi7EUIIufxHxbYMYEr+oeV2FPhO8hwAfDLo3fkY/J4F+EpwO9IBcMJX27vzeUclAAAAAD47DGZ9DP6gBAAA+MahdwcAAAA4DnEHAAAAOA5xBwAAADgOcQcAAAA4DnEHAAAAOA5xBwAAADgOcQcAAAA4DnEHAAAAOA5xBwAAADgOcQcAAAA4DnEHAAAAOA5xBwAAADgOjwgFbmJ8GPqCXcbS14VfFCmjQssU30S1VWnrAQDga4beHeCgwrGGxpciL8h/8xD9V6SSIvGotNcAAPD1Q9wBDiocSsouQ4upShZJQoVXlrg5Eg8AwDcBcQc4qJy9L0U6dUorX1r3DwAAfCsQd4Czyu7doZ06JU7fKfJ/4ZJF+oEAAOCbwLAs/mb9ijAMIYQU+ZqUuBLKCYcUAOCzCQ8P7969OyEkNDTU2dn5Szfnf3Bl1jcAZ+XiKtC5spyWL8dKAPhkMBwMXxYGs6oHw/zbYUBfFFksXIZhim5VuHDZtRWvp8QdcUN5DikAAEB5oHenGqhGRoq8KK0Mw/ynjKrzpnANhd8qknWK11PkBQeU55ASQshytsQyhQ/CRwezSttX8S8HwNdPdcMFKicnRygUqqmpfZHGpKWlCQQCvV/0vsjeAQpD3PmsCp+zVWdTnFOrAocUvl3VMuG9tEGigoKCI0eOzJ8/38rKys3NrVu3bs2bN6/67ipk8eLFO3fupGPHAF8WpipXg/J0D5RxAi7cJ1Fi+e+wK+KjH5Ms/2YGtL6JKQtf7YVmxY/eV9vUcir8iYr0xJRfv379Tp48GRUV1eJoiyI1qOq8f//+smXL5s6dm5ubm5+fb21tbWRkVMXGly0tLU2hUNStW1e1RqFQODk5XbK7RL6RHwSoOkxV5jLVQFJpawoPNhUuUKSSIhsWKVNkGItwJdmU6KOH9F/L2SIFilRClXHYi6wsLVNWWmnn5i9+zi7xLtJfpCVlKO0ofdmmXrx4sU2bNnp6lRmgqa6v++bNm0+ePFl2mTlz5jRo0KBLly5F1iuVSkIIj/fvxM2HDx8eOnTI2dm5ZcuWfn5+lpaWI0aMWL9+vUgkmj59enJy8rZt2xiG8fDwaNGiBSHk6tWrJ0+eHDRoUIcOHfLz8zdt2vThwwdXV9eYmBg/Pz8TE5MzZ84QQvbt2xcXF6elpZWSklItHxmgqlioPoSwOKLVq7RDSpYTsvwbONalNbLE9Rs2bDAwMGjcuPGMGTM+c6uqfjDlcvmdO3eysrIqXcPevXsjIiI+2qpP9HV//fr19evX09LSCq989erVrVu3bt26FRcXR9cEBweLxeK+ffvevHnz6tWr6enpdH1CQsKFCxciIiKUSiVdY29vb25uHh8ff+HChYYNGx44cKB44+liTk7OlStXzMzMWrVqdevWLZZl16xZo6ent3btWisrKwMDg1mzZikUCpZl8/Pzhw0b5unpOWjQIEJIVFRUiV9KmUw2c+ZMAwODJk2aeHt7//jjjykpKSzLRkVFDR48eOjQoX379p03bx7LsnK5fMiQISKRSEtLy9XVtXHjxt7e3jKZTCAQ7Ny5Mz09PTMzc9q0aYSQS5cu3bt3r23btk5OThs2bGjVqtWdO3dYlo2MjHR2dhYIBH369Hn06NGrV69Ylj148KCRkdH9+/cPHjxICPlWflqhWly+fJmmi9DQ0C/dlv/At2C50JMu/n3qf2Uc9qJvFfoFqlQqZTJZdna2VCr99N8LFVOhuHPnzp1x48YRQgYOHPiZW1WJs5FSqaTnYOrt27eEkJMnT0okkszMTIlEoiqWk5OTl5dXeFtapvDK9PR0W1vbUaNG5efnZ2Zm0i9laXHn9OnTw4YN8/Ly8vb2vnv3Lsuyq1evDgoKYlk2Nzd35syZdCXLsqdPnx4+fPiECRO8vLzu3btHVwYFBY0fP37atGnjxo1bsGABy7Lv3r1r3bo1IcTe3p7mEpZlU1NTrays6C/uYcOG0ZVr1qwhhJibm/v6+rZr1y45OZll2VevXrVq1Wrt2rUdO3bcvXs3LXnw4EE7OztbW1s1NbU1a9bs3bu3+Ceii5cuXSKEbNq0ae3atYSQhw8f3r17d/To0Tweb+3atUuXLiWE3L9/n2VZDw+PESNGsCwbHx9fRtyhL/r06TNy5EjV+vz8fE1NzcmTJ7Msm5OT061bt8ePH9O39u3bZ2trm5GRQRezs7P5fP6NGzc2bNhw7do1qVTq5eVF3zp58mReXt6rV6+aNGnSu3dvujIqKqp79+4fPnxQ7WvNmjVLly6lr4cMGYK48135auMOBrPg6/XRUaTs7Gxtbe2nT5927dq1e/fuc+bM6dChw2dp2ifRrl27du3avX//Pi8vr/D6d+/ehYeHm5iYtG3btvD6e/fuxcXF2dvb165dm65JTU29ceMGwzDa2to9evSgK1NSUsLDw8VisZ2dXRlDMEqlMiwsLCsrSyQS1axZ087OjhCiUCjOnTuXlZUlEAjq1q3btWvXkydPyuVydXX1mJiY8PDwuXPnduzYkRBy5MgRQshvv/1Gz4ijR4+mDZg1a1ZMTMz79+9nzJjh6elJ69y9e/e9e/fu378/d+7cwYMHE0J27txJOzamT5+el5c3ceLETp06ldbU+fPn169fXyaTXbp0qVevXiKRaPny5TNmzHB3dw8ODt60aZOtra21tXVCQsKgQYNatWr18uXLjIwMb29vQkhMTMyiRYuOHj3q6+t77Nix48ePE0JOnz6dm5v76tWrgoKCqVOnurq66urqHj9+/OHDh8+ePdPR0QkODqa7njJlSlhY2Lp169q0aTN9+nRtbe2EhAQHBwdra+vOnTtraGgEBAR4eHjUqFFj6NCh58+fj4uLS0tL09TUZD52+4TGjRtnZWURQmQymbW1tZubm4WFxU8//UQIKSgoGD9+fERExLlz57Zt20YIYcsxwurh4XHz5s3Ca+RyuZOTEyFES0vL2tp63Lhx169fJ4To6up26NBBV1eXFhOLxa1atbp27VpsbOyBAwcsLS3pNxgd2AoJCXn37l1cXBwNZ4SQf/75p2PHjpqamnSxoKAgODh4zJgxH20hwGf1pfPWt6HEDobc3NzCf91+DVSNzM3NHTp0qJOTk5ub2/jx4z9zM7Zv356YmFhGgeJ9NiUe4bJqWE7IcnLkyBGWZceOHUsIefnyperdvLw8mUymWiwoKJBKpfn5+Wlpafn5+ar1crn83bt3RcZfsrKy3r179/79+3fv3uXk5NCVGRkZCQkJaWlpBQUFLMumpaX169evT58+R48evX37tq2trZ+fX2ntLL4yMTGRLCf9+/f39PTs27fv6NGjVa1NTEw0MjLq169f4fIeHh6EkPr1648dO5aOlWRkZHh5eTVo0IAU6grKz8/v27cvwzD0wuPg4GCWZXNzc3v27Pnjjz+qq6sPHjy4tFaR5WTr1q2EkLFjx4rF4iZNmtCul59//rldu3bBwcHDhg1jGObEiRNhYWE03yxcuPDo0aPNmjULCQmhTWIY5uLFi4Wrffv27eDBgwsKCm7fvu3q6kpXzpgxw9vbm2XZf/75Z9CgQaqShJByDmbVq1fvxYsXcrn8woULSUlJLMva2NjQEUCZTKalpRUYGMiybGxsLCHk9evX3t7ejo6OdPN//vmH9ugMGzZMdehat25Naxg3bpy6ujr9q9Tf319LS8vR0dHDwyM1NZV+je7evUsIKfztvXr1akKISCRSU1Pj8/murq6qb5sff/xRtd8SPxFdpCNT9vb2e/fuJYTQrqng4OAdO3bQYitWrDA1NWVZVkdHh3Y+vXjxgh6uMnp3HB0df/zxR9X6/Px8NTW1gIAAujhlypQ9e/bQ1126dCnyW+LAgQP0W8vMzEz1ebdv3z5z5kyWZa9cuUII6dmzJy3cokWLqVOnFhQUvH//nn5/7tmzhxASGBgYGRmpr6+P3p3vylfbu4PbDFZA4fvdMQzR1NTg83lFVn6Rf0Xk5ub2798/PT1dV1c3JCSEzhz8bNLS0jZt2jR79uzyFJZIJDt27Hjz5k3l9jV9+nRCCD0DqYSFhbm7u8+ePfvZs2eEkEuXLg0fPnz+/PlDhw6tXbv2+PHjFQoFISQmJmb58uW1atVq0aJFXFwc3fb58+fW1ta1atWqXbt2rVq1Ro0aRQh59uxZjx49Nm/ebGhoGBgYSAgRi8VJSUmnTp0aNGhQ//79GzZsqPrTtjzoTVDOnTtHM8S+fftu3LhB3+Lz+QLBf/pcr1+/TjPWkSNHIiIiZDIZISQ8PDw0NPTYsWMsy+ro6NDeoBkzZqSnpyuVSqlUunPnThcXl5ycHHq63bdv39mzZ+Pj49nSewXu3bvn4+Ozc+dO2j1z7do1Qsjjx489PDz69+9/4MCBpk2bRkZGOjg41KtXr2/fvqtWrRo4cKBCoXj69Glpdero6Njb2+/fv9/X1zc1NZUQcuLEic2bN9PP2LZt28OHD5f/uKn069fPxsZGKBQ+efKETrwtw/79+2NiYhwdHemipqbmmjVrrKys0tPTt2zZQldKJBInJ6eNGzfSibe0C8Tc3Dw7O7t169YikcjAwCA8PLzE+q2trQkhfn5++fn5crl8zZo1mZmZFfo4V65c2b59++XLl7W0tFQrBQLB1atXCSFyufzWrVtCoZAQ4uDgEBIS8vfff9NemSlTppRW55s3b+Lj46Ojo/fs2fPLL79kZWWpqak5OjqOHj36119/nTt37o0bNwYOHEgIefny5cuXL6Oionbu3Ll169acnBxCiJmZmVgsbtas2YABA1R1isXiTZs2MQzTrVs3QsiFCxdCQ0MPHToUFRW1detWNTW1evXqvXv3jhDi5eU1Z84cT09PBweH/v37V+hoAHwiGMyqEMbT0/PQoUN//PGHl5fX3Llzf/7552qpNz8/XyKRqDqTCSHp6env3r3j8/nGxsZ8Pp8QkpaWtmfPnsGDB9epU+evv/7S0NCgowBF4k5eXt6AAQOMjIx2795NCGnbtm1QUBB9KzU11c3Nzdvbe9CgQTo6Ojt27Lh06VLDhg2zsrK0tLTOnj17+vTpGzdubNy4kfYQEEKkUmlAQIClpSUhZObMmbdv3zYwMPj999/r1au3devWHTt2TJ48+eLFixoaGjExMX/++aepqencuXNjY2N1dHS8vb0lEsnq1asNDQ0jIiKmTZtmYWGRk5NjZmZGyCrapA8fPvz00093794lZGcljlu3bt06duw4cODAe/fu0TWnTp0aP358SEhIZGSkk5PTo0ePjIyMDAwMjh49evDgQRcXl/Hjx+vp6W3atOn27dsnTpx48uTJzZs3e/bsGRkZWaNGjdDQ0IyMjL///vvWrVsxMTFz5szJyclxcXEZN27clClTNDU1V69eTYcqzp4927p1a2dn5wULFjRr1qxCzdbX1yeENG3atE+fPj169LC1tfX396enEF1d3Xr16tFARgh59eqVu7s7wzDjxo3j8/kaGhp//fXX8OHDe/bsOWvWrClTptSpU+fFixc0P82fP3/cuHGjRo2KjIxs2rTpoEGDYmJibt26ZWhoOG7cOJZlGzVqVMaQyokTJ2hw6dmz5+3bt42MjGJjY2/fvr1s2TJCSH5+fkFBAd2cZVn6PVk2iUQyefJkhUIxbdo0XV1d2mFAc6Eqdb169apmzZo6Ojp0kUai/Pz8Fy9emJubl1bzggUL+vfvL5FIBg4cWKtWraFDh9atWzczMzM5OTk1NVWhUCQlJWVnZ9OLjz58+DB69OiuXbvm5uaqUqmHh0eDBg1ycnISEhKMjY3XrVs3c+bM3r17//XXX7du3fL397ewsJg1a1ZBQcHkyZPDw8NFIhHd1srKasKECQ4ODmKxuGbNmh07dly7du3Ro0cXLlzo7+/fqFEjGxsbmsIHDhx44cIFQoiFhcWwYcMWLVpU2sdZt27d/v3779+/f/PmzQYNGhw4cMDa2vr48ePnzp3buXOnu7v7w4cPtbS0Ll26FBAQcOrUqenTp6urq4eHh2tra1ufsC6xzrt37w4aNCg4OHjr1q1+fn70x9nf3//hw4cSiaRJkyZTp06lh10kEvn7+zMMQ7++tKSNjU1ERIS2tnbdunV79+5NB7N69+59+fJlelR5PJ5CoWjZsqVcLg8PD+fxeCzLMgyj+iXm4+PTu3fvmjVrtmrVao/Pno9+twB8cl+sX+mb8v+DL4QQ8vDhw1WrVhFCdu3apSrw22+/0Q5bspxERkbSF//eX2s5efjw4b/1/H8Z1b+MjAyynAwfPpwurl69+n/vUsv/7QdWrad/TdatW5f2rhcZEtq0aRMhpPjYyt69ezt06DBhwoSmTZuamZm9e/eOXgyyZMmSvn37urm59ezZc+7cuZcvXzY1NV20aJGrq+uAAQMcHBxmz57NsuzFixe7du16+vTp7du3t2jRIiMj48KFC/Qs+8cff9DuGdX80IkTJyYlJclkMolEQju3T58+LRQK8/LyMjMz58+frxrMyszMPHXqFFvZwaz3798zDLN3716GYehg1v3793/66afVq1dPmDCB/H8n/LZt206ePMmybGpqqpmZmZOTE61kz549q1evHj9+PI/HS01NZVl269athoaG9CP8+uuvLMu+fftWIBDQcwAhREtLSzWjs1mzZitWrPhoO0tb37JlS/q6yKDDkCFDXFxc6OuoqChCiL29PV1UKpV0AK5169ZGRkYbN26MiIiwtbWlE3svX76ckZFBI2bNmjUXL1785MkTQggdRVIqla9fvz5//nyJraLtGTdu3PPnzyMiImbPnh0eHs6y7MCBA+nm69evV31fubm5ubm50Q0bN268atUqlmUzMjJEItHQoUOXLl26YMGCFStWPHjwoGXLlvSvAjs7u1q1avn4+Mhkst9//71169bLli1btGiRUCikQzYFBQXOzs7a2to+Pj6DBw/u27evXC4vbTCLEGJiYtK5c2cDA4Nnz56xLPvhwwd6qw9XV9fhw4cTQoKDg1+9ekUIYRiGpjQ6gkb7/FQrVT9HZ86cmT179syZM//880+6o23bts2cOXP+/PkzZsygw6aqr8LKlSunTp2qGhhiWfb48eMTJ05UzYZmWdbPz2/evHmLFi2aNm0anUZd/MirFmUyWUFBAR0cp+0p/L/qS09fKxQK1TD6p7jIrtphMOu78tUOZqF3p2IMDQ2PHj1Kh4dYliWEpKWltW/fPmFUAiFEvEYsWSBpFdyK3ulr3JtxrYJbRQ6ItDpmxbZkVbfcsD1ne+/eveDg4N53emdkZBBCCgoKCCFkOVm4fCEh5GLXiw7XHMhyEj8yvn379mmT0+i249+O31Fvh7e3t4uLi0KhKM9f2CqrV6/u16/fhg0b4uPju3fvfv/+fUdHR1tbW19f38OHDwsEgg8fPsyfP3/t2rWdOnVauXLlwYMHtbS00tLSfHx81q9fP3fuXHt7+3r16kkkksTExDdv3jg6Ou7evTs2NpaO+MyaNeuff/6xtrYuKChITk4WCASFB2UMDAysrKyMjY2XLl06Z86c/5/jSHR0dFxdXSv95dDT0/vll18EAgH7/70F169fj42NffXqlVQqLd6TUbt27c6dO9N5D6tXr758+fKePXtu3ry5Y8cOWsDFxWX37t1jx47l8/n0IilCCB2hGDt2bFJS0tWrVxMSEuhVPCXKzc29d+9erVq1PnoH25ycnKioqEuXLoWFhdHrfeLi4mivXk5Ozj///FOrVi1LS8slS5acOHGCjmts2rRJQ0PDy8srLi7O1NTUycnp1KlTt27dGj9+/N69e48dO7Zo0aLNmzfz+Xx9ff2GDRs2b9588eLFR44csbS0nDhx4rlz51xcXHr27Flie4YMGbJnz573799LJJLHjx+bm5t369Zt3bp1M2bMGDNmTHJysru7+9SpU6OjozMzM/l8/ty5c/38/GxtbS9fvqyhoTFjxozt27f7+fk9evSooKDA29u7VatWffv2PXLkiFgsFovFjRs3DgkJmTp16oQJE+Ry+S+//CISiUJDQ9u3b08IEYvFQUFBo0ePDggI2LJlS5cuXcr49qZjWHK5nH5MQoimpuaRI0fevn1raGgoFAqXLFliamrau3fviRMnLl68WC6XZ2VlzZo1Sy6XT5gwYcqUKb6+vgUFBSkpKT/99JNUKhUIBL169erVq1fhvUyaNKnEvTMMU7yrhkbAwmvmzJlT9jdAYYV/XuiLwv/TnaoOiOqWOaX54nd1AvgKIe5UgLa29tmzZ+3s7KZNm5aZmSmXywkhmpqaY8eOHTVqVINdDSQLJLSkk5PTOXKOEPLgwQPVr57sWdnaG7UJIbecbxFnQgf4G//ZmBBytMVRQkjDhg0TSSIhxOGaA92EDrrT1+wydu7cua2Pt554f2IZjXR2dtbS0qJtI4TcunUrKCho/fr1SqWyTp06hBATE5O5c+f+G7DKLTU1NTExMTIyMi8v79y5c3T45vTp03QIhhBSu3ZtGiOuXr0aEhJC5yoqFIqCggINDY3Dhw9v3br11atXz54969y5MyH/m+0hlUqFQiEhlfkFLZVKp0yZcuvWLUJITk5OVlbWzJkzjx496ubmNm3atMePH+/Zs2fp0qUymWzXrl2Ojo4pKSmhoaF0TG358uUuLi5+fn60D2Dy5Ml+fn7h4eFRUVE2NjZKpZKuqV279pAhQ1avXv369eukpCSRSGRpaZmXl7dkyZLMzEw6+MXn81esWCEWiwkh0dHRdnZ2zs7OoaGhZTc+ISGhZcuWlpaWBw4cGDJkCCEkLS3NxcWFXgbcrl27KVOmbN261dfXt1mzZn///TchpE+fPl5eXgzDnD9/PiIi4vz582KxeNu2bQUFBZmZmcOGDWvUqNH169d5PF54eDidyLxixYoxY8YEBgaamZmVMdWDELJgwYIFCxYUWWliYhISElJ4jZmZWeFZLHSIiho9evTo0aMLF161ahXtCi1iypQpxRujqan5119/ldFClRKjpL6+Ph0lJISYmpoSQtLT0+Pi4pYvX37lypXNmze3b99eIBC4ubnt3bt30aJFLMtGRUW1a9dOQ0OjPDv9RKo9mnxDt4sE+JwQdypALpfTW46ampqq8sSVK1cuXbq0SLaIkP8OQhVz6dKlnjd61qhR45jVMbqGXcYW/kUwderUefnzCCFtQtp06tRp7dq1+fn53t7elkcsy9/I5s2bu7m5PXjwgGVZhmECAwMPHTq0fv16gUDw8OFDQsjVq1enT59+7tw5QkhaWtqbN2/EYvHff/9tbGwsFAqjo6NjYmKSkpKioqLU1NRoQiKEtG7dWltb28TEhE6DZRhm165d4eHh6urq7dq1s7a2joyMvHnz5pgxY5ycnFxdXXv16rVo0aKDBw/WqlXr119/3bt3b05Ojru7e506dehcSCo9Pd3Q0NDHx4eQn8r/GVUWL17s5+d37ty53r17r169evbs2adPn967d+/p06ffvn3r7OxMR3kePnwok8mmT5/u6enZsmVLNTW1ly9fHj9+3M/P79mzZ3w+v1evXo0aNTIyMrp37561tfXr169ZlhWJRA0bNjxy5EhAQMC9e/dmzZpF764mEAgUCsWUKVPmzp0rl8slEgmfz1eNdllbW3t7e6urq3+08RYWFg8fPszNzdXW1qZrOnbs+OLFCzpxh2VZ1Wl4xIgRRbbt2LEjvTyqsA4dOpR4Hb6xsfH8+fMrclw54s8//5w2bdq8efPoRF16GH/44YezZ8/GxMRIpdKuXbvS66q+lK8wmgBwFeJOBSgUivT09M2bNxNCZs+e/eHDB0LIunXraHc6If/LOiVOCO3/oD/p/L/F9+/f/2+r5SRyQGSzZs28cr1qb6t93+3+fXJ/24ZtG7U3zsqe9W+dPkzDQw2zsrLs7e179OhBbz5Woj179gwfPrxjx45CoVBLS+vo0aOEEH9///Xr13t6et69e7dHjx5du3alN4EdOnRozZo1r1+/3qJFC4FA4OXlFRERERIScuLECULI1KlTP3z4EBoaGhgYePTo0X79+o0bN87FxcXBwYEQsnDhwrlz5w4ZMkQul7u7u7u5udHrhpYtWxYTE5OVleXm5kbHqgICAtLT01NSUrS0tC5cuGBh8W9TtbS09u/fr8oKFeXn50cIWb58eeGV9LKaIgdE9ZrO8CCEtGjRonfv3oWLpaWl/f7775cvX7a3tyeEbNy4MSUlhQYIa2vrwl0afD7fxMSkxCa9efPm0aNH9CY0ZYuOjnZ3d//zzz8Lr6zcowmgRKampsX72Bo0aECzPgB8V/CI0HL5//TCtG/fPiwsLDo62sHBITs7+9SpU3TOhKqkpqZmRkZG//79z5w506pVqwcPHjRq1OjVq1djxoxp2bLljBkzCCG2trY6OjpCofDEiRMJCQkmJiZisVgulysUCgMDg4SEBDU1taCgoODgYIFAYGVlNWvWLELI+vXro6KixGJxVlZWly5dVGMB9IFcRV4rFAoayDZv3qy6ulWhUOTk5IhEIrFYzOfz6XgWnf8oEolkMplQKJTL5UKhkM/n0+t7+Xy+RCJhGIbOSqazHFRTB+j8CZFIVNnjWVT5vxnRPQ7wbUFX1nfiq31EKOJOuTDMt/QIbgCArw3iznfiq407GMwC4AicTgAASoO4U0HLcUapJsuZqp+epVJpREREmzZtCt+OtiroHQRKXBMfH5+QkMDj8erWrUvvrP8p5OXlDR8+XF9ff9euXXTNsWPH/vzzzzNnzvTv37+0C5cwtAcAUDbEne9DRUfiiqS6ahnIKykpymQyeiFSkfv0lMf9+/fXr18fEBBw+/btwlckvXv3bvr06SYmJitXrlStZFlWoVBUdBcqL1++7NWrV3R0NCFk8ODB9FESn4JEInnz5k3hudvu7u7u7u4eHh75+fmfaKcAAJyHuFMuLEsYH0LItzGDp3ivCeNTgZEOxocpMqGrxM2XLl26YsWKgIAAes+YStSZkZHh6upK75ozePDgil6iRe+h9+jRo3HjxkVGRqrWCwSCf/75p0jhAwcO0FuwlL9+lZ07d06dOnXnzp0jR450c3Mr8rjyqvjnn3+ePn3q4OBQv359uub69eve3t5GRkbVtQsAACCIOxX1ZadHsCybnZ2tesBQicoY18jOzpbL5ZW+1DkpKeno0aP0eUCEkKFDh/r5+VXl3N+nTx9bW9tly5YxDLN79+6bN29269aN9mHQ+9Z8+PBBLBarMhB9BoWOjg59epGamlqjRo00NTVVtz8mhOTm5sbHx/P5fHo5G3XhwgUvL6+6detev349Ly/PxcVFKBSePHmyRYsWO3fufPbs2bRp06RSaeHeoMI2bdpkZWU1cuRIQsiyZctodxS9w2FKSkpeXp6WllZQUNDGjRtPnz5tZWW1ZcuWO3fuTJs2bdiwYV5eXmPGjMnIyNDW1tbT00tKSurZs+f06dPz8vLmzp179erVqKgoCwuLKVOm0KdeaGho7Ny508DAoPjl9AAAUGl4Ivq35Nq1a/SkWznz5893d3ev3Lbv3r1zdXXdvn37r7/+um7duj///NPc3FxHR0cqlfr7+//888/0Ec3U2bNn16xZo5p9UpqoqCgvLy9nZ2cnJ6dmzZrNmTPnzp07Y8aMoU9TDwoK6tevX2xsLC185swZT0/PBg0aDBs2TCL59+7VGhoaLi4ub9++VdUZExOzcePGp0+fFu4o0tbWVldX19HRadq0aZMmTQwMDBo0aJCfn0+f4JiZmXnhwoVnz54VfkRrYc+ePdu2bRt9bW1t3a5dO0LItm3bdu3a5e3tbWtrGxUVdebMmUGDBoWHh9MHievo6ERHR9+5c0dTU7NZs2ZhYWFjxozZsWNHQUHBnTt3CCFr1qy5fPnyw4cPHz16lJSUpHqou4ODQ9OmTVU3sQQAgGqB3p0Ki4+Pz8nJqVGjhupGc+np6UlJSQzDaGlpGRsbZ2VlvXr1qvDNaYyNjWUy2cuXLwvfoobP55uamvJ4vHfv3iUmJiqVylq1ahkZGaWmptLZG/QZxQKBwMTEpEaNGoSQP/74IyQkZOvWrQzDCASCCRMmSCSS58+fT5o0KTc3t3///nZ2dqU129fX9/z58zk5OWPHjpVIJHXr1qWPe5w3b97ly5ebNGlSu3btLVu2lPY4npcvX0ZFRYlEog0bNkgkkhYtWvz444/q6upLly7V1tbOz8/38/O7ffs2fdzBjBkz6O15fvvtt7/++qu0B4ZbW1vTZ60TQoRCYWRk5N27d0NDQ+mGGhoat27dorNt7t27N2/evN69e0+ZMsXZ2fnNmzeqg6+mprZz586FCxfSxTZt2uzfv19TU7Nwt1OHDh3MzMy6dOlCn59K9enTZ/jw4enp6SzL6unphYaGtmjRggSVfPToM1AJIXK5PD09XSgU+vn5HT58uG/fvn379n306NG+ffv8/f11dXXp/YrMzMx69OhRr149QgiPxzMzM3NxcSGELF269NChQ7Qq2illaWl5586dL/scAwAAzkPvTsXcu3evffv2rVq16tix44MHDwghEolk2LBhVlZWLVu2tLW1jYuLu3TpkpWVVYv/Z2VlNXHixOTkZDc3txaF0AdZ5+Xl9e/fnz7jqUOHDvHx8RkZGb169WrRooWlpeXixYs7duw4efJkQkhOTk50dLSBgUFOTk5aWlpubi4hJC4urkWLFh06dBg+fPg///xDn8NVoqVLlzo4ODRr1mzXrl3+/v5r164lhMyfP3/dunVjx44NDAyMjY1VPcm2uLZt23p4eHTr1i0+Pv7Nmzfnz58nhMhkss6dO8fFxb18+bJt27b0PtE7duwwMjIaOHBgly5dHjx4UOSuwaWRyWRmZmYTJ05kGIZlWalU2qtXr1mzZtHUYm1tPWvWLB6PV+T5TaUxNDSksaMMenp63bt3d3NzS0pKWrZsWRlP/SSEqB4J6e/v/9NPPzEMk5yc3LRpU1UBuVyuqak5e/ZsVcwSi8XFH+1kbGxMA5xSqVTd67lWrVr37t0rz+cihCgUCtVH++uvv6ZPn/78+fNybgsA8N1C3KmYHj16dO7c+ddffxWLxXRU4saNGw8ePBgxYkRiYmKTJk127tw5YMCAtm3bjhw5kj524PTp0y9fvpRKpSYmJmPGjLl48SIh5N69e25ublKp9OrVqzdu3JgxYwZ9sk/Pnj3NzMyCg4MJIefPnz9z5syWLVuOHTsmlUpr1KjRpUuXevXqLVy40MfHh4741KhRo379+m3atLG3tx8/fnybNm3KaDzDMPTpFjwej/bi/PXXXz4+PrSboV+/fqp5OeUkk8noMyKEQmGPHj0ePXpECAkNDU1ISAgLC7t//36PHj2KPCa6sPDw8LNnzxZZaWZmplQqaTvj4+PpYNbGjRv9/Pz09PQ8PDzK2bZnz54VXuTz+S9fvqTjVvTZYYSQESNGPH/+/LfffqPzZkqzY8cOmUwWHh5+8+bNgICAevXq6erq/vbbb2PHjr169eovv/xy/PhxmpZon9PFixdXr1598+ZN2pEjFArpfpVK5dOnT2NjY+/fv0+TUKNGjWbPnq265osQsmvXrkOHDl26dElLS0v1wM527doFBQWdP39eX1+/ffv2qsG7EydO/PLLLwkJCeU8JgAA3y0MZlWMsbExHbPo379/dnY2IWTFihW9e/emf6n7+vo6OjquXLlSU1NTR0eHPkurbt26WVlZW7ZsEYlEurq6dGWjRo20tLR4PJ6fnx+Px1u4cKFMJlNTU2MYRqFQ1K5dmxBiaGhICNHT0yvjzteGhoYXL17cvn370KFDCSHlfKA0IUQikYjFYi0trdOnT//0008aGhpSqVT19PUSMQxD55Tk5ubyeDx1dXWGYaRSKX1XJpP9/PPPY8eOHTlyZERERGhoqEAgOHr06OHDh21sbEqssEGDBhs2bOjWrVtGRsbp06fpYzEWLlzYt2/f3bt3x8TEHDp06OzZszVq1Pj5558tLCyio6OvXLlCCJk0adKKFStsbGzopWEpKSnu7u5t2rRZsmTJtWvXNm3aFBsbm5CQcOrUKRsbGzrONXXq1JEjRzo5Of399999+vQ5efIk/dJ4eXmdPHmy7OdgjBs3jmVZep/QCRMm/Pzzz4SQiRMnGhkZLVq06M2bN4GBgXRS1LBhwzQ0NJ48eaKtrR0aGnr79m3attq1az9//tze3p7H4/Xt2/fJkyfDhw93cnIKCgqSy+XTpk1r1KgR3ZempuauXbv4fH5+fr6qw4k+D4SutLa2po86pw3r0qWLpaUluVGerzkAwPcLcadiIiMj4+Pj27Rp06BBA3rWEQgE0dHReXl5pU2/MDIysre39/Hx6dOnT5G35HL5xo0bZ8+efezYMTq59fLly3w+/6NP9nj16tX58+etra0jIiJu3rz5xx9/+Pn5TZgwITAwkFiVulXbtm1v37595MgRqVT622+/bdu2bfv27V26dNm1a5dQKJw5c2bx5ykW1qVLl0mTJnl5eQUEBJibmz948EBDQ2PKlCnNmjWztrZeunRpy5YtCSHDhw//448/hEKhnp6eg4MDnb9SohcvXqxYsUJLS4tl2UWLFtFOpj59+qxYsSIsLEwulwcEBFy7dk0gEISEhPj4+CQnJ/P5fHd3d2NjYxsbm3v37t25c8fc3NzKyiojI4POX1YoFNnZ2c2aNWvevHlmZqZqUvOQIUN69+69a9euCxcuqO6+I5PJoqKiOnXqVPbRJoR4e3sPHjyYZVl6URjl6urao0cPhUJR+CaHAwYMUL02NzcnhBgYGFj8/zNRe/bs2bNnT/q6Tp06EydOLLIjGluL8PT0LLFVdnZ2ZczWAgAAFcSditHU1Fy+fPnvv/8+adIkkUgUEBCwZMkSekF1zZo1379/v2zZMj6fL5PJAgICzMzMhELh0aNH6eRZgUCwf/9+enIdPHhwbGxs3bp1J0yY8OjRIycnpxo1avD5/JiYmAkTJvj7+xNCRowYsXHjxr/++isvL+/QoUMjR44cO3bsnTt3xo0bl5CQEBkZefHiRX19/ZCQkBEjRggEgqysrIMHDx7beKy0xo8dOzYvL8/Dw6N3796LFi0Si8XNmjU7c+ZMfn4+y7InT55s3759GZ99/PjxpqamOTk57u7u9GL4ffv2ZWZm1qxZU1dX9/Tp0/TKpubNm9PxrBo1aqgeP14ikUi0dOnSbt26vXz58pdffnF3d7e0tBSLxYsXL1aVUd3Up/iwl7W1Ne2kKcze3p4+z7wIHo9Xs2bNuXPn0sW3b98yDBMbG3vu3LknT56U0UiKz+eXeN0WvWAeAAC+cog7FRMWFubk5OTu7t66desffvhBIBDY2dmdOnVq3759hJC+ffvOnj37wYMH1tbWbdu2jY2NnTJlyrNnz4YNG/bhwwcbGxsrK6v69etPnjzZwsKCDqDUqlUrNTXVx8cnKSlp5cqVderUIYRMmDAhIyPD0NCwa9euampqRkZGP/zwAyGEXtKcn58vEAjoyFeLFi0GDRpE84qGhkZp11WpTJs2berUqapJPISQXr16leeDl3A7n0v//+I+IWf+//WF/5a5+pFqhUKho6MjISQpKalt27a///572dNoqsuJEycmTJigrq5+/Pjxxo0bf4Y9AgDAF4QnopcXHksEXzk8IhQAvriv9onouDILAAAAOA6DWeWFP50BAAC+UejdAQAAAI5D3AEAAACOQ9wBAAAAjkPcAQAAAI5D3AEAAACOQ9wBAAAAjkPcAQAAAI5D3AEAAACOQ9wBAAAAjkPcAQAAAI5D3AEAAACOQ9wBAAAAjkPcAQAAAI5D3AEAAACOQ9wBAAAAjkPcAQAAAI5D3AEAAACOE3zpBgAAAPcplUqlUkkI4fF49EVxPB4vIyNj48aN1btrhmF4vJL/tp89e7aOjk717g6+Tog7AADfktTU1OjoaD6fTxcVCkVBQYHqXbFYLJFIPmkD1NTUdu/efebMGbFYXM5NFAqFsbGxpaWlTCZ7+vRpixYtWJYtXoxhGIlE8ujRo2psrVwub9So0erVqxUKRfF31dXVq3Ff8DVD3AGAbwnLsjKZrNKbFxQUlNa1UJhAIHj16tWqVasEggr8kmRZVk1NjXzKk6hQKHz48OGFCxdUaywsLCZNmiSVSgkhSqXy8OHDw4YNK89nrApra+u2bduWv7xMJrOxsenWrZtCobh8+bKjo+OnaxtAiRB3AKAaREREJCQkVCgcFMaybHn6JPh8flpa2vz58yu3F4VC0atXLwMDgxL/0C+MdjOkpKQwDFP++iUSiZOTE5/P/+GHH1S9L9WLZVmhUKiurk57R1iWFYvFhYdjxo0bV6NGjU+x62rB5/ORdeCLQNwBqKSMjAyZTFah0yFFT6USiaQS25aHSCRas2ZNVFSUUCika3g8nqGhIZ/Pz8jIoIva2toljiZUjkAguHLlSlxcXOXijlKp1NXV9fPz4/P5ZbdKoVDo6+v/+eeflWunQqFwdHTU09Or3ObfhK856wB8QYg7wE1paWkHDhyoyl/YSqWyjFOvQCDw8/NLTEysXOXt27fv2rVrVQZlysAwTH5+vpGREY1TSqVSS0vL19e3Ro0aJ0+eZBhGLBY7OjpW7wwPsVhclaPNMIwqnAEAVDvEHShVamoqn89PSUnh8XgKhSI/P7+6amYYRi6XT5s2bdCgQcHBwTk5ORXq51AqlUZGRjKZzMjIqMQ5CjweLysr6+bNm6VdjvFRBQUFEydO7NmzZxmZ4NChQ5Wrn07bNDQ0rFzbquKHH35QvcYkTQD4fiDufA5yuZy+UJ2bGYbJysq6fv26nZ3dmjVrqmVQg8fjVeN0AYFAEBoaqqGhER4eTggxMjLy9vau3t4IR0fH7Ozs7t27V3RDmUzm6ur64cMHV1dX1bEtgmGYTzR5AgAAvjmIOxUwevTo4ODgiv5NrFAobGxsFAqFQCCoV6+eao6kTCZLTU0NCgqKioqqetskEknXrl1nzJhRjSMUnp6edCKkUqlUV1dv2LBhddVcjSo9NxYAAL4fOFVUwODBg62trSt6fpXJZH379pXL5QKBwMTE5BO1DQAAAEqDuFMBLi4uLi4uX7oVAAAAUDF4ZhYAAABwHOIOAAAAcBziDgAAAHAc4g4AAABwHOIOAAAAcBziDgAAAHAc4g4AAABwHOIOAAAAcBziDgAAAHAc4g4AAABwHOIOAAAAcBziDgAAAHAc4g4AAABwHOIOAAAAcBziDgAAAHCc4Es3AAAAvgssy37qXTAMEx4efunSJZFIRBdZlmUYprTyU6dO1dbW/tStgq8B4g4AwDdDoVA8evRIqVRWV4UMw+Tl5fF4PDU1NYlEwjCMRCIpUobH43348GH69OkjRow4cOCATCar6F4UCoWhoaGZmZlCoaimhpeMz+fHx8cnJCTI5XI7O7vRo0ePHTvWx8fHzMysxGaLxeJP2h74eiDuAMA3g3YP0D/WWZatUG+BRCL5ROdasVgcGBh4+fJloVBIOxU+0UmUYZiCgoIdO3ZUInCUYfz48ampqcHBwY6Ojnl5ef379y/xQHl5ebEsO3r06ErsQiaTdejQoWfPnlVubIXFxcV9/p3CVwhxBwAq79q1a1lZWTxeVWcBsiwrl8uVSmVp/RYMw2RmZp45c0apVP7www/a2tr+/v6nTp0qZ7CQy+Vdu3Y1NDT8FImHx+MlJyenp6fLZDJnZ2cejzd48GA+n1/tOyKEMAwzb968MkZnKopl2Tp16shksvfv39esWVOpVOro6FRX5QBfD8QdgE8oLy9PQ0MjJydHoVDI5XK6sqCgoFpOV2KxOCwsTFtb28bGRiaTsSy7YMGC9PT0qocPimVZfX19hmEyMjLq1atXvACPxzt27FhaWloVT+1KpdLAwKBz586NGze2tLQsreuCYZh+/foRQvLz8/Pz8/v27du/f/9y7kIulzs4ODRs2LAq7eQwkUikqan5pVsB8Akh7gBH7N+/v2XLljKZrGXLlvv3769cJdU7RsDn8/ft2zds2LDVq1e/efOmWbNmSqWSz+f36tWrWnbEMEx+fj7DMGfOnKHDOjKZrBpPWhKJZOjQoXw+f//+/b6+viVGtJ9//rla0hWPxxMIBDwe7xN1igDAd475DFPl4VshlUppb0RiYmKlux9yc3OVSqVIJKLDE8UL8Pn8hQsXvnz5UiCoZNo2MzMTi8VFvnXPnz/fuHFjuVzeuHHj8+fPV7RO2oe/YcMGoVBYjT8UIpFIKpWqqakRQurVq6dUKnk8nqWlZXXVDwDwVQkPD+/evTshJDQ01NnZ+Us353/Qu/PJVXRCZYl4PN62bdvMzc1jYmKSkpJUfwHzeDzaYVDlZhI+n5+UlBQdHW1ubv7bb79Vup5JkybVrl07IiJCU1OzefPmJSaexo0bm5iYVK5+lmXnzZtXu3btIpMwRCKRQqFgGEahUNDpogAAABTiTgWMGzcuJCREXV29/JsolUp9ff22bdtWcYIkj8eLjIysWbNmRkZGbm6u6soUb2/vPXv2+Pn51a1bt+pzMOmAglwunzRpUqUrMTU1FQqFGRkZAoGgRo0aVWxSGYp3DtE1GA0BAIAiEHcqwNXVtUmTJkKhsPyb0LGVQYMGfbpWzZ49+9NVXmk1a9b80k0AAAD4F+JOBbi5ubm5uX3pVgAAAEDF4JlZAAAAwHGIOwAAAMBxiDsAAADAcYg7AAAAwHGIOwAAAMBxiDsAAADAcYg7AAAAwHGIOwAAAMBxiDsAAADAcYg7AAAAwHGIOwAAAMBxiDsAAADAcYg7AAAAwHGIOwAAAMBxgi/dAAAAgCrJz8/n8XhKpTIzM5NhmHJuxbKsWCzW09P7pG2DrwTiDgAAfBJ79+5NT0/n8ao6jMCyLMuypb0rEokiIiJEIlFBQUFgYGCFanZ0dLxw4UIVmwffBMQdAICqSk5O1tPTE4lEhJCkpKQGDRo8f/68oKCg/D0NKnl5eUqlUlNTUyqVSqVSNTU1iURSjU0VCoVLliyJiYlhWXbMmDGnTp16//49TSRKpVJfX5/P5ysUiubNmysUiqrsiGGYW7du5eXlVeIgFCaVSpcsWWJlZSWVSksr06ZNG5ZlGYaZNWtWGcGoCKVSqaurW5W2wTcEcQcAOIJl2TLOiIQQoVC4ZcuWFy9eCASV/NUnFApZli2yOZ/PP3PmTIcOHeiwyMmTJ3v37r179+73799XYhcTJkxo2LDhqlWrnJ2dO3XqdOjQIVdXV6FQWLkGl6hdu3Y2NjaEEKlU6ujoqFovk8latmwpFAplMtmQIUOUSmUVd1T1fh2A6sKUPwgDwPcgMTHx6dOnfD6/tAIsyxYUFFTxV4dYLD506JCNjY25ufnVq1c3bdokFourUqFSqdTT07OzsyvjJM3j8WJjY3Nzcyvd3zBs2DCFQtG3b19Vj4tIJHr06NGbN2969+6dn59PCBGLxRKJRENDoxIne5ZlGzZsKBKJXr58qa+vr6Wl9e7dO319/Sp2kAB8NuHh4d27dyeEhIaGOjs7f+nm/A/iDkDJJBKJXC6v9OYsy9KTXxF8Pj85OXnp0qXV9Ycvy7Kamposy2ppaVW9NqFQeO/evWvXrpV2fmVZVl1dfeXKlerq6lX87cHn85VKJcuyPB6vjHRVTgqFom7duv37969iPZXYb35+frUcfAAO+GrjDgaz4Bt2+fLl169f0zOlUqmUyWTVVbNYLN69e3d4eHjlBhHoFR+enp507KPwWwzDyGSyapyNIZFIunbtKpPJPD09q14by7IikUgsFpcRZRiG0dbWRn8DtXr16uTk5G3btn3phgBAWRB3vmEfPnwQCoVisTglJUUmk1X99KNUKuVyuVwuLz4cIBaLd+7cGRYWVpURB5ZlDQ0NaVdE1VpKCCECgSA8PPzNmzd8Pl8qlbZu3XrBggVlT92okJkzZ86ZM6dy29LQ0KNHj6p3WsBXztPTs3bt2l+6FQDwEYg71Uwmk23btq1yV2SUoXg+EIlEYWFhDRs2NDMzW716dXp6etX3UqdOnaZNmzZu3Lh+/fpFLspgGKagoKBNmzZV+VwymWzatGnm5ubV1Q0jFotVeYJhGPQ3wOfXrFmzL90EAPg4xJ0KyMnJmTVrVtn9KAqFIiIiolr6WiilUlmnTp3u3bubmZk9fvy4devWJiYmCoWCZVlbW1ulUqlQKM6dO1ctd7bQ0NDQ1dXV0dHR0NColsaXpoqTUgEAACoEcacCatSosXPnzo8WQx8DAADAVwVxp2IQZQAAAL45uAcUAAAAcBziDgAAAHAc4g4AAABwHOIOAAAAcBziDgAAAHAc4g4AAABwHOIOAAAAcBziDgAAAHAc4g4AAABwHOIOAAAAcBziDgAAAHAc4g4AAABwHOIOAAAAcBziDgAAAHAc4g4AAHyPWJbNzc390q2Az0TwpRsAAABALl68+PjxY6FQWJVKlEolj8djWVahUHy0sEwmCwgICAwMbNq0aVV2Ct8ExB0AgG/G27dveTyeUqn88OEDwzCF38rLyxMIBGKxuKCgQCqVFnm3uvD5/AULFrx8+VIgqNjpQ6lUWlhYZGZm6urqqqursyxbpNr79++/fv2az+dXum1KpXLs2LE7d+40Nzdfvny5VCr96CZr1qzR0NCo9B7hG4K4AwDfHdoHIJfLS3xXIBBcu3btxIkTIpGo0rugoaQqJ+/iRCLRn3/+qa6unp+fHxsbW+TdMWPGxMfHX7p0ycXFpWPHjuU52VeOpaWlhYVFRbeSy+WzZ89+9OiRpaVl/fr1i/e+CAQCHq8a5lesXLmy6pUA9yDuAEBlSCSSu3fvNm3a9Pnz559oFyzL5uXllfauSCS6e/fumjVrxGJxharl8Xjm5uZPnz7t1q2bSCQq0s1ACyQnJ798+bLSZ18ejzdq1Ch/f/9Nmzbp6OgolcrK1VMEy7L9+/enr4t0rrAsa2JikpeX9+bNm4YNG+rq6lbLHqtdgwYN6IvqDYIAH4W4A1BJLMvKZLJqqUoul0skEpFIpFAoqnhqFIlE169f37t3Lw0BDMOoq6vT0MDj8bS1taulwTweLysr648//nB0dAwLC6uWOovT0dFZunRpGYMyPB5vyZIlFa2Wx+P16NHj0qVL3t7eampqVWtjWWbMmPHpKi+Rmpqanp7eZ94pwDcBcQe4LCUlJTw8vKKTDFQUCkVpgUYgECQkJCxdurTSlavI5fLmzZt37do1IiLCyMhIX1+/KomHYZicnJy0tDQ6YVNdXd3Dw2PPnj08Hq9evXqzZs2qrp4GhmF+/vlniURS0c6V8uPxePr6+p+o8koMxwDAtwtxh/uysrIkEkm1zFvMzc0t3vNfdWKxOCgo6MiRI/TEyefzGzZsWPUG8/n8xMTECxcuVC6RSKXSjh07Tp8+XSKRFH9XLpcbGhoGBARUsZG0KlNT0zZt2jx+/LhBgwafYhhixIgR1V4nAMA3BHHnc0tLSztw4EB1jVvTv9TLiCAikeiXX36JiYmp+r4EAoGXl1fxSyqqjmGY/Px8Y2NjhmGUSqW2traPj49QKKz6jvh8flX6Hng8XtU7b8rP0tLys+0LAOC7grhTAbm5ucOGDYuNjS1PWGFZ1sLCQlNTs/A5m854uHnzZrVcgMCy7IgRIyQSyYABA0rsgaB2795dLemKx+PZ2Nh8ztM/AABAtcCpqwI0NTXnzZt37Nix8lyeKpfLp0yZYmhoWORiV4ZhcEkCAADA54S4UzG2tra2trYV2gTdIQAAAF8WnpkFAAAAHIe4AwAAAByHuAMAAAAch7gDAAAAHIe4AwAAAByHuAMAAAAch7gDAAAAHIe4AwAAAByHuAMAAAAch7gDAAAAHIe4AwAAAByHuAMAAAAch7gDAAAAHIe4AwAAAByHuAMAAAAch7gDAAAAHIe4AwAAAByHuAMAAAAch7gDAAAAHIe4AwAAAByHuAMAAAAch7gDAAAAHIe4AwAAAByHuAMAAAAch7gDAAAAHIe4AwAAAByHuAMAAAAch7gDAAAAHIe4AwAAAByHuAMAAAAcJ/jSDfguZKzIIITUXFJT9Vq1CAAAAJ8aenc+N6QcAACAzwy9O9WvSP+NarFwH09pi0W2qrmkJnqDAAAAqgi9O9Ws+LiVKqbUXFKzcGQpXKa0rYrUAAAAAJWAuPMVUXXkFEazDhIPAABApWEw67PKWJFRRnBBpgEAAPgU0LtTzUobxiKldN4UKUD/kWIzfgAAAKDSGJZlv3QbAAAAgAvCw8O7d+9OCAkNDXV2dv7SzfkfDGZV2NfQ3YJhLwAAgPLDYBYAAABwHHp3KqnE/pXY2NguXboIhcLw8PCmTZuWuGGJ15aXPYW5+OYAAABQfujdqU4KhYJhmLFjxzZp0uRLtwUAAAD+hd6dSsrOzt62bZu7u3tISMioUaPq1KmTnp7u4+PDsqyamhrDMLTY1q1bT5w4wbKsVCpduXKlnZ1daRU6OTmxLLt58+bc3NyFCxfyeDxdXd3s7GyGYQoKCuRyedOmTX/77bfP9fkAAAC4A707lWRoaBgcHNy2bdt58+atWLGCEHLr1i0NDQ2BQPD333/TMg8fPpw2bVpYWBghhM/nOzs737hxo7QKL1y44ODgYGlp2b59e2tr6+Tk5OnTp8fGxj58+NDU1LR58+ZHjhxZt/K5AfUAABTWSURBVG7d5/l0AAAAXILenUqqXbv2hQsXkpKS5s6dO3HiREKIq6urq6urh4dHfn4+LWNlZTVw4MCnT5+eOnVKqVR6eHj8/vvvW822llihh4dH3759f/rpJx0dnf79+wuFws6dOwcHB2/YsGHHjh2EkNu3b0skEqL22T4iAAAAR6B3p5Lq1auno6NjYWFRt27dhg0bllFMW1tbTU1NQ0PDycnp4MGDpZXU09Pr3bv3qVOngoKCBgwYQO+HZGRkJBKJPskHAAAA+G4g7lSSQqEoTzFLS8uCgoLMzExCSGBg4PDhw1VvPXr06PHjx6pFc3PzV69esSwbFRWVmppqampKCOHxeO/evbt27dqNGzeysrJ4PHy9AAAAKgynz0r68OGDh4eHh4fH3bt36ZoTJ054eHjcunXr7t277u7u+/btI4QYGBhERkYOHDhw5syZBQUFkydPVtXg4+Pj4OCQkJBAFydPnly3bt1Vq1ZNmjTJ1NR01KhRhBA1NbUbN27Y2dl16dJlyJAhQ4YM+cwfEwAAgAMQdypp7969mZmZcrn8xYsXPj4+hJDu3burq6vb2Nh07tw5OztbJpMRQq5duyYUCufNm7dw4cKLFy927NhRVUOHDh2mTp1qbGxMFwUCwbNnz9zc3NavX6+a7Hz9+nUHBwelUpmenr5y5UoLC4vP/TkBAAC+fZiqXEk2Njbnz58nhHTp0iUrK4sQUqNGDdqjU5hMJrOysurSpYuamlqRoag5c+YUKayjo0NfqObrBAYGBgYGenp6urm5fYpPAQAA8D1A706VjB079saNGw8fPnz//n3xd7Oysu7fv//PP//UrVs3Nze3EvXb2dkZGxvfu3evnFOFAAAAoDj07lSeQqGYMmVKdna2trZ2jRo1ihfg8/nr1q0Ti8V8Pl9DQ6MSuxgyZIidnZ2RkVGVGwsAAPD9YugFz1B+X8NTq/BEdAAA+AqFh4d3796dEBIaGurs7Pylm/M/GMwCAAAAjsNgVoWhZwUAAODbgt4dAAAA4DjEHQAAAOA4xB0AAADgOMQdAAAA4DjEHQD43kkkki/dBAD4tBB3AOB7d/DgQdyBDIDbcCE6AHzvvLy8vnQTAODTQu8OAAAAcBziDgAAAHAc4g4AAABwHOIOAAAAcBziDgAAAHAc4g4AAABwHOIOAAAAcBziDgAAAHAc4g4AAABwHOIOAAAAcBziDgAAAHAc4g4AAABwHOIOAAAAcBziDgAAAHAc4g4AAABwHOIOAAAAcBziDgAAAHAc4g4AAABwHOIOAAAAcBziDgAAAHAc4g4AAABwHOIOAAAAcBziDgAAAHAc4g4AAABwHOIOAAAAcBziDgAAAHAc4g4AAABwHOIOAAAAcBziDgAAAHAc4g4AAABwHOIOAAAAcBziDgAAAHAc4g4AAABwHOIOAAAAcBziDgAAAHAc4g4AAABwHOIOAAAAcBziDgAAAHAc4g4AAABwHOIOAAAAcBziDgAAAHAc4g4AAABwHOIOAAAAcBziDgAAAHAc4g4AAABwHOIOAAAAcBziDgAAAHAc4g4AAABwHOIOAAAAcJzgSzcAAAC4JmNFBiGk5pKaqteqRYAvAr07AADwCSHlwNcAvTsAAFAlRfpvVIuF+3hKWyyyVc0lNdEbBJ8CencAAKDyio9bqWJKzSU1C0eWwmVK26pIDQDVBXEHAAC+DFVHTmE06yDxQPXCYBYAAHwqGSsyygguyDTw2aB3BwAAKq+0YSxSSudNkQL0Hyk24wegejEsy37pNgAAAAAXhIeHd+/enRASGhrq7Oz8pZvzPxjMAgCAsqC75ZPCiN7ngcEsAAAA4Dj07gAAwMehE6Laodvsc0LvDgAAAHAc4g4AAABwHAazAACgXKRSaUxMTLNmzcRiceH1aWlpb968MTQ01NPTI4RkZ2ffvHlzy5Yt9evXJ4QkJSV5enp26tTJ0NBQXV2dEPL69WuWZY2MjAghBQUFz58/p4s6OjrFd5qZmZmYmCiTySQSCcMwGhoaQqHQzMyMYZjCxeRyeUxMjEKhyMvL09DQEAgEpqamfD5fVSA3NzcuLk5PT69hw4aFN0xMTExPT1dTUzM1NU1KSmrQoAEhRKFQxMTEKJVKlmVVn/fNmzcGBga0zry8vMTExDt37qxbt45hGF9fX3Nzc2NjY5FIVLjylJSU1NRUHu/fngWWZdXV1Zs0aVLZrwBUAQsAAFC6dN/0dN90lmVfvHhBCPH29l68ePG8efN+/vnnmzdvLliwwM7OjhDy+++/sywrkUhcXV0JIT4+PnTz6OjoRo0aEULu3LlD1wwdOrRevXrx8fEsyy5ZsoSejH744QeFQsGy7KVLlzZs2EBLyuXyAQMGEEK6du26du3aZcuWaWpqMgxz+PDhIo3csWMHIaRWrVqrVq3S09NjGGbSpEnR0dGqApcvXyaENG/efP78+aGhoTt27Fi4cGFkZKS5uTkhxMjISKlUjho1ihbOyMjQ1NSkDVN9EA8Pjz/++IO+nj17dvHz6Zo1a/5z3NLTO3ToUKRMnTp1Fi9enJGRUfjAcgk9zoSQ0NDQL92W/0DcAQCAsqjOymfOnCl85ubxeFpaWqpFGnfGjBlDF319fVU1zJo1ixDSrVs3ujhq1ChCiJWV1fz58wvXVr9+fV9fX11d3datW9OSkydPpu/u2rWLZVmZTFa3bl1CiIuLi0QiUdW/bds22oNy4sQJlmWPHTtGt1LFJpZl27dvr9oX7SIihNSoUYOuOXHixO7duwkhTZo0OXPmjIeHh6qwmZlZRETEL7/8Qgjx8vJiWZa+JoT07NkzIyPD39+fdv9YWFi8f/9etcfevXvTYnw+v2vXrp06dVL1SHl6erKIO58XBrMAAKBc/Pz8CCHm5uaLFy+WyWQsyyqVSjq4I5PJHB0dnz59eurUqRI3fP36dVBQUEhIiJubG1355MmTx48fa2hoqELAu3fvfHx8FAqFak1iYiIhxMTExNvbmxAiEAjGjh27YsWK0NDQ8ePH//HHH4QQf3//KVOmsCxrZ2fXrVs3Qkj37t27d+9++fLlOXPm1K5de8SIEYSQtLQ0Qkj79u1nzpyZl5d34MCBH3/8UaFQ8Pl8uVxua2ubnJxcu3btuLi43r17C4VCsVjMsqxUKo2Oju7YsSPLsoQQoVCYk5OzZ88e2rzJkyfr6uqOHDlSqVR6eXk9efJk4MCBoaGhYrE4PDz86tWrhBALC4uVK1f26dNHLpcfO3YsISFh8eLF586d+/vvv5uSpp/qSwXFIO4AAEAFHDx40NrausS3QkJCUlJSiq/PyMi4dOmSUqkcPHjwmzdvGIbR0tIKCwtr2bKlVCotXLJGjRorV64MCAgghMTGxj5+/NjT0/PVq1dKpZL23/j6+j548ODkyZPXr1+nm0RERNAsYmFhoa2tTQjR1dU9evRo06ZNMzIy7ty5Q+MOLePp6enp6UlfaGhoFN714cOH09LSWrdu3bRp02nTpuXn5+fm5h4+fNjY2HjBggWmpqapqakikWjUqFGRkZGEEJFI1KlTJ7rt6NGjnz59um7duqtXr4aHhzs7O0dFRX348IHut0uXLkKhUCgUDhs2jBCSkpKydevWmJgYxJ3PCVdmAQBABRQUFJT2VufOnQ8fPlyrVi1CyB9//PHs2TNCCMuyvr6+79+/J4RIJJK0tDRNTc2BAwd26NBBQ0ND97+ysrL27dunpqZGCGnWrJm1tfW7d+8UCkXhvRSeDhwbGxscHFy8JapOI5o5aDMEAgHNOrRAkU06depkbGw8fvz4I0eOdO3a1cnJacCAAYGBgWvXrtXR0VmwYAEhRFdXNzc3t8TPTtusVCp37dpFCFEN8z19+rRHjx6Ojo5hYWF0zYgRI3g8XpEPBZ8aencAAKACBg8ePH36dLlcTggRCAS098LW1pYQUqtWLQ8PDyMjI1dX1/j4+F69es2ePTsiIuLAgQOqzSdOnHjp0iW2pMc1xsTE+Pv7x8fH0zk9P/30019//UUIMTExUV3cVMT79+9L7E9S8ff3nzx5so2NDY/Hk8vlvXr1GjlyZH5+fr169UaPHl24ZHZ29uXLl42NjUus5/Xr1xoaGr6+vnS87KNGjRr122+//f3334SQqKioqKiou3fvLl26VE1N7dixY7q6uq1atSIvy1MTVA/EHQAAKJedO3dmZ2ezLJuXlycWizdt2nT27Nns7OxatWp17959z549dOZvx44dg4KC6CzjjRs3CoVCCwuLp0+f0ohD/y9yGblMJpsxY8axY8eSk5PnzJmzevVqlmUDAwOLt+Hx48c3btwghBQUFOTk5JQWgwqjezQ3N1dXV5dKpVu3bpXJZB07dszMzNy+fbtQKLSzs9uyZUvxHCOVSmNjYxcsWPDixYv4+Phff/2VEKJUKst5uJo2bUrjDpWZmUmnbBNCHB0dW7VqlXECd1X+fBB3AACgXJo1a1Z4MTAwcN++faNGjXr37t2RI0d69OgxYcIE+pa9vf3jx49VJZOTkw0NDWmH0A8//FC85qlTp+7YsaNVq1arV6+mnS6lpQpLS8vOnTsHBQUlJiZu3bq1b9++5Wx8kcvKCCEXL15MTEzMy8t7/Pixu7u7o6NjkQKbNm1SXTvm7+8/cuTIcu6Lat68eYXKwyeFuTsAAFBJtra2hoaGFdqkV69ehRefPn1aq1ata9euRUVFXblyRTXAxOPx6IVgxVVXjHBwcKDTjAghT548KfLuqlWrVFnH19dXlXXWr19fnsr9/f2XLl36+++/Z2Zm+vr6mpqaFn43LCwsJCSkSq2HCkLcAQCAj7t27dqgQYM8PT3pxeGUSCQSCP4dJaDjU35+fgMGDIiMjCzcPVNQUEBHlEQiUZHhp19//fX9+/dPnjyZNWuW6s5+lIuLi42NTfGW+Pr6Dhs2TFNTs3///s2aNbO3t6fri1zkpaKpqRkSEjJw4MAxY8bk5eWVWGbDhg2FFzdt2rR48WL62sjIaPjw4aq39PX1S6xBhc5ZTk9Pb9So0YQJE3R0dJYsWRIeHt62bVs1NTXVjZ63bt1adj1QvRB3AADg4+Li4oKCgg4fPpyVlaVaaWxsTJ+6QAhRV1e/ePHivHnzjh8/3rp163Xr1qmK/fTTT/RCpAULFtA7LKvcvn2bvrh161Z+fn7ht/T09Ep83sKFCxeCg4MNDAwsLCzU1NTobZEJIXv37i08V4YaN26chYVFRETEsWPH9u3bp7p8vWwRERGq1/379zcxMVEt1qxZs0WLFqVtKBaLS+yUqlev3rVr1zIzM1XjfXfu3ClPS6C6IO4AAEAF0EvKCSE5OTkBAQF3796lixKJxN7eXjVWtWPHjvv377979+7EiRPnz58nhLRq1YreLbBEenp6hR9xRdWuXVtNTS0+Pn7z5s30sVkXLlzw8PDIy8urXbs2LTNlyhRVhKLTg6RS6atXr+RyuZaW1vTp01W1KRSKvXv3lrj3Eh8KQY0bN67wor6+/unTp+vVqyeVSletWpWamvrmzZu7d+/u27ePx+M1atRIS0srPz//zJkz6urqSUlJqampdEN1dXWxWExvvUM/b2l7hE8BU5UBAODj2rRpY2BgkJKSMnz48JkzZ/J4vD179jx69Ii+q6+v36ZNGz6ff+TIkTVr1rx9+3bv3r2F70bYsmXLc+fOGRgYlFb/qlWrit8LZ+vWrd7e3s7OzjNnzrx48SIh5NatW1lZWc2bNw8KCqJlLC0tz58/37Nnz1evXu3bty8iIiI6Onr79u2EkJEjR1pYWBSu8NmzZ1u2bFENe2VnZ9MXRYqVzcjIaNasWYmJibt27dqyZYtqfZcuXa5du0YIYVk2LCysffv2DRs2bNy48axZsyZOnBgcHPzixQtV+VWrVpEX5d8nVNnnfmoFAAB8U1SPdrp9+3abNm3o3JQi6EOgVAoKCp48eXLkyJGmTZuOHTv2yZMnSUlJJVZ+8eLFli1bEkL2799fWgNu377dqlUrS0vLOnXqWFhYWFhYFH4gF/Xw4cN27doZGxvr6emJxWKGYZo1a3bt2jX67rt37w4fPqx6QlYRvXr1ok8nVYmOjm7a9N9bHkdFRZXWMHppOlWnTp3bt28XfnfQoEH0LYZhTExMCoe5du3a5efn45lZnxPDlnSvJwAAACpjRQYhpOaSmnRx//79qamphQeelErl6NGja9asWeldbN++3c7O7qNdLFeuXGnVqpWurm5pBVJSUuLj458/f/7+/fvx48cXSWbnz5+PjIykDwdVkclkLVu2dHFxKVLVkydPTp8+zefzR44cWdr05Li4uGPHjgmFQplMZmdnV+T55+/fv6cjXIQQiUQiFArpa7lc3rhxY3d39yIHlhvCw8O7d+9OCAkNDXV2dv7SzfkfxB0AACgLPSvDJ4K483lg7g4AAABUDwsLi4MHD7Is26pVqy/dlv9A7w4AAABwHC5EBwAAAI5D3AEAAACOQ9wBAAAAjkPcAQAAAI5D3AEAAACOQ9wBAAAAjkPcAQAAAI5D3AEAAACOQ9wBAAAAjkPcAQAAAI5D3AEAAACOQ9wBAAAAjkPcAQAAAI5D3AEAAACOQ9wBAAAAjkPcAQAAAI5D3AEAAACOQ9wBAAAAjkPcAQAAAI5D3AEAAACOQ9wBAAAAjkPcAQAAAI5D3AEAAACOQ9wBAAAAjkPcAQAAAI5D3AEAAACOQ9wBAAAAjkPcAQAAAI5D3AEAAACOQ9wBAAAAjkPcAQAAAI5D3AEAAACOQ9wBAAAAjkPcAQAAAI5D3AEAAACOQ9wBAAAAjkPcAQAAAI5D3AEAAACOQ9wBAAAAjkPcAQAAAI5D3AEAAACOQ9wBAAAAjkPcAQAAAI5D3AEAAACOQ9wBAAAAjkPcAQAAAI5D3AEAAACOQ9wBAAAAjkPcAQAAAI5D3AEAAACOQ9wBAAAAjkPcAQAAAI5D3AEAAACOQ9wBAAAAjkPcAQAAAI5D3AEAAACOQ9wBAAAAjkPcAQAAAI5D3AEAAACOQ9wBAAAAjkPcAQAAAI5D3AEAAACOQ9wBAAAAjkPcAQAAAI5D3AEAAACOQ9wBAAAAjkPcAQAAAI5D3AEAAACOQ9wBAAAAjkPcAQAAAI5D3AEAAACOQ9wBAAAAjvs/jMB45U475iQAAAAASUVORK5CYII=", - "text/plain": [ - "" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "draw = ImageDraw.Draw(image, \"RGBA\")\n", "\n", @@ -838,7 +783,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/", @@ -847,18 +792,7 @@ "id": "RhINSBw9I24G", "outputId": "28738ce2-617c-47d3-b8c9-f949d3066d60" }, - "outputs": [ - { - "data": { - "text/plain": [ - "'[CLS] project objective : date : btf - to : files confidential epb : cc : odl project initiation form september 16, 1980 project code : project name : project leader : work requested by : e. p. barbee r. s. sprinkle, iii to develop cigarette cigarette to utilize a filter tip with longitudinal grooves from the mouth end to the tobacco end in con - junction with perforated tipping paper. other personnel assigned : j. e. mann, jr. d. e. cawthon approved by : r. s. sprinkle, iii bmc / rmi / epb jem / dec prc / drb [SEP] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD]'" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "batch = next(iter(train_dataloader))\n", "input_ids = batch[0][0]\n", @@ -893,7 +827,7 @@ } ], "source": [ - "from peft import get_peft_config, LoraModel, get_peft_model, LoraConfig, TaskType\n", + "from peft import get_peft_config, PeftModel, get_peft_model, LoraConfig, TaskType\n", "peft_config = LoraConfig(\n", " task_type=TaskType.TOKEN_CLS,\n", " inference_mode=False,\n", @@ -907,7 +841,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/", @@ -926,465 +860,7 @@ "id": "xIdOsFBiTsuw", "outputId": "95e8811c-025a-41a0-9d03-4285a17f2a9b" }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Some weights of the model checkpoint at microsoft/layoutlm-base-uncased were not used when initializing LayoutLMForTokenClassification: ['cls.predictions.transform.dense.bias', 'cls.predictions.decoder.bias', 'cls.predictions.bias', 'cls.predictions.decoder.weight', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.dense.weight']\n", - "- This IS expected if you are initializing LayoutLMForTokenClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n", - "- This IS NOT expected if you are initializing LayoutLMForTokenClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n", - "Some weights of LayoutLMForTokenClassification were not initialized from the model checkpoint at microsoft/layoutlm-base-uncased and are newly initialized: ['classifier.weight', 'classifier.bias']\n", - "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n" - ] - }, - { - "data": { - "text/plain": [ - "PETModelForTokenClassification(\n", - " (base_model): LoRAModel(\n", - " (model): LayoutLMForTokenClassification(\n", - " (layoutlm): LayoutLMModel(\n", - " (embeddings): LayoutLMEmbeddings(\n", - " (word_embeddings): Embedding(30522, 768, padding_idx=0)\n", - " (position_embeddings): Embedding(512, 768)\n", - " (x_position_embeddings): Embedding(1024, 768)\n", - " (y_position_embeddings): Embedding(1024, 768)\n", - " (h_position_embeddings): Embedding(1024, 768)\n", - " (w_position_embeddings): Embedding(1024, 768)\n", - " (token_type_embeddings): Embedding(2, 768)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (encoder): LayoutLMEncoder(\n", - " (layer): ModuleList(\n", - " (0): LayoutLMLayer(\n", - " (attention): LayoutLMAttention(\n", - " (self): LayoutLMSelfAttention(\n", - " (query): Linear(\n", - " in_features=768, out_features=768, bias=True\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=768, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=768, bias=False)\n", - " )\n", - " (key): Linear(in_features=768, out_features=768, bias=True)\n", - " (value): Linear(\n", - " in_features=768, out_features=768, bias=True\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=768, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=768, bias=False)\n", - " )\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (output): LayoutLMSelfOutput(\n", - " (dense): Linear(in_features=768, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " (intermediate): LayoutLMIntermediate(\n", - " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", - " (intermediate_act_fn): GELUActivation()\n", - " )\n", - " (output): LayoutLMOutput(\n", - " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " (1): LayoutLMLayer(\n", - " (attention): LayoutLMAttention(\n", - " (self): LayoutLMSelfAttention(\n", - " (query): Linear(\n", - " in_features=768, out_features=768, bias=True\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=768, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=768, bias=False)\n", - " )\n", - " (key): Linear(in_features=768, out_features=768, bias=True)\n", - " (value): Linear(\n", - " in_features=768, out_features=768, bias=True\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=768, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=768, bias=False)\n", - " )\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (output): LayoutLMSelfOutput(\n", - " (dense): Linear(in_features=768, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " (intermediate): LayoutLMIntermediate(\n", - " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", - " (intermediate_act_fn): GELUActivation()\n", - " )\n", - " (output): LayoutLMOutput(\n", - " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " (2): LayoutLMLayer(\n", - " (attention): LayoutLMAttention(\n", - " (self): LayoutLMSelfAttention(\n", - " (query): Linear(\n", - " in_features=768, out_features=768, bias=True\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=768, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=768, bias=False)\n", - " )\n", - " (key): Linear(in_features=768, out_features=768, bias=True)\n", - " (value): Linear(\n", - " in_features=768, out_features=768, bias=True\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=768, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=768, bias=False)\n", - " )\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (output): LayoutLMSelfOutput(\n", - " (dense): Linear(in_features=768, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " (intermediate): LayoutLMIntermediate(\n", - " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", - " (intermediate_act_fn): GELUActivation()\n", - " )\n", - " (output): LayoutLMOutput(\n", - " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " (3): LayoutLMLayer(\n", - " (attention): LayoutLMAttention(\n", - " (self): LayoutLMSelfAttention(\n", - " (query): Linear(\n", - " in_features=768, out_features=768, bias=True\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=768, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=768, bias=False)\n", - " )\n", - " (key): Linear(in_features=768, out_features=768, bias=True)\n", - " (value): Linear(\n", - " in_features=768, out_features=768, bias=True\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=768, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=768, bias=False)\n", - " )\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (output): LayoutLMSelfOutput(\n", - " (dense): Linear(in_features=768, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " (intermediate): LayoutLMIntermediate(\n", - " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", - " (intermediate_act_fn): GELUActivation()\n", - " )\n", - " (output): LayoutLMOutput(\n", - " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " (4): LayoutLMLayer(\n", - " (attention): LayoutLMAttention(\n", - " (self): LayoutLMSelfAttention(\n", - " (query): Linear(\n", - " in_features=768, out_features=768, bias=True\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=768, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=768, bias=False)\n", - " )\n", - " (key): Linear(in_features=768, out_features=768, bias=True)\n", - " (value): Linear(\n", - " in_features=768, out_features=768, bias=True\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=768, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=768, bias=False)\n", - " )\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (output): LayoutLMSelfOutput(\n", - " (dense): Linear(in_features=768, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " (intermediate): LayoutLMIntermediate(\n", - " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", - " (intermediate_act_fn): GELUActivation()\n", - " )\n", - " (output): LayoutLMOutput(\n", - " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " (5): LayoutLMLayer(\n", - " (attention): LayoutLMAttention(\n", - " (self): LayoutLMSelfAttention(\n", - " (query): Linear(\n", - " in_features=768, out_features=768, bias=True\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=768, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=768, bias=False)\n", - " )\n", - " (key): Linear(in_features=768, out_features=768, bias=True)\n", - " (value): Linear(\n", - " in_features=768, out_features=768, bias=True\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=768, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=768, bias=False)\n", - " )\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (output): LayoutLMSelfOutput(\n", - " (dense): Linear(in_features=768, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " (intermediate): LayoutLMIntermediate(\n", - " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", - " (intermediate_act_fn): GELUActivation()\n", - " )\n", - " (output): LayoutLMOutput(\n", - " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " (6): LayoutLMLayer(\n", - " (attention): LayoutLMAttention(\n", - " (self): LayoutLMSelfAttention(\n", - " (query): Linear(\n", - " in_features=768, out_features=768, bias=True\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=768, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=768, bias=False)\n", - " )\n", - " (key): Linear(in_features=768, out_features=768, bias=True)\n", - " (value): Linear(\n", - " in_features=768, out_features=768, bias=True\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=768, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=768, bias=False)\n", - " )\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (output): LayoutLMSelfOutput(\n", - " (dense): Linear(in_features=768, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " (intermediate): LayoutLMIntermediate(\n", - " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", - " (intermediate_act_fn): GELUActivation()\n", - " )\n", - " (output): LayoutLMOutput(\n", - " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " (7): LayoutLMLayer(\n", - " (attention): LayoutLMAttention(\n", - " (self): LayoutLMSelfAttention(\n", - " (query): Linear(\n", - " in_features=768, out_features=768, bias=True\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=768, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=768, bias=False)\n", - " )\n", - " (key): Linear(in_features=768, out_features=768, bias=True)\n", - " (value): Linear(\n", - " in_features=768, out_features=768, bias=True\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=768, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=768, bias=False)\n", - " )\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (output): LayoutLMSelfOutput(\n", - " (dense): Linear(in_features=768, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " (intermediate): LayoutLMIntermediate(\n", - " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", - " (intermediate_act_fn): GELUActivation()\n", - " )\n", - " (output): LayoutLMOutput(\n", - " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " (8): LayoutLMLayer(\n", - " (attention): LayoutLMAttention(\n", - " (self): LayoutLMSelfAttention(\n", - " (query): Linear(\n", - " in_features=768, out_features=768, bias=True\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=768, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=768, bias=False)\n", - " )\n", - " (key): Linear(in_features=768, out_features=768, bias=True)\n", - " (value): Linear(\n", - " in_features=768, out_features=768, bias=True\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=768, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=768, bias=False)\n", - " )\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (output): LayoutLMSelfOutput(\n", - " (dense): Linear(in_features=768, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " (intermediate): LayoutLMIntermediate(\n", - " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", - " (intermediate_act_fn): GELUActivation()\n", - " )\n", - " (output): LayoutLMOutput(\n", - " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " (9): LayoutLMLayer(\n", - " (attention): LayoutLMAttention(\n", - " (self): LayoutLMSelfAttention(\n", - " (query): Linear(\n", - " in_features=768, out_features=768, bias=True\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=768, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=768, bias=False)\n", - " )\n", - " (key): Linear(in_features=768, out_features=768, bias=True)\n", - " (value): Linear(\n", - " in_features=768, out_features=768, bias=True\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=768, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=768, bias=False)\n", - " )\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (output): LayoutLMSelfOutput(\n", - " (dense): Linear(in_features=768, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " (intermediate): LayoutLMIntermediate(\n", - " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", - " (intermediate_act_fn): GELUActivation()\n", - " )\n", - " (output): LayoutLMOutput(\n", - " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " (10): LayoutLMLayer(\n", - " (attention): LayoutLMAttention(\n", - " (self): LayoutLMSelfAttention(\n", - " (query): Linear(\n", - " in_features=768, out_features=768, bias=True\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=768, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=768, bias=False)\n", - " )\n", - " (key): Linear(in_features=768, out_features=768, bias=True)\n", - " (value): Linear(\n", - " in_features=768, out_features=768, bias=True\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=768, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=768, bias=False)\n", - " )\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (output): LayoutLMSelfOutput(\n", - " (dense): Linear(in_features=768, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " (intermediate): LayoutLMIntermediate(\n", - " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", - " (intermediate_act_fn): GELUActivation()\n", - " )\n", - " (output): LayoutLMOutput(\n", - " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " (11): LayoutLMLayer(\n", - " (attention): LayoutLMAttention(\n", - " (self): LayoutLMSelfAttention(\n", - " (query): Linear(\n", - " in_features=768, out_features=768, bias=True\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=768, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=768, bias=False)\n", - " )\n", - " (key): Linear(in_features=768, out_features=768, bias=True)\n", - " (value): Linear(\n", - " in_features=768, out_features=768, bias=True\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=768, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=768, bias=False)\n", - " )\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (output): LayoutLMSelfOutput(\n", - " (dense): Linear(in_features=768, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " (intermediate): LayoutLMIntermediate(\n", - " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", - " (intermediate_act_fn): GELUActivation()\n", - " )\n", - " (output): LayoutLMOutput(\n", - " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " )\n", - " )\n", - " (pooler): LayoutLMPooler(\n", - " (dense): Linear(in_features=768, out_features=768, bias=True)\n", - " (activation): Tanh()\n", - " )\n", - " )\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " (classifier): Linear(in_features=768, out_features=13, bias=True)\n", - " )\n", - " )\n", - ")" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "from transformers import LayoutLMForTokenClassification\n", "import torch\n", @@ -1401,48 +877,9 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Parameter containing:\n", - "tensor([[-0.0224, 0.0299, -0.0252, ..., -0.0109, 0.0444, 0.0079],\n", - " [-0.0067, 0.0337, -0.0272, ..., -0.0594, 0.1091, 0.0007],\n", - " [ 0.0390, 0.0659, 0.0154, ..., -0.0101, 0.0157, -0.0150],\n", - " ...,\n", - " [ 0.0020, 0.0183, 0.0587, ..., 0.0487, -0.0090, -0.0306],\n", - " [-0.0447, 0.0733, 0.0809, ..., -0.0755, 0.0394, 0.0626],\n", - " [ 0.0111, -0.0696, -0.0267, ..., -0.0041, -0.0576, -0.0373]],\n", - " device='cuda:0')\n", - "Parameter containing:\n", - "tensor([[-0.0253, 0.0085, 0.0225, ..., 0.0184, -0.0036, -0.0280],\n", - " [ 0.0029, -0.0113, -0.0316, ..., 0.0294, -0.0333, -0.0033],\n", - " [ 0.0151, 0.0121, 0.0291, ..., -0.0328, -0.0295, -0.0161],\n", - " ...,\n", - " [-0.0248, -0.0324, -0.0055, ..., 0.0129, -0.0264, -0.0004],\n", - " [ 0.0191, 0.0314, 0.0033, ..., -0.0264, 0.0292, 0.0079],\n", - " [ 0.0197, 0.0346, -0.0040, ..., 0.0037, 0.0151, -0.0032]],\n", - " device='cuda:0', requires_grad=True)\n", - "Parameter containing:\n", - "tensor([[ 3.3286e-03, 2.5395e-04, 1.2631e-02, ..., -2.2320e-02,\n", - " 3.2886e-02, 1.8957e-02],\n", - " [ 3.3295e-02, -2.8352e-02, -1.4806e-02, ..., -6.0922e-04,\n", - " -9.3333e-05, -1.5491e-02],\n", - " [ 1.4130e-02, -4.7829e-03, -1.1069e-02, ..., 2.9516e-02,\n", - " -4.1938e-03, -9.6518e-04],\n", - " ...,\n", - " [-7.7986e-04, -1.9359e-02, 1.9118e-02, ..., -1.6913e-02,\n", - " -3.6974e-03, -1.3698e-02],\n", - " [ 1.1183e-02, -1.6715e-02, 7.0737e-03, ..., -2.1076e-02,\n", - " -1.6323e-02, -4.1332e-03],\n", - " [-2.9230e-02, 5.7393e-05, 1.9348e-02, ..., -1.4401e-02,\n", - " 2.9383e-03, 9.1288e-03]], device='cuda:0', requires_grad=True)\n" - ] - } - ], + "outputs": [], "source": [ "print(model.model.layoutlm.encoder.layer[0].attention.self.query.weight)\n", "print(model.model.layoutlm.encoder.layer[0].attention.self.query.lora_A.weight)\n", @@ -1460,7 +897,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/" @@ -1468,1514 +905,7 @@ "id": "Yu0qePs2cRKo", "outputId": "cdbb9a03-eb9b-4740-bbe3-da06b9192bae" }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 10%|█████▍ | 1/10 [00:01<00:09, 1.11s/it]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 0 steps: 2.6514651775360107\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:03<00:00, 2.87it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.52it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 10 steps: 2.384181022644043\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.75it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.52it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 20 steps: 2.183215856552124\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.74it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.51it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 30 steps: 1.7022178173065186\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.74it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.51it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 40 steps: 1.1768041849136353\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.74it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.54it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 50 steps: 0.9286957383155823\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.74it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.53it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 60 steps: 0.7891858816146851\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.74it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.50it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 70 steps: 0.6700936555862427\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.74it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.51it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 80 steps: 0.5903935432434082\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.74it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.52it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 90 steps: 0.46567460894584656\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.74it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.51it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 100 steps: 0.43472540378570557\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.74it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.50it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 110 steps: 0.4344632029533386\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.73it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.51it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 120 steps: 0.29618480801582336\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.73it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.51it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 130 steps: 0.3432932496070862\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.73it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.51it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 140 steps: 0.26484620571136475\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.73it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.52it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 150 steps: 0.3635620176792145\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.73it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.52it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 160 steps: 0.28532707691192627\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.73it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.51it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 170 steps: 0.2699059247970581\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.73it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.50it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 180 steps: 0.25385379791259766\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.73it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.51it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 190 steps: 0.23770077526569366\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.73it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.50it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 200 steps: 0.18621960282325745\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.73it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.50it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 210 steps: 0.22020076215267181\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.73it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.52it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 220 steps: 0.16716673970222473\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.73it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.50it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 230 steps: 0.12251073122024536\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.73it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.51it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 240 steps: 0.12013913691043854\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.73it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.51it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 250 steps: 0.09276405721902847\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.73it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.52it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 260 steps: 0.11919382959604263\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.73it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.51it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 270 steps: 0.12163368612527847\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.73it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.51it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 280 steps: 0.09964565187692642\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.73it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.49it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 290 steps: 0.2966587543487549\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.73it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.53it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 300 steps: 0.1136971265077591\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.73it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.52it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 310 steps: 0.15149033069610596\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.50it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 320 steps: 0.11433174461126328\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.52it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 330 steps: 0.08340273052453995\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.51it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 340 steps: 0.05601643770933151\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.51it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 350 steps: 0.06016877666115761\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.52it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 360 steps: 0.06651942431926727\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.51it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 370 steps: 0.38789188861846924\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.73it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.51it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 380 steps: 0.16247011721134186\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.49it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 390 steps: 0.10822084546089172\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.49it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 400 steps: 0.10201780498027802\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.73it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.52it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 410 steps: 0.09159471094608307\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.73it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.50it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 420 steps: 0.052561257034540176\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.51it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 430 steps: 0.07446825504302979\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.52it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 440 steps: 0.055142153054475784\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.73it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.51it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 450 steps: 0.04064352810382843\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.51it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 460 steps: 0.047722212970256805\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.49it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 470 steps: 0.03962016478180885\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.50it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 480 steps: 0.05285123735666275\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.50it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 490 steps: 0.04547346383333206\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.49it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 500 steps: 0.035933323204517365\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.51it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 510 steps: 0.01684512570500374\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.73it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.52it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 520 steps: 0.018589401617646217\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.48it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 530 steps: 0.034401122480630875\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.49it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 540 steps: 0.029914753511548042\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.50it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 550 steps: 0.019160285592079163\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.49it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 560 steps: 0.030683889985084534\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.50it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 570 steps: 0.0281855259090662\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.50it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 580 steps: 0.019448189064860344\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.50it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 590 steps: 0.01689102128148079\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.50it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 600 steps: 0.012563714757561684\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.50it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 610 steps: 0.017075713723897934\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.52it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 620 steps: 0.011337793432176113\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.50it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 630 steps: 0.01142528560012579\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.71it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.49it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 640 steps: 0.012161443941295147\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.50it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 650 steps: 0.01025047805160284\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.49it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 660 steps: 0.008623240515589714\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.48it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 670 steps: 0.020196430385112762\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.50it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 680 steps: 0.013619341887533665\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.50it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 690 steps: 0.014648614451289177\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.49it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 700 steps: 0.012998558580875397\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.50it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 710 steps: 0.021816464141011238\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.71it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.49it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 720 steps: 0.01582351326942444\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.50it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 730 steps: 0.014046266674995422\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.50it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 740 steps: 0.007438092492520809\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.47it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 750 steps: 0.01699301227927208\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.49it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 760 steps: 0.013121521100401878\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.51it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 770 steps: 0.01430545188486576\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.49it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 780 steps: 0.016542932018637657\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.50it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 790 steps: 0.008827055804431438\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.50it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 800 steps: 0.014699053950607777\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.50it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 810 steps: 0.01083141379058361\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.50it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 820 steps: 0.011546650901436806\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.50it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 830 steps: 0.007675974629819393\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.71it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.50it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 840 steps: 0.011567591689527035\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.50it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 850 steps: 0.006588640157133341\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.50it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 860 steps: 0.012320021167397499\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.49it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 870 steps: 0.021453894674777985\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.50it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 880 steps: 0.010092893615365028\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.49it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 890 steps: 0.009351171553134918\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.49it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 900 steps: 0.004404681734740734\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.48it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 910 steps: 0.002058168640360236\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.71it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.50it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 920 steps: 0.0021678945049643517\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.51it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 930 steps: 0.006458617746829987\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.49it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 940 steps: 0.007828252390027046\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.49it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 950 steps: 0.008966044522821903\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.71it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.51it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 960 steps: 0.014970815740525723\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.71it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.49it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 970 steps: 0.0066721560433506966\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.50it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 980 steps: 0.004333506338298321\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s]\n", - "Training: 10%|█████▍ | 1/10 [00:00<00:02, 3.51it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loss after 990 steps: 0.006984252016991377\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Training: 100%|█████████████████████████████████████████████████████| 10/10 [00:02<00:00, 3.72it/s]\n" - ] - } - ], + "outputs": [], "source": [ "from transformers import AdamW, get_linear_schedule_with_warmup\n", "from tqdm import tqdm\n", @@ -3140,17 +1070,7 @@ "metadata": {}, "outputs": [], "source": [ - "from peft import get_peft_model_state_dict\n", - "to_return = get_peft_model_state_dict(model)\n" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": {}, - "outputs": [], - "source": [ - "torch.save(to_return, \"layoutlm_funsd.pt\")" + "model.save_pretrained(\"peft_layoutlm\")\n" ] }, { @@ -3167,7 +1087,7 @@ } ], "source": [ - "!du -h \"layoutlm_funsd.pt\"" + "!du -h \"peft_layoutlm/adapter_model.bin\"" ] }, { diff --git a/src/peft/__init__.py b/src/peft/__init__.py index b76dad8..c35eb32 100644 --- a/src/peft/__init__.py +++ b/src/peft/__init__.py @@ -40,13 +40,13 @@ from .tuners import ( PromptTuningInit, ) from .utils import ( + TRANSFORMERS_MODELS_TO_PREFIX_TUNING_POSTPROCESS_MAPPING, PeftConfig, PeftType, PromptLearningConfig, TaskType, bloom_model_postprocess_past_key_value, get_peft_model_state_dict, - peft_model_load_and_dispatch, set_peft_model_state_dict, shift_tokens_right, ) diff --git a/src/peft/peft_model.py b/src/peft/peft_model.py index 32937ae..49bfce1 100644 --- a/src/peft/peft_model.py +++ b/src/peft/peft_model.py @@ -18,6 +18,9 @@ import os import warnings import torch +from accelerate import dispatch_model, infer_auto_device_map +from accelerate.hooks import AlignDevicesHook, add_hook_to_module, remove_hook_from_submodules +from accelerate.utils import get_balanced_memory from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss from transformers import PreTrainedModel from transformers.modeling_outputs import SequenceClassifierOutput, TokenClassifierOutput @@ -27,6 +30,7 @@ from huggingface_hub import hf_hub_download from .tuners import LoraModel, PrefixEncoder, PromptEmbedding, PromptEncoder from .utils import ( + TRANSFORMERS_MODELS_TO_PREFIX_TUNING_POSTPROCESS_MAPPING, WEIGHTS_NAME, PeftConfig, PeftType, @@ -94,7 +98,14 @@ class PeftModel(PushToHubMixin, torch.nn.Module): raise ValueError(f"Provided path ({save_directory}) should be a directory, not a file") os.makedirs(save_directory, exist_ok=True) - # save the config + for param in self.parameters(): + param.requires_grad = False # freeze the model + + # save only the trainable weights + output_state_dict = get_peft_model_state_dict(self, kwargs.get("state_dict", None)) + torch.save(output_state_dict, os.path.join(save_directory, WEIGHTS_NAME)) + + # save the config and change the inference mode to `True` if self.peft_config.base_model_name_or_path is None: self.peft_config.base_model_name_or_path = ( self.base_model.__dict__.get("name_or_path", None) @@ -104,13 +115,6 @@ class PeftModel(PushToHubMixin, torch.nn.Module): self.peft_config.inference_mode = True self.peft_config.save_pretrained(save_directory) - for param in self.parameters(): - param.requires_grad = False # freeze the model - - # save only the trainable weights - output_state_dict = get_peft_model_state_dict(self, kwargs.get("state_dict", None)) - torch.save(output_state_dict, os.path.join(save_directory, WEIGHTS_NAME)) - @classmethod def from_pretrained(cls, model, model_id, **kwargs): r""" @@ -131,6 +135,9 @@ class PeftModel(PushToHubMixin, torch.nn.Module): # load the config config = PEFT_TYPE_TO_CONFIG_MAPPING[PeftConfig.from_pretrained(model_id).peft_type].from_pretrained(model_id) + if getattr(model, "hf_device_map", None) is not None: + remove_hook_from_submodules(model) + if config.task_type not in MODEL_TYPE_TO_PEFT_MODEL_MAPPING.keys(): model = cls(model, config) else: @@ -150,7 +157,30 @@ class PeftModel(PushToHubMixin, torch.nn.Module): adapters_weights = torch.load(filename) # load the weights into the model - return set_peft_model_state_dict(model, adapters_weights) + model = set_peft_model_state_dict(model, adapters_weights) + if getattr(model, "hf_device_map", None) is not None: + device_map = kwargs.get("device_map", "auto") + max_memory = kwargs.get("max_memory", None) + no_split_module_classes = model._no_split_modules + if device_map != "sequential": + max_memory = get_balanced_memory( + model, + max_memory=max_memory, + no_split_module_classes=no_split_module_classes, + low_zero=(device_map == "balanced_low_0"), + ) + if isinstance(device_map, str): + device_map = infer_auto_device_map( + model, max_memory=max_memory, no_split_module_classes=no_split_module_classes + ) + model = dispatch_model(model, device_map=device_map) + hook = AlignDevicesHook(io_same_device=True) + if model.peft_config.peft_type == PeftType.LORA: + add_hook_to_module(model.base_model.model, hook) + else: + remove_hook_from_submodules(model.prompt_encoder) + add_hook_to_module(model.base_model, hook) + return model def _setup_prompt_encoder(self): num_transformer_submodules = 0 @@ -218,8 +248,8 @@ class PeftModel(PushToHubMixin, torch.nn.Module): past_key_values = past_key_values.permute([2, 0, 3, 1, 4]).split( self.peft_config.num_transformer_submodules * 2 ) - if self.peft_config.postprocess_past_key_value_function is not None: - post_process_fn = self.peft_config.postprocess_past_key_value_function + if TRANSFORMERS_MODELS_TO_PREFIX_TUNING_POSTPROCESS_MAPPING.get(self.config.model_type, None) is not None: + post_process_fn = TRANSFORMERS_MODELS_TO_PREFIX_TUNING_POSTPROCESS_MAPPING[self.config.model_type] past_key_values = post_process_fn(past_key_values) return past_key_values else: @@ -538,17 +568,22 @@ class PeftModelForCausalLM(PeftModel): ) kwargs["token_type_ids"] = None - if self.peft_config.peft_type == PeftType.PREFIX_TUNING: - batch_size = kwargs["input_ids"].shape[0] - past_key_values = self.get_prompt(batch_size) - kwargs["past_key_values"] = past_key_values - return self.base_model.generate(**kwargs) - else: - raise NotImplementedError + return self.base_model.generate(**kwargs) def prepare_inputs_for_generation(self, *args, **kwargs): model_kwargs = self.base_model_prepare_inputs_for_generation(*args, **kwargs) - model_kwargs["past_key_values"] = kwargs.get("past", None) or kwargs.get("past_key_values", None) + if isinstance(self.peft_config, PromptLearningConfig): + if model_kwargs["past_key_values"] is None and self.peft_config.peft_type == PeftType.PREFIX_TUNING: + past_key_values = self.get_prompt(batch_size=model_kwargs["input_ids"].shape[0]) + model_kwargs["past_key_values"] = past_key_values + else: + if model_kwargs["past_key_values"] is None: + prompts = self.get_prompt(batch_size=model_kwargs["input_ids"].shape[0]) + model_kwargs["inputs_embeds"] = torch.cat( + (prompts, self.word_embeddings(model_kwargs["input_ids"])), dim=1 + ) + model_kwargs["input_ids"] = None + return model_kwargs @@ -682,25 +717,16 @@ class PeftModelForSeq2SeqLM(PeftModel): kwargs["token_type_ids"] = None if self.peft_config.peft_type == PeftType.PREFIX_TUNING: - batch_size = kwargs["input_ids"].shape[0] - past_key_values = self.get_prompt(batch_size) - kwargs["past_key_values"] = past_key_values return self.base_model.generate(**kwargs) else: raise NotImplementedError def prepare_inputs_for_generation(self, *args, **kwargs): model_kwargs = self.base_model_prepare_inputs_for_generation(*args, **kwargs) - model_kwargs["past_key_values"] = kwargs.get("past", None) or kwargs.get("past_key_values", None) - return model_kwargs - - def _prepare_encoder_decoder_kwargs_for_generation(self, inputs_tensor, model_kwargs, model_input_name=None): - past_key_values = model_kwargs.get("past_key_values", None) - model_kwargs["past_key_values"] = None - model_kwargs = self.base_model_prepare_encoder_decoder_kwargs_for_generation( - inputs_tensor, model_kwargs, model_input_name - ) - model_kwargs["past_key_values"] = past_key_values + if model_kwargs["past_key_values"] is None and self.peft_config.peft_type == PeftType.PREFIX_TUNING: + batch_size = model_kwargs["decoder_input_ids"].shape[0] + past_key_values = self.get_prompt(batch_size) + model_kwargs["past_key_values"] = past_key_values return model_kwargs diff --git a/src/peft/tuners/lora.py b/src/peft/tuners/lora.py index 0974a32..a2667fa 100644 --- a/src/peft/tuners/lora.py +++ b/src/peft/tuners/lora.py @@ -12,7 +12,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import importlib import math import warnings from dataclasses import asdict, dataclass, field @@ -29,15 +28,6 @@ import bitsandbytes as bnb from ..utils import PeftConfig, PeftType, transpose -def is_loralib_available(): - return importlib.util.find_spec("loralib") is not None - - -if is_loralib_available(): - import loralib as lora # noqa: F401 - from loralib import mark_only_lora_as_trainable - - @dataclass class LoraConfig(PeftConfig): """ @@ -108,8 +98,6 @@ class LoraModel(torch.nn.Module): """ def __init__(self, config, model): - if not is_loralib_available(): - raise ImportError("LoRA requires `loralib` to be installed. Please run `pip install loralib`.") super().__init__() self.peft_config = config self.model = model @@ -432,3 +420,22 @@ class Linear8bitLt(bnb.nn.Linear8bitLt, LoraLayer): if self.r > 0: result += self.lora_B(self.lora_A(self.lora_dropout(x))) * self.scaling return result + + +# had to adapt it for `lora_only` to work +def mark_only_lora_as_trainable(model: nn.Module, bias: str = "none") -> None: + for n, p in model.named_parameters(): + if "lora_" not in n: + p.requires_grad = False + if bias == "none": + return + elif bias == "all": + for n, p in model.named_parameters(): + if "bias" in n: + p.requires_grad = True + elif bias == "lora_only": + for m in model.modules(): + if isinstance(m, LoraLayer) and hasattr(m, "bias") and m.bias is not None: + m.bias.requires_grad = True + else: + raise NotImplementedError diff --git a/src/peft/tuners/prefix_tuning.py b/src/peft/tuners/prefix_tuning.py index d925c49..fcb207c 100644 --- a/src/peft/tuners/prefix_tuning.py +++ b/src/peft/tuners/prefix_tuning.py @@ -15,7 +15,6 @@ from dataclasses import dataclass, field -from typing import Callable, Optional import torch @@ -30,7 +29,6 @@ class PrefixTuningConfig(PromptLearningConfig): Args: encoder_hidden_size (`int`): The hidden size of the prompt encoder. prefix_projection (`bool`): Whether to project the prefix embeddings. - postprocess_past_key_value_function (`Callable`, *optional*): The function to postprocess the past key value. """ encoder_hidden_size: int = field( @@ -41,10 +39,6 @@ class PrefixTuningConfig(PromptLearningConfig): default=False, metadata={"help": "Whether to project the prefix tokens"}, ) - postprocess_past_key_value_function: Optional[Callable] = field( - default=None, - metadata={"help": "The function to postprocess the past key value"}, - ) def __post_init__(self): self.peft_type = PeftType.PREFIX_TUNING diff --git a/src/peft/utils/__init__.py b/src/peft/utils/__init__.py index c418d3d..55cdce7 100644 --- a/src/peft/utils/__init__.py +++ b/src/peft/utils/__init__.py @@ -19,5 +19,11 @@ from .adapters_utils import CONFIG_NAME, WEIGHTS_NAME from .config import PeftConfig, PeftType, PromptLearningConfig, TaskType -from .other import _set_trainable, bloom_model_postprocess_past_key_value, shift_tokens_right, transpose -from .save_and_load import get_peft_model_state_dict, peft_model_load_and_dispatch, set_peft_model_state_dict +from .other import ( + TRANSFORMERS_MODELS_TO_PREFIX_TUNING_POSTPROCESS_MAPPING, + _set_trainable, + bloom_model_postprocess_past_key_value, + shift_tokens_right, + transpose, +) +from .save_and_load import get_peft_model_state_dict, set_peft_model_state_dict diff --git a/src/peft/utils/other.py b/src/peft/utils/other.py index 14ab90e..3f8627e 100644 --- a/src/peft/utils/other.py +++ b/src/peft/utils/other.py @@ -30,6 +30,11 @@ def bloom_model_postprocess_past_key_value(past_key_values): return tuple(zip(keys, values)) +TRANSFORMERS_MODELS_TO_PREFIX_TUNING_POSTPROCESS_MAPPING = { + "bloom": bloom_model_postprocess_past_key_value, +} + + # copied from transformers.models.bart.modeling_bart def shift_tokens_right(input_ids: torch.Tensor, pad_token_id: int, decoder_start_token_id: int): """ diff --git a/src/peft/utils/save_and_load.py b/src/peft/utils/save_and_load.py index 9b6962c..c6596c7 100644 --- a/src/peft/utils/save_and_load.py +++ b/src/peft/utils/save_and_load.py @@ -77,35 +77,3 @@ def set_peft_model_state_dict(model, peft_model_state_dict): {"weight": peft_model_state_dict["prompt_embeddings"]}, strict=True ) return model - - -def peft_model_load_and_dispatch(model, peft_model_state_dict, peft_config, max_memory=None): - """ - Load the Peft model state dict and dispatch the model to the correct device. - - Args: - model ([`PeftModel`]): The Pre-trained base model which has already been sharded and dispatched - using `accelerate` functionalities. - peft_model_state_dict (`dict`): The state dict of the Peft model. - max_memory (`Dict`, *optional*): - A dictionary device identifier to maximum memory. Will default to the maximum memory available for each GPU - and the available CPU RAM if unset. - """ - from accelerate import dispatch_model, infer_auto_device_map - from accelerate.hooks import AlignDevicesHook, add_hook_to_module, remove_hook_from_submodules - - from ..mapping import get_peft_model - - remove_hook_from_submodules(model) - model = get_peft_model(model, peft_config) - model.print_trainable_parameters() - set_peft_model_state_dict(model, peft_model_state_dict) - device_map = infer_auto_device_map(model, max_memory=max_memory, no_split_module_classes=model._no_split_modules) - model = dispatch_model(model, device_map=device_map) - hook = AlignDevicesHook(io_same_device=True) - if model.peft_config.peft_type == PeftType.LORA: - add_hook_to_module(model.base_model.model, hook) - else: - remove_hook_from_submodules(model.prompt_encoder) - add_hook_to_module(model.base_model, hook) - return model