From 64dbdf1e843745a95695eed1735dbd93df7ddc7e Mon Sep 17 00:00:00 2001 From: Nick Walton Date: Mon, 9 Sep 2019 15:33:06 -0600 Subject: [PATCH] local branc --- cortex.sh | 468 ++++++++++++++++++++++++++++++ generator.py | 180 ++++++++++++ gpt2/CONTRIBUTORS.md | 17 ++ gpt2/DEVELOPERS.md | 85 ++++++ gpt2/LICENSE | 21 ++ gpt2/README.md | 61 ++++ gpt2/__init__.py | 0 gpt2/download_model.py | 28 ++ gpt2/models/117M/.gitignore | 1 + gpt2/models/117M/checkpoint | 2 + gpt2/models/117M/hparams.json | 7 + gpt2/models/117M/model.ckpt.index | Bin 0 -> 5215 bytes gpt2/models/117M/model.ckpt.meta | Bin 0 -> 471155 bytes gpt2/src/model.py | 176 +++++++++++ gpt2/src/sample.py | 79 +++++ main.py | 105 ++++--- 16 files changed, 1192 insertions(+), 38 deletions(-) create mode 100755 cortex.sh create mode 100644 generator.py create mode 100644 gpt2/CONTRIBUTORS.md create mode 100644 gpt2/DEVELOPERS.md create mode 100644 gpt2/LICENSE create mode 100644 gpt2/README.md create mode 100644 gpt2/__init__.py create mode 100644 gpt2/download_model.py create mode 100644 gpt2/models/117M/.gitignore create mode 100644 gpt2/models/117M/checkpoint create mode 100644 gpt2/models/117M/hparams.json create mode 100644 gpt2/models/117M/model.ckpt.index create mode 100644 gpt2/models/117M/model.ckpt.meta create mode 100644 gpt2/src/model.py create mode 100644 gpt2/src/sample.py diff --git a/cortex.sh b/cortex.sh new file mode 100755 index 0000000..d16f778 --- /dev/null +++ b/cortex.sh @@ -0,0 +1,468 @@ +#!/bin/bash + +# Copyright 2019 Cortex Labs, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +#################### +### FLAG PARSING ### +#################### + +flag_help=false +positional_args=() + +while [[ $# -gt 0 ]]; do + key="$1" + case $key in + -c|--config) + export CORTEX_CONFIG="$2" + shift + shift + ;; + -h|--help) + flag_help="true" + shift + ;; + *) + positional_args+=("$1") + shift + ;; + esac +done + +set -- "${positional_args[@]}" +positional_args=() +for i in "$@"; do + case $i in + -c=*|--config=*) + export CORTEX_CONFIG="${i#*=}" + shift + ;; + -h=*|--help=*) + flag_help="true" + ;; + *) + positional_args+=("$1") + shift + ;; + esac +done + +set -- "${positional_args[@]}" +if [ "$flag_help" == "true" ]; then + show_help + exit 0 +fi + +for arg in "$@"; do + if [[ "$arg" == -* ]]; then + echo "unknown flag: $arg" + show_help + exit 1 + fi +done + +##################### +### CONFIGURATION ### +##################### + +if [ "$CORTEX_CONFIG" != "" ]; then + if [ ! -f "$CORTEX_CONFIG" ]; then + echo "Cortex config file does not exist: $CORTEX_CONFIG" + exit 1 + fi + source $CORTEX_CONFIG +fi + +set -u + +export CORTEX_VERSION_STABLE=0.7.3 + +# Defaults +export AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID:-""}" + +if [ "$AWS_ACCESS_KEY_ID" = "" ]; then + echo -e "\nPlease set AWS_ACCESS_KEY_ID" + exit 1 +fi + +export AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY:-""}" + +if [ "$AWS_SECRET_ACCESS_KEY" = "" ]; then + echo -e "\nPlease set AWS_SECRET_ACCESS_KEY" + exit 1 +fi + +export CORTEX_LOG_GROUP="${CORTEX_LOG_GROUP:-cortex}" +export CORTEX_BUCKET="${CORTEX_BUCKET:-""}" +export CORTEX_REGION="${CORTEX_REGION:-us-west-2}" +export CORTEX_ZONES="${CORTEX_ZONES:-""}" + +export CORTEX_CLUSTER="${CORTEX_CLUSTER:-cortex}" +export CORTEX_NODE_TYPE="${CORTEX_NODE_TYPE:-t3.large}" +export CORTEX_NODES_MIN="${CORTEX_NODES_MIN:-2}" +export CORTEX_NODES_MAX="${CORTEX_NODES_MAX:-5}" +export CORTEX_NAMESPACE="${CORTEX_NAMESPACE:-cortex}" + +export CORTEX_IMAGE_MANAGER="${CORTEX_IMAGE_MANAGER:-cortexlabs/manager:$CORTEX_VERSION_STABLE}" +export CORTEX_IMAGE_FLUENTD="${CORTEX_IMAGE_FLUENTD:-cortexlabs/fluentd:$CORTEX_VERSION_STABLE}" +export CORTEX_IMAGE_NGINX_BACKEND="${CORTEX_IMAGE_NGINX_BACKEND:-cortexlabs/nginx-backend:$CORTEX_VERSION_STABLE}" +export CORTEX_IMAGE_NGINX_CONTROLLER="${CORTEX_IMAGE_NGINX_CONTROLLER:-cortexlabs/nginx-controller:$CORTEX_VERSION_STABLE}" +export CORTEX_IMAGE_OPERATOR="${CORTEX_IMAGE_OPERATOR:-cortexlabs/operator:$CORTEX_VERSION_STABLE}" +export CORTEX_IMAGE_SPARK="${CORTEX_IMAGE_SPARK:-cortexlabs/spark:$CORTEX_VERSION_STABLE}" +export CORTEX_IMAGE_SPARK_OPERATOR="${CORTEX_IMAGE_SPARK_OPERATOR:-cortexlabs/spark-operator:$CORTEX_VERSION_STABLE}" +export CORTEX_IMAGE_TF_SERVE="${CORTEX_IMAGE_TF_SERVE:-cortexlabs/tf-serve:$CORTEX_VERSION_STABLE}" +export CORTEX_IMAGE_TF_TRAIN="${CORTEX_IMAGE_TF_TRAIN:-cortexlabs/tf-train:$CORTEX_VERSION_STABLE}" +export CORTEX_IMAGE_TF_API="${CORTEX_IMAGE_TF_API:-cortexlabs/tf-api:$CORTEX_VERSION_STABLE}" +export CORTEX_IMAGE_PYTHON_PACKAGER="${CORTEX_IMAGE_PYTHON_PACKAGER:-cortexlabs/python-packager:$CORTEX_VERSION_STABLE}" +export CORTEX_IMAGE_TF_SERVE_GPU="${CORTEX_IMAGE_TF_SERVE_GPU:-cortexlabs/tf-serve-gpu:$CORTEX_VERSION_STABLE}" +export CORTEX_IMAGE_TF_TRAIN_GPU="${CORTEX_IMAGE_TF_TRAIN_GPU:-cortexlabs/tf-train-gpu:$CORTEX_VERSION_STABLE}" +export CORTEX_IMAGE_ONNX_SERVE="${CORTEX_IMAGE_ONNX_SERVE:-cortexlabs/onnx-serve:$CORTEX_VERSION_STABLE}" +export CORTEX_IMAGE_ONNX_SERVE_GPU="${CORTEX_IMAGE_ONNX_SERVE_GPU:-cortexlabs/onnx-serve-gpu:$CORTEX_VERSION_STABLE}" +export CORTEX_IMAGE_CLUSTER_AUTOSCALER="${CORTEX_IMAGE_CLUSTER_AUTOSCALER:-cortexlabs/cluster-autoscaler:$CORTEX_VERSION_STABLE}" +export CORTEX_IMAGE_NVIDIA="${CORTEX_IMAGE_NVIDIA:-cortexlabs/nvidia:$CORTEX_VERSION_STABLE}" +export CORTEX_IMAGE_METRICS_SERVER="${CORTEX_IMAGE_METRICS_SERVER:-cortexlabs/metrics-server:$CORTEX_VERSION_STABLE}" + +export CORTEX_ENABLE_TELEMETRY="${CORTEX_ENABLE_TELEMETRY:-""}" + +########################## +### TOP-LEVEL COMMANDS ### +########################## + +function install_eks() { + echo + docker run -it --entrypoint /root/install_eks.sh \ + -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ + -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ + -e CORTEX_CLUSTER=$CORTEX_CLUSTER \ + -e CORTEX_REGION=$CORTEX_REGION \ + -e CORTEX_NODE_TYPE=$CORTEX_NODE_TYPE \ + -e CORTEX_NODES_MIN=$CORTEX_NODES_MIN \ + -e CORTEX_NODES_MAX=$CORTEX_NODES_MAX \ + $CORTEX_IMAGE_MANAGER +} + +function uninstall_eks() { + echo + docker run -it --entrypoint /root/uninstall_eks.sh \ + -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ + -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ + -e CORTEX_CLUSTER=$CORTEX_CLUSTER \ + -e CORTEX_REGION=$CORTEX_REGION \ + $CORTEX_IMAGE_MANAGER +} + +function install_cortex() { + echo + docker run -it --entrypoint /root/install_cortex.sh \ + -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ + -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ + -e CORTEX_CLUSTER=$CORTEX_CLUSTER \ + -e CORTEX_REGION=$CORTEX_REGION \ + -e CORTEX_NAMESPACE=$CORTEX_NAMESPACE \ + -e CORTEX_NODE_TYPE=$CORTEX_NODE_TYPE \ + -e CORTEX_LOG_GROUP=$CORTEX_LOG_GROUP \ + -e CORTEX_BUCKET=$CORTEX_BUCKET \ + -e CORTEX_IMAGE_FLUENTD=$CORTEX_IMAGE_FLUENTD \ + -e CORTEX_IMAGE_NGINX_BACKEND=$CORTEX_IMAGE_NGINX_BACKEND \ + -e CORTEX_IMAGE_NGINX_CONTROLLER=$CORTEX_IMAGE_NGINX_CONTROLLER \ + -e CORTEX_IMAGE_OPERATOR=$CORTEX_IMAGE_OPERATOR \ + -e CORTEX_IMAGE_SPARK=$CORTEX_IMAGE_SPARK \ + -e CORTEX_IMAGE_SPARK_OPERATOR=$CORTEX_IMAGE_SPARK_OPERATOR \ + -e CORTEX_IMAGE_TF_SERVE=$CORTEX_IMAGE_TF_SERVE \ + -e CORTEX_IMAGE_TF_TRAIN=$CORTEX_IMAGE_TF_TRAIN \ + -e CORTEX_IMAGE_TF_API=$CORTEX_IMAGE_TF_API \ + -e CORTEX_IMAGE_PYTHON_PACKAGER=$CORTEX_IMAGE_PYTHON_PACKAGER \ + -e CORTEX_IMAGE_TF_SERVE_GPU=$CORTEX_IMAGE_TF_SERVE_GPU \ + -e CORTEX_IMAGE_TF_TRAIN_GPU=$CORTEX_IMAGE_TF_TRAIN_GPU \ + -e CORTEX_IMAGE_ONNX_SERVE=$CORTEX_IMAGE_ONNX_SERVE \ + -e CORTEX_IMAGE_ONNX_SERVE_GPU=$CORTEX_IMAGE_ONNX_SERVE_GPU \ + -e CORTEX_IMAGE_CLUSTER_AUTOSCALER=$CORTEX_IMAGE_CLUSTER_AUTOSCALER \ + -e CORTEX_IMAGE_NVIDIA=$CORTEX_IMAGE_NVIDIA \ + -e CORTEX_IMAGE_METRICS_SERVER=$CORTEX_IMAGE_METRICS_SERVER \ + -e CORTEX_ENABLE_TELEMETRY=$CORTEX_ENABLE_TELEMETRY \ + $CORTEX_IMAGE_MANAGER +} + +function uninstall_operator() { + echo + docker run -it --entrypoint /root/uninstall_operator.sh \ + -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ + -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ + -e CORTEX_CLUSTER=$CORTEX_CLUSTER \ + -e CORTEX_REGION=$CORTEX_REGION \ + -e CORTEX_NAMESPACE=$CORTEX_NAMESPACE \ + $CORTEX_IMAGE_MANAGER +} + +function info() { + echo + docker run -it --entrypoint /root/info.sh \ + -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ + -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ + -e CORTEX_CLUSTER=$CORTEX_CLUSTER \ + -e CORTEX_REGION=$CORTEX_REGION \ + -e CORTEX_NAMESPACE=$CORTEX_NAMESPACE \ + $CORTEX_IMAGE_MANAGER +} + +################ +### CHECK OS ### +################ + +case "$OSTYPE" in + darwin*) PARSED_OS="darwin" ;; + linux*) PARSED_OS="linux" ;; + *) echo -e "\nerror: only mac and linux are supported"; exit 1 ;; +esac + +############################# +### DEPENDENCY MANAGEMENT ### +############################# + +function check_dep_curl() { + if ! command -v curl >/dev/null; then + echo -e "\nerror: please install \`curl\`" + exit 1 + fi +} + +function install_cli() { + set -e + + check_dep_curl + + echo -e "\nInstalling the Cortex CLI (/usr/local/bin/cortex) ..." + + CORTEX_SH_TMP_DIR="$HOME/.cortex-sh-tmp" + rm -rf $CORTEX_SH_TMP_DIR && mkdir -p $CORTEX_SH_TMP_DIR + curl -s -o $CORTEX_SH_TMP_DIR/cortex https://s3-us-west-2.amazonaws.com/get-cortex/$CORTEX_VERSION_STABLE/cli/$PARSED_OS/cortex + chmod +x $CORTEX_SH_TMP_DIR/cortex + + if [ $(id -u) = 0 ]; then + mv -f $CORTEX_SH_TMP_DIR/cortex /usr/local/bin/cortex + else + ask_sudo + sudo mv -f $CORTEX_SH_TMP_DIR/cortex /usr/local/bin/cortex + fi + + rm -rf $CORTEX_SH_TMP_DIR + echo "✓ Installed the Cortex CLI" + + bash_profile_path=$(get_bash_profile) + if [ ! "$bash_profile_path" = "" ]; then + if ! grep -Fxq "source <(cortex completion)" "$bash_profile_path"; then + echo + read -p "Would you like to modify your bash profile ($bash_profile_path) to enable cortex command completion and the cx alias? [Y/n] " -n 1 -r + echo + if [[ $REPLY =~ ^[Yy]$ ]]; then + echo -e "\nsource <(cortex completion)" >> $bash_profile_path + echo "✓ Your bash profile ($bash_profile_path) has been updated" + echo + echo "Note: \`bash_completion\` must be installed on your system for cortex command completion to function properly" + echo + echo "Command to update your current terminal session:" + echo " source $bash_profile_path" + else + echo "Your bash profile has not been modified. If you would like to modify it manually, add this line to your bash profile:" + echo " source <(cortex completion)" + echo "Note: \`bash_completion\` must be installed on your system for cortex command completion to function properly" + fi + fi + else + echo -e "\nIf your would like to enable cortex command completion and the cx alias, add this line to your bash profile:" + echo " source <(cortex completion)" + echo "Note: \`bash_completion\` must be installed on your system for cortex command completion to function properly" + fi +} + +function uninstall_cli() { + set -e + + rm -rf $HOME/.cortex + + if ! command -v cortex >/dev/null; then + echo -e "\nThe Cortex CLI is not installed" + return + fi + + if [[ ! -f /usr/local/bin/cortex ]]; then + echo -e "\nThe Cortex CLI was not found at /usr/local/bin/cortex, please uninstall it manually" + return + fi + + if [ $(id -u) = 0 ]; then + rm /usr/local/bin/cortex + else + ask_sudo + sudo rm /usr/local/bin/cortex + fi + echo -e "\n✓ Uninstalled the Cortex CLI" + + bash_profile_path=$(get_bash_profile) + if [ ! "$bash_profile_path" = "" ]; then + if grep -Fxq "source <(cortex completion)" "$bash_profile_path"; then + echo + read -p "Would you like to remove \"source <(cortex completion)\" from your bash profile ($bash_profile_path)? [Y/n] " -n 1 -r + echo + if [[ $REPLY =~ ^[Yy]$ ]]; then + sed '/^source <(cortex completion)$/d' "$bash_profile_path" > "${bash_profile_path}_cortex_modified" && mv -f "${bash_profile_path}_cortex_modified" "$bash_profile_path" + echo "✓ Your bash profile ($bash_profile_path) has been updated" + fi + fi + fi +} + +function get_bash_profile() { + if [ "$PARSED_OS" = "darwin" ]; then + if [ -f $HOME/.bash_profile ]; then + echo $HOME/.bash_profile + return + elif [ -f $HOME/.bashrc ]; then + echo $HOME/.bashrc + return + fi + else + if [ -f $HOME/.bashrc ]; then + echo $HOME/.bashrc + return + elif [ -f $HOME/.bash_profile ]; then + echo $HOME/.bash_profile + return + fi + fi + + echo "" +} + +function ask_sudo() { + if ! sudo -n true 2>/dev/null; then + echo -e "\nPlease enter your sudo password" + fi +} + +function prompt_for_telemetry() { + if [ "$CORTEX_ENABLE_TELEMETRY" != "true" ] && [ "$CORTEX_ENABLE_TELEMETRY" != "false" ]; then + while true + do + echo + read -p "Would you like to help improve Cortex by anonymously sending error reports and usage stats to the dev team? [Y/n] " -n 1 -r + echo + if [[ $REPLY =~ ^[Yy]$ ]]; then + export CORTEX_ENABLE_TELEMETRY=true + break + elif [[ $REPLY =~ ^[Nn]$ ]]; then + export CORTEX_ENABLE_TELEMETRY=false + break + fi + echo "Unexpected value, please enter \"Y\" or \"n\"" + done + fi +} + +############ +### HELP ### +############ + +function show_help() { + echo " +Usage: + ./cortex.sh command [sub-command] [flags] + +Available Commands: + install install Cortex + uninstall uninstall Cortex + update update Cortex + info information about Cortex + + install cli install the Cortex CLI + uninstall cli uninstall the Cortex CLI + +Flags: + -c, --config path to a Cortex config file + -h, --help +" +} + +###################### +### ARG PROCESSING ### +###################### + +arg1=${1:-""} +arg2=${2:-""} +arg3=${3:-""} + +if [ -z "$arg1" ]; then + show_help + exit 0 +fi + +if [ "$arg1" = "install" ]; then + if [ ! "$arg3" = "" ]; then + echo -e "\nerror: too many arguments for install command" + show_help + exit 1 + elif [ "$arg2" = "" ]; then + prompt_for_telemetry && install_eks && install_cortex && info + elif [ "$arg2" = "cli" ]; then + install_cli + elif [ "$arg2" = "" ]; then + echo -e "\nerror: missing subcommand for install" + show_help + exit 1 + else + echo -e "\nerror: invalid subcommand for install: $arg2" + show_help + exit 1 + fi +elif [ "$arg1" = "uninstall" ]; then + if [ ! "$arg3" = "" ]; then + echo -e "\nerror: too many arguments for uninstall command" + show_help + exit 1 + elif [ "$arg2" = "" ]; then + uninstall_eks + elif [ "$arg2" = "cli" ]; then + uninstall_cli + elif [ "$arg2" = "" ]; then + echo -e "\nerror: missing subcommand for uninstall" + show_help + exit 1 + else + echo -e "\nerror: invalid subcommand for uninstall: $arg2" + show_help + exit 1 + fi +elif [ "$arg1" = "update" ]; then + if [ ! "$arg2" = "" ]; then + echo -e "\nerror: too many arguments for get command" + show_help + exit 1 + else + uninstall_operator && install_cortex + fi +elif [ "$arg1" = "info" ]; then + if [ ! "$arg2" = "" ]; then + echo -e "\nerror: too many arguments for get command" + show_help + exit 1 + else + info + fi +else + echo -e "\nerror: unknown command: $arg1" + show_help + exit 1 +fi diff --git a/generator.py b/generator.py new file mode 100644 index 0000000..ac1e8b0 --- /dev/null +++ b/generator.py @@ -0,0 +1,180 @@ +import json +import os +import numpy as np +import tensorflow as tf + +import gpt2.src.model as model +from tensorflow.contrib import predictor +import gpt2.src.sample as sample +import gpt2.src.encoder as encoder +from utils import * + +pos_action_starts = ["You attack", "You tell", "You use", "You go"] + + +class StoryGenerator(): + + def __init__(self, sess, length=75, temperature=0.9, top_k=40): + + seed = None + batch_size=1 + model_path='gpt2/models/117M' + self.sess = sess + + self.enc = encoder.get_encoder(model_path) + hparams = model.default_hparams() + with open(os.path.join(model_path, 'hparams.json')) as f: + hparams.override_from_dict(json.load(f)) + + self.context = tf.placeholder(tf.int32, [batch_size, None]) + np.random.seed(seed) + tf.set_random_seed(seed) + self.output = sample.sample_sequence( + hparams=hparams, length=length, + context=self.context, + batch_size=batch_size, + ) + + saver = tf.train.Saver() + ckpt = tf.train.latest_checkpoint(model_path) + saver.restore(self.sess, ckpt) + + + def generate(self, prompt): + context_tokens = self.enc.encode(prompt) + out = self.sess.run(self.output, feed_dict={ + self.context: [context_tokens for _ in range(1)] + })[:, len(context_tokens):] + + text = self.enc.decode(out[0]) + return text + + def generate_story_block(self, prompt): + block = self.generate(prompt) + block = cut_trailing_sentence(block) + block = story_replace(block) + + return block + + def generate_action_options(self, prompt, action_starts=pos_action_starts): + + possible_actions = [] + for phrase in action_starts: + action = phrase + self.generate(prompt + phrase) + action = first_sentence(action) + possible_actions.append(action) + + return possible_actions + + def generate_action_result(self, prompt, phrase): + action = phrase + self.generate(prompt + phrase) + action_result = cut_trailing_sentence(action) + action_result = story_replace(action_result) + + action = first_sentence(action) + + + return action, action_result + + +def save_model(): + length=75 + temperature=0.9 + top_k=40 + + with tf.Session() as sess: + seed = None + batch_size=None + model_path='gpt2/models/117M' + + hparams = model.default_hparams() + with open(os.path.join(model_path, 'hparams.json')) as f: + hparams.override_from_dict(json.load(f)) + + context = tf.placeholder(tf.int32, [batch_size, None]) + np.random.seed(seed) + tf.set_random_seed(seed) + output = sample.sample_sequence( + hparams=hparams, length=length, + context=context, + batch_size=batch_size, + ) + print("***********************",type(output)) + + saver = tf.train.Saver() + ckpt = tf.train.latest_checkpoint(model_path) + saver.restore(sess, ckpt) + + tf.saved_model.simple_save(sess, "./saved2", inputs={"context": context}, outputs={"output": output}) + + +def generate_gpu_config(memory_fraction): + config = tf.ConfigProto() + config.gpu_options.allow_growth = True + config.gpu_options.per_process_gpu_memory_fraction = memory_fraction + return config + +def run_interactive(): + pass + + +def load_model(): + # Set your memory fraction equal to a value less than 1, 0.6 is a good starting point. + # If no fraction is defined, the tensorflow algorithm may run into gpu out of memory problems. + fraction = 0.6 + config = config=generate_gpu_config(fraction) + path_to_graph = "./saved" + + #tf.saved_model.loader.load( + # session, + # [tf.saved_model.tag_constants.SERVING], + # path_to_graph) + + #output = session.graph.get_tensor_by_name('output:0') + #context = session.graph.get_tensor_by_name('context:0') + model_path = 'gpt2/models/117M' + enc = encoder.get_encoder(model_path) + + predict_fn = predictor.from_saved_model(path_to_graph, config=config) + context_tokens = [enc.encode("hello")] + predictions = predict_fn({"context": context_tokens}) + output = enc.decode(predictions["output"][0]) + print(output) + + return (output, session) + +if __name__ == '__main__': + save_model() + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/gpt2/CONTRIBUTORS.md b/gpt2/CONTRIBUTORS.md new file mode 100644 index 0000000..eab7132 --- /dev/null +++ b/gpt2/CONTRIBUTORS.md @@ -0,0 +1,17 @@ +# Contributors (alphabetically) + +* **[madisonmay](https://github.com/madisonmay)** + + Added Dockerfiles + +* **[Margaret Mitchell et al](https://arxiv.org/abs/1810.03993)** + + Our [usage](./README.md#usage) writeup was loosely inspired by the paper + [Model Cards for Model Reporting](https://arxiv.org/abs/1810.03993) + and related conversations with some of the authors. + +* **[webproduktion01](https://github.com/webproduktion01)** + + Ported download script to python. + +**[Full code contributors list](https://github.com/openai/gpt-2/contributors).** diff --git a/gpt2/DEVELOPERS.md b/gpt2/DEVELOPERS.md new file mode 100644 index 0000000..078999b --- /dev/null +++ b/gpt2/DEVELOPERS.md @@ -0,0 +1,85 @@ +# Installation + +Git clone this repository, and `cd` into directory for remaining commands +``` +git clone https://github.com/openai/gpt-2.git && cd gpt-2 +``` + +Then, follow instructions for either native or Docker installation. + +## Native Installation + +All steps can optionally be done in a virtual environment using tools such as `virtualenv` or `conda`. + +Install tensorflow 1.12 (with GPU support, if you have a GPU and want everything to run faster) +``` +pip3 install tensorflow==1.12.0 +``` +or +``` +pip3 install tensorflow-gpu==1.12.0 +``` + +Install other python packages: +``` +pip3 install -r requirements.txt +``` + +Download the model data +``` +python3 download_model.py 117M +``` + +## Docker Installation + +Build the Dockerfile and tag the created image as `gpt-2`: +``` +docker build --tag gpt-2 -f Dockerfile.gpu . # or Dockerfile.cpu +``` + +Start an interactive bash session from the `gpt-2` docker image. + +You can opt to use the `--runtime=nvidia` flag if you have access to a NVIDIA GPU +and a valid install of [nvidia-docker 2.0](https://github.com/nvidia/nvidia-docker/wiki/Installation-(version-2.0)). +``` +docker run --runtime=nvidia -it gpt-2 bash +``` + +# Running + +| WARNING: Samples are unfiltered and may contain offensive content. | +| --- | + +Some of the examples below may include Unicode text characters. Set the environment variable: +``` +export PYTHONIOENCODING=UTF-8 +``` +to override the standard stream settings in UTF-8 mode. + +## Unconditional sample generation + +To generate unconditional samples from the small model: +``` +python3 src/generate_unconditional_samples.py | tee /tmp/samples +``` +There are various flags for controlling the samples: +``` +python3 src/generate_unconditional_samples.py --top_k 40 --temperature 0.7 | tee /tmp/samples +``` + +To check flag descriptions, use: +``` +python3 src/generate_unconditional_samples.py -- --help +``` + +## Conditional sample generation + +To give the model custom prompts, you can use: +``` +python3 src/interactive_conditional_samples.py --top_k 40 +``` + +To check flag descriptions, use: +``` +python3 src/interactive_conditional_samples.py -- --help +``` diff --git a/gpt2/LICENSE b/gpt2/LICENSE new file mode 100644 index 0000000..cb36e12 --- /dev/null +++ b/gpt2/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2019 OpenAI + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/gpt2/README.md b/gpt2/README.md new file mode 100644 index 0000000..c1be039 --- /dev/null +++ b/gpt2/README.md @@ -0,0 +1,61 @@ +# gpt-2 + +Code and samples from the paper ["Language Models are Unsupervised Multitask Learners"](https://d4mucfpksywv.cloudfront.net/better-language-models/language-models.pdf). + +For now, we have only released a smaller (117M parameter) version of GPT-2. + +See more details in our [blog post](https://blog.openai.com/better-language-models/). + +## Usage + +This repository is meant to be a starting point for researchers and engineers to experiment with GPT-2-117M. While GPT-2-117M is less proficient than GPT-2-1.5B, it is useful for a wide range of research and applications which could also apply to larger models. + +### Some caveats + +- GPT-2-117M robustness and worst case behaviors are not well-understood. As with any machine-learned model, carefully evaluate GPT-2-117M for your use case, especially if used without fine-tuning or in safety-critical applications where reliability is important. +- The dataset our GPT-2-117M was trained on contains many texts with [biases](https://twitter.com/TomerUllman/status/1101485289720242177) and factual inaccuracies, and thus GPT-2-117M is likely to be biased and inaccurate as well. +- To avoid having samples mistaken as human-written, we recommend clearly labeling samples as synthetic before wide dissemination. Our models are often incoherent or inaccurate in subtle ways, which takes more than a quick read for a human to notice. + +### Work with us + +Please [let us know](mailto:languagequestions@openai.com) if you’re doing interesting research with or working on applications of GPT-2-117M! We’re especially interested in hearing from and potentially working with those who are studying +- Potential malicious use cases and defenses against them (e.g. the detectability of synthetic text) +- The extent of problematic content (e.g. bias) being baked into the models and effective mitigations + +## Development + +See [DEVELOPERS.md](./DEVELOPERS.md) + +## Contributors + +See [CONTRIBUTORS.md](./CONTRIBUTORS.md) + +## GPT-2 samples + +| WARNING: Samples are unfiltered and may contain offensive content. | +| --- | + +While we have not yet released GPT-2 itself, you can see some samples from it in the `gpt-2-samples` folder. +We show unconditional samples with default settings (temperature 1 and no truncation), with temperature 0.7, and with truncation with top_k 40. +We show conditional samples, with contexts drawn from `WebText`'s test set, with default settings (temperature 1 and no truncation), with temperature 0.7, and with truncation with top_k 40. + +## Citation + +Please use the following bibtex entry: +``` +@article{radford2019language, + title={Language Models are Unsupervised Multitask Learners}, + author={Radford, Alec and Wu, Jeff and Child, Rewon and Luan, David and Amodei, Dario and Sutskever, Ilya}, + year={2019} +} +``` + +## Future work + +We may release code for evaluating the models on various benchmarks. + +We are still considering release of the larger models. + +## License + +[MIT](./LICENSE) diff --git a/gpt2/__init__.py b/gpt2/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/gpt2/download_model.py b/gpt2/download_model.py new file mode 100644 index 0000000..30ba84a --- /dev/null +++ b/gpt2/download_model.py @@ -0,0 +1,28 @@ +import os +import sys +import requests +from tqdm import tqdm + +if len(sys.argv) != 2: + print('You must enter the model name as a parameter, e.g.: download_model.py 117M') + sys.exit(1) + +model = sys.argv[1] + +subdir = os.path.join('models', model) +if not os.path.exists(subdir): + os.makedirs(subdir) +subdir = subdir.replace('\\','/') # needed for Windows + +for filename in ['checkpoint','encoder.json','hparams.json','model.ckpt.data-00000-of-00001', 'model.ckpt.index', 'model.ckpt.meta', 'vocab.bpe']: + + r = requests.get("https://storage.googleapis.com/gpt-2/" + subdir + "/" + filename, stream=True) + + with open(os.path.join(subdir, filename), 'wb') as f: + file_size = int(r.headers["content-length"]) + chunk_size = 1000 + with tqdm(ncols=100, desc="Fetching " + filename, total=file_size, unit_scale=True) as pbar: + # 1k for chunk_size, since Ethernet packet size is around 1500 bytes + for chunk in r.iter_content(chunk_size=chunk_size): + f.write(chunk) + pbar.update(chunk_size) diff --git a/gpt2/models/117M/.gitignore b/gpt2/models/117M/.gitignore new file mode 100644 index 0000000..ada5d9f --- /dev/null +++ b/gpt2/models/117M/.gitignore @@ -0,0 +1 @@ +model* \ No newline at end of file diff --git a/gpt2/models/117M/checkpoint b/gpt2/models/117M/checkpoint new file mode 100644 index 0000000..febd7d5 --- /dev/null +++ b/gpt2/models/117M/checkpoint @@ -0,0 +1,2 @@ +model_checkpoint_path: "model.ckpt" +all_model_checkpoint_paths: "model.ckpt" diff --git a/gpt2/models/117M/hparams.json b/gpt2/models/117M/hparams.json new file mode 100644 index 0000000..cc4af87 --- /dev/null +++ b/gpt2/models/117M/hparams.json @@ -0,0 +1,7 @@ +{ + "n_vocab": 50257, + "n_ctx": 1024, + "n_embd": 768, + "n_head": 12, + "n_layer": 12 +} diff --git a/gpt2/models/117M/model.ckpt.index b/gpt2/models/117M/model.ckpt.index new file mode 100644 index 0000000000000000000000000000000000000000..24729aa0232341273ef983aabc2355455b8e8b14 GIT binary patch literal 5215 zcmai&e^gUt7{}i+8e;;32^;>NWHrkeh}VEbE7Qn~QZZ8urX&gSD@D^%&J9Ec{$Psc z^#?knXyvh1p)Nh;GK*QJvqYC<&FXYkX^qTWYKPwE-VNdIeQSTfIrsTK-}~I>^FHtW zfsjlt85km$AazVh*|Q6Z^$U~q^C~Jz_4#@Hzxi^B>Nb^B&Z=~5CR34d*Sa{#uov*b zC>73vfQ*l7*-RbFTBS_n^BHTRW$N<9Wkvd)WSN$ASfx5Po^dXz-ySC!9_)Z6Zji7h z33E1aNpZMLU0j-%+{eRh4+W3r$>}GfBTB9)CL=A&4`+C!PBq#o|)-g8PG9T5-*c>w70*3NbkAg&rOlX#}pJ^L?FtF1yrJGkVX zAV3Zw7CC;EO_b#pBJL}ZAJ72#Kw@A9HFaodfLyDw1e*KJq?)-TK(1qID7#=&{Vp0H zFG#>+4@xPJE7i0q(X!svg8944NS{g+I4{D=+h&&65EPr#|sibW6!^bh{k*E9w_KyWo`=5 z1EVqeumsS2w(H&fBs50jz2I^D+l*Kd8nX~7eQo>iL1=73EXVQUSW%W!h}^$mPb`gv z2P6i@eRRfR8Vg4qmOvi|WEV~&v2YwuhO$#jongJUjby#BaRzT20YGlZ1LH%FYp3AN zZ8UXrbJvXpK*lrE{}Iy;%#NiVO8`)A_UsQxRay?EL6sZ32lOX;U_f?|0H7xHr{g66=fMe=lvE{RX;@A$IKuxFo&mxWYhLB z_5_wdACvbTk&*T>cHIOhyQ*8hYYuK$OAwPRq0DefPCxl<3>EwwtLrHQL9bs z`N)ck`Ds_kP9RRX5lmSTWEl68?XeLd*$S)vm`*alT)GW z&SkS2X@Hy{0gnqa^3pM&F;qa#4byl)^+n}0KwJZs03e6v-peFF+>vZ3DCMWPhlw7T zg%u>g;(_nC_(&|Qr)GkOJ__ZEy6O-jyGOPx4`SgKV(uBATt~{ljb=>nea0u~+J0Ga zJB^IVi3>0=Q-6HBmPW=466k7W>X|YU8Lw#$T*UHOzBU>e<3=n2kM6qGv6u`)C1X6D z3uIbdIaD%EkN}y32inh)$T(k~4+XJ>@8!_Qct8SV!kQaalE`>A=7Yzuob%&EWY|r7 zEE=6`Q9c1Q!?FAJAFf%%N3^07A-1D|7W=I@1g9P}j&+a@#;$uEj4j!2kqS{5Bwh57}A)yZj@o@^Vcdl$I5@mUW z$llnz12jHt2R@~N*)^^)g~rDX66j+}+RQ8xANQ7}Q1-UE%Pq87@qq+9&K(S&hgsZ5 zW#Q{s0W5qK3&#-qh*{XrVhONV+@boM#KOL26%=%)jJtO0Hq&!T`fbJqc>ri{j`A@Q zAlECEV0{0=+1Es^_6m`ajZ>cr0%Y9EXHB}%l_knD3z3ZtH}9YUvVp|FtdF?VMgzog zSOU#0DNfp_lxg@2?ES9IxvC^kH#al$l~_qapLe;yLz$J=fXi~Ob#<6hlS)NoZ1PN(I|O%4Yzp`#0+byLQCVM>_unn1C*z literal 0 HcmV?d00001 diff --git a/gpt2/models/117M/model.ckpt.meta b/gpt2/models/117M/model.ckpt.meta new file mode 100644 index 0000000000000000000000000000000000000000..2541738718793eb0c18a26c418951f35d098500a GIT binary patch literal 471155 zcmd44Uyx?Ubsx6h%wjOx072jbNG^wzD6Rew8e3ZIEJ&(0QUrofB}^GKMZlyZOB~Mb z41pnbXO}ax_#;)SD3?{TW2H)_q?BBVZ8>tBiWEmqRVsOK5=T`@C>mXPoH!8{M`TciRFL!33tbBe*W-q zx#bq`-&nlq|NH>{+<`xT0sg%42?sx3J^$d*(Vb`AGkZ@z^Za7t>edsR*S42Wy7%4I z=g-cL-#S@7Kb-?*0Bys_h+fme?Y z56+MF&)z)|Ng^!?;M}Jg%5sfv2bs@r(QiczxVd* z2j{OpzF%I~T2U3_T?-yghxe0JmNo5zRm@32lh?cn+5@yYp%iwr54&FXEIehK-!I>ZY;}aNf_`qw$;qe3L z4*ZvQhp2$>06bSZ@TzF?nDZ3E@(EM(+MOre6MLIiVQ#+eu73I8-aB#y z;3VjrXU%??(%!g!#eKm&_2v5q_m19vc>nO|w4CsKe&2Y(Juxk|(;qum?`&LOxX-%H zFF$-Q&3gYQ+X+5~O7*mJd)v<}o`+Qi=D*jNFT>3E)1ooIyl^jk1(u_S58gP3KMqb0 zzx43*0jwXeG=*;a*{rOISI>@)4yW-1ncjF7-i9Cc@RyIVRr1cZ+uFPO{4we@EBlj|i} zymwrDBP%)XKM&nCX#D))=@D#o&TTyc3*h+u>rXD9bx%*E_D>ET9G$^0FSus|oqZUu z{j>Xixx4=4&U5a`>AkbPt*~*O!MYXyZ|UwIpPj$xx3eFCEpg!m{$n8Uvu~fCCt371 zuKRB5je~bri))Cm49{FAA zc~nHy&QG}XWget!*+}?H?uj=Z-9PrL;jnxmY*J^#{dg{)#<;M16xw-wF?cHoH&|lrpxu+gHgq`c=8{ynGoi%p9Cmk(b zcy+nGxx5OkY~0vhT!DYCUb(^v!{JX5hDGa6_{9Cz!u`n9{Ckg%ZhrCP`1}~oiNA1k zdJ{$ww)Xv#u&3XQG7blw-A}r${d@Nh&d!!g_w4c+I62;VB{x!G&9il7bA+|OzHl$* zRvWB6@K$sbOPe!TkAl&;eEL-fLcg->d1As^?0&Qm)>XH0=Zd$!b}ihTogW?^y}K$u z?vH-pC+OhS`T0K=?#C_ss1}-Px9!*!VfRVD0eYow3mo)+vhBdo7OJ`yITKZ$m&dQ?M85rawg+)gS-M+g21#A{W|CZi`v}2XT zj(PUX*O9-ofDI+ARkPbUR#CoCIOP1tefCe>GsrzSZ4Q^&??SlmH)Q0K?hog4 z><{NhmHpvo{--tehbC*r{Q)?eX@6*PW8NQBe%v4Zj&XnZzi@wO^0UJJ&}=VE(bm+4 ze$}^y?=QB654#WDd1`re`+xnbo5jA6lz7wqAGkp@xhpn^=G&TV@&?iTavJ517z~|_ z`|QH~A?^?@!hDBly?GNvk#~spi(p?RNjJAT7}bVp7SI3-{U3$qN!{W=SO*yXvuHYO#-B;Zxa0uyN7v~0M07A1hk*DYi&Jse{JFZ z?6fbzY3tpi)ARkeP7e-`At11S{%}8fp?>`Io43CFGY9|97ry-R@`r8j?WEl<_QRrJ z-*nGUtVEX^Kf8P?odG`j>I?DUw7BDvSNUHrsF}LCTR%E2+==@Tqq9stz8!i7+|igx zlHc5YIy?5fHT{WV`2#29ohO1-3{J$4PV^q09PPjT@cd@-p{K(Jn--OGjJn5t=cnM5 z`7zOW^2n#w*V!DumWCv;9wGs$Wy1Gn?fm5r$J5#EsaLPTxLvuk2|I%OZx?A0V#~`I zec0h+lSYBj->JxRrzVOtY>VCJV`P8^BzCt$6vB^VZHoG1{#V=6!)7_>2DXk3P-H3t=-iQ5LY0T#GKYYV|WRe1`2N0~i zImKy_4%3asY}|#Q!K5G0y!v#W*Bjs20?mGoQ3%nb=q~s~`e^y;vq7n<3qfT%xxb!= zaq7S_N1f9^M#1gq{oE;;^Cw>)m=5Rbe9(sf`OP}ca|~SEgisFz_ToUA)dWn6 zNliZV>Qh-w7FXPn`$Uz<3ficXS+vPy$yL6#Z%u!5BQ3a0W=rPs=L)*FQ#E;#IVzrQTy>8 z(;L*Lno6q*tm^s7D(&7#n;CfL-#YRt1!35@2G*v*GA()6UVSobatr2*-IR@nD0ynM z{8(7X6{_wRQrIGJx|~AvX}2TXr+rb3Ie#;-)U=N2rU_XeF=TRZczE;udRMp{YU_xD znASAapt;ctx}|kM{qE*1>?Bxb!F=?)j=>9kO z)*p=O`Vn*gJ|u8VdSN?)!}>dVScgcWcRKV>BamMj&EjOR?p@Sr{e5W~@1bcNZEzy% zv?>X8{v+e%JMH-XbbUwa|bIe{Eko@M&z{E~fw8@;s4|0xS8pZB0z88+!cE7tw zp0?@q4wEJ3weG|oWcoXUvktlJv9ZNZ@CV@pUmohE<@<8lV&<}Z?luR&(a z{l>z5|Fk~7{nE|*C!jTNv1E@g_}u!WsyA}x$#wV9YD&^_m)Zw-PvOO1sd&yE1SlH1O_-2HtH?QzqMmx3`Ow+h0FLk_u)GqLYFaQ zJMMz!;&mFI_d-10a3qN4cv{oWAYIqz>~g*o_D6S~Z^%U_B~ed*CKm+Z1N&*({9_X~ZHW>b0i z`Y-AP?qW<=GNjG|fx+gB z9_5r7!8=ULvO|Fvq-dKZr?N*t(yPMjkf9>P3r}{0WZ#;64JJ~VGj}d&f~1m(D>@pu zkyNMMA-88+dLS=jC(q4;^YfFN_x8gdAx4<=@%xkgdi~Xp#t_x!ji*y;_ec))ru%Q; zx|vGxJ!av6@3NNb{Uh1p{*?Qk!)?j+jOvueP)<( zn(0@WvAYBPDHnz!UEKX=3-`HUK6wt{g>ZW3pFaPSYzcF}x#50#nA5KW#Sr)V^z88! z^=xE7AdXi_C-2PuEgz{~dNU*3J&f+#kiqWd^761;U`?3Tr+GxWv#Ji^0yLo4CSkdM zwQw(u%4Kp9tR&8z)u#Ie3g8du>6)!{>=;H;4;H;`AnKuV*cuqV;@k(9rzRHy$<3H3 zxj$K?s0)}JtPk>-hK>aG+Rm^Jj7(_VQ()51))~?L%^V6Ux-Hxam2O|4TKy;aT9e>S zIS40{z^om*?_sH3bZVT2m+tp#y3bt@>nQSFe+&daWxSi`9ca=Qwq!e4>j=Gce_Ye_ z!Xcp#`Q4NZ-5-^f>e;n_1uwu@uUg8?Tf`RTaapusqGzWFSJlN*=?sU4^;$wR<24U^Otk+u1 zEO*oWl|>5U3XKDen#>+us4=Toe;7}Zy#U`5+4!KNTq1tnIFk9huLBCU>HdCC!AL|s zhGk1pb+jw8-?O9Q5XR&a5!`Zrft>Aw3bPPWnO_nT1ViDAkkV2)k6{8!>Q$)Da5GnP zNSk#@KxEt12>hdkWIVLJoGM48B$W#F&Wza0Dzy7r;iBDhFUhRl-rxP^ohwv8Uh*dR zZE>j$QfE#LVkI{Z{w*hh*H~~QfbT}kCUvX%q2!JD^!{7Ws2 z7Gqc4cNZxUB8@;}uL%MOqn8qS#7EN}fP=!ekljxgLj?7u?mS1m=ug96UV*(vER%Uj zyXi|UB$d=<0o}i{0snfLcn6JL77#v&cO+7`GTkNCe@HnS>76LAl=zf>&YuRBJrki! z1WhknG$pdJ{j^`ww}V8!B}XZ!A#8pzpRTha4pb=EbnnW;Rb`9U#lN~Lmg3KZagUFQ zRY}T08SmVZt4RKECWU-Y8XdJK2{q04{gGxOnl&Mk*#mxec z3n=+~Qq0z*q$;RuFa$irh)jx^^QSLGF@<@c%l0A^lh_7A43lE+)f6*X1o~dzTo>Y+ zFs+mg#oS|xsmny_n|-ZpDdwI*G56|)X^ zxBF*`4eaCYV|P9hHZZ7E1ApR@jr?PJZK$IV(xv=GIgI#(eKj5wXl2rri&gMaXY$tW ziv^P(buZjmMkXVJUWxo*PbEzK-MILd4b-`<653AXTL(R#`-*$LH@&Hi3x&9w?fo@g}W_s z`cCo;UDh8B-V`|{xrag3*u4OLWvf2$8M5pQ}hX0rKr%CmnY{;`3>#VrhB@U2but~aDZGci4hNe6%FV(Iqde1a1 zS?Dh?s<*x1M|rnvLtIx}RBy^eb0yq%$=4Or=C}!YAL7&-!~<6KJeR<}UTdDMFQ&yy zTZHQDvlYhFez`ahc6a&V>js@zo}e+F}qG?DRHR$I5f zCAU^+cgmvDbR8T?1?1F7F3^-CtA zg%O7*^30Ju3AZ~G>A@mf$ho*n+QdSa_wWEwCL{fejgNept`YSsVm~X_Mwmyiapaq^ zW!vRtiQ4+}?@5`3w$&9VQe9PUWTf&?!!={EB>5qOTH>$6_=rGnlBirk0$ZIiQ)#~^ zY5SId_8a9Y6%?4r*bKj7X`Dp<3@TeY|WNzp0?4-)Uh6>adIoMFb zQ;5cwy!Lgk<*w&@DHKuVZSJA|KrUzJr;m>gp*BCnw?mPBjnT4Cs|?2P6nQb}=G~IH zqlE1h8I(s>bTRKW9huBq7Tr%!9*Fy*NYSQ-uk$wZbux2%e53cLP#QpTf2-kyzfFCe zVrfo7th+iVzME2_?q0tymQkddKPcaX9T$POb-bOU#uMFXEkN`#x3T8*{X=R@#lCtO zgG0jOf{J*%W@JToYvyPs+zE%V>I&BO}3=>Rp-eUp&Y z{P!MLmHhYKXQ3AFXJqnT|0`CQcVNQX1fb==Pdu^w_Z4%1rju z9-;_CVDq2@=e_Pm<^9{KL22~V%}w)v3DNw#^r@;NR4 zJrG?p|9xs>rn~oA{`*9L=|0=K8d=g|v|zbs885sfJW@jOAgOYry`kZbZUZH!62i&4CYW=Xi}S)Tvi8q7wO)H>vx|rR2b~ z;Sou0bDk%)9C%I}6|BgCPogSw;CT~m?dljlAvJyrPTH};vyUgF8wyyq5^nAnHz+t0 zdDMHRKqUAZqKwntI%S;T+1~x3pJ~jO+Whf*jog+=QWErQu zo83E<7UueT#XSp69%Ugh;IX5DqKuROpwvK1KgPAJrP!{}z;+oY;JaHt%)ir;URGgP zbXm2ey_*+Z#tD4An2#A20#7lP`e7RY*>m!HWfaL&fyNfjamOT#6i7nYZ3@S6#*_|Z zi5mAFQOIfUW~-1BSO<^q+8WM$$U;uj$buDHE#y=QmxCf%D^Y?67_%VqnE50TKgJ~Y zH$2k{In}w$9b13E-8&@kn#LKp3{mjeM_N>J+Vi#zW+2ssUOKpo3Qw|iXYTFpq${Pl9X@?mvP;PT#zDabc`~xNx_# zdpMIb$FP1l*Kqoz+gV=cHJq@yUH}VXoYbahU6(&{`JS6(j%r;O^s0RIC z*bV6p(7Jk2x&!wdf2f==%DFXNrm{wNt{Ww)dlaru#hKiHPVY!%iE&?nO^*%nBB8#G z9|?tVqp+ro*U^=wpcn_h+M?veXcsjU|p zRQh$Ay_Yqxh$vy-)4>Fc>K?Bs9F`H*k5NDl9hR(@Obx`528`DYeE}S=xfAr=qZ4m+`bQT#Ry0Ao_m%pTiF0dKVPQTQbba)$kb6Md-Uap z@1bW%J14&K3b)}Z-Xnlt;XQ(emz(gPCigqMyMY-T3tNx=UE%9$mFL%cNd|#Q;1-+Gg69q0ccGVA;rLD?>r^+-f2>7-mBVu_0Jh~U6W$G zdsIphXbLHU2u7ut!HglS>xbvfgOj%bP7eO7h-zes+OSpC1=P zXXCu?@9}QAzp+uc%lj&pf+Mqtb7w{6!TrE4Et^dK<-*+@RFr9XzCHL=NM>JYhF@Ig z|1qr&L6=JAIJAqrNWReggp)F^pud{O<^QG$pR@vYj#8b!4&PSY)M1 z9+wNsR)?D~-=nP~nJfxKRh{T*#4Gr9)=_r=ijJDK&Rnc4FZETGzxl*igh`jX% z%9yb3eu-XzCM4llH`z3*IHTK9eKzWN+;Z=e8Xbg%G`7O^*f|%D9ul(EAz#@hWr25clrg7B?%kcQL8^QWk%mYG>P z^QPQL%1z{pI5@?5O)p8qGi7$C=Xt?SKd|%tS}E$O#fp*IGE+!`aB4-yq-94?%`Bp8 z5-&AyRhzNT%7C>3sGB&63ygCweB%RG?6 zq?&V}@$H)**rrf&jgUn3*oc~_js#F`Er8Kwk02n!6V+qWVaQ~MrbKm<1(^AQ({IXR ziB0?9kVsU=Cpl$2{M$9JYU|i8Pm7vNsb;E_K$Okh+G5 zbptv@OD>tYn%2)0Z!&+zN-hn2wIGKDlXcy1oZgpSDPXy`mRu4v3l~@kPE1QKiLY&%jwP4m=QM$5@|5YS+3SC; zK$u;{?$VM=l1-Ch+aQNU)2$2lOHy))e-<6oQ#2^$;mlQXi9imkR54g1jQL>Z=WC(dT zDSL)@Jwp!5PiIDUjm-x&1b5%Bdw8s-+ZiIFbes!+Y;6IplD``>hP1Qasx8cMcBTKR#P}a7H*L1@Pe2 zaEK}7a3mBD3Jl{#%<3%SI+M5jcISbFId-re3@Cjcz+VXDaOBb>T;VOw5+;#dc&x^I z$>l)Ou7DgSPiBQ2j_tAlBZl6CjpKWXI#&s05Xj-!WcHt9xH>c!)Bc1ZHANN8NW+;Q z0x=WmPHXNZKn_QG5TAPMh=YQbn!rjV)inVjhgstc5_EOwN<>R$tusK-lJ$U(dU*c9 zqoaUuT^oPMH5Nk}1e4%E(^Wl2nL`e@uG2(&wSEtNRiNVr)ohIDe!7GuX+0=xI08AWk*z=u$5(@nkj%yDP<4KKe0X%Y ze|G=)UYJ?;-pL8F8i{EPp(sYm%x=8&rv!31`V}|5m^1b;!R)2t@kfpGgVXbLyeX%s zadkn+VbT0d>kA?#i}vSPVF=_fZ$GUu!aEmqD+y0}=h8zCIn3L=xiaK%YzOjZFb8S% zbsHgvvE8ZOr?9hQr%1-mG@m>hX~*oWmIyfpcrhjl+YjPBpc-m)2v&9 z<~z!S(Rm@v!bv>&>Wa>A8s|A6ha&|w5iMvUJju|xlAVDgL<%7}gdC2og)TQsW2^!> z92*b0dw4i0(}(P}BoK0#wymx}k!oFTNg5pi68v8RIULFRArp|pLZWg732fO(W;KIa zG{UI$EeXhB!qqKFIIxu{h$4#(zIP~T({te-XxR}#cRTtE&-MuG&Ssiks<{Wdx( z7R^%U1ag?LA|Qv!lRdeF1Cz*-`KU=ChhyUfzck3$TR9yFXbSwYK+<2$uzfVOt>IsZfBrGkcmejkc#a{lqWUdk7+w?biJ`=B_b5XSt z_i1G&avG~vT{mT(#m;Fq;*~2-%YW|+nAK()kZ{R_M-!%Jk7{2j!E~K21r_=4-A^@N{ z9#@t8_uglE{(Ju`pZ}iM;{iP~X^YeH-zT0}4zN8PT@5IgOd;lqP z;OY16^u%78GKC!GLQUi^sE89-&En)6)g#kI!RUHa%YmoDN$GtpXMD7{^6tz|OB8aL z%O|F`NXTLSQtXU8EX%I?X^IECyWE%19YX$fs3))Hzz4z9bKt|{nAM?6d^87sOUr?0 z!y}U1=1i<3ki+Vjfcy8!G(1HPd=gce1Fx~q6DxZg@>JCVSEbVoIShQJ8w#?tLVK=| zl<|%sG7)#OppJ@!o6gl_C2a4-ElALCcOS^qcm41Dl!q2$HC*6*X24!@Nhi;)w3&;7 zI$m_bK$U$Ui4g(%V@A~BEY=lvXvBO?tu#wId1>^j_xN{ORsm{#sey|5k~FYSR=-WD z)%a}kr)Xf;jI=N(wWzoiIbuH&)ZsXA?UGIa#49yWk#*BlwDfTf4J>R!F_p-rNDE6X zRV1jxQHt%7PGl9V=S;K$c>bt|DVbp*@Dv5=aM%OLwv*pe%KU&oQ*X#eo|0p#z# z7?KEbGF`T3OtoYXbvOzG!h;-(c4AEFK%UdKjSu2*m@*K0*DB`(1_GcC=f2o>H&M>1 zj^~@!m)5XR?_G7^i*inhrdG}=;VkoVPU#C}oGl~R47p|j64YTFXJ5eQjmLL)QVq%3 zMp_he5~#yuLRDSCO1TtY@ky3GQc#DVi4K^;b#&{45g z#3P7_SmuqO4&Ms!FPDZoOvtsOK_R@57NZ7OA#JV~!y2eC%}|FajYi~+2C%FgmoXuK ziHt6=)rEkf4jVb-rJGjgaV6AYBad2yrGq+bYag)UO^g|J*w{vlQl<62g-uFZ>=`|iR4M8({kkrTX-T}59lcXMX#h)a zts{P!-W&YHN2aWk!=^z=gcNm{cA6dOSI|(UK#jEn#W}26km))_9j4sN!#-}$I7AtB zn0-jA)0|Rxspiqk+BN<7Wg0PGs|HWHp0!SCB&}>M&za zL{1gGC1*<81Z&oZrl`Y&k?3CLSxWkStrRYzD%cD=If6P|l0?@w_8rabj5=I?7^6x3 zm_u6Rx$m3Z(#s1;4(XUBF<`th)Zwy2aJW$1{F${FL`q<{+V?95*y7gzd+OpU!bh0p zmsv}}35>QG+m9IFAL$ zhIm)=&w^{*CRuo~XFY~+&2@s&JFy3LUi(VOIO=e2cuE#SKcDYWphvFWmA%CNbu5lb zk1hapclu zwC3*E^ehh|!ETB^zO&C%4=X*=OnPopgkuEe2{SVwU){eg=ZA<2-Wl9{Zc^8vo$GRev^xhOndq~^?a>!5%WPkV&y`Sz&C=9-TNM}K zY6?b=9No<>qJP3|W$)*MqCxr6(Pt49cGL*T!~Dz8m_G-1TiO8PkV!#Sp;>Me^pobCRc0dxxw|gMNx-2zb5rY)M4>4 zRqsB|p;cikzQcAJ7(pEtOj@*=I2Y|)wnkKOtA&*!IvKcIjc9MvgSgx9M1q}2QHPlf zB2jdTD299C5h>Zl8P7c;J=f!Pf;z16l~TUb!MoARfoq-{j-U?fWKke^bCwzUUUkSV zeDW+gi&n4L9ycC>I?R|9-bQ(fr=vq|6>A)}GU=z}VkI4&|HSj#De5reIr|(`aYnbJ z`fSwkNKuCw3u#=0>v7I&&rqfSi~A@7Ct2!vKZLvoQfli|N(yfpEG=MWbgH~yzv8MKm z9Q021FilZUZOEND<<+L=dBILUu*2818|RRZg=f`DY;L+roRDC?X1>G*EIZOlQBQ5O z*+|_~G_YYa_GO}np4UoIj~(JE>SX`#T-R7h9ZeP@Nl{Ncb~F%E)bj_WJ!lc~EQ*(n zlTrh#De8%Dg9bK($+!rU6!qBnMW?6})L}wVpbp0^ASbDlHY z<|K8oJ7otkVJ#1-Z6hLS>ameZnmV#FN&(Z<37cTP!v661-udyvll|k9LnxhkmQ2}M z)6^MFk*3aioYU0V7bTb2eN$p}>+*s_)Zs`pWIa!5>XxzYxCWx3npD$aqIx6;Ruxvu zEIre!pblfplcz2PQkYb8qB_1!Bas%tl0@~`2rCMnifbW(v2i1b>ahv%Aac~<_&ZNj zr@SCV(>M{-VI=8I(AU9tDf;8xnL@JbCaOo8A>rA7Na<%lu zbj$G6fEYoiF^wJMk^&@l*7Kq`k4SUXB|8T(L}e3L+6f0HDXqF>=4x6$R}$w=if)>T zh3Ve2_M*nDA^=i#N$|ux0=5xQ&SC=!YYI-qu95|NL8~r_KFIT0s?s7}%T<@e@AcJ9 zIp?E6FwkfHpl6act-2)n7XF1ww=YnwNYy3ITf_Tc7Mt#bSPOMnFv)bE>RQX|GKxAZ zX!bp1ONu%yzP4#Pq7KW?OFzZ{qnV%~zso$=g5I!9{3F@%g{6GL}02G#?4p-=+P*Q-1zZjF{6Ra(N zm|`WX$FQdTiB??V6hsg!Rn_7zMK*H9CG{!frNVx@a>XSLk(gFo;xuy{OBZSkWa$s( zDY6$FrKrQ4i!?kUl+{Ub98fTZI$WWR3YM-5@E3hBtF2urnw3{r7N_d%%3%U79ik3L z@(gvD>{-=%54$CJ&!*eYk-XUK1)>g93KDggK2uPK<6{Zuy!+aPfDLEU&b^(l{znd& zQ-eu;y#OmD2#DFFZZ$@VS9fit#@%-px!8T|U?huLk9*C8k@M*O@p-;h!9QwmGS-^v z1a&wzpBFs_@md#nt^3{svH|~kWEt<1p~v)1iaIRm(%SMV7mRm3qYleYC)D9Q^;<$9 zSOu?m*08Yrx8x`VH3WC!g`y7g9?C7=reRFP5YB#P|Ka2FM~~0gl&MmtGz%#JX;S2w6dBJe zof-YXq?j4#ud`JTxqT5)O7v2g2bZUqB+o@vOhg^lIc^G_!A#`!gP9HliK9@DOG6zl zDH{}XudbL8bvXBUwG|Vt;>(RoK^-o5N7Uis5vf-RuG;nBhbiiCBoq${4CY15>O6Tf z&`ZJOEj!qZpU57cIN%GXzi_Z(f;t>2dDt+#A6OccVc+Tlwm!-ir%1a3b(lPv6?Hha z%L0rk8?g#hE&z3y zHO^2$Yj-9!ut#QZ0PJ15FvnV-2NRT_4%5~XBB)B4qYl&0ZgSLN_Go3);n+$8by)XZ zc|*ki!=%};-Zn^nbDbvItCvbO8zXh=vsMsYMI@%E!&+x6sKazbN#^2osG{hf6QpoX zkkv>OsKbqy_5c+1LnlY??fWfj|G~l8J8%PTWA_=ieROh|{=V4#RAHKb$USxE+A^9U zi?r@>*kD*CpFMl~^!Vf**8CO7_H=iT-a0-(0>H)5{rks{&X9{M1a+8-#~(FjMjaN- z&$PZE>ab{ko)w0m4)gZY8Y5f=VPQ_flV|kwkfRRsc2`Foj_pAH4CWxQ)6mEw?htht z+no}rgq{OoOtIvfcZ zP=}LP>AFMGk^*1A1hUX`;%>XYyvQ|jW9usFu)<>tN4R@+W7~h75Qz!uaHNx5$d`vY zED59YLYRdDby$8EeHU%JkYLrOu@d;i0B0Z!PcT!}OWF{88* zb(nNrpbitRu&OWlac|%hnhrov+~Nn~mG0P*itC#^o~FH}m3!=qvxI;{JD{c*M$`)M3)ulho3;w+yXaWEIJy$Lqx z*`tSNM>`_zDhaXf>WrwvEd^p;V30?S3E}Wn{6QIYxPI)d#mW4!w~9y%neFM)WYm^q zBU}#Zuq2B1RehB5{b2cVJ3!RQ%ZsToO8$Geo6JJRa|S{P>Tv8&Dx*|SV4OOd7Sy^* z*izm@|K97Ryvyj0n*TmE)nJ@fW~!aWEUAQ%XU#0+j;;ko9Uc%ctIf0)M=PNY+d2Kp zIO?#y@etQdi_?%c!RnS1*y*m`t?v|auo(O>MIE-Yp8fYl{(DzRHIbJ2@7-nw^55~s zr78b?BCY1X_qeL$zxO_4sKWz;U_~7kP4?2LCMBV$!=izD4sd51i0ZduGW;aQG8wp9 z6E6rZbe~*0k(uAXs(^+$*H>Sb&g;o;2 zEmx~G^WOvGdj9+PC47Q9ESO}v-|#ku0fH5tRf8~Zb(UNBY~fe)-zSlD^WUe}GU~AC zFtazNN)fu+_JkpF+0(|iu&dZzHUB;K(xlj=bko3Cw0frHzXy(@sn4xRJ(@RO9iE^L z^WlnA(bBItDPJyg;7hJ?h*&O7*U?N-hgJ4`;N|UuM@PxXo=YqQb+|$o1tS7NX~7Sm z7-g0+6;9%w*T!#2CGLA^QB`x`1BIB{B2kCc&d9@Z>_IwW-fA%vZ^*MO`G73kN}N}7 z-~-zb|Cmf4U8pguLw`7$15Z(hRk`eoB)2)wlLU3RLK_vV$bnCy>S)&tIq+NHc}T^Z zfX_5Kva~`%9jkEeTr`5JE9co4bb@>H7gy;?6GYIesx@$SL>$4ATOUj6X) zV)=xRu!p^Tt&&dAGTX$9%axZ0L!87^lyvf<69y`)gapq?Vnl%cP-Kk*Xp#0TySgl> z<2xdDX4#Q3E}MICseuinvM-altVvl^3=n&n1}f(v3v+WSuA+fe)ZwVLqNEdas+{eu zOQFppOt;DskkgM(r3SW3I{Aju#%!Y6)WC-hFF1v1*)Hj{ez=C94##ov6b0&V*aOJ6 z)8GtcVW~#bF6e})!;v+J$xV@AwmI=~P7_crvSHhM5Op|`YL#<>9d;emVIpTR1uwCj zQ^F9|$~hIN!yPWyuh*4_h@cMRIQzVbHwfzRq=J||oK-M4({oEjD?=1>3hiPdRq3-{ zxtLREc`|D;9VN3(Kzd`Rr%WW%tV!#QEap@}9Yz8uEDIf3%qc(sE5)3`d?-64(ynb2 z@sz~jgRs;bn)HO1RZ7MJ_R>ZZQHN1dUjIH%P>17_z>TCTk03Ux_Z^@jfKWoMY8o2QbXA zSPe>n*)b5aK79CK|K#E6gM<5!YJ!7ET*W#Mseg=)eK_HSQR@3@RVzSb6}EoDvW+Z79cCO0cYxN_tm0-yO(RKBhY3r{xiu~DwsF?| z1Z$xVQ}&o@vn{7_6us`pU&f!T-H~crR3yJnQHLq721Q2HVfGi6gAiFkTtzutrpK^W#!I1(6c!u=ah-rfb|4k^3VH!8w9DOqv(rBh2#4 zti_=uH=g7pD4x{dDjdSY{K3r5e&FaA$vxamFNY;CP?Dd;h>`Lnp4Iv+CS8JyE{%a0 zixmxM5JMd%LzKdEYC(kDe~=CF@8_TM$5DMcyIvfuBndIAzD91AdkSDDM&`VqYje?1a&xzpl1gP^(*uuF8~-AE;U_9G5BxhkRm}HF4for3fnvg zv^+yp>U8}Gord{AAw>|us1y-(cp3-=br{EOZ?tu(aKK<&;p{iX5OsLsFElOvt6o%* zCb?wke+{U^c`36^A4DBSk-(jx*WpC_EVHaYt%RZlCPozvQHLjbF(rdJ1vC$_or45**vut&57ze0b@(sP zh~0NLT!P6p^Eu!m{0y5YIZcWl*O=cSnkGceyIJWGOI9uGZkH8M%t`T%pbndblI@gP z#FDC>xDp$lo+i*R>$vV|)biUY>M(DDDSV-TGbg`U1%H>K z4s(7@>W!$w;$y1blUot|qBY-N3&otbTk(vy2H6qRVcwX`kW;mpI2X-AZKhu-ZndyR zM<;_hLwmp}vO!UYnG7OPbgLDKclgH(P64nXouqF;j zVt6ugu#=#0*?X~MJC=i}v%Z38;>JTzhZ%9DV^X;dqj!xBafq(mw^ zXblWFAj$PhHYo|}a7eXE89O{Tu-y-ndjlDun6J0jK^;yX&!3{6T4uX6J506q7|j}O zq!K4$<(_OtGFfc{!B^ ziX?NAdb%!^SW6};VQqB(!8G;QNF_}jS;2wtoJ?bSG@=d@HZLtrozWC&>a52Zb(npj zj5MqZ?;IUH+CMygaJJldap7wizIl8IrMO?(MA?|Kp%HaB5`~dYY3i1-&RdmjBrs7u zk^`#>6V+?J;nHJWB(RgTZfuzcQkYb8!8?3=4b}r%c#ZXvoAKK zn#AA3kjV~BiRz|N$$Y`-S2PVIy*9iKzDv;`@6JroyvRpH9Ui2KiR!T#kW?~zHEtwT zc?L0&hXp?!Xl`FR>M*C+@)@%W;f>4rqQzEV$^+L7L0mz|Cxtr%8D=-|cn6J44e(t~ zbbDxupbp#lI9&;K*v{!dbn3Iw3u|vY=ot_0ccxLu#2QB(7A$#^7@s1j!{WV` zrX?u$RCUSzsV34~b;;gL3{mBsa6pSpLZ(%h%v?=dUy3)GKVwyw2EJlps@Vnb2Md=< z5ec4{M__<%akA24Oi{<#G0!w&ipPOgT@rne9)n6(bJZpJIrZkVRXFeKt!gr&wTchl z+->PsfUCI42P!B@2Y;p8z0*V-q!nf#t-8c{Yj_{bI;Oi8)XLY5({w}~mY-8i@5ikMhHBVdv7}X(B%8uZ=ZZ_}x6z@V=QeYdcjH&I;*#3@K$`r3#mrEL zD|AsPt;Lb}eL{VG0be>N>M-Y11hJaqn=3ByCUT4rH=$l|NP`&mJ1IuC?t-)93az-r zX@W*gW{)n^7|7Be%2Q;QS*W<3s#;Ej6x8APSWH`z z!T(wTnzZL$y7LU+nIfdZR`2gVPIdD5!G4edU^8tJ)Zxeh%eC#K&<@2^5p_6HQnwnt zqI$DRcq5W~m7)%FvZ(dA*Gw2W40V`))ZS#Q#nlPwaBMy=TMXh=mxcFotX5tAw-&kL zf}~4p%ctxa-t~++EI*wYbvQO3)DT>I7m7N}dnmW~y4ABdY56BvDnj@Yby(*Z z_f58>HX%Z2wSNM5AV(L3LQ#k3R!M?7TvDb=nbItz0HjHgXHsMwMbny{1TUouhaD+T zikX4_=3F5Wby)OLmef5(Ra*AdaZR#Uo;D%!)c33B`i~gLx6NI*T|1y%bDd z80v7OG&|}rX;+{QlP7>W3?P|ro<2UgcW{2>PcF5RI~0PBfZ1)jpCRG6V8^fkBZl6? zGvCWR$!eqs>TqN=2Nf}x7n+NwC1FU3@>gcGAb{r*yWq6uUIOZH>?e3eCF*c3++3bZ zSm~rq_j4K1hc(VnLA&U})geOEVb=OQn4ko8n6{n}LDTJc`fLJImD6{UCr2G-UrngP z4Tn{LUQB5VB=I`7a;d1pb((0e8c*;w_nF_D{68eCx9XIYCwzAATB$e-!w zAa&Psf;t?T&q;G)>x(@m89UQ_S_ySH(gKGb_Myvbb?^Bm$vFy4WuGdh|0HH5|AB>M zfo!%?e;MjPE9p!{Uh8wi40Rxt&ICK~RSy z1vL>ZjJQuk6K9Srzucf5*U};CFx3cB8lz@5mdQ*1V&fxUrXpXspOt2C$lYrdFR|B> zK-A&LcxaoW4$~)4c%v$}6^85=Tqmc#q~`d&-E*t!<7WF5ErP!WQg~qsikt4d(-sg5O-P!iWX6a2`d71 zm^|5&OE@rzEVUI;M;(rh7pTL8=jMJ3|h&riy@k)%6{~m6+vQTl|;M{Dbn0sSWhLIgX9gf0CMK~JCy=N2< z&5|Y+`p~V|E}3gYn>Q)yu$}SI{P)OrX+nCB5i=pHM>j41eQLHP;PKM!1H({u$3qi) zbloCZyUXcU#!-ju!xi)_O{ga7uxQf{O-^9v3+^UA}#aZ zyCbto{(IDSlZ@2-_a0Z3{P*5xdj5O=tKt0jApvOl?^6RT2N-lQr2A&3P{OHqhF#zG_F>u&!0B)N<_ zEI%*v--i>|H6>kPo-ov5$!0JAJ?ckul*YCGu9p8ENJUeh7jLTDwzUABp8rlzhq)Xg zRkZXg&Nf6HR=M$37Ol^^VsyXmMFgaeM3pHkS z=nqG8;3?{`Dwj(_9j?$u1uJsklc+k{m2%+oV#H@*XP1p9ZA2XgKGO{aSy~~X4p+Kj z%EBj;o?yO)OZ6T^uGl=#QTfdN!^h{39-r@rrQmFN%{_T10t@$wy+F(6dQU|`9r&1S z;)*qDp*a+0|u19p4eL zGs}*QaoH4Oseuink{w)H%^IZFv^y0Iyu^}D9#yxmtBMA;OFBWbQUeuPn?yH@;$){tBhueyypSKy=)!Pl3sKD)iSymH5RqcHrl z@JBsN$qWmDFCryShr=E~ww?T5$=RF*oe*_63J(IL9E)~hOzFVr-XqF6B@Eu>)>PJs zMEgSDfss)PqMTDg04^n%#Vh?RYDYV zidEvKp%Wbq+(@eO2x99gq7L6i+}=0nv3AZYJp^@_kTaKYsyiN<;IWicEBFSd^eeLZ zv?Hj)v>kRPfUHD<=}QAx4ctE3+HN`H?a?bpY{6nRQ*m8Dj&2H-g|1fT(E|$8O`r-| zOi_nvzZ+5<2wL6FMVoBB_uAdtID?OaK?SEZ#I9=902f6aHi{#!c?osc_?;%kjHO|0 zLq@5r3e#DECJ|HAVH-cwl6Y%u&_R9~b=XGcrk}@Y&=)P_SlXUVuPEv;{hA%=iCJxn zNsVH(Qp)A&sv3~GPEm(x8}hJ^yB5yNj5^Fdqz)Fgn3~0|AHPf**a9Y@sKbn7;SSKc zdQrLqR~>(-RDR{$nrwL6eAQ3?YoQJkiE&?n;;phW+4TYlOzwtHgrN>oUJZ(jsKe|- zs>oBBC2PNQ1#WTQ)Z1TKICS0S=~p-r6uoWw1`sXYI5Y1O)L|kiHLf&Q0EM6qmn6}( zjeSoC6M{Niez)X~N1cA+>gat37 z4%a-!<(c^TPk-(w*?AK-mtr`)&+c_rVLOhKrsN^2F=VH}`%{RmyRNs2&IND)Mk*|hZv$^f8S&^N504ii>JTbH5&2C2f8 zqO#u)$v2-M;HKqy*Z!ZswCE4Y{?x7-l(XHLO6 zw-|ytJoO`Iu~2K89SeElO#{Dy|M>~hH~yRXAffi7GPTM*P?K4$Z{2tP9{gWx2LcI##f$dR+_x2M&z+9MDz zK`o}J!<A$p@}b{1)0U7?hsDR$ zox(U4ATm*YeH9H>eK-_#Sa2(u>XGhD98Y2Y2McmdmM8*u zC)&B{dJhNk-886ze@07sQ+`eAji|%D@o4$}%uDXw6|&_h>abwaqRpgqyM53aPS7n@ z{E7&C)WIkn9}Uxwp$;<{M55?cD-WlLI;?#eeT-dq5H3#4XTs`ZmBJYCG7crI6Vzd? zQ*a0!j(bI^vt?H~-I^J?Y#*K%d4m0Ay+bB=*!B~#tN^TWv9JZL4e87Iz;w|Na za7&*36sol6K2p?S#xor2CYuI}>=1^c+fjWM9gjT)Bl*yXI?PxQPP`Oq>bMsN)QX`F z>!eYw@O}u914xNfb|4P~@4XE+j$Phh`IP&%O9|;AIm~;XJ`aQw>hP2?mQaWFe&hkS zaY8X)UtR}wIDKr+LT_I;gFKX!6`fk{nWc>s_0&XZkj|8%o(RaDS@t2ju=5Ah*z9Ae zSo)|Mj%{iGV$Q0}e2EQMb_CVT!cue7B`NBWdc$VyQ=7|LEH#kZiORvq!rYujBTZ4y zqN`kzNYAAPDza|cgO)zdp@BtoS!f_J{wyptHSl4+n-`_zlZh`fjhUbh6N&b=KpYrp~@7xy+(7Q$8zw07H#cT@qwkVFQ2PDx%8_@NEb|RYN|c z>XMyL9-CT=)0I$%?VM`il;sQBuRAHoVsAX?+3K;Tkd`$LK`i+5)u>W{L_BI~S_)95 zs!R4yRpEUu#&zt?$b9wzE$XaQlU!-lB{Nsk*0+*4cT#lIOibjq&n^H`bxH8VJOTrB zi!)d?Jzb!+nnp}><^KuRq7i+N9)nbMNxYV;E{WgkbE5?|EE=h}?#;ceMZ0L#B|#GG z+JnR8_7@M2PR@_dKiH{syJzdiwI^9k1AV5b!<;weL_EBi=}w5XP=^JRO!pOEgn2p` z?!t>%&^SvkDeAD`*rw@-IxIggJ!J!Z9Ck?r6x6C>{~BhpN;XZ3O}rNaooEXF%-v8o zujruWSx$3h#}m|HE{8}Jz3Rf44`zNw9ag_J#;-8Ez{}kO5@Z5~Rhu6OU=3Kztm0CI zE(#?Di1>>!gk|C$y#Xwe&s4Nay9?f7|B9>U4lByTN2@*v~iH+mN`*}E3{F;(scp;B8ZB36$5&M1jmh& zFApEru8w68+&xQzIvmN*jXE5gy+G7qNW`gRCUa!|LLpuWa!l-2 zrb}@&Q!W_qdPW_VpH3K>t-ccwidVejpsYX-C`^KCP(yI-T`1}>@1fk{>xt%TqYg{L zsEk4k5_MR97!{JQx2>_DdYBg%0(c-t7ld+2sKX^?s+1|sLJB~d6j`GqTBI>Z{rMP4__ZnHQ3)wE|C?=u~>l||%yEz2Iipf!jb(#1|CaWQ|6FBTb zF;&!IDS*0SPK>Xsm;`k=k_H^Zd+#i-ei{Cofl>4l&Uv&Lsa1x3_h*7`h{pagZ8 zww@3{(^WnCOrQ?4M;vvSJsQLUJ$0!xKy2kwQHSd^(O%U(W_~;J|ImxXtpolB$*OKp z&BlmMHNi>Rss57<>Fi+wuAmOn5hZzJ)1iu@e+ty$M$Zs+xbe~+EMT+41GagX4GNP{7mZ$q7IAp=UHJ0>M(CVtuewo7Z&CuJn5ZF52tme^5wMBz)i;lVctr0 z-3I$WY-*4{gE>g6BldnvZml_KP8vhu$20K-6Juck-!X`cGn3 z@*h}eiawbqo9>qvE?p>ygeao#@@e`vsS|A7$`|61-%O?D0uK&KhB_P@P*H~!9xnuS zIMPW+HGw*8bRoAq8GheL5X2O9SQ5rF;tM>aBv6OtchPszwhIYfY#OTtKUEUc;YcB* zeocm|&_sBW(Q_p`*nFEKie8GS!?Cr{Wl|cWcGJF0UTUEZ$HqhM9u_Yt+pyP?K-6K{ zHb)(%PYCL8{5k8LRc>S?9ik4$*8GqO)ZzFfNmQ;NK^UvhM$}=_L4i60DAL>)#Z z2OI(^J_TvyLNW#FFd=$-6$-^*tH{1-u-R(8%E+050)RdBbJuP{FYPl-^;R0a|S{P>Tv8&D#B4uV4OOd7F0`` z)KrY9m;XL=Zx~7*eTLh>2F5LwnFji!nT6ctv~XJf`(SOR+Ag&grygjSx~Ucvb=a;7 zUl~Uowl{tj$}^E#NofUdA`pxAeJ59|)fDovw$%Lh?g)bEIvtCO{P*ssnn*M1u;^TC zl7{mCraWDesb|+IHUGWWxk~(3#b>(T$dX=9?kVXi6B)aTZu9#``KawBy+H+)5F zk@y#fF;YcKUnEB414x+zA3Dpp##Ml)h4pf(x3nC1+Pn|EPT zEc?p!L3zP~4Ub52oAW&R_j?LPipdu_@PSD+2VP?zCBi3P#{!5Y4-j=2Mk?JW8-%%Lqw)wG2))7id{f2U=zZtc8=} z=H?ppo3wkHe@Oj`0m-9^Ivna1Kp#kAMEszTb>)6#OIl6q)gUQ?`I;D&S1wsW9Y4%X zezM2ULcG+#hEdsXP`^z%UI4Vzz!}X*OE>!JTap5m{0_-&3Ze-ib&fhb+enLIP6Bn9oycpR<2}`IrqaSa%;gZVVry(? zc?b@VaR_4CQx@T(sKfN7#*pP$RYDy$ey52sqYfL}kX4$j3S*KgMIE;BGsUNFjSaeA zruPOvv7!!BwtIHiq)HTZn10QU^j0w!=b~yTNdU5%#z#Bo|8tsxOy->M(c|`v?_|_r z+U>gD+r-&CpjpMJI{7K;FymMp#)dC$H9BcqO>4ilD?b8GTa*&5dU1Z$xVQ}&o@ zv&FY@6dlFUrUc6nt8{lU)M5Im^cg6li>Slw!_s>-IJ_%33xbeDu*nhBVa8xzZ%K5? zsyM9?B&frL5f-6IdOBACg`f_XBr$e^qnZvT1a-Llu$?;Aclgs=dU-uekt`8Ps@GcB z*lkEj8n%9^zz9)E6lH;K{`^)fq=Z)s^Ao{z>#j(&obn`KQ2>fMOyn)XN0{X&86dG3 z`N$AB3eyI22KyKwzew^fGrb&^P$W2YWJh@-&uTsGp{T<|QfVN@VntF*8UQ<+6m?i5 zj2eVFsZ=(^zn^~=T<23azsJ(sJ%jJ;xc`I(B%} zbWMunYE^kUUj=m-r6^E`$%8_Q-ldmUcw1)>b$Du6K^-O(M_ZSo0tTtVm7*$xu@yZ8 z_bQ+Nq3P*F$`vlsB$q6`8v1^Y_6AXhQ6vI&I6n}I7MK{PC%4=X*$=l2lBw27M;O{- zAddvI#J!MWPKDVmShtp%aIurn60w%k5V})jHaWOO^RjR#1Ua?Xrn&76)IGtOrmr#) zu%@pBb(ptWKXb!f{KLbechgX>!{I&av zaSc5;C9o6JVa~5fy%BX-d`#7Q!a39xve$RvB^+0+yA*X;Flo_d;#{;&zPruTI)XaP z*ojUCbB6YS#mZ2JHKOPg;b#SZ>_gOH?aSz6?7D++;J8EW+$HE(2SV-eC=6aN>({w$;0jsH2BugFd zhme5*DYbPf@-B3$V=&2_JwzQ2$zgCggP{&j8Ds5!pyHTlQ`F(ltb;n7KAt~CJ+;hs z>B3Bu@whF1#OiHku$DpE;{M6O1MqJyuk1F`UK2kwK{)*;cjlB=o1SMEcKU&xoYz7f zPA#sG;!aZ3Q!BLu;q<(@kD?A*#7I)qW9zHhjD2bY)*?w!Pi!#_v=F;sEmbtInxdZi zAT>~twL$q;h%AS@RW+Nj)WB+rdg_RY%jUFC4g->+9vi>t6m^0+OehM};kX6lBz5wp zlDwQcW6OsH#jIi!J479hBy*B_x-OMiOC~8{t#^)1-o`iRzIYSXEdpv-C{+ z9CW^o*F4kT&K0__WgbXjQq6hY_%@&po8+0gOcK>&BWj{L5*Qmdl6oJT$iCQ=Y7&1B zLnb>U>Vw;c2NTtiq&God2j68%yvHXwWjy}0ZlZc@CPbO0p%WeTCtgfckFU!!h^?ns zb?GygjylXKwgNoteBG#%AzEw&pVmPg7G#*^%G{&@hBQTdDl@iKF+mWrvd~f{|ULRhKwNqrs~IxB7;k z;Hm18N)>}O!k7Dmsx^3%v)M5 zlr;crz3E8Br3!6SuykF3zX+ltUd4bTBEfOn+SM^4f{%})4oC6~b(qb!+9`3F>K$Es z^~vqc<&`^&O}C#Td9m3GL>;CSB%mJxssZ@*6;f zsWNF5QHLWXb*s_QBNG6M&xU;k{-tid*7AGR5!7K$7PTJtnh7I^p$_wp+IrpsVxTqE zp}L<~CUZdkLLpuWx@hcHrb_`{Q!W_qdPW_VpU#Xr9Geen2=2rSMIGinlw17Vs#uCY zE&rsdq7c4B9o9LQDnSf~m_;U>yN>Eh$r_OlcNU z0Mew$8XXz;D@BQN-INSPiy)}Ok@BRN8R)Nb8)3ja-gJmMEP5$2T`o^CNuGTo2jp$^AKWL9JhmEsYpR|%H~4&FaL zTW(wr2)$o||Jm66Gld26ard!1A6afdX~~W44fr==ENyJR;{M(u}C3RMg?fr3bmf`+=oF$u2f-Rsnk-<-1y>U4c4Go&f4_&8CH+cflqTfgfrTD*fnrN?HD%EU^u#v9sh_w{!1W)V;>afnIhBUA2KI`1wqqmMvGF5^)OvU4m8WeSy4V-<7 zUU6TSUb!&Up!u2B7epNv?6Vc7Tm%<BHgjqO= zCtqEW-?g1-ZF7-A*nS`_-;nwx6VZ|Zn^A`)J87U7_7jFf)Zy4#=rSpdQCawuB&8PW zaBMu}?&0C2Oe?b2l0ej9+BQcWrcVg!aQrzBzg2E@$Pjfnw&sUSpbp0;NuqKE3Bp)~ zHlhxb4hqy^!WE(plOOl&I+9rg@JwOMG*O4m=)$#=BSjsKtQV-mR07h}Qn|A_&e~wZ zD3ix>?)qUBsKbm&v9$n`xiR~g5Y*w=c!4@hh~8d>LNVAXvTr)1)Rqae5>V9P$a;Y~ zj9ryOt@1YaP=6pzy(;Q3VN?t^FuKZM5Y*w=_T;gE^BQwU3ENAe4##h^mujDQ%v!M@ z>KOAouIiM!(sIII8tQP-MHO{e2(j+!jHtuV=RqGl3zt>=L6?9!96Or!y30WwmPFCM z5*`;+L_{5yAGeJxOJts+4#(DG;i3x*6%*89PI1=!_o=K$D~|2+--jYN1a;WXIIYZN zN$-k$Yvv!f=oeMffS?ZB`5e!GuX0+T3s*V)%?(#1YfsJAb6IbGWgK;QKn$zZaNqKW zKo2ww&Z!JF-vlZaEcu-y0~M?J?_IfM|NT57E%V>I9ITT69<`{mR)svZEj9nW$5kc& zz4w`(|K9(~=f9gegQkvsSy1!eC!SdTyFIC0T#dJJUZ(n*5}qeUotFO|{G4V5gEJAKGX~gOBUQeAtC^qEw@;pNw794l;-zVws6t@y;qq4$kQpsoU%1mU^tNHJd z&0hZd^x8_iH}c=NwEXu#Dw_I}*_@8&dNgk>)L}l_kt$mHA~7N#K*}8Wl4}$e&j#zI zDO1#8l|3J|qS>KyiABqSr`>{MdVxC38LZ{N^H$lx0UN zmWJOeW8>jyXGa|lm-nG%X27;qcWvLca9$Y2?J`kCK^-qTVW6^-PVk&0Mnu3qYLc#B z(#a3Nc!5S)P{(&f?98$wxK3Fl+ZruJ0~e5F}VqA+ucMtr#hbRd}9mdEG9zaF#{v^svkRE2&X_DCc*@ByC^y}=Qtzk zFpe{Dx%tN9yF2}n&hyJQ4x*ToKpif9*((=wQc#DPOi3O=!rof9nA6@(mTQeM%S0g0 zEz|JHVonO`FdZ=2A(3`%8y->2NktuwM8SzHbvPDtGLg!%aOWiUpCsk=@AGiq6jRB= zlaSFPsmddWiM&J9;m=$y>M$W^4r4V?@Mv?r7}h|AX@)vXX|OW^bDgpRPF9Y~886eZ z>(MJP%&=I^RCTw9yUSojR_Adg)M29(2ZC0&b0rd6??c2&4L?~$L1RzcJnhW7;6u4i zQHPD<$g!$~I&Az-6Jy5GFt!n+RHm-w^=#KDONu&d<7ZkDZ;cJQU#9m4Ke3_?Q?`31 zUsQm9v>>ZJYArj`7su~vp{1G7gv~=nP={&XM&)GskSxbG>SS*2@awT(@zbG zjHtuxL#oJq@0E36hW!WEDe5p|u&=k|Oljq>;d!`DP=^U4(LKupC`2n-RV-@vwNe1e z5!B(5B)Yb7&Z4=!^Qm%=V7zhTua74=Bu!Rh1h71a+7+J&hAO9pgB8ni+LCivZQH zbH0fRC#9Gd0K9&g7l%AQg%m+)=mKRUtVxkvttzFs3hFRQQJ@Z!2Za>vyhbDI4-Eave*DOuV zw^ga@zM6s!bZcdgU_{5$+>sP@*en#-Tc-VQ7I9|OVKe>cEVL^`XZMrnfdqBf%G0zw zOD}PEjR8p(p3?qDhIDq(CA*eU)M4Iv=3UYJgpnqj%r?m@c^&q_l|D>IP=|RBq|Zg3 zCnm{f)M4>4^}BEfBCs#=Z*f1g_e1xXAD@4)qj$My zWUsDGzm2HFOq>l@qE%wLGjTkHouCeL7Ma4Yp`tSfrxoGFI$`r%2%| zAgIGkdW_w!SMfS7VcemHsKZ+8175~~Yn~evbyz2h0%@a6*{3CFt0rcuQi-Hz%rnIzKXl%NhX zv88btb3G2IRXAWJv|gPws#Q%&ZJkOE$j&w{vh;Wa0J4)(LVCzgOc^^oW31f|R2&m0 zG(q^Y>!1#&kLx+W-}x!-hHI|GBtbZ}+}oua$+M}6ei%>O$ZYcXhX@Ii2*{l|B{`?( z*@c~cVCVb(Zfxn}3j6Rc=B&!hm)L-1N7hdePLvc4Y}kx_0nBABmKw-Y)Y<8$r6+?o zCZb#{wj@naPoH%(5L49i2V?f2WdP>Tz-o$m;u{w_3rnp5A}Q*zaXk?{S$^xMs1wv- zLQ$X&$1NZysgpOA;>ameZnmV#FN&(Z< z37cTP775_oXK->A(y-;rm9Q!Pc@O|s!R4} zVu)%$%}wIKRry~n+)FX~Y36ELKhs4H^JlE;(!f_Nw|#C&(5g#@hUlM++ZVad1fFI2jnr$MVWkZ8Jh zK8WiLsk+1)Xm}sYV$+=v8nAH7k+P(y!-7es`vZIxiZ1Ig%!fKGIJRjzq7KW?OFzXx zw{?RqeMa3olqNupU9}4a(oo~gk z;`&D_E^!JXh?U?p`Ad~c*i zE^#txc*a6m0}94ahby#E!P0dB{vwEqcohSBg9OJ_>v?rH{ovYL;!yns;!g-fLVP1F zK_#$FKnFTJ|KJgzt_5_UXZ9aHK7aK1d_SB}&X!l*#+@q!bvX8_f;vnb^i?o91!HW( zk^7XCfQF7m7wSvv69MY8PaaFDCGhR6TDMoTt^( z^H>tp;n;kxAI>0N>p3Glt-eembEJ!}J*}EKh6k6L8nq zd{9H!{37abT@>T+*9h*)dnj{M>NIQOPk=tEnkwQ?)12(_LiiGOSm(G25TUA|$Rv2r zrBZfsbU`Q-by%OC-to2p|N8#JnlJ!nL{NuI%2X*+nuQdAG%4~-ii|f!LsigEl~RLi zikX4_I=2yqR>-Pi3iCi8gr@Q}t`jgsDIs13bvV8;DJG&0%Mas9IlMO76_cY5>oNgd zya2^iQHP}f>WUdrhg+gCC?-K2PD65jaPOVv)i1+;Gwf%4L}o?(QMfKR4ltazHSZHn zSj7}|IPwq=3Jm5&%<9Z$oyl8%o~p1d88&hRHtee;m1QzI=SCflz4zc*0(F=?0n}j- z-W#X)Ji2g!C|s*u=f6pa!~}IXw#x#HDI1i~thSoMv;=iHewO{`>AqR~J)V|?AtlOR zwPEBUV71?@0G>sX{ zP(drK&w~j{P=_h+2@y2ij;GHiFjYByCwX$zVfNL^sKc?9OGO>7)3n&DwRCT&tS&c-2I^1|^ z59XeZ3|xQ!dwk*&D|jBzV%V1ut2nNqn*?>3ipL){W=0(r&Cj&HAnLGaf1VYFpbqo) z(;6cja6z|{@Z=diJ>;mvyxmcJ^RG(x2;|RT4iXG`*8v48_r$vg_a7fE-K*Q%o69SA zHeicG)ZxfqPMQ;2U+fgg*qP?jN~pt;7C7{<4;p)XrKlA^K-6JucM2t^x|`TH$$wy> z70e=NzrT!^*kXeQ)Zs*$t~>ccJo4N1I1@ey>Tqmeg4GqM!wQeYxSJ1kIMPWzgv&!6 zmW0uhCd|S~Jo)O1{7!YIwXH;gT$_&_;9M7mr<#ZsrVTvFsKb(-+~5FuDWVR?))Gc; z@>`WjN(H)b)pGn9gd8LwmIrBeL_%&SCtzXJBO&lu{A$r0(Ce( zNfMPSNMP7HW|TIf4wDWF)M3IEq7IWENoAe3!t!HHLLH7RskpvLbJkBAhbswUAudpd zsRX1wrgDb@H@Yv{@WbE<7jxGSt3VxQOqM;ePuI*mK^=~b7pTL8=M&tc47W!eHcvPlZh|@-+nziYa9(5XD4juhn$2FS3qu`_Od_F6Lme(DQzyJgyGlZ= zyE-H4ut`~1oPs*zYyO~&I=q@cs2Wd%%=UC?GKwSFSORspPIo*g1*tK@8JdZreI-0D zL_caqR^&KO1@2z&<-Sd~Z?ydPaOeEOLd67iIF3Lnqf}2|n$%R*qnH0a6v5H* z-zPE+#vedJY}yD0_%2OL={~B~OmbTOdzVk1|4z=nRXV0L%s32XcRVz)&+{&lwY!{t zWgK;QP|y4cd5!$_aR_3;pPK*Pl}q;D&m+<@|GmpW%zy8s28EonEnKwhvx1@y3z5}l zU?u;(+ZlhF{ESQ*?0;n_)3k;F{(xRH4~O2gcCsAcBC)rc!7lE&+lEdli!(5YTK;>K zs{nNq7D?)Ang8A!mfltvN2FEK$=<EOob+COmb|*okb8**W%wRuaA~=WWgW z_rSQG|2|%_>Gx~8&+p>P9Co4MjlAhlJ>~4}1Go&le{%2uG7pzm?3-M9N#FhPLfV>9 zg$wXFUDK8P_er8`wd+@kFl)jpK*VbPdt}q3*n|1+QKwqtsO7(h&!W}Mtw}wumWsY} zZ+ZPtu;KpJ!WH?^lqy>KwX;B(15dw?9@RX*S%!Pp)lE@{`5aG25ql2&fYq$$z|(Fu zbKt{^wH$cf%F5!I7JrI5%;ghPTee~zM01SgVL8tCEaLt!`KwxZ^g968%oYCA-SAV= z0yPId2&SF`A0Efvh!9*}pZv@d>ykXp0p8l82C(;wq$7qk*X;ah`6S0G=F1}%h-iW^&WAyxIC54r@i7S z9`ts0)ZxSyGhh`9)52U~_pV_K6a{s>=!AjFDj~shk{A)7KNMLvuVPyIIDC6;i=YlC zVFw5fQXy1!q!qj+M>hk%MM)<>lwC&y=dvbcNhe^7X(01}w2TcupwejU%cZmycMDxr zG_Wun#lVS@PO7Ze)WDT&L{U`C^CfIU*%4f7Ey8RKkSyucLmjSUM8K4YsUFK$pAD%V z&fAFLp9QlL{Z(g7$qWmDFCrBS{-gqaVGkhNPJZuV%lshfa9j%@CdaX8C&rWx^bVk% za!v_@ceyo{bs`BHq3^&54!Ui36Xl#P8Fe@c(2p$`u^e?cK8rp5;p|7sqYQaYMbRnj z{`%#d5Oo;G8MtJq!}XC)cAag=ohasXA*jQ#8`*HebQB^}l1C7JoidSfZ!BVkq&|#V zhelYgHO4Fxfw57O#het>VLD*4Lvs{!Qc;H^QSbE>l8wcjvJh=k6H$k8;Jp5Qo}dmR z&HY=Fs=S74ql&1*pS@hvVM5Ma#;JjVN1N-#uw)hKM)EI19i}vxMFv@khYug@pFBK$ zaB!a$aI$h-hI{PMD=^Gn&dcRgn|LnNVcKK0Zqu*-1l`E1kuUeF8!o}+(l#4X90*$7 z9_)QU9geeevjw$ESzR;4b={|n_z)RfGrmGmhbfJQIC89N1~FzV4I}M=C>5!PZd84W zI&9--S`u%K4Z2^Z_Xa=lk*U0h8@ton|AX)BI=>~4bCw;blw*>WiQ-sa4QH(aSnLSu zFdH&)*hi}Xdzev&*@x7@!WL8W?9>nWojq_GjMnjvq7E~T#bHeAs+o5imq`;T>M&uc z;UlnVjqWE{3w4;XH;-x?yZ}O@8wcI%?vH3LeOA)KXP}HOq7KtevIV>!X0i^9uH4O@ z{mR0j>z4Gauean(Y2~lU$s?DLe2JvgxKiE6UArabeF^@GJvBl$De7=Z5`#)Os_9?? zMs<(Z0T(|{E+eSJB~$(mkGD#aU%3%s8!%oO>aa!>WufRw3?e14TkZSxB1qRM>M&_u zgpV-GFRO9X9&+PJJ~9+fGJ*|f3xJ>w6K3=2T9IIk7%5NUS*=$o39Tu(eB2la5i25| z*cN(sH*f^1=3}%N@hYTD0!5fCoZkZ35dVJuIY*hI`gFFx;3Pvyh*|YD;yCmaz|L!5 z2^mKno<=(x;OOVe^jeBI7=WUv!xP~PKpoBvbJSs?)_IIbm3rUZHBpDNC}F{y8FiR6 z%~6NR1A>=3{R-X5^SfucPkQJOb$DtbF95A+BBThqv<~Vp_Got0VbXL>ia;}CH)j!y zN)b_qrzRBCVZzF2>xep>8KeqVi&tn)7=gP3&PYE0Vfd4i4!wYWri2F|Ds+GPUjyp! z*n$W7{+y)O9)UWXWhE3XFkw^dEMuT2x7-lfXX)*AQ_ZrgaB~f6DPr2F7g7x2o85xu z{jLFYH%Ty2Zg~)W;S4Z2De5r)m>mMxX_mPL?aXL;$!*@c;|)^qKP%iGVKke0sCSB&fq?X%4suS5q*~dS}{3^iSgEOH*nSE@@RknR#zL#hge_QHS|!Efsq# zP>0Q5YU0eO!{%0^voLup2D)yNuOq0#R-Pt@;qB3B0PAewDecZ3e|sg3yRW=t9d}#j z|M7y{8l+1l^RC#bn*4-w0v1!$VKIEdc__~lljJk%u=u#{Kr{)qR_D9twvCXnG7OP zbgLc2#9K@*bM=b(x+x*gSLM#m#& ztP<2=Cbl##7vy@32Q0jJPI{k7qgvtp5b_>KiBxtV4{`%-|8>W5n1+;~4u|BhWai)H2(p8)>QO{aGP~{q805jA?>!qA7RglvkUc=LI|c zzz$#2ZrrmlYEUS<7MbW)B_;@GzQhJBJA!Ivq1x&aO;L~38#ZH~+FaHmNl{O1F%2Y+ zqD9*fDOJZXREFU6mZE`0xLFJwrl_Y*DH>=IvZ_d3In;EQT$0FqwKdRc{2BT}Qq&`H zpHy5X2&aImITgKHevF_F6N&gl>vVlA1Z zgf)|cn3x3L$W@|aBNfzPWMz~Brl}J)wFK}qzGP)Ov@Letd_Pljcb;-^rue!7rrz@cj+d0+3Da#kMUvrSf-gw9sOOs=>RdYkM$pTE= zP1>1|Az60G(%7LIRl0c~$h0&q1*lThCHtqU@IDvgI`(E{h-yHKOfsTXm&{yE>*q@1 z%vG1n#6lo?L~Bg_PWrf@sKbJ%`gzPoLOGZX44?GZH-L&Ask$WkAUy`D7L9lfCTpY4F3e>XKyBq}U`cet`EDm!#?v=O{X; z%{k#i(Hep}tWrg*l->xf;t=-WVyD+Rj#X*KBkJO!;wm28OMpy)MC?-c8u8%`)vU+cE6)M{iIPBOp)GQTsSPG!7m=Se2 z?+qd|Kcb(lTksKe~h%BaJ!l}kk(uG2(&RZsW!+Uft{Ye;?T+JHk- zN=g90A^~Lh!#d1o@CTv$qqK(=7^jyz;YCk=+Nb6X^d4+ha;;YcMlII zS^|`)8ByIg5OtWgt*$_k>e_N6BS}z)$&d-u;rJv;RIVU_E$f(3+K4(#x-L+M30H_Z zOn%HJhrefSJdVP#M%Z#Hu5a>q)(#x4Ok!qNI=GS`7UBYRm`WgDSc*TY+%;3daWsrB zjRip+Caeh5Ve({8F5$o=Qne4oH>6|=)L}yO_9_&L!B&xd=NoK-IviOqP>191&{a9q zDsR|p>u5HWgP;yaMyja8gwg-6y|;_eEV&NDrn{HJ;pOgL#dNs((xmZ%Y+nf{wU zBrvt46)2FDjkvTD#a5?hra3#>nV!*f4`)fp1}(se?8J^>$w+J?f#Cp&9mYu%C~*QK zN)!i40K-6xJoq6G31A?}gCG201bMPkRrma#Q+4a!@9T*KaHQSkRDD(Fr%s(Zb?V%6 zi~#4V!wVb=qUtcrJ$fwCDWTj^#`z8i)}2v%-G1b%FX|G}({jSUvZ}+hG{>iB&z_#0 zq-;MQc{f?z#!4MiQFXXq6n<`g)#3ih z%cDa^o{}YLN+zORN<7v4s>239S#{WSwIrKOs2E`3D&Cm-L=r@HVc0M%Uw{1jz1!D{=EQg#J?*UYxpWJ z_iu+dQXY~!4WjC>mi;xBi|giWN2L7aF#a9b+9l~h{CjOx)+6Cdayfk^pUM-Iaa$DzpGKzuMuzBaX z73+5 zGGGI1bSimEM$Va`DZE=V7*R@ptg-4avgulEgSO5^n@}ecVoqjD z2q+($uT=X+dLu=ZI`mWwY%cgX)RfiWzV2@Qe1IPbK-u8+>wNpAJ8mt5c z27J?0>{)To#&#IaM>H%>Nn zttSah^L>W)^T1Vy0k!nhs%z1Xh;d9QAx4u-xfCFSp-Jadtvbwz^i527q#TJ@&sXp{ znS!pKFY1@o-(8bA%YhKYn)KGBs}2J`rt0tnmg`(+t~$)@&2bZ7dvR|Eb4adjrA;y? zQ+2q;bnoMKNuETFULb=Er*IWS96?!|^y^iVIqh9nvDW-%i2z6(yOz&WL~+*~VO^#! zF%Xs@6L#Ha1U?rZ6O^gL7#i%+$(+zwn7cK(>M$={mcO46RfnOcU}K@$D@Y%uTy^-? ztF1aLz;!=kHLBpr>89SWW)yByRfi>v*0VzLXrub<#n|s`8yH=G6H12J0ll(~$#vIN zby!louByXw&IT8w7nVxWru~`Jm#V|c8#}RAiXTMP4Mr8#ELtonRWD?+>ahG#gB_&v zuGc(O4XG-gjqNKl) z=RK>g^lPmYobNE{sW#bpN2kY6PPR9}+1pI^{5S6UeH=3S;9Ah)6{pvC+-wt{o<;4hximd+WSIDE1SqRCQRQr$l?P zFn0J07FHdWxONCKR~=RZr68|Vby&_Ci)QO9fvbb4IxMHMZmd9VMRujWt<&^8TphUT zus|eyR}x2Zt6FO9do6r`l2wO2NOo;A?RYT}Rfqk+c5Z=bhyQb`8|&xB#~9qPji;m; z#;dA2906rsu$!M%_4tT{#creVi&+Av;CHz+wwT^LA90i)p$5_;-ePQ5N}iN-3`Yds z+ur;V$CggMB}Em9Mb%+}EZwdH4vvKE1U;Q8leoH!s5&fwy%+qy<8$^rud1f!{ZdB0Ql9|OS<7=VZdC#5f{a$ zRsBB0Yj6MbA5w2BAs6BEBhN)yb(mPdwr-zDaMj^)ilFN7*pXQ^dka}sr8SZQ7dcyc zzZB!D!$dne=`^bVruaf9i$$qAOn1sYUUAi7W{IgfOdUqh3KDbIhN}*b(?W>DobdKp zM@)(%;VAKO@03u2Znt2gM4h7GWy>*`P|n4K<;G|kp2GghAXjm^4OWUQq^INq7ry@gJ=aYMPBfJ zlH6`_!E&>(Cxy~{uY1omy^5j)ns=)e zO#v6!(L;r9wMJ^GYcpJKc}LsZo&?HH5q~+|cODTraOyIwP>m5Y7rCeg`SrIUR)PGpMtgQ<1_ghaPC*->S&1@iw;BReV((VYrgV zv6Me&f8}xZR}OztW+)aQ$>$KV6WoqrU#H_y?5__*nht>iuFE{v zA>wz5OCqy_=rFj~WNPAchnT%z9xx!i1c#wBKu3(V>6kja1RX`UfYrQUB4kWN)Gfp#LUDl!qQ3pEomySJ{OdEMhqMCdP%*{Xot0C%0gO$L5 z*OO>e??62~3B^-dfxV68b&9l#~4inC>U&$J6juYU8v zHF|mG(Nl*HDT=EU*Ui6{FOfZ4NK_q$Ny0#N2GF-|_?#mG)nN+q#bT%lwCg~1iObxr z$bsq%5+6V@@Sc0tQP0nHEyn|tdJHh@lKB~BT{7);8Cdqr1SGZ|y85{)vo2Y@ zjens+Hc3G%OO2VjPrhWqh-j^>$hxF40tdBz zylOm9&2X!4<_(TjX+x;&RX4YmV&+rErEu(O?^m+wa5#PP02IRq_jZFNw(x5+5tYce zRDsJ(!3QG!LsNBFcj9{kRvm79D*(D2uaqX(S!7(&5X>W1bA2h}lAeN)q#e_GR4Y#s z3#txlm`S2WvnOUMJY>b4oiAD2M2^U~q_IHC=RIZ35U?#-b+`hX1q?qI&_7J95?;kp zz2W{*`Hw-L=+&|_b;6DdDyj|x_;ptuhG{Rb>ac`hs}9S)pz1JmTgEa=zv`+R>PWo~(lp3$UJ z)nPkytUBCc3Dtt!yi$rb`YN!ZxGPLw#4x!Nq3UoaD|#xuisG)CLVLuoTNFL}QGbf8 ziiY{N>Try)(YLL&P)Yt#JVn6+ySj-fuc+#!z?TVtVux<4Txv2k(ambY7sTT zq(Xn|OpeG^hYgj+Ja~12xyM>E6ilu<9An(%b0~^hlv{$Ss}9FJ$sXZqW#NWh{+g^h z>;yBcI&2wG7fh%++}fob8dFpq2GFUF;r;iwUwbG0n+l~k+_~zo>k{8Otz3RsRviXF za8XcYUd*nh&D+4gzsytK`u$ah0n1$EioXx^)ga3OhMw=O>I)-(94O{BRfk2-TB{Dj zyy}4wqd^5sHZ3dP?}@6zFpT`?{7hO*Cb%sbLuyQa)m;jT@DWvJ0PbiEyn?F30582y z*{Z|J0I4wSnirQ#16LhZldGzrU6V&8E=9p)HT!yMf{Loca(1i;ieJaeK2vpAb?K_Z zs%vH~P!7C?Szc+?;X2IO_G&G@IiFkkkElA_fJ(+l?3BV{!ZLWQb)VRjRfi+6m8!$= z)3SL)+}L=jlGQ&=)!_()t2*5H(O%NRA3uBV`J>bKkB&chbawRQ`0V|3xxJJkGKYI7 z4#7(ROUBimrbY-H|o z#bdCnw?x%p$)0@GkX47f6hNe1xazQ(Ug?)eb!cg*XtX-|>(MZx>afNib&PSq71=`O zQR)Ld)KncdoijBJSL}s(Qv3|ZAT8%BR~_bgbl@SFUVrU%I>bohwRa2Cg~`xROtm!+#K2i+@3(i8h)8Te9jf z0P3hZjBJ&9hdzjBd9x~A%IfMZrB?>>KVdjI%51x%!DFby|+@rGW> zb&9`wcAo1LZ^)X(b?3#at2zt>qVr;(?})}M9N9ykuGn$lOlup6qFyYnRlVvkKuBh) z4@4`Hn0lysu0?0Y#e)1MTy+>`!y38hxAIYnOR5gTqhXH(EGE;TfaMWs$f)-I~)nS;&RUH;USFAb=Gli9Xak0A>wd!yXRfid1-p^0$sVA!r^G`2+ zslPQC7h<*QFkqDtHI~HcRfl2j(PNQL3FVH85v)6-E~@GfuKnPxo|i>kvK;#%Y1$Fv?(@$a(`PtiPwHjH)zQ;)n<@$X`5 zR}(9B1vdh5k&T$#N>Tj#B%V6{{o*mrrHON#w+q&u#54-l?u~z+oV-@82gF4v(l0;FPqs{!kmmx%VlXnG*b#tDb<@4#16)Xi8Va4@~}@$Xa1 z(#PHWRdw}q<{3*F#utb2?|}DIm@shi;>0Q?TKQ6VvaJ~ZUJ#Gt-{+@n5>w=*X{y7240Q_kLr|nS)8mYl~MA> z{7|^Z@hZ|E13$xR-pq+Yv68CZwgi_MV;+Q1;V1Wns5)Gk1=vGMQW4+s$-~ha#lRQ# zLuiYwIvk>5FUvpyk=nuovhk!th%e|RT~jOnldL+dF{1aHn5hue5qEaRz_X}DG4N7$ zk4%u;Ixp9v>agYja$Ur^xfu9@WMY+%fsfE<%T7yI*C*!#R~=4x;ui`wwL*KZh$yZ; zn4;!u`AR+)k#iqUv7_?Mqo*&!j`$97Y}%cH1Zp$#f~Bhm|3CuUDV=21Vc6|~0pA0< z^1*O(MH;s!FM;C;6I2~Wyt}Y$;)7CZQ#!rUs>47|P<6PR0pi#x3L{Q7)t4?NN$mS4 zC(n)^JbH4L;t)z?!Ml$he2}6Hwn*!Us>6USndcK!>&=a_3f8rc$%qmKn7H^G4!M!oy-Z1g+5Tkixk^xV5X(W zFn=Q1+NP&qW1+$oWG1Ctb@=wHtvW2gbw6VXW2Y6d+bjhu;Z8kvrAj12iX_-0FTj|( zcA}>NURvqqS2kE>u zu#swMbi$tTN+qhpbxMq^I^2iPs1kSQEqYed3-&Ro=T_xdiT0G%AB90y9hRfjm0oDO zzBkoUE75xEJD0&b2dYnES7p^r`R#;P6)z^D>aZWU__+iw_wp+}+7{?>?|({~QP59S9rgsJ%f*KQ zK2^pQzk)mVhY}H}G3woB|hZ4dC zUW|esdsl0)sz}dOQFT}V$AvI&MfLaD$%G(Qx@6Vi2rx?!2MZJZi8sQ(Pt6AFn)pnd z-M|;FIy|D6?7a3TA@i#a)5N3Gf&F}$q5`iah%+CcWYysj@C8&Iripdc;jy_=b(qGa zT;E+wL+dK44v&*0o+XzfRUP(s-&{~wU$H8me~5ayV(M|7*xd!|lwnfUVIqgn?4wFl z9UiA31;7u(UeZObe?n$1U@oQ0q>8G;o{O%!aP?~mnVtdhGgXHbOvA;*f}EIJKl}upcwOwfmieDVXS{xh$5|Z2DY#s}5_rgT(HOst)T}8{C}X zBIM5y@-FNZFx)v?>7%;KNB18epPg-ge)!z>&f}-|?|wOCYd*FpEM1>r6Q?E=cTHb9 z=kYmVgRDB-&6ISwj^`g%9qt|(v6HF}chA!2ECgcdv7(#5vg&XzPNTzcKU}YdcqL9c zPwC{|$vYgyW@CofbP0#7I;@FTj>Eva(#_vKKOiihaxC>x@Q-B9W3?Aeq>#N&jGU*c z!23raanwN18?G zdcP>Oa7|D3)9L9)qIRsRemh7ThQau z?`517ur8_&M?^_KBbLHdzXMMrRvlJ-26deevlSd}EQ~=^9aaF0x6$6>r7(%FiX-N> zuo->8Ixc-WRQ*rNub>6Ubgdh0nvk8~b`1OGIvy_%Ty?W;9Lc4I%eu(3fIo70bc zJ!H>6qeXx);bjhBO9;z(4@%8L*Gpi-W}Jq$E^ASIBmqp(cA~nkvw6MyKw-y;*P1rj0ZFnJsZj{eHZ`$5tlkk)Ve z?T?qVenWn4E`Lf2OG<4#h^oT^qNzFzM*sz>i(mQ>>#pe6EK(go>hF_39Y9i$I({yN zsG&(JP@8-HaF{wwQmH!3sIcnkJyfykut4*bg{do;CQMyXOjU=~h+w6=7O_hn2Cg~` zKugr~7^dE{*0crtR)Pc70bH^@Ia{XpOfLyk=b1;V79Ua+S1AUXf4xH0VVEQgRA&HP zD?+j@0@Yy(^2K7P3AF1#b&1Q|t;m7u3^JRbKa+fy_`=3L6fzD?t5zL`Y04|pxO9fl z!~jus7=CW=Abo=PbifPEomXCUSVL?+oybSBMxq2WW9#uL-I*3w9X4Q;<*Falwpo%aqKz)Q#1sKk-A3?qMqH(u`3P zRfo+?Q^%6wmCU+iW>Ox3ewO;1RW&?)d)BKfU>>*%{xQ3s3=P&}fLWK!&nWAXX|L~% z<}Gdpv*OF##5p^1LS|jEcpLvhg>2MH8?6jGX6ioql2wQG1i_6MZ+_iui=7ZFRUI}+ zD%tP#Ma-=_Y+(A0V?AoB>aZEzC+S>u*bbKgJky@C8A}yQc*A3`Ewe6JG`j>_JYCH| zw29K$@I?qp&=`S(T0dT`D>`0O9S)(gSKS!%#muLSOW{~}znThXW?!~;xr>ZT;q=J^ zPz)d3+YOe?C++%Q}BUE{}7(AFkh7J#PP*wH$Q4f6(h&vJcEtnI8f)pM&r!lNdx9+?YgU3%@(r5p|W<@ArVvs<6zzcz@f z!@yzy%Tmv0%~&Jn+2cp&cIPWx^~o{6vSvDKjGCk*au-_SRg}F2UJzWFTOQ+sG{30o zupK&99j3c#XEh34epnQDh3ShJCU;_A(Dy7sx??73Z^j+#Qm+Q{I2yW(}rM*Y>c5`a@Aq8E`V{9&!H%4-P#h&S5|e{31(Pz z*fO9lm{4^XH(!%SQWljc52ETYfR3sTJKVYIuJFBN)mNbY7X!{U}QJ z0=)D-bpcg})#Pic3i_|m2Xkun_0$9vRfpy5SP>LI)suau>agn4RfkpA%vzuvcnz~O zst(tMr51RdnLUKM}kZ>I)+pDpAXq?x~-1tCoQ$$rappr2XLtUxra0Iqebr^nH zHjjuK8!uI|`lqQn9D#6EhZ{fI%M0ZT{{Eh*MJ&qpO7`TdhO9aq6ckp{S5?tmb=XY5 zCjIiq?4`Y|ZbE-O8b(wd*7&22G2S_ZUYKX^=-4^xnaaqSnuaU(q5!VD>M+kc22vS2 z8;BAiNaKDgRfiiM{wsLAZ;7hI04q84IF4$vx^PwYxau&^J3b|I_zxm$@h>Q}LZ}7) ze`>?V-*i+RMz%`5Bf^pjUI~dwkF&tzS2rSKG3or*vT?U#VuU1)vTpEd6^KREVW1FQ z$gi&Ium#M{3u6|R61=53f(zKgb9C~DQBMAxeh1B4R#WdqU5il-i`o@>#WafQHJ zN3tBvOLQT*nHpnB)nP!j#O}3{m-Oe7;HtxN?oa~()TMlj7En|j7E5NT4nq%e)M}7| z*2J2Hu(|56Al+0Q7FcmsU(qNs>$JsJTs%e9VVJ2|b+|h`ysDqmCkaLoCxB^Nh^oVq z1N6aC3p?I}EhnRJ33OK-7O0r2!=i_)4l|OBQvXtL)nS;&RUH;USFAb=GrhX1!wfL* zXRbQTq}rtpabv!&XQ{dBFd$N`IxG-10=#6_? zdiLz;*@>w-9J_V%<#IFqldpR@a93X->et}ca-ubJmnTPDssR3xcasfT(AY7D9Wt@* z>V&GpEdf$q;B=Am&ia}^=oM5QhJ`>I%Tyh%^K>%Voqe#%v2Vs1N-D(-TiM=Kq)qlut{V>!%&su&Qx%J@Qgj z9iD;Cy5rv`@myd0J22E!kP8hzfWTNb0wYo*Ur;_j-NmH*^<@0H~xKc^0N~E zo&;=JuB8&|I==+5*#}|#`y}e*zjyKPlS3nx`uO+BX(0X`FHl`^y5po>ak`ho`1dKS zD)H}A!{Ye&=}{_8(~J#25l=M=Rv7<2;-q4LyC#diMV^7ME6Y_4oY$Y$%c{dh?kA&y zM^b(K`_!_m9REH7sMO->;&vGS4tP(62?O+t+gGCMuwIfC9d{5zo8CD_fp z@FJpn21-Hvd%;Kr`eF20{H~Ti)pxl$8>a7OzG#`iKV5T{E&bYEz{kLQtoh=u<8i0L zlrKjy@N#+^Ah#~Ih^oVS6W|p{WQ#Y?I+RW`%aM;#j<8?A9ynT~82ADKLR)Or;gA@6 zS=JJeV&F-KPp(%!v-qZ7%jp!xz!$XB`^RYd#7u>#j<^$I;HPm!a-tY`wLB)sZJmL2 zqUvzCCPyO9&BeeYt0triBi@i+9mqCiy+TGOt~#9X#MQB>6|m}Xg_J%}BxbVaYxzn( z7m;%xPoXB9N}iGiUr^fZ41(-bb6&#hhZ6V5-CXdg9-e~ z86b|GqA=oQQ+>Ibx{}I>s>6U+}U ztN`p2CE*FT`nzi~XE_jpSd-&K^4jS{I7qQh=TyhCYX$p+Vc@F6%-)2hsyaN1V6@U? z=k%?EN#=ARRfmBLGMvIy5OD-$ZPKq-P3E+BUBz1Sn#kKHyhPPum^zH1DfHwR8pGMOBy%bvk>&3vMAc!5jfHAI!|ft@;HtxSUTxK30j~QQ zt5Ky-PB-;5WwS^(oD^HK>ac{N?gZ&YSk%jv37mQ{_HRGw$y&WxcU2wk#85w7Eypnz z6jtw3)!-XZb+{A9=!K>7XthsojALryiK@`+c(GDNgi&uaD;P_%Wz}KX-V!Rz)r3`t zJI6(Aq*@xCbJ$m^UV)j=K*f!8RaPDD!)H{9yYm)3tLeR&L)6REdWrs9T$)u9tu9o- zmA)4VcU7uBHJKW%eM|O#%MfI-=geiFKY-%jP}N~cc3tk9SFc`f#14QyQbpBag|Tr5 zM5Inx|D*h)^4MsV{FR{dr;b>4ShNTCRs>371pz5$dW$x@&;1Ysuf$RTLjH?ch z_?4;08>tpJfGIw*%wm!0oJ$Q?9cGr8s>9S_1g#*^ws5(4O8-=$|gd@y4Vp1Fl zM~RPnrvw>vy9FEd>l6hqTaLkm@|@_UQWWbzX#xtqk>LV$*I+-^t2hVK>&wl#SZep+ zDtIocI;_`BU8yuTBi~lVMYv4BTD2WDme~usEC3#=IF0Igf9^I&oOGVjNyU_%|1-pBmHxHraxj8uOJ~m@st#)kY^0FAPmG+Ws>5b*Rpm|{et=_QM%5A1&y~>8 zD60+|tc^1!Ci&u|>91@IKN4+6#W;wn!y4Z%@#d<-X0R0R>Fn+OixdiMGn7qM9X3ezNOL>zh!kD_5{Rn9 z3LS7Vj2l~so$pzIWJz5-EoI2s8KwMlmnGfKJKRDLXBNu$T5 zKWV%DAh+>EzL|?bU|H3ODCuY9tS|yS@HBElG0|sGuMpnKTLX53s5-2WG~Pygi!a9x zAR)dgjxc20rq-`~NzxIjE~*YIoYS>#v}r7=5*&sRJ32P7xjklj2rp;#gVic_Z%mdQ z{=wn$(f!BAXJ^|fQR()l9zVT*_sekV)9CO)uY7R!oj;@lJQ`0umvfzHbdJ%~$};W3 z#o3nYJ>G}E!?^_rOM5&!GU!Vq9y0Gx?jn7`dHg2^G#eQ+a}qmZSeCo6YOZ-8j1Bv- zHB}1coRrjlyD}0+Z8dg*`~N{0-Se|uzk5y?1CIA`bH2c}A~FUqj4o{55z{m`XG!Y4 z^wx}#5x5!R_;54`SC4K^-!S4kvIJ?W5yR22(}&$Uqne)#XoDhp6uyohQXz5?9nAj; zrvk*6-*ei?R3r|tnR)&uiyv>TNK#(p*51|7v5n#iq3c+5MJvjclN|hR<(*&RFYe%GPUEx@gTSRh&uy<2#YzRtxnf zN%5kRorAg83H{;je|=-Py@1K*4<4L+6LGCoD_)HTU#|Gy*&KdyKCaP2Oyyl`sklkG z-IzLnOSG1%H;kKX>)ITS$v3j$FK*JS?;L@{*R3_Nb!V|C&7yola{+@fj&AtX%>l#P z`*|~ZgbwG6TUPZ8_dd$O#`?K2{B3njSDH`x6ohftE6IV+e~uixJ7G1tetY_%hfhyZ!+!$Ejm_7fX`;n!ZN*GCI02&Lw$%G~O)1Z5j`b zeldwX74wT2C%<)tJKCF}wn4+q5>_#uLza_%>z>;l{;#UgJKsi)ww?a9nW1|s(ef=9 zMB!KMZFoqb<`o%tj*l9pOJuzHjJ7SFfy~AO&-Yqh#jGzs?|1Zb$6(oCZ_WO4BOqse zJbH1kIk+aiDSbooIKk(D{@1Y?6=yfI>);Eqo*}@@n2IpL3z={qH3(pVQZE&)H?q+H z8UAKfn$0U&d!2Wt=*>Pve*U_bOOG+qQ;%m+(c}EN3w|eHq=y^^71qLQS*3UGd)ao% z*QV*P6F5mDL~Srr~ZaSHcl{n$oC z!fV4{*q}-z1}>{waRwFfE6YS>ea*1;;nSPYU6iZA`AH34XkFRA%pY1gz)?nf{1Ge` z1wLTA#+r3^ixHv<#+S4Efil~t%qy5lALlRfn0$m6Q11Ap(+_kE_nJk^>KzAbK#*A~ zEhmap-rc1%iO3-ux+cl|z?~el#_46{B5NL}ykZ!tg2V~XQabTNI{X>&z^PO*!B~%5 z%G-Q#tx71B8l3nZ!rICW<+UY>{8~bd>9B6C*^p3SH^dUUSC!BSYoVnwWHF4PC7~Bv z{J1Hhs`_zFLiZXH3iXd$e5k5U66Ktx#vN$q$M@gge(jy~Z>q+I5+ycBqfbO8ty~Lv zNG}ZHL8%Vh9Cl~55Q#|^QMtDMqq^kB-o|4y!N6<1ZIg!VQ zL`w$TudHy6FPPmWstBc&o6stt(O9^!a>#$w9Z%^iBwG*Ji-bzvRK3Eaf~fZJ8)sJq zOO-`lTlA=MW%|xCo_;;`O8=5qIt6Z>&|CWc%Gm4^p}E8(S^6 z-Iak;n&W0`ltqqncjrgR{71f)Z8gsBO_4IZ0hA5f`t7O~kUk=0Q0fFr*2;XaVbSE_ z?`}Hpr{b}xjBIs4krf%sHr~TzjZ&zM{6ofprl3%9ila`H+2Hmqj1uf23Gq6hfw8MKb~TsyWL~&sCH^&XBJ8Rd3WNbARdl~TCU7hEZ?|S zFcocuD>85h2XOv|2?gruNWN62BRX%iqO6=$DWk5n9>_^+I}+y{4F!v)?ni53YxuPd z7X&0L5pTaP|K`L)sTumT9yNQyg8;Rh>*f97O0GL#ZKG1m6$>WYU?qu}umT5X1x`84 zuPA%4?*F*plF5zR_gQ1;@l9+O%gJsqB>U9jINJ#+fxoMs}ixSvc2?@ zb(LFw)UiM@%1!KF&Nt9yg_TpjfjW_@vT}`+4L2&(R>m00rmB#BW&IRY^DbRj zapkk>bCjvBtga393w%$|dXChonJ))yiZ#}%YL8)oWgU4k5s+s-unhzE%Xir7nR=a1 zE@9yQWUiNDQsQRm3ZWaPoC@5Z^;#4L?oXmVJ8&N`k1R#)jH>6U4+HlnG1bYa=5X2L z>gp+19o%8y{v@tH*I#J)mhotHBo8a%CeQ1>aOX;KLi zKhB^6_p4F!rsiB1>!UnAbJo{JHOGPbuq2a#`{VC|zb+fgJf7IPLG543kV3a!FtT?wRTr*paT8d zyDJ6nN13nU;QfL*y$e-X3&ACo-nqg1m&4$FwMr%euP$ai_i$;;SAzF7a27QmypQ~v zlCz0$yonO)2#S!DM=$G`fD&+vks_ z+ID%b($wmV7tWiF*2`tt9!Zg@bd+6$So_sm20rgqW}i9Lb#|Wg0~wxC%A^DwpQNbv z`P_7lqP#|-b?Z_MWn#?1?TG1v*IH^4S)YV0mEc`T7~>B{OA@wOc4r`r%f1v{DvUj5 zY9x&A+fciHHt$u)aay6C#a@$H83?2M{L>LL6VaAby1Z8xU$Hs4kVWWd(F{2Q52QoV zr-M9_=$AfQ(>jwNJ(llP1!j7T9G|2OZV?r{G<^q?&3c7pjJGdC9w_m~gviW=VY6P< ziz1^9TzRoi+Pa?TN}D9f`W5jQv~=l3YpmUDJHxD3bwt;>;HUDz7kQizE#;egmb`1F zY0*SAmh`S&;FYi>HyKs0iaVT%qkbt{0n7ff z&i~ft@RgZZMvpm_YQ^S9w;v!i=DZfCFh^iY6cjK?Vr!A`S z{0~*+G~;hQ2I4T|AMMOs^98FAdIdVgI7n6{YQ`pmjw!J|g^=6@==TZHzqc{`aIkwC zjs|g{Ha$&Od$(}xmcU}q-;JeQI~<&(+E*(hppC0;DVb~?WSv{)-tfQ0GVL4>(IVg5 z>0F8IAFn#RdQpv?UM)x4)#1;m95^Jx&Jz(*H(P|8DX6XvzdXDZ2yXa{6@PmdK2cy_bpV62Y{*zc<&1+R-N#*xC9zB-msaMCMR5;xB zxa6bqDXNl}PsHvLKzgH!_y`YFQ5W3rrjbJir^FDg^)OFU6Suq>vb2>Sh+DE0Cx?&> zlCKEVj)Js?7pbemza-s}8AP(G+aI{}7J~lv@CmKS$X*w&LxR|pN%4ut#Q9&`2mw`V zD+cYTN;8kupqnebV`wh;@J{Qa>KCL}5jdCB;93eop9OM&?jGS`5m2c@FW7yFq`3q+ zW|B^r#mwRdnhQH*X7j_b{<_d1GvT|cfSI{6wAm}!O^Ety$~Xs^|3&G#&+8;=LmTPK z4BP9QmarHc!xD~d>Fci!eQu?%%292X_bLv@^c{hSN$+KFS62O%C~zgDhrh5n{9%0}cZk9ax1$KD z?qx%EucL<9x;BGe={ua|xrDm4qHJ{L%fW2EB!w`(g{+6AJ4K{tB6fNYY}a)@*5Ud#DA<}R3#18 zA?faDBl~w|%RtkSGlMgWJe5H9Ukp4}AWkOOsY3PzidFh3_)FowYPcu6*%rGU=SM^R zr#0EvF3K5rIYGDH%oSts|3OXCotrXV@_Reot1{oq*bJ)gSKar9U#|(ab5~ZVoB^!m zHf_7<%2o@VmO_QKY};nEd47vd8{FLyB~*6P1$Uo)?^lsy7dzuoxfC-0Y zl~ppwg(T2A+)Z-${>jO+qX&i1tJ;0_NtfeU80jL^9^lmqOfx#HkNOTk5o#~^nBEAg1*TVDXc=m+fZ9-d zMKu?gR%1M(%!00@Ah64Y|m4h z2OGs(4om0Xb4t~XuDvCL?O_&`l%3+z8N|E=OqQdagB-o#;yJ2T4Ymi&^GlQsw&yMm zwuiQrD@-%PGT5JZ@W2sPFmizH{iRP)hzirY^$ODyuUj+j#=63^2H72Q-%3_srko0< zpH-YpCaN-Ul&xET39Eh3G>0OuD_l%|E~9oE?_K?v>hSy3aB`KTs|JEhRMn5`jMp%F z(CMGOeBq&u)QSC6BEl9_m^L_Nm~o;i^TU?F;e!|@s_Gvbv6T~5_0KaWQI(l>>O>tm ztrAt48S1?GA>*obA+qO|MTGoV+$#KEL+#hkgPDRc>!SgVaVQciuat8RE49bGcb)Fa@HB@ z$w(%t3g@?>{zBO=enU(qxT-K+0nCh{*C_oc>GPr-c#v)l^T7xi; zW|X8#{|XS{BvoPe(P~MmCM!&9ph>hw%c#OycsQ%{!p(E)!HhX7lT>L$aFxu{*~I;G z)+J6>n63b4QKPRO^e+>?q@%J_)VP0C{$nVpNYRvyTXtzWx}=|~Ot11o`I2p0@&Kxr zY!elxfiP3y8Qs4g!e$=?Sz%g&a22Ly=kKw;P7cxxKL^9$t1C7kky>@VUw!jTYpclRX5Ill4{=nL00zx!U(Xe;Y-mxntHA< zZ3mC_dx_c>f2w6tW&K{5zsO_q5kiIOIy>ek@)H%7G?DiFU$;yKYk&%Mm!;CKK#@vW zVLC>-ri)t(A_Z2~@3m{381t$sOncB$Iw{)2^1!K75^lh~S#P2$n8ehogi_7Mb?VHz zgc{S~6-uaxxrrrIRG2oaGZAaE772ZI6{ekp#ucV54;m7>{tD9oJXNLi6{a2jyIi^1 z^$AlXJue@B@F+bsU){*t;%C$U*u3^*&OS+@ICnn<1>82%f1w!9=9Mq23ex}yZWXEk zkKN?7l^b|oT!m>sbgcG_$C0-j=U12(6Wa>YqU-txuAX4+nI$a{R+Sd@2)AT~X_#0& zW{Ow;(Fo5KrfV?poR<}*VN!a)q%RMl1MXMW+A=Sgb84zt1vDB9ucE>k{LZj(uR+sLp@%y|8YLstVI}plsN_TCilT#9+fJ(#8~&i;F51}9dw8a(>(F$q!n>dh_9oeV9`l~ ztS}9*WaZ{sg=u83EOyVi02pb;V#-6TD@?--afRssPiF;Af|FIHg*`Z=G&S@=0sulU z#4D^YZ6UM&!>}RFfA|>DWOH{-UMtRFAQ7M8iuLD0?sH!t4Bi13eyZR zB`G-mAmr$tv)w`jdmVHeePxMdg=xStuZgWNExJU7X(o&K+`lw~s4xu^`3lnl?y420 zVXCl%^ZHg@R+wgpd5en*(@d^g^AJ7eZp#NKd`RuZPb%D6LG=pL0#|EtFIxU6{dkj zOk`YP+HkVrMuiH~EjGvpkId3{%d^JjjV)U5K~$Iq(4%+hRaBS;fM|}H3e)R+VUQK3 zEmStetoKH&Rn;ED3ezN(+Ay1l%!5x{qrm<0<-B^PR=3o;Pp0REaF7+Ifgv4Pi_T=; z{;uGyLPax!ReD{&H0y;u4BQ{#)Z;_1%M?~~F!yGt^^et)=7)j%lb9|D+y^duT$fdt z?iYxkTTo%TfA%77sH(R5f*pYsX|GV^z4SHoMKP-y)vCgDKl1cG-~;z3MId5p5V$`% z&!WJ6;NXPjD$(A$!gM!pYJvNc^yF*kO&91nJuC&ix7An8<-&~0R7EUz z&6XqCli;h8dvj3k5AWBBfmDTQyOgFZpS`;ywL#$i)Z(ljxDWEWJaGSV9JmjdPX;vu z{0o$?FmS(sT{UojoTQ!)L%#$yGx+NY(^`OZaw}oSv}asxD|CeTUse^S4bIbn`y=h2kys0+6ZlLX2Y0qBHoJw_d$sIWW1STpTCKpg;f-|Uyur;CgW$k`KkF+0&yFb z2r3M~@}0m&XV7IO2;TRI^Qoh+1ykPa%VF@ooIgL2No#yvtdSL_wLBx`B6iQdF5x+l zKKMV5f~q3|CWH5dWGN=93ey@@2(xh&ruDCYAa}*qqL703Cs!*gD!we%684iYs0z~> zr$+F8VIBmRR2XkR8=pix4Bl6(WFqj^c_|Nr_cd_dhdy|}{4x&S*OS1D2_4W%in@aL zFAp!c!gN|N{KCRzm!J~UdR1}ji14xrTdI^;U2QogR6l4iF-n3e~ z=$vl}xO79A7;|zvV)}ATPN(upuzc>Tu;4KYW7DYVU{U-zX_RqwaJjD%VoDghz^zmA z0?RcLMwdd>uAj|)6>_}2#w#VpO7P;Otbs62EHN!W z+7i>{c#t2I6&(^I*Y~#9m%~Stm;=RvHX39pUrz!uiVT!r?ZrFE#NnQyJ`u`OP8LsOo?gWu`Mwz1S47KErH{Kvw##}| zD=`g_6Ns4rY}Tu9D^md;T%#*B9@4{yyEZk^)5p2D~6QMCfUo~&1KiD@9WVB`SX z>q{5mvc&W*SIYdFyd1Cd)*?$x3wYi6u39A*<$P1gu4YwClO?7l5OtSGA(RiEJ~=vl z`uxf9WA$Lxjk8w*{jX)oiaTWA{+Po!)e3JiR@Hl10n242raL(qz2;1nnC_g~ypVNs zV72ElRiS#2C8j&kPTPO1@p;a?inF_vr3nxoVOZ=_JK$u;urMSej8Y+D? zO20Ne2;Cfsy$F5e$`aEGXX8@{$t826S+`kEJ9Wc=-6Km(i{Xuju1;pf{sbr-twRJF z7tgDMEHNz+T$7Y{yjmIIZelA-OiOHgWLlJ%RzsyskIA>Ymmrqc76q++qDo9F6gxy% zTt~anZy_p6ObcAV^?NaoHL}FChsn-!v4!!rBuh;Dq2Mx~d_Q0{pq%Qm#I&D$#?#dd zBX@I)6yvZpl~vr5Y|c*`5k9ToJ(qO-6f{Q+~ax)*AA># zhRRiuiKMy9BuY$=bC6DGX{bHR%)$m*{5~XP#bhRU{UxTw+*M|(64S(pxde$4)8iab ziD`jOAD;*d&#<&uZRzX(8cvj$Ci=4z`X`bKG|~r91)H1$}i zD~J$#`e3IICEU4O!VKm(8tpOD$ThH~N=%QWPtri3*OHK z^PD5Eu&U+Clkxd8a9hRUC`(Lhk{fPO9i6`trmmE9^;JrinC`|j@|9KGjv}UdJi#ZI zYjixSNX`kb1Jv3T6{2wmwkuR?dh)aO)Ux_fBEQZ6ywJ&koh z&XA%mCJ(a2bT3w;%093T#Yss>?DQVk=~3Acxo)~qpf(0sVp=mmxoH;2Z{WY2K}MGc zGbk~wX|T>4cLf>Q&n2eK@apj~XGLOHPM3T(eLAwlw87hWbt+^}o#&v4=&szqe1j}8 zt#E6&C%dPQ9h6bPQ#0UakR_%yl6Bc{IJQazdQ$i5gOnLWIgxh(eZxOc;n?B?a`92| zl8X}4nqYMbjIPQlN?)9v96f$||NTd&4@;G~`C?Lwn35%?4Vqno-5Y-2zc!JZuBZ;U z{Az2m5n(>VuH956rWGFmD!W#^x8X>Xn2yGRgKpBXL?^{H<#|bAzElM`n%;UY#rd%~ zV}APfRh5{I;WAUG;IaB45;%M35(k`mJj z?Q{_w&7PR4;L40U!I?QFYW($rEHSOHK*~+1eW!4BeeNwwOvkWUz{D3XVtEp)WReTf zW6-Zj*MwyN0q>&3bctL;SAq#)6d9c&Oc;kgsQ<_2@X^uml__LV{oEL9>6I>aB}9!W z=IQ+WbWG1jzub=+52D7ysPSghcq?kW9W~yOjr0zrNTSxnaYCP27$m}d{n~Yx2=@nu z&))qEB*J|?{nsSJ{c}|0JiS1DqxBWT7pTnjr-n~%e{(__aw=!dt-beE5W9t^GnMSKGZaGO!M}_Tg3~vfH&^HG3 zDI8*DvFByzJClDw=@9dEfk9#J+^1MBjCh+Ydl#&~zQs(Rj!1>5z9^6#@HGL{GHW8o zB(GjU`@I3aJ1P*xR|IQc)UPW@$)@D&s@{{||NX#wN+!vwm#iImz9R`O;OFsi_<2Qu zN-QsBGELpL=g_?{XcEXrE%6<{5s)n=r+nYWdZb7wFR-IOToRDIx{`g%z6p3=fl5q4 zb6yfrFi^m3oJb#C-@q?-@IqqpiPghHHm}9LjZ3dZ&s)EOikaZ>-AU3_$i+8M`Ym`U zl-r?o^B(U-B&EZ@v4OHa-cD26P5jIqb}NC}fdzeeeo`#3^pN-F(bE^_&t9A#mG_&o z?T-y_-2G_&0{F`IYghi>uWe;xaPmX;QV-+Z?adn-!!a&!dKr2Dx~gXuvlJ^_$$-nI z>ty)pVHnSw3<6d1;(Xd)AK+H^JeM@c{+zYs8F$mJC;Rs~esvDmV$Sb~h8@8syn7;% zV?OWt2H406liz*xAf;QpvBit}&hw`au0J|W@`8L{*P{M1E@}p`{aXGvs%8ed1DAfF z0{1|*5}y5T?vkSF@c@1tm2UqiIbBNC7%03I6PQ0@$&%YIwWpY1B!rgKAixv%faW%+ zX{d}@`jb8egoGwBytNlX_-mV}Z4E$E0pX+XJU+gE^4`u9@k^)k(ps zZSt;ta(Lt9@#9C&&K{i|JvlymKUM3wbnWBAaPQ>dqf>0z$Ui$hd3lrz`(WdxoND~! z^a1{TD_wf6ntWsU=-rPL-^SLJfBNAp1wVEA;H2ypDU}~Td++(9)A#lC>E001osy6D z30_{`l?4x=vEW^>pd&mIyt?wKoxWjpj^RJ72;Tl|!2=Xy$wK@qf`{Fd(4*AqWTE)< z5XwTIm~Qom>BhCsJ28EI817yH7u?1`XVU$rr}vM~lcp>6tU&QBK0(u62J8x)cl$~@ z_!ERbvWM1=*~D@4KAuuIPr;A(Q!G|`g&%X8ZkIvuw>8B4Lou6G-acSR=NnhT4`}HeZ+plFIxx1NI z*LnVI41a%P_?a2NrxD!#!44|Wghg9-$MPNdxKT-5UDDfJ(KhZI!-wB}e0=ZZaVqYX z%pG@cxY_gWIp2^?DR&L_H(6@d%UdgSKoK)b3*FQPs~utKg7pfIFo4!`gyoEIy`sHY zvz-tg9buh&W$HW|{^QCKc3_S$D=o481Zc*`7}8KV#;|)YMIOcnmNlWu+2k#)4!de& znO(IpJj5DK7wvNW(;Ijz5#CmaVb&+w0et39!*F)|%@cj4{xoXw zH{Sl}l@D)Udiz5~BTGsDa5aYXpIjZTmMOoM=KUjk*UwK*&z?R%Iz4`Ja)!5&|LiJ4 z_z$kW{qJ8*FtYyq&5gG)?grisp*;G%O8ZOS{_OeFU%DRrzO=u58$|uR8`qNyayR|Z zG(6M|DVLCKc=$Fy*N#uZ=_8JNQRBBLL!e*ugUPm46x;^;4p`CtjXfLTT zB<;|^v_tX?rX3oXc6{4cw2MZj9UBeps`=&Hl<3zke5E5A0TBEMpdWl@@9om!+m?rK z>z2Iyw&mg5^o)(mT9^>EiZ#OD0+J}_2^SGpAgH3nD=+2um_fQD8e#cBBP<_C5Qs*S z1l^+eK(as-<63Gyh*rS|Y_)t)?fiQPbZ~n?w+FvB^V9Ev2e|F`tR8>{-V3Ec>H%or zy^yl)q_0cL$6TxjL?iD7Yy{*`52$9Q1>!jz?Yceqy`i0cZ)o4Qv_k{aUK%9r(7?3s zN3@GZrX3p%?W&n+fv^=vyKWDDZ)m6A8`=*n?a;uqmj+2YG%)Q45$&RpX~#xGyJ}`y zAUem-OOHhIabB zq5Y<%9U7SS(jaMv2B!UHM7wBY+Og5lu9}$^h_!OG>-OOHhIabBq5YPn9U7SS(jaMv z2B!U1M7wBY+Og5lu9}$^2q|;4>-OOHhIabBq5Za{9U7SS(jaMv2B!UXM7wBY+Og5l zu9}$^h@5k@>-OOHhIabBq5Y1f9U7SS(jaMv2B!T^M7wBY+Og5lu9{!ItpaD%-yXjG zk)-bixbXd;*Yn5N;}g|zc;k& zcBUO~3Pc%R6m)~69j^&QCR+I4NY~}4k!i8u1L7+k(FlOxM*x568iZFy zd_1O0!u2amh+4%O;cx94sdhmXEuQ6r?ubTMKF|ou2NDFLkt9L4 zC_a!Z5XHEbnh&B?@Bv#bA5=U4p00XkFX;B*_hx?jJ@CL@8AKWQ~d+>WhJN@3!j#ow^6|aoCLDG&_Mxq^We2R9_$h2dl zp8roGe(?VB0L%VJdes5@}-y7QT%1ET*l~FfH+VRRrwBwCW(JmU9 zc5F1Xt7fK!u6l-c-5&hj&`!TMwBwbLNX09oZjiL&m62%28=sFL%_H zQT^@V4qqARf2J#A$`5n-!TGnIr7PcG+8F-rjlJL8_`UdN_~XO(m&;L#Pf5WQkB=Ul zJWFw6C#UznwSDEM(tpn%oqtPRm0tfvahbw*z7MC+y~{`UA0MBcZGUq3#P-L^2W{nt zDQxEKy!v4G>z^4uTYi>8N6w!t!=lw~~m!gqfI@k!vp$)2< z#Y2?2i1X8x?ubSJgxY<~JjUfWOuBWz4efhRc%!#Z@BwaLf>(GSYUkh6J<99_-5&hj%ul~B?fjGv*DscIypNIS!}ZJjBHhbL?D0CK_JU~Sy?~8? z9PS0x%(T!w%FwRcgWnt4>Gy_qypIv7cpuXZQV-yLjA+N}l%icUGVR!CXjjcl3*Dm( z?Yceqy`i0cZ)nH+7?FziG2I|($NL!3j@KzgyJ%$EvC+`3nwb{5M;Y36d+>WhJN@3! zj`uMl74KuZLDG)*F`^x>Q;K%c$h2dlp|F z9q(gAJ6@+0?V^!s$3{cDYGzvK9%X3P?ZNL2?eu#?JKo2LRJ@Pr21z^K$B1^kPAS?& zBh!wJhIZA=w9q}u(5~Bq-y7QL_l9=7j}fVOAJYwzcD#=f?RcG1w2MZj9UBeps+nn_ zdz7JFw+FvBwA1em?RXy}Qt>{f8zk*`A0yiFI;ChAjZ8Z>8roI!#XjNv)ve(R=1Nj| z#k1Fv?>*|R_INqtQSVQEcz9DU()LH_3aRd^t?T8^)I!@8{y$%H-=qAmuMPjpYj@Mf z6yN*jdpuZ|4E(-vAN|b@cw9HBduk8kX49hoXrq;sQ(xUH*-~2BS+;}pKg0j9F}zVU z$cTCR;^Ol7zvuy5!+p8)ajkIf?grB?An0XWe0D0sXQ{TaJ)|5;ij4H7e2t9pNOffF zwOCiO&n-z8Nz5Ce#x%M`*q^}OFT(!Hi!hoyF@jx-aC!I}W)TjE41BodH+&;R2P`pP zBp2${;nUmSCvt*O{6_`LdPq{LdhTl@JA4mo_@)R0gYV%Sd>>vAzBglhlL_kZy&2Ho zPePh;0SS)3w`%x${@z-?8>i$aIh|r+hX2CI?`^_2`rgFx_%<{2he?J%H|%U*)iTWS zM=>5P(n~QPqlkvzF?io;;q7(Dou+K`k+O>RzM8f7cp~BP~Ut;gqw&u#BS>(NrstrfYPwYSEe&(IM2mmd4wiX=>A!-4eHUbV3#_EP?GNF^OL z8U75_ZzsVj?6nLSGrB@@Ly1QQ3{g7#j~0J_ifRmkb=dDZ>6&=-bD*+k{6g?KiuNQC z0>+Hn4hPNc&`KLU<^_fFBa76&TL?Z^WBh}KI9w0@6&81`^5vXeUGE5g;NmNw-_kdm zN~1|$;{-!^@#gUu`y&@-c<3>{wIWiLW3|Z#k1-#*Xf(o8S)guiH|Ji7oT|y=G3Mje zU>xwceb!DbeARSm@?l7~QzgfHe7vGzgAa$Ev$4F8?Q zoR8K3bFWAC+LzNlG*(d}-nQA}V->9fB)Ou@_x75S+&JO&#Pb)?+ctZAsD{`FOVGZW z_F}(q5g?iUe{1E4k5nJ%Fg_soU2rg;fo)6S8d;@p4WHlsoU}^QN67^Z{32=ipDhJ^ zs8%QlmZE*C_M@|KF~C>j7K4vfPfBf1+?>A{kq>}XQWy0jHhwYqn5|F{EXJ*C{UBFi zc$4hmx2#<8`D^2Pf`j39Q#1V_$@gc5@8ABEk?%44!-8e>IP(c?arQ^>Ui%2{zL z_ZOVBO)b~FlWSz08)2Uh+8F;}8QSM;>-MdXS-40Qn`b{bX339Gv3d5}e&f~wW~Pos zzNLS*-7&i#9J3lsuomrO)(o#;IpKyI?V8q6I9Qu!pO0Cx7zmfzwpii1+qF_7+?WS` zG5FA}P!KG}oj!X@b7J8(D60~w1$V9;1ZOciTHINDa3f`V#~^=i*WkFAi!FJNP$hPG zoP)Et>e@NjT@1K~*vdieaptp_?1!pz=U{Id^fJ4qxlyg1cED$_#o60D2m3uZSJ<_9 zi3V%)9Pp7#b_2QsLYwE{par~n-Zu>n2{hgi&gMDb<2J(G>y(4`VcW8Y1bVCJn5)j_ zIpCv~Yz72YQSJw8ztFbK42_Jlc@FrXjqwkbq3u2HMrPqcfX!pyunzc`MMCHshKk8^ za66XN)(&aAc^b4EWc)(#Asgc#EJXW$+l|b^g#aIqU5GKwKm&yCj#oO1qF z@~>`(d;%Mc{Q-Q~b~aCAyyR^5Jm&LPVH~-34%;uaQy4F^fIP;0>Kcp#9v?QhPE(wx z$*C-Wd&4^9BNktR@B^^W#5ru=WE(F*a@zUweAT^S9rEECU>@j^_Q~3Zc||>RX(<1O zb;!pmz5*eaYRbIrDQ>(1>51nrfPcd}AZ|y~OA;+sBJ5o+UMzLbs z++2L5nwjS>!?rt8Z~7xuL-Ffyv+YxEm;`3#3jSSw9={NLq@ry=;40hUCKVHDhJ2f8 zDp)rnrXrt$UkE;6WBh}K*l$@T8tsJ}q3XV2pNNgmx#`bc{5sd!Ha8cayEx^7)o|zT zO@Hp<=ffW3zL;YizW}}9==K=%xr>%Uff(5~H!mgz3gN{RdW`wJH5hx_=4Sh>ZI~~B zRKXS_t@ocWB$(Y#*)-3qejBv$KV{w{33n(HdYL=#lniwGZ=E6pXiR zZt<~-)&X*Ozk9+cvlS zkxDrTDz?q7z9ZF4GoR;?ug4v!xBQW+0SD`_&+E`IWXv=b{JRX9NZnh%%`HAs(Ka9q zYTH44%hN;1I8B9+Rc)JFe89%|2Mf`5D0h>zaFdL7k2}R~`EwVuZ#YZg)=d_r#*u~N z*Ti~jjotF+E~W?Z7%x6|amocmcpme)ij1e_QRcVBsoJm(^epqa9Z8@2uIe@)sv-8l5=?&2Q#DCg_It^$ zER}N?-1bK*Zv0nS8~?&{+n>9bUCUVtx7&B!#+i&$ z#$w()8{N5k+n>9b{>x)5oX6AXRnE~SdV$H{Fuud*E?NqZj?9M}-)Wz@Q$#GYdpyQ` z-WrU(ZF8sg0qxBD1FOwxA|E>2<_;gPXdz&Nw$Qe@vqGqG!+el1V_|o1Qg`@h4KVk5 zu@Wcd$FCefIR>4{vAG2 z(KaALYTKcGvi9(LeYVXVK44?~gN3-$#q#K?F^}bi%F4dUF|QvhxhW`>g+F*Pedf-# z%VM$MOWQU<_23x9CKFp!hXXZ+QGh#XsN zaOYDt!2M;w^TXDs>H_78I_T0!5$?PI$s=!w;4>CY1w;fiyI+F#wY6a@=$YrQONpd3 z`12VX;vX%;!ZUU??~bDVW&VW65aQ2RN*WBtj$esa8C`6!$)E5T3j8Un$3$z+xX6G@|bNLg<2xW(AzLpP=%Sd88F+1pLQD(b6zC#>aa z*Pq3hHrwmi-Mv6}lm2U+c{pc9l3M4qKZ|$$S&W&z1DxCU--gYDbFK$xK8w+QDD&qB zE2nyP+t=MGoELg^oISkDXRyIJuz7Y5C)a;jPMns=d7EdKk6g4HFavvR^X#?{+=dw> z=bbMjSG~=%%g1eid!$p^KH@&yS5YYCdn%()_^3sj0U1?Q?(OSr!{*U5&tHVV=Go7!a13LA%%nR@Y~TcI?hR0pPAaA#T8#GD z+p`#js8tptuzB|US&Zc}{8_x$e9&F&(LsRmM)5Y!oA8^hXt+u1w z*$;bs7Nh-8<_~S2z4kdgh4Z>>o;^N;4bFkhv)4X0 z-SU;lvGe7i?ILl0o;^Np1KcB>(mre(Hjkid6@?PmJR(CB+6)M+rrcAWsMe(?P|184 zGzku563KUjv8GpZMjh0LDz2zl3nNFjxX6cq`o**qzW)IyubOdWH( zwakIdv(Ikf`toJvw4DaQUtHME!sEVhQ(l}p1&BEz0DIOl+s|L#b}?s z-4v{{7=g_rQcB^uhGnw6{g84=k>%Mq#BhQIybiXUw|PVoDT8xx7Vj@Ui?KS`0x|-e z`7BmAhc?fC`{Fyr`gPen`+No)oFhKB&);dB*JJaDL{MlqV8ylA=1Iw*T57&wH^_PC z%ShnoIpBM3fP0`*Qc|cs{8mvKZNtsIZ5{B97HtM(R86_3JW;JBXxI~a=J|^d_<2OO zs1W~X8TMN?kHT@`LUh{}%6!b1nO><-oNZx|W-+<(6x??ON_wA{>Idj)-0V8%M;hBVyMPvFnJ~ zbwunsB6b}SyN(El&YmM;&k?cbM>!()91(kth&@Ndo+Dz<5wYipaKQH+5&MpaeMiK; zpT`lg?}*rUMC>~v_8k%XjtB?pz!7oah&XUW95^Bl{LGGs14qQcEi3Pbw&A92a6@j} zU)(X8VbA;^NuO2~N%%~I8?tYIabO!P=1K0%q!#leb>~OnO|7UldPFvSj>~8X>DwQBstSlBr!7$7V{)z<`-^Ki+PeTGsI$kU@^7_6N5u=C>{8w=L$kE#|i^ z=C>{8w=L$kE#|i!Vit3IHr%n8-?5nAv6$bnnBTFO-?5nAv6$bnnBTFOr>OB%jPG|h zhTq;8e)#^=({G+UKR9^e{wwbsY=3yz z`rzos)s3AG4I3X^-Prr~Pd$ERg$mb*P*xTArUAKe&5i?+?;|>pyk+=={;~<44~iBCki|Byg|9z8&F^i4Ncp;b9-(@FO0F8Uy~!#(-57kO8A)Ch(s#(~KR+gi0@< zup5tUGa{`=>=qdG(x7v5mv=m<{&{wk;W!uf@=qeD1P*NNqDra;D=Jw zv8e*fi8}p0h9%TKmAxq1U#fl6gP>8%+HRm0)xPbPh;oK7 z?aS#ZA}`cF{<;W4)&;0bbq6w`(hDe{+Baj{j1X#Hjmx|U!4Wqqy?_j=eLZP~Bdqoz zr@XXqunC!fYMjs2S?VEWkD;{x%Hi3gQ_Eb3o-8IfY_k7Njfgxwco;E@x zERHiUS|U`HPp$Tgyaj5X+M~VDqxN9}w-*Ank3Y3)pW0)(gxbe!??&y*W@Ig@T-66e zqS{w|61bOtR{LU5COUvaf*(5JKf<=B=!F#2MNI z4$|0DAG1pvwJ)2IwWxAc9}tOZU-e1gUjA9_i$R&_01gR$*atYE+LzN+L|&+U z{PifALeuIFWCChh(+en|+BdyzMhLa9#${fF;D{TQUO)!bzMizg5mx({*Ec~&^^Q%* z1lJA_Q5DngCE?Mwf*(pz$EFG_+Q|l1huWv~*-q`79t4fL&vpZ~sP=8QB-)F;Q2Tmd zW8%z>qUzPa7c;7RMPFEfK29 zr&jw#J_EH+?a^N7QTs4~+Y5o($DdlYPwg>XLhWPPV59bBGqM&{uId9KQSGZf3Eay+ zt9>yj6CJ=I!4LZY2UPoVx{AmPwU56ZB~xfx-GNL%O>24q1yuW{x6KHl_SLw|ix3=f zqtXk=pxW1yRye|HAG4??=&0VY37O#90V1km`n@DP+E(yGDeBl%fkivn!0J%@lv&uR zeba-WQTN$ypcd7>?UqD)u@`Dz56n!6A<}x+x&a|nZf4%fiVL+*185UCNMlcxGtgb* z40O-uOc@w*X6$JrM8e`Y1EVEERr%CvzsP5x_NhJE3q5KdCUAQpQ2Y2(tM;iqrc0=O zOrL7hzHCO;qRLf$KqRVt)hB^_`De8+24$iHI3)OCAK-v$UrtvMd7<|4*P~<#O{+VQ z38-mJFQ9;G-}JT_A=JJamw6F_BW_fB0U1>LdeRCI<;?l5H#vO+YQvB+PB@3XfO6c?dyS=2{A-k?^-t? zgv!m#TUl|T_GtiZ0tadAsd5IoYn*}Z`J5>OL(YsnZG=cz9A{v(M5rpCTJ0D44Aefg zM|+`1?ZX6aF9d2Ie`?h}wa0V`wU7A}joO#Z$XZmnst<@nwXgala4-L?_Qjw~bO470 zKkNe>Q0>d5N-rRTYF|%U z;RvgJOoo@Bqk6|CWP)o4h^UI`_mc2vTfq;dsAE$F7VTsMt3&Nm>Pe^eO%H-b-DkUj zT2%YCTN3TXUZ{ONFf$>BNb6ne282+#nRzQKF4R5^piST)jXhP)KzEHZ&^@0sWnjpe zv8Rm?35(+ljFt#he( z5U73psa5;b9@8b%J|+z_YF{=ZYfK&Vq39cO=qAI4}OTwdV1wWLcj!hL(5JKf<=B=!_Q2R81Hi3gQ_Eb3o-8IfY_k7Njfgxwco;E@xERHiU zS|U`HPp$Tgd0ZKfn_5)Hs79%x_L$oVAu(7Mo zA?UAi2>KUrsEmv`GsJ zjS<2kB1VkjnovNC$n+pHYTbYa+KP5`n<2th)B`h5Vu-|*S~nns7Ll2^A|gU0%eR5M zza8W;p+{zBRDja@SNj1eQi5jciUfolqRrufja_vPL4TD)(7%8~Wn|2uv1^mm#n^3qukOr3R$mX~c zh^1pfjg$=b`$8NOVpJwR0!14l?gTvOn2js4An9%c9IP;6co8xg?lE&swfFZ~6_;+N2ZUjhg$&p?2XX@ls zM3Zg|=C2q4U;NX*u`zt|{Q2>t)0~6p=zGbSPLI+_kBS04Zg^3wEE0PX;fHyh(7?nd z6b07~t7TwS7Hkk(@y2y~yF<}7JqWti4H)27w4=MXq8=!>qUmWPM8^?sMLSv|#HUfj z8Kq4Q{yC4{VKC}PYlXL2XJV4 z!#=Gv@#q1^QC<$KAPI2j;u zhBg7JQPYEqQtJjezyr~amP_Onf>8T1)UBJG4)Hz0&+-^|;$7qr?h@)oFlYLE6p zkJ^U`++GOOKK|62ThuNY&HGNMeYDZdqmFFGwMLbz`hZAO`>Ia@_p8imUku7b2XJV4 z!#=;*c@csmZd7^!8C3gv(h5gd z?U#%$TJ2*KG6B^-wF5*{`_wK8XYq5eq^QGKqS3%|YM*`|!xCy==O~hD-}E4vs7q`& zP>agVc1uLO*bC*R2WBS35Q!JHZa@f?o0+$=;t^+P6F5j?Pn9#!UE>UN&*w}T7;r8R(wRnKCfs%-GXLh=j#)21ZMSs`9DTev!{W?NfWS7kbn_OyKrH zp!V^nR_#-JOqWpmn5oC8ec6nxMU|`ifJjvPs!sy<^3Q5t49Y|Ya7ggOKEMIhzMQTi z@s{*xgiyJec`GX})IJTMP2eDnJyp&?ca1a9J)bjWV91%Vr;QK^i{lK8 zmIzhlQ>**Hr}mgGq4qITk5T)w8Ci=eSM>prsP%i4Ne9;D>#H1FC&FT}9-D+Q(myk|{K;?m#A>rZv5Q0;+w}+h&AN`)XX~ zMF@_#QRxL_Q0?nUD;#09UoyIAwU1561XTOf4iHflQ@bQQ+E(yGDeBl%fkivn!0J%@ zlyb?UqD)u@`Dz56n!6A<}x+x&a|nZf4%fiVL+*185UCNMlcx zGtgb*40O-uOc@w*X6$JrM8e`Y1EVEERr%CvzsP5x_NhJE3q5KdCUAQpQ2Y2(tM;iq zrc0=O%+zDlzHCO;qRLf$KqRVt)hB^_`De8+24$iHI3)OCAK-v$UrtvMd7<|4*P~<# zO{+VQ38-mJFQ9;G-}JT_A=JJamw6F_BW_fB0U1>LdeRCJQ{@bF*Ej>+^Ep!nhMXCD+6a-bIL^RmiBMHOwc0Q8 z8K`|~kM=^3+J_0;UI^4a{?w{{YLDp>Y9BN87_~2(k+rCDRUZ(EYG3t9;9mY&?TbN~ z=l~80e%J>%pxT$yRYYE>ef;$(nL^X*4rBsqTGIJ z%VuOPs$A6vM55YPeG<5re^&cqP$oKnLxLan0S>74<#ZL17iu4WJxZp~w7LVCfST6y z0t%@1O>dhKLhY+@nHM2A;zp$xkU_PtC#`UV)qct7qSZb&ArnyTQ#(LJRZQ)Y@Mv4X z52dJMQw0|7WCN>1?NiDjr}j+`f=1nEyMbC%`?gyW?ZsZGeLXNUA%;lnUF!yfP`R0T zD=RM4J`JEv;2@1XRn9=>=p^?dwS^9AULz zGP-EBk4?w~RQuEp5K$FVyCgi?R`5e9>ey6)MLXHR>QMWXa>%KD(}SQ<_t|cs7S+D( zmPC887iwP*%uI+O(t6jr0U=awX5Pw*3$;%JXcIU{V^5Vc&|TvUbkFBZ85nYA>}exJ z!s0jsqa{LB`P6E^$Y-GTsXf{YJ!&5&aC;$8`}k9<_NhIlOQ?O!)MM1XY)00i%2j

YhsA)|vpnz)M^tKrx z)V>;*c@csmZd7^!8C3gv(h5gJmJR&IHi4&P8rX(x@Erh@zUZeBl6c-)9$qHbPD_;f zjtab*`jGaY0qvOYqzJ2oi zdMAhIEbz;s4})qyEE{Fnpk;3!2Ucds^FWKHI1qeCD6{n=!Nc==L>vkh9vymmPv#Nv zo=g@A8g!JJ2hIgbV1eI$=62j2aVHQ}MPRyZR>vV0pjUJpp} zunn2vI{+$)B#n>^2#*Q)rP>16R)goa0F8*@$zy`P?excl=|b?NZ)`tMO~-`om&o|- zAdd+>GBcwBl-9r64@i*`G*ee3Amk8j4i9YXs&feXs~m#<1sp0PV-Ag76G)Npcn-mM ziBw%?%`s6VFghlvBitih#{^8__egY1;E(b0p!a0oF@fKN+G9etBb!M*t4<)6jtSK% z8SM9kI3~oXOnd~2HbmSBc+fE+XRK@mc}(EXM{$)}S@$3_NGqFeK!c76)7@r>JSNn@ Z%##=*ai!J`2+=X2=dEz&_jxj={||kw$*uqZ literal 0 HcmV?d00001 diff --git a/gpt2/src/model.py b/gpt2/src/model.py new file mode 100644 index 0000000..c66c23d --- /dev/null +++ b/gpt2/src/model.py @@ -0,0 +1,176 @@ +import numpy as np +import tensorflow as tf +from tensorflow.contrib.training import HParams + +def default_hparams(): + return HParams( + n_vocab=0, + n_ctx=1024, + n_embd=768, + n_head=12, + n_layer=12, + ) + +def shape_list(x): + """Deal with dynamic shape in tensorflow cleanly.""" + static = x.shape.as_list() + dynamic = tf.shape(x) + return [dynamic[i] if s is None else s for i, s in enumerate(static)] + +def softmax(x, axis=-1): + x = x - tf.reduce_max(x, axis=axis, keepdims=True) + ex = tf.exp(x) + return ex / tf.reduce_sum(ex, axis=axis, keepdims=True) + +def gelu(x): + return 0.5*x*(1+tf.tanh(np.sqrt(2/np.pi)*(x+0.044715*tf.pow(x, 3)))) + +def norm(x, scope, *, axis=-1, epsilon=1e-5): + """Normalize to mean = 0, std = 1, then do a diagonal affine transform.""" + with tf.variable_scope(scope): + n_state = x.shape[-1].value + g = tf.get_variable('g', [n_state], initializer=tf.constant_initializer(1)) + b = tf.get_variable('b', [n_state], initializer=tf.constant_initializer(0)) + u = tf.reduce_mean(x, axis=axis, keepdims=True) + s = tf.reduce_mean(tf.square(x-u), axis=axis, keepdims=True) + x = (x - u) * tf.rsqrt(s + epsilon) + x = x*g + b + return x + +def split_states(x, n): + """Reshape the last dimension of x into [n, x.shape[-1]/n].""" + *start, m = shape_list(x) + return tf.reshape(x, start + [n, m//n]) + +def merge_states(x): + """Smash the last two dimensions of x into a single dimension.""" + *start, a, b = shape_list(x) + return tf.reshape(x, start + [a*b]) + +def conv1d(x, scope, nf, *, w_init_stdev=0.02): + with tf.variable_scope(scope): + *start, nx = shape_list(x) + w = tf.get_variable('w', [1, nx, nf], initializer=tf.random_normal_initializer(stddev=w_init_stdev)) + b = tf.get_variable('b', [nf], initializer=tf.constant_initializer(0)) + c = tf.reshape(tf.matmul(tf.reshape(x, [-1, nx]), tf.reshape(w, [-1, nf]))+b, start+[nf]) + return c + +def attention_mask(nd, ns, *, dtype): + """1's in the lower triangle, counting from the lower right corner. + + Same as tf.matrix_band_part(tf.ones([nd, ns]), -1, ns-nd), but doesn't produce garbage on TPUs. + """ + i = tf.range(nd)[:,None] + j = tf.range(ns) + m = i >= j - ns + nd + return tf.cast(m, dtype) + + +def attn(x, scope, n_state, *, past, hparams): + assert x.shape.ndims == 3 # Should be [batch, sequence, features] + assert n_state % hparams.n_head == 0 + if past is not None: + assert past.shape.ndims == 5 # Should be [batch, 2, heads, sequence, features], where 2 is [k, v] + + def split_heads(x): + # From [batch, sequence, features] to [batch, heads, sequence, features] + return tf.transpose(split_states(x, hparams.n_head), [0, 2, 1, 3]) + + def merge_heads(x): + # Reverse of split_heads + return merge_states(tf.transpose(x, [0, 2, 1, 3])) + + def mask_attn_weights(w): + # w has shape [batch, heads, dst_sequence, src_sequence], where information flows from src to dst. + _, _, nd, ns = shape_list(w) + b = attention_mask(nd, ns, dtype=w.dtype) + b = tf.reshape(b, [1, 1, nd, ns]) + w = w*b - tf.cast(1e10, w.dtype)*(1-b) + return w + + def multihead_attn(q, k, v): + # q, k, v have shape [batch, heads, sequence, features] + w = tf.matmul(q, k, transpose_b=True) + w = w * tf.rsqrt(tf.cast(v.shape[-1].value, w.dtype)) + + w = mask_attn_weights(w) + w = softmax(w) + a = tf.matmul(w, v) + return a + + with tf.variable_scope(scope): + c = conv1d(x, 'c_attn', n_state*3) + q, k, v = map(split_heads, tf.split(c, 3, axis=2)) + present = tf.stack([k, v], axis=1) + if past is not None: + pk, pv = tf.unstack(past, axis=1) + k = tf.concat([pk, k], axis=-2) + v = tf.concat([pv, v], axis=-2) + a = multihead_attn(q, k, v) + a = merge_heads(a) + a = conv1d(a, 'c_proj', n_state) + return a, present + + +def mlp(x, scope, n_state, *, hparams): + with tf.variable_scope(scope): + nx = x.shape[-1].value + h = gelu(conv1d(x, 'c_fc', n_state)) + h2 = conv1d(h, 'c_proj', nx) + return h2 + + +def block(x, scope, *, past, hparams): + with tf.variable_scope(scope): + nx = x.shape[-1].value + a, present = attn(norm(x, 'ln_1'), 'attn', nx, past=past, hparams=hparams) + x = x + a + m = mlp(norm(x, 'ln_2'), 'mlp', nx*4, hparams=hparams) + x = x + m + return x, present + +def past_shape(*, hparams, batch_size=None, sequence=None): + return [batch_size, hparams.n_layer, 2, hparams.n_head, sequence, hparams.n_embd // hparams.n_head] + +def expand_tile(value, size): + """Add a new axis of given size.""" + value = tf.convert_to_tensor(value, name='value') + ndims = value.shape.ndims + return tf.tile(tf.expand_dims(value, axis=0), [size] + [1]*ndims) + +def positions_for(tokens, past_length): + batch_size = tf.shape(tokens)[0] + nsteps = tf.shape(tokens)[1] + return expand_tile(past_length + tf.range(nsteps), batch_size) + + +def model(hparams, X, past=None, scope='model', reuse=False): + with tf.variable_scope(scope, reuse=reuse): + results = {} + batch, sequence = shape_list(X) + + wpe = tf.get_variable('wpe', [hparams.n_ctx, hparams.n_embd], + initializer=tf.random_normal_initializer(stddev=0.01)) + wte = tf.get_variable('wte', [hparams.n_vocab, hparams.n_embd], + initializer=tf.random_normal_initializer(stddev=0.02)) + past_length = 0 if past is None else tf.shape(past)[-2] + h = tf.gather(wte, X) + tf.gather(wpe, positions_for(X, past_length)) + + # Transformer + presents = [] + pasts = tf.unstack(past, axis=1) if past is not None else [None] * hparams.n_layer + assert len(pasts) == hparams.n_layer + for layer, past in enumerate(pasts): + h, present = block(h, 'h%d' % layer, past=past, hparams=hparams) + presents.append(present) + results['present'] = tf.stack(presents, axis=1) + h = norm(h, 'ln_f') + + # Language model loss. Do tokens 1): - generate_cache() - else: - app.run(host='0.0.0.0', port=8080) + app.run(host='0.0.0.0', port=8080)