From 04593dbc795b492ae6685949af01a2a168b9e27b Mon Sep 17 00:00:00 2001 From: Nick Walton Date: Tue, 9 Apr 2019 12:59:26 -0600 Subject: [PATCH] split repo, this one is just server --- download_model.py | 28 ---- generator.py | 78 --------- gpt2/CONTRIBUTORS.md | 17 -- gpt2/DEVELOPERS.md | 85 ---------- gpt2/LICENSE | 21 --- gpt2/README.md | 61 ------- gpt2/__init__.py | 0 gpt2/__pycache__/__init__.cpython-37.pyc | Bin 134 -> 0 bytes gpt2/download_model.py | 28 ---- gpt2/src/__pycache__/encoder.cpython-37.pyc | Bin 4961 -> 0 bytes gpt2/src/__pycache__/model.cpython-37.pyc | Bin 6708 -> 0 bytes gpt2/src/__pycache__/sample.cpython-37.pyc | Bin 2603 -> 0 bytes gpt2/src/encoder.py | 117 ------------- gpt2/src/model.py | 174 -------------------- gpt2/src/sample.py | 79 --------- 15 files changed, 688 deletions(-) delete mode 100644 download_model.py delete mode 100644 generator.py delete mode 100644 gpt2/CONTRIBUTORS.md delete mode 100644 gpt2/DEVELOPERS.md delete mode 100644 gpt2/LICENSE delete mode 100644 gpt2/README.md delete mode 100644 gpt2/__init__.py delete mode 100644 gpt2/__pycache__/__init__.cpython-37.pyc delete mode 100644 gpt2/download_model.py delete mode 100644 gpt2/src/__pycache__/encoder.cpython-37.pyc delete mode 100644 gpt2/src/__pycache__/model.cpython-37.pyc delete mode 100644 gpt2/src/__pycache__/sample.cpython-37.pyc delete mode 100644 gpt2/src/encoder.py delete mode 100644 gpt2/src/model.py delete mode 100644 gpt2/src/sample.py diff --git a/download_model.py b/download_model.py deleted file mode 100644 index c5a8b89..0000000 --- a/download_model.py +++ /dev/null @@ -1,28 +0,0 @@ -import os -import sys -import requests -from tqdm import tqdm - -if len(sys.argv) != 2: - print('You must enter the model name as a parameter, e.g.: download_model.py 117M') - sys.exit(1) - -model = sys.argv[1] - -subdir = os.path.join('gpt2','models', model) -if not os.path.exists(subdir): - os.makedirs(subdir) -subdir = subdir.replace('\\','/') # needed for Windows - -for filename in ['checkpoint','encoder.json','hparams.json','model.ckpt.data-00000-of-00001', 'model.ckpt.index', 'model.ckpt.meta', 'vocab.bpe']: - - r = requests.get("https://storage.googleapis.com/gpt-2/" + subdir + "/" + filename, stream=True) - - with open(os.path.join(subdir, filename), 'wb') as f: - file_size = int(r.headers["content-length"]) - chunk_size = 1000 - with tqdm(ncols=100, desc="Fetching " + filename, total=file_size, unit_scale=True) as pbar: - # 1k for chunk_size, since Ethernet packet size is around 1500 bytes - for chunk in r.iter_content(chunk_size=chunk_size): - f.write(chunk) - pbar.update(chunk_size) diff --git a/generator.py b/generator.py deleted file mode 100644 index 0e6d80d..0000000 --- a/generator.py +++ /dev/null @@ -1,78 +0,0 @@ -import json -import os -import numpy as np -import tensorflow as tf - -import gpt2.src.model as model -import gpt2.src.sample as sample -import gpt2.src.encoder as encoder -from utils import * - -pos_action_starts = ["You attack", "You tell", "You use", "You go"] - - -class StoryGenerator(): - - def __init__(self, sess, length=75, temperature=0.9, top_k=40): - - seed = None - batch_size=1 - model_path='gpt2/models/117M' - self.sess = sess - - self.enc = encoder.get_encoder(model_path) - hparams = model.default_hparams() - with open(os.path.join(model_path, 'hparams.json')) as f: - hparams.override_from_dict(json.load(f)) - - self.context = tf.placeholder(tf.int32, [batch_size, None]) - np.random.seed(seed) - tf.set_random_seed(seed) - self.output = sample.sample_sequence( - hparams=hparams, length=length, - context=self.context, - batch_size=batch_size, - ) - - saver = tf.train.Saver() - ckpt = tf.train.latest_checkpoint(model_path) - saver.restore(self.sess, ckpt) - - - def generate(self, prompt): - context_tokens = self.enc.encode(prompt) - out = self.sess.run(self.output, feed_dict={ - self.context: [context_tokens for _ in range(1)] - })[:, len(context_tokens):] - - text = self.enc.decode(out[0]) - return text - - def generate_story_block(self, prompt): - block = self.generate(prompt) - block = cut_trailing_sentence(block) - block = story_replace(block) - - return block - - def generate_action_options(self, prompt, action_starts=pos_action_starts): - - possible_actions = [] - for phrase in action_starts: - action = phrase + self.generate(prompt + phrase) - action = first_sentence(action) - possible_actions.append(action) - - return possible_actions - - def generate_action_result(self, prompt, phrase): - action = phrase + self.generate(prompt + phrase) - action_result = cut_trailing_sentence(action) - action_result = story_replace(action_result) - - action = first_sentence(action) - - - return action, action_result - - diff --git a/gpt2/CONTRIBUTORS.md b/gpt2/CONTRIBUTORS.md deleted file mode 100644 index eab7132..0000000 --- a/gpt2/CONTRIBUTORS.md +++ /dev/null @@ -1,17 +0,0 @@ -# Contributors (alphabetically) - -* **[madisonmay](https://github.com/madisonmay)** - - Added Dockerfiles - -* **[Margaret Mitchell et al](https://arxiv.org/abs/1810.03993)** - - Our [usage](./README.md#usage) writeup was loosely inspired by the paper - [Model Cards for Model Reporting](https://arxiv.org/abs/1810.03993) - and related conversations with some of the authors. - -* **[webproduktion01](https://github.com/webproduktion01)** - - Ported download script to python. - -**[Full code contributors list](https://github.com/openai/gpt-2/contributors).** diff --git a/gpt2/DEVELOPERS.md b/gpt2/DEVELOPERS.md deleted file mode 100644 index 078999b..0000000 --- a/gpt2/DEVELOPERS.md +++ /dev/null @@ -1,85 +0,0 @@ -# Installation - -Git clone this repository, and `cd` into directory for remaining commands -``` -git clone https://github.com/openai/gpt-2.git && cd gpt-2 -``` - -Then, follow instructions for either native or Docker installation. - -## Native Installation - -All steps can optionally be done in a virtual environment using tools such as `virtualenv` or `conda`. - -Install tensorflow 1.12 (with GPU support, if you have a GPU and want everything to run faster) -``` -pip3 install tensorflow==1.12.0 -``` -or -``` -pip3 install tensorflow-gpu==1.12.0 -``` - -Install other python packages: -``` -pip3 install -r requirements.txt -``` - -Download the model data -``` -python3 download_model.py 117M -``` - -## Docker Installation - -Build the Dockerfile and tag the created image as `gpt-2`: -``` -docker build --tag gpt-2 -f Dockerfile.gpu . # or Dockerfile.cpu -``` - -Start an interactive bash session from the `gpt-2` docker image. - -You can opt to use the `--runtime=nvidia` flag if you have access to a NVIDIA GPU -and a valid install of [nvidia-docker 2.0](https://github.com/nvidia/nvidia-docker/wiki/Installation-(version-2.0)). -``` -docker run --runtime=nvidia -it gpt-2 bash -``` - -# Running - -| WARNING: Samples are unfiltered and may contain offensive content. | -| --- | - -Some of the examples below may include Unicode text characters. Set the environment variable: -``` -export PYTHONIOENCODING=UTF-8 -``` -to override the standard stream settings in UTF-8 mode. - -## Unconditional sample generation - -To generate unconditional samples from the small model: -``` -python3 src/generate_unconditional_samples.py | tee /tmp/samples -``` -There are various flags for controlling the samples: -``` -python3 src/generate_unconditional_samples.py --top_k 40 --temperature 0.7 | tee /tmp/samples -``` - -To check flag descriptions, use: -``` -python3 src/generate_unconditional_samples.py -- --help -``` - -## Conditional sample generation - -To give the model custom prompts, you can use: -``` -python3 src/interactive_conditional_samples.py --top_k 40 -``` - -To check flag descriptions, use: -``` -python3 src/interactive_conditional_samples.py -- --help -``` diff --git a/gpt2/LICENSE b/gpt2/LICENSE deleted file mode 100644 index cb36e12..0000000 --- a/gpt2/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2019 OpenAI - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/gpt2/README.md b/gpt2/README.md deleted file mode 100644 index c1be039..0000000 --- a/gpt2/README.md +++ /dev/null @@ -1,61 +0,0 @@ -# gpt-2 - -Code and samples from the paper ["Language Models are Unsupervised Multitask Learners"](https://d4mucfpksywv.cloudfront.net/better-language-models/language-models.pdf). - -For now, we have only released a smaller (117M parameter) version of GPT-2. - -See more details in our [blog post](https://blog.openai.com/better-language-models/). - -## Usage - -This repository is meant to be a starting point for researchers and engineers to experiment with GPT-2-117M. While GPT-2-117M is less proficient than GPT-2-1.5B, it is useful for a wide range of research and applications which could also apply to larger models. - -### Some caveats - -- GPT-2-117M robustness and worst case behaviors are not well-understood. As with any machine-learned model, carefully evaluate GPT-2-117M for your use case, especially if used without fine-tuning or in safety-critical applications where reliability is important. -- The dataset our GPT-2-117M was trained on contains many texts with [biases](https://twitter.com/TomerUllman/status/1101485289720242177) and factual inaccuracies, and thus GPT-2-117M is likely to be biased and inaccurate as well. -- To avoid having samples mistaken as human-written, we recommend clearly labeling samples as synthetic before wide dissemination. Our models are often incoherent or inaccurate in subtle ways, which takes more than a quick read for a human to notice. - -### Work with us - -Please [let us know](mailto:languagequestions@openai.com) if you’re doing interesting research with or working on applications of GPT-2-117M! We’re especially interested in hearing from and potentially working with those who are studying -- Potential malicious use cases and defenses against them (e.g. the detectability of synthetic text) -- The extent of problematic content (e.g. bias) being baked into the models and effective mitigations - -## Development - -See [DEVELOPERS.md](./DEVELOPERS.md) - -## Contributors - -See [CONTRIBUTORS.md](./CONTRIBUTORS.md) - -## GPT-2 samples - -| WARNING: Samples are unfiltered and may contain offensive content. | -| --- | - -While we have not yet released GPT-2 itself, you can see some samples from it in the `gpt-2-samples` folder. -We show unconditional samples with default settings (temperature 1 and no truncation), with temperature 0.7, and with truncation with top_k 40. -We show conditional samples, with contexts drawn from `WebText`'s test set, with default settings (temperature 1 and no truncation), with temperature 0.7, and with truncation with top_k 40. - -## Citation - -Please use the following bibtex entry: -``` -@article{radford2019language, - title={Language Models are Unsupervised Multitask Learners}, - author={Radford, Alec and Wu, Jeff and Child, Rewon and Luan, David and Amodei, Dario and Sutskever, Ilya}, - year={2019} -} -``` - -## Future work - -We may release code for evaluating the models on various benchmarks. - -We are still considering release of the larger models. - -## License - -[MIT](./LICENSE) diff --git a/gpt2/__init__.py b/gpt2/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/gpt2/__pycache__/__init__.cpython-37.pyc b/gpt2/__pycache__/__init__.cpython-37.pyc deleted file mode 100644 index b59cf9edfb25f9867a46ca775ef716bfa086cffe..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmZ?b<>g`k0;NT(V?gv{5CH>>K!yVl7qb9~6oz01O-8?!3`HPe1o2B>KO;XkRX;B? zIlDYDrzAg5KRvTV-^EuqIJKxOwMaj`pu|W&K0Y%qvm`!Vub}c4hfQvNN@-529mt?! HAZ7pnJ}w=% diff --git a/gpt2/download_model.py b/gpt2/download_model.py deleted file mode 100644 index 30ba84a..0000000 --- a/gpt2/download_model.py +++ /dev/null @@ -1,28 +0,0 @@ -import os -import sys -import requests -from tqdm import tqdm - -if len(sys.argv) != 2: - print('You must enter the model name as a parameter, e.g.: download_model.py 117M') - sys.exit(1) - -model = sys.argv[1] - -subdir = os.path.join('models', model) -if not os.path.exists(subdir): - os.makedirs(subdir) -subdir = subdir.replace('\\','/') # needed for Windows - -for filename in ['checkpoint','encoder.json','hparams.json','model.ckpt.data-00000-of-00001', 'model.ckpt.index', 'model.ckpt.meta', 'vocab.bpe']: - - r = requests.get("https://storage.googleapis.com/gpt-2/" + subdir + "/" + filename, stream=True) - - with open(os.path.join(subdir, filename), 'wb') as f: - file_size = int(r.headers["content-length"]) - chunk_size = 1000 - with tqdm(ncols=100, desc="Fetching " + filename, total=file_size, unit_scale=True) as pbar: - # 1k for chunk_size, since Ethernet packet size is around 1500 bytes - for chunk in r.iter_content(chunk_size=chunk_size): - f.write(chunk) - pbar.update(chunk_size) diff --git a/gpt2/src/__pycache__/encoder.cpython-37.pyc b/gpt2/src/__pycache__/encoder.cpython-37.pyc deleted file mode 100644 index c23a36bf6ab621533c368b197ccaa846bad3497f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4961 zcmbtYU2_~q744p{on5UJRSuB= z;waNWx&bR}TzNZKC-a1f2*RM7DkZaFniwNZ#F7YRlR>5xY?m#XwSLr>fs#8?Wzl*p zgY}`rY$3B?H&vTKngrWVpA}`nK!#9aKiv)zDQVj{R6QBIVpJN>>h?nwb~Tm{qJ(5s zu#LnXdm-HD* zMWPU*qZp-=R1Ly-A;`l0h2V{Aw~e99Nyh6`N6`IHF9f?XFei)CbQ22c5Vp;Nb*XnH zR6g|=HYSyyX5#nAlVlYbIvft^Jh0PlKhp4Q*p*A&G|?)JgDDehIJ5F#upX+73?3{M ze8kT`;vYZcfd4;$Blg#m^75zh@@I4T+h0}`NjzleyL3lC5~mniMa$;V^?>>NO(2xe z$$x8YIjigs48do2`>JIX*5cKICu#J9a^rA&2~Okhj3INX2KSSH+d)4}liS7>9>!ea-o63nxXPCH|(EiT6ZRGeMMO*NtA z6%IWVt_qW$EbLT?!h;n7kirpBR~Pm{G;BGAwVoALH&e4%z2NJoI;&@BWbVn`d{TvM zz-Cr>lRJEt`@Dg^iKbq__{3~vG8gS8TJ|k;njL~7Ko=ZTVeP$!I|M%w2dLsKsDdB! z+{*0@%MgPNQh4z$OpM@n4t`YzeLzqLD6{>+I($!TXd)}5comBc6VrEbOQ2SA2pRyA zDLS)2Zx0dSlgU|dsa%DCN$}bURYX9&63e8g`xQ^LN&p#PC_2yzTx>&+!VZOi4ISFL zaE&hQI=0;D5GD(U`X?<~?@8S;qA!zOn&_D8!HCZqQ>p;ZT^g)@Hg4WU%NEf|hP1+v zQfA0>N4#y9Nd(EIOd)QWK-!*YP|_%B-=S0?RZ)X6VuVeX+(N&{vOPY}x7z#0tJy@SW3PgINR)0o(-6 z6$niNXKabZPA5tt-RVpP&Sb*>t&(~LTdG&7n}oknOMMMvSEuy(!Nn}UsB;i}esKqF zkY9}B{G!N%ch-i#{9tssyjd|fKl}MwW=7GijP75~*Rso(-Z{Uv+M>JGTu~b{N8y4W z2U$@A+Z(@oQmIsBCNzsCg>$D8EuY4M>dT}v8DBk%uCNgVMIB_?0flU4g==bx!cwxR zk!_A`ND$MQc57gu<_ke;F?`R8ov_kDo>J(LcvM`(W-# zKXr-u=tm_O@ejczy9aj8`KQ3?rl;oVnuAtWQP~hDk1TBw3aS3c7KlQA!}sPio<3n4 zrZ7A5h5eZ|a&iZA9aJcH?y%eeuh4an%{pdw~oEs&HWQ= zRL^V2HPDrx*NMUmWz|RiU6uqK=>v7n_`P7&`#bEXEV1q}n)N)(-AzmVv)Vzp&#_I0 zXS@^cIfnP1d0#vBv71;Z+F-lUe-bI2^2t@cMhSFd92}sq#EPG!>j zaJDs%986xWtrVxpoor8=zk!xLhYo~2H!|@^arhJ9f6FpF&+%2Fz7C0&Wk_2+Pxnva z;Zc3LClk3hRM)$-B(3kTgY(tlrAcE`sHmh5TvJU>i2J`(j96=|Slh6j0m6T%r@jw4 zajKYFrg^?v!lkHf0A67ntJm?Q2+u%TnRef$k@soD&4&0=YdIy}91SS84O^NGGj)R` zi7l(W2~O%Zx?iAW=JON4nCCC^OZ;H=al1@#K`x5aDg6Mcpyelg7LZf9#(nGx`aK$29zvJDLi}X__7$T4sr<+mD}ln)!9r|FbAX* z&_}DX^o8nRWm=kY_@ub~UzMShGghYH-KVP(O{r1>3v!gm;5g&hI%QB)b>V*DjWeft zZI^!a#@_r8Qz|1E;xmW6gV7%CKwskw>F&amC(G3+ST5jYt46sp2?w&%DH@&5AQjs& z-8Va(t?e)_->5esYVc){qG$uN?Nn!WK93g^gBj&d6zomE<<}IAPOM3=siWOQ%RWX& ze+_tG;j0}WI-$Q4Sf3-pwR7x(UXU}8KiCi1N9^9Tq)n1klYVA@4p^Y1_fFWzK9w!& zR1i^3_$T;#foPN;0LWMl)sML&W3s1MLhn81`U9sHr-NkbA|x6mrQewE7k zO3kNyi50wIz8N0=3qLGRe4~goTlK<8@jX^pX+}9x_e+2|=By5qC%)8_424n5xT@tA z4d6h=_vYM4+bDC52|l> zpZn;0Uf((A;;osPs)gURpZ(K=?q$pR6<HxXlY&iclOgA-OL03ItZCjiqENHHA3lyuQyCjG z{?RfO+@$&4&E<{ZAX<)lovq!lpAO^YZZBQF_m?;BM`kB7%iU4>+Hzt#%Y&hg`mNF9 zd{#&6;dVc5Z;XTo4{IgdiKXo6pQ+)!!|n3NuTqJ;l2_4W)~5AT9a&?Q+E4Lzq<&<5 zWUW)2_7gvKGIgNtSqBuFv){TGh5cZ+mu>|5aU2eMogmo=M^Vs=gEWehp;_+_cY{tp z3giCc*7U3V8boDX;8$Nl)5Vt_UVUJX)!5E#ZEgAH>c>{5E?Y6)|K5J&;(KaE1k-eK zyLB^Hy<8Y zlM#72Ok`AUr|Wqoinj-m3Dc-)=L#xJ(lG6HimfO!i?t2EnpaPkZ}6)nG?raeRb?*W z>G=Qf>u@XlQ34$BHC)MSXfoK=+2aPT@FG6%49y_y z_x7V89R>q9T=13P=Jg;+H6CtV57Lb&4)ibx1KkU|!x)YmuCMoEj5lGNtYdo1EP>rz zbxnXfe5=-rdudNrHMv?7kb}*M;LH__S!q@Z)iicO(+k)7Q9J1jN0FK5Lf%f;-;VNn zH%i+l{rO^N7{i6)w0(B*+KIVh<8ZM@+o6ea&rr?y1hot2l$EJa(L==Z*4Q39 zsiIfX7c-#Y6{~AQK{oMe|4Z*j1^z+UK|f4V$a(-A>+Veo2jp0j!~>i5z@4hii$;qEQFq0DXEr`}5H z6mQP(WP>N*%akh^VdgAaU3lk&r!U_hgvkcYr{E~v9iDQPoM9^rl3v{HN2k5duw}l4 zElgN4Z{Rssp(yS5SOVI1VMkPLasvY=EX^r%2XEc)FaGm8|F!?gUD#)RSBwZ_b+pqV zR1iVvyi;WQ1TRwu{##|G(A`9ruDGg0yv@AK?mBeb47hbzI5Hj9(d;Q4{KS#rxKG7# zc?YJIGp}nO5%)@232!#vj_j;_$;w>LrYy&B<*CaO_xzPy?VdOqf{F$U0J1M&zlB)H zia2D`7rQW&MPVk(c*-xsZS`=_rfrA)v)*KGawO%Qmj+=v*zQk!$ovUucD!!xa)fBX zj`z%0`Bpek5!ecyNH8eDUw3Y4#;ByoCF-i7UV!(!uFN~2ir;Xvv@hD304J?a_!1x`rGe)Cuzwy zR^Y%Ac0DMi8|7{X&fhFS)EZZkdpdm#RGfiOau|e~jc@m!DYUk|gKWkJLvBnI98$AK}VH1ZNohIlPE=Uq`?~U^;Nd zehME*-A&jybUmXRht7fhz!_J@)o~5fpj!XBEeq6 zbtkC_^VCUFe#W(G1$eU(NdBA~pk^(5|1wM=K21^2jCy9@w2u3n#;BcIMO-?&W69G0 zz@8le(*juD!~8#xL)8i}mMwmh-}>||o*6aWBIQgG$QJ(!4@DS#;3D8mLJosXkx?NZ za#QE2n<1kD9D`1g10n#Sixzs2UvcCSd)+0Zj!cjcrK@$h=sdJD0xJgGkO2m$%v4QCdl`ZU^+-PJ@{-1F@WNBo}o7a<{29w%48sBZO!3$80DD$<$1XPa;F+_$XY}qg-v}YDZ*bzCxlh-{B~t zPLW?X5LnV4O+%#fAv%f(0jUJGuY6=y`&ZA&<(Vm~rH)J$hr$OWw+pb&@&BS&%Fzqd z-E7t;>150TCvUu&z{3cXUpHoG#LdnG$agS3cLw1INfUD;$(=e|d3l>@CmtBkI$_$J zD~bk&MDwSd9A z;*uEZQV3tiEJ@KU*vtZ?GiTRb-}U`^wGPTvp)H_YfZQlMvw<1YtcI>c*-LbN3lBsp z!QY~NFHj`*oiQ> z>Vo{5?D?1}4s4YwhBWBL=cPrxNl76SkiIdEm)z@gMMIGlHWTsb(^3L&4W|@a-$!@C zs>TxGU4?C2!DR-Zp#HC}g=uG_ouKl|{Umy{9mSof!{N*gc+qmiY8c;8wIEYqtq#;d zoZIBZDh8I(v{wpGbPN?zcowrdWH(~bF{wi~f&!;dP^Ig%rKEX+?jlMkLrqT0DsF{~ z$yNx%1@JLoAt_)2W234j@UgO{2~)Qyv${%qU2K*I2NH%$`=5DJ>mUr`XgA>T2dj{7 zZwCb~#n2L)1q8Y{l8PkgpT(Aw(}Z~@wg8fLI&2q*RRtD4#MW{*?e(L)hJzF4igK{x z$g((kz$Xj`ofpE%XiohI{gV@*29>pg_o*akK^`uV#7#ViwM(Lm1ep?mdXP33=`th$ z3c>=9OyJZkHJwQ!5@7Nq{U|oi@0fRmnwQey7K+cjCd%nYaW~yCRN-m~P>5rX^9;-& zcR{_6`n-+ffJ2O&oc=VB@tenZ&pkP8e7nQ#^T+eGspIY-n-ZnMHMKglf1-X2_w8B0 zATV*D;HD)x_HcksthJ!#F4)~HiK?MyqyawO9XPJF33r|LmthJ;e`Rt$7;8VSLohxf zpVAcwu8Ow`aziDxSS9n*Dy+SFfXE1^uXQbCo;+Kun18`HYIsIwduYd1jd~Z~sw2A2 zq%+yffxIC<{`a);1GV>hI!oF;>*1^5paBKVWi@2Kjf*@5u=LEK3wjONX#MGYHhZL$ z_2@soOGzwb_2L^q;Kpq$y?}YM2N;W;E@m}kuLLLI6484OGoH)lvI>vBA&GezHP2C$ zM%TfZ`D`9*E!+mwJ%i*Icy{fXXgi5=XLl4$s^llAlFggY5pJw3(U+d|eR5Y^!!T}} zpcumdyzgv$PM-xoLx2(lTt{MIn9Aj){t&0pN%P_v0F-o#rKzEH82Y+lpfa@gB7~Q^ zAr2CZ*pgWznLv{}Fi!J5zVgUXkS#BTqfr!V6O!^pHj=v0XPOHFN4ff-z*7m~gvp#o zC&(~Ra0x0FSe0?|n9UxW9-38u61|gEkKTa^at>dxTH0u@_fabT4TnFAbY>s4f54Tn zT(%IaT~(JXkeT2W^@8K7YZAkk;4=%V>RbeU5fA{$Enx;-mutM;yoi4aY!60{%}aRA ztEcDXBGT38PYEp`6$;n27E(GKn#EoDXMr5#7dbAIHaXoa1a+IQJS&lqErp{*Gzmu{ zT|s66r9u_l(HTxpf!)6ihu`i;ci`H^QN84KwqNl}RnK?k>b~cD4d1W%E#LPmbB>3Qs|EGrMK?O<&9kq%%9c7Pm$#lkkcIRuD;SZml27HG7F)Mzw4 z-4l`{t%ROSR))q0Ar~KgSovo7U*tdJ!fQ@@%Q=T6U$IBBuvnN4R#&rFEVAmWVt;by zPKTkr_1zDT{|WA2^l{x{Fdw0t`yi4@Ua&px-(ngF#@RI73ptZQ2Ip)qlH5NbW<=jc zGM24#u@_6e%hKdIR$x0rK`%6Vx6lK0^AX4rx)Y&$OR>u?_=PwSe%0h_X4x~2E@;M; zfP~9XM(4=|0~Zr_6~IDkM{}$*ME8wWCJV2kn?HhBcEpdc$`a?e6pzJ%EqG0*Vqd`) zq`^V3$rj+Ag-c$G328)_&yQjmTnZU(V3*!jw3BkjC(r+f1}>8Jbf!kIG02{Yf21uZ z?12+y=^|UrvWaVz>Lh%E$9XYV#_`N`rsFbeK2xE| zQ7$g3(b$^r*?&IRKCGr{yBrTECwXD3avOr}Pyh7(uF}U!Z;xj7!)>F7+a{mR3bi#m zb#dl5K~Xk65XLv~ggoZ!{Oolv%bNAK?ibZCFU*6jM&-trUJJzgl6Jgg@HpSmcfos- za^1h?^%^*r+%NL!zRVwN;LD)1vtPa3*!9GhnXA@Pr&VFu(hnX3lDc6E^tDacbwQ4$3I=6GJG@;U;l^BW$b%Lw;H%Kicn^6xQuJsd?R!-A#6*ACqn! zjsjSSY<$%6Db>3aSDZ$cY{aiQ$wn9*1@eyUU&fDvMF&r}?(VX`u+P|+*biW|wsW?< zJ5$5)!RbIf$%nQ$9Uwmj)xqGFUk7<92P=2JsecYf4asHH2qDxxI`_aJZ&}T-aX8^f zAlt)5PzMs*zHtHNJmwq)a0O+32syo@I(imT-q!$~4CzoDhpi)2G*qTE-bq8(I-KP? zpCVhobm1&F)nvt(jIvl@@S&4V%%w1X)4PB`cXpV;4;;sfq@sosU0%^yloe zhoA30M6t+@k%s%OZIsQ-VLnr?Lsl9-js3DN_HZ-$4J^OcE~Z&Ex3jrzAU+_Q+?)O} zLtJRAnvv+b4u}Cy_=XVt?2V>gWCod{%AxwP6p)U)`+``L?_VB(Zy+H=)O1n6&2T37 zh@r9@yby?>nL{D)B_8g64Z*NM9WH_6OQ0J1A+Q%wP-Ki1h>H>4K(;o4FA2n6l9d(( zX~b7waV_cy(ulrCbU}1bw7s_)7tskY;B%kglnU1^=hLh?sd2r=pP8@idVm?v|J#)N zkV<))_{ix50+-k-J1DB$`m*4<(|KXXWi=h=g^R|e{q2V?26l(Jbu9x2t22c#QsjUi zt&UyX7<0W6)nXM%vx3uVLuuH+={;!adqk*w+>Y*kB~L%V*s5H;&fn%ezAiR+|EJ= j - ns + nd - return tf.cast(m, dtype) - - -def attn(x, scope, n_state, *, past, hparams): - assert x.shape.ndims == 3 # Should be [batch, sequence, features] - assert n_state % hparams.n_head == 0 - if past is not None: - assert past.shape.ndims == 5 # Should be [batch, 2, heads, sequence, features], where 2 is [k, v] - - def split_heads(x): - # From [batch, sequence, features] to [batch, heads, sequence, features] - return tf.transpose(split_states(x, hparams.n_head), [0, 2, 1, 3]) - - def merge_heads(x): - # Reverse of split_heads - return merge_states(tf.transpose(x, [0, 2, 1, 3])) - - def mask_attn_weights(w): - # w has shape [batch, heads, dst_sequence, src_sequence], where information flows from src to dst. - _, _, nd, ns = shape_list(w) - b = attention_mask(nd, ns, dtype=w.dtype) - b = tf.reshape(b, [1, 1, nd, ns]) - w = w*b - tf.cast(1e10, w.dtype)*(1-b) - return w - - def multihead_attn(q, k, v): - # q, k, v have shape [batch, heads, sequence, features] - w = tf.matmul(q, k, transpose_b=True) - w = w * tf.rsqrt(tf.cast(v.shape[-1].value, w.dtype)) - - w = mask_attn_weights(w) - w = softmax(w) - a = tf.matmul(w, v) - return a - - with tf.variable_scope(scope): - c = conv1d(x, 'c_attn', n_state*3) - q, k, v = map(split_heads, tf.split(c, 3, axis=2)) - present = tf.stack([k, v], axis=1) - if past is not None: - pk, pv = tf.unstack(past, axis=1) - k = tf.concat([pk, k], axis=-2) - v = tf.concat([pv, v], axis=-2) - a = multihead_attn(q, k, v) - a = merge_heads(a) - a = conv1d(a, 'c_proj', n_state) - return a, present - - -def mlp(x, scope, n_state, *, hparams): - with tf.variable_scope(scope): - nx = x.shape[-1].value - h = gelu(conv1d(x, 'c_fc', n_state)) - h2 = conv1d(h, 'c_proj', nx) - return h2 - - -def block(x, scope, *, past, hparams): - with tf.variable_scope(scope): - nx = x.shape[-1].value - a, present = attn(norm(x, 'ln_1'), 'attn', nx, past=past, hparams=hparams) - x = x + a - m = mlp(norm(x, 'ln_2'), 'mlp', nx*4, hparams=hparams) - x = x + m - return x, present - -def past_shape(*, hparams, batch_size=None, sequence=None): - return [batch_size, hparams.n_layer, 2, hparams.n_head, sequence, hparams.n_embd // hparams.n_head] - -def expand_tile(value, size): - """Add a new axis of given size.""" - value = tf.convert_to_tensor(value, name='value') - ndims = value.shape.ndims - return tf.tile(tf.expand_dims(value, axis=0), [size] + [1]*ndims) - -def positions_for(tokens, past_length): - batch_size = tf.shape(tokens)[0] - nsteps = tf.shape(tokens)[1] - return expand_tile(past_length + tf.range(nsteps), batch_size) - - -def model(hparams, X, past=None, scope='model', reuse=False): - with tf.variable_scope(scope, reuse=reuse): - results = {} - batch, sequence = shape_list(X) - - wpe = tf.get_variable('wpe', [hparams.n_ctx, hparams.n_embd], - initializer=tf.random_normal_initializer(stddev=0.01)) - wte = tf.get_variable('wte', [hparams.n_vocab, hparams.n_embd], - initializer=tf.random_normal_initializer(stddev=0.02)) - past_length = 0 if past is None else tf.shape(past)[-2] - h = tf.gather(wte, X) + tf.gather(wpe, positions_for(X, past_length)) - - # Transformer - presents = [] - pasts = tf.unstack(past, axis=1) if past is not None else [None] * hparams.n_layer - assert len(pasts) == hparams.n_layer - for layer, past in enumerate(pasts): - h, present = block(h, 'h%d' % layer, past=past, hparams=hparams) - presents.append(present) - results['present'] = tf.stack(presents, axis=1) - h = norm(h, 'ln_f') - - # Language model loss. Do tokens