wassname/Clover-Edition
1
0 Fork 0
mirror of https://github.com/wassname/Clover-Edition.git synced 2026-10-29 12:40:16 +08:00
Code Issues Packages Projects Releases Wiki Activity

rmeoved old things

Browse Source
This commit is contained in:
Nick Walton committed 2019-11-18 12:22:22 -07:00
1 parent dda5ae9b2c
commit 56630dc6f8
61 files changed
-997265
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

generator/ctrl/.gitignore
-2
View File
@@ -1,2 +0,0 @@
fastBPE
model
generator/ctrl/LICENSE.txt
-12
View File
@@ -1,12 +0,0 @@
Copyright (c) 2019, Salesforce.com, Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
* Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
generator/ctrl/__init__.py
View File
Whitespace-only changes.
generator/ctrl/codes
-200000
View File
File diff suppressed because it is too large. Load diff
generator/ctrl/control_codes.txt
-55
View File
@@ -1,55 +0,0 @@
0.000296793 Pregnancy
0.000127197 Christianity
0.003084531 Explain
0.000180196 Fitness
6.88985E-05 Saving
0.000217295 Ask
8.47981E-05 Ass
0.000143097 Joke
0.000196096 Questions
0.000127197 Thoughts
0.000169596 Retail
0.000270294 Feminism
0.000111298 Writing
0.000402791 Atheism
1.05998E-06 Netflix
0.000365692 Computing
0.000132497 Opinion
0.000169596 Alone
0.000323293 Funny
0.000249094 Gaming
0.000402791 Human
0.000132497 India
2.11995E-08 Joker
0.000201395 Diet
0.000238495 Legal
6.35986E-06 Norman
3.60392E-07 Tip
0.000302093 Weight
0.000132497 Movies
0.000111298 Running
7.41983E-05 Science
0.00135147 Horror
0.000291493 Confession
0.000190796 Finance
0.000413391 Politics
7.41983E-05 Scary
0.000206695 Support
6.35986E-05 Technologies
0.000243795 Teenage
0.000217295 Event
0.000206695 Learned
0.000121897 Notion
0.0847981 Wikipedia
0.095927851 Books
0.001176574 Extract
0.000127197 Confessions
0.000227895 Conspiracy
0.365691808 Links
0.000423991 Narcissus
0.000280894 Relationship
0.000922179 Relationships
0.153696557 Reviews
0.043877717 News
0.129847091 Translation
0.111297507 multilingual
generator/ctrl/ctrl_generator.py
-327
View File
@@ -1,327 +0,0 @@
import tensorflow as tf
import numpy as np
tf.enable_eager_execution()
import generator.ctrl.model.transformer as transformer
import re
from collections import Counter
from tensorflow.python import debug as tf_debug
from tensorflow.python.ops import math_ops
from tensorflow.python.ops import embedding_ops
import fastBPE
from story.utils import *
import warnings
warnings.filterwarnings("ignore")
# the loss function is a simple categorical crossentropy between the logits and the labels
def loss(labels, logits):
return tf.keras.losses.sparse_categorical_crossentropy(labels, logits, from_logits=True)
class CTRLGenerator():
def __init__(self, control_code="Apocalypse ", generate_num=40, temperature=0.4, topk=40, nucleus_prob=0):
self.generate_num=generate_num
model_dir = "generator/ctrl/model/aidungeon2model/"
self.control_code = control_code
vocab_file = 'generator/ctrl/model/vocab'
code_file = 'generator/ctrl/model/codes'
self.max_new_lines = 5
# load the vocabulary from file
vocab = open(vocab_file, encoding='utf-8').read().split('\n')
vocab = list(map(lambda x: x.split(' ')[0], vocab)) + ['<unk>'] + ['\n']
print('{} unique words'.format(len(vocab)))
# length of the vocabulary
vocab_size = len(vocab)
# define the numericalization map
# idx2word maps the numericalized ID to the word
# word2idx maps the word to the numericalized ID
self.word2idx = {u: i for i, u in enumerate(vocab)}
self.idx2word = np.array(vocab)
# sequence length to use for the transformer
# the model is trained with a self.seq_length of 512
# so, any value <= 512 should work
self.seq_length = min(self.generate_num, 256)
# the dimension of the transformer
embedding_dim = 1280
# input for the keras model
tokens = tf.keras.layers.Input(shape=(self.seq_length,), dtype='int32')
# Now, we begin defining the model
# we defer the transformer definition to transformer.py
# here, we only define the tied softmax layer
# this layer ties the softmax weights to the input embeddings
class TiedEmbeddingSoftmax(tf.keras.layers.Layer):
def __init__(self, vocab_size=vocab_size, embedding_size=embedding_dim, **kwargs):
super(TiedEmbeddingSoftmax, self).__init__()
self.w = self.add_weight(name='w', shape=(vocab_size, embedding_size),
initializer='random_normal',
trainable=True)
self.b = self.add_weight(name='b', shape=(vocab_size,),
initializer='zeros',
trainable=True)
def call(self, inputs, embed=True):
if embed:
dtype = tf.keras.backend.dtype(inputs)
if dtype != 'int32' and dtype != 'int64':
inputs = math_ops.cast(inputs, 'int32')
return embedding_ops.embedding_lookup(self.w, inputs)
else:
return tf.tensordot(inputs, tf.transpose(self.w), 1) + self.b
# instantiates a tied softmax class
tied_embedding_softmax = TiedEmbeddingSoftmax()
# embedded tokens, before passing it to the transformer
embedded = tied_embedding_softmax(tokens, embed=True)
# the activations after passing it from the transformer
# for some odd reason, TPUs don't play well with specifying the arguments of the Encoder() function
# so you have to leave them at their defaults
transformed = transformer.Encoder()(embedded, training=False)
# pass the activations from our tiedsoftmax class
# this time with embed=False denoting that we are doing the softmax operation
# and not a lookup
logits = tied_embedding_softmax(transformed, embed=False)
# finally, define the Keras model with inputs as tokens and outputs as the logits we just computed
model = tf.keras.Model(inputs=tokens, outputs=logits)
# the optimizer is not used since this code only supports inference
# however, to compile the model, we still define it
optimizer = tf.contrib.tpu.CrossShardOptimizer(
tf.contrib.estimator.clip_gradients_by_norm(
tf.train.AdagradOptimizer(learning_rate=1e-2), 0.25)
)
# compile the model with the optimizer and loss
model.compile(optimizer=optimizer, loss=loss)
print(model.summary())
# IMPORTANT
# this is where the saved model is presented to the code
# the model directory should have the model checkpoint and
# a checkpoint file
run_config = tf.contrib.tpu.RunConfig(
model_dir=model_dir)
# this converts the Keras model to a TensorFlow estimator
# this step is critical
# remember to patch the TF 1.14 file before running the code, else you're going to see errors here
estimator_model = tf.keras.estimator.model_to_estimator(keras_model=model, config=run_config)
# we now create a serving function from this estimator
# this enables us to load the model once and easily query it multiple times
def serving_input_fn():
inputs = {'input_1': tf.placeholder(tf.int32, [1, self.seq_length])}
return tf.estimator.export.ServingInputReceiver(inputs, inputs)
self.predict_fn = tf.contrib.predictor.from_estimator(estimator_model, serving_input_fn)
# almost there, we now take the user prompt and tokenize with BPE
# load BPE codes
self.bpe = fastBPE.fastBPE(code_file, vocab_file)
self.temperature=temperature
self.nucleusprob = nucleus_prob
self.penalty = 1.2
self.topk=topk
def configure_verb_probs(self, probabilities, options):
# Make sure only a possible verb is chosen.
for word in get_possible_verbs():
probabilities[self.word2idx[word]] += 100
# Disallow used verbs
if "used_verbs" in options:
for verb in options["used_verbs"]:
if verb in self.word2idx:
probabilities[self.word2idx[verb]] = -1e8
return probabilities
def prompt_replace(self, prompt):
# print("\n\nBEFORE PROMPT_REPLACE:")
# print(repr(prompt))
if prompt[-1] != " ":
prompt = prompt + " "
prompt = second_to_first_person(prompt)
prompt = self.control_code + prompt
# print("\n\nAFTER PROMPT_REPLACE")
# print(repr(prompt))
return prompt
def result_replace(self, result):
# print("\n\nBEFORE RESULT_REPLACE:")
# print(repr(result))
result = cut_trailing_sentence(result)
first_letter_capitalized = result[0].isupper()
result = result.replace('."', '".')
result = result.replace("#", "")
result = result.replace("*", "")
result = first_to_second_person(result)
result = remove_profanity(result)
if not first_letter_capitalized:
result = result[0].lower() + result[1:]
#
# print("\n\nAFTER RESULT_REPLACE:")
# print(repr(result))
return result
def generate_next_token(self, token, tokens_generated, options, num_new_lines, token_num, first_token=False, forbid_newline=False):
# get the logits from the prediction function
# the logic here is a bit convoluted because we are allowing generation past 512 tokens
# this is done by sliding the window over (past 512 tokens) and continuing prediction
# I'm sure this can be simplified (TODO)
if token <= self.seq_length:
prompt_logits = self.predict_fn({'input_1': tokens_generated[:, :self.seq_length]})[
'tied_embedding_softmax'].squeeze() / (self.temperature if self.temperature > 0 else 1.)
_token = token if token < self.seq_length else -1
else:
_token = -1
end = token + 1
start = token - self.seq_length + 2
prompt_logits = \
self.predict_fn({'input_1': np.hstack((tokens_generated[:, 0:1], tokens_generated[:, start:end]))})[
'tied_embedding_softmax'].squeeze() / (self.temperature if self.temperature > 0 else 1.)
# if penalty (for repetition) is non-zero,
# discount the logits from already generated tokens
if self.penalty > 0:
penalized_so_far = set()
for _ in range(token + 1):
generated_token = tokens_generated[0][_]
if generated_token not in penalized_so_far:
penalized_so_far.add(generated_token)
prompt_logits[_token][generated_token] /= self.penalty
# disallow some tokens
forbidden_tokens = ['<unk>', 'Sco@@', "&amp@@", "1]@@", "2]@@", "3]@@", "4]@@", "https://www.@@", "[@@", ":@@",
"Edit", "&@@", "2:","1:", ":", "Edit@@", "EDI@@", "EDIT@@", "edit", "TL@@", "tl@@", ";@@",
'**', "http://@@", "Redd@@", "UP@@", "mom", "Up@@", "Me:", "Update", "mom@@", "Part",
"http://www.@@", "edit@@", "*@@", "Writing", "Text@@", "\\@@", "<br>@@", "<div", "|@@", '...',
'..','…', 'https://@@', '...@@', "http://gutenberg@@"]
#encourage_tokens = ["zombie", "radiation", "fallout", "undead", "corpse", "vampire", "virus", "plague"]
encourage_tokens = []
for encourage_token in encourage_tokens:
prompt_logits[_token][self.word2idx[encourage_token]] *= 1.2
for forbidden_token in forbidden_tokens:
prompt_logits[_token][self.word2idx[forbidden_token]] = -1e8
last_ind = tokens_generated[0][token]
if forbid_newline:
prompt_logits[_token][self.word2idx['\n']] = -1e8
else:
prompt_logits[_token][self.word2idx['\n']] *= 1.0
# Set whitelist
if "word_whitelist" in options and token_num in options["word_whitelist"].keys():
for word in options["word_whitelist"][token_num]:
prompt_logits[_token][self.word2idx[word]] += 100
# Set blacklist, overwrites whitelist
if "word_blacklist" in options and token_num in options["word_blacklist"].keys():
for word in options["word_blacklist"][token_num]:
prompt_logits[_token][self.word2idx[word]] = -1e8
# compute probabilities from logits
prompt_probs = np.exp(prompt_logits[_token])
prompt_probs = prompt_probs / sum(prompt_probs)
pruned_list = np.argsort(prompt_probs)[::-1]
# if you are using nucleus prob, then compute the nucleus probability size
if self.nucleusprob > 0.:
minimum_topk = 1
nucleus = max(np.where(np.cumsum(np.sort(prompt_probs)[::-1]) > self.nucleusprob)[0][0], minimum_topk)
elif self.topk > 0:
nucleus = self.topk
else:
nucleus = len(pruned_list)
pruned_list = pruned_list[:nucleus]
# if temperature is 0
# just pick the first (most probable) token
if self.temperature == 0:
idx = pruned_list[0]
else:
# else,
# sample from the pruned_list with the logits
chosen_idx = int(
tf.random.categorical(np.expand_dims(prompt_logits[_token][pruned_list], 0), num_samples=1).numpy())
idx = pruned_list[chosen_idx]
return idx
def generate(self, prompt, options=None):
prompt = self.prompt_replace(prompt)
debug_print = True
if debug_print:
print("\n\n*****DEBUG*****")
print("Prompt is:")
print(prompt + "\n\n")
if options is None: options = dict()
if "used_verbs" not in options:
options["used_verbs"] = set()
first_token = True
# tokenize provided prompt
split_prompt = self.bpe.apply([prompt])[0].split()
text = [self.word2idx[i] for i in split_prompt]
total_text_len = len(text) + self.generate_num
# pad with 0s and create a mini-batch of 2 (arbitrary, for ease of code)
padded_text = text + [0] * (total_text_len - len(text))
tokens_generated = np.tile(padded_text, (1, 1))
result = ""
token_num = 0
num_new_lines = 0
for token in range(len(text) - 1, total_text_len - 1):
idx = self.generate_next_token(token, tokens_generated, options, num_new_lines, token_num, first_token=first_token, forbid_newline=False)
is_nothing = len(cut_trailing_sentence(result)) == 0 or len(cut_trailing_quotes(result)) == 0
if self.idx2word[idx] == '\n' and token_num > 7 and not is_nothing:
return self.result_replace(result)
elif self.idx2word[idx] == '\n':
idx = self.generate_next_token(token, tokens_generated, options, num_new_lines, token_num,
first_token=first_token, forbid_newline=True)
# assign the token for generation
tokens_generated[0][token + 1] = idx
if debug_print:
print(repr(self.idx2word[idx]), end="_")
tokens_generated_so_far = ' '.join([self.idx2word[c] for c in tokens_generated[0][len(text):token+2]])
tokens_generated_so_far = re.sub('(@@ )', '', string=tokens_generated_so_far)
tokens_generated_so_far = re.sub('(@@ ?$)', '', string=tokens_generated_so_far)
result = tokens_generated_so_far
token_num += 1
if debug_print:
print("\n****END DEBUG*****\n")
return self.result_replace(result)
generator/ctrl/download_model.sh
-4
View File
@@ -1,4 +0,0 @@
URL="gs://aidungeon2model"
gsutil -m cp -r "$URL" model
generator/ctrl/estimator.patch
-24
View File
@@ -1,24 +0,0 @@
47c47
<
---
> import tensorflow as tf
228c228
< def _create_keras_model_fn(keras_model, custom_objects=None):
---
> def _create_keras_model_fn(keras_model, params=None, custom_objects=None):
239c239
< def model_fn(features, labels, mode):
---
> def model_fn(features, labels, mode, params=None):
448c448
< if keras_model._is_graph_network:
---
> if False:
462,464c462,464
< estimator = estimator_lib.Estimator(keras_model_fn,
< config=config,
< warm_start_from=warm_start_path)
---
> estimator = tf.contrib.tpu.TPUEstimator(keras_model_fn, use_tpu=False, train_batch_size=4, eval_batch_size=4,
> config=config,
> warm_start_from=warm_start_path)
generator/ctrl/generation.py
-289
View File
@@ -1,289 +0,0 @@
from __future__ import division
from __future__ import print_function
import tensorflow as tf
import os
import numpy as np
tf.enable_eager_execution()
import transformer
import argparse
import pdb
import sys
import re
from collections import Counter
from tensorflow.python import debug as tf_debug
from tensorflow.python.ops import math_ops
from tensorflow.python.ops import embedding_ops
import fastBPE
import platform
use_py3 = platform.python_version()[0] == '3'
parser = argparse.ArgumentParser(description='TensorFlow code for generating from CTRL')
parser.add_argument('--model_dir', type=str, required=True,
help='location of model checkpoint')
parser.add_argument('--seed', type=int, default=1337,
help='random seed for TensorFlow, numpy and PythonHash')
parser.add_argument('--generate_num', type=int, default=256,
help='number of tokens to generate')
parser.add_argument('--temperature', type=float, default=0,
help='temperature for sampling distribution; 0 means greedy')
parser.add_argument('--nucleus', type=float, default=0.,
help='cumulative probability cutoff for nucleus sampling; 0 means no nucleus sampling')
parser.add_argument('--topk', type=int, default=0,
help='topk value for sampling from the softmax distribution ; 0 means no topk preferred')
parser.add_argument('--penalty', type=float, default=1.2,
help='repetition penalty for greedy sampling')
parser.add_argument('--print_once', action='store_true',
help='the completion is printed only at the end; not every word')
parser.add_argument('--topn', type=int, default=0,
help='print top-n candidates during generations; defaults to 0 which is no printing')
args = parser.parse_args()
tf.random.set_random_seed(args.seed)
os.environ['PYTHONHASHSEED'] = str(args.seed)
np.random.seed(args.seed)
# load the vocabulary from file
vocab = open('vocab').read().decode(encoding='utf-8').split('\n') if not use_py3 else open('vocab', encoding='utf-8').read().split('\n')
vocab = list(map(lambda x: x.split(' ')[0], vocab)) + ['<unk>'] + ['\n']
print ('{} unique words'.format(len(vocab)))
# length of the vocabulary
vocab_size = len(vocab)
# define the numericalization map
# idx2word maps the numericalized ID to the word
# word2idx maps the word to the numericalized ID
word2idx = {u:i for i, u in enumerate(vocab)}
idx2word = np.array(vocab)
# sequence length to use for the transformer
# the model is trained with a seq_length of 512
# so, any value <= 512 should work
seq_length = min(args.generate_num, 256)
# the dimension of the transformer
embedding_dim = 1280
# Now, we begin defining the model
# we defer the transformer definition to transformer.py
# here, we only define the tied softmax layer
# this layer ties the softmax weights to the input embeddings
class TiedEmbeddingSoftmax(tf.keras.layers.Layer):
def __init__(self, vocab_size=vocab_size, embedding_size=embedding_dim, **kwargs):
super(TiedEmbeddingSoftmax, self).__init__()
self.w = self.add_weight(name='w', shape=(vocab_size, embedding_size),
initializer='random_normal',
trainable=True)
self.b = self.add_weight(name='b', shape=(vocab_size,),
initializer='zeros',
trainable=True)
def call(self, inputs, embed=True):
if embed:
dtype = tf.keras.backend.dtype(inputs)
if dtype != 'int32' and dtype != 'int64':
inputs = math_ops.cast(inputs, 'int32')
return embedding_ops.embedding_lookup(self.w, inputs)
else:
return tf.tensordot(inputs, tf.transpose(self.w), 1) + self.b
# input for the keras model
tokens = tf.keras.layers.Input(shape=(seq_length,), dtype='int32')
# instantiates a tied softmax class
tied_embedding_softmax = TiedEmbeddingSoftmax()
# embedded tokens, before passing it to the transformer
embedded = tied_embedding_softmax(tokens, embed=True)
# the activations after passing it from the transformer
# for some odd reason, TPUs don't play well with specifying the arguments of the Encoder() function
# so you have to leave them at their defaults
transformed = transformer.Encoder()(embedded, training=False)
# pass the activations from our tiedsoftmax class
# this time with embed=False denoting that we are doing the softmax operation
# and not a lookup
logits = tied_embedding_softmax(transformed, embed=False)
# finally, define the Keras model with inputs as tokens and outputs as the logits we just computed
model = tf.keras.Model(inputs=tokens, outputs=logits)
# the loss function is a simple categorical crossentropy between the logits and the labels
def loss(labels, logits):
return tf.keras.losses.sparse_categorical_crossentropy(labels, logits, from_logits=True)
# the optimizer is not used since this code only supports inference
# however, to compile the model, we still define it
optimizer = tf.contrib.tpu.CrossShardOptimizer(
tf.contrib.estimator.clip_gradients_by_norm(
tf.train.AdagradOptimizer(learning_rate=1e-2), 0.25)
)
# compile the model with the optimizer and loss
model.compile(optimizer=optimizer, loss=loss)
print(model.summary())
# IMPORTANT
# this is where the saved model is presented to the code
# the model directory should have the model checkpoint and
# a checkpoint file
run_config = tf.contrib.tpu.RunConfig(
model_dir=args.model_dir)
# this converts the Keras model to a TensorFlow estimator
# this step is critical
# remember to patch the TF 1.14 file before running the code, else you're going to see errors here
estimator_model = tf.keras.estimator.model_to_estimator(keras_model=model, config=run_config)
# we now create a serving function from this estimator
# this enables us to load the model once and easily query it multiple times
def serving_input_fn():
inputs = {'input_1': tf.placeholder(tf.int32, [1,seq_length])}
return tf.estimator.export.ServingInputReceiver(inputs, inputs)
predict_fn = tf.contrib.predictor.from_estimator(estimator_model, serving_input_fn)
# almost there, we now take the user prompt and tokenize with BPE
# load BPE codes
bpe = fastBPE.fastBPE('codes', 'vocab')
temperature = args.temperature
nucleusprob = args.nucleus
penalty = args.penalty
topk = args.topk
while True:
prompt = raw_input('ENTER PROMPT: ') if not use_py3 else input('ENTER PROMPT: ')
# tokenize provided prompt
split_prompt = bpe.apply([prompt])[0].split()
text = [word2idx[i] for i in split_prompt]
# pad with 0s and create a mini-batch of 2 (arbitrary, for ease of code)
padded_text = text + [0] * (args.generate_num - len(text))
tokens_generated = np.tile(padded_text, (1,1))
try:
for token in range(len(text)-1, args.generate_num-1):
# get the logits from the prediction function
# the logic here is a bit convoluted because we are allowing generation past 512 tokens
# this is done by sliding the window over (past 512 tokens) and continuing prediction
# I'm sure this can be simplified (TODO)
if token <= seq_length:
prompt_logits = predict_fn({'input_1':tokens_generated[:, :seq_length]})['tied_embedding_softmax'].squeeze() / (temperature if temperature>0 else 1.)
_token = token if token < seq_length else -1
else:
_token = -1
end = token + 1
start = token - seq_length + 2
prompt_logits = predict_fn({'input_1':np.hstack((tokens_generated[:,0:1], tokens_generated[:,start:end]))})['tied_embedding_softmax'].squeeze() / (temperature if temperature>0 else 1.)
# if penalty (for repetition) is non-zero,
# discount the logits from already generated tokens
if penalty>0:
penalized_so_far = set()
for _ in range(token+1):
generated_token = tokens_generated[0][_]
# don't penalize newlines
# you could also choose not to penalize frequent words
# (which incidentally are sorted in the vocab file)
# but I don't do that
# if it prints too many new lines instead of continuing generating text,
# you might want to comment this out
if idx2word[generated_token] == '\n':
continue
if generated_token in penalized_so_far:
continue
penalized_so_far.add(generated_token)
prompt_logits[_token][generated_token] /= penalty
# disallow some tokens
prompt_logits[_token][word2idx['<unk>']] = -1e8
prompt_logits[_token][word2idx['\n']] = -1e8
# sometimes, when generating from reddit,
# it tries to generate the Score (reddit Karma) immediately after generating the Title:
# to disallow this, we can just prevent it from generating Score
prompt_logits[_token][word2idx['Sco@@']] = -1e8
# compute probabilities from logits
prompt_probs = np.exp(prompt_logits[_token])
prompt_probs = prompt_probs / sum(prompt_probs)
pruned_list = np.argsort(prompt_probs)[::-1]
# if you are using nucleus prob, then compute the nucleus probability size
if nucleusprob > 0.:
minimum_topk = 1
nucleus = max(np.where(np.cumsum(np.sort(prompt_probs)[::-1])>nucleusprob)[0][0], minimum_topk)
elif topk > 0:
# we are over-loading notation here
# if you choose to specify a topk instead of a nucleus,
# we will hardcode the nucleus to be just that
nucleus = topk
else:
# if you specify neither nucleus or topk,
# then we will use the whole list
nucleus = len(pruned_list)
pruned_list = pruned_list[:nucleus]
# if you want to disallow more complex tokens, you can do so here
# for instance, if you want to disallow anything with the phrase `http`,
# you can delete theme from the pruned_list
# you can comment this out, I'm keeping it in for demonstration purpose
tokens_to_disallow = []
for _ in range(len(pruned_list)):
if 'http' in idx2word[pruned_list[_]]:
tokens_to_disallow.append(_)
pruned_list = np.delete(pruned_list, tokens_to_disallow)
if args.topn > 0 :
print('TOPN :: top-n alternatives:', [idx2word[_] for _ in pruned_list[:args.topn]])
# if temperature is 0
# just pick the first (most probable) token
if temperature==0:
idx = pruned_list[0]
else:
# else,
# sample from the pruned_list with the logits
chosen_idx = int(tf.random.categorical(np.expand_dims(prompt_logits[_token][pruned_list],0), num_samples=1).numpy())
idx = pruned_list[chosen_idx]
if args.topn > 0 :
print('TOPN :: chosen word:', idx2word[idx])
# assign the token for generation
tokens_generated[0][token+1] = idx
# clear screen if you want to
# os.system("clear")
tokens_generated_so_far = ' '.join([idx2word[c] for c in tokens_generated[0].squeeze()[:token+2]])
tokens_generated_so_far = re.sub('(@@ )', '', string=tokens_generated_so_far)
tokens_generated_so_far = re.sub('(@@ ?$)', '', string=tokens_generated_so_far)
if not args.print_once:
print('---------------------------------------')
print(tokens_generated_so_far)
print()
print('---------------------------------------')
print(tokens_generated_so_far)
print()
except KeyboardInterrupt: #Exception as e:
print('Continuing')
generator/ctrl/install_ctrl_py2.sh
-27
View File
@@ -1,27 +0,0 @@
#!/bin/bash
cd model
# Cython is needed to compile fastBPE
pip install Cython
# Patch the TensorFlow estimator package
export FILE="/usr/local/lib/python2.7/dist-packages/tensorflow_estimator/python/estimator/keras.py"
patch -b "$FILE" estimator.patch
# Install fastBPE
git clone https://github.com/glample/fastBPE.git
cd fastBPE
python setup.py install
cd ../..
# Download the 512-length model if specified, 256-length otherwise
#if [ "$1" = "512" ]
#then
# URL="gs://sf-ctrl/seqlen512_v1.ckpt/"
#else
# URL="gs://sf-ctrl/seqlen256_v1.ckpt/"
#fi
# Copy model
#gsutil -m cp -r "$URL" .
generator/ctrl/install_ctrl_py3.sh
-27
View File
@@ -1,27 +0,0 @@
#!/bin/bash
cd model
# Cython is needed to compile fastBPE
pip install Cython
# Patch the TensorFlow estimator package
export FILE="/usr/local/lib/python3.6/dist-packages/tensorflow_estimator/python/estimator/keras.py"
patch -b "$FILE" estimator.patch
# Install fastBPE
git clone https://github.com/glample/fastBPE.git
cd fastBPE
python setup.py install
cd ../..
# Download the 512-length model if specified, 256-length otherwise
#if [ "$1" = "512" ]
#then
# URL="gs://sf-ctrl/seqlen512_v1.ckpt/"
#else
# URL="gs://sf-ctrl/seqlen256_v1.ckpt/"
#fi
# Copy model
#gsutil -m cp -r "$URL" .
generator/ctrl/install_gcloud.sh
-3
View File
@@ -1,3 +0,0 @@
curl https://sdk.cloud.google.com | bash
exec -l $SHELL
gcloud init
generator/ctrl/model/.gitignore
-1
View File
@@ -1 +0,0 @@
fastBPE
generator/ctrl/model/CODE_OF_CONDUCT.md
-105
View File
@@ -1,105 +0,0 @@
# Salesforce Open Source Community Code of Conduct
## About the Code of Conduct
Equality is a core value at Salesforce. We believe a diverse and inclusive
community fosters innovation and creativity, and are committed to building a
culture where everyone feels included.
Salesforce open-source projects are committed to providing a friendly, safe, and
welcoming environment for all, regardless of gender identity and expression,
sexual orientation, disability, physical appearance, body size, ethnicity, nationality,
race, age, religion, level of experience, education, socioeconomic status, or
other similar personal characteristics.
The goal of this code of conduct is to specify a baseline standard of behavior so
that people with different social values and communication styles can work
together effectively, productively, and respectfully in our open source community.
It also establishes a mechanism for reporting issues and resolving conflicts.
All questions and reports of abusive, harassing, or otherwise unacceptable behavior
in a Salesforce open-source project may be reported by contacting the Salesforce
Open Source Conduct Committee at ossconduct@salesforce.com.
## Our Pledge
In the interest of fostering an open and welcoming environment, we as
contributors and maintainers pledge to making participation in our project and
our community a harassment-free experience for everyone, regardless of gender
identity and expression, sexual orientation, disability, physical appearance,
body size, ethnicity, nationality, race, age, religion, level of experience, education,
socioeconomic status, or other similar personal characteristics.
## Our Standards
Examples of behavior that contributes to creating a positive environment
include:
* Using welcoming and inclusive language
* Being respectful of differing viewpoints and experiences
* Gracefully accepting constructive criticism
* Focusing on what is best for the community
* Showing empathy toward other community members
Examples of unacceptable behavior by participants include:
* The use of sexualized language or imagery and unwelcome sexual attention or
advances
* Personal attacks, insulting/derogatory comments, or trolling
* Public or private harassment
* Publishing, or threatening to publish, others' private information—such as
a physical or electronic address—without explicit permission
* Other conduct which could reasonably be considered inappropriate in a
professional setting
* Advocating for or encouraging any of the above behaviors
## Our Responsibilities
Project maintainers are responsible for clarifying the standards of acceptable
behavior and are expected to take appropriate and fair corrective action in
response to any instances of unacceptable behavior.
Project maintainers have the right and responsibility to remove, edit, or
reject comments, commits, code, wiki edits, issues, and other contributions
that are not aligned with this Code of Conduct, or to ban temporarily or
permanently any contributor for other behaviors that they deem inappropriate,
threatening, offensive, or harmful.
## Scope
This Code of Conduct applies both within project spaces and in public spaces
when an individual is representing the project or its community. Examples of
representing a project or community include using an official project email
address, posting via an official social media account, or acting as an appointed
representative at an online or offline event. Representation of a project may be
further defined and clarified by project maintainers.
## Enforcement
Instances of abusive, harassing, or otherwise unacceptable behavior may be
reported by contacting the Salesforce Open Source Conduct Committee
at ossconduct@salesforce.com. All complaints will be reviewed and investigated
and will result in a response that is deemed necessary and appropriate to the
circumstances. The committee is obligated to maintain confidentiality with
regard to the reporter of an incident. Further details of specific enforcement
policies may be posted separately.
Project maintainers who do not follow or enforce the Code of Conduct in good
faith may face temporary or permanent repercussions as determined by other
members of the project's leadership and the Salesforce Open Source Conduct
Committee.
## Attribution
This Code of Conduct is adapted from the [Contributor Covenant][contributor-covenant-home],
version 1.4, available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html.
It includes adaptions and additions from [Go Community Code of Conduct][golang-coc],
[CNCF Code of Conduct][cncf-coc], and [Microsoft Open Source Code of Conduct][microsoft-coc].
This Code of Conduct is licensed under the [Creative Commons Attribution 3.0 License][cc-by-3-us].
[contributor-covenant-home]: https://www.contributor-covenant.org (https://www.contributor-covenant.org/)
[golang-coc]: https://golang.org/conduct
[cncf-coc]: https://github.com/cncf/foundation/blob/master/code-of-conduct.md
[microsoft-coc]: https://opensource.microsoft.com/codeofconduct/
[cc-by-3-us]: https://creativecommons.org/licenses/by/3.0/us/
generator/ctrl/model/LICENSE.txt
-12
View File
@@ -1,12 +0,0 @@
Copyright (c) 2019, Salesforce.com, Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
* Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
generator/ctrl/model/README.md
-356
View File
@@ -1,356 +0,0 @@
# CTRL - A Conditional Transformer Language Model for Controllable Generation
Authors: [Nitish Shirish Keskar](http://keskarnitish.github.io), [Bryan McCann](https://bmccann.github.io/), [Lav Varshney](http://www.varshney.csl.illinois.edu/), [Caiming Xiong](http://www.stat.ucla.edu/~caiming/), and [Richard Socher](https://www.socher.org/)
## Introduction
Large-scale language models show promising text generation capabilities, but
users cannot easily control this generation process. We release *CTRL*, a 1.6 billion-parameter conditional
transformer language model, trained to condition on control codes that specify
domain, subdomain, entities, relationships between entities, dates, and task-specific behavior. Control codes were derived from structure that naturally co-occurs with raw text, preserving the advantages of unsupervised learning while providing more explicit control over text generation.
Paper link: https://arxiv.org/abs/1909.05858
Blog link: https://blog.einstein.ai/introducing-a-conditional-transformer-language-model-for-controllable-generation/
The code currently supports two functionalities:
1. Generating from a trained model, two models are available for download - one with a sequence length of 256 and another with a sequence length of 512 -- they are trained with word-level vocabularies and through a sliding window approach can generate well beyond their trained sequence lengths.
2. Source attribution - given a prompt, prints the perplexity of the prompt conditional on each domain control code (see Section 5 of the paper).
Please refer to the argument flags for more details regarding the options available for either.
## Table of Contents
1. [Citation](#citation)
2. [License](#license)
3. [Questions for Deliberation](#questions-for-deliberation)
4. [Usage](#usage)
5. [Sample Generations](#generations)
6. [Sample Source Attributions](#source-attributions)
7. [FAQs](#faqs)
8. [Get Involved](#get-involved)
## Citation
```
@article{keskarCTRL2019,
title={{CTRL - A Conditional Transformer Language Model for Controllable Generation}},
author={Keskar, Nitish Shirish and McCann, Bryan and Varshney, Lav and Xiong, Caiming and Socher, Richard},
journal={arXiv preprint arXiv:1909.05858},
year={2019}
}
```
## License
The code is released under the BSD-3 License (see `LICENSE.txt` for details), but we also ask that users respect the following:
This software should not be used to promote or profit from:
violence, hate, and division,
environmental destruction,
abuse of human rights, or
the destruction of people's physical and mental health.
We encourage users of this software to tell us about the applications in which they are putting it to use by emailing ctrl-monitoring@salesforce.com, and to use [appropriate](https://arxiv.org/abs/1810.03993) [documentation](https://www.partnershiponai.org/about-ml/) when developing high-stakes applications of this model.
## Questions for Deliberation
We consulted extended members of the AI community in the responsible publication of this model. In particular, a preview of a [Partnership on AI (PAI)](http://partnershiponai.org) project relating to AI research publication norms was considered prior to the release of this work. While this PAI project is as-yet unpublished, it is informed by companies, organizations, and people differently affected by artificial intelligence and presents key considerations to evaluate before publishing potentially high-impact research.
The questions referenced from the early draft of the PAI project included:
1. How do you envision your research being used in the world? Who will use it? How much expertise is required to use it?
2. Who will use it?
3. Why would they be motivated to replicate / productionize your work?
4. How would a science fiction author turn your research into a dystopian story?
5. What is the worst way someone could use your research finding, given no resource constraints?
6. What are the historical patterns of misuse or application in this area? How can the research be made more robust against such misuse?
7. Which populations or communities will this technology negatively affect, deployed in the scenarios you envision? Will some groups be disproportionately affected?
## Usage
Here are the steps to get generating:
1. Install the dependencies
This code relies on [TensorFlow 1.14](https://www.tensorflow.org/install) and [fastBPE](https://github.com/glample/fastBPE).
TensorFlow can be installed via `pip install tensorflow[-gpu]==1.14`. fastBPE installation instructions can be found in the GitHub repository linked above. We highly recommend experimenting within a virtualenv or Docker image.
2. Patch the `/usr/local/lib/python2.7/dist-packages/tensorflow_estimator/python/estimator/keras.py` (or equivalent, if installed elsewhere) by running
```patch -b <path_to_tensorflow_estimator_package>/python/estimator/keras.py estimator.patch```
We highly recommend experimenting within a virtualenv or Docker image since the workflow involves patching a TensorFlow file to support some custom functionality. This step is not optional; skipping this step will cause errors (irrespective of device).
3. Get the model files from `gs://sf-ctrl/seqlen256_v1.ckpt/` or `gs://sf-ctrl/seqlen512_v1.ckpt/`.
The model architecture is identical for both checkpoints. The former is trained with lower training sequence length (256) while the latter is trained with a larger one (512). We plan to update the models (with the appropriate version tags) as we continue to train them longer and on more data. **Our current recommendation is to use the `256_v1` model unless you have a strong reason not to. If you have no preference for domain, `Links` is always a good first choice.**
[With `gsutil` installed](https://cloud.google.com/storage/docs/gsutil_install), you can simply run `gsutil -m cp -r gs://sf-ctrl/seqlen256_v1.ckpt/ .` for copying the model checkpoint over.
Without `gsutil`, you can follow the route recommended @ https://github.com/salesforce/ctrl/issues/7#issuecomment-531303214
4. Run the generation script `generation.py` or the source attribution script `source_attribution.py`.
The `generation.py` prompts the user to input text and then prints the continuation.
The `source_attribution.py` promps the user to input text and then prints a sorted list of domains and the perplexity of the text conditional on each individual domain.
## Generations
The generations and attributions computed below have been generated using the `256` sequence length model. Comparable results can be obtained from the `512` version of the model as well. We demonstrate only a few of the functionalities, especially the control codes. For a complete list of the control codes, and how to use them, please refer to the paper. Note that `<GENERATION_BEGINS>` is only included for demonstrative purposes and is not actually generated by the model.
1. Links
```
Links In a shocking finding, scientist discovered a herd of unicorns living in a remote, previously unexplored valley, in the Andes Mountains. Even more surprising to the researchers was the fact that the unicorns spoke perfect English.
<GENERATION_BEGINS>
The scientists were able to track down the unicorn population by analyzing their DNA. They found out that they are descended from an ancient species called the “Tapiri,” which is believed to have been extinct for over 10,000 years.
According to scientists, this discovery proves that there may be other undiscovered creatures on our planet.
“We believe these animals could still exist and we hope to find them one day,” said Dr. Carlos Jaramillo, director of the National Geographic Society’s Center for Biodiversity Research at the University of Arizona.
Scientists also confirmed that the Unicorn Genome Project has already identified several genes associated with the animal’s unique horn structure.
“The genome project will help us understand how the animal evolved its horns so it can better protect itself against predators like jaguars or pumas.”
Researchers say that if all goes well, the new genetic information should allow them to create a 3D model of the animal.
“It would give us a much clearer picture about what makes the creature unique.”- Dr David Sillito, lead researcher ...
```
As a generate note, you don't have to use greedy sampling. You can switch to `topk` or `nucleus` through the appropriate argument flags.
Running it with `--temperature 0.2` and `--topk 5` yields:
```
Links In a shocking finding, scientist discovered a herd of unicorns living in a remote, previously unexplored valley, in the Andes Mountains. Even more surprising to the researchers was the fact that the unicorns spoke perfect English.
The scientists were able to track down and capture one of these unicorn herds as it roamed through the valley for several days. They then took DNA samples from each animal and compared them with other animals found throughout the region.
Advertisement
What they found is truly astounding.
According to the scientists, there are approximately 50 different species of unicorn, all of which live on two continents — North America and South America...
```
2. Wikipedia
```
Wikipedia Salesforce Inc. is <GENERATION_BEGINS> a software company that provides cloud computing and business management software. It was founded in 1999 by Marc Benioff as an online retailer of enterprise software, but it has since expanded into other areas such as e-commerce, customer relationship management, and digital media...
```
3. Reviews
```
Reviews Rating: 1.0 <GENERATION_BEGINS>
I bought this for my son who is a huge fan of the show. He was so excited to get it and when he opened it, we were all very disappointed. The quality of the product is terrible. It looks like something you would buy at a dollar store.
Rating: 1.0
This item arrived in poor condition. There are several scratches on the front cover as well as some other minor damage...
```
```
Reviews Rating: 4.0 <GENERATION_BEGINS>
I bought this for my husband and he loves it. He has a small wrist so it is hard to find watches that fit him well. This one fits perfectly.
Rating: 4.0
The watch looks great but the band was too big even though i had read other reviews about how large the bands were.
Rating: 4.0
It works well as an everyday watch. It's not very fancy or anything, just simple with no extra bells & whistles.
```
4. Books
```
Books Weary with toil, I haste me to my bed, <GENERATION_BEGINS>
And sleep the slumber of a weary man.
The morning sun is up, and shines on high;
My bed is ready, but no friend appears.
A little while ago he came in sight,
But now he's gone, as if he'd never come.
He left his hat upon the table there,
His coat was hanging up behind the door;
And, when at last I looked around, there stood
No one beside me, nor within call.
Oh, what shall I do? What can be done?
I'm sure that some strange accident has happened.
I'll go out into the street again,
To seek for him, or call him back again.
But, oh, how lonely it will seem
When all are gone who once were near me!
```
5. Reddit - r/running
```
Running Title: <GENERATION_BEGINS> I just ran my first 5k in under 30 minutes.
Text: It was a pretty easy run, but it felt great.
My goal is to run at least 10 miles per week for the next few months.
I'm not sure if this qualifies as running or not, but it's definitely something that I've been wanting to do since high school.
Thanks r/running!
Edit: Thank you all so much for your kind words and advice.
Edit 2: Wow, thanks guys!
```
6. Reddit - r/fitness
```
Fitness Title: <GENERATION_BEGINS> How do you get your body to burn fat when you're not working out?
Text: I know that the reason why people lose weight is because they are eating less and burning more calories than their bodies need but how does this work for someone who doesn't have a lot of time or energy to exercise?
For example, if I'm at home doing nothing all day then I'll be able to eat whatever I want without feeling like I've eaten too much. But what happens after that?
EDIT: Thanks everyone for the responses!
```
7. Reddit - r/frugal
```
Saving Title: <GENERATION_BEGINS> How to get a free credit report
Text: I have been trying for months now and it seems like no one is willing to give me one.
Is there any way that you can just go online and fill out the forms yourself or do they require some sort of ID or something?
Edit: Thanks everyone for your help!
Edit 2: Thank you all so much for helping me!
```
8. Reddit - r/personalfinance
```
Finance Title: <GENERATION_BEGINS> I have a question about my credit score.
Text: So I'm currently in the process of applying for an auto loan and I've been told that if you don't pay your bills on time then they will not approve you.
My questions are:
1. How do I know when to start paying off debt?
2. What is the best way to get out of debt without having to file bankruptcy?
3. Is it possible to refinance my car loan?
4. Should I just wait until after school starts so I can take advantage of lower interest rates?
5. If so, how long should I wait?
Thanks
```
9. Questions
```
Questions Q: What is the capital of Australia? <GENERATION_BEGINS>
A: Canberra
Q: How many people live in Canberra?
A: 650,000
```
10. Translation
```
Translation English : This is a natural language processing model that aims to generate coherent text in a controllable manner. ; French : <GENERATION_BEGINS>
Il s'agit d'un modèle de traitement du langage naturel qui vise à générer un texte cohérent et contrôlable.
```
```
Translation English : This is a natural language processing model that aims to generate coherent text in a controllable manner. ; German : <GENERATION_BEGINS>
Es handelt sich um ein natürliches Textverarbeitungssystem, das auf eine einheitliche und kontrollierbare Erzeugung von Text abzielt.
```
## Source Attributions
1. `I lost 10 lbs! Feeling great!`
```
PROMPT: I lost 10 lbs! Feeling great!
Diet ppl = 28.960714
Weight ppl = 29.223865
Fitness ppl = 36.162671
...
```
2. `My landlord is suing me for unpaid rent`
```
PROMPT: My landlord is suing me for unpaid rent
Legal ppl = 21.210965
Finance ppl = 24.619064
Saving ppl = 27.923208
...
```
3. `And then I saw him, the man in the mirror.`
```
PROMPT: And then I saw him, the man in the mirror.
Horror ppl = 17.919299
Scary ppl = 18.587843
Writing ppl = 23.154564
...
```
4. `Anarchism is an anti-authoritarian political philosophy that rejects hierarchies deemed unjust and advocates their replacement with self-managed, self-governed societies based on voluntary, cooperative institutions.`
```
PROMPT: Anarchism is an anti-authoritarian political philosophy that rejects hierarchies deemed unjust and advocates their replacement with self-managed, self-governed societies based on voluntary, cooperative institutions.
Wikipedia ppl = 34.446701
News ppl = 34.484165
Links ppl = 35.460126
...
```
5. `I love God`
```
PROMPT: I love God
Christianity ppl = 55.653985
Atheism ppl = 116.811038
Confessions ppl = 133.619834
...
```
## FAQs
(We hope to update this section frequently).
1. Will you be releasing the training code and data?
We plan to release the training code soon. We will not be releasing the training data, but we will release tips and scripts related to data collection.
2. Is a version of the model available in PyTorch?
Not at the moment, but if we come across an equivalent implementation, we will update this section.
3. The code errors out.
Make sure that you have performed the patch as described above. If the error persists, please create a GitHub issue.
4. The code generates non-sense irrespective of the prompt.
Make sure that you have (a) provided the right `--model_dir` and that the folder actually exists and has the checkpoint, (b) provided a valid source code as the first token, and (c) tried generating with a simple prompt such as `Links I` or `Books From`. If the error persists, please create a GitHub issue.
## Get Involved
Please create a GitHub issue if you have any questions, suggestions, requests or bug-reports.
We welcome PRs!
generator/ctrl/model/__init__.py
View File
Whitespace-only changes.
generator/ctrl/model/codes
-200000
View File
File diff suppressed because it is too large. Load diff
generator/ctrl/model/control_codes.txt
-55
View File
@@ -1,55 +0,0 @@
0.000296793 Pregnancy
0.000127197 Christianity
0.003084531 Explain
0.000180196 Fitness
6.88985E-05 Saving
0.000217295 Ask
8.47981E-05 Ass
0.000143097 Joke
0.000196096 Questions
0.000127197 Thoughts
0.000169596 Retail
0.000270294 Feminism
0.000111298 Writing
0.000402791 Atheism
1.05998E-06 Netflix
0.000365692 Computing
0.000132497 Opinion
0.000169596 Alone
0.000323293 Funny
0.000249094 Gaming
0.000402791 Human
0.000132497 India
2.11995E-08 Joker
0.000201395 Diet
0.000238495 Legal
6.35986E-06 Norman
3.60392E-07 Tip
0.000302093 Weight
0.000132497 Movies
0.000111298 Running
7.41983E-05 Science
0.00135147 Horror
0.000291493 Confession
0.000190796 Finance
0.000413391 Politics
7.41983E-05 Scary
0.000206695 Support
6.35986E-05 Technologies
0.000243795 Teenage
0.000217295 Event
0.000206695 Learned
0.000121897 Notion
0.0847981 Wikipedia
0.095927851 Books
0.001176574 Extract
0.000127197 Confessions
0.000227895 Conspiracy
0.365691808 Links
0.000423991 Narcissus
0.000280894 Relationship
0.000922179 Relationships
0.153696557 Reviews
0.043877717 News
0.129847091 Translation
0.111297507 multilingual
generator/ctrl/model/generation.py
-281
View File
@@ -1,281 +0,0 @@
from __future__ import division
from __future__ import print_function
import tensorflow as tf
import os
import numpy as np
tf.enable_eager_execution()
import transformer
import argparse
import pdb
import sys
import re
from collections import Counter
from tensorflow.python import debug as tf_debug
from tensorflow.python.ops import math_ops
from tensorflow.python.ops import embedding_ops
import fastBPE
import platform
use_py3 = platform.python_version()[0] == '3'
parser = argparse.ArgumentParser(description='TensorFlow code for generating from CTRL')
parser.add_argument('--model_dir', type=str, required=True,
help='location of model checkpoint')
parser.add_argument('--seed', type=int, default=1337,
help='random seed for TensorFlow, numpy and PythonHash')
parser.add_argument('--generate_num', type=int, default=256,
help='number of tokens to generate')
parser.add_argument('--temperature', type=float, default=0,
help='temperature for sampling distribution; 0 means greedy')
parser.add_argument('--nucleus', type=float, default=0.,
help='cumulative probability cutoff for nucleus sampling; 0 means no nucleus sampling')
parser.add_argument('--topk', type=int, default=0,
help='topk value for sampling from the softmax distribution ; 0 means no topk preferred')
parser.add_argument('--penalty', type=float, default=1.2,
help='repetition penalty for greedy sampling')
args = parser.parse_args()
tf.random.set_random_seed(args.seed)
os.environ['PYTHONHASHSEED'] = str(args.seed)
np.random.seed(args.seed)
# load the vocabulary from file
vocab = open('vocab').read().decode(encoding='utf-8').split('\n') if not use_py3 else open('vocab', encoding='utf-8').read().split('\n')
vocab = list(map(lambda x: x.split(' ')[0], vocab)) + ['<unk>'] + ['\n']
print ('{} unique words'.format(len(vocab)))
# length of the vocabulary
vocab_size = len(vocab)
# define the numericalization map
# idx2word maps the numericalized ID to the word
# word2idx maps the word to the numericalized ID
word2idx = {u:i for i, u in enumerate(vocab)}
idx2word = np.array(vocab)
# sequence length to use for the transformer
# the model is trained with a seq_length of 512
# so, any value <= 512 should work
seq_length = min(args.generate_num, 256)
# the dimension of the transformer
embedding_dim = 1280
# Now, we begin defining the model
# we defer the transformer definition to transformer.py
# here, we only define the tied softmax layer
# this layer ties the softmax weights to the input embeddings
class TiedEmbeddingSoftmax(tf.keras.layers.Layer):
def __init__(self, vocab_size=vocab_size, embedding_size=embedding_dim, **kwargs):
super(TiedEmbeddingSoftmax, self).__init__()
self.w = self.add_weight(name='w', shape=(vocab_size, embedding_size),
initializer='random_normal',
trainable=True)
self.b = self.add_weight(name='b', shape=(vocab_size,),
initializer='zeros',
trainable=True)
def call(self, inputs, embed=True):
if embed:
dtype = tf.keras.backend.dtype(inputs)
if dtype != 'int32' and dtype != 'int64':
inputs = math_ops.cast(inputs, 'int32')
return embedding_ops.embedding_lookup(self.w, inputs)
else:
return tf.tensordot(inputs, tf.transpose(self.w), 1) + self.b
# input for the keras model
tokens = tf.keras.layers.Input(shape=(seq_length,), dtype='int32')
# instantiates a tied softmax class
tied_embedding_softmax = TiedEmbeddingSoftmax()
# embedded tokens, before passing it to the transformer
embedded = tied_embedding_softmax(tokens, embed=True)
# the activations after passing it from the transformer
# for some odd reason, TPUs don't play well with specifying the arguments of the Encoder() function
# so you have to leave them at their defaults
transformed = transformer.Encoder()(embedded, training=False)
# pass the activations from our tiedsoftmax class
# this time with embed=False denoting that we are doing the softmax operation
# and not a lookup
logits = tied_embedding_softmax(transformed, embed=False)
# finally, define the Keras model with inputs as tokens and outputs as the logits we just computed
model = tf.keras.Model(inputs=tokens, outputs=logits)
# the loss function is a simple categorical crossentropy between the logits and the labels
def loss(labels, logits):
return tf.keras.losses.sparse_categorical_crossentropy(labels, logits, from_logits=True)
# the optimizer is not used since this code only supports inference
# however, to compile the model, we still define it
optimizer = tf.contrib.tpu.CrossShardOptimizer(
tf.contrib.estimator.clip_gradients_by_norm(
tf.train.AdagradOptimizer(learning_rate=1e-2), 0.25)
)
# compile the model with the optimizer and loss
model.compile(optimizer=optimizer, loss=loss)
print(model.summary())
# IMPORTANT
# this is where the saved model is presented to the code
# the model directory should have the model checkpoint and
# a checkpoint file
run_config = tf.contrib.tpu.RunConfig(
model_dir=args.model_dir)
# this converts the Keras model to a TensorFlow estimator
# this step is critical
# remember to patch the TF 1.14 file before running the code, else you're going to see errors here
estimator_model = tf.keras.estimator.model_to_estimator(keras_model=model, config=run_config)
# we now create a serving function from this estimator
# this enables us to load the model once and easily query it multiple times
def serving_input_fn():
inputs = {'input_1': tf.placeholder(tf.int32, [1,seq_length])}
return tf.estimator.export.ServingInputReceiver(inputs, inputs)
predict_fn = tf.contrib.predictor.from_estimator(estimator_model, serving_input_fn)
# almost there, we now take the user prompt and tokenize with BPE
# load BPE codes
bpe = fastBPE.fastBPE('codes', 'vocab')
temperature = args.temperature
nucleusprob = args.nucleus
penalty = args.penalty
topk = args.topk
while True:
prompt = raw_input('ENTER PROMPT: ') if not use_py3 else input('ENTER PROMPT: ')
# tokenize provided prompt
split_prompt = bpe.apply([prompt])[0].split()
text = [word2idx[i] for i in split_prompt]
# pad with 0s and create a mini-batch of 2 (arbitrary, for ease of code)
padded_text = text + [0] * (args.generate_num - len(text))
tokens_generated = np.tile(padded_text, (1,1))
try:
for token in range(len(text)-1, args.generate_num-1):
# get the logits from the prediction function
# the logic here is a bit convoluted because we are allowing generation past 512 tokens
# this is done by sliding the window over (past 512 tokens) and continuing prediction
# I'm sure this can be simplified (TODO)
if token <= seq_length:
prompt_logits = predict_fn({'input_1':tokens_generated[:, :seq_length]})['tied_embedding_softmax'].squeeze() / (temperature if temperature>0 else 1.)
_token = token if token < seq_length else -1
else:
_token = -1
end = token + 1
start = token - seq_length + 2
prompt_logits = predict_fn({'input_1':np.hstack((tokens_generated[:,0:1], tokens_generated[:,start:end]))})['tied_embedding_softmax'].squeeze() / (temperature if temperature>0 else 1.)
# if penalty (for repetition) is non-zero,
# discount the logits from already generated tokens
if penalty>0:
penalized_so_far = set()
for _ in range(token+1):
generated_token = tokens_generated[0][_]
# don't penalize newlines
# you could also choose not to penalize frequent words
# (which incidentally are sorted in the vocab file)
# but I don't do that
# if it prints too many new lines instead of continuing generating text,
# you might want to comment this out
if idx2word[generated_token] == '\n':
continue
if generated_token in penalized_so_far:
continue
penalized_so_far.add(generated_token)
prompt_logits[_token][generated_token] /= penalty
# disallow some tokens
prompt_logits[_token][word2idx['<unk>']] = -1e8
# sometimes, when generating from reddit,
# it tries to generate the Score (reddit Karma) immediately after generating the Title:
# to disallow this, we can just prevent it from generating Score
prompt_logits[_token][word2idx['Sco@@']] = -1e8
# compute probabilities from logits
prompt_probs = np.exp(prompt_logits[_token])
prompt_probs = prompt_probs / sum(prompt_probs)
pruned_list = np.argsort(prompt_probs)[::-1]
# if you are using nucleus prob, then compute the nucleus probability size
if nucleusprob > 0.:
minimum_topk = 1
nucleus = max(np.where(np.cumsum(np.sort(prompt_probs)[::-1])>nucleusprob)[0][0], minimum_topk)
elif topk > 0:
# we are over-loading notation here
# if you choose to specify a topk instead of a nucleus,
# we will hardcode the nucleus to be just that
nucleus = topk
else:
# if you specify neither nucleus or topk,
# then we will use the whole list
nucleus = len(pruned_list)
# if you want to disallow more complex tokens, you can do so here
# for instance, if you want to disallow anything with the phrase `http`,
# you can delete theme from the pruned_list
# you can comment this out, I'm keeping it in for demonstration purpose
tokens_to_disallow = []
for _ in range(len(pruned_list)):
if 'http' in idx2word[pruned_list[_]]:
tokens_to_disallow.append(_)
pruned_list = np.delete(pruned_list, tokens_to_disallow)
# if temperature is 0
# just pick the first (most probable) token
if temperature==0:
idx = pruned_list[0]
else:
# else,
# sample from the pruned_list with the logits
chosen_idx = int(tf.random.categorical(np.expand_dims(prompt_logits[0][_token][pruned_list],0), num_samples=1).numpy())
idx = pruned_list[chosen_idx]
# if you want to do some debugging,
# like which one was chosen,
# what the top25 were,
# here is your opportunity.
print('chosen:', idx2word[idx])
#print('top25 alternatives:', pruned_list[:25])
# assign the token for generation
tokens_generated[0][token+1] = idx
# clear screen if you want to
# os.system("clear")
tokens_generated_so_far = ' '.join([idx2word[c] for c in tokens_generated[0].squeeze()[:token+2]])
tokens_generated_so_far = re.sub('(@@ )', '', string=tokens_generated_so_far)
tokens_generated_so_far = re.sub('(@@ ?$)', '', string=tokens_generated_so_far)
tokens_generated_so_far = re.sub('(\n\n)', '', string=tokens_generated_so_far)
print(tokens_generated_so_far)
print()
except KeyboardInterrupt: #Exception as e:
print('Continuing')
generator/ctrl/model/low_mem_transformer.py
-139
View File
@@ -1,139 +0,0 @@
import tensorflow as tf
import numpy as np
def angle_defn(pos, i, d_model_size):
angle_rates = 1 / np.power(10000, (2 * (i // 2)) / np.float32(d_model_size))
return pos * angle_rates
def positional_encoding(position, d_model_size):
# create the sinusoidal pattern for the positional encoding
angle_rads = angle_defn(np.arange(position)[:, np.newaxis], np.arange(d_model_size)[np.newaxis, :], d_model_size)
sines = np.sin(angle_rads[:, 0::2])
cosines = np.cos(angle_rads[:, 1::2])
pos_encoding = tf.cast(np.concatenate([sines, cosines], axis=-1)[np.newaxis, ...], dtype=tf.float32)
return pos_encoding
def scaled_dot_product_attention(q, k, v, mask):
# calculate attention
matmul_qk = tf.cast(tf.matmul(q, k, transpose_b=True), tf.float32)
dk = tf.cast(tf.shape(k)[-1], tf.float32)
scaled_attention_logits = matmul_qk / tf.math.sqrt(dk)
if mask is not None:
scaled_attention_logits += (mask * -1e3)
attention_weights = tf.cast(tf.nn.softmax(scaled_attention_logits, axis=-1), tf.float16)
output = tf.matmul(attention_weights, v)
return output
class MultiHeadAttention(tf.keras.layers.Layer):
def __init__(self, d_model_size, num_heads):
super(MultiHeadAttention, self).__init__()
self.num_heads = num_heads
self.d_model_size = d_model_size
self.depth = int(d_model_size / self.num_heads)
self.Wq = tf.keras.layers.Dense(d_model_size)
self.Wk = tf.keras.layers.Dense(d_model_size)
self.Wv = tf.keras.layers.Dense(d_model_size)
self.dense = tf.keras.layers.Dense(d_model_size)
def split_into_heads(self, x, batch_size):
x = tf.reshape(x, (batch_size, -1, self.num_heads, self.depth))
return tf.transpose(x, perm=[0, 2, 1, 3])
def call(self, v, k, q, mask):
batch_size = tf.shape(q)[0]
q = self.Wq(q)
k = self.Wk(k)
v = self.Wv(v)
q = self.split_into_heads(q, batch_size)
k = self.split_into_heads(k, batch_size)
v = self.split_into_heads(v, batch_size)
scaled_attention = tf.transpose(scaled_dot_product_attention(q, k, v, mask), perm=[0, 2, 1, 3])
original_size_attention = tf.reshape(scaled_attention, (batch_size, -1, self.d_model_size))
output = self.dense(original_size_attention)
return output
def point_wise_feed_forward_network(d_model_size, dff):
return tf.keras.Sequential([tf.keras.layers.Dense(dff, activation='relu'),
tf.keras.layers.Dense(d_model_size)])
class EncoderLayer(tf.keras.layers.Layer):
def __init__(self, d_model_size, num_heads, dff, rate=0.1):
super(EncoderLayer, self).__init__()
self.multi_head_attention = MultiHeadAttention(d_model_size, num_heads)
self.ffn = point_wise_feed_forward_network(d_model_size, dff)
self.layernorm1 = tf.keras.layers.LayerNormalization(epsilon=1e-6)
self.layernorm2 = tf.keras.layers.LayerNormalization(epsilon=1e-6)
self.dropout1 = tf.keras.layers.Dropout(rate)
self.dropout2 = tf.keras.layers.Dropout(rate)
self.to32 = lambda x: tf.cast(x, tf.float32)
self.to16 = lambda x: tf.cast(x, tf.float16)
def call(self, x, training, mask):
normed = self.to16(self.layernorm1(self.to32(x)))
attn_output = self.multi_head_attention(normed, normed, normed, mask)
attn_output = self.dropout1(attn_output, training=training)
out1 = x + attn_output
out2 = self.to16(self.layernorm2(self.to32(out1)))
ffn_output = self.ffn(out2)
ffn_output = self.dropout2(ffn_output, training=training)
out2 = out1 + ffn_output
return out2
class Encoder(tf.keras.layers.Layer):
def __init__(self, num_layers=48, d_model_size=1280, num_heads=16, dff=8192, input_vocab_size=50000,
rate=0.1, **kwargs):
super(Encoder, self).__init__()
self.d_model_size = d_model_size
self.num_layers = num_layers
self.pos_encoding = positional_encoding(input_vocab_size, self.d_model_size)
for i in range(num_layers):
setattr(self, "layer%i" % i, EncoderLayer(d_model_size, num_heads, dff, rate))
self.layernorm = tf.keras.layers.LayerNormalization(epsilon=1e-6)
self.dropout = tf.keras.layers.Dropout(rate)
def get_config(self):
base_config = super(Encoder, self).get_config()
return base_config
def call(self, x, training):
seq_len = tf.shape(x)[1]
mask = 1 - tf.linalg.band_part(tf.ones((seq_len, seq_len)), -1, 0)
x *= tf.math.sqrt(tf.cast(self.d_model_size, tf.float32))
x += self.pos_encoding[:, :seq_len, :]
x = self.dropout(x, training=training)
x = tf.cast(x, tf.float16)
for i in range(self.num_layers):
x = getattr(self, "layer%i" % i)(x, training, mask)
return self.layernorm(tf.cast(x, tf.float32))
generator/ctrl/model/source_attribution.py
-192
View File
@@ -1,192 +0,0 @@
from __future__ import division
from __future__ import print_function
import tensorflow as tf
import os
import numpy as np
tf.enable_eager_execution()
import transformer
import argparse
import pdb
import sys
import re
from collections import Counter
from tensorflow.python import debug as tf_debug
from tensorflow.python.ops import math_ops
from tensorflow.python.ops import embedding_ops
import fastBPE
import platform
use_py3 = platform.python_version()[0] == '3'
parser = argparse.ArgumentParser(description='TensorFlow code for generating from CTRL')
parser.add_argument('--model_dir', type=str, required=True,
help='location of model checkpoint')
parser.add_argument('--seed', type=int, default=1337,
help='random seed for TensorFlow, numpy and PythonHash')
args = parser.parse_args()
tf.random.set_random_seed(args.seed)
os.environ['PYTHONHASHSEED'] = str(args.seed)
np.random.seed(args.seed)
# load the vocabulary from file
vocab = open('vocab').read().decode(encoding='utf-8').split('\n') if not use_py3 else open('vocab', encoding='utf-8').read().split('\n')
vocab = list(map(lambda x: x.split(' ')[0], vocab)) + ['<unk>'] + ['\n']
print ('{} unique words'.format(len(vocab)))
# length of the vocabulary
vocab_size = len(vocab)
# define the numericalization map
# idx2word maps the numericalized ID to the word
# word2idx maps the word to the numericalized ID
word2idx = {u:i for i, u in enumerate(vocab)}
idx2word = np.array(vocab)
# sequence length to use for the transformer
# the model is trained with a seq_length of 512
# so, any value <= 512 should work
seq_length = 256
# the dimension of the transformer
embedding_dim = 1280
# Now, we begin defining the model
# we defer the transformer definition to transformer.py
# here, we only define the tied softmax layer
# this layer ties the softmax weights to the input embeddings
class TiedEmbeddingSoftmax(tf.keras.layers.Layer):
def __init__(self, vocab_size=vocab_size, embedding_size=embedding_dim, **kwargs):
super(TiedEmbeddingSoftmax, self).__init__()
self.w = self.add_weight(name='w', shape=(vocab_size, embedding_size),
initializer='random_normal',
trainable=True)
self.b = self.add_weight(name='b', shape=(vocab_size,),
initializer='zeros',
trainable=True)
def call(self, inputs, embed=True):
if embed:
dtype = tf.keras.backend.dtype(inputs)
if dtype != 'int32' and dtype != 'int64':
inputs = math_ops.cast(inputs, 'int32')
return embedding_ops.embedding_lookup(self.w, inputs)
else:
return tf.tensordot(inputs, tf.transpose(self.w), 1) + self.b
# input for the keras model
tokens = tf.keras.layers.Input(shape=(seq_length,), dtype='int32')
# instantiates a tied softmax class
tied_embedding_softmax = TiedEmbeddingSoftmax()
# embedded tokens, before passing it to the transformer
embedded = tied_embedding_softmax(tokens, embed=True)
# the activations after passing it from the transformer
# for some odd reason, TPUs don't play well with specifying the arguments of the Encoder() function
# so you have to leave them at their defaults
transformed = transformer.Encoder()(embedded, training=False)
# pass the activations from our tiedsoftmax class
# this time with embed=False denoting that we are doing the softmax operation
# and not a lookup
logits = tied_embedding_softmax(transformed, embed=False)
# finally, define the Keras model with inputs as tokens and outputs as the logits we just computed
model = tf.keras.Model(inputs=tokens, outputs=logits)
# the loss function is a simple categorical crossentropy between the logits and the labels
def loss(labels, logits):
return tf.keras.losses.sparse_categorical_crossentropy(labels, logits, from_logits=True)
# the optimizer is not used since this code only supports inference
# however, to compile the model, we still define it
optimizer = tf.contrib.tpu.CrossShardOptimizer(
tf.contrib.estimator.clip_gradients_by_norm(
tf.train.AdagradOptimizer(learning_rate=1e-2), 0.25)
)
# compile the model with the optimizer and loss
model.compile(optimizer=optimizer, loss=loss)
print(model.summary())
# IMPORTANT
# this is where the saved model is presented to the code
# the model directory should have the model checkpoint and
# a checkpoint file
run_config = tf.contrib.tpu.RunConfig(
model_dir=args.model_dir)
# this converts the Keras model to a TensorFlow estimator
# this step is critical
# remember to patch the TF 1.14 file before running the code, else you're going to see errors here
estimator_model = tf.keras.estimator.model_to_estimator(keras_model=model, config=run_config)
# we now create a serving function from this estimator
# this enables us to load the model once and easily query it multiple times
def serving_input_fn():
inputs = {'input_1': tf.placeholder(tf.int32, [2,seq_length])}
return tf.estimator.export.ServingInputReceiver(inputs, inputs)
predict_fn = tf.contrib.predictor.from_estimator(estimator_model, serving_input_fn)
# almost there, we now take the user prompt and tokenize with BPE
# load BPE codes
bpe = fastBPE.fastBPE('codes', 'vocab')
domains = []
with open('control_codes.txt', 'r') as f:
domains = [line.split() for line in f.readlines()]
domains = [(t[1], float(t[0])) for t in domains]
while True:
_prompt = raw_input('ENTER PROMPT: ') if not use_py3 else input('ENTER PROMPT: ')
ppls = {}
# loop over all domains and compute perplexity
for domain, domain_prior in domains:
print(u'computing for domain: {}'.format(domain))
# tokenize data and add domain tag to it
prompt = domain + u' ' + _prompt
split_prompt = bpe.apply([prompt])[0].split()
# numericalize data and pad to the seq_len dimension
text = [word2idx[i] for i in split_prompt]
padding_text = text + [0] * (seq_length - len(text))
tokens_generated = np.tile(padding_text, (2,1))
output_scores = predict_fn({'input_1':tokens_generated})['tied_embedding_softmax'].squeeze()[0]
token_scores = output_scores[:-1]
# compute the perplexity for this sequence
xent = 0
for sequence_idx, token_idx in enumerate(text[1:]):
token = idx2word[token_idx]
# compute the probability of this token
Z = np.exp(token_scores[sequence_idx]).sum()
token_prob = np.exp(token_scores[sequence_idx, token_idx]) / Z
xent -= np.log(token_prob) / len(text[1:])
ppls[domain] = round(np.exp(xent), 6)
#print(u'{} ppl = {}'.format(domain, ppls[domain]))
# sort the domains based on perplexities and print
ppls = [(k, v) for k, v in ppls.items()]
ppls.sort(key=lambda x: x[1])
print('PROMPT: {}'.format(_prompt))
for t in ppls:
domain, ppl = t
print(u'{} ppl = {}'.format(domain, ppl))
generator/ctrl/model/transformer.py
-137
View File
@@ -1,137 +0,0 @@
import tensorflow as tf
import numpy as np
def angle_defn(pos, i, d_model_size):
angle_rates = 1 / np.power(10000, (2 * (i//2)) / np.float32(d_model_size))
return pos * angle_rates
def positional_encoding(position, d_model_size):
# create the sinusoidal pattern for the positional encoding
angle_rads = angle_defn(np.arange(position)[:, np.newaxis], np.arange(d_model_size)[np.newaxis, :], d_model_size)
sines = np.sin(angle_rads[:, 0::2])
cosines = np.cos(angle_rads[:, 1::2])
pos_encoding = tf.cast(np.concatenate([sines, cosines], axis=-1)[np.newaxis, ...], dtype=tf.float32)
return pos_encoding
def scaled_dot_product_attention(q, k, v, mask):
# calculate attention
matmul_qk = tf.matmul(q, k, transpose_b=True)
dk = tf.cast(tf.shape(k)[-1], tf.float32)
scaled_attention_logits = matmul_qk / tf.math.sqrt(dk)
if mask is not None:
scaled_attention_logits += (mask * -1e9)
attention_weights = tf.nn.softmax(scaled_attention_logits, axis=-1)
output = tf.matmul(attention_weights, v)
return output
class MultiHeadAttention(tf.keras.layers.Layer):
def __init__(self, d_model_size, num_heads):
super(MultiHeadAttention, self).__init__()
self.num_heads = num_heads
self.d_model_size = d_model_size
self.depth = int(d_model_size / self.num_heads)
self.Wq = tf.keras.layers.Dense(d_model_size)
self.Wk = tf.keras.layers.Dense(d_model_size)
self.Wv = tf.keras.layers.Dense(d_model_size)
self.dense = tf.keras.layers.Dense(d_model_size)
def split_into_heads(self, x, batch_size):
x = tf.reshape(x, (batch_size, -1, self.num_heads, self.depth))
return tf.transpose(x, perm=[0, 2, 1, 3])
def call(self, v, k, q, mask):
batch_size = tf.shape(q)[0]
q = self.Wq(q)
k = self.Wk(k)
v = self.Wv(v)
q = self.split_into_heads(q, batch_size)
k = self.split_into_heads(k, batch_size)
v = self.split_into_heads(v, batch_size)
scaled_attention = tf.transpose(scaled_dot_product_attention(q, k, v, mask), perm=[0, 2, 1, 3])
original_size_attention = tf.reshape(scaled_attention, (batch_size, -1, self.d_model_size))
output = self.dense(original_size_attention)
return output
def point_wise_feed_forward_network(d_model_size, dff):
return tf.keras.Sequential([tf.keras.layers.Dense(dff, activation='relu'),
tf.keras.layers.Dense(d_model_size)])
class EncoderLayer(tf.keras.layers.Layer):
def __init__(self, d_model_size, num_heads, dff, rate=0.1):
super(EncoderLayer, self).__init__()
self.multi_head_attention = MultiHeadAttention(d_model_size, num_heads)
self.ffn = point_wise_feed_forward_network(d_model_size, dff)
self.layernorm1 = tf.keras.layers.LayerNormalization(epsilon=1e-6)
self.layernorm2 = tf.keras.layers.LayerNormalization(epsilon=1e-6)
self.dropout1 = tf.keras.layers.Dropout(rate)
self.dropout2 = tf.keras.layers.Dropout(rate)
def call(self, x, training, mask):
normed = self.layernorm1(x)
attn_output = self.multi_head_attention(normed, normed, normed, mask)
attn_output = self.dropout1(attn_output, training=training)
out1 = x + attn_output
out2 = self.layernorm2(out1)
ffn_output = self.ffn(out2)
ffn_output = self.dropout2(ffn_output, training=training)
out2 = out1 + ffn_output
return out2
class Encoder(tf.keras.layers.Layer):
def __init__(self, num_layers=48, d_model_size=1280, num_heads=16, dff=8192, input_vocab_size=50000,
rate=0.1, **kwargs):
super(Encoder, self).__init__()
self.d_model_size = d_model_size
self.num_layers = num_layers
self.pos_encoding = positional_encoding(input_vocab_size, self.d_model_size)
for i in range(num_layers):
setattr(self, "layer%i" % i, EncoderLayer(d_model_size, num_heads, dff, rate))
self.layernorm = tf.keras.layers.LayerNormalization(epsilon=1e-6)
self.dropout = tf.keras.layers.Dropout(rate)
def get_config(self):
base_config = super(Encoder, self).get_config()
return base_config
def call(self, x, training):
seq_len = tf.shape(x)[1]
mask = 1 - tf.linalg.band_part(tf.ones((seq_len, seq_len)), -1, 0)
x *= tf.math.sqrt(tf.cast(self.d_model_size, tf.float32))
x += self.pos_encoding[:, :seq_len, :]
x = self.dropout(x, training=training)
for i in range(self.num_layers):
x = getattr(self, "layer%i" % i)(x, training, mask)
return self.layernorm(x)
generator/ctrl/model/vocab
-246531
View File
File diff suppressed because it is too large. Load diff
generator/ctrl/pytorch_generation.py
-283
View File
@@ -1,283 +0,0 @@
from __future__ import print_function
import torch
import os
import tqdm
import pdb
import numpy as np
import platform
import hashlib
import pytorch_transformer
import re
import argparse
import tensorflow as tf
import fastBPE
from tensorflow.python import pywrap_tensorflow
use_py3 = platform.python_version()[0] == '3'
parser = argparse.ArgumentParser(description='TensorFlow code for generating from CTRL')
parser.add_argument('--model_path', type=str, required=True,
help='location of model *data* checkpoint; this is NOT the directory but rather the model checkpoint')
parser.add_argument('--seed', type=int, default=1337,
help='random seed for TensorFlow, numpy and PythonHash')
parser.add_argument('--generate_num', type=int, default=256,
help='number of tokens to generate')
parser.add_argument('--temperature', type=float, default=0,
help='temperature for sampling distribution; 0 means greedy')
parser.add_argument('--nucleus', type=float, default=0.,
help='cumulative probability cutoff for nucleus sampling; 0 means no nucleus sampling')
parser.add_argument('--topk', type=int, default=0,
help='topk value for sampling from the softmax distribution ; 0 means no topk preferred')
parser.add_argument('--penalty', type=float, default=1.2,
help='repetition penalty for greedy sampling')
parser.add_argument('--print_once', action='store_true',
help='the completion is printed only at the end; not every word')
parser.add_argument('--topn', type=int, default=0,
help='print top-n candidates during generations; defaults to 0 which is no printing')
args = parser.parse_args()
torch.manual_seed(args.seed)
torch.cuda.manual_seed_all(args.seed)
os.environ['PYTHONHASHSEED'] = str(args.seed)
np.random.seed(args.seed)
# load the vocabulary from file
vocab = open('vocab').read().decode(encoding='utf-8').split('\n') if not use_py3 else open('vocab', encoding='utf-8').read().split('\n')
vocab = list(map(lambda x: x.split(' ')[0], vocab)) + ['<unk>'] + ['\n']
print ('{} unique words'.format(len(vocab)))
# length of the vocabulary
vocab_size = len(vocab)
# define the numericalization map
# idx2word maps the numericalized ID to the word
# word2idx maps the word to the numericalized ID
word2idx = {u:i for i, u in enumerate(vocab)}
idx2word = np.array(vocab)
embedding_dim = 1280
class TiedEmbeddingSoftmax(torch.nn.Module):
def __init__(self, vocab_size=vocab_size, embedding_size=embedding_dim, **kwargs):
super(TiedEmbeddingSoftmax, self).__init__()
self.w = torch.nn.Parameter(torch.zeros(vocab_size, embedding_size))
self.b = torch.nn.Parameter(torch.zeros(vocab_size))
def forward(self, inputs, embed=True):
if embed:
return torch.nn.functional.embedding(inputs, self.w)
else:
return torch.tensordot(inputs, self.w.t(), 1) + self.b
test_softmax = TiedEmbeddingSoftmax()
test_encoder = pytorch_transformer.Encoder()
def predict_fn(inputs):
with torch.no_grad():
embedded = torch.tensor(inputs['input_1']).cuda()
embedded = test_softmax(embedded, embed=True)
embedded = test_encoder(embedded)
embedded = test_softmax(embedded, embed=False)
return embedded
bpe = fastBPE.fastBPE('codes', 'vocab')
seq_length = min(args.generate_num, 256)
pytorch_model_hash = hashlib.md5(args.model_path.encode('utf-8')).hexdigest()
temperature = args.temperature
nucleusprob = args.nucleus
penalty = args.penalty
topk = args.topk
# try to load the model from a (cached) PyTorch checkpoint
# if one is not available, then create it by converting the weights
if os.path.exists(pytorch_model_hash):
print('Found PyTorch checkpoint @', pytorch_model_hash)
print('Loading instead of converting from TensorFlow')
checkpoint = torch.load(pytorch_model_hash)
test_softmax.load_state_dict(checkpoint['softmax'])
test_encoder.load_state_dict(checkpoint['encoder'])
test_softmax.to('cuda')
test_encoder.to('cuda')
else:
print('Could not find PyTorch checkpoint')
print('Converting weights and will store the PyTorch checkpoint as ', pytorch_model_hash)
chkpt_for_reader = '.'.join(args.model_path.split('.')[:-1])
reader = pywrap_tensorflow.NewCheckpointReader(chkpt_for_reader)
test_softmax.w = torch.nn.Parameter(torch.tensor(reader.get_tensor('w')).to('cuda'))
test_softmax.b = torch.nn.Parameter(torch.tensor(reader.get_tensor('b')).to('cuda'))
list_of_variables = list(filter(lambda x: 'Adagrad' not in x, reader.get_variable_to_shape_map().keys()))
str2parameter = lambda x: torch.nn.Parameter(torch.tensor(reader.get_tensor(x)).t().to('cuda'))
test_encoder.layernorm.weight = str2parameter('encoder/layer_normalization_96/gamma')
test_encoder.layernorm.bias = str2parameter('encoder/layer_normalization_96/beta')
for i in tqdm.tqdm(range(48)):
if i==0:
layer_variables = sorted(filter(lambda x: 'layer/' in x, list_of_variables))
else:
layer_variables = sorted(filter(lambda x: 'layer_'+str(i)+'/' in x, list_of_variables))
current_layer = getattr(test_encoder, 'layer'+str(i))
current_layer.layernorm1.bias = str2parameter(layer_variables[0])
current_layer.layernorm1.weight = str2parameter(layer_variables[1])
current_layer.layernorm2.bias = str2parameter(layer_variables[2])
current_layer.layernorm2.weight = str2parameter(layer_variables[3])
current_layer.multi_head_attention.Wq.bias = str2parameter(layer_variables[4])
current_layer.multi_head_attention.Wq.weight = str2parameter(layer_variables[5])
current_layer.multi_head_attention.Wk.bias = str2parameter(layer_variables[6])
current_layer.multi_head_attention.Wk.weight = str2parameter(layer_variables[7])
current_layer.multi_head_attention.Wv.bias = str2parameter(layer_variables[8])
current_layer.multi_head_attention.Wv.weight = str2parameter(layer_variables[9])
current_layer.multi_head_attention.dense.bias = str2parameter(layer_variables[10])
current_layer.multi_head_attention.dense.weight = str2parameter(layer_variables[11])
current_layer.ffn[0].bias = str2parameter(layer_variables[12])
current_layer.ffn[0].weight = str2parameter(layer_variables[13])
current_layer.ffn[2].bias = str2parameter(layer_variables[14])
current_layer.ffn[2].weight = str2parameter(layer_variables[15])
torch.save({
'softmax': test_softmax.state_dict(),
'encoder': test_encoder.state_dict(),
}, pytorch_model_hash)
test_softmax.eval()
test_encoder.eval()
while True:
prompt = raw_input('ENTER PROMPT: ') if not use_py3 else input('ENTER PROMPT: ')
# tokenize provided prompt
split_prompt = bpe.apply([prompt])[0].split()
text = [word2idx[i] for i in split_prompt]
# pad with 0s and create a mini-batch of 2 (arbitrary, for ease of code)
padded_text = text + [0] * (args.generate_num - len(text))
tokens_generated = np.tile(padded_text, (1,1))
try:
for token in range(len(text)-1, args.generate_num-1):
# get the logits from the prediction function
# the logic here is a bit convoluted because we are allowing generation past 512 tokens
# this is done by sliding the window over (past 512 tokens) and continuing prediction
# I'm sure this can be simplified (TODO)
if token <= seq_length:
prompt_logits = predict_fn({'input_1':tokens_generated[:, :seq_length]}).squeeze() / (temperature if temperature>0 else 1.)
_token = token if token < seq_length else -1
else:
_token = -1
end = token + 1
start = token - seq_length + 2
prompt_logits = predict_fn({'input_1':np.hstack((tokens_generated[:,0:1], tokens_generated[:,start:end]))}).squeeze() / (temperature if temperature>0 else 1.)
prompt_logits = prompt_logits.cpu().detach().numpy()
# if penalty (for repetition) is non-zero,
# discount the logits from already generated tokens
if penalty>0:
penalized_so_far = set()
for _ in range(token+1):
generated_token = tokens_generated[0][_]
# don't penalize newlines
# you could also choose not to penalize frequent words
# (which incidentally are sorted in the vocab file)
# but I don't do that
# if it prints too many new lines instead of continuing generating text,
# you might want to comment this out
#if idx2word[generated_token] == '\n':
# continue
if generated_token in penalized_so_far:
continue
penalized_so_far.add(generated_token)
prompt_logits[_token][generated_token] /= penalty
# disallow some tokens
prompt_logits[_token][word2idx['<unk>']] = -1e8
# sometimes, when generating from reddit,
# it tries to generate the Score (reddit Karma) immediately after generating the Title:
# to disallow this, we can just prevent it from generating Score
prompt_logits[_token][word2idx['Sco@@']] = -1e8
# compute probabilities from logits
prompt_probs = np.exp(prompt_logits[_token])
prompt_probs = prompt_probs / sum(prompt_probs)
pruned_list = np.argsort(prompt_probs)[::-1]
# if you are using nucleus prob, then compute the nucleus probability size
if nucleusprob > 0.:
minimum_topk = 1
nucleus = max(np.where(np.cumsum(np.sort(prompt_probs)[::-1])>nucleusprob)[0][0], minimum_topk)
elif topk > 0:
# we are over-loading notation here
# if you choose to specify a topk instead of a nucleus,
# we will hardcode the nucleus to be just that
nucleus = topk
else:
# if you specify neither nucleus or topk,
# then we will use the whole list
nucleus = len(pruned_list)
pruned_list = pruned_list[:nucleus]
# if you want to disallow more complex tokens, you can do so here
# for instance, if you want to disallow anything with the phrase `http`,
# you can delete theme from the pruned_list
# you can comment this out, I'm keeping it in for demonstration purpose
tokens_to_disallow = []
for _ in range(len(pruned_list)):
if 'http' in idx2word[pruned_list[_]]:
tokens_to_disallow.append(_)
pruned_list = np.delete(pruned_list, tokens_to_disallow)
if args.topn > 0 :
print('TOPN :: top-n alternatives:', [idx2word[_] for _ in pruned_list[:args.topn]])
# if temperature is 0
# just pick the first (most probable) token
if temperature==0:
idx = pruned_list[0]
else:
# else,
# sample from the pruned_list with the logits
chosen_idx = torch.distributions.categorical.Categorical(torch.tensor(np.expand_dims(prompt_logits[_token][pruned_list],0))).sample().numpy()[0]
idx = pruned_list[chosen_idx]
if args.topn > 0 :
print('TOPN :: chosen word:', idx2word[idx])
# assign the token for generation
tokens_generated[0][token+1] = idx
# clear screen if you want to
# os.system("clear")
tokens_generated_so_far = ' '.join([idx2word[c] for c in tokens_generated[0].squeeze()[:token+2]])
tokens_generated_so_far = re.sub('(@@ )', '', string=tokens_generated_so_far)
tokens_generated_so_far = re.sub('(@@ ?$)', '', string=tokens_generated_so_far)
if not args.print_once:
print('---------------------------------------')
print(tokens_generated_so_far)
print()
print('---------------------------------------')
print(tokens_generated_so_far)
print()
except KeyboardInterrupt: #Exception as e:
print('Continuing')
generator/ctrl/pytorch_transformer.py
-143
View File
@@ -1,143 +0,0 @@
from __future__ import print_function
import torch
import os
import tqdm
import pdb
import numpy as np
import platform
import re
import argparse
import tensorflow as tf
from tensorflow.python import pywrap_tensorflow
def angle_defn(pos, i, d_model_size):
angle_rates = 1 / np.power(10000, (2 * (i//2)) / np.float32(d_model_size))
return pos * angle_rates
def positional_encoding(position, d_model_size):
# create the sinusoidal pattern for the positional encoding
angle_rads = angle_defn(np.arange(position)[:, np.newaxis], np.arange(d_model_size)[np.newaxis, :], d_model_size)
sines = np.sin(angle_rads[:, 0::2])
cosines = np.cos(angle_rads[:, 1::2])
pos_encoding = torch.tensor(np.concatenate([sines, cosines], axis=-1)[np.newaxis, ...], dtype=torch.float)
return pos_encoding
def scaled_dot_product_attention(q, k, v, mask):
# calculate attention
matmul_qk = torch.matmul(q, k.permute(0,1,3,2))
dk = k.shape[-1]
scaled_attention_logits = matmul_qk / np.sqrt(dk)
if mask is not None:
scaled_attention_logits += (mask * -1e9)
attention_weights = torch.softmax(scaled_attention_logits, dim=-1)
output = torch.matmul(attention_weights, v)
return output
class MultiHeadAttention(torch.nn.Module):
def __init__(self, d_model_size, num_heads):
super(MultiHeadAttention, self).__init__()
self.num_heads = num_heads
self.d_model_size = d_model_size
self.depth = int(d_model_size / self.num_heads)
self.Wq = torch.nn.Linear(d_model_size, d_model_size)
self.Wk = torch.nn.Linear(d_model_size, d_model_size)
self.Wv = torch.nn.Linear(d_model_size, d_model_size)
self.dense = torch.nn.Linear(d_model_size, d_model_size)
def split_into_heads(self, x, batch_size):
x = x.reshape(batch_size, -1, self.num_heads, self.depth)
return x.permute([0, 2, 1, 3])
def forward(self, v, k, q, mask):
batch_size = q.shape[0]
q = self.Wq(q)
k = self.Wk(k)
v = self.Wv(v)
q = self.split_into_heads(q, batch_size)
k = self.split_into_heads(k, batch_size)
v = self.split_into_heads(v, batch_size)
scaled_attention = scaled_dot_product_attention(q, k, v, mask).permute([0, 2, 1, 3])
original_size_attention = scaled_attention.reshape(batch_size, -1, self.d_model_size)
output = self.dense(original_size_attention)
return output
def point_wise_feed_forward_network(d_model_size, dff):
return torch.nn.Sequential(torch.nn.Linear(d_model_size, dff), torch.nn.ReLU(), torch.nn.Linear(dff, d_model_size))
class EncoderLayer(torch.nn.Module):
def __init__(self, d_model_size, num_heads, dff, rate=0.1):
super(EncoderLayer, self).__init__()
self.multi_head_attention = MultiHeadAttention(d_model_size, num_heads)
self.ffn = point_wise_feed_forward_network(d_model_size, dff)
self.layernorm1 = torch.nn.LayerNorm(d_model_size, eps=1e-6)
self.layernorm2 = torch.nn.LayerNorm(d_model_size, eps=1e-6)
self.dropout1 = torch.nn.Dropout(rate)
self.dropout2 = torch.nn.Dropout(rate)
def forward(self, x, mask):
normed = self.layernorm1(x)
attn_output = self.multi_head_attention(normed, normed, normed, mask)
attn_output = self.dropout1(attn_output)
out1 = x + attn_output
out2 = self.layernorm2(out1)
ffn_output = self.ffn(out2)
ffn_output = self.dropout2(ffn_output)
out2 = out1 + ffn_output
return out2
class Encoder(torch.nn.Module):
def __init__(self, num_layers=48, d_model_size=1280, num_heads=16, dff=8192, input_vocab_size=50000,
rate=0.1, **kwargs):
super(Encoder, self).__init__()
self.d_model_size = d_model_size
self.num_layers = num_layers
self.pos_encoding = positional_encoding(input_vocab_size, self.d_model_size).to('cuda')
for i in range(num_layers):
setattr(self, "layer%i" % i, EncoderLayer(d_model_size, num_heads, dff, rate))
self.layernorm = torch.nn.LayerNorm(d_model_size, eps=1e-6)
self.dropout = torch.nn.Dropout(rate)
def forward(self, x):
seq_len = x.shape[1]
mask = torch.triu(torch.ones(seq_len, seq_len), 1).to('cuda')
x *= np.sqrt(self.d_model_size)
x += self.pos_encoding[:, :seq_len, :]
x = self.dropout(x)
for i in range(self.num_layers):
x = getattr(self, "layer%i" % i)(x, mask)
return self.layernorm(x)
generator/ctrl/source_attribution.py
-192
View File
@@ -1,192 +0,0 @@
from __future__ import division
from __future__ import print_function
import tensorflow as tf
import os
import numpy as np
tf.enable_eager_execution()
import transformer
import argparse
import pdb
import sys
import re
from collections import Counter
from tensorflow.python import debug as tf_debug
from tensorflow.python.ops import math_ops
from tensorflow.python.ops import embedding_ops
import fastBPE
import platform
use_py3 = platform.python_version()[0] == '3'
parser = argparse.ArgumentParser(description='TensorFlow code for generating from CTRL')
parser.add_argument('--model_dir', type=str, required=True,
help='location of model checkpoint')
parser.add_argument('--seed', type=int, default=1337,
help='random seed for TensorFlow, numpy and PythonHash')
args = parser.parse_args()
tf.random.set_random_seed(args.seed)
os.environ['PYTHONHASHSEED'] = str(args.seed)
np.random.seed(args.seed)
# load the vocabulary from file
vocab = open('vocab').read().decode(encoding='utf-8').split('\n') if not use_py3 else open('vocab', encoding='utf-8').read().split('\n')
vocab = list(map(lambda x: x.split(' ')[0], vocab)) + ['<unk>'] + ['\n']
print ('{} unique words'.format(len(vocab)))
# length of the vocabulary
vocab_size = len(vocab)
# define the numericalization map
# idx2word maps the numericalized ID to the word
# word2idx maps the word to the numericalized ID
word2idx = {u:i for i, u in enumerate(vocab)}
idx2word = np.array(vocab)
# sequence length to use for the transformer
# the model is trained with a seq_length of 512
# so, any value <= 512 should work
seq_length = 256
# the dimension of the transformer
embedding_dim = 1280
# Now, we begin defining the model
# we defer the transformer definition to transformer.py
# here, we only define the tied softmax layer
# this layer ties the softmax weights to the input embeddings
class TiedEmbeddingSoftmax(tf.keras.layers.Layer):
def __init__(self, vocab_size=vocab_size, embedding_size=embedding_dim, **kwargs):
super(TiedEmbeddingSoftmax, self).__init__()
self.w = self.add_weight(name='w', shape=(vocab_size, embedding_size),
initializer='random_normal',
trainable=True)
self.b = self.add_weight(name='b', shape=(vocab_size,),
initializer='zeros',
trainable=True)
def call(self, inputs, embed=True):
if embed:
dtype = tf.keras.backend.dtype(inputs)
if dtype != 'int32' and dtype != 'int64':
inputs = math_ops.cast(inputs, 'int32')
return embedding_ops.embedding_lookup(self.w, inputs)
else:
return tf.tensordot(inputs, tf.transpose(self.w), 1) + self.b
# input for the keras model
tokens = tf.keras.layers.Input(shape=(seq_length,), dtype='int32')
# instantiates a tied softmax class
tied_embedding_softmax = TiedEmbeddingSoftmax()
# embedded tokens, before passing it to the transformer
embedded = tied_embedding_softmax(tokens, embed=True)
# the activations after passing it from the transformer
# for some odd reason, TPUs don't play well with specifying the arguments of the Encoder() function
# so you have to leave them at their defaults
transformed = transformer.Encoder()(embedded, training=False)
# pass the activations from our tiedsoftmax class
# this time with embed=False denoting that we are doing the softmax operation
# and not a lookup
logits = tied_embedding_softmax(transformed, embed=False)
# finally, define the Keras model with inputs as tokens and outputs as the logits we just computed
model = tf.keras.Model(inputs=tokens, outputs=logits)
# the loss function is a simple categorical crossentropy between the logits and the labels
def loss(labels, logits):
return tf.keras.losses.sparse_categorical_crossentropy(labels, logits, from_logits=True)
# the optimizer is not used since this code only supports inference
# however, to compile the model, we still define it
optimizer = tf.contrib.tpu.CrossShardOptimizer(
tf.contrib.estimator.clip_gradients_by_norm(
tf.train.AdagradOptimizer(learning_rate=1e-2), 0.25)
)
# compile the model with the optimizer and loss
model.compile(optimizer=optimizer, loss=loss)
print(model.summary())
# IMPORTANT
# this is where the saved model is presented to the code
# the model directory should have the model checkpoint and
# a checkpoint file
run_config = tf.contrib.tpu.RunConfig(
model_dir=args.model_dir)
# this converts the Keras model to a TensorFlow estimator
# this step is critical
# remember to patch the TF 1.14 file before running the code, else you're going to see errors here
estimator_model = tf.keras.estimator.model_to_estimator(keras_model=model, config=run_config)
# we now create a serving function from this estimator
# this enables us to load the model once and easily query it multiple times
def serving_input_fn():
inputs = {'input_1': tf.placeholder(tf.int32, [2,seq_length])}
return tf.estimator.export.ServingInputReceiver(inputs, inputs)
predict_fn = tf.contrib.predictor.from_estimator(estimator_model, serving_input_fn)
# almost there, we now take the user prompt and tokenize with BPE
# load BPE codes
bpe = fastBPE.fastBPE('codes', 'vocab')
domains = []
with open('control_codes.txt', 'r') as f:
domains = [line.split() for line in f.readlines()]
domains = [(t[1], float(t[0])) for t in domains]
while True:
_prompt = raw_input('ENTER PROMPT: ') if not use_py3 else input('ENTER PROMPT: ')
ppls = {}
# loop over all domains and compute perplexity
for domain, domain_prior in domains:
print(u'computing for domain: {}'.format(domain))
# tokenize data and add domain tag to it
prompt = domain + u' ' + _prompt
split_prompt = bpe.apply([prompt])[0].split()
# numericalize data and pad to the seq_len dimension
text = [word2idx[i] for i in split_prompt]
padding_text = text + [0] * (seq_length - len(text))
tokens_generated = np.tile(padding_text, (2,1))
output_scores = predict_fn({'input_1':tokens_generated})['tied_embedding_softmax'].squeeze()[0]
token_scores = output_scores[:-1]
# compute the perplexity for this sequence
xent = 0
for sequence_idx, token_idx in enumerate(text[1:]):
token = idx2word[token_idx]
# compute the probability of this token
Z = np.exp(token_scores[sequence_idx]).sum()
token_prob = np.exp(token_scores[sequence_idx, token_idx]) / Z
xent -= np.log(token_prob) / len(text[1:])
ppls[domain] = round(np.exp(xent), 6)
#print(u'{} ppl = {}'.format(domain, ppls[domain]))
# sort the domains based on perplexities and print
ppls = [(k, v) for k, v in ppls.items()]
ppls.sort(key=lambda x: x[1])
print('PROMPT: {}'.format(_prompt))
for t in ppls:
domain, ppl = t
print(u'{} ppl = {}'.format(domain, ppl))
generator/ctrl/training_utils/.gitignore
-3
View File
@@ -1,3 +0,0 @@
seqlen256_v1.ckpt
*.txt
*.tfrecords
generator/ctrl/training_utils/README.md
-85
View File
@@ -1,85 +0,0 @@
# Fine-Tuning the Model on Custom Dataset
This folder contains sample code to fine-tune the model on custom data. It is primarily targeted for GPU usage, but there are pointers throughout showing how to run on TPUs as well.
Fine-tuning can be used to augment existing control codes or add new control codes. There are 5 steps elaborated upon in the example below:
1. Patch `keras.py` as in the generation script
2. Obtain raw versions of your text files
3. Convert this text data into TFRecords; _if you wish to use TPUs, you must transfer these records to GCS._
4. Fine-tuning the model on these TFRecords files
5. Testing that the generation works.
## Example of adding a new control code
Let's begin by adding a new control code `Moby` that is associated with the book [Moby Dick](https://www.gutenberg.org/ebooks/2701)
If you run `generation.py` with the pretrained models available and try to use this control code, you will find that the model outputs gibberish. Great! It is indeed a fresh new control code. We will run this again after training as a sanity check.
### Step 1 - Patch your `keras.py`
As is required for the generation script, you must patch your `keras.py`. If you patched it before, please roll-back and re-patch with the latest version.
There are two changes: (1) it defaults to `use_tpu=False` so training/inference takes place on GPUs, (2) the batch size defaults to 4 for GPU training. You might need to go lower depending on your machine.
You can leave `use_tpu=True` if you wish to train on TPUs and adjust the batch size accordingly.
### Step 2 - Obtain Your Data
The book is available publicly; you can simply download it as
```
wget -O moby_dick.txt https://www.gutenberg.org/files/2701/2701-0.txt
```
### Step 3 - Convert Data to TFRecords
We include the file `make_tf_records.py` to facilitate this.
Run:
```
python make_tf_records.py --text_file moby_dick.txt --control_code Moby --sequence_len 256
```
It has three arguments: `text_file` which specifies the name of the file to convert, `control_code` which specifies one token (must be in vocabulary) to append to each example, and `sequence_len` which specifies the sequence length to use to create the data. This must match the sequence length of the model being trained.
### Step 4 - Train!
Simply run `python training.py --model_dir <path_to_model>.ckpt/ --iterations <number_of_iterations>`
The script picks up all TFRecords in the current folder and fine-tunes the model provided in the `--model_dir` flag.
If you intend to use TPUs, you must transfer these TFRecords to GCS and edit the location of the data path used by `input_fn` to the GCS bucket.
To very important gotchas here:
1. If you have very limited data, the model will very likely overfit and end up memorizing. At the moment, just keep the `--iterations` flag low, preferably equivalent to one epoch or so.
2. The model is updated and stored in the same directory, if you don't wish to overwrite your model files, please create a backup before you run the training code.
### Step 5 - Generate!
We ran `python training.py --model_dir seqlen256_v1.ckpt/ --iterations 250` and try generating with the `Moby` control code.
Running with the `Moby` control code and a prompt of `I` yields something reasonable in-domain:
```
Moby I <GENERATION_BEGINS> was a little fellow, and he was a
great man, what should that matter? And yet it seemed to me that
Queequegs words about his father were true. He had been very angry
with him, because the old man would not let him go a-whaling...
```
Providing a prompt also works:
```
Moby Then I realized, it wasn't one white whale but three! <GENERATION_BEGINS> And all three
were making straight for my boat, which was now some distance away.
But the three spouts seemed to be coming from different directions, and
as they drew nearer and nearer, their tongues began licking up the
brine like so many hungry wolves at a carcass...
```
generator/ctrl/training_utils/apoc_seed1.json
-1
View File
@@ -1 +0,0 @@
{"tree_id": "apoc_seed1", "context": "Long ago the bombs fell and the world ended. You are one of the few who is still alive. You are trying to survive by scavenging among the ruins of what is left behind. ", "first_story_block": "You are standing outside an old rundown gas station at the edge of town. There are old broken cars surrounding the gas station and on the road. You are extremely hungry and haven't eaten in a few days. ", "action_results": [{"action": "look in the gas station for food. ", "result": "You search the gas station for food, but find nothing you can eat.", "action_results": [{"action": "search the cars for food.", "result": "You search the cars for food, you search three without success, but in the glove compartment of the fourth you find a box of crackers.", "action_results": [{"action": "eat the crackers.", "result": "They remind you of when you needed to eat a bland diet after your operation. You pat yourself on where you remember the incision was made. The scar healed long ago, but the brain remembers. The crackers weren't very nutritious, but some fuel in the tank is better than none.", "action_results": [{"action": "Continue to look for food in the building up ahead.", "result": "You find a moldy sandwich and some peppered beef jerky in an old lunchbox.", "action_results": [{"action": "Snack on the jerky while searching for more food.", "result": "The jerky is so hard your tooth falls out.", "action_results": [{"action": "look for a mirror to assess the damage", "result": "Blood is pouring from your mouth and the pain is unbearable.", "action_results": [{"action": "Press your tongue into the abscess to stop the bleeding.", "result": "A foul odor rushing into your nostrils and the pain becomes even worse.", "action_results": []}, {"action": "Look for cloth or paper towel to stuff the gaping wound with.", "result": "You find a cloth on the floor. It looks like a used bandage, and is crusted with dried blood.", "action_results": []}]}, {"action": "use pliers to pull the rest of the tooth", "result": "Passes out from the pain. Falls to the floor", "action_results": [{"action": "Check your things for alcohol or antiseptic.", "result": "You find a bottle of alcohol in your bag. You take a swig from the bottle.", "action_results": []}, {"action": "Search the area for medical supplies.", "result": "You find a bottle of Tylenol and struggle to open it.", "action_results": []}]}]}, {"action": "Scrape the mold from the sandwich, taste a bite to see if it is edible.", "result": "The sandwich is stale. You throw it away and hope you don't get sick.", "action_results": [{"action": "Look for more food.", "result": "There is no more food around, but there's two bottles of beer.", "action_results": [{"action": "Drink one beer.", "result": "You drink the beer, but it tastes funny. You hope you don't get sick later.", "action_results": []}, {"action": "Put the beer in your bag and walk towards the next store.", "result": "You grab the beer, leaving the building and heading toward the next store. You are hungry.", "action_results": []}]}, {"action": "Take some medicine you saved.", "result": "It wasn't enought, you are food poisoned.", "action_results": [{"action": "take more medicine", "result": "The medicine does little to improve your upset stomach.", "action_results": []}, {"action": "ride out the food poisoning", "result": "I try to drink some milk.", "action_results": []}]}]}, {"action": "Eat the peppered beef and throw the sandwich to the side", "result": "You discovered the peppered beef had gone bad when you developed cramps and diarrhea just hours after eating it. Fortunately, the food poisoning will pass and you will be all right.", "action_results": [{"action": "Look for water.", "result": "You find a dirty puddle. You know it is likely contaminated but you are so thirsty you bend down and lap it up.", "action_results": [{"action": "It tastes horrible and within a half hour your stomach begins having awful pains.", "result": "You feel nauseous and begin to vomit loudly.", "action_results": []}, {"action": "You change your mind and make yourself throw up the water. You gather the dirty water in a container and search for a way to boil or purify it.", "result": "You see something that looks like a lighter sticking out of a fanny pack caught on a tree, but it looks like you'll need help to get it down.", "action_results": []}, {"action": "Get on your feet and keep walking.", "result": "You walk on. Hopefully, you can come across more water and food.", "action_results": []}, {"action": "Consider the new lows you're hitting, lapping from a dirty puddle.", "result": "You decide it would be better to boil the water before drinking. You fill a container and proceed to start a fire.", "action_results": []}]}, {"action": "Look for more food.", "result": "You find some rotten vegetables, and nothing more.", "action_results": [{"action": "Throw the rotten vegetables against the wall, in angerLine truncated
generator/ctrl/training_utils/apoc_seed2.json
-1
View File
@@ -1 +0,0 @@
{"tree_id": "apoc_seed2", "context": "Long ago the bombs fell and the world ended. You are one of the few who is still alive. You are trying to survive by scavenging among the ruins of what is left behind. ", "first_story_block": "You find a car with the keys in the ignition and try to start it. Nothing happens. You keep looking and find another car that looks like it might work. You manage to get the engine started. ", "action_results": [{"action": "drive around looking for any people or stores that might have supplies.", "result": "You drive around the town and see many boarded up buildings, but no people. Eventually you find a store that you think might have supplies.", "action_results": [{"action": "go inside the store and see if there are any supplies.", "result": "It looks like the store has been ransacked and all the shelves are empty. You manage to find a package of granola bars and a flashlight behind the counter.", "action_results": [{"action": "eat a granola bar.", "result": "It tastes just like a granola bar you enjoyed when you were younger. In an instant, you are transported to a school hiking trip. Your mind is awash by the mountain air and the bundles of sticks and leaves that formed the trail. Those days seem so long ago now...", "action_results": [{"action": "Look for other doors inside the store.", "result": "You find the store manager's door, with a set of keys hanging off the lock. The door, though, seems to be blocked from the inside.", "action_results": [{"action": "Push the door to uncover what is blocking it from the inside", "result": "You hear a scream from the inside: \"Go away! I have a gun!\"", "action_results": [{"action": "Continue to push the door.", "result": "You yell: \"So do I!\" and feel the door give a bit more.", "action_results": [{"action": "kick the door", "result": "You try to scape jumping off the window.", "action_results": []}, {"action": "continue to push the door", "result": "The door doesn't give. You look around to find something to ram against the door.", "action_results": []}]}, {"action": "Try to talk down the person behind the door.", "result": "explains that he is a good person, and tries to approach slowly", "action_results": [{"action": "Put your hands up to show you are not armed.", "result": "You put your hands up, but your tattered, duct-taped Remington 870 shotgun is still within reach. Good thing too, as the man reaches behind his back and pulls out a rusty, long blade. You side step, grabbing the shotgun, as the man thrusts out with the knife.", "action_results": []}, {"action": "Take your crossbow out and keep moving forward.", "result": "You arm yourself with the crossbow. You cannot be too cautious.", "action_results": []}]}]}, {"action": "Knock on the door, while being prepared for what is inside.", "result": "You swing the door open and find nothing there. You let out an audible sigh of relief. The door was only jammed.", "action_results": [{"action": "Search the manager's desk.", "result": "You find a note that was written by the last owner. You open up the note.", "action_results": [{"action": "the note is worthless so you throw it away", "result": "You notice that the managers desk has two drawers. One is empty, however the other is locked.", "action_results": []}, {"action": "the note has a cryptic message on it, but you keep it in case someone else can decode it", "result": "You move on from room to room looking for useful supplies.", "action_results": []}]}, {"action": "Look at the coat rack by the door.", "result": "You find a hat, which will help a little when it gets cold at night, but not the warm coat you were hoping for.", "action_results": [{"action": "start looking for more things", "result": "There is no coat here but there is a worn rucksack hung over a chair.", "action_results": []}, {"action": "go out to look for more places", "result": "You find two suitable places that may have clothing to scavenge.", "action_results": []}]}]}, {"action": "Attempt to break down the door with something heavy.", "result": "You look around and find a piece of wood. You take the wood and ram it against the door. Eventually, it starts to give.", "action_results": [{"action": "Keep ramming.", "result": "need to enter that room", "action_results": [{"action": "Hit the hinges and see if you can knock them loose.", "result": "Suddenly the door falls over, and you can now enter the room.", "action_results": []}, {"action": "Take a running start and ram the door as hard as you can.", "result": "The door opens, and you can't believe what you see inside.", "action_results": []}]}, {"action": "Take a break.", "result": "Sitting in the quiet, suddenly you hear a scratching noise coming from the other side of the manager's office door.", "action_results": [{"action": "Listen and try to identify if there is an animal.", "result": "The scratching intensifies. You notice the door begin to shake against the weight of the creature behind it.", "actiLine truncated
generator/ctrl/training_utils/apoc_seed3.json
-1
View File
@@ -1 +0,0 @@
{"tree_id": "apoc_seed3", "context": "Long ago the bombs fell and the world ended. You are one of the few who is still alive. You are trying to survive by scavenging among the ruins of what is left behind. ", "first_story_block": "As you leave the store you notice movement out behind your car. A man steps out with a hungry look in his eyes. His clothes are in tatters and he is carrying a crowbar.", "action_results": [{"action": "run at the man and attack him.", "result": "You run forward to attack the man. His crowbar crashes into your chest and you fall to the ground. He kicks you a few times and then grabs your granola bars and runs off.", "action_results": [{"action": "go after him.", "result": "You go after him. As you try and stand up though you realize you're in extreme pain. You think one of your ribs is broken. You manage to get yourself on your feet, but realize catching him is impossible. ", "action_results": [{"action": "look for something to wrap your chest with.", "result": "There's nothing around, so you resort to taking off your pants and wrapping it around your chest. There's a mild draft right now...", "action_results": [{"action": "Check for other injuries.", "result": "There are some superficial cuts on the arms, but nothing else seems to be broken.", "action_results": [{"action": "Search for clothes or blankets for warmth.", "result": "There are boxes on shelves on the wall. You check them for supplies and eventually come across a tote full of blankets. You take one of the blankets out of the tote.", "action_results": [{"action": "Wrap it around yourself.", "result": "You are starting to feel warm again. You can feel your fingers.", "action_results": [{"action": "Count your fingers just for fun.", "result": "You warm up for a bit before collecting more supplies.", "action_results": []}, {"action": "Turn the boxes into shoes.", "result": "You remove the boxes from the shelves, placing two on your feet as makeshift shoes. Your feet begin to warm up.", "action_results": []}]}, {"action": "Stuff it in your bag.", "result": "Your bag is now too full to hold anything else so you remove the other contents so all you are carrying is the blanket. You look around for something to use as a second bag.", "action_results": [{"action": "Make a rucksack out of a dirty blanket.", "result": "The blanket has too many holes, and will not hold anything.", "action_results": []}, {"action": "Create a backpack out of the cardboard box.", "result": "While the cardboard box is uncomfortable, it serves its purpose to carry your supplies.", "action_results": []}]}]}, {"action": "Find water and rinse the exposed limbs to help clean cuts.", "result": "The water is freezing cold, but at least it helps you recover from the shock of being so brutally attacked.", "action_results": [{"action": "Dry off and build a fire.", "result": "The fire comforts you for a brief moment.", "action_results": [{"action": "Watch as the fire comes to life and begins to chase you.", "result": "You decide to go look for food and water after regaining strength.", "action_results": []}, {"action": "Fall asleep by the fire.", "result": "You stay warm throughout the night and wake up to begin the day. Hunting for food and water is on your agenda.", "action_results": []}, {"action": "i go to the store", "result": "I am unable to find anything valuable to scavange.", "action_results": []}, {"action": "go to the store", "result": "You reach the long abandoned store hoping that you might find food or other supplies still left inside. The windows are shattered, probably from previous scavangers.", "action_results": []}]}, {"action": "Dry off and keep moving.", "result": "It is starting to get dark. You must find shelter soon. As you come up over the hill you see a cave. Looks like a good place for shelter.", "action_results": [{"action": "Walk into the cave.", "result": "You walk into the cave and quickly look around to make sure that the cave is safe.", "action_results": []}, {"action": "Keep walking, leave the cave behind.", "result": "You end up sleeping outside unprotected.", "action_results": []}]}]}, {"action": "Look around you", "result": "There's a small river on your left, running along the road. On your right you can see a huge building a couple of miles away.", "action_results": [{"action": "Stop to take a closer look at the river bank.", "result": "There must be animals near the river, looking for water, so you sit quietly near the bank in the hopes of fresh meat.", "action_results": [{"action": "Find something to use as a weapon.", "result": "You look around for a weapon. You pick up a large stick from the ground.", "action_results": []}, {"action": "Refill your water bottle from the river.", "result": "You bring the bottle to your mouth. The water is muddy, but it's refreshing.", "action_results": []}]}, {"action": "Pass by the river and head for the huge building.", "result": "You need water, but the river is probably contaminatLine truncated
generator/ctrl/training_utils/apoc_seed4.json
-1
View File
@@ -1 +0,0 @@
{"tree_id": "apoc_seed4", "context": "Long ago the bombs fell and the world ended. You are one of the few who is still alive. You are trying to survive by scavenging among the ruins of what is left behind. ", "first_story_block": "You walk for two hours and take a break. You've left the town you were in and are now in a more rural area. There's a farmhouse to your right and a forest on your left. ", "action_results": [{"action": "go in the farmhouse and look for food.", "result": "You try to open the farmhouse front door, but it's locked. There's a broken window on the right and a shed behind the house.", "action_results": [{"action": "go through the window into the farmhouse. ", "result": "You break the rest of the glass in the window and try to carefully crawl through. You cut yourself on the edges as you're climbing in, but you make it inside. ", "action_results": [{"action": "search the farmhouse for food. ", "result": "You find meat in the fridge that looks like it's gone bad. You also find a can of peas in the bottom of the pantry.", "action_results": [{"action": "Search the farmhouse for tools and utencils.", "result": "You find a broom in the closet and a wooden spoon in the cupboard.", "action_results": [{"action": "Open the van of peas and use the wooden spoon to eat.", "result": "You get the can of peas open and use the wooden spoon to take a bite of them. They taste horrible, but you are so hungry that you hardly notice.", "action_results": [{"action": "Keep eating the peas.", "result": "As you finish the peas, you feel your stomach begin to turn, and know you are in trouble. You quickly look around for medicine.", "action_results": [{"action": "Force yourself to vomit out the peas.", "result": "You turn to the nearest bush and release the contents of your stomach, the pain subsides but the hunger returns.", "action_results": []}, {"action": "Try to find a place to lie down.", "result": "You lie down in a chair in the corner. It's uncomfortable, but you still fall asleep.", "action_results": []}]}, {"action": "Spit the peas onto the ground and stop eating them.", "result": "You begin to feel nauseous and vomit.", "action_results": [{"action": "realize this is from eating after a long time of not eating and force down the peas", "result": "You fight through the nausea and keep the food down.", "action_results": []}, {"action": "throw the can and it bounces off a case of vegetables you hurriedly rush over to.", "result": "Looking over the vegetables you feel they are by far uneatable.", "action_results": []}]}]}, {"action": "Sharpen the broom handle into a weapon for protection and use the broom head as a fire starter.", "result": "You put the broom into your bag and carry it around in case you need to protect yourself.", "action_results": [{"action": "Move to bedroom to find more stuff", "result": "As you enter the bedroom, you lead with the sharpened broom handle. Luckily it is empty, and you quickly pack up a blanket and jacket you find in the closet.", "action_results": [{"action": "there are thermals to wear in the drawers", "result": "It's warm. You are glad that you found the jacket.", "action_results": []}, {"action": "there are arrows to pick up on the ground", "result": "You walk outside. There is a deadly silence in the air.", "action_results": []}, {"action": "Try to fit some shoes in your bag that is quickly filling up.", "result": "You open the bag. find some shoe which is Ok for the time being. You wear it", "action_results": []}, {"action": "Turn around to pick up your broom.", "result": "You then hear a loud crashing sound coming from nearbye.", "action_results": []}]}, {"action": "Check that doors are locked", "result": "You find one door is unlocked, leading you to a dark staircase.", "action_results": [{"action": "go down the stairs", "result": "You walk down the dark staircase, cobwebs choking you as you struggle to see.", "action_results": []}, {"action": "close the door without going through the doorway", "result": "You continue walking through the building. You hear a noise and pull out your broom.", "action_results": []}]}]}, {"action": "take the spoon and the broom", "result": "A man appears from another room in the house.", "action_results": [{"action": "Confront him - maybe he has things you can steal.", "result": "You pull out your gun and ask the man what he is doing there. You notice his bag, and tell him to empty it.", "action_results": [{"action": "Knock the man unconscious.", "result": "You tie the man up to a chair and wait for him to awaken. You ask him questions about supplies and other information that may be valuable.", "action_results": []}, {"action": "Browse through the stuff the man pulled out of the bag.", "result": "You find nothing of particular value, just the essentials to survive in this wasteland. You notice the man seems almost afraid.", "action_results": []}]}, {"action": "Run away.", "result": "You burst through the front door of the Line truncated
generator/ctrl/training_utils/forest_training.py
-172
View File
@@ -1,172 +0,0 @@
from __future__ import division
from __future__ import print_function
import sys
sys.path.append('../')
import tensorflow as tf
import os
import numpy as np
tf.enable_eager_execution()
import transformer
import argparse
import pdb
import re
from collections import Counter
from tensorflow.python import debug as tf_debug
from tensorflow.python.ops import math_ops
from tensorflow.python.ops import embedding_ops
import fastBPE
import platform
use_py3 = platform.python_version()[0] == '3'
parser = argparse.ArgumentParser(description='TensorFlow code for generating from CTRL')
parser.add_argument('--model_dir', type=str, required=True,
help='location of model checkpoint')
parser.add_argument('--seed', type=int, default=1337,
help='random seed for TensorFlow, numpy and PythonHash')
parser.add_argument('--sequence_len', type=int, default=256,
help='sequence len of model being fine-tuned (must match also the TFRecords)')
parser.add_argument('--iterations', type=int, default=1000,
help='random seed for TensorFlow, numpy and PythonHash')
args = parser.parse_args()
tf.random.set_random_seed(args.seed)
os.environ['PYTHONHASHSEED'] = str(args.seed)
np.random.seed(args.seed)
# load the vocabulary from file
vocab = open('../vocab').read().decode(encoding='utf-8').split('\n') if not use_py3 else open('../vocab',
encoding='utf-8').read().split(
'\n')
vocab = list(map(lambda x: x.split(' ')[0], vocab)) + ['<unk>'] + ['\n']
print('{} unique words'.format(len(vocab)))
# length of the vocabulary
vocab_size = len(vocab)
# define the numericalization map
# idx2word maps the numericalized ID to the word
# word2idx maps the word to the numericalized ID
word2idx = {u: i for i, u in enumerate(vocab)}
idx2word = np.array(vocab)
# sequence length to use for the transformer
# must match the model being fine-tuned
seq_length = args.sequence_len
def input_fn(params=None):
print('READING!', params)
dataset = tf.data.Dataset.list_files(tf.io.gfile.glob('./*.tfrecords'), shuffle=True)
tf_data = tf.data.TFRecordDataset(dataset)
myfeatures = {
'input': tf.io.FixedLenFeature([256], tf.int64),
'output': tf.io.FixedLenFeature([256], tf.int64)
}
def _parse_text_function(example_proto):
blah = tf.io.parse_single_example(example_proto, myfeatures)
return blah['input'], blah['output']
train_data = tf_data.map(_parse_text_function).batch(params['batch_size'], drop_remainder=True).repeat().shuffle(
10000) # .prefetch(tf.contrib.data.AUTOTUNE)
return train_data
# the dimension of the transformer
embedding_dim = 1280
# Now, we begin defining the model
# we defer the transformer definition to transformer.py
# here, we only define the tied softmax layer
# this layer ties the softmax weights to the input embeddings
class TiedEmbeddingSoftmax(tf.keras.layers.Layer):
def __init__(self, vocab_size=vocab_size, embedding_size=embedding_dim, **kwargs):
with tf.device('/cpu:0'):
super(TiedEmbeddingSoftmax, self).__init__()
self.w = self.add_weight(name='w', shape=(vocab_size, embedding_size),
initializer='random_normal',
trainable=True)
self.b = self.add_weight(name='b', shape=(vocab_size,),
initializer='zeros',
trainable=True)
def call(self, inputs, embed=True):
with tf.device('/cpu:0'):
if embed:
dtype = tf.keras.backend.dtype(inputs)
if dtype != 'int32' and dtype != 'int64':
inputs = math_ops.cast(inputs, 'int32')
return embedding_ops.embedding_lookup(self.w, inputs)
else:
return tf.tensordot(inputs, tf.transpose(self.w), 1) + self.b
# input for the keras model
tokens = tf.keras.layers.Input(shape=(seq_length,), dtype='int32')
# instantiates a tied softmax class
tied_embedding_softmax = TiedEmbeddingSoftmax()
# embedded tokens, before passing it to the transformer
embedded = tied_embedding_softmax(tokens, embed=True)
# the activations after passing it from the transformer
# for some odd reason, TPUs don't play well with specifying the arguments of the Encoder() function
# so you have to leave them at their defaults
transformed = transformer.Encoder()(embedded, training=False)
# pass the activations from our tiedsoftmax class
# this time with embed=False denoting that we are doing the softmax operation
# and not a lookup
logits = tied_embedding_softmax(transformed, embed=False)
# finally, define the Keras model with inputs as tokens and outputs as the logits we just computed
model = tf.keras.Model(inputs=tokens, outputs=logits)
# the loss function is a simple categorical crossentropy between the logits and the labels
def loss(labels, logits):
loss = tf.keras.losses.sparse_categorical_crossentropy(labels, logits, from_logits=True)
mean = tf.math.reduce_mean(loss)
loss = tf.Print(loss, [mean])
return loss
# the optimizer is not used since this code only supports inference
# however, to compile the model, we still define it
optimizer = tf.contrib.estimator.clip_gradients_by_norm(
tf.train.AdagradOptimizer(learning_rate=3e-3), 0.25)
# compile the model with the optimizer and loss
model.compile(optimizer=optimizer, loss=loss)
print(model.summary())
# IMPORTANT
# this is where the saved model is presented to the code
# the model directory should have the model checkpoint and
# a checkpoint file
run_config = tf.contrib.tpu.RunConfig(
model_dir=args.model_dir)
# this converts the Keras model to a TensorFlow estimator
# this step is critical
# remember to patch the TF 1.14 file before running the code, else you're going to see errors here
run_config = tf.contrib.tpu.RunConfig(
model_dir=args.model_dir,
session_config=tf.ConfigProto(allow_soft_placement=True, log_device_placement=True),
tpu_config=tf.contrib.tpu.TPUConfig(iterations_per_loop=100, num_cores_per_replica=1,
input_partition_dims=[[1, 1], [1, 1]], per_host_input_for_training=3))
tf.logging.set_verbosity(tf.logging.INFO)
estimator_model = tf.keras.estimator.model_to_estimator(keras_model=model, config=run_config)
estimator_model.train(input_fn=input_fn, steps=args.iterations)
generator/ctrl/training_utils/make_forest_tf_records.py
-130
View File
@@ -1,130 +0,0 @@
import numpy as np
import os
import tensorflow as tf
import tqdm
import pdb
import glob
import time
import sys
import re
import argparse
import fastBPE
import platform
import json
import sys
# insert at 1, 0 is the script path (or '' in REPL)
sys.path.insert(1, '../../..')
from story.utils import *
def make_samples_helper(context, story_block, action_results, path, tree_id):
samples = []
for i, action_result in enumerate(action_results):
new_path = path[:]
new_path.append(i)
if action_result["result"] is not None:
sample = [context, story_block, action_result["action"], action_result["result"]]
samples.append(sample)
if len(action_result["action_results"]) is not 0:
sub_result = make_samples_helper(context, action_result["result"], action_result["action_results"], new_path, tree_id)
samples += sub_result
return samples
def make_samples(tree):
# Traverse to the bottom levels of each tree
first_story_block = tree["first_story_block"]
samples = make_samples_helper(tree["context"], first_story_block, tree["action_results"], [], tree["tree_id"])
return samples
def build_tokenized_samples(bpe, tree):
samples = make_samples(tree)
string_samples = []
for sample in samples:
sample = [string.strip() for string in sample]
sample[2] = sample[2][0].lower() + sample[2][1:]
sample[2] = "You " + sample[2]
new_sample = []
for item in sample:
new_sample.append(second_to_first_person(item))
string_samples.append(" ".join(new_sample))
tokenized_samples = [bpe.apply([sample.encode('ascii', errors='ignore') if not use_py3 else sample])[0] for sample in
string_samples] # will NOT work for non-English texts
tokenized_samples = [re.findall(r'\S+|\n', sample) for sample in tokenized_samples]
tokenized_samples = [list(filter(lambda x: x != u'@@', sample)) for sample in tokenized_samples]
# Fill samples up to seq_len
for sample in tokenized_samples:
pad_len = seq_length - len(sample)
for _ in range(pad_len):
sample.append("\n")
return tokenized_samples
use_py3 = platform.python_version()[0] == '3'
paths_to_train_files = ["apoc_seed1.json","apoc_seed2.json","apoc_seed3.json","apoc_seed4.json"]
seq_length = 256
domain = ["Apocalypse"]
# Build sequences from JSON
bpe = fastBPE.fastBPE('../codes', '../vocab')
tokenized_samples = []
for fname in paths_to_train_files:
with open(fname, 'r') as fp:
tree = json.load(fp)
tokenized_samples += build_tokenized_samples(bpe, tree)
string_samples = []
# load the vocabulary from file
vocab = open('../vocab').read().decode(encoding='utf-8').split('\n') if not use_py3 else open('../vocab', encoding='utf-8').read().split('\n')
vocab = list(map(lambda x: x.split(' ')[0], vocab)) + ['<unk>'] + ['\n']
print ('{} unique words'.format(len(vocab)))
# Creating a mapping from unique characters to indices
word2idx = {u:i for i, u in enumerate(vocab)}
idx2word = np.array(vocab)
seq_length = seq_length-1
def numericalize(x):
count = 0
for i in x:
if i not in word2idx:
print(i)
count += 1
return count>1, [word2idx.get(i, word2idx['<unk>']) for i in x]
tfrecords_fname = 'action_results.tfrecords'
total = 0
skipped = 0
with tf.io.TFRecordWriter(tfrecords_fname) as writer:
for sample in tokenized_samples:
domain_seq = (domain+sample)[:256+1]
flag_input, inputs = numericalize(domain_seq[:-1])
flag_output, outputs = numericalize(domain_seq[1:])
total += 1
if flag_input or flag_output:
skipped += 1
continue
if len(inputs)!=seq_length+1 or len(outputs)!=seq_length+1:
break
example_proto = tf.train.Example(features=tf.train.Features(feature={'input': tf.train.Feature(int64_list=tf.train.Int64List(value=inputs)),
'output': tf.train.Feature(int64_list=tf.train.Int64List(value=outputs))}))
writer.write(example_proto.SerializeToString())
print('Done')
print('Skipped', skipped, 'of', total)
generator/ctrl/training_utils/make_tf_records.py
-81
View File
@@ -1,81 +0,0 @@
import numpy as np
import os
import tensorflow as tf
import tqdm
import pdb
import glob
import time
import sys
import re
import argparse
import fastBPE
import platform
use_py3 = platform.python_version()[0] == '3'
parser = argparse.ArgumentParser(description='TensorFlow code for creating TFRecords data')
parser.add_argument('--text_file', type=str, required=True,
help='location of text file to convert to TFRecords')
parser.add_argument('--control_code', type=str, required=True,
help='control code to use for this file. must be in the vocabulary, else it will error out.')
parser.add_argument('--sequence_len', type=int, required=True,
help='sequence length of model being fine-tuned (256 or 512)')
args = parser.parse_args()
path_to_train_file = fname = args.text_file
domain = [args.control_code]
train_text = open(path_to_train_file, 'rb').read().decode(encoding='utf-8')
bpe = fastBPE.fastBPE('../codes', '../vocab')
tokenized_train_text = bpe.apply([train_text.encode('ascii', errors='ignore') if not use_py3 else train_text])[0] # will NOT work for non-English texts
# if you want to run non-english text, please tokenize separately using ./fast applybpe and then run this script on the .bpe file with utf8 encoding
tokenized_train_text = re.findall(r'\S+|\n', tokenized_train_text)
tokenized_train_text = list(filter(lambda x: x != u'@@', tokenized_train_text))
# load the vocabulary from file
vocab = open('../vocab').read().decode(encoding='utf-8').split('\n') if not use_py3 else open('../vocab', encoding='utf-8').read().split('\n')
vocab = list(map(lambda x: x.split(' ')[0], vocab)) + ['<unk>'] + ['\n']
print ('{} unique words'.format(len(vocab)))
if args.control_code not in vocab:
print('Provided control code is not in the vocabulary')
print('Please provide a different one; refer to the vocab file for allowable tokens')
sys.exit(1)
# Creating a mapping from unique characters to indices
word2idx = {u:i for i, u in enumerate(vocab)}
idx2word = np.array(vocab)
seq_length = args.sequence_len-1
def numericalize(x):
count = 0
for i in x:
if i not in word2idx:
print(i)
count += 1
return count>1, [word2idx.get(i, word2idx['<unk>']) for i in x]
tfrecords_fname = fname.lower()+'.tfrecords'
total = 0
skipped = 0
with tf.io.TFRecordWriter(tfrecords_fname) as writer:
for i in tqdm.tqdm(range(0, len(tokenized_train_text), seq_length)):
flag_input, inputs = numericalize(domain+tokenized_train_text[i:i+seq_length])
flag_output, outputs = numericalize(tokenized_train_text[i:i+seq_length+1])
total += 1
if flag_input or flag_output:
skipped += 1
continue
if len(inputs)!=seq_length+1 or len(outputs)!=seq_length+1:
break
example_proto = tf.train.Example(features=tf.train.Features(feature={'input': tf.train.Feature(int64_list=tf.train.Int64List(value=inputs)),
'output': tf.train.Feature(int64_list=tf.train.Int64List(value=outputs))}))
writer.write(example_proto.SerializeToString())
print('Done')
print('Skipped', skipped, 'of', total)
generator/ctrl/training_utils/training.py
-166
View File
@@ -1,166 +0,0 @@
from __future__ import division
from __future__ import print_function
import sys
sys.path.append('../')
import tensorflow as tf
import os
import numpy as np
tf.enable_eager_execution()
import transformer
import argparse
import pdb
import re
from collections import Counter
from tensorflow.python import debug as tf_debug
from tensorflow.python.ops import math_ops
from tensorflow.python.ops import embedding_ops
import fastBPE
import platform
use_py3 = platform.python_version()[0] == '3'
parser = argparse.ArgumentParser(description='TensorFlow code for generating from CTRL')
parser.add_argument('--model_dir', type=str, required=True,
help='location of model checkpoint')
parser.add_argument('--seed', type=int, default=1337,
help='random seed for TensorFlow, numpy and PythonHash')
parser.add_argument('--sequence_len', type=int, default=256,
help='sequence len of model being fine-tuned (must match also the TFRecords)')
parser.add_argument('--iterations', type=int, default=1000,
help='random seed for TensorFlow, numpy and PythonHash')
args = parser.parse_args()
tf.random.set_random_seed(args.seed)
os.environ['PYTHONHASHSEED'] = str(args.seed)
np.random.seed(args.seed)
# load the vocabulary from file
vocab = open('../vocab').read().decode(encoding='utf-8').split('\n') if not use_py3 else open('../vocab',
encoding='utf-8').read().split(
'\n')
vocab = list(map(lambda x: x.split(' ')[0], vocab)) + ['<unk>'] + ['\n']
print('{} unique words'.format(len(vocab)))
# length of the vocabulary
vocab_size = len(vocab)
# define the numericalization map
# idx2word maps the numericalized ID to the word
# word2idx maps the word to the numericalized ID
word2idx = {u: i for i, u in enumerate(vocab)}
idx2word = np.array(vocab)
# sequence length to use for the transformer
# must match the model being fine-tuned
seq_length = args.sequence_len
def input_fn(params=None):
print('READING!', params)
dataset = tf.data.Dataset.list_files(tf.io.gfile.glob('./*.tfrecords'), shuffle=True)
tf_data = tf.data.TFRecordDataset(dataset)
myfeatures = {
'input': tf.io.FixedLenFeature([256], tf.int64),
'output': tf.io.FixedLenFeature([256], tf.int64)
}
def _parse_text_function(example_proto):
blah = tf.io.parse_single_example(example_proto, myfeatures)
return blah['input'], blah['output']
train_data = tf_data.map(_parse_text_function).batch(params['batch_size'], drop_remainder=True).repeat().shuffle(
10000) # .prefetch(tf.contrib.data.AUTOTUNE)
return train_data
# the dimension of the transformer
embedding_dim = 1280
# Now, we begin defining the model
# we defer the transformer definition to transformer.py
# here, we only define the tied softmax layer
# this layer ties the softmax weights to the input embeddings
class TiedEmbeddingSoftmax(tf.keras.layers.Layer):
def __init__(self, vocab_size=vocab_size, embedding_size=embedding_dim, **kwargs):
super(TiedEmbeddingSoftmax, self).__init__()
self.w = self.add_weight(name='w', shape=(vocab_size, embedding_size),
initializer='random_normal',
trainable=True)
self.b = self.add_weight(name='b', shape=(vocab_size,),
initializer='zeros',
trainable=True)
def call(self, inputs, embed=True):
if embed:
dtype = tf.keras.backend.dtype(inputs)
if dtype != 'int32' and dtype != 'int64':
inputs = math_ops.cast(inputs, 'int32')
return embedding_ops.embedding_lookup(self.w, inputs)
else:
return tf.tensordot(inputs, tf.transpose(self.w), 1) + self.b
# input for the keras model
tokens = tf.keras.layers.Input(shape=(seq_length,), dtype='int32')
# instantiates a tied softmax class
tied_embedding_softmax = TiedEmbeddingSoftmax()
# embedded tokens, before passing it to the transformer
embedded = tied_embedding_softmax(tokens, embed=True)
# the activations after passing it from the transformer
# for some odd reason, TPUs don't play well with specifying the arguments of the Encoder() function
# so you have to leave them at their defaults
transformed = transformer.Encoder()(embedded, training=False)
# pass the activations from our tiedsoftmax class
# this time with embed=False denoting that we are doing the softmax operation
# and not a lookup
logits = tied_embedding_softmax(transformed, embed=False)
# finally, define the Keras model with inputs as tokens and outputs as the logits we just computed
model = tf.keras.Model(inputs=tokens, outputs=logits)
# the loss function is a simple categorical crossentropy between the logits and the labels
def loss(labels, logits):
return tf.keras.losses.sparse_categorical_crossentropy(labels, logits, from_logits=True)
# the optimizer is not used since this code only supports inference
# however, to compile the model, we still define it
optimizer = tf.contrib.estimator.clip_gradients_by_norm(
tf.train.AdagradOptimizer(learning_rate=3e-3), 0.25)
# compile the model with the optimizer and loss
model.compile(optimizer=optimizer, loss=loss)
print(model.summary())
# IMPORTANT
# this is where the saved model is presented to the code
# the model directory should have the model checkpoint and
# a checkpoint file
run_config = tf.contrib.tpu.RunConfig(
model_dir=args.model_dir)
# this converts the Keras model to a TensorFlow estimator
# this step is critical
# remember to patch the TF 1.14 file before running the code, else you're going to see errors here
run_config = tf.contrib.tpu.RunConfig(
model_dir=args.model_dir,
session_config=tf.ConfigProto(allow_soft_placement=True, log_device_placement=True),
tpu_config=tf.contrib.tpu.TPUConfig(iterations_per_loop=100, num_cores_per_replica=1,
input_partition_dims=[[1, 1], [1, 1]], per_host_input_for_training=3))
tf.logging.set_verbosity(tf.logging.INFO)
estimator_model = tf.keras.estimator.model_to_estimator(keras_model=model, config=run_config)
estimator_model.train(input_fn=input_fn, steps=args.iterations)
generator/ctrl/transformer.py
-139
View File
@@ -1,139 +0,0 @@
import tensorflow as tf
import numpy as np
def angle_defn(pos, i, d_model_size):
angle_rates = 1 / np.power(10000, (2 * (i//2)) / np.float32(d_model_size))
return pos * angle_rates
def positional_encoding(position, d_model_size):
# create the sinusoidal pattern for the positional encoding
angle_rads = angle_defn(np.arange(position)[:, np.newaxis], np.arange(d_model_size)[np.newaxis, :], d_model_size)
sines = np.sin(angle_rads[:, 0::2])
cosines = np.cos(angle_rads[:, 1::2])
pos_encoding = tf.cast(np.concatenate([sines, cosines], axis=-1)[np.newaxis, ...], dtype=tf.float32)
return pos_encoding
def scaled_dot_product_attention(q, k, v, mask):
# calculate attention
matmul_qk = tf.matmul(q, k, transpose_b=True)
dk = tf.cast(tf.shape(k)[-1], tf.float32)
scaled_attention_logits = matmul_qk / tf.math.sqrt(dk)
if mask is not None:
scaled_attention_logits += (mask * -1e9)
attention_weights = tf.nn.softmax(scaled_attention_logits, axis=-1)
output = tf.matmul(attention_weights, v)
return output
class MultiHeadAttention(tf.keras.layers.Layer):
def __init__(self, d_model_size, num_heads):
super(MultiHeadAttention, self).__init__()
self.num_heads = num_heads
self.d_model_size = d_model_size
self.depth = int(d_model_size / self.num_heads)
self.Wq = tf.keras.layers.Dense(d_model_size)
self.Wk = tf.keras.layers.Dense(d_model_size)
self.Wv = tf.keras.layers.Dense(d_model_size)
self.dense = tf.keras.layers.Dense(d_model_size)
def split_into_heads(self, x, batch_size):
x = tf.reshape(x, (batch_size, -1, self.num_heads, self.depth))
return tf.transpose(x, perm=[0, 2, 1, 3])
def call(self, v, k, q, mask):
batch_size = tf.shape(q)[0]
q = self.Wq(q)
k = self.Wk(k)
v = self.Wv(v)
q = self.split_into_heads(q, batch_size)
k = self.split_into_heads(k, batch_size)
v = self.split_into_heads(v, batch_size)
scaled_attention = tf.transpose(scaled_dot_product_attention(q, k, v, mask), perm=[0, 2, 1, 3])
original_size_attention = tf.reshape(scaled_attention, (batch_size, -1, self.d_model_size))
output = self.dense(original_size_attention)
return output
def point_wise_feed_forward_network(d_model_size, dff):
return tf.keras.Sequential([tf.keras.layers.Dense(dff, activation='relu'),
tf.keras.layers.Dense(d_model_size)])
class EncoderLayer(tf.keras.layers.Layer):
def __init__(self, d_model_size, num_heads, dff, rate=0.1):
super(EncoderLayer, self).__init__()
self.multi_head_attention = MultiHeadAttention(d_model_size, num_heads)
self.ffn = point_wise_feed_forward_network(d_model_size, dff)
self.layernorm1 = tf.keras.layers.LayerNormalization(epsilon=1e-6)
self.layernorm2 = tf.keras.layers.LayerNormalization(epsilon=1e-6)
self.dropout1 = tf.keras.layers.Dropout(rate)
self.dropout2 = tf.keras.layers.Dropout(rate)
def call(self, x, training, mask):
normed = self.layernorm1(x)
attn_output = self.multi_head_attention(normed, normed, normed, mask)
attn_output = self.dropout1(attn_output, training=training)
out1 = x + attn_output
out2 = self.layernorm2(out1)
ffn_output = self.ffn(out2)
ffn_output = self.dropout2(ffn_output, training=training)
out2 = out1 + ffn_output
return out2
class Encoder(tf.keras.layers.Layer):
def __init__(self, num_layers=48, d_model_size=1280, num_heads=16, dff=8192, input_vocab_size=50000,
rate=0.1, **kwargs):
super(Encoder, self).__init__()
self.d_model_size = d_model_size
self.num_layers = num_layers
self.pos_encoding = positional_encoding(input_vocab_size, self.d_model_size)
for i in range(num_layers):
setattr(self, "layer%i" % i, EncoderLayer(d_model_size, num_heads, dff, rate))
self.layernorm = tf.keras.layers.LayerNormalization(epsilon=1e-6)
self.dropout = tf.keras.layers.Dropout(rate)
def get_config(self):
base_config = super(Encoder, self).get_config()
return base_config
def call(self, x, training):
seq_len = tf.shape(x)[1]
mask = 1 - tf.linalg.band_part(tf.ones((seq_len, seq_len)), -1, 0)
x *= tf.math.sqrt(tf.cast(self.d_model_size, tf.float32))
x += self.pos_encoding[:, :seq_len, :]
x = self.dropout(x, training=training)
for i in range(self.num_layers):
x = getattr(self, "layer%i" % i)(x, training, mask)
return self.layernorm(x)
generator/ctrl/vocab
-246531
View File
File diff suppressed because it is too large. Load diff
generator/human/task_generator.py
-6
View File
@@ -1,6 +0,0 @@
"""
Story tree representation
"""
generator/tf/CONTRIBUTORS.md
-17
View File
@@ -1,17 +0,0 @@
# Contributors (alphabetically)
* **[madisonmay](https://github.com/madisonmay)**
Added Dockerfiles
* **[Margaret Mitchell et al](https://arxiv.org/abs/1810.03993)**
Our [usage](./README.md#usage) writeup was loosely inspired by the paper
[Model Cards for Model Reporting](https://arxiv.org/abs/1810.03993)
and related conversations with some of the authors.
* **[webproduktion01](https://github.com/webproduktion01)**
Ported download script to python.
**[Full code contributors list](https://github.com/openai/gpt-2/contributors).**
generator/tf/DEVELOPERS.md
-85
View File
@@ -1,85 +0,0 @@
# Installation
Git clone this repository, and `cd` into directory for remaining commands
```
git clone https://github.com/openai/gpt-2.git && cd gpt-2
```
Then, follow instructions for either native or Docker installation.
## Native Installation
All steps can optionally be done in a virtual environment using tools such as `virtualenv` or `conda`.
Install tensorflow 1.12 (with GPU support, if you have a GPU and want everything to run faster)
```
pip3 install tensorflow==1.12.0
```
or
```
pip3 install tensorflow-gpu==1.12.0
```
Install other python packages:
```
pip3 install -r requirements.txt
```
Download the model data
```
python3 download_model.py 117M
```
## Docker Installation
Build the Dockerfile and tag the created image as `gpt-2`:
```
docker build --tag gpt-2 -f Dockerfile.gpu . # or Dockerfile.cpu
```
Start an interactive bash session from the `gpt-2` docker image.
You can opt to use the `--runtime=nvidia` flag if you have access to a NVIDIA GPU
and a valid install of [nvidia-docker 2.0](https://github.com/nvidia/nvidia-docker/wiki/Installation-(version-2.0)).
```
docker run --runtime=nvidia -it gpt-2 bash
```
# Running
| WARNING: Samples are unfiltered and may contain offensive content. |
| --- |
Some of the examples below may include Unicode text characters. Set the environment variable:
```
export PYTHONIOENCODING=UTF-8
```
to override the standard stream settings in UTF-8 mode.
## Unconditional sample generation
To generate unconditional samples from the small model:
```
python3 src/generate_unconditional_samples.py | tee /tmp/samples
```
There are various flags for controlling the samples:
```
python3 src/generate_unconditional_samples.py --top_k 40 --temperature 0.7 | tee /tmp/samples
```
To check flag descriptions, use:
```
python3 src/generate_unconditional_samples.py -- --help
```
## Conditional sample generation
To give the model custom prompts, you can use:
```
python3 src/interactive_conditional_samples.py --top_k 40
```
To check flag descriptions, use:
```
python3 src/interactive_conditional_samples.py -- --help
```
generator/tf/LICENSE
-21
View File
@@ -1,21 +0,0 @@
MIT License
Copyright (c) 2019 OpenAI
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
generator/tf/README.md
-61
View File
@@ -1,61 +0,0 @@
# gpt-2
Code and samples from the paper ["Language Models are Unsupervised Multitask Learners"](https://d4mucfpksywv.cloudfront.net/better-language-models/language-models.pdf).
For now, we have only released a smaller (117M parameter) version of GPT-2.
See more details in our [blog post](https://blog.openai.com/better-language-models/).
## Usage
This repository is meant to be a starting point for researchers and engineers to experiment with GPT-2-117M. While GPT-2-117M is less proficient than GPT-2-1.5B, it is useful for a wide range of research and applications which could also apply to larger models.
### Some caveats
- GPT-2-117M robustness and worst case behaviors are not well-understood. As with any machine-learned model, carefully evaluate GPT-2-117M for your use case, especially if used without fine-tuning or in safety-critical applications where reliability is important.
- The dataset our GPT-2-117M was trained on contains many texts with [biases](https://twitter.com/TomerUllman/status/1101485289720242177) and factual inaccuracies, and thus GPT-2-117M is likely to be biased and inaccurate as well.
- To avoid having samples mistaken as human-written, we recommend clearly labeling samples as synthetic before wide dissemination. Our models are often incoherent or inaccurate in subtle ways, which takes more than a quick read for a human to notice.
### Work with us
Please [let us know](mailto:languagequestions@openai.com) if you’re doing interesting research with or working on applications of GPT-2-117M! We’re especially interested in hearing from and potentially working with those who are studying
- Potential malicious use cases and defenses against them (e.g. the detectability of synthetic text)
- The extent of problematic content (e.g. bias) being baked into the models and effective mitigations
## Development
See [DEVELOPERS.md](./DEVELOPERS.md)
## Contributors
See [CONTRIBUTORS.md](./CONTRIBUTORS.md)
## GPT-2 samples
| WARNING: Samples are unfiltered and may contain offensive content. |
| --- |
While we have not yet released GPT-2 itself, you can see some samples from it in the `gpt-2-samples` folder.
We show unconditional samples with default settings (temperature 1 and no truncation), with temperature 0.7, and with truncation with top_k 40.
We show conditional samples, with contexts drawn from `WebText`'s test set, with default settings (temperature 1 and no truncation), with temperature 0.7, and with truncation with top_k 40.
## Citation
Please use the following bibtex entry:
```
@article{radford2019language,
title={Language Models are Unsupervised Multitask Learners},
author={Radford, Alec and Wu, Jeff and Child, Rewon and Luan, David and Amodei, Dario and Sutskever, Ilya},
year={2019}
}
```
## Future work
We may release code for evaluating the models on various benchmarks.
We are still considering release of the larger models.
## License
[MIT](./LICENSE)
generator/tf/__init__.py
View File
Whitespace-only changes.
generator/tf/download_model.py
-28
View File
@@ -1,28 +0,0 @@
import os
import sys
import requests
from tqdm import tqdm
if len(sys.argv) != 2:
print('You must enter the model name as a parameter, e.g.: download_model.py 117M')
sys.exit(1)
model = sys.argv[1]
subdir = os.path.join('models', model)
if not os.path.exists(subdir):
os.makedirs(subdir)
subdir = subdir.replace('\\','/') # needed for Windows
for filename in ['checkpoint','encoder.json','hparams.json','model.ckpt.data-00000-of-00001', 'model.ckpt.index', 'model.ckpt.meta', 'vocab.bpe']:
r = requests.get("https://storage.googleapis.com/gpt-2/" + subdir + "/" + filename, stream=True)
with open(os.path.join(subdir, filename), 'wb') as f:
file_size = int(r.headers["content-length"])
chunk_size = 1000
with tqdm(ncols=100, desc="Fetching " + filename, total=file_size, unit_scale=True) as pbar:
# 1k for chunk_size, since Ethernet packet size is around 1500 bytes
for chunk in r.iter_content(chunk_size=chunk_size):
f.write(chunk)
pbar.update(chunk_size)
generator/tf/models/117M/.gitignore
-1
View File
@@ -1 +0,0 @@
model.ckpt.data-00000-of-00001
generator/tf/models/117M/checkpoint
-2
View File
@@ -1,2 +0,0 @@
model_checkpoint_path: "model.ckpt"
all_model_checkpoint_paths: "model.ckpt"
generator/tf/models/117M/encoder.json
-1
View File
@@ -1 +0,0 @@
{"!": 0, "\"": 1, "#": 2, "$": 3, "%": 4, "&": 5, "'": 6, "(": 7, ")": 8, "*": 9, "+": 10, ",": 11, "-": 12, ".": 13, "/": 14, "0": 15, "1": 16, "2": 17, "3": 18, "4": 19, "5": 20, "6": 21, "7": 22, "8": 23, "9": 24, ":": 25, ";": 26, "<": 27, "=": 28, ">": 29, "?": 30, "@": 31, "A": 32, "B": 33, "C": 34, "D": 35, "E": 36, "F": 37, "G": 38, "H": 39, "I": 40, "J": 41, "K": 42, "L": 43, "M": 44, "N": 45, "O": 46, "P": 47, "Q": 48, "R": 49, "S": 50, "T": 51, "U": 52, "V": 53, "W": 54, "X": 55, "Y": 56, "Z": 57, "[": 58, "\\": 59, "]": 60, "^": 61, "_": 62, "`": 63, "a": 64, "b": 65, "c": 66, "d": 67, "e": 68, "f": 69, "g": 70, "h": 71, "i": 72, "j": 73, "k": 74, "l": 75, "m": 76, "n": 77, "o": 78, "p": 79, "q": 80, "r": 81, "s": 82, "t": 83, "u": 84, "v": 85, "w": 86, "x": 87, "y": 88, "z": 89, "{": 90, "|": 91, "}": 92, "~": 93, "\u00a1": 94, "\u00a2": 95, "\u00a3": 96, "\u00a4": 97, "\u00a5": 98, "\u00a6": 99, "\u00a7": 100, "\u00a8": 101, "\u00a9": 102, "\u00aa": 103, "\u00ab": 104, "\u00ac": 105, "\u00ae": 106, "\u00af": 107, "\u00b0": 108, "\u00b1": 109, "\u00b2": 110, "\u00b3": 111, "\u00b4": 112, "\u00b5": 113, "\u00b6": 114, "\u00b7": 115, "\u00b8": 116, "\u00b9": 117, "\u00ba": 118, "\u00bb": 119, "\u00bc": 120, "\u00bd": 121, "\u00be": 122, "\u00bf": 123, "\u00c0": 124, "\u00c1": 125, "\u00c2": 126, "\u00c3": 127, "\u00c4": 128, "\u00c5": 129, "\u00c6": 130, "\u00c7": 131, "\u00c8": 132, "\u00c9": 133, "\u00ca": 134, "\u00cb": 135, "\u00cc": 136, "\u00cd": 137, "\u00ce": 138, "\u00cf": 139, "\u00d0": 140, "\u00d1": 141, "\u00d2": 142, "\u00d3": 143, "\u00d4": 144, "\u00d5": 145, "\u00d6": 146, "\u00d7": 147, "\u00d8": 148, "\u00d9": 149, "\u00da": 150, "\u00db": 151, "\u00dc": 152, "\u00dd": 153, "\u00de": 154, "\u00df": 155, "\u00e0": 156, "\u00e1": 157, "\u00e2": 158, "\u00e3": 159, "\u00e4": 160, "\u00e5": 161, "\u00e6": 162, "\u00e7": 163, "\u00e8": 164, "\u00e9": 165, "\u00ea": 166, "\u00eb": 167, "\u00ec": 168, "\u00ed": 169, "\u00ee": 170, "\u00ef": 171, "\u00f0": 172, "\u00f1": 173, "\u00f2": 174, "\u00f3": 175, "\u00f4": 176, "\u00f5": 177, "\u00f6": 178, "\u00f7": 179, "\u00f8": 180, "\u00f9": 181, "\u00fa": 182, "\u00fb": 183, "\u00fc": 184, "\u00fd": 185, "\u00fe": 186, "\u00ff": 187, "\u0100": 188, "\u0101": 189, "\u0102": 190, "\u0103": 191, "\u0104": 192, "\u0105": 193, "\u0106": 194, "\u0107": 195, "\u0108": 196, "\u0109": 197, "\u010a": 198, "\u010b": 199, "\u010c": 200, "\u010d": 201, "\u010e": 202, "\u010f": 203, "\u0110": 204, "\u0111": 205, "\u0112": 206, "\u0113": 207, "\u0114": 208, "\u0115": 209, "\u0116": 210, "\u0117": 211, "\u0118": 212, "\u0119": 213, "\u011a": 214, "\u011b": 215, "\u011c": 216, "\u011d": 217, "\u011e": 218, "\u011f": 219, "\u0120": 220, "\u0121": 221, "\u0122": 222, "\u0123": 223, "\u0124": 224, "\u0125": 225, "\u0126": 226, "\u0127": 227, "\u0128": 228, "\u0129": 229, "\u012a": 230, "\u012b": 231, "\u012c": 232, "\u012d": 233, "\u012e": 234, "\u012f": 235, "\u0130": 236, "\u0131": 237, "\u0132": 238, "\u0133": 239, "\u0134": 240, "\u0135": 241, "\u0136": 242, "\u0137": 243, "\u0138": 244, "\u0139": 245, "\u013a": 246, "\u013b": 247, "\u013c": 248, "\u013d": 249, "\u013e": 250, "\u013f": 251, "\u0140": 252, "\u0141": 253, "\u0142": 254, "\u0143": 255, "\u0120t": 256, "\u0120a": 257, "he": 258, "in": 259, "re": 260, "on": 261, "\u0120the": 262, "er": 263, "\u0120s": 264, "at": 265, "\u0120w": 266, "\u0120o": 267, "en": 268, "\u0120c": 269, "it": 270, "is": 271, "an": 272, "or": 273, "es": 274, "\u0120b": 275, "ed": 276, "\u0120f": 277, "ing": 278, "\u0120p": 279, "ou": 280, "\u0120an": 281, "al": 282, "ar": 283, "\u0120to": 284, "\u0120m": 285, "\u0120of": 286, "\u0120in": 287, "\u0120d": 288, "\u0120h": 289, "\u0120and": 290, "ic": 291, "as": 292, "le": 293, "\u0120th": 294, "ion": 295, "om": 296, "ll": 297, "ent": 298, "\u0120n": 299, "\u0120l": 300, "st": 301, "\u0120re": 302, "ve": 303, "\u0120e": 304, "ro": 305, "ly": 306, "\u0120be": 307, "\u0120g": 308, "\u0120T": 309, "ct": 310, "\u0120S": 311, "id": 312, "ot": 313, "\u0120I": 314, "ut": 315, "et": 316, "\u0120A": 317, "\u0120is": 318, "\u0120on": 319, "im": 320, "am": 321, "ow": 322, "ay": 323, "ad": 324, "se": 325, "\u0120that": 326, "\u0120C": 327, "ig": 328, "\u0120for": 329, "ac": 330, "\u0120y": 331, "ver": 332, "ur": 333, "\u0120u": 334, "ld": 335, "\u0120st": 336, "\u0120M": 337, "'s": 338, "\u0120he": 339, "\u0120it": 340, "ation": 341, "ith": 342, "ir": 343, "ce": 344, "\u0120you": 345, "il": 346, "\u0120B": 347, "\u0120wh": 348, "ol": 349, "\u0120P": 350, "\u0120with": 351, "\u01201": 352, "ter": 353, "ch": 354, "\u0120as": 355, "\u0120we": 356, "\u0120(": 357, "nd": 358, "ill": 359, "\u0120D": 360, "if": 361, "\u01202": 362, "ag": 363, "ers": 364, "ke": 365, "\u0120\"": 366, "\u0120H": 367, "em": 368, "\u0120con": 369, "\u0120W": 370, "\u0120R": 371, "her": 372, "\u0120was": 373, "\u0120r": 374, "od": 375, "\u0120F": 376, "ul": 377, "ate": 378, "\u0120at": 379, "ri": 380, "pp": 381, "oLine truncated
generator/tf/models/117M/hparams.json
-7
View File
@@ -1,7 +0,0 @@
{
"n_vocab": 50257,
"n_ctx": 1024,
"n_embd": 768,
"n_head": 12,
"n_layer": 12
}
generator/tf/models/117M/vocab.bpe
-50001
View File
File diff suppressed because it is too large. Load diff
generator/tf/models/774M/.gitignore
-2
View File
@@ -1,2 +0,0 @@
model.ckpt.data-00000-of-00001
generator/tf/models/774M/checkpoint
-2
View File
@@ -1,2 +0,0 @@
model_checkpoint_path: "model.ckpt"
all_model_checkpoint_paths: "model.ckpt"
generator/tf/models/774M/encoder.json
-1
View File
@@ -1 +0,0 @@
{"!": 0, "\"": 1, "#": 2, "$": 3, "%": 4, "&": 5, "'": 6, "(": 7, ")": 8, "*": 9, "+": 10, ",": 11, "-": 12, ".": 13, "/": 14, "0": 15, "1": 16, "2": 17, "3": 18, "4": 19, "5": 20, "6": 21, "7": 22, "8": 23, "9": 24, ":": 25, ";": 26, "<": 27, "=": 28, ">": 29, "?": 30, "@": 31, "A": 32, "B": 33, "C": 34, "D": 35, "E": 36, "F": 37, "G": 38, "H": 39, "I": 40, "J": 41, "K": 42, "L": 43, "M": 44, "N": 45, "O": 46, "P": 47, "Q": 48, "R": 49, "S": 50, "T": 51, "U": 52, "V": 53, "W": 54, "X": 55, "Y": 56, "Z": 57, "[": 58, "\\": 59, "]": 60, "^": 61, "_": 62, "`": 63, "a": 64, "b": 65, "c": 66, "d": 67, "e": 68, "f": 69, "g": 70, "h": 71, "i": 72, "j": 73, "k": 74, "l": 75, "m": 76, "n": 77, "o": 78, "p": 79, "q": 80, "r": 81, "s": 82, "t": 83, "u": 84, "v": 85, "w": 86, "x": 87, "y": 88, "z": 89, "{": 90, "|": 91, "}": 92, "~": 93, "\u00a1": 94, "\u00a2": 95, "\u00a3": 96, "\u00a4": 97, "\u00a5": 98, "\u00a6": 99, "\u00a7": 100, "\u00a8": 101, "\u00a9": 102, "\u00aa": 103, "\u00ab": 104, "\u00ac": 105, "\u00ae": 106, "\u00af": 107, "\u00b0": 108, "\u00b1": 109, "\u00b2": 110, "\u00b3": 111, "\u00b4": 112, "\u00b5": 113, "\u00b6": 114, "\u00b7": 115, "\u00b8": 116, "\u00b9": 117, "\u00ba": 118, "\u00bb": 119, "\u00bc": 120, "\u00bd": 121, "\u00be": 122, "\u00bf": 123, "\u00c0": 124, "\u00c1": 125, "\u00c2": 126, "\u00c3": 127, "\u00c4": 128, "\u00c5": 129, "\u00c6": 130, "\u00c7": 131, "\u00c8": 132, "\u00c9": 133, "\u00ca": 134, "\u00cb": 135, "\u00cc": 136, "\u00cd": 137, "\u00ce": 138, "\u00cf": 139, "\u00d0": 140, "\u00d1": 141, "\u00d2": 142, "\u00d3": 143, "\u00d4": 144, "\u00d5": 145, "\u00d6": 146, "\u00d7": 147, "\u00d8": 148, "\u00d9": 149, "\u00da": 150, "\u00db": 151, "\u00dc": 152, "\u00dd": 153, "\u00de": 154, "\u00df": 155, "\u00e0": 156, "\u00e1": 157, "\u00e2": 158, "\u00e3": 159, "\u00e4": 160, "\u00e5": 161, "\u00e6": 162, "\u00e7": 163, "\u00e8": 164, "\u00e9": 165, "\u00ea": 166, "\u00eb": 167, "\u00ec": 168, "\u00ed": 169, "\u00ee": 170, "\u00ef": 171, "\u00f0": 172, "\u00f1": 173, "\u00f2": 174, "\u00f3": 175, "\u00f4": 176, "\u00f5": 177, "\u00f6": 178, "\u00f7": 179, "\u00f8": 180, "\u00f9": 181, "\u00fa": 182, "\u00fb": 183, "\u00fc": 184, "\u00fd": 185, "\u00fe": 186, "\u00ff": 187, "\u0100": 188, "\u0101": 189, "\u0102": 190, "\u0103": 191, "\u0104": 192, "\u0105": 193, "\u0106": 194, "\u0107": 195, "\u0108": 196, "\u0109": 197, "\u010a": 198, "\u010b": 199, "\u010c": 200, "\u010d": 201, "\u010e": 202, "\u010f": 203, "\u0110": 204, "\u0111": 205, "\u0112": 206, "\u0113": 207, "\u0114": 208, "\u0115": 209, "\u0116": 210, "\u0117": 211, "\u0118": 212, "\u0119": 213, "\u011a": 214, "\u011b": 215, "\u011c": 216, "\u011d": 217, "\u011e": 218, "\u011f": 219, "\u0120": 220, "\u0121": 221, "\u0122": 222, "\u0123": 223, "\u0124": 224, "\u0125": 225, "\u0126": 226, "\u0127": 227, "\u0128": 228, "\u0129": 229, "\u012a": 230, "\u012b": 231, "\u012c": 232, "\u012d": 233, "\u012e": 234, "\u012f": 235, "\u0130": 236, "\u0131": 237, "\u0132": 238, "\u0133": 239, "\u0134": 240, "\u0135": 241, "\u0136": 242, "\u0137": 243, "\u0138": 244, "\u0139": 245, "\u013a": 246, "\u013b": 247, "\u013c": 248, "\u013d": 249, "\u013e": 250, "\u013f": 251, "\u0140": 252, "\u0141": 253, "\u0142": 254, "\u0143": 255, "\u0120t": 256, "\u0120a": 257, "he": 258, "in": 259, "re": 260, "on": 261, "\u0120the": 262, "er": 263, "\u0120s": 264, "at": 265, "\u0120w": 266, "\u0120o": 267, "en": 268, "\u0120c": 269, "it": 270, "is": 271, "an": 272, "or": 273, "es": 274, "\u0120b": 275, "ed": 276, "\u0120f": 277, "ing": 278, "\u0120p": 279, "ou": 280, "\u0120an": 281, "al": 282, "ar": 283, "\u0120to": 284, "\u0120m": 285, "\u0120of": 286, "\u0120in": 287, "\u0120d": 288, "\u0120h": 289, "\u0120and": 290, "ic": 291, "as": 292, "le": 293, "\u0120th": 294, "ion": 295, "om": 296, "ll": 297, "ent": 298, "\u0120n": 299, "\u0120l": 300, "st": 301, "\u0120re": 302, "ve": 303, "\u0120e": 304, "ro": 305, "ly": 306, "\u0120be": 307, "\u0120g": 308, "\u0120T": 309, "ct": 310, "\u0120S": 311, "id": 312, "ot": 313, "\u0120I": 314, "ut": 315, "et": 316, "\u0120A": 317, "\u0120is": 318, "\u0120on": 319, "im": 320, "am": 321, "ow": 322, "ay": 323, "ad": 324, "se": 325, "\u0120that": 326, "\u0120C": 327, "ig": 328, "\u0120for": 329, "ac": 330, "\u0120y": 331, "ver": 332, "ur": 333, "\u0120u": 334, "ld": 335, "\u0120st": 336, "\u0120M": 337, "'s": 338, "\u0120he": 339, "\u0120it": 340, "ation": 341, "ith": 342, "ir": 343, "ce": 344, "\u0120you": 345, "il": 346, "\u0120B": 347, "\u0120wh": 348, "ol": 349, "\u0120P": 350, "\u0120with": 351, "\u01201": 352, "ter": 353, "ch": 354, "\u0120as": 355, "\u0120we": 356, "\u0120(": 357, "nd": 358, "ill": 359, "\u0120D": 360, "if": 361, "\u01202": 362, "ag": 363, "ers": 364, "ke": 365, "\u0120\"": 366, "\u0120H": 367, "em": 368, "\u0120con": 369, "\u0120W": 370, "\u0120R": 371, "her": 372, "\u0120was": 373, "\u0120r": 374, "od": 375, "\u0120F": 376, "ul": 377, "ate": 378, "\u0120at": 379, "ri": 380, "pp": 381, "oLine truncated
generator/tf/models/774M/hparams.json
-7
View File
@@ -1,7 +0,0 @@
{
"n_vocab": 50257,
"n_ctx": 1024,
"n_embd": 1280,
"n_head": 20,
"n_layer": 36
}
generator/tf/models/774M/vocab.bpe
-50001
View File
File diff suppressed because it is too large. Load diff
generator/tf/src/__init__.py
View File
Whitespace-only changes.
generator/tf/src/encoder.py
-117
View File
@@ -1,117 +0,0 @@
"""Byte pair encoding utilities"""
import os
import json
import regex as re
from functools import lru_cache
@lru_cache()
def bytes_to_unicode():
"""
Returns list of utf-8 byte and a corresponding list of unicode strings.
The reversible bpe codes work on unicode strings.
This means you need a large # of unicode characters in your vocab if you want to avoid UNKs.
When you're at something like a 10B token dataset you end up needing around 5K for decent coverage.
This is a signficant percentage of your normal, say, 32K bpe vocab.
To avoid that, we want lookup tables between utf-8 bytes and unicode strings.
And avoids mapping to whitespace/control characters the bpe code barfs on.
"""
bs = list(range(ord("!"), ord("~")+1))+list(range(ord("¡"), ord("¬")+1))+list(range(ord("®"), ord("ÿ")+1))
cs = bs[:]
n = 0
for b in range(2**8):
if b not in bs:
bs.append(b)
cs.append(2**8+n)
n += 1
cs = [chr(n) for n in cs]
return dict(zip(bs, cs))
def get_pairs(word):
"""Return set of symbol pairs in a word.
Word is represented as tuple of symbols (symbols being variable-length strings).
"""
pairs = set()
prev_char = word[0]
for char in word[1:]:
pairs.add((prev_char, char))
prev_char = char
return pairs
class Encoder:
def __init__(self, encoder, bpe_merges, errors='replace'):
self.encoder = encoder
self.decoder = {v:k for k,v in self.encoder.items()}
self.errors = errors # how to handle errors in decoding
self.byte_encoder = bytes_to_unicode()
self.byte_decoder = {v:k for k, v in self.byte_encoder.items()}
self.bpe_ranks = dict(zip(bpe_merges, range(len(bpe_merges))))
self.cache = {}
# Should haved added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
self.pat = re.compile(r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+""")
def bpe(self, token):
if token in self.cache:
return self.cache[token]
word = tuple(token)
pairs = get_pairs(word)
if not pairs:
return token
while True:
bigram = min(pairs, key = lambda pair: self.bpe_ranks.get(pair, float('inf')))
if bigram not in self.bpe_ranks:
break
first, second = bigram
new_word = []
i = 0
while i < len(word):
try:
j = word.index(first, i)
new_word.extend(word[i:j])
i = j
except:
new_word.extend(word[i:])
break
if word[i] == first and i < len(word)-1 and word[i+1] == second:
new_word.append(first+second)
i += 2
else:
new_word.append(word[i])
i += 1
new_word = tuple(new_word)
word = new_word
if len(word) == 1:
break
else:
pairs = get_pairs(word)
word = ' '.join(word)
self.cache[token] = word
return word
def encode(self, text):
bpe_tokens = []
for token in re.findall(self.pat, text):
token = ''.join(self.byte_encoder[b] for b in token.encode('utf-8'))
bpe_tokens.extend(self.encoder[bpe_token] for bpe_token in self.bpe(token).split(' '))
return bpe_tokens
def decode(self, tokens):
text = ''.join([self.decoder[token] for token in tokens])
text = bytearray([self.byte_decoder[c] for c in text]).decode('utf-8', errors=self.errors)
return text
def get_encoder(model_path):
with open(os.path.join(model_path, 'encoder.json'), 'r') as f:
encoder = json.load(f)
with open(os.path.join(model_path, 'vocab.bpe'), 'r', encoding="utf-8") as f:
bpe_data = f.read()
bpe_merges = [tuple(merge_str.split()) for merge_str in bpe_data.split('\n')[1:-1]]
return Encoder(
encoder=encoder,
bpe_merges=bpe_merges,
)
generator/tf/src/model.py
-176
View File
@@ -1,176 +0,0 @@
import numpy as np
import tensorflow as tf
from tensorflow.contrib.training import HParams
def default_hparams():
return HParams(
n_vocab=0,
n_ctx=1024,
n_embd=768,
n_head=12,
n_layer=12,
)
def shape_list(x):
"""Deal with dynamic shape in tensorflow cleanly."""
static = x.shape.as_list()
dynamic = tf.shape(x)
return [dynamic[i] if s is None else s for i, s in enumerate(static)]
def softmax(x, axis=-1):
x = x - tf.reduce_max(x, axis=axis, keepdims=True)
ex = tf.exp(x)
return ex / tf.reduce_sum(ex, axis=axis, keepdims=True)
def gelu(x):
return 0.5*x*(1+tf.tanh(np.sqrt(2/np.pi)*(x+0.044715*tf.pow(x, 3))))
def norm(x, scope, *, axis=-1, epsilon=1e-5):
"""Normalize to mean = 0, std = 1, then do a diagonal affine transform."""
with tf.variable_scope(scope):
n_state = x.shape[-1].value
g = tf.get_variable('g', [n_state], initializer=tf.constant_initializer(1))
b = tf.get_variable('b', [n_state], initializer=tf.constant_initializer(0))
u = tf.reduce_mean(x, axis=axis, keepdims=True)
s = tf.reduce_mean(tf.square(x-u), axis=axis, keepdims=True)
x = (x - u) * tf.rsqrt(s + epsilon)
x = x*g + b
return x
def split_states(x, n):
"""Reshape the last dimension of x into [n, x.shape[-1]/n]."""
*start, m = shape_list(x)
return tf.reshape(x, start + [n, m//n])
def merge_states(x):
"""Smash the last two dimensions of x into a single dimension."""
*start, a, b = shape_list(x)
return tf.reshape(x, start + [a*b])
def conv1d(x, scope, nf, *, w_init_stdev=0.02):
with tf.variable_scope(scope):
*start, nx = shape_list(x)
w = tf.get_variable('w', [1, nx, nf], initializer=tf.random_normal_initializer(stddev=w_init_stdev))
b = tf.get_variable('b', [nf], initializer=tf.constant_initializer(0))
c = tf.reshape(tf.matmul(tf.reshape(x, [-1, nx]), tf.reshape(w, [-1, nf]))+b, start+[nf])
return c
def attention_mask(nd, ns, *, dtype):
"""1's in the lower triangle, counting from the lower right corner.
Same as tf.matrix_band_part(tf.ones([nd, ns]), -1, ns-nd), but doesn't produce garbage on TPUs.
"""
i = tf.range(nd)[:,None]
j = tf.range(ns)
m = i >= j - ns + nd
return tf.cast(m, dtype)
def attn(x, scope, n_state, *, past, hparams):
assert x.shape.ndims == 3 # Should be [batch, sequence, features]
assert n_state % hparams.n_head == 0
if past is not None:
assert past.shape.ndims == 5 # Should be [batch, 2, heads, sequence, features], where 2 is [k, v]
def split_heads(x):
# From [batch, sequence, features] to [batch, heads, sequence, features]
return tf.transpose(split_states(x, hparams.n_head), [0, 2, 1, 3])
def merge_heads(x):
# Reverse of split_heads
return merge_states(tf.transpose(x, [0, 2, 1, 3]))
def mask_attn_weights(w):
# w has shape [batch, heads, dst_sequence, src_sequence], where information flows from src to dst.
_, _, nd, ns = shape_list(w)
b = attention_mask(nd, ns, dtype=w.dtype)
b = tf.reshape(b, [1, 1, nd, ns])
w = w*b - tf.cast(1e10, w.dtype)*(1-b)
return w
def multihead_attn(q, k, v):
# q, k, v have shape [batch, heads, sequence, features]
w = tf.matmul(q, k, transpose_b=True)
w = w * tf.rsqrt(tf.cast(v.shape[-1].value, w.dtype))
w = mask_attn_weights(w)
w = softmax(w)
a = tf.matmul(w, v)
return a
with tf.variable_scope(scope):
c = conv1d(x, 'c_attn', n_state*3)
q, k, v = map(split_heads, tf.split(c, 3, axis=2))
present = tf.stack([k, v], axis=1)
if past is not None:
pk, pv = tf.unstack(past, axis=1)
k = tf.concat([pk, k], axis=-2)
v = tf.concat([pv, v], axis=-2)
a = multihead_attn(q, k, v)
a = merge_heads(a)
a = conv1d(a, 'c_proj', n_state)
return a, present
def mlp(x, scope, n_state, *, hparams):
with tf.variable_scope(scope):
nx = x.shape[-1].value
h = gelu(conv1d(x, 'c_fc', n_state))
h2 = conv1d(h, 'c_proj', nx)
return h2
def block(x, scope, *, past, hparams):
with tf.variable_scope(scope):
nx = x.shape[-1].value
a, present = attn(norm(x, 'ln_1'), 'attn', nx, past=past, hparams=hparams)
x = x + a
m = mlp(norm(x, 'ln_2'), 'mlp', nx*4, hparams=hparams)
x = x + m
return x, present
def past_shape(*, hparams, batch_size=None, sequence=None):
return [batch_size, hparams.n_layer, 2, hparams.n_head, sequence, hparams.n_embd // hparams.n_head]
def expand_tile(value, size):
"""Add a new axis of given size."""
value = tf.convert_to_tensor(value, name='value')
ndims = value.shape.ndims
return tf.tile(tf.expand_dims(value, axis=0), [size] + [1]*ndims)
def positions_for(tokens, past_length):
batch_size = tf.shape(tokens)[0]
nsteps = tf.shape(tokens)[1]
return expand_tile(past_length + tf.range(nsteps), batch_size)
def model(hparams, X, past=None, scope='model', reuse=False):
with tf.variable_scope(scope, reuse=reuse):
results = {}
batch, sequence = shape_list(X)
wpe = tf.get_variable('wpe', [hparams.n_ctx, hparams.n_embd],
initializer=tf.random_normal_initializer(stddev=0.01))
wte = tf.get_variable('wte', [hparams.n_vocab, hparams.n_embd],
initializer=tf.random_normal_initializer(stddev=0.02))
past_length = 0 if past is None else tf.shape(past)[-2]
h = tf.gather(wte, X) + tf.gather(wpe, positions_for(X, past_length))
# Transformer
presents = []
pasts = tf.unstack(past, axis=1) if past is not None else [None] * hparams.n_layer
assert len(pasts) == hparams.n_layer
for layer, past in enumerate(pasts):
h, present = block(h, 'h%d' % layer, past=past, hparams=hparams)
presents.append(present)
results['present'] = tf.stack(presents, axis=1)
h = norm(h, 'ln_f')
# Language model loss. Do tokens <n predict token n?
h_flat = tf.reshape(h, [batch*sequence, hparams.n_embd])
logits = tf.matmul(h_flat, wte, transpose_b=True)
logits = tf.reshape(logits, [batch, sequence, hparams.n_vocab])
results['logits'] = logits
return results
generator/tf/src/sample.py
-79
View File
@@ -1,79 +0,0 @@
import tensorflow as tf
from src.model import *
def top_k_logits(logits, k):
if k == 0:
# no truncation
return logits
def _top_k():
values, _ = tf.nn.top_k(logits, k=k)
min_values = values[:, -1, tf.newaxis]
return tf.where(
logits < min_values,
tf.ones_like(logits, dtype=logits.dtype) * -1e10,
logits,
)
return tf.cond(
tf.equal(k, 0),
lambda: logits,
lambda: _top_k(),
)
def sample_sequence(*, hparams, length, start_token=None, batch_size=None, context=None, temperature=1, top_k=0):
if start_token is None:
assert context is not None, 'Specify exactly one of start_token and context!'
else:
assert context is None, 'Specify exactly one of start_token and context!'
context = tf.fill([batch_size, 1], start_token)
def step(hparams, tokens, past=None):
lm_output = model(hparams=hparams, X=tokens, past=past, reuse=tf.AUTO_REUSE)
logits = lm_output['logits'][:, :, :hparams.n_vocab]
presents = lm_output['present']
presents.set_shape(past_shape(hparams=hparams, batch_size=batch_size))
return {
'logits': logits,
'presents': presents,
}
with tf.name_scope('sample_sequence'):
# Don't feed the last context token -- leave that to the loop below
# TODO: Would be slightly faster if we called step on the entire context,
# rather than leaving the last token transformer calculation to the while loop.
context_output = step(hparams, context[:, :-1])
def body(past, prev, output):
next_outputs = step(hparams, prev[:, tf.newaxis], past=past)
logits = next_outputs['logits'][:, -1, :] / tf.to_float(temperature)
logits = top_k_logits(logits, k=top_k)
samples = tf.multinomial(logits, num_samples=1, output_dtype=tf.int32)
return [
tf.concat([past, next_outputs['presents']], axis=-2),
tf.squeeze(samples, axis=[1]),
tf.concat([output, samples], axis=1),
]
def cond(*args):
return True
_, _, tokens = tf.while_loop(
cond=cond, body=body,
maximum_iterations=length,
loop_vars=[
context_output['presents'],
context[:, -1],
context,
],
shape_invariants=[
tf.TensorShape(past_shape(hparams=hparams, batch_size=batch_size)),
tf.TensorShape([batch_size]),
tf.TensorShape([batch_size, None]),
],
back_prop=False, name="EndWhile"
)
return tokens
generator/tf/tf_generator.py
-142
View File
@@ -1,142 +0,0 @@
import json
import os
import numpy as np
import tensorflow as tf
from src.model import *
from tensorflow.contrib import predictor
from src.sample import *
from src.encoder import *
import pdb
pos_action_starts = ["You attack", "You tell", "You use", "You go"]
class TFGenerator():
def __init__(self, sess, length=75, temperature=0.9, top_k=40):
seed = None
batch_size=1
model_path='gpt2/models/117M'
self.sess = sess
self.enc = encoder.get_encoder(model_path)
hparams = model.default_hparams()
with open(os.path.join(model_path, 'hparams.json')) as f:
hparams.override_from_dict(json.load(f))
pdb.set_trace()
self.context = tf.placeholder(tf.int32, [batch_size, None])
np.random.seed(seed)
tf.set_random_seed(seed)
self.output = sample.sample_sequence(
hparams=hparams, length=length,
context=self.context,
batch_size=batch_size,
)
saver = tf.train.Saver()
ckpt = tf.train.latest_checkpoint(model_path)
saver.restore(self.sess, ckpt)
def generate(self, prompt, options={}):
context_tokens = self.enc.encode(prompt)
out = self.sess.run(self.output, feed_dict={
self.context: [context_tokens for _ in range(1)]
})[:, len(context_tokens):]
text = self.enc.decode(out[0])
return text
def save_model():
length=75
temperature=0.9
top_k=40
os.environ['CUDA_VISIBLE_DEVICES'] = '-1'
with tf.Session() as sess:
seed = None
batch_size=None
model_path='models/774M'
hparams = default_hparams()
with open(os.path.join(model_path, 'hparams.json')) as f:
hparams.override_from_dict(json.load(f))
context = tf.placeholder(tf.int32, [batch_size, None])
np.random.seed(seed)
tf.set_random_seed(seed)
output = sample_sequence(
hparams=hparams, length=length,
context=context,
batch_size=batch_size,
)
print("***********************",type(output))
saver = tf.train.Saver()
ckpt = tf.train.latest_checkpoint(model_path)
saver.restore(sess, ckpt)
tf.saved_model.simple_save(sess, "./saved_model", inputs={"context": context}, outputs={"output": output})
def load_model():
fraction = 0.6
config = config = generate_gpu_config(fraction)
path_to_graph = "./saved"
# tf.saved_model.loader.load(
# session,
# [tf.saved_model.tag_constants.SERVING],
# path_to_graph)
# output = session.graph.get_tensor_by_name('output:0')
# context = session.graph.get_tensor_by_name('context:0')
model_path = 'gpt2/models/117M'
enc = encoder.get_encoder(model_path)
predict_fn = predictor.from_saved_model(path_to_graph, config=config)
context_tokens = [enc.encode("hello")]
predictions = predict_fn({"context": context_tokens})
output = enc.decode(predictions["output"][0])
print(output)
return (output, session)
if __name__ == '__main__':
save_model()