wassname/Clover-Edition
1
0
Fork 0
mirror of https://github.com/wassname/Clover-Edition.git synced 2026-09-09 11:13:26 +08:00
Code Issues Packages Projects Releases Wiki Activity

rmeoved old things

Browse Source
This commit is contained in:
Nick Walton
2019-11-18 12:22:22 -07:00
parent dda5ae9b2c
commit 56630dc6f8
61 changed files with 0 additions and 997265 deletions
Download Patch File Download Diff File Expand all files Collapse all files
generator/ctrl/.gitignore
-2
View File
@@ -1,2 +0,0 @@
fastBPE
model
generator/ctrl/LICENSE.txt
-12
View File
@@ -1,12 +0,0 @@
Copyright (c) 2019, Salesforce.com, Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
* Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
generator/ctrl/__init__.py
View File
generator/ctrl/codes
-200000
View File
File diff suppressed because it is too large Load Diff
generator/ctrl/control_codes.txt
-55
View File
@@ -1,55 +0,0 @@
0.000296793 Pregnancy
0.000127197 Christianity
0.003084531 Explain
0.000180196 Fitness
6.88985E-05 Saving
0.000217295 Ask
8.47981E-05 Ass
0.000143097 Joke
0.000196096 Questions
0.000127197 Thoughts
0.000169596 Retail
0.000270294 Feminism
0.000111298 Writing
0.000402791 Atheism
1.05998E-06 Netflix
0.000365692 Computing
0.000132497 Opinion
0.000169596 Alone
0.000323293 Funny
0.000249094 Gaming
0.000402791 Human
0.000132497 India
2.11995E-08 Joker
0.000201395 Diet
0.000238495 Legal
6.35986E-06 Norman
3.60392E-07 Tip
0.000302093 Weight
0.000132497 Movies
0.000111298 Running
7.41983E-05 Science
0.00135147 Horror
0.000291493 Confession
0.000190796 Finance
0.000413391 Politics
7.41983E-05 Scary
0.000206695 Support
6.35986E-05 Technologies
0.000243795 Teenage
0.000217295 Event
0.000206695 Learned
0.000121897 Notion
0.0847981 Wikipedia
0.095927851 Books
0.001176574 Extract
0.000127197 Confessions
0.000227895 Conspiracy
0.365691808 Links
0.000423991 Narcissus
0.000280894 Relationship
0.000922179 Relationships
0.153696557 Reviews
0.043877717 News
0.129847091 Translation
0.111297507 multilingual
generator/ctrl/ctrl_generator.py
-327
View File
@@ -1,327 +0,0 @@
import tensorflow as tf
import numpy as np
tf.enable_eager_execution()
import generator.ctrl.model.transformer as transformer
import re
from collections import Counter
from tensorflow.python import debug as tf_debug
from tensorflow.python.ops import math_ops
from tensorflow.python.ops import embedding_ops
import fastBPE
from story.utils import *
import warnings
warnings.filterwarnings("ignore")
# the loss function is a simple categorical crossentropy between the logits and the labels
def loss(labels, logits):
return tf.keras.losses.sparse_categorical_crossentropy(labels, logits, from_logits=True)
class CTRLGenerator():
def __init__(self, control_code="Apocalypse ", generate_num=40, temperature=0.4, topk=40, nucleus_prob=0):
self.generate_num=generate_num
model_dir = "generator/ctrl/model/aidungeon2model/"
self.control_code = control_code
vocab_file = 'generator/ctrl/model/vocab'
code_file = 'generator/ctrl/model/codes'
self.max_new_lines = 5
# load the vocabulary from file
vocab = open(vocab_file, encoding='utf-8').read().split('\n')
vocab = list(map(lambda x: x.split(' ')[0], vocab)) + ['<unk>'] + ['\n']
print('{} unique words'.format(len(vocab)))
# length of the vocabulary
vocab_size = len(vocab)
# define the numericalization map
# idx2word maps the numericalized ID to the word
# word2idx maps the word to the numericalized ID
self.word2idx = {u: i for i, u in enumerate(vocab)}
self.idx2word = np.array(vocab)
# sequence length to use for the transformer
# the model is trained with a self.seq_length of 512
# so, any value <= 512 should work
self.seq_length = min(self.generate_num, 256)
# the dimension of the transformer
embedding_dim = 1280
# input for the keras model
tokens = tf.keras.layers.Input(shape=(self.seq_length,), dtype='int32')
# Now, we begin defining the model
# we defer the transformer definition to transformer.py
# here, we only define the tied softmax layer
# this layer ties the softmax weights to the input embeddings
class TiedEmbeddingSoftmax(tf.keras.layers.Layer):
def __init__(self, vocab_size=vocab_size, embedding_size=embedding_dim, **kwargs):
super(TiedEmbeddingSoftmax, self).__init__()
self.w = self.add_weight(name='w', shape=(vocab_size, embedding_size),
initializer='random_normal',
trainable=True)
self.b = self.add_weight(name='b', shape=(vocab_size,),
initializer='zeros',
trainable=True)
def call(self, inputs, embed=True):
if embed:
dtype = tf.keras.backend.dtype(inputs)
if dtype != 'int32' and dtype != 'int64':
inputs = math_ops.cast(inputs, 'int32')
return embedding_ops.embedding_lookup(self.w, inputs)
else:
return tf.tensordot(inputs, tf.transpose(self.w), 1) + self.b
# instantiates a tied softmax class
tied_embedding_softmax = TiedEmbeddingSoftmax()
# embedded tokens, before passing it to the transformer
embedded = tied_embedding_softmax(tokens, embed=True)
# the activations after passing it from the transformer
# for some odd reason, TPUs don't play well with specifying the arguments of the Encoder() function
# so you have to leave them at their defaults
transformed = transformer.Encoder()(embedded, training=False)
# pass the activations from our tiedsoftmax class
# this time with embed=False denoting that we are doing the softmax operation
# and not a lookup
logits = tied_embedding_softmax(transformed, embed=False)
# finally, define the Keras model with inputs as tokens and outputs as the logits we just computed
model = tf.keras.Model(inputs=tokens, outputs=logits)
# the optimizer is not used since this code only supports inference
# however, to compile the model, we still define it
optimizer = tf.contrib.tpu.CrossShardOptimizer(
tf.contrib.estimator.clip_gradients_by_norm(
tf.train.AdagradOptimizer(learning_rate=1e-2), 0.25)
)
# compile the model with the optimizer and loss
model.compile(optimizer=optimizer, loss=loss)
print(model.summary())
# IMPORTANT
# this is where the saved model is presented to the code
# the model directory should have the model checkpoint and
# a checkpoint file
run_config = tf.contrib.tpu.RunConfig(
model_dir=model_dir)
# this converts the Keras model to a TensorFlow estimator
# this step is critical
# remember to patch the TF 1.14 file before running the code, else you're going to see errors here
estimator_model = tf.keras.estimator.model_to_estimator(keras_model=model, config=run_config)
# we now create a serving function from this estimator
# this enables us to load the model once and easily query it multiple times
def serving_input_fn():
inputs = {'input_1': tf.placeholder(tf.int32, [1, self.seq_length])}
return tf.estimator.export.ServingInputReceiver(inputs, inputs)
self.predict_fn = tf.contrib.predictor.from_estimator(estimator_model, serving_input_fn)
# almost there, we now take the user prompt and tokenize with BPE
# load BPE codes
self.bpe = fastBPE.fastBPE(code_file, vocab_file)
self.temperature=temperature
self.nucleusprob = nucleus_prob
self.penalty = 1.2
self.topk=topk
def configure_verb_probs(self, probabilities, options):
# Make sure only a possible verb is chosen.
for word in get_possible_verbs():
probabilities[self.word2idx[word]] += 100
# Disallow used verbs
if "used_verbs" in options:
for verb in options["used_verbs"]:
if verb in self.word2idx:
probabilities[self.word2idx[verb]] = -1e8
return probabilities
def prompt_replace(self, prompt):
# print("\n\nBEFORE PROMPT_REPLACE:")
# print(repr(prompt))
if prompt[-1] != " ":
prompt = prompt + " "
prompt = second_to_first_person(prompt)
prompt = self.control_code + prompt
# print("\n\nAFTER PROMPT_REPLACE")
# print(repr(prompt))
return prompt
def result_replace(self, result):
# print("\n\nBEFORE RESULT_REPLACE:")
# print(repr(result))
result = cut_trailing_sentence(result)
first_letter_capitalized = result[0].isupper()
result = result.replace('."', '".')
result = result.replace("#", "")
result = result.replace("*", "")
result = first_to_second_person(result)
result = remove_profanity(result)
if not first_letter_capitalized:
result = result[0].lower() + result[1:]
#
# print("\n\nAFTER RESULT_REPLACE:")
# print(repr(result))
return result
def generate_next_token(self, token, tokens_generated, options, num_new_lines, token_num, first_token=False, forbid_newline=False):
# get the logits from the prediction function
# the logic here is a bit convoluted because we are allowing generation past 512 tokens
# this is done by sliding the window over (past 512 tokens) and continuing prediction
# I'm sure this can be simplified (TODO)
if token <= self.seq_length:
prompt_logits = self.predict_fn({'input_1': tokens_generated[:, :self.seq_length]})[
'tied_embedding_softmax'].squeeze() / (self.temperature if self.temperature > 0 else 1.)
_token = token if token < self.seq_length else -1
else:
_token = -1
end = token + 1
start = token - self.seq_length + 2
prompt_logits = \
self.predict_fn({'input_1': np.hstack((tokens_generated[:, 0:1], tokens_generated[:, start:end]))})[
'tied_embedding_softmax'].squeeze() / (self.temperature if self.temperature > 0 else 1.)
# if penalty (for repetition) is non-zero,
# discount the logits from already generated tokens
if self.penalty > 0:
penalized_so_far = set()
for _ in range(token + 1):
generated_token = tokens_generated[0][_]
if generated_token not in penalized_so_far:
penalized_so_far.add(generated_token)
prompt_logits[_token][generated_token] /= self.penalty
# disallow some tokens
forbidden_tokens = ['<unk>', 'Sco@@', "&amp@@", "1]@@", "2]@@", "3]@@", "4]@@", "https://www.@@", "[@@", ":@@",
"Edit", "&@@", "2:","1:", ":", "Edit@@", "EDI@@", "EDIT@@", "edit", "TL@@", "tl@@", ";@@",
'**', "http://@@", "Redd@@", "UP@@", "mom", "Up@@", "Me:", "Update", "mom@@", "Part",
"http://www.@@", "edit@@", "*@@", "Writing", "Text@@", "\\@@", "<br>@@", "<div", "|@@", '...',
'..','…', 'https://@@', '...@@', "http://gutenberg@@"]
#encourage_tokens = ["zombie", "radiation", "fallout", "undead", "corpse", "vampire", "virus", "plague"]
encourage_tokens = []
for encourage_token in encourage_tokens:
prompt_logits[_token][self.word2idx[encourage_token]] *= 1.2
for forbidden_token in forbidden_tokens:
prompt_logits[_token][self.word2idx[forbidden_token]] = -1e8
last_ind = tokens_generated[0][token]
if forbid_newline:
prompt_logits[_token][self.word2idx['\n']] = -1e8
else:
prompt_logits[_token][self.word2idx['\n']] *= 1.0
# Set whitelist
if "word_whitelist" in options and token_num in options["word_whitelist"].keys():
for word in options["word_whitelist"][token_num]:
prompt_logits[_token][self.word2idx[word]] += 100
# Set blacklist, overwrites whitelist
if "word_blacklist" in options and token_num in options["word_blacklist"].keys():
for word in options["word_blacklist"][token_num]:
prompt_logits[_token][self.word2idx[word]] = -1e8
# compute probabilities from logits
prompt_probs = np.exp(prompt_logits[_token])
prompt_probs = prompt_probs / sum(prompt_probs)
pruned_list = np.argsort(prompt_probs)[::-1]
# if you are using nucleus prob, then compute the nucleus probability size
if self.nucleusprob > 0.:
minimum_topk = 1
nucleus = max(np.where(np.cumsum(np.sort(prompt_probs)[::-1]) > self.nucleusprob)[0][0], minimum_topk)
elif self.topk > 0:
nucleus = self.topk
else:
nucleus = len(pruned_list)
pruned_list = pruned_list[:nucleus]
# if temperature is 0
# just pick the first (most probable) token
if self.temperature == 0:
idx = pruned_list[0]
else:
# else,
# sample from the pruned_list with the logits
chosen_idx = int(
tf.random.categorical(np.expand_dims(prompt_logits[_token][pruned_list], 0), num_samples=1).numpy())
idx = pruned_list[chosen_idx]
return idx
def generate(self, prompt, options=None):
prompt = self.prompt_replace(prompt)
debug_print = True
if debug_print:
print("\n\n*****DEBUG*****")
print("Prompt is:")
print(prompt + "\n\n")
if options is None: options = dict()
if "used_verbs" not in options:
options["used_verbs"] = set()
first_token = True
# tokenize provided prompt
split_prompt = self.bpe.apply([prompt])[0].split()
text = [self.word2idx[i] for i in split_prompt]
total_text_len = len(text) + self.generate_num
# pad with 0s and create a mini-batch of 2 (arbitrary, for ease of code)
padded_text = text + [0] * (total_text_len - len(text))
tokens_generated = np.tile(padded_text, (1, 1))
result = ""
token_num = 0
num_new_lines = 0
for token in range(len(text) - 1, total_text_len - 1):
idx = self.generate_next_token(token, tokens_generated, options, num_new_lines, token_num, first_token=first_token, forbid_newline=False)
is_nothing = len(cut_trailing_sentence(result)) == 0 or len(cut_trailing_quotes(result)) == 0
if self.idx2word[idx] == '\n' and token_num > 7 and not is_nothing:
return self.result_replace(result)
elif self.idx2word[idx] == '\n':
idx = self.generate_next_token(token, tokens_generated, options, num_new_lines, token_num,
first_token=first_token, forbid_newline=True)
# assign the token for generation
tokens_generated[0][token + 1] = idx
if debug_print:
print(repr(self.idx2word[idx]), end="_")
tokens_generated_so_far = ' '.join([self.idx2word[c] for c in tokens_generated[0][len(text):token+2]])
tokens_generated_so_far = re.sub('(@@ )', '', string=tokens_generated_so_far)
tokens_generated_so_far = re.sub('(@@ ?$)', '', string=tokens_generated_so_far)
result = tokens_generated_so_far
token_num += 1
if debug_print:
print("\n****END DEBUG*****\n")
return self.result_replace(result)
generator/ctrl/download_model.sh
-4
View File
@@ -1,4 +0,0 @@
URL="gs://aidungeon2model"
gsutil -m cp -r "$URL" model
generator/ctrl/estimator.patch
-24
View File
@@ -1,24 +0,0 @@
47c47
<
---
> import tensorflow as tf
228c228
< def _create_keras_model_fn(keras_model, custom_objects=None):
---
> def _create_keras_model_fn(keras_model, params=None, custom_objects=None):
239c239
< def model_fn(features, labels, mode):
---
> def model_fn(features, labels, mode, params=None):
448c448
< if keras_model._is_graph_network:
---
> if False:
462,464c462,464
< estimator = estimator_lib.Estimator(keras_model_fn,
< config=config,
< warm_start_from=warm_start_path)
---
> estimator = tf.contrib.tpu.TPUEstimator(keras_model_fn, use_tpu=False, train_batch_size=4, eval_batch_size=4,
> config=config,
> warm_start_from=warm_start_path)
generator/ctrl/generation.py
-289
View File
@@ -1,289 +0,0 @@
from __future__ import division
from __future__ import print_function
import tensorflow as tf
import os
import numpy as np
tf.enable_eager_execution()
import transformer
import argparse
import pdb
import sys
import re
from collections import Counter
from tensorflow.python import debug as tf_debug
from tensorflow.python.ops import math_ops
from tensorflow.python.ops import embedding_ops
import fastBPE
import platform
use_py3 = platform.python_version()[0] == '3'
parser = argparse.ArgumentParser(description='TensorFlow code for generating from CTRL')
parser.add_argument('--model_dir', type=str, required=True,
help='location of model checkpoint')
parser.add_argument('--seed', type=int, default=1337,
help='random seed for TensorFlow, numpy and PythonHash')
parser.add_argument('--generate_num', type=int, default=256,
help='number of tokens to generate')
parser.add_argument('--temperature', type=float, default=0,
help='temperature for sampling distribution; 0 means greedy')
parser.add_argument('--nucleus', type=float, default=0.,
help='cumulative probability cutoff for nucleus sampling; 0 means no nucleus sampling')
parser.add_argument('--topk', type=int, default=0,
help='topk value for sampling from the softmax distribution ; 0 means no topk preferred')
parser.add_argument('--penalty', type=float, default=1.2,
help='repetition penalty for greedy sampling')
parser.add_argument('--print_once', action='store_true',
help='the completion is printed only at the end; not every word')
parser.add_argument('--topn', type=int, default=0,
help='print top-n candidates during generations; defaults to 0 which is no printing')
args = parser.parse_args()
tf.random.set_random_seed(args.seed)
os.environ['PYTHONHASHSEED'] = str(args.seed)
np.random.seed(args.seed)
# load the vocabulary from file
vocab = open('vocab').read().decode(encoding='utf-8').split('\n') if not use_py3 else open('vocab', encoding='utf-8').read().split('\n')
vocab = list(map(lambda x: x.split(' ')[0], vocab)) + ['<unk>'] + ['\n']
print ('{} unique words'.format(len(vocab)))
# length of the vocabulary
vocab_size = len(vocab)
# define the numericalization map
# idx2word maps the numericalized ID to the word
# word2idx maps the word to the numericalized ID
word2idx = {u:i for i, u in enumerate(vocab)}
idx2word = np.array(vocab)
# sequence length to use for the transformer
# the model is trained with a seq_length of 512
# so, any value <= 512 should work
seq_length = min(args.generate_num, 256)
# the dimension of the transformer
embedding_dim = 1280
# Now, we begin defining the model
# we defer the transformer definition to transformer.py
# here, we only define the tied softmax layer
# this layer ties the softmax weights to the input embeddings
class TiedEmbeddingSoftmax(tf.keras.layers.Layer):
def __init__(self, vocab_size=vocab_size, embedding_size=embedding_dim, **kwargs):
super(TiedEmbeddingSoftmax, self).__init__()
self.w = self.add_weight(name='w', shape=(vocab_size, embedding_size),
initializer='random_normal',
trainable=True)
self.b = self.add_weight(name='b', shape=(vocab_size,),
initializer='zeros',
trainable=True)
def call(self, inputs, embed=True):
if embed:
dtype = tf.keras.backend.dtype(inputs)
if dtype != 'int32' and dtype != 'int64':
inputs = math_ops.cast(inputs, 'int32')
return embedding_ops.embedding_lookup(self.w, inputs)
else:
return tf.tensordot(inputs, tf.transpose(self.w), 1) + self.b
# input for the keras model
tokens = tf.keras.layers.Input(shape=(seq_length,), dtype='int32')
# instantiates a tied softmax class
tied_embedding_softmax = TiedEmbeddingSoftmax()
# embedded tokens, before passing it to the transformer
embedded = tied_embedding_softmax(tokens, embed=True)
# the activations after passing it from the transformer
# for some odd reason, TPUs don't play well with specifying the arguments of the Encoder() function
# so you have to leave them at their defaults
transformed = transformer.Encoder()(embedded, training=False)
# pass the activations from our tiedsoftmax class
# this time with embed=False denoting that we are doing the softmax operation
# and not a lookup
logits = tied_embedding_softmax(transformed, embed=False)
# finally, define the Keras model with inputs as tokens and outputs as the logits we just computed
model = tf.keras.Model(inputs=tokens, outputs=logits)
# the loss function is a simple categorical crossentropy between the logits and the labels
def loss(labels, logits):
return tf.keras.losses.sparse_categorical_crossentropy(labels, logits, from_logits=True)
# the optimizer is not used since this code only supports inference
# however, to compile the model, we still define it
optimizer = tf.contrib.tpu.CrossShardOptimizer(
tf.contrib.estimator.clip_gradients_by_norm(
tf.train.AdagradOptimizer(learning_rate=1e-2), 0.25)
)
# compile the model with the optimizer and loss
model.compile(optimizer=optimizer, loss=loss)
print(model.summary())
# IMPORTANT
# this is where the saved model is presented to the code
# the model directory should have the model checkpoint and
# a checkpoint file
run_config = tf.contrib.tpu.RunConfig(
model_dir=args.model_dir)
# this converts the Keras model to a TensorFlow estimator
# this step is critical
# remember to patch the TF 1.14 file before running the code, else you're going to see errors here
estimator_model = tf.keras.estimator.model_to_estimator(keras_model=model, config=run_config)
# we now create a serving function from this estimator
# this enables us to load the model once and easily query it multiple times
def serving_input_fn():
inputs = {'input_1': tf.placeholder(tf.int32, [1,seq_length])}
return tf.estimator.export.ServingInputReceiver(inputs, inputs)
predict_fn = tf.contrib.predictor.from_estimator(estimator_model, serving_input_fn)
# almost there, we now take the user prompt and tokenize with BPE
# load BPE codes
bpe = fastBPE.fastBPE('codes', 'vocab')
temperature = args.temperature
nucleusprob = args.nucleus
penalty = args.penalty
topk = args.topk
while True:
prompt = raw_input('ENTER PROMPT: ') if not use_py3 else input('ENTER PROMPT: ')
# tokenize provided prompt
split_prompt = bpe.apply([prompt])[0].split()
text = [word2idx[i] for i in split_prompt]
# pad with 0s and create a mini-batch of 2 (arbitrary, for ease of code)
padded_text = text + [0] * (args.generate_num - len(text))
tokens_generated = np.tile(padded_text, (1,1))
try:
for token in range(len(text)-1, args.generate_num-1):
# get the logits from the prediction function
# the logic here is a bit convoluted because we are allowing generation past 512 tokens
# this is done by sliding the window over (past 512 tokens) and continuing prediction
# I'm sure this can be simplified (TODO)
if token <= seq_length:
prompt_logits = predict_fn({'input_1':tokens_generated[:, :seq_length]})['tied_embedding_softmax'].squeeze() / (temperature if temperature>0 else 1.)
_token = token if token < seq_length else -1
else:
_token = -1
end = token + 1
start = token - seq_length + 2
prompt_logits = predict_fn({'input_1':np.hstack((tokens_generated[:,0:1], tokens_generated[:,start:end]))})['tied_embedding_softmax'].squeeze() / (temperature if temperature>0 else 1.)
# if penalty (for repetition) is non-zero,
# discount the logits from already generated tokens
if penalty>0:
penalized_so_far = set()
for _ in range(token+1):
generated_token = tokens_generated[0][_]
# don't penalize newlines
# you could also choose not to penalize frequent words
# (which incidentally are sorted in the vocab file)
# but I don't do that
# if it prints too many new lines instead of continuing generating text,
# you might want to comment this out
if idx2word[generated_token] == '\n':
continue
if generated_token in penalized_so_far:
continue
penalized_so_far.add(generated_token)
prompt_logits[_token][generated_token] /= penalty
# disallow some tokens
prompt_logits[_token][word2idx['<unk>']] = -1e8
prompt_logits[_token][word2idx['\n']] = -1e8
# sometimes, when generating from reddit,
# it tries to generate the Score (reddit Karma) immediately after generating the Title:
# to disallow this, we can just prevent it from generating Score
prompt_logits[_token][word2idx['Sco@@']] = -1e8
# compute probabilities from logits
prompt_probs = np.exp(prompt_logits[_token])
prompt_probs = prompt_probs / sum(prompt_probs)
pruned_list = np.argsort(prompt_probs)[::-1]
# if you are using nucleus prob, then compute the nucleus probability size
if nucleusprob > 0.:
minimum_topk = 1
nucleus = max(np.where(np.cumsum(np.sort(prompt_probs)[::-1])>nucleusprob)[0][0], minimum_topk)
elif topk > 0:
# we are over-loading notation here
# if you choose to specify a topk instead of a nucleus,
# we will hardcode the nucleus to be just that
nucleus = topk
else:
# if you specify neither nucleus or topk,
# then we will use the whole list
nucleus = len(pruned_list)
pruned_list = pruned_list[:nucleus]
# if you want to disallow more complex tokens, you can do so here
# for instance, if you want to disallow anything with the phrase `http`,
# you can delete theme from the pruned_list
# you can comment this out, I'm keeping it in for demonstration purpose
tokens_to_disallow = []
for _ in range(len(pruned_list)):
if 'http' in idx2word[pruned_list[_]]:
tokens_to_disallow.append(_)
pruned_list = np.delete(pruned_list, tokens_to_disallow)
if args.topn > 0 :
print('TOPN :: top-n alternatives:', [idx2word[_] for _ in pruned_list[:args.topn]])
# if temperature is 0
# just pick the first (most probable) token
if temperature==0:
idx = pruned_list[0]
else:
# else,
# sample from the pruned_list with the logits
chosen_idx = int(tf.random.categorical(np.expand_dims(prompt_logits[_token][pruned_list],0), num_samples=1).numpy())
idx = pruned_list[chosen_idx]
if args.topn > 0 :
print('TOPN :: chosen word:', idx2word[idx])
# assign the token for generation
tokens_generated[0][token+1] = idx
# clear screen if you want to
# os.system("clear")
tokens_generated_so_far = ' '.join([idx2word[c] for c in tokens_generated[0].squeeze()[:token+2]])
tokens_generated_so_far = re.sub('(@@ )', '', string=tokens_generated_so_far)
tokens_generated_so_far = re.sub('(@@ ?$)', '', string=tokens_generated_so_far)
if not args.print_once:
print('---------------------------------------')
print(tokens_generated_so_far)
print()
print('---------------------------------------')
print(tokens_generated_so_far)
print()
except KeyboardInterrupt: #Exception as e:
print('Continuing')
generator/ctrl/install_ctrl_py2.sh
-27
View File
@@ -1,27 +0,0 @@
#!/bin/bash
cd model
# Cython is needed to compile fastBPE
pip install Cython
# Patch the TensorFlow estimator package
export FILE="/usr/local/lib/python2.7/dist-packages/tensorflow_estimator/python/estimator/keras.py"
patch -b "$FILE" estimator.patch
# Install fastBPE
git clone https://github.com/glample/fastBPE.git
cd fastBPE
python setup.py install
cd ../..
# Download the 512-length model if specified, 256-length otherwise
#if [ "$1" = "512" ]
#then
# URL="gs://sf-ctrl/seqlen512_v1.ckpt/"
#else
# URL="gs://sf-ctrl/seqlen256_v1.ckpt/"
#fi
# Copy model
#gsutil -m cp -r "$URL" .
generator/ctrl/install_ctrl_py3.sh
-27
View File
@@ -1,27 +0,0 @@
#!/bin/bash
cd model
# Cython is needed to compile fastBPE
pip install Cython
# Patch the TensorFlow estimator package
export FILE="/usr/local/lib/python3.6/dist-packages/tensorflow_estimator/python/estimator/keras.py"
patch -b "$FILE" estimator.patch
# Install fastBPE
git clone https://github.com/glample/fastBPE.git
cd fastBPE
python setup.py install
cd ../..
# Download the 512-length model if specified, 256-length otherwise
#if [ "$1" = "512" ]
#then
# URL="gs://sf-ctrl/seqlen512_v1.ckpt/"
#else
# URL="gs://sf-ctrl/seqlen256_v1.ckpt/"
#fi
# Copy model
#gsutil -m cp -r "$URL" .
generator/ctrl/install_gcloud.sh
-3
View File
@@ -1,3 +0,0 @@
curl https://sdk.cloud.google.com | bash
exec -l $SHELL
gcloud init
generator/ctrl/model/.gitignore
-1
View File
@@ -1 +0,0 @@
fastBPE
generator/ctrl/model/CODE_OF_CONDUCT.md
-105
View File
@@ -1,105 +0,0 @@
# Salesforce Open Source Community Code of Conduct
## About the Code of Conduct
Equality is a core value at Salesforce. We believe a diverse and inclusive
community fosters innovation and creativity, and are committed to building a
culture where everyone feels included.
Salesforce open-source projects are committed to providing a friendly, safe, and
welcoming environment for all, regardless of gender identity and expression,
sexual orientation, disability, physical appearance, body size, ethnicity, nationality,
race, age, religion, level of experience, education, socioeconomic status, or
other similar personal characteristics.
The goal of this code of conduct is to specify a baseline standard of behavior so
that people with different social values and communication styles can work
together effectively, productively, and respectfully in our open source community.
It also establishes a mechanism for reporting issues and resolving conflicts.
All questions and reports of abusive, harassing, or otherwise unacceptable behavior
in a Salesforce open-source project may be reported by contacting the Salesforce
Open Source Conduct Committee at ossconduct@salesforce.com.
## Our Pledge
In the interest of fostering an open and welcoming environment, we as
contributors and maintainers pledge to making participation in our project and
our community a harassment-free experience for everyone, regardless of gender
identity and expression, sexual orientation, disability, physical appearance,
body size, ethnicity, nationality, race, age, religion, level of experience, education,
socioeconomic status, or other similar personal characteristics.
## Our Standards
Examples of behavior that contributes to creating a positive environment
include:
* Using welcoming and inclusive language
* Being respectful of differing viewpoints and experiences
* Gracefully accepting constructive criticism
* Focusing on what is best for the community
* Showing empathy toward other community members
Examples of unacceptable behavior by participants include:
* The use of sexualized language or imagery and unwelcome sexual attention or
advances
* Personal attacks, insulting/derogatory comments, or trolling
* Public or private harassment
* Publishing, or threatening to publish, others' private information—such as
a physical or electronic address—without explicit permission
* Other conduct which could reasonably be considered inappropriate in a
professional setting
* Advocating for or encouraging any of the above behaviors
## Our Responsibilities
Project maintainers are responsible for clarifying the standards of acceptable
behavior and are expected to take appropriate and fair corrective action in
response to any instances of unacceptable behavior.
Project maintainers have the right and responsibility to remove, edit, or
reject comments, commits, code, wiki edits, issues, and other contributions
that are not aligned with this Code of Conduct, or to ban temporarily or
permanently any contributor for other behaviors that they deem inappropriate,
threatening, offensive, or harmful.
## Scope
This Code of Conduct applies both within project spaces and in public spaces
when an individual is representing the project or its community. Examples of
representing a project or community include using an official project email
address, posting via an official social media account, or acting as an appointed
representative at an online or offline event. Representation of a project may be
further defined and clarified by project maintainers.
## Enforcement
Instances of abusive, harassing, or otherwise unacceptable behavior may be
reported by contacting the Salesforce Open Source Conduct Committee
at ossconduct@salesforce.com. All complaints will be reviewed and investigated
and will result in a response that is deemed necessary and appropriate to the
circumstances. The committee is obligated to maintain confidentiality with
regard to the reporter of an incident. Further details of specific enforcement
policies may be posted separately.
Project maintainers who do not follow or enforce the Code of Conduct in good
faith may face temporary or permanent repercussions as determined by other
members of the project's leadership and the Salesforce Open Source Conduct
Committee.
## Attribution
This Code of Conduct is adapted from the [Contributor Covenant][contributor-covenant-home],
version 1.4, available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html.
It includes adaptions and additions from [Go Community Code of Conduct][golang-coc],
[CNCF Code of Conduct][cncf-coc], and [Microsoft Open Source Code of Conduct][microsoft-coc].
This Code of Conduct is licensed under the [Creative Commons Attribution 3.0 License][cc-by-3-us].
[contributor-covenant-home]: https://www.contributor-covenant.org (https://www.contributor-covenant.org/)
[golang-coc]: https://golang.org/conduct
[cncf-coc]: https://github.com/cncf/foundation/blob/master/code-of-conduct.md
[microsoft-coc]: https://opensource.microsoft.com/codeofconduct/
[cc-by-3-us]: https://creativecommons.org/licenses/by/3.0/us/
generator/ctrl/model/LICENSE.txt
-12
View File
@@ -1,12 +0,0 @@
Copyright (c) 2019, Salesforce.com, Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
* Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
generator/ctrl/model/README.md
-356
View File
@@ -1,356 +0,0 @@
# CTRL - A Conditional Transformer Language Model for Controllable Generation
Authors: [Nitish Shirish Keskar](http://keskarnitish.github.io), [Bryan McCann](https://bmccann.github.io/), [Lav Varshney](http://www.varshney.csl.illinois.edu/), [Caiming Xiong](http://www.stat.ucla.edu/~caiming/), and [Richard Socher](https://www.socher.org/)
## Introduction
Large-scale language models show promising text generation capabilities, but
users cannot easily control this generation process. We release *CTRL*, a 1.6 billion-parameter conditional
transformer language model, trained to condition on control codes that specify
domain, subdomain, entities, relationships between entities, dates, and task-specific behavior. Control codes were derived from structure that naturally co-occurs with raw text, preserving the advantages of unsupervised learning while providing more explicit control over text generation.
Paper link: https://arxiv.org/abs/1909.05858
Blog link: https://blog.einstein.ai/introducing-a-conditional-transformer-language-model-for-controllable-generation/
The code currently supports two functionalities:
1. Generating from a trained model, two models are available for download - one with a sequence length of 256 and another with a sequence length of 512 -- they are trained with word-level vocabularies and through a sliding window approach can generate well beyond their trained sequence lengths.
2. Source attribution - given a prompt, prints the perplexity of the prompt conditional on each domain control code (see Section 5 of the paper).
Please refer to the argument flags for more details regarding the options available for either.
## Table of Contents
1. [Citation](#citation)
2. [License](#license)
3. [Questions for Deliberation](#questions-for-deliberation)
4. [Usage](#usage)
5. [Sample Generations](#generations)
6. [Sample Source Attributions](#source-attributions)
7. [FAQs](#faqs)
8. [Get Involved](#get-involved)
## Citation
```
@article{keskarCTRL2019,
title={{CTRL - A Conditional Transformer Language Model for Controllable Generation}},
author={Keskar, Nitish Shirish and McCann, Bryan and Varshney, Lav and Xiong, Caiming and Socher, Richard},
journal={arXiv preprint arXiv:1909.05858},
year={2019}
}
```
## License
The code is released under the BSD-3 License (see `LICENSE.txt` for details), but we also ask that users respect the following:
This software should not be used to promote or profit from:
violence, hate, and division,
environmental destruction,
abuse of human rights, or
the destruction of people's physical and mental health.
We encourage users of this software to tell us about the applications in which they are putting it to use by emailing ctrl-monitoring@salesforce.com, and to use [appropriate](https://arxiv.org/abs/1810.03993) [documentation](https://www.partnershiponai.org/about-ml/) when developing high-stakes applications of this model.
## Questions for Deliberation
We consulted extended members of the AI community in the responsible publication of this model. In particular, a preview of a [Partnership on AI (PAI)](http://partnershiponai.org) project relating to AI research publication norms was considered prior to the release of this work. While this PAI project is as-yet unpublished, it is informed by companies, organizations, and people differently affected by artificial intelligence and presents key considerations to evaluate before publishing potentially high-impact research.
The questions referenced from the early draft of the PAI project included:
1. How do you envision your research being used in the world? Who will use it? How much expertise is required to use it?
2. Who will use it?
3. Why would they be motivated to replicate / productionize your work?
4. How would a science fiction author turn your research into a dystopian story?
5. What is the worst way someone could use your research finding, given no resource constraints?
6. What are the historical patterns of misuse or application in this area? How can the research be made more robust against such misuse?
7. Which populations or communities will this technology negatively affect, deployed in the scenarios you envision? Will some groups be disproportionately affected?
## Usage
Here are the steps to get generating:
1. Install the dependencies
This code relies on [TensorFlow 1.14](https://www.tensorflow.org/install) and [fastBPE](https://github.com/glample/fastBPE).
TensorFlow can be installed via `pip install tensorflow[-gpu]==1.14`. fastBPE installation instructions can be found in the GitHub repository linked above. We highly recommend experimenting within a virtualenv or Docker image.
2. Patch the `/usr/local/lib/python2.7/dist-packages/tensorflow_estimator/python/estimator/keras.py` (or equivalent, if installed elsewhere) by running
```patch -b <path_to_tensorflow_estimator_package>/python/estimator/keras.py estimator.patch```
We highly recommend experimenting within a virtualenv or Docker image since the workflow involves patching a TensorFlow file to support some custom functionality. This step is not optional; skipping this step will cause errors (irrespective of device).
3. Get the model files from `gs://sf-ctrl/seqlen256_v1.ckpt/` or `gs://sf-ctrl/seqlen512_v1.ckpt/`.
The model architecture is identical for both checkpoints. The former is trained with lower training sequence length (256) while the latter is trained with a larger one (512). We plan to update the models (with the appropriate version tags) as we continue to train them longer and on more data. **Our current recommendation is to use the `256_v1` model unless you have a strong reason not to. If you have no preference for domain, `Links` is always a good first choice.**
[With `gsutil` installed](https://cloud.google.com/storage/docs/gsutil_install), you can simply run `gsutil -m cp -r gs://sf-ctrl/seqlen256_v1.ckpt/ .` for copying the model checkpoint over.
Without `gsutil`, you can follow the route recommended @ https://github.com/salesforce/ctrl/issues/7#issuecomment-531303214
4. Run the generation script `generation.py` or the source attribution script `source_attribution.py`.
The `generation.py` prompts the user to input text and then prints the continuation.
The `source_attribution.py` promps the user to input text and then prints a sorted list of domains and the perplexity of the text conditional on each individual domain.
## Generations
The generations and attributions computed below have been generated using the `256` sequence length model. Comparable results can be obtained from the `512` version of the model as well. We demonstrate only a few of the functionalities, especially the control codes. For a complete list of the control codes, and how to use them, please refer to the paper. Note that `<GENERATION_BEGINS>` is only included for demonstrative purposes and is not actually generated by the model.
1. Links
```
Links In a shocking finding, scientist discovered a herd of unicorns living in a remote, previously unexplored valley, in the Andes Mountains. Even more surprising to the researchers was the fact that the unicorns spoke perfect English.
<GENERATION_BEGINS>
The scientists were able to track down the unicorn population by analyzing their DNA. They found out that they are descended from an ancient species called the “Tapiri,” which is believed to have been extinct for over 10,000 years.
According to scientists, this discovery proves that there may be other undiscovered creatures on our planet.
“We believe these animals could still exist and we hope to find them one day,” said Dr. Carlos Jaramillo, director of the National Geographic Society’s Center for Biodiversity Research at the University of Arizona.
Scientists also confirmed that the Unicorn Genome Project has already identified several genes associated with the animal’s unique horn structure.
“The genome project will help us understand how the animal evolved its horns so it can better protect itself against predators like jaguars or pumas.”
Researchers say that if all goes well, the new genetic information should allow them to create a 3D model of the animal.
“It would give us a much clearer picture about what makes the creature unique.”- Dr David Sillito, lead researcher ...
```
As a generate note, you don't have to use greedy sampling. You can switch to `topk` or `nucleus` through the appropriate argument flags.
Running it with `--temperature 0.2` and `--topk 5` yields:
```
Links In a shocking finding, scientist discovered a herd of unicorns living in a remote, previously unexplored valley, in the Andes Mountains. Even more surprising to the researchers was the fact that the unicorns spoke perfect English.
The scientists were able to track down and capture one of these unicorn herds as it roamed through the valley for several days. They then took DNA samples from each animal and compared them with other animals found throughout the region.
Advertisement
What they found is truly astounding.
According to the scientists, there are approximately 50 different species of unicorn, all of which live on two continents — North America and South America...
```
2. Wikipedia
```
Wikipedia Salesforce Inc. is <GENERATION_BEGINS> a software company that provides cloud computing and business management software. It was founded in 1999 by Marc Benioff as an online retailer of enterprise software, but it has since expanded into other areas such as e-commerce, customer relationship management, and digital media...
```
3. Reviews
```
Reviews Rating: 1.0 <GENERATION_BEGINS>
I bought this for my son who is a huge fan of the show. He was so excited to get it and when he opened it, we were all very disappointed. The quality of the product is terrible. It looks like something you would buy at a dollar store.
Rating: 1.0
This item arrived in poor condition. There are several scratches on the front cover as well as some other minor damage...
```
```
Reviews Rating: 4.0 <GENERATION_BEGINS>
I bought this for my husband and he loves it. He has a small wrist so it is hard to find watches that fit him well. This one fits perfectly.
Rating: 4.0
The watch looks great but the band was too big even though i had read other reviews about how large the bands were.
Rating: 4.0
It works well as an everyday watch. It's not very fancy or anything, just simple with no extra bells & whistles.
```
4. Books
```
Books Weary with toil, I haste me to my bed, <GENERATION_BEGINS>
And sleep the slumber of a weary man.
The morning sun is up, and shines on high;
My bed is ready, but no friend appears.
A little while ago he came in sight,
But now he's gone, as if he'd never come.
He left his hat upon the table there,
His coat was hanging up behind the door;
And, when at last I looked around, there stood
No one beside me, nor within call.
Oh, what shall I do? What can be done?
I'm sure that some strange accident has happened.
I'll go out into the street again,
To seek for him, or call him back again.
But, oh, how lonely it will seem
When all are gone who once were near me!
```
5. Reddit - r/running
```
Running Title: <GENERATION_BEGINS> I just ran my first 5k in under 30 minutes.
Text: It was a pretty easy run, but it felt great.
My goal is to run at least 10 miles per week for the next few months.
I'm not sure if this qualifies as running or not, but it's definitely something that I've been wanting to do since high school.
Thanks r/running!
Edit: Thank you all so much for your kind words and advice.
Edit 2: Wow, thanks guys!
```
6. Reddit - r/fitness
```
Fitness Title: <GENERATION_BEGINS> How do you get your body to burn fat when you're not working out?
Text: I know that the reason why people lose weight is because they are eating less and burning more calories than their bodies need but how does this work for someone who doesn't have a lot of time or energy to exercise?
For example, if I'm at home doing nothing all day then I'll be able to eat whatever I want without feeling like I've eaten too much. But what happens after that?
EDIT: Thanks everyone for the responses!
```
7. Reddit - r/frugal
```
Saving Title: <GENERATION_BEGINS> How to get a free credit report
Text: I have been trying for months now and it seems like no one is willing to give me one.
Is there any way that you can just go online and fill out the forms yourself or do they require some sort of ID or something?
Edit: Thanks everyone for your help!
Edit 2: Thank you all so much for helping me!
```
8. Reddit - r/personalfinance
```
Finance Title: <GENERATION_BEGINS> I have a question about my credit score.
Text: So I'm currently in the process of applying for an auto loan and I've been told that if you don't pay your bills on time then they will not approve you.
My questions are:
1. How do I know when to start paying off debt?
2. What is the best way to get out of debt without having to file bankruptcy?
3. Is it possible to refinance my car loan?
4. Should I just wait until after school starts so I can take advantage of lower interest rates?
5. If so, how long should I wait?
Thanks
```
9. Questions
```
Questions Q: What is the capital of Australia? <GENERATION_BEGINS>
A: Canberra
Q: How many people live in Canberra?
A: 650,000
```
10. Translation
```
Translation English : This is a natural language processing model that aims to generate coherent text in a controllable manner. ; French : <GENERATION_BEGINS>
Il s'agit d'un modèle de traitement du langage naturel qui vise à générer un texte cohérent et contrôlable.
```
```
Translation English : This is a natural language processing model that aims to generate coherent text in a controllable manner. ; German : <GENERATION_BEGINS>
Es handelt sich um ein natürliches Textverarbeitungssystem, das auf eine einheitliche und kontrollierbare Erzeugung von Text abzielt.
```
## Source Attributions
1. `I lost 10 lbs! Feeling great!`
```
PROMPT: I lost 10 lbs! Feeling great!
Diet ppl = 28.960714
Weight ppl = 29.223865
Fitness ppl = 36.162671
...
```
2. `My landlord is suing me for unpaid rent`
```
PROMPT: My landlord is suing me for unpaid rent
Legal ppl = 21.210965
Finance ppl = 24.619064
Saving ppl = 27.923208
...
```
3. `And then I saw him, the man in the mirror.`
```
PROMPT: And then I saw him, the man in the mirror.
Horror ppl = 17.919299
Scary ppl = 18.587843
Writing ppl = 23.154564
...
```
4. `Anarchism is an anti-authoritarian political philosophy that rejects hierarchies deemed unjust and advocates their replacement with self-managed, self-governed societies based on voluntary, cooperative institutions.`
```
PROMPT: Anarchism is an anti-authoritarian political philosophy that rejects hierarchies deemed unjust and advocates their replacement with self-managed, self-governed societies based on voluntary, cooperative institutions.
Wikipedia ppl = 34.446701
News ppl = 34.484165
Links ppl = 35.460126
...
```
5. `I love God`
```
PROMPT: I love God
Christianity ppl = 55.653985
Atheism ppl = 116.811038
Confessions ppl = 133.619834
...
```
## FAQs
(We hope to update this section frequently).
1. Will you be releasing the training code and data?
We plan to release the training code soon. We will not be releasing the training data, but we will release tips and scripts related to data collection.
2. Is a version of the model available in PyTorch?
Not at the moment, but if we come across an equivalent implementation, we will update this section.
3. The code errors out.
Make sure that you have performed the patch as described above. If the error persists, please create a GitHub issue.
4. The code generates non-sense irrespective of the prompt.
Make sure that you have (a) provided the right `--model_dir` and that the folder actually exists and has the checkpoint, (b) provided a valid source code as the first token, and (c) tried generating with a simple prompt such as `Links I` or `Books From`. If the error persists, please create a GitHub issue.
## Get Involved
Please create a GitHub issue if you have any questions, suggestions, requests or bug-reports.
We welcome PRs!
generator/ctrl/model/__init__.py
View File
generator/ctrl/model/codes
-200000
View File
File diff suppressed because it is too large Load Diff
generator/ctrl/model/control_codes.txt
-55
View File
@@ -1,55 +0,0 @@
0.000296793 Pregnancy
0.000127197 Christianity
0.003084531 Explain
0.000180196 Fitness
6.88985E-05 Saving
0.000217295 Ask
8.47981E-05 Ass
0.000143097 Joke
0.000196096 Questions
0.000127197 Thoughts
0.000169596 Retail
0.000270294 Feminism
0.000111298 Writing
0.000402791 Atheism
1.05998E-06 Netflix
0.000365692 Computing
0.000132497 Opinion
0.000169596 Alone
0.000323293 Funny
0.000249094 Gaming
0.000402791 Human
0.000132497 India
2.11995E-08 Joker
0.000201395 Diet
0.000238495 Legal
6.35986E-06 Norman
3.60392E-07 Tip
0.000302093 Weight
0.000132497 Movies
0.000111298 Running
7.41983E-05 Science
0.00135147 Horror
0.000291493 Confession
0.000190796 Finance
0.000413391 Politics
7.41983E-05 Scary
0.000206695 Support
6.35986E-05 Technologies
0.000243795 Teenage
0.000217295 Event
0.000206695 Learned
0.000121897 Notion
0.0847981 Wikipedia
0.095927851 Books
0.001176574 Extract
0.000127197 Confessions
0.000227895 Conspiracy
0.365691808 Links
0.000423991 Narcissus
0.000280894 Relationship
0.000922179 Relationships
0.153696557 Reviews
0.043877717 News
0.129847091 Translation
0.111297507 multilingual
generator/ctrl/model/generation.py
-281
View File
@@ -1,281 +0,0 @@
from __future__ import division
from __future__ import print_function
import tensorflow as tf
import os
import numpy as np
tf.enable_eager_execution()
import transformer
import argparse
import pdb
import sys
import re
from collections import Counter
from tensorflow.python import debug as tf_debug
from tensorflow.python.ops import math_ops
from tensorflow.python.ops import embedding_ops
import fastBPE
import platform
use_py3 = platform.python_version()[0] == '3'
parser = argparse.ArgumentParser(description='TensorFlow code for generating from CTRL')
parser.add_argument('--model_dir', type=str, required=True,
help='location of model checkpoint')
parser.add_argument('--seed', type=int, default=1337,
help='random seed for TensorFlow, numpy and PythonHash')
parser.add_argument('--generate_num', type=int, default=256,
help='number of tokens to generate')
parser.add_argument('--temperature', type=float, default=0,
help='temperature for sampling distribution; 0 means greedy')
parser.add_argument('--nucleus', type=float, default=0.,
help='cumulative probability cutoff for nucleus sampling; 0 means no nucleus sampling')
parser.add_argument('--topk', type=int, default=0,
help='topk value for sampling from the softmax distribution ; 0 means no topk preferred')
parser.add_argument('--penalty', type=float, default=1.2,
help='repetition penalty for greedy sampling')
args = parser.parse_args()
tf.random.set_random_seed(args.seed)
os.environ['PYTHONHASHSEED'] = str(args.seed)
np.random.seed(args.seed)
# load the vocabulary from file
vocab = open('vocab').read().decode(encoding='utf-8').split('\n') if not use_py3 else open('vocab', encoding='utf-8').read().split('\n')
vocab = list(map(lambda x: x.split(' ')[0], vocab)) + ['<unk>'] + ['\n']
print ('{} unique words'.format(len(vocab)))
# length of the vocabulary
vocab_size = len(vocab)
# define the numericalization map
# idx2word maps the numericalized ID to the word
# word2idx maps the word to the numericalized ID
word2idx = {u:i for i, u in enumerate(vocab)}
idx2word = np.array(vocab)
# sequence length to use for the transformer
# the model is trained with a seq_length of 512
# so, any value <= 512 should work
seq_length = min(args.generate_num, 256)
# the dimension of the transformer
embedding_dim = 1280
# Now, we begin defining the model
# we defer the transformer definition to transformer.py
# here, we only define the tied softmax layer
# this layer ties the softmax weights to the input embeddings
class TiedEmbeddingSoftmax(tf.keras.layers.Layer):
def __init__(self, vocab_size=vocab_size, embedding_size=embedding_dim, **kwargs):
super(TiedEmbeddingSoftmax, self).__init__()
self.w = self.add_weight(name='w', shape=(vocab_size, embedding_size),
initializer='random_normal',
trainable=True)
self.b = self.add_weight(name='b', shape=(vocab_size,),
initializer='zeros',
trainable=True)
def call(self, inputs, embed=True):
if embed:
dtype = tf.keras.backend.dtype(inputs)
if dtype != 'int32' and dtype != 'int64':
inputs = math_ops.cast(inputs, 'int32')
return embedding_ops.embedding_lookup(self.w, inputs)
else:
return tf.tensordot(inputs, tf.transpose(self.w), 1) + self.b
# input for the keras model
tokens = tf.keras.layers.Input(shape=(seq_length,), dtype='int32')
# instantiates a tied softmax class
tied_embedding_softmax = TiedEmbeddingSoftmax()
# embedded tokens, before passing it to the transformer
embedded = tied_embedding_softmax(tokens, embed=True)
# the activations after passing it from the transformer
# for some odd reason, TPUs don't play well with specifying the arguments of the Encoder() function
# so you have to leave them at their defaults
transformed = transformer.Encoder()(embedded, training=False)
# pass the activations from our tiedsoftmax class
# this time with embed=False denoting that we are doing the softmax operation
# and not a lookup
logits = tied_embedding_softmax(transformed, embed=False)
# finally, define the Keras model with inputs as tokens and outputs as the logits we just computed
model = tf.keras.Model(inputs=tokens, outputs=logits)
# the loss function is a simple categorical crossentropy between the logits and the labels
def loss(labels, logits):
return tf.keras.losses.sparse_categorical_crossentropy(labels, logits, from_logits=True)
# the optimizer is not used since this code only supports inference
# however, to compile the model, we still define it
optimizer = tf.contrib.tpu.CrossShardOptimizer(
tf.contrib.estimator.clip_gradients_by_norm(
tf.train.AdagradOptimizer(learning_rate=1e-2), 0.25)
)
# compile the model with the optimizer and loss
model.compile(optimizer=optimizer, loss=loss)
print(model.summary())
# IMPORTANT
# this is where the saved model is presented to the code
# the model directory should have the model checkpoint and
# a checkpoint file
run_config = tf.contrib.tpu.RunConfig(
model_dir=args.model_dir)
# this converts the Keras model to a TensorFlow estimator
# this step is critical
# remember to patch the TF 1.14 file before running the code, else you're going to see errors here
estimator_model = tf.keras.estimator.model_to_estimator(keras_model=model, config=run_config)
# we now create a serving function from this estimator
# this enables us to load the model once and easily query it multiple times
def serving_input_fn():
inputs = {'input_1': tf.placeholder(tf.int32, [1,seq_length])}
return tf.estimator.export.ServingInputReceiver(inputs, inputs)
predict_fn = tf.contrib.predictor.from_estimator(estimator_model, serving_input_fn)
# almost there, we now take the user prompt and tokenize with BPE
# load BPE codes
bpe = fastBPE.fastBPE('codes', 'vocab')
temperature = args.temperature
nucleusprob = args.nucleus
penalty = args.penalty
topk = args.topk
while True:
prompt = raw_input('ENTER PROMPT: ') if not use_py3 else input('ENTER PROMPT: ')
# tokenize provided prompt
split_prompt = bpe.apply([prompt])[0].split()
text = [word2idx[i] for i in split_prompt]
# pad with 0s and create a mini-batch of 2 (arbitrary, for ease of code)
padded_text = text + [0] * (args.generate_num - len(text))
tokens_generated = np.tile(padded_text, (1,1))
try:
for token in range(len(text)-1, args.generate_num-1):
# get the logits from the prediction function
# the logic here is a bit convoluted because we are allowing generation past 512 tokens
# this is done by sliding the window over (past 512 tokens) and continuing prediction
# I'm sure this can be simplified (TODO)
if token <= seq_length:
prompt_logits = predict_fn({'input_1':tokens_generated[:, :seq_length]})['tied_embedding_softmax'].squeeze() / (temperature if temperature>0 else 1.)
_token = token if token < seq_length else -1
else:
_token = -1
end = token + 1
start = token - seq_length + 2
prompt_logits = predict_fn({'input_1':np.hstack((tokens_generated[:,0:1], tokens_generated[:,start:end]))})['tied_embedding_softmax'].squeeze() / (temperature if temperature>0 else 1.)
# if penalty (for repetition) is non-zero,
# discount the logits from already generated tokens
if penalty>0:
penalized_so_far = set()
for _ in range(token+1):
generated_token = tokens_generated[0][_]
# don't penalize newlines
# you could also choose not to penalize frequent words
# (which incidentally are sorted in the vocab file)
# but I don't do that
# if it prints too many new lines instead of continuing generating text,
# you might want to comment this out
if idx2word[generated_token] == '\n':
continue
if generated_token in penalized_so_far:
continue
penalized_so_far.add(generated_token)
prompt_logits[_token][generated_token] /= penalty
# disallow some tokens
prompt_logits[_token][word2idx['<unk>']] = -1e8
# sometimes, when generating from reddit,
# it tries to generate the Score (reddit Karma) immediately after generating the Title:
# to disallow this, we can just prevent it from generating Score
prompt_logits[_token][word2idx['Sco@@']] = -1e8
# compute probabilities from logits
prompt_probs = np.exp(prompt_logits[_token])
prompt_probs = prompt_probs / sum(prompt_probs)
pruned_list = np.argsort(prompt_probs)[::-1]
# if you are using nucleus prob, then compute the nucleus probability size
if nucleusprob > 0.:
minimum_topk = 1
nucleus = max(np.where(np.cumsum(np.sort(prompt_probs)[::-1])>nucleusprob)[0][0], minimum_topk)
elif topk > 0:
# we are over-loading notation here
# if you choose to specify a topk instead of a nucleus,
# we will hardcode the nucleus to be just that
nucleus = topk
else:
# if you specify neither nucleus or topk,
# then we will use the whole list
nucleus = len(pruned_list)
# if you want to disallow more complex tokens, you can do so here
# for instance, if you want to disallow anything with the phrase `http`,
# you can delete theme from the pruned_list
# you can comment this out, I'm keeping it in for demonstration purpose
tokens_to_disallow = []
for _ in range(len(pruned_list)):
if 'http' in idx2word[pruned_list[_]]:
tokens_to_disallow.append(_)
pruned_list = np.delete(pruned_list, tokens_to_disallow)
# if temperature is 0
# just pick the first (most probable) token
if temperature==0:
idx = pruned_list[0]
else:
# else,
# sample from the pruned_list with the logits
chosen_idx = int(tf.random.categorical(np.expand_dims(prompt_logits[0][_token][pruned_list],0), num_samples=1).numpy())
idx = pruned_list[chosen_idx]
# if you want to do some debugging,
# like which one was chosen,
# what the top25 were,
# here is your opportunity.
print('chosen:', idx2word[idx])
#print('top25 alternatives:', pruned_list[:25])
# assign the token for generation
tokens_generated[0][token+1] = idx
# clear screen if you want to
# os.system("clear")
tokens_generated_so_far = ' '.join([idx2word[c] for c in tokens_generated[0].squeeze()[:token+2]])
tokens_generated_so_far = re.sub('(@@ )', '', string=tokens_generated_so_far)
tokens_generated_so_far = re.sub('(@@ ?$)', '', string=tokens_generated_so_far)
tokens_generated_so_far = re.sub('(\n\n)', '', string=tokens_generated_so_far)
print(tokens_generated_so_far)
print()
except KeyboardInterrupt: #Exception as e:
print('Continuing')
generator/ctrl/model/low_mem_transformer.py
-139
View File
@@ -1,139 +0,0 @@
import tensorflow as tf
import numpy as np
def angle_defn(pos, i, d_model_size):
angle_rates = 1 / np.power(10000, (2 * (i // 2)) / np.float32(d_model_size))
return pos * angle_rates
def positional_encoding(position, d_model_size):
# create the sinusoidal pattern for the positional encoding
angle_rads = angle_defn(np.arange(position)[:, np.newaxis], np.arange(d_model_size)[np.newaxis, :], d_model_size)
sines = np.sin(angle_rads[:, 0::2])
cosines = np.cos(angle_rads[:, 1::2])
pos_encoding = tf.cast(np.concatenate([sines, cosines], axis=-1)[np.newaxis, ...], dtype=tf.float32)
return pos_encoding
def scaled_dot_product_attention(q, k, v, mask):
# calculate attention
matmul_qk = tf.cast(tf.matmul(q, k, transpose_b=True), tf.float32)
dk = tf.cast(tf.shape(k)[-1], tf.float32)
scaled_attention_logits = matmul_qk / tf.math.sqrt(dk)
if mask is not None:
scaled_attention_logits += (mask * -1e3)
attention_weights = tf.cast(tf.nn.softmax(scaled_attention_logits, axis=-1), tf.float16)
output = tf.matmul(attention_weights, v)
return output
class MultiHeadAttention(tf.keras.layers.Layer):
def __init__(self, d_model_size, num_heads):
super(MultiHeadAttention, self).__init__()
self.num_heads = num_heads
self.d_model_size = d_model_size
self.depth = int(d_model_size / self.num_heads)
self.Wq = tf.keras.layers.Dense(d_model_size)
self.Wk = tf.keras.layers.Dense(d_model_size)
self.Wv = tf.keras.layers.Dense(d_model_size)
self.dense = tf.keras.layers.Dense(d_model_size)
def split_into_heads(self, x, batch_size):
x = tf.reshape(x, (batch_size, -1, self.num_heads, self.depth))
return tf.transpose(x, perm=[0, 2, 1, 3])
def call(self, v, k, q, mask):
batch_size = tf.shape(q)[0]
q = self.Wq(q)
k = self.Wk(k)
v = self.Wv(v)
q = self.split_into_heads(q, batch_size)
k = self.split_into_heads(k, batch_size)
v = self.split_into_heads(v, batch_size)
scaled_attention = tf.transpose(scaled_dot_product_attention(q, k, v, mask), perm=[0, 2, 1, 3])
original_size_attention = tf.reshape(scaled_attention, (batch_size, -1, self.d_model_size))
output = self.dense(original_size_attention)
return output
def point_wise_feed_forward_network(d_model_size, dff):
return tf.keras.Sequential([tf.keras.layers.Dense(dff, activation='relu'),
tf.keras.layers.Dense(d_model_size)])
class EncoderLayer(tf.keras.layers.Layer):
def __init__(self, d_model_size, num_heads, dff, rate=0.1):
super(EncoderLayer, self).__init__()
self.multi_head_attention = MultiHeadAttention(d_model_size, num_heads)
self.ffn = point_wise_feed_forward_network(d_model_size, dff)
self.layernorm1 = tf.keras.layers.LayerNormalization(epsilon=1e-6)
self.layernorm2 = tf.keras.layers.LayerNormalization(epsilon=1e-6)
self.dropout1 = tf.keras.layers.Dropout(rate)
self.dropout2 = tf.keras.layers.Dropout(rate)
self.to32 = lambda x: tf.cast(x, tf.float32)
self.to16 = lambda x: tf.cast(x, tf.float16)
def call(self, x, training, mask):
normed = self.to16(self.layernorm1(self.to32(x)))
attn_output = self.multi_head_attention(normed, normed, normed, mask)
attn_output = self.dropout1(attn_output, training=training)
out1 = x + attn_output
out2 = self.to16(self.layernorm2(self.to32(out1)))
ffn_output = self.ffn(out2)
ffn_output = self.dropout2(ffn_output, training=training)
out2 = out1 + ffn_output
return out2
class Encoder(tf.keras.layers.Layer):
def __init__(self, num_layers=48, d_model_size=1280, num_heads=16, dff=8192, input_vocab_size=50000,
rate=0.1, **kwargs):
super(Encoder, self).__init__()
self.d_model_size = d_model_size
self.num_layers = num_layers
self.pos_encoding = positional_encoding(input_vocab_size, self.d_model_size)
for i in range(num_layers):
setattr(self, "layer%i" % i, EncoderLayer(d_model_size, num_heads, dff, rate))
self.layernorm = tf.keras.layers.LayerNormalization(epsilon=1e-6)
self.dropout = tf.keras.layers.Dropout(rate)
def get_config(self):
base_config = super(Encoder, self).get_config()
return base_config
def call(self, x, training):
seq_len = tf.shape(x)[1]
mask = 1 - tf.linalg.band_part(tf.ones((seq_len, seq_len)), -1, 0)
x *= tf.math.sqrt(tf.cast(self.d_model_size, tf.float32))
x += self.pos_encoding[:, :seq_len, :]
x = self.dropout(x, training=training)
x = tf.cast(x, tf.float16)
for i in range(self.num_layers):
x = getattr(self, "layer%i" % i)(x, training, mask)
return self.layernorm(tf.cast(x, tf.float32))
generator/ctrl/model/source_attribution.py
-192
View File
@@ -1,192 +0,0 @@
from __future__ import division
from __future__ import print_function
import tensorflow as tf
import os
import numpy as np
tf.enable_eager_execution()
import transformer
import argparse
import pdb
import sys
import re
from collections import Counter
from tensorflow.python import debug as tf_debug
from tensorflow.python.ops import math_ops
from tensorflow.python.ops import embedding_ops
import fastBPE
import platform
use_py3 = platform.python_version()[0] == '3'
parser = argparse.ArgumentParser(description='TensorFlow code for generating from CTRL')
parser.add_argument('--model_dir', type=str, required=True,
help='location of model checkpoint')
parser.add_argument('--seed', type=int, default=1337,
help='random seed for TensorFlow, numpy and PythonHash')
args = parser.parse_args()
tf.random.set_random_seed(args.seed)
os.environ['PYTHONHASHSEED'] = str(args.seed)
np.random.seed(args.seed)
# load the vocabulary from file
vocab = open('vocab').read().decode(encoding='utf-8').split('\n') if not use_py3 else open('vocab', encoding='utf-8').read().split('\n')
vocab = list(map(lambda x: x.split(' ')[0], vocab)) + ['<unk>'] + ['\n']
print ('{} unique words'.format(len(vocab)))
# length of the vocabulary
vocab_size = len(vocab)
# define the numericalization map
# idx2word maps the numericalized ID to the word
# word2idx maps the word to the numericalized ID
word2idx = {u:i for i, u in enumerate(vocab)}
idx2word = np.array(vocab)
# sequence length to use for the transformer
# the model is trained with a seq_length of 512
# so, any value <= 512 should work
seq_length = 256
# the dimension of the transformer
embedding_dim = 1280
# Now, we begin defining the model
# we defer the transformer definition to transformer.py
# here, we only define the tied softmax layer
# this layer ties the softmax weights to the input embeddings
class TiedEmbeddingSoftmax(tf.keras.layers.Layer):
def __init__(self, vocab_size=vocab_size, embedding_size=embedding_dim, **kwargs):
super(TiedEmbeddingSoftmax, self).__init__()
self.w = self.add_weight(name='w', shape=(vocab_size, embedding_size),
initializer='random_normal',
trainable=True)
self.b = self.add_weight(name='b', shape=(vocab_size,),
initializer='zeros',
trainable=True)
def call(self, inputs, embed=True):
if embed:
dtype = tf.keras.backend.dtype(inputs)
if dtype != 'int32' and dtype != 'int64':
inputs = math_ops.cast(inputs, 'int32')
return embedding_ops.embedding_lookup(self.w, inputs)
else:
return tf.tensordot(inputs, tf.transpose(self.w), 1) + self.b
# input for the keras model
tokens = tf.keras.layers.Input(shape=(seq_length,), dtype='int32')
# instantiates a tied softmax class
tied_embedding_softmax = TiedEmbeddingSoftmax()
# embedded tokens, before passing it to the transformer
embedded = tied_embedding_softmax(tokens, embed=True)
# the activations after passing it from the transformer
# for some odd reason, TPUs don't play well with specifying the arguments of the Encoder() function
# so you have to leave them at their defaults
transformed = transformer.Encoder()(embedded, training=False)
# pass the activations from our tiedsoftmax class
# this time with embed=False denoting that we are doing the softmax operation
# and not a lookup
logits = tied_embedding_softmax(transformed, embed=False)
# finally, define the Keras model with inputs as tokens and outputs as the logits we just computed
model = tf.keras.Model(inputs=tokens, outputs=logits)
# the loss function is a simple categorical crossentropy between the logits and the labels
def loss(labels, logits):
return tf.keras.losses.sparse_categorical_crossentropy(labels, logits, from_logits=True)
# the optimizer is not used since this code only supports inference
# however, to compile the model, we still define it
optimizer = tf.contrib.tpu.CrossShardOptimizer(
tf.contrib.estimator.clip_gradients_by_norm(
tf.train.AdagradOptimizer(learning_rate=1e-2), 0.25)
)
# compile the model with the optimizer and loss
model.compile(optimizer=optimizer, loss=loss)
print(model.summary())
# IMPORTANT
# this is where the saved model is presented to the code
# the model directory should have the model checkpoint and
# a checkpoint file
run_config = tf.contrib.tpu.RunConfig(
model_dir=args.model_dir)
# this converts the Keras model to a TensorFlow estimator
# this step is critical
# remember to patch the TF 1.14 file before running the code, else you're going to see errors here
estimator_model = tf.keras.estimator.model_to_estimator(keras_model=model, config=run_config)
# we now create a serving function from this estimator
# this enables us to load the model once and easily query it multiple times
def serving_input_fn():
inputs = {'input_1': tf.placeholder(tf.int32, [2,seq_length])}
return tf.estimator.export.ServingInputReceiver(inputs, inputs)
predict_fn = tf.contrib.predictor.from_estimator(estimator_model, serving_input_fn)
# almost there, we now take the user prompt and tokenize with BPE
# load BPE codes
bpe = fastBPE.fastBPE('codes', 'vocab')
domains = []
with open('control_codes.txt', 'r') as f:
domains = [line.split() for line in f.readlines()]
domains = [(t[1], float(t[0])) for t in domains]
while True:
_prompt = raw_input('ENTER PROMPT: ') if not use_py3 else input('ENTER PROMPT: ')
ppls = {}
# loop over all domains and compute perplexity
for domain, domain_prior in domains:
print(u'computing for domain: {}'.format(domain))
# tokenize data and add domain tag to it
prompt = domain + u' ' + _prompt
split_prompt = bpe.apply([prompt])[0].split()
# numericalize data and pad to the seq_len dimension
text = [word2idx[i] for i in split_prompt]
padding_text = text + [0] * (seq_length - len(text))
tokens_generated = np.tile(padding_text, (2,1))
output_scores = predict_fn({'input_1':tokens_generated})['tied_embedding_softmax'].squeeze()[0]
token_scores = output_scores[:-1]
# compute the perplexity for this sequence
xent = 0
for sequence_idx, token_idx in enumerate(text[1:]):
token = idx2word[token_idx]
# compute the probability of this token
Z = np.exp(token_scores[sequence_idx]).sum()
token_prob = np.exp(token_scores[sequence_idx, token_idx]) / Z
xent -= np.log(token_prob) / len(text[1:])
ppls[domain] = round(np.exp(xent), 6)
#print(u'{} ppl = {}'.format(domain, ppls[domain]))
# sort the domains based on perplexities and print
ppls = [(k, v) for k, v in ppls.items()]
ppls.sort(key=lambda x: x[1])
print('PROMPT: {}'.format(_prompt))
for t in ppls:
domain, ppl = t
print(u'{} ppl = {}'.format(domain, ppl))
generator/ctrl/model/transformer.py
-137
View File
@@ -1,137 +0,0 @@
import tensorflow as tf
import numpy as np
def angle_defn(pos, i, d_model_size):
angle_rates = 1 / np.power(10000, (2 * (i//2)) / np.float32(d_model_size))
return pos * angle_rates
def positional_encoding(position, d_model_size):
# create the sinusoidal pattern for the positional encoding
angle_rads = angle_defn(np.arange(position)[:, np.newaxis], np.arange(d_model_size)[np.newaxis, :], d_model_size)
sines = np.sin(angle_rads[:, 0::2])
cosines = np.cos(angle_rads[:, 1::2])
pos_encoding = tf.cast(np.concatenate([sines, cosines], axis=-1)[np.newaxis, ...], dtype=tf.float32)
return pos_encoding
def scaled_dot_product_attention(q, k, v, mask):
# calculate attention
matmul_qk = tf.matmul(q, k, transpose_b=True)
dk = tf.cast(tf.shape(k)[-1], tf.float32)
scaled_attention_logits = matmul_qk / tf.math.sqrt(dk)
if mask is not None:
scaled_attention_logits += (mask * -1e9)
attention_weights = tf.nn.softmax(scaled_attention_logits, axis=-1)
output = tf.matmul(attention_weights, v)
return output
class MultiHeadAttention(tf.keras.layers.Layer):
def __init__(self, d_model_size, num_heads):
super(MultiHeadAttention, self).__init__()
self.num_heads = num_heads
self.d_model_size = d_model_size
self.depth = int(d_model_size / self.num_heads)
self.Wq = tf.keras.layers.Dense(d_model_size)
self.Wk = tf.keras.layers.Dense(d_model_size)
self.Wv = tf.keras.layers.Dense(d_model_size)
self.dense = tf.keras.layers.Dense(d_model_size)
def split_into_heads(self, x, batch_size):
x = tf.reshape(x, (batch_size, -1, self.num_heads, self.depth))
return tf.transpose(x, perm=[0, 2, 1, 3])
def call(self, v, k, q, mask):
batch_size = tf.shape(q)[0]
q = self.Wq(q)
k = self.Wk(k)
v = self.Wv(v)
q = self.split_into_heads(q, batch_size)
k = self.split_into_heads(k, batch_size)
v = self.split_into_heads(v, batch_size)
scaled_attention = tf.transpose(scaled_dot_product_attention(q, k, v, mask), perm=[0, 2, 1, 3])
original_size_attention = tf.reshape(scaled_attention, (batch_size, -1, self.d_model_size))
output = self.dense(original_size_attention)
return output
def point_wise_feed_forward_network(d_model_size, dff):
return tf.keras.Sequential([tf.keras.layers.Dense(dff, activation='relu'),
tf.keras.layers.Dense(d_model_size)])
class EncoderLayer(tf.keras.layers.Layer):
def __init__(self, d_model_size, num_heads, dff, rate=0.1):
super(EncoderLayer, self).__init__()
self.multi_head_attention = MultiHeadAttention(d_model_size, num_heads)
self.ffn = point_wise_feed_forward_network(d_model_size, dff)
self.layernorm1 = tf.keras.layers.LayerNormalization(epsilon=1e-6)
self.layernorm2 = tf.keras.layers.LayerNormalization(epsilon=1e-6)
self.dropout1 = tf.keras.layers.Dropout(rate)
self.dropout2 = tf.keras.layers.Dropout(rate)
def call(self, x, training, mask):
normed = self.layernorm1(x)
attn_output = self.multi_head_attention(normed, normed, normed, mask)
attn_output = self.dropout1(attn_output, training=training)
out1 = x + attn_output
out2 = self.layernorm2(out1)
ffn_output = self.ffn(out2)
ffn_output = self.dropout2(ffn_output, training=training)
out2 = out1 + ffn_output
return out2
class Encoder(tf.keras.layers.Layer):
def __init__(self, num_layers=48, d_model_size=1280, num_heads=16, dff=8192, input_vocab_size=50000,
rate=0.1, **kwargs):
super(Encoder, self).__init__()
self.d_model_size = d_model_size
self.num_layers = num_layers
self.pos_encoding = positional_encoding(input_vocab_size, self.d_model_size)
for i in range(num_layers):
setattr(self, "layer%i" % i, EncoderLayer(d_model_size, num_heads, dff, rate))
self.layernorm = tf.keras.layers.LayerNormalization(epsilon=1e-6)
self.dropout = tf.keras.layers.Dropout(rate)
def get_config(self):
base_config = super(Encoder, self).get_config()
return base_config
def call(self, x, training):
seq_len = tf.shape(x)[1]
mask = 1 - tf.linalg.band_part(tf.ones((seq_len, seq_len)), -1, 0)
x *= tf.math.sqrt(tf.cast(self.d_model_size, tf.float32))
x += self.pos_encoding[:, :seq_len, :]
x = self.dropout(x, training=training)
for i in range(self.num_layers):
x = getattr(self, "layer%i" % i)(x, training, mask)
return self.layernorm(x)
generator/ctrl/model/vocab
-246531
View File
File diff suppressed because it is too large Load Diff
generator/ctrl/pytorch_generation.py
-283
View File
@@ -1,283 +0,0 @@
from __future__ import print_function
import torch
import os
import tqdm
import pdb
import numpy as np
import platform
import hashlib
import pytorch_transformer
import re
import argparse
import tensorflow as tf
import fastBPE
from tensorflow.python import pywrap_tensorflow
use_py3 = platform.python_version()[0] == '3'
parser = argparse.ArgumentParser(description='TensorFlow code for generating from CTRL')
parser.add_argument('--model_path', type=str, required=True,
help='location of model *data* checkpoint; this is NOT the directory but rather the model checkpoint')
parser.add_argument('--seed', type=int, default=1337,
help='random seed for TensorFlow, numpy and PythonHash')
parser.add_argument('--generate_num', type=int, default=256,
help='number of tokens to generate')
parser.add_argument('--temperature', type=float, default=0,
help='temperature for sampling distribution; 0 means greedy')
parser.add_argument('--nucleus', type=float, default=0.,
help='cumulative probability cutoff for nucleus sampling; 0 means no nucleus sampling')
parser.add_argument('--topk', type=int, default=0,
help='topk value for sampling from the softmax distribution ; 0 means no topk preferred')
parser.add_argument('--penalty', type=float, default=1.2,
help='repetition penalty for greedy sampling')
parser.add_argument('--print_once', action='store_true',
help='the completion is printed only at the end; not every word')
parser.add_argument('--topn', type=int, default=0,
help='print top-n candidates during generations; defaults to 0 which is no printing')
args = parser.parse_args()
torch.manual_seed(args.seed)
torch.cuda.manual_seed_all(args.seed)
os.environ['PYTHONHASHSEED'] = str(args.seed)
np.random.seed(args.seed)
# load the vocabulary from file
vocab = open('vocab').read().decode(encoding='utf-8').split('\n') if not use_py3 else open('vocab', encoding='utf-8').read().split('\n')
vocab = list(map(lambda x: x.split(' ')[0], vocab)) + ['<unk>'] + ['\n']
print ('{} unique words'.format(len(vocab)))
# length of the vocabulary
vocab_size = len(vocab)
# define the numericalization map
# idx2word maps the numericalized ID to the word
# word2idx maps the word to the numericalized ID
word2idx = {u:i for i, u in enumerate(vocab)}
idx2word = np.array(vocab)
embedding_dim = 1280
class TiedEmbeddingSoftmax(torch.nn.Module):
def __init__(self, vocab_size=vocab_size, embedding_size=embedding_dim, **kwargs):
super(TiedEmbeddingSoftmax, self).__init__()
self.w = torch.nn.Parameter(torch.zeros(vocab_size, embedding_size))
self.b = torch.nn.Parameter(torch.zeros(vocab_size))
def forward(self, inputs, embed=True):
if embed:
return torch.nn.functional.embedding(inputs, self.w)
else:
return torch.tensordot(inputs, self.w.t(), 1) + self.b
test_softmax = TiedEmbeddingSoftmax()
test_encoder = pytorch_transformer.Encoder()
def predict_fn(inputs):
with torch.no_grad():
embedded = torch.tensor(inputs['input_1']).cuda()
embedded = test_softmax(embedded, embed=True)
embedded = test_encoder(embedded)
embedded = test_softmax(embedded, embed=False)
return embedded
bpe = fastBPE.fastBPE('codes', 'vocab')
seq_length = min(args.generate_num, 256)
pytorch_model_hash = hashlib.md5(args.model_path.encode('utf-8')).hexdigest()
temperature = args.temperature
nucleusprob = args.nucleus
penalty = args.penalty
topk = args.topk
# try to load the model from a (cached) PyTorch checkpoint
# if one is not available, then create it by converting the weights
if os.path.exists(pytorch_model_hash):
print('Found PyTorch checkpoint @', pytorch_model_hash)
print('Loading instead of converting from TensorFlow')
checkpoint = torch.load(pytorch_model_hash)
test_softmax.load_state_dict(checkpoint['softmax'])
test_encoder.load_state_dict(checkpoint['encoder'])
test_softmax.to('cuda')
test_encoder.to('cuda')
else:
print('Could not find PyTorch checkpoint')
print('Converting weights and will store the PyTorch checkpoint as ', pytorch_model_hash)
chkpt_for_reader = '.'.join(args.model_path.split('.')[:-1])
reader = pywrap_tensorflow.NewCheckpointReader(chkpt_for_reader)
test_softmax.w = torch.nn.Parameter(torch.tensor(reader.get_tensor('w')).to('cuda'))
test_softmax.b = torch.nn.Parameter(torch.tensor(reader.get_tensor('b')).to('cuda'))
list_of_variables = list(filter(lambda x: 'Adagrad' not in x, reader.get_variable_to_shape_map().keys()))
str2parameter = lambda x: torch.nn.Parameter(torch.tensor(reader.get_tensor(x)).t().to('cuda'))
test_encoder.layernorm.weight = str2parameter('encoder/layer_normalization_96/gamma')
test_encoder.layernorm.bias = str2parameter('encoder/layer_normalization_96/beta')
for i in tqdm.tqdm(range(48)):
if i==0:
layer_variables = sorted(filter(lambda x: 'layer/' in x, list_of_variables))
else:
layer_variables = sorted(filter(lambda x: 'layer_'+str(i)+'/' in x, list_of_variables))
current_layer = getattr(test_encoder, 'layer'+str(i))
current_layer.layernorm1.bias = str2parameter(layer_variables[0])
current_layer.layernorm1.weight = str2parameter(layer_variables[1])
current_layer.layernorm2.bias = str2parameter(layer_variables[2])
current_layer.layernorm2.weight = str2parameter(layer_variables[3])
current_layer.multi_head_attention.Wq.bias = str2parameter(layer_variables[4])
current_layer.multi_head_attention.Wq.weight = str2parameter(layer_variables[5])
current_layer.multi_head_attention.Wk.bias = str2parameter(layer_variables[6])
current_layer.multi_head_attention.Wk.weight = str2parameter(layer_variables[7])
current_layer.multi_head_attention.Wv.bias = str2parameter(layer_variables[8])
current_layer.multi_head_attention.Wv.weight = str2parameter(layer_variables[9])
current_layer.multi_head_attention.dense.bias = str2parameter(layer_variables[10])
current_layer.multi_head_attention.dense.weight = str2parameter(layer_variables[11])
current_layer.ffn[0].bias = str2parameter(layer_variables[12])
current_layer.ffn[0].weight = str2parameter(layer_variables[13])
current_layer.ffn[2].bias = str2parameter(layer_variables[14])
current_layer.ffn[2].weight = str2parameter(layer_variables[15])
torch.save({
'softmax': test_softmax.state_dict(),
'encoder': test_encoder.state_dict(),
}, pytorch_model_hash)
test_softmax.eval()
test_encoder.eval()
while True:
prompt = raw_input('ENTER PROMPT: ') if not use_py3 else input('ENTER PROMPT: ')
# tokenize provided prompt
split_prompt = bpe.apply([prompt])[0].split()
text = [word2idx[i] for i in split_prompt]
# pad with 0s and create a mini-batch of 2 (arbitrary, for ease of code)
padded_text = text + [0] * (args.generate_num - len(text))
tokens_generated = np.tile(padded_text, (1,1))
try:
for token in range(len(text)-1, args.generate_num-1):
# get the logits from the prediction function
# the logic here is a bit convoluted because we are allowing generation past 512 tokens
# this is done by sliding the window over (past 512 tokens) and continuing prediction
# I'm sure this can be simplified (TODO)
if token <= seq_length:
prompt_logits = predict_fn({'input_1':tokens_generated[:, :seq_length]}).squeeze() / (temperature if temperature>0 else 1.)
_token = token if token < seq_length else -1
else:
_token = -1
end = token + 1
start = token - seq_length + 2
prompt_logits = predict_fn({'input_1':np.hstack((tokens_generated[:,0:1], tokens_generated[:,start:end]))}).squeeze() / (temperature if temperature>0 else 1.)
prompt_logits = prompt_logits.cpu().detach().numpy()
# if penalty (for repetition) is non-zero,
# discount the logits from already generated tokens
if penalty>0:
penalized_so_far = set()
for _ in range(token+1):
generated_token = tokens_generated[0][_]
# don't penalize newlines
# you could also choose not to penalize frequent words
# (which incidentally are sorted in the vocab file)
# but I don't do that
# if it prints too many new lines instead of continuing generating text,
# you might want to comment this out
#if idx2word[generated_token] == '\n':
# continue
if generated_token in penalized_so_far:
continue
penalized_so_far.add(generated_token)
prompt_logits[_token][generated_token] /= penalty
# disallow some tokens
prompt_logits[_token][word2idx['<unk>']] = -1e8
# sometimes, when generating from reddit,
# it tries to generate the Score (reddit Karma) immediately after generating the Title:
# to disallow this, we can just prevent it from generating Score
prompt_logits[_token][word2idx['Sco@@']] = -1e8
# compute probabilities from logits
prompt_probs = np.exp(prompt_logits[_token])
prompt_probs = prompt_probs / sum(prompt_probs)
pruned_list = np.argsort(prompt_probs)[::-1]
# if you are using nucleus prob, then compute the nucleus probability size
if nucleusprob > 0.:
minimum_topk = 1
nucleus = max(np.where(np.cumsum(np.sort(prompt_probs)[::-1])>nucleusprob)[0][0], minimum_topk)
elif topk > 0:
# we are over-loading notation here
# if you choose to specify a topk instead of a nucleus,
# we will hardcode the nucleus to be just that
nucleus = topk
else:
# if you specify neither nucleus or topk,
# then we will use the whole list
nucleus = len(pruned_list)
pruned_list = pruned_list[:nucleus]
# if you want to disallow more complex tokens, you can do so here
# for instance, if you want to disallow anything with the phrase `http`,
# you can delete theme from the pruned_list
# you can comment this out, I'm keeping it in for demonstration purpose
tokens_to_disallow = []
for _ in range(len(pruned_list)):
if 'http' in idx2word[pruned_list[_]]:
tokens_to_disallow.append(_)
pruned_list = np.delete(pruned_list, tokens_to_disallow)
if args.topn > 0 :
print('TOPN :: top-n alternatives:', [idx2word[_] for _ in pruned_list[:args.topn]])
# if temperature is 0
# just pick the first (most probable) token
if temperature==0:
idx = pruned_list[0]
else:
# else,
# sample from the pruned_list with the logits
chosen_idx = torch.distributions.categorical.Categorical(torch.tensor(np.expand_dims(prompt_logits[_token][pruned_list],0))).sample().numpy()[0]
idx = pruned_list[chosen_idx]
if args.topn > 0 :
print('TOPN :: chosen word:', idx2word[idx])
# assign the token for generation
tokens_generated[0][token+1] = idx
# clear screen if you want to
# os.system("clear")
tokens_generated_so_far = ' '.join([idx2word[c] for c in tokens_generated[0].squeeze()[:token+2]])
tokens_generated_so_far = re.sub('(@@ )', '', string=tokens_generated_so_far)
tokens_generated_so_far = re.sub('(@@ ?$)', '', string=tokens_generated_so_far)
if not args.print_once:
print('---------------------------------------')
print(tokens_generated_so_far)
print()
print('---------------------------------------')
print(tokens_generated_so_far)
print()
except KeyboardInterrupt: #Exception as e:
print('Continuing')
generator/ctrl/pytorch_transformer.py
-143
View File
@@ -1,143 +0,0 @@
from __future__ import print_function
import torch
import os
import tqdm
import pdb
import numpy as np
import platform
import re
import argparse
import tensorflow as tf
from tensorflow.python import pywrap_tensorflow
def angle_defn(pos, i, d_model_size):
angle_rates = 1 / np.power(10000, (2 * (i//2)) / np.float32(d_model_size))
return pos * angle_rates
def positional_encoding(position, d_model_size):
# create the sinusoidal pattern for the positional encoding
angle_rads = angle_defn(np.arange(position)[:, np.newaxis], np.arange(d_model_size)[np.newaxis, :], d_model_size)
sines = np.sin(angle_rads[:, 0::2])
cosines = np.cos(angle_rads[:, 1::2])
pos_encoding = torch.tensor(np.concatenate([sines, cosines], axis=-1)[np.newaxis, ...], dtype=torch.float)
return pos_encoding
def scaled_dot_product_attention(q, k, v, mask):
# calculate attention
matmul_qk = torch.matmul(q, k.permute(0,1,3,2))
dk = k.shape[-1]
scaled_attention_logits = matmul_qk / np.sqrt(dk)
if mask is not None:
scaled_attention_logits += (mask * -1e9)
attention_weights = torch.softmax(scaled_attention_logits, dim=-1)
output = torch.matmul(attention_weights, v)
return output
class MultiHeadAttention(torch.nn.Module):
def __init__(self, d_model_size, num_heads):
super(MultiHeadAttention, self).__init__()
self.num_heads = num_heads
self.d_model_size = d_model_size
self.depth = int(d_model_size / self.num_heads)
self.Wq = torch.nn.Linear(d_model_size, d_model_size)
self.Wk = torch.nn.Linear(d_model_size, d_model_size)
self.Wv = torch.nn.Linear(d_model_size, d_model_size)
self.dense = torch.nn.Linear(d_model_size, d_model_size)
def split_into_heads(self, x, batch_size):
x = x.reshape(batch_size, -1, self.num_heads, self.depth)
return x.permute([0, 2, 1, 3])
def forward(self, v, k, q, mask):
batch_size = q.shape[0]
q = self.Wq(q)
k = self.Wk(k)
v = self.Wv(v)
q = self.split_into_heads(q, batch_size)
k = self.split_into_heads(k, batch_size)
v = self.split_into_heads(v, batch_size)
scaled_attention = scaled_dot_product_attention(q, k, v, mask).permute([0, 2, 1, 3])
original_size_attention = scaled_attention.reshape(batch_size, -1, self.d_model_size)
output = self.dense(original_size_attention)
return output
def point_wise_feed_forward_network(d_model_size, dff):
return torch.nn.Sequential(torch.nn.Linear(d_model_size, dff), torch.nn.ReLU(), torch.nn.Linear(dff, d_model_size))
class EncoderLayer(torch.nn.Module):
def __init__(self, d_model_size, num_heads, dff, rate=0.1):
super(EncoderLayer, self).__init__()
self.multi_head_attention = MultiHeadAttention(d_model_size, num_heads)
self.ffn = point_wise_feed_forward_network(d_model_size, dff)
self.layernorm1 = torch.nn.LayerNorm(d_model_size, eps=1e-6)
self.layernorm2 = torch.nn.LayerNorm(d_model_size, eps=1e-6)
self.dropout1 = torch.nn.Dropout(rate)
self.dropout2 = torch.nn.Dropout(rate)
def forward(self, x, mask):
normed = self.layernorm1(x)
attn_output = self.multi_head_attention(normed, normed, normed, mask)
attn_output = self.dropout1(attn_output)
out1 = x + attn_output
out2 = self.layernorm2(out1)
ffn_output = self.ffn(out2)
ffn_output = self.dropout2(ffn_output)
out2 = out1 + ffn_output
return out2
class Encoder(torch.nn.Module):
def __init__(self, num_layers=48, d_model_size=1280, num_heads=16, dff=8192, input_vocab_size=50000,
rate=0.1, **kwargs):
super(Encoder, self).__init__()
self.d_model_size = d_model_size
self.num_layers = num_layers
self.pos_encoding = positional_encoding(input_vocab_size, self.d_model_size).to('cuda')
for i in range(num_layers):
setattr(self, "layer%i" % i, EncoderLayer(d_model_size, num_heads, dff, rate))
self.layernorm = torch.nn.LayerNorm(d_model_size, eps=1e-6)
self.dropout = torch.nn.Dropout(rate)
def forward(self, x):
seq_len = x.shape[1]
mask = torch.triu(torch.ones(seq_len, seq_len), 1).to('cuda')
x *= np.sqrt(self.d_model_size)
x += self.pos_encoding[:, :seq_len, :]
x = self.dropout(x)
for i in range(self.num_layers):
x = getattr(self, "layer%i" % i)(x, mask)
return self.layernorm(x)
generator/ctrl/source_attribution.py
-192
View File
@@ -1,192 +0,0 @@
from __future__ import division
from __future__ import print_function
import tensorflow as tf
import os
import numpy as np
tf.enable_eager_execution()
import transformer
import argparse
import pdb
import sys
import re
from collections import Counter
from tensorflow.python import debug as tf_debug
from tensorflow.python.ops import math_ops
from tensorflow.python.ops import embedding_ops
import fastBPE
import platform
use_py3 = platform.python_version()[0] == '3'
parser = argparse.ArgumentParser(description='TensorFlow code for generating from CTRL')
parser.add_argument('--model_dir', type=str, required=True,
help='location of model checkpoint')
parser.add_argument('--seed', type=int, default=1337,
help='random seed for TensorFlow, numpy and PythonHash')
args = parser.parse_args()
tf.random.set_random_seed(args.seed)
os.environ['PYTHONHASHSEED'] = str(args.seed)
np.random.seed(args.seed)
# load the vocabulary from file
vocab = open('vocab').read().decode(encoding='utf-8').split('\n') if not use_py3 else open('vocab', encoding='utf-8').read().split('\n')
vocab = list(map(lambda x: x.split(' ')[0], vocab)) + ['<unk>'] + ['\n']
print ('{} unique words'.format(len(vocab)))
# length of the vocabulary
vocab_size = len(vocab)
# define the numericalization map
# idx2word maps the numericalized ID to the word
# word2idx maps the word to the numericalized ID
word2idx = {u:i for i, u in enumerate(vocab)}
idx2word = np.array(vocab)
# sequence length to use for the transformer
# the model is trained with a seq_length of 512
# so, any value <= 512 should work
seq_length = 256
# the dimension of the transformer
embedding_dim = 1280
# Now, we begin defining the model
# we defer the transformer definition to transformer.py
# here, we only define the tied softmax layer
# this layer ties the softmax weights to the input embeddings
class TiedEmbeddingSoftmax(tf.keras.layers.Layer):
def __init__(self, vocab_size=vocab_size, embedding_size=embedding_dim, **kwargs):
super(TiedEmbeddingSoftmax, self).__init__()
self.w = self.add_weight(name='w', shape=(vocab_size, embedding_size),
initializer='random_normal',
trainable=True)
self.b = self.add_weight(name='b', shape=(vocab_size,),
initializer='zeros',
trainable=True)
def call(self, inputs, embed=True):
if embed:
dtype = tf.keras.backend.dtype(inputs)
if dtype != 'int32' and dtype != 'int64':
inputs = math_ops.cast(inputs, 'int32')
return embedding_ops.embedding_lookup(self.w, inputs)
else:
return tf.tensordot(inputs, tf.transpose(self.w), 1) + self.b
# input for the keras model
tokens = tf.keras.layers.Input(shape=(seq_length,), dtype='int32')
# instantiates a tied softmax class
tied_embedding_softmax = TiedEmbeddingSoftmax()
# embedded tokens, before passing it to the transformer
embedded = tied_embedding_softmax(tokens, embed=True)
# the activations after passing it from the transformer
# for some odd reason, TPUs don't play well with specifying the arguments of the Encoder() function
# so you have to leave them at their defaults
transformed = transformer.Encoder()(embedded, training=False)
# pass the activations from our tiedsoftmax class
# this time with embed=False denoting that we are doing the softmax operation
# and not a lookup
logits = tied_embedding_softmax(transformed, embed=False)
# finally, define the Keras model with inputs as tokens and outputs as the logits we just computed
model = tf.keras.Model(inputs=tokens, outputs=logits)
# the loss function is a simple categorical crossentropy between the logits and the labels
def loss(labels, logits):
return tf.keras.losses.sparse_categorical_crossentropy(labels, logits, from_logits=True)
# the optimizer is not used since this code only supports inference
# however, to compile the model, we still define it
optimizer = tf.contrib.tpu.CrossShardOptimizer(
tf.contrib.estimator.clip_gradients_by_norm(
tf.train.AdagradOptimizer(learning_rate=1e-2), 0.25)
)
# compile the model with the optimizer and loss
model.compile(optimizer=optimizer, loss=loss)
print(model.summary())
# IMPORTANT
# this is where the saved model is presented to the code
# the model directory should have the model checkpoint and
# a checkpoint file
run_config = tf.contrib.tpu.RunConfig(
model_dir=args.model_dir)
# this converts the Keras model to a TensorFlow estimator
# this step is critical
# remember to patch the TF 1.14 file before running the code, else you're going to see errors here
estimator_model = tf.keras.estimator.model_to_estimator(keras_model=model, config=run_config)
# we now create a serving function from this estimator
# this enables us to load the model once and easily query it multiple times
def serving_input_fn():
inputs = {'input_1': tf.placeholder(tf.int32, [2,seq_length])}
return tf.estimator.export.ServingInputReceiver(inputs, inputs)
predict_fn = tf.contrib.predictor.from_estimator(estimator_model, serving_input_fn)
# almost there, we now take the user prompt and tokenize with BPE
# load BPE codes
bpe = fastBPE.fastBPE('codes', 'vocab')
domains = []
with open('control_codes.txt', 'r') as f:
domains = [line.split() for line in f.readlines()]
domains = [(t[1], float(t[0])) for t in domains]
while True:
_prompt = raw_input('ENTER PROMPT: ') if not use_py3 else input('ENTER PROMPT: ')
ppls = {}
# loop over all domains and compute perplexity
for domain, domain_prior in domains:
print(u'computing for domain: {}'.format(domain))
# tokenize data and add domain tag to it
prompt = domain + u' ' + _prompt
split_prompt = bpe.apply([prompt])[0].split()
# numericalize data and pad to the seq_len dimension
text = [word2idx[i] for i in split_prompt]
padding_text = text + [0] * (seq_length - len(text))
tokens_generated = np.tile(padding_text, (2,1))
output_scores = predict_fn({'input_1':tokens_generated})['tied_embedding_softmax'].squeeze()[0]
token_scores = output_scores[:-1]
# compute the perplexity for this sequence
xent = 0
for sequence_idx, token_idx in enumerate(text[1:]):
token = idx2word[token_idx]
# compute the probability of this token
Z = np.exp(token_scores[sequence_idx]).sum()
token_prob = np.exp(token_scores[sequence_idx, token_idx]) / Z
xent -= np.log(token_prob) / len(text[1:])
ppls[domain] = round(np.exp(xent), 6)
#print(u'{} ppl = {}'.format(domain, ppls[domain]))
# sort the domains based on perplexities and print
ppls = [(k, v) for k, v in ppls.items()]
ppls.sort(key=lambda x: x[1])
print('PROMPT: {}'.format(_prompt))
for t in ppls:
domain, ppl = t
print(u'{} ppl = {}'.format(domain, ppl))
generator/ctrl/training_utils/.gitignore
-3
View File
@@ -1,3 +0,0 @@
seqlen256_v1.ckpt
*.txt
*.tfrecords
generator/ctrl/training_utils/README.md
-85
View File
@@ -1,85 +0,0 @@
# Fine-Tuning the Model on Custom Dataset
This folder contains sample code to fine-tune the model on custom data. It is primarily targeted for GPU usage, but there are pointers throughout showing how to run on TPUs as well.
Fine-tuning can be used to augment existing control codes or add new control codes. There are 5 steps elaborated upon in the example below:
1. Patch `keras.py` as in the generation script
2. Obtain raw versions of your text files
3. Convert this text data into TFRecords; _if you wish to use TPUs, you must transfer these records to GCS._
4. Fine-tuning the model on these TFRecords files
5. Testing that the generation works.
## Example of adding a new control code
Let's begin by adding a new control code `Moby` that is associated with the book [Moby Dick](https://www.gutenberg.org/ebooks/2701)
If you run `generation.py` with the pretrained models available and try to use this control code, you will find that the model outputs gibberish. Great! It is indeed a fresh new control code. We will run this again after training as a sanity check.
### Step 1 - Patch your `keras.py`
As is required for the generation script, you must patch your `keras.py`. If you patched it before, please roll-back and re-patch with the latest version.
There are two changes: (1) it defaults to `use_tpu=False` so training/inference takes place on GPUs, (2) the batch size defaults to 4 for GPU training. You might need to go lower depending on your machine.
You can leave `use_tpu=True` if you wish to train on TPUs and adjust the batch size accordingly.
### Step 2 - Obtain Your Data
The book is available publicly; you can simply download it as
```
wget -O moby_dick.txt https://www.gutenberg.org/files/2701/2701-0.txt
```
### Step 3 - Convert Data to TFRecords
We include the file `make_tf_records.py` to facilitate this.
Run:
```
python make_tf_records.py --text_file moby_dick.txt --control_code Moby --sequence_len 256
```
It has three arguments: `text_file` which specifies the name of the file to convert, `control_code` which specifies one token (must be in vocabulary) to append to each example, and `sequence_len` which specifies the sequence length to use to create the data. This must match the sequence length of the model being trained.
### Step 4 - Train!
Simply run `python training.py --model_dir <path_to_model>.ckpt/ --iterations <number_of_iterations>`
The script picks up all TFRecords in the current folder and fine-tunes the model provided in the `--model_dir` flag.
If you intend to use TPUs, you must transfer these TFRecords to GCS and edit the location of the data path used by `input_fn` to the GCS bucket.
To very important gotchas here:
1. If you have very limited data, the model will very likely overfit and end up memorizing. At the moment, just keep the `--iterations` flag low, preferably equivalent to one epoch or so.
2. The model is updated and stored in the same directory, if you don't wish to overwrite your model files, please create a backup before you run the training code.
### Step 5 - Generate!
We ran `python training.py --model_dir seqlen256_v1.ckpt/ --iterations 250` and try generating with the `Moby` control code.
Running with the `Moby` control code and a prompt of `I` yields something reasonable in-domain:
```
Moby I <GENERATION_BEGINS> was a little fellow, and he was a
great man, what should that matter? And yet it seemed to me that
Queequegs words about his father were true. He had been very angry
with him, because the old man would not let him go a-whaling...
```
Providing a prompt also works:
```
Moby Then I realized, it wasn't one white whale but three! <GENERATION_BEGINS> And all three
were making straight for my boat, which was now some distance away.
But the three spouts seemed to be coming from different directions, and
as they drew nearer and nearer, their tongues began licking up the
brine like so many hungry wolves at a carcass...
```
generator/ctrl/training_utils/apoc_seed1.json
-1
View File
File diff suppressed because one or more lines are too long
generator/ctrl/training_utils/apoc_seed2.json
-1
View File
File diff suppressed because one or more lines are too long
generator/ctrl/training_utils/apoc_seed3.json
-1
View File
File diff suppressed because one or more lines are too long
generator/ctrl/training_utils/apoc_seed4.json
-1
View File
File diff suppressed because one or more lines are too long
generator/ctrl/training_utils/forest_training.py
-172
View File
@@ -1,172 +0,0 @@
from __future__ import division
from __future__ import print_function
import sys
sys.path.append('../')
import tensorflow as tf
import os
import numpy as np
tf.enable_eager_execution()
import transformer
import argparse
import pdb
import re
from collections import Counter
from tensorflow.python import debug as tf_debug
from tensorflow.python.ops import math_ops
from tensorflow.python.ops import embedding_ops
import fastBPE
import platform
use_py3 = platform.python_version()[0] == '3'
parser = argparse.ArgumentParser(description='TensorFlow code for generating from CTRL')
parser.add_argument('--model_dir', type=str, required=True,
help='location of model checkpoint')
parser.add_argument('--seed', type=int, default=1337,
help='random seed for TensorFlow, numpy and PythonHash')
parser.add_argument('--sequence_len', type=int, default=256,
help='sequence len of model being fine-tuned (must match also the TFRecords)')
parser.add_argument('--iterations', type=int, default=1000,
help='random seed for TensorFlow, numpy and PythonHash')
args = parser.parse_args()
tf.random.set_random_seed(args.seed)
os.environ['PYTHONHASHSEED'] = str(args.seed)
np.random.seed(args.seed)
# load the vocabulary from file
vocab = open('../vocab').read().decode(encoding='utf-8').split('\n') if not use_py3 else open('../vocab',
encoding='utf-8').read().split(
'\n')
vocab = list(map(lambda x: x.split(' ')[0], vocab)) + ['<unk>'] + ['\n']
print('{} unique words'.format(len(vocab)))
# length of the vocabulary
vocab_size = len(vocab)
# define the numericalization map
# idx2word maps the numericalized ID to the word
# word2idx maps the word to the numericalized ID
word2idx = {u: i for i, u in enumerate(vocab)}
idx2word = np.array(vocab)
# sequence length to use for the transformer
# must match the model being fine-tuned
seq_length = args.sequence_len
def input_fn(params=None):
print('READING!', params)
dataset = tf.data.Dataset.list_files(tf.io.gfile.glob('./*.tfrecords'), shuffle=True)
tf_data = tf.data.TFRecordDataset(dataset)
myfeatures = {
'input': tf.io.FixedLenFeature([256], tf.int64),
'output': tf.io.FixedLenFeature([256], tf.int64)
}
def _parse_text_function(example_proto):
blah = tf.io.parse_single_example(example_proto, myfeatures)
return blah['input'], blah['output']
train_data = tf_data.map(_parse_text_function).batch(params['batch_size'], drop_remainder=True).repeat().shuffle(
10000) # .prefetch(tf.contrib.data.AUTOTUNE)
return train_data
# the dimension of the transformer
embedding_dim = 1280
# Now, we begin defining the model
# we defer the transformer definition to transformer.py
# here, we only define the tied softmax layer
# this layer ties the softmax weights to the input embeddings
class TiedEmbeddingSoftmax(tf.keras.layers.Layer):
def __init__(self, vocab_size=vocab_size, embedding_size=embedding_dim, **kwargs):
with tf.device('/cpu:0'):
super(TiedEmbeddingSoftmax, self).__init__()
self.w = self.add_weight(name='w', shape=(vocab_size, embedding_size),
initializer='random_normal',
trainable=True)
self.b = self.add_weight(name='b', shape=(vocab_size,),
initializer='zeros',
trainable=True)
def call(self, inputs, embed=True):
with tf.device('/cpu:0'):
if embed:
dtype = tf.keras.backend.dtype(inputs)
if dtype != 'int32' and dtype != 'int64':
inputs = math_ops.cast(inputs, 'int32')
return embedding_ops.embedding_lookup(self.w, inputs)
else:
return tf.tensordot(inputs, tf.transpose(self.w), 1) + self.b
# input for the keras model
tokens = tf.keras.layers.Input(shape=(seq_length,), dtype='int32')
# instantiates a tied softmax class
tied_embedding_softmax = TiedEmbeddingSoftmax()
# embedded tokens, before passing it to the transformer
embedded = tied_embedding_softmax(tokens, embed=True)
# the activations after passing it from the transformer
# for some odd reason, TPUs don't play well with specifying the arguments of the Encoder() function
# so you have to leave them at their defaults
transformed = transformer.Encoder()(embedded, training=False)
# pass the activations from our tiedsoftmax class
# this time with embed=False denoting that we are doing the softmax operation
# and not a lookup
logits = tied_embedding_softmax(transformed, embed=False)
# finally, define the Keras model with inputs as tokens and outputs as the logits we just computed
model = tf.keras.Model(inputs=tokens, outputs=logits)
# the loss function is a simple categorical crossentropy between the logits and the labels
def loss(labels, logits):
loss = tf.keras.losses.sparse_categorical_crossentropy(labels, logits, from_logits=True)
mean = tf.math.reduce_mean(loss)
loss = tf.Print(loss, [mean])
return loss
# the optimizer is not used since this code only supports inference
# however, to compile the model, we still define it
optimizer = tf.contrib.estimator.clip_gradients_by_norm(
tf.train.AdagradOptimizer(learning_rate=3e-3), 0.25)
# compile the model with the optimizer and loss
model.compile(optimizer=optimizer, loss=loss)
print(model.summary())
# IMPORTANT
# this is where the saved model is presented to the code
# the model directory should have the model checkpoint and
# a checkpoint file
run_config = tf.contrib.tpu.RunConfig(
model_dir=args.model_dir)
# this converts the Keras model to a TensorFlow estimator
# this step is critical
# remember to patch the TF 1.14 file before running the code, else you're going to see errors here
run_config = tf.contrib.tpu.RunConfig(
model_dir=args.model_dir,
session_config=tf.ConfigProto(allow_soft_placement=True, log_device_placement=True),
tpu_config=tf.contrib.tpu.TPUConfig(iterations_per_loop=100, num_cores_per_replica=1,
input_partition_dims=[[1, 1], [1, 1]], per_host_input_for_training=3))
tf.logging.set_verbosity(tf.logging.INFO)
estimator_model = tf.keras.estimator.model_to_estimator(keras_model=model, config=run_config)
estimator_model.train(input_fn=input_fn, steps=args.iterations)
generator/ctrl/training_utils/make_forest_tf_records.py
-130
View File
@@ -1,130 +0,0 @@
import numpy as np
import os
import tensorflow as tf
import tqdm
import pdb
import glob
import time
import sys
import re
import argparse
import fastBPE
import platform
import json
import sys
# insert at 1, 0 is the script path (or '' in REPL)
sys.path.insert(1, '../../..')
from story.utils import *
def make_samples_helper(context, story_block, action_results, path, tree_id):
samples = []
for i, action_result in enumerate(action_results):
new_path = path[:]
new_path.append(i)
if action_result["result"] is not None:
sample = [context, story_block, action_result["action"], action_result["result"]]
samples.append(sample)
if len(action_result["action_results"]) is not 0:
sub_result = make_samples_helper(context, action_result["result"], action_result["action_results"], new_path, tree_id)
samples += sub_result
return samples
def make_samples(tree):
# Traverse to the bottom levels of each tree
first_story_block = tree["first_story_block"]
samples = make_samples_helper(tree["context"], first_story_block, tree["action_results"], [], tree["tree_id"])
return samples
def build_tokenized_samples(bpe, tree):
samples = make_samples(tree)
string_samples = []
for sample in samples:
sample = [string.strip() for string in sample]
sample[2] = sample[2][0].lower() + sample[2][1:]
sample[2] = "You " + sample[2]
new_sample = []
for item in sample:
new_sample.append(second_to_first_person(item))
string_samples.append(" ".join(new_sample))
tokenized_samples = [bpe.apply([sample.encode('ascii', errors='ignore') if not use_py3 else sample])[0] for sample in
string_samples] # will NOT work for non-English texts
tokenized_samples = [re.findall(r'\S+|\n', sample) for sample in tokenized_samples]
tokenized_samples = [list(filter(lambda x: x != u'@@', sample)) for sample in tokenized_samples]
# Fill samples up to seq_len
for sample in tokenized_samples:
pad_len = seq_length - len(sample)
for _ in range(pad_len):
sample.append("\n")
return tokenized_samples
use_py3 = platform.python_version()[0] == '3'
paths_to_train_files = ["apoc_seed1.json","apoc_seed2.json","apoc_seed3.json","apoc_seed4.json"]
seq_length = 256
domain = ["Apocalypse"]
# Build sequences from JSON
bpe = fastBPE.fastBPE('../codes', '../vocab')
tokenized_samples = []
for fname in paths_to_train_files:
with open(fname, 'r') as fp:
tree = json.load(fp)
tokenized_samples += build_tokenized_samples(bpe, tree)
string_samples = []
# load the vocabulary from file
vocab = open('../vocab').read().decode(encoding='utf-8').split('\n') if not use_py3 else open('../vocab', encoding='utf-8').read().split('\n')
vocab = list(map(lambda x: x.split(' ')[0], vocab)) + ['<unk>'] + ['\n']
print ('{} unique words'.format(len(vocab)))
# Creating a mapping from unique characters to indices
word2idx = {u:i for i, u in enumerate(vocab)}
idx2word = np.array(vocab)
seq_length = seq_length-1
def numericalize(x):
count = 0
for i in x:
if i not in word2idx:
print(i)
count += 1
return count>1, [word2idx.get(i, word2idx['<unk>']) for i in x]
tfrecords_fname = 'action_results.tfrecords'
total = 0
skipped = 0
with tf.io.TFRecordWriter(tfrecords_fname) as writer:
for sample in tokenized_samples:
domain_seq = (domain+sample)[:256+1]
flag_input, inputs = numericalize(domain_seq[:-1])
flag_output, outputs = numericalize(domain_seq[1:])
total += 1
if flag_input or flag_output:
skipped += 1
continue
if len(inputs)!=seq_length+1 or len(outputs)!=seq_length+1:
break
example_proto = tf.train.Example(features=tf.train.Features(feature={'input': tf.train.Feature(int64_list=tf.train.Int64List(value=inputs)),
'output': tf.train.Feature(int64_list=tf.train.Int64List(value=outputs))}))
writer.write(example_proto.SerializeToString())
print('Done')
print('Skipped', skipped, 'of', total)
generator/ctrl/training_utils/make_tf_records.py
-81
View File
@@ -1,81 +0,0 @@
import numpy as np
import os
import tensorflow as tf
import tqdm
import pdb
import glob
import time
import sys
import re
import argparse
import fastBPE
import platform
use_py3 = platform.python_version()[0] == '3'
parser = argparse.ArgumentParser(description='TensorFlow code for creating TFRecords data')
parser.add_argument('--text_file', type=str, required=True,
help='location of text file to convert to TFRecords')
parser.add_argument('--control_code', type=str, required=True,
help='control code to use for this file. must be in the vocabulary, else it will error out.')
parser.add_argument('--sequence_len', type=int, required=True,
help='sequence length of model being fine-tuned (256 or 512)')
args = parser.parse_args()
path_to_train_file = fname = args.text_file
domain = [args.control_code]
train_text = open(path_to_train_file, 'rb').read().decode(encoding='utf-8')
bpe = fastBPE.fastBPE('../codes', '../vocab')
tokenized_train_text = bpe.apply([train_text.encode('ascii', errors='ignore') if not use_py3 else train_text])[0] # will NOT work for non-English texts
# if you want to run non-english text, please tokenize separately using ./fast applybpe and then run this script on the .bpe file with utf8 encoding
tokenized_train_text = re.findall(r'\S+|\n', tokenized_train_text)
tokenized_train_text = list(filter(lambda x: x != u'@@', tokenized_train_text))
# load the vocabulary from file
vocab = open('../vocab').read().decode(encoding='utf-8').split('\n') if not use_py3 else open('../vocab', encoding='utf-8').read().split('\n')
vocab = list(map(lambda x: x.split(' ')[0], vocab)) + ['<unk>'] + ['\n']
print ('{} unique words'.format(len(vocab)))
if args.control_code not in vocab:
print('Provided control code is not in the vocabulary')
print('Please provide a different one; refer to the vocab file for allowable tokens')
sys.exit(1)
# Creating a mapping from unique characters to indices
word2idx = {u:i for i, u in enumerate(vocab)}
idx2word = np.array(vocab)
seq_length = args.sequence_len-1
def numericalize(x):
count = 0
for i in x:
if i not in word2idx:
print(i)
count += 1
return count>1, [word2idx.get(i, word2idx['<unk>']) for i in x]
tfrecords_fname = fname.lower()+'.tfrecords'
total = 0
skipped = 0
with tf.io.TFRecordWriter(tfrecords_fname) as writer:
for i in tqdm.tqdm(range(0, len(tokenized_train_text), seq_length)):
flag_input, inputs = numericalize(domain+tokenized_train_text[i:i+seq_length])
flag_output, outputs = numericalize(tokenized_train_text[i:i+seq_length+1])
total += 1
if flag_input or flag_output:
skipped += 1
continue
if len(inputs)!=seq_length+1 or len(outputs)!=seq_length+1:
break
example_proto = tf.train.Example(features=tf.train.Features(feature={'input': tf.train.Feature(int64_list=tf.train.Int64List(value=inputs)),
'output': tf.train.Feature(int64_list=tf.train.Int64List(value=outputs))}))
writer.write(example_proto.SerializeToString())
print('Done')
print('Skipped', skipped, 'of', total)
generator/ctrl/training_utils/training.py
-166
View File
@@ -1,166 +0,0 @@
from __future__ import division
from __future__ import print_function
import sys
sys.path.append('../')
import tensorflow as tf
import os
import numpy as np
tf.enable_eager_execution()
import transformer
import argparse
import pdb
import re
from collections import Counter
from tensorflow.python import debug as tf_debug
from tensorflow.python.ops import math_ops
from tensorflow.python.ops import embedding_ops
import fastBPE
import platform
use_py3 = platform.python_version()[0] == '3'
parser = argparse.ArgumentParser(description='TensorFlow code for generating from CTRL')
parser.add_argument('--model_dir', type=str, required=True,
help='location of model checkpoint')
parser.add_argument('--seed', type=int, default=1337,
help='random seed for TensorFlow, numpy and PythonHash')
parser.add_argument('--sequence_len', type=int, default=256,
help='sequence len of model being fine-tuned (must match also the TFRecords)')
parser.add_argument('--iterations', type=int, default=1000,
help='random seed for TensorFlow, numpy and PythonHash')
args = parser.parse_args()
tf.random.set_random_seed(args.seed)
os.environ['PYTHONHASHSEED'] = str(args.seed)
np.random.seed(args.seed)
# load the vocabulary from file
vocab = open('../vocab').read().decode(encoding='utf-8').split('\n') if not use_py3 else open('../vocab',
encoding='utf-8').read().split(
'\n')
vocab = list(map(lambda x: x.split(' ')[0], vocab)) + ['<unk>'] + ['\n']
print('{} unique words'.format(len(vocab)))
# length of the vocabulary
vocab_size = len(vocab)
# define the numericalization map
# idx2word maps the numericalized ID to the word
# word2idx maps the word to the numericalized ID
word2idx = {u: i for i, u in enumerate(vocab)}
idx2word = np.array(vocab)
# sequence length to use for the transformer
# must match the model being fine-tuned
seq_length = args.sequence_len
def input_fn(params=None):
print('READING!', params)
dataset = tf.data.Dataset.list_files(tf.io.gfile.glob('./*.tfrecords'), shuffle=True)
tf_data = tf.data.TFRecordDataset(dataset)
myfeatures = {
'input': tf.io.FixedLenFeature([256], tf.int64),
'output': tf.io.FixedLenFeature([256], tf.int64)
}
def _parse_text_function(example_proto):
blah = tf.io.parse_single_example(example_proto, myfeatures)
return blah['input'], blah['output']
train_data = tf_data.map(_parse_text_function).batch(params['batch_size'], drop_remainder=True).repeat().shuffle(
10000) # .prefetch(tf.contrib.data.AUTOTUNE)
return train_data
# the dimension of the transformer
embedding_dim = 1280
# Now, we begin defining the model
# we defer the transformer definition to transformer.py
# here, we only define the tied softmax layer
# this layer ties the softmax weights to the input embeddings
class TiedEmbeddingSoftmax(tf.keras.layers.Layer):
def __init__(self, vocab_size=vocab_size, embedding_size=embedding_dim, **kwargs):
super(TiedEmbeddingSoftmax, self).__init__()
self.w = self.add_weight(name='w', shape=(vocab_size, embedding_size),
initializer='random_normal',
trainable=True)
self.b = self.add_weight(name='b', shape=(vocab_size,),
initializer='zeros',
trainable=True)
def call(self, inputs, embed=True):
if embed:
dtype = tf.keras.backend.dtype(inputs)
if dtype != 'int32' and dtype != 'int64':
inputs = math_ops.cast(inputs, 'int32')
return embedding_ops.embedding_lookup(self.w, inputs)
else:
return tf.tensordot(inputs, tf.transpose(self.w), 1) + self.b
# input for the keras model
tokens = tf.keras.layers.Input(shape=(seq_length,), dtype='int32')
# instantiates a tied softmax class
tied_embedding_softmax = TiedEmbeddingSoftmax()
# embedded tokens, before passing it to the transformer
embedded = tied_embedding_softmax(tokens, embed=True)
# the activations after passing it from the transformer
# for some odd reason, TPUs don't play well with specifying the arguments of the Encoder() function
# so you have to leave them at their defaults
transformed = transformer.Encoder()(embedded, training=False)
# pass the activations from our tiedsoftmax class
# this time with embed=False denoting that we are doing the softmax operation
# and not a lookup
logits = tied_embedding_softmax(transformed, embed=False)
# finally, define the Keras model with inputs as tokens and outputs as the logits we just computed
model = tf.keras.Model(inputs=tokens, outputs=logits)
# the loss function is a simple categorical crossentropy between the logits and the labels
def loss(labels, logits):
return tf.keras.losses.sparse_categorical_crossentropy(labels, logits, from_logits=True)
# the optimizer is not used since this code only supports inference
# however, to compile the model, we still define it
optimizer = tf.contrib.estimator.clip_gradients_by_norm(
tf.train.AdagradOptimizer(learning_rate=3e-3), 0.25)
# compile the model with the optimizer and loss
model.compile(optimizer=optimizer, loss=loss)
print(model.summary())
# IMPORTANT
# this is where the saved model is presented to the code
# the model directory should have the model checkpoint and
# a checkpoint file
run_config = tf.contrib.tpu.RunConfig(
model_dir=args.model_dir)
# this converts the Keras model to a TensorFlow estimator
# this step is critical
# remember to patch the TF 1.14 file before running the code, else you're going to see errors here
run_config = tf.contrib.tpu.RunConfig(
model_dir=args.model_dir,
session_config=tf.ConfigProto(allow_soft_placement=True, log_device_placement=True),
tpu_config=tf.contrib.tpu.TPUConfig(iterations_per_loop=100, num_cores_per_replica=1,
input_partition_dims=[[1, 1], [1, 1]], per_host_input_for_training=3))
tf.logging.set_verbosity(tf.logging.INFO)
estimator_model = tf.keras.estimator.model_to_estimator(keras_model=model, config=run_config)
estimator_model.train(input_fn=input_fn, steps=args.iterations)
generator/ctrl/transformer.py
-139
View File
@@ -1,139 +0,0 @@
import tensorflow as tf
import numpy as np
def angle_defn(pos, i, d_model_size):
angle_rates = 1 / np.power(10000, (2 * (i//2)) / np.float32(d_model_size))
return pos * angle_rates
def positional_encoding(position, d_model_size):
# create the sinusoidal pattern for the positional encoding
angle_rads = angle_defn(np.arange(position)[:, np.newaxis], np.arange(d_model_size)[np.newaxis, :], d_model_size)
sines = np.sin(angle_rads[:, 0::2])
cosines = np.cos(angle_rads[:, 1::2])
pos_encoding = tf.cast(np.concatenate([sines, cosines], axis=-1)[np.newaxis, ...], dtype=tf.float32)
return pos_encoding
def scaled_dot_product_attention(q, k, v, mask):
# calculate attention
matmul_qk = tf.matmul(q, k, transpose_b=True)
dk = tf.cast(tf.shape(k)[-1], tf.float32)
scaled_attention_logits = matmul_qk / tf.math.sqrt(dk)
if mask is not None:
scaled_attention_logits += (mask * -1e9)
attention_weights = tf.nn.softmax(scaled_attention_logits, axis=-1)
output = tf.matmul(attention_weights, v)
return output
class MultiHeadAttention(tf.keras.layers.Layer):
def __init__(self, d_model_size, num_heads):
super(MultiHeadAttention, self).__init__()
self.num_heads = num_heads
self.d_model_size = d_model_size
self.depth = int(d_model_size / self.num_heads)
self.Wq = tf.keras.layers.Dense(d_model_size)
self.Wk = tf.keras.layers.Dense(d_model_size)
self.Wv = tf.keras.layers.Dense(d_model_size)
self.dense = tf.keras.layers.Dense(d_model_size)
def split_into_heads(self, x, batch_size):
x = tf.reshape(x, (batch_size, -1, self.num_heads, self.depth))
return tf.transpose(x, perm=[0, 2, 1, 3])
def call(self, v, k, q, mask):
batch_size = tf.shape(q)[0]
q = self.Wq(q)
k = self.Wk(k)
v = self.Wv(v)
q = self.split_into_heads(q, batch_size)
k = self.split_into_heads(k, batch_size)
v = self.split_into_heads(v, batch_size)
scaled_attention = tf.transpose(scaled_dot_product_attention(q, k, v, mask), perm=[0, 2, 1, 3])
original_size_attention = tf.reshape(scaled_attention, (batch_size, -1, self.d_model_size))
output = self.dense(original_size_attention)
return output
def point_wise_feed_forward_network(d_model_size, dff):
return tf.keras.Sequential([tf.keras.layers.Dense(dff, activation='relu'),
tf.keras.layers.Dense(d_model_size)])
class EncoderLayer(tf.keras.layers.Layer):
def __init__(self, d_model_size, num_heads, dff, rate=0.1):
super(EncoderLayer, self).__init__()
self.multi_head_attention = MultiHeadAttention(d_model_size, num_heads)
self.ffn = point_wise_feed_forward_network(d_model_size, dff)
self.layernorm1 = tf.keras.layers.LayerNormalization(epsilon=1e-6)
self.layernorm2 = tf.keras.layers.LayerNormalization(epsilon=1e-6)
self.dropout1 = tf.keras.layers.Dropout(rate)
self.dropout2 = tf.keras.layers.Dropout(rate)
def call(self, x, training, mask):
normed = self.layernorm1(x)
attn_output = self.multi_head_attention(normed, normed, normed, mask)
attn_output = self.dropout1(attn_output, training=training)
out1 = x + attn_output
out2 = self.layernorm2(out1)
ffn_output = self.ffn(out2)
ffn_output = self.dropout2(ffn_output, training=training)
out2 = out1 + ffn_output
return out2
class Encoder(tf.keras.layers.Layer):
def __init__(self, num_layers=48, d_model_size=1280, num_heads=16, dff=8192, input_vocab_size=50000,
rate=0.1, **kwargs):
super(Encoder, self).__init__()
self.d_model_size = d_model_size
self.num_layers = num_layers
self.pos_encoding = positional_encoding(input_vocab_size, self.d_model_size)
for i in range(num_layers):
setattr(self, "layer%i" % i, EncoderLayer(d_model_size, num_heads, dff, rate))
self.layernorm = tf.keras.layers.LayerNormalization(epsilon=1e-6)
self.dropout = tf.keras.layers.Dropout(rate)
def get_config(self):
base_config = super(Encoder, self).get_config()
return base_config
def call(self, x, training):
seq_len = tf.shape(x)[1]
mask = 1 - tf.linalg.band_part(tf.ones((seq_len, seq_len)), -1, 0)
x *= tf.math.sqrt(tf.cast(self.d_model_size, tf.float32))
x += self.pos_encoding[:, :seq_len, :]
x = self.dropout(x, training=training)
for i in range(self.num_layers):
x = getattr(self, "layer%i" % i)(x, training, mask)
return self.layernorm(x)
generator/ctrl/vocab
-246531
View File
File diff suppressed because it is too large Load Diff
generator/human/task_generator.py
-6
View File
@@ -1,6 +0,0 @@
"""
Story tree representation
"""
generator/tf/CONTRIBUTORS.md
-17
View File
@@ -1,17 +0,0 @@
# Contributors (alphabetically)
* **[madisonmay](https://github.com/madisonmay)**
Added Dockerfiles
* **[Margaret Mitchell et al](https://arxiv.org/abs/1810.03993)**
Our [usage](./README.md#usage) writeup was loosely inspired by the paper
[Model Cards for Model Reporting](https://arxiv.org/abs/1810.03993)
and related conversations with some of the authors.
* **[webproduktion01](https://github.com/webproduktion01)**
Ported download script to python.
**[Full code contributors list](https://github.com/openai/gpt-2/contributors).**
generator/tf/DEVELOPERS.md
-85
View File
@@ -1,85 +0,0 @@
# Installation
Git clone this repository, and `cd` into directory for remaining commands
```
git clone https://github.com/openai/gpt-2.git && cd gpt-2
```
Then, follow instructions for either native or Docker installation.
## Native Installation
All steps can optionally be done in a virtual environment using tools such as `virtualenv` or `conda`.
Install tensorflow 1.12 (with GPU support, if you have a GPU and want everything to run faster)
```
pip3 install tensorflow==1.12.0
```
or
```
pip3 install tensorflow-gpu==1.12.0
```
Install other python packages:
```
pip3 install -r requirements.txt
```
Download the model data
```
python3 download_model.py 117M
```
## Docker Installation
Build the Dockerfile and tag the created image as `gpt-2`:
```
docker build --tag gpt-2 -f Dockerfile.gpu . # or Dockerfile.cpu
```
Start an interactive bash session from the `gpt-2` docker image.
You can opt to use the `--runtime=nvidia` flag if you have access to a NVIDIA GPU
and a valid install of [nvidia-docker 2.0](https://github.com/nvidia/nvidia-docker/wiki/Installation-(version-2.0)).
```
docker run --runtime=nvidia -it gpt-2 bash
```
# Running
| WARNING: Samples are unfiltered and may contain offensive content. |
| --- |
Some of the examples below may include Unicode text characters. Set the environment variable:
```
export PYTHONIOENCODING=UTF-8
```
to override the standard stream settings in UTF-8 mode.
## Unconditional sample generation
To generate unconditional samples from the small model:
```
python3 src/generate_unconditional_samples.py | tee /tmp/samples
```
There are various flags for controlling the samples:
```
python3 src/generate_unconditional_samples.py --top_k 40 --temperature 0.7 | tee /tmp/samples
```
To check flag descriptions, use:
```
python3 src/generate_unconditional_samples.py -- --help
```
## Conditional sample generation
To give the model custom prompts, you can use:
```
python3 src/interactive_conditional_samples.py --top_k 40
```
To check flag descriptions, use:
```
python3 src/interactive_conditional_samples.py -- --help
```
generator/tf/LICENSE
-21
View File
@@ -1,21 +0,0 @@
MIT License
Copyright (c) 2019 OpenAI
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
generator/tf/README.md
-61
View File
@@ -1,61 +0,0 @@
# gpt-2
Code and samples from the paper ["Language Models are Unsupervised Multitask Learners"](https://d4mucfpksywv.cloudfront.net/better-language-models/language-models.pdf).
For now, we have only released a smaller (117M parameter) version of GPT-2.
See more details in our [blog post](https://blog.openai.com/better-language-models/).
## Usage
This repository is meant to be a starting point for researchers and engineers to experiment with GPT-2-117M. While GPT-2-117M is less proficient than GPT-2-1.5B, it is useful for a wide range of research and applications which could also apply to larger models.
### Some caveats
- GPT-2-117M robustness and worst case behaviors are not well-understood. As with any machine-learned model, carefully evaluate GPT-2-117M for your use case, especially if used without fine-tuning or in safety-critical applications where reliability is important.
- The dataset our GPT-2-117M was trained on contains many texts with [biases](https://twitter.com/TomerUllman/status/1101485289720242177) and factual inaccuracies, and thus GPT-2-117M is likely to be biased and inaccurate as well.
- To avoid having samples mistaken as human-written, we recommend clearly labeling samples as synthetic before wide dissemination. Our models are often incoherent or inaccurate in subtle ways, which takes more than a quick read for a human to notice.
### Work with us
Please [let us know](mailto:languagequestions@openai.com) if you’re doing interesting research with or working on applications of GPT-2-117M! We’re especially interested in hearing from and potentially working with those who are studying
- Potential malicious use cases and defenses against them (e.g. the detectability of synthetic text)
- The extent of problematic content (e.g. bias) being baked into the models and effective mitigations
## Development
See [DEVELOPERS.md](./DEVELOPERS.md)
## Contributors
See [CONTRIBUTORS.md](./CONTRIBUTORS.md)
## GPT-2 samples
| WARNING: Samples are unfiltered and may contain offensive content. |
| --- |
While we have not yet released GPT-2 itself, you can see some samples from it in the `gpt-2-samples` folder.
We show unconditional samples with default settings (temperature 1 and no truncation), with temperature 0.7, and with truncation with top_k 40.
We show conditional samples, with contexts drawn from `WebText`'s test set, with default settings (temperature 1 and no truncation), with temperature 0.7, and with truncation with top_k 40.
## Citation
Please use the following bibtex entry:
```
@article{radford2019language,
title={Language Models are Unsupervised Multitask Learners},
author={Radford, Alec and Wu, Jeff and Child, Rewon and Luan, David and Amodei, Dario and Sutskever, Ilya},
year={2019}
}
```
## Future work
We may release code for evaluating the models on various benchmarks.
We are still considering release of the larger models.
## License
[MIT](./LICENSE)
generator/tf/__init__.py
View File
generator/tf/download_model.py
-28
View File
@@ -1,28 +0,0 @@
import os
import sys
import requests
from tqdm import tqdm
if len(sys.argv) != 2:
print('You must enter the model name as a parameter, e.g.: download_model.py 117M')
sys.exit(1)
model = sys.argv[1]
subdir = os.path.join('models', model)
if not os.path.exists(subdir):
os.makedirs(subdir)
subdir = subdir.replace('\\','/') # needed for Windows
for filename in ['checkpoint','encoder.json','hparams.json','model.ckpt.data-00000-of-00001', 'model.ckpt.index', 'model.ckpt.meta', 'vocab.bpe']:
r = requests.get("https://storage.googleapis.com/gpt-2/" + subdir + "/" + filename, stream=True)
with open(os.path.join(subdir, filename), 'wb') as f:
file_size = int(r.headers["content-length"])
chunk_size = 1000
with tqdm(ncols=100, desc="Fetching " + filename, total=file_size, unit_scale=True) as pbar:
# 1k for chunk_size, since Ethernet packet size is around 1500 bytes
for chunk in r.iter_content(chunk_size=chunk_size):
f.write(chunk)
pbar.update(chunk_size)
generator/tf/models/117M/.gitignore
-1
View File
@@ -1 +0,0 @@
model.ckpt.data-00000-of-00001
generator/tf/models/117M/checkpoint
-2
View File
@@ -1,2 +0,0 @@
model_checkpoint_path: "model.ckpt"
all_model_checkpoint_paths: "model.ckpt"
generator/tf/models/117M/encoder.json
-1
View File
File diff suppressed because one or more lines are too long
generator/tf/models/117M/hparams.json
-7
View File
@@ -1,7 +0,0 @@
{
"n_vocab": 50257,
"n_ctx": 1024,
"n_embd": 768,
"n_head": 12,
"n_layer": 12
}
generator/tf/models/117M/vocab.bpe
-50001
View File
File diff suppressed because it is too large Load Diff
generator/tf/models/774M/.gitignore
-2
View File
@@ -1,2 +0,0 @@
model.ckpt.data-00000-of-00001
generator/tf/models/774M/checkpoint
-2
View File
@@ -1,2 +0,0 @@
model_checkpoint_path: "model.ckpt"
all_model_checkpoint_paths: "model.ckpt"
generator/tf/models/774M/encoder.json
-1
View File
File diff suppressed because one or more lines are too long
generator/tf/models/774M/hparams.json
-7
View File
@@ -1,7 +0,0 @@
{
"n_vocab": 50257,
"n_ctx": 1024,
"n_embd": 1280,
"n_head": 20,
"n_layer": 36
}
generator/tf/models/774M/vocab.bpe
-50001
View File
File diff suppressed because it is too large Load Diff
generator/tf/src/__init__.py
View File
generator/tf/src/encoder.py
-117
View File
@@ -1,117 +0,0 @@
"""Byte pair encoding utilities"""
import os
import json
import regex as re
from functools import lru_cache
@lru_cache()
def bytes_to_unicode():
"""
Returns list of utf-8 byte and a corresponding list of unicode strings.
The reversible bpe codes work on unicode strings.
This means you need a large # of unicode characters in your vocab if you want to avoid UNKs.
When you're at something like a 10B token dataset you end up needing around 5K for decent coverage.
This is a signficant percentage of your normal, say, 32K bpe vocab.
To avoid that, we want lookup tables between utf-8 bytes and unicode strings.
And avoids mapping to whitespace/control characters the bpe code barfs on.
"""
bs = list(range(ord("!"), ord("~")+1))+list(range(ord("¡"), ord("¬")+1))+list(range(ord("®"), ord("ÿ")+1))
cs = bs[:]
n = 0
for b in range(2**8):
if b not in bs:
bs.append(b)
cs.append(2**8+n)
n += 1
cs = [chr(n) for n in cs]
return dict(zip(bs, cs))
def get_pairs(word):
"""Return set of symbol pairs in a word.
Word is represented as tuple of symbols (symbols being variable-length strings).
"""
pairs = set()
prev_char = word[0]
for char in word[1:]:
pairs.add((prev_char, char))
prev_char = char
return pairs
class Encoder:
def __init__(self, encoder, bpe_merges, errors='replace'):
self.encoder = encoder
self.decoder = {v:k for k,v in self.encoder.items()}
self.errors = errors # how to handle errors in decoding
self.byte_encoder = bytes_to_unicode()
self.byte_decoder = {v:k for k, v in self.byte_encoder.items()}
self.bpe_ranks = dict(zip(bpe_merges, range(len(bpe_merges))))
self.cache = {}
# Should haved added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
self.pat = re.compile(r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+""")
def bpe(self, token):
if token in self.cache:
return self.cache[token]
word = tuple(token)
pairs = get_pairs(word)
if not pairs:
return token
while True:
bigram = min(pairs, key = lambda pair: self.bpe_ranks.get(pair, float('inf')))
if bigram not in self.bpe_ranks:
break
first, second = bigram
new_word = []
i = 0
while i < len(word):
try:
j = word.index(first, i)
new_word.extend(word[i:j])
i = j
except:
new_word.extend(word[i:])
break
if word[i] == first and i < len(word)-1 and word[i+1] == second:
new_word.append(first+second)
i += 2
else:
new_word.append(word[i])
i += 1
new_word = tuple(new_word)
word = new_word
if len(word) == 1:
break
else:
pairs = get_pairs(word)
word = ' '.join(word)
self.cache[token] = word
return word
def encode(self, text):
bpe_tokens = []
for token in re.findall(self.pat, text):
token = ''.join(self.byte_encoder[b] for b in token.encode('utf-8'))
bpe_tokens.extend(self.encoder[bpe_token] for bpe_token in self.bpe(token).split(' '))
return bpe_tokens
def decode(self, tokens):
text = ''.join([self.decoder[token] for token in tokens])
text = bytearray([self.byte_decoder[c] for c in text]).decode('utf-8', errors=self.errors)
return text
def get_encoder(model_path):
with open(os.path.join(model_path, 'encoder.json'), 'r') as f:
encoder = json.load(f)
with open(os.path.join(model_path, 'vocab.bpe'), 'r', encoding="utf-8") as f:
bpe_data = f.read()
bpe_merges = [tuple(merge_str.split()) for merge_str in bpe_data.split('\n')[1:-1]]
return Encoder(
encoder=encoder,
bpe_merges=bpe_merges,
)
generator/tf/src/model.py
-176
View File
@@ -1,176 +0,0 @@
import numpy as np
import tensorflow as tf
from tensorflow.contrib.training import HParams
def default_hparams():
return HParams(
n_vocab=0,
n_ctx=1024,
n_embd=768,
n_head=12,
n_layer=12,
)
def shape_list(x):
"""Deal with dynamic shape in tensorflow cleanly."""
static = x.shape.as_list()
dynamic = tf.shape(x)
return [dynamic[i] if s is None else s for i, s in enumerate(static)]
def softmax(x, axis=-1):
x = x - tf.reduce_max(x, axis=axis, keepdims=True)
ex = tf.exp(x)
return ex / tf.reduce_sum(ex, axis=axis, keepdims=True)
def gelu(x):
return 0.5*x*(1+tf.tanh(np.sqrt(2/np.pi)*(x+0.044715*tf.pow(x, 3))))
def norm(x, scope, *, axis=-1, epsilon=1e-5):
"""Normalize to mean = 0, std = 1, then do a diagonal affine transform."""
with tf.variable_scope(scope):
n_state = x.shape[-1].value
g = tf.get_variable('g', [n_state], initializer=tf.constant_initializer(1))
b = tf.get_variable('b', [n_state], initializer=tf.constant_initializer(0))
u = tf.reduce_mean(x, axis=axis, keepdims=True)
s = tf.reduce_mean(tf.square(x-u), axis=axis, keepdims=True)
x = (x - u) * tf.rsqrt(s + epsilon)
x = x*g + b
return x
def split_states(x, n):
"""Reshape the last dimension of x into [n, x.shape[-1]/n]."""
*start, m = shape_list(x)
return tf.reshape(x, start + [n, m//n])
def merge_states(x):
"""Smash the last two dimensions of x into a single dimension."""
*start, a, b = shape_list(x)
return tf.reshape(x, start + [a*b])
def conv1d(x, scope, nf, *, w_init_stdev=0.02):
with tf.variable_scope(scope):
*start, nx = shape_list(x)
w = tf.get_variable('w', [1, nx, nf], initializer=tf.random_normal_initializer(stddev=w_init_stdev))
b = tf.get_variable('b', [nf], initializer=tf.constant_initializer(0))
c = tf.reshape(tf.matmul(tf.reshape(x, [-1, nx]), tf.reshape(w, [-1, nf]))+b, start+[nf])
return c
def attention_mask(nd, ns, *, dtype):
"""1's in the lower triangle, counting from the lower right corner.
Same as tf.matrix_band_part(tf.ones([nd, ns]), -1, ns-nd), but doesn't produce garbage on TPUs.
"""
i = tf.range(nd)[:,None]
j = tf.range(ns)
m = i >= j - ns + nd
return tf.cast(m, dtype)
def attn(x, scope, n_state, *, past, hparams):
assert x.shape.ndims == 3 # Should be [batch, sequence, features]
assert n_state % hparams.n_head == 0
if past is not None:
assert past.shape.ndims == 5 # Should be [batch, 2, heads, sequence, features], where 2 is [k, v]
def split_heads(x):
# From [batch, sequence, features] to [batch, heads, sequence, features]
return tf.transpose(split_states(x, hparams.n_head), [0, 2, 1, 3])
def merge_heads(x):
# Reverse of split_heads
return merge_states(tf.transpose(x, [0, 2, 1, 3]))
def mask_attn_weights(w):
# w has shape [batch, heads, dst_sequence, src_sequence], where information flows from src to dst.
_, _, nd, ns = shape_list(w)
b = attention_mask(nd, ns, dtype=w.dtype)
b = tf.reshape(b, [1, 1, nd, ns])
w = w*b - tf.cast(1e10, w.dtype)*(1-b)
return w
def multihead_attn(q, k, v):
# q, k, v have shape [batch, heads, sequence, features]
w = tf.matmul(q, k, transpose_b=True)
w = w * tf.rsqrt(tf.cast(v.shape[-1].value, w.dtype))
w = mask_attn_weights(w)
w = softmax(w)
a = tf.matmul(w, v)
return a
with tf.variable_scope(scope):
c = conv1d(x, 'c_attn', n_state*3)
q, k, v = map(split_heads, tf.split(c, 3, axis=2))
present = tf.stack([k, v], axis=1)
if past is not None:
pk, pv = tf.unstack(past, axis=1)
k = tf.concat([pk, k], axis=-2)
v = tf.concat([pv, v], axis=-2)
a = multihead_attn(q, k, v)
a = merge_heads(a)
a = conv1d(a, 'c_proj', n_state)
return a, present
def mlp(x, scope, n_state, *, hparams):
with tf.variable_scope(scope):
nx = x.shape[-1].value
h = gelu(conv1d(x, 'c_fc', n_state))
h2 = conv1d(h, 'c_proj', nx)
return h2
def block(x, scope, *, past, hparams):
with tf.variable_scope(scope):
nx = x.shape[-1].value
a, present = attn(norm(x, 'ln_1'), 'attn', nx, past=past, hparams=hparams)
x = x + a
m = mlp(norm(x, 'ln_2'), 'mlp', nx*4, hparams=hparams)
x = x + m
return x, present
def past_shape(*, hparams, batch_size=None, sequence=None):
return [batch_size, hparams.n_layer, 2, hparams.n_head, sequence, hparams.n_embd // hparams.n_head]
def expand_tile(value, size):
"""Add a new axis of given size."""
value = tf.convert_to_tensor(value, name='value')
ndims = value.shape.ndims
return tf.tile(tf.expand_dims(value, axis=0), [size] + [1]*ndims)
def positions_for(tokens, past_length):
batch_size = tf.shape(tokens)[0]
nsteps = tf.shape(tokens)[1]
return expand_tile(past_length + tf.range(nsteps), batch_size)
def model(hparams, X, past=None, scope='model', reuse=False):
with tf.variable_scope(scope, reuse=reuse):
results = {}
batch, sequence = shape_list(X)
wpe = tf.get_variable('wpe', [hparams.n_ctx, hparams.n_embd],
initializer=tf.random_normal_initializer(stddev=0.01))
wte = tf.get_variable('wte', [hparams.n_vocab, hparams.n_embd],
initializer=tf.random_normal_initializer(stddev=0.02))
past_length = 0 if past is None else tf.shape(past)[-2]
h = tf.gather(wte, X) + tf.gather(wpe, positions_for(X, past_length))
# Transformer
presents = []
pasts = tf.unstack(past, axis=1) if past is not None else [None] * hparams.n_layer
assert len(pasts) == hparams.n_layer
for layer, past in enumerate(pasts):
h, present = block(h, 'h%d' % layer, past=past, hparams=hparams)
presents.append(present)
results['present'] = tf.stack(presents, axis=1)
h = norm(h, 'ln_f')
# Language model loss. Do tokens <n predict token n?
h_flat = tf.reshape(h, [batch*sequence, hparams.n_embd])
logits = tf.matmul(h_flat, wte, transpose_b=True)
logits = tf.reshape(logits, [batch, sequence, hparams.n_vocab])
results['logits'] = logits
return results
generator/tf/src/sample.py
-79
View File
@@ -1,79 +0,0 @@
import tensorflow as tf
from src.model import *
def top_k_logits(logits, k):
if k == 0:
# no truncation
return logits
def _top_k():
values, _ = tf.nn.top_k(logits, k=k)
min_values = values[:, -1, tf.newaxis]
return tf.where(
logits < min_values,
tf.ones_like(logits, dtype=logits.dtype) * -1e10,
logits,
)
return tf.cond(
tf.equal(k, 0),
lambda: logits,
lambda: _top_k(),
)
def sample_sequence(*, hparams, length, start_token=None, batch_size=None, context=None, temperature=1, top_k=0):
if start_token is None:
assert context is not None, 'Specify exactly one of start_token and context!'
else:
assert context is None, 'Specify exactly one of start_token and context!'
context = tf.fill([batch_size, 1], start_token)
def step(hparams, tokens, past=None):
lm_output = model(hparams=hparams, X=tokens, past=past, reuse=tf.AUTO_REUSE)
logits = lm_output['logits'][:, :, :hparams.n_vocab]
presents = lm_output['present']
presents.set_shape(past_shape(hparams=hparams, batch_size=batch_size))
return {
'logits': logits,
'presents': presents,
}
with tf.name_scope('sample_sequence'):
# Don't feed the last context token -- leave that to the loop below
# TODO: Would be slightly faster if we called step on the entire context,
# rather than leaving the last token transformer calculation to the while loop.
context_output = step(hparams, context[:, :-1])
def body(past, prev, output):
next_outputs = step(hparams, prev[:, tf.newaxis], past=past)
logits = next_outputs['logits'][:, -1, :] / tf.to_float(temperature)
logits = top_k_logits(logits, k=top_k)
samples = tf.multinomial(logits, num_samples=1, output_dtype=tf.int32)
return [
tf.concat([past, next_outputs['presents']], axis=-2),
tf.squeeze(samples, axis=[1]),
tf.concat([output, samples], axis=1),
]
def cond(*args):
return True
_, _, tokens = tf.while_loop(
cond=cond, body=body,
maximum_iterations=length,
loop_vars=[
context_output['presents'],
context[:, -1],
context,
],
shape_invariants=[
tf.TensorShape(past_shape(hparams=hparams, batch_size=batch_size)),
tf.TensorShape([batch_size]),
tf.TensorShape([batch_size, None]),
],
back_prop=False, name="EndWhile"
)
return tokens
generator/tf/tf_generator.py
-142
View File
@@ -1,142 +0,0 @@
import json
import os
import numpy as np
import tensorflow as tf
from src.model import *
from tensorflow.contrib import predictor
from src.sample import *
from src.encoder import *
import pdb
pos_action_starts = ["You attack", "You tell", "You use", "You go"]
class TFGenerator():
def __init__(self, sess, length=75, temperature=0.9, top_k=40):
seed = None
batch_size=1
model_path='gpt2/models/117M'
self.sess = sess
self.enc = encoder.get_encoder(model_path)
hparams = model.default_hparams()
with open(os.path.join(model_path, 'hparams.json')) as f:
hparams.override_from_dict(json.load(f))
pdb.set_trace()
self.context = tf.placeholder(tf.int32, [batch_size, None])
np.random.seed(seed)
tf.set_random_seed(seed)
self.output = sample.sample_sequence(
hparams=hparams, length=length,
context=self.context,
batch_size=batch_size,
)
saver = tf.train.Saver()
ckpt = tf.train.latest_checkpoint(model_path)
saver.restore(self.sess, ckpt)
def generate(self, prompt, options={}):
context_tokens = self.enc.encode(prompt)
out = self.sess.run(self.output, feed_dict={
self.context: [context_tokens for _ in range(1)]
})[:, len(context_tokens):]
text = self.enc.decode(out[0])
return text
def save_model():
length=75
temperature=0.9
top_k=40
os.environ['CUDA_VISIBLE_DEVICES'] = '-1'
with tf.Session() as sess:
seed = None
batch_size=None
model_path='models/774M'
hparams = default_hparams()
with open(os.path.join(model_path, 'hparams.json')) as f:
hparams.override_from_dict(json.load(f))
context = tf.placeholder(tf.int32, [batch_size, None])
np.random.seed(seed)
tf.set_random_seed(seed)
output = sample_sequence(
hparams=hparams, length=length,
context=context,
batch_size=batch_size,
)
print("***********************",type(output))
saver = tf.train.Saver()
ckpt = tf.train.latest_checkpoint(model_path)
saver.restore(sess, ckpt)
tf.saved_model.simple_save(sess, "./saved_model", inputs={"context": context}, outputs={"output": output})
def load_model():
fraction = 0.6
config = config = generate_gpu_config(fraction)
path_to_graph = "./saved"
# tf.saved_model.loader.load(
# session,
# [tf.saved_model.tag_constants.SERVING],
# path_to_graph)
# output = session.graph.get_tensor_by_name('output:0')
# context = session.graph.get_tensor_by_name('context:0')
model_path = 'gpt2/models/117M'
enc = encoder.get_encoder(model_path)
predict_fn = predictor.from_saved_model(path_to_graph, config=config)
context_tokens = [enc.encode("hello")]
predictions = predict_fn({"context": context_tokens})
output = enc.decode(predictions["output"][0])
print(output)
return (output, session)
if __name__ == '__main__':
save_model()