mirror of
https://github.com/wassname/Clover-Edition.git
synced 2026-09-17 11:50:25 +08:00
readd web files
This commit is contained in:
Nick Walton
@@ -0,0 +1,19 @@
|
||||
runtime: python37
|
||||
entrypoint: gunicorn --timeout=300 --graceful-timeout=300 -b :$PORT main:app
|
||||
instance_class: F1
|
||||
|
||||
automatic_scaling:
|
||||
min_instances: 1
|
||||
|
||||
handlers:
|
||||
# This configures Google App Engine to serve the files in the app's static
|
||||
# directory.
|
||||
- url: /static
|
||||
static_dir: static
|
||||
|
||||
# This handler routes all requests not caught above to your main app. It is
|
||||
# required when static routes are defined, but can be omitted (along with
|
||||
# the entire handlers section) when there are no static files defined.
|
||||
- url: /.*
|
||||
script: auto
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
steps:
|
||||
- name: "gcr.io/cloud-builders/gcloud"
|
||||
args: ["app", "deploy", "--no-promote"]
|
||||
timeout: "1600s"
|
||||
+4
-20
@@ -2,26 +2,10 @@ from story.story_manager import *
|
||||
# from generator.web.web_generator import *
|
||||
# from generator.ctrl.ctrl_generator import *
|
||||
from generator.gpt2.gpt2_generator import *
|
||||
import textwrap
|
||||
|
||||
CRED_FILE = "./AI-Adventure-2bb65e3a4e2f.json"
|
||||
|
||||
|
||||
def console_print(text, width=75):
|
||||
last_newline = 0
|
||||
i = 0
|
||||
while i < len(text):
|
||||
if text[i] == "\n":
|
||||
last_newline = 0
|
||||
elif last_newline > width:
|
||||
text = text[:i] + "\n" + text[i:]
|
||||
else:
|
||||
last_newline += 1
|
||||
i += 1
|
||||
|
||||
|
||||
|
||||
|
||||
def play_unconstrained():
|
||||
generator = GPT2Generator()
|
||||
prompt = get_story_start("knight")
|
||||
@@ -30,8 +14,8 @@ def play_unconstrained():
|
||||
story_manager.start_new_story(prompt, context=context)
|
||||
|
||||
print("\n")
|
||||
console_print(context)
|
||||
console_print(str(story_manager.story))
|
||||
print(context)
|
||||
print(str(story_manager.story))
|
||||
while True:
|
||||
action = input("> ")
|
||||
|
||||
@@ -46,10 +30,10 @@ def play_unconstrained():
|
||||
|
||||
result = story_manager.act(action)
|
||||
if player_died(result):
|
||||
console_print(result + "\nGAME OVER")
|
||||
print(result + "\nGAME OVER")
|
||||
break
|
||||
else:
|
||||
console_print(result)
|
||||
print(result)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
fastBPE
|
||||
model
|
||||
@@ -0,0 +1,12 @@
|
||||
Copyright (c) 2019, Salesforce.com, Inc.
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
|
||||
|
||||
* Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
|
||||
|
||||
* Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
+200000
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,55 @@
|
||||
0.000296793 Pregnancy
|
||||
0.000127197 Christianity
|
||||
0.003084531 Explain
|
||||
0.000180196 Fitness
|
||||
6.88985E-05 Saving
|
||||
0.000217295 Ask
|
||||
8.47981E-05 Ass
|
||||
0.000143097 Joke
|
||||
0.000196096 Questions
|
||||
0.000127197 Thoughts
|
||||
0.000169596 Retail
|
||||
0.000270294 Feminism
|
||||
0.000111298 Writing
|
||||
0.000402791 Atheism
|
||||
1.05998E-06 Netflix
|
||||
0.000365692 Computing
|
||||
0.000132497 Opinion
|
||||
0.000169596 Alone
|
||||
0.000323293 Funny
|
||||
0.000249094 Gaming
|
||||
0.000402791 Human
|
||||
0.000132497 India
|
||||
2.11995E-08 Joker
|
||||
0.000201395 Diet
|
||||
0.000238495 Legal
|
||||
6.35986E-06 Norman
|
||||
3.60392E-07 Tip
|
||||
0.000302093 Weight
|
||||
0.000132497 Movies
|
||||
0.000111298 Running
|
||||
7.41983E-05 Science
|
||||
0.00135147 Horror
|
||||
0.000291493 Confession
|
||||
0.000190796 Finance
|
||||
0.000413391 Politics
|
||||
7.41983E-05 Scary
|
||||
0.000206695 Support
|
||||
6.35986E-05 Technologies
|
||||
0.000243795 Teenage
|
||||
0.000217295 Event
|
||||
0.000206695 Learned
|
||||
0.000121897 Notion
|
||||
0.0847981 Wikipedia
|
||||
0.095927851 Books
|
||||
0.001176574 Extract
|
||||
0.000127197 Confessions
|
||||
0.000227895 Conspiracy
|
||||
0.365691808 Links
|
||||
0.000423991 Narcissus
|
||||
0.000280894 Relationship
|
||||
0.000922179 Relationships
|
||||
0.153696557 Reviews
|
||||
0.043877717 News
|
||||
0.129847091 Translation
|
||||
0.111297507 multilingual
|
||||
@@ -0,0 +1,327 @@
|
||||
import tensorflow as tf
|
||||
import numpy as np
|
||||
|
||||
tf.enable_eager_execution()
|
||||
import generator.ctrl.model.transformer as transformer
|
||||
import re
|
||||
from collections import Counter
|
||||
from tensorflow.python import debug as tf_debug
|
||||
from tensorflow.python.ops import math_ops
|
||||
from tensorflow.python.ops import embedding_ops
|
||||
import fastBPE
|
||||
from story.utils import *
|
||||
import warnings
|
||||
warnings.filterwarnings("ignore")
|
||||
|
||||
# the loss function is a simple categorical crossentropy between the logits and the labels
|
||||
def loss(labels, logits):
|
||||
return tf.keras.losses.sparse_categorical_crossentropy(labels, logits, from_logits=True)
|
||||
|
||||
|
||||
class CTRLGenerator():
|
||||
|
||||
def __init__(self, control_code="Apocalypse ", generate_num=40, temperature=0.4, topk=40, nucleus_prob=0):
|
||||
|
||||
self.generate_num=generate_num
|
||||
model_dir = "generator/ctrl/model/aidungeon2model/"
|
||||
self.control_code = control_code
|
||||
vocab_file = 'generator/ctrl/model/vocab'
|
||||
code_file = 'generator/ctrl/model/codes'
|
||||
|
||||
self.max_new_lines = 5
|
||||
|
||||
# load the vocabulary from file
|
||||
vocab = open(vocab_file, encoding='utf-8').read().split('\n')
|
||||
vocab = list(map(lambda x: x.split(' ')[0], vocab)) + ['<unk>'] + ['\n']
|
||||
print('{} unique words'.format(len(vocab)))
|
||||
|
||||
# length of the vocabulary
|
||||
vocab_size = len(vocab)
|
||||
|
||||
# define the numericalization map
|
||||
# idx2word maps the numericalized ID to the word
|
||||
# word2idx maps the word to the numericalized ID
|
||||
self.word2idx = {u: i for i, u in enumerate(vocab)}
|
||||
self.idx2word = np.array(vocab)
|
||||
|
||||
# sequence length to use for the transformer
|
||||
# the model is trained with a self.seq_length of 512
|
||||
# so, any value <= 512 should work
|
||||
self.seq_length = min(self.generate_num, 256)
|
||||
|
||||
# the dimension of the transformer
|
||||
embedding_dim = 1280
|
||||
|
||||
# input for the keras model
|
||||
tokens = tf.keras.layers.Input(shape=(self.seq_length,), dtype='int32')
|
||||
|
||||
# Now, we begin defining the model
|
||||
# we defer the transformer definition to transformer.py
|
||||
# here, we only define the tied softmax layer
|
||||
# this layer ties the softmax weights to the input embeddings
|
||||
class TiedEmbeddingSoftmax(tf.keras.layers.Layer):
|
||||
|
||||
def __init__(self, vocab_size=vocab_size, embedding_size=embedding_dim, **kwargs):
|
||||
super(TiedEmbeddingSoftmax, self).__init__()
|
||||
self.w = self.add_weight(name='w', shape=(vocab_size, embedding_size),
|
||||
initializer='random_normal',
|
||||
trainable=True)
|
||||
self.b = self.add_weight(name='b', shape=(vocab_size,),
|
||||
initializer='zeros',
|
||||
trainable=True)
|
||||
|
||||
def call(self, inputs, embed=True):
|
||||
if embed:
|
||||
dtype = tf.keras.backend.dtype(inputs)
|
||||
if dtype != 'int32' and dtype != 'int64':
|
||||
inputs = math_ops.cast(inputs, 'int32')
|
||||
return embedding_ops.embedding_lookup(self.w, inputs)
|
||||
else:
|
||||
return tf.tensordot(inputs, tf.transpose(self.w), 1) + self.b
|
||||
|
||||
# instantiates a tied softmax class
|
||||
tied_embedding_softmax = TiedEmbeddingSoftmax()
|
||||
|
||||
# embedded tokens, before passing it to the transformer
|
||||
embedded = tied_embedding_softmax(tokens, embed=True)
|
||||
|
||||
# the activations after passing it from the transformer
|
||||
# for some odd reason, TPUs don't play well with specifying the arguments of the Encoder() function
|
||||
# so you have to leave them at their defaults
|
||||
transformed = transformer.Encoder()(embedded, training=False)
|
||||
|
||||
# pass the activations from our tiedsoftmax class
|
||||
# this time with embed=False denoting that we are doing the softmax operation
|
||||
# and not a lookup
|
||||
logits = tied_embedding_softmax(transformed, embed=False)
|
||||
|
||||
# finally, define the Keras model with inputs as tokens and outputs as the logits we just computed
|
||||
model = tf.keras.Model(inputs=tokens, outputs=logits)
|
||||
|
||||
# the optimizer is not used since this code only supports inference
|
||||
# however, to compile the model, we still define it
|
||||
optimizer = tf.contrib.tpu.CrossShardOptimizer(
|
||||
tf.contrib.estimator.clip_gradients_by_norm(
|
||||
tf.train.AdagradOptimizer(learning_rate=1e-2), 0.25)
|
||||
)
|
||||
|
||||
# compile the model with the optimizer and loss
|
||||
model.compile(optimizer=optimizer, loss=loss)
|
||||
print(model.summary())
|
||||
|
||||
# IMPORTANT
|
||||
# this is where the saved model is presented to the code
|
||||
# the model directory should have the model checkpoint and
|
||||
# a checkpoint file
|
||||
run_config = tf.contrib.tpu.RunConfig(
|
||||
model_dir=model_dir)
|
||||
|
||||
# this converts the Keras model to a TensorFlow estimator
|
||||
# this step is critical
|
||||
# remember to patch the TF 1.14 file before running the code, else you're going to see errors here
|
||||
estimator_model = tf.keras.estimator.model_to_estimator(keras_model=model, config=run_config)
|
||||
|
||||
# we now create a serving function from this estimator
|
||||
# this enables us to load the model once and easily query it multiple times
|
||||
def serving_input_fn():
|
||||
inputs = {'input_1': tf.placeholder(tf.int32, [1, self.seq_length])}
|
||||
return tf.estimator.export.ServingInputReceiver(inputs, inputs)
|
||||
|
||||
self.predict_fn = tf.contrib.predictor.from_estimator(estimator_model, serving_input_fn)
|
||||
|
||||
# almost there, we now take the user prompt and tokenize with BPE
|
||||
# load BPE codes
|
||||
self.bpe = fastBPE.fastBPE(code_file, vocab_file)
|
||||
|
||||
self.temperature=temperature
|
||||
self.nucleusprob = nucleus_prob
|
||||
self.penalty = 1.2
|
||||
self.topk=topk
|
||||
|
||||
def configure_verb_probs(self, probabilities, options):
|
||||
|
||||
# Make sure only a possible verb is chosen.
|
||||
for word in get_possible_verbs():
|
||||
probabilities[self.word2idx[word]] += 100
|
||||
|
||||
# Disallow used verbs
|
||||
if "used_verbs" in options:
|
||||
for verb in options["used_verbs"]:
|
||||
if verb in self.word2idx:
|
||||
probabilities[self.word2idx[verb]] = -1e8
|
||||
|
||||
return probabilities
|
||||
|
||||
def prompt_replace(self, prompt):
|
||||
# print("\n\nBEFORE PROMPT_REPLACE:")
|
||||
# print(repr(prompt))
|
||||
if prompt[-1] != " ":
|
||||
prompt = prompt + " "
|
||||
|
||||
prompt = second_to_first_person(prompt)
|
||||
|
||||
prompt = self.control_code + prompt
|
||||
# print("\n\nAFTER PROMPT_REPLACE")
|
||||
# print(repr(prompt))
|
||||
return prompt
|
||||
|
||||
def result_replace(self, result):
|
||||
# print("\n\nBEFORE RESULT_REPLACE:")
|
||||
# print(repr(result))
|
||||
|
||||
result = cut_trailing_sentence(result)
|
||||
first_letter_capitalized = result[0].isupper()
|
||||
result = result.replace('."', '".')
|
||||
result = result.replace("#", "")
|
||||
result = result.replace("*", "")
|
||||
result = first_to_second_person(result)
|
||||
result = remove_profanity(result)
|
||||
|
||||
if not first_letter_capitalized:
|
||||
result = result[0].lower() + result[1:]
|
||||
|
||||
#
|
||||
# print("\n\nAFTER RESULT_REPLACE:")
|
||||
# print(repr(result))
|
||||
|
||||
return result
|
||||
|
||||
def generate_next_token(self, token, tokens_generated, options, num_new_lines, token_num, first_token=False, forbid_newline=False):
|
||||
|
||||
# get the logits from the prediction function
|
||||
# the logic here is a bit convoluted because we are allowing generation past 512 tokens
|
||||
# this is done by sliding the window over (past 512 tokens) and continuing prediction
|
||||
# I'm sure this can be simplified (TODO)
|
||||
if token <= self.seq_length:
|
||||
prompt_logits = self.predict_fn({'input_1': tokens_generated[:, :self.seq_length]})[
|
||||
'tied_embedding_softmax'].squeeze() / (self.temperature if self.temperature > 0 else 1.)
|
||||
_token = token if token < self.seq_length else -1
|
||||
else:
|
||||
_token = -1
|
||||
end = token + 1
|
||||
start = token - self.seq_length + 2
|
||||
prompt_logits = \
|
||||
self.predict_fn({'input_1': np.hstack((tokens_generated[:, 0:1], tokens_generated[:, start:end]))})[
|
||||
'tied_embedding_softmax'].squeeze() / (self.temperature if self.temperature > 0 else 1.)
|
||||
|
||||
# if penalty (for repetition) is non-zero,
|
||||
# discount the logits from already generated tokens
|
||||
if self.penalty > 0:
|
||||
penalized_so_far = set()
|
||||
for _ in range(token + 1):
|
||||
generated_token = tokens_generated[0][_]
|
||||
if generated_token not in penalized_so_far:
|
||||
penalized_so_far.add(generated_token)
|
||||
prompt_logits[_token][generated_token] /= self.penalty
|
||||
|
||||
# disallow some tokens
|
||||
forbidden_tokens = ['<unk>', 'Sco@@', "&@@", "1]@@", "2]@@", "3]@@", "4]@@", "https://www.@@", "[@@", ":@@",
|
||||
"Edit", "&@@", "2:","1:", ":", "Edit@@", "EDI@@", "EDIT@@", "edit", "TL@@", "tl@@", ";@@",
|
||||
'**', "http://@@", "Redd@@", "UP@@", "mom", "Up@@", "Me:", "Update", "mom@@", "Part",
|
||||
"http://www.@@", "edit@@", "*@@", "Writing", "Text@@", "\\@@", "<br>@@", "<div", "|@@", '...',
|
||||
'..','…', 'https://@@', '...@@', "http://gutenberg@@"]
|
||||
|
||||
#encourage_tokens = ["zombie", "radiation", "fallout", "undead", "corpse", "vampire", "virus", "plague"]
|
||||
encourage_tokens = []
|
||||
for encourage_token in encourage_tokens:
|
||||
prompt_logits[_token][self.word2idx[encourage_token]] *= 1.2
|
||||
|
||||
for forbidden_token in forbidden_tokens:
|
||||
prompt_logits[_token][self.word2idx[forbidden_token]] = -1e8
|
||||
|
||||
last_ind = tokens_generated[0][token]
|
||||
|
||||
if forbid_newline:
|
||||
prompt_logits[_token][self.word2idx['\n']] = -1e8
|
||||
else:
|
||||
prompt_logits[_token][self.word2idx['\n']] *= 1.0
|
||||
|
||||
# Set whitelist
|
||||
if "word_whitelist" in options and token_num in options["word_whitelist"].keys():
|
||||
for word in options["word_whitelist"][token_num]:
|
||||
prompt_logits[_token][self.word2idx[word]] += 100
|
||||
|
||||
# Set blacklist, overwrites whitelist
|
||||
if "word_blacklist" in options and token_num in options["word_blacklist"].keys():
|
||||
for word in options["word_blacklist"][token_num]:
|
||||
prompt_logits[_token][self.word2idx[word]] = -1e8
|
||||
|
||||
# compute probabilities from logits
|
||||
prompt_probs = np.exp(prompt_logits[_token])
|
||||
prompt_probs = prompt_probs / sum(prompt_probs)
|
||||
pruned_list = np.argsort(prompt_probs)[::-1]
|
||||
# if you are using nucleus prob, then compute the nucleus probability size
|
||||
if self.nucleusprob > 0.:
|
||||
minimum_topk = 1
|
||||
nucleus = max(np.where(np.cumsum(np.sort(prompt_probs)[::-1]) > self.nucleusprob)[0][0], minimum_topk)
|
||||
elif self.topk > 0:
|
||||
nucleus = self.topk
|
||||
else:
|
||||
nucleus = len(pruned_list)
|
||||
|
||||
pruned_list = pruned_list[:nucleus]
|
||||
|
||||
# if temperature is 0
|
||||
# just pick the first (most probable) token
|
||||
if self.temperature == 0:
|
||||
idx = pruned_list[0]
|
||||
else:
|
||||
# else,
|
||||
# sample from the pruned_list with the logits
|
||||
chosen_idx = int(
|
||||
tf.random.categorical(np.expand_dims(prompt_logits[_token][pruned_list], 0), num_samples=1).numpy())
|
||||
idx = pruned_list[chosen_idx]
|
||||
|
||||
return idx
|
||||
|
||||
def generate(self, prompt, options=None):
|
||||
prompt = self.prompt_replace(prompt)
|
||||
|
||||
debug_print = True
|
||||
|
||||
if debug_print:
|
||||
print("\n\n*****DEBUG*****")
|
||||
print("Prompt is:")
|
||||
print(prompt + "\n\n")
|
||||
|
||||
if options is None: options = dict()
|
||||
if "used_verbs" not in options:
|
||||
options["used_verbs"] = set()
|
||||
|
||||
first_token = True
|
||||
|
||||
# tokenize provided prompt
|
||||
split_prompt = self.bpe.apply([prompt])[0].split()
|
||||
text = [self.word2idx[i] for i in split_prompt]
|
||||
total_text_len = len(text) + self.generate_num
|
||||
|
||||
# pad with 0s and create a mini-batch of 2 (arbitrary, for ease of code)
|
||||
padded_text = text + [0] * (total_text_len - len(text))
|
||||
tokens_generated = np.tile(padded_text, (1, 1))
|
||||
result = ""
|
||||
|
||||
token_num = 0
|
||||
num_new_lines = 0
|
||||
for token in range(len(text) - 1, total_text_len - 1):
|
||||
idx = self.generate_next_token(token, tokens_generated, options, num_new_lines, token_num, first_token=first_token, forbid_newline=False)
|
||||
is_nothing = len(cut_trailing_sentence(result)) == 0 or len(cut_trailing_quotes(result)) == 0
|
||||
if self.idx2word[idx] == '\n' and token_num > 7 and not is_nothing:
|
||||
return self.result_replace(result)
|
||||
elif self.idx2word[idx] == '\n':
|
||||
idx = self.generate_next_token(token, tokens_generated, options, num_new_lines, token_num,
|
||||
first_token=first_token, forbid_newline=True)
|
||||
# assign the token for generation
|
||||
|
||||
tokens_generated[0][token + 1] = idx
|
||||
if debug_print:
|
||||
print(repr(self.idx2word[idx]), end="_")
|
||||
|
||||
tokens_generated_so_far = ' '.join([self.idx2word[c] for c in tokens_generated[0][len(text):token+2]])
|
||||
tokens_generated_so_far = re.sub('(@@ )', '', string=tokens_generated_so_far)
|
||||
tokens_generated_so_far = re.sub('(@@ ?$)', '', string=tokens_generated_so_far)
|
||||
result = tokens_generated_so_far
|
||||
|
||||
token_num += 1
|
||||
if debug_print:
|
||||
print("\n****END DEBUG*****\n")
|
||||
return self.result_replace(result)
|
||||
Executable
+4
@@ -0,0 +1,4 @@
|
||||
|
||||
URL="gs://aidungeon2model"
|
||||
|
||||
gsutil -m cp -r "$URL" model
|
||||
@@ -0,0 +1,24 @@
|
||||
47c47
|
||||
<
|
||||
---
|
||||
> import tensorflow as tf
|
||||
228c228
|
||||
< def _create_keras_model_fn(keras_model, custom_objects=None):
|
||||
---
|
||||
> def _create_keras_model_fn(keras_model, params=None, custom_objects=None):
|
||||
239c239
|
||||
< def model_fn(features, labels, mode):
|
||||
---
|
||||
> def model_fn(features, labels, mode, params=None):
|
||||
448c448
|
||||
< if keras_model._is_graph_network:
|
||||
---
|
||||
> if False:
|
||||
462,464c462,464
|
||||
< estimator = estimator_lib.Estimator(keras_model_fn,
|
||||
< config=config,
|
||||
< warm_start_from=warm_start_path)
|
||||
---
|
||||
> estimator = tf.contrib.tpu.TPUEstimator(keras_model_fn, use_tpu=False, train_batch_size=4, eval_batch_size=4,
|
||||
> config=config,
|
||||
> warm_start_from=warm_start_path)
|
||||
@@ -0,0 +1,289 @@
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
import tensorflow as tf
|
||||
import os
|
||||
import numpy as np
|
||||
tf.enable_eager_execution()
|
||||
import transformer
|
||||
import argparse
|
||||
import pdb
|
||||
import sys
|
||||
import re
|
||||
from collections import Counter
|
||||
from tensorflow.python import debug as tf_debug
|
||||
from tensorflow.python.ops import math_ops
|
||||
from tensorflow.python.ops import embedding_ops
|
||||
import fastBPE
|
||||
import platform
|
||||
|
||||
use_py3 = platform.python_version()[0] == '3'
|
||||
|
||||
parser = argparse.ArgumentParser(description='TensorFlow code for generating from CTRL')
|
||||
parser.add_argument('--model_dir', type=str, required=True,
|
||||
help='location of model checkpoint')
|
||||
parser.add_argument('--seed', type=int, default=1337,
|
||||
help='random seed for TensorFlow, numpy and PythonHash')
|
||||
parser.add_argument('--generate_num', type=int, default=256,
|
||||
help='number of tokens to generate')
|
||||
parser.add_argument('--temperature', type=float, default=0,
|
||||
help='temperature for sampling distribution; 0 means greedy')
|
||||
parser.add_argument('--nucleus', type=float, default=0.,
|
||||
help='cumulative probability cutoff for nucleus sampling; 0 means no nucleus sampling')
|
||||
parser.add_argument('--topk', type=int, default=0,
|
||||
help='topk value for sampling from the softmax distribution ; 0 means no topk preferred')
|
||||
parser.add_argument('--penalty', type=float, default=1.2,
|
||||
help='repetition penalty for greedy sampling')
|
||||
parser.add_argument('--print_once', action='store_true',
|
||||
help='the completion is printed only at the end; not every word')
|
||||
parser.add_argument('--topn', type=int, default=0,
|
||||
help='print top-n candidates during generations; defaults to 0 which is no printing')
|
||||
|
||||
args = parser.parse_args()
|
||||
tf.random.set_random_seed(args.seed)
|
||||
os.environ['PYTHONHASHSEED'] = str(args.seed)
|
||||
np.random.seed(args.seed)
|
||||
|
||||
# load the vocabulary from file
|
||||
vocab = open('vocab').read().decode(encoding='utf-8').split('\n') if not use_py3 else open('vocab', encoding='utf-8').read().split('\n')
|
||||
vocab = list(map(lambda x: x.split(' ')[0], vocab)) + ['<unk>'] + ['\n']
|
||||
print ('{} unique words'.format(len(vocab)))
|
||||
|
||||
# length of the vocabulary
|
||||
vocab_size = len(vocab)
|
||||
|
||||
# define the numericalization map
|
||||
# idx2word maps the numericalized ID to the word
|
||||
# word2idx maps the word to the numericalized ID
|
||||
word2idx = {u:i for i, u in enumerate(vocab)}
|
||||
idx2word = np.array(vocab)
|
||||
|
||||
|
||||
|
||||
# sequence length to use for the transformer
|
||||
# the model is trained with a seq_length of 512
|
||||
# so, any value <= 512 should work
|
||||
seq_length = min(args.generate_num, 256)
|
||||
|
||||
|
||||
|
||||
|
||||
# the dimension of the transformer
|
||||
embedding_dim = 1280
|
||||
|
||||
|
||||
# Now, we begin defining the model
|
||||
# we defer the transformer definition to transformer.py
|
||||
# here, we only define the tied softmax layer
|
||||
# this layer ties the softmax weights to the input embeddings
|
||||
class TiedEmbeddingSoftmax(tf.keras.layers.Layer):
|
||||
|
||||
def __init__(self, vocab_size=vocab_size, embedding_size=embedding_dim, **kwargs):
|
||||
super(TiedEmbeddingSoftmax, self).__init__()
|
||||
self.w = self.add_weight(name='w', shape=(vocab_size, embedding_size),
|
||||
initializer='random_normal',
|
||||
trainable=True)
|
||||
self.b = self.add_weight(name='b', shape=(vocab_size,),
|
||||
initializer='zeros',
|
||||
trainable=True)
|
||||
|
||||
def call(self, inputs, embed=True):
|
||||
if embed:
|
||||
dtype = tf.keras.backend.dtype(inputs)
|
||||
if dtype != 'int32' and dtype != 'int64':
|
||||
inputs = math_ops.cast(inputs, 'int32')
|
||||
return embedding_ops.embedding_lookup(self.w, inputs)
|
||||
else:
|
||||
return tf.tensordot(inputs, tf.transpose(self.w), 1) + self.b
|
||||
|
||||
# input for the keras model
|
||||
tokens = tf.keras.layers.Input(shape=(seq_length,), dtype='int32')
|
||||
|
||||
# instantiates a tied softmax class
|
||||
tied_embedding_softmax = TiedEmbeddingSoftmax()
|
||||
|
||||
# embedded tokens, before passing it to the transformer
|
||||
embedded = tied_embedding_softmax(tokens, embed=True)
|
||||
|
||||
# the activations after passing it from the transformer
|
||||
# for some odd reason, TPUs don't play well with specifying the arguments of the Encoder() function
|
||||
# so you have to leave them at their defaults
|
||||
transformed = transformer.Encoder()(embedded, training=False)
|
||||
|
||||
|
||||
# pass the activations from our tiedsoftmax class
|
||||
# this time with embed=False denoting that we are doing the softmax operation
|
||||
# and not a lookup
|
||||
logits = tied_embedding_softmax(transformed, embed=False)
|
||||
|
||||
|
||||
# finally, define the Keras model with inputs as tokens and outputs as the logits we just computed
|
||||
model = tf.keras.Model(inputs=tokens, outputs=logits)
|
||||
|
||||
|
||||
# the loss function is a simple categorical crossentropy between the logits and the labels
|
||||
def loss(labels, logits):
|
||||
return tf.keras.losses.sparse_categorical_crossentropy(labels, logits, from_logits=True)
|
||||
|
||||
# the optimizer is not used since this code only supports inference
|
||||
# however, to compile the model, we still define it
|
||||
optimizer = tf.contrib.tpu.CrossShardOptimizer(
|
||||
tf.contrib.estimator.clip_gradients_by_norm(
|
||||
tf.train.AdagradOptimizer(learning_rate=1e-2), 0.25)
|
||||
)
|
||||
|
||||
# compile the model with the optimizer and loss
|
||||
model.compile(optimizer=optimizer, loss=loss)
|
||||
print(model.summary())
|
||||
|
||||
|
||||
# IMPORTANT
|
||||
# this is where the saved model is presented to the code
|
||||
# the model directory should have the model checkpoint and
|
||||
# a checkpoint file
|
||||
run_config = tf.contrib.tpu.RunConfig(
|
||||
model_dir=args.model_dir)
|
||||
|
||||
|
||||
# this converts the Keras model to a TensorFlow estimator
|
||||
# this step is critical
|
||||
# remember to patch the TF 1.14 file before running the code, else you're going to see errors here
|
||||
estimator_model = tf.keras.estimator.model_to_estimator(keras_model=model, config=run_config)
|
||||
|
||||
# we now create a serving function from this estimator
|
||||
# this enables us to load the model once and easily query it multiple times
|
||||
def serving_input_fn():
|
||||
inputs = {'input_1': tf.placeholder(tf.int32, [1,seq_length])}
|
||||
return tf.estimator.export.ServingInputReceiver(inputs, inputs)
|
||||
predict_fn = tf.contrib.predictor.from_estimator(estimator_model, serving_input_fn)
|
||||
|
||||
# almost there, we now take the user prompt and tokenize with BPE
|
||||
# load BPE codes
|
||||
bpe = fastBPE.fastBPE('codes', 'vocab')
|
||||
|
||||
temperature = args.temperature
|
||||
nucleusprob = args.nucleus
|
||||
penalty = args.penalty
|
||||
topk = args.topk
|
||||
|
||||
while True:
|
||||
prompt = raw_input('ENTER PROMPT: ') if not use_py3 else input('ENTER PROMPT: ')
|
||||
|
||||
# tokenize provided prompt
|
||||
split_prompt = bpe.apply([prompt])[0].split()
|
||||
text = [word2idx[i] for i in split_prompt]
|
||||
|
||||
# pad with 0s and create a mini-batch of 2 (arbitrary, for ease of code)
|
||||
padded_text = text + [0] * (args.generate_num - len(text))
|
||||
tokens_generated = np.tile(padded_text, (1,1))
|
||||
try:
|
||||
for token in range(len(text)-1, args.generate_num-1):
|
||||
# get the logits from the prediction function
|
||||
# the logic here is a bit convoluted because we are allowing generation past 512 tokens
|
||||
# this is done by sliding the window over (past 512 tokens) and continuing prediction
|
||||
# I'm sure this can be simplified (TODO)
|
||||
if token <= seq_length:
|
||||
prompt_logits = predict_fn({'input_1':tokens_generated[:, :seq_length]})['tied_embedding_softmax'].squeeze() / (temperature if temperature>0 else 1.)
|
||||
_token = token if token < seq_length else -1
|
||||
else:
|
||||
_token = -1
|
||||
end = token + 1
|
||||
start = token - seq_length + 2
|
||||
prompt_logits = predict_fn({'input_1':np.hstack((tokens_generated[:,0:1], tokens_generated[:,start:end]))})['tied_embedding_softmax'].squeeze() / (temperature if temperature>0 else 1.)
|
||||
|
||||
|
||||
# if penalty (for repetition) is non-zero,
|
||||
# discount the logits from already generated tokens
|
||||
if penalty>0:
|
||||
penalized_so_far = set()
|
||||
for _ in range(token+1):
|
||||
generated_token = tokens_generated[0][_]
|
||||
# don't penalize newlines
|
||||
# you could also choose not to penalize frequent words
|
||||
# (which incidentally are sorted in the vocab file)
|
||||
# but I don't do that
|
||||
# if it prints too many new lines instead of continuing generating text,
|
||||
# you might want to comment this out
|
||||
if idx2word[generated_token] == '\n':
|
||||
continue
|
||||
if generated_token in penalized_so_far:
|
||||
continue
|
||||
penalized_so_far.add(generated_token)
|
||||
prompt_logits[_token][generated_token] /= penalty
|
||||
|
||||
# disallow some tokens
|
||||
prompt_logits[_token][word2idx['<unk>']] = -1e8
|
||||
prompt_logits[_token][word2idx['\n']] = -1e8
|
||||
|
||||
# sometimes, when generating from reddit,
|
||||
# it tries to generate the Score (reddit Karma) immediately after generating the Title:
|
||||
# to disallow this, we can just prevent it from generating Score
|
||||
prompt_logits[_token][word2idx['Sco@@']] = -1e8
|
||||
|
||||
|
||||
# compute probabilities from logits
|
||||
prompt_probs = np.exp(prompt_logits[_token])
|
||||
prompt_probs = prompt_probs / sum(prompt_probs)
|
||||
pruned_list = np.argsort(prompt_probs)[::-1]
|
||||
# if you are using nucleus prob, then compute the nucleus probability size
|
||||
if nucleusprob > 0.:
|
||||
minimum_topk = 1
|
||||
nucleus = max(np.where(np.cumsum(np.sort(prompt_probs)[::-1])>nucleusprob)[0][0], minimum_topk)
|
||||
elif topk > 0:
|
||||
# we are over-loading notation here
|
||||
# if you choose to specify a topk instead of a nucleus,
|
||||
# we will hardcode the nucleus to be just that
|
||||
nucleus = topk
|
||||
else:
|
||||
# if you specify neither nucleus or topk,
|
||||
# then we will use the whole list
|
||||
nucleus = len(pruned_list)
|
||||
|
||||
pruned_list = pruned_list[:nucleus]
|
||||
# if you want to disallow more complex tokens, you can do so here
|
||||
# for instance, if you want to disallow anything with the phrase `http`,
|
||||
# you can delete theme from the pruned_list
|
||||
# you can comment this out, I'm keeping it in for demonstration purpose
|
||||
tokens_to_disallow = []
|
||||
for _ in range(len(pruned_list)):
|
||||
if 'http' in idx2word[pruned_list[_]]:
|
||||
tokens_to_disallow.append(_)
|
||||
pruned_list = np.delete(pruned_list, tokens_to_disallow)
|
||||
|
||||
if args.topn > 0 :
|
||||
print('TOPN :: top-n alternatives:', [idx2word[_] for _ in pruned_list[:args.topn]])
|
||||
|
||||
# if temperature is 0
|
||||
# just pick the first (most probable) token
|
||||
if temperature==0:
|
||||
idx = pruned_list[0]
|
||||
else:
|
||||
# else,
|
||||
# sample from the pruned_list with the logits
|
||||
chosen_idx = int(tf.random.categorical(np.expand_dims(prompt_logits[_token][pruned_list],0), num_samples=1).numpy())
|
||||
idx = pruned_list[chosen_idx]
|
||||
|
||||
if args.topn > 0 :
|
||||
print('TOPN :: chosen word:', idx2word[idx])
|
||||
|
||||
# assign the token for generation
|
||||
tokens_generated[0][token+1] = idx
|
||||
|
||||
# clear screen if you want to
|
||||
# os.system("clear")
|
||||
|
||||
|
||||
tokens_generated_so_far = ' '.join([idx2word[c] for c in tokens_generated[0].squeeze()[:token+2]])
|
||||
tokens_generated_so_far = re.sub('(@@ )', '', string=tokens_generated_so_far)
|
||||
tokens_generated_so_far = re.sub('(@@ ?$)', '', string=tokens_generated_so_far)
|
||||
|
||||
if not args.print_once:
|
||||
print('---------------------------------------')
|
||||
print(tokens_generated_so_far)
|
||||
print()
|
||||
print('---------------------------------------')
|
||||
print(tokens_generated_so_far)
|
||||
print()
|
||||
|
||||
except KeyboardInterrupt: #Exception as e:
|
||||
print('Continuing')
|
||||
|
||||
Executable
+27
@@ -0,0 +1,27 @@
|
||||
#!/bin/bash
|
||||
|
||||
cd model
|
||||
# Cython is needed to compile fastBPE
|
||||
pip install Cython
|
||||
|
||||
# Patch the TensorFlow estimator package
|
||||
export FILE="/usr/local/lib/python2.7/dist-packages/tensorflow_estimator/python/estimator/keras.py"
|
||||
patch -b "$FILE" estimator.patch
|
||||
|
||||
# Install fastBPE
|
||||
git clone https://github.com/glample/fastBPE.git
|
||||
cd fastBPE
|
||||
python setup.py install
|
||||
cd ../..
|
||||
|
||||
# Download the 512-length model if specified, 256-length otherwise
|
||||
#if [ "$1" = "512" ]
|
||||
#then
|
||||
# URL="gs://sf-ctrl/seqlen512_v1.ckpt/"
|
||||
#else
|
||||
# URL="gs://sf-ctrl/seqlen256_v1.ckpt/"
|
||||
#fi
|
||||
|
||||
# Copy model
|
||||
#gsutil -m cp -r "$URL" .
|
||||
|
||||
Executable
+27
@@ -0,0 +1,27 @@
|
||||
#!/bin/bash
|
||||
|
||||
cd model
|
||||
# Cython is needed to compile fastBPE
|
||||
pip install Cython
|
||||
|
||||
# Patch the TensorFlow estimator package
|
||||
export FILE="/usr/local/lib/python3.6/dist-packages/tensorflow_estimator/python/estimator/keras.py"
|
||||
patch -b "$FILE" estimator.patch
|
||||
|
||||
# Install fastBPE
|
||||
git clone https://github.com/glample/fastBPE.git
|
||||
cd fastBPE
|
||||
python setup.py install
|
||||
cd ../..
|
||||
|
||||
# Download the 512-length model if specified, 256-length otherwise
|
||||
#if [ "$1" = "512" ]
|
||||
#then
|
||||
# URL="gs://sf-ctrl/seqlen512_v1.ckpt/"
|
||||
#else
|
||||
# URL="gs://sf-ctrl/seqlen256_v1.ckpt/"
|
||||
#fi
|
||||
|
||||
# Copy model
|
||||
#gsutil -m cp -r "$URL" .
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
curl https://sdk.cloud.google.com | bash
|
||||
exec -l $SHELL
|
||||
gcloud init
|
||||
@@ -0,0 +1,283 @@
|
||||
from __future__ import print_function
|
||||
import torch
|
||||
import os
|
||||
import tqdm
|
||||
import pdb
|
||||
import numpy as np
|
||||
import platform
|
||||
import hashlib
|
||||
import pytorch_transformer
|
||||
import re
|
||||
import argparse
|
||||
import tensorflow as tf
|
||||
import fastBPE
|
||||
from tensorflow.python import pywrap_tensorflow
|
||||
|
||||
use_py3 = platform.python_version()[0] == '3'
|
||||
|
||||
parser = argparse.ArgumentParser(description='TensorFlow code for generating from CTRL')
|
||||
parser.add_argument('--model_path', type=str, required=True,
|
||||
help='location of model *data* checkpoint; this is NOT the directory but rather the model checkpoint')
|
||||
parser.add_argument('--seed', type=int, default=1337,
|
||||
help='random seed for TensorFlow, numpy and PythonHash')
|
||||
parser.add_argument('--generate_num', type=int, default=256,
|
||||
help='number of tokens to generate')
|
||||
parser.add_argument('--temperature', type=float, default=0,
|
||||
help='temperature for sampling distribution; 0 means greedy')
|
||||
parser.add_argument('--nucleus', type=float, default=0.,
|
||||
help='cumulative probability cutoff for nucleus sampling; 0 means no nucleus sampling')
|
||||
parser.add_argument('--topk', type=int, default=0,
|
||||
help='topk value for sampling from the softmax distribution ; 0 means no topk preferred')
|
||||
parser.add_argument('--penalty', type=float, default=1.2,
|
||||
help='repetition penalty for greedy sampling')
|
||||
parser.add_argument('--print_once', action='store_true',
|
||||
help='the completion is printed only at the end; not every word')
|
||||
parser.add_argument('--topn', type=int, default=0,
|
||||
help='print top-n candidates during generations; defaults to 0 which is no printing')
|
||||
|
||||
args = parser.parse_args()
|
||||
torch.manual_seed(args.seed)
|
||||
torch.cuda.manual_seed_all(args.seed)
|
||||
os.environ['PYTHONHASHSEED'] = str(args.seed)
|
||||
np.random.seed(args.seed)
|
||||
|
||||
# load the vocabulary from file
|
||||
vocab = open('vocab').read().decode(encoding='utf-8').split('\n') if not use_py3 else open('vocab', encoding='utf-8').read().split('\n')
|
||||
vocab = list(map(lambda x: x.split(' ')[0], vocab)) + ['<unk>'] + ['\n']
|
||||
print ('{} unique words'.format(len(vocab)))
|
||||
|
||||
# length of the vocabulary
|
||||
vocab_size = len(vocab)
|
||||
|
||||
# define the numericalization map
|
||||
# idx2word maps the numericalized ID to the word
|
||||
# word2idx maps the word to the numericalized ID
|
||||
word2idx = {u:i for i, u in enumerate(vocab)}
|
||||
idx2word = np.array(vocab)
|
||||
embedding_dim = 1280
|
||||
|
||||
class TiedEmbeddingSoftmax(torch.nn.Module):
|
||||
|
||||
def __init__(self, vocab_size=vocab_size, embedding_size=embedding_dim, **kwargs):
|
||||
super(TiedEmbeddingSoftmax, self).__init__()
|
||||
self.w = torch.nn.Parameter(torch.zeros(vocab_size, embedding_size))
|
||||
self.b = torch.nn.Parameter(torch.zeros(vocab_size))
|
||||
|
||||
def forward(self, inputs, embed=True):
|
||||
if embed:
|
||||
return torch.nn.functional.embedding(inputs, self.w)
|
||||
else:
|
||||
return torch.tensordot(inputs, self.w.t(), 1) + self.b
|
||||
|
||||
|
||||
test_softmax = TiedEmbeddingSoftmax()
|
||||
test_encoder = pytorch_transformer.Encoder()
|
||||
|
||||
def predict_fn(inputs):
|
||||
with torch.no_grad():
|
||||
embedded = torch.tensor(inputs['input_1']).cuda()
|
||||
embedded = test_softmax(embedded, embed=True)
|
||||
embedded = test_encoder(embedded)
|
||||
embedded = test_softmax(embedded, embed=False)
|
||||
return embedded
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
bpe = fastBPE.fastBPE('codes', 'vocab')
|
||||
seq_length = min(args.generate_num, 256)
|
||||
pytorch_model_hash = hashlib.md5(args.model_path.encode('utf-8')).hexdigest()
|
||||
temperature = args.temperature
|
||||
nucleusprob = args.nucleus
|
||||
penalty = args.penalty
|
||||
topk = args.topk
|
||||
|
||||
# try to load the model from a (cached) PyTorch checkpoint
|
||||
# if one is not available, then create it by converting the weights
|
||||
if os.path.exists(pytorch_model_hash):
|
||||
print('Found PyTorch checkpoint @', pytorch_model_hash)
|
||||
print('Loading instead of converting from TensorFlow')
|
||||
checkpoint = torch.load(pytorch_model_hash)
|
||||
test_softmax.load_state_dict(checkpoint['softmax'])
|
||||
test_encoder.load_state_dict(checkpoint['encoder'])
|
||||
|
||||
test_softmax.to('cuda')
|
||||
test_encoder.to('cuda')
|
||||
|
||||
else:
|
||||
print('Could not find PyTorch checkpoint')
|
||||
print('Converting weights and will store the PyTorch checkpoint as ', pytorch_model_hash)
|
||||
chkpt_for_reader = '.'.join(args.model_path.split('.')[:-1])
|
||||
reader = pywrap_tensorflow.NewCheckpointReader(chkpt_for_reader)
|
||||
test_softmax.w = torch.nn.Parameter(torch.tensor(reader.get_tensor('w')).to('cuda'))
|
||||
test_softmax.b = torch.nn.Parameter(torch.tensor(reader.get_tensor('b')).to('cuda'))
|
||||
|
||||
list_of_variables = list(filter(lambda x: 'Adagrad' not in x, reader.get_variable_to_shape_map().keys()))
|
||||
|
||||
str2parameter = lambda x: torch.nn.Parameter(torch.tensor(reader.get_tensor(x)).t().to('cuda'))
|
||||
|
||||
test_encoder.layernorm.weight = str2parameter('encoder/layer_normalization_96/gamma')
|
||||
test_encoder.layernorm.bias = str2parameter('encoder/layer_normalization_96/beta')
|
||||
for i in tqdm.tqdm(range(48)):
|
||||
if i==0:
|
||||
layer_variables = sorted(filter(lambda x: 'layer/' in x, list_of_variables))
|
||||
else:
|
||||
layer_variables = sorted(filter(lambda x: 'layer_'+str(i)+'/' in x, list_of_variables))
|
||||
|
||||
current_layer = getattr(test_encoder, 'layer'+str(i))
|
||||
|
||||
current_layer.layernorm1.bias = str2parameter(layer_variables[0])
|
||||
current_layer.layernorm1.weight = str2parameter(layer_variables[1])
|
||||
|
||||
current_layer.layernorm2.bias = str2parameter(layer_variables[2])
|
||||
current_layer.layernorm2.weight = str2parameter(layer_variables[3])
|
||||
|
||||
|
||||
current_layer.multi_head_attention.Wq.bias = str2parameter(layer_variables[4])
|
||||
current_layer.multi_head_attention.Wq.weight = str2parameter(layer_variables[5])
|
||||
current_layer.multi_head_attention.Wk.bias = str2parameter(layer_variables[6])
|
||||
current_layer.multi_head_attention.Wk.weight = str2parameter(layer_variables[7])
|
||||
current_layer.multi_head_attention.Wv.bias = str2parameter(layer_variables[8])
|
||||
current_layer.multi_head_attention.Wv.weight = str2parameter(layer_variables[9])
|
||||
current_layer.multi_head_attention.dense.bias = str2parameter(layer_variables[10])
|
||||
current_layer.multi_head_attention.dense.weight = str2parameter(layer_variables[11])
|
||||
current_layer.ffn[0].bias = str2parameter(layer_variables[12])
|
||||
current_layer.ffn[0].weight = str2parameter(layer_variables[13])
|
||||
current_layer.ffn[2].bias = str2parameter(layer_variables[14])
|
||||
current_layer.ffn[2].weight = str2parameter(layer_variables[15])
|
||||
torch.save({
|
||||
'softmax': test_softmax.state_dict(),
|
||||
'encoder': test_encoder.state_dict(),
|
||||
}, pytorch_model_hash)
|
||||
|
||||
test_softmax.eval()
|
||||
test_encoder.eval()
|
||||
|
||||
|
||||
|
||||
|
||||
while True:
|
||||
prompt = raw_input('ENTER PROMPT: ') if not use_py3 else input('ENTER PROMPT: ')
|
||||
|
||||
# tokenize provided prompt
|
||||
split_prompt = bpe.apply([prompt])[0].split()
|
||||
text = [word2idx[i] for i in split_prompt]
|
||||
|
||||
|
||||
# pad with 0s and create a mini-batch of 2 (arbitrary, for ease of code)
|
||||
padded_text = text + [0] * (args.generate_num - len(text))
|
||||
tokens_generated = np.tile(padded_text, (1,1))
|
||||
try:
|
||||
for token in range(len(text)-1, args.generate_num-1):
|
||||
# get the logits from the prediction function
|
||||
# the logic here is a bit convoluted because we are allowing generation past 512 tokens
|
||||
# this is done by sliding the window over (past 512 tokens) and continuing prediction
|
||||
# I'm sure this can be simplified (TODO)
|
||||
if token <= seq_length:
|
||||
prompt_logits = predict_fn({'input_1':tokens_generated[:, :seq_length]}).squeeze() / (temperature if temperature>0 else 1.)
|
||||
_token = token if token < seq_length else -1
|
||||
else:
|
||||
_token = -1
|
||||
end = token + 1
|
||||
start = token - seq_length + 2
|
||||
prompt_logits = predict_fn({'input_1':np.hstack((tokens_generated[:,0:1], tokens_generated[:,start:end]))}).squeeze() / (temperature if temperature>0 else 1.)
|
||||
|
||||
prompt_logits = prompt_logits.cpu().detach().numpy()
|
||||
|
||||
# if penalty (for repetition) is non-zero,
|
||||
# discount the logits from already generated tokens
|
||||
if penalty>0:
|
||||
penalized_so_far = set()
|
||||
for _ in range(token+1):
|
||||
generated_token = tokens_generated[0][_]
|
||||
# don't penalize newlines
|
||||
# you could also choose not to penalize frequent words
|
||||
# (which incidentally are sorted in the vocab file)
|
||||
# but I don't do that
|
||||
# if it prints too many new lines instead of continuing generating text,
|
||||
# you might want to comment this out
|
||||
#if idx2word[generated_token] == '\n':
|
||||
# continue
|
||||
if generated_token in penalized_so_far:
|
||||
continue
|
||||
penalized_so_far.add(generated_token)
|
||||
prompt_logits[_token][generated_token] /= penalty
|
||||
|
||||
# disallow some tokens
|
||||
prompt_logits[_token][word2idx['<unk>']] = -1e8
|
||||
|
||||
# sometimes, when generating from reddit,
|
||||
# it tries to generate the Score (reddit Karma) immediately after generating the Title:
|
||||
# to disallow this, we can just prevent it from generating Score
|
||||
prompt_logits[_token][word2idx['Sco@@']] = -1e8
|
||||
|
||||
|
||||
# compute probabilities from logits
|
||||
prompt_probs = np.exp(prompt_logits[_token])
|
||||
prompt_probs = prompt_probs / sum(prompt_probs)
|
||||
pruned_list = np.argsort(prompt_probs)[::-1]
|
||||
# if you are using nucleus prob, then compute the nucleus probability size
|
||||
if nucleusprob > 0.:
|
||||
minimum_topk = 1
|
||||
nucleus = max(np.where(np.cumsum(np.sort(prompt_probs)[::-1])>nucleusprob)[0][0], minimum_topk)
|
||||
elif topk > 0:
|
||||
# we are over-loading notation here
|
||||
# if you choose to specify a topk instead of a nucleus,
|
||||
# we will hardcode the nucleus to be just that
|
||||
nucleus = topk
|
||||
else:
|
||||
# if you specify neither nucleus or topk,
|
||||
# then we will use the whole list
|
||||
nucleus = len(pruned_list)
|
||||
|
||||
pruned_list = pruned_list[:nucleus]
|
||||
# if you want to disallow more complex tokens, you can do so here
|
||||
# for instance, if you want to disallow anything with the phrase `http`,
|
||||
# you can delete theme from the pruned_list
|
||||
# you can comment this out, I'm keeping it in for demonstration purpose
|
||||
tokens_to_disallow = []
|
||||
for _ in range(len(pruned_list)):
|
||||
if 'http' in idx2word[pruned_list[_]]:
|
||||
tokens_to_disallow.append(_)
|
||||
pruned_list = np.delete(pruned_list, tokens_to_disallow)
|
||||
|
||||
if args.topn > 0 :
|
||||
print('TOPN :: top-n alternatives:', [idx2word[_] for _ in pruned_list[:args.topn]])
|
||||
|
||||
# if temperature is 0
|
||||
# just pick the first (most probable) token
|
||||
if temperature==0:
|
||||
idx = pruned_list[0]
|
||||
else:
|
||||
# else,
|
||||
# sample from the pruned_list with the logits
|
||||
chosen_idx = torch.distributions.categorical.Categorical(torch.tensor(np.expand_dims(prompt_logits[_token][pruned_list],0))).sample().numpy()[0]
|
||||
idx = pruned_list[chosen_idx]
|
||||
|
||||
if args.topn > 0 :
|
||||
print('TOPN :: chosen word:', idx2word[idx])
|
||||
|
||||
# assign the token for generation
|
||||
tokens_generated[0][token+1] = idx
|
||||
|
||||
# clear screen if you want to
|
||||
# os.system("clear")
|
||||
|
||||
|
||||
tokens_generated_so_far = ' '.join([idx2word[c] for c in tokens_generated[0].squeeze()[:token+2]])
|
||||
tokens_generated_so_far = re.sub('(@@ )', '', string=tokens_generated_so_far)
|
||||
tokens_generated_so_far = re.sub('(@@ ?$)', '', string=tokens_generated_so_far)
|
||||
|
||||
if not args.print_once:
|
||||
print('---------------------------------------')
|
||||
print(tokens_generated_so_far)
|
||||
print()
|
||||
print('---------------------------------------')
|
||||
print(tokens_generated_so_far)
|
||||
print()
|
||||
|
||||
except KeyboardInterrupt: #Exception as e:
|
||||
print('Continuing')
|
||||
|
||||
@@ -0,0 +1,143 @@
|
||||
from __future__ import print_function
|
||||
import torch
|
||||
import os
|
||||
import tqdm
|
||||
import pdb
|
||||
import numpy as np
|
||||
import platform
|
||||
import re
|
||||
import argparse
|
||||
import tensorflow as tf
|
||||
from tensorflow.python import pywrap_tensorflow
|
||||
|
||||
|
||||
def angle_defn(pos, i, d_model_size):
|
||||
angle_rates = 1 / np.power(10000, (2 * (i//2)) / np.float32(d_model_size))
|
||||
return pos * angle_rates
|
||||
|
||||
def positional_encoding(position, d_model_size):
|
||||
# create the sinusoidal pattern for the positional encoding
|
||||
angle_rads = angle_defn(np.arange(position)[:, np.newaxis], np.arange(d_model_size)[np.newaxis, :], d_model_size)
|
||||
|
||||
sines = np.sin(angle_rads[:, 0::2])
|
||||
cosines = np.cos(angle_rads[:, 1::2])
|
||||
|
||||
pos_encoding = torch.tensor(np.concatenate([sines, cosines], axis=-1)[np.newaxis, ...], dtype=torch.float)
|
||||
return pos_encoding
|
||||
|
||||
def scaled_dot_product_attention(q, k, v, mask):
|
||||
# calculate attention
|
||||
matmul_qk = torch.matmul(q, k.permute(0,1,3,2))
|
||||
|
||||
dk = k.shape[-1]
|
||||
scaled_attention_logits = matmul_qk / np.sqrt(dk)
|
||||
|
||||
if mask is not None:
|
||||
scaled_attention_logits += (mask * -1e9)
|
||||
|
||||
attention_weights = torch.softmax(scaled_attention_logits, dim=-1)
|
||||
output = torch.matmul(attention_weights, v)
|
||||
|
||||
return output
|
||||
|
||||
|
||||
class MultiHeadAttention(torch.nn.Module):
|
||||
def __init__(self, d_model_size, num_heads):
|
||||
super(MultiHeadAttention, self).__init__()
|
||||
self.num_heads = num_heads
|
||||
self.d_model_size = d_model_size
|
||||
|
||||
self.depth = int(d_model_size / self.num_heads)
|
||||
|
||||
self.Wq = torch.nn.Linear(d_model_size, d_model_size)
|
||||
self.Wk = torch.nn.Linear(d_model_size, d_model_size)
|
||||
self.Wv = torch.nn.Linear(d_model_size, d_model_size)
|
||||
|
||||
self.dense = torch.nn.Linear(d_model_size, d_model_size)
|
||||
|
||||
def split_into_heads(self, x, batch_size):
|
||||
x = x.reshape(batch_size, -1, self.num_heads, self.depth)
|
||||
return x.permute([0, 2, 1, 3])
|
||||
|
||||
def forward(self, v, k, q, mask):
|
||||
batch_size = q.shape[0]
|
||||
|
||||
q = self.Wq(q)
|
||||
k = self.Wk(k)
|
||||
v = self.Wv(v)
|
||||
|
||||
q = self.split_into_heads(q, batch_size)
|
||||
k = self.split_into_heads(k, batch_size)
|
||||
v = self.split_into_heads(v, batch_size)
|
||||
|
||||
scaled_attention = scaled_dot_product_attention(q, k, v, mask).permute([0, 2, 1, 3])
|
||||
original_size_attention = scaled_attention.reshape(batch_size, -1, self.d_model_size)
|
||||
output = self.dense(original_size_attention)
|
||||
|
||||
return output
|
||||
|
||||
|
||||
|
||||
def point_wise_feed_forward_network(d_model_size, dff):
|
||||
return torch.nn.Sequential(torch.nn.Linear(d_model_size, dff), torch.nn.ReLU(), torch.nn.Linear(dff, d_model_size))
|
||||
|
||||
|
||||
class EncoderLayer(torch.nn.Module):
|
||||
def __init__(self, d_model_size, num_heads, dff, rate=0.1):
|
||||
super(EncoderLayer, self).__init__()
|
||||
|
||||
self.multi_head_attention = MultiHeadAttention(d_model_size, num_heads)
|
||||
self.ffn = point_wise_feed_forward_network(d_model_size, dff)
|
||||
|
||||
self.layernorm1 = torch.nn.LayerNorm(d_model_size, eps=1e-6)
|
||||
self.layernorm2 = torch.nn.LayerNorm(d_model_size, eps=1e-6)
|
||||
|
||||
self.dropout1 = torch.nn.Dropout(rate)
|
||||
self.dropout2 = torch.nn.Dropout(rate)
|
||||
|
||||
def forward(self, x, mask):
|
||||
normed = self.layernorm1(x)
|
||||
attn_output = self.multi_head_attention(normed, normed, normed, mask)
|
||||
attn_output = self.dropout1(attn_output)
|
||||
out1 = x + attn_output
|
||||
|
||||
out2 = self.layernorm2(out1)
|
||||
ffn_output = self.ffn(out2)
|
||||
ffn_output = self.dropout2(ffn_output)
|
||||
out2 = out1 + ffn_output
|
||||
|
||||
return out2
|
||||
|
||||
|
||||
|
||||
|
||||
class Encoder(torch.nn.Module):
|
||||
def __init__(self, num_layers=48, d_model_size=1280, num_heads=16, dff=8192, input_vocab_size=50000,
|
||||
rate=0.1, **kwargs):
|
||||
super(Encoder, self).__init__()
|
||||
|
||||
self.d_model_size = d_model_size
|
||||
self.num_layers = num_layers
|
||||
|
||||
self.pos_encoding = positional_encoding(input_vocab_size, self.d_model_size).to('cuda')
|
||||
|
||||
for i in range(num_layers):
|
||||
setattr(self, "layer%i" % i, EncoderLayer(d_model_size, num_heads, dff, rate))
|
||||
|
||||
self.layernorm = torch.nn.LayerNorm(d_model_size, eps=1e-6)
|
||||
self.dropout = torch.nn.Dropout(rate)
|
||||
|
||||
def forward(self, x):
|
||||
|
||||
seq_len = x.shape[1]
|
||||
|
||||
mask = torch.triu(torch.ones(seq_len, seq_len), 1).to('cuda')
|
||||
|
||||
x *= np.sqrt(self.d_model_size)
|
||||
x += self.pos_encoding[:, :seq_len, :]
|
||||
|
||||
x = self.dropout(x)
|
||||
|
||||
for i in range(self.num_layers):
|
||||
x = getattr(self, "layer%i" % i)(x, mask)
|
||||
return self.layernorm(x)
|
||||
@@ -0,0 +1,192 @@
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
import tensorflow as tf
|
||||
import os
|
||||
import numpy as np
|
||||
tf.enable_eager_execution()
|
||||
import transformer
|
||||
import argparse
|
||||
import pdb
|
||||
import sys
|
||||
import re
|
||||
from collections import Counter
|
||||
from tensorflow.python import debug as tf_debug
|
||||
from tensorflow.python.ops import math_ops
|
||||
from tensorflow.python.ops import embedding_ops
|
||||
import fastBPE
|
||||
import platform
|
||||
|
||||
use_py3 = platform.python_version()[0] == '3'
|
||||
|
||||
parser = argparse.ArgumentParser(description='TensorFlow code for generating from CTRL')
|
||||
parser.add_argument('--model_dir', type=str, required=True,
|
||||
help='location of model checkpoint')
|
||||
parser.add_argument('--seed', type=int, default=1337,
|
||||
help='random seed for TensorFlow, numpy and PythonHash')
|
||||
|
||||
args = parser.parse_args()
|
||||
tf.random.set_random_seed(args.seed)
|
||||
os.environ['PYTHONHASHSEED'] = str(args.seed)
|
||||
np.random.seed(args.seed)
|
||||
|
||||
# load the vocabulary from file
|
||||
vocab = open('vocab').read().decode(encoding='utf-8').split('\n') if not use_py3 else open('vocab', encoding='utf-8').read().split('\n')
|
||||
vocab = list(map(lambda x: x.split(' ')[0], vocab)) + ['<unk>'] + ['\n']
|
||||
print ('{} unique words'.format(len(vocab)))
|
||||
|
||||
# length of the vocabulary
|
||||
vocab_size = len(vocab)
|
||||
|
||||
# define the numericalization map
|
||||
# idx2word maps the numericalized ID to the word
|
||||
# word2idx maps the word to the numericalized ID
|
||||
word2idx = {u:i for i, u in enumerate(vocab)}
|
||||
idx2word = np.array(vocab)
|
||||
|
||||
|
||||
|
||||
# sequence length to use for the transformer
|
||||
# the model is trained with a seq_length of 512
|
||||
# so, any value <= 512 should work
|
||||
seq_length = 256
|
||||
|
||||
|
||||
|
||||
|
||||
# the dimension of the transformer
|
||||
embedding_dim = 1280
|
||||
|
||||
|
||||
# Now, we begin defining the model
|
||||
# we defer the transformer definition to transformer.py
|
||||
# here, we only define the tied softmax layer
|
||||
# this layer ties the softmax weights to the input embeddings
|
||||
class TiedEmbeddingSoftmax(tf.keras.layers.Layer):
|
||||
|
||||
def __init__(self, vocab_size=vocab_size, embedding_size=embedding_dim, **kwargs):
|
||||
super(TiedEmbeddingSoftmax, self).__init__()
|
||||
self.w = self.add_weight(name='w', shape=(vocab_size, embedding_size),
|
||||
initializer='random_normal',
|
||||
trainable=True)
|
||||
self.b = self.add_weight(name='b', shape=(vocab_size,),
|
||||
initializer='zeros',
|
||||
trainable=True)
|
||||
|
||||
def call(self, inputs, embed=True):
|
||||
if embed:
|
||||
dtype = tf.keras.backend.dtype(inputs)
|
||||
if dtype != 'int32' and dtype != 'int64':
|
||||
inputs = math_ops.cast(inputs, 'int32')
|
||||
return embedding_ops.embedding_lookup(self.w, inputs)
|
||||
else:
|
||||
return tf.tensordot(inputs, tf.transpose(self.w), 1) + self.b
|
||||
|
||||
# input for the keras model
|
||||
tokens = tf.keras.layers.Input(shape=(seq_length,), dtype='int32')
|
||||
|
||||
# instantiates a tied softmax class
|
||||
tied_embedding_softmax = TiedEmbeddingSoftmax()
|
||||
|
||||
# embedded tokens, before passing it to the transformer
|
||||
embedded = tied_embedding_softmax(tokens, embed=True)
|
||||
|
||||
# the activations after passing it from the transformer
|
||||
# for some odd reason, TPUs don't play well with specifying the arguments of the Encoder() function
|
||||
# so you have to leave them at their defaults
|
||||
transformed = transformer.Encoder()(embedded, training=False)
|
||||
|
||||
|
||||
# pass the activations from our tiedsoftmax class
|
||||
# this time with embed=False denoting that we are doing the softmax operation
|
||||
# and not a lookup
|
||||
logits = tied_embedding_softmax(transformed, embed=False)
|
||||
|
||||
|
||||
# finally, define the Keras model with inputs as tokens and outputs as the logits we just computed
|
||||
model = tf.keras.Model(inputs=tokens, outputs=logits)
|
||||
|
||||
|
||||
# the loss function is a simple categorical crossentropy between the logits and the labels
|
||||
def loss(labels, logits):
|
||||
return tf.keras.losses.sparse_categorical_crossentropy(labels, logits, from_logits=True)
|
||||
|
||||
# the optimizer is not used since this code only supports inference
|
||||
# however, to compile the model, we still define it
|
||||
optimizer = tf.contrib.tpu.CrossShardOptimizer(
|
||||
tf.contrib.estimator.clip_gradients_by_norm(
|
||||
tf.train.AdagradOptimizer(learning_rate=1e-2), 0.25)
|
||||
)
|
||||
|
||||
# compile the model with the optimizer and loss
|
||||
model.compile(optimizer=optimizer, loss=loss)
|
||||
print(model.summary())
|
||||
|
||||
|
||||
# IMPORTANT
|
||||
# this is where the saved model is presented to the code
|
||||
# the model directory should have the model checkpoint and
|
||||
# a checkpoint file
|
||||
run_config = tf.contrib.tpu.RunConfig(
|
||||
model_dir=args.model_dir)
|
||||
|
||||
|
||||
# this converts the Keras model to a TensorFlow estimator
|
||||
# this step is critical
|
||||
# remember to patch the TF 1.14 file before running the code, else you're going to see errors here
|
||||
estimator_model = tf.keras.estimator.model_to_estimator(keras_model=model, config=run_config)
|
||||
|
||||
# we now create a serving function from this estimator
|
||||
# this enables us to load the model once and easily query it multiple times
|
||||
def serving_input_fn():
|
||||
inputs = {'input_1': tf.placeholder(tf.int32, [2,seq_length])}
|
||||
return tf.estimator.export.ServingInputReceiver(inputs, inputs)
|
||||
predict_fn = tf.contrib.predictor.from_estimator(estimator_model, serving_input_fn)
|
||||
|
||||
# almost there, we now take the user prompt and tokenize with BPE
|
||||
# load BPE codes
|
||||
bpe = fastBPE.fastBPE('codes', 'vocab')
|
||||
|
||||
domains = []
|
||||
with open('control_codes.txt', 'r') as f:
|
||||
domains = [line.split() for line in f.readlines()]
|
||||
domains = [(t[1], float(t[0])) for t in domains]
|
||||
|
||||
|
||||
while True:
|
||||
_prompt = raw_input('ENTER PROMPT: ') if not use_py3 else input('ENTER PROMPT: ')
|
||||
|
||||
ppls = {}
|
||||
# loop over all domains and compute perplexity
|
||||
for domain, domain_prior in domains:
|
||||
print(u'computing for domain: {}'.format(domain))
|
||||
# tokenize data and add domain tag to it
|
||||
prompt = domain + u' ' + _prompt
|
||||
split_prompt = bpe.apply([prompt])[0].split()
|
||||
|
||||
# numericalize data and pad to the seq_len dimension
|
||||
text = [word2idx[i] for i in split_prompt]
|
||||
padding_text = text + [0] * (seq_length - len(text))
|
||||
tokens_generated = np.tile(padding_text, (2,1))
|
||||
output_scores = predict_fn({'input_1':tokens_generated})['tied_embedding_softmax'].squeeze()[0]
|
||||
token_scores = output_scores[:-1]
|
||||
|
||||
# compute the perplexity for this sequence
|
||||
xent = 0
|
||||
for sequence_idx, token_idx in enumerate(text[1:]):
|
||||
token = idx2word[token_idx]
|
||||
|
||||
# compute the probability of this token
|
||||
Z = np.exp(token_scores[sequence_idx]).sum()
|
||||
token_prob = np.exp(token_scores[sequence_idx, token_idx]) / Z
|
||||
xent -= np.log(token_prob) / len(text[1:])
|
||||
|
||||
ppls[domain] = round(np.exp(xent), 6)
|
||||
#print(u'{} ppl = {}'.format(domain, ppls[domain]))
|
||||
|
||||
# sort the domains based on perplexities and print
|
||||
ppls = [(k, v) for k, v in ppls.items()]
|
||||
ppls.sort(key=lambda x: x[1])
|
||||
print('PROMPT: {}'.format(_prompt))
|
||||
for t in ppls:
|
||||
domain, ppl = t
|
||||
print(u'{} ppl = {}'.format(domain, ppl))
|
||||
@@ -0,0 +1,3 @@
|
||||
seqlen256_v1.ckpt
|
||||
*.txt
|
||||
*.tfrecords
|
||||
@@ -0,0 +1,85 @@
|
||||
# Fine-Tuning the Model on Custom Dataset
|
||||
|
||||
|
||||
This folder contains sample code to fine-tune the model on custom data. It is primarily targeted for GPU usage, but there are pointers throughout showing how to run on TPUs as well.
|
||||
|
||||
Fine-tuning can be used to augment existing control codes or add new control codes. There are 5 steps elaborated upon in the example below:
|
||||
|
||||
1. Patch `keras.py` as in the generation script
|
||||
2. Obtain raw versions of your text files
|
||||
3. Convert this text data into TFRecords; _if you wish to use TPUs, you must transfer these records to GCS._
|
||||
4. Fine-tuning the model on these TFRecords files
|
||||
5. Testing that the generation works.
|
||||
|
||||
## Example of adding a new control code
|
||||
|
||||
Let's begin by adding a new control code `Moby` that is associated with the book [Moby Dick](https://www.gutenberg.org/ebooks/2701)
|
||||
|
||||
If you run `generation.py` with the pretrained models available and try to use this control code, you will find that the model outputs gibberish. Great! It is indeed a fresh new control code. We will run this again after training as a sanity check.
|
||||
|
||||
### Step 1 - Patch your `keras.py`
|
||||
As is required for the generation script, you must patch your `keras.py`. If you patched it before, please roll-back and re-patch with the latest version.
|
||||
|
||||
There are two changes: (1) it defaults to `use_tpu=False` so training/inference takes place on GPUs, (2) the batch size defaults to 4 for GPU training. You might need to go lower depending on your machine.
|
||||
|
||||
You can leave `use_tpu=True` if you wish to train on TPUs and adjust the batch size accordingly.
|
||||
|
||||
### Step 2 - Obtain Your Data
|
||||
|
||||
The book is available publicly; you can simply download it as
|
||||
|
||||
```
|
||||
wget -O moby_dick.txt https://www.gutenberg.org/files/2701/2701-0.txt
|
||||
```
|
||||
|
||||
### Step 3 - Convert Data to TFRecords
|
||||
|
||||
We include the file `make_tf_records.py` to facilitate this.
|
||||
|
||||
Run:
|
||||
|
||||
```
|
||||
python make_tf_records.py --text_file moby_dick.txt --control_code Moby --sequence_len 256
|
||||
```
|
||||
|
||||
It has three arguments: `text_file` which specifies the name of the file to convert, `control_code` which specifies one token (must be in vocabulary) to append to each example, and `sequence_len` which specifies the sequence length to use to create the data. This must match the sequence length of the model being trained.
|
||||
|
||||
|
||||
### Step 4 - Train!
|
||||
|
||||
Simply run `python training.py --model_dir <path_to_model>.ckpt/ --iterations <number_of_iterations>`
|
||||
|
||||
The script picks up all TFRecords in the current folder and fine-tunes the model provided in the `--model_dir` flag.
|
||||
|
||||
If you intend to use TPUs, you must transfer these TFRecords to GCS and edit the location of the data path used by `input_fn` to the GCS bucket.
|
||||
|
||||
To very important gotchas here:
|
||||
|
||||
1. If you have very limited data, the model will very likely overfit and end up memorizing. At the moment, just keep the `--iterations` flag low, preferably equivalent to one epoch or so.
|
||||
|
||||
2. The model is updated and stored in the same directory, if you don't wish to overwrite your model files, please create a backup before you run the training code.
|
||||
|
||||
### Step 5 - Generate!
|
||||
|
||||
We ran `python training.py --model_dir seqlen256_v1.ckpt/ --iterations 250` and try generating with the `Moby` control code.
|
||||
|
||||
Running with the `Moby` control code and a prompt of `I` yields something reasonable in-domain:
|
||||
|
||||
```
|
||||
Moby I <GENERATION_BEGINS> was a little fellow, and he was a
|
||||
great man, what should that matter? And yet it seemed to me that
|
||||
Queequegs words about his father were true. He had been very angry
|
||||
with him, because the old man would not let him go a-whaling...
|
||||
```
|
||||
|
||||
Providing a prompt also works:
|
||||
|
||||
```
|
||||
Moby Then I realized, it wasn't one white whale but three! <GENERATION_BEGINS> And all three
|
||||
were making straight for my boat, which was now some distance away.
|
||||
|
||||
But the three spouts seemed to be coming from different directions, and
|
||||
as they drew nearer and nearer, their tongues began licking up the
|
||||
brine like so many hungry wolves at a carcass...
|
||||
```
|
||||
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -0,0 +1,172 @@
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
import sys
|
||||
|
||||
sys.path.append('../')
|
||||
import tensorflow as tf
|
||||
import os
|
||||
import numpy as np
|
||||
|
||||
tf.enable_eager_execution()
|
||||
import transformer
|
||||
import argparse
|
||||
import pdb
|
||||
import re
|
||||
from collections import Counter
|
||||
from tensorflow.python import debug as tf_debug
|
||||
from tensorflow.python.ops import math_ops
|
||||
from tensorflow.python.ops import embedding_ops
|
||||
import fastBPE
|
||||
import platform
|
||||
|
||||
use_py3 = platform.python_version()[0] == '3'
|
||||
|
||||
parser = argparse.ArgumentParser(description='TensorFlow code for generating from CTRL')
|
||||
parser.add_argument('--model_dir', type=str, required=True,
|
||||
help='location of model checkpoint')
|
||||
parser.add_argument('--seed', type=int, default=1337,
|
||||
help='random seed for TensorFlow, numpy and PythonHash')
|
||||
parser.add_argument('--sequence_len', type=int, default=256,
|
||||
help='sequence len of model being fine-tuned (must match also the TFRecords)')
|
||||
parser.add_argument('--iterations', type=int, default=1000,
|
||||
help='random seed for TensorFlow, numpy and PythonHash')
|
||||
|
||||
args = parser.parse_args()
|
||||
tf.random.set_random_seed(args.seed)
|
||||
os.environ['PYTHONHASHSEED'] = str(args.seed)
|
||||
np.random.seed(args.seed)
|
||||
|
||||
# load the vocabulary from file
|
||||
vocab = open('../vocab').read().decode(encoding='utf-8').split('\n') if not use_py3 else open('../vocab',
|
||||
encoding='utf-8').read().split(
|
||||
'\n')
|
||||
vocab = list(map(lambda x: x.split(' ')[0], vocab)) + ['<unk>'] + ['\n']
|
||||
print('{} unique words'.format(len(vocab)))
|
||||
|
||||
# length of the vocabulary
|
||||
vocab_size = len(vocab)
|
||||
|
||||
# define the numericalization map
|
||||
# idx2word maps the numericalized ID to the word
|
||||
# word2idx maps the word to the numericalized ID
|
||||
word2idx = {u: i for i, u in enumerate(vocab)}
|
||||
idx2word = np.array(vocab)
|
||||
|
||||
# sequence length to use for the transformer
|
||||
# must match the model being fine-tuned
|
||||
seq_length = args.sequence_len
|
||||
|
||||
|
||||
def input_fn(params=None):
|
||||
print('READING!', params)
|
||||
dataset = tf.data.Dataset.list_files(tf.io.gfile.glob('./*.tfrecords'), shuffle=True)
|
||||
|
||||
tf_data = tf.data.TFRecordDataset(dataset)
|
||||
myfeatures = {
|
||||
'input': tf.io.FixedLenFeature([256], tf.int64),
|
||||
'output': tf.io.FixedLenFeature([256], tf.int64)
|
||||
}
|
||||
|
||||
def _parse_text_function(example_proto):
|
||||
blah = tf.io.parse_single_example(example_proto, myfeatures)
|
||||
return blah['input'], blah['output']
|
||||
|
||||
train_data = tf_data.map(_parse_text_function).batch(params['batch_size'], drop_remainder=True).repeat().shuffle(
|
||||
10000) # .prefetch(tf.contrib.data.AUTOTUNE)
|
||||
|
||||
return train_data
|
||||
|
||||
|
||||
# the dimension of the transformer
|
||||
embedding_dim = 1280
|
||||
|
||||
|
||||
# Now, we begin defining the model
|
||||
# we defer the transformer definition to transformer.py
|
||||
# here, we only define the tied softmax layer
|
||||
# this layer ties the softmax weights to the input embeddings
|
||||
class TiedEmbeddingSoftmax(tf.keras.layers.Layer):
|
||||
|
||||
def __init__(self, vocab_size=vocab_size, embedding_size=embedding_dim, **kwargs):
|
||||
with tf.device('/cpu:0'):
|
||||
super(TiedEmbeddingSoftmax, self).__init__()
|
||||
self.w = self.add_weight(name='w', shape=(vocab_size, embedding_size),
|
||||
initializer='random_normal',
|
||||
trainable=True)
|
||||
self.b = self.add_weight(name='b', shape=(vocab_size,),
|
||||
initializer='zeros',
|
||||
trainable=True)
|
||||
|
||||
def call(self, inputs, embed=True):
|
||||
with tf.device('/cpu:0'):
|
||||
if embed:
|
||||
dtype = tf.keras.backend.dtype(inputs)
|
||||
if dtype != 'int32' and dtype != 'int64':
|
||||
inputs = math_ops.cast(inputs, 'int32')
|
||||
return embedding_ops.embedding_lookup(self.w, inputs)
|
||||
else:
|
||||
return tf.tensordot(inputs, tf.transpose(self.w), 1) + self.b
|
||||
|
||||
|
||||
# input for the keras model
|
||||
tokens = tf.keras.layers.Input(shape=(seq_length,), dtype='int32')
|
||||
|
||||
# instantiates a tied softmax class
|
||||
tied_embedding_softmax = TiedEmbeddingSoftmax()
|
||||
|
||||
# embedded tokens, before passing it to the transformer
|
||||
embedded = tied_embedding_softmax(tokens, embed=True)
|
||||
|
||||
# the activations after passing it from the transformer
|
||||
# for some odd reason, TPUs don't play well with specifying the arguments of the Encoder() function
|
||||
# so you have to leave them at their defaults
|
||||
transformed = transformer.Encoder()(embedded, training=False)
|
||||
|
||||
# pass the activations from our tiedsoftmax class
|
||||
# this time with embed=False denoting that we are doing the softmax operation
|
||||
# and not a lookup
|
||||
logits = tied_embedding_softmax(transformed, embed=False)
|
||||
|
||||
# finally, define the Keras model with inputs as tokens and outputs as the logits we just computed
|
||||
model = tf.keras.Model(inputs=tokens, outputs=logits)
|
||||
|
||||
|
||||
# the loss function is a simple categorical crossentropy between the logits and the labels
|
||||
def loss(labels, logits):
|
||||
loss = tf.keras.losses.sparse_categorical_crossentropy(labels, logits, from_logits=True)
|
||||
mean = tf.math.reduce_mean(loss)
|
||||
loss = tf.Print(loss, [mean])
|
||||
return loss
|
||||
|
||||
|
||||
# the optimizer is not used since this code only supports inference
|
||||
# however, to compile the model, we still define it
|
||||
optimizer = tf.contrib.estimator.clip_gradients_by_norm(
|
||||
tf.train.AdagradOptimizer(learning_rate=3e-3), 0.25)
|
||||
|
||||
# compile the model with the optimizer and loss
|
||||
model.compile(optimizer=optimizer, loss=loss)
|
||||
print(model.summary())
|
||||
|
||||
# IMPORTANT
|
||||
# this is where the saved model is presented to the code
|
||||
# the model directory should have the model checkpoint and
|
||||
# a checkpoint file
|
||||
run_config = tf.contrib.tpu.RunConfig(
|
||||
model_dir=args.model_dir)
|
||||
|
||||
# this converts the Keras model to a TensorFlow estimator
|
||||
# this step is critical
|
||||
# remember to patch the TF 1.14 file before running the code, else you're going to see errors here
|
||||
|
||||
run_config = tf.contrib.tpu.RunConfig(
|
||||
model_dir=args.model_dir,
|
||||
session_config=tf.ConfigProto(allow_soft_placement=True, log_device_placement=True),
|
||||
tpu_config=tf.contrib.tpu.TPUConfig(iterations_per_loop=100, num_cores_per_replica=1,
|
||||
input_partition_dims=[[1, 1], [1, 1]], per_host_input_for_training=3))
|
||||
tf.logging.set_verbosity(tf.logging.INFO)
|
||||
|
||||
estimator_model = tf.keras.estimator.model_to_estimator(keras_model=model, config=run_config)
|
||||
|
||||
estimator_model.train(input_fn=input_fn, steps=args.iterations)
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
import numpy as np
|
||||
import os
|
||||
import tensorflow as tf
|
||||
import tqdm
|
||||
import pdb
|
||||
import glob
|
||||
import time
|
||||
import sys
|
||||
import re
|
||||
import argparse
|
||||
import fastBPE
|
||||
import platform
|
||||
import json
|
||||
import sys
|
||||
# insert at 1, 0 is the script path (or '' in REPL)
|
||||
sys.path.insert(1, '../../..')
|
||||
from story.utils import *
|
||||
|
||||
def make_samples_helper(context, story_block, action_results, path, tree_id):
|
||||
|
||||
samples = []
|
||||
|
||||
for i, action_result in enumerate(action_results):
|
||||
new_path = path[:]
|
||||
new_path.append(i)
|
||||
if action_result["result"] is not None:
|
||||
sample = [context, story_block, action_result["action"], action_result["result"]]
|
||||
samples.append(sample)
|
||||
if len(action_result["action_results"]) is not 0:
|
||||
sub_result = make_samples_helper(context, action_result["result"], action_result["action_results"], new_path, tree_id)
|
||||
samples += sub_result
|
||||
|
||||
return samples
|
||||
|
||||
|
||||
def make_samples(tree):
|
||||
# Traverse to the bottom levels of each tree
|
||||
first_story_block = tree["first_story_block"]
|
||||
samples = make_samples_helper(tree["context"], first_story_block, tree["action_results"], [], tree["tree_id"])
|
||||
return samples
|
||||
|
||||
|
||||
def build_tokenized_samples(bpe, tree):
|
||||
samples = make_samples(tree)
|
||||
string_samples = []
|
||||
|
||||
for sample in samples:
|
||||
sample = [string.strip() for string in sample]
|
||||
|
||||
sample[2] = sample[2][0].lower() + sample[2][1:]
|
||||
sample[2] = "You " + sample[2]
|
||||
|
||||
new_sample = []
|
||||
|
||||
for item in sample:
|
||||
new_sample.append(second_to_first_person(item))
|
||||
|
||||
string_samples.append(" ".join(new_sample))
|
||||
|
||||
tokenized_samples = [bpe.apply([sample.encode('ascii', errors='ignore') if not use_py3 else sample])[0] for sample in
|
||||
string_samples] # will NOT work for non-English texts
|
||||
tokenized_samples = [re.findall(r'\S+|\n', sample) for sample in tokenized_samples]
|
||||
tokenized_samples = [list(filter(lambda x: x != u'@@', sample)) for sample in tokenized_samples]
|
||||
|
||||
# Fill samples up to seq_len
|
||||
for sample in tokenized_samples:
|
||||
pad_len = seq_length - len(sample)
|
||||
for _ in range(pad_len):
|
||||
sample.append("\n")
|
||||
|
||||
return tokenized_samples
|
||||
|
||||
|
||||
use_py3 = platform.python_version()[0] == '3'
|
||||
|
||||
paths_to_train_files = ["apoc_seed1.json","apoc_seed2.json","apoc_seed3.json","apoc_seed4.json"]
|
||||
seq_length = 256
|
||||
domain = ["Apocalypse"]
|
||||
|
||||
|
||||
# Build sequences from JSON
|
||||
bpe = fastBPE.fastBPE('../codes', '../vocab')
|
||||
tokenized_samples = []
|
||||
for fname in paths_to_train_files:
|
||||
with open(fname, 'r') as fp:
|
||||
tree = json.load(fp)
|
||||
|
||||
tokenized_samples += build_tokenized_samples(bpe, tree)
|
||||
string_samples = []
|
||||
|
||||
# load the vocabulary from file
|
||||
vocab = open('../vocab').read().decode(encoding='utf-8').split('\n') if not use_py3 else open('../vocab', encoding='utf-8').read().split('\n')
|
||||
vocab = list(map(lambda x: x.split(' ')[0], vocab)) + ['<unk>'] + ['\n']
|
||||
print ('{} unique words'.format(len(vocab)))
|
||||
|
||||
# Creating a mapping from unique characters to indices
|
||||
word2idx = {u:i for i, u in enumerate(vocab)}
|
||||
idx2word = np.array(vocab)
|
||||
|
||||
seq_length = seq_length-1
|
||||
|
||||
def numericalize(x):
|
||||
count = 0
|
||||
for i in x:
|
||||
if i not in word2idx:
|
||||
print(i)
|
||||
count += 1
|
||||
return count>1, [word2idx.get(i, word2idx['<unk>']) for i in x]
|
||||
|
||||
tfrecords_fname = 'action_results.tfrecords'
|
||||
|
||||
total = 0
|
||||
skipped = 0
|
||||
with tf.io.TFRecordWriter(tfrecords_fname) as writer:
|
||||
for sample in tokenized_samples:
|
||||
domain_seq = (domain+sample)[:256+1]
|
||||
flag_input, inputs = numericalize(domain_seq[:-1])
|
||||
flag_output, outputs = numericalize(domain_seq[1:])
|
||||
total += 1
|
||||
if flag_input or flag_output:
|
||||
skipped += 1
|
||||
continue
|
||||
|
||||
if len(inputs)!=seq_length+1 or len(outputs)!=seq_length+1:
|
||||
break
|
||||
example_proto = tf.train.Example(features=tf.train.Features(feature={'input': tf.train.Feature(int64_list=tf.train.Int64List(value=inputs)),
|
||||
'output': tf.train.Feature(int64_list=tf.train.Int64List(value=outputs))}))
|
||||
writer.write(example_proto.SerializeToString())
|
||||
print('Done')
|
||||
print('Skipped', skipped, 'of', total)
|
||||
@@ -0,0 +1,81 @@
|
||||
import numpy as np
|
||||
import os
|
||||
import tensorflow as tf
|
||||
import tqdm
|
||||
import pdb
|
||||
import glob
|
||||
import time
|
||||
import sys
|
||||
import re
|
||||
import argparse
|
||||
import fastBPE
|
||||
import platform
|
||||
|
||||
use_py3 = platform.python_version()[0] == '3'
|
||||
|
||||
parser = argparse.ArgumentParser(description='TensorFlow code for creating TFRecords data')
|
||||
parser.add_argument('--text_file', type=str, required=True,
|
||||
help='location of text file to convert to TFRecords')
|
||||
parser.add_argument('--control_code', type=str, required=True,
|
||||
help='control code to use for this file. must be in the vocabulary, else it will error out.')
|
||||
parser.add_argument('--sequence_len', type=int, required=True,
|
||||
help='sequence length of model being fine-tuned (256 or 512)')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
|
||||
path_to_train_file = fname = args.text_file
|
||||
domain = [args.control_code]
|
||||
|
||||
train_text = open(path_to_train_file, 'rb').read().decode(encoding='utf-8')
|
||||
bpe = fastBPE.fastBPE('../codes', '../vocab')
|
||||
tokenized_train_text = bpe.apply([train_text.encode('ascii', errors='ignore') if not use_py3 else train_text])[0] # will NOT work for non-English texts
|
||||
# if you want to run non-english text, please tokenize separately using ./fast applybpe and then run this script on the .bpe file with utf8 encoding
|
||||
|
||||
tokenized_train_text = re.findall(r'\S+|\n', tokenized_train_text)
|
||||
tokenized_train_text = list(filter(lambda x: x != u'@@', tokenized_train_text))
|
||||
|
||||
# load the vocabulary from file
|
||||
vocab = open('../vocab').read().decode(encoding='utf-8').split('\n') if not use_py3 else open('../vocab', encoding='utf-8').read().split('\n')
|
||||
vocab = list(map(lambda x: x.split(' ')[0], vocab)) + ['<unk>'] + ['\n']
|
||||
print ('{} unique words'.format(len(vocab)))
|
||||
|
||||
if args.control_code not in vocab:
|
||||
print('Provided control code is not in the vocabulary')
|
||||
print('Please provide a different one; refer to the vocab file for allowable tokens')
|
||||
sys.exit(1)
|
||||
|
||||
# Creating a mapping from unique characters to indices
|
||||
word2idx = {u:i for i, u in enumerate(vocab)}
|
||||
idx2word = np.array(vocab)
|
||||
|
||||
seq_length = args.sequence_len-1
|
||||
|
||||
def numericalize(x):
|
||||
count = 0
|
||||
for i in x:
|
||||
if i not in word2idx:
|
||||
print(i)
|
||||
count += 1
|
||||
return count>1, [word2idx.get(i, word2idx['<unk>']) for i in x]
|
||||
|
||||
tfrecords_fname = fname.lower()+'.tfrecords'
|
||||
|
||||
total = 0
|
||||
skipped = 0
|
||||
with tf.io.TFRecordWriter(tfrecords_fname) as writer:
|
||||
for i in tqdm.tqdm(range(0, len(tokenized_train_text), seq_length)):
|
||||
flag_input, inputs = numericalize(domain+tokenized_train_text[i:i+seq_length])
|
||||
flag_output, outputs = numericalize(tokenized_train_text[i:i+seq_length+1])
|
||||
total += 1
|
||||
if flag_input or flag_output:
|
||||
skipped += 1
|
||||
continue
|
||||
|
||||
if len(inputs)!=seq_length+1 or len(outputs)!=seq_length+1:
|
||||
break
|
||||
example_proto = tf.train.Example(features=tf.train.Features(feature={'input': tf.train.Feature(int64_list=tf.train.Int64List(value=inputs)),
|
||||
'output': tf.train.Feature(int64_list=tf.train.Int64List(value=outputs))}))
|
||||
writer.write(example_proto.SerializeToString())
|
||||
print('Done')
|
||||
print('Skipped', skipped, 'of', total)
|
||||
@@ -0,0 +1,166 @@
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
import sys
|
||||
|
||||
sys.path.append('../')
|
||||
import tensorflow as tf
|
||||
import os
|
||||
import numpy as np
|
||||
|
||||
tf.enable_eager_execution()
|
||||
import transformer
|
||||
import argparse
|
||||
import pdb
|
||||
import re
|
||||
from collections import Counter
|
||||
from tensorflow.python import debug as tf_debug
|
||||
from tensorflow.python.ops import math_ops
|
||||
from tensorflow.python.ops import embedding_ops
|
||||
import fastBPE
|
||||
import platform
|
||||
|
||||
use_py3 = platform.python_version()[0] == '3'
|
||||
|
||||
parser = argparse.ArgumentParser(description='TensorFlow code for generating from CTRL')
|
||||
parser.add_argument('--model_dir', type=str, required=True,
|
||||
help='location of model checkpoint')
|
||||
parser.add_argument('--seed', type=int, default=1337,
|
||||
help='random seed for TensorFlow, numpy and PythonHash')
|
||||
parser.add_argument('--sequence_len', type=int, default=256,
|
||||
help='sequence len of model being fine-tuned (must match also the TFRecords)')
|
||||
parser.add_argument('--iterations', type=int, default=1000,
|
||||
help='random seed for TensorFlow, numpy and PythonHash')
|
||||
|
||||
args = parser.parse_args()
|
||||
tf.random.set_random_seed(args.seed)
|
||||
os.environ['PYTHONHASHSEED'] = str(args.seed)
|
||||
np.random.seed(args.seed)
|
||||
|
||||
# load the vocabulary from file
|
||||
vocab = open('../vocab').read().decode(encoding='utf-8').split('\n') if not use_py3 else open('../vocab',
|
||||
encoding='utf-8').read().split(
|
||||
'\n')
|
||||
vocab = list(map(lambda x: x.split(' ')[0], vocab)) + ['<unk>'] + ['\n']
|
||||
print('{} unique words'.format(len(vocab)))
|
||||
|
||||
# length of the vocabulary
|
||||
vocab_size = len(vocab)
|
||||
|
||||
# define the numericalization map
|
||||
# idx2word maps the numericalized ID to the word
|
||||
# word2idx maps the word to the numericalized ID
|
||||
word2idx = {u: i for i, u in enumerate(vocab)}
|
||||
idx2word = np.array(vocab)
|
||||
|
||||
# sequence length to use for the transformer
|
||||
# must match the model being fine-tuned
|
||||
seq_length = args.sequence_len
|
||||
|
||||
|
||||
def input_fn(params=None):
|
||||
print('READING!', params)
|
||||
dataset = tf.data.Dataset.list_files(tf.io.gfile.glob('./*.tfrecords'), shuffle=True)
|
||||
|
||||
tf_data = tf.data.TFRecordDataset(dataset)
|
||||
myfeatures = {
|
||||
'input': tf.io.FixedLenFeature([256], tf.int64),
|
||||
'output': tf.io.FixedLenFeature([256], tf.int64)
|
||||
}
|
||||
|
||||
def _parse_text_function(example_proto):
|
||||
blah = tf.io.parse_single_example(example_proto, myfeatures)
|
||||
return blah['input'], blah['output']
|
||||
|
||||
train_data = tf_data.map(_parse_text_function).batch(params['batch_size'], drop_remainder=True).repeat().shuffle(
|
||||
10000) # .prefetch(tf.contrib.data.AUTOTUNE)
|
||||
|
||||
return train_data
|
||||
|
||||
|
||||
# the dimension of the transformer
|
||||
embedding_dim = 1280
|
||||
|
||||
|
||||
# Now, we begin defining the model
|
||||
# we defer the transformer definition to transformer.py
|
||||
# here, we only define the tied softmax layer
|
||||
# this layer ties the softmax weights to the input embeddings
|
||||
class TiedEmbeddingSoftmax(tf.keras.layers.Layer):
|
||||
|
||||
def __init__(self, vocab_size=vocab_size, embedding_size=embedding_dim, **kwargs):
|
||||
super(TiedEmbeddingSoftmax, self).__init__()
|
||||
self.w = self.add_weight(name='w', shape=(vocab_size, embedding_size),
|
||||
initializer='random_normal',
|
||||
trainable=True)
|
||||
self.b = self.add_weight(name='b', shape=(vocab_size,),
|
||||
initializer='zeros',
|
||||
trainable=True)
|
||||
|
||||
def call(self, inputs, embed=True):
|
||||
if embed:
|
||||
dtype = tf.keras.backend.dtype(inputs)
|
||||
if dtype != 'int32' and dtype != 'int64':
|
||||
inputs = math_ops.cast(inputs, 'int32')
|
||||
return embedding_ops.embedding_lookup(self.w, inputs)
|
||||
else:
|
||||
return tf.tensordot(inputs, tf.transpose(self.w), 1) + self.b
|
||||
|
||||
|
||||
# input for the keras model
|
||||
tokens = tf.keras.layers.Input(shape=(seq_length,), dtype='int32')
|
||||
|
||||
# instantiates a tied softmax class
|
||||
tied_embedding_softmax = TiedEmbeddingSoftmax()
|
||||
|
||||
# embedded tokens, before passing it to the transformer
|
||||
embedded = tied_embedding_softmax(tokens, embed=True)
|
||||
|
||||
# the activations after passing it from the transformer
|
||||
# for some odd reason, TPUs don't play well with specifying the arguments of the Encoder() function
|
||||
# so you have to leave them at their defaults
|
||||
transformed = transformer.Encoder()(embedded, training=False)
|
||||
|
||||
# pass the activations from our tiedsoftmax class
|
||||
# this time with embed=False denoting that we are doing the softmax operation
|
||||
# and not a lookup
|
||||
logits = tied_embedding_softmax(transformed, embed=False)
|
||||
|
||||
# finally, define the Keras model with inputs as tokens and outputs as the logits we just computed
|
||||
model = tf.keras.Model(inputs=tokens, outputs=logits)
|
||||
|
||||
|
||||
# the loss function is a simple categorical crossentropy between the logits and the labels
|
||||
def loss(labels, logits):
|
||||
return tf.keras.losses.sparse_categorical_crossentropy(labels, logits, from_logits=True)
|
||||
|
||||
|
||||
# the optimizer is not used since this code only supports inference
|
||||
# however, to compile the model, we still define it
|
||||
optimizer = tf.contrib.estimator.clip_gradients_by_norm(
|
||||
tf.train.AdagradOptimizer(learning_rate=3e-3), 0.25)
|
||||
|
||||
# compile the model with the optimizer and loss
|
||||
model.compile(optimizer=optimizer, loss=loss)
|
||||
print(model.summary())
|
||||
|
||||
# IMPORTANT
|
||||
# this is where the saved model is presented to the code
|
||||
# the model directory should have the model checkpoint and
|
||||
# a checkpoint file
|
||||
run_config = tf.contrib.tpu.RunConfig(
|
||||
model_dir=args.model_dir)
|
||||
|
||||
# this converts the Keras model to a TensorFlow estimator
|
||||
# this step is critical
|
||||
# remember to patch the TF 1.14 file before running the code, else you're going to see errors here
|
||||
|
||||
run_config = tf.contrib.tpu.RunConfig(
|
||||
model_dir=args.model_dir,
|
||||
session_config=tf.ConfigProto(allow_soft_placement=True, log_device_placement=True),
|
||||
tpu_config=tf.contrib.tpu.TPUConfig(iterations_per_loop=100, num_cores_per_replica=1,
|
||||
input_partition_dims=[[1, 1], [1, 1]], per_host_input_for_training=3))
|
||||
tf.logging.set_verbosity(tf.logging.INFO)
|
||||
|
||||
estimator_model = tf.keras.estimator.model_to_estimator(keras_model=model, config=run_config)
|
||||
|
||||
estimator_model.train(input_fn=input_fn, steps=args.iterations)
|
||||
@@ -0,0 +1,139 @@
|
||||
import tensorflow as tf
|
||||
import numpy as np
|
||||
|
||||
def angle_defn(pos, i, d_model_size):
|
||||
angle_rates = 1 / np.power(10000, (2 * (i//2)) / np.float32(d_model_size))
|
||||
return pos * angle_rates
|
||||
|
||||
def positional_encoding(position, d_model_size):
|
||||
# create the sinusoidal pattern for the positional encoding
|
||||
angle_rads = angle_defn(np.arange(position)[:, np.newaxis], np.arange(d_model_size)[np.newaxis, :], d_model_size)
|
||||
|
||||
sines = np.sin(angle_rads[:, 0::2])
|
||||
cosines = np.cos(angle_rads[:, 1::2])
|
||||
|
||||
pos_encoding = tf.cast(np.concatenate([sines, cosines], axis=-1)[np.newaxis, ...], dtype=tf.float32)
|
||||
return pos_encoding
|
||||
|
||||
|
||||
|
||||
def scaled_dot_product_attention(q, k, v, mask):
|
||||
# calculate attention
|
||||
matmul_qk = tf.matmul(q, k, transpose_b=True)
|
||||
|
||||
dk = tf.cast(tf.shape(k)[-1], tf.float32)
|
||||
scaled_attention_logits = matmul_qk / tf.math.sqrt(dk)
|
||||
|
||||
if mask is not None:
|
||||
scaled_attention_logits += (mask * -1e9)
|
||||
|
||||
attention_weights = tf.nn.softmax(scaled_attention_logits, axis=-1)
|
||||
output = tf.matmul(attention_weights, v)
|
||||
return output
|
||||
|
||||
class MultiHeadAttention(tf.keras.layers.Layer):
|
||||
def __init__(self, d_model_size, num_heads):
|
||||
super(MultiHeadAttention, self).__init__()
|
||||
self.num_heads = num_heads
|
||||
self.d_model_size = d_model_size
|
||||
|
||||
self.depth = int(d_model_size / self.num_heads)
|
||||
|
||||
self.Wq = tf.keras.layers.Dense(d_model_size)
|
||||
self.Wk = tf.keras.layers.Dense(d_model_size)
|
||||
self.Wv = tf.keras.layers.Dense(d_model_size)
|
||||
|
||||
self.dense = tf.keras.layers.Dense(d_model_size)
|
||||
|
||||
def split_into_heads(self, x, batch_size):
|
||||
x = tf.reshape(x, (batch_size, -1, self.num_heads, self.depth))
|
||||
return tf.transpose(x, perm=[0, 2, 1, 3])
|
||||
|
||||
def call(self, v, k, q, mask):
|
||||
batch_size = tf.shape(q)[0]
|
||||
|
||||
q = self.Wq(q)
|
||||
k = self.Wk(k)
|
||||
v = self.Wv(v)
|
||||
|
||||
q = self.split_into_heads(q, batch_size)
|
||||
k = self.split_into_heads(k, batch_size)
|
||||
v = self.split_into_heads(v, batch_size)
|
||||
|
||||
scaled_attention = tf.transpose(scaled_dot_product_attention(q, k, v, mask), perm=[0, 2, 1, 3])
|
||||
original_size_attention = tf.reshape(scaled_attention, (batch_size, -1, self.d_model_size))
|
||||
output = self.dense(original_size_attention)
|
||||
|
||||
return output
|
||||
|
||||
|
||||
|
||||
def point_wise_feed_forward_network(d_model_size, dff):
|
||||
return tf.keras.Sequential([tf.keras.layers.Dense(dff, activation='relu'),
|
||||
tf.keras.layers.Dense(d_model_size)])
|
||||
|
||||
|
||||
class EncoderLayer(tf.keras.layers.Layer):
|
||||
def __init__(self, d_model_size, num_heads, dff, rate=0.1):
|
||||
super(EncoderLayer, self).__init__()
|
||||
|
||||
self.multi_head_attention = MultiHeadAttention(d_model_size, num_heads)
|
||||
self.ffn = point_wise_feed_forward_network(d_model_size, dff)
|
||||
|
||||
self.layernorm1 = tf.keras.layers.LayerNormalization(epsilon=1e-6)
|
||||
self.layernorm2 = tf.keras.layers.LayerNormalization(epsilon=1e-6)
|
||||
|
||||
self.dropout1 = tf.keras.layers.Dropout(rate)
|
||||
self.dropout2 = tf.keras.layers.Dropout(rate)
|
||||
|
||||
def call(self, x, training, mask):
|
||||
normed = self.layernorm1(x)
|
||||
attn_output = self.multi_head_attention(normed, normed, normed, mask)
|
||||
attn_output = self.dropout1(attn_output, training=training)
|
||||
out1 = x + attn_output
|
||||
|
||||
out2 = self.layernorm2(out1)
|
||||
ffn_output = self.ffn(out2)
|
||||
ffn_output = self.dropout2(ffn_output, training=training)
|
||||
out2 = out1 + ffn_output
|
||||
|
||||
return out2
|
||||
|
||||
|
||||
|
||||
|
||||
class Encoder(tf.keras.layers.Layer):
|
||||
def __init__(self, num_layers=48, d_model_size=1280, num_heads=16, dff=8192, input_vocab_size=50000,
|
||||
rate=0.1, **kwargs):
|
||||
super(Encoder, self).__init__()
|
||||
|
||||
self.d_model_size = d_model_size
|
||||
self.num_layers = num_layers
|
||||
|
||||
self.pos_encoding = positional_encoding(input_vocab_size, self.d_model_size)
|
||||
|
||||
for i in range(num_layers):
|
||||
setattr(self, "layer%i" % i, EncoderLayer(d_model_size, num_heads, dff, rate))
|
||||
|
||||
self.layernorm = tf.keras.layers.LayerNormalization(epsilon=1e-6)
|
||||
self.dropout = tf.keras.layers.Dropout(rate)
|
||||
|
||||
def get_config(self):
|
||||
base_config = super(Encoder, self).get_config()
|
||||
return base_config
|
||||
|
||||
def call(self, x, training):
|
||||
|
||||
seq_len = tf.shape(x)[1]
|
||||
|
||||
mask = 1 - tf.linalg.band_part(tf.ones((seq_len, seq_len)), -1, 0)
|
||||
|
||||
x *= tf.math.sqrt(tf.cast(self.d_model_size, tf.float32))
|
||||
x += self.pos_encoding[:, :seq_len, :]
|
||||
|
||||
x = self.dropout(x, training=training)
|
||||
|
||||
for i in range(self.num_layers):
|
||||
x = getattr(self, "layer%i" % i)(x, training, mask)
|
||||
return self.layernorm(x)
|
||||
|
||||
+246531
File diff suppressed because it is too large
Load Diff
@@ -11,9 +11,11 @@ from generator.gpt2.src import model, sample, encoder
|
||||
import json
|
||||
import numpy as np
|
||||
|
||||
tf.logging.set_verbosity(tf.logging.ERROR)
|
||||
|
||||
class GPT2Generator:
|
||||
|
||||
def __init__(self, generate_num=100, temperature=0.3, top_k=40, top_p=0.8):
|
||||
def __init__(self, generate_num=80, temperature=0.3, top_k=40, top_p=0.8):
|
||||
self.generate_num=generate_num
|
||||
self.temp = temperature
|
||||
self.top_k = top_k
|
||||
@@ -88,7 +90,7 @@ class GPT2Generator:
|
||||
|
||||
def generate(self, prompt, options=None, seed=1):
|
||||
|
||||
debug_print=False
|
||||
debug_print=True
|
||||
prefix = self.prompt_replace(prompt)
|
||||
|
||||
if debug_print:
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
"""
|
||||
Story tree representation
|
||||
|
||||
|
||||
|
||||
"""
|
||||
@@ -0,0 +1,17 @@
|
||||
# Contributors (alphabetically)
|
||||
|
||||
* **[madisonmay](https://github.com/madisonmay)**
|
||||
|
||||
Added Dockerfiles
|
||||
|
||||
* **[Margaret Mitchell et al](https://arxiv.org/abs/1810.03993)**
|
||||
|
||||
Our [usage](./README.md#usage) writeup was loosely inspired by the paper
|
||||
[Model Cards for Model Reporting](https://arxiv.org/abs/1810.03993)
|
||||
and related conversations with some of the authors.
|
||||
|
||||
* **[webproduktion01](https://github.com/webproduktion01)**
|
||||
|
||||
Ported download script to python.
|
||||
|
||||
**[Full code contributors list](https://github.com/openai/gpt-2/contributors).**
|
||||
@@ -0,0 +1,85 @@
|
||||
# Installation
|
||||
|
||||
Git clone this repository, and `cd` into directory for remaining commands
|
||||
```
|
||||
git clone https://github.com/openai/gpt-2.git && cd gpt-2
|
||||
```
|
||||
|
||||
Then, follow instructions for either native or Docker installation.
|
||||
|
||||
## Native Installation
|
||||
|
||||
All steps can optionally be done in a virtual environment using tools such as `virtualenv` or `conda`.
|
||||
|
||||
Install tensorflow 1.12 (with GPU support, if you have a GPU and want everything to run faster)
|
||||
```
|
||||
pip3 install tensorflow==1.12.0
|
||||
```
|
||||
or
|
||||
```
|
||||
pip3 install tensorflow-gpu==1.12.0
|
||||
```
|
||||
|
||||
Install other python packages:
|
||||
```
|
||||
pip3 install -r requirements.txt
|
||||
```
|
||||
|
||||
Download the model data
|
||||
```
|
||||
python3 download_model.py 117M
|
||||
```
|
||||
|
||||
## Docker Installation
|
||||
|
||||
Build the Dockerfile and tag the created image as `gpt-2`:
|
||||
```
|
||||
docker build --tag gpt-2 -f Dockerfile.gpu . # or Dockerfile.cpu
|
||||
```
|
||||
|
||||
Start an interactive bash session from the `gpt-2` docker image.
|
||||
|
||||
You can opt to use the `--runtime=nvidia` flag if you have access to a NVIDIA GPU
|
||||
and a valid install of [nvidia-docker 2.0](https://github.com/nvidia/nvidia-docker/wiki/Installation-(version-2.0)).
|
||||
```
|
||||
docker run --runtime=nvidia -it gpt-2 bash
|
||||
```
|
||||
|
||||
# Running
|
||||
|
||||
| WARNING: Samples are unfiltered and may contain offensive content. |
|
||||
| --- |
|
||||
|
||||
Some of the examples below may include Unicode text characters. Set the environment variable:
|
||||
```
|
||||
export PYTHONIOENCODING=UTF-8
|
||||
```
|
||||
to override the standard stream settings in UTF-8 mode.
|
||||
|
||||
## Unconditional sample generation
|
||||
|
||||
To generate unconditional samples from the small model:
|
||||
```
|
||||
python3 src/generate_unconditional_samples.py | tee /tmp/samples
|
||||
```
|
||||
There are various flags for controlling the samples:
|
||||
```
|
||||
python3 src/generate_unconditional_samples.py --top_k 40 --temperature 0.7 | tee /tmp/samples
|
||||
```
|
||||
|
||||
To check flag descriptions, use:
|
||||
```
|
||||
python3 src/generate_unconditional_samples.py -- --help
|
||||
```
|
||||
|
||||
## Conditional sample generation
|
||||
|
||||
To give the model custom prompts, you can use:
|
||||
```
|
||||
python3 src/interactive_conditional_samples.py --top_k 40
|
||||
```
|
||||
|
||||
To check flag descriptions, use:
|
||||
```
|
||||
python3 src/interactive_conditional_samples.py -- --help
|
||||
```
|
||||
@@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2019 OpenAI
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
@@ -0,0 +1,61 @@
|
||||
# gpt-2
|
||||
|
||||
Code and samples from the paper ["Language Models are Unsupervised Multitask Learners"](https://d4mucfpksywv.cloudfront.net/better-language-models/language-models.pdf).
|
||||
|
||||
For now, we have only released a smaller (117M parameter) version of GPT-2.
|
||||
|
||||
See more details in our [blog post](https://blog.openai.com/better-language-models/).
|
||||
|
||||
## Usage
|
||||
|
||||
This repository is meant to be a starting point for researchers and engineers to experiment with GPT-2-117M. While GPT-2-117M is less proficient than GPT-2-1.5B, it is useful for a wide range of research and applications which could also apply to larger models.
|
||||
|
||||
### Some caveats
|
||||
|
||||
- GPT-2-117M robustness and worst case behaviors are not well-understood. As with any machine-learned model, carefully evaluate GPT-2-117M for your use case, especially if used without fine-tuning or in safety-critical applications where reliability is important.
|
||||
- The dataset our GPT-2-117M was trained on contains many texts with [biases](https://twitter.com/TomerUllman/status/1101485289720242177) and factual inaccuracies, and thus GPT-2-117M is likely to be biased and inaccurate as well.
|
||||
- To avoid having samples mistaken as human-written, we recommend clearly labeling samples as synthetic before wide dissemination. Our models are often incoherent or inaccurate in subtle ways, which takes more than a quick read for a human to notice.
|
||||
|
||||
### Work with us
|
||||
|
||||
Please [let us know](mailto:languagequestions@openai.com) if you’re doing interesting research with or working on applications of GPT-2-117M! We’re especially interested in hearing from and potentially working with those who are studying
|
||||
- Potential malicious use cases and defenses against them (e.g. the detectability of synthetic text)
|
||||
- The extent of problematic content (e.g. bias) being baked into the models and effective mitigations
|
||||
|
||||
## Development
|
||||
|
||||
See [DEVELOPERS.md](./DEVELOPERS.md)
|
||||
|
||||
## Contributors
|
||||
|
||||
See [CONTRIBUTORS.md](./CONTRIBUTORS.md)
|
||||
|
||||
## GPT-2 samples
|
||||
|
||||
| WARNING: Samples are unfiltered and may contain offensive content. |
|
||||
| --- |
|
||||
|
||||
While we have not yet released GPT-2 itself, you can see some samples from it in the `gpt-2-samples` folder.
|
||||
We show unconditional samples with default settings (temperature 1 and no truncation), with temperature 0.7, and with truncation with top_k 40.
|
||||
We show conditional samples, with contexts drawn from `WebText`'s test set, with default settings (temperature 1 and no truncation), with temperature 0.7, and with truncation with top_k 40.
|
||||
|
||||
## Citation
|
||||
|
||||
Please use the following bibtex entry:
|
||||
```
|
||||
@article{radford2019language,
|
||||
title={Language Models are Unsupervised Multitask Learners},
|
||||
author={Radford, Alec and Wu, Jeff and Child, Rewon and Luan, David and Amodei, Dario and Sutskever, Ilya},
|
||||
year={2019}
|
||||
}
|
||||
```
|
||||
|
||||
## Future work
|
||||
|
||||
We may release code for evaluating the models on various benchmarks.
|
||||
|
||||
We are still considering release of the larger models.
|
||||
|
||||
## License
|
||||
|
||||
[MIT](./LICENSE)
|
||||
@@ -0,0 +1,28 @@
|
||||
import os
|
||||
import sys
|
||||
import requests
|
||||
from tqdm import tqdm
|
||||
|
||||
if len(sys.argv) != 2:
|
||||
print('You must enter the model name as a parameter, e.g.: download_model.py 117M')
|
||||
sys.exit(1)
|
||||
|
||||
model = sys.argv[1]
|
||||
|
||||
subdir = os.path.join('models', model)
|
||||
if not os.path.exists(subdir):
|
||||
os.makedirs(subdir)
|
||||
subdir = subdir.replace('\\','/') # needed for Windows
|
||||
|
||||
for filename in ['checkpoint','encoder.json','hparams.json','model.ckpt.data-00000-of-00001', 'model.ckpt.index', 'model.ckpt.meta', 'vocab.bpe']:
|
||||
|
||||
r = requests.get("https://storage.googleapis.com/gpt-2/" + subdir + "/" + filename, stream=True)
|
||||
|
||||
with open(os.path.join(subdir, filename), 'wb') as f:
|
||||
file_size = int(r.headers["content-length"])
|
||||
chunk_size = 1000
|
||||
with tqdm(ncols=100, desc="Fetching " + filename, total=file_size, unit_scale=True) as pbar:
|
||||
# 1k for chunk_size, since Ethernet packet size is around 1500 bytes
|
||||
for chunk in r.iter_content(chunk_size=chunk_size):
|
||||
f.write(chunk)
|
||||
pbar.update(chunk_size)
|
||||
@@ -0,0 +1 @@
|
||||
model.ckpt.data-00000-of-00001
|
||||
@@ -0,0 +1,2 @@
|
||||
model_checkpoint_path: "model.ckpt"
|
||||
all_model_checkpoint_paths: "model.ckpt"
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"n_vocab": 50257,
|
||||
"n_ctx": 1024,
|
||||
"n_embd": 768,
|
||||
"n_head": 12,
|
||||
"n_layer": 12
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,2 @@
|
||||
model.ckpt.data-00000-of-00001
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
model_checkpoint_path: "model.ckpt"
|
||||
all_model_checkpoint_paths: "model.ckpt"
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"n_vocab": 50257,
|
||||
"n_ctx": 1024,
|
||||
"n_embd": 1280,
|
||||
"n_head": 20,
|
||||
"n_layer": 36
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,117 @@
|
||||
"""Byte pair encoding utilities"""
|
||||
|
||||
import os
|
||||
import json
|
||||
import regex as re
|
||||
from functools import lru_cache
|
||||
|
||||
@lru_cache()
|
||||
def bytes_to_unicode():
|
||||
"""
|
||||
Returns list of utf-8 byte and a corresponding list of unicode strings.
|
||||
The reversible bpe codes work on unicode strings.
|
||||
This means you need a large # of unicode characters in your vocab if you want to avoid UNKs.
|
||||
When you're at something like a 10B token dataset you end up needing around 5K for decent coverage.
|
||||
This is a signficant percentage of your normal, say, 32K bpe vocab.
|
||||
To avoid that, we want lookup tables between utf-8 bytes and unicode strings.
|
||||
And avoids mapping to whitespace/control characters the bpe code barfs on.
|
||||
"""
|
||||
bs = list(range(ord("!"), ord("~")+1))+list(range(ord("¡"), ord("¬")+1))+list(range(ord("®"), ord("ÿ")+1))
|
||||
cs = bs[:]
|
||||
n = 0
|
||||
for b in range(2**8):
|
||||
if b not in bs:
|
||||
bs.append(b)
|
||||
cs.append(2**8+n)
|
||||
n += 1
|
||||
cs = [chr(n) for n in cs]
|
||||
return dict(zip(bs, cs))
|
||||
|
||||
def get_pairs(word):
|
||||
"""Return set of symbol pairs in a word.
|
||||
|
||||
Word is represented as tuple of symbols (symbols being variable-length strings).
|
||||
"""
|
||||
pairs = set()
|
||||
prev_char = word[0]
|
||||
for char in word[1:]:
|
||||
pairs.add((prev_char, char))
|
||||
prev_char = char
|
||||
return pairs
|
||||
|
||||
class Encoder:
|
||||
def __init__(self, encoder, bpe_merges, errors='replace'):
|
||||
self.encoder = encoder
|
||||
self.decoder = {v:k for k,v in self.encoder.items()}
|
||||
self.errors = errors # how to handle errors in decoding
|
||||
self.byte_encoder = bytes_to_unicode()
|
||||
self.byte_decoder = {v:k for k, v in self.byte_encoder.items()}
|
||||
self.bpe_ranks = dict(zip(bpe_merges, range(len(bpe_merges))))
|
||||
self.cache = {}
|
||||
|
||||
# Should haved added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
|
||||
self.pat = re.compile(r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+""")
|
||||
|
||||
def bpe(self, token):
|
||||
if token in self.cache:
|
||||
return self.cache[token]
|
||||
word = tuple(token)
|
||||
pairs = get_pairs(word)
|
||||
|
||||
if not pairs:
|
||||
return token
|
||||
|
||||
while True:
|
||||
bigram = min(pairs, key = lambda pair: self.bpe_ranks.get(pair, float('inf')))
|
||||
if bigram not in self.bpe_ranks:
|
||||
break
|
||||
first, second = bigram
|
||||
new_word = []
|
||||
i = 0
|
||||
while i < len(word):
|
||||
try:
|
||||
j = word.index(first, i)
|
||||
new_word.extend(word[i:j])
|
||||
i = j
|
||||
except:
|
||||
new_word.extend(word[i:])
|
||||
break
|
||||
|
||||
if word[i] == first and i < len(word)-1 and word[i+1] == second:
|
||||
new_word.append(first+second)
|
||||
i += 2
|
||||
else:
|
||||
new_word.append(word[i])
|
||||
i += 1
|
||||
new_word = tuple(new_word)
|
||||
word = new_word
|
||||
if len(word) == 1:
|
||||
break
|
||||
else:
|
||||
pairs = get_pairs(word)
|
||||
word = ' '.join(word)
|
||||
self.cache[token] = word
|
||||
return word
|
||||
|
||||
def encode(self, text):
|
||||
bpe_tokens = []
|
||||
for token in re.findall(self.pat, text):
|
||||
token = ''.join(self.byte_encoder[b] for b in token.encode('utf-8'))
|
||||
bpe_tokens.extend(self.encoder[bpe_token] for bpe_token in self.bpe(token).split(' '))
|
||||
return bpe_tokens
|
||||
|
||||
def decode(self, tokens):
|
||||
text = ''.join([self.decoder[token] for token in tokens])
|
||||
text = bytearray([self.byte_decoder[c] for c in text]).decode('utf-8', errors=self.errors)
|
||||
return text
|
||||
|
||||
def get_encoder(model_path):
|
||||
with open(os.path.join(model_path, 'encoder.json'), 'r') as f:
|
||||
encoder = json.load(f)
|
||||
with open(os.path.join(model_path, 'vocab.bpe'), 'r', encoding="utf-8") as f:
|
||||
bpe_data = f.read()
|
||||
bpe_merges = [tuple(merge_str.split()) for merge_str in bpe_data.split('\n')[1:-1]]
|
||||
return Encoder(
|
||||
encoder=encoder,
|
||||
bpe_merges=bpe_merges,
|
||||
)
|
||||
@@ -0,0 +1,176 @@
|
||||
import numpy as np
|
||||
import tensorflow as tf
|
||||
from tensorflow.contrib.training import HParams
|
||||
|
||||
def default_hparams():
|
||||
return HParams(
|
||||
n_vocab=0,
|
||||
n_ctx=1024,
|
||||
n_embd=768,
|
||||
n_head=12,
|
||||
n_layer=12,
|
||||
)
|
||||
|
||||
def shape_list(x):
|
||||
"""Deal with dynamic shape in tensorflow cleanly."""
|
||||
static = x.shape.as_list()
|
||||
dynamic = tf.shape(x)
|
||||
return [dynamic[i] if s is None else s for i, s in enumerate(static)]
|
||||
|
||||
def softmax(x, axis=-1):
|
||||
x = x - tf.reduce_max(x, axis=axis, keepdims=True)
|
||||
ex = tf.exp(x)
|
||||
return ex / tf.reduce_sum(ex, axis=axis, keepdims=True)
|
||||
|
||||
def gelu(x):
|
||||
return 0.5*x*(1+tf.tanh(np.sqrt(2/np.pi)*(x+0.044715*tf.pow(x, 3))))
|
||||
|
||||
def norm(x, scope, *, axis=-1, epsilon=1e-5):
|
||||
"""Normalize to mean = 0, std = 1, then do a diagonal affine transform."""
|
||||
with tf.variable_scope(scope):
|
||||
n_state = x.shape[-1].value
|
||||
g = tf.get_variable('g', [n_state], initializer=tf.constant_initializer(1))
|
||||
b = tf.get_variable('b', [n_state], initializer=tf.constant_initializer(0))
|
||||
u = tf.reduce_mean(x, axis=axis, keepdims=True)
|
||||
s = tf.reduce_mean(tf.square(x-u), axis=axis, keepdims=True)
|
||||
x = (x - u) * tf.rsqrt(s + epsilon)
|
||||
x = x*g + b
|
||||
return x
|
||||
|
||||
def split_states(x, n):
|
||||
"""Reshape the last dimension of x into [n, x.shape[-1]/n]."""
|
||||
*start, m = shape_list(x)
|
||||
return tf.reshape(x, start + [n, m//n])
|
||||
|
||||
def merge_states(x):
|
||||
"""Smash the last two dimensions of x into a single dimension."""
|
||||
*start, a, b = shape_list(x)
|
||||
return tf.reshape(x, start + [a*b])
|
||||
|
||||
def conv1d(x, scope, nf, *, w_init_stdev=0.02):
|
||||
with tf.variable_scope(scope):
|
||||
*start, nx = shape_list(x)
|
||||
w = tf.get_variable('w', [1, nx, nf], initializer=tf.random_normal_initializer(stddev=w_init_stdev))
|
||||
b = tf.get_variable('b', [nf], initializer=tf.constant_initializer(0))
|
||||
c = tf.reshape(tf.matmul(tf.reshape(x, [-1, nx]), tf.reshape(w, [-1, nf]))+b, start+[nf])
|
||||
return c
|
||||
|
||||
def attention_mask(nd, ns, *, dtype):
|
||||
"""1's in the lower triangle, counting from the lower right corner.
|
||||
|
||||
Same as tf.matrix_band_part(tf.ones([nd, ns]), -1, ns-nd), but doesn't produce garbage on TPUs.
|
||||
"""
|
||||
i = tf.range(nd)[:,None]
|
||||
j = tf.range(ns)
|
||||
m = i >= j - ns + nd
|
||||
return tf.cast(m, dtype)
|
||||
|
||||
|
||||
def attn(x, scope, n_state, *, past, hparams):
|
||||
assert x.shape.ndims == 3 # Should be [batch, sequence, features]
|
||||
assert n_state % hparams.n_head == 0
|
||||
if past is not None:
|
||||
assert past.shape.ndims == 5 # Should be [batch, 2, heads, sequence, features], where 2 is [k, v]
|
||||
|
||||
def split_heads(x):
|
||||
# From [batch, sequence, features] to [batch, heads, sequence, features]
|
||||
return tf.transpose(split_states(x, hparams.n_head), [0, 2, 1, 3])
|
||||
|
||||
def merge_heads(x):
|
||||
# Reverse of split_heads
|
||||
return merge_states(tf.transpose(x, [0, 2, 1, 3]))
|
||||
|
||||
def mask_attn_weights(w):
|
||||
# w has shape [batch, heads, dst_sequence, src_sequence], where information flows from src to dst.
|
||||
_, _, nd, ns = shape_list(w)
|
||||
b = attention_mask(nd, ns, dtype=w.dtype)
|
||||
b = tf.reshape(b, [1, 1, nd, ns])
|
||||
w = w*b - tf.cast(1e10, w.dtype)*(1-b)
|
||||
return w
|
||||
|
||||
def multihead_attn(q, k, v):
|
||||
# q, k, v have shape [batch, heads, sequence, features]
|
||||
w = tf.matmul(q, k, transpose_b=True)
|
||||
w = w * tf.rsqrt(tf.cast(v.shape[-1].value, w.dtype))
|
||||
|
||||
w = mask_attn_weights(w)
|
||||
w = softmax(w)
|
||||
a = tf.matmul(w, v)
|
||||
return a
|
||||
|
||||
with tf.variable_scope(scope):
|
||||
c = conv1d(x, 'c_attn', n_state*3)
|
||||
q, k, v = map(split_heads, tf.split(c, 3, axis=2))
|
||||
present = tf.stack([k, v], axis=1)
|
||||
if past is not None:
|
||||
pk, pv = tf.unstack(past, axis=1)
|
||||
k = tf.concat([pk, k], axis=-2)
|
||||
v = tf.concat([pv, v], axis=-2)
|
||||
a = multihead_attn(q, k, v)
|
||||
a = merge_heads(a)
|
||||
a = conv1d(a, 'c_proj', n_state)
|
||||
return a, present
|
||||
|
||||
|
||||
def mlp(x, scope, n_state, *, hparams):
|
||||
with tf.variable_scope(scope):
|
||||
nx = x.shape[-1].value
|
||||
h = gelu(conv1d(x, 'c_fc', n_state))
|
||||
h2 = conv1d(h, 'c_proj', nx)
|
||||
return h2
|
||||
|
||||
|
||||
def block(x, scope, *, past, hparams):
|
||||
with tf.variable_scope(scope):
|
||||
nx = x.shape[-1].value
|
||||
a, present = attn(norm(x, 'ln_1'), 'attn', nx, past=past, hparams=hparams)
|
||||
x = x + a
|
||||
m = mlp(norm(x, 'ln_2'), 'mlp', nx*4, hparams=hparams)
|
||||
x = x + m
|
||||
return x, present
|
||||
|
||||
def past_shape(*, hparams, batch_size=None, sequence=None):
|
||||
return [batch_size, hparams.n_layer, 2, hparams.n_head, sequence, hparams.n_embd // hparams.n_head]
|
||||
|
||||
def expand_tile(value, size):
|
||||
"""Add a new axis of given size."""
|
||||
value = tf.convert_to_tensor(value, name='value')
|
||||
ndims = value.shape.ndims
|
||||
return tf.tile(tf.expand_dims(value, axis=0), [size] + [1]*ndims)
|
||||
|
||||
def positions_for(tokens, past_length):
|
||||
batch_size = tf.shape(tokens)[0]
|
||||
nsteps = tf.shape(tokens)[1]
|
||||
return expand_tile(past_length + tf.range(nsteps), batch_size)
|
||||
|
||||
|
||||
def model(hparams, X, past=None, scope='model', reuse=False):
|
||||
with tf.variable_scope(scope, reuse=reuse):
|
||||
results = {}
|
||||
batch, sequence = shape_list(X)
|
||||
|
||||
wpe = tf.get_variable('wpe', [hparams.n_ctx, hparams.n_embd],
|
||||
initializer=tf.random_normal_initializer(stddev=0.01))
|
||||
wte = tf.get_variable('wte', [hparams.n_vocab, hparams.n_embd],
|
||||
initializer=tf.random_normal_initializer(stddev=0.02))
|
||||
past_length = 0 if past is None else tf.shape(past)[-2]
|
||||
h = tf.gather(wte, X) + tf.gather(wpe, positions_for(X, past_length))
|
||||
|
||||
# Transformer
|
||||
presents = []
|
||||
pasts = tf.unstack(past, axis=1) if past is not None else [None] * hparams.n_layer
|
||||
assert len(pasts) == hparams.n_layer
|
||||
for layer, past in enumerate(pasts):
|
||||
h, present = block(h, 'h%d' % layer, past=past, hparams=hparams)
|
||||
presents.append(present)
|
||||
results['present'] = tf.stack(presents, axis=1)
|
||||
h = norm(h, 'ln_f')
|
||||
|
||||
# Language model loss. Do tokens <n predict token n?
|
||||
h_flat = tf.reshape(h, [batch*sequence, hparams.n_embd])
|
||||
logits = tf.matmul(h_flat, wte, transpose_b=True)
|
||||
logits = tf.reshape(logits, [batch, sequence, hparams.n_vocab])
|
||||
results['logits'] = logits
|
||||
return results
|
||||
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
import tensorflow as tf
|
||||
|
||||
from src.model import *
|
||||
|
||||
def top_k_logits(logits, k):
|
||||
if k == 0:
|
||||
# no truncation
|
||||
return logits
|
||||
|
||||
def _top_k():
|
||||
values, _ = tf.nn.top_k(logits, k=k)
|
||||
min_values = values[:, -1, tf.newaxis]
|
||||
return tf.where(
|
||||
logits < min_values,
|
||||
tf.ones_like(logits, dtype=logits.dtype) * -1e10,
|
||||
logits,
|
||||
)
|
||||
return tf.cond(
|
||||
tf.equal(k, 0),
|
||||
lambda: logits,
|
||||
lambda: _top_k(),
|
||||
)
|
||||
|
||||
|
||||
def sample_sequence(*, hparams, length, start_token=None, batch_size=None, context=None, temperature=1, top_k=0):
|
||||
if start_token is None:
|
||||
assert context is not None, 'Specify exactly one of start_token and context!'
|
||||
else:
|
||||
assert context is None, 'Specify exactly one of start_token and context!'
|
||||
context = tf.fill([batch_size, 1], start_token)
|
||||
|
||||
def step(hparams, tokens, past=None):
|
||||
lm_output = model(hparams=hparams, X=tokens, past=past, reuse=tf.AUTO_REUSE)
|
||||
|
||||
logits = lm_output['logits'][:, :, :hparams.n_vocab]
|
||||
presents = lm_output['present']
|
||||
presents.set_shape(past_shape(hparams=hparams, batch_size=batch_size))
|
||||
return {
|
||||
'logits': logits,
|
||||
'presents': presents,
|
||||
}
|
||||
|
||||
with tf.name_scope('sample_sequence'):
|
||||
# Don't feed the last context token -- leave that to the loop below
|
||||
# TODO: Would be slightly faster if we called step on the entire context,
|
||||
# rather than leaving the last token transformer calculation to the while loop.
|
||||
context_output = step(hparams, context[:, :-1])
|
||||
|
||||
def body(past, prev, output):
|
||||
next_outputs = step(hparams, prev[:, tf.newaxis], past=past)
|
||||
logits = next_outputs['logits'][:, -1, :] / tf.to_float(temperature)
|
||||
logits = top_k_logits(logits, k=top_k)
|
||||
samples = tf.multinomial(logits, num_samples=1, output_dtype=tf.int32)
|
||||
return [
|
||||
tf.concat([past, next_outputs['presents']], axis=-2),
|
||||
tf.squeeze(samples, axis=[1]),
|
||||
tf.concat([output, samples], axis=1),
|
||||
]
|
||||
|
||||
def cond(*args):
|
||||
return True
|
||||
|
||||
_, _, tokens = tf.while_loop(
|
||||
cond=cond, body=body,
|
||||
maximum_iterations=length,
|
||||
loop_vars=[
|
||||
context_output['presents'],
|
||||
context[:, -1],
|
||||
context,
|
||||
],
|
||||
shape_invariants=[
|
||||
tf.TensorShape(past_shape(hparams=hparams, batch_size=batch_size)),
|
||||
tf.TensorShape([batch_size]),
|
||||
tf.TensorShape([batch_size, None]),
|
||||
],
|
||||
back_prop=False, name="EndWhile"
|
||||
)
|
||||
|
||||
return tokens
|
||||
@@ -0,0 +1,142 @@
|
||||
import json
|
||||
import os
|
||||
import numpy as np
|
||||
import tensorflow as tf
|
||||
|
||||
from src.model import *
|
||||
from tensorflow.contrib import predictor
|
||||
from src.sample import *
|
||||
from src.encoder import *
|
||||
import pdb
|
||||
|
||||
pos_action_starts = ["You attack", "You tell", "You use", "You go"]
|
||||
|
||||
|
||||
class TFGenerator():
|
||||
|
||||
def __init__(self, sess, length=75, temperature=0.9, top_k=40):
|
||||
|
||||
seed = None
|
||||
batch_size=1
|
||||
model_path='gpt2/models/117M'
|
||||
self.sess = sess
|
||||
|
||||
self.enc = encoder.get_encoder(model_path)
|
||||
hparams = model.default_hparams()
|
||||
with open(os.path.join(model_path, 'hparams.json')) as f:
|
||||
hparams.override_from_dict(json.load(f))
|
||||
|
||||
pdb.set_trace()
|
||||
|
||||
self.context = tf.placeholder(tf.int32, [batch_size, None])
|
||||
np.random.seed(seed)
|
||||
tf.set_random_seed(seed)
|
||||
self.output = sample.sample_sequence(
|
||||
hparams=hparams, length=length,
|
||||
context=self.context,
|
||||
batch_size=batch_size,
|
||||
)
|
||||
|
||||
saver = tf.train.Saver()
|
||||
ckpt = tf.train.latest_checkpoint(model_path)
|
||||
saver.restore(self.sess, ckpt)
|
||||
|
||||
def generate(self, prompt, options={}):
|
||||
context_tokens = self.enc.encode(prompt)
|
||||
out = self.sess.run(self.output, feed_dict={
|
||||
self.context: [context_tokens for _ in range(1)]
|
||||
})[:, len(context_tokens):]
|
||||
|
||||
text = self.enc.decode(out[0])
|
||||
return text
|
||||
|
||||
def save_model():
|
||||
length=75
|
||||
temperature=0.9
|
||||
top_k=40
|
||||
os.environ['CUDA_VISIBLE_DEVICES'] = '-1'
|
||||
|
||||
with tf.Session() as sess:
|
||||
seed = None
|
||||
batch_size=None
|
||||
model_path='models/774M'
|
||||
|
||||
hparams = default_hparams()
|
||||
with open(os.path.join(model_path, 'hparams.json')) as f:
|
||||
hparams.override_from_dict(json.load(f))
|
||||
|
||||
context = tf.placeholder(tf.int32, [batch_size, None])
|
||||
np.random.seed(seed)
|
||||
tf.set_random_seed(seed)
|
||||
output = sample_sequence(
|
||||
hparams=hparams, length=length,
|
||||
context=context,
|
||||
batch_size=batch_size,
|
||||
)
|
||||
print("***********************",type(output))
|
||||
|
||||
saver = tf.train.Saver()
|
||||
ckpt = tf.train.latest_checkpoint(model_path)
|
||||
saver.restore(sess, ckpt)
|
||||
|
||||
tf.saved_model.simple_save(sess, "./saved_model", inputs={"context": context}, outputs={"output": output})
|
||||
|
||||
def load_model():
|
||||
fraction = 0.6
|
||||
config = config = generate_gpu_config(fraction)
|
||||
path_to_graph = "./saved"
|
||||
|
||||
# tf.saved_model.loader.load(
|
||||
# session,
|
||||
# [tf.saved_model.tag_constants.SERVING],
|
||||
# path_to_graph)
|
||||
|
||||
# output = session.graph.get_tensor_by_name('output:0')
|
||||
# context = session.graph.get_tensor_by_name('context:0')
|
||||
model_path = 'gpt2/models/117M'
|
||||
enc = encoder.get_encoder(model_path)
|
||||
|
||||
predict_fn = predictor.from_saved_model(path_to_graph, config=config)
|
||||
context_tokens = [enc.encode("hello")]
|
||||
predictions = predict_fn({"context": context_tokens})
|
||||
output = enc.decode(predictions["output"][0])
|
||||
print(output)
|
||||
|
||||
return (output, session)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
save_model()
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
from generator.tf.src.encoder import *
|
||||
import googleapiclient.discovery
|
||||
import traceback
|
||||
|
||||
project = "ai-adventure"
|
||||
model = "generator_v1"
|
||||
version = "version2"
|
||||
|
||||
class WebGenerator():
|
||||
|
||||
def __init__(self, credentials_file):
|
||||
os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = credentials_file
|
||||
model_path = './generator/tf/models/117M'
|
||||
self.enc = get_encoder(model_path)
|
||||
|
||||
def predict(self, context_tokens):
|
||||
service = googleapiclient.discovery.build('ml', 'v1')
|
||||
name = 'projects/{}/models/{}'.format(project, model)
|
||||
instance = context_tokens
|
||||
|
||||
if version is not None:
|
||||
name += '/versions/{}'.format(version)
|
||||
|
||||
response = service.projects().predict(
|
||||
name=name,
|
||||
body={'instances': [{'context': instance}]}
|
||||
).execute()
|
||||
|
||||
if 'error' in response:
|
||||
raise RuntimeError(response['error'])
|
||||
|
||||
return response['predictions']
|
||||
|
||||
def generate(self, prompt, options={}):
|
||||
|
||||
while (True):
|
||||
context_tokens = self.enc.encode(prompt)
|
||||
try:
|
||||
pred = self.predict(context_tokens)
|
||||
pred = pred[0]["output"][len(context_tokens):]
|
||||
output = self.enc.decode(pred)
|
||||
return output
|
||||
except:
|
||||
traceback.print_exc()
|
||||
print("generate request failed, trying again")
|
||||
continue
|
||||
@@ -0,0 +1,70 @@
|
||||
from flask import g
|
||||
from flask import session
|
||||
import os
|
||||
from story.utils import *
|
||||
import json
|
||||
from flask import Flask, render_template, request, abort
|
||||
from story.story_manager import *
|
||||
from generator.web.web_generator import *
|
||||
from other.cacher import *
|
||||
import numpy as np
|
||||
|
||||
app = Flask(__name__)
|
||||
app.secret_key = '#d\xe0\xd1\xfb\xee\xa4\xbb\xd0\xf0/e)\xb5g\xdd<`\xc7\xa5\xb0-\xb8d0S'
|
||||
CRED_FILE = "./AI-Adventure-2bb65e3a4e2f.json"
|
||||
generator = WebGenerator(CRED_FILE)
|
||||
story_manager = ConstrainedStoryManager(generator)
|
||||
|
||||
def get_response_string(story_text, possible_actions):
|
||||
string_list = ["\n\n", story_text, "\n\nOptions:" + "\n"]
|
||||
for i, action in enumerate(possible_actions):
|
||||
string_list.append(str(i) + ") " + action + "\n")
|
||||
string_list.append("\nWhich action do you choose? ")
|
||||
|
||||
response = "".join(string_list)
|
||||
return response
|
||||
|
||||
# Shows about. (Should also link to paper when published)
|
||||
@app.route('/about.html')
|
||||
def about():
|
||||
return render_template('about.html')
|
||||
|
||||
# Bread and butter of app, updates story and returns based on choice
|
||||
@app.route('/generate', methods=['POST'])
|
||||
def generate():
|
||||
action = request.form["action"]
|
||||
|
||||
# If there is no story in session, make a new one
|
||||
if "story" not in session or session["story"] is None:
|
||||
print("Starting new story")
|
||||
seed = np.random.randint(100)
|
||||
story_manager.enable_caching(credentials_file=CRED_FILE, seed=seed, bucket_name="dungeon-cache")
|
||||
prompt = get_story_start("classic")
|
||||
story_manager.start_new_story(prompt, seed)
|
||||
possible_actions = story_manager.get_possible_actions()
|
||||
response = get_response_string(str(story_manager.story), possible_actions)
|
||||
|
||||
# If there is a story in session continue from it.
|
||||
else:
|
||||
print("Using existing story")
|
||||
story = session["story"]
|
||||
story_manager.load_story(story, from_json=True)
|
||||
|
||||
result, possible_actions = story_manager.act(action)
|
||||
if result is None:
|
||||
response = "\nInvalid choice. Must be a number from 0 to 3. \n" + "\nWhich action do you choose? "
|
||||
else:
|
||||
response = get_response_string(result, possible_actions)
|
||||
|
||||
session["story"] = story_manager.json_story()
|
||||
print("Returning response")
|
||||
return response
|
||||
|
||||
# Routes to index
|
||||
@app.route('/')
|
||||
def root():
|
||||
session["story"] = None
|
||||
return render_template('index.html')
|
||||
|
||||
if __name__ == '__main__':
|
||||
app.run(host='0.0.0.0', port=8080)
|
||||
@@ -0,0 +1,224 @@
|
||||
"""
|
||||
format of tree is
|
||||
dict {
|
||||
tree_id: tree_id_text
|
||||
context: context text?
|
||||
first_story_block
|
||||
action_results: [act_res1, act_res2, act_res3...]
|
||||
}
|
||||
|
||||
where each action_result's format is:
|
||||
dict{
|
||||
action: action_text
|
||||
result: result_text
|
||||
action_results: [act_res1, act_res2, act_res3...]
|
||||
}
|
||||
"""
|
||||
|
||||
import csv
|
||||
import json
|
||||
import os
|
||||
|
||||
def data_to_forest(filename):
|
||||
|
||||
trees = []
|
||||
rows = []
|
||||
|
||||
with open(filename, newline='') as f:
|
||||
reader = csv.reader(f)
|
||||
for row in reader:
|
||||
rows.append(row)
|
||||
|
||||
for i in range(1, len(rows[0])):
|
||||
tree = {}
|
||||
tree["tree_id"] = rows[0][i]
|
||||
tree["context"] = rows[1][i]
|
||||
tree["first_story_block"] = rows[2][i]
|
||||
tree["action_results"] = []
|
||||
current_action_results = tree["action_results"]
|
||||
row_ind = 3
|
||||
while row_ind < len(rows):
|
||||
action_result = {}
|
||||
action_result["action"] = rows[row_ind][i]
|
||||
if row_ind+1 < len(rows):
|
||||
action_result["result"] = rows[row_ind+1][i]
|
||||
else:
|
||||
action_result["result"] = None
|
||||
action_result["action_results"] = []
|
||||
current_action_results.append(action_result)
|
||||
current_action_results = action_result["action_results"]
|
||||
row_ind += 2
|
||||
trees.append(tree)
|
||||
|
||||
return trees
|
||||
|
||||
|
||||
def build_action_samples_helper(context, story_block, action_results, path, tree_id):
|
||||
|
||||
samples = []
|
||||
|
||||
for i, action_result in enumerate(action_results):
|
||||
new_path = path[:]
|
||||
new_path.append(i)
|
||||
if len(action_result["action_results"]) is 0 and action_result["result"] is not None:
|
||||
row = [tree_id, "".join(str(x) for x in new_path), context, story_block, action_result["action"], action_result["result"]]
|
||||
samples.append(row)
|
||||
else:
|
||||
sub_result = build_action_samples_helper(context, action_result["result"], action_result["action_results"], new_path, tree_id)
|
||||
samples += sub_result
|
||||
|
||||
return samples
|
||||
|
||||
|
||||
def make_write_actions_batch(forest, filename):
|
||||
# Traverse to the bottom levels of each tree
|
||||
with open(filename, mode='w', newline='') as file:
|
||||
writer = csv.writer(file, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
|
||||
writer.writerow(["tree_id", "path", "context", "story_block_1", "previous_action", "story_block_2"])
|
||||
|
||||
for tree in forest:
|
||||
first_story_block = tree["first_story_block"]
|
||||
samples = build_action_samples_helper(tree["context"], first_story_block, tree["action_results"], [], tree["tree_id"])
|
||||
|
||||
for sample in samples:
|
||||
writer.writerow(sample)
|
||||
|
||||
|
||||
def build_result_samples_helper(context, story_block, parent_action_result, path, tree_id):
|
||||
|
||||
samples = []
|
||||
action_results = parent_action_result["action_results"]
|
||||
|
||||
for i, action_result in enumerate(action_results):
|
||||
new_path = path[:]
|
||||
new_path.append(i)
|
||||
if action_result["result"] is None:
|
||||
row = [tree_id, "".join(str(x) for x in new_path), context, story_block, parent_action_result["action"], parent_action_result["result"], action_result["action"]]
|
||||
samples.append(row)
|
||||
else:
|
||||
sub_result = build_result_samples_helper(context, parent_action_result["result"], action_result, new_path, tree_id)
|
||||
samples += sub_result
|
||||
|
||||
return samples
|
||||
|
||||
|
||||
def make_write_results_batch(forest, filename):
|
||||
|
||||
with open(filename, mode='w', newline='') as file:
|
||||
writer = csv.writer(file, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
|
||||
writer.writerow(["tree_id", "path", "context", "story_block_1", "previous_action_1", "story_block_2", "previous_action_2"])
|
||||
|
||||
for tree in forest:
|
||||
first_story_block = tree["first_story_block"]
|
||||
samples = []
|
||||
for i, action_result in enumerate(tree["action_results"]):
|
||||
path = [i]
|
||||
samples += build_result_samples_helper(tree["context"], first_story_block, action_result, path, tree["tree_id"])
|
||||
|
||||
for sample in samples:
|
||||
writer.writerow(sample)
|
||||
|
||||
|
||||
def save_tree(tree, filename):
|
||||
with open(filename, 'w') as fp:
|
||||
json.dump(tree, fp)
|
||||
|
||||
def save_forest(forest, forest_name):
|
||||
|
||||
if not os.path.exists("./" + forest_name):
|
||||
os.mkdir("./" + forest_name)
|
||||
for tree in forest:
|
||||
save_tree(tree, "./" + forest_name + "/" + tree["tree_id"] + ".json")
|
||||
|
||||
def load_tree(filename):
|
||||
with open(filename, 'r') as fp:
|
||||
tree = json.load(fp)
|
||||
return tree
|
||||
|
||||
def load_forest(forest_name):
|
||||
|
||||
files = os.listdir("./" + forest_name)
|
||||
forest = []
|
||||
for file in files:
|
||||
forest.append(load_tree("./" + forest_name + "/" + file))
|
||||
return forest
|
||||
|
||||
|
||||
def csv_to_dict(file):
|
||||
update_dict = {}
|
||||
field_names = []
|
||||
|
||||
with open(file, newline='') as f:
|
||||
reader = csv.reader(f)
|
||||
for row in reader:
|
||||
if len(update_dict) is 0:
|
||||
for item in row:
|
||||
update_dict[item] = []
|
||||
field_names.append(item)
|
||||
|
||||
else:
|
||||
for i, item in enumerate(row):
|
||||
update_dict[field_names[i]].append(item)
|
||||
|
||||
return update_dict
|
||||
|
||||
|
||||
def update_forest_with_results(forest_name, update_file):
|
||||
update_dict = csv_to_dict(update_file)
|
||||
tree_dict = {}
|
||||
tree_filenames = os.listdir("./" + forest_name)
|
||||
|
||||
for file_name in tree_filenames:
|
||||
tree = load_tree("./" + forest_name + "/" + file_name)
|
||||
tree_dict[tree["tree_id"]] = tree
|
||||
|
||||
for i in range(len(update_dict["Input.tree_id"])):
|
||||
tree = tree_dict[update_dict["Input.tree_id"][i]]
|
||||
current_action_results = tree
|
||||
for choice in update_dict["Input.path"][i]:
|
||||
choice_num = int(choice)
|
||||
current_action_results = current_action_results["action_results"][choice_num]
|
||||
|
||||
current_action_results["result"] = update_dict["Answer.result"][i]
|
||||
|
||||
return tree_dict.values()
|
||||
|
||||
def update_forest_with_actions(forest_name, update_file):
|
||||
update_dict = csv_to_dict(update_file)
|
||||
tree_dict = {}
|
||||
tree_filenames = os.listdir("./" + forest_name)
|
||||
|
||||
for file_name in tree_filenames:
|
||||
tree = load_tree("./" + forest_name + "/" + file_name)
|
||||
tree_dict[tree["tree_id"]] = tree
|
||||
|
||||
for i in range(len(update_dict["Input.tree_id"])):
|
||||
tree = tree_dict[update_dict["Input.tree_id"][i]]
|
||||
current_action_results = tree
|
||||
for choice in update_dict["Input.path"][i]:
|
||||
choice_num = int(choice)
|
||||
current_action_results = current_action_results["action_results"][choice_num]
|
||||
|
||||
|
||||
current_action_results["action_results"].append(
|
||||
{"action": update_dict["Answer.action_1"][i], "result": None , "action_results":[]})
|
||||
current_action_results["action_results"].append(
|
||||
{"action": update_dict["Answer.action_2"][i], "result": None, "action_results": []})
|
||||
|
||||
return tree_dict.values()
|
||||
|
||||
old_forest_name = "seed_forest_1.8"
|
||||
new_forest_name = "seed_forest_1.9"
|
||||
update_type = "results"
|
||||
update_file = "mech_turk_" + update_type + "5.csv"
|
||||
|
||||
if update_type is "results":
|
||||
new_forest = update_forest_with_results(old_forest_name, update_file)
|
||||
save_forest(new_forest, new_forest_name)
|
||||
make_write_actions_batch(new_forest, "actions_batch6.csv")
|
||||
else:
|
||||
new_forest = update_forest_with_actions(old_forest_name, update_file)
|
||||
save_forest(new_forest, new_forest_name)
|
||||
make_write_results_batch(new_forest, "results_batch5.csv")
|
||||
|
||||
print("Done")
|
||||
@@ -0,0 +1,3 @@
|
||||
<span id="a">Adventurer@DungeonDream</span>:<span id="b">~</span><span id="c">$</span>
|
||||
Welcome to the Dungeon.<!-- laglaglaglag--><p>Here you will embark on an epic journey through the dreams of an AI to claim victory on your quest. </p><!-- qowifjqwoeijfoqweijfqweoifjqweofijqweoqwoijefoqwijefoijfqiwoefjj -->
|
||||
<p>Good luck</p>
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 15 KiB |
@@ -0,0 +1,285 @@
|
||||
start_text = "<span id='a'>Adventurer@AIDungeon</span>:<span id='b'>~</span><span id='c'>$</span> ./EnterDungeon \n <br/><!-- laglaglaglaglaglaglaglaglaglaglag-->"
|
||||
|
||||
var acceptInput=false
|
||||
var action_waiting = false
|
||||
var inputStr = ""
|
||||
var typing = false
|
||||
var action_list = ["You attack", "You tell", "You use", "You go"]
|
||||
var prompt_num = 0
|
||||
var seed_max = 100
|
||||
var seed_min = 0;
|
||||
prompts = ["You enter a dungeon with your trusty sword and shield. You are searching for the evil necromancer who killed your family. You've heard that he resides at the bottom of the dungeon, guarded by legions of the undead. You enter the first door and see"]
|
||||
|
||||
if(seed == -1){
|
||||
var seed = Math.floor(Math.random() * (+seed_max - +seed_min)) + +seed_min;
|
||||
}
|
||||
|
||||
|
||||
//var seed = 999
|
||||
console.log("Seed is ", seed)
|
||||
|
||||
function isMobileDevice() {
|
||||
return /Android|webOS|iPhone|iPad|iPod|BlackBerry|IEMobile|Opera Mini/i.test(navigator.userAgent)
|
||||
};
|
||||
|
||||
|
||||
function buttonCheck(){
|
||||
if(typing == true){
|
||||
setTimeout(buttonCheck, 500);
|
||||
}
|
||||
else{
|
||||
document.getElementById('buttons').style.visibility='visible';
|
||||
}
|
||||
}
|
||||
|
||||
var StoryTracker = {
|
||||
lastActionResult: "",
|
||||
actions: [],
|
||||
results: [],
|
||||
choices: [],
|
||||
|
||||
// Requests the first story
|
||||
getFirstStory:function(){
|
||||
console.log("Requesting first story")
|
||||
Typer.appendToText(prompts[prompt_num])
|
||||
StoryTracker.requestFirstStory()
|
||||
|
||||
},
|
||||
|
||||
addFirstStory:function(story){
|
||||
StoryTracker.lastActionResult = story
|
||||
StoryTracker.makeActionRequests()
|
||||
Typer.appendToText(story)
|
||||
},
|
||||
|
||||
// Called after requesting options, prints generating msg if waits too lng
|
||||
actionWait:function(){
|
||||
|
||||
if(action_waiting == true){
|
||||
if(typing == true || acceptInput == true){
|
||||
setTimeout(StoryTracker.actionWait, 10000);
|
||||
}
|
||||
else{
|
||||
Typer.appendToText("\n\n Generating options... (~20s)")
|
||||
}
|
||||
}
|
||||
|
||||
},
|
||||
|
||||
// Callback for action request
|
||||
addNextAction:function(action_result){
|
||||
|
||||
// Response receieved no longer waiting
|
||||
var action_results = JSON.parse(action_result)
|
||||
Typer.appendToText("\n\nOptions:")
|
||||
|
||||
action_waiting = false
|
||||
StoryTracker.actions = []
|
||||
StoryTracker.results = []
|
||||
|
||||
for (i = 0; i < 4; i++){
|
||||
|
||||
action_result = action_results[i]
|
||||
|
||||
action = action_result[0]
|
||||
result = action_result[1]
|
||||
|
||||
StoryTracker.results.push(result)
|
||||
var print_action = "\n" + String(i) + ") " + action
|
||||
Typer.appendToText(print_action)
|
||||
|
||||
if (i == 3){
|
||||
Typer.appendToText("\nWhich action do you choose? ")
|
||||
|
||||
// Now we wait for the user to give input to us.
|
||||
acceptInput = true
|
||||
|
||||
if(isMobileDevice()){
|
||||
setTimeout(buttonCheck, 500);
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
// Make a request to the server for result actions
|
||||
makeActionRequests:function(){
|
||||
action_waiting = true
|
||||
setTimeout(StoryTracker.actionWait, 10000);
|
||||
StoryTracker.requestActions(StoryTracker.lastActionResult, JSON.stringify(StoryTracker.choices))
|
||||
},
|
||||
|
||||
requestFirstStory:function(){
|
||||
$.post("/generate", {actions: false, seed, prompt_num},
|
||||
StoryTracker.addFirstStory)
|
||||
},
|
||||
|
||||
requestActions:function(last_action_result, choices){
|
||||
$.post("/generate", {actions: true, seed, prompt_num, last_action_result, choices},
|
||||
StoryTracker.addNextAction)
|
||||
},
|
||||
|
||||
// Called once a choice has been made by button or entering.
|
||||
processInput:function(){
|
||||
var choice_int = parseInt(inputStr, 10)
|
||||
if(choice_int >= 0 && choice_int <= 3){
|
||||
|
||||
console.log("choice_int is %d", choice_int)
|
||||
StoryTracker.choices.push(choice_int)
|
||||
StoryTracker.lastActionResult = StoryTracker.results[choice_int]
|
||||
StoryTracker.makeActionRequests(StoryTracker.firstStory + StoryTracker.lastStory)
|
||||
action_waiting = true
|
||||
Typer.appendToText("\n")
|
||||
Typer.appendToText(StoryTracker.lastActionResult)
|
||||
}
|
||||
else{
|
||||
|
||||
Typer.appendToText("Invalid choice. Must be a number from 0 to 3. \n")
|
||||
Typer.appendToText("\nWhich action do you choose? ")
|
||||
acceptInput = true
|
||||
|
||||
|
||||
if(isMobileDevice()){
|
||||
setTimeout(buttonCheck, 500);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
inputStr = ""
|
||||
}
|
||||
}
|
||||
|
||||
// Used to control the terminal like screen typing
|
||||
var Typer={
|
||||
text: null,
|
||||
index:0,
|
||||
speed:2,
|
||||
|
||||
content:function(){
|
||||
return $("#console").html()
|
||||
},
|
||||
|
||||
appendToText:function(str){
|
||||
str = str.replace(".", "." + "<!-- laglaglag-->")
|
||||
typing = true
|
||||
Typer.text = Typer.text + str;
|
||||
},
|
||||
|
||||
removeChar:function(){
|
||||
var cont=Typer.content()
|
||||
$("#console").html($("#console").html().substring(0,cont.length-1))
|
||||
Typer.text = Typer.text.substring(0, Typer.text.length-1)
|
||||
Typer.index = Typer.index - 1
|
||||
},
|
||||
|
||||
addText:function(){
|
||||
|
||||
if (Typer.index <= Typer.text.length) {
|
||||
var cont=Typer.content()
|
||||
if(cont.substring(cont.length-1,cont.length)=="|")
|
||||
$("#console").html($("#console").html().substring(0,cont.length-1))
|
||||
|
||||
if (Typer.text.substring(Typer.index, Typer.index + Typer.speed).includes(".")){
|
||||
Typer.index += 1
|
||||
}
|
||||
else{
|
||||
Typer.index+=Typer.speed
|
||||
}
|
||||
var text=Typer.text.substring(0,Typer.index)
|
||||
var rtn= new RegExp("\n", "g")
|
||||
|
||||
$("#console").html(text.replace(rtn,"<br/>"))
|
||||
}
|
||||
else{
|
||||
typing = false
|
||||
}
|
||||
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
function writeAppend(str){
|
||||
$("#console").append(str)
|
||||
|
||||
}
|
||||
|
||||
|
||||
function startTyping(){
|
||||
addTextTimer = setInterval("typeWords()", 20)
|
||||
}
|
||||
|
||||
|
||||
function typeWords() {
|
||||
Typer.addText()
|
||||
}
|
||||
|
||||
document.addEventListener("keydown", KeyCheck);
|
||||
|
||||
function KeyCheck(evt) {
|
||||
evt = evt || window.event
|
||||
var charCode = evt.keyCode || evt.which
|
||||
if(charCode == 8){
|
||||
if(inputStr.length > 0){
|
||||
console.log(inputStr)
|
||||
inputStr = inputStr.substring(0, inputStr.length-1)
|
||||
console.log(inputStr)
|
||||
Typer.removeChar()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
document.onkeypress = function(evt) {
|
||||
|
||||
if(acceptInput && !isMobileDevice()){
|
||||
evt = evt || window.event
|
||||
var charCode = evt.keyCode || evt.which
|
||||
|
||||
if(charCode == 13){
|
||||
acceptInput = false
|
||||
Typer.appendToText("\n")
|
||||
StoryTracker.processInput(inputStr)
|
||||
}
|
||||
else{
|
||||
var charStr = String.fromCharCode(charCode)
|
||||
Typer.appendToText(charStr)
|
||||
inputStr = inputStr + charStr
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
function onButtonClick(num){
|
||||
|
||||
document.getElementById('buttons').style.visibility='hidden';
|
||||
|
||||
if (acceptInput == true){
|
||||
acceptInput = false
|
||||
num = String(num)
|
||||
Typer.appendToText(num)
|
||||
Typer.appendToText("\n")
|
||||
inputStr = num
|
||||
StoryTracker.processInput()
|
||||
}
|
||||
}
|
||||
|
||||
function start(){
|
||||
|
||||
Typer.speed=1
|
||||
Typer.text = ""
|
||||
Typer.appendToText(start_text)
|
||||
|
||||
StoryTracker.getFirstStory()
|
||||
|
||||
startTyping()
|
||||
|
||||
console.log("Not mobile device");
|
||||
document.getElementById('buttons').style.visibility='hidden';
|
||||
}
|
||||
|
||||
|
||||
$(document).ready(function() {
|
||||
start()
|
||||
})
|
||||
|
||||
@@ -0,0 +1,138 @@
|
||||
start_text = "<span id='a'>Adventurer@AIDungeon</span>:<span id='b'>~</span><span id='c'>$</span> ./EnterDungeon <br/><!-- laglaglaglaglaglaglaglaglaglaglag-->"
|
||||
|
||||
function isMobileDevice() {
|
||||
return /Android|webOS|iPhone|iPad|iPod|BlackBerry|IEMobile|Opera Mini/i.test(navigator.userAgent)
|
||||
};
|
||||
|
||||
// Used to control the terminal like screen typing
|
||||
var Typer={
|
||||
text:null,
|
||||
inputStr:"",
|
||||
index:0,
|
||||
speed:2,
|
||||
acceptInput:false,
|
||||
inputReady:false,
|
||||
|
||||
content:function(){
|
||||
return $("#console").html()
|
||||
},
|
||||
|
||||
appendToText:function(str){
|
||||
str = str.replace(".", "." + "<!-- laglaglag-->")
|
||||
typing = true
|
||||
Typer.text = Typer.text + str;
|
||||
},
|
||||
|
||||
removeChar:function(){
|
||||
var cont=Typer.content()
|
||||
$("#console").html($("#console").html().substring(0,cont.length-1))
|
||||
Typer.text = Typer.text.substring(0, Typer.text.length-1)
|
||||
Typer.index = Typer.index - 1
|
||||
},
|
||||
|
||||
addText:function(){
|
||||
|
||||
if (Typer.index <= Typer.text.length) {
|
||||
var cont=Typer.content()
|
||||
if(cont.substring(cont.length-1,cont.length)=="|")
|
||||
$("#console").html($("#console").html().substring(0,cont.length-1))
|
||||
|
||||
if (Typer.text.substring(Typer.index, Typer.index + Typer.speed).includes(".")){
|
||||
Typer.index += 1
|
||||
}
|
||||
else{
|
||||
Typer.index+=Typer.speed
|
||||
}
|
||||
var text=Typer.text.substring(0,Typer.index)
|
||||
var rtn= new RegExp("\n", "g")
|
||||
|
||||
$("#console").html(text.replace(rtn,"<br/>"))
|
||||
}
|
||||
else{
|
||||
typing = false
|
||||
}
|
||||
|
||||
},
|
||||
|
||||
sendInput:function(){
|
||||
request = Typer.inputStr
|
||||
$.post("/generate", {action: request}, receiveResponse)
|
||||
Typer.inputStr = ""
|
||||
},
|
||||
|
||||
startTyping:function(){
|
||||
addTextTimer = setInterval("Typer.addText()", 20)
|
||||
},
|
||||
|
||||
|
||||
KeyCheck:function(evt) {
|
||||
evt = evt || window.event
|
||||
var charCode = evt.keyCode || evt.which
|
||||
if(charCode == 8){
|
||||
if(Typer.inputStr.length > 0){
|
||||
console.log(Typer.inputStr)
|
||||
Typer.inputStr = Typer.inputStr.substring(0, Typer.inputStr.length-1)
|
||||
console.log(Typer.inputStr)
|
||||
Typer.removeChar()
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
onKeyPressFunc:function(evt) {
|
||||
|
||||
if(Typer.acceptInput && !isMobileDevice()){
|
||||
evt = evt || window.event
|
||||
var charCode = evt.keyCode || evt.which
|
||||
|
||||
if(charCode == 13){
|
||||
Typer.acceptInput = false
|
||||
Typer.sendInput()
|
||||
}
|
||||
else{
|
||||
var charStr = String.fromCharCode(charCode)
|
||||
Typer.appendToText(charStr)
|
||||
Typer.inputStr = Typer.inputStr + charStr
|
||||
}
|
||||
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
|
||||
function onButtonClick(num){
|
||||
|
||||
document.getElementById('buttons').style.visibility='hidden';
|
||||
|
||||
if (Typer.acceptInput == true){
|
||||
Typer.acceptInput = false
|
||||
num = String(num)
|
||||
Typer.appendToText(num)
|
||||
Typer.inputStr = num
|
||||
Typer.sendInput
|
||||
}
|
||||
}
|
||||
|
||||
function receiveResponse(text){
|
||||
|
||||
Typer.appendToText(text)
|
||||
Typer.acceptInput = true
|
||||
}
|
||||
|
||||
function start(){
|
||||
|
||||
Typer.speed=1
|
||||
Typer.text = ""
|
||||
Typer.appendToText(start_text)
|
||||
Typer.startTyping()
|
||||
request_str = ""
|
||||
$.post("/generate", {action: request_str}, receiveResponse)
|
||||
document.getElementById('buttons').style.visibility='hidden';
|
||||
}
|
||||
|
||||
document.onkeypress = Typer.onKeyPressFunc
|
||||
document.addEventListener("keydown", Typer.KeyCheck);
|
||||
$(document).ready(function() {
|
||||
start()
|
||||
})
|
||||
|
||||
@@ -0,0 +1,131 @@
|
||||
body {
|
||||
background-color: #000
|
||||
}
|
||||
|
||||
#console {
|
||||
font-family: courier, monospace;
|
||||
color: #fff;
|
||||
max-width: 800px;
|
||||
width:80%;
|
||||
margin-left:auto;
|
||||
margin-right:auto;
|
||||
margin-top:calc(30px + 4.0vh);
|
||||
font-size:14px;
|
||||
background-color: #000;
|
||||
}
|
||||
|
||||
.about {
|
||||
font-family: courier, monospace;
|
||||
max-width: 800px;
|
||||
width:80%;
|
||||
margin-left:auto;
|
||||
margin-right:auto;
|
||||
margin-top:100px;
|
||||
font-size:14px;
|
||||
}
|
||||
|
||||
.about text{
|
||||
color: #fff;
|
||||
}
|
||||
.about h2{
|
||||
color: #0bc;
|
||||
font-size:16px;
|
||||
}
|
||||
|
||||
a {
|
||||
color: #0bc;
|
||||
text-decoration: none;
|
||||
}
|
||||
|
||||
#a {
|
||||
color: #0f0;
|
||||
}
|
||||
|
||||
#c {
|
||||
color: #0bc;
|
||||
}
|
||||
|
||||
#b {
|
||||
color: #ff0096;
|
||||
}
|
||||
|
||||
#k {
|
||||
animation: change 1s;
|
||||
}
|
||||
|
||||
#op{
|
||||
color: #888888
|
||||
}
|
||||
|
||||
#SI{
|
||||
background-color:transparent;
|
||||
}
|
||||
|
||||
@keyframes change {
|
||||
0% { color: #333; }
|
||||
50% { color: #0f0; }
|
||||
99% { color: black; }
|
||||
}
|
||||
|
||||
#buttons {
|
||||
position: relative;
|
||||
bottom: 0px;
|
||||
height: calc(100px + 12.0vw);
|
||||
max-width: 800px;
|
||||
width:80%;
|
||||
color: 334;
|
||||
background-color:#000;
|
||||
margin-left:auto;
|
||||
margin-right:auto;
|
||||
display: flex;
|
||||
}
|
||||
|
||||
button {
|
||||
border-color: #fff;
|
||||
color: #fff;
|
||||
margin-top: 60px;
|
||||
margin-left: 3.0vw;
|
||||
margin-right: 3.0vw;
|
||||
margin-bottom: 30px;
|
||||
background-color: #111;
|
||||
width: 14.0vw;
|
||||
height: 12.0vw;
|
||||
text-align: center;
|
||||
border-radius: 25px;
|
||||
font-size:30px;
|
||||
|
||||
}
|
||||
|
||||
button:focus{
|
||||
background-color: #0bc;
|
||||
}
|
||||
|
||||
|
||||
/* Add a black background color to the top navigation */
|
||||
.topnav {
|
||||
background-color: #000;
|
||||
max-width: 800px;
|
||||
width:80%;
|
||||
margin-left:auto;
|
||||
margin-right:auto;
|
||||
|
||||
}
|
||||
|
||||
/* Style the links inside the navigation bar */
|
||||
.topnav a {
|
||||
font-family: courier, monospace;
|
||||
display: inline-block;
|
||||
color: #0bc;
|
||||
text-align: center;
|
||||
padding: 10px 5px;
|
||||
text-decoration: none;
|
||||
font-size:18px;
|
||||
}
|
||||
|
||||
/* Change the color of links on hover */
|
||||
.topnav a:hover {
|
||||
background-color: #0bc;
|
||||
color: white;
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
from flask import g
|
||||
from flask import session
|
||||
import os
|
||||
from story.utils import *
|
||||
import json
|
||||
from flask import Flask, render_template, request, abort
|
||||
from story.story_manager import *
|
||||
from generator.web.web_generator import *
|
||||
from other.cacher import *
|
||||
import numpy as np
|
||||
|
||||
app = Flask(__name__)
|
||||
app.secret_key = '#d\xe0\xd1\xfb\xee\xa4\xbb\xd0\xf0/e)\xb5g\xdd<`\xc7\xa5\xb0-\xb8d0S'
|
||||
CRED_FILE = "./AI-Adventure-2bb65e3a4e2f.json"
|
||||
|
||||
# Bread and butter of app, updates story and returns based on choice
|
||||
@app.route('/generate', methods=['POST'])
|
||||
def generate():
|
||||
action = request.form["action"]
|
||||
|
||||
# If there is no story in session, make a new one
|
||||
if "prompt" not in session or session["prompt"] is None:
|
||||
session["prompt"] = get_story_start("classic")
|
||||
response = "Continue the initial story block:\n\n" + session["prompt"]
|
||||
|
||||
# If there is a story in session continue from it.
|
||||
elif "story" not in session or session["story"] is None:
|
||||
story_start = session["prompt"] + action
|
||||
story = Story(story_start)
|
||||
|
||||
session["story"] = story.to_json()
|
||||
response = "Enter the action then two newlines then the result:\n\n> "
|
||||
|
||||
else:
|
||||
story = Story("")
|
||||
story = story.initialize_from_json(session["story"])
|
||||
action_result = action.split("\n")
|
||||
story.add_to_story(action, result)
|
||||
session["story"] = story.to_json()
|
||||
|
||||
response = "Enter the action then two newlines then the result:\n\n> "
|
||||
|
||||
print("Returning response")
|
||||
return response
|
||||
|
||||
# Routes to index
|
||||
@app.route('/')
|
||||
def root():
|
||||
session["story"] = None
|
||||
return render_template('index.html')
|
||||
|
||||
if __name__ == '__main__':
|
||||
app.run(host='0.0.0.0', port=8080)
|
||||
@@ -0,0 +1,43 @@
|
||||
<html>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<head>
|
||||
<!-- Google Analytics -->
|
||||
<script>
|
||||
(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
|
||||
(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
|
||||
m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
|
||||
})(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
|
||||
|
||||
ga('create', 'UA-139423787-1', 'auto');
|
||||
ga('send', 'pageview');
|
||||
</script>
|
||||
<!-- End Google Analytics -->
|
||||
|
||||
<title>About</title>
|
||||
<link rel="stylesheet" type="text/css" href="static/style.css">
|
||||
</head>
|
||||
<div class="topnav">
|
||||
<a href="../">Home</a>
|
||||
<a href="http://patreon.com/AIDungeon">Support</a>
|
||||
</div>
|
||||
<div class="about">
|
||||
<h2> About AI Dungeon </h2>
|
||||
<text>
|
||||
AI Dungeon is an AI generated text adventure that uses deep learning to create each adventure. It uses OpenAI's new <a href="https://openai.com/blog/better-language-models/">GPT-2 model</a>, which has 117 million parameters, to generate each story block and possible action.
|
||||
|
||||
<br><br> The first couple sentences of AIDungeon and the action verbs are handcrafted, but everything else is not. For each choice that is made, the initial prompt, the last story block, and the last action are fed into the neural network. The resulting story and action options are then output by the model.
|
||||
|
||||
<br><br> In order to speed up the experience some of the first sets of stories, choices and results have been pre-generated and cached. After enough choices, however, it will start taking longer (around 20s) to generate each result.
|
||||
|
||||
<br><br> As you can probably tell there's still a ways to go before AI will be your group's dungeon master, but even after running hundreds of adventures it still manages to surprise me in interesting ways. I've had a lot of fun making this and hope you enjoy it too.
|
||||
|
||||
<br><br> Warning: The GPT-2 model was trained on a huge amount of internet text so there might be offensive content.
|
||||
|
||||
<br><br> If you want to contact me about suggested improvements, comments, etc... feel free to email me about them at aidungeon.io@gmail.com.
|
||||
|
||||
<br><br> AI Dungeon was created by Nick Walton with the support of the <a href="https://pcc.cs.byu.edu/">BYU Percepton Cognition and Control Lab</a>, Alan Walton and Max Robinson.
|
||||
</text>
|
||||
</div>
|
||||
|
||||
|
||||
</html>
|
||||
@@ -0,0 +1,37 @@
|
||||
|
||||
<html>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<head>
|
||||
<!-- Google Analytics -->
|
||||
<script>
|
||||
(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
|
||||
(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
|
||||
m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
|
||||
})(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
|
||||
|
||||
ga('create', 'UA-139423787-1', 'auto');
|
||||
ga('send', 'pageview');
|
||||
</script>
|
||||
<!-- End Google Analytics -->
|
||||
<link rel="shortcut icon" href="{{ url_for('static', filename='favicon.ico') }}">
|
||||
<title>AI Dungeon</title>
|
||||
<script type="text/javascript" src="https://ajax.googleapis.com/ajax/libs/jquery/1.4.2/jquery.min.js"></script>
|
||||
<link rel="stylesheet" type="text/css" href="static/style.css">
|
||||
</head>
|
||||
<div class="topnav">
|
||||
<a href="about.html">About</a>
|
||||
<a href="http://patreon.com/AIDungeon/overview">Support</a>
|
||||
<a href=".">Restart</a>
|
||||
</div>
|
||||
<body>
|
||||
<script type="text/javascript" src="static/script.js">
|
||||
</script>
|
||||
<div id="console"></div>
|
||||
</body>
|
||||
<div id="buttons">
|
||||
<button onclick="onButtonClick(0)">0</button>
|
||||
<button onclick="onButtonClick(1)">1</button>
|
||||
<button onclick="onButtonClick(2)">2</button>
|
||||
<button onclick="onButtonClick(3)">3</button>
|
||||
</div>
|
||||
</html>
|
||||
Reference in New Issue
Block a user