mirror of
https://github.com/wassname/Clover-Edition.git
synced 2026-10-29 12:40:16 +08:00
update finetune
This commit is contained in:
Nick Walton committed
1 parent
de9a7da9ed
commit
5132fb5bbd
4 files changed
+50011
-259
No files matched your search
@@ -0,0 +1 @@
|
||||
{"!": 0, "\"": 1, "#": 2, "$": 3, "%": 4, "&": 5, "'": 6, "(": 7, ")": 8, "*": 9, "+": 10, ",": 11, "-": 12, ".": 13, "/": 14, "0": 15, "1": 16, "2": 17, "3": 18, "4": 19, "5": 20, "6": 21, "7": 22, "8": 23, "9": 24, ":": 25, ";": 26, "<": 27, "=": 28, ">": 29, "?": 30, "@": 31, "A": 32, "B": 33, "C": 34, "D": 35, "E": 36, "F": 37, "G": 38, "H": 39, "I": 40, "J": 41, "K": 42, "L": 43, "M": 44, "N": 45, "O": 46, "P": 47, "Q": 48, "R": 49, "S": 50, "T": 51, "U": 52, "V": 53, "W": 54, "X": 55, "Y": 56, "Z": 57, "[": 58, "\\": 59, "]": 60, "^": 61, "_": 62, "`": 63, "a": 64, "b": 65, "c": 66, "d": 67, "e": 68, "f": 69, "g": 70, "h": 71, "i": 72, "j": 73, "k": 74, "l": 75, "m": 76, "n": 77, "o": 78, "p": 79, "q": 80, "r": 81, "s": 82, "t": 83, "u": 84, "v": 85, "w": 86, "x": 87, "y": 88, "z": 89, "{": 90, "|": 91, "}": 92, "~": 93, "\u00a1": 94, "\u00a2": 95, "\u00a3": 96, "\u00a4": 97, "\u00a5": 98, "\u00a6": 99, "\u00a7": 100, "\u00a8": 101, "\u00a9": 102, "\u00aa": 103, "\u00ab": 104, "\u00ac": 105, "\u00ae": 106, "\u00af": 107, "\u00b0": 108, "\u00b1": 109, "\u00b2": 110, "\u00b3": 111, "\u00b4": 112, "\u00b5": 113, "\u00b6": 114, "\u00b7": 115, "\u00b8": 116, "\u00b9": 117, "\u00ba": 118, "\u00bb": 119, "\u00bc": 120, "\u00bd": 121, "\u00be": 122, "\u00bf": 123, "\u00c0": 124, "\u00c1": 125, "\u00c2": 126, "\u00c3": 127, "\u00c4": 128, "\u00c5": 129, "\u00c6": 130, "\u00c7": 131, "\u00c8": 132, "\u00c9": 133, "\u00ca": 134, "\u00cb": 135, "\u00cc": 136, "\u00cd": 137, "\u00ce": 138, "\u00cf": 139, "\u00d0": 140, "\u00d1": 141, "\u00d2": 142, "\u00d3": 143, "\u00d4": 144, "\u00d5": 145, "\u00d6": 146, "\u00d7": 147, "\u00d8": 148, "\u00d9": 149, "\u00da": 150, "\u00db": 151, "\u00dc": 152, "\u00dd": 153, "\u00de": 154, "\u00df": 155, "\u00e0": 156, "\u00e1": 157, "\u00e2": 158, "\u00e3": 159, "\u00e4": 160, "\u00e5": 161, "\u00e6": 162, "\u00e7": 163, "\u00e8": 164, "\u00e9": 165, "\u00ea": 166, "\u00eb": 167, "\u00ec": 168, "\u00ed": 169, "\u00ee": 170, "\u00ef": 171, "\u00f0": 172, "\u00f1": 173, "\u00f2": 174, "\u00f3": 175, "\u00f4": 176, "\u00f5": 177, "\u00f6": 178, "\u00f7": 179, "\u00f8": 180, "\u00f9": 181, "\u00fa": 182, "\u00fb": 183, "\u00fc": 184, "\u00fd": 185, "\u00fe": 186, "\u00ff": 187, "\u0100": 188, "\u0101": 189, "\u0102": 190, "\u0103": 191, "\u0104": 192, "\u0105": 193, "\u0106": 194, "\u0107": 195, "\u0108": 196, "\u0109": 197, "\u010a": 198, "\u010b": 199, "\u010c": 200, "\u010d": 201, "\u010e": 202, "\u010f": 203, "\u0110": 204, "\u0111": 205, "\u0112": 206, "\u0113": 207, "\u0114": 208, "\u0115": 209, "\u0116": 210, "\u0117": 211, "\u0118": 212, "\u0119": 213, "\u011a": 214, "\u011b": 215, "\u011c": 216, "\u011d": 217, "\u011e": 218, "\u011f": 219, "\u0120": 220, "\u0121": 221, "\u0122": 222, "\u0123": 223, "\u0124": 224, "\u0125": 225, "\u0126": 226, "\u0127": 227, "\u0128": 228, "\u0129": 229, "\u012a": 230, "\u012b": 231, "\u012c": 232, "\u012d": 233, "\u012e": 234, "\u012f": 235, "\u0130": 236, "\u0131": 237, "\u0132": 238, "\u0133": 239, "\u0134": 240, "\u0135": 241, "\u0136": 242, "\u0137": 243, "\u0138": 244, "\u0139": 245, "\u013a": 246, "\u013b": 247, "\u013c": 248, "\u013d": 249, "\u013e": 250, "\u013f": 251, "\u0140": 252, "\u0141": 253, "\u0142": 254, "\u0143": 255, "\u0120t": 256, "\u0120a": 257, "he": 258, "in": 259, "re": 260, "on": 261, "\u0120the": 262, "er": 263, "\u0120s": 264, "at": 265, "\u0120w": 266, "\u0120o": 267, "en": 268, "\u0120c": 269, "it": 270, "is": 271, "an": 272, "or": 273, "es": 274, "\u0120b": 275, "ed": 276, "\u0120f": 277, "ing": 278, "\u0120p": 279, "ou": 280, "\u0120an": 281, "al": 282, "ar": 283, "\u0120to": 284, "\u0120m": 285, "\u0120of": 286, "\u0120in": 287, "\u0120d": 288, "\u0120h": 289, "\u0120and": 290, "ic": 291, "as": 292, "le": 293, "\u0120th": 294, "ion": 295, "om": 296, "ll": 297, "ent": 298, "\u0120n": 299, "\u0120l": 300, "st": 301, "\u0120re": 302, "ve": 303, "\u0120e": 304, "ro": 305, "ly": 306, "\u0120be": 307, "\u0120g": 308, "\u0120T": 309, "ct": 310, "\u0120S": 311, "id": 312, "ot": 313, "\u0120I": 314, "ut": 315, "et": 316, "\u0120A": 317, "\u0120is": 318, "\u0120on": 319, "im": 320, "am": 321, "ow": 322, "ay": 323, "ad": 324, "se": 325, "\u0120that": 326, "\u0120C": 327, "ig": 328, "\u0120for": 329, "ac": 330, "\u0120y": 331, "ver": 332, "ur": 333, "\u0120u": 334, "ld": 335, "\u0120st": 336, "\u0120M": 337, "'s": 338, "\u0120he": 339, "\u0120it": 340, "ation": 341, "ith": 342, "ir": 343, "ce": 344, "\u0120you": 345, "il": 346, "\u0120B": 347, "\u0120wh": 348, "ol": 349, "\u0120P": 350, "\u0120with": 351, "\u01201": 352, "ter": 353, "ch": 354, "\u0120as": 355, "\u0120we": 356, "\u0120(": 357, "nd": 358, "ill": 359, "\u0120D": 360, "if": 361, "\u01202": 362, "ag": 363, "ers": 364, "ke": 365, "\u0120\"": 366, "\u0120H": 367, "em": 368, "\u0120con": 369, "\u0120W": 370, "\u0120R": 371, "her": 372, "\u0120was": 373, "\u0120r": 374, "od": 375, "\u0120F": 376, "ul": 377, "ate": 378, "\u0120at": 379, "ri": 380, "pp": 381, "oLine truncated
|
||||
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"n_vocab": 50257,
|
||||
"n_ctx": 1024,
|
||||
"n_embd": 1600,
|
||||
"n_head": 25,
|
||||
"n_layer": 48
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -1,274 +1,17 @@
|
||||
import tarfile
|
||||
import os
|
||||
import json
|
||||
import requests
|
||||
import sys
|
||||
import shutil
|
||||
import re
|
||||
from tqdm import tqdm, trange
|
||||
import numpy as np
|
||||
import tensorflow as tf
|
||||
from tensorflow.core.protobuf import rewriter_config_pb2
|
||||
from tensorflow.python.client import device_lib
|
||||
import time
|
||||
from datetime import datetime
|
||||
import csv
|
||||
import argparse
|
||||
|
||||
# if in Google Colaboratory
|
||||
try:
|
||||
from google.colab import drive
|
||||
except:
|
||||
pass
|
||||
|
||||
import gpt_2_simple as gpt2
|
||||
|
||||
from gpt_2_simple.src import model, sample, encoder, memory_saving_gradients
|
||||
from gpt_2_simple.src.load_dataset import load_dataset, Sampler
|
||||
from gpt_2_simple.src.accumulate import AccumulatingOptimizer
|
||||
|
||||
|
||||
def get_available_gpus():
|
||||
local_device_protos = device_lib.list_local_devices()
|
||||
return [x.name for x in local_device_protos if x.device_type == 'GPU']
|
||||
|
||||
|
||||
def finetune(sess,
|
||||
dataset,
|
||||
steps=-1,
|
||||
model_name='124M',
|
||||
model_dir='models',
|
||||
combine=50000,
|
||||
batch_size=1,
|
||||
learning_rate=0.0001,
|
||||
accumulate_gradients=5,
|
||||
restore_from='latest',
|
||||
run_name='run1',
|
||||
checkpoint_dir='checkpoint',
|
||||
sample_every=100,
|
||||
sample_length=1023,
|
||||
sample_num=1,
|
||||
save_every=1000,
|
||||
print_every=1,
|
||||
max_checkpoints=1,
|
||||
use_memory_saving_gradients=False,
|
||||
only_train_transformer_layers=False,
|
||||
optimizer='adam',
|
||||
overwrite=False):
|
||||
"""Finetunes the model on the given dataset.
|
||||
Adapted from https://github.com/nshepperd/gpt-2/blob/finetuning/train.py.
|
||||
See that file for parameter definitions.
|
||||
"""
|
||||
|
||||
SAMPLE_DIR = 'samples'
|
||||
|
||||
checkpoint_path = os.path.join(checkpoint_dir, run_name)
|
||||
|
||||
def maketree(path):
|
||||
try:
|
||||
os.makedirs(path)
|
||||
except:
|
||||
pass
|
||||
|
||||
maketree(checkpoint_path)
|
||||
files = [f for f in os.listdir(checkpoint_path)]
|
||||
for file in ['hparams.json', 'encoder.json', 'vocab.bpe']:
|
||||
try:
|
||||
shutil.copyfile(os.path.join(model_dir, model_name, file),
|
||||
os.path.join(checkpoint_path, file))
|
||||
except FileNotFoundError as fnf_error:
|
||||
print("You need to download the GPT-2 model first via download_gpt2()")
|
||||
raise (fnf_error)
|
||||
|
||||
enc = encoder.get_encoder(checkpoint_path)
|
||||
hparams = model.default_hparams()
|
||||
with open(os.path.join(checkpoint_path, 'hparams.json')) as f:
|
||||
hparams.override_from_dict(json.load(f))
|
||||
|
||||
if sample_length > hparams.n_ctx:
|
||||
raise ValueError(
|
||||
"Can't get samples longer than window size: %s" % hparams.n_ctx)
|
||||
|
||||
if model_name not in ['117M', '124M']:
|
||||
use_memory_saving_gradients = True
|
||||
only_train_transformer_layers = True
|
||||
accumulate_gradients = 1
|
||||
|
||||
context = tf.compat.v1.placeholder(tf.int32, [batch_size, None])
|
||||
output = model.model(hparams=hparams, X=context)
|
||||
loss = tf.reduce_mean(
|
||||
input_tensor=tf.nn.sparse_softmax_cross_entropy_with_logits(
|
||||
labels=context[:, 1:], logits=output['logits'][:, :-1]))
|
||||
|
||||
tf_sample = sample.sample_sequence(
|
||||
hparams=hparams,
|
||||
length=sample_length,
|
||||
context=context,
|
||||
batch_size=batch_size,
|
||||
temperature=1.0,
|
||||
top_k=40)
|
||||
|
||||
all_vars = [v for v in tf.compat.v1.trainable_variables() if 'model' in v.name]
|
||||
train_vars = [v for v in all_vars if '/h' in v.name] if only_train_transformer_layers else all_vars
|
||||
|
||||
if optimizer == 'adam':
|
||||
opt = tf.compat.v1.train.AdamOptimizer(learning_rate=learning_rate)
|
||||
elif optimizer == 'sgd':
|
||||
opt = tf.compat.v1.train.GradientDescentOptimizer(learning_rate=learning_rate)
|
||||
|
||||
if accumulate_gradients > 1:
|
||||
if use_memory_saving_gradients:
|
||||
exit("Memory saving gradients are not implemented for gradient accumulation yet.")
|
||||
opt = AccumulatingOptimizer(
|
||||
opt=opt,
|
||||
var_list=train_vars)
|
||||
opt_reset = opt.reset()
|
||||
opt_compute = opt.compute_gradients(loss)
|
||||
opt_apply = opt.apply_gradients()
|
||||
summary_loss = tf.compat.v1.summary.scalar('loss', opt_apply)
|
||||
else:
|
||||
if use_memory_saving_gradients:
|
||||
opt_grads = memory_saving_gradients.gradients(loss, train_vars)
|
||||
else:
|
||||
opt_grads = tf.gradients(ys=loss, xs=train_vars)
|
||||
opt_grads = list(zip(opt_grads, train_vars))
|
||||
opt_apply = opt.apply_gradients(opt_grads)
|
||||
summary_loss = tf.compat.v1.summary.scalar('loss', loss)
|
||||
|
||||
summary_log = tf.compat.v1.summary.FileWriter(checkpoint_path)
|
||||
|
||||
saver = tf.compat.v1.train.Saver(
|
||||
var_list=all_vars,
|
||||
max_to_keep=max_checkpoints)
|
||||
sess.run(tf.compat.v1.global_variables_initializer())
|
||||
|
||||
if restore_from == 'latest':
|
||||
ckpt = tf.train.latest_checkpoint(checkpoint_path)
|
||||
if ckpt is None:
|
||||
# Get fresh GPT weights if new run.
|
||||
ckpt = tf.train.latest_checkpoint(
|
||||
os.path.join(model_dir, model_name))
|
||||
elif restore_from == 'fresh':
|
||||
ckpt = tf.train.latest_checkpoint(
|
||||
os.path.join(model_dir, model_name))
|
||||
else:
|
||||
ckpt = tf.train.latest_checkpoint(restore_from)
|
||||
print('Loading checkpoint', ckpt)
|
||||
saver.restore(sess, ckpt)
|
||||
|
||||
print('Loading dataset...')
|
||||
chunks = load_dataset(enc, dataset, combine)
|
||||
data_sampler = Sampler(chunks)
|
||||
print('dataset has', data_sampler.total_size, 'tokens')
|
||||
print('Training...')
|
||||
|
||||
counter = 1
|
||||
counter_path = os.path.join(checkpoint_path, 'counter')
|
||||
if os.path.exists(counter_path) and restore_from == 'latest':
|
||||
# Load the step number if we're resuming a run
|
||||
# Add 1 so we don't immediately try to save again
|
||||
with open(counter_path, 'r') as fp:
|
||||
counter = int(fp.read()) + 1
|
||||
counter_base = counter
|
||||
|
||||
def save():
|
||||
maketree(checkpoint_path)
|
||||
print(
|
||||
'Saving',
|
||||
os.path.join(checkpoint_path,
|
||||
'model-{}').format(counter - 1))
|
||||
saver.save(
|
||||
sess,
|
||||
os.path.join(checkpoint_path, 'model'),
|
||||
global_step=counter - 1)
|
||||
with open(counter_path, 'w') as fp:
|
||||
fp.write(str(counter - 1) + '\n')
|
||||
|
||||
def generate_samples():
|
||||
context_tokens = data_sampler.sample(1)
|
||||
all_text = []
|
||||
index = 0
|
||||
while index < sample_num:
|
||||
out = sess.run(
|
||||
tf_sample,
|
||||
feed_dict={context: batch_size * [context_tokens]})
|
||||
for i in range(min(sample_num - index, batch_size)):
|
||||
text = enc.decode(out[i])
|
||||
text = '======== SAMPLE {} ========\n{}\n'.format(
|
||||
index + 1, text)
|
||||
all_text.append(text)
|
||||
index += 1
|
||||
print(text)
|
||||
maketree(os.path.join(SAMPLE_DIR, run_name))
|
||||
with open(
|
||||
os.path.join(SAMPLE_DIR, run_name,
|
||||
'samples-{}').format(counter), 'w') as fp:
|
||||
fp.write('\n'.join(all_text))
|
||||
|
||||
def sample_batch():
|
||||
return [data_sampler.sample(1024) for _ in range(batch_size)]
|
||||
|
||||
if overwrite and restore_from == 'latest':
|
||||
for file in files:
|
||||
if file.startswith('model') or file.startswith('events'):
|
||||
os.remove(os.path.join(checkpoint_path, file))
|
||||
save()
|
||||
|
||||
avg_loss = (0.0, 0.0)
|
||||
start_time = time.time()
|
||||
|
||||
if steps:
|
||||
steps = int(steps)
|
||||
|
||||
try:
|
||||
while True:
|
||||
if steps > 0 and counter == (counter_base + steps):
|
||||
save()
|
||||
return
|
||||
if (counter - 1) % save_every == 0 and counter > 1:
|
||||
save()
|
||||
if (counter - 1) % sample_every == 0 and counter > 1:
|
||||
generate_samples()
|
||||
|
||||
if accumulate_gradients > 1:
|
||||
sess.run(opt_reset)
|
||||
for _ in range(accumulate_gradients):
|
||||
sess.run(
|
||||
opt_compute, feed_dict={context: sample_batch()})
|
||||
(v_loss, v_summary) = sess.run((opt_apply, summary_loss))
|
||||
else:
|
||||
(_, v_loss, v_summary) = sess.run(
|
||||
(opt_apply, loss, summary_loss),
|
||||
feed_dict={context: sample_batch()})
|
||||
|
||||
summary_log.add_summary(v_summary, counter)
|
||||
|
||||
if counter % print_every == 0:
|
||||
avg_loss = (avg_loss[0] * 0.99 + v_loss,
|
||||
avg_loss[1] * 0.99 + 1.0)
|
||||
|
||||
print(
|
||||
'[{counter} | {time:2.2f}] loss={loss:2.2f} avg={avg:2.2f}'
|
||||
.format(
|
||||
counter=counter,
|
||||
time=time.time() - start_time,
|
||||
loss=v_loss,
|
||||
avg=avg_loss[0] / avg_loss[1]))
|
||||
|
||||
counter += 1
|
||||
except KeyboardInterrupt:
|
||||
print('interrupted')
|
||||
save()
|
||||
|
||||
model_name = "1558M"
|
||||
if not os.path.isdir(os.path.join("models", model_name)):
|
||||
print("Downloading ", model_name, " model...")
|
||||
gpt2.download_gpt2(model_name=model_name) # model is saved into current directory under /models/124M/
|
||||
|
||||
file_name = "merged_first_person.txt"
|
||||
file_name = "merged-first-person.txt"
|
||||
|
||||
sess = gpt2.start_tf_sess()
|
||||
finetune(sess,
|
||||
gpt2.finetune(sess,
|
||||
file_name,
|
||||
model_name=model_name,
|
||||
steps=10000)
|
||||
|
||||
Reference in new issue
Block a user