mirror of
https://github.com/wassname/stampy-chat.git
synced 2026-09-11 12:50:34 +08:00
Merge remote-tracking branch 'origin/creating_embeddings_dataset' into main
This commit is contained in:
Fraser
@@ -161,13 +161,60 @@ class Dataset:
|
||||
else:
|
||||
raise MissingDataException("Entry has no source.")
|
||||
|
||||
random_number = random.random()
|
||||
if random_number > self.fraction_of_articles_to_use:
|
||||
continue
|
||||
|
||||
# if we specified custom sources, only include articles from those sources
|
||||
if (self.custom_sources is not None) and (entry['source'] not in self.custom_sources):
|
||||
continue
|
||||
|
||||
|
||||
if entry["source"] == 'alignment forum':
|
||||
if int(entry['score'].replace('−', '-')) < 70: continue
|
||||
elif entry["source"] == 'lesswrong':
|
||||
if int(entry['score'].replace('−', '-')) < 150: continue
|
||||
elif entry["source"] == 'arxiv':
|
||||
if 'citation_level' != '0': continue
|
||||
|
||||
# Dict describing the proportion of each source we want:
|
||||
# E.g.: {'arxiv': 0.5, 'youtube': 0.5, 'lesswrong': 1.0}
|
||||
desired_source_proportions = {
|
||||
"https://aipulse.org": 1,
|
||||
"ebook": 0.2,
|
||||
"https://qualiacomputing.com": 0.02,
|
||||
"alignment forum": 1,
|
||||
"lesswrong": .5,
|
||||
"manual": 1,
|
||||
"arxiv": 0.1,
|
||||
"https://deepmindsafetyresearch.medium.com/": 1,
|
||||
"waitbutwhy.com": 1,
|
||||
"GitHub": 1,
|
||||
"https://aiimpacts.org": 0.3,
|
||||
"arbital.com": 0.2,
|
||||
"carado.moe": 0.3,
|
||||
"nonarxiv_papers": 0.3,
|
||||
"https://vkrakovna.wordpress.com": .5,
|
||||
"https://jsteinhardt.wordpress.com": .5,
|
||||
"audio-transcripts": 0.2,
|
||||
"https://intelligence.org": .2,
|
||||
"youtube": 0.07,
|
||||
"reports": 0.4,
|
||||
"https://aisafety.camp": 1,
|
||||
"curriculum": 1,
|
||||
"https://www.yudkowsky.net": 1,
|
||||
"distill": 1,
|
||||
"Cold Takes": 0.5,
|
||||
"printouts": 1,
|
||||
"gwern.net": 1,
|
||||
"generative.ink": 1,
|
||||
"greaterwrong.com": 0.2
|
||||
}
|
||||
|
||||
random_number = random.random()
|
||||
if random_number > desired_source_proportions[entry['source']]:
|
||||
continue
|
||||
|
||||
# if we specified a fraction of articles to use, only use that fraction from the remaining articles
|
||||
random_number = random.random()
|
||||
if random_number < self.fraction_of_articles_to_use:
|
||||
continue
|
||||
|
||||
# Get title, author, date, URL, tags, and text
|
||||
title, author, date_published, url, tags, text = self.extract_info_from_article(entry)
|
||||
@@ -220,7 +267,7 @@ class Dataset:
|
||||
embeddings[i] = embedding['embedding']
|
||||
return batch_idx, embeddings
|
||||
|
||||
batch_size = 200
|
||||
batch_size = 500
|
||||
rate_limit = 3500 / 60 # Maximum embeddings per second
|
||||
|
||||
start = time.time()
|
||||
|
||||
@@ -7,6 +7,6 @@ LEN_EMBEDDINGS = 1536
|
||||
MAX_LEN_PROMPT = 4095 # This may be 8191, unsure.
|
||||
|
||||
current_file_path = Path(__file__).resolve()
|
||||
PATH_TO_RAW_DATA = str(current_file_path.parent / 'dataset' / 'data' / 'alignment_texts.jsonl')
|
||||
PATH_TO_DATASET_PKL = str(current_file_path.parent / 'dataset' / 'data' / 'dataset.pkl')
|
||||
PATH_TO_DATASET_DICT_PKL = str(current_file_path.parent / 'dataset' / 'data' / 'dataset_dict.pkl')
|
||||
PATH_TO_RAW_DATA = str(current_file_path.parent / 'data' / 'alignment_texts.jsonl')
|
||||
PATH_TO_DATASET_PKL = str(current_file_path.parent / 'data' / 'dataset.pkl')
|
||||
PATH_TO_DATASET_DICT_PKL = str(current_file_path.parent / 'data' / 'dataset_dict.pkl')
|
||||
@@ -38,7 +38,8 @@ class TokenSplitter:
|
||||
|
||||
def _text_splitter(self, text: str, signature: str) -> List[str]:
|
||||
"""Splits text into blocks of tokens according to chatgpt's tokenizer."""
|
||||
enc = self.encoding.encode # takes a string and returns a list of ints (tokens)
|
||||
# enc = self.encoding.encode # takes a string and returns a list of ints (tokens)
|
||||
enc = self.encoding.encode_ordinary # takes a string and returns a list of ints (tokens)
|
||||
dec = self.encoding.decode # takes a list of ints (tokens) and returns a string
|
||||
tok_len = lambda x: len(enc(x)) # length of a string in tokens
|
||||
|
||||
|
||||
+46
-44
@@ -27,8 +27,7 @@ if str(src_path) not in sys.path:
|
||||
|
||||
from dataset import create_dataset
|
||||
#from assistant import semantic_search
|
||||
from settings import PATH_TO_DATASET_PKL, EMBEDDING_MODEL, PATH_TO_DATASET_DICT_PKL
|
||||
|
||||
from settings import EMBEDDING_MODEL
|
||||
|
||||
import numpy as np
|
||||
import matplotlib.pyplot as plt
|
||||
@@ -43,54 +42,57 @@ def load_rawdata_into_pkl():
|
||||
print(answer)
|
||||
"""
|
||||
# List of possible sources:
|
||||
all_sources = ["https://aipulse.org", "ebook", "https://qualiacomputing.com", "alignment forum", "lesswrong", "manual", "arxiv", "https://deepmindsafetyresearch.medium.com", "waitbutwhy.com", "GitHub", "https://aiimpacts.org", "arbital.com", "carado.moe", "nonarxiv_papers", "https://vkrakovna.wordpress.com", "https://jsteinhardt.wordpress.com", "audio-transcripts", "https://intelligence.org", "youtube", "reports", "https://aisafety.camp", "curriculum", "https://www.yudkowsky.net", "distill", "Cold Takes", "printouts", "gwern.net", "generative.ink", "greaterwrong.com"] # These sources do not have a source field in the .jsonl file
|
||||
all_sources = ["https://aipulse.org", "ebook", "https://qualiacomputing.com", "alignment forum", "lesswrong", "manual", "arxiv", "https://deepmindsafetyresearch.medium.com", "waitbutwhy.com", "GitHub", "https://aiimpacts.org", "arbital.com", "carado.moe", "nonarxiv_papers", "https://vkrakovna.wordpress.com", "https://jsteinhardt.wordpress.com", "audio-transcripts", "https://intelligence.org", "youtube", "reports", "https://aisafety.camp", "curriculum", "https://www.yudkowsky.net", "distill",
|
||||
"Cold Takes", "printouts", "gwern.net", "generative.ink", "greaterwrong.com"] # These last do not have a source field in the .jsonl file
|
||||
|
||||
# List of sources we are using for the test run:
|
||||
custom_sources = [
|
||||
# "https://aipulse.org",
|
||||
# "ebook",
|
||||
# "https://qualiacomputing.com",
|
||||
# "alignment forum",
|
||||
# "lesswrong",
|
||||
"https://aipulse.org",
|
||||
"ebook",
|
||||
"https://qualiacomputing.com",
|
||||
"alignment forum",
|
||||
"lesswrong",
|
||||
"manual",
|
||||
# "arxiv",
|
||||
# "https://deepmindsafetyresearch.medium.com",
|
||||
"arxiv",
|
||||
"https://deepmindsafetyresearch.medium.com/",
|
||||
"waitbutwhy.com",
|
||||
# "GitHub",
|
||||
# "https://aiimpacts.org",
|
||||
# "arbital.com",
|
||||
# "carado.moe",
|
||||
# "nonarxiv_papers",
|
||||
# "https://vkrakovna.wordpress.com",
|
||||
"GitHub",
|
||||
"https://aiimpacts.org",
|
||||
"arbital.com",
|
||||
"carado.moe",
|
||||
"nonarxiv_papers",
|
||||
"https://vkrakovna.wordpress.com",
|
||||
"https://jsteinhardt.wordpress.com",
|
||||
# "audio-transcripts",
|
||||
# "https://intelligence.org",
|
||||
# "youtube",
|
||||
# "reports",
|
||||
"audio-transcripts",
|
||||
"https://intelligence.org",
|
||||
"youtube",
|
||||
"reports",
|
||||
"https://aisafety.camp",
|
||||
"curriculum",
|
||||
"https://www.yudkowsky.net",
|
||||
# "distill",
|
||||
# "Cold Takes",
|
||||
# "printouts",
|
||||
# "gwern.net",
|
||||
# "generative.ink",
|
||||
# "greaterwrong.com"
|
||||
"distill",
|
||||
"Cold Takes",
|
||||
"printouts",
|
||||
"gwern.net",
|
||||
"generative.ink",
|
||||
"greaterwrong.com"
|
||||
]
|
||||
|
||||
dataset = create_dataset.Dataset(
|
||||
custom_sources=custom_sources,
|
||||
rate_limit_per_minute=3500,
|
||||
min_tokens_per_block=200, max_tokens_per_block=300,
|
||||
# fraction_of_articles_to_use=1/2000
|
||||
fraction_of_articles_to_use=1/100,
|
||||
)
|
||||
dataset.get_alignment_texts()
|
||||
print(len(dataset.embedding_strings))
|
||||
dataset.get_embeddings()
|
||||
# dataset.save_embeddings("data/embeddings.npy")
|
||||
|
||||
dataset.save_class()
|
||||
# # dataset = pickle.load(open("dataset.pkl", "rb"))
|
||||
print(len(dataset.embedding_strings))
|
||||
print(dataset.total_word_count)
|
||||
print(dataset.total_block_count)
|
||||
print(dataset.articles_count)
|
||||
|
||||
dataset.get_embeddings()
|
||||
dataset.save_data()
|
||||
|
||||
@retry(wait=wait_random_exponential(min=1, max=20), stop=stop_after_attempt(4))
|
||||
def get_embedding(text: str) -> np.ndarray:
|
||||
@@ -98,7 +100,7 @@ def get_embedding(text: str) -> np.ndarray:
|
||||
return np.array(result["data"][0]["embedding"])
|
||||
|
||||
def print_out_dataset_stuff():
|
||||
with open(PATH_TO_DATASET, 'rb') as f:
|
||||
with open(PATH_TO_DATASET_PKL, 'rb') as f:
|
||||
dataset = pickle.load(f)
|
||||
|
||||
embeddings_len = len(dataset.embedding_strings)
|
||||
@@ -154,18 +156,18 @@ def plot_likelihood(embeddings, num_buckets=200):
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# load_rawdata_into_pkl()
|
||||
load_rawdata_into_pkl()
|
||||
# print_out_dataset_stuff()
|
||||
|
||||
with open(PATH_TO_DATASET_PKL, 'rb') as f:
|
||||
dataset = pickle.load(f)
|
||||
# with open(PATH_TO_DATASET_PKL, 'rb') as f:
|
||||
# dataset = pickle.load(f)
|
||||
|
||||
dataset_dict = {
|
||||
"embedding_strings": dataset.embedding_strings,
|
||||
"embeddings": dataset.embeddings,
|
||||
"embeddings_metadata_index": dataset.embeddings_metadata_index,
|
||||
"metadata": dataset.metadata
|
||||
}
|
||||
# dataset_dict = {
|
||||
# "embedding_strings": dataset.embedding_strings,
|
||||
# "embeddings": dataset.embeddings,
|
||||
# "embeddings_metadata_index": dataset.embeddings_metadata_index,
|
||||
# "metadata": dataset.metadata
|
||||
# }
|
||||
|
||||
with open(PATH_TO_DATASET_DICT_PKL, 'wb') as f:
|
||||
pickle.dump(dataset_dict, f)
|
||||
# with open(PATH_TO_DATASET_DICT_PKL, 'wb') as f:
|
||||
# pickle.dump(dataset_dict, f)
|
||||
Reference in New Issue
Block a user