From a31174de04ebe80780f1b1f620d685617f9e70d3 Mon Sep 17 00:00:00 2001 From: Thomas Lemoine Date: Thu, 30 Mar 2023 02:08:47 -0400 Subject: [PATCH 1/2] fixed variable name --- src/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main.py b/src/main.py index 4b92ba3..d6f6988 100644 --- a/src/main.py +++ b/src/main.py @@ -98,7 +98,7 @@ def get_embedding(text: str) -> np.ndarray: return np.array(result["data"][0]["embedding"]) def print_out_dataset_stuff(): - with open(PATH_TO_DATASET, 'rb') as f: + with open(PATH_TO_DATASET_PKL, 'rb') as f: dataset = pickle.load(f) embeddings_len = len(dataset.embedding_strings) From 8d225a18d7d6669dc4b993ac6c915e1092dce503 Mon Sep 17 00:00:00 2001 From: henri123lemoine Date: Thu, 30 Mar 2023 02:26:08 -0400 Subject: [PATCH 2/2] Repaired text_splitter and settings. create_dataset success --- src/dataset/create_dataset.py | 57 +++++++++++++++++++++-- src/dataset/settings.py | 6 +-- src/dataset/text_splitter.py | 3 +- src/main.py | 88 ++++++++++++++++++----------------- 4 files changed, 102 insertions(+), 52 deletions(-) diff --git a/src/dataset/create_dataset.py b/src/dataset/create_dataset.py index 7379ec4..8929be7 100644 --- a/src/dataset/create_dataset.py +++ b/src/dataset/create_dataset.py @@ -161,13 +161,60 @@ class Dataset: else: raise MissingDataException("Entry has no source.") - random_number = random.random() - if random_number > self.fraction_of_articles_to_use: - continue - # if we specified custom sources, only include articles from those sources if (self.custom_sources is not None) and (entry['source'] not in self.custom_sources): continue + + + if entry["source"] == 'alignment forum': + if int(entry['score'].replace('−', '-')) < 70: continue + elif entry["source"] == 'lesswrong': + if int(entry['score'].replace('−', '-')) < 150: continue + elif entry["source"] == 'arxiv': + if 'citation_level' != '0': continue + + # Dict describing the proportion of each source we want: + # E.g.: {'arxiv': 0.5, 'youtube': 0.5, 'lesswrong': 1.0} + desired_source_proportions = { + "https://aipulse.org": 1, + "ebook": 0.2, + "https://qualiacomputing.com": 0.02, + "alignment forum": 1, + "lesswrong": .5, + "manual": 1, + "arxiv": 0.1, + "https://deepmindsafetyresearch.medium.com/": 1, + "waitbutwhy.com": 1, + "GitHub": 1, + "https://aiimpacts.org": 0.3, + "arbital.com": 0.2, + "carado.moe": 0.3, + "nonarxiv_papers": 0.3, + "https://vkrakovna.wordpress.com": .5, + "https://jsteinhardt.wordpress.com": .5, + "audio-transcripts": 0.2, + "https://intelligence.org": .2, + "youtube": 0.07, + "reports": 0.4, + "https://aisafety.camp": 1, + "curriculum": 1, + "https://www.yudkowsky.net": 1, + "distill": 1, + "Cold Takes": 0.5, + "printouts": 1, + "gwern.net": 1, + "generative.ink": 1, + "greaterwrong.com": 0.2 + } + + random_number = random.random() + if random_number > desired_source_proportions[entry['source']]: + continue + + # if we specified a fraction of articles to use, only use that fraction from the remaining articles + random_number = random.random() + if random_number < self.fraction_of_articles_to_use: + continue # Get title, author, date, URL, tags, and text title, author, date_published, url, tags, text = self.extract_info_from_article(entry) @@ -220,7 +267,7 @@ class Dataset: embeddings[i] = embedding['embedding'] return batch_idx, embeddings - batch_size = 200 + batch_size = 500 rate_limit = 3500 / 60 # Maximum embeddings per second start = time.time() diff --git a/src/dataset/settings.py b/src/dataset/settings.py index 2362cc7..28cb3da 100644 --- a/src/dataset/settings.py +++ b/src/dataset/settings.py @@ -7,6 +7,6 @@ LEN_EMBEDDINGS = 1536 MAX_LEN_PROMPT = 4095 # This may be 8191, unsure. current_file_path = Path(__file__).resolve() -PATH_TO_RAW_DATA = str(current_file_path.parent / 'dataset' / 'data' / 'alignment_texts.jsonl') -PATH_TO_DATASET_PKL = str(current_file_path.parent / 'dataset' / 'data' / 'dataset.pkl') -PATH_TO_DATASET_DICT_PKL = str(current_file_path.parent / 'dataset' / 'data' / 'dataset_dict.pkl') \ No newline at end of file +PATH_TO_RAW_DATA = str(current_file_path.parent / 'data' / 'alignment_texts.jsonl') +PATH_TO_DATASET_PKL = str(current_file_path.parent / 'data' / 'dataset.pkl') +PATH_TO_DATASET_DICT_PKL = str(current_file_path.parent / 'data' / 'dataset_dict.pkl') \ No newline at end of file diff --git a/src/dataset/text_splitter.py b/src/dataset/text_splitter.py index d4229ae..221a7b2 100644 --- a/src/dataset/text_splitter.py +++ b/src/dataset/text_splitter.py @@ -38,7 +38,8 @@ class TokenSplitter: def _text_splitter(self, text: str, signature: str) -> List[str]: """Splits text into blocks of tokens according to chatgpt's tokenizer.""" - enc = self.encoding.encode # takes a string and returns a list of ints (tokens) + # enc = self.encoding.encode # takes a string and returns a list of ints (tokens) + enc = self.encoding.encode_ordinary # takes a string and returns a list of ints (tokens) dec = self.encoding.decode # takes a list of ints (tokens) and returns a string tok_len = lambda x: len(enc(x)) # length of a string in tokens diff --git a/src/main.py b/src/main.py index d6f6988..8f8913c 100644 --- a/src/main.py +++ b/src/main.py @@ -27,8 +27,7 @@ if str(src_path) not in sys.path: from dataset import create_dataset #from assistant import semantic_search -from settings import PATH_TO_DATASET_PKL, EMBEDDING_MODEL, PATH_TO_DATASET_DICT_PKL - +from settings import EMBEDDING_MODEL import numpy as np import matplotlib.pyplot as plt @@ -43,54 +42,57 @@ def load_rawdata_into_pkl(): print(answer) """ # List of possible sources: - all_sources = ["https://aipulse.org", "ebook", "https://qualiacomputing.com", "alignment forum", "lesswrong", "manual", "arxiv", "https://deepmindsafetyresearch.medium.com", "waitbutwhy.com", "GitHub", "https://aiimpacts.org", "arbital.com", "carado.moe", "nonarxiv_papers", "https://vkrakovna.wordpress.com", "https://jsteinhardt.wordpress.com", "audio-transcripts", "https://intelligence.org", "youtube", "reports", "https://aisafety.camp", "curriculum", "https://www.yudkowsky.net", "distill", "Cold Takes", "printouts", "gwern.net", "generative.ink", "greaterwrong.com"] # These sources do not have a source field in the .jsonl file + all_sources = ["https://aipulse.org", "ebook", "https://qualiacomputing.com", "alignment forum", "lesswrong", "manual", "arxiv", "https://deepmindsafetyresearch.medium.com", "waitbutwhy.com", "GitHub", "https://aiimpacts.org", "arbital.com", "carado.moe", "nonarxiv_papers", "https://vkrakovna.wordpress.com", "https://jsteinhardt.wordpress.com", "audio-transcripts", "https://intelligence.org", "youtube", "reports", "https://aisafety.camp", "curriculum", "https://www.yudkowsky.net", "distill", + "Cold Takes", "printouts", "gwern.net", "generative.ink", "greaterwrong.com"] # These last do not have a source field in the .jsonl file # List of sources we are using for the test run: custom_sources = [ - # "https://aipulse.org", - # "ebook", - # "https://qualiacomputing.com", - # "alignment forum", - # "lesswrong", + "https://aipulse.org", + "ebook", + "https://qualiacomputing.com", + "alignment forum", + "lesswrong", "manual", - # "arxiv", - # "https://deepmindsafetyresearch.medium.com", + "arxiv", + "https://deepmindsafetyresearch.medium.com/", "waitbutwhy.com", - # "GitHub", - # "https://aiimpacts.org", - # "arbital.com", - # "carado.moe", - # "nonarxiv_papers", - # "https://vkrakovna.wordpress.com", + "GitHub", + "https://aiimpacts.org", + "arbital.com", + "carado.moe", + "nonarxiv_papers", + "https://vkrakovna.wordpress.com", "https://jsteinhardt.wordpress.com", - # "audio-transcripts", - # "https://intelligence.org", - # "youtube", - # "reports", + "audio-transcripts", + "https://intelligence.org", + "youtube", + "reports", "https://aisafety.camp", "curriculum", "https://www.yudkowsky.net", - # "distill", - # "Cold Takes", - # "printouts", - # "gwern.net", - # "generative.ink", - # "greaterwrong.com" + "distill", + "Cold Takes", + "printouts", + "gwern.net", + "generative.ink", + "greaterwrong.com" ] dataset = create_dataset.Dataset( custom_sources=custom_sources, rate_limit_per_minute=3500, min_tokens_per_block=200, max_tokens_per_block=300, - # fraction_of_articles_to_use=1/2000 + fraction_of_articles_to_use=1/100, ) dataset.get_alignment_texts() - print(len(dataset.embedding_strings)) - dataset.get_embeddings() - # dataset.save_embeddings("data/embeddings.npy") - dataset.save_class() - # # dataset = pickle.load(open("dataset.pkl", "rb")) + print(len(dataset.embedding_strings)) + print(dataset.total_word_count) + print(dataset.total_block_count) + print(dataset.articles_count) + + dataset.get_embeddings() + dataset.save_data() @retry(wait=wait_random_exponential(min=1, max=20), stop=stop_after_attempt(4)) def get_embedding(text: str) -> np.ndarray: @@ -154,18 +156,18 @@ def plot_likelihood(embeddings, num_buckets=200): if __name__ == "__main__": - # load_rawdata_into_pkl() + load_rawdata_into_pkl() # print_out_dataset_stuff() - with open(PATH_TO_DATASET_PKL, 'rb') as f: - dataset = pickle.load(f) + # with open(PATH_TO_DATASET_PKL, 'rb') as f: + # dataset = pickle.load(f) - dataset_dict = { - "embedding_strings": dataset.embedding_strings, - "embeddings": dataset.embeddings, - "embeddings_metadata_index": dataset.embeddings_metadata_index, - "metadata": dataset.metadata - } + # dataset_dict = { + # "embedding_strings": dataset.embedding_strings, + # "embeddings": dataset.embeddings, + # "embeddings_metadata_index": dataset.embeddings_metadata_index, + # "metadata": dataset.metadata + # } - with open(PATH_TO_DATASET_DICT_PKL, 'wb') as f: - pickle.dump(dataset_dict, f) \ No newline at end of file + # with open(PATH_TO_DATASET_DICT_PKL, 'wb') as f: + # pickle.dump(dataset_dict, f) \ No newline at end of file