diff --git a/src/dataset.py b/src/dataset.py index dc26257..fc939b5 100644 --- a/src/dataset.py +++ b/src/dataset.py @@ -17,10 +17,13 @@ from tenacity import ( from text_splitter import TokenSplitter, split_into_sentences from settings import PATH_TO_DATA, PATH_TO_EMBEDDINGS, PATH_TO_DATASET, EMBEDDING_MODEL, LEN_EMBEDDINGS - +import os +from tqdm.auto import tqdm import openai + openai.api_key = os.environ.get('OPENAI_API_KEY') + error_count_dict = { "Entry has no source.": 0, "Entry has no title.": 0, @@ -54,7 +57,7 @@ class Dataset: self.metadata: List[Tuple[str]] = [] # List of tuples, each containing the title of an article, its URL, and text. E.g.: [('title', 'url', 'text'), ...] self.embedding_strings: List[str] = [] # List of strings, each being a few paragraphs from a single article (not exceeding 1000 words). - self.embeddings_metadata_index: List[int] # List of integers, each being the index of the article from which the embedding string was taken. + self.embeddings_metadata_index: List[int] = [] # List of integers, each being the index of the article from which the embedding string was taken. self.articles_count: DefaultDict[str, int] = defaultdict(int) # Number of articles per source. E.g.: {'source1': 10, 'source2': 20, 'total': 30} @@ -131,7 +134,7 @@ class Dataset: def get_alignment_texts(self): text_splitter = TokenSplitter(self.min_tokens_per_block, self.max_tokens_per_block) with jsonlines.open(self.jsonl_data_path, "r") as reader: - for entry in reader: + for entry in tqdm(reader): try: if 'source' not in entry: if 'url' in entry and entry['url'] == "https://www.cold-takes.com/": @@ -211,7 +214,7 @@ class Dataset: with concurrent.futures.ThreadPoolExecutor() as executor: futures = [executor.submit(get_embedding_at_index, text, i) for i, text in enumerate(self.embedding_strings)] num_completed = 0 - for future in concurrent.futures.as_completed(futures): + for future in tqdm(concurrent.futures.as_completed(futures)): i, embedding = future.result() self.embeddings[i] = embedding num_completed += 1 @@ -238,8 +241,6 @@ class Dataset: pass """ - - def save_embeddings(self, path: str): np.save(path, self.embeddings) @@ -280,22 +281,22 @@ if __name__ == "__main__": # List of sources we are using for the test run: custom_sources = [ - "https://aipulse.org", - "ebook", + # "https://aipulse.org", + # "ebook", # "https://qualiacomputing.com", # "alignment forum", # "lesswrong", "manual", # "arxiv", - "https://deepmindsafetyresearch.medium.com", + # "https://deepmindsafetyresearch.medium.com", "waitbutwhy.com", "GitHub", # "https://aiimpacts.org", # "arbital.com", - "carado.moe", + # "carado.moe", # "nonarxiv_papers", - "https://vkrakovna.wordpress.com", - "https://jsteinhardt.wordpress.com", + # "https://vkrakovna.wordpress.com", + # "https://jsteinhardt.wordpress.com", "audio-transcripts", # "https://intelligence.org", # "youtube", @@ -320,10 +321,10 @@ if __name__ == "__main__": # fraction_of_articles_to_use=1/2000 ) dataset.get_alignment_texts() - # dataset.get_embeddings() - # dataset.save_embeddings("embeddings.npy") + dataset.get_embeddings() + dataset.save_embeddings("embeddings.npy") - # dataset.save_class("dataset.pkl") + dataset.save_class("data/dataset.pkl") # dataset = pickle.load(open("dataset.pkl", "rb")) \ No newline at end of file diff --git a/src/settings.py b/src/settings.py index e80339d..571faff 100644 --- a/src/settings.py +++ b/src/settings.py @@ -6,7 +6,7 @@ COMPLETIONS_MODEL = "text-davinci-003" LEN_EMBEDDINGS = 1536 MAX_LEN_PROMPT = 4095 # This may be 8191, unsure. -project_path = Path(__file__).parent.parent.parent +project_path = Path(__file__).parent.parent#.parent PATH_TO_DATA = project_path / "src" / "data" / "alignment_texts.jsonl" # Path to the dataset .jsonl file. PATH_TO_EMBEDDINGS = project_path / "src" / "data" / "embeddings.npy" # Path to the saved embeddings (.npy) file. PATH_TO_DATASET = project_path / "src" / "data" / "dataset.pkl" # Path to the saved dataset (.pkl) file, containing the dataset class object. diff --git a/src/text_splitter.py b/src/text_splitter.py index 7a9a5c7..67f130b 100644 --- a/src/text_splitter.py +++ b/src/text_splitter.py @@ -8,7 +8,7 @@ import nltk # Download the Punkt tokenizer if you haven't already -# nltk.download("punkt") +nltk.download("punkt") def split_into_sentences(text: str) -> List[str]: """ diff --git a/web/api/requirements.txt b/web/api/requirements.txt new file mode 100644 index 0000000..9220901 --- /dev/null +++ b/web/api/requirements.txt @@ -0,0 +1,4 @@ +openai +typing +numpy +tenacity \ No newline at end of file diff --git a/web/src/pages/index.tsx b/web/src/pages/index.tsx index 85fa319..37a1617 100644 --- a/web/src/pages/index.tsx +++ b/web/src/pages/index.tsx @@ -26,25 +26,23 @@ const Home: NextPage = () => { so if anyone else is considering this, you can message us to coordinate sharing the embeddings to avoid redundancy.
-Python serverless test:
+ +Get the most semantic similar results to a query:
loading...
: ( + + {loading ?loading...
: ( // display results in list