Updated main, success

.gitignore

@@ -135,5 +135,6 @@ dmypy.json
 *.pickle
 *.pkl
 *.DS_Store
+src/tmp.py
 
 .vercel/

src/main.py

@@ -27,7 +27,7 @@ if str(src_path) not in sys.path:
 
 from dataset import create_dataset
 #from assistant import semantic_search
-from settings import EMBEDDING_MODEL
+from settings import EMBEDDING_MODEL, PATH_TO_DATASET_DICT_PKL
 
 import numpy as np
 import matplotlib.pyplot as plt
@@ -82,7 +82,7 @@ def load_rawdata_into_pkl():
         custom_sources=custom_sources, 
         rate_limit_per_minute=3500, 
         min_tokens_per_block=200, max_tokens_per_block=300, 
-        fraction_of_articles_to_use=1/100,
+        # fraction_of_articles_to_use=1/150,
     )
     dataset.get_alignment_texts()
     
@@ -92,7 +92,7 @@ def load_rawdata_into_pkl():
     print(dataset.articles_count)
     
     dataset.get_embeddings()
-    dataset.save_data()    
+    dataset.save_data()
     
 @retry(wait=wait_random_exponential(min=1, max=20), stop=stop_after_attempt(4))
 def get_embedding(text: str) -> np.ndarray:
@@ -156,18 +156,8 @@ def plot_likelihood(embeddings, num_buckets=200):
 
 
 if __name__ == "__main__":
-    load_rawdata_into_pkl()
+    # load_rawdata_into_pkl()
     # print_out_dataset_stuff()
     
-    # with open(PATH_TO_DATASET_PKL, 'rb') as f:
-    #     dataset = pickle.load(f)
-    
-    # dataset_dict = {
-    #     "embedding_strings": dataset.embedding_strings,
-    #     "embeddings": dataset.embeddings,
-    #     "embeddings_metadata_index": dataset.embeddings_metadata_index,
-    #     "metadata": dataset.metadata
-    # }
-    
-    # with open(PATH_TO_DATASET_DICT_PKL, 'wb') as f:
-    #     pickle.dump(dataset_dict, f)
+    with open(PATH_TO_DATASET_DICT_PKL, 'rb') as f:
+        dataset = pickle.load(f)