diff --git a/.gitignore b/.gitignore index c2d9be9..dd28442 100644 --- a/.gitignore +++ b/.gitignore @@ -135,5 +135,6 @@ dmypy.json *.pickle *.pkl *.DS_Store +src/tmp.py .vercel/ \ No newline at end of file diff --git a/src/main.py b/src/main.py index 8f8913c..900a14f 100644 --- a/src/main.py +++ b/src/main.py @@ -27,7 +27,7 @@ if str(src_path) not in sys.path: from dataset import create_dataset #from assistant import semantic_search -from settings import EMBEDDING_MODEL +from settings import EMBEDDING_MODEL, PATH_TO_DATASET_DICT_PKL import numpy as np import matplotlib.pyplot as plt @@ -82,7 +82,7 @@ def load_rawdata_into_pkl(): custom_sources=custom_sources, rate_limit_per_minute=3500, min_tokens_per_block=200, max_tokens_per_block=300, - fraction_of_articles_to_use=1/100, + # fraction_of_articles_to_use=1/150, ) dataset.get_alignment_texts() @@ -92,7 +92,7 @@ def load_rawdata_into_pkl(): print(dataset.articles_count) dataset.get_embeddings() - dataset.save_data() + dataset.save_data() @retry(wait=wait_random_exponential(min=1, max=20), stop=stop_after_attempt(4)) def get_embedding(text: str) -> np.ndarray: @@ -156,18 +156,8 @@ def plot_likelihood(embeddings, num_buckets=200): if __name__ == "__main__": - load_rawdata_into_pkl() + # load_rawdata_into_pkl() # print_out_dataset_stuff() - # with open(PATH_TO_DATASET_PKL, 'rb') as f: - # dataset = pickle.load(f) - - # dataset_dict = { - # "embedding_strings": dataset.embedding_strings, - # "embeddings": dataset.embeddings, - # "embeddings_metadata_index": dataset.embeddings_metadata_index, - # "metadata": dataset.metadata - # } - - # with open(PATH_TO_DATASET_DICT_PKL, 'wb') as f: - # pickle.dump(dataset_dict, f) \ No newline at end of file + with open(PATH_TO_DATASET_DICT_PKL, 'rb') as f: + dataset = pickle.load(f) \ No newline at end of file