diff --git a/src/Embeddings Search/dataset.py b/src/Embeddings Search/dataset.py index 336c1ad..7b5618f 100644 --- a/src/Embeddings Search/dataset.py +++ b/src/Embeddings Search/dataset.py @@ -1,9 +1,7 @@ import jsonlines import numpy as np from typing import List, Dict, Tuple -import re import time -import random import pickle import openai from tenacity import ( @@ -13,22 +11,12 @@ from tenacity import ( ) # for exponential backoff import config - from text_splitter import TextSplitter, split_into_sentences +from settings import PATH_TO_DATA, PATH_TO_EMBEDDINGS, PATH_TO_DATASET, EMBEDDING_MODEL -LEN_EMBEDDINGS = 1536 -PATH_TO_DATA = r"C:\Users\Henri\Documents\GitHub\AlignmentSearch\src\Embeddings Search\data\alignment_texts.jsonl" -PATH_TO_EMBEDDINGS = r"C:\Users\Henri\Documents\GitHub\AlignmentSearch\src\Embeddings Search\data\embeddings.npy" -PATH_TO_DATASET = r"C:\Users\Henri\Documents\GitHub\AlignmentSearch\src\Embeddings Search\data\dataset.pkl" - -COMPLETIONS_MODEL = "text-davinci-003" -EMBEDDING_MODEL = "text-embedding-ada-002" - openai.api_key = config.OPENAI_API_KEY -MAX_LEN_PROMPT = 5000 - error_count_dict = { "Entry has no source.": 0, "Entry has no title.": 0, diff --git a/src/Embeddings Search/main.py b/src/Embeddings Search/main.py index d6c8e40..9e44f33 100644 --- a/src/Embeddings Search/main.py +++ b/src/Embeddings Search/main.py @@ -1,37 +1,13 @@ -import jsonlines import numpy as np -from typing import List, Dict, Tuple -import re -import time -import random import pickle import openai -from tenacity import ( - retry, - stop_after_attempt, - wait_random_exponential, -) # for exponential backoff import config - -from dataset import Dataset from semantic_search import AlignmentSearch - -from settings import DATA_PATH - - -LEN_EMBEDDINGS = 1536 -PATH_TO_DATA = r"C:\Users\Henri\Documents\GitHub\AlignmentSearch\src\Embeddings Search\data\alignment_texts.jsonl" -PATH_TO_EMBEDDINGS = r"C:\Users\Henri\Documents\GitHub\AlignmentSearch\src\Embeddings Search\data\embeddings.npy" -PATH_TO_DATASET = r"C:\Users\Henri\Documents\GitHub\AlignmentSearch\src\Embeddings Search\data\dataset.pkl" - -COMPLETIONS_MODEL = "text-davinci-003" -EMBEDDING_MODEL = "text-embedding-ada-002" +from settings import PATH_TO_DATASET openai.api_key = config.OPENAI_API_KEY -MAX_LEN_PROMPT = 5000 - def main(): with open(PATH_TO_DATASET, 'rb') as f: diff --git a/src/Embeddings Search/semantic_search.py b/src/Embeddings Search/semantic_search.py index 3d1db9e..5aedc09 100644 --- a/src/Embeddings Search/semantic_search.py +++ b/src/Embeddings Search/semantic_search.py @@ -1,9 +1,5 @@ -import jsonlines import numpy as np -from typing import List, Dict, Tuple -import re -import time -import random +from typing import List, Tuple import pickle import openai from tenacity import ( @@ -11,23 +7,13 @@ from tenacity import ( stop_after_attempt, wait_random_exponential, ) # for exponential backoff -from dataset import Dataset import config - - -LEN_EMBEDDINGS = 1536 -PATH_TO_DATA = r"C:\Users\Henri\Documents\GitHub\AlignmentSearch\src\Embeddings Search\data\alignment_texts.jsonl" -PATH_TO_EMBEDDINGS = r"C:\Users\Henri\Documents\GitHub\AlignmentSearch\src\Embeddings Search\data\embeddings.npy" -PATH_TO_DATASET = r"C:\Users\Henri\Documents\GitHub\AlignmentSearch\src\Embeddings Search\data\dataset.pkl" - -COMPLETIONS_MODEL = "text-davinci-003" -EMBEDDING_MODEL = "text-embedding-ada-002" +from dataset import Dataset +from settings import PATH_TO_DATASET, EMBEDDING_MODEL, COMPLETIONS_MODEL, MAX_LEN_PROMPT openai.api_key = config.OPENAI_API_KEY -MAX_LEN_PROMPT = 5000 - class AlignmentSearch: def __init__(self, diff --git a/src/Embeddings Search/settings.py b/src/Embeddings Search/settings.py index e3fad0d..2d9a53b 100644 --- a/src/Embeddings Search/settings.py +++ b/src/Embeddings Search/settings.py @@ -2,6 +2,8 @@ EMBEDDING_MODEL = "text-embedding-ada-002" COMPLETIONS_MODEL = "text-davinci-003" LEN_EMBEDDINGS = 1536 -MAX_LEN_PROMPT = 5000 +MAX_LEN_PROMPT = 8191 -DATA_PATH = r"C:\Users\Henri\Documents\GitHub\AlignmentSearch\data\alignment_texts.jsonl" # Path to the dataset .jsonl file. \ No newline at end of file +PATH_TO_DATA = r"C:\Users\Henri\Documents\GitHub\AlignmentSearch\data\alignment_texts.jsonl" # Path to the dataset .jsonl file. +PATH_TO_EMBEDDINGS = r"C:\Users\Henri\Documents\GitHub\AlignmentSearch\src\Embeddings Search\data\embeddings.npy" # Path to the saved embeddings (.npy) file. +PATH_TO_DATASET = r"C:\Users\Henri\Documents\GitHub\AlignmentSearch\src\Embeddings Search\data\dataset.pkl" # Path to the saved dataset (.pkl) file.