diff --git a/src/dataset.py b/src/dataset.py index dc26257..741e28d 100644 --- a/src/dataset.py +++ b/src/dataset.py @@ -17,10 +17,12 @@ from tenacity import ( from text_splitter import TokenSplitter, split_into_sentences from settings import PATH_TO_DATA, PATH_TO_EMBEDDINGS, PATH_TO_DATASET, EMBEDDING_MODEL, LEN_EMBEDDINGS +import os +from tqdm.auto import tqdm -import openai openai.api_key = os.environ.get('OPENAI_API_KEY') + error_count_dict = { "Entry has no source.": 0, "Entry has no title.": 0, @@ -131,7 +133,7 @@ class Dataset: def get_alignment_texts(self): text_splitter = TokenSplitter(self.min_tokens_per_block, self.max_tokens_per_block) with jsonlines.open(self.jsonl_data_path, "r") as reader: - for entry in reader: + for entry in tqdm(reader): try: if 'source' not in entry: if 'url' in entry and entry['url'] == "https://www.cold-takes.com/": @@ -238,8 +240,6 @@ class Dataset: pass """ - - def save_embeddings(self, path: str): np.save(path, self.embeddings) @@ -280,22 +280,22 @@ if __name__ == "__main__": # List of sources we are using for the test run: custom_sources = [ - "https://aipulse.org", - "ebook", + # "https://aipulse.org", + # "ebook", # "https://qualiacomputing.com", # "alignment forum", # "lesswrong", "manual", # "arxiv", - "https://deepmindsafetyresearch.medium.com", + # "https://deepmindsafetyresearch.medium.com", "waitbutwhy.com", "GitHub", # "https://aiimpacts.org", # "arbital.com", - "carado.moe", + # "carado.moe", # "nonarxiv_papers", - "https://vkrakovna.wordpress.com", - "https://jsteinhardt.wordpress.com", + # "https://vkrakovna.wordpress.com", + # "https://jsteinhardt.wordpress.com", "audio-transcripts", # "https://intelligence.org", # "youtube", diff --git a/src/settings.py b/src/settings.py index e80339d..571faff 100644 --- a/src/settings.py +++ b/src/settings.py @@ -6,7 +6,7 @@ COMPLETIONS_MODEL = "text-davinci-003" LEN_EMBEDDINGS = 1536 MAX_LEN_PROMPT = 4095 # This may be 8191, unsure. -project_path = Path(__file__).parent.parent.parent +project_path = Path(__file__).parent.parent#.parent PATH_TO_DATA = project_path / "src" / "data" / "alignment_texts.jsonl" # Path to the dataset .jsonl file. PATH_TO_EMBEDDINGS = project_path / "src" / "data" / "embeddings.npy" # Path to the saved embeddings (.npy) file. PATH_TO_DATASET = project_path / "src" / "data" / "dataset.pkl" # Path to the saved dataset (.pkl) file, containing the dataset class object. diff --git a/src/text_splitter.py b/src/text_splitter.py index bcf0c7a..a00f674 100644 --- a/src/text_splitter.py +++ b/src/text_splitter.py @@ -8,7 +8,7 @@ import nltk # Download the Punkt tokenizer if you haven't already -# nltk.download("punkt") +nltk.download("punkt") def split_into_sentences(text: str) -> List[str]: """ diff --git a/web/api/requirements.txt b/web/api/requirements.txt new file mode 100644 index 0000000..9220901 --- /dev/null +++ b/web/api/requirements.txt @@ -0,0 +1,4 @@ +openai +typing +numpy +tenacity \ No newline at end of file