diff --git a/src/dataset/pinecone_db_handler.py b/src/dataset/pinecone_db_handler.py index e62e91d..7ecef06 100644 --- a/src/dataset/pinecone_db_handler.py +++ b/src/dataset/pinecone_db_handler.py @@ -1,4 +1,4 @@ -# dataset/pinecone_handler.py +# dataset/pinecone_db_handler.py import json import pinecone diff --git a/src/dataset/settings.py b/src/dataset/settings.py index ec04ef2..dcd4c91 100644 --- a/src/dataset/settings.py +++ b/src/dataset/settings.py @@ -1,3 +1,5 @@ +# dataset/settings.py + from pathlib import Path ### FILE PATHS ### @@ -16,7 +18,4 @@ EMBEDDINGS_RATE_LIMIT = 3500 PINECONE_INDEX_NAME = "stampy-chat-embeddings-test" PINECONE_VALUES_DIMS = EMBEDDINGS_DIMS PINECONE_METRIC = "cosine" -PINECONE_METADATA_ENTRIES = ["entry_id", "source", "title", "authors", "text"] - -### MISC ### -RESET_DBS = False \ No newline at end of file +PINECONE_METADATA_ENTRIES = ["entry_id", "source", "title", "authors", "text"] \ No newline at end of file diff --git a/src/dataset/sql_db_handler.py b/src/dataset/sql_db_handler.py index 9ba223d..008a252 100644 --- a/src/dataset/sql_db_handler.py +++ b/src/dataset/sql_db_handler.py @@ -1,3 +1,5 @@ +# dataset/sql_db_handler.py + import sqlite3 from typing import List, Dict, Any diff --git a/src/dataset/text_splitter.py b/src/dataset/text_splitter.py index 221a7b2..2a99f6a 100644 --- a/src/dataset/text_splitter.py +++ b/src/dataset/text_splitter.py @@ -1,3 +1,5 @@ +# dataset/text_splitter.py + import re from typing import List import tiktoken diff --git a/src/main.py b/src/main.py index 770ac39..a38c918 100644 --- a/src/main.py +++ b/src/main.py @@ -10,17 +10,18 @@ else: import openai openai.api_key = os.environ.get('OPENAI_API_KEY') +import logging +logging.basicConfig(level=logging.INFO) + from dataset.update_dataset import ARDUpdater -from dataset.settings import RESET_DBS + def update_sql_and_pinecone_dbs(): updater = ARDUpdater( min_tokens_per_block=200, max_tokens_per_block=300, ) - if RESET_DBS: - updater.reset_dbs() - updater.update(['gwern_blog']) + updater.update() def main():