Added filenames on line 1 and removed RESET_DBS, fixed logging bug

src/dataset/pinecone_db_handler.py

@@ -1,4 +1,4 @@
-# dataset/pinecone_handler.py
+# dataset/pinecone_db_handler.py
 
 import json
 import pinecone

src/dataset/settings.py

@@ -1,3 +1,5 @@
+# dataset/settings.py
+
 from pathlib import Path
 
 ### FILE PATHS ###
@@ -16,7 +18,4 @@ EMBEDDINGS_RATE_LIMIT = 3500
 PINECONE_INDEX_NAME = "stampy-chat-embeddings-test"
 PINECONE_VALUES_DIMS = EMBEDDINGS_DIMS
 PINECONE_METRIC = "cosine"
-PINECONE_METADATA_ENTRIES = ["entry_id", "source", "title", "authors", "text"]
-
-### MISC ###
-RESET_DBS = False
+PINECONE_METADATA_ENTRIES = ["entry_id", "source", "title", "authors", "text"]

src/dataset/sql_db_handler.py

@@ -1,3 +1,5 @@
+# dataset/sql_db_handler.py
+
 import sqlite3
 from typing import List, Dict, Any
 

src/dataset/text_splitter.py

@@ -1,3 +1,5 @@
+# dataset/text_splitter.py
+
 import re
 from typing import List
 import tiktoken

src/main.py

@@ -10,17 +10,18 @@ else:
 import openai
 openai.api_key = os.environ.get('OPENAI_API_KEY')
 
+import logging
+logging.basicConfig(level=logging.INFO)
+
 from dataset.update_dataset import ARDUpdater
-from dataset.settings import RESET_DBS
+
 
 def update_sql_and_pinecone_dbs():
     updater = ARDUpdater(
         min_tokens_per_block=200,
         max_tokens_per_block=300,
     )
-    if RESET_DBS:
-        updater.reset_dbs()
-    updater.update(['gwern_blog'])
+    updater.update()
 
 
 def main():