From 560b132e6778ccf514b5d6dcc1a32f7af04508d1 Mon Sep 17 00:00:00 2001
From: henri123lemoine <henri123lemoine@gmail.com>
Date: Tue, 27 Jun 2023 07:21:45 -0400
Subject: [PATCH] Added filenames on line 1 and removed RESET_DBS, fixed
 logging bug

---
 src/dataset/pinecone_db_handler.py | 2 +-
 src/dataset/settings.py            | 7 +++----
 src/dataset/sql_db_handler.py      | 2 ++
 src/dataset/text_splitter.py       | 2 ++
 src/main.py                        | 9 +++++----
 5 files changed, 13 insertions(+), 9 deletions(-)

diff --git a/src/dataset/pinecone_db_handler.py b/src/dataset/pinecone_db_handler.py
index e62e91d..7ecef06 100644
--- a/src/dataset/pinecone_db_handler.py
+++ b/src/dataset/pinecone_db_handler.py
@@ -1,4 +1,4 @@
-# dataset/pinecone_handler.py
+# dataset/pinecone_db_handler.py
 
 import json
 import pinecone
diff --git a/src/dataset/settings.py b/src/dataset/settings.py
index ec04ef2..dcd4c91 100644
--- a/src/dataset/settings.py
+++ b/src/dataset/settings.py
@@ -1,3 +1,5 @@
+# dataset/settings.py
+
 from pathlib import Path
 
 ### FILE PATHS ###
@@ -16,7 +18,4 @@ EMBEDDINGS_RATE_LIMIT = 3500
 PINECONE_INDEX_NAME = "stampy-chat-embeddings-test"
 PINECONE_VALUES_DIMS = EMBEDDINGS_DIMS
 PINECONE_METRIC = "cosine"
-PINECONE_METADATA_ENTRIES = ["entry_id", "source", "title", "authors", "text"]
-
-### MISC ###
-RESET_DBS = False
\ No newline at end of file
+PINECONE_METADATA_ENTRIES = ["entry_id", "source", "title", "authors", "text"]
\ No newline at end of file
diff --git a/src/dataset/sql_db_handler.py b/src/dataset/sql_db_handler.py
index 9ba223d..008a252 100644
--- a/src/dataset/sql_db_handler.py
+++ b/src/dataset/sql_db_handler.py
@@ -1,3 +1,5 @@
+# dataset/sql_db_handler.py
+
 import sqlite3
 from typing import List, Dict, Any
 
diff --git a/src/dataset/text_splitter.py b/src/dataset/text_splitter.py
index 221a7b2..2a99f6a 100644
--- a/src/dataset/text_splitter.py
+++ b/src/dataset/text_splitter.py
@@ -1,3 +1,5 @@
+# dataset/text_splitter.py
+
 import re
 from typing import List
 import tiktoken
diff --git a/src/main.py b/src/main.py
index 770ac39..a38c918 100644
--- a/src/main.py
+++ b/src/main.py
@@ -10,17 +10,18 @@ else:
 import openai
 openai.api_key = os.environ.get('OPENAI_API_KEY')
 
+import logging
+logging.basicConfig(level=logging.INFO)
+
 from dataset.update_dataset import ARDUpdater
-from dataset.settings import RESET_DBS
+
 
 def update_sql_and_pinecone_dbs():
     updater = ARDUpdater(
         min_tokens_per_block=200,
         max_tokens_per_block=300,
     )
-    if RESET_DBS:
-        updater.reset_dbs()
-    updater.update(['gwern_blog'])
+    updater.update()
 
 
 def main():