Merge branch 'main' of https://github.com/FraserLee/AlignmentSearch

src/dataset.py

@@ -17,10 +17,13 @@ from tenacity import (
 
 from text_splitter import TokenSplitter, split_into_sentences
 from settings import PATH_TO_DATA, PATH_TO_EMBEDDINGS, PATH_TO_DATASET, EMBEDDING_MODEL, LEN_EMBEDDINGS
-
+import os
+from tqdm.auto import tqdm
 import openai
+
 openai.api_key = os.environ.get('OPENAI_API_KEY')
 
+
 error_count_dict = {
     "Entry has no source.": 0,
     "Entry has no title.": 0,
@@ -54,7 +57,7 @@ class Dataset:
         
         self.metadata: List[Tuple[str]] = []  # List of tuples, each containing the title of an article, its URL, and text. E.g.: [('title', 'url', 'text'), ...]
         self.embedding_strings: List[str] = []  # List of strings, each being a few paragraphs from a single article (not exceeding 1000 words).
-        self.embeddings_metadata_index: List[int] # List of integers, each being the index of the article from which the embedding string was taken.
+        self.embeddings_metadata_index: List[int] = [] # List of integers, each being the index of the article from which the embedding string was taken.
 
         self.articles_count: DefaultDict[str, int] = defaultdict(int)  # Number of articles per source. E.g.: {'source1': 10, 'source2': 20, 'total': 30}
 
@@ -131,7 +134,7 @@ class Dataset:
     def get_alignment_texts(self):
         text_splitter = TokenSplitter(self.min_tokens_per_block, self.max_tokens_per_block)
         with jsonlines.open(self.jsonl_data_path, "r") as reader:
-            for entry in reader:
+            for entry in tqdm(reader):
                 try:
                     if 'source' not in entry: 
                         if 'url' in entry and entry['url'] == "https://www.cold-takes.com/": 
@@ -211,7 +214,7 @@ class Dataset:
         with concurrent.futures.ThreadPoolExecutor() as executor:
             futures = [executor.submit(get_embedding_at_index, text, i) for i, text in enumerate(self.embedding_strings)]
             num_completed = 0
-            for future in concurrent.futures.as_completed(futures):
+            for future in tqdm(concurrent.futures.as_completed(futures)):
                 i, embedding = future.result()
                 self.embeddings[i] = embedding
                 num_completed += 1
@@ -238,8 +241,6 @@ class Dataset:
                             pass
                 """
 
-            
-
     def save_embeddings(self, path: str):
         np.save(path, self.embeddings)
         
@@ -280,22 +281,22 @@ if __name__ == "__main__":
 
     # List of sources we are using for the test run:
     custom_sources = [
-        "https://aipulse.org", 
-        "ebook", 
+        # "https://aipulse.org", 
+        # "ebook", 
         # "https://qualiacomputing.com", 
         # "alignment forum", 
         # "lesswrong", 
         "manual", 
         # "arxiv", 
-        "https://deepmindsafetyresearch.medium.com", 
+        # "https://deepmindsafetyresearch.medium.com", 
         "waitbutwhy.com", 
         "GitHub", 
         # "https://aiimpacts.org", 
         # "arbital.com", 
-        "carado.moe", 
+        # "carado.moe", 
         # "nonarxiv_papers", 
-        "https://vkrakovna.wordpress.com", 
-        "https://jsteinhardt.wordpress.com", 
+        # "https://vkrakovna.wordpress.com", 
+        # "https://jsteinhardt.wordpress.com", 
         "audio-transcripts", 
         # "https://intelligence.org", 
         # "youtube", 
@@ -320,10 +321,10 @@ if __name__ == "__main__":
         # fraction_of_articles_to_use=1/2000
     )
     dataset.get_alignment_texts()
-    # dataset.get_embeddings()
-    # dataset.save_embeddings("embeddings.npy")
+    dataset.get_embeddings()
+    dataset.save_embeddings("embeddings.npy")
     
-    # dataset.save_class("dataset.pkl")
+    dataset.save_class("data/dataset.pkl")
     # dataset = pickle.load(open("dataset.pkl", "rb"))
     
     

src/settings.py

@@ -6,7 +6,7 @@ COMPLETIONS_MODEL = "text-davinci-003"
 LEN_EMBEDDINGS = 1536
 MAX_LEN_PROMPT = 4095 # This may be 8191, unsure.
 
-project_path = Path(__file__).parent.parent.parent
+project_path = Path(__file__).parent.parent#.parent
 PATH_TO_DATA = project_path / "src" / "data" / "alignment_texts.jsonl" # Path to the dataset .jsonl file.
 PATH_TO_EMBEDDINGS = project_path / "src" / "data" / "embeddings.npy" # Path to the saved embeddings (.npy) file.
 PATH_TO_DATASET = project_path / "src" / "data" / "dataset.pkl" # Path to the saved dataset (.pkl) file, containing the dataset class object.

src/text_splitter.py

@@ -8,7 +8,7 @@ import nltk
 
 
 # Download the Punkt tokenizer if you haven't already
-# nltk.download("punkt")
+nltk.download("punkt")
 
 def split_into_sentences(text: str) -> List[str]:
     """

web/api/requirements.txt

@@ -0,0 +1,4 @@
+openai
+typing
+numpy
+tenacity

web/src/pages/index.tsx

@@ -26,25 +26,23 @@ const Home: NextPage = () => {
                     so if anyone else is considering this, you can message us to 
                     coordinate sharing the embeddings to avoid redundancy.
                 </p>
-                <p>Python serverless test:</p>
+
+                <p className="mt-4">Get the most semantic similar results to a query:</p>
                 <SearchBox />
             </main>
         </>
     );
 };
 
-// round trip test. If this works, our heavier usecase probably will (famous last words)
+// Round trip test. If this works, our heavier usecase probably will (famous last words)
 // The one real difference is we'll want to send back a series of results as we get
 // them back from OpenAI - I think we can just do this with a websocket, which
-// shouldn't be too different.
-
-// a search-box the person can type in, where they then can hit enter to search.
-// The query gets sent to api/search, and a list of links is returned, which are
-// then displayed below.
+// shouldn't be too much harder.
 
 const SearchBox: React.FC = () => {
-    const [query, setQuery] = useState("");
-    const [results, setResults] = useState<{title: string, url: string}[]>([]);
+
+    const [query,   setQuery]   = useState("");
+    const [results, setResults] = useState<{title: string,  url: string}[]>([]);
     const [loading, setLoading] = useState(false);
 
     const embeddings = async (query: String) => {
@@ -71,7 +69,7 @@ const SearchBox: React.FC = () => {
 
     return (
         <>
-            <form className="flex" onSubmit={async (e) => {
+            <form className="flex mb-2" onSubmit={async (e) => { // store in a form so that <enter> submits
                 e.preventDefault();
                 setResults(await embeddings(query));
             }}>
@@ -86,10 +84,11 @@ const SearchBox: React.FC = () => {
                     {loading ? "Loading..." : "Search"}
                 </button>
             </form>
-            {loading ? <p>loading...</p> : (
+
+            {loading ? <p>loading...</p> : ( // display results in list
                 <ul>
                     {results.map((result) => (
-                        <li key={result.url}>
+                        <li key={result.url} className="my-1">
                             <a href={result.url}>{result.title}</a>
                         </li>
                     ))}

web/src/styles/globals.css

@@ -3,7 +3,7 @@
 @tailwind utilities;
 
 h1 {
-  @apply text-4xl font-bold mb-4;
+  @apply text-4xl font-bold my-4;
 }
 
 main {