From ea5bf2e707ea2829f52bc3bb804e9dd5420195df Mon Sep 17 00:00:00 2001 From: henri123lemoine Date: Fri, 24 Mar 2023 22:21:29 -0400 Subject: [PATCH 1/3] Solved error in dataset.py. --- src/dataset.py | 20 ++++++++++---------- src/settings.py | 2 +- src/text_splitter.py | 2 +- web/api/requirements.txt | 4 ++++ 4 files changed, 16 insertions(+), 12 deletions(-) create mode 100644 web/api/requirements.txt diff --git a/src/dataset.py b/src/dataset.py index dc26257..741e28d 100644 --- a/src/dataset.py +++ b/src/dataset.py @@ -17,10 +17,12 @@ from tenacity import ( from text_splitter import TokenSplitter, split_into_sentences from settings import PATH_TO_DATA, PATH_TO_EMBEDDINGS, PATH_TO_DATASET, EMBEDDING_MODEL, LEN_EMBEDDINGS +import os +from tqdm.auto import tqdm -import openai openai.api_key = os.environ.get('OPENAI_API_KEY') + error_count_dict = { "Entry has no source.": 0, "Entry has no title.": 0, @@ -131,7 +133,7 @@ class Dataset: def get_alignment_texts(self): text_splitter = TokenSplitter(self.min_tokens_per_block, self.max_tokens_per_block) with jsonlines.open(self.jsonl_data_path, "r") as reader: - for entry in reader: + for entry in tqdm(reader): try: if 'source' not in entry: if 'url' in entry and entry['url'] == "https://www.cold-takes.com/": @@ -238,8 +240,6 @@ class Dataset: pass """ - - def save_embeddings(self, path: str): np.save(path, self.embeddings) @@ -280,22 +280,22 @@ if __name__ == "__main__": # List of sources we are using for the test run: custom_sources = [ - "https://aipulse.org", - "ebook", + # "https://aipulse.org", + # "ebook", # "https://qualiacomputing.com", # "alignment forum", # "lesswrong", "manual", # "arxiv", - "https://deepmindsafetyresearch.medium.com", + # "https://deepmindsafetyresearch.medium.com", "waitbutwhy.com", "GitHub", # "https://aiimpacts.org", # "arbital.com", - "carado.moe", + # "carado.moe", # "nonarxiv_papers", - "https://vkrakovna.wordpress.com", - "https://jsteinhardt.wordpress.com", + # "https://vkrakovna.wordpress.com", + # "https://jsteinhardt.wordpress.com", "audio-transcripts", # "https://intelligence.org", # "youtube", diff --git a/src/settings.py b/src/settings.py index e80339d..571faff 100644 --- a/src/settings.py +++ b/src/settings.py @@ -6,7 +6,7 @@ COMPLETIONS_MODEL = "text-davinci-003" LEN_EMBEDDINGS = 1536 MAX_LEN_PROMPT = 4095 # This may be 8191, unsure. -project_path = Path(__file__).parent.parent.parent +project_path = Path(__file__).parent.parent#.parent PATH_TO_DATA = project_path / "src" / "data" / "alignment_texts.jsonl" # Path to the dataset .jsonl file. PATH_TO_EMBEDDINGS = project_path / "src" / "data" / "embeddings.npy" # Path to the saved embeddings (.npy) file. PATH_TO_DATASET = project_path / "src" / "data" / "dataset.pkl" # Path to the saved dataset (.pkl) file, containing the dataset class object. diff --git a/src/text_splitter.py b/src/text_splitter.py index bcf0c7a..a00f674 100644 --- a/src/text_splitter.py +++ b/src/text_splitter.py @@ -8,7 +8,7 @@ import nltk # Download the Punkt tokenizer if you haven't already -# nltk.download("punkt") +nltk.download("punkt") def split_into_sentences(text: str) -> List[str]: """ diff --git a/web/api/requirements.txt b/web/api/requirements.txt new file mode 100644 index 0000000..9220901 --- /dev/null +++ b/web/api/requirements.txt @@ -0,0 +1,4 @@ +openai +typing +numpy +tenacity \ No newline at end of file From c8a33ff11488203d63b41fdff5708bbc3f2a1da6 Mon Sep 17 00:00:00 2001 From: henri123lemoine Date: Fri, 24 Mar 2023 22:23:19 -0400 Subject: [PATCH 2/3] Solved dataset.py errors. --- src/dataset.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/dataset.py b/src/dataset.py index 741e28d..fc939b5 100644 --- a/src/dataset.py +++ b/src/dataset.py @@ -19,6 +19,7 @@ from text_splitter import TokenSplitter, split_into_sentences from settings import PATH_TO_DATA, PATH_TO_EMBEDDINGS, PATH_TO_DATASET, EMBEDDING_MODEL, LEN_EMBEDDINGS import os from tqdm.auto import tqdm +import openai openai.api_key = os.environ.get('OPENAI_API_KEY') @@ -56,7 +57,7 @@ class Dataset: self.metadata: List[Tuple[str]] = [] # List of tuples, each containing the title of an article, its URL, and text. E.g.: [('title', 'url', 'text'), ...] self.embedding_strings: List[str] = [] # List of strings, each being a few paragraphs from a single article (not exceeding 1000 words). - self.embeddings_metadata_index: List[int] # List of integers, each being the index of the article from which the embedding string was taken. + self.embeddings_metadata_index: List[int] = [] # List of integers, each being the index of the article from which the embedding string was taken. self.articles_count: DefaultDict[str, int] = defaultdict(int) # Number of articles per source. E.g.: {'source1': 10, 'source2': 20, 'total': 30} @@ -213,7 +214,7 @@ class Dataset: with concurrent.futures.ThreadPoolExecutor() as executor: futures = [executor.submit(get_embedding_at_index, text, i) for i, text in enumerate(self.embedding_strings)] num_completed = 0 - for future in concurrent.futures.as_completed(futures): + for future in tqdm(concurrent.futures.as_completed(futures)): i, embedding = future.result() self.embeddings[i] = embedding num_completed += 1 @@ -320,10 +321,10 @@ if __name__ == "__main__": # fraction_of_articles_to_use=1/2000 ) dataset.get_alignment_texts() - # dataset.get_embeddings() - # dataset.save_embeddings("embeddings.npy") + dataset.get_embeddings() + dataset.save_embeddings("embeddings.npy") - # dataset.save_class("dataset.pkl") + dataset.save_class("data/dataset.pkl") # dataset = pickle.load(open("dataset.pkl", "rb")) \ No newline at end of file From eaf17cfba0529e1d76e00a3869467d328def1ec8 Mon Sep 17 00:00:00 2001 From: Fraser Date: Fri, 24 Mar 2023 22:27:04 -0400 Subject: [PATCH 3/3] clean up index.tsx --- web/src/pages/index.tsx | 23 +++++++++++------------ web/src/styles/globals.css | 2 +- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/web/src/pages/index.tsx b/web/src/pages/index.tsx index 85fa319..37a1617 100644 --- a/web/src/pages/index.tsx +++ b/web/src/pages/index.tsx @@ -26,25 +26,23 @@ const Home: NextPage = () => { so if anyone else is considering this, you can message us to coordinate sharing the embeddings to avoid redundancy.

-

Python serverless test:

+ +

Get the most semantic similar results to a query:

); }; -// round trip test. If this works, our heavier usecase probably will (famous last words) +// Round trip test. If this works, our heavier usecase probably will (famous last words) // The one real difference is we'll want to send back a series of results as we get // them back from OpenAI - I think we can just do this with a websocket, which -// shouldn't be too different. - -// a search-box the person can type in, where they then can hit enter to search. -// The query gets sent to api/search, and a list of links is returned, which are -// then displayed below. +// shouldn't be too much harder. const SearchBox: React.FC = () => { - const [query, setQuery] = useState(""); - const [results, setResults] = useState<{title: string, url: string}[]>([]); + + const [query, setQuery] = useState(""); + const [results, setResults] = useState<{title: string, url: string}[]>([]); const [loading, setLoading] = useState(false); const embeddings = async (query: String) => { @@ -71,7 +69,7 @@ const SearchBox: React.FC = () => { return ( <> -
{ + { // store in a form so that submits e.preventDefault(); setResults(await embeddings(query)); }}> @@ -86,10 +84,11 @@ const SearchBox: React.FC = () => { {loading ? "Loading..." : "Search"} - {loading ?

loading...

: ( + + {loading ?

loading...

: ( // display results in list
    {results.map((result) => ( -
  • +
  • {result.title}
    • ))} diff --git a/web/src/styles/globals.css b/web/src/styles/globals.css index 46d0311..c35c1b9 100644 --- a/web/src/styles/globals.css +++ b/web/src/styles/globals.css @@ -3,7 +3,7 @@ @tailwind utilities; h1 { - @apply text-4xl font-bold mb-4; + @apply text-4xl font-bold my-4; } main {