mirror of
https://github.com/wassname/stampy-chat.git
synced 2026-09-11 12:50:34 +08:00
Added a search+q&a class
This commit is contained in:
henri123lemoine
+147
-421
@@ -93,24 +93,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"HI\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"print(\"HI\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"execution_count": 6,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@@ -119,7 +102,9 @@
|
||||
"from typing import List, Dict, Tuple\n",
|
||||
"import re\n",
|
||||
"\n",
|
||||
"import matplotlib.pyplot as plt"
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"import openai\n",
|
||||
"import config"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -132,12 +117,19 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 14,
|
||||
"execution_count": 50,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"LEN_EMBEDDINGS = 1536\n",
|
||||
"PATH_TO_DATA = r\"C:\\Users\\Henri\\Documents\\GitHub\\AlignmentSearch\\data\\alignment_texts.jsonl\""
|
||||
"PATH_TO_DATA = r\"C:\\Users\\Henri\\Documents\\GitHub\\AlignmentSearch\\data\\alignment_texts.jsonl\"\n",
|
||||
"\n",
|
||||
"COMPLETIONS_MODEL = \"text-davinci-003\"\n",
|
||||
"EMBEDDING_MODEL = \"text-embedding-ada-002\"\n",
|
||||
"\n",
|
||||
"openai.api_key = config.OPENAI_API_KEY\n",
|
||||
"\n",
|
||||
"MAX_LEN_PROMPT = 5000"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -150,7 +142,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 9,
|
||||
"execution_count": 8,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@@ -201,7 +193,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 10,
|
||||
"execution_count": 9,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@@ -231,7 +223,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 17,
|
||||
"execution_count": 51,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@@ -239,10 +231,9 @@
|
||||
" def __init__(self,\n",
|
||||
" path: str, # Path to the dataset .jsonl file.\n",
|
||||
" sources: List[str] = None, # List of sources to include. If None, include all sources.\n",
|
||||
" max_paragraph_length: Tuple[int, int] = None, # (max number of words in a paragraph, max number of characters in a paragraph)\n",
|
||||
" load_data: bool = False, # Whether to load the data from the .jsonl file at initialization.\n",
|
||||
" load_embeddings: bool = False, # Whether to load the embeddings from the .npy file at initialization. (only available if load_data is True)\n",
|
||||
" max_paragraph_length: Tuple[int, int] = None # (max number of words in a paragraph, max number of characters in a paragraph)\n",
|
||||
" ):\n",
|
||||
"\n",
|
||||
" self.path = path\n",
|
||||
" self.sources = sources\n",
|
||||
" self.max_paragraph_length = max_paragraph_length\n",
|
||||
@@ -263,12 +254,7 @@
|
||||
" self.total_sentence_count = 0\n",
|
||||
" self.total_paragraph_count = 0\n",
|
||||
" \n",
|
||||
" if load_data:\n",
|
||||
" self.load()\n",
|
||||
" if load_embeddings:\n",
|
||||
" self.load_embeddings()\n",
|
||||
" \n",
|
||||
" def load(self):\n",
|
||||
" def get_alignment_texts(self):\n",
|
||||
" with jsonlines.open(self.path, \"r\") as reader:\n",
|
||||
" for entry in reader:\n",
|
||||
" try:\n",
|
||||
@@ -285,6 +271,7 @@
|
||||
" else:\n",
|
||||
" continue\n",
|
||||
" \n",
|
||||
" # BIG PROBLEM: Very often, the post will have no URL, so this will fail.\n",
|
||||
" self.data.append((entry['title'], entry['url'], entry['text']))\n",
|
||||
" paragraphs = split_article(entry['text'])\n",
|
||||
" self.embed_split.extend(paragraphs)\n",
|
||||
@@ -295,115 +282,128 @@
|
||||
" self.total_paragraph_count += len(paragraphs)\n",
|
||||
" except KeyError: # TO BE CHANGED\n",
|
||||
" pass\n",
|
||||
" \n",
|
||||
" def get_embedding(text: str) -> np.ndarray:\n",
|
||||
" result = openai.Embedding.create(model=EMBEDDING_MODEL, input=text)\n",
|
||||
" return result[\"data\"][0][\"embedding\"]\n",
|
||||
"\n",
|
||||
" def get_embeddings(self):\n",
|
||||
" self.embeddings = np.array([self.get_embedding(text) for text in self.embed_split])\n",
|
||||
" \n",
|
||||
" def save_embeddings(self, path: str):\n",
|
||||
" np.save(path, self.embeddings)\n",
|
||||
" \n",
|
||||
" def load_embeddings(self):\n",
|
||||
" raise NotImplementedError"
|
||||
" def load_embeddings(self, path: str):\n",
|
||||
" self.embeddings = np.load(path)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 18,
|
||||
"execution_count": 52,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# worthwhile_sources = [\n",
|
||||
"# 'https://aipulse.org',\n",
|
||||
"# 'ebook',\n",
|
||||
"# 'https://qualiacomputing.com',\n",
|
||||
"# 'alignment forum',\n",
|
||||
"# 'lesswrong',\n",
|
||||
"# 'manual',\n",
|
||||
"# 'arxiv',\n",
|
||||
"# 'https://deepmindsafetyresearch.medium.com/',\n",
|
||||
"# 'waitbutwhy.com',\n",
|
||||
"# 'GitHub',\n",
|
||||
"# 'https://aiimpacts.org',\n",
|
||||
"# 'arbital.com',\n",
|
||||
"# 'carado.moe',\n",
|
||||
"# 'nonarxiv_papers',\n",
|
||||
"# 'https://vkrakovna.wordpress.com',\n",
|
||||
"# 'https://jsteinhardt.wordpress.com',\n",
|
||||
"# 'audio-transcripts',\n",
|
||||
"# 'https://intelligence.org',\n",
|
||||
"# 'youtube',\n",
|
||||
"# 'reports',\n",
|
||||
"# 'https://aisafety.camp',\n",
|
||||
"# 'curriculum',\n",
|
||||
"# 'https://www.yudkowsky.net',\n",
|
||||
"# 'distill'\n",
|
||||
"# ]"
|
||||
"class SearchAndAnswer:\n",
|
||||
" def __init__(self,\n",
|
||||
" dataset: Dataset, # Dataset object containing the data.\n",
|
||||
" ):\n",
|
||||
" self.dataset = dataset\n",
|
||||
" \n",
|
||||
" def get_embedding(self, text: str) -> np.ndarray:\n",
|
||||
" result = openai.Embedding.create(model=EMBEDDING_MODEL, input=text)\n",
|
||||
" return result[\"data\"][0][\"embedding\"]\n",
|
||||
" \n",
|
||||
" def get_top_k(self, query: str, k: int=5) -> List[Tuple[str, str, str]]:\n",
|
||||
" # Receives a query (str) and returns the top k articles (List[Tuple[str, str, str]]) that are most similar to the query.\n",
|
||||
" # Each tuple contains the title of an article, its URL, and text.\n",
|
||||
" query_embedding = self.get_embedding(query)\n",
|
||||
" similarities = np.dot(self.dataset.embeddings, query_embedding)\n",
|
||||
" top_k_indices = np.argsort(similarities)[::-1][:k]\n",
|
||||
" top_k = [self.dataset.data[i] for i in top_k_indices]\n",
|
||||
" return top_k\n",
|
||||
" \n",
|
||||
" def construct_prompt(self, question: str, texts: List[Tuple[str, str, str]]) -> str:\n",
|
||||
" # Receives a question (str) and a list of articles (List[Tuple[str, str, str]]) and returns a prompt (str) to be used for text generation.\n",
|
||||
" context = \"\\n\".join(texts)[:MAX_LEN_PROMPT]\n",
|
||||
" header = \"\"\"Answer the question as truthfully as possible using the provided context, and if the answer is not contained within the text below, say \"I don't know.\"\\n\\nContext:\\n\"\"\"\n",
|
||||
" return header + \"\".join(context) + \"\\n\\n Q: \" + question + \"\\n A:\"\n",
|
||||
" \n",
|
||||
" def answer_question(self, question: str, texts: List[Tuple[str, str, str]]) -> str:\n",
|
||||
" # Receives a question (str) and a list of articles (List[Tuple[str, str, str]]) and returns an answer (str) to the question.\n",
|
||||
" prompt = self.construct_prompt(question, texts)\n",
|
||||
" COMPLETIONS_API_PARAMS = {\n",
|
||||
" \"temperature\": 0.0,\n",
|
||||
" \"max_tokens\": 500,\n",
|
||||
" \"model\": COMPLETIONS_MODEL,\n",
|
||||
" }\n",
|
||||
" answer = openai.Completion.create(prompt=prompt, **COMPLETIONS_API_PARAMS)[\"choices\"][0][\"text\"].strip(\" \\n\")\n",
|
||||
" return answer"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 19,
|
||||
"execution_count": 54,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"dataset = Dataset(path=PATH_TO_DATA, sources=None, load_data=True, load_embeddings=False)"
|
||||
"dataset = Dataset(path=PATH_TO_DATA, sources=['https://www.yudkowsky.net'])\n",
|
||||
"dataset.get_alignment_texts()\n",
|
||||
"# dataset.get_embeddings()\n",
|
||||
"# dataset.save_embeddings(EMBEDDINGS_PATH)\n",
|
||||
"# # dataset.load_embeddings(EMBEDDINGS_PATH)\n",
|
||||
"\n",
|
||||
"# search_and_answer = SearchAndAnswer(dataset)\n",
|
||||
"\n",
|
||||
"# while True:\n",
|
||||
"# question = input(\"Enter a question: \")\n",
|
||||
"# if question == \"quit\":\n",
|
||||
"# break\n",
|
||||
"# top_k = search_and_answer.get_top_k(question)\n",
|
||||
"# answer = search_and_answer.answer_question(question, top_k)\n",
|
||||
"# print(answer)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 26,
|
||||
"execution_count": 59,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"74845\n"
|
||||
]
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"{'https://www.yudkowsky.net': 23, 'total': 23}"
|
||||
]
|
||||
},
|
||||
"execution_count": 59,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"len_embeds = []\n",
|
||||
"for embed in dataset.embed_split:\n",
|
||||
" len_embeds.append(len(embed.split()))\n",
|
||||
"# Find argmax\n",
|
||||
"print(np.argmax(len_embeds)) # RESPONSE: 12502\n",
|
||||
"# print(max(len_embeds)) # RESPONSE: 12502; This is bad news"
|
||||
"dataset.num_articles"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 39,
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Source Truth Empirical \n",
|
||||
"total 41614 39713 \n",
|
||||
"https://aipulse.org 23 23 \n",
|
||||
"ebook 23 22 \n",
|
||||
"https://qualiacomput 278 278 \n",
|
||||
"alignment forum 2138 2138 \n",
|
||||
"lesswrong 28479 28259 \n",
|
||||
"manual 132 1 \n",
|
||||
"arxiv 8007 7012 \n",
|
||||
"https://deepmindsafe 10 10 \n",
|
||||
"waitbutwhy.com 2 2 \n",
|
||||
"GitHub 0 1 \n",
|
||||
"https://aiimpacts.or 227 227 \n",
|
||||
"arbital.com 223 223 \n",
|
||||
"carado.moe 59 59 \n",
|
||||
"nonarxiv_papers 323 244 \n",
|
||||
"https://vkrakovna.wo 43 43 \n",
|
||||
"https://jsteinhardt. 39 39 \n",
|
||||
"audio-transcripts 25 37 \n",
|
||||
"https://intelligence 479 479 \n",
|
||||
"youtube 457 457 \n",
|
||||
"reports 323 78 \n",
|
||||
"https://aisafety.cam 8 8 \n",
|
||||
"curriculum 0 1 \n",
|
||||
"https://www.yudkowsk 23 23 \n",
|
||||
"distill 49 49 \n",
|
||||
"\n",
|
||||
" Truth Empirical \n",
|
||||
"Word Count 53550146 44501538 \n",
|
||||
"Character Count 351767163 294346152 \n"
|
||||
"Source Truth Empirical Difference\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"ename": "NameError",
|
||||
"evalue": "name 'dataset' is not defined",
|
||||
"output_type": "error",
|
||||
"traceback": [
|
||||
"\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
|
||||
"\u001b[1;31mNameError\u001b[0m Traceback (most recent call last)",
|
||||
"Cell \u001b[1;32mIn[1], line 51\u001b[0m\n\u001b[0;32m 49\u001b[0m \u001b[39m# Print table. First row has Truth and Empirical findings.\u001b[39;00m\n\u001b[0;32m 50\u001b[0m \u001b[39mprint\u001b[39m(\u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39m{\u001b[39;00m\u001b[39m'\u001b[39m\u001b[39mSource\u001b[39m\u001b[39m'\u001b[39m\u001b[39m:\u001b[39;00m\u001b[39m<20\u001b[39m\u001b[39m}\u001b[39;00m\u001b[39m \u001b[39m\u001b[39m{\u001b[39;00m\u001b[39m'\u001b[39m\u001b[39mTruth\u001b[39m\u001b[39m'\u001b[39m\u001b[39m:\u001b[39;00m\u001b[39m<10\u001b[39m\u001b[39m}\u001b[39;00m\u001b[39m \u001b[39m\u001b[39m{\u001b[39;00m\u001b[39m'\u001b[39m\u001b[39mEmpirical\u001b[39m\u001b[39m'\u001b[39m\u001b[39m:\u001b[39;00m\u001b[39m<10\u001b[39m\u001b[39m}\u001b[39;00m\u001b[39m \u001b[39m\u001b[39m{\u001b[39;00m\u001b[39m'\u001b[39m\u001b[39mDifference\u001b[39m\u001b[39m'\u001b[39m\u001b[39m:\u001b[39;00m\u001b[39m<10\u001b[39m\u001b[39m}\u001b[39;00m\u001b[39m\"\u001b[39m)\n\u001b[1;32m---> 51\u001b[0m \u001b[39mfor\u001b[39;00m source \u001b[39min\u001b[39;00m dataset\u001b[39m.\u001b[39mnum_articles:\n\u001b[0;32m 52\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m 53\u001b[0m \u001b[39mprint\u001b[39m(\u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39m{\u001b[39;00msource[:\u001b[39m20\u001b[39m]\u001b[39m:\u001b[39;00m\u001b[39m<20\u001b[39m\u001b[39m}\u001b[39;00m\u001b[39m \u001b[39m\u001b[39m{\u001b[39;00mnum_articles_truth[source]\u001b[39m:\u001b[39;00m\u001b[39m<10\u001b[39m\u001b[39m}\u001b[39;00m\u001b[39m \u001b[39m\u001b[39m{\u001b[39;00mdataset\u001b[39m.\u001b[39mnum_articles[source]\u001b[39m:\u001b[39;00m\u001b[39m<10\u001b[39m\u001b[39m}\u001b[39;00m\u001b[39m \u001b[39m\u001b[39m{\u001b[39;00mnum_articles_truth[source] \u001b[39m-\u001b[39m dataset\u001b[39m.\u001b[39mnum_articles[source]\u001b[39m:\u001b[39;00m\u001b[39m<10\u001b[39m\u001b[39m}\u001b[39;00m\u001b[39m\"\u001b[39m)\n",
|
||||
"\u001b[1;31mNameError\u001b[0m: name 'dataset' is not defined"
|
||||
]
|
||||
}
|
||||
],
|
||||
@@ -432,23 +432,23 @@
|
||||
" 'https://qualiacomputing.com': 278,\n",
|
||||
" 'alignment forum': 2138,\n",
|
||||
" 'lesswrong': 28252 + 227,\n",
|
||||
" 'manual': 132, # Stampy.ai?\n",
|
||||
" 'manual': \"?\",\n",
|
||||
" 'arxiv': 707 + 1679 + 1000 + 4621,\n",
|
||||
" 'https://deepmindsafetyresearch.medium.com/': 10,\n",
|
||||
" 'waitbutwhy.com': 2,\n",
|
||||
" 'GitHub': 0, # Huh?\n",
|
||||
" 'GitHub': \"?\",\n",
|
||||
" 'https://aiimpacts.org': 227,\n",
|
||||
" 'arbital.com': 223,\n",
|
||||
" 'carado.moe': 59,\n",
|
||||
" 'nonarxiv_papers': 323,\n",
|
||||
" 'nonarxiv_papers': \"?\",\n",
|
||||
" 'https://vkrakovna.wordpress.com': 43,\n",
|
||||
" 'https://jsteinhardt.wordpress.com': 39,\n",
|
||||
" 'audio-transcripts': 25 + 12,\n",
|
||||
" 'https://intelligence.org': 479,\n",
|
||||
" 'youtube': 457,\n",
|
||||
" 'reports': 323,\n",
|
||||
" 'reports': \"?\",\n",
|
||||
" 'https://aisafety.camp': 8,\n",
|
||||
" 'curriculum': 0, # Huh?\n",
|
||||
" 'curriculum': \"?\",\n",
|
||||
" 'https://www.yudkowsky.net': 23,\n",
|
||||
" 'distill': 49,\n",
|
||||
" 'total': 2138+28252+707+1679+1000+4621+23+227+23+8+59+111+10+17+7+479+39+278+43+2+23+420+323+49+457+25+12+223+227+132 \n",
|
||||
@@ -457,23 +457,19 @@
|
||||
"char_count_truth = 351_767_163\n",
|
||||
"\n",
|
||||
"# Print table. First row has Truth and Empirical findings.\n",
|
||||
"print(f\"{'Source':<20} {'Truth':<10} {'Empirical':<10}\")\n",
|
||||
"print(f\"{'Source':<20} {'Truth':<10} {'Empirical':<10} {'Difference':<10}\")\n",
|
||||
"for source in dataset.num_articles:\n",
|
||||
" print(f\"{source[:20]:<20} {num_articles_truth[source]:<10} {dataset.num_articles[source]:<10}\")\n",
|
||||
" try:\n",
|
||||
" print(f\"{source[:20]:<20} {num_articles_truth[source]:<10} {dataset.num_articles[source]:<10} {num_articles_truth[source] - dataset.num_articles[source]:<10}\")\n",
|
||||
" except TypeError:\n",
|
||||
" print(f\"{source[:20]:<20} {num_articles_truth[source]:<10} {dataset.num_articles[source]:<10} {'UNKNOWN':<10}\")\n",
|
||||
"\n",
|
||||
"# Compare true and empirical word counts and character counts\n",
|
||||
"print(f\"\\n{'':<20} {'Truth':<10} {'Empirical':<10}\")\n",
|
||||
"print(f\"{'Word Count':<20} {word_count_truth:<10} {dataset.total_word_count:<10}\")\n",
|
||||
"print(f\"{'Character Count':<20} {char_count_truth:<10} {dataset.total_char_count:<10}\")"
|
||||
"print(f\"\\n{'':<20} {'Truth':<10} {'Empirical':<10} {'Difference':<10}\")\n",
|
||||
"print(f\"{'Word Count':<20} {word_count_truth:<10} {dataset.total_word_count:<10} {word_count_truth - dataset.total_word_count:<10}\")\n",
|
||||
"print(f\"{'Character Count':<20} {char_count_truth:<10} {dataset.total_char_count:<10} {char_count_truth - dataset.total_char_count:<10}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 150,
|
||||
@@ -520,319 +516,49 @@
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 143,
|
||||
"attachments": {},
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"187.5"
|
||||
]
|
||||
},
|
||||
"execution_count": 143,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"3000*500/8000"
|
||||
"## Random tests"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 63,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# 1$ per 2000 embeds"
|
||||
"import json"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 37,
|
||||
"execution_count": 72,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import numpy as np"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 39,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Make a 1D array of 1536 (very small) random numbers which add up to 1, and a 2D array of shape (1536, 100000), and for each row the sum of the elements is 1.\n",
|
||||
"arr1 = np.random.rand(1536)\n",
|
||||
"arr1 /= arr1.sum()\n",
|
||||
"\n",
|
||||
"arr2 = np.random.rand(1536, 100000)\n",
|
||||
"arr2 /= arr2.sum(axis=0)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 41,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"67.8 ms ± 2.33 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# Now, calculate the amount of time it takes to take the dot product of the two arrays.\n",
|
||||
"%timeit arr1.dot(arr2)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 59,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"arr3 = arr1.dot(arr2)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 61,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"(100000,)"
|
||||
]
|
||||
},
|
||||
"execution_count": 61,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"arr3.shape"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 60,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"ename": "IndexError",
|
||||
"evalue": "invalid index to scalar variable.",
|
||||
"output_type": "error",
|
||||
"traceback": [
|
||||
"\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
|
||||
"\u001b[1;31mIndexError\u001b[0m Traceback (most recent call last)",
|
||||
"Cell \u001b[1;32mIn[60], line 3\u001b[0m\n\u001b[0;32m 1\u001b[0m \u001b[39m# top k argmax\u001b[39;00m\n\u001b[0;32m 2\u001b[0m k \u001b[39m=\u001b[39m \u001b[39m10\u001b[39m\n\u001b[1;32m----> 3\u001b[0m top_k \u001b[39m=\u001b[39m np\u001b[39m.\u001b[39;49margmax(arr3, axis\u001b[39m=\u001b[39;49m\u001b[39m0\u001b[39;49m)[:k]\n",
|
||||
"\u001b[1;31mIndexError\u001b[0m: invalid index to scalar variable."
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# top k argmax\n",
|
||||
"k = 10\n",
|
||||
"top_k = np.argmax(arr3, axis=0)[:k]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 66,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import openai\n",
|
||||
"import config"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 67,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"EMBEDDING_MODEL = \"text-embedding-ada-002\"\n",
|
||||
"openai.api_key = config.OPENAI_API_KEY"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 129,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def random_unit_vectors(d, n):\n",
|
||||
" vec = np.random.randn(d, n)\n",
|
||||
" norm = np.linalg.norm(vec, axis=0)\n",
|
||||
" return vec / norm\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 133,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"d = 1536\n",
|
||||
"n = 1000\n",
|
||||
"rand_mat = random_unit_vectors(d, n)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 134,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"(1536, 1000)"
|
||||
]
|
||||
},
|
||||
"execution_count": 134,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"rand_mat.shape"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 73,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def get_embedding(text: str, model: str=EMBEDDING_MODEL) -> list[float]:\n",
|
||||
" result = openai.Embedding.create(\n",
|
||||
" model=model,\n",
|
||||
" input=text\n",
|
||||
" )\n",
|
||||
" return np.array(result[\"data\"][0][\"embedding\"])"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 103,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"a = get_embedding(\"JSON stands for JavaScript Object Notation. It means that a script (executable) file which is made of text in a programming language, is used to store and transfer the data. Python supports JSON through a built-in package called json. To use this feature, we import the json package in Python script. The text in JSON is done through quoted-string which contains value in key-value mapping within { }. It is similar to the dictionary in Python.\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 108,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"(1536, 100000)"
|
||||
]
|
||||
},
|
||||
"execution_count": 108,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"arr2.shape"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 107,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"array([0.02933221, 0.02961996, 0.02955815, ..., 0.02957929, 0.02946535,\n",
|
||||
" 0.02934775])"
|
||||
]
|
||||
},
|
||||
"execution_count": 107,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# Find the distance of a to origin\n",
|
||||
"np.linalg.norm(arr2, axis=0)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 97,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"77 ms ± 2.21 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"%timeit a.dot(arr2)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 92,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"array([0.00081222])"
|
||||
]
|
||||
},
|
||||
"execution_count": 92,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# %timeit np.argmax(a.dot(arr2)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 98,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"b = a.dot(arr2)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 102,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"(97841,)"
|
||||
]
|
||||
},
|
||||
"execution_count": 102,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"b[b<0].shape"
|
||||
"with jsonlines.open(PATH_TO_DATA, \"r\") as reader, open(\"yud.txt\", \"w\", encoding=\"utf-8\") as writer:\n",
|
||||
" for entry in reader:\n",
|
||||
" try:\n",
|
||||
" if 'source' in entry and entry['source'] == 'https://www.yudkowsky.net':\n",
|
||||
" if 'title' in entry:\n",
|
||||
" writer.write(f\"Title: {entry['title']}\\n\")\n",
|
||||
" else:\n",
|
||||
" writer.write(f\"NO TITLE\\n\")\n",
|
||||
" if 'text' in entry:\n",
|
||||
" writer.write(f\"Text: {entry['text']}\\n\")\n",
|
||||
" else:\n",
|
||||
" writer.write(f\"NO TEXT\\n\")\n",
|
||||
" if 'url' in entry:\n",
|
||||
" writer.write(f\"URL: {entry['url']}\\n\")\n",
|
||||
" else:\n",
|
||||
" writer.write(f\"NO URL\\n\")\n",
|
||||
" writer.write(\"\\n\\n\")\n",
|
||||
" else:\n",
|
||||
" continue\n",
|
||||
" except KeyError:\n",
|
||||
" pass"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user