diff --git a/src/testing.ipynb b/src/testing.ipynb index 5180a66..6ab251b 100644 --- a/src/testing.ipynb +++ b/src/testing.ipynb @@ -93,24 +93,7 @@ }, { "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "HI\n" - ] - } - ], - "source": [ - "print(\"HI\")" - ] - }, - { - "cell_type": "code", - "execution_count": 5, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ @@ -119,7 +102,9 @@ "from typing import List, Dict, Tuple\n", "import re\n", "\n", - "import matplotlib.pyplot as plt" + "import matplotlib.pyplot as plt\n", + "import openai\n", + "import config" ] }, { @@ -132,12 +117,19 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 50, "metadata": {}, "outputs": [], "source": [ "LEN_EMBEDDINGS = 1536\n", - "PATH_TO_DATA = r\"C:\\Users\\Henri\\Documents\\GitHub\\AlignmentSearch\\data\\alignment_texts.jsonl\"" + "PATH_TO_DATA = r\"C:\\Users\\Henri\\Documents\\GitHub\\AlignmentSearch\\data\\alignment_texts.jsonl\"\n", + "\n", + "COMPLETIONS_MODEL = \"text-davinci-003\"\n", + "EMBEDDING_MODEL = \"text-embedding-ada-002\"\n", + "\n", + "openai.api_key = config.OPENAI_API_KEY\n", + "\n", + "MAX_LEN_PROMPT = 5000" ] }, { @@ -150,7 +142,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 8, "metadata": {}, "outputs": [], "source": [ @@ -201,7 +193,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ @@ -231,7 +223,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 51, "metadata": {}, "outputs": [], "source": [ @@ -239,10 +231,9 @@ " def __init__(self,\n", " path: str, # Path to the dataset .jsonl file.\n", " sources: List[str] = None, # List of sources to include. If None, include all sources.\n", - " max_paragraph_length: Tuple[int, int] = None, # (max number of words in a paragraph, max number of characters in a paragraph)\n", - " load_data: bool = False, # Whether to load the data from the .jsonl file at initialization.\n", - " load_embeddings: bool = False, # Whether to load the embeddings from the .npy file at initialization. (only available if load_data is True)\n", + " max_paragraph_length: Tuple[int, int] = None # (max number of words in a paragraph, max number of characters in a paragraph)\n", " ):\n", + "\n", " self.path = path\n", " self.sources = sources\n", " self.max_paragraph_length = max_paragraph_length\n", @@ -263,12 +254,7 @@ " self.total_sentence_count = 0\n", " self.total_paragraph_count = 0\n", " \n", - " if load_data:\n", - " self.load()\n", - " if load_embeddings:\n", - " self.load_embeddings()\n", - " \n", - " def load(self):\n", + " def get_alignment_texts(self):\n", " with jsonlines.open(self.path, \"r\") as reader:\n", " for entry in reader:\n", " try:\n", @@ -285,6 +271,7 @@ " else:\n", " continue\n", " \n", + " # BIG PROBLEM: Very often, the post will have no URL, so this will fail.\n", " self.data.append((entry['title'], entry['url'], entry['text']))\n", " paragraphs = split_article(entry['text'])\n", " self.embed_split.extend(paragraphs)\n", @@ -295,115 +282,128 @@ " self.total_paragraph_count += len(paragraphs)\n", " except KeyError: # TO BE CHANGED\n", " pass\n", + " \n", + " def get_embedding(text: str) -> np.ndarray:\n", + " result = openai.Embedding.create(model=EMBEDDING_MODEL, input=text)\n", + " return result[\"data\"][0][\"embedding\"]\n", + "\n", + " def get_embeddings(self):\n", + " self.embeddings = np.array([self.get_embedding(text) for text in self.embed_split])\n", + " \n", + " def save_embeddings(self, path: str):\n", + " np.save(path, self.embeddings)\n", " \n", - " def load_embeddings(self):\n", - " raise NotImplementedError" + " def load_embeddings(self, path: str):\n", + " self.embeddings = np.load(path)" ] }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 52, "metadata": {}, "outputs": [], "source": [ - "# worthwhile_sources = [\n", - "# 'https://aipulse.org',\n", - "# 'ebook',\n", - "# 'https://qualiacomputing.com',\n", - "# 'alignment forum',\n", - "# 'lesswrong',\n", - "# 'manual',\n", - "# 'arxiv',\n", - "# 'https://deepmindsafetyresearch.medium.com/',\n", - "# 'waitbutwhy.com',\n", - "# 'GitHub',\n", - "# 'https://aiimpacts.org',\n", - "# 'arbital.com',\n", - "# 'carado.moe',\n", - "# 'nonarxiv_papers',\n", - "# 'https://vkrakovna.wordpress.com',\n", - "# 'https://jsteinhardt.wordpress.com',\n", - "# 'audio-transcripts',\n", - "# 'https://intelligence.org',\n", - "# 'youtube',\n", - "# 'reports',\n", - "# 'https://aisafety.camp',\n", - "# 'curriculum',\n", - "# 'https://www.yudkowsky.net',\n", - "# 'distill'\n", - "# ]" + "class SearchAndAnswer:\n", + " def __init__(self,\n", + " dataset: Dataset, # Dataset object containing the data.\n", + " ):\n", + " self.dataset = dataset\n", + " \n", + " def get_embedding(self, text: str) -> np.ndarray:\n", + " result = openai.Embedding.create(model=EMBEDDING_MODEL, input=text)\n", + " return result[\"data\"][0][\"embedding\"]\n", + " \n", + " def get_top_k(self, query: str, k: int=5) -> List[Tuple[str, str, str]]:\n", + " # Receives a query (str) and returns the top k articles (List[Tuple[str, str, str]]) that are most similar to the query.\n", + " # Each tuple contains the title of an article, its URL, and text.\n", + " query_embedding = self.get_embedding(query)\n", + " similarities = np.dot(self.dataset.embeddings, query_embedding)\n", + " top_k_indices = np.argsort(similarities)[::-1][:k]\n", + " top_k = [self.dataset.data[i] for i in top_k_indices]\n", + " return top_k\n", + " \n", + " def construct_prompt(self, question: str, texts: List[Tuple[str, str, str]]) -> str:\n", + " # Receives a question (str) and a list of articles (List[Tuple[str, str, str]]) and returns a prompt (str) to be used for text generation.\n", + " context = \"\\n\".join(texts)[:MAX_LEN_PROMPT]\n", + " header = \"\"\"Answer the question as truthfully as possible using the provided context, and if the answer is not contained within the text below, say \"I don't know.\"\\n\\nContext:\\n\"\"\"\n", + " return header + \"\".join(context) + \"\\n\\n Q: \" + question + \"\\n A:\"\n", + " \n", + " def answer_question(self, question: str, texts: List[Tuple[str, str, str]]) -> str:\n", + " # Receives a question (str) and a list of articles (List[Tuple[str, str, str]]) and returns an answer (str) to the question.\n", + " prompt = self.construct_prompt(question, texts)\n", + " COMPLETIONS_API_PARAMS = {\n", + " \"temperature\": 0.0,\n", + " \"max_tokens\": 500,\n", + " \"model\": COMPLETIONS_MODEL,\n", + " }\n", + " answer = openai.Completion.create(prompt=prompt, **COMPLETIONS_API_PARAMS)[\"choices\"][0][\"text\"].strip(\" \\n\")\n", + " return answer" ] }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 54, "metadata": {}, "outputs": [], "source": [ - "dataset = Dataset(path=PATH_TO_DATA, sources=None, load_data=True, load_embeddings=False)" + "dataset = Dataset(path=PATH_TO_DATA, sources=['https://www.yudkowsky.net'])\n", + "dataset.get_alignment_texts()\n", + "# dataset.get_embeddings()\n", + "# dataset.save_embeddings(EMBEDDINGS_PATH)\n", + "# # dataset.load_embeddings(EMBEDDINGS_PATH)\n", + "\n", + "# search_and_answer = SearchAndAnswer(dataset)\n", + "\n", + "# while True:\n", + "# question = input(\"Enter a question: \")\n", + "# if question == \"quit\":\n", + "# break\n", + "# top_k = search_and_answer.get_top_k(question)\n", + "# answer = search_and_answer.answer_question(question, top_k)\n", + "# print(answer)" ] }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 59, "metadata": {}, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "74845\n" - ] + "data": { + "text/plain": [ + "{'https://www.yudkowsky.net': 23, 'total': 23}" + ] + }, + "execution_count": 59, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ - "len_embeds = []\n", - "for embed in dataset.embed_split:\n", - " len_embeds.append(len(embed.split()))\n", - "# Find argmax\n", - "print(np.argmax(len_embeds)) # RESPONSE: 12502\n", - "# print(max(len_embeds)) # RESPONSE: 12502; This is bad news" + "dataset.num_articles" ] }, { "cell_type": "code", - "execution_count": 39, + "execution_count": 1, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Source Truth Empirical \n", - "total 41614 39713 \n", - "https://aipulse.org 23 23 \n", - "ebook 23 22 \n", - "https://qualiacomput 278 278 \n", - "alignment forum 2138 2138 \n", - "lesswrong 28479 28259 \n", - "manual 132 1 \n", - "arxiv 8007 7012 \n", - "https://deepmindsafe 10 10 \n", - "waitbutwhy.com 2 2 \n", - "GitHub 0 1 \n", - "https://aiimpacts.or 227 227 \n", - "arbital.com 223 223 \n", - "carado.moe 59 59 \n", - "nonarxiv_papers 323 244 \n", - "https://vkrakovna.wo 43 43 \n", - "https://jsteinhardt. 39 39 \n", - "audio-transcripts 25 37 \n", - "https://intelligence 479 479 \n", - "youtube 457 457 \n", - "reports 323 78 \n", - "https://aisafety.cam 8 8 \n", - "curriculum 0 1 \n", - "https://www.yudkowsk 23 23 \n", - "distill 49 49 \n", - "\n", - " Truth Empirical \n", - "Word Count 53550146 44501538 \n", - "Character Count 351767163 294346152 \n" + "Source Truth Empirical Difference\n" + ] + }, + { + "ename": "NameError", + "evalue": "name 'dataset' is not defined", + "output_type": "error", + "traceback": [ + "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[1;31mNameError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[1;32mIn[1], line 51\u001b[0m\n\u001b[0;32m 49\u001b[0m \u001b[39m# Print table. First row has Truth and Empirical findings.\u001b[39;00m\n\u001b[0;32m 50\u001b[0m \u001b[39mprint\u001b[39m(\u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39m{\u001b[39;00m\u001b[39m'\u001b[39m\u001b[39mSource\u001b[39m\u001b[39m'\u001b[39m\u001b[39m:\u001b[39;00m\u001b[39m<20\u001b[39m\u001b[39m}\u001b[39;00m\u001b[39m \u001b[39m\u001b[39m{\u001b[39;00m\u001b[39m'\u001b[39m\u001b[39mTruth\u001b[39m\u001b[39m'\u001b[39m\u001b[39m:\u001b[39;00m\u001b[39m<10\u001b[39m\u001b[39m}\u001b[39;00m\u001b[39m \u001b[39m\u001b[39m{\u001b[39;00m\u001b[39m'\u001b[39m\u001b[39mEmpirical\u001b[39m\u001b[39m'\u001b[39m\u001b[39m:\u001b[39;00m\u001b[39m<10\u001b[39m\u001b[39m}\u001b[39;00m\u001b[39m \u001b[39m\u001b[39m{\u001b[39;00m\u001b[39m'\u001b[39m\u001b[39mDifference\u001b[39m\u001b[39m'\u001b[39m\u001b[39m:\u001b[39;00m\u001b[39m<10\u001b[39m\u001b[39m}\u001b[39;00m\u001b[39m\"\u001b[39m)\n\u001b[1;32m---> 51\u001b[0m \u001b[39mfor\u001b[39;00m source \u001b[39min\u001b[39;00m dataset\u001b[39m.\u001b[39mnum_articles:\n\u001b[0;32m 52\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m 53\u001b[0m \u001b[39mprint\u001b[39m(\u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39m{\u001b[39;00msource[:\u001b[39m20\u001b[39m]\u001b[39m:\u001b[39;00m\u001b[39m<20\u001b[39m\u001b[39m}\u001b[39;00m\u001b[39m \u001b[39m\u001b[39m{\u001b[39;00mnum_articles_truth[source]\u001b[39m:\u001b[39;00m\u001b[39m<10\u001b[39m\u001b[39m}\u001b[39;00m\u001b[39m \u001b[39m\u001b[39m{\u001b[39;00mdataset\u001b[39m.\u001b[39mnum_articles[source]\u001b[39m:\u001b[39;00m\u001b[39m<10\u001b[39m\u001b[39m}\u001b[39;00m\u001b[39m \u001b[39m\u001b[39m{\u001b[39;00mnum_articles_truth[source] \u001b[39m-\u001b[39m dataset\u001b[39m.\u001b[39mnum_articles[source]\u001b[39m:\u001b[39;00m\u001b[39m<10\u001b[39m\u001b[39m}\u001b[39;00m\u001b[39m\"\u001b[39m)\n", + "\u001b[1;31mNameError\u001b[0m: name 'dataset' is not defined" ] } ], @@ -432,23 +432,23 @@ " 'https://qualiacomputing.com': 278,\n", " 'alignment forum': 2138,\n", " 'lesswrong': 28252 + 227,\n", - " 'manual': 132, # Stampy.ai?\n", + " 'manual': \"?\",\n", " 'arxiv': 707 + 1679 + 1000 + 4621,\n", " 'https://deepmindsafetyresearch.medium.com/': 10,\n", " 'waitbutwhy.com': 2,\n", - " 'GitHub': 0, # Huh?\n", + " 'GitHub': \"?\",\n", " 'https://aiimpacts.org': 227,\n", " 'arbital.com': 223,\n", " 'carado.moe': 59,\n", - " 'nonarxiv_papers': 323,\n", + " 'nonarxiv_papers': \"?\",\n", " 'https://vkrakovna.wordpress.com': 43,\n", " 'https://jsteinhardt.wordpress.com': 39,\n", " 'audio-transcripts': 25 + 12,\n", " 'https://intelligence.org': 479,\n", " 'youtube': 457,\n", - " 'reports': 323,\n", + " 'reports': \"?\",\n", " 'https://aisafety.camp': 8,\n", - " 'curriculum': 0, # Huh?\n", + " 'curriculum': \"?\",\n", " 'https://www.yudkowsky.net': 23,\n", " 'distill': 49,\n", " 'total': 2138+28252+707+1679+1000+4621+23+227+23+8+59+111+10+17+7+479+39+278+43+2+23+420+323+49+457+25+12+223+227+132 \n", @@ -457,23 +457,19 @@ "char_count_truth = 351_767_163\n", "\n", "# Print table. First row has Truth and Empirical findings.\n", - "print(f\"{'Source':<20} {'Truth':<10} {'Empirical':<10}\")\n", + "print(f\"{'Source':<20} {'Truth':<10} {'Empirical':<10} {'Difference':<10}\")\n", "for source in dataset.num_articles:\n", - " print(f\"{source[:20]:<20} {num_articles_truth[source]:<10} {dataset.num_articles[source]:<10}\")\n", + " try:\n", + " print(f\"{source[:20]:<20} {num_articles_truth[source]:<10} {dataset.num_articles[source]:<10} {num_articles_truth[source] - dataset.num_articles[source]:<10}\")\n", + " except TypeError:\n", + " print(f\"{source[:20]:<20} {num_articles_truth[source]:<10} {dataset.num_articles[source]:<10} {'UNKNOWN':<10}\")\n", "\n", "# Compare true and empirical word counts and character counts\n", - "print(f\"\\n{'':<20} {'Truth':<10} {'Empirical':<10}\")\n", - "print(f\"{'Word Count':<20} {word_count_truth:<10} {dataset.total_word_count:<10}\")\n", - "print(f\"{'Character Count':<20} {char_count_truth:<10} {dataset.total_char_count:<10}\")" + "print(f\"\\n{'':<20} {'Truth':<10} {'Empirical':<10} {'Difference':<10}\")\n", + "print(f\"{'Word Count':<20} {word_count_truth:<10} {dataset.total_word_count:<10} {word_count_truth - dataset.total_word_count:<10}\")\n", + "print(f\"{'Character Count':<20} {char_count_truth:<10} {dataset.total_char_count:<10} {char_count_truth - dataset.total_char_count:<10}\")" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, { "cell_type": "code", "execution_count": 150, @@ -520,319 +516,49 @@ ] }, { - "cell_type": "code", - "execution_count": 143, + "attachments": {}, + "cell_type": "markdown", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "187.5" - ] - }, - "execution_count": 143, - "metadata": {}, - "output_type": "execute_result" - } - ], "source": [ - "3000*500/8000" + "## Random tests" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 63, "metadata": {}, "outputs": [], "source": [ - "# 1$ per 2000 embeds" + "import json" ] }, { "cell_type": "code", - "execution_count": 37, + "execution_count": 72, "metadata": {}, "outputs": [], "source": [ - "import numpy as np" - ] - }, - { - "cell_type": "code", - "execution_count": 39, - "metadata": {}, - "outputs": [], - "source": [ - "# Make a 1D array of 1536 (very small) random numbers which add up to 1, and a 2D array of shape (1536, 100000), and for each row the sum of the elements is 1.\n", - "arr1 = np.random.rand(1536)\n", - "arr1 /= arr1.sum()\n", - "\n", - "arr2 = np.random.rand(1536, 100000)\n", - "arr2 /= arr2.sum(axis=0)" - ] - }, - { - "cell_type": "code", - "execution_count": 41, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "67.8 ms ± 2.33 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)\n" - ] - } - ], - "source": [ - "# Now, calculate the amount of time it takes to take the dot product of the two arrays.\n", - "%timeit arr1.dot(arr2)" - ] - }, - { - "cell_type": "code", - "execution_count": 59, - "metadata": {}, - "outputs": [], - "source": [ - "arr3 = arr1.dot(arr2)" - ] - }, - { - "cell_type": "code", - "execution_count": 61, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(100000,)" - ] - }, - "execution_count": 61, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "arr3.shape" - ] - }, - { - "cell_type": "code", - "execution_count": 60, - "metadata": {}, - "outputs": [ - { - "ename": "IndexError", - "evalue": "invalid index to scalar variable.", - "output_type": "error", - "traceback": [ - "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[1;31mIndexError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[1;32mIn[60], line 3\u001b[0m\n\u001b[0;32m 1\u001b[0m \u001b[39m# top k argmax\u001b[39;00m\n\u001b[0;32m 2\u001b[0m k \u001b[39m=\u001b[39m \u001b[39m10\u001b[39m\n\u001b[1;32m----> 3\u001b[0m top_k \u001b[39m=\u001b[39m np\u001b[39m.\u001b[39;49margmax(arr3, axis\u001b[39m=\u001b[39;49m\u001b[39m0\u001b[39;49m)[:k]\n", - "\u001b[1;31mIndexError\u001b[0m: invalid index to scalar variable." - ] - } - ], - "source": [ - "# top k argmax\n", - "k = 10\n", - "top_k = np.argmax(arr3, axis=0)[:k]" - ] - }, - { - "cell_type": "code", - "execution_count": 66, - "metadata": {}, - "outputs": [], - "source": [ - "import openai\n", - "import config" - ] - }, - { - "cell_type": "code", - "execution_count": 67, - "metadata": {}, - "outputs": [], - "source": [ - "EMBEDDING_MODEL = \"text-embedding-ada-002\"\n", - "openai.api_key = config.OPENAI_API_KEY" - ] - }, - { - "cell_type": "code", - "execution_count": 129, - "metadata": {}, - "outputs": [], - "source": [ - "def random_unit_vectors(d, n):\n", - " vec = np.random.randn(d, n)\n", - " norm = np.linalg.norm(vec, axis=0)\n", - " return vec / norm\n" - ] - }, - { - "cell_type": "code", - "execution_count": 133, - "metadata": {}, - "outputs": [], - "source": [ - "d = 1536\n", - "n = 1000\n", - "rand_mat = random_unit_vectors(d, n)" - ] - }, - { - "cell_type": "code", - "execution_count": 134, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(1536, 1000)" - ] - }, - "execution_count": 134, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "rand_mat.shape" - ] - }, - { - "cell_type": "code", - "execution_count": 73, - "metadata": {}, - "outputs": [], - "source": [ - "def get_embedding(text: str, model: str=EMBEDDING_MODEL) -> list[float]:\n", - " result = openai.Embedding.create(\n", - " model=model,\n", - " input=text\n", - " )\n", - " return np.array(result[\"data\"][0][\"embedding\"])" - ] - }, - { - "cell_type": "code", - "execution_count": 103, - "metadata": {}, - "outputs": [], - "source": [ - "a = get_embedding(\"JSON stands for JavaScript Object Notation. It means that a script (executable) file which is made of text in a programming language, is used to store and transfer the data. Python supports JSON through a built-in package called json. To use this feature, we import the json package in Python script. The text in JSON is done through quoted-string which contains value in key-value mapping within { }. It is similar to the dictionary in Python.\")" - ] - }, - { - "cell_type": "code", - "execution_count": 108, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(1536, 100000)" - ] - }, - "execution_count": 108, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "arr2.shape" - ] - }, - { - "cell_type": "code", - "execution_count": 107, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([0.02933221, 0.02961996, 0.02955815, ..., 0.02957929, 0.02946535,\n", - " 0.02934775])" - ] - }, - "execution_count": 107, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# Find the distance of a to origin\n", - "np.linalg.norm(arr2, axis=0)" - ] - }, - { - "cell_type": "code", - "execution_count": 97, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "77 ms ± 2.21 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)\n" - ] - } - ], - "source": [ - "%timeit a.dot(arr2)" - ] - }, - { - "cell_type": "code", - "execution_count": 92, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([0.00081222])" - ] - }, - "execution_count": 92, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# %timeit np.argmax(a.dot(arr2)" - ] - }, - { - "cell_type": "code", - "execution_count": 98, - "metadata": {}, - "outputs": [], - "source": [ - "b = a.dot(arr2)" - ] - }, - { - "cell_type": "code", - "execution_count": 102, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(97841,)" - ] - }, - "execution_count": 102, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "b[b<0].shape" + "with jsonlines.open(PATH_TO_DATA, \"r\") as reader, open(\"yud.txt\", \"w\", encoding=\"utf-8\") as writer:\n", + " for entry in reader:\n", + " try:\n", + " if 'source' in entry and entry['source'] == 'https://www.yudkowsky.net':\n", + " if 'title' in entry:\n", + " writer.write(f\"Title: {entry['title']}\\n\")\n", + " else:\n", + " writer.write(f\"NO TITLE\\n\")\n", + " if 'text' in entry:\n", + " writer.write(f\"Text: {entry['text']}\\n\")\n", + " else:\n", + " writer.write(f\"NO TEXT\\n\")\n", + " if 'url' in entry:\n", + " writer.write(f\"URL: {entry['url']}\\n\")\n", + " else:\n", + " writer.write(f\"NO URL\\n\")\n", + " writer.write(\"\\n\\n\")\n", + " else:\n", + " continue\n", + " except KeyError:\n", + " pass" ] }, {