From aa39a22bf2fb203d566f4eb36c949af050883990 Mon Sep 17 00:00:00 2001 From: henri123lemoine Date: Fri, 3 Feb 2023 09:53:40 -0500 Subject: [PATCH] Updated testing.ipynb --- src/testing.ipynb | 211 ++++++++++++++++++---------------------------- 1 file changed, 80 insertions(+), 131 deletions(-) diff --git a/src/testing.ipynb b/src/testing.ipynb index 90a469e..5180a66 100644 --- a/src/testing.ipynb +++ b/src/testing.ipynb @@ -71,13 +71,6 @@ "```" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "My thoughts are: what's relevant to embed is only the text." - ] - }, { "attachments": {}, "cell_type": "markdown", @@ -90,13 +83,6 @@ "- https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, { "attachments": {}, "cell_type": "markdown", @@ -107,7 +93,24 @@ }, { "cell_type": "code", - "execution_count": 202, + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "HI\n" + ] + } + ], + "source": [ + "print(\"HI\")" + ] + }, + { + "cell_type": "code", + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -129,11 +132,12 @@ }, { "cell_type": "code", - "execution_count": 211, + "execution_count": 14, "metadata": {}, "outputs": [], "source": [ - "LEN_EMBEDDINGS = 1536" + "LEN_EMBEDDINGS = 1536\n", + "PATH_TO_DATA = r\"C:\\Users\\Henri\\Documents\\GitHub\\AlignmentSearch\\data\\alignment_texts.jsonl\"" ] }, { @@ -146,7 +150,7 @@ }, { "cell_type": "code", - "execution_count": 212, + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ @@ -197,7 +201,7 @@ }, { "cell_type": "code", - "execution_count": 213, + "execution_count": 10, "metadata": {}, "outputs": [], "source": [ @@ -227,51 +231,9 @@ }, { "cell_type": "code", - "execution_count": 160, + "execution_count": 17, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "' \\ndef process_text(self, text: str) -> List[str]:\\n # Receives one entry[\\'text\\'] and returns a list of sections, each section being a few appended paragraphs that do not exceed 5000 words.\\n # This is done to avoid the 8000 token limit of OpenAI embeddings.\\n sections = []\\n section = \"\"\\n for paragraph in text:\\n if len(section) + len(paragraph) > 5000:\\n sections.append(section)\\n section = \"\"\\n section += paragraph\\n sections.append(section)\\n return sections\\n'" - ] - }, - "execution_count": 160, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "\"\"\" \n", - "def process_text(self, text: str) -> List[str]:\n", - " # Receives one entry['text'] and returns a list of sections, each section being a few appended paragraphs that do not exceed 5000 words.\n", - " # This is done to avoid the 8000 token limit of OpenAI embeddings.\n", - " sections = []\n", - " section = \"\"\n", - " for paragraph in text:\n", - " if len(section) + len(paragraph) > 5000:\n", - " sections.append(section)\n", - " section = \"\"\n", - " section += paragraph\n", - " sections.append(section)\n", - " return sections\n", - "\"\"\"" - ] - }, - { - "cell_type": "code", - "execution_count": 210, - "metadata": {}, - "outputs": [ - { - "ename": "SyntaxError", - "evalue": "'continue' not properly in loop (3525560596.py, line 17)", - "output_type": "error", - "traceback": [ - "\u001b[1;36m Cell \u001b[1;32mIn[210], line 17\u001b[1;36m\u001b[0m\n\u001b[1;33m continue\u001b[0m\n\u001b[1;37m ^\u001b[0m\n\u001b[1;31mSyntaxError\u001b[0m\u001b[1;31m:\u001b[0m 'continue' not properly in loop\n" - ] - } - ], + "outputs": [], "source": [ "class Dataset:\n", " def __init__(self,\n", @@ -290,10 +252,11 @@ " \n", " self.num_articles: Dict[str, int] = {} # Number of articles per source. E.g.: {'source1': 10, 'source2': 20, 'total': 30}\n", " if sources is None:\n", - " continue\n", + " self.num_articles['total'] = 0\n", " else:\n", - " for source in sources: self.num_articles[source] = 0\n", - " self.num_articles['total'] = 0\n", + " for source in sources: \n", + " self.num_articles[source] = 0\n", + " self.num_articles['total'] = 0\n", " \n", " self.total_char_count = 0\n", " self.total_word_count = 0\n", @@ -330,8 +293,8 @@ " self.total_word_count += len(entry['text'].split())\n", " self.total_sentence_count += len(split_into_sentences(entry['text']))\n", " self.total_paragraph_count += len(paragraphs)\n", - " except KeyError:\n", - " print(f\"KeyError: {entry['url']}\")\n", + " except KeyError: # TO BE CHANGED\n", + " pass\n", " \n", " def load_embeddings(self):\n", " raise NotImplementedError" @@ -339,60 +302,57 @@ }, { "cell_type": "code", - "execution_count": 195, + "execution_count": 18, "metadata": {}, "outputs": [], "source": [ - "alignment_dataset = r\"C:\\Users\\Henri\\Documents\\GitHub\\AlignmentSearch\\data\\alignment_texts.jsonl\"\n", - "\n", - "worthwhile_sources = [\n", - " 'https://aipulse.org',\n", - " 'ebook',\n", - " 'https://qualiacomputing.com',\n", - " 'alignment forum',\n", - " 'lesswrong',\n", - " 'manual',\n", - " 'arxiv',\n", - " 'https://deepmindsafetyresearch.medium.com/',\n", - " 'waitbutwhy.com',\n", - " 'GitHub',\n", - " 'https://aiimpacts.org',\n", - " 'arbital.com',\n", - " 'carado.moe',\n", - " 'nonarxiv_papers',\n", - " 'https://vkrakovna.wordpress.com',\n", - " 'https://jsteinhardt.wordpress.com',\n", - " 'audio-transcripts',\n", - " 'https://intelligence.org',\n", - " 'youtube',\n", - " 'reports',\n", - " 'https://aisafety.camp',\n", - " 'curriculum',\n", - " 'https://www.yudkowsky.net',\n", - " 'distill'\n", - "]" + "# worthwhile_sources = [\n", + "# 'https://aipulse.org',\n", + "# 'ebook',\n", + "# 'https://qualiacomputing.com',\n", + "# 'alignment forum',\n", + "# 'lesswrong',\n", + "# 'manual',\n", + "# 'arxiv',\n", + "# 'https://deepmindsafetyresearch.medium.com/',\n", + "# 'waitbutwhy.com',\n", + "# 'GitHub',\n", + "# 'https://aiimpacts.org',\n", + "# 'arbital.com',\n", + "# 'carado.moe',\n", + "# 'nonarxiv_papers',\n", + "# 'https://vkrakovna.wordpress.com',\n", + "# 'https://jsteinhardt.wordpress.com',\n", + "# 'audio-transcripts',\n", + "# 'https://intelligence.org',\n", + "# 'youtube',\n", + "# 'reports',\n", + "# 'https://aisafety.camp',\n", + "# 'curriculum',\n", + "# 'https://www.yudkowsky.net',\n", + "# 'distill'\n", + "# ]" ] }, { "cell_type": "code", - "execution_count": 184, + "execution_count": 19, "metadata": {}, "outputs": [], "source": [ - "dataset = Dataset(path=alignment_dataset, sources=worthwhile_sources)\n", - "dataset.load()" + "dataset = Dataset(path=PATH_TO_DATA, sources=None, load_data=True, load_embeddings=False)" ] }, { "cell_type": "code", - "execution_count": 207, + "execution_count": 26, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "12502\n" + "74845\n" ] } ], @@ -400,34 +360,14 @@ "len_embeds = []\n", "for embed in dataset.embed_split:\n", " len_embeds.append(len(embed.split()))\n", - "print(max(len_embeds))" + "# Find argmax\n", + "print(np.argmax(len_embeds)) # RESPONSE: 12502\n", + "# print(max(len_embeds)) # RESPONSE: 12502; This is bad news" ] }, { "cell_type": "code", - "execution_count": 208, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjkAAAGdCAYAAADwjmIIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAp6klEQVR4nO3df1TVdZ7H8Rc/5ILlvfgjQEZUyvJHkhom3n4408rxmkytk7ujxjpmpquLbUjjr80ha3YW17Ypm0y3bUfas5rpnrQJCiP8NSZqkqRYMv2wscYuWgZXyQDls390+I430UJB5OPzcc735P1+3vd7P5/3Ce/rfP1+v4QYY4wAAAAsE9raEwAAAGgJhBwAAGAlQg4AALASIQcAAFiJkAMAAKxEyAEAAFYi5AAAACsRcgAAgJXCW3sCram+vl6HDh1Shw4dFBIS0trTAQAAP4AxRseOHVN8fLxCQ89+vuayDjmHDh1SQkJCa08DAACch08//VTdunU76/hlHXI6dOgg6dsmud3uVp4NAAD4IQKBgBISEpzv8bO5rENOwz9Rud1uQg4AAG3M911qwoXHAADASoQcAABgJUIOAACwEiEHAABYiZADAACsRMgBAABWIuQAAAArEXIAAICVCDkAAMBKhBwAAGAlQg4AALASIQcAAFiJkAMAAKxEyAEAAFYKb+0JoGX0nJsf9PqThWmtNBMAAFoHZ3IAAICVCDkAAMBKhBwAAGAlQg4AALASIQcAAFiJkAMAAKxEyAEAAFYi5AAAACsRcgAAgJUIOQAAwEqEHAAAYCVCDgAAsBIhBwAAWImQAwAArETIAQAAViLkAAAAKxFyAACAlQg5AADASoQcAABgJUIOAACwEiEHAABYiZADAACsRMgBAABWIuQAAAArEXIAAICVCDkAAMBKhBwAAGClJoWcnJwc3XTTTerQoYNiYmI0evRolZeXB9X85Cc/UUhISNA2bdq0oJqDBw8qLS1N7du3V0xMjGbNmqWTJ08G1WzatEk33nijXC6XevXqpdzc3DPms2TJEvXs2VORkZFKSUnRzp07m7IcAABgsSaFnM2bNysjI0Pbt29XYWGh6urqNGLECFVXVwfVTZkyRZ9//rmzLVq0yBk7deqU0tLSVFtbq23btumFF15Qbm6usrOznZoDBw4oLS1Nt99+u0pLS5WZman7779f69evd2peeuklZWVl6ZFHHtE777yjAQMGyOfz6fDhw+fbCwAAYJEQY4w53zcfOXJEMTEx2rx5s4YNGybp2zM5AwcO1FNPPdXoe15//XX99Kc/1aFDhxQbGytJWrZsmebMmaMjR44oIiJCc+bMUX5+vsrKypz3jRs3TpWVlSooKJAkpaSk6KabbtIzzzwjSaqvr1dCQoIeeOABzZ079wfNPxAIyOPxqKqqSm63+3zbcEnqOTc/6PUnC9NaaSYAADSvH/r9fUHX5FRVVUmSOnXqFLR/xYoV6tKli/r376958+bp66+/dsaKi4uVlJTkBBxJ8vl8CgQC2rdvn1OTmpoadEyfz6fi4mJJUm1trUpKSoJqQkNDlZqa6tQ0pqamRoFAIGgDAAB2Cj/fN9bX1yszM1O33HKL+vfv7+y/55571KNHD8XHx2vPnj2aM2eOysvL9fLLL0uS/H5/UMCR5Lz2+/3nrAkEAjpx4oS++uornTp1qtGa/fv3n3XOOTk5evTRR893yQAAoA0575CTkZGhsrIybd26NWj/1KlTnT8nJSWpa9euGj58uD766CNdc8015z/TZjBv3jxlZWU5rwOBgBISElpxRgAAoKWcV8iZMWOG8vLytGXLFnXr1u2ctSkpKZKkDz/8UNdcc43i4uLOuAuqoqJCkhQXF+f8t2Hf6TVut1tRUVEKCwtTWFhYozUNx2iMy+WSy+X6YYsEAABtWpOuyTHGaMaMGVq7dq02bNigxMTE731PaWmpJKlr166SJK/Xq7179wbdBVVYWCi3261+/fo5NUVFRUHHKSwslNfrlSRFREQoOTk5qKa+vl5FRUVODQAAuLw16UxORkaGVq5cqVdeeUUdOnRwrqHxeDyKiorSRx99pJUrV2rUqFHq3Lmz9uzZo5kzZ2rYsGG64YYbJEkjRoxQv379NGHCBC1atEh+v1/z589XRkaGc5Zl2rRpeuaZZzR79mzdd9992rBhg1avXq38/L/eMZSVlaWJEydq8ODBGjJkiJ566ilVV1dr0qRJzdUbAADQhjUp5CxdulTSt7eJn2758uW69957FRERoTfffNMJHAkJCRozZozmz5/v1IaFhSkvL0/Tp0+X1+vVFVdcoYkTJ+qxxx5zahITE5Wfn6+ZM2dq8eLF6tatm55//nn5fD6nZuzYsTpy5Iiys7Pl9/s1cOBAFRQUnHExMgAAuDxd0HNy2jqekwMAQNtzUZ6TAwAAcKki5AAAACsRcgAAgJUIOQAAwEqEHAAAYCVCDgAAsBIhBwAAWImQAwAArETIAQAAViLkAAAAKxFyAACAlQg5AADASoQcAABgJUIOAACwEiEHAABYiZADAACsRMgBAABWIuQAAAArEXIAAICVCDkAAMBKhBwAAGAlQg4AALASIQcAAFiJkAMAAKxEyAEAAFYi5AAAACsRcgAAgJUIOQAAwEqEHAAAYCVCDgAAsBIhBwAAWImQAwAArETIAQAAViLkAAAAKxFyAACAlQg5AADASoQcAABgJUIOAACwEiEHAABYiZADAACsRMgBAABWIuQAAAArEXIAAICVCDkAAMBKhBwAAGAlQg4AALASIQcAAFiJkAMAAKxEyAEAAFYi5AAAACsRcgAAgJUIOQAAwEqEHAAAYKUmhZycnBzddNNN6tChg2JiYjR69GiVl5cH1XzzzTfKyMhQ586ddeWVV2rMmDGqqKgIqjl48KDS0tLUvn17xcTEaNasWTp58mRQzaZNm3TjjTfK5XKpV69eys3NPWM+S5YsUc+ePRUZGamUlBTt3LmzKcsBAAAWa1LI2bx5szIyMrR9+3YVFhaqrq5OI0aMUHV1tVMzc+ZMvfrqq1qzZo02b96sQ4cO6e6773bGT506pbS0NNXW1mrbtm164YUXlJubq+zsbKfmwIEDSktL0+23367S0lJlZmbq/vvv1/r1652al156SVlZWXrkkUf0zjvvaMCAAfL5fDp8+PCF9AMAAFgixBhjzvfNR44cUUxMjDZv3qxhw4apqqpKV111lVauXKm/+7u/kyTt379fffv2VXFxsYYOHarXX39dP/3pT3Xo0CHFxsZKkpYtW6Y5c+boyJEjioiI0Jw5c5Sfn6+ysjLns8aNG6fKykoVFBRIklJSUnTTTTfpmWeekSTV19crISFBDzzwgObOnfuD5h8IBOTxeFRVVSW3232+bbgk9ZybH/T6k4VprTQTAACa1w/9/r6ga3KqqqokSZ06dZIklZSUqK6uTqmpqU5Nnz591L17dxUXF0uSiouLlZSU5AQcSfL5fAoEAtq3b59Tc/oxGmoajlFbW6uSkpKgmtDQUKWmpjo1jampqVEgEAjaAACAnc475NTX1yszM1O33HKL+vfvL0ny+/2KiIhQdHR0UG1sbKz8fr9Tc3rAaRhvGDtXTSAQ0IkTJ/TFF1/o1KlTjdY0HKMxOTk58ng8zpaQkND0hQMAgDbhvENORkaGysrKtGrVquacT4uaN2+eqqqqnO3TTz9t7SkBAIAWEn4+b5oxY4by8vK0ZcsWdevWzdkfFxen2tpaVVZWBp3NqaioUFxcnFPz3bugGu6+Or3mu3dkVVRUyO12KyoqSmFhYQoLC2u0puEYjXG5XHK5XE1fMAAAaHOadCbHGKMZM2Zo7dq12rBhgxITE4PGk5OT1a5dOxUVFTn7ysvLdfDgQXm9XkmS1+vV3r17g+6CKiwslNvtVr9+/Zya04/RUNNwjIiICCUnJwfV1NfXq6ioyKkBAACXtyadycnIyNDKlSv1yiuvqEOHDs71Lx6PR1FRUfJ4PJo8ebKysrLUqVMnud1uPfDAA/J6vRo6dKgkacSIEerXr58mTJigRYsWye/3a/78+crIyHDOskybNk3PPPOMZs+erfvuu08bNmzQ6tWrlZ//1zuGsrKyNHHiRA0ePFhDhgzRU089perqak2aNKm5egMAANqwJoWcpUuXSpJ+8pOfBO1fvny57r33XknSk08+qdDQUI0ZM0Y1NTXy+Xx69tlnndqwsDDl5eVp+vTp8nq9uuKKKzRx4kQ99thjTk1iYqLy8/M1c+ZMLV68WN26ddPzzz8vn8/n1IwdO1ZHjhxRdna2/H6/Bg4cqIKCgjMuRgYAAJenC3pOTlvHc3IAAGh7LspzcgAAAC5VhBwAAGAlQg4AALASIQcAAFiJkAMAAKxEyAEAAFYi5AAAACsRcgAAgJUIOQAAwEqEHAAAYCVCDgAAsBIhBwAAWImQAwAArETIAQAAViLkAAAAKxFyAACAlQg5AADASoQcAABgJUIOAACwEiEHAABYiZADAACsRMgBAABWIuQAAAArEXIAAICVCDkAAMBKhBwAAGAlQg4AALASIQcAAFiJkAMAAKxEyAEAAFYi5AAAACsRcgAAgJUIOQAAwEqEHAAAYCVCDgAAsBIhBwAAWImQAwAArETIAQAAViLkAAAAKxFyAACAlQg5AADASoQcAABgJUIOAACwEiEHAABYiZADAACsRMgBAABWIuQAAAArEXIAAICVCDkAAMBKhBwAAGAlQg4AALASIQcAAFiJkAMAAKzU5JCzZcsW3XnnnYqPj1dISIjWrVsXNH7vvfcqJCQkaBs5cmRQzdGjR5Weni63263o6GhNnjxZx48fD6rZs2ePbrvtNkVGRiohIUGLFi06Yy5r1qxRnz59FBkZqaSkJL322mtNXQ4AALBUk0NOdXW1BgwYoCVLlpy1ZuTIkfr888+d7cUXXwwaT09P1759+1RYWKi8vDxt2bJFU6dOdcYDgYBGjBihHj16qKSkRI8//rgWLFig5557zqnZtm2bxo8fr8mTJ2v37t0aPXq0Ro8erbKysqYuCQAAWCjEGGPO+80hIVq7dq1Gjx7t7Lv33ntVWVl5xhmeBu+//7769eunt99+W4MHD5YkFRQUaNSoUfrss88UHx+vpUuX6uGHH5bf71dERIQkae7cuVq3bp32798vSRo7dqyqq6uVl5fnHHvo0KEaOHCgli1b9oPmHwgE5PF4VFVVJbfbfR4duHT1nJsf9PqThWmtNBMAAJrXD/3+bpFrcjZt2qSYmBj17t1b06dP15dffumMFRcXKzo62gk4kpSamqrQ0FDt2LHDqRk2bJgTcCTJ5/OpvLxcX331lVOTmpoa9Lk+n0/FxcVnnVdNTY0CgUDQBgAA7NTsIWfkyJH6n//5HxUVFenf//3ftXnzZt1xxx06deqUJMnv9ysmJiboPeHh4erUqZP8fr9TExsbG1TT8Pr7ahrGG5OTkyOPx+NsCQkJF7ZYAABwyQpv7gOOGzfO+XNSUpJuuOEGXXPNNdq0aZOGDx/e3B/XJPPmzVNWVpbzOhAIEHQAALBUi99CfvXVV6tLly768MMPJUlxcXE6fPhwUM3Jkyd19OhRxcXFOTUVFRVBNQ2vv6+mYbwxLpdLbrc7aAMAAHZq8ZDz2Wef6csvv1TXrl0lSV6vV5WVlSopKXFqNmzYoPr6eqWkpDg1W7ZsUV1dnVNTWFio3r17q2PHjk5NUVFR0GcVFhbK6/W29JIAAEAb0OR/rjp+/LhzVkaSDhw4oNLSUnXq1EmdOnXSo48+qjFjxiguLk4fffSRZs+erV69esnn80mS+vbtq5EjR2rKlClatmyZ6urqNGPGDI0bN07x8fGSpHvuuUePPvqoJk+erDlz5qisrEyLFy/Wk08+6Xzugw8+qB//+Md64oknlJaWplWrVmnXrl1Bt5m3pu/e3SRxhxMAABdTk8/k7Nq1S4MGDdKgQYMkSVlZWRo0aJCys7MVFhamPXv26K677tJ1112nyZMnKzk5WX/84x/lcrmcY6xYsUJ9+vTR8OHDNWrUKN16661B4cTj8eiNN97QgQMHlJycrIceekjZ2dlBz9K5+eabtXLlSj333HMaMGCA/u///k/r1q1T//79L6QfAADAEhf0nJy2riWfk9PaZ3J4Tg4AwFat+pwcAACA1kbIAQAAViLkAAAAKxFyAACAlQg5AADASoQcAABgJUIOAACwEiEHAABYiZADAACsRMgBAABWIuQAAAArEXIAAICVCDkAAMBKhBwAAGAlQg4AALASIQcAAFiJkAMAAKxEyAEAAFYi5AAAACsRcgAAgJUIOQAAwEqEHAAAYCVCDgAAsBIhBwAAWImQAwAArETIAQAAViLkAAAAKxFyAACAlQg5AADASoQcAABgJUIOAACwEiEHAABYiZADAACsRMgBAABWIuQAAAArEXIAAICVCDkAAMBKhBwAAGAlQg4AALASIQcAAFiJkAMAAKxEyAEAAFYi5AAAACsRcgAAgJUIOQAAwEqEHAAAYCVCDgAAsBIhBwAAWImQAwAArETIAQAAViLkAAAAKxFyAACAlQg5AADASk0OOVu2bNGdd96p+Ph4hYSEaN26dUHjxhhlZ2era9euioqKUmpqqj744IOgmqNHjyo9PV1ut1vR0dGaPHmyjh8/HlSzZ88e3XbbbYqMjFRCQoIWLVp0xlzWrFmjPn36KDIyUklJSXrttdeauhwAAGCpJoec6upqDRgwQEuWLGl0fNGiRXr66ae1bNky7dixQ1dccYV8Pp+++eYbpyY9PV379u1TYWGh8vLytGXLFk2dOtUZDwQCGjFihHr06KGSkhI9/vjjWrBggZ577jmnZtu2bRo/frwmT56s3bt3a/To0Ro9erTKysqauiQAAGChEGOMOe83h4Ro7dq1Gj16tKRvz+LEx8froYce0i9/+UtJUlVVlWJjY5Wbm6tx48bp/fffV79+/fT2229r8ODBkqSCggKNGjVKn332meLj47V06VI9/PDD8vv9ioiIkCTNnTtX69at0/79+yVJY8eOVXV1tfLy8pz5DB06VAMHDtSyZct+0PwDgYA8Ho+qqqrkdrvPtw2N6jk3/4x9nyxMa9bPaMrnX8zPBgCgJf3Q7+9mvSbnwIED8vv9Sk1NdfZ5PB6lpKSouLhYklRcXKzo6Ggn4EhSamqqQkNDtWPHDqdm2LBhTsCRJJ/Pp/Lycn311VdOzemf01DT8DmNqampUSAQCNoAAICdmjXk+P1+SVJsbGzQ/tjYWGfM7/crJiYmaDw8PFydOnUKqmnsGKd/xtlqGsYbk5OTI4/H42wJCQlNXSIAAGgjLqu7q+bNm6eqqipn+/TTT1t7SgAAoIU0a8iJi4uTJFVUVATtr6iocMbi4uJ0+PDhoPGTJ0/q6NGjQTWNHeP0zzhbTcN4Y1wul9xud9AGAADs1KwhJzExUXFxcSoqKnL2BQIB7dixQ16vV5Lk9XpVWVmpkpISp2bDhg2qr69XSkqKU7NlyxbV1dU5NYWFherdu7c6duzo1Jz+OQ01DZ8DAAAub00OOcePH1dpaalKS0slfXuxcWlpqQ4ePKiQkBBlZmbqX//1X/WHP/xBe/fu1S9+8QvFx8c7d2D17dtXI0eO1JQpU7Rz50699dZbmjFjhsaNG6f4+HhJ0j333KOIiAhNnjxZ+/bt00svvaTFixcrKyvLmceDDz6ogoICPfHEE9q/f78WLFigXbt2acaMGRfeFQAA0OaFN/UNu3bt0u233+68bggeEydOVG5urmbPnq3q6mpNnTpVlZWVuvXWW1VQUKDIyEjnPStWrNCMGTM0fPhwhYaGasyYMXr66aedcY/HozfeeEMZGRlKTk5Wly5dlJ2dHfQsnZtvvlkrV67U/Pnz9S//8i+69tprtW7dOvXv3/+8GgEAAOxyQc/Jaet4Tg4AAG1PqzwnBwAA4FJByAEAAFYi5AAAACsRcgAAgJUIOQAAwEqEHAAAYCVCDgAAsBIhBwAAWImQAwAArETIAQAAViLkAAAAKxFyAACAlQg5AADASoQcAABgJUIOAACwEiEHAABYiZADAACsFN7aE8CF6zk3v7WnAADAJYczOQAAwEqEHAAAYCVCDgAAsBIhBwAAWImQAwAArETIAQAAViLkAAAAKxFyAACAlQg5AADASoQcAABgJUIOAACwEiEHAABYiZADAACsRMgBAABWIuQAAAArhbf2BC4nPefmB73+ZGFaK80EAAD7cSYHAABYiZADAACsRMgBAABWIuQAAAArEXIAAICVCDkAAMBKhBwAAGAlQg4AALASIQcAAFiJkAMAAKxEyAEAAFYi5AAAACsRcgAAgJUIOQAAwEqEHAAAYCVCDgAAsBIhBwAAWImQAwAArETIAQAAVmr2kLNgwQKFhIQEbX369HHGv/nmG2VkZKhz58668sorNWbMGFVUVAQd4+DBg0pLS1P79u0VExOjWbNm6eTJk0E1mzZt0o033iiXy6VevXopNze3uZcCAADasBY5k3P99dfr888/d7atW7c6YzNnztSrr76qNWvWaPPmzTp06JDuvvtuZ/zUqVNKS0tTbW2ttm3bphdeeEG5ubnKzs52ag4cOKC0tDTdfvvtKi0tVWZmpu6//36tX7++JZYDAADaoPAWOWh4uOLi4s7YX1VVpf/+7//WypUr9Td/8zeSpOXLl6tv377avn27hg4dqjfeeEPvvfee3nzzTcXGxmrgwIH69a9/rTlz5mjBggWKiIjQsmXLlJiYqCeeeEKS1LdvX23dulVPPvmkfD5fSywJAAC0MS1yJueDDz5QfHy8rr76aqWnp+vgwYOSpJKSEtXV1Sk1NdWp7dOnj7p3767i4mJJUnFxsZKSkhQbG+vU+Hw+BQIB7du3z6k5/RgNNQ3HOJuamhoFAoGgDQAA2KnZQ05KSopyc3NVUFCgpUuX6sCBA7rtttt07Ngx+f1+RUREKDo6Oug9sbGx8vv9kiS/3x8UcBrGG8bOVRMIBHTixImzzi0nJ0cej8fZEhISLnS5AADgEtXs/1x1xx13OH++4YYblJKSoh49emj16tWKiopq7o9rknnz5ikrK8t5HQgE2mTQ6Tk3v7WnAADAJa/FbyGPjo7Wddddpw8//FBxcXGqra1VZWVlUE1FRYVzDU9cXNwZd1s1vP6+Grfbfc4g5XK55Ha7gzYAAGCnFg85x48f10cffaSuXbsqOTlZ7dq1U1FRkTNeXl6ugwcPyuv1SpK8Xq/27t2rw4cPOzWFhYVyu93q16+fU3P6MRpqGo4BAADQ7CHnl7/8pTZv3qxPPvlE27Zt089+9jOFhYVp/Pjx8ng8mjx5srKysrRx40aVlJRo0qRJ8nq9Gjp0qCRpxIgR6tevnyZMmKB3331X69ev1/z585WRkSGXyyVJmjZtmj7++GPNnj1b+/fv17PPPqvVq1dr5syZzb0cAADQRjX7NTmfffaZxo8fry+//FJXXXWVbr31Vm3fvl1XXXWVJOnJJ59UaGioxowZo5qaGvl8Pj377LPO+8PCwpSXl6fp06fL6/Xqiiuu0MSJE/XYY485NYmJicrPz9fMmTO1ePFidevWTc8//zy3jwMAAEeIMca09iRaSyAQkMfjUVVVVbNfn/NDLg7+ZGFaix27uT4LAIBLzQ/9/uZ3VwEAACsRcgAAgJUIOQAAwEqEHAAAYKUW+QWd+GEau4D4uxcI83RjAADOD2dyAACAlQg5AADASoQcAABgJa7JucRwDQ4AAM2DMzkAAMBKhBwAAGAlQg4AALASIQcAAFiJkAMAAKxEyAEAAFYi5AAAACsRcgAAgJUIOQAAwEqEHAAAYCVCDgAAsBIhBwAAWImQAwAArETIAQAAViLkAAAAKxFyAACAlQg5AADASoQcAABgJUIOAACwEiEHAABYiZADAACsRMgBAABWIuQAAAArEXIAAICVCDkAAMBKhBwAAGAlQg4AALASIQcAAFiJkAMAAKxEyAEAAFYi5AAAACsRcgAAgJUIOQAAwEqEHAAAYCVCDgAAsBIhBwAAWImQAwAArETIAQAAViLkAAAAKxFyAACAlQg5AADASoQcAABgJUIOAACwEiEHAABYqc2HnCVLlqhnz56KjIxUSkqKdu7c2dpTAgAAl4A2HXJeeuklZWVl6ZFHHtE777yjAQMGyOfz6fDhw609NQAA0MradMj57W9/qylTpmjSpEnq16+fli1bpvbt2+v3v/99a08NAAC0svDWnsD5qq2tVUlJiebNm+fsCw0NVWpqqoqLixt9T01NjWpqapzXVVVVkqRAINDs86uv+brZj3khWmKNAAC0hobvNGPMOevabMj54osvdOrUKcXGxgbtj42N1f79+xt9T05Ojh599NEz9ickJLTIHC8lnqdaewYAADSvY8eOyePxnHW8zYac8zFv3jxlZWU5r+vr63X06FF17txZISEhzfY5gUBACQkJ+vTTT+V2u5vtuLagP+dGf74fPTo3+nNu9Ofc2kJ/jDE6duyY4uPjz1nXZkNOly5dFBYWpoqKiqD9FRUViouLa/Q9LpdLLpcraF90dHRLTVFut/uS/R/kUkB/zo3+fD96dG7059zoz7ld6v051xmcBm32wuOIiAglJyerqKjI2VdfX6+ioiJ5vd5WnBkAALgUtNkzOZKUlZWliRMnavDgwRoyZIieeuopVVdXa9KkSa09NQAA0MradMgZO3asjhw5ouzsbPn9fg0cOFAFBQVnXIx8sblcLj3yyCNn/NMYvkV/zo3+fD96dG7059zoz7nZ1J8Q8333XwEAALRBbfaaHAAAgHMh5AAAACsRcgAAgJUIOQAAwEqEnBawZMkS9ezZU5GRkUpJSdHOnTtbe0rNLicnRzfddJM6dOigmJgYjR49WuXl5UE133zzjTIyMtS5c2ddeeWVGjNmzBkPbzx48KDS0tLUvn17xcTEaNasWTp58mRQzaZNm3TjjTfK5XKpV69eys3NbenlNbuFCxcqJCREmZmZzr7LvT9/+ctf9A//8A/q3LmzoqKilJSUpF27djnjxhhlZ2era9euioqKUmpqqj744IOgYxw9elTp6elyu92Kjo7W5MmTdfz48aCaPXv26LbbblNkZKQSEhK0aNGii7K+C3Hq1Cn96le/UmJioqKionTNNdfo17/+ddDv6bmc+rNlyxbdeeedio+PV0hIiNatWxc0fjF7sWbNGvXp00eRkZFKSkrSa6+91uzrPR/n6lFdXZ3mzJmjpKQkXXHFFYqPj9cvfvELHTp0KOgYVvbIoFmtWrXKREREmN///vdm3759ZsqUKSY6OtpUVFS09tSalc/nM8uXLzdlZWWmtLTUjBo1ynTv3t0cP37cqZk2bZpJSEgwRUVFZteuXWbo0KHm5ptvdsZPnjxp+vfvb1JTU83u3bvNa6+9Zrp06WLmzZvn1Hz88cemffv2Jisry7z33nvmd7/7nQkLCzMFBQUXdb0XYufOnaZnz57mhhtuMA8++KCz/3Luz9GjR02PHj3Mvffea3bs2GE+/vhjs379evPhhx86NQsXLjQej8esW7fOvPvuu+auu+4yiYmJ5sSJE07NyJEjzYABA8z27dvNH//4R9OrVy8zfvx4Z7yqqsrExsaa9PR0U1ZWZl588UUTFRVl/vM///OirrepfvOb35jOnTubvLw8c+DAAbNmzRpz5ZVXmsWLFzs1l1N/XnvtNfPwww+bl19+2Ugya9euDRq/WL146623TFhYmFm0aJF57733zPz58027du3M3r17W7wH3+dcPaqsrDSpqanmpZdeMvv37zfFxcVmyJAhJjk5OegYNvaIkNPMhgwZYjIyMpzXp06dMvHx8SYnJ6cVZ9XyDh8+bCSZzZs3G2O+/aFq166dWbNmjVPz/vvvG0mmuLjYGPPtD2VoaKjx+/1OzdKlS43b7TY1NTXGGGNmz55trr/++qDPGjt2rPH5fC29pGZx7Ngxc+2115rCwkLz4x//2Ak5l3t/5syZY2699dazjtfX15u4uDjz+OOPO/sqKyuNy+UyL774ojHGmPfee89IMm+//bZT8/rrr5uQkBDzl7/8xRhjzLPPPms6duzo9Kvhs3v37t3cS2pWaWlp5r777gvad/fdd5v09HRjzOXdn+9+gV/MXvz85z83aWlpQfNJSUkx//iP/9isa7xQjQXB79q5c6eRZP785z8bY+ztEf9c1Yxqa2tVUlKi1NRUZ19oaKhSU1NVXFzcijNreVVVVZKkTp06SZJKSkpUV1cX1Is+ffqoe/fuTi+Ki4uVlJQU9PBGn8+nQCCgffv2OTWnH6Ohpq30MyMjQ2lpaWes4XLvzx/+8AcNHjxYf//3f6+YmBgNGjRI//Vf/+WMHzhwQH6/P2htHo9HKSkpQf2Jjo7W4MGDnZrU1FSFhoZqx44dTs2wYcMUERHh1Ph8PpWXl+urr75q6WWet5tvvllFRUX605/+JEl69913tXXrVt1xxx2S6M/pLmYv2urPW2OqqqoUEhLi/P5GW3tEyGlGX3zxhU6dOnXGE5djY2Pl9/tbaVYtr76+XpmZmbrlllvUv39/SZLf71dERMQZvwD19F74/f5Ge9Uwdq6aQCCgEydOtMRyms2qVav0zjvvKCcn54yxy70/H3/8sZYuXaprr71W69ev1/Tp0/XP//zPeuGFFyT9dX3n+lny+/2KiYkJGg8PD1enTp2a1MNL0dy5czVu3Dj16dNH7dq106BBg5SZman09HRJ9Od0F7MXZ6tpK71q8M0332jOnDkaP3688ws4be1Rm/61Drg0ZGRkqKysTFu3bm3tqVwyPv30Uz344IMqLCxUZGRka0/nklNfX6/Bgwfr3/7t3yRJgwYNUllZmZYtW6aJEye28uxa3+rVq7VixQqtXLlS119/vUpLS5WZman4+Hj6gwtSV1enn//85zLGaOnSpa09nRbHmZxm1KVLF4WFhZ1xh0xFRYXi4uJaaVYta8aMGcrLy9PGjRvVrVs3Z39cXJxqa2tVWVkZVH96L+Li4hrtVcPYuWrcbreioqKaeznNpqSkRIcPH9aNN96o8PBwhYeHa/PmzXr66acVHh6u2NjYy7o/Xbt2Vb9+/YL29e3bVwcPHpT01/Wd62cpLi5Ohw8fDho/efKkjh492qQeXopmzZrlnM1JSkrShAkTNHPmTOes4OXen9NdzF6craat9Koh4Pz5z39WYWGhcxZHsrdHhJxmFBERoeTkZBUVFTn76uvrVVRUJK/X24oza37GGM2YMUNr167Vhg0blJiYGDSenJysdu3aBfWivLxcBw8edHrh9Xq1d+/eoB+shh+8hi9Ar9cbdIyGmku9n8OHD9fevXtVWlrqbIMHD1Z6errz58u5P7fccssZjxz405/+pB49ekiSEhMTFRcXF7S2QCCgHTt2BPWnsrJSJSUlTs2GDRtUX1+vlJQUp2bLli2qq6tzagoLC9W7d2917NixxdZ3ob7++muFhgb/9RwWFqb6+npJ9Od0F7MXbfXnTfprwPnggw/05ptvqnPnzkHj1vaoVS53ttiqVauMy+Uyubm55r333jNTp0410dHRQXfI2GD69OnG4/GYTZs2mc8//9zZvv76a6dm2rRppnv37mbDhg1m165dxuv1Gq/X64w33CI9YsQIU1paagoKCsxVV13V6C3Ss2bNMu+//75ZsmRJm7hFujGn311lzOXdn507d5rw8HDzm9/8xnzwwQdmxYoVpn379uZ///d/nZqFCxea6Oho88orr5g9e/aYv/3bv230tuBBgwaZHTt2mK1bt5prr7026JbXyspKExsbayZMmGDKysrMqlWrTPv27S+5W6S/a+LEieZHP/qRcwv5yy+/bLp06WJmz57t1FxO/Tl27JjZvXu32b17t5Fkfvvb35rdu3c7dwZdrF689dZbJjw83PzHf/yHef/9980jjzxyydxCfq4e1dbWmrvuust069bNlJaWBv2dffqdUjb2iJDTAn73u9+Z7t27m4iICDNkyBCzffv21p5Ss5PU6LZ8+XKn5sSJE+af/umfTMeOHU379u3Nz372M/P5558HHeeTTz4xd9xxh4mKijJdunQxDz30kKmrqwuq2bhxoxk4cKCJiIgwV199ddBntCXfDTmXe39effVV079/f+NyuUyfPn3Mc889FzReX19vfvWrX5nY2FjjcrnM8OHDTXl5eVDNl19+acaPH2+uvPJK43a7zaRJk8yxY8eCat59911z6623GpfLZX70ox+ZhQsXtvjaLlQgEDAPPvig6d69u4mMjDRXX321efjhh4O+kC6n/mzcuLHRv28mTpxojLm4vVi9erW57rrrTEREhLn++utNfn5+i627Kc7VowMHDpz17+yNGzc6x7CxRyHGnPYITQAAAEtwTQ4AALASIQcAAFiJkAMAAKxEyAEAAFYi5AAAACsRcgAAgJUIOQAAwEqEHAAAYCVCDgAAsBIhBwAAWImQAwAArETIAQAAVvp/G2gv8eafKWIAAAAASUVORK5CYII=", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "# Plot the number of characters per embedding\n", - "plt.hist([len(embed.split()) for embed in dataset.embed_split], bins=100)\n", - "plt.savefig(\"embed_char_count.png\")" - ] - }, - { - "cell_type": "code", - "execution_count": 193, + "execution_count": 39, "metadata": {}, "outputs": [ { @@ -435,11 +375,12 @@ "output_type": "stream", "text": [ "Source Truth Empirical \n", + "total 41614 39713 \n", "https://aipulse.org 23 23 \n", "ebook 23 22 \n", "https://qualiacomput 278 278 \n", "alignment forum 2138 2138 \n", - "lesswrong 28252 28259 \n", + "lesswrong 28479 28259 \n", "manual 132 1 \n", "arxiv 8007 7012 \n", "https://deepmindsafe 10 10 \n", @@ -459,7 +400,10 @@ "curriculum 0 1 \n", "https://www.yudkowsk 23 23 \n", "distill 49 49 \n", - "total 41614 39713 \n" + "\n", + " Truth Empirical \n", + "Word Count 53550146 44501538 \n", + "Character Count 351767163 294346152 \n" ] } ], @@ -487,7 +431,7 @@ " 'ebook': 23,\n", " 'https://qualiacomputing.com': 278,\n", " 'alignment forum': 2138,\n", - " 'lesswrong': 28252, # +227?\n", + " 'lesswrong': 28252 + 227,\n", " 'manual': 132, # Stampy.ai?\n", " 'arxiv': 707 + 1679 + 1000 + 4621,\n", " 'https://deepmindsafetyresearch.medium.com/': 10,\n", @@ -499,7 +443,7 @@ " 'nonarxiv_papers': 323,\n", " 'https://vkrakovna.wordpress.com': 43,\n", " 'https://jsteinhardt.wordpress.com': 39,\n", - " 'audio-transcripts': 25,\n", + " 'audio-transcripts': 25 + 12,\n", " 'https://intelligence.org': 479,\n", " 'youtube': 457,\n", " 'reports': 323,\n", @@ -515,7 +459,12 @@ "# Print table. First row has Truth and Empirical findings.\n", "print(f\"{'Source':<20} {'Truth':<10} {'Empirical':<10}\")\n", "for source in dataset.num_articles:\n", - " print(f\"{source[:20]:<20} {num_articles_truth[source]:<10} {dataset.num_articles[source]:<10}\")" + " print(f\"{source[:20]:<20} {num_articles_truth[source]:<10} {dataset.num_articles[source]:<10}\")\n", + "\n", + "# Compare true and empirical word counts and character counts\n", + "print(f\"\\n{'':<20} {'Truth':<10} {'Empirical':<10}\")\n", + "print(f\"{'Word Count':<20} {word_count_truth:<10} {dataset.total_word_count:<10}\")\n", + "print(f\"{'Character Count':<20} {char_count_truth:<10} {dataset.total_char_count:<10}\")" ] }, {