diff --git a/src/dataset.py b/src/dataset.py index 45af899..19a11db 100644 --- a/src/dataset.py +++ b/src/dataset.py @@ -15,7 +15,6 @@ from tenacity import ( ) # for exponential backoff import tiktoken -import config from pathlib import Path import config from text_splitter import TokenSplitter, split_into_sentences diff --git a/src/main.py b/src/main.py index 9e44f33..0f88830 100644 --- a/src/main.py +++ b/src/main.py @@ -16,7 +16,6 @@ def main(): prompt = "What would be an idea to solve the Alignment Problem? Name the Lesswrong post by Quintin Pope that discusses this idea." answer = AS.search_and_answer(prompt, 3, HyDE=False) print(answer) - if __name__ == "__main__": main() \ No newline at end of file diff --git a/src/testing.ipynb b/src/testing.ipynb index 54a561b..ab5a5c2 100644 --- a/src/testing.ipynb +++ b/src/testing.ipynb @@ -837,13 +837,11 @@ " \n", " def construct_messages(self, question: str, blocks: List[str] = None, mode: str = \"balanced\") -> str:\n", " # Receives a question (str) and a list of blocks and returns a prompt (str) to be used for text generation.\n", + " context = \"\"\n", " if blocks:\n", - " context = \"\"\n", " for i, block in enumerate(blocks):\n", " context += f'Context #{i+1}: \"\"\"{block}\"\"\"\\n\\n'\n", " context = self.limit_tokens(context, 2000)\n", - " else:\n", - " context = \"\"\n", "\n", " if mode == \"creative\":\n", " raise NotImplementedError\n", @@ -908,6 +906,36 @@ "Embedding created for query: Recognizing Human Actions in Data\n", "IRL is a promising approach to learning human values in part because of the easy availability of data. For supervised learning, humans need to produce many labeled instances specialized for a task. IRL, by contrast, is an unsupervised/semi-supervised approach where any record of human behavior is a potential data source. Facebook’s logs of user behavior provide trillions of data-points. YouTube videos, history books, and literature are a trove of data on human behavior in both actual and imagined scenarios. However, while there is lots of existing data that is informative about human preferences, we argue that exploiting this data for IRL will be a difficult, complex task with current techniques.\n" ] + }, + { + "ename": "KeyboardInterrupt", + "evalue": "", + "output_type": "error", + "traceback": [ + "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[1;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", + "Cell \u001b[1;32mIn[156], line 6\u001b[0m\n\u001b[0;32m 3\u001b[0m query \u001b[39m=\u001b[39m \u001b[39m\"\"\"\u001b[39m\u001b[39mRecognizing Human Actions in Data\u001b[39m\n\u001b[0;32m 4\u001b[0m \u001b[39mIRL is a promising approach to learning human values in part because of the easy availability of data. For supervised learning, humans need to produce many labeled instances specialized for a task. IRL, by contrast, is an unsupervised/semi-supervised approach where any record of human behavior is a potential data source. Facebook’s logs of user behavior provide trillions of data-points. YouTube videos, history books, and literature are a trove of data on human behavior in both actual and imagined scenarios. However, while there is lots of existing data that is informative about human preferences, we argue that exploiting this data for IRL will be a difficult, complex task with current techniques.\u001b[39m\u001b[39m\"\"\"\u001b[39m\n\u001b[0;32m 5\u001b[0m \u001b[39m# top_k_sources = AS.get_top_k(query, k)\u001b[39;00m\n\u001b[1;32m----> 6\u001b[0m answer, top_k_sources \u001b[39m=\u001b[39m AS\u001b[39m.\u001b[39;49msearch_and_answer(query, k)\u001b[39m#, HyDE=True)\u001b[39;00m\n\u001b[0;32m 7\u001b[0m \u001b[39mprint\u001b[39m(answer)\n", + "Cell \u001b[1;32mIn[142], line 93\u001b[0m, in \u001b[0;36mAlignmentSearch.search_and_answer\u001b[1;34m(self, question, k, HyDE)\u001b[0m\n\u001b[0;32m 91\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m 92\u001b[0m top_k \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mget_top_k(question, k)\n\u001b[1;32m---> 93\u001b[0m answer \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49manswer_question(question, top_k)\n\u001b[0;32m 94\u001b[0m \u001b[39mreturn\u001b[39;00m answer, top_k\n", + "Cell \u001b[1;32mIn[142], line 76\u001b[0m, in \u001b[0;36mAlignmentSearch.answer_question\u001b[1;34m(self, question, blocks)\u001b[0m\n\u001b[0;32m 73\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39manswer_question\u001b[39m(\u001b[39mself\u001b[39m, question: \u001b[39mstr\u001b[39m, blocks: List[\u001b[39mstr\u001b[39m]) \u001b[39m-\u001b[39m\u001b[39m>\u001b[39m \u001b[39mstr\u001b[39m:\n\u001b[0;32m 74\u001b[0m \u001b[39m# Receives a question (str) and a list of blocks and returns an answer (str) to the question.\u001b[39;00m\n\u001b[0;32m 75\u001b[0m messages \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mconstruct_messages(question, blocks, mode\u001b[39m=\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mbalanced\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[1;32m---> 76\u001b[0m answer \u001b[39m=\u001b[39m openai\u001b[39m.\u001b[39;49mChatCompletion\u001b[39m.\u001b[39;49mcreate(\n\u001b[0;32m 77\u001b[0m model\u001b[39m=\u001b[39;49mCOMPLETIONS_MODEL, \n\u001b[0;32m 78\u001b[0m messages\u001b[39m=\u001b[39;49mmessages\n\u001b[0;32m 79\u001b[0m )\n\u001b[0;32m 80\u001b[0m \u001b[39mreturn\u001b[39;00m answer[\u001b[39m\"\u001b[39m\u001b[39mchoices\u001b[39m\u001b[39m\"\u001b[39m][\u001b[39m0\u001b[39m][\u001b[39m\"\u001b[39m\u001b[39mmessage\u001b[39m\u001b[39m\"\u001b[39m][\u001b[39m\"\u001b[39m\u001b[39mcontent\u001b[39m\u001b[39m\"\u001b[39m]\n", + "File \u001b[1;32m~\\AppData\\Roaming\\Python\\Python310\\site-packages\\openai\\api_resources\\chat_completion.py:25\u001b[0m, in \u001b[0;36mChatCompletion.create\u001b[1;34m(cls, *args, **kwargs)\u001b[0m\n\u001b[0;32m 23\u001b[0m \u001b[39mwhile\u001b[39;00m \u001b[39mTrue\u001b[39;00m:\n\u001b[0;32m 24\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m---> 25\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39msuper\u001b[39m()\u001b[39m.\u001b[39mcreate(\u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n\u001b[0;32m 26\u001b[0m \u001b[39mexcept\u001b[39;00m TryAgain \u001b[39mas\u001b[39;00m e:\n\u001b[0;32m 27\u001b[0m \u001b[39mif\u001b[39;00m timeout \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m \u001b[39mand\u001b[39;00m time\u001b[39m.\u001b[39mtime() \u001b[39m>\u001b[39m start \u001b[39m+\u001b[39m timeout:\n", + "File \u001b[1;32m~\\AppData\\Roaming\\Python\\Python310\\site-packages\\openai\\api_resources\\abstract\\engine_api_resource.py:153\u001b[0m, in \u001b[0;36mEngineAPIResource.create\u001b[1;34m(cls, api_key, api_base, api_type, request_id, api_version, organization, **params)\u001b[0m\n\u001b[0;32m 127\u001b[0m \u001b[39m@classmethod\u001b[39m\n\u001b[0;32m 128\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mcreate\u001b[39m(\n\u001b[0;32m 129\u001b[0m \u001b[39mcls\u001b[39m,\n\u001b[1;32m (...)\u001b[0m\n\u001b[0;32m 136\u001b[0m \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mparams,\n\u001b[0;32m 137\u001b[0m ):\n\u001b[0;32m 138\u001b[0m (\n\u001b[0;32m 139\u001b[0m deployment_id,\n\u001b[0;32m 140\u001b[0m engine,\n\u001b[1;32m (...)\u001b[0m\n\u001b[0;32m 150\u001b[0m api_key, api_base, api_type, api_version, organization, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mparams\n\u001b[0;32m 151\u001b[0m )\n\u001b[1;32m--> 153\u001b[0m response, _, api_key \u001b[39m=\u001b[39m requestor\u001b[39m.\u001b[39;49mrequest(\n\u001b[0;32m 154\u001b[0m \u001b[39m\"\u001b[39;49m\u001b[39mpost\u001b[39;49m\u001b[39m\"\u001b[39;49m,\n\u001b[0;32m 155\u001b[0m url,\n\u001b[0;32m 156\u001b[0m params\u001b[39m=\u001b[39;49mparams,\n\u001b[0;32m 157\u001b[0m headers\u001b[39m=\u001b[39;49mheaders,\n\u001b[0;32m 158\u001b[0m stream\u001b[39m=\u001b[39;49mstream,\n\u001b[0;32m 159\u001b[0m request_id\u001b[39m=\u001b[39;49mrequest_id,\n\u001b[0;32m 160\u001b[0m request_timeout\u001b[39m=\u001b[39;49mrequest_timeout,\n\u001b[0;32m 161\u001b[0m )\n\u001b[0;32m 163\u001b[0m \u001b[39mif\u001b[39;00m stream:\n\u001b[0;32m 164\u001b[0m \u001b[39m# must be an iterator\u001b[39;00m\n\u001b[0;32m 165\u001b[0m \u001b[39massert\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39misinstance\u001b[39m(response, OpenAIResponse)\n", + "File \u001b[1;32m~\\AppData\\Roaming\\Python\\Python310\\site-packages\\openai\\api_requestor.py:216\u001b[0m, in \u001b[0;36mAPIRequestor.request\u001b[1;34m(self, method, url, params, headers, files, stream, request_id, request_timeout)\u001b[0m\n\u001b[0;32m 205\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mrequest\u001b[39m(\n\u001b[0;32m 206\u001b[0m \u001b[39mself\u001b[39m,\n\u001b[0;32m 207\u001b[0m method,\n\u001b[1;32m (...)\u001b[0m\n\u001b[0;32m 214\u001b[0m request_timeout: Optional[Union[\u001b[39mfloat\u001b[39m, Tuple[\u001b[39mfloat\u001b[39m, \u001b[39mfloat\u001b[39m]]] \u001b[39m=\u001b[39m \u001b[39mNone\u001b[39;00m,\n\u001b[0;32m 215\u001b[0m ) \u001b[39m-\u001b[39m\u001b[39m>\u001b[39m Tuple[Union[OpenAIResponse, Iterator[OpenAIResponse]], \u001b[39mbool\u001b[39m, \u001b[39mstr\u001b[39m]:\n\u001b[1;32m--> 216\u001b[0m result \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mrequest_raw(\n\u001b[0;32m 217\u001b[0m method\u001b[39m.\u001b[39;49mlower(),\n\u001b[0;32m 218\u001b[0m url,\n\u001b[0;32m 219\u001b[0m params\u001b[39m=\u001b[39;49mparams,\n\u001b[0;32m 220\u001b[0m supplied_headers\u001b[39m=\u001b[39;49mheaders,\n\u001b[0;32m 221\u001b[0m files\u001b[39m=\u001b[39;49mfiles,\n\u001b[0;32m 222\u001b[0m stream\u001b[39m=\u001b[39;49mstream,\n\u001b[0;32m 223\u001b[0m request_id\u001b[39m=\u001b[39;49mrequest_id,\n\u001b[0;32m 224\u001b[0m request_timeout\u001b[39m=\u001b[39;49mrequest_timeout,\n\u001b[0;32m 225\u001b[0m )\n\u001b[0;32m 226\u001b[0m resp, got_stream \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_interpret_response(result, stream)\n\u001b[0;32m 227\u001b[0m \u001b[39mreturn\u001b[39;00m resp, got_stream, \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mapi_key\n", + "File \u001b[1;32m~\\AppData\\Roaming\\Python\\Python310\\site-packages\\openai\\api_requestor.py:516\u001b[0m, in \u001b[0;36mAPIRequestor.request_raw\u001b[1;34m(self, method, url, params, supplied_headers, files, stream, request_id, request_timeout)\u001b[0m\n\u001b[0;32m 514\u001b[0m _thread_context\u001b[39m.\u001b[39msession \u001b[39m=\u001b[39m _make_session()\n\u001b[0;32m 515\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m--> 516\u001b[0m result \u001b[39m=\u001b[39m _thread_context\u001b[39m.\u001b[39;49msession\u001b[39m.\u001b[39;49mrequest(\n\u001b[0;32m 517\u001b[0m method,\n\u001b[0;32m 518\u001b[0m abs_url,\n\u001b[0;32m 519\u001b[0m headers\u001b[39m=\u001b[39;49mheaders,\n\u001b[0;32m 520\u001b[0m data\u001b[39m=\u001b[39;49mdata,\n\u001b[0;32m 521\u001b[0m files\u001b[39m=\u001b[39;49mfiles,\n\u001b[0;32m 522\u001b[0m stream\u001b[39m=\u001b[39;49mstream,\n\u001b[0;32m 523\u001b[0m timeout\u001b[39m=\u001b[39;49mrequest_timeout \u001b[39mif\u001b[39;49;00m request_timeout \u001b[39melse\u001b[39;49;00m TIMEOUT_SECS,\n\u001b[0;32m 524\u001b[0m )\n\u001b[0;32m 525\u001b[0m \u001b[39mexcept\u001b[39;00m requests\u001b[39m.\u001b[39mexceptions\u001b[39m.\u001b[39mTimeout \u001b[39mas\u001b[39;00m e:\n\u001b[0;32m 526\u001b[0m \u001b[39mraise\u001b[39;00m error\u001b[39m.\u001b[39mTimeout(\u001b[39m\"\u001b[39m\u001b[39mRequest timed out: \u001b[39m\u001b[39m{}\u001b[39;00m\u001b[39m\"\u001b[39m\u001b[39m.\u001b[39mformat(e)) \u001b[39mfrom\u001b[39;00m \u001b[39me\u001b[39;00m\n", + "File \u001b[1;32mc:\\Python310\\lib\\site-packages\\requests\\sessions.py:587\u001b[0m, in \u001b[0;36mSession.request\u001b[1;34m(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)\u001b[0m\n\u001b[0;32m 582\u001b[0m send_kwargs \u001b[39m=\u001b[39m {\n\u001b[0;32m 583\u001b[0m \u001b[39m\"\u001b[39m\u001b[39mtimeout\u001b[39m\u001b[39m\"\u001b[39m: timeout,\n\u001b[0;32m 584\u001b[0m \u001b[39m\"\u001b[39m\u001b[39mallow_redirects\u001b[39m\u001b[39m\"\u001b[39m: allow_redirects,\n\u001b[0;32m 585\u001b[0m }\n\u001b[0;32m 586\u001b[0m send_kwargs\u001b[39m.\u001b[39mupdate(settings)\n\u001b[1;32m--> 587\u001b[0m resp \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39msend(prep, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39msend_kwargs)\n\u001b[0;32m 589\u001b[0m \u001b[39mreturn\u001b[39;00m resp\n", + "File \u001b[1;32mc:\\Python310\\lib\\site-packages\\requests\\sessions.py:701\u001b[0m, in \u001b[0;36mSession.send\u001b[1;34m(self, request, **kwargs)\u001b[0m\n\u001b[0;32m 698\u001b[0m start \u001b[39m=\u001b[39m preferred_clock()\n\u001b[0;32m 700\u001b[0m \u001b[39m# Send the request\u001b[39;00m\n\u001b[1;32m--> 701\u001b[0m r \u001b[39m=\u001b[39m adapter\u001b[39m.\u001b[39msend(request, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n\u001b[0;32m 703\u001b[0m \u001b[39m# Total elapsed time of the request (approximately)\u001b[39;00m\n\u001b[0;32m 704\u001b[0m elapsed \u001b[39m=\u001b[39m preferred_clock() \u001b[39m-\u001b[39m start\n", + "File \u001b[1;32mc:\\Python310\\lib\\site-packages\\requests\\adapters.py:489\u001b[0m, in \u001b[0;36mHTTPAdapter.send\u001b[1;34m(self, request, stream, timeout, verify, cert, proxies)\u001b[0m\n\u001b[0;32m 487\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m 488\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m chunked:\n\u001b[1;32m--> 489\u001b[0m resp \u001b[39m=\u001b[39m conn\u001b[39m.\u001b[39;49murlopen(\n\u001b[0;32m 490\u001b[0m method\u001b[39m=\u001b[39;49mrequest\u001b[39m.\u001b[39;49mmethod,\n\u001b[0;32m 491\u001b[0m url\u001b[39m=\u001b[39;49murl,\n\u001b[0;32m 492\u001b[0m body\u001b[39m=\u001b[39;49mrequest\u001b[39m.\u001b[39;49mbody,\n\u001b[0;32m 493\u001b[0m headers\u001b[39m=\u001b[39;49mrequest\u001b[39m.\u001b[39;49mheaders,\n\u001b[0;32m 494\u001b[0m redirect\u001b[39m=\u001b[39;49m\u001b[39mFalse\u001b[39;49;00m,\n\u001b[0;32m 495\u001b[0m assert_same_host\u001b[39m=\u001b[39;49m\u001b[39mFalse\u001b[39;49;00m,\n\u001b[0;32m 496\u001b[0m preload_content\u001b[39m=\u001b[39;49m\u001b[39mFalse\u001b[39;49;00m,\n\u001b[0;32m 497\u001b[0m decode_content\u001b[39m=\u001b[39;49m\u001b[39mFalse\u001b[39;49;00m,\n\u001b[0;32m 498\u001b[0m retries\u001b[39m=\u001b[39;49m\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mmax_retries,\n\u001b[0;32m 499\u001b[0m timeout\u001b[39m=\u001b[39;49mtimeout,\n\u001b[0;32m 500\u001b[0m )\n\u001b[0;32m 502\u001b[0m \u001b[39m# Send the request.\u001b[39;00m\n\u001b[0;32m 503\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m 504\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mhasattr\u001b[39m(conn, \u001b[39m\"\u001b[39m\u001b[39mproxy_pool\u001b[39m\u001b[39m\"\u001b[39m):\n", + "File \u001b[1;32mc:\\Python310\\lib\\site-packages\\urllib3\\connectionpool.py:703\u001b[0m, in \u001b[0;36mHTTPConnectionPool.urlopen\u001b[1;34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)\u001b[0m\n\u001b[0;32m 700\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_prepare_proxy(conn)\n\u001b[0;32m 702\u001b[0m \u001b[39m# Make the request on the httplib connection object.\u001b[39;00m\n\u001b[1;32m--> 703\u001b[0m httplib_response \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_make_request(\n\u001b[0;32m 704\u001b[0m conn,\n\u001b[0;32m 705\u001b[0m method,\n\u001b[0;32m 706\u001b[0m url,\n\u001b[0;32m 707\u001b[0m timeout\u001b[39m=\u001b[39;49mtimeout_obj,\n\u001b[0;32m 708\u001b[0m body\u001b[39m=\u001b[39;49mbody,\n\u001b[0;32m 709\u001b[0m headers\u001b[39m=\u001b[39;49mheaders,\n\u001b[0;32m 710\u001b[0m chunked\u001b[39m=\u001b[39;49mchunked,\n\u001b[0;32m 711\u001b[0m )\n\u001b[0;32m 713\u001b[0m \u001b[39m# If we're going to release the connection in ``finally:``, then\u001b[39;00m\n\u001b[0;32m 714\u001b[0m \u001b[39m# the response doesn't need to know about the connection. Otherwise\u001b[39;00m\n\u001b[0;32m 715\u001b[0m \u001b[39m# it will also try to release it and we'll have a double-release\u001b[39;00m\n\u001b[0;32m 716\u001b[0m \u001b[39m# mess.\u001b[39;00m\n\u001b[0;32m 717\u001b[0m response_conn \u001b[39m=\u001b[39m conn \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m release_conn \u001b[39melse\u001b[39;00m \u001b[39mNone\u001b[39;00m\n", + "File \u001b[1;32mc:\\Python310\\lib\\site-packages\\urllib3\\connectionpool.py:449\u001b[0m, in \u001b[0;36mHTTPConnectionPool._make_request\u001b[1;34m(self, conn, method, url, timeout, chunked, **httplib_request_kw)\u001b[0m\n\u001b[0;32m 444\u001b[0m httplib_response \u001b[39m=\u001b[39m conn\u001b[39m.\u001b[39mgetresponse()\n\u001b[0;32m 445\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mBaseException\u001b[39;00m \u001b[39mas\u001b[39;00m e:\n\u001b[0;32m 446\u001b[0m \u001b[39m# Remove the TypeError from the exception chain in\u001b[39;00m\n\u001b[0;32m 447\u001b[0m \u001b[39m# Python 3 (including for exceptions like SystemExit).\u001b[39;00m\n\u001b[0;32m 448\u001b[0m \u001b[39m# Otherwise it looks like a bug in the code.\u001b[39;00m\n\u001b[1;32m--> 449\u001b[0m six\u001b[39m.\u001b[39;49mraise_from(e, \u001b[39mNone\u001b[39;49;00m)\n\u001b[0;32m 450\u001b[0m \u001b[39mexcept\u001b[39;00m (SocketTimeout, BaseSSLError, SocketError) \u001b[39mas\u001b[39;00m e:\n\u001b[0;32m 451\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_raise_timeout(err\u001b[39m=\u001b[39me, url\u001b[39m=\u001b[39murl, timeout_value\u001b[39m=\u001b[39mread_timeout)\n", + "File \u001b[1;32m:3\u001b[0m, in \u001b[0;36mraise_from\u001b[1;34m(value, from_value)\u001b[0m\n", + "File \u001b[1;32mc:\\Python310\\lib\\site-packages\\urllib3\\connectionpool.py:444\u001b[0m, in \u001b[0;36mHTTPConnectionPool._make_request\u001b[1;34m(self, conn, method, url, timeout, chunked, **httplib_request_kw)\u001b[0m\n\u001b[0;32m 441\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mTypeError\u001b[39;00m:\n\u001b[0;32m 442\u001b[0m \u001b[39m# Python 3\u001b[39;00m\n\u001b[0;32m 443\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m--> 444\u001b[0m httplib_response \u001b[39m=\u001b[39m conn\u001b[39m.\u001b[39;49mgetresponse()\n\u001b[0;32m 445\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mBaseException\u001b[39;00m \u001b[39mas\u001b[39;00m e:\n\u001b[0;32m 446\u001b[0m \u001b[39m# Remove the TypeError from the exception chain in\u001b[39;00m\n\u001b[0;32m 447\u001b[0m \u001b[39m# Python 3 (including for exceptions like SystemExit).\u001b[39;00m\n\u001b[0;32m 448\u001b[0m \u001b[39m# Otherwise it looks like a bug in the code.\u001b[39;00m\n\u001b[0;32m 449\u001b[0m six\u001b[39m.\u001b[39mraise_from(e, \u001b[39mNone\u001b[39;00m)\n", + "File \u001b[1;32mc:\\Python310\\lib\\http\\client.py:1374\u001b[0m, in \u001b[0;36mHTTPConnection.getresponse\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m 1372\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m 1373\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m-> 1374\u001b[0m response\u001b[39m.\u001b[39;49mbegin()\n\u001b[0;32m 1375\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mConnectionError\u001b[39;00m:\n\u001b[0;32m 1376\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mclose()\n", + "File \u001b[1;32mc:\\Python310\\lib\\http\\client.py:318\u001b[0m, in \u001b[0;36mHTTPResponse.begin\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m 316\u001b[0m \u001b[39m# read until we get a non-100 response\u001b[39;00m\n\u001b[0;32m 317\u001b[0m \u001b[39mwhile\u001b[39;00m \u001b[39mTrue\u001b[39;00m:\n\u001b[1;32m--> 318\u001b[0m version, status, reason \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_read_status()\n\u001b[0;32m 319\u001b[0m \u001b[39mif\u001b[39;00m status \u001b[39m!=\u001b[39m CONTINUE:\n\u001b[0;32m 320\u001b[0m \u001b[39mbreak\u001b[39;00m\n", + "File \u001b[1;32mc:\\Python310\\lib\\http\\client.py:279\u001b[0m, in \u001b[0;36mHTTPResponse._read_status\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m 278\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39m_read_status\u001b[39m(\u001b[39mself\u001b[39m):\n\u001b[1;32m--> 279\u001b[0m line \u001b[39m=\u001b[39m \u001b[39mstr\u001b[39m(\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mfp\u001b[39m.\u001b[39;49mreadline(_MAXLINE \u001b[39m+\u001b[39;49m \u001b[39m1\u001b[39;49m), \u001b[39m\"\u001b[39m\u001b[39miso-8859-1\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[0;32m 280\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mlen\u001b[39m(line) \u001b[39m>\u001b[39m _MAXLINE:\n\u001b[0;32m 281\u001b[0m \u001b[39mraise\u001b[39;00m LineTooLong(\u001b[39m\"\u001b[39m\u001b[39mstatus line\u001b[39m\u001b[39m\"\u001b[39m)\n", + "File \u001b[1;32mc:\\Python310\\lib\\socket.py:705\u001b[0m, in \u001b[0;36mSocketIO.readinto\u001b[1;34m(self, b)\u001b[0m\n\u001b[0;32m 703\u001b[0m \u001b[39mwhile\u001b[39;00m \u001b[39mTrue\u001b[39;00m:\n\u001b[0;32m 704\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m--> 705\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_sock\u001b[39m.\u001b[39;49mrecv_into(b)\n\u001b[0;32m 706\u001b[0m \u001b[39mexcept\u001b[39;00m timeout:\n\u001b[0;32m 707\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_timeout_occurred \u001b[39m=\u001b[39m \u001b[39mTrue\u001b[39;00m\n", + "File \u001b[1;32mc:\\Python310\\lib\\ssl.py:1274\u001b[0m, in \u001b[0;36mSSLSocket.recv_into\u001b[1;34m(self, buffer, nbytes, flags)\u001b[0m\n\u001b[0;32m 1270\u001b[0m \u001b[39mif\u001b[39;00m flags \u001b[39m!=\u001b[39m \u001b[39m0\u001b[39m:\n\u001b[0;32m 1271\u001b[0m \u001b[39mraise\u001b[39;00m \u001b[39mValueError\u001b[39;00m(\n\u001b[0;32m 1272\u001b[0m \u001b[39m\"\u001b[39m\u001b[39mnon-zero flags not allowed in calls to recv_into() on \u001b[39m\u001b[39m%s\u001b[39;00m\u001b[39m\"\u001b[39m \u001b[39m%\u001b[39m\n\u001b[0;32m 1273\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m\u001b[39m__class__\u001b[39m)\n\u001b[1;32m-> 1274\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mread(nbytes, buffer)\n\u001b[0;32m 1275\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m 1276\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39msuper\u001b[39m()\u001b[39m.\u001b[39mrecv_into(buffer, nbytes, flags)\n", + "File \u001b[1;32mc:\\Python310\\lib\\ssl.py:1130\u001b[0m, in \u001b[0;36mSSLSocket.read\u001b[1;34m(self, len, buffer)\u001b[0m\n\u001b[0;32m 1128\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m 1129\u001b[0m \u001b[39mif\u001b[39;00m buffer \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[1;32m-> 1130\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_sslobj\u001b[39m.\u001b[39;49mread(\u001b[39mlen\u001b[39;49m, buffer)\n\u001b[0;32m 1131\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m 1132\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_sslobj\u001b[39m.\u001b[39mread(\u001b[39mlen\u001b[39m)\n", + "\u001b[1;31mKeyboardInterrupt\u001b[0m: " + ] } ], "source": [ @@ -1027,7 +1055,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 158, "metadata": {}, "outputs": [ { @@ -1035,22 +1063,18 @@ "output_type": "stream", "text": [ "Source Truth Empirical Difference\n", - "alignment forum 2138 4 2134 \n", - "manual ? 0 UNKNOWN \n", - "waitbutwhy.com 2 0 2 \n", - "https://aiimpacts.or 227 0 227 \n", - "arbital.com 223 0 223 \n", - "https://intelligence 479 4 475 \n", - "reports ? 0 UNKNOWN \n", - "https://aisafety.cam 8 0 8 \n", - "curriculum ? 0 UNKNOWN \n", - "https://www.yudkowsk 23 0 23 \n", - "distill 49 0 49 \n", - "total 41614 8 41606 \n", + "https://aipulse.org 23 23 0 \n", + "manual ? 1 UNKNOWN \n", + "waitbutwhy.com 2 2 0 \n", + "https://vkrakovna.wo 43 43 0 \n", + "https://jsteinhardt. 39 39 0 \n", + "https://aisafety.cam 8 8 0 \n", + "curriculum ? 1 UNKNOWN \n", + "https://www.yudkowsk 23 23 0 \n", "\n", " Truth Empirical Difference\n", - "Word Count 53550146 11997 53538149 \n", - "Character Count 351767163 72919 351694244 \n" + "Word Count 53550146 335374 53214772 \n", + "Character Count 351767163 2096402 349670761 \n" ] } ], @@ -1087,11 +1111,11 @@ "\n", "# Print table. First row has Truth and Empirical findings.\n", "print(f\"{'Source':<20} {'Truth':<10} {'Empirical':<10} {'Difference':<10}\")\n", - "for source in dataset.num_articles:\n", + "for source in dataset.articles_count:\n", " try:\n", - " print(f\"{source[:20]:<20} {num_articles_truth[source]:<10} {dataset.num_articles[source]:<10} {num_articles_truth[source] - dataset.num_articles[source]:<10}\")\n", + " print(f\"{source[:20]:<20} {num_articles_truth[source]:<10} {dataset.articles_count[source]:<10} {num_articles_truth[source] - dataset.articles_count[source]:<10}\")\n", " except TypeError:\n", - " print(f\"{source[:20]:<20} {num_articles_truth[source]:<10} {dataset.num_articles[source]:<10} {'UNKNOWN':<10}\")\n", + " print(f\"{source[:20]:<20} {num_articles_truth[source]:<10} {dataset.articles_count[source]:<10} {'UNKNOWN':<10}\")\n", "\n", "# Compare true and empirical word counts and character counts\n", "print(f\"\\n{'':<20} {'Truth':<10} {'Empirical':<10} {'Difference':<10}\")\n", diff --git a/src/text_splitter.py b/src/text_splitter.py index bec3297..f720ae0 100644 --- a/src/text_splitter.py +++ b/src/text_splitter.py @@ -2,130 +2,104 @@ import re from typing import List import tiktoken -# FROM https://stackoverflow.com/a/31505798/16185542 -# -*- coding: utf-8 -*- -alphabets= "([A-Za-z])" -prefixes = "(Mr|St|Mrs|Ms|Dr)[.]" -suffixes = "(Inc|Ltd|Jr|Sr|Co)" -starters = "(Mr|Mrs|Ms|Dr|Prof|Capt|Cpt|Lt|He\s|She\s|It\s|They\s|Their\s|Our\s|We\s|But\s|However\s|That\s|This\s|Wherever)" -acronyms = "([A-Z][.][A-Z][.](?:[A-Z][.])?)" -websites = "[.](com|net|org|io|gov|edu|me)" -digits = "([0-9])" +import re +from typing import List +import nltk -def split_into_sentences(text): - text = " " + text + " " - text = text.replace("\n"," ") - text = text.replace("?!", "?") - text = re.sub(prefixes,"\\1",text) - text = re.sub(websites,"\\1",text) - text = re.sub(digits + "[.]" + digits,"\\1\\2",text) - if "..." in text: text = text.replace("...","") - if "Ph.D" in text: text = text.replace("Ph.D.","PhD") - text = re.sub("\s" + alphabets + "[.] "," \\1 ",text) - text = re.sub(acronyms+" "+starters,"\\1 \\2",text) - text = re.sub(alphabets + "[.]" + alphabets + "[.]" + alphabets + "[.]","\\1\\2\\3",text) - text = re.sub(alphabets + "[.]" + alphabets + "[.]","\\1\\2",text) - text = re.sub(" "+suffixes+"[.] "+starters," \\1 \\2",text) - text = re.sub(" "+suffixes+"[.]"," \\1",text) - text = re.sub(" " + alphabets + "[.]"," \\1",text) - if "”" in text: text = text.replace(".”","”.") - if "\"" in text: text = text.replace(".\"","\".") - if "!" in text: text = text.replace("!\"","\"!") - if "?" in text: text = text.replace("?\"","\"?") - text = text.replace(".",".") - text = text.replace("?","?") - text = text.replace("!","!") - text = text.replace("",".") - sentences = text.split("") - sentences = sentences[:-1] - sentences = [s.strip() for s in sentences] - - if sentences == []: - sentences = [text.strip()] +# Download the Punkt tokenizer if you haven't already +# nltk.download("punkt") + +def split_into_sentences(text: str) -> List[str]: + """ + Splits the input text into sentences. + + :param text: The input text to be split. + :return: A list of sentences. + """ + text = text.replace("\n", " ") # Replace newline characters with spaces + sentences = nltk.sent_tokenize(text) # Use the Punkt tokenizer from the NLTK library to split the text into sentences + sentences = [s.strip() for s in sentences] # Strip leading and trailing whitespace from each sentence return sentences class TokenSplitter: - """splits text into blocks of tokens according to chatgpt's tokenizer""" + """Splits text into blocks of tokens according to chatgpt's tokenizer.""" + def __init__(self, min_tokens: int = 500, max_tokens: int = 750): self.encoding = tiktoken.get_encoding("cl100k_base") self.min_tokens = min_tokens self.max_tokens = max_tokens - self.blocks = [] - self.signature = "{url, title, author} unknown" - + self.default_signature = "{url, title, author} unknown" + def _text_splitter(self, text: str, signature: str) -> List[str]: + """Splits text into blocks of tokens according to chatgpt's tokenizer.""" + enc = self.encoding.encode # takes a string and returns a list of ints (tokens) + dec = self.encoding.decode # takes a list of ints (tokens) and returns a string + tok_len = lambda x: len(enc(x)) # length of a string in tokens - def _text_splitter(self, text: str) -> List[str]: - """splits text into blocks of tokens according to chatgpt's tokenizer""" - # Do not call this function outside of split() - - enc = self.encoding.encode # takes a string and returns a list of ints (tokens) - dec = self.encoding.decode # takes a list of ints (tokens) and returns a string - tok_len = lambda x: len(enc(x)) # length of a string in tokens - - max_tokens = self.max_tokens - tok_len(self.signature) - 10 # 10 to be safe + max_tokens = self.max_tokens - tok_len(signature) - 10 # 10 to be safe assert max_tokens > 0, "max_tokens is too small for the signature" - - min_tokens = self.min_tokens - tok_len(self.signature) - 10 # 10 to be safe + + min_tokens = self.min_tokens - tok_len(signature) - 10 # 10 to be safe assert min_tokens > 0, "min_tokens is too small for the signature" + blocks = [] current_block = "" paragraphs = text.split("\n\n") + for paragraph in paragraphs: sentences = split_into_sentences(paragraph) if current_block != "": current_block += "\n\n" for sentence in sentences: - potential_new_block = current_block + " " + sentence + potential_new_block = f"{current_block} {sentence}" if tok_len(potential_new_block) <= max_tokens: current_block = potential_new_block else: - self.blocks.append(current_block) + blocks.append(current_block) if tok_len(sentence) < max_tokens: current_block = sentence else: - self.blocks.append(dec(enc(sentence)[:max_tokens])) + blocks.append(dec(enc(sentence)[:max_tokens])) current_block = "" if tok_len(current_block) > min_tokens: - self.blocks.append(current_block) + blocks.append(current_block) current_block = "" if current_block != "": - if len(self.blocks) == 0: - self.blocks.append(current_block) - return - latest_block = self.blocks[-1] - len_cur_block = tok_len(current_block) - latest_plus_current = latest_block + current_block - - if len_cur_block > min_tokens: - self.blocks.append(current_block) - + if len(blocks) == 0: + blocks.append(current_block) else: - #select the last self.max_tokens tokens from the latest block - last_block = dec(enc(latest_plus_current)[-max_tokens:]) - self.blocks.append(last_block) + latest_block = blocks[-1] + len_cur_block = tok_len(current_block) + latest_plus_current = latest_block + current_block - - - def split(self, text: str, signature: str) -> List[str]: - self.signature = signature - self._text_splitter(text) - blocks = self.blocks - self.blocks = [] - self.signature = "{url, title, author} unknown" - - # check all block elements are strings + if len_cur_block > min_tokens: + blocks.append(current_block) + + else: + # select the last self.max_tokens tokens from the latest block + last_block = dec(enc(latest_plus_current)[-max_tokens:]) + blocks.append(last_block) + + return blocks + + def split(self, text: str, signature: str = None) -> List[str]: + if signature is None: + signature = self.default_signature + + blocks = self._text_splitter(text, signature) + + # Check all block elements are strings assert all([isinstance(block, str) for block in blocks]), "block elements are not strings" output = [f"{block}\n - {signature}" for block in blocks] - #check all output elements are strings + # Check all output elements are strings assert all([isinstance(block, str) for block in output]), "output elements are not strings" return output @@ -236,6 +210,6 @@ Anyway, at this point we’re getting into specifics of portals, so I’ll cut o signature = "Title: Humans are very reliable agents, Author: alyssavance, URL: https://www.lesswrong.com/posts/28zsuPaJpKAGSX4zq" - splitting = TokenSplitter(max_tokens=500, min_tokens=400) + splitting = TokenSplitter(max_tokens=250, min_tokens=330) blocks = splitting.split(text, signature) print("\n\n\n".join(blocks)) \ No newline at end of file