From 0f61660c0c66db053bba8eb853544787b57857b5 Mon Sep 17 00:00:00 2001 From: henri123lemoine Date: Thu, 16 Mar 2023 11:29:58 -0400 Subject: [PATCH 1/2] Added tests --- src/Embeddings Search/testing.ipynb | 328 +++++++++++++++++++++------- 1 file changed, 247 insertions(+), 81 deletions(-) diff --git a/src/Embeddings Search/testing.ipynb b/src/Embeddings Search/testing.ipynb index 473ba99..44f7d40 100644 --- a/src/Embeddings Search/testing.ipynb +++ b/src/Embeddings Search/testing.ipynb @@ -71,62 +71,6 @@ "```" ] }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "https://aipulse.org: title links link authors author text (tags)\n", - "\n", - "None: title url text\n", - "\n", - "ebook: title book_title authors text (publication_date)\n", - "\n", - "https://qualiacomputing.com: title link authors author text (tags)\n", - "\n", - "alignment forum: title url authors text (tags)\n", - "\n", - "lesswrong: title authors url text (tags score date_published)\n", - "\n", - "manual: title authors text (date_published)\n", - "\n", - "arxiv: title authors url text (citation_level alignment_text confidence_score date_published)\n", - "\n", - "https://deepmindsafetyresearch.medium.com/: title url text\n", - "\n", - "waitbutwhy.com: title authors text (date_published)\n", - "\n", - "GitHub: book_title authors author text\n", - "\n", - "https://aiimpacts.org: title link authors author text (tags)\n", - "\n", - "arbital.com: title authors url text (date_published)\n", - "\n", - "carado.moe: title authors text (date_published)\n", - "\n", - "nonarxiv_papers: title authors doi text (date_published)\n", - "\n", - "https://vkrakovna.wordpress.com: title link authors author text (tags)\n", - "\n", - "https://jsteinhardt.wordpress.com: title link authors author text (tags)\n", - "\n", - "audio-transcripts: title authors text (date_published)\n", - "\n", - "https://intelligence.org: title link authors author text (tags)\n", - "\n", - "youtube: title authors url text (date_published)\n", - "\n", - "reports: title authors doi text (date_published)\n", - "\n", - "https://aisafety.camp: title link authors author text (tags)\n", - "\n", - "curriculum: title authors text (date_published)\n", - "\n", - "https://www.yudkowsky.net: title link authors author text (tags)\n", - "\n", - "distill: title authors doi text (date_published)" - ] - }, { "attachments": {}, "cell_type": "markdown", @@ -134,9 +78,19 @@ "source": [ "Useful links:\n", "\n", - "- https://github.com/openai/openai-cookbook/blob/main/examples/Semantic_text_search_using_embeddings.ipynb\n", + "- Semantic Search OpenAI Cookbook: https://github.com/openai/openai-cookbook/blob/main/examples/Semantic_text_search_using_embeddings.ipynb\n", "\n", - "- https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb" + "- Question-Answering OpenAI Cookbook: https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb\n", + "\n", + "- Pinecone: https://app.pinecone.io\n", + "\n", + "- Retrieval Enhanced Generative Question Answering with Pinecone: https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/pinecone/Gen_QA.ipynb\n", + "\n", + "- Moderation: https://platform.openai.com/docs/guides/moderation/quickstart\n", + "\n", + "- 5k Bounty: https://www.lesswrong.com/posts/SLRLuiuDykfTdmesK/speed-running-everyone-through-the-bad-alignement-bingo\n", + "\n", + "- Handling rate-limits Cookbook: https://github.com/openai/openai-cookbook/blob/main/examples/How_to_handle_rate_limits.ipynb" ] }, { @@ -149,7 +103,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -181,7 +135,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -208,7 +162,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -259,7 +213,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -269,7 +223,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -360,7 +314,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ @@ -375,7 +329,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 7, "metadata": {}, "outputs": [], "source": [ @@ -2948,29 +2902,241 @@ "print(embeddings_2.shape)" ] }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import json\n", + "import argparse\n", + "import openai\n", + "from typing import List\n", + "\n", + "from rich.console import Console\n", + "from rich.markdown import Markdown#, MarkdownIt\n", + "from rich.live import Live\n", + "\n", + "# import readline\n", + "try:\n", + " import rlcompleter\n", + "except ImportError:\n", + " pass" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "assistant_prompt = \"You are a helpful assistant, and you help users by answering questions and providing information about AI Alignment, on which you are extremely knowledgeable. Answer the user's question even if you are not certain of the answer; it is supremely important that you do attempt to offer an answer related to the user's query.\"\n", + "\n", + "question = \"What are the most important things to know about AI Alignment?\"\n", + "\n", + "messages = [\n", + " {\"role\": \"system\", \"content\": assistant_prompt},\n", + " {\"role\": \"user\", \"content\": question},\n", + "]" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "ename": "NameError", + "evalue": "name '__file__' is not defined", + "output_type": "error", + "traceback": [ + "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[1;31mNameError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[1;32mIn[16], line 1\u001b[0m\n\u001b[1;32m----> 1\u001b[0m \u001b[39mclass\u001b[39;00m \u001b[39mConfig\u001b[39;00m:\n\u001b[0;32m 2\u001b[0m sep \u001b[39m=\u001b[39m Markdown(\u001b[39m\"\u001b[39m\u001b[39m---\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[0;32m 3\u001b[0m baseDir \u001b[39m=\u001b[39m os\u001b[39m.\u001b[39mpath\u001b[39m.\u001b[39mdirname(os\u001b[39m.\u001b[39mpath\u001b[39m.\u001b[39mrealpath(\u001b[39m__file__\u001b[39m))\n", + "Cell \u001b[1;32mIn[16], line 3\u001b[0m, in \u001b[0;36mConfig\u001b[1;34m()\u001b[0m\n\u001b[0;32m 1\u001b[0m \u001b[39mclass\u001b[39;00m \u001b[39mConfig\u001b[39;00m:\n\u001b[0;32m 2\u001b[0m sep \u001b[39m=\u001b[39m Markdown(\u001b[39m\"\u001b[39m\u001b[39m---\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[1;32m----> 3\u001b[0m baseDir \u001b[39m=\u001b[39m os\u001b[39m.\u001b[39mpath\u001b[39m.\u001b[39mdirname(os\u001b[39m.\u001b[39mpath\u001b[39m.\u001b[39mrealpath(\u001b[39m__file__\u001b[39;49m))\n\u001b[0;32m 4\u001b[0m default \u001b[39m=\u001b[39m os\u001b[39m.\u001b[39mpath\u001b[39m.\u001b[39mjoin(baseDir, \u001b[39m\"\u001b[39m\u001b[39mconfig.json\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[0;32m 6\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39m__init__\u001b[39m(\u001b[39mself\u001b[39m) \u001b[39m-\u001b[39m\u001b[39m>\u001b[39m \u001b[39mNone\u001b[39;00m:\n", + "\u001b[1;31mNameError\u001b[0m: name '__file__' is not defined" + ] + } + ], + "source": [ + "class Config:\n", + " sep = Markdown(\"---\")\n", + " baseDir = os.path.dirname(os.path.realpath(__file__))\n", + " default = os.path.join(baseDir, \"config.json\")\n", + "\n", + " def __init__(self) -> None:\n", + " self.cfg = {}\n", + " self.history = []\n", + "\n", + " def load(self, file):\n", + " with open(file, \"r\") as f:\n", + " self.cfg = json.load(f)\n", + " self.history.extend(self.cfg.get(\"history\", []))\n", + "\n", + " @property\n", + " def key(self):\n", + " return self.cfg.get(\"key\", os.environ.get(\"OPENAI_API_KEY\", \"\"))\n", + "\n", + " @property\n", + " def model(self):\n", + " return self.cfg.get(\"model\", \"gpt-3.5-turbo\")\n", + "\n", + " @property\n", + " def prompt(self):\n", + " return self.cfg.get(\"prompt\", [])\n", + "\n", + " @property\n", + " def stream(self):\n", + " return self.cfg.get(\"stream\", False)\n", + "\n", + " @property\n", + " def response(self):\n", + " return self.cfg.get(\"response\", False)\n", + "\n", + " @property\n", + " def proxy(self):\n", + " return self.cfg.get(\"proxy\", \"\")\n" + ] + }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "try:\n", - " response = openai.ChatCompletion.create(\n", - " model=kConfig.model,\n", - " messages=messages,\n", - " stream=True)\n", - " with Live(md, auto_refresh=False) as lv:\n", - " for part in response:\n", - " finish_reason = part[\"choices\"][0][\"finish_reason\"]\n", - " if \"content\" in part[\"choices\"][0][\"delta\"]:\n", - " content = part[\"choices\"][0][\"delta\"][\"content\"]\n", - " answer += content\n", - " md.markup = answer\n", - " md.parsed = parser.parse(md.markup)\n", - " lv.refresh()\n", - " elif finish_reason:\n", - " pass\n" + "c = Console()\n", + "\n", + "def query_openai_stream(data: dict):\n", + " assistant_prompt = \"You are a helpful assistant, and you help users by answering questions and providing information about AI Alignment, on which you are extremely knowledgeable. Answer the user's question even if you are not certain of the answer; it is supremely important that you do attempt to offer an answer related to the user's query.\"\n", + "\n", + " question = \"What are the most important things to know about AI Alignment?\"\n", + "\n", + " messages = [\n", + " {\"role\": \"system\", \"content\": assistant_prompt},\n", + " {\"role\": \"user\", \"content\": question},\n", + " ]\n", + " md = Markdown(\"\")\n", + " parser = MarkdownIt().enable(\"strikethrough\")\n", + " answer = \"\"\n", + " try:\n", + " response = openai.ChatCompletion.create(\n", + " model=COMPLETIONS_MODEL,\n", + " messages=messages,\n", + " stream=True)\n", + " with Live(md, auto_refresh=False) as lv:\n", + " for part in response:\n", + " finish_reason = part[\"choices\"][0][\"finish_reason\"]\n", + " if \"content\" in part[\"choices\"][0][\"delta\"]:\n", + " content = part[\"choices\"][0][\"delta\"][\"content\"]\n", + " answer += content\n", + " md.markup = answer\n", + " md.parsed = parser.parse(md.markup)\n", + " lv.refresh()\n", + " elif finish_reason:\n", + " pass\n", + " except KeyboardInterrupt:\n", + " c.print(\"Canceled\")\n", + " except openai.error.OpenAIError as e:\n", + " c.print(e)\n", + " answer = \"\"\n", + " except Exception as e:\n", + " c.print(e)\n", + " c.print(Config.sep)\n", + " return answer" ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "import time\n", + "import asyncio\n", + "from rich.console import Console\n", + "from rich.markdown import Markdown# , MarkdownIt\n", + "from markdown_it import MarkdownIt\n", + "from rich.live import Live\n", + "\n", + "text = \"\"\"\\\n", + "As an AI language model, I don't have the ability to directly perform syntax highlighting, but I can show you an example of how to use Markdown syntax for code blocks:\n", + "```python\n", + "def welcome(name):\n", + " print(f\"Hello, {name}!\")\n", + "# Call the function\n", + "welcome(\"Alice\")\n", + "```\n", + "This creates a code block with syntax highlighting for Python code.\n", + "To make this work, you need to include the name of the programming language immediately after the first set of backticks.\n", + "In this case, we've specified that we're writing Python code by including `python` after the backticks.\n", + "\"\"\"\n", + "\n", + "c = Console()\n", + "\n", + "def test_live():\n", + " md = Markdown(\"\")\n", + " parser = MarkdownIt().enable(\"strikethrough\")\n", + " parts = [text[i:i+5] for i in range(0, len(text), 5)]\n", + " with Live(md, refresh_per_second=4): # update 4 times a second to feel fluid\n", + " for part in parts:\n", + " time.sleep(0.1)\n", + " md.markup += part\n", + " md.parsed = parser.parse(md.markup)\n", + "\n", + "async def test_stream():\n", + " # NOTE: aiohttp does not support socks5 proxy yet\n", + " openai.proxy = \"http://127.0.0.1:1080\"\n", + " messages = [\n", + " { \"role\": \"system\", \"content\": \"Use triple backticks with the language name for every code block in your markdown response, if any.\" },\n", + " { \"role\": \"user\", \"content\": \"python example to unzip file to dir\" },\n", + " ]\n", + "\n", + " async for part in await openai.ChatCompletion.acreate(\n", + " model=COMPLETIONS_MODEL,\n", + " messages=messages,\n", + " stream=True\n", + " ):\n", + " finish_reason = part[\"choices\"][0][\"finish_reason\"]\n", + " if \"content\" in part[\"choices\"][0][\"delta\"]:\n", + " content = part[\"choices\"][0][\"delta\"][\"content\"]\n", + " print(content, end=\"\")\n", + " elif finish_reason:\n", + " print(finish_reason)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "ename": "RuntimeError", + "evalue": "This event loop is already running", + "output_type": "error", + "traceback": [ + "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[1;31mRuntimeError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[1;32mIn[6], line 3\u001b[0m\n\u001b[0;32m 1\u001b[0m \u001b[39m# asyncio.run(test_stream())\u001b[39;00m\n\u001b[0;32m 2\u001b[0m loop \u001b[39m=\u001b[39m asyncio\u001b[39m.\u001b[39mget_event_loop()\n\u001b[1;32m----> 3\u001b[0m loop\u001b[39m.\u001b[39;49mrun_until_complete(test_stream())\n", + "File \u001b[1;32mc:\\Python310\\lib\\asyncio\\base_events.py:622\u001b[0m, in \u001b[0;36mBaseEventLoop.run_until_complete\u001b[1;34m(self, future)\u001b[0m\n\u001b[0;32m 611\u001b[0m \u001b[39m\"\"\"Run until the Future is done.\u001b[39;00m\n\u001b[0;32m 612\u001b[0m \n\u001b[0;32m 613\u001b[0m \u001b[39mIf the argument is a coroutine, it is wrapped in a Task.\u001b[39;00m\n\u001b[1;32m (...)\u001b[0m\n\u001b[0;32m 619\u001b[0m \u001b[39mReturn the Future's result, or raise its exception.\u001b[39;00m\n\u001b[0;32m 620\u001b[0m \u001b[39m\"\"\"\u001b[39;00m\n\u001b[0;32m 621\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_check_closed()\n\u001b[1;32m--> 622\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_check_running()\n\u001b[0;32m 624\u001b[0m new_task \u001b[39m=\u001b[39m \u001b[39mnot\u001b[39;00m futures\u001b[39m.\u001b[39misfuture(future)\n\u001b[0;32m 625\u001b[0m future \u001b[39m=\u001b[39m tasks\u001b[39m.\u001b[39mensure_future(future, loop\u001b[39m=\u001b[39m\u001b[39mself\u001b[39m)\n", + "File \u001b[1;32mc:\\Python310\\lib\\asyncio\\base_events.py:582\u001b[0m, in \u001b[0;36mBaseEventLoop._check_running\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m 580\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39m_check_running\u001b[39m(\u001b[39mself\u001b[39m):\n\u001b[0;32m 581\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mis_running():\n\u001b[1;32m--> 582\u001b[0m \u001b[39mraise\u001b[39;00m \u001b[39mRuntimeError\u001b[39;00m(\u001b[39m'\u001b[39m\u001b[39mThis event loop is already running\u001b[39m\u001b[39m'\u001b[39m)\n\u001b[0;32m 583\u001b[0m \u001b[39mif\u001b[39;00m events\u001b[39m.\u001b[39m_get_running_loop() \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[0;32m 584\u001b[0m \u001b[39mraise\u001b[39;00m \u001b[39mRuntimeError\u001b[39;00m(\n\u001b[0;32m 585\u001b[0m \u001b[39m'\u001b[39m\u001b[39mCannot run the event loop while another loop is running\u001b[39m\u001b[39m'\u001b[39m)\n", + "\u001b[1;31mRuntimeError\u001b[0m: This event loop is already running" + ] + } + ], + "source": [ + "asyncio.run(test_stream())\n", + "# loop = asyncio.get_event_loop()\n", + "# loop.run_until_complete(test_stream())\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { From b6ad0bb4e035a06a5632d1e7a28e78d332b6dde7 Mon Sep 17 00:00:00 2001 From: henri123lemoine Date: Thu, 16 Mar 2023 11:33:43 -0400 Subject: [PATCH 2/2] added tmp.py to .gitignore --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 070eec4..06f1333 100644 --- a/.gitignore +++ b/.gitignore @@ -133,4 +133,5 @@ dmypy.json *config.py *embeddings.npy *.pickle -*.pkl \ No newline at end of file +*.pkl +src/Embeddings Search/tmp.py \ No newline at end of file