From fbb0c3bbeaf82bf8d76befe2e8fd266dc7b83830 Mon Sep 17 00:00:00 2001 From: henri123lemoine Date: Tue, 14 Mar 2023 08:18:05 -0400 Subject: [PATCH] Updated the text_splitter.py file to remove an unnecessary function, and updated the testing notebook. --- .gitignore | 2 + src/Embeddings Search/testing.ipynb | 658 ++++++++++++------------- src/Embeddings Search/text_splitter.py | 9 +- 3 files changed, 311 insertions(+), 358 deletions(-) diff --git a/.gitignore b/.gitignore index a651d8f..070eec4 100644 --- a/.gitignore +++ b/.gitignore @@ -132,3 +132,5 @@ dmypy.json *alignment_texts.jsonl *config.py *embeddings.npy +*.pickle +*.pkl \ No newline at end of file diff --git a/src/Embeddings Search/testing.ipynb b/src/Embeddings Search/testing.ipynb index 3dc7f03..b77edcf 100644 --- a/src/Embeddings Search/testing.ipynb +++ b/src/Embeddings Search/testing.ipynb @@ -71,6 +71,62 @@ "```" ] }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "https://aipulse.org: title links link authors author text (tags)\n", + "\n", + "None: title url text\n", + "\n", + "ebook: title book_title authors text (publication_date)\n", + "\n", + "https://qualiacomputing.com: title link authors author text (tags)\n", + "\n", + "alignment forum: title url authors text (tags)\n", + "\n", + "lesswrong: title authors url text (tags score date_published)\n", + "\n", + "manual: title authors text (date_published)\n", + "\n", + "arxiv: title authors url text (citation_level alignment_text confidence_score date_published)\n", + "\n", + "https://deepmindsafetyresearch.medium.com/: title url text\n", + "\n", + "waitbutwhy.com: title authors text (date_published)\n", + "\n", + "GitHub: book_title authors author text\n", + "\n", + "https://aiimpacts.org: title link authors author text (tags)\n", + "\n", + "arbital.com: title authors url text (date_published)\n", + "\n", + "carado.moe: title authors text (date_published)\n", + "\n", + "nonarxiv_papers: title authors doi text (date_published)\n", + "\n", + "https://vkrakovna.wordpress.com: title link authors author text (tags)\n", + "\n", + "https://jsteinhardt.wordpress.com: title link authors author text (tags)\n", + "\n", + "audio-transcripts: title authors text (date_published)\n", + "\n", + "https://intelligence.org: title link authors author text (tags)\n", + "\n", + "youtube: title authors url text (date_published)\n", + "\n", + "reports: title authors doi text (date_published)\n", + "\n", + "https://aisafety.camp: title link authors author text (tags)\n", + "\n", + "curriculum: title authors text (date_published)\n", + "\n", + "https://www.yudkowsky.net: title link authors author text (tags)\n", + "\n", + "distill: title authors doi text (date_published)" + ] + }, { "attachments": {}, "cell_type": "markdown", @@ -93,7 +149,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 143, "metadata": {}, "outputs": [], "source": [ @@ -101,8 +157,15 @@ "import numpy as np\n", "from typing import List, Dict, Tuple\n", "import re\n", - "import matplotlib.pyplot as plt\n", + "import time\n", + "import random\n", + "import pickle\n", "import openai\n", + "from tenacity import (\n", + " retry,\n", + " stop_after_attempt,\n", + " wait_random_exponential,\n", + ") # for exponential backoff\n", "\n", "import config" ] @@ -117,12 +180,14 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 119, "metadata": {}, "outputs": [], "source": [ "LEN_EMBEDDINGS = 1536\n", - "PATH_TO_DATA = r\"C:\\Users\\Henri\\Documents\\GitHub\\AlignmentSearch\\src\\data\\alignment_texts.jsonl\"\n", + "PATH_TO_DATA = r\"C:\\Users\\Henri\\Documents\\GitHub\\AlignmentSearch\\src\\Embeddings Search\\data\\alignment_texts.jsonl\"\n", + "PATH_TO_EMBEDDINGS = r\"C:\\Users\\Henri\\Documents\\GitHub\\AlignmentSearch\\src\\Embeddings Search\\data\\embeddings.npy\"\n", + "PATH_TO_DATASET = r\"C:\\Users\\Henri\\Documents\\GitHub\\AlignmentSearch\\src\\Embeddings Search\\data\\dataset.pkl\"\n", "\n", "COMPLETIONS_MODEL = \"text-davinci-003\"\n", "EMBEDDING_MODEL = \"text-embedding-ada-002\"\n", @@ -193,29 +258,7 @@ }, { "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [], - "source": [ - "def split_article(text: str) -> List[str]:\n", - " # Receives one text (str) and returns a list of sections (List[str]), each section being a few appended paragraphs that do not exceed 1000 words.\n", - " # This is done to avoid the 8000 token limit of OpenAI embeddings.\n", - " sections = []\n", - " section = \"\"\n", - " paragraphs = text.split('\\n')\n", - " for paragraph in paragraphs:\n", - " if paragraph == \"\": continue\n", - " if len(section.split()) + len(paragraph.split()) > 1000 or len(section) + len(paragraph) > 7000:\n", - " sections.append(section)\n", - " section = \"\"\n", - " section += f\"{paragraph}\\n\"\n", - " sections.append(section)\n", - " return sections" - ] - }, - { - "cell_type": "code", - "execution_count": 19, + "execution_count": 13, "metadata": {}, "outputs": [], "source": [ @@ -223,6 +266,84 @@ " pass" ] }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "class TextSplitter:\n", + " def __init__(self, block_maxsize: int = 800, block_minsize: int = 500):\n", + " self.block_maxsize = block_maxsize\n", + " self.block_minsize = block_minsize\n", + " self.blocks = []\n", + " self.current_block = []\n", + " self.current_block_len = 0\n", + "\n", + "\n", + " def add_sentence_to_blocks(self, sentence):\n", + " sentence_len = len(sentence)\n", + " sentence_fits_in_current_block = self.current_block_len + sentence_len <= self.block_maxsize\n", + " current_block_is_big_enough = self.current_block_len >= self.block_minsize\n", + " sentence_fits_in_standalone_block = sentence_len <= self.block_maxsize\n", + "\n", + " if sentence_fits_in_current_block:\n", + " self.current_block.append(sentence)\n", + " self.current_block_len += sentence_len + 1 # +1 for the space\n", + " return\n", + " \n", + " if current_block_is_big_enough and sentence_fits_in_standalone_block:\n", + " self.blocks.append(\" \".join(self.current_block))\n", + " self.current_block = [sentence]\n", + " self.current_block_len = sentence_len + 1 # +1 for the space\n", + " return\n", + " \n", + " #special cases:TODO refactor\n", + " #case 1: current_block_len < block_minsize and current_block_len + sentence_len > block_maxsize\n", + " #case 2: current_block_len > block_minsize but sentence_len > block_maxsize\n", + " shorter_sentence = sentence[self.block_maxsize - self.current_block_len]\n", + " self.current_block.append(shorter_sentence)\n", + " self.blocks.append(\" \".join(self.current_block))\n", + " self.current_block = []\n", + " self.current_block_len = 0 \n", + " \n", + "\n", + " def add_paragraph_to_blocks(self, paragraph):\n", + " paragraph_len = len(paragraph)\n", + " if self.current_block_len + paragraph_len > self.block_maxsize:\n", + " sentences = split_into_sentences(paragraph)\n", + " for sentence in sentences:\n", + " self.add_sentence_to_blocks(sentence)\n", + " return\n", + " \n", + " if self.block_minsize <= self.current_block_len + paragraph_len <= self.block_maxsize:\n", + " self.current_block.append(paragraph)\n", + " self.blocks.append(\"\\n\\n\".join(self.current_block))\n", + " self.current_block = []\n", + " self.current_block_len = 0\n", + " return\n", + " \n", + " if self.current_block_len + paragraph_len < self.block_minsize:\n", + " self.current_block.append(paragraph)\n", + " self.current_block_len += paragraph_len + 2 # +2 for the \\n\\n\n", + " return\n", + " \n", + " def add_text_to_blocks(self, text):\n", + " paragraphs = text.split(\"\\n\\n\")\n", + " for paragraph in paragraphs:\n", + " self.add_paragraph_to_blocks(paragraph)\n", + " if self.current_block != []:\n", + " self.blocks.append(\"\\n\\n\".join(self.current_block))\n", + "\n", + "\n", + " def split(self, text: str, signature: str) -> List[str]:\n", + " \"\"\"Split text into multiple blocks and add signature to each block.\"\"\"\n", + " # signature has the format : \"link, title, author\"\n", + " self.add_text_to_blocks(text)\n", + " \n", + " return [f\"{block}\\n - {signature}\" for block in self.blocks]\n" + ] + }, { "attachments": {}, "cell_type": "markdown", @@ -233,7 +354,7 @@ }, { "cell_type": "code", - "execution_count": 61, + "execution_count": 15, "metadata": {}, "outputs": [], "source": [ @@ -241,13 +362,14 @@ " \"Entry has no source.\": 0,\n", " \"Entry has no title.\": 0,\n", " \"Entry has no text.\": 0,\n", - " \"Entry has no URL.\": 0\n", + " \"Entry has no URL.\": 0,\n", + " \"Entry has wrong citation level.\": 0\n", "}" ] }, { "cell_type": "code", - "execution_count": 69, + "execution_count": 138, "metadata": {}, "outputs": [], "source": [ @@ -255,13 +377,20 @@ " def __init__(self,\n", " path: str, # Path to the dataset .jsonl file.\n", " sources: List[str] = None, # List of sources to include. If None, include all sources.\n", - " max_paragraph_length: Tuple[int, int] = None # (max number of words in a paragraph, max number of characters in a paragraph)\n", + " rate_limit_per_minute: int = 60, # Rate limit for the OpenAI API.\n", + " block_min_max_size: Tuple[int, int] = None, # Tuple of (min_block_size, max_block_size), used for the text splitter. If None, use default values.\n", " ):\n", " self.path = path\n", " self.sources = sources\n", - " self.max_paragraph_length = max_paragraph_length\n", - " \n", - " self.data: List[Tuple[str, str, str]] = [] # List of tuples, each containing the title of an article, its URL, and text. E.g.: [('title', 'url', 'text'), ...]\n", + " self.rate_limit_per_minute = rate_limit_per_minute\n", + " self.delay_in_seconds = 60.0 / self.rate_limit_per_minute\n", + " \n", + " # Set up text splitter\n", + " if block_min_max_size is None: self.block_min_max_size = (400, 600)\n", + " else: self.block_min_max_size = block_min_max_size\n", + " self.text_splitter = TextSplitter(block_maxsize=self.block_min_max_size[1], block_minsize=self.block_min_max_size[0])\n", + " \n", + " self.data: List[Tuple[str]] = [] # List of tuples, each containing the title of an article, its URL, and text. E.g.: [('title', 'url', 'text'), ...]\n", " self.embed_split: List[str] = [] # List of strings, each being a few paragraphs from a single article (not exceeding 1000 words).\n", " \n", " self.num_articles: Dict[str, int] = {} # Number of articles per source. E.g.: {'source1': 10, 'source2': 20, 'total': 30}\n", @@ -275,7 +404,7 @@ " self.total_char_count = 0\n", " self.total_word_count = 0\n", " self.total_sentence_count = 0\n", - " self.total_paragraph_count = 0\n", + " self.total_block_count = 0\n", " \n", " def get_info_tmp(self):\n", " self.sources_so_far = []\n", @@ -287,60 +416,155 @@ " if entry['source'] not in self.sources_so_far:\n", " self.sources_so_far.append(entry['source'])\n", " self.info_types[entry['source']] = entry.keys()\n", + " \n", + " if 'tags' in entry:\n", + " print(entry['tags'])\n", + " \n", + " \"\"\"\n", + " {\n", + " 'text', \n", + " 'title', 'book_title', # If there is both, take title, otherwise take book_title\n", + " 'author', 'authors', # If there is both, take author, otherwise take authors, otherwise take author\n", + " 'citation_level', # must be 0 or 1\n", + " 'date_published', 'published', # take first 10 chars of date_published, if it exists; else take first 16 chars of published, if it exists\n", + " 'doi', 'link', 'links', 'url', # if link, take link; elif url, take url; elif doi, take doi\n", + " 'tags'\n", + " }\n", + " \"\"\"\n", " \n", " def get_alignment_texts(self):\n", " with jsonlines.open(self.path, \"r\") as reader:\n", " for entry in reader:\n", + " # Only get one in a thousand articles\n", + " if random.randint(0, 3000) != 19: continue\n", " try:\n", " if 'source' not in entry: raise MissingDataException(\"Entry has no source.\")\n", " \n", " if self.sources is None:\n", - " if entry['source'] not in self.num_articles:\n", - " self.num_articles[entry['source']] = 1\n", - " else:\n", - " self.num_articles[entry['source']] += 1\n", + " if entry['source'] not in self.num_articles: self.num_articles[entry['source']] = 1\n", + " else: self.num_articles[entry['source']] += 1\n", " self.num_articles['total'] += 1\n", " else:\n", " if entry['source'] in self.sources:\n", " self.num_articles[entry['source']] += 1\n", " self.num_articles['total'] += 1\n", - " else:\n", - " continue\n", + " else: continue\n", " \n", - " if 'title' not in entry: raise MissingDataException(\"Entry has no title.\")\n", - " if 'link' not in entry: raise MissingDataException(\"Entry has no link.\")\n", - " if 'text' not in entry: raise MissingDataException(\"Entry has no text.\")\n", + " text=title=author=citation_level=date_published=url=tags=None\n", + " \n", + " # Get text\n", + " if 'text' in entry and entry['text'] != '': text = entry['text']\n", + " else: raise MissingDataException(f\"Entry has no text.\")\n", + " \n", + " # Get title\n", + " if 'title' in entry and 'book_title' in entry and entry['title'] != '': title = entry['title']\n", + " elif 'book_title' in entry and entry['book_title'] != '': title = entry['book_title']\n", + " else: title = None\n", + " \n", + " # Get author\n", + " if 'author' in entry and 'authors' in entry and entry['author'] != '': author = entry['author']\n", + " elif 'authors' in entry and entry['authors'] != '': author = entry['authors']\n", + " elif 'author' in entry and entry['author'] != '': author = entry['author']\n", + " else: author = None\n", + " \n", + " # Get citation level\n", + " if 'citation_level' in entry:\n", + " if entry['citation_level'] != 0: raise MissingDataException(f\"Entry has citation_level {entry['citation_level']}.\")\n", + " \n", + " # Get date published\n", + " if 'date_published' in entry and entry['date_published'] != '': date_published = entry['date_published'][:10]\n", + " elif 'published' in entry and entry['published'] != '': date_published = entry['published'][:16]\n", + " else: date_published = None\n", + " \n", + " # Get URL\n", + " if 'link' in entry and entry['link'] != '': url = entry['link']\n", + " elif 'url' in entry and entry['url'] != '': url = entry['url']\n", + " elif 'doi' in entry and entry['doi'] != '': url = entry['doi']\n", + " else: url = None\n", + " \n", + " # Get tags\n", + " if 'tags' in entry and entry['tags'] != '':\n", + " if type(entry['tags']) == list: tags = ', '.join([val['term'] for val in entry['tags']])\n", + " elif type(entry['tags']) == str: tags = entry['tags']\n", + " else: tags = None\n", + " \n", + " signature = \"\"\n", + " if title: signature += f\"Title: {title}, \"\n", + " if author: signature += f\"Author: {author}, \"\n", + " if date_published: signature += f\"Date published: {date_published}, \"\n", + " if url: signature += f\"URL: {url}, \"\n", + " if tags: signature += f\"Tags: {tags}, \"\n", + " signature = signature[:-2]\n", "\n", - " self.data.append((entry['title'], entry['link'], entry['text']))\n", - " paragraphs = split_article(entry['text'])\n", - " self.embed_split.extend(paragraphs)\n", + " self.data.append((title, author, date_published, url, tags, text))\n", + " \n", + " blocks = self.text_splitter.split(text, signature)\n", + " self.embed_split.extend(blocks)\n", " \n", " self.total_char_count += len(entry['text'])\n", " self.total_word_count += len(entry['text'].split())\n", " self.total_sentence_count += len(split_into_sentences(entry['text']))\n", - " self.total_paragraph_count += len(paragraphs)\n", - " except KeyError:\n", - " pass\n", + " self.total_block_count += len(blocks)\n", + " \n", " except MissingDataException as e:\n", + " if str(e) not in error_count_dict:\n", + " error_count_dict[str(e)] = 0\n", " error_count_dict[str(e)] += 1\n", "\n", - " def get_embedding(text: str) -> np.ndarray:\n", + " @retry(wait=wait_random_exponential(min=1, max=100), stop=stop_after_attempt(10))\n", + " def get_embedding(self, text: str, delay_in_seconds: float = 0) -> np.ndarray:\n", + " time.sleep(delay_in_seconds)\n", " result = openai.Embedding.create(model=EMBEDDING_MODEL, input=text)\n", " return result[\"data\"][0][\"embedding\"]\n", "\n", " def get_embeddings(self):\n", - " self.embeddings = np.array([self.get_embedding(text) for text in self.embed_split])\n", + " self.embeddings = np.array([self.get_embedding(text, delay_in_seconds=self.delay_in_seconds) for text in self.embed_split])\n", " \n", " def save_embeddings(self, path: str):\n", " np.save(path, self.embeddings)\n", " \n", " def load_embeddings(self, path: str):\n", - " self.embeddings = np.load(path)" + " self.embeddings = np.load(path)\n", + " \n", + " def save_class(self, path: str):\n", + " with open(path, 'wb') as f:\n", + " pickle.dump(self, f)" ] }, { "cell_type": "code", - "execution_count": 70, + "execution_count": 139, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1416\n" + ] + } + ], + "source": [ + "dataset = Dataset(path=PATH_TO_DATA, sources=None)\n", + "dataset.get_alignment_texts()\n", + "dataset.get_embeddings()\n", + "dataset.save_embeddings(PATH_TO_EMBEDDINGS)\n", + "dataset.save_class(PATH_TO_DATASET)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# with open(PATH_TO_DATASET, 'rb') as f:\n", + "# dataset = pickle.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 157, "metadata": {}, "outputs": [], "source": [ @@ -349,7 +573,8 @@ " dataset: Dataset, # Dataset object containing the data.\n", " ):\n", " self.dataset = dataset\n", - " \n", + " \n", + " @retry(wait=wait_random_exponential(min=1, max=100), stop=stop_after_attempt(10))\n", " def get_embedding(self, text: str) -> np.ndarray:\n", " result = openai.Embedding.create(model=EMBEDDING_MODEL, input=text)\n", " return result[\"data\"][0][\"embedding\"]\n", @@ -359,11 +584,13 @@ " # Each tuple contains the title of an article, its URL, and text.\n", " query_embedding = self.get_embedding(query)\n", " similarities = np.dot(self.dataset.embeddings, query_embedding)\n", + " print(similarities.shape)\n", " top_k_indices = np.argsort(similarities)[::-1][:k]\n", - " top_k = [self.dataset.data[i] for i in top_k_indices]\n", + " print(top_k_indices)\n", + " top_k = [self.dataset.embed_split[i] for i in top_k_indices]\n", " return top_k\n", " \n", - " def construct_prompt(self, question: str, texts: List[Tuple[str, str, str]]) -> str:\n", + " def construct_prompt(self, question: str, texts: List[Tuple[str]]) -> str:\n", " # Receives a question (str) and a list of articles (List[Tuple[str, str, str]]) and returns a prompt (str) to be used for text generation.\n", " context = \"\\n\".join(texts)[:MAX_LEN_PROMPT]\n", " header = \"\"\"Answer the question as truthfully as possible using the provided context, and if the answer is not contained within the text below, say \"I don't know.\"\\n\\nContext:\\n\"\"\"\n", @@ -380,227 +607,56 @@ " answer = openai.Completion.create(prompt=prompt, **COMPLETIONS_API_PARAMS)[\"choices\"][0][\"text\"].strip(\" \\n\")\n", " return answer\n", " \n", - " def search_and_answer(self, question: str, k: int=10) -> str:\n", + " def search_and_answer(self, question: str, k: int=10, HyDE: bool=False) -> str:\n", " # Receives a question (str) and returns an answer (str) to the question.\n", - " top_k = self.get_top_k(question, k)\n", + " if HyDE:\n", + " raise NotImplementedError\n", + " else:\n", + " top_k = self.get_top_k(question, k)\n", " answer = self.answer_question(question, top_k)\n", - " return answer\n", - " \n", - " def summarize(self, article: str) -> str:\n", - " COMPLETIONS_API_PARAMS = {\n", - " \"temperature\": 0.0,\n", - " \"max_tokens\": 300,\n", - " \"model\": COMPLETIONS_MODEL,\n", - " }\n", - " raise NotImplementedError" + " return answer\n" ] }, { "cell_type": "code", - "execution_count": 71, - "metadata": {}, - "outputs": [], - "source": [ - "dataset = Dataset(path=PATH_TO_DATA, sources=['https://www.yudkowsky.net'])\n", - "# dataset.get_info_tmp()\n", - "dataset.get_alignment_texts()\n", - "# dataset.get_embeddings()\n", - "# dataset.save_embeddings(EMBEDDINGS_PATH)\n", - "# # dataset.load_embeddings(EMBEDDINGS_PATH)\n", - "\n", - "# search_and_answer = SearchAndAnswer(dataset)\n", - "\n", - "# while True:\n", - "# question = input(\"Enter a question: \")\n", - "# if question == \"quit\":\n", - "# break\n", - "# top_k = search_and_answer.get_top_k(question)\n", - "# answer = search_and_answer.answer_question(question, top_k)\n", - "# print(answer)" - ] - }, - { - "cell_type": "code", - "execution_count": 82, + "execution_count": 158, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "112\n" - ] - }, - { - "ename": "", - "evalue": "", - "output_type": "error", - "traceback": [ - "\u001b[1;31mThe Kernel crashed while executing code in the the current cell or a previous cell. Please review the code in the cell(s) to identify a possible cause of the failure. Click here for more info. View Jupyter log for further details." + "(1416,)\n", + "[ 82 916 1171]\n", + "The need for a high quality alignment dataset for very capable models.\n" ] } ], "source": [ - "print(len(dataset.embed_split))" + "SA = SearchAndAnswer(dataset=dataset)\n", + "prompt = \"Name a problem in AI Alignment.\"\n", + "answer = SA.search_and_answer(prompt, 3, HyDE=False)\n", + "print(answer)" ] }, { "cell_type": "code", - "execution_count": 65, + "execution_count": 147, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "['https://aipulse.org', 'None', 'ebook', 'https://qualiacomputing.com', 'alignment forum', 'lesswrong', 'manual', 'arxiv', 'https://deepmindsafetyresearch.medium.com/', 'waitbutwhy.com', 'GitHub', 'https://aiimpacts.org', 'arbital.com', 'carado.moe', 'nonarxiv_papers', 'https://vkrakovna.wordpress.com', 'https://jsteinhardt.wordpress.com', 'audio-transcripts', 'https://intelligence.org', 'youtube', 'reports', 'https://aisafety.camp', 'curriculum', 'https://www.yudkowsky.net', 'distill']\n" - ] - } - ], - "source": [ - "print(dataset.sources_so_far)" - ] - }, - { - "cell_type": "code", - "execution_count": 68, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "https://aipulse.org dict_keys(['title', 'title_detail', 'links', 'link', 'authors', 'author', 'author_detail', 'published', 'published_parsed', 'tags', 'id', 'guidislink', 'summary', 'summary_detail', 'content', 'post-id', 'text', 'source', 'source_type'])\n", - "None dict_keys(['text', 'url', 'title', 'source'])\n", - "ebook dict_keys(['epub_version', 'title', 'language', 'description', 'authors', 'publisher', 'publication_date', 'identifiers', 'subject', 'file_size_in_bytes', 'cover_image_extension', 'toc', 'source', 'source_filetype', 'converted_with', 'book_title', 'date_published', 'chapter_names', 'text'])\n", - "https://qualiacomputing.com dict_keys(['title', 'title_detail', 'links', 'link', 'comments', 'authors', 'author', 'author_detail', 'published', 'published_parsed', 'tags', 'id', 'guidislink', 'summary', 'summary_detail', 'content', 'wfw_commentrss', 'slash_comments', 'post-id', 'text', 'source', 'source_type'])\n", - "alignment forum dict_keys(['id', 'title', 'authors', 'date_published', 'score', 'omega_karma', 'votes', 'tags', 'url', 'text', 'source', 'comments'])\n", - "lesswrong dict_keys(['id', 'title', 'authors', 'date_published', 'score', 'omega_karma', 'votes', 'tags', 'url', 'text', 'source', 'comments'])\n", - "manual dict_keys(['source', 'source_type', 'title', 'authors', 'date_published', 'text'])\n", - "arxiv dict_keys(['source', 'source_type', 'converted_with', 'paper_version', 'title', 'authors', 'date_published', 'data_last_modified', 'url', 'abstract', 'author_comment', 'journal_ref', 'doi', 'primary_category', 'categories', 'citation_level', 'alignment_text', 'confidence_score', 'main_tex_filename', 'text', 'bibliography_bbl', 'bibliography_bib', 'arxiv_citations'])\n", - "https://deepmindsafetyresearch.medium.com/ dict_keys(['source', 'source_type', 'url', 'title', 'content', 'text'])\n", - "waitbutwhy.com dict_keys(['source', 'source_type', 'title', 'authors', 'date_published', 'text'])\n", - "GitHub dict_keys(['source', 'source_filetype', 'converted_with', 'book_title', 'author', 'date_published', 'text'])\n", - "https://aiimpacts.org dict_keys(['title', 'title_detail', 'links', 'link', 'comments', 'authors', 'author', 'author_detail', 'published', 'published_parsed', 'tags', 'id', 'guidislink', 'summary', 'summary_detail', 'content', 'wfw_commentrss', 'slash_comments', 'text', 'source', 'source_type'])\n", - "arbital.com dict_keys(['source', 'source_type', 'converted_with', 'title', 'authors', 'date_published', 'url', 'text'])\n", - "carado.moe dict_keys(['source', 'source_type', 'title', 'authors', 'date_published', 'text'])\n", - "nonarxiv_papers dict_keys(['source', 'source_filetype', 'converted_with', 'paper_version', 'title', 'authors', 'date_published', 'abstract', 'journal_ref', 'doi', 'citation_level', 'text', 'bibliography_bib', 'source_file'])\n", - "https://vkrakovna.wordpress.com dict_keys(['title', 'title_detail', 'links', 'link', 'comments', 'authors', 'author', 'author_detail', 'published', 'published_parsed', 'tags', 'id', 'guidislink', 'summary', 'summary_detail', 'content', 'wfw_commentrss', 'slash_comments', 'media_content', 'text', 'source', 'source_type'])\n", - "https://jsteinhardt.wordpress.com dict_keys(['title', 'title_detail', 'links', 'link', 'comments', 'authors', 'author', 'author_detail', 'published', 'published_parsed', 'tags', 'id', 'guidislink', 'summary', 'summary_detail', 'content', 'wfw_commentrss', 'slash_comments', 'media_content', 'text', 'source', 'source_type'])\n", - "audio-transcripts dict_keys(['source', 'source_filetype', 'cleaned', 'converted_with', 'title', 'authors', 'date_published', 'text'])\n", - "https://intelligence.org dict_keys(['title', 'title_detail', 'links', 'link', 'comments', 'authors', 'author', 'author_detail', 'published', 'published_parsed', 'tags', 'id', 'guidislink', 'summary', 'summary_detail', 'content', 'wfw_commentrss', 'slash_comments', 'text', 'source', 'source_type'])\n", - "youtube dict_keys(['source', 'source_type', 'converted_with', 'title', 'authors', 'date_published', 'url', 'text'])\n", - "reports dict_keys(['source', 'source_filetype', 'converted_with', 'paper_version', 'title', 'authors', 'date_published', 'abstract', 'journal_ref', 'doi', 'citation_level', 'text', 'bibliography_bib', 'source_file'])\n", - "https://aisafety.camp dict_keys(['title', 'title_detail', 'links', 'link', 'authors', 'author', 'author_detail', 'published', 'published_parsed', 'tags', 'id', 'guidislink', 'summary', 'summary_detail', 'content', 'post-id', 'text', 'source', 'source_type'])\n", - "curriculum dict_keys(['source', 'source_type', 'title', 'authors', 'date_published', 'text'])\n", - "https://www.yudkowsky.net dict_keys(['title', 'title_detail', 'links', 'link', 'authors', 'author', 'author_detail', 'published', 'published_parsed', 'tags', 'id', 'guidislink', 'summary', 'summary_detail', 'content', 'text', 'source', 'source_type'])\n", - "distill dict_keys(['source', 'source_type', 'converted_with', 'title', 'authors', 'date_published', 'abstract', 'journal_ref', 'doi', 'text', 'bibliography_bib'])\n" - ] - } - ], - "source": [ - "for dataset_source in dataset.sources_so_far:\n", - " print(dataset_source, dataset.info_types[dataset_source])" - ] - }, - { - "cell_type": "code", - "execution_count": 41, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "missing info: {'Entry has no source.': 487, 'Entry has no title.': 0, 'Entry has no text.': 0, 'Entry has no URL.': 23}\n" - ] - } - ], - "source": [ - "print(f\"missing info: {error_count_dict}\")" - ] - }, - { - "cell_type": "code", - "execution_count": 35, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0" - ] - }, - "execution_count": 35, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "dataset.total_word_count" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[]" - ] - }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "dataset.data" - ] - }, - { - "cell_type": "code", - "execution_count": 59, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'https://www.yudkowsky.net': 23, 'total': 23}" - ] - }, - "execution_count": 59, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "dataset.num_articles" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Source Truth Empirical Difference\n" - ] - }, - { - "ename": "NameError", - "evalue": "name 'dataset' is not defined", - "output_type": "error", - "traceback": [ - "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[1;31mNameError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[1;32mIn[1], line 51\u001b[0m\n\u001b[0;32m 49\u001b[0m \u001b[39m# Print table. First row has Truth and Empirical findings.\u001b[39;00m\n\u001b[0;32m 50\u001b[0m \u001b[39mprint\u001b[39m(\u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39m{\u001b[39;00m\u001b[39m'\u001b[39m\u001b[39mSource\u001b[39m\u001b[39m'\u001b[39m\u001b[39m:\u001b[39;00m\u001b[39m<20\u001b[39m\u001b[39m}\u001b[39;00m\u001b[39m \u001b[39m\u001b[39m{\u001b[39;00m\u001b[39m'\u001b[39m\u001b[39mTruth\u001b[39m\u001b[39m'\u001b[39m\u001b[39m:\u001b[39;00m\u001b[39m<10\u001b[39m\u001b[39m}\u001b[39;00m\u001b[39m \u001b[39m\u001b[39m{\u001b[39;00m\u001b[39m'\u001b[39m\u001b[39mEmpirical\u001b[39m\u001b[39m'\u001b[39m\u001b[39m:\u001b[39;00m\u001b[39m<10\u001b[39m\u001b[39m}\u001b[39;00m\u001b[39m \u001b[39m\u001b[39m{\u001b[39;00m\u001b[39m'\u001b[39m\u001b[39mDifference\u001b[39m\u001b[39m'\u001b[39m\u001b[39m:\u001b[39;00m\u001b[39m<10\u001b[39m\u001b[39m}\u001b[39;00m\u001b[39m\"\u001b[39m)\n\u001b[1;32m---> 51\u001b[0m \u001b[39mfor\u001b[39;00m source \u001b[39min\u001b[39;00m dataset\u001b[39m.\u001b[39mnum_articles:\n\u001b[0;32m 52\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m 53\u001b[0m \u001b[39mprint\u001b[39m(\u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39m{\u001b[39;00msource[:\u001b[39m20\u001b[39m]\u001b[39m:\u001b[39;00m\u001b[39m<20\u001b[39m\u001b[39m}\u001b[39;00m\u001b[39m \u001b[39m\u001b[39m{\u001b[39;00mnum_articles_truth[source]\u001b[39m:\u001b[39;00m\u001b[39m<10\u001b[39m\u001b[39m}\u001b[39;00m\u001b[39m \u001b[39m\u001b[39m{\u001b[39;00mdataset\u001b[39m.\u001b[39mnum_articles[source]\u001b[39m:\u001b[39;00m\u001b[39m<10\u001b[39m\u001b[39m}\u001b[39;00m\u001b[39m \u001b[39m\u001b[39m{\u001b[39;00mnum_articles_truth[source] \u001b[39m-\u001b[39m dataset\u001b[39m.\u001b[39mnum_articles[source]\u001b[39m:\u001b[39;00m\u001b[39m<10\u001b[39m\u001b[39m}\u001b[39;00m\u001b[39m\"\u001b[39m)\n", - "\u001b[1;31mNameError\u001b[0m: name 'dataset' is not defined" + "Source Truth Empirical Difference\n", + "total 41614 14 41600 \n", + "lesswrong 28479 11 28468 \n", + "alignment forum 2138 1 2137 \n", + "arxiv 8007 2 8005 \n", + "\n", + " Truth Empirical Difference\n", + "Word Count 53550146 23344 53526802 \n", + "Character Count 351767163 143048 351624115 \n" ] } ], @@ -666,104 +722,6 @@ "print(f\"{'Word Count':<20} {word_count_truth:<10} {dataset.total_word_count:<10} {word_count_truth - dataset.total_word_count:<10}\")\n", "print(f\"{'Character Count':<20} {char_count_truth:<10} {dataset.total_char_count:<10} {char_count_truth - dataset.total_char_count:<10}\")" ] - }, - { - "cell_type": "code", - "execution_count": 150, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "348457759 characters\n", - "~104537328 tokens\n", - "~69691552 words\n", - "~348457 paragraphs\n", - "~12912 embeddings using method 1\n", - "~116152 embeddings using method 2\n", - "~139383 pages\n", - "~46 cost using method 1\n", - "~627 cost using method 2\n" - ] - } - ], - "source": [ - "\n", - "num_words = dataset.data_length / 5\n", - "num_tokens = num_words * 1.5\n", - "num_paragraphs = num_words // 200\n", - "num_embeds_method_1 = num_tokens // 8096\n", - "num_embeds_method_2 = num_words // 600\n", - "cost_per_embed = 1/(3000*500/8096)\n", - "cost_per_page = 1/3000\n", - "num_pages = num_words // 500\n", - "cost_1 = num_pages * cost_per_page\n", - "cost_2 = num_embeds_method_2 * cost_per_embed\n", - "\n", - "print(f\"{dataset.data_length} characters\")\n", - "print(f\"~{num_tokens:.0f} tokens\")\n", - "print(f\"~{num_words:.0f} words\")\n", - "print(f\"~{num_paragraphs:.0f} paragraphs\")\n", - "print(f\"~{num_embeds_method_1:.0f} embeddings using method 1\")\n", - "print(f\"~{num_embeds_method_2:.0f} embeddings using method 2\")\n", - "print(f\"~{num_pages:.0f} pages\")\n", - "print(f\"~{cost_1:.0f} cost using method 1\")\n", - "print(f\"~{cost_2:.0f} cost using method 2\")" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Random tests" - ] - }, - { - "cell_type": "code", - "execution_count": 63, - "metadata": {}, - "outputs": [], - "source": [ - "import json" - ] - }, - { - "cell_type": "code", - "execution_count": 72, - "metadata": {}, - "outputs": [], - "source": [ - "with jsonlines.open(PATH_TO_DATA, \"r\") as reader, open(\"aipulse.txt\", \"w\", encoding=\"utf-8\") as writer:\n", - " for entry in reader:\n", - " try:\n", - " if 'source' in entry and entry['source'] == 'https://aipulse.org':\n", - " if 'title' in entry:\n", - " writer.write(f\"Title: {entry['title']}\\n\")\n", - " else:\n", - " writer.write(f\"NO TITLE\\n\")\n", - " if 'text' in entry:\n", - " writer.write(f\"Text: {entry['text']}\\n\")\n", - " else:\n", - " writer.write(f\"NO TEXT\\n\")\n", - " if 'url' in entry:\n", - " writer.write(f\"URL: {entry['url']}\\n\")\n", - " else:\n", - " writer.write(f\"NO URL\\n\")\n", - " writer.write(\"\\n\\n\")\n", - " else:\n", - " continue\n", - " except KeyError:\n", - " pass" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { diff --git a/src/Embeddings Search/text_splitter.py b/src/Embeddings Search/text_splitter.py index d9723a9..26058a0 100644 --- a/src/Embeddings Search/text_splitter.py +++ b/src/Embeddings Search/text_splitter.py @@ -44,13 +44,6 @@ def split_into_sentences(text): sentences = [text.strip()] return sentences -def select_first_n_chars(text: str, n: int) -> str: - """Select first n characters from a string if there are more than n characters, otherwise return the whole string""" - if len(text) > n: - return text[:n] - return text - - class TextSplitter: def __init__(self, block_maxsize: int = 800, block_minsize: int = 500): @@ -81,7 +74,7 @@ class TextSplitter: #special cases:TODO refactor #case 1: current_block_len < block_minsize and current_block_len + sentence_len > block_maxsize #case 2: current_block_len > block_minsize but sentence_len > block_maxsize - shorter_sentence = select_first_n_chars(sentence, self.block_maxsize - self.current_block_len) + shorter_sentence = sentence[self.block_maxsize - self.current_block_len] self.current_block.append(shorter_sentence) self.blocks.append(" ".join(self.current_block)) self.current_block = []