wassname/stampy-chat
1
0
Fork 0
mirror of https://github.com/wassname/stampy-chat.git synced 2026-09-11 12:50:34 +08:00
Code Issues Packages Projects Releases Wiki Activity

Updated the text_splitter.py file to remove an unnecessary function, and updated the testing notebook.

Browse Source
This commit is contained in:
henri123lemoine
2023-03-14 08:18:05 -04:00
parent 719674541c
commit fbb0c3bbea
3 changed files with 311 additions and 358 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+2
View File
@@ -132,3 +132,5 @@ dmypy.json
*alignment_texts.jsonl
*config.py
*embeddings.npy
*.pickle
*.pkl
src/Embeddings Search/testing.ipynb
+308 -350
View File
@@ -71,6 +71,62 @@
"```"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
"https://aipulse.org: title links link authors author text (tags)\n",
"\n",
"None: title url text\n",
"\n",
"ebook: title book_title authors text (publication_date)\n",
"\n",
"https://qualiacomputing.com: title link authors author text (tags)\n",
"\n",
"alignment forum: title url authors text (tags)\n",
"\n",
"lesswrong: title authors url text (tags score date_published)\n",
"\n",
"manual: title authors text (date_published)\n",
"\n",
"arxiv: title authors url text (citation_level alignment_text confidence_score date_published)\n",
"\n",
"https://deepmindsafetyresearch.medium.com/: title url text\n",
"\n",
"waitbutwhy.com: title authors text (date_published)\n",
"\n",
"GitHub: book_title authors author text\n",
"\n",
"https://aiimpacts.org: title link authors author text (tags)\n",
"\n",
"arbital.com: title authors url text (date_published)\n",
"\n",
"carado.moe: title authors text (date_published)\n",
"\n",
"nonarxiv_papers: title authors doi text (date_published)\n",
"\n",
"https://vkrakovna.wordpress.com: title link authors author text (tags)\n",
"\n",
"https://jsteinhardt.wordpress.com: title link authors author text (tags)\n",
"\n",
"audio-transcripts: title authors text (date_published)\n",
"\n",
"https://intelligence.org: title link authors author text (tags)\n",
"\n",
"youtube: title authors url text (date_published)\n",
"\n",
"reports: title authors doi text (date_published)\n",
"\n",
"https://aisafety.camp: title link authors author text (tags)\n",
"\n",
"curriculum: title authors text (date_published)\n",
"\n",
"https://www.yudkowsky.net: title link authors author text (tags)\n",
"\n",
"distill: title authors doi text (date_published)"
]
},
{
"attachments": {},
"cell_type": "markdown",
@@ -93,7 +149,7 @@
},
{
"cell_type": "code",
"execution_count": 1,
"execution_count": 143,
"metadata": {},
"outputs": [],
"source": [
@@ -101,8 +157,15 @@
"import numpy as np\n",
"from typing import List, Dict, Tuple\n",
"import re\n",
"import matplotlib.pyplot as plt\n",
"import time\n",
"import random\n",
"import pickle\n",
"import openai\n",
"from tenacity import (\n",
" retry,\n",
" stop_after_attempt,\n",
" wait_random_exponential,\n",
") # for exponential backoff\n",
"\n",
"import config"
]
@@ -117,12 +180,14 @@
},
{
"cell_type": "code",
"execution_count": 8,
"execution_count": 119,
"metadata": {},
"outputs": [],
"source": [
"LEN_EMBEDDINGS = 1536\n",
"PATH_TO_DATA = r\"C:\\Users\\Henri\\Documents\\GitHub\\AlignmentSearch\\src\\data\\alignment_texts.jsonl\"\n",
"PATH_TO_DATA = r\"C:\\Users\\Henri\\Documents\\GitHub\\AlignmentSearch\\src\\Embeddings Search\\data\\alignment_texts.jsonl\"\n",
"PATH_TO_EMBEDDINGS = r\"C:\\Users\\Henri\\Documents\\GitHub\\AlignmentSearch\\src\\Embeddings Search\\data\\embeddings.npy\"\n",
"PATH_TO_DATASET = r\"C:\\Users\\Henri\\Documents\\GitHub\\AlignmentSearch\\src\\Embeddings Search\\data\\dataset.pkl\"\n",
"\n",
"COMPLETIONS_MODEL = \"text-davinci-003\"\n",
"EMBEDDING_MODEL = \"text-embedding-ada-002\"\n",
@@ -193,29 +258,7 @@
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [],
"source": [
"def split_article(text: str) -> List[str]:\n",
" # Receives one text (str) and returns a list of sections (List[str]), each section being a few appended paragraphs that do not exceed 1000 words.\n",
" # This is done to avoid the 8000 token limit of OpenAI embeddings.\n",
" sections = []\n",
" section = \"\"\n",
" paragraphs = text.split('\\n')\n",
" for paragraph in paragraphs:\n",
" if paragraph == \"\": continue\n",
" if len(section.split()) + len(paragraph.split()) > 1000 or len(section) + len(paragraph) > 7000:\n",
" sections.append(section)\n",
" section = \"\"\n",
" section += f\"{paragraph}\\n\"\n",
" sections.append(section)\n",
" return sections"
]
},
{
"cell_type": "code",
"execution_count": 19,
"execution_count": 13,
"metadata": {},
"outputs": [],
"source": [
@@ -223,6 +266,84 @@
" pass"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {},
"outputs": [],
"source": [
"class TextSplitter:\n",
" def __init__(self, block_maxsize: int = 800, block_minsize: int = 500):\n",
" self.block_maxsize = block_maxsize\n",
" self.block_minsize = block_minsize\n",
" self.blocks = []\n",
" self.current_block = []\n",
" self.current_block_len = 0\n",
"\n",
"\n",
" def add_sentence_to_blocks(self, sentence):\n",
" sentence_len = len(sentence)\n",
" sentence_fits_in_current_block = self.current_block_len + sentence_len <= self.block_maxsize\n",
" current_block_is_big_enough = self.current_block_len >= self.block_minsize\n",
" sentence_fits_in_standalone_block = sentence_len <= self.block_maxsize\n",
"\n",
" if sentence_fits_in_current_block:\n",
" self.current_block.append(sentence)\n",
" self.current_block_len += sentence_len + 1 # +1 for the space\n",
" return\n",
" \n",
" if current_block_is_big_enough and sentence_fits_in_standalone_block:\n",
" self.blocks.append(\" \".join(self.current_block))\n",
" self.current_block = [sentence]\n",
" self.current_block_len = sentence_len + 1 # +1 for the space\n",
" return\n",
" \n",
" #special cases:TODO refactor\n",
" #case 1: current_block_len < block_minsize and current_block_len + sentence_len > block_maxsize\n",
" #case 2: current_block_len > block_minsize but sentence_len > block_maxsize\n",
" shorter_sentence = sentence[self.block_maxsize - self.current_block_len]\n",
" self.current_block.append(shorter_sentence)\n",
" self.blocks.append(\" \".join(self.current_block))\n",
" self.current_block = []\n",
" self.current_block_len = 0 \n",
" \n",
"\n",
" def add_paragraph_to_blocks(self, paragraph):\n",
" paragraph_len = len(paragraph)\n",
" if self.current_block_len + paragraph_len > self.block_maxsize:\n",
" sentences = split_into_sentences(paragraph)\n",
" for sentence in sentences:\n",
" self.add_sentence_to_blocks(sentence)\n",
" return\n",
" \n",
" if self.block_minsize <= self.current_block_len + paragraph_len <= self.block_maxsize:\n",
" self.current_block.append(paragraph)\n",
" self.blocks.append(\"\\n\\n\".join(self.current_block))\n",
" self.current_block = []\n",
" self.current_block_len = 0\n",
" return\n",
" \n",
" if self.current_block_len + paragraph_len < self.block_minsize:\n",
" self.current_block.append(paragraph)\n",
" self.current_block_len += paragraph_len + 2 # +2 for the \\n\\n\n",
" return\n",
" \n",
" def add_text_to_blocks(self, text):\n",
" paragraphs = text.split(\"\\n\\n\")\n",
" for paragraph in paragraphs:\n",
" self.add_paragraph_to_blocks(paragraph)\n",
" if self.current_block != []:\n",
" self.blocks.append(\"\\n\\n\".join(self.current_block))\n",
"\n",
"\n",
" def split(self, text: str, signature: str) -> List[str]:\n",
" \"\"\"Split text into multiple blocks and add signature to each block.\"\"\"\n",
" # signature has the format : \"link, title, author\"\n",
" self.add_text_to_blocks(text)\n",
" \n",
" return [f\"{block}\\n - {signature}\" for block in self.blocks]\n"
]
},
{
"attachments": {},
"cell_type": "markdown",
@@ -233,7 +354,7 @@
},
{
"cell_type": "code",
"execution_count": 61,
"execution_count": 15,
"metadata": {},
"outputs": [],
"source": [
@@ -241,13 +362,14 @@
" \"Entry has no source.\": 0,\n",
" \"Entry has no title.\": 0,\n",
" \"Entry has no text.\": 0,\n",
" \"Entry has no URL.\": 0\n",
" \"Entry has no URL.\": 0,\n",
" \"Entry has wrong citation level.\": 0\n",
"}"
]
},
{
"cell_type": "code",
"execution_count": 69,
"execution_count": 138,
"metadata": {},
"outputs": [],
"source": [
@@ -255,13 +377,20 @@
" def __init__(self,\n",
" path: str, # Path to the dataset .jsonl file.\n",
" sources: List[str] = None, # List of sources to include. If None, include all sources.\n",
" max_paragraph_length: Tuple[int, int] = None # (max number of words in a paragraph, max number of characters in a paragraph)\n",
" rate_limit_per_minute: int = 60, # Rate limit for the OpenAI API.\n",
" block_min_max_size: Tuple[int, int] = None, # Tuple of (min_block_size, max_block_size), used for the text splitter. If None, use default values.\n",
" ):\n",
" self.path = path\n",
" self.sources = sources\n",
" self.max_paragraph_length = max_paragraph_length\n",
" \n",
" self.data: List[Tuple[str, str, str]] = [] # List of tuples, each containing the title of an article, its URL, and text. E.g.: [('title', 'url', 'text'), ...]\n",
" self.rate_limit_per_minute = rate_limit_per_minute\n",
" self.delay_in_seconds = 60.0 / self.rate_limit_per_minute\n",
" \n",
" # Set up text splitter\n",
" if block_min_max_size is None: self.block_min_max_size = (400, 600)\n",
" else: self.block_min_max_size = block_min_max_size\n",
" self.text_splitter = TextSplitter(block_maxsize=self.block_min_max_size[1], block_minsize=self.block_min_max_size[0])\n",
" \n",
" self.data: List[Tuple[str]] = [] # List of tuples, each containing the title of an article, its URL, and text. E.g.: [('title', 'url', 'text'), ...]\n",
" self.embed_split: List[str] = [] # List of strings, each being a few paragraphs from a single article (not exceeding 1000 words).\n",
" \n",
" self.num_articles: Dict[str, int] = {} # Number of articles per source. E.g.: {'source1': 10, 'source2': 20, 'total': 30}\n",
@@ -275,7 +404,7 @@
" self.total_char_count = 0\n",
" self.total_word_count = 0\n",
" self.total_sentence_count = 0\n",
" self.total_paragraph_count = 0\n",
" self.total_block_count = 0\n",
" \n",
" def get_info_tmp(self):\n",
" self.sources_so_far = []\n",
@@ -287,60 +416,155 @@
" if entry['source'] not in self.sources_so_far:\n",
" self.sources_so_far.append(entry['source'])\n",
" self.info_types[entry['source']] = entry.keys()\n",
" \n",
" if 'tags' in entry:\n",
" print(entry['tags'])\n",
" \n",
" \"\"\"\n",
" {\n",
" 'text', \n",
" 'title', 'book_title', # If there is both, take title, otherwise take book_title\n",
" 'author', 'authors', # If there is both, take author, otherwise take authors, otherwise take author\n",
" 'citation_level', # must be 0 or 1\n",
" 'date_published', 'published', # take first 10 chars of date_published, if it exists; else take first 16 chars of published, if it exists\n",
" 'doi', 'link', 'links', 'url', # if link, take link; elif url, take url; elif doi, take doi\n",
" 'tags'\n",
" }\n",
" \"\"\"\n",
" \n",
" def get_alignment_texts(self):\n",
" with jsonlines.open(self.path, \"r\") as reader:\n",
" for entry in reader:\n",
" # Only get one in a thousand articles\n",
" if random.randint(0, 3000) != 19: continue\n",
" try:\n",
" if 'source' not in entry: raise MissingDataException(\"Entry has no source.\")\n",
" \n",
" if self.sources is None:\n",
" if entry['source'] not in self.num_articles:\n",
" self.num_articles[entry['source']] = 1\n",
" else:\n",
" self.num_articles[entry['source']] += 1\n",
" if entry['source'] not in self.num_articles: self.num_articles[entry['source']] = 1\n",
" else: self.num_articles[entry['source']] += 1\n",
" self.num_articles['total'] += 1\n",
" else:\n",
" if entry['source'] in self.sources:\n",
" self.num_articles[entry['source']] += 1\n",
" self.num_articles['total'] += 1\n",
" else:\n",
" continue\n",
" else: continue\n",
" \n",
" if 'title' not in entry: raise MissingDataException(\"Entry has no title.\")\n",
" if 'link' not in entry: raise MissingDataException(\"Entry has no link.\")\n",
" if 'text' not in entry: raise MissingDataException(\"Entry has no text.\")\n",
" text=title=author=citation_level=date_published=url=tags=None\n",
" \n",
" # Get text\n",
" if 'text' in entry and entry['text'] != '': text = entry['text']\n",
" else: raise MissingDataException(f\"Entry has no text.\")\n",
" \n",
" # Get title\n",
" if 'title' in entry and 'book_title' in entry and entry['title'] != '': title = entry['title']\n",
" elif 'book_title' in entry and entry['book_title'] != '': title = entry['book_title']\n",
" else: title = None\n",
" \n",
" # Get author\n",
" if 'author' in entry and 'authors' in entry and entry['author'] != '': author = entry['author']\n",
" elif 'authors' in entry and entry['authors'] != '': author = entry['authors']\n",
" elif 'author' in entry and entry['author'] != '': author = entry['author']\n",
" else: author = None\n",
" \n",
" # Get citation level\n",
" if 'citation_level' in entry:\n",
" if entry['citation_level'] != 0: raise MissingDataException(f\"Entry has citation_level {entry['citation_level']}.\")\n",
" \n",
" # Get date published\n",
" if 'date_published' in entry and entry['date_published'] != '': date_published = entry['date_published'][:10]\n",
" elif 'published' in entry and entry['published'] != '': date_published = entry['published'][:16]\n",
" else: date_published = None\n",
" \n",
" # Get URL\n",
" if 'link' in entry and entry['link'] != '': url = entry['link']\n",
" elif 'url' in entry and entry['url'] != '': url = entry['url']\n",
" elif 'doi' in entry and entry['doi'] != '': url = entry['doi']\n",
" else: url = None\n",
" \n",
" # Get tags\n",
" if 'tags' in entry and entry['tags'] != '':\n",
" if type(entry['tags']) == list: tags = ', '.join([val['term'] for val in entry['tags']])\n",
" elif type(entry['tags']) == str: tags = entry['tags']\n",
" else: tags = None\n",
" \n",
" signature = \"\"\n",
" if title: signature += f\"Title: {title}, \"\n",
" if author: signature += f\"Author: {author}, \"\n",
" if date_published: signature += f\"Date published: {date_published}, \"\n",
" if url: signature += f\"URL: {url}, \"\n",
" if tags: signature += f\"Tags: {tags}, \"\n",
" signature = signature[:-2]\n",
"\n",
" self.data.append((entry['title'], entry['link'], entry['text']))\n",
" paragraphs = split_article(entry['text'])\n",
" self.embed_split.extend(paragraphs)\n",
" self.data.append((title, author, date_published, url, tags, text))\n",
" \n",
" blocks = self.text_splitter.split(text, signature)\n",
" self.embed_split.extend(blocks)\n",
" \n",
" self.total_char_count += len(entry['text'])\n",
" self.total_word_count += len(entry['text'].split())\n",
" self.total_sentence_count += len(split_into_sentences(entry['text']))\n",
" self.total_paragraph_count += len(paragraphs)\n",
" except KeyError:\n",
" pass\n",
" self.total_block_count += len(blocks)\n",
" \n",
" except MissingDataException as e:\n",
" if str(e) not in error_count_dict:\n",
" error_count_dict[str(e)] = 0\n",
" error_count_dict[str(e)] += 1\n",
"\n",
" def get_embedding(text: str) -> np.ndarray:\n",
" @retry(wait=wait_random_exponential(min=1, max=100), stop=stop_after_attempt(10))\n",
" def get_embedding(self, text: str, delay_in_seconds: float = 0) -> np.ndarray:\n",
" time.sleep(delay_in_seconds)\n",
" result = openai.Embedding.create(model=EMBEDDING_MODEL, input=text)\n",
" return result[\"data\"][0][\"embedding\"]\n",
"\n",
" def get_embeddings(self):\n",
" self.embeddings = np.array([self.get_embedding(text) for text in self.embed_split])\n",
" self.embeddings = np.array([self.get_embedding(text, delay_in_seconds=self.delay_in_seconds) for text in self.embed_split])\n",
" \n",
" def save_embeddings(self, path: str):\n",
" np.save(path, self.embeddings)\n",
" \n",
" def load_embeddings(self, path: str):\n",
" self.embeddings = np.load(path)"
" self.embeddings = np.load(path)\n",
" \n",
" def save_class(self, path: str):\n",
" with open(path, 'wb') as f:\n",
" pickle.dump(self, f)"
]
},
{
"cell_type": "code",
"execution_count": 70,
"execution_count": 139,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"1416\n"
]
}
],
"source": [
"dataset = Dataset(path=PATH_TO_DATA, sources=None)\n",
"dataset.get_alignment_texts()\n",
"dataset.get_embeddings()\n",
"dataset.save_embeddings(PATH_TO_EMBEDDINGS)\n",
"dataset.save_class(PATH_TO_DATASET)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# with open(PATH_TO_DATASET, 'rb') as f:\n",
"# dataset = pickle.load(f)"
]
},
{
"cell_type": "code",
"execution_count": 157,
"metadata": {},
"outputs": [],
"source": [
@@ -349,7 +573,8 @@
" dataset: Dataset, # Dataset object containing the data.\n",
" ):\n",
" self.dataset = dataset\n",
" \n",
" \n",
" @retry(wait=wait_random_exponential(min=1, max=100), stop=stop_after_attempt(10))\n",
" def get_embedding(self, text: str) -> np.ndarray:\n",
" result = openai.Embedding.create(model=EMBEDDING_MODEL, input=text)\n",
" return result[\"data\"][0][\"embedding\"]\n",
@@ -359,11 +584,13 @@
" # Each tuple contains the title of an article, its URL, and text.\n",
" query_embedding = self.get_embedding(query)\n",
" similarities = np.dot(self.dataset.embeddings, query_embedding)\n",
" print(similarities.shape)\n",
" top_k_indices = np.argsort(similarities)[::-1][:k]\n",
" top_k = [self.dataset.data[i] for i in top_k_indices]\n",
" print(top_k_indices)\n",
" top_k = [self.dataset.embed_split[i] for i in top_k_indices]\n",
" return top_k\n",
" \n",
" def construct_prompt(self, question: str, texts: List[Tuple[str, str, str]]) -> str:\n",
" def construct_prompt(self, question: str, texts: List[Tuple[str]]) -> str:\n",
" # Receives a question (str) and a list of articles (List[Tuple[str, str, str]]) and returns a prompt (str) to be used for text generation.\n",
" context = \"\\n\".join(texts)[:MAX_LEN_PROMPT]\n",
" header = \"\"\"Answer the question as truthfully as possible using the provided context, and if the answer is not contained within the text below, say \"I don't know.\"\\n\\nContext:\\n\"\"\"\n",
@@ -380,227 +607,56 @@
" answer = openai.Completion.create(prompt=prompt, **COMPLETIONS_API_PARAMS)[\"choices\"][0][\"text\"].strip(\" \\n\")\n",
" return answer\n",
" \n",
" def search_and_answer(self, question: str, k: int=10) -> str:\n",
" def search_and_answer(self, question: str, k: int=10, HyDE: bool=False) -> str:\n",
" # Receives a question (str) and returns an answer (str) to the question.\n",
" top_k = self.get_top_k(question, k)\n",
" if HyDE:\n",
" raise NotImplementedError\n",
" else:\n",
" top_k = self.get_top_k(question, k)\n",
" answer = self.answer_question(question, top_k)\n",
" return answer\n",
" \n",
" def summarize(self, article: str) -> str:\n",
" COMPLETIONS_API_PARAMS = {\n",
" \"temperature\": 0.0,\n",
" \"max_tokens\": 300,\n",
" \"model\": COMPLETIONS_MODEL,\n",
" }\n",
" raise NotImplementedError"
" return answer\n"
]
},
{
"cell_type": "code",
"execution_count": 71,
"metadata": {},
"outputs": [],
"source": [
"dataset = Dataset(path=PATH_TO_DATA, sources=['https://www.yudkowsky.net'])\n",
"# dataset.get_info_tmp()\n",
"dataset.get_alignment_texts()\n",
"# dataset.get_embeddings()\n",
"# dataset.save_embeddings(EMBEDDINGS_PATH)\n",
"# # dataset.load_embeddings(EMBEDDINGS_PATH)\n",
"\n",
"# search_and_answer = SearchAndAnswer(dataset)\n",
"\n",
"# while True:\n",
"# question = input(\"Enter a question: \")\n",
"# if question == \"quit\":\n",
"# break\n",
"# top_k = search_and_answer.get_top_k(question)\n",
"# answer = search_and_answer.answer_question(question, top_k)\n",
"# print(answer)"
]
},
{
"cell_type": "code",
"execution_count": 82,
"execution_count": 158,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"112\n"
]
},
{
"ename": "",
"evalue": "",
"output_type": "error",
"traceback": [
"\u001b[1;31mThe Kernel crashed while executing code in the the current cell or a previous cell. Please review the code in the cell(s) to identify a possible cause of the failure. Click <a href='https://aka.ms/vscodeJupyterKernelCrash'>here</a> for more info. View Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
"(1416,)\n",
"[ 82 916 1171]\n",
"The need for a high quality alignment dataset for very capable models.\n"
]
}
],
"source": [
"print(len(dataset.embed_split))"
"SA = SearchAndAnswer(dataset=dataset)\n",
"prompt = \"Name a problem in AI Alignment.\"\n",
"answer = SA.search_and_answer(prompt, 3, HyDE=False)\n",
"print(answer)"
]
},
{
"cell_type": "code",
"execution_count": 65,
"execution_count": 147,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"['https://aipulse.org', 'None', 'ebook', 'https://qualiacomputing.com', 'alignment forum', 'lesswrong', 'manual', 'arxiv', 'https://deepmindsafetyresearch.medium.com/', 'waitbutwhy.com', 'GitHub', 'https://aiimpacts.org', 'arbital.com', 'carado.moe', 'nonarxiv_papers', 'https://vkrakovna.wordpress.com', 'https://jsteinhardt.wordpress.com', 'audio-transcripts', 'https://intelligence.org', 'youtube', 'reports', 'https://aisafety.camp', 'curriculum', 'https://www.yudkowsky.net', 'distill']\n"
]
}
],
"source": [
"print(dataset.sources_so_far)"
]
},
{
"cell_type": "code",
"execution_count": 68,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"https://aipulse.org dict_keys(['title', 'title_detail', 'links', 'link', 'authors', 'author', 'author_detail', 'published', 'published_parsed', 'tags', 'id', 'guidislink', 'summary', 'summary_detail', 'content', 'post-id', 'text', 'source', 'source_type'])\n",
"None dict_keys(['text', 'url', 'title', 'source'])\n",
"ebook dict_keys(['epub_version', 'title', 'language', 'description', 'authors', 'publisher', 'publication_date', 'identifiers', 'subject', 'file_size_in_bytes', 'cover_image_extension', 'toc', 'source', 'source_filetype', 'converted_with', 'book_title', 'date_published', 'chapter_names', 'text'])\n",
"https://qualiacomputing.com dict_keys(['title', 'title_detail', 'links', 'link', 'comments', 'authors', 'author', 'author_detail', 'published', 'published_parsed', 'tags', 'id', 'guidislink', 'summary', 'summary_detail', 'content', 'wfw_commentrss', 'slash_comments', 'post-id', 'text', 'source', 'source_type'])\n",
"alignment forum dict_keys(['id', 'title', 'authors', 'date_published', 'score', 'omega_karma', 'votes', 'tags', 'url', 'text', 'source', 'comments'])\n",
"lesswrong dict_keys(['id', 'title', 'authors', 'date_published', 'score', 'omega_karma', 'votes', 'tags', 'url', 'text', 'source', 'comments'])\n",
"manual dict_keys(['source', 'source_type', 'title', 'authors', 'date_published', 'text'])\n",
"arxiv dict_keys(['source', 'source_type', 'converted_with', 'paper_version', 'title', 'authors', 'date_published', 'data_last_modified', 'url', 'abstract', 'author_comment', 'journal_ref', 'doi', 'primary_category', 'categories', 'citation_level', 'alignment_text', 'confidence_score', 'main_tex_filename', 'text', 'bibliography_bbl', 'bibliography_bib', 'arxiv_citations'])\n",
"https://deepmindsafetyresearch.medium.com/ dict_keys(['source', 'source_type', 'url', 'title', 'content', 'text'])\n",
"waitbutwhy.com dict_keys(['source', 'source_type', 'title', 'authors', 'date_published', 'text'])\n",
"GitHub dict_keys(['source', 'source_filetype', 'converted_with', 'book_title', 'author', 'date_published', 'text'])\n",
"https://aiimpacts.org dict_keys(['title', 'title_detail', 'links', 'link', 'comments', 'authors', 'author', 'author_detail', 'published', 'published_parsed', 'tags', 'id', 'guidislink', 'summary', 'summary_detail', 'content', 'wfw_commentrss', 'slash_comments', 'text', 'source', 'source_type'])\n",
"arbital.com dict_keys(['source', 'source_type', 'converted_with', 'title', 'authors', 'date_published', 'url', 'text'])\n",
"carado.moe dict_keys(['source', 'source_type', 'title', 'authors', 'date_published', 'text'])\n",
"nonarxiv_papers dict_keys(['source', 'source_filetype', 'converted_with', 'paper_version', 'title', 'authors', 'date_published', 'abstract', 'journal_ref', 'doi', 'citation_level', 'text', 'bibliography_bib', 'source_file'])\n",
"https://vkrakovna.wordpress.com dict_keys(['title', 'title_detail', 'links', 'link', 'comments', 'authors', 'author', 'author_detail', 'published', 'published_parsed', 'tags', 'id', 'guidislink', 'summary', 'summary_detail', 'content', 'wfw_commentrss', 'slash_comments', 'media_content', 'text', 'source', 'source_type'])\n",
"https://jsteinhardt.wordpress.com dict_keys(['title', 'title_detail', 'links', 'link', 'comments', 'authors', 'author', 'author_detail', 'published', 'published_parsed', 'tags', 'id', 'guidislink', 'summary', 'summary_detail', 'content', 'wfw_commentrss', 'slash_comments', 'media_content', 'text', 'source', 'source_type'])\n",
"audio-transcripts dict_keys(['source', 'source_filetype', 'cleaned', 'converted_with', 'title', 'authors', 'date_published', 'text'])\n",
"https://intelligence.org dict_keys(['title', 'title_detail', 'links', 'link', 'comments', 'authors', 'author', 'author_detail', 'published', 'published_parsed', 'tags', 'id', 'guidislink', 'summary', 'summary_detail', 'content', 'wfw_commentrss', 'slash_comments', 'text', 'source', 'source_type'])\n",
"youtube dict_keys(['source', 'source_type', 'converted_with', 'title', 'authors', 'date_published', 'url', 'text'])\n",
"reports dict_keys(['source', 'source_filetype', 'converted_with', 'paper_version', 'title', 'authors', 'date_published', 'abstract', 'journal_ref', 'doi', 'citation_level', 'text', 'bibliography_bib', 'source_file'])\n",
"https://aisafety.camp dict_keys(['title', 'title_detail', 'links', 'link', 'authors', 'author', 'author_detail', 'published', 'published_parsed', 'tags', 'id', 'guidislink', 'summary', 'summary_detail', 'content', 'post-id', 'text', 'source', 'source_type'])\n",
"curriculum dict_keys(['source', 'source_type', 'title', 'authors', 'date_published', 'text'])\n",
"https://www.yudkowsky.net dict_keys(['title', 'title_detail', 'links', 'link', 'authors', 'author', 'author_detail', 'published', 'published_parsed', 'tags', 'id', 'guidislink', 'summary', 'summary_detail', 'content', 'text', 'source', 'source_type'])\n",
"distill dict_keys(['source', 'source_type', 'converted_with', 'title', 'authors', 'date_published', 'abstract', 'journal_ref', 'doi', 'text', 'bibliography_bib'])\n"
]
}
],
"source": [
"for dataset_source in dataset.sources_so_far:\n",
" print(dataset_source, dataset.info_types[dataset_source])"
]
},
{
"cell_type": "code",
"execution_count": 41,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"missing info: {'Entry has no source.': 487, 'Entry has no title.': 0, 'Entry has no text.': 0, 'Entry has no URL.': 23}\n"
]
}
],
"source": [
"print(f\"missing info: {error_count_dict}\")"
]
},
{
"cell_type": "code",
"execution_count": 35,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"0"
]
},
"execution_count": 35,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"dataset.total_word_count"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"[]"
]
},
"execution_count": 15,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"dataset.data"
]
},
{
"cell_type": "code",
"execution_count": 59,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'https://www.yudkowsky.net': 23, 'total': 23}"
]
},
"execution_count": 59,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"dataset.num_articles"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Source Truth Empirical Difference\n"
]
},
{
"ename": "NameError",
"evalue": "name 'dataset' is not defined",
"output_type": "error",
"traceback": [
"\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[1;31mNameError\u001b[0m Traceback (most recent call last)",
"Cell \u001b[1;32mIn[1], line 51\u001b[0m\n\u001b[0;32m 49\u001b[0m \u001b[39m# Print table. First row has Truth and Empirical findings.\u001b[39;00m\n\u001b[0;32m 50\u001b[0m \u001b[39mprint\u001b[39m(\u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39m{\u001b[39;00m\u001b[39m'\u001b[39m\u001b[39mSource\u001b[39m\u001b[39m'\u001b[39m\u001b[39m:\u001b[39;00m\u001b[39m<20\u001b[39m\u001b[39m}\u001b[39;00m\u001b[39m \u001b[39m\u001b[39m{\u001b[39;00m\u001b[39m'\u001b[39m\u001b[39mTruth\u001b[39m\u001b[39m'\u001b[39m\u001b[39m:\u001b[39;00m\u001b[39m<10\u001b[39m\u001b[39m}\u001b[39;00m\u001b[39m \u001b[39m\u001b[39m{\u001b[39;00m\u001b[39m'\u001b[39m\u001b[39mEmpirical\u001b[39m\u001b[39m'\u001b[39m\u001b[39m:\u001b[39;00m\u001b[39m<10\u001b[39m\u001b[39m}\u001b[39;00m\u001b[39m \u001b[39m\u001b[39m{\u001b[39;00m\u001b[39m'\u001b[39m\u001b[39mDifference\u001b[39m\u001b[39m'\u001b[39m\u001b[39m:\u001b[39;00m\u001b[39m<10\u001b[39m\u001b[39m}\u001b[39;00m\u001b[39m\"\u001b[39m)\n\u001b[1;32m---> 51\u001b[0m \u001b[39mfor\u001b[39;00m source \u001b[39min\u001b[39;00m dataset\u001b[39m.\u001b[39mnum_articles:\n\u001b[0;32m 52\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m 53\u001b[0m \u001b[39mprint\u001b[39m(\u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39m{\u001b[39;00msource[:\u001b[39m20\u001b[39m]\u001b[39m:\u001b[39;00m\u001b[39m<20\u001b[39m\u001b[39m}\u001b[39;00m\u001b[39m \u001b[39m\u001b[39m{\u001b[39;00mnum_articles_truth[source]\u001b[39m:\u001b[39;00m\u001b[39m<10\u001b[39m\u001b[39m}\u001b[39;00m\u001b[39m \u001b[39m\u001b[39m{\u001b[39;00mdataset\u001b[39m.\u001b[39mnum_articles[source]\u001b[39m:\u001b[39;00m\u001b[39m<10\u001b[39m\u001b[39m}\u001b[39;00m\u001b[39m \u001b[39m\u001b[39m{\u001b[39;00mnum_articles_truth[source] \u001b[39m-\u001b[39m dataset\u001b[39m.\u001b[39mnum_articles[source]\u001b[39m:\u001b[39;00m\u001b[39m<10\u001b[39m\u001b[39m}\u001b[39;00m\u001b[39m\"\u001b[39m)\n",
"\u001b[1;31mNameError\u001b[0m: name 'dataset' is not defined"
"Source Truth Empirical Difference\n",
"total 41614 14 41600 \n",
"lesswrong 28479 11 28468 \n",
"alignment forum 2138 1 2137 \n",
"arxiv 8007 2 8005 \n",
"\n",
" Truth Empirical Difference\n",
"Word Count 53550146 23344 53526802 \n",
"Character Count 351767163 143048 351624115 \n"
]
}
],
@@ -666,104 +722,6 @@
"print(f\"{'Word Count':<20} {word_count_truth:<10} {dataset.total_word_count:<10} {word_count_truth - dataset.total_word_count:<10}\")\n",
"print(f\"{'Character Count':<20} {char_count_truth:<10} {dataset.total_char_count:<10} {char_count_truth - dataset.total_char_count:<10}\")"
]
},
{
"cell_type": "code",
"execution_count": 150,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"348457759 characters\n",
"~104537328 tokens\n",
"~69691552 words\n",
"~348457 paragraphs\n",
"~12912 embeddings using method 1\n",
"~116152 embeddings using method 2\n",
"~139383 pages\n",
"~46 cost using method 1\n",
"~627 cost using method 2\n"
]
}
],
"source": [
"\n",
"num_words = dataset.data_length / 5\n",
"num_tokens = num_words * 1.5\n",
"num_paragraphs = num_words // 200\n",
"num_embeds_method_1 = num_tokens // 8096\n",
"num_embeds_method_2 = num_words // 600\n",
"cost_per_embed = 1/(3000*500/8096)\n",
"cost_per_page = 1/3000\n",
"num_pages = num_words // 500\n",
"cost_1 = num_pages * cost_per_page\n",
"cost_2 = num_embeds_method_2 * cost_per_embed\n",
"\n",
"print(f\"{dataset.data_length} characters\")\n",
"print(f\"~{num_tokens:.0f} tokens\")\n",
"print(f\"~{num_words:.0f} words\")\n",
"print(f\"~{num_paragraphs:.0f} paragraphs\")\n",
"print(f\"~{num_embeds_method_1:.0f} embeddings using method 1\")\n",
"print(f\"~{num_embeds_method_2:.0f} embeddings using method 2\")\n",
"print(f\"~{num_pages:.0f} pages\")\n",
"print(f\"~{cost_1:.0f} cost using method 1\")\n",
"print(f\"~{cost_2:.0f} cost using method 2\")"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
"## Random tests"
]
},
{
"cell_type": "code",
"execution_count": 63,
"metadata": {},
"outputs": [],
"source": [
"import json"
]
},
{
"cell_type": "code",
"execution_count": 72,
"metadata": {},
"outputs": [],
"source": [
"with jsonlines.open(PATH_TO_DATA, \"r\") as reader, open(\"aipulse.txt\", \"w\", encoding=\"utf-8\") as writer:\n",
" for entry in reader:\n",
" try:\n",
" if 'source' in entry and entry['source'] == 'https://aipulse.org':\n",
" if 'title' in entry:\n",
" writer.write(f\"Title: {entry['title']}\\n\")\n",
" else:\n",
" writer.write(f\"NO TITLE\\n\")\n",
" if 'text' in entry:\n",
" writer.write(f\"Text: {entry['text']}\\n\")\n",
" else:\n",
" writer.write(f\"NO TEXT\\n\")\n",
" if 'url' in entry:\n",
" writer.write(f\"URL: {entry['url']}\\n\")\n",
" else:\n",
" writer.write(f\"NO URL\\n\")\n",
" writer.write(\"\\n\\n\")\n",
" else:\n",
" continue\n",
" except KeyError:\n",
" pass"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
src/Embeddings Search/text_splitter.py
+1 -8
View File
@@ -44,13 +44,6 @@ def split_into_sentences(text):
sentences = [text.strip()]
return sentences
def select_first_n_chars(text: str, n: int) -> str:
"""Select first n characters from a string if there are more than n characters, otherwise return the whole string"""
if len(text) > n:
return text[:n]
return text
class TextSplitter:
def __init__(self, block_maxsize: int = 800, block_minsize: int = 500):
@@ -81,7 +74,7 @@ class TextSplitter:
#special cases:TODO refactor
#case 1: current_block_len < block_minsize and current_block_len + sentence_len > block_maxsize
#case 2: current_block_len > block_minsize but sentence_len > block_maxsize
shorter_sentence = select_first_n_chars(sentence, self.block_maxsize - self.current_block_len)
shorter_sentence = sentence[self.block_maxsize - self.current_block_len]
self.current_block.append(shorter_sentence)
self.blocks.append(" ".join(self.current_block))
self.current_block = []