wassname/scrape_r_rational
1
0
Fork 0
mirror of https://github.com/wassname/scrape_r_rational.git synced 2026-09-09 11:33:49 +08:00
Code Issues Packages Projects Releases Wiki Activity

misc

Browse Source
This commit is contained in:
wassname
2024-12-25 18:25:40 +08:00
parent 95555ddc32
commit 1092c4a31c
4 changed files with 5182 additions and 1279 deletions
Download Patch File Download Diff File Expand all files Collapse all files
nbs/mjc_001_download.ipynb
+155 -806
View File
File diff suppressed because it is too large Load Diff
nbs/mjc_004_process.ipynb
+4070 -472
View File
File diff suppressed because it is too large Load Diff
nbs/mjc_005_llm.ipynb
+888
View File
@@ -0,0 +1,888 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import json\n",
"import re\n",
"from pathlib import Path\n",
"from typing import List, Tuple, Optional\n",
"import pandas as pd\n",
"from tqdm.auto import tqdm\n",
"tqdm.pandas()"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"10257 threads\n"
]
}
],
"source": [
"fs = sorted(Path(\"../data/json2\").glob(\"*.json\"))\n",
"print(f\"{len(fs)} threads\")"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>score</th>\n",
" <th>n_links</th>\n",
" <th>n_comments</th>\n",
" <th>comments</th>\n",
" <th>thread_urls</th>\n",
" <th>first_link_utc</th>\n",
" <th>last_link_utc</th>\n",
" <th>url</th>\n",
" </tr>\n",
" <tr>\n",
" <th>title</th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>Worth the Candle</th>\n",
" <td>1445</td>\n",
" <td>108</td>\n",
" <td>108</td>\n",
" <td>[{'author_flair_text': 'Self-Appointed Court S...</td>\n",
" <td>[https://reddit.com/r/rational/comments/zvov5c...</td>\n",
" <td>2017-07-28 13:17:36</td>\n",
" <td>2024-04-17 19:03:01</td>\n",
" <td>[https://archiveofourown.org/works/11478249/ch...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>Alexander Wales - The Metropolitan Man, Shadows of the Limelight</th>\n",
" <td>1408</td>\n",
" <td>22</td>\n",
" <td>22</td>\n",
" <td>[{'author_flair_text': 'Time flies like an arr...</td>\n",
" <td>[https://reddit.com/r/rational/comments/zvov5c...</td>\n",
" <td>2015-04-18 18:13:47</td>\n",
" <td>2021-04-29 20:00:04</td>\n",
" <td>[https://www.patreon.com/alexanderwales]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>Reddit - Dive into anything</th>\n",
" <td>1330</td>\n",
" <td>72</td>\n",
" <td>72</td>\n",
" <td>[{'author_flair_text': 'Team Glimglam', 'body'...</td>\n",
" <td>[https://reddit.com/r/rational/comments/zvov5c...</td>\n",
" <td>2014-09-04 21:14:23</td>\n",
" <td>2024-07-31 01:19:04</td>\n",
" <td>[https://reddit.com/r/motheroflearning/comment...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>Mother of Learning</th>\n",
" <td>859</td>\n",
" <td>84</td>\n",
" <td>84</td>\n",
" <td>[{'author_flair_text': 'Utopian Smut Peddler',...</td>\n",
" <td>[https://reddit.com/r/rational/comments/zvov5c...</td>\n",
" <td>2014-07-26 08:21:12</td>\n",
" <td>2024-05-21 07:04:42</td>\n",
" <td>[https://www.fictionpress.com/s/2961893/1/Moth...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>The Metropolitan Man</th>\n",
" <td>801</td>\n",
" <td>58</td>\n",
" <td>58</td>\n",
" <td>[{'author_flair_text': 'Time flies like an arr...</td>\n",
" <td>[https://reddit.com/r/rational/comments/zvov5c...</td>\n",
" <td>2014-05-31 03:39:09</td>\n",
" <td>2023-06-26 19:21:24</td>\n",
" <td>[https://www.fanfiction.net/s/10360716/1/The-M...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>...</th>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>I know Bouletcorp did one...</th>\n",
" <td>16</td>\n",
" <td>2</td>\n",
" <td>2</td>\n",
" <td>[{'author_flair_text': 'Time flies like an arr...</td>\n",
" <td>[https://reddit.com/r/rational/comments/zvov5c...</td>\n",
" <td>2015-04-30 04:07:01</td>\n",
" <td>2015-10-29 01:28:38</td>\n",
" <td>[http://english.bouletcorp.com/2014/09/05/king...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>The Writing on the Wall</th>\n",
" <td>16</td>\n",
" <td>3</td>\n",
" <td>3</td>\n",
" <td>[{'author_flair_text': None, 'body': '[CORDYCE...</td>\n",
" <td>[https://reddit.com/r/rational/comments/zvov5c...</td>\n",
" <td>2016-06-22 23:40:41</td>\n",
" <td>2019-10-31 18:48:58</td>\n",
" <td>[https://www.fimfiction.net/story/42409/the-wr...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>The actual arguments I heard against trusting fact checkers</th>\n",
" <td>16</td>\n",
" <td>2</td>\n",
" <td>2</td>\n",
" <td>[{'author_flair_text': None, 'body': 'The best...</td>\n",
" <td>[https://reddit.com/r/rational/comments/zvov5c...</td>\n",
" <td>2016-11-28 18:49:35</td>\n",
" <td>2017-01-16 19:08:14</td>\n",
" <td>[https://www.reddit.com/r/AskTrumpSupporters/c...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>The flash fiction designation is usually reserved for stories that are under 1,500 words</th>\n",
" <td>16</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>[{'author_flair_text': None, 'body': '1500-300...</td>\n",
" <td>[https://reddit.com/r/rational/comments/zvov5c...</td>\n",
" <td>2019-04-28 05:47:47</td>\n",
" <td>2019-04-28 05:47:47</td>\n",
" <td>[https://www.writersdigest.com/writing-article...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>The Possibility of a Superhero: E. Paul Zehr, James Kakalios: 8601420885141: Amazon.com: Books</th>\n",
" <td>16</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>[{'author_flair_text': None, 'body': 'I've got...</td>\n",
" <td>[https://reddit.com/r/rational/comments/zvov5c...</td>\n",
" <td>2018-01-06 04:50:40</td>\n",
" <td>2018-01-06 04:50:40</td>\n",
" <td>[https://www.amazon.com/Becoming-Batman-Possib...</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>2591 rows × 8 columns</p>\n",
"</div>"
],
"text/plain": [
" score n_links \\\n",
"title \n",
"Worth the Candle 1445 108 \n",
"Alexander Wales - The Metropolitan Man, Shadows... 1408 22 \n",
"Reddit - Dive into anything 1330 72 \n",
"Mother of Learning 859 84 \n",
"The Metropolitan Man 801 58 \n",
"... ... ... \n",
"I know Bouletcorp did one... 16 2 \n",
"The Writing on the Wall 16 3 \n",
"The actual arguments I heard against trusting f... 16 2 \n",
"The flash fiction designation is usually reserv... 16 1 \n",
"The Possibility of a Superhero: E. Paul Zehr, J... 16 1 \n",
"\n",
" n_comments \\\n",
"title \n",
"Worth the Candle 108 \n",
"Alexander Wales - The Metropolitan Man, Shadows... 22 \n",
"Reddit - Dive into anything 72 \n",
"Mother of Learning 84 \n",
"The Metropolitan Man 58 \n",
"... ... \n",
"I know Bouletcorp did one... 2 \n",
"The Writing on the Wall 3 \n",
"The actual arguments I heard against trusting f... 2 \n",
"The flash fiction designation is usually reserv... 1 \n",
"The Possibility of a Superhero: E. Paul Zehr, J... 1 \n",
"\n",
" comments \\\n",
"title \n",
"Worth the Candle [{'author_flair_text': 'Self-Appointed Court S... \n",
"Alexander Wales - The Metropolitan Man, Shadows... [{'author_flair_text': 'Time flies like an arr... \n",
"Reddit - Dive into anything [{'author_flair_text': 'Team Glimglam', 'body'... \n",
"Mother of Learning [{'author_flair_text': 'Utopian Smut Peddler',... \n",
"The Metropolitan Man [{'author_flair_text': 'Time flies like an arr... \n",
"... ... \n",
"I know Bouletcorp did one... [{'author_flair_text': 'Time flies like an arr... \n",
"The Writing on the Wall [{'author_flair_text': None, 'body': '[CORDYCE... \n",
"The actual arguments I heard against trusting f... [{'author_flair_text': None, 'body': 'The best... \n",
"The flash fiction designation is usually reserv... [{'author_flair_text': None, 'body': '1500-300... \n",
"The Possibility of a Superhero: E. Paul Zehr, J... [{'author_flair_text': None, 'body': 'I've got... \n",
"\n",
" thread_urls \\\n",
"title \n",
"Worth the Candle [https://reddit.com/r/rational/comments/zvov5c... \n",
"Alexander Wales - The Metropolitan Man, Shadows... [https://reddit.com/r/rational/comments/zvov5c... \n",
"Reddit - Dive into anything [https://reddit.com/r/rational/comments/zvov5c... \n",
"Mother of Learning [https://reddit.com/r/rational/comments/zvov5c... \n",
"The Metropolitan Man [https://reddit.com/r/rational/comments/zvov5c... \n",
"... ... \n",
"I know Bouletcorp did one... [https://reddit.com/r/rational/comments/zvov5c... \n",
"The Writing on the Wall [https://reddit.com/r/rational/comments/zvov5c... \n",
"The actual arguments I heard against trusting f... [https://reddit.com/r/rational/comments/zvov5c... \n",
"The flash fiction designation is usually reserv... [https://reddit.com/r/rational/comments/zvov5c... \n",
"The Possibility of a Superhero: E. Paul Zehr, J... [https://reddit.com/r/rational/comments/zvov5c... \n",
"\n",
" first_link_utc \\\n",
"title \n",
"Worth the Candle 2017-07-28 13:17:36 \n",
"Alexander Wales - The Metropolitan Man, Shadows... 2015-04-18 18:13:47 \n",
"Reddit - Dive into anything 2014-09-04 21:14:23 \n",
"Mother of Learning 2014-07-26 08:21:12 \n",
"The Metropolitan Man 2014-05-31 03:39:09 \n",
"... ... \n",
"I know Bouletcorp did one... 2015-04-30 04:07:01 \n",
"The Writing on the Wall 2016-06-22 23:40:41 \n",
"The actual arguments I heard against trusting f... 2016-11-28 18:49:35 \n",
"The flash fiction designation is usually reserv... 2019-04-28 05:47:47 \n",
"The Possibility of a Superhero: E. Paul Zehr, J... 2018-01-06 04:50:40 \n",
"\n",
" last_link_utc \\\n",
"title \n",
"Worth the Candle 2024-04-17 19:03:01 \n",
"Alexander Wales - The Metropolitan Man, Shadows... 2021-04-29 20:00:04 \n",
"Reddit - Dive into anything 2024-07-31 01:19:04 \n",
"Mother of Learning 2024-05-21 07:04:42 \n",
"The Metropolitan Man 2023-06-26 19:21:24 \n",
"... ... \n",
"I know Bouletcorp did one... 2015-10-29 01:28:38 \n",
"The Writing on the Wall 2019-10-31 18:48:58 \n",
"The actual arguments I heard against trusting f... 2017-01-16 19:08:14 \n",
"The flash fiction designation is usually reserv... 2019-04-28 05:47:47 \n",
"The Possibility of a Superhero: E. Paul Zehr, J... 2018-01-06 04:50:40 \n",
"\n",
" url \n",
"title \n",
"Worth the Candle [https://archiveofourown.org/works/11478249/ch... \n",
"Alexander Wales - The Metropolitan Man, Shadows... [https://www.patreon.com/alexanderwales] \n",
"Reddit - Dive into anything [https://reddit.com/r/motheroflearning/comment... \n",
"Mother of Learning [https://www.fictionpress.com/s/2961893/1/Moth... \n",
"The Metropolitan Man [https://www.fanfiction.net/s/10360716/1/The-M... \n",
"... ... \n",
"I know Bouletcorp did one... [http://english.bouletcorp.com/2014/09/05/king... \n",
"The Writing on the Wall [https://www.fimfiction.net/story/42409/the-wr... \n",
"The actual arguments I heard against trusting f... [https://www.reddit.com/r/AskTrumpSupporters/c... \n",
"The flash fiction designation is usually reserv... [https://www.writersdigest.com/writing-article... \n",
"The Possibility of a Superhero: E. Paul Zehr, J... [https://www.amazon.com/Becoming-Batman-Possib... \n",
"\n",
"[2591 rows x 8 columns]"
]
},
"execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"from IPython.display import display\n",
"# load output from\n",
"df3 = pd.read_parquet(\"../outputs/df_links4.parquet\")\n",
"df3"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Extra get a llm summary of each link [WIP]\n",
"\n",
"Grab all md's that mention a story, ask claude to summarize\n",
"\n",
"We could also get total karma per mention"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": []
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"import dotenv\n",
"from anycache import anycache\n",
"\n",
"dotenv.load_dotenv()\n",
"from openai import OpenAI\n",
"\n",
"client = OpenAI()\n",
"\n",
"import tiktoken\n",
"\n",
"enc = tiktoken.encoding_for_model(\"gpt-4o-mini\")\n",
"cost = 0.150 / 1e6"
]
},
{
"cell_type": "code",
"execution_count": 41,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"('2009-11-25T02:34:03', '2024-12-16T15:00:13')"
]
},
"execution_count": 41,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# load all md posts \n",
"md_posts = []\n",
"fs = sorted(Path(\"../data/cache2\").glob(\"*.md\"))\n",
"for f in fs:\n",
" s = f.open().read()\n",
" md_posts.append(s)\n",
"\n",
"\n",
"# order by date\n",
"def md2date(s: str) -> str:\n",
" return s.split('* Created: ')[1].split('\\n')[0]\n",
"\n",
"md_posts = sorted(md_posts, key=md2date)\n",
"md2date(md_posts[0]), md2date(md_posts[-1])"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 45,
"metadata": {},
"outputs": [],
"source": [
"\n",
"def get_post_context(urls: List[str], char_budget=100000, min_size=1000) -> str:\n",
" # TODO maybe I should just get comment with links, and children?\n",
" assert len(urls) > 0\n",
" # from comments\n",
" # return df3.loc[title].comments\n",
"\n",
" # or I could just all markdowns with\n",
"\n",
" # TODO use langchain chunking?\n",
"\n",
" matches = []\n",
" for ii, post in enumerate(md_posts):\n",
" for url in urls:\n",
" if url in post:\n",
" matches.append(post)\n",
" break\n",
"\n",
" budget_pp = char_budget / len(matches)\n",
" budget_pp = max(budget_pp, min_size)\n",
" s = \"\"\n",
" for i in range(len(matches)):\n",
" post = matches[i]\n",
"\n",
" for url in urls:\n",
" if url in post:\n",
" ind = post.index(url)\n",
"\n",
" i0 = int(max(0, ind - budget_pp // 4))\n",
" i1 = int(min(len(post), ind + budget_pp // 4 * 3))\n",
" post_chunk = post[i0:i1]\n",
" if i0 > 0:\n",
" post_chunk = \"...\" + post_chunk\n",
" if i1 < len(post):\n",
" post_chunk = post_chunk + \"...\"\n",
"\n",
" s += f\"\\n\\n----- Thread {ii} -----\\n\\n\" + post_chunk\n",
"\n",
" # if too large get first N//2 and last N//2\n",
" if len(s) > char_budget:\n",
" s = s[:char_budget // 2] + \"...\" + s[-char_budget // 2:]\n",
" return s\n",
"\n",
"\n",
"# url = df3.url[0].split('\\n')\n",
"# print(url)\n",
"# c = get_context(url, 400000)\n",
"# print(c[:1000])"
]
},
{
"cell_type": "code",
"execution_count": 46,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array(['https://www.amazon.com/Becoming-Batman-Possibility-Paul-Zehr/dp/0801890632'],\n",
" dtype=object)"
]
},
"execution_count": 46,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"df3.iloc[-1].url"
]
},
{
"cell_type": "code",
"execution_count": 49,
"metadata": {},
"outputs": [],
"source": [
"# QC test with long and short context\n",
"# urls = ['https://archiveofourown.org/works/11478249/chapters/25740126',\n",
"# 'https://www.royalroad.com/fiction/25137/worth-the-candle',\n",
"# 'http://archiveofourown.org/works/11478249?view_full_work=true',\n",
"# 'http://archiveofourown.org/works/11478249/chapters/25740126',\n",
"# 'https://archiveofourown.org/works/11478249']\n",
"# c = get_post_context(urls, 400000)\n",
"# print(urls)\n",
"# print(c)\n",
"\n",
"# # urls = ['https://www.amazon.com/Becoming-Batman-Possibility-Paul-Zehr/dp/0801890632']\n",
"# # c = get_post_context(urls, 400000)\n",
"# # print(urls)\n",
"# # print(c)\n"
]
},
{
"cell_type": "code",
"execution_count": 117,
"metadata": {},
"outputs": [],
"source": [
"\n",
"\n",
"from pydantic import BaseModel, Field\n",
"\n",
"\n",
"class FictionInfo(BaseModel):\n",
" title: str\n",
" description: str = Field(description=\"A few paragraphs of very concise, informative, dense, description of the work\")\n",
" tags: List[str] = Field(\n",
" description=\"\"\"Long list of common descriptors: format (web serial, fanfic, lightnovel, short, complete, comic), genre (scifi, fantasy)\n",
" Key elements (rational, timeloop, litrpg, progression, cultivation, isekai)\n",
" Content notes (grimdark, romance, harem, queer, funny, NSFW)\n",
" \"\"\"\n",
" )\n",
"\n",
" # status: Optional[str] = Field(description=\"complete/ongoing/hiatus/abandoned\")\n",
" # type: str = Field(description='e.g. fanfiction, original, comic, etc.')\n",
"\n",
" reviews_quotes: List[str] = Field(\n",
" description=\"Directly and fully quote excerpts from every single users' comments about the fiction\"\n",
" )\n",
" reviews_summary: Optional[str] = Field(\n",
" description=\"Structured, dry, and concise summary of reviews\"\n",
" )\n",
" reccomendations: Optional[str] = Field(\n",
" description=\"Why users recommend it\"\n",
" )\n",
" disrecommendations: Optional[str] = Field(\n",
" description=\"Why users disrecommend it\"\n",
" )\n",
" why: Optional[str] = Field(\n",
" description=\"Why/when might readers of r/rational like it\"\n",
" )\n",
" if_you_liked_x_you_will_like_this: List[str] = Field(\n",
" description=\"Fans of X will also like the work. Exhaustively list ALL of X mentioned in the discussions.\"\n",
" )\n",
"\n",
" quality: float = Field(\n",
" # ge=0.0, le=10.0,\n",
" description=\"Overall user sentiment out of 10\"\n",
" )\n",
" rationality: Optional[float] = Field(\n",
" # ge=0.0, le=10.0,\n",
" description=\"Systematic worldbuilding, character competence, logical consistency. Where HPMOR is a 10 and Worm is a 5.\"\n",
" )\n",
" rating_writing: Optional[float]\n",
" rating_plot: Optional[float]\n",
" rating_character: Optional[float]\n",
" rating_worldbuilding: Optional[float]\n",
"\n",
"\n",
"f_cache_llm = Path(\"../outputs/.anycache3\")\n",
"\n",
"\n",
"# @anycache(f_cache_llm)\n",
"def get_llm_summary(title: str, urls: str, context: str):\n",
" chat_completion = client.beta.chat.completions.parse(\n",
" messages=[\n",
" {\n",
" \"role\": \"system\",\n",
" \"content\": \"You are Gwern Branwern, an internet librarian who specializes in rational fiction. You are summarising community reccomendations from r/rational into a dry, informative, concise, and structured format for your own personal notes. Because it's private you can be consise, frank, and opinionated.\",\n",
" },\n",
" {\n",
" \"role\": \"user\",\n",
" \"content\": f\"\"\"Using the given structure, summarize the parts of these discussions where they talk about {title} (urls: {urls}).\n",
"\n",
"### Context:\n",
"\n",
"{context}\"\"\",\n",
" },\n",
" ],\n",
" model=\"gpt-4o-mini\",\n",
" response_format=FictionInfo,\n",
" )\n",
"\n",
" return chat_completion.choices[0].message.parsed.__dict__"
]
},
{
"cell_type": "code",
"execution_count": 118,
"metadata": {},
"outputs": [],
"source": [
"# import shutil\n",
"# shutil.rmtree(f_cache_llm, ignore_errors=True)"
]
},
{
"cell_type": "code",
"execution_count": 119,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'properties': {'title': {'title': 'Title', 'type': 'string'},\n",
" 'description': {'description': 'A few paragraphs of very concise, informative, dense, description of the work',\n",
" 'title': 'Description',\n",
" 'type': 'string'},\n",
" 'tags': {'description': 'Long list of common descriptors: format (web serial, fanfic, lightnovel, short, complete, comic), genre (scifi, fantasy)\\n Key elements (rational, timeloop, litrpg, progression, cultivation, isekai)\\n Content notes (grimdark, romance, harem, queer, funny, NSFW)\\n ',\n",
" 'items': {'type': 'string'},\n",
" 'title': 'Tags',\n",
" 'type': 'array'},\n",
" 'reviews_quotes': {'description': \"Directly and fully quote excerpts from every single users' comments about the fiction\",\n",
" 'items': {'type': 'string'},\n",
" 'title': 'Reviews Quotes',\n",
" 'type': 'array'},\n",
" 'reviews_summary': {'anyOf': [{'type': 'string'}, {'type': 'null'}],\n",
" 'description': 'Structured, dry, and concise summary of reviews',\n",
" 'title': 'Reviews Summary'},\n",
" 'reccomendations': {'anyOf': [{'type': 'string'}, {'type': 'null'}],\n",
" 'description': 'Why users recommend it',\n",
" 'title': 'Reccomendations'},\n",
" 'disrecommendations': {'anyOf': [{'type': 'string'}, {'type': 'null'}],\n",
" 'description': 'Why users disrecommend it',\n",
" 'title': 'Disrecommendations'},\n",
" 'why': {'anyOf': [{'type': 'string'}, {'type': 'null'}],\n",
" 'description': 'Why/when might readers of r/rational like it',\n",
" 'title': 'Why'},\n",
" 'if_you_liked_x_you_will_like_this': {'description': 'Fans of X will also like the work. Exhaustively list ALL of X mentioned in the discussions.',\n",
" 'items': {'type': 'string'},\n",
" 'title': 'If You Liked X You Will Like This',\n",
" 'type': 'array'},\n",
" 'quality': {'description': 'Overall user sentiment out of 10',\n",
" 'title': 'Quality',\n",
" 'type': 'number'},\n",
" 'rationality': {'anyOf': [{'type': 'number'}, {'type': 'null'}],\n",
" 'description': 'Systematic worldbuilding, character competence, logical consistency. Where HPMOR is a 10 and Worm is a 5.',\n",
" 'title': 'Rationality'},\n",
" 'rating_writing': {'anyOf': [{'type': 'number'}, {'type': 'null'}],\n",
" 'title': 'Rating Writing'},\n",
" 'rating_plot': {'anyOf': [{'type': 'number'}, {'type': 'null'}],\n",
" 'title': 'Rating Plot'},\n",
" 'rating_character': {'anyOf': [{'type': 'number'}, {'type': 'null'}],\n",
" 'title': 'Rating Character'},\n",
" 'rating_worldbuilding': {'anyOf': [{'type': 'number'}, {'type': 'null'}],\n",
" 'title': 'Rating Worldbuilding'}},\n",
" 'required': ['title',\n",
" 'description',\n",
" 'tags',\n",
" 'reviews_quotes',\n",
" 'reviews_summary',\n",
" 'reccomendations',\n",
" 'disrecommendations',\n",
" 'why',\n",
" 'if_you_liked_x_you_will_like_this',\n",
" 'quality',\n",
" 'rationality',\n",
" 'rating_writing',\n",
" 'rating_plot',\n",
" 'rating_character',\n",
" 'rating_worldbuilding'],\n",
" 'title': 'FictionInfo',\n",
" 'type': 'object',\n",
" 'additionalProperties': False}"
]
},
"execution_count": 119,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"from openai.lib._pydantic import to_strict_json_schema\n",
"\n",
"to_strict_json_schema(FictionInfo)"
]
},
{
"cell_type": "code",
"execution_count": 120,
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "61da806424a046a0bfaa3b90ec004a0c",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
" 0%| | 0/1000 [00:00<?, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Input Tokens: 13825. Input Cost: 0.0021 USD, for title=Worth the Candle with urls=['https://archiveofourown.org/works/11478249/chapters/25740126'\n",
" 'https://www.royalroad.com/fiction/25137/worth-the-candle'\n",
" 'http://archiveofourown.org/works/11478249?view_full_work=true'\n",
" 'http://archiveofourown.org/works/11478249/chapters/25740126'\n",
" 'https://archiveofourown.org/works/11478249']\n",
"Content: \n",
"\n",
"----- Thread 10261 -----\n",
"\n",
"## [RT][WIP] Worth the Candle, Ch 1\n",
"\n",
"* Author: u/cthulhuraejepsen *Fruit flies like a banana**\n",
"* URL: http://archiveofourown.org/works/11478249/chapters/25740126\n",
"* Score: 33\n",
"\n",
"* Created: 2017-07-14T05:23:38\n",
"\n",
"### Post:\n",
"\n",
"[Link to content](http://archiveofourown.org/works/11478249/chapters/25740126)\n",
"\n",
"### Comments:\n",
"\n",
"> **u/cthulhuraejepsen** [+9] *Fruit flies like a banana**\n",
"> \n",
"> This is a self-insert litRPG portal fantasy, and I would have made it a Groundhog Day loop if I thought I could fit that in.\n",
"> \n",
"\n",
"> **u/LucidityWaver** [+8] *\n",
"> \n",
"> > loosely based on my personal experience of falling into a portal to another world and discovering that I had a character sheet attached to my soul. This story uses some custom CSS for the character sheet and definition tags to allow mouseover on various game terms\n",
"> I can tell I'm going to love this already....\n",
"\n",
"----- Thread 10261 -----\n",
"\n",
"..., another four chapters, so if you were one of those people who doesn't read serials \n"
]
},
{
"data": {
"text/plain": [
"{'title': 'Worth the Candle',\n",
" 'description': \"*Worth the Candle* is a self-insert litRPG portal fantasy novel written by Alexander Wales. The story follows Juniper, a teenage boy who unexpectedly finds himself in a fantasy realm reminiscent of various tabletop games he has played and run as a Dungeon Master. In this world, Juniper possesses a character sheet and must navigate the challenges and mechanics of a complex, game-like system while dealing with themes of agency, friendship, and the consequences of his actions. The work explores the protagonist's growth through various trials, character interactions, and the use of humor interspersed with serious topics such as trauma and morality, making it an engaging read for fans of the genre.\",\n",
" 'tags': ['web serial',\n",
" 'litRPG',\n",
" 'fantasy',\n",
" 'self-insert',\n",
" 'portal fantasy',\n",
" 'grimdark',\n",
" 'humor',\n",
" 'progression'],\n",
" 'reviews_quotes': ['\"This is quickly becoming my favorite litrpg. You need to work a little harder to catch up to \\'Wandering Inn.\\'\" - u/josephwdye',\n",
" '\"One of the things I love about *Worth the Candle* is that it tends to avoid shortcuts and oversimplifications.\" - u/Noumero',\n",
" '\"The plot is rather interesting, as are the characters. Great work!\" - u/Noumero',\n",
" '\"I am dealing with a little bit of seasonal affective disorder right now, but this story is still excellent!\" - u/cthulhuraejepsen',\n",
" '\"I\\'m glad to see the author maintain a high update speed!\" - u/EliezerYudkowsky'],\n",
" 'reviews_summary': \"Overall, readers appreciate the balance of comedy and serious themes, intricate worldbuilding, and character development. There is a strong sentiment regarding the pacing and the author's update frequency, which is positively noted.\",\n",
" 'reccomendations': 'Highly recommended for readers who enjoy complex worldbuilding, humor alongside serious themes, and character growth in a litrPG setting. Particularly appealing to fans of tabletop RPGs and game mechanics.',\n",
" 'disrecommendations': 'May not appeal to readers who dislike self-insert narratives or genre-savvy approaches to storytelling. Those averse to grim topics or violence may also find aspects challenging.',\n",
" 'why': 'Fans of *Worm* or *HPMOR* who enjoy well-developed characters and logical consistency in fantasy worlds will likely appreciate *Worth the Candle* for its nuanced approach to familiar tropes.',\n",
" 'if_you_liked_x_you_will_like_this': ['Worm',\n",
" 'HPMOR',\n",
" 'The Wandering Inn',\n",
" 'Mother of Learning',\n",
" 'The Forerunner Initiative',\n",
" 'The Erogamer'],\n",
" 'quality': 9.0,\n",
" 'rationality': 8.0,\n",
" 'rating_writing': 8.0,\n",
" 'rating_plot': 8.0,\n",
" 'rating_character': 9.0,\n",
" 'rating_worldbuilding': 9.0,\n",
" 'title2': 'Worth the Candle',\n",
" 'url': array(['https://archiveofourown.org/works/11478249/chapters/25740126',\n",
" 'https://www.royalroad.com/fiction/25137/worth-the-candle',\n",
" 'http://archiveofourown.org/works/11478249?view_full_work=true',\n",
" 'http://archiveofourown.org/works/11478249/chapters/25740126',\n",
" 'https://archiveofourown.org/works/11478249'], dtype=object)}"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"ename": "ZeroDivisionError",
"evalue": "division by zero",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mZeroDivisionError\u001b[0m Traceback (most recent call last)",
"Cell \u001b[0;32mIn[120], line 26\u001b[0m\n\u001b[1;32m 22\u001b[0m display(llm_data)\n\u001b[1;32m 24\u001b[0m llm_info\u001b[38;5;241m.\u001b[39mappend(llm_data)\n\u001b[0;32m---> 26\u001b[0m \u001b[38;5;241;43m1\u001b[39;49m\u001b[38;5;241;43m/\u001b[39;49m\u001b[38;5;241;43m0\u001b[39;49m\n",
"\u001b[0;31mZeroDivisionError\u001b[0m: division by zero"
]
}
],
"source": [
"llm_info = []\n",
"\n",
"\n",
"l = min(1000, len(df3))\n",
"for i in tqdm(range(0, l)):\n",
" title = df3.index[i]\n",
" urls = df3.iloc[i].url\n",
"\n",
" context = get_post_context(urls, char_budget=50000)\n",
" tokens = len(enc.encode(context))\n",
" print(\n",
" f\"Input Tokens: {tokens}. Input Cost: {cost * tokens:.4f} USD, for title={title} with urls={urls}\"\n",
" )\n",
"\n",
" llm_data = get_llm_summary(title, urls, context)\n",
"\n",
" llm_data[\"title2\"] = title\n",
" llm_data[\"url\"] = urls\n",
"\n",
" if i==0:\n",
" print(f\"Content: {context[:1000]}\")\n",
" display(llm_data)\n",
"\n",
" llm_info.append(llm_data)\n",
"\n",
" 1/0"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"1"
]
},
"execution_count": 12,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"i"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"df_llm = pd.DataFrame(llm_info)\n",
"df_llm\n",
"# also join with df4"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": ".venv",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.12"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
poetry.lock
Generated
+69 -1
View File
@@ -848,6 +848,20 @@ http2 = ["h2 (>=3,<5)"]
socks = ["socksio (==1.*)"]
zstd = ["zstandard (>=0.18.0)"]
[[package]]
name = "humanize"
version = "4.11.0"
description = "Python humanize utilities"
optional = false
python-versions = ">=3.9"
files = [
{file = "humanize-4.11.0-py3-none-any.whl", hash = "sha256:b53caaec8532bcb2fff70c8826f904c35943f8cecaca29d272d9df38092736c0"},
{file = "humanize-4.11.0.tar.gz", hash = "sha256:e66f36020a2d5a974c504bd2555cf770621dbdbb6d82f94a6857c0b1ea2608be"},
]
[package.extras]
tests = ["freezegun", "pytest", "pytest-cov"]
[[package]]
name = "idna"
version = "3.8"
@@ -2042,6 +2056,60 @@ files = [
[package.extras]
tests = ["pytest"]
[[package]]
name = "pyarrow"
version = "18.1.0"
description = "Python library for Apache Arrow"
optional = false
python-versions = ">=3.9"
files = [
{file = "pyarrow-18.1.0-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:e21488d5cfd3d8b500b3238a6c4b075efabc18f0f6d80b29239737ebd69caa6c"},
{file = "pyarrow-18.1.0-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:b516dad76f258a702f7ca0250885fc93d1fa5ac13ad51258e39d402bd9e2e1e4"},
{file = "pyarrow-18.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4f443122c8e31f4c9199cb23dca29ab9427cef990f283f80fe15b8e124bcc49b"},
{file = "pyarrow-18.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0a03da7f2758645d17b7b4f83c8bffeae5bbb7f974523fe901f36288d2eab71"},
{file = "pyarrow-18.1.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:ba17845efe3aa358ec266cf9cc2800fa73038211fb27968bfa88acd09261a470"},
{file = "pyarrow-18.1.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:3c35813c11a059056a22a3bef520461310f2f7eea5c8a11ef9de7062a23f8d56"},
{file = "pyarrow-18.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:9736ba3c85129d72aefa21b4f3bd715bc4190fe4426715abfff90481e7d00812"},
{file = "pyarrow-18.1.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:eaeabf638408de2772ce3d7793b2668d4bb93807deed1725413b70e3156a7854"},
{file = "pyarrow-18.1.0-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:3b2e2239339c538f3464308fd345113f886ad031ef8266c6f004d49769bb074c"},
{file = "pyarrow-18.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f39a2e0ed32a0970e4e46c262753417a60c43a3246972cfc2d3eb85aedd01b21"},
{file = "pyarrow-18.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e31e9417ba9c42627574bdbfeada7217ad8a4cbbe45b9d6bdd4b62abbca4c6f6"},
{file = "pyarrow-18.1.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:01c034b576ce0eef554f7c3d8c341714954be9b3f5d5bc7117006b85fcf302fe"},
{file = "pyarrow-18.1.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:f266a2c0fc31995a06ebd30bcfdb7f615d7278035ec5b1cd71c48d56daaf30b0"},
{file = "pyarrow-18.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:d4f13eee18433f99adefaeb7e01d83b59f73360c231d4782d9ddfaf1c3fbde0a"},
{file = "pyarrow-18.1.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:9f3a76670b263dc41d0ae877f09124ab96ce10e4e48f3e3e4257273cee61ad0d"},
{file = "pyarrow-18.1.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:da31fbca07c435be88a0c321402c4e31a2ba61593ec7473630769de8346b54ee"},
{file = "pyarrow-18.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:543ad8459bc438efc46d29a759e1079436290bd583141384c6f7a1068ed6f992"},
{file = "pyarrow-18.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0743e503c55be0fdb5c08e7d44853da27f19dc854531c0570f9f394ec9671d54"},
{file = "pyarrow-18.1.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:d4b3d2a34780645bed6414e22dda55a92e0fcd1b8a637fba86800ad737057e33"},
{file = "pyarrow-18.1.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:c52f81aa6f6575058d8e2c782bf79d4f9fdc89887f16825ec3a66607a5dd8e30"},
{file = "pyarrow-18.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:0ad4892617e1a6c7a551cfc827e072a633eaff758fa09f21c4ee548c30bcaf99"},
{file = "pyarrow-18.1.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:84e314d22231357d473eabec709d0ba285fa706a72377f9cc8e1cb3c8013813b"},
{file = "pyarrow-18.1.0-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:f591704ac05dfd0477bb8f8e0bd4b5dc52c1cadf50503858dce3a15db6e46ff2"},
{file = "pyarrow-18.1.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:acb7564204d3c40babf93a05624fc6a8ec1ab1def295c363afc40b0c9e66c191"},
{file = "pyarrow-18.1.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:74de649d1d2ccb778f7c3afff6085bd5092aed4c23df9feeb45dd6b16f3811aa"},
{file = "pyarrow-18.1.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:f96bd502cb11abb08efea6dab09c003305161cb6c9eafd432e35e76e7fa9b90c"},
{file = "pyarrow-18.1.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:36ac22d7782554754a3b50201b607d553a8d71b78cdf03b33c1125be4b52397c"},
{file = "pyarrow-18.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:25dbacab8c5952df0ca6ca0af28f50d45bd31c1ff6fcf79e2d120b4a65ee7181"},
{file = "pyarrow-18.1.0-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:6a276190309aba7bc9d5bd2933230458b3521a4317acfefe69a354f2fe59f2bc"},
{file = "pyarrow-18.1.0-cp313-cp313t-macosx_12_0_x86_64.whl", hash = "sha256:ad514dbfcffe30124ce655d72771ae070f30bf850b48bc4d9d3b25993ee0e386"},
{file = "pyarrow-18.1.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aebc13a11ed3032d8dd6e7171eb6e86d40d67a5639d96c35142bd568b9299324"},
{file = "pyarrow-18.1.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d6cf5c05f3cee251d80e98726b5c7cc9f21bab9e9783673bac58e6dfab57ecc8"},
{file = "pyarrow-18.1.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:11b676cd410cf162d3f6a70b43fb9e1e40affbc542a1e9ed3681895f2962d3d9"},
{file = "pyarrow-18.1.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:b76130d835261b38f14fc41fdfb39ad8d672afb84c447126b84d5472244cfaba"},
{file = "pyarrow-18.1.0-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:0b331e477e40f07238adc7ba7469c36b908f07c89b95dd4bd3a0ec84a3d1e21e"},
{file = "pyarrow-18.1.0-cp39-cp39-macosx_12_0_x86_64.whl", hash = "sha256:2c4dd0c9010a25ba03e198fe743b1cc03cd33c08190afff371749c52ccbbaf76"},
{file = "pyarrow-18.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4f97b31b4c4e21ff58c6f330235ff893cc81e23da081b1a4b1c982075e0ed4e9"},
{file = "pyarrow-18.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4a4813cb8ecf1809871fd2d64a8eff740a1bd3691bbe55f01a3cf6c5ec869754"},
{file = "pyarrow-18.1.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:05a5636ec3eb5cc2a36c6edb534a38ef57b2ab127292a716d00eabb887835f1e"},
{file = "pyarrow-18.1.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:73eeed32e724ea3568bb06161cad5fa7751e45bc2228e33dcb10c614044165c7"},
{file = "pyarrow-18.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:a1880dd6772b685e803011a6b43a230c23b566859a6e0c9a276c1e0faf4f4052"},
{file = "pyarrow-18.1.0.tar.gz", hash = "sha256:9386d3ca9c145b5539a1cfc75df07757dff870168c959b473a0bccbc3abc8c73"},
]
[package.extras]
test = ["cffi", "hypothesis", "pandas", "pytest", "pytz"]
[[package]]
name = "pycodestyle"
version = "2.12.1"
@@ -3302,4 +3370,4 @@ h11 = ">=0.9.0,<1"
[metadata]
lock-version = "2.0"
python-versions = ">=3.10,<4.0"
content-hash = "35f407a3a2ca5df109fc3ff7741828f016f1fa3fc90b0c651b4414a2bac71905"
content-hash = "5660dc3e3a96458dc7e86b6b3d98691b96a882602dc3ee37eeb2e041da44b02f"