From 54e055cbda2d46f7db60695884ce75e01f330d92 Mon Sep 17 00:00:00 2001
From: wassname <1103714+wassname@users.noreply.github.com>
Date: Wed, 25 Dec 2024 09:05:38 +0800
Subject: [PATCH] wip

---
 README.md                 |    4 +
 nbs/mjc_002_llm.ipynb     |  160 ---
 nbs/mjc_003_nonllm.ipynb  |  237 ----
 nbs/mjc_004_process.ipynb | 2257 ++++++++++++++++---------------------
 poetry.lock               | 1119 +++++++++++++++++-
 pyproject.toml            |    2 +
 6 files changed, 2115 insertions(+), 1664 deletions(-)
 delete mode 100644 nbs/mjc_002_llm.ipynb
 delete mode 100644 nbs/mjc_003_nonllm.ipynb

diff --git a/README.md b/README.md
index 768f5d09..69d7bb17 100644
--- a/README.md
+++ b/README.md
@@ -14,6 +14,10 @@ Project plan:
 - [x] use [statistics](https://github.com/wassname/scrape_r_rational/blob/main/nbs/links.csv)
 - [ ] Use llm to get reccomendations, sentiment, karma etc
 - [x] share
+- [ ] comment md to html
+- [ ] comment expand
+- [ ] threads where it's mentioned
+- [ ] have llm check if the titles are valid novel titles?
 
 
 ## Install requirements
diff --git a/nbs/mjc_002_llm.ipynb b/nbs/mjc_002_llm.ipynb
deleted file mode 100644
index 5b570920..00000000
--- a/nbs/mjc_002_llm.ipynb
+++ /dev/null
@@ -1,160 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": 1,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# TODO have an llm extract reccomendations, with the id. that way we can get the karma and sentiment too"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 2,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# dotenv\n",
-    "import os\n",
-    "from dotenv import load_dotenv\n",
-    "load_dotenv()  # take environment variables from .env\n",
-    "\n",
-    "from pathlib import Path"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 3,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "prompt = \"\"\"The following is a reddit thread.\n",
-    "\n",
-    "{thread}\n",
-    "\n",
-    "Please extract all the recommendations from this thread, and provide the following information for each recommendation:\n",
-    "\n",
-    "- The recommendation itself\n",
-    "- The karma of the recommendation\n",
-    "- The sentiment of the recommendation\n",
-    "- The thing that was recommended\n",
-    "\n",
-    "\"\"\""
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 4,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "threads = sorted(Path(\"../outputs\").glob(\"*.md\"))\n",
-    "thread = threads[0].read_text()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 5,
-   "metadata": {},
-   "outputs": [
-    {
-     "ename": "BadRequestError",
-     "evalue": "Error code: 400 - {'error': {'message': \"This model's maximum context length is 8192 tokens. However, your messages resulted in 9289 tokens. Please reduce the length of the messages.\", 'type': 'invalid_request_error', 'param': 'messages', 'code': 'context_length_exceeded'}}",
-     "output_type": "error",
-     "traceback": [
-      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
-      "\u001b[0;31mBadRequestError\u001b[0m                           Traceback (most recent call last)",
-      "Cell \u001b[0;32mIn[5], line 9\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mopenai\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m OpenAI\n\u001b[1;32m      4\u001b[0m client \u001b[38;5;241m=\u001b[39m OpenAI(\n\u001b[1;32m      5\u001b[0m     \u001b[38;5;66;03m# This is the default and can be omitted\u001b[39;00m\n\u001b[1;32m      6\u001b[0m     api_key\u001b[38;5;241m=\u001b[39mos\u001b[38;5;241m.\u001b[39menviron\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mOPENAI_API_KEY\u001b[39m\u001b[38;5;124m\"\u001b[39m),\n\u001b[1;32m      7\u001b[0m )\n\u001b[0;32m----> 9\u001b[0m chat_completion \u001b[38;5;241m=\u001b[39m \u001b[43mclient\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mchat\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcompletions\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcreate\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m     10\u001b[0m \u001b[43m    \u001b[49m\u001b[43mmessages\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m[\u001b[49m\n\u001b[1;32m     11\u001b[0m \u001b[43m        \u001b[49m\u001b[43m{\u001b[49m\n\u001b[1;32m     12\u001b[0m \u001b[43m            \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mrole\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43muser\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m     13\u001b[0m \u001b[43m            \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mcontent\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mprompt\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mformat\u001b[49m\u001b[43m(\u001b[49m\u001b[43mthread\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mthread\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     14\u001b[0m \u001b[43m        \u001b[49m\u001b[43m}\u001b[49m\n\u001b[1;32m     15\u001b[0m \u001b[43m    \u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     16\u001b[0m \u001b[43m    \u001b[49m\u001b[43mmodel\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mgpt-4\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m     17\u001b[0m \u001b[43m)\u001b[49m\n\u001b[1;32m     18\u001b[0m chat_completion\n",
-      "File \u001b[0;32m/media/wassname/SGIronWolf/projects5/rrational/.venv/lib/python3.10/site-packages/openai/_utils/_utils.py:274\u001b[0m, in \u001b[0;36mrequired_args.<locals>.inner.<locals>.wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    272\u001b[0m             msg \u001b[38;5;241m=\u001b[39m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mMissing required argument: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mquote(missing[\u001b[38;5;241m0\u001b[39m])\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    273\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mTypeError\u001b[39;00m(msg)\n\u001b[0;32m--> 274\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
-      "File \u001b[0;32m/media/wassname/SGIronWolf/projects5/rrational/.venv/lib/python3.10/site-packages/openai/resources/chat/completions.py:668\u001b[0m, in \u001b[0;36mCompletions.create\u001b[0;34m(self, messages, model, frequency_penalty, function_call, functions, logit_bias, logprobs, max_tokens, n, parallel_tool_calls, presence_penalty, response_format, seed, service_tier, stop, stream, stream_options, temperature, tool_choice, tools, top_logprobs, top_p, user, extra_headers, extra_query, extra_body, timeout)\u001b[0m\n\u001b[1;32m    633\u001b[0m \u001b[38;5;129m@required_args\u001b[39m([\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmessages\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmodel\u001b[39m\u001b[38;5;124m\"\u001b[39m], [\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmessages\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmodel\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mstream\u001b[39m\u001b[38;5;124m\"\u001b[39m])\n\u001b[1;32m    634\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mcreate\u001b[39m(\n\u001b[1;32m    635\u001b[0m     \u001b[38;5;28mself\u001b[39m,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    665\u001b[0m     timeout: \u001b[38;5;28mfloat\u001b[39m \u001b[38;5;241m|\u001b[39m httpx\u001b[38;5;241m.\u001b[39mTimeout \u001b[38;5;241m|\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;241m|\u001b[39m NotGiven \u001b[38;5;241m=\u001b[39m NOT_GIVEN,\n\u001b[1;32m    666\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m ChatCompletion \u001b[38;5;241m|\u001b[39m Stream[ChatCompletionChunk]:\n\u001b[1;32m    667\u001b[0m     validate_response_format(response_format)\n\u001b[0;32m--> 668\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_post\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    669\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m/chat/completions\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m    670\u001b[0m \u001b[43m        \u001b[49m\u001b[43mbody\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmaybe_transform\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    671\u001b[0m \u001b[43m            \u001b[49m\u001b[43m{\u001b[49m\n\u001b[1;32m    672\u001b[0m \u001b[43m                \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mmessages\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mmessages\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    673\u001b[0m \u001b[43m                \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mmodel\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mmodel\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    674\u001b[0m \u001b[43m                \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mfrequency_penalty\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mfrequency_penalty\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    675\u001b[0m \u001b[43m                \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mfunction_call\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mfunction_call\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    676\u001b[0m \u001b[43m                \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mfunctions\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mfunctions\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    677\u001b[0m \u001b[43m                \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mlogit_bias\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mlogit_bias\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    678\u001b[0m \u001b[43m                \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mlogprobs\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mlogprobs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    679\u001b[0m \u001b[43m                \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mmax_tokens\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mmax_tokens\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    680\u001b[0m \u001b[43m                \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mn\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    681\u001b[0m \u001b[43m                \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mparallel_tool_calls\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mparallel_tool_calls\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    682\u001b[0m \u001b[43m                \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mpresence_penalty\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mpresence_penalty\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    683\u001b[0m \u001b[43m                \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mresponse_format\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mresponse_format\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    684\u001b[0m \u001b[43m                \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mseed\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mseed\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    685\u001b[0m \u001b[43m                \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mservice_tier\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mservice_tier\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    686\u001b[0m \u001b[43m                \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mstop\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mstop\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    687\u001b[0m \u001b[43m                \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mstream\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mstream\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    688\u001b[0m \u001b[43m                \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mstream_options\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mstream_options\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    689\u001b[0m \u001b[43m                \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mtemperature\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mtemperature\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    690\u001b[0m \u001b[43m                \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mtool_choice\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mtool_choice\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    691\u001b[0m \u001b[43m                \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mtools\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mtools\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    692\u001b[0m \u001b[43m                \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mtop_logprobs\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mtop_logprobs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    693\u001b[0m \u001b[43m                \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mtop_p\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mtop_p\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    694\u001b[0m \u001b[43m                \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43muser\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43muser\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    695\u001b[0m \u001b[43m            \u001b[49m\u001b[43m}\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    696\u001b[0m \u001b[43m            \u001b[49m\u001b[43mcompletion_create_params\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mCompletionCreateParams\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    697\u001b[0m \u001b[43m        \u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    698\u001b[0m \u001b[43m        \u001b[49m\u001b[43moptions\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmake_request_options\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    699\u001b[0m \u001b[43m            \u001b[49m\u001b[43mextra_headers\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mextra_headers\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mextra_query\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mextra_query\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mextra_body\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mextra_body\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtimeout\u001b[49m\n\u001b[1;32m    700\u001b[0m \u001b[43m        \u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    701\u001b[0m \u001b[43m        \u001b[49m\u001b[43mcast_to\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mChatCompletion\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    702\u001b[0m \u001b[43m        \u001b[49m\u001b[43mstream\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstream\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01mor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m    703\u001b[0m \u001b[43m        \u001b[49m\u001b[43mstream_cls\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mStream\u001b[49m\u001b[43m[\u001b[49m\u001b[43mChatCompletionChunk\u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    704\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n",
-      "File \u001b[0;32m/media/wassname/SGIronWolf/projects5/rrational/.venv/lib/python3.10/site-packages/openai/_base_client.py:1260\u001b[0m, in \u001b[0;36mSyncAPIClient.post\u001b[0;34m(self, path, cast_to, body, options, files, stream, stream_cls)\u001b[0m\n\u001b[1;32m   1246\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mpost\u001b[39m(\n\u001b[1;32m   1247\u001b[0m     \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m   1248\u001b[0m     path: \u001b[38;5;28mstr\u001b[39m,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m   1255\u001b[0m     stream_cls: \u001b[38;5;28mtype\u001b[39m[_StreamT] \u001b[38;5;241m|\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[1;32m   1256\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m ResponseT \u001b[38;5;241m|\u001b[39m _StreamT:\n\u001b[1;32m   1257\u001b[0m     opts \u001b[38;5;241m=\u001b[39m FinalRequestOptions\u001b[38;5;241m.\u001b[39mconstruct(\n\u001b[1;32m   1258\u001b[0m         method\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mpost\u001b[39m\u001b[38;5;124m\"\u001b[39m, url\u001b[38;5;241m=\u001b[39mpath, json_data\u001b[38;5;241m=\u001b[39mbody, files\u001b[38;5;241m=\u001b[39mto_httpx_files(files), \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39moptions\n\u001b[1;32m   1259\u001b[0m     )\n\u001b[0;32m-> 1260\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m cast(ResponseT, \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcast_to\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mopts\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mstream\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstream\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mstream_cls\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstream_cls\u001b[49m\u001b[43m)\u001b[49m)\n",
-      "File \u001b[0;32m/media/wassname/SGIronWolf/projects5/rrational/.venv/lib/python3.10/site-packages/openai/_base_client.py:937\u001b[0m, in \u001b[0;36mSyncAPIClient.request\u001b[0;34m(self, cast_to, options, remaining_retries, stream, stream_cls)\u001b[0m\n\u001b[1;32m    928\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mrequest\u001b[39m(\n\u001b[1;32m    929\u001b[0m     \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m    930\u001b[0m     cast_to: Type[ResponseT],\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    935\u001b[0m     stream_cls: \u001b[38;5;28mtype\u001b[39m[_StreamT] \u001b[38;5;241m|\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[1;32m    936\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m ResponseT \u001b[38;5;241m|\u001b[39m _StreamT:\n\u001b[0;32m--> 937\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_request\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    938\u001b[0m \u001b[43m        \u001b[49m\u001b[43mcast_to\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcast_to\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    939\u001b[0m \u001b[43m        \u001b[49m\u001b[43moptions\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43moptions\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    940\u001b[0m \u001b[43m        \u001b[49m\u001b[43mstream\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstream\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    941\u001b[0m \u001b[43m        \u001b[49m\u001b[43mstream_cls\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstream_cls\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    942\u001b[0m \u001b[43m        \u001b[49m\u001b[43mremaining_retries\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mremaining_retries\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    943\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n",
-      "File \u001b[0;32m/media/wassname/SGIronWolf/projects5/rrational/.venv/lib/python3.10/site-packages/openai/_base_client.py:1041\u001b[0m, in \u001b[0;36mSyncAPIClient._request\u001b[0;34m(self, cast_to, options, remaining_retries, stream, stream_cls)\u001b[0m\n\u001b[1;32m   1038\u001b[0m         err\u001b[38;5;241m.\u001b[39mresponse\u001b[38;5;241m.\u001b[39mread()\n\u001b[1;32m   1040\u001b[0m     log\u001b[38;5;241m.\u001b[39mdebug(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mRe-raising status error\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m-> 1041\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_make_status_error_from_response(err\u001b[38;5;241m.\u001b[39mresponse) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m   1043\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_process_response(\n\u001b[1;32m   1044\u001b[0m     cast_to\u001b[38;5;241m=\u001b[39mcast_to,\n\u001b[1;32m   1045\u001b[0m     options\u001b[38;5;241m=\u001b[39moptions,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m   1049\u001b[0m     retries_taken\u001b[38;5;241m=\u001b[39moptions\u001b[38;5;241m.\u001b[39mget_max_retries(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmax_retries) \u001b[38;5;241m-\u001b[39m retries,\n\u001b[1;32m   1050\u001b[0m )\n",
-      "\u001b[0;31mBadRequestError\u001b[0m: Error code: 400 - {'error': {'message': \"This model's maximum context length is 8192 tokens. However, your messages resulted in 9289 tokens. Please reduce the length of the messages.\", 'type': 'invalid_request_error', 'param': 'messages', 'code': 'context_length_exceeded'}}"
-     ]
-    }
-   ],
-   "source": [
-    "import os\n",
-    "from openai import OpenAI\n",
-    "\n",
-    "client = OpenAI(\n",
-    "    # This is the default and can be omitted\n",
-    "    api_key=os.environ.get(\"OPENAI_API_KEY\"),\n",
-    ")\n",
-    "\n",
-    "chat_completion = client.chat.completions.create(\n",
-    "    messages=[\n",
-    "        {\n",
-    "            \"role\": \"user\",\n",
-    "            \"content\": prompt.format(thread=thread),\n",
-    "        }\n",
-    "    ],\n",
-    "    model=\"gpt-3.5-turbo\",\n",
-    ")\n",
-    "chat_completion"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "print(thread)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "print(chat_completion.choices[0].message.content)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": ".venv",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.10.12"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 2
-}
diff --git a/nbs/mjc_003_nonllm.ipynb b/nbs/mjc_003_nonllm.ipynb
deleted file mode 100644
index 5b1409cb..00000000
--- a/nbs/mjc_003_nonllm.ipynb
+++ /dev/null
@@ -1,237 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": 1,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import pandas as pd\n",
-    "import numpy as np\n",
-    "import matplotlib.pyplot as plt\n",
-    "\n",
-    "from pathlib import Path\n",
-    "import json, re, os\n",
-    "from tqdm.auto import tqdm\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 2,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "39"
-      ]
-     },
-     "execution_count": 2,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "fs = sorted(Path('../data/cache2').glob('*.md'))\n",
-    "len(fs)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# Python script to extract links from Markdown using libraries\n",
-    "# import markdown\n",
-    "# from bs4 import BeautifulSoup\n",
-    "\n",
-    "# # now extract links from these\n",
-    "# def extract_links_library(markdown_text):\n",
-    "#     # Convert Markdown to HTML\n",
-    "#     html = markdown.markdown(markdown_text)\n",
-    "    \n",
-    "#     # Parse HTML with BeautifulSoup\n",
-    "#     soup = BeautifulSoup(html, 'html.parser')\n",
-    "    \n",
-    "#     # Find all <a> tags and extract href attributes\n",
-    "#     links = [a['href'] for a in soup.find_all('a', href=True)]\n",
-    "    \n",
-    "#     return links\n",
-    "\n",
-    "def extract_links_re(s:str):\n",
-    "    return re.findall(r'\\[.*?\\]\\((.*?)\\)', s)\n",
-    "\n",
-    "data = []\n",
-    "for f in fs:\n",
-    "    s = f.open().read()\n",
-    "    data += extract_links_re(s)\n",
-    "\n",
-    "# print(s)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 21,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "(946,)"
-      ]
-     },
-     "execution_count": 21,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "df_links = pd.Series(data).value_counts()\n",
-    "df_links.shape"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 23,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "39               https://www.reddit.com/r/rational/wiki\n",
-       "38    https://www.reddit.com/r/rational/search?q=%22...\n",
-       "6     https://www.royalroad.com/fiction/36735/the-pe...\n",
-       "6     https://old.reddit.com/user/DomesticatedDungeo...\n",
-       "5     https://www.royalroad.com/fiction/52324/violen...\n",
-       "                            ...                        \n",
-       "2     https://www.royalroad.com/fiction/58187/player...\n",
-       "2               https://adarkroom.doublespeakgames.com/\n",
-       "2     https://www.royalroad.com/fiction/10286/the-go...\n",
-       "2     https://www.royalroad.com/fiction/4293/the-iro...\n",
-       "2                     https://twigserial.wordpress.com/\n",
-       "Length: 107, dtype: object"
-      ]
-     },
-     "execution_count": 23,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "blocklist = [\n",
-    "#   'reddit',\n",
-    "    'redact', 'pastebin', 'wikipedia', 'docs.google', 'discord','tvtropes.org']\n",
-    "df_links = df_links[~df_links.index.str.contains('|'.join(blocklist), regex=True)]\n",
-    "df_links = df_links[df_links.index.str.startswith('http')]\n",
-    "df_links = df_links[df_links > 1]\n",
-    "pd.Series(df_links.index, index=df_links.values)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "TODO\n",
-    "- [ ] dedup\n",
-    "- [ ] convert to title"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 6,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "<Axes: ylabel='Frequency'>"
-      ]
-     },
-     "execution_count": 6,
-     "metadata": {},
-     "output_type": "execute_result"
-    },
-    {
-     "data": {
-      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjoAAAGdCAYAAAAbudkLAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAiKElEQVR4nO3de3BU9f3/8deGkHAxhEskIQQIVcCJSKIhZFJlCpKKgUGEtkOrSERKqy4tGq0TplOoY+cb1JJB6hmxKkSmU6G0Sq0UKgYEtShJICBGEZRLgFygSEKiBNg9vz/8ur9vykXYnM3ZfPb5mNkZ95zjyXvPRvOcs2d3PbZt2wIAADBQlNsDAAAAhAqhAwAAjEXoAAAAYxE6AADAWIQOAAAwFqEDAACMRegAAABjEToAAMBY0W4P4Da/36+jR48qLi5OHo/H7XEAAMBlsG1bp06dUnJysqKiLn7eJuJD5+jRoxowYIDbYwAAgCBUV1crJSXlousjPnTi4uIkfX2gevTo4fI0AADgcjQ2NmrAgAGBv+MXE/Gh883LVT169CB0AADoYL7tshMuRgYAAMYidAAAgLEIHQAAYCxCBwAAGIvQAQAAxiJ0AACAsQgdAABgLEIHAAAYi9ABAADGInQAAICxCB0AAGCsiA0dy7KUlpamrKwst0cBAAAh4rFt23Z7CDc1NjYqPj5eDQ0NfKknAAAdxOX+/Y74by8PpdTCtY7s58DCiY7sBwCASBOxL10BAADzEToAAMBYhA4AADAWoQMAAIxF6AAAAGMROgAAwFiEDgAAMBahAwAAjEXoAAAAYxE6AADAWIQOAAAwFqEDAACMRegAAABjEToAAMBYhA4AADAWoQMAAIxF6AAAAGMROgAAwFiEDgAAMBahAwAAjEXoAAAAYxE6AADAWB0+dE6ePKmRI0cqIyNDw4cP1wsvvOD2SAAAIExEuz1AW8XFxWnLli3q1q2bmpubNXz4cE2dOlV9+vRxezQAAOCyDn9Gp1OnTurWrZskqaWlRbZty7Ztl6cCAADhwPXQ2bJliyZNmqTk5GR5PB6tWbPmvG0sy1Jqaqq6dOmi7Oxsbdu2rdX6kydPKj09XSkpKfrVr36lhISEdpoeAACEM9dDp7m5Wenp6bIs64LrV61apYKCAi1YsEDbt29Xenq6xo8fr/r6+sA2PXv21M6dO7V//379+c9/Vl1dXXuNDwAAwpjroZOXl6ff/e53mjJlygXXFxcXa/bs2Zo5c6bS0tK0dOlSdevWTcuWLTtv28TERKWnp+udd9656M9raWlRY2NjqxsAADCT66FzKWfOnFFFRYVyc3MDy6KiopSbm6utW7dKkurq6nTq1ClJUkNDg7Zs2aJhw4ZddJ9FRUWKj48P3AYMGBDaBwEAAFwT1qFz/Phx+Xw+JSYmtlqemJio2tpaSdLBgwc1evRopaena/To0frFL36hG2644aL7nDdvnhoaGgK36urqkD4GAADgng7/9vJRo0apsrLysrePjY1VbGxs6AYCAABhI6zP6CQkJKhTp07nXVxcV1enpKQkl6YCAAAdRViHTkxMjDIzM1VaWhpY5vf7VVpaqpycHBcnAwAAHYHrL101NTVp3759gfv79+9XZWWlevfurYEDB6qgoED5+fkaOXKkRo0apcWLF6u5uVkzZ85s08+1LEuWZcnn87X1IQAAgDDlsV3+GOG3335bY8eOPW95fn6+SkpKJEnPPvusnn76adXW1iojI0NLlixRdna2Iz+/sbFR8fHxamhoUI8ePRzZ5zdSC9c6sp8DCyc6sh8AAExxuX+/XQ8dtxE6AAB0PJf79zusr9EBAABoC0IHAAAYK2JDx7IspaWlKSsry+1RAABAiERs6Hi9XlVVVamsrMztUQAAQIhEbOgAAADzEToAAMBYhA4AADAWoQMAAIwVsaHDu64AADBfxIYO77oCAMB8ERs6AADAfIQOAAAwFqEDAACMRegAAABjEToAAMBYERs6vL0cAADzRWzo8PZyAADMF7GhAwAAzEfoAAAAYxE6AADAWIQOAAAwFqEDAACMRegAAABjRWzo8Dk6AACYL2JDh8/RAQDAfBEbOgAAwHyEDgAAMBahAwAAjEXoAAAAYxE6AADAWIQOAAAwFqEDAACMRegAAABjRWzo8MnIAACYL2JDh09GBgDAfBEbOgAAwHyEDgAAMBahAwAAjEXoAAAAYxE6AADAWIQOAAAwFqEDAACMFe32APh2qYVrHdnPgYUTHdkPAAAdBWd0AACAsQgdAABgLEIHAAAYK2JDhy/1BADAfBEbOnypJwAA5ovY0AEAAOYjdAAAgLEIHQAAYCxCBwAAGIvQAQAAxiJ0AACAsQgdAABgLEIHAAAYi9ABAADGInQAAICxCB0AAGAsQgcAABiL0AEAAMYidAAAgLEIHQAAYCxCBwAAGCtiQ8eyLKWlpSkrK8vtUQAAQIhEbOh4vV5VVVWprKzM7VEAAECIRGzoAAAA8xE6AADAWIQOAAAwFqEDAACMRegAAABjEToAAMBYhA4AADAWoQMAAIxF6AAAAGMROgAAwFiEDgAAMBahAwAAjEXoAAAAYxE6AADAWNFuD4D2k1q41pH9HFg40ZH9AAAQapzRAQAAxiJ0AACAsQgdAABgLEIHAAAYi9ABAADGInQAAICxCB0AAGAsQgcAABirw4dOdXW1xowZo7S0NI0YMUKrV692eyQAABAmOvwnI0dHR2vx4sXKyMhQbW2tMjMzNWHCBHXv3t3t0QAAgMs6fOj069dP/fr1kyQlJSUpISFBJ06cIHQAAID7L11t2bJFkyZNUnJysjwej9asWXPeNpZlKTU1VV26dFF2dra2bdt2wX1VVFTI5/NpwIABIZ4aAAB0BK6HTnNzs9LT02VZ1gXXr1q1SgUFBVqwYIG2b9+u9PR0jR8/XvX19a22O3HihGbMmKE//vGP7TE2AADoAFx/6SovL095eXkXXV9cXKzZs2dr5syZkqSlS5dq7dq1WrZsmQoLCyVJLS0tuvPOO1VYWKjvfve7l/x5LS0tamlpCdxvbGx04FEAAIBw5PoZnUs5c+aMKioqlJubG1gWFRWl3Nxcbd26VZJk27buvfde3Xrrrbrnnnu+dZ9FRUWKj48P3HiZCwAAc4V16Bw/flw+n0+JiYmtlicmJqq2tlaS9N5772nVqlVas2aNMjIylJGRoQ8//PCi+5w3b54aGhoCt+rq6pA+BgAA4B7XX7pqq1tuuUV+v/+yt4+NjVVsbGwIJwIAAOEirM/oJCQkqFOnTqqrq2u1vK6uTklJSS5NBQAAOoqgQufzzz93eo4LiomJUWZmpkpLSwPL/H6/SktLlZOT06Z9W5altLQ0ZWVltXVMAAAQpoIKnWuvvVZjx47Vn/70J50+fbpNAzQ1NamyslKVlZWSpP3796uyslKHDh2SJBUUFOiFF17Qyy+/rI8//lgPPPCAmpubA+/CCpbX61VVVZXKysratB8AABC+ggqd7du3a8SIESooKFBSUpJ+/vOfX/RD/L5NeXm5brzxRt14442Svg6bG2+8UfPnz5ckTZs2Tb///e81f/58ZWRkqLKyUuvXrz/vAmUAAID/5rFt2w72Xz537pxef/11lZSUaP369Ro6dKjuu+8+3XPPPbr66qudnDNkGhsbFR8fr4aGBvXo0cPRfacWrnV0f+HiwMKJbo8AAIhwl/v3u00XI0dHR2vq1KlavXq1nnzySe3bt0+PPvqoBgwYoBkzZqimpqYtuwcAAGiTNoVOeXm5HnzwQfXr10/FxcV69NFH9dlnn2nDhg06evSoJk+e7NScjuNiZAAAzBfUS1fFxcVavny59uzZowkTJuinP/2pJkyYoKio/99Nhw8fVmpqqs6dO+fowE7jpasrx0tXAAC3Xe7f76A+MPC5557Tfffdp3vvvVf9+vW74DZ9+/bVSy+9FMzuAQAAHBFU6Ozdu/dbt4mJiVF+fn4wuwcAAHBEUNfoLF++XKtXrz5v+erVq/Xyyy+3eSgAAAAnBBU6RUVFSkhIOG9537599T//8z9tHgoAAMAJQb10dejQIQ0ePPi85YMGDQp8onG4syxLlmXJ5/O5PUqH49RF1lzUDAAItaDO6PTt21e7du06b/nOnTvVp0+fNg/VHvgKCAAAzBdU6PzkJz/RL3/5S23atEk+n08+n08bN27U3Llz9eMf/9jpGQEAAIIS1EtXTzzxhA4cOKBx48YpOvrrXfj9fs2YMYNrdAAAQNgIKnRiYmK0atUqPfHEE9q5c6e6du2qG264QYMGDXJ6PgAAgKAFFTrfGDp0qIYOHerULAAAAI4KKnR8Pp9KSkpUWlqq+vp6+f3+Vus3btzoyHAAAABtEVTozJ07VyUlJZo4caKGDx8uj8fj9Fwhx9vLAQAwX1Bf6pmQkKAVK1ZowoQJoZipXfGlnu7hc3QAAMG63L/fQb29PCYmRtdee23QwwEAALSHoELnkUce0TPPPKMgTgYBAAC0m6Cu0Xn33Xe1adMmrVu3Ttdff706d+7cav2rr77qyHAAAABtEVTo9OzZU1OmTHF6FgAAAEcFFTrLly93eg4AAADHBXWNjiSdO3dOb731lp5//nmdOnVKknT06FE1NTU5NhwAAEBbBHVG5+DBg7r99tt16NAhtbS06Pvf/77i4uL05JNPqqWlRUuXLnV6TsfxOToAAJgvqDM6c+fO1ciRI/XFF1+oa9eugeVTpkxRaWmpY8OFktfrVVVVlcrKytweBQAAhEhQZ3Teeecd/fvf/1ZMTEyr5ampqTpy5IgjgwEAALRVUGd0/H7/BV/yOXz4sOLi4to8FAAAgBOCCp3bbrtNixcvDtz3eDxqamrSggULjPhaCAAAYIagXrpatGiRxo8fr7S0NJ0+fVp33XWX9u7dq4SEBL3yyitOzwgAABCUoEInJSVFO3fu1MqVK7Vr1y41NTVp1qxZuvvuu1tdnAwAAOCmoEJHkqKjozV9+nQnZwEAAHBUUKGzYsWKS66fMWNGUMMAAAA4KajQmTt3bqv7Z8+e1ZdffqmYmBh169aN0AEAAGEhqHddffHFF61uTU1N2rNnj2655ZYOczGyZVlKS0tTVlaW26MAAIAQ8di2bTu1s/Lyck2fPl2ffPKJU7sMucbGRsXHx6uhoUE9evRwdN+phWsd3R8u7MDCiW6PAABoZ5f79zvoL/W8kOjoaB09etTJXQIAAAQtqGt0Xn/99Vb3bdtWTU2Nnn32Wd18882ODAYAANBWQYXOnXfe2eq+x+PR1VdfrVtvvVWLFi1yYi4AAIA2Cyp0/H6/03MAAAA4ztFrdAAAAMJJUGd0CgoKLnvb4uLiYH4EAABAmwUVOjt27NCOHTt09uxZDRs2TJL06aefqlOnTrrpppsC23k8HmemBAAACEJQoTNp0iTFxcXp5ZdfVq9evSR9/SGCM2fO1OjRo/XII484OiQAAEAwgrpGZ9GiRSoqKgpEjiT16tVLv/vd73jXFQAACBtBhU5jY6OOHTt23vJjx47p1KlTbR4KAADACUGFzpQpUzRz5ky9+uqrOnz4sA4fPqy//e1vmjVrlqZOner0jAAAAEEJ6hqdpUuX6tFHH9Vdd92ls2fPfr2j6GjNmjVLTz/9tKMDhoplWbIsSz6fz+1RAABAiLTpSz2bm5v12WefSZKuueYade/e3bHB2gtf6tnx8aWeABB52uVLPWtqalRTU6MhQ4aoe/fucvCL0AEAANosqND5z3/+o3Hjxmno0KGaMGGCampqJEmzZs3ireUAACBsBBU6Dz/8sDp37qxDhw6pW7dugeXTpk3T+vXrHRsOAACgLYK6GPnNN9/Uv/71L6WkpLRaPmTIEB08eNCRwQAAANoqqDM6zc3Nrc7kfOPEiROKjY1t81AAAABOCCp0Ro8erRUrVgTuezwe+f1+PfXUUxo7dqxjwwEAALRFUC9dPfXUUxo3bpzKy8t15swZPfbYY/roo4904sQJvffee07PCAAAEJSgzugMHz5cn376qW655RZNnjxZzc3Nmjp1qnbs2KFrrrnG6RkBAACCcsVndM6ePavbb79dS5cu1a9//etQzAQAAOCIKz6j07lzZ+3atSsUswAAADgqqJeupk+frpdeesnpWQAAABwV1MXI586d07Jly/TWW28pMzPzvO+4Ki4udmQ4AACAtrii0Pn888+Vmpqq3bt366abbpIkffrpp6228Xg8zk0HAADQBlcUOkOGDFFNTY02bdok6euvfFiyZIkSExNDMhwAAEBbXFHo/Pe3k69bt07Nzc2ODgRcqdTCtY7s58DCiY7sBwAQPoK6GPkb/x0+AAAA4eSKQsfj8Zx3DU5HvSbHsiylpaUpKyvL7VEAAECIXPFLV/fee2/giztPnz6t+++//7x3Xb366qvOTRgiXq9XXq9XjY2Nio+Pd3scAAAQAlcUOvn5+a3uT58+3dFhAAAAnHRFobN8+fJQzQEAAOC4Nl2MDAAAEM4IHQAAYCxCBwAAGIvQAQAAxiJ0AACAsQgdAABgLEIHAAAYi9ABAADGInQAAICxCB0AAGAsQgcAABiL0AEAAMYidAAAgLEIHQAAYCxCBwAAGIvQAQAAxiJ0AACAsQgdAABgLEIHAAAYK9rtAQCEVmrhWkf2c2DhREf2AwDtiTM6AADAWEaEzpQpU9SrVy/98Ic/dHsUAAAQRowInblz52rFihVujwEAAMKMEaEzZswYxcXFuT0GAAAIM66HzpYtWzRp0iQlJyfL4/FozZo1521jWZZSU1PVpUsXZWdna9u2be0/KAAA6HBcD53m5malp6fLsqwLrl+1apUKCgq0YMECbd++Xenp6Ro/frzq6+vbeVIAANDRuP728ry8POXl5V10fXFxsWbPnq2ZM2dKkpYuXaq1a9dq2bJlKiwsvOKf19LSopaWlsD9xsbGKx8aAAB0CK6HzqWcOXNGFRUVmjdvXmBZVFSUcnNztXXr1qD2WVRUpMcff9ypEWEQPm8GAMzj+ktXl3L8+HH5fD4lJia2Wp6YmKja2trA/dzcXP3oRz/SP//5T6WkpFwygubNm6eGhobArbq6OmTzAwAAd4X1GZ3L9dZbb132trGxsYqNjQ3hNAAAIFyE9RmdhIQEderUSXV1da2W19XVKSkpyaWpAABARxHWoRMTE6PMzEyVlpYGlvn9fpWWlionJ8fFyQAAQEfg+ktXTU1N2rdvX+D+/v37VVlZqd69e2vgwIEqKChQfn6+Ro4cqVGjRmnx4sVqbm4OvAsrWJZlybIs+Xy+tj4EoBUuagaA8OF66JSXl2vs2LGB+wUFBZKk/Px8lZSUaNq0aTp27Jjmz5+v2tpaZWRkaP369eddoHylvF6vvF6vGhsbFR8f36Z9AQCA8OR66IwZM0a2bV9ymzlz5mjOnDntNBEAADBFWF+jAwAA0BaEDgAAMFbEho5lWUpLS1NWVpbbowAAgBCJ2NDxer2qqqpSWVmZ26MAAIAQidjQAQAA5iN0AACAsQgdAABgLEIHAAAYK2JDh3ddAQBgvogNHd51BQCA+SI2dAAAgPkIHQAAYCxCBwAAGIvQAQAAxiJ0AACAsSI2dHh7OQAA5ovY0OHt5QAAmC9iQwcAAJiP0AEAAMYidAAAgLEIHQAAYCxCBwAAGIvQAQAAxorY0OFzdAAAMF/Ehg6fowMAgPkiNnQAAID5CB0AAGAsQgcAABiL0AEAAMYidAAAgLEIHQAAYCxCBwAAGIvQAQAAxorY0OGTkQEAMF/Ehg6fjAwAgPkiNnQAAID5CB0AAGAsQgcAABiL0AEAAMYidAAAgLEIHQAAYCxCBwAAGIvQAQAAxiJ0AACAsQgdAABgLEIHAAAYK9rtAdxiWZYsy5LP53N7FKBDSC1c6/YI5zmwcKLbIwAIcxF7Rocv9QQAwHwRGzoAAMB8hA4AADAWoQMAAIxF6AAAAGMROgAAwFiEDgAAMBahAwAAjEXoAAAAYxE6AADAWIQOAAAwFqEDAACMRegAAABjEToAAMBYhA4AADAWoQMAAIxF6AAAAGNFuz2AWyzLkmVZ8vl8bo8CwGWphWsd2c+BhRMd2Q/ah1PPu1PC7ffHlP8uIvaMjtfrVVVVlcrKytweBQAAhEjEhg4AADAfoQMAAIxF6AAAAGMROgAAwFiEDgAAMBahAwAAjEXoAAAAYxE6AADAWIQOAAAwFqEDAACMRegAAABjEToAAMBYhA4AADAWoQMAAIxF6AAAAGMROgAAwFiEDgAAMBahAwAAjEXoAAAAYxE6AADAWIQOAAAwFqEDAACMRegAAABjGRE6b7zxhoYNG6YhQ4boxRdfdHscAAAQJqLdHqCtzp07p4KCAm3atEnx8fHKzMzUlClT1KdPH7dHAwAALuvwZ3S2bdum66+/Xv3799dVV12lvLw8vfnmm26PBQAAwoDrobNlyxZNmjRJycnJ8ng8WrNmzXnbWJal1NRUdenSRdnZ2dq2bVtg3dGjR9W/f//A/f79++vIkSPtMToAAAhzrodOc3Oz0tPTZVnWBdevWrVKBQUFWrBggbZv36709HSNHz9e9fX1Qf28lpYWNTY2troBAAAzuX6NTl5envLy8i66vri4WLNnz9bMmTMlSUuXLtXatWu1bNkyFRYWKjk5udUZnCNHjmjUqFEX3V9RUZEef/xx5x4AECKphWvdHiHshdsxCrd5Diyc6Mh+wu1xAVfC9TM6l3LmzBlVVFQoNzc3sCwqKkq5ubnaunWrJGnUqFHavXu3jhw5oqamJq1bt07jx4+/6D7nzZunhoaGwK26ujrkjwMAALjD9TM6l3L8+HH5fD4lJia2Wp6YmKhPPvlEkhQdHa1FixZp7Nix8vv9euyxxy75jqvY2FjFxsaGdG4AABAewjp0Ltcdd9yhO+64w+0xAABAmAnrl64SEhLUqVMn1dXVtVpeV1enpKQkl6YCAAAdRViHTkxMjDIzM1VaWhpY5vf7VVpaqpycnDbt27IspaWlKSsrq61jAgCAMOX6S1dNTU3at29f4P7+/ftVWVmp3r17a+DAgSooKFB+fr5GjhypUaNGafHixWpubg68CytYXq9XXq9XjY2Nio+Pb+vDAAAAYcj10CkvL9fYsWMD9wsKCiRJ+fn5Kikp0bRp03Ts2DHNnz9ftbW1ysjI0Pr168+7QBkAAOC/uR46Y8aMkW3bl9xmzpw5mjNnTjtNBAAATBHW1+gAAAC0RcSGDhcjAwBgvogNHa/Xq6qqKpWVlbk9CgAACJGIDR0AAGA+QgcAABiL0AEAAMYidAAAgLFc/xwdt1iWJcuydO7cOUlSY2Oj4z/D3/Kl4/sEgMvl1P/X+H9Z+wjF36G2cOp5D9Xj+ma/3/ZZfB7727Yw3OHDhzVgwAC3xwAAAEGorq5WSkrKRddHfOj4/X4dPXpUcXFxOnXqlAYMGKDq6mr16NHD7dEiTmNjI8ffRRx/d3H83cXxd1cwx9+2bZ06dUrJycmKirr4lTgR+9LVN6KiogIl6PF4JEk9evTgF91FHH93cfzdxfF3F8ffXVd6/C/nS7m5GBkAABiL0AEAAMYidP6P2NhYLViwQLGxsW6PEpE4/u7i+LuL4+8ujr+7Qnn8I/5iZAAAYC7O6AAAAGMROgAAwFiEDgAAMBahAwAAjEXo/C/LspSamqouXbooOztb27Ztc3skY23ZskWTJk1ScnKyPB6P1qxZ02q9bduaP3+++vXrp65duyo3N1d79+51Z1jDFBUVKSsrS3Fxcerbt6/uvPNO7dmzp9U2p0+fltfrVZ8+fXTVVVfpBz/4gerq6lya2CzPPfecRowYEfhQtJycHK1bty6wnmPfvhYuXCiPx6OHHnoosIznIHR++9vfyuPxtLpdd911gfWhOvaEjqRVq1apoKBACxYs0Pbt25Wenq7x48ervr7e7dGM1NzcrPT0dFmWdcH1Tz31lJYsWaKlS5fqgw8+UPfu3TV+/HidPn26nSc1z+bNm+X1evX+++9rw4YNOnv2rG677TY1NzcHtnn44Yf1j3/8Q6tXr9bmzZt19OhRTZ061cWpzZGSkqKFCxeqoqJC5eXluvXWWzV58mR99NFHkjj27amsrEzPP/+8RowY0Wo5z0FoXX/99aqpqQnc3n333cC6kB17G/aoUaNsr9cbuO/z+ezk5GS7qKjIxakigyT7tddeC9z3+/12UlKS/fTTTweWnTx50o6NjbVfeeUVFyY0W319vS3J3rx5s23bXx/rzp0726tXrw5s8/HHH9uS7K1bt7o1ptF69eplv/jiixz7dnTq1Cl7yJAh9oYNG+zvfe979ty5c23b5vc/1BYsWGCnp6dfcF0oj33En9E5c+aMKioqlJubG1gWFRWl3Nxcbd261cXJItP+/ftVW1vb6vmIj49XdnY2z0cINDQ0SJJ69+4tSaqoqNDZs2dbHf/rrrtOAwcO5Pg7zOfzaeXKlWpublZOTg7Hvh15vV5NnDix1bGW+P1vD3v37lVycrK+853v6O6779ahQ4ckhfbYR/yXeh4/flw+n0+JiYmtlicmJuqTTz5xaarIVVtbK0kXfD6+WQdn+P1+PfTQQ7r55ps1fPhwSV8f/5iYGPXs2bPVthx/53z44YfKycnR6dOnddVVV+m1115TWlqaKisrOfbtYOXKldq+fbvKysrOW8fvf2hlZ2erpKREw4YNU01NjR5//HGNHj1au3fvDumxj/jQASKV1+vV7t27W71GjtAbNmyYKisr1dDQoL/+9a/Kz8/X5s2b3R4rIlRXV2vu3LnasGGDunTp4vY4EScvLy/wzyNGjFB2drYGDRqkv/zlL+ratWvIfm7Ev3SVkJCgTp06nXdld11dnZKSklyaKnJ9c8x5PkJrzpw5euONN7Rp0yalpKQEliclJenMmTM6efJkq+05/s6JiYnRtddeq8zMTBUVFSk9PV3PPPMMx74dVFRUqL6+XjfddJOio6MVHR2tzZs3a8mSJYqOjlZiYiLPQTvq2bOnhg4dqn379oX09z/iQycmJkaZmZkqLS0NLPP7/SotLVVOTo6Lk0WmwYMHKykpqdXz0djYqA8++IDnwwG2bWvOnDl67bXXtHHjRg0ePLjV+szMTHXu3LnV8d+zZ48OHTrE8Q8Rv9+vlpYWjn07GDdunD788ENVVlYGbiNHjtTdd98d+Geeg/bT1NSkzz77TP369Qvt73+bLmU2xMqVK+3Y2Fi7pKTErqqqsn/2s5/ZPXv2tGtra90ezUinTp2yd+zYYe/YscOWZBcXF9s7duywDx48aNu2bS9cuNDu2bOn/fe//93etWuXPXnyZHvw4MH2V1995fLkHd8DDzxgx8fH22+//bZdU1MTuH355ZeBbe6//3574MCB9saNG+3y8nI7JyfHzsnJcXFqcxQWFtqbN2+29+/fb+/atcsuLCy0PR6P/eabb9q2zbF3w/9915Vt8xyE0iOPPGK//fbb9v79++333nvPzs3NtRMSEuz6+nrbtkN37Amd//WHP/zBHjhwoB0TE2OPGjXKfv/9990eyVibNm2yJZ13y8/Pt23767eY/+Y3v7ETExPt2NhYe9y4cfaePXvcHdoQFzrukuzly5cHtvnqq6/sBx980O7Vq5fdrVs3e8qUKXZNTY17QxvkvvvuswcNGmTHxMTYV199tT1u3LhA5Ng2x94N/x06PAehM23aNLtfv352TEyM3b9/f3vatGn2vn37AutDdew9tm3bbTsnBAAAEJ4i/hodAABgLkIHAAAYi9ABAADGInQAAICxCB0AAGAsQgcAABiL0AEAAMYidAAAgLEIHQAAYCxCBwAAGIvQAQAAxiJ0AACAsf4fhRDEg0e1gFwAAAAASUVORK5CYII=",
-      "text/plain": [
-       "<Figure size 640x480 with 1 Axes>"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    }
-   ],
-   "source": [
-    "df_links.plot.hist(bins=25, logy=True)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 7,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "df_links.to_markdown('../outputs/links2.md')"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [
-    {
-     "ename": "AttributeError",
-     "evalue": "'Series' object has no attribute 'to_html'",
-     "output_type": "error",
-     "traceback": [
-      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
-      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
-      "\u001b[0;32m/tmp/ipykernel_2936525/1764501736.py\u001b[0m in \u001b[0;36m?\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mdf_links\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_html\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'../outputs/links2.html'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
-      "\u001b[0;32m/media/wassname/SGIronWolf/projects5/rrational/.venv/lib/python3.10/site-packages/pandas/core/generic.py\u001b[0m in \u001b[0;36m?\u001b[0;34m(self, name)\u001b[0m\n\u001b[1;32m   6295\u001b[0m             \u001b[0;32mand\u001b[0m \u001b[0mname\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_accessors\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   6296\u001b[0m             \u001b[0;32mand\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_info_axis\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_can_hold_identifiers_and_holds_name\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   6297\u001b[0m         ):\n\u001b[1;32m   6298\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 6299\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mobject\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__getattribute__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
-      "\u001b[0;31mAttributeError\u001b[0m: 'Series' object has no attribute 'to_html'"
-     ]
-    }
-   ],
-   "source": []
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Extra get a llm summary of each link\n",
-    "\n",
-    "Grab all md's that mention a story, ask claude to summarize\n",
-    "\n",
-    "We could also get total karma per mention"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": []
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": ".venv",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.10.12"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 2
-}
diff --git a/nbs/mjc_004_process.ipynb b/nbs/mjc_004_process.ipynb
index fae12501..4c00f754 100644
--- a/nbs/mjc_004_process.ipynb
+++ b/nbs/mjc_004_process.ipynb
@@ -1,70 +1,67 @@
 {
  "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": 1,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import pandas as pd\n",
-    "import numpy as np\n",
-    "import matplotlib.pyplot as plt\n",
-    "\n",
-    "from pathlib import Path\n",
-    "import json, re, os\n",
-    "from tqdm.auto import tqdm\n"
-   ]
-  },
   {
    "cell_type": "code",
    "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
-    "# We will get a list of title from markdown links\n",
-    "from collections import defaultdict\n",
+    "import json\n",
     "import re\n",
-    "titles_md = defaultdict(list)\n",
+    "from pathlib import Path\n",
     "\n",
-    "\n",
-    "title_blocklist = ['patreon', 'here', 'link', 'this', 'delete', 'this one', 'vote', 'image', 'comic explanation', 'this link', 'this thread', 'redact', '(link)', 'this post', 'CLICK THIS LINK', \"here's\", 'wiki', 'report', 'source', 'the wiki', 'reddit - dive into anything', 'top posts', 'a patreon campaign', 'table of contents', 'fanfiction.net', 'this video', 'here.']\n",
-    "                   \n",
-    "\n",
-    "def remember_md_link_titles(markdown_text):\n",
-    "    # Regular expression to match markdown links\n",
-    "    markdown_link_pattern = re.compile(r'\\[([^\\]]+)\\]\\((http[s]?://[^\\)]+)\\)')\n",
-    "\n",
-    "    # Find all markdown links\n",
-    "    for match in markdown_link_pattern.findall(markdown_text):\n",
-    "        title, url = match\n",
-    "        # tidy title of * / _ and remove leading/trailing whitespace\n",
-    "        title = title.strip().strip('*').strip('_')\n",
-    "\n",
-    "        if len(title)<3:\n",
-    "            continue\n",
-    "\n",
-    "        if title.startswith('http'):\n",
-    "            continue\n",
-    "        if title.startswith('^'):\n",
-    "            continue\n",
-    "\n",
-    "        if title.lower() in title_blocklist:\n",
-    "            continue\n",
-    "\n",
-    "\n",
-    "        titles_md[url].append(title)\n",
-    "\n"
+    "import pandas as pd\n",
+    "from tqdm.auto import tqdm"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "10257 threads\n"
+     ]
+    }
+   ],
+   "source": [
+    "fs = sorted(Path(\"../data/json2\").glob(\"*.json\"))\n",
+    "print(f\"{len(fs)} threads\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Extract titles from markdown links"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
    "metadata": {},
    "outputs": [
     {
      "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "f7709599023b4987882d10b664fafb55",
+       "version_major": 2,
+       "version_minor": 0
+      },
       "text/plain": [
-       "10257"
+       "  0%|          | 0/10257 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": [
+       "13739"
       ]
      },
      "execution_count": 3,
@@ -73,229 +70,656 @@
     }
    ],
    "source": [
-    "fs = sorted(Path('../data/json2').glob('*.json'))\n",
-    "len(fs)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 26,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "45067"
-      ]
-     },
-     "execution_count": 26,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "\n",
-    "def extract_links_re(s:str):\n",
-    "    return re.findall(r'\\[.*?\\]\\((.*?)\\)', s)\n",
-    "\n",
-    "\n",
-    "\n",
+    "# We will get a list of title from markdown links\n",
+    "import re\n",
     "from collections import defaultdict\n",
-    "karma = defaultdict(int) # track score\n",
-    "gossip = defaultdict(list) # track number and content of comments... FIXME should include subcomments?\n",
-    "gossip_threads = defaultdict(list)\n",
-    "dates = defaultdict(list)\n",
     "\n",
-    "banned_suffixes = ['jpeg', 'png', 'jpg']\n",
+    "titles_md = defaultdict(list)\n",
     "\n",
-    "data = []\n",
-    "for f in fs:\n",
+    "\n",
+    "title_blocklist = [\n",
+    "    \"patreon\",\n",
+    "    \"here\",\n",
+    "    \"link\",\n",
+    "    \"this\",\n",
+    "    \"delete\",\n",
+    "    \"this one\",\n",
+    "    \"vote\",\n",
+    "    \"image\",\n",
+    "    \"comic explanation\",\n",
+    "    \"this link\",\n",
+    "    \"this thread\",\n",
+    "    \"redact\",\n",
+    "    \"(link)\",\n",
+    "    \"this post\",\n",
+    "    \"CLICK THIS LINK\",\n",
+    "    \"here's\",\n",
+    "    \"wiki\",\n",
+    "    \"report\",\n",
+    "    \"source\",\n",
+    "    \"the wiki\",\n",
+    "    \"reddit - dive into anything\",\n",
+    "    \"top posts\",\n",
+    "    \"a patreon campaign\",\n",
+    "    \"table of contents\",\n",
+    "    \"fanfiction.net\",\n",
+    "    \"this video\",\n",
+    "    \"here.\",\n",
+    "    \"threads\",\n",
+    "    \"code\",\n",
+    "    \"link index\",\n",
+    "    \"some discussions\",\n",
+    "    \"for\",\n",
+    "    \"details\",\n",
+    "    \"pushshift\",\n",
+    "    \"Reddit - Dive into anything\",\n",
+    "    \"table of contents\",\n",
+    "    \"rationalreads\",\n",
+    "    \"calibre\",\n",
+    "    \"goodreads\",\n",
+    "]\n",
+    "\n",
+    "title_should_not_have = [\n",
+    "    \"vote\",\n",
+    "    \"comments\",\n",
+    "    \"link\",\n",
+    "    \"here\",\n",
+    "    \"image\",\n",
+    "    \"chapter\",\n",
+    "    \"reddit\",\n",
+    "    \"wiki\",\n",
+    "    \"pateron\",\n",
+    "    \"source\",\n",
+    "]\n",
+    "\n",
+    "\n",
+    "def remember_md_link_titles(markdown_text):\n",
+    "    # other rules\n",
+    "    # 1. should have space\n",
+    "    # not subreddit?\n",
+    "    # should have capital\n",
+    "\n",
+    "    # Regular expression to match markdown links\n",
+    "    markdown_link_pattern = re.compile(r\"\\[([^\\]]+)\\]\\((http[s]?://[^\\)]+)\\)\")\n",
+    "\n",
+    "    # Find all markdown links\n",
+    "    for match in markdown_link_pattern.findall(markdown_text):\n",
+    "        title, url = match\n",
+    "        # tidy title of * / _ and remove leading/trailing whitespace\n",
+    "        title = title.strip().strip(\"*\").strip(\"_\")\n",
+    "\n",
+    "        if \"r/\" in title:\n",
+    "            continue\n",
+    "\n",
+    "        if \" \" not in title:\n",
+    "            continue\n",
+    "\n",
+    "        if title.lower() == title:\n",
+    "            continue\n",
+    "\n",
+    "        if len(title) < 4:\n",
+    "            continue\n",
+    "\n",
+    "        if title.startswith(\"http\"):\n",
+    "            continue\n",
+    "        if title.startswith(\"^\"):\n",
+    "            continue\n",
+    "\n",
+    "        if title.lower() in title_blocklist:\n",
+    "            continue\n",
+    "\n",
+    "        if any([x in title.lower() for x in title_should_not_have]):\n",
+    "            continue\n",
+    "\n",
+    "        titles_md[url].append(title)\n",
+    "\n",
+    "\n",
+    "for f in tqdm(fs):\n",
     "    s = json.loads(f.open().read())\n",
     "\n",
-    "    for comment in s['comments']:\n",
-    "        text = comment['body']\n",
-    "        links = extract_links_re(text)\n",
-    "        links = [l for l in links if not any(l.endswith(s) for s in banned_suffixes)]\n",
+    "    for comment in s[\"comments\"]:\n",
+    "        text = comment[\"body\"]\n",
     "        remember_md_link_titles(text)\n",
-    "        for l in links:\n",
-    "            karma[l] += comment['score'] / len(links)\n",
-    "            gossip[l].append(comment['body'])\n",
-    "            gossip_threads[l].append('https://reddit.com' + comment['permalink'])\n",
-    "            dates[l].append(s['created_utc'])\n",
-    "        data += links\n",
     "\n",
-    "len(data)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 5,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# data"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 6,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "25797"
-      ]
-     },
-     "execution_count": 6,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
     "len(titles_md)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": null,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "https://www.patreon.com/alexanderwales                                                                                  880.000000\n",
-       "https://redact.dev/home                                                                                                 422.000000\n",
-       "https://archiveofourown.org/works/11478249/chapters/25740126                                                            328.040476\n",
-       "https://www.fanfiction.net/s/10360716/1/The-Metropolitan-Man                                                            300.614797\n",
-       "https://archiveofourown.org/works/6178036/chapters/14154868                                                             291.982143\n",
-       "http://topwebfiction.com/vote.php?for=a-practical-guide-to-evil                                                         257.333333\n",
-       "https://twigserial.wordpress.com/                                                                                       244.894859\n",
-       "https://docs.google.com/spreadsheets/d/1PaLrwVYgxp_SYHtkred7ybpSJPHL88lf4zB0zMKmk1E/edit?usp=sharing                    225.500000\n",
-       "https://redact.dev                                                                                                      214.000000\n",
-       "https://www.fictionpress.com/s/2961893/1/Mother-of-Learning                                                             209.998729\n",
-       "https://www.fanfiction.net/s/10677106/1/Seventh-Horcrux                                                                 207.737103\n",
-       "http://unsongbook.com/                                                                                                  196.986742\n",
-       "https://www.royalroad.com/fiction/40182/only-villains-do-that                                                           195.305556\n",
-       "https://discord.gg/sM99CF3                                                                                              166.628175\n",
-       "https://forums.sufficientvelocity.com/threads/marked-for-death-a-rational-naruto-quest.24481/                           165.971780\n",
-       "https://www.fanfiction.net/s/5193644/1/Time-Braid                                                                       163.726136\n",
-       "https://docs.google.com/document/d/1EUSMDHdRdbvQJii5uoSezbjtvJpxdF6Da8zqvuW42bg/edit?usp=sharing                        161.437698\n",
-       "https://www.royalroad.com/fiction/25225/delve                                                                           159.377273\n",
-       "https://parahumans.wordpress.com/                                                                                       159.085083\n",
-       "https://palewebserial.wordpress.com/about/                                                                              155.833333\n",
-       "https://wertifloke.wordpress.com/2015/01/25/chapter-1/                                                                  152.853878\n",
-       "https://github.com/trambelus/UserSim                                                                                    149.162162\n",
-       "https://forums.spacebattles.com/threads/purple-days-asoiaf-joffrey-timeloop-au.450894/                                  148.886905\n",
-       "https://www.royalroad.com/fiction/26727/arkendrithyst                                                                   148.280556\n",
-       "/r/User_Simulator                                                                                                       148.162162\n",
-       "https://www.fanfiction.net/s/9658524/1/Branches-on-the-Tree-of-Time                                                     147.401810\n",
-       "https://forums.sufficientvelocity.com/threads/dungeon-keeper-ami-sailor-moon-dungeon-keeper-story-only-thread.30066/    138.841919\n",
-       "https://www.reddit.com/r/rational/wiki/index                                                                            137.023810\n",
-       "https://archiveofourown.org/works/20177950/chapters/47807593                                                            135.342857\n",
-       "https://thefifthdefiance.com/2015/11/02/introduction/                                                                   134.500000\n",
-       "https://patreon.com/alexanderwales                                                                                      131.000000\n",
-       "https://www.royalroad.com/fiction/21410/super-minion                                                                    129.055051\n",
-       "http://yudkowsky.tumblr.com/writing                                                                                     126.783333\n",
-       "https://archiveofourown.org/works/9402014/chapters/21285149                                                             118.233333\n",
-       "https://www.royalroad.com/fiction/81002/the-years-of-apocalypse-a-time-loop-progression                                 117.305556\n",
-       "https://www.fanfiction.net/s/5193644                                                                                    116.619421\n",
-       "http://docfuture.tumblr.com/post/82363551272/fall-of-doc-future-contents                                                115.239524\n",
-       "https://www.royalroad.com/fiction/40290/demesne                                                                         113.904762\n",
-       "https://www.royalroad.com/fiction/32123/just-a-bystander                                                                112.913793\n",
-       "http://archiveofourown.org/works/11478249?view_full_work=true                                                           112.166667\n",
-       "https://forums.spacebattles.com/threads/the-last-angel.244209/                                                          109.506349\n",
-       "/sp                                                                                                                     109.254823\n",
-       "http://rationalreads.com/                                                                                               108.833333\n",
-       "https://qntm.org/ra                                                                                                     108.102381\n",
-       "http://topwebfiction.com/vote.php?for=worth-the-candle                                                                  108.066667\n",
-       "https://imgur.com/t3edwYW                                                                                               108.000000\n",
-       "https://en.wikipedia.org/wiki/Magic_realism                                                                             108.000000\n",
-       "http://luminous.elcenia.com/                                                                                            106.916234\n",
-       "https://practicalguidetoevil.wordpress.com/                                                                             106.265693\n",
-       "https://www.royalroad.com/fiction/50836/to-the-far-shore                                                                105.000000\n",
-       "dtype: float64"
+       "Time Braid                                     94\n",
+       "Mother of Learning                             66\n",
+       "Worth the Candle                               64\n",
+       "With This Ring                                 37\n",
+       "Marked for Death                               34\n",
+       "The Waves Arisen                               34\n",
+       "The Metropolitan Man                           33\n",
+       "What is this?                                  32\n",
+       "A Hero's War                                   30\n",
+       "Friendship is Optimal                          30\n",
+       "Dungeon Keeper Ami                             29\n",
+       "A Practical Guide to Evil                      28\n",
+       "Seventh Horcrux                                28\n",
+       "A Young Woman's Political Record               26\n",
+       "Harry Potter and the Natural 20                26\n",
+       "Purple Days                                    25\n",
+       "The Last Angel                                 25\n",
+       "Branches on the Tree of Time                   25\n",
+       "Super Minion                                   24\n",
+       "Forge of Destiny                               24\n",
+       "The Wandering Inn                              23\n",
+       "The Fall of Doc Future                         23\n",
+       "Hard Reset                                     22\n",
+       "The Games We Play                              22\n",
+       "Shadows of the Limelight                       22\n",
+       "The Erogamer                                   22\n",
+       "More Books                                     20\n",
+       "Harry Potter and the Methods of Rationality    20\n",
+       "Lord of the Mysteries                          19\n",
+       "Fairy Dance of Death                           18\n",
+       "The Daily Grind                                18\n",
+       "Only Villains Do That                          18\n",
+       "Dungeon Crawler Carl                           18\n",
+       "Fine Structure                                 18\n",
+       "The Two Year Emperor                           17\n",
+       "A Bad Name                                     17\n",
+       "Hard Reset 2: Reset Harder                     17\n",
+       "Animorphs: The Reckoning                       16\n",
+       "The Gods are Bastards                          16\n",
+       "Transdimensional Brain Chip                    16\n",
+       "Dreaming of Sunshine                           16\n",
+       "To the Stars                                   16\n",
+       "A Journey of Black and Red                     15\n",
+       "Chunin Exam Day                                15\n",
+       "Significant Digits                             15\n",
+       "The Gods Are Bastards                          15\n",
+       "Crystal Society                                15\n",
+       "Post Human                                     15\n",
+       "Metropolitan Man                               15\n",
+       "Strong Female Protagonist                      14\n",
+       "Name: count, dtype: int64"
       ]
      },
-     "execution_count": 7,
+     "execution_count": 4,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
    "source": [
-    "# QC top links\n",
-    "pd.Series(karma).sort_values(ascending=False).head(50)"
+    "# QC: get top markdown titles\n",
+    "import itertools\n",
+    "\n",
+    "l = sorted(itertools.chain(*titles_md.values()))\n",
+    "pd.Series(l).value_counts().head(50)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Extract links"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
    "metadata": {},
    "outputs": [
     {
      "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "ffa3d559f6174a6d963af58b952f82bf",
+       "version_major": 2,
+       "version_minor": 0
+      },
       "text/plain": [
-       "(32085,)"
+       "  0%|          | 0/10257 [00:00<?, ?it/s]"
       ]
      },
-     "execution_count": 8,
      "metadata": {},
-     "output_type": "execute_result"
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "46353 unique links found\n"
+     ]
     }
    ],
    "source": [
-    "df_links = pd.Series(data).value_counts()\n",
-    "df_links.shape"
+    "def extract_links_re(s: str):\n",
+    "    return re.findall(r\"\\[.*?\\]\\((.*?)\\)\", s)\n",
+    "\n",
+    "\n",
+    "from collections import defaultdict\n",
+    "\n",
+    "# TODO consolidate into a metadata object\n",
+    "link_metadata = {\n",
+    "    \"karma\": defaultdict(int),  # track score\n",
+    "    \"gossip\": defaultdict(list),  # track number and content of comments...\n",
+    "    \"gossip_threads\": defaultdict(list),  # threads\n",
+    "    \"dates\": defaultdict(list),  # comment dates\n",
+    "}\n",
+    "\n",
+    "\n",
+    "def stem_url(u):\n",
+    "    parts = u.strip().rstrip(\"/\").split(\"/\")\n",
+    "    return \"/\".join(parts[:-1])\n",
+    "\n",
+    "\n",
+    "# banned_link_suffixes = ['jpeg', 'png', 'jpg']\n",
+    "\n",
+    "data = []\n",
+    "for f in tqdm(fs):\n",
+    "    s = json.loads(f.open().read())\n",
+    "\n",
+    "    for comment in s[\"comments\"]:\n",
+    "        text = comment[\"body\"]\n",
+    "        links = extract_links_re(text)\n",
+    "        for l in links:\n",
+    "            data.append(\n",
+    "                dict(\n",
+    "                    url=l,\n",
+    "                    score_mean=comment[\"score\"],\n",
+    "                    created_utc=s[\"created_utc\"],\n",
+    "                    comment_url=\"https://reddit.com\" + comment[\"permalink\"],\n",
+    "                    comment=comment[\"body\"],\n",
+    "                    thread_url=stem_url(comment[\"permalink\"]),\n",
+    "                )\n",
+    "            )\n",
+    "\n",
+    "            # karma[l] += comment['score'] / len(links)\n",
+    "            # gossip[l].append(comment['body'])\n",
+    "            # gossip_threads[l].append('https://reddit.com' + comment['permalink'])\n",
+    "            # dates[l].append(s['created_utc'])\n",
+    "        # data += links\n",
+    "\n",
+    "print(f\"{len(data)} unique links found\")"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": null,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "63    https://archiveofourown.org/works/11478249/cha...\n",
-       "61                 https://www.fanfiction.net/s/5193644\n",
-       "60    https://www.fictionpress.com/s/2961893/1/Mothe...\n",
-       "55    https://www.fanfiction.net/s/10360716/1/The-Me...\n",
-       "48                    https://twigserial.wordpress.com/\n",
-       "                            ...                        \n",
-       "2     https://www.fimfiction.net/story/132779/onto-t...\n",
-       "2     https://archiveofourown.org/works/22022296/cha...\n",
-       "2                            http://craphound.com/down/\n",
-       "2     http://www.youngwizards.com/ErrantryWiki/index...\n",
-       "2     https://forums.sufficientvelocity.com/threads/...\n",
-       "Length: 3129, dtype: object"
+       "url\n",
+       "https://github.com/trambelus/UserSim                                                                231\n",
+       "/r/User_Simulator                                                                                   230\n",
+       "/sp                                                                                                 219\n",
+       "#sfw                                                                                                164\n",
+       "#or                                                                                                 164\n",
+       "http://www.np.reddit.com/r/autowikibot/wiki/index                                                   126\n",
+       "http://www.np.reddit.com/r/autowikibot/comments/1x013o/for_moderators_switches_commands_and_css/    126\n",
+       "http://www.np.reddit.com/r/autowikibot/comments/1ux484/ask_wikibot/                                 126\n",
+       "http://topwebfiction.com/vote.php?for=a-practical-guide-to-evil                                     104\n",
+       "https://redact.dev/home                                                                              79\n",
+       "Name: count, dtype: int64"
       ]
      },
-     "execution_count": 9,
+     "execution_count": 14,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>url</th>\n",
+       "      <th>score_mean</th>\n",
+       "      <th>created_utc</th>\n",
+       "      <th>comment_url</th>\n",
+       "      <th>comment</th>\n",
+       "      <th>thread_url</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>https://readcomiconline.li/Comic/Squarriors-2014</td>\n",
+       "      <td>3</td>\n",
+       "      <td>1970-01-01 00:00:01.672671609</td>\n",
+       "      <td>https://reddit.com/r/rational/comments/101em0c...</td>\n",
+       "      <td>Thanks for the recs. You've linked to the seco...</td>\n",
+       "      <td>/r/rational/comments/101em0c/d_monday_request_...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>https://readcomiconline.li/Comic/Alt-Life</td>\n",
+       "      <td>27</td>\n",
+       "      <td>1970-01-01 00:00:01.672671609</td>\n",
+       "      <td>https://reddit.com/r/rational/comments/101em0c...</td>\n",
+       "      <td>I've been reading a lot of comics lately. The ...</td>\n",
+       "      <td>/r/rational/comments/101em0c/d_monday_request_...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>https://readcomiconline.li/Comic/Pride-of-Baghdad</td>\n",
+       "      <td>27</td>\n",
+       "      <td>1970-01-01 00:00:01.672671609</td>\n",
+       "      <td>https://reddit.com/r/rational/comments/101em0c...</td>\n",
+       "      <td>I've been reading a lot of comics lately. The ...</td>\n",
+       "      <td>/r/rational/comments/101em0c/d_monday_request_...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>https://readcomiconline.li/Comic/%C3%86ther-Em...</td>\n",
+       "      <td>27</td>\n",
+       "      <td>1970-01-01 00:00:01.672671609</td>\n",
+       "      <td>https://reddit.com/r/rational/comments/101em0c...</td>\n",
+       "      <td>I've been reading a lot of comics lately. The ...</td>\n",
+       "      <td>/r/rational/comments/101em0c/d_monday_request_...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>https://readcomiconline.li/Comic/Guinea-Pigs</td>\n",
+       "      <td>27</td>\n",
+       "      <td>1970-01-01 00:00:01.672671609</td>\n",
+       "      <td>https://reddit.com/r/rational/comments/101em0c...</td>\n",
+       "      <td>I've been reading a lot of comics lately. The ...</td>\n",
+       "      <td>/r/rational/comments/101em0c/d_monday_request_...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>46348</th>\n",
+       "      <td>https://questionablequesting.com/threads/with-...</td>\n",
+       "      <td>52</td>\n",
+       "      <td>1970-01-01 00:00:01.672066808</td>\n",
+       "      <td>https://reddit.com/r/rational/comments/zvov5c/...</td>\n",
+       "      <td>I thought I would post a \"My favourite reads f...</td>\n",
+       "      <td>/r/rational/comments/zvov5c/d_monday_request_a...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>46349</th>\n",
+       "      <td>https://www.royalroad.com/fiction/40373/vigor-...</td>\n",
+       "      <td>52</td>\n",
+       "      <td>1970-01-01 00:00:01.672066808</td>\n",
+       "      <td>https://reddit.com/r/rational/comments/zvov5c/...</td>\n",
+       "      <td>I thought I would post a \"My favourite reads f...</td>\n",
+       "      <td>/r/rational/comments/zvov5c/d_monday_request_a...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>46350</th>\n",
+       "      <td>https://forums.spacebattles.com/threads/dark-l...</td>\n",
+       "      <td>52</td>\n",
+       "      <td>1970-01-01 00:00:01.672066808</td>\n",
+       "      <td>https://reddit.com/r/rational/comments/zvov5c/...</td>\n",
+       "      <td>I thought I would post a \"My favourite reads f...</td>\n",
+       "      <td>/r/rational/comments/zvov5c/d_monday_request_a...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>46351</th>\n",
+       "      <td>https://forums.spacebattles.com/threads/legend...</td>\n",
+       "      <td>5</td>\n",
+       "      <td>1970-01-01 00:00:01.672066808</td>\n",
+       "      <td>https://reddit.com/r/rational/comments/zvov5c/...</td>\n",
+       "      <td>I read [Legends never die ](https://forums.spa...</td>\n",
+       "      <td>/r/rational/comments/zvov5c/d_monday_request_a...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>46352</th>\n",
+       "      <td>https://tvtropes.org/pmwiki/pmwiki.php/WhatAnI...</td>\n",
+       "      <td>4</td>\n",
+       "      <td>1970-01-01 00:00:01.672066808</td>\n",
+       "      <td>https://reddit.com/r/rational/comments/zvov5c/...</td>\n",
+       "      <td>Any examples of fiction (preferably rational a...</td>\n",
+       "      <td>/r/rational/comments/zvov5c/d_monday_request_a...</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>46353 rows × 6 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                                     url  score_mean  \\\n",
+       "0       https://readcomiconline.li/Comic/Squarriors-2014           3   \n",
+       "1              https://readcomiconline.li/Comic/Alt-Life          27   \n",
+       "2      https://readcomiconline.li/Comic/Pride-of-Baghdad          27   \n",
+       "3      https://readcomiconline.li/Comic/%C3%86ther-Em...          27   \n",
+       "4           https://readcomiconline.li/Comic/Guinea-Pigs          27   \n",
+       "...                                                  ...         ...   \n",
+       "46348  https://questionablequesting.com/threads/with-...          52   \n",
+       "46349  https://www.royalroad.com/fiction/40373/vigor-...          52   \n",
+       "46350  https://forums.spacebattles.com/threads/dark-l...          52   \n",
+       "46351  https://forums.spacebattles.com/threads/legend...           5   \n",
+       "46352  https://tvtropes.org/pmwiki/pmwiki.php/WhatAnI...           4   \n",
+       "\n",
+       "                        created_utc  \\\n",
+       "0     1970-01-01 00:00:01.672671609   \n",
+       "1     1970-01-01 00:00:01.672671609   \n",
+       "2     1970-01-01 00:00:01.672671609   \n",
+       "3     1970-01-01 00:00:01.672671609   \n",
+       "4     1970-01-01 00:00:01.672671609   \n",
+       "...                             ...   \n",
+       "46348 1970-01-01 00:00:01.672066808   \n",
+       "46349 1970-01-01 00:00:01.672066808   \n",
+       "46350 1970-01-01 00:00:01.672066808   \n",
+       "46351 1970-01-01 00:00:01.672066808   \n",
+       "46352 1970-01-01 00:00:01.672066808   \n",
+       "\n",
+       "                                             comment_url  \\\n",
+       "0      https://reddit.com/r/rational/comments/101em0c...   \n",
+       "1      https://reddit.com/r/rational/comments/101em0c...   \n",
+       "2      https://reddit.com/r/rational/comments/101em0c...   \n",
+       "3      https://reddit.com/r/rational/comments/101em0c...   \n",
+       "4      https://reddit.com/r/rational/comments/101em0c...   \n",
+       "...                                                  ...   \n",
+       "46348  https://reddit.com/r/rational/comments/zvov5c/...   \n",
+       "46349  https://reddit.com/r/rational/comments/zvov5c/...   \n",
+       "46350  https://reddit.com/r/rational/comments/zvov5c/...   \n",
+       "46351  https://reddit.com/r/rational/comments/zvov5c/...   \n",
+       "46352  https://reddit.com/r/rational/comments/zvov5c/...   \n",
+       "\n",
+       "                                                 comment  \\\n",
+       "0      Thanks for the recs. You've linked to the seco...   \n",
+       "1      I've been reading a lot of comics lately. The ...   \n",
+       "2      I've been reading a lot of comics lately. The ...   \n",
+       "3      I've been reading a lot of comics lately. The ...   \n",
+       "4      I've been reading a lot of comics lately. The ...   \n",
+       "...                                                  ...   \n",
+       "46348  I thought I would post a \"My favourite reads f...   \n",
+       "46349  I thought I would post a \"My favourite reads f...   \n",
+       "46350  I thought I would post a \"My favourite reads f...   \n",
+       "46351  I read [Legends never die ](https://forums.spa...   \n",
+       "46352  Any examples of fiction (preferably rational a...   \n",
+       "\n",
+       "                                              thread_url  \n",
+       "0      /r/rational/comments/101em0c/d_monday_request_...  \n",
+       "1      /r/rational/comments/101em0c/d_monday_request_...  \n",
+       "2      /r/rational/comments/101em0c/d_monday_request_...  \n",
+       "3      /r/rational/comments/101em0c/d_monday_request_...  \n",
+       "4      /r/rational/comments/101em0c/d_monday_request_...  \n",
+       "...                                                  ...  \n",
+       "46348  /r/rational/comments/zvov5c/d_monday_request_a...  \n",
+       "46349  /r/rational/comments/zvov5c/d_monday_request_a...  \n",
+       "46350  /r/rational/comments/zvov5c/d_monday_request_a...  \n",
+       "46351  /r/rational/comments/zvov5c/d_monday_request_a...  \n",
+       "46352  /r/rational/comments/zvov5c/d_monday_request_a...  \n",
+       "\n",
+       "[46353 rows x 6 columns]"
+      ]
+     },
+     "execution_count": 15,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
    "source": [
-    "blocklist = [\n",
-    "#   'reddit',\n",
-    "    'redact', 'pastebin', 'wikipedia', 'docs.google', 'discord','tvtropes.org', 'ask_wikibot', 'autowiki', 'banned',\n",
+    "df_links = pd.DataFrame(data)\n",
+    "# check for dups\n",
+    "# print(df_links['url'].value_counts().head(10))\n",
+    "\n",
+    "df_links[\"created_utc\"] = pd.to_datetime(df_links[\"created_utc\"])\n",
+    "df_links"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "#### Filter links"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "46353 before blocklist\n"
+     ]
+    },
+    {
+     "ename": "AttributeError",
+     "evalue": "Can only use .str accessor with string values!",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
+      "Cell \u001b[0;32mIn[17], line 45\u001b[0m\n\u001b[1;32m     41\u001b[0m link_allowlist \u001b[38;5;241m=\u001b[39m [\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mhfy\u001b[39m\u001b[38;5;124m\"\u001b[39m]\n\u001b[1;32m     43\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mlen\u001b[39m(df_links)\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m before blocklist\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m     44\u001b[0m df_links \u001b[38;5;241m=\u001b[39m df_links[\n\u001b[0;32m---> 45\u001b[0m     \u001b[38;5;241m~\u001b[39m\u001b[43mdf_links\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mindex\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstr\u001b[49m\u001b[38;5;241m.\u001b[39mcontains(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m|\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;241m.\u001b[39mjoin(link_blocklist), regex\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[1;32m     46\u001b[0m     \u001b[38;5;241m|\u001b[39m df_links\u001b[38;5;241m.\u001b[39mindex\u001b[38;5;241m.\u001b[39mstr\u001b[38;5;241m.\u001b[39mcontains(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m|\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;241m.\u001b[39mjoin(link_allowlist), regex\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[1;32m     47\u001b[0m ]\n\u001b[1;32m     48\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mlen\u001b[39m(df_links)\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m after blocklist\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m     49\u001b[0m df_links \u001b[38;5;241m=\u001b[39m df_links[df_links\u001b[38;5;241m.\u001b[39mindex\u001b[38;5;241m.\u001b[39mstr\u001b[38;5;241m.\u001b[39mstartswith(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mhttp\u001b[39m\u001b[38;5;124m\"\u001b[39m)]\n",
+      "File \u001b[0;32m/media/wassname/SGIronWolf/projects5/rrational/.venv/lib/python3.10/site-packages/pandas/core/accessor.py:224\u001b[0m, in \u001b[0;36mCachedAccessor.__get__\u001b[0;34m(self, obj, cls)\u001b[0m\n\u001b[1;32m    221\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m obj \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m    222\u001b[0m     \u001b[38;5;66;03m# we're accessing the attribute of the class, i.e., Dataset.geo\u001b[39;00m\n\u001b[1;32m    223\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_accessor\n\u001b[0;32m--> 224\u001b[0m accessor_obj \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_accessor\u001b[49m\u001b[43m(\u001b[49m\u001b[43mobj\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    225\u001b[0m \u001b[38;5;66;03m# Replace the property with the accessor object. Inspired by:\u001b[39;00m\n\u001b[1;32m    226\u001b[0m \u001b[38;5;66;03m# https://www.pydanny.com/cached-property.html\u001b[39;00m\n\u001b[1;32m    227\u001b[0m \u001b[38;5;66;03m# We need to use object.__setattr__ because we overwrite __setattr__ on\u001b[39;00m\n\u001b[1;32m    228\u001b[0m \u001b[38;5;66;03m# NDFrame\u001b[39;00m\n\u001b[1;32m    229\u001b[0m \u001b[38;5;28mobject\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;21m__setattr__\u001b[39m(obj, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_name, accessor_obj)\n",
+      "File \u001b[0;32m/media/wassname/SGIronWolf/projects5/rrational/.venv/lib/python3.10/site-packages/pandas/core/strings/accessor.py:191\u001b[0m, in \u001b[0;36mStringMethods.__init__\u001b[0;34m(self, data)\u001b[0m\n\u001b[1;32m    188\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__init__\u001b[39m(\u001b[38;5;28mself\u001b[39m, data) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m    189\u001b[0m     \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mpandas\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mcore\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01marrays\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mstring_\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m StringDtype\n\u001b[0;32m--> 191\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_inferred_dtype \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_validate\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdata\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    192\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_is_categorical \u001b[38;5;241m=\u001b[39m \u001b[38;5;28misinstance\u001b[39m(data\u001b[38;5;241m.\u001b[39mdtype, CategoricalDtype)\n\u001b[1;32m    193\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_is_string \u001b[38;5;241m=\u001b[39m \u001b[38;5;28misinstance\u001b[39m(data\u001b[38;5;241m.\u001b[39mdtype, StringDtype)\n",
+      "File \u001b[0;32m/media/wassname/SGIronWolf/projects5/rrational/.venv/lib/python3.10/site-packages/pandas/core/strings/accessor.py:245\u001b[0m, in \u001b[0;36mStringMethods._validate\u001b[0;34m(data)\u001b[0m\n\u001b[1;32m    242\u001b[0m inferred_dtype \u001b[38;5;241m=\u001b[39m lib\u001b[38;5;241m.\u001b[39minfer_dtype(values, skipna\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[1;32m    244\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m inferred_dtype \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m allowed_types:\n\u001b[0;32m--> 245\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mAttributeError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mCan only use .str accessor with string values!\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m    246\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m inferred_dtype\n",
+      "\u001b[0;31mAttributeError\u001b[0m: Can only use .str accessor with string values!"
+     ]
+    }
+   ],
+   "source": [
+    "banned_link_suffixes = [\"jpeg\", \"png\", \"jpg\"]\n",
+    "\n",
+    "link_blocklist = [\n",
+    "    #   'reddit',\n",
+    "    \"redact\",\n",
+    "    \"pastebin\",\n",
+    "    \"wikipedia\",\n",
+    "    \"docs.google\",\n",
+    "    \"discord\",\n",
+    "    \"tvtropes.org\",\n",
+    "    \"ask_wikibot\",\n",
+    "    \"autowiki\",\n",
+    "    \"banned\",\n",
     "    # pateron.com ?\n",
-    "    'reddit.com/user/', 'xkcd', 'ebay', 'youtubot',\n",
-    "    'reddit.com/r/rational',\n",
-    "    # 'sneakpeekbot', 'RemindMeBot', \n",
+    "    \"reddit.com/user/\",\n",
+    "    \"xkcd\",\n",
+    "    \"ebay\",\n",
+    "    \"youtubot\",\n",
+    "    \"reddit.com/r/rational\",\n",
+    "    # 'sneakpeekbot', 'RemindMeBot',\n",
     "    # 'WikiSummarizerBot', 'bot/', 'Bot/',\n",
-    "    'knowyourmeme.com', 'UserSim',\n",
-    "    'vote.php',\n",
-    "    ]\n",
+    "    \"knowyourmeme.com\",\n",
+    "    \"UserSim\",\n",
+    "    \"vote.php\",\n",
+    "    \"youtube\",\n",
+    "    \"github\",\n",
+    "    \"imgur\",\n",
+    "    \"wikisummarizer\",\n",
+    "    \"mozilla.org\",\n",
+    "    \"reddit.com/message\",\n",
+    "    \"autotldr\",\n",
+    "    \"/top/\",\n",
+    "    \"redd.it\",\n",
+    "    \"reddit.com/u\",\n",
+    "    \"fanficfare\",\n",
+    "    \"bot.com\",\n",
+    "    \"greasyfork\",\n",
+    "]\n",
     "\n",
     "\n",
-    "allowlist = ['hfy']\n",
-    "df_links = df_links[~df_links.index.str.contains('|'.join(blocklist), regex=True) | df_links.index.str.contains('|'.join(allowlist), regex=True)]\n",
-    "df_links = df_links[df_links.index.str.startswith('http')]\n",
+    "link_allowlist = [\"hfy\"]\n",
+    "\n",
+    "print(f\"{len(df_links)} before blocklist\")\n",
+    "df_links = df_links[\n",
+    "    ~df_links.index.str.contains(\"|\".join(link_blocklist), regex=True)\n",
+    "    | df_links.index.str.contains(\"|\".join(link_allowlist), regex=True)\n",
+    "]\n",
+    "print(f\"{len(df_links)} after blocklist\")\n",
+    "df_links = df_links[df_links.index.str.startswith(\"http\")]\n",
+    "for suffix in banned_link_suffixes:\n",
+    "    df_links = df_links[df_links.index.str.endswith(suffix)]\n",
+    "print(f\"{len(df_links)} after suffixes\")\n",
+    "\n",
+    "# must have more than one mention?\n",
     "df_links = df_links[df_links > 1]\n",
+    "print(f\"{len(df_links)} after count\")\n",
     "\n",
-    "# if it has reddit and bot in the title, it's probably a bot\n",
-    "df_links = df_links[~(df_links.index.str.contains('reddit', case=False) & df_links.index.str.contains('bot', case=False))]\n",
+    "# if it has reddit, github, wiki and bot in the title, it's probably a bot\n",
+    "df_links = df_links[\n",
+    "    ~(\n",
+    "        df_links.index.str.contains(\"reddit\", case=False)\n",
+    "        & df_links.index.str.contains(\"bot\", case=False)\n",
+    "    )\n",
+    "]\n",
+    "df_links = df_links[\n",
+    "    ~(\n",
+    "        df_links.index.str.contains(\"wiki\", case=False)\n",
+    "        & df_links.index.str.contains(\"bot\", case=False)\n",
+    "    )\n",
+    "]\n",
+    "df_links = df_links[\n",
+    "    ~(\n",
+    "        df_links.index.str.contains(\"github\", case=False)\n",
+    "        & df_links.index.str.contains(\"bot\", case=False)\n",
+    "    )\n",
+    "]\n",
+    "print(f\"{len(df_links)} after bot\")\n",
     "\n",
     "# QC\n",
     "pd.Series(df_links.index, index=df_links.values)"
@@ -303,60 +727,41 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "<Axes: ylabel='Frequency'>"
-      ]
-     },
-     "execution_count": 10,
-     "metadata": {},
-     "output_type": "execute_result"
-    },
-    {
-     "data": {
-      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjoAAAGdCAYAAAAbudkLAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAjdElEQVR4nO3de3BU5eH/8c9CSCBcwiWSEG6hCtgVSTS3SYGpSGoEigitQ6tIpAxWu1Q0XhqmU6lTvw3KkEHqKfEGkbEVpKNUpeAl3NSihHAT03LRCBFygSIJiRIge35/8GM1BjBsNpzNs+/XzM645xxPPvuQ2Xzm2efscdm2bQsAAMBA7ZwOAAAA0FooOgAAwFgUHQAAYCyKDgAAMBZFBwAAGIuiAwAAjEXRAQAAxqLoAAAAY4U5HcBpXq9Xhw8fVteuXeVyuZyOAwAAmsG2bZ04cUJxcXFq1+7C8zYhX3QOHz6s/v37Ox0DAAD4oaysTP369bvg/pAvOl27dpV0dqC6devmcBoAANAcNTU16t+/v+/v+IWEfNE593FVt27dKDoAALQx37fshMXIAADAWBQdAABgLIoOAAAwFkUHAAAYi6IDAACMRdEBAADGougAAABjhWzRsSxLbrdbKSkpTkcBAACtxGXbtu10CCfV1NQoKipK1dXVfGEgAABtRHP/fofsjA4AADAfRQcAABiLogMAAIxF0QEAAMYK+buXt6b4nNUBOc/n88YH5DwAAIQaZnQAAICxKDoAAMBYFB0AAGAsig4AADAWRQcAABiLogMAAIxF0QEAAMai6AAAAGNRdAAAgLEoOgAAwFghW3Qsy5Lb7VZKSorTUQAAQCsJ2aLj8XhUUlKioqIip6MAAIBWErJFBwAAmI+iAwAAjEXRAQAAxqLoAAAAY1F0AACAsSg6AADAWBQdAABgLIoOAAAwFkUHAAAYi6IDAACMRdEBAADGougAAABjUXQAAICxKDoAAMBYFB0AAGAsig4AADAWRQcAABiLogMAAIxF0QEAAMai6AAAAGNRdAAAgLEoOgAAwFhtvugcP35cycnJSkxM1LBhw/Tcc885HQkAAASJMKcDtFTXrl21adMmRUZGqq6uTsOGDdPkyZPVq1cvp6MBAACHtfkZnfbt2ysyMlKSVF9fL9u2Zdu2w6kAAEAwcLzobNq0SRMmTFBcXJxcLpdWrVrV5BjLshQfH6+OHTsqLS1NW7ZsabT/+PHjSkhIUL9+/fTwww8rOjr6MqUHAADBzPGiU1dXp4SEBFmWdd79K1asUHZ2tubOnatt27YpISFBmZmZqqqq8h3TvXt37dy5U6Wlpfr73/+uysrKyxUfAAAEMceLztixY/X4449r0qRJ592fl5enmTNnavr06XK73crPz1dkZKSWLFnS5NiYmBglJCTovffeu+DPq6+vV01NTaMHAAAwk+NF52JOnTql4uJiZWRk+La1a9dOGRkZ2rx5sySpsrJSJ06ckCRVV1dr06ZNGjp06AXPmZubq6ioKN+jf//+rfsiAACAY4K66Bw9elQNDQ2KiYlptD0mJkYVFRWSpAMHDmjUqFFKSEjQqFGj9Nvf/lbXXnvtBc85Z84cVVdX+x5lZWWt+hoAAIBz2vzl5ampqdqxY0ezj4+IiFBERETrBQIAAEEjqGd0oqOj1b59+yaLiysrKxUbG+tQKgAA0FYEddEJDw9XUlKSCgsLfdu8Xq8KCwuVnp7eonNbliW3262UlJSWxgQAAEHK8Y+uamtrtX//ft/z0tJS7dixQz179tSAAQOUnZ2trKwsJScnKzU1VQsXLlRdXZ2mT5/eop/r8Xjk8XhUU1OjqKiolr4MAAAQhBwvOlu3btXo0aN9z7OzsyVJWVlZKigo0JQpU3TkyBE9+uijqqioUGJiotauXdtkgTIAAMB3uewQv1/CuRmd6upqdevWLaDnjs9ZHZDzfD5vfEDOAwCAKZr79zuo1+gAAAC0RMgWHRYjAwBgvpAtOh6PRyUlJSoqKnI6CgAAaCUhW3QAAID5KDoAAMBYFB0AAGAsig4AADBWyBYdrroCAMB8IVt0uOoKAADzhWzRAQAA5qPoAAAAY1F0AACAsSg6AADAWCFbdLjqCgAA84Vs0eGqKwAAzBeyRQcAAJiPogMAAIxF0QEAAMai6AAAAGNRdAAAgLEoOgAAwFghW3T4Hh0AAMwXskWH79EBAMB8IVt0AACA+Sg6AADAWBQdAABgLIoOAAAwFkUHAAAYi6IDAACMRdEBAADGougAAABjhWzR4ZuRAQAwX8gWHb4ZGQAA84Vs0QEAAOaj6AAAAGNRdAAAgLEoOgAAwFgUHQAAYCyKDgAAMBZFBwAAGIuiAwAAjEXRAQAAxqLoAAAAY4Vs0eFeVwAAmC9kiw73ugIAwHwhW3QAAID5KDoAAMBYFB0AAGAsig4AADBWmNMB8P3ic1YH5DyfzxsfkPMAANBWMKMDAACMRdEBAADGougAAABjUXQAAICxKDoAAMBYFB0AAGAsig4AADAWRQcAABiLogMAAIxF0QEAAMai6AAAAGOFbNGxLEtut1spKSlORwEAAK0kZIuOx+NRSUmJioqKnI4CAABaScgWHQAAYD6KDgAAMBZFBwAAGIuiAwAAjEXRAQAAxqLoAAAAY1F0AACAsSg6AADAWBQdAABgLIoOAAAwVpjTAXD5xOesDsh5Pp83PiDnAQCgtTGjAwAAjEXRAQAAxqLoAAAAY1F0AACAsSg6AADAWBQdAABgLIoOAAAwFkUHAAAYi6IDAACMRdEBAADGavNFp6ysTDfccIPcbreGDx+ulStXOh0JAAAEiTZ/r6uwsDAtXLhQiYmJqqioUFJSksaNG6fOnTs7HQ0AADiszRedPn36qE+fPpKk2NhYRUdH69ixYxQdAADg30dXn332WcACbNq0SRMmTFBcXJxcLpdWrVrV5BjLshQfH6+OHTsqLS1NW7ZsOe+5iouL1dDQoP79+wcsHwAAaLv8KjpXXXWVRo8erZdeekknT55sUYC6ujolJCTIsqzz7l+xYoWys7M1d+5cbdu2TQkJCcrMzFRVVVWj444dO6Zp06bp2WefbVEeAABgDr+KzrZt2zR8+HBlZ2crNjZWv/71ry84y/J9xo4dq8cff1yTJk067/68vDzNnDlT06dPl9vtVn5+viIjI7VkyRLfMfX19br11luVk5OjH/3oRxf9efX19aqpqWn0AAAAZvKr6CQmJuqpp57S4cOHtWTJEpWXl2vkyJEaNmyY8vLydOTIkYCEO3XqlIqLi5WRkfFN4HbtlJGRoc2bN0uSbNvWXXfdpRtvvFF33nnn954zNzdXUVFRvgcfcwEAYK4WXV4eFhamyZMna+XKlXriiSe0f/9+PfTQQ+rfv7+mTZum8vLyFoU7evSoGhoaFBMT02h7TEyMKioqJEkffPCBVqxYoVWrVikxMVGJiYn6+OOPL3jOOXPmqLq62vcoKytrUUYAABC8WnTV1datW7VkyRItX75cnTt31kMPPaQZM2boiy++0GOPPaaJEyf6/ZFWc40cOVJer7fZx0dERCgiIqIVEwEAgGDhV9HJy8vT0qVLtWfPHo0bN07Lli3TuHHj1K7d2QmiQYMGqaCgQPHx8S0KFx0drfbt26uysrLR9srKSsXGxrbo3AAAwHx+fXS1ePFi3X777Tpw4IBWrVqln/70p76Sc07v3r31wgsvtChceHi4kpKSVFhY6Nvm9XpVWFio9PT0Fp0bAACYz68ZnX379n3vMeHh4crKyvre42pra7V//37f89LSUu3YsUM9e/bUgAEDlJ2draysLCUnJys1NVULFy5UXV2dpk+f7k90H8uyZFmWGhoaWnQeAAAQvFy2bduX+j8tXbpUXbp00W233dZo+8qVK/XVV181q+Ccs2HDBo0ePbrJ9qysLBUUFEiSnn76ac2fP18VFRVKTEzUokWLlJaWdqmxz6umpkZRUVGqrq5Wt27dAnLOc+JzVgf0fMHi83njnY4AAAhxzf377VfRGTJkiJ555pkmBWXjxo26++67tWfPnktP7BCKzqWj6AAAnNbcv99+rdE5ePCgBg0a1GT7wIEDdfDgQX9OCQAAEHB+FZ3evXtr165dTbbv3LlTvXr1anEoAACAQPCr6Pzyl7/Ufffdp/Xr16uhoUENDQ1at26dZs+erV/84heBztgqLMuS2+1WSkqK01EAAEAr8WuNzqlTp3TnnXdq5cqVCgs7e+GW1+vVtGnTlJ+fr/Dw8IAHbS2s0bl0rNEBADituX+//bq8PDw8XCtWrNCf/vQn7dy5U506ddK1116rgQMH+h0YAAAg0Fp0C4ghQ4ZoyJAhgcoCAAAQUH4VnYaGBhUUFKiwsFBVVVVN7jW1bt26gIQDAABoCb+KzuzZs1VQUKDx48dr2LBhcrlcgc4FAADQYn4VneXLl+uVV17RuHHjAp3nsuEWEAAAmM+vy8vDw8N11VVXBTrLZeXxeFRSUqKioiKnowAAgFbiV9F58MEH9dRTT8mPK9MBAAAuG78+unr//fe1fv16rVmzRtdcc406dOjQaP+rr74akHAIToH6fiC+jwcA0Nr8Kjrdu3fXpEmTAp0FAAAgoPwqOkuXLg10DgAAgIDza42OJJ05c0bvvvuunnnmGZ04cUKSdPjwYdXW1gYsXGviXlcAAJjPrxmdAwcO6Oabb9bBgwdVX1+vn/zkJ+rataueeOIJ1dfXKz8/P9A5A87j8cjj8fjulQEAAMzj14zO7NmzlZycrC+//FKdOnXybZ80aZIKCwsDFg4AAKAl/JrRee+99/Tvf/+7yV3K4+PjdejQoYAEAwAAaCm/ZnS8Xu95v1H4iy++UNeuXVscCgAAIBD8Kjo33XSTFi5c6HvucrlUW1uruXPntunbQgAAALP49dHVggULlJmZKbfbrZMnT+r222/Xvn37FB0drZdffjnQGQEAAPziV9Hp16+fdu7cqeXLl2vXrl2qra3VjBkzdMcddzRanAwAAOAkv4qOJIWFhWnq1KmBzHJZcfdyAADM51fRWbZs2UX3T5s2za8wlxPfowMAgPn8KjqzZ89u9Pz06dP66quvFB4ersjIyDZRdAAAgPn8uurqyy+/bPSora3Vnj17NHLkSBYjAwCAoOH3va6+a/DgwZo3b16T2R4AAACnBKzoSGcXKB8+fDiQpwQAAPCbX2t0Xn/99UbPbdtWeXm5nn76aY0YMSIgwQAAAFrKr6Jz6623Nnrucrl0xRVX6MYbb9SCBQsCkQsAAKDF/Co6Xq830DkAAAACLqBrdNoSy7LkdruVkpLidBQAANBK/JrRyc7ObvaxeXl5/vyIVscXBgIAYD6/is727du1fft2nT59WkOHDpUk7d27V+3bt9f111/vO87lcgUmJQAAgB/8KjoTJkxQ165d9eKLL6pHjx6Szn6J4PTp0zVq1Cg9+OCDAQ0JAADgD7/W6CxYsEC5ubm+kiNJPXr00OOPP85VVwAAIGj4VXRqamp05MiRJtuPHDmiEydOtDgUAABAIPj10dWkSZM0ffp0LViwQKmpqZKkjz76SA8//LAmT54c0IAwV3zO6oCc5/N54wNyHgCAefwqOvn5+XrooYd0++236/Tp02dPFBamGTNmaP78+QENCAAA4C+/ik5kZKT++te/av78+fr0008lSVdeeaU6d+4c0HAAAAAt0aIvDCwvL1d5ebkGDx6szp07y7btQOUCAABoMb+Kzv/+9z+NGTNGQ4YM0bhx41ReXi5JmjFjBpeWAwCAoOFX0XnggQfUoUMHHTx4UJGRkb7tU6ZM0dq1awMWDgAAoCX8WqPz9ttv66233lK/fv0abR88eLAOHDgQkGAAAAAt5deMTl1dXaOZnHOOHTumiIiIFoe6HLipJwAA5vOr6IwaNUrLli3zPXe5XPJ6vXryySc1evTogIVrTR6PRyUlJSoqKnI6CgAAaCV+fXT15JNPasyYMdq6datOnTqlRx55RJ988omOHTumDz74INAZAQAA/OLXjM6wYcO0d+9ejRw5UhMnTlRdXZ0mT56s7du368orrwx0RgAAAL9c8ozO6dOndfPNNys/P1+///3vWyMTAABAQFzyjE6HDh20a9eu1sgCAAAQUH59dDV16lS98MILgc4CAAAQUH4tRj5z5oyWLFmid999V0lJSU3ucZWXlxeQcAAAAC1xSUXns88+U3x8vHbv3q3rr79ekrR3795Gx7hcrsClAwAAaIFLKjqDBw9WeXm51q9fL+nsLR8WLVqkmJiYVgkHAADQEpe0Rue7dydfs2aN6urqAhoIAAAgUPxajHzOd4sPAABAMLmkouNyuZqswWFNDgAACFaXtEbHtm3dddddvht3njx5Uvfcc0+Tq65effXVwCUEAADw0yUVnaysrEbPp06dGtAwAAAAgXRJRWfp0qWtlQMAACDgWrQYGQAAIJiFbNGxLEtut1spKSlORwEAAK0kZIuOx+NRSUmJioqKnI4CAABaScgWHQAAYD6KDgAAMBZFBwAAGIuiAwAAjEXRAQAAxqLoAAAAY1F0AACAsSg6AADAWJd0ryvAZPE5qwNyns/njQ/IeQAALceMDgAAMBZFBwAAGIuiAwAAjEXRAQAAxqLoAAAAY1F0AACAsSg6AADAWBQdAABgLIoOAAAwFkUHAAAYi6IDAACMRdEBAADGougAAABjGVF0Jk2apB49eujnP/+501EAAEAQMaLozJ49W8uWLXM6BgAACDJhTgcIhBtuuEEbNmxwOgYcEp+z2ukIAIAg5fiMzqZNmzRhwgTFxcXJ5XJp1apVTY6xLEvx8fHq2LGj0tLStGXLlssfFAAAtDmOF526ujolJCTIsqzz7l+xYoWys7M1d+5cbdu2TQkJCcrMzFRVVdVlTgoAANoaxz+6Gjt2rMaOHXvB/Xl5eZo5c6amT58uScrPz9fq1au1ZMkS5eTkXPLPq6+vV319ve95TU3NpYcGAABtguMzOhdz6tQpFRcXKyMjw7etXbt2ysjI0ObNm/06Z25urqKionyP/v37ByouAAAIMkFddI4ePaqGhgbFxMQ02h4TE6OKigrf84yMDN12223617/+pX79+l20BM2ZM0fV1dW+R1lZWavlBwAAznL8o6tAePfdd5t9bEREhCIiIloxDQAACBZBPaMTHR2t9u3bq7KystH2yspKxcbGOpQKAAC0FUFddMLDw5WUlKTCwkLfNq/Xq8LCQqWnp7fo3JZlye12KyUlpaUxAQBAkHL8o6va2lrt37/f97y0tFQ7duxQz549NWDAAGVnZysrK0vJyclKTU3VwoULVVdX57sKy18ej0cej0c1NTWKiopq6csAAABByPGis3XrVo0ePdr3PDs7W5KUlZWlgoICTZkyRUeOHNGjjz6qiooKJSYmau3atU0WKAMAAHyXy7Zt2+kQTjo3o1NdXa1u3boF9NzcmiA0fT5vvNMRAMB4zf37HdRrdAAAAFrC8Y+unGJZlizLUkNDg9NRgPMK1IwgM0wAQlnIzuh4PB6VlJSoqKjI6SgAAKCVhGzRAQAA5qPoAAAAY1F0AACAsSg6AADAWFx1xVVXMBxXbwEIZSE7o8NVVwAAmC9kiw4AADAfRQcAABiLogMAAIxF0QEAAMai6AAAAGOFbNGxLEtut1spKSlORwEAAK0kZIsOl5cDAGC+kC06AADAfBQdAABgLIoOAAAwFkUHAAAYi6IDAACMRdEBAADGCnM6gFMsy5JlWWpoaHA6CgwTn7Pa6QgAgP8vZGd0+B4dAADMF7JFBwAAmI+iAwAAjEXRAQAAxqLoAAAAY1F0AACAsSg6AADAWBQdAABgrJAtOpZlye12KyUlxekoAACglYRs0eELAwEAMF/IFh0AAGA+ig4AADAWRQcAABiLogMAAIxF0QEAAMai6AAAAGNRdAAAgLEoOgAAwFgUHQAAYCyKDgAAMFaY0wGcYlmWLMtSQ0OD01GANiE+Z3XAzvX5vPEBOxcAXEzIzuhwrysAAMwXskUHAACYj6IDAACMRdEBAADGougAAABjUXQAAICxKDoAAMBYFB0AAGAsig4AADAWRQcAABiLogMAAIxF0QEAAMai6AAAAGNRdAAAgLEoOgAAwFgUHQAAYCyKDgAAMFaY0wGcYlmWLMtSQ0OD01GAkBOfs9rpCI18Pm98QM4TqNcVqDzBhvFpW0z59wrZGR2Px6OSkhIVFRU5HQUAALSSkC06AADAfBQdAABgLIoOAAAwFkUHAAAYi6IDAACMRdEBAADGougAAABjUXQAAICxKDoAAMBYFB0AAGAsig4AADAWRQcAABiLogMAAIxF0QEAAMai6AAAAGNRdAAAgLEoOgAAwFgUHQAAYCyKDgAAMBZFBwAAGIuiAwAAjEXRAQAAxjKi6Lz55psaOnSoBg8erOeff97pOAAAIEiEOR2gpc6cOaPs7GytX79eUVFRSkpK0qRJk9SrVy+nowEAAIe1+RmdLVu26JprrlHfvn3VpUsXjR07Vm+//bbTsQAAQBBwvOhs2rRJEyZMUFxcnFwul1atWtXkGMuyFB8fr44dOyotLU1btmzx7Tt8+LD69u3re963b18dOnTockQHAABBzvGiU1dXp4SEBFmWdd79K1asUHZ2tubOnatt27YpISFBmZmZqqqq8uvn1dfXq6amptEDAACYyfE1OmPHjtXYsWMvuD8vL08zZ87U9OnTJUn5+flavXq1lixZopycHMXFxTWawTl06JBSU1MveL7c3Fw99thjgXsBANq8+JzVTkdoJNjyfD5vvNMRglqg/r0CNc7B9vvjNMdndC7m1KlTKi4uVkZGhm9bu3btlJGRoc2bN0uSUlNTtXv3bh06dEi1tbVas2aNMjMzL3jOOXPmqLq62vcoKytr9dcBAACc4fiMzsUcPXpUDQ0NiomJabQ9JiZG//3vfyVJYWFhWrBggUaPHi2v16tHHnnkoldcRUREKCIiolVzAwCA4BDURae5brnlFt1yyy1OxwAAAEEmqD+6io6OVvv27VVZWdloe2VlpWJjYx1KBQAA2oqgLjrh4eFKSkpSYWGhb5vX61VhYaHS09NbdG7LsuR2u5WSktLSmAAAIEg5/tFVbW2t9u/f73teWlqqHTt2qGfPnhowYICys7OVlZWl5ORkpaamauHChaqrq/NdheUvj8cjj8ejmpoaRUVFtfRlAACAIOR40dm6datGjx7te56dnS1JysrKUkFBgaZMmaIjR47o0UcfVUVFhRITE7V27domC5QBAAC+y/Gic8MNN8i27YseM2vWLM2aNesyJQIAAKYI6jU6AAAALRGyRYfFyAAAmC9ki47H41FJSYmKioqcjgIAAFpJyBYdAABgPooOAAAwFkUHAAAYi6IDAACM5fj36DjFsixZlqUzZ85IkmpqagL+M7z1XwX8nABwuQXq/TFQ74mt8X7dEsH2uoLtb09r/XudO+/3fRefy/6+Iwz3xRdfqH///k7HAAAAfigrK1O/fv0uuD/ki47X69Xhw4fVtWtXuVyuRvtqamrUv39/lZWVqVu3bg4lDH6MU/MwTs3DODUP49Q8jFPztMVxsm1bJ06cUFxcnNq1u/BKnJD96Oqcdu3aXbQJSlK3bt3azD+8kxin5mGcmodxah7GqXkYp+Zpa+PUnJtysxgZAAAYi6IDAACMRdG5iIiICM2dO1cRERFORwlqjFPzME7Nwzg1D+PUPIxT85g8TiG/GBkAAJiLGR0AAGAsig4AADAWRQcAABiLogMAAIxF0bkIy7IUHx+vjh07Ki0tTVu2bHE6kqM2bdqkCRMmKC4uTi6XS6tWrWq037ZtPfroo+rTp486deqkjIwM7du3z5mwDsnNzVVKSoq6du2q3r1769Zbb9WePXsaHXPy5El5PB716tVLXbp00c9+9jNVVlY6lNgZixcv1vDhw31fTpaenq41a9b49jNG5zdv3jy5XC7df//9vm2MlfTHP/5RLper0ePqq6/27WeMvnHo0CFNnTpVvXr1UqdOnXTttddq69atvv0mvo9TdC5gxYoVys7O1ty5c7Vt2zYlJCQoMzNTVVVVTkdzTF1dnRISEmRZ1nn3P/nkk1q0aJHy8/P10UcfqXPnzsrMzNTJkycvc1LnbNy4UR6PRx9++KHeeecdnT59WjfddJPq6up8xzzwwAN64403tHLlSm3cuFGHDx/W5MmTHUx9+fXr10/z5s1TcXGxtm7dqhtvvFETJ07UJ598IokxOp+ioiI988wzGj58eKPtjNVZ11xzjcrLy32P999/37ePMTrryy+/1IgRI9ShQwetWbNGJSUlWrBggXr06OE7xsj3cRvnlZqaans8Ht/zhoYGOy4uzs7NzXUwVfCQZL/22mu+516v146NjbXnz5/v23b8+HE7IiLCfvnllx1IGByqqqpsSfbGjRtt2z47Jh06dLBXrlzpO+Y///mPLcnevHmzUzGDQo8ePeznn3+eMTqPEydO2IMHD7bfeecd+8c//rE9e/Zs27b5fTpn7ty5dkJCwnn3MUbf+N3vfmePHDnygvtNfR9nRuc8Tp06peLiYmVkZPi2tWvXThkZGdq8ebODyYJXaWmpKioqGo1ZVFSU0tLSQnrMqqurJUk9e/aUJBUXF+v06dONxunqq6/WgAEDQnacGhoatHz5ctXV1Sk9PZ0xOg+Px6Px48c3GhOJ36dv27dvn+Li4vSDH/xAd9xxhw4ePCiJMfq2119/XcnJybrtttvUu3dvXXfddXruued8+019H6fonMfRo0fV0NCgmJiYRttjYmJUUVHhUKrgdm5cGLNveL1e3X///RoxYoSGDRsm6ew4hYeHq3v37o2ODcVx+vjjj9WlSxdFRETonnvu0WuvvSa3280Yfcfy5cu1bds25ebmNtnHWJ2VlpamgoICrV27VosXL1ZpaalGjRqlEydOMEbf8tlnn2nx4sUaPHiw3nrrLd17772677779OKLL0oy93085O9eDrQWj8ej3bt3N1orgG8MHTpUO3bsUHV1tf7xj38oKytLGzdudDpWUCkrK9Ps2bP1zjvvqGPHjk7HCVpjx471/ffw4cOVlpamgQMH6pVXXlGnTp0cTBZcvF6vkpOT9ec//1mSdN1112n37t3Kz89XVlaWw+laDzM65xEdHa327ds3WZVfWVmp2NhYh1IFt3PjwpidNWvWLL355ptav369+vXr59seGxurU6dO6fjx442OD8VxCg8P11VXXaWkpCTl5uYqISFBTz31FGP0LcXFxaqqqtL111+vsLAwhYWFaePGjVq0aJHCwsIUExPDWJ1H9+7dNWTIEO3fv5/fp2/p06eP3G53o20//OEPfR/zmfo+TtE5j/DwcCUlJamwsNC3zev1qrCwUOnp6Q4mC16DBg1SbGxsozGrqanRRx99FFJjZtu2Zs2apddee03r1q3ToEGDGu1PSkpShw4dGo3Tnj17dPDgwZAap/Pxer2qr69njL5lzJgx+vjjj7Vjxw7fIzk5WXfccYfvvxmrpmpra/Xpp5+qT58+/D59y4gRI5p83cXevXs1cOBASQa/jzu9GjpYLV++3I6IiLALCgrskpIS++6777a7d+9uV1RUOB3NMSdOnLC3b99ub9++3ZZk5+Xl2du3b7cPHDhg27Ztz5s3z+7evbv9z3/+0961a5c9ceJEe9CgQfbXX3/tcPLL595777WjoqLsDRs22OXl5b7HV1995TvmnnvusQcMGGCvW7fO3rp1q52enm6np6c7mPryy8nJsTdu3GiXlpbau3btsnNycmyXy2W//fbbtm0zRhfz7auubJuxsm3bfvDBB+0NGzbYpaWl9gcffGBnZGTY0dHRdlVVlW3bjNE5W7ZsscPCwuz/+7//s/ft22f/7W9/syMjI+2XXnrJd4yJ7+MUnYv4y1/+Yg8YMMAODw+3U1NT7Q8//NDpSI5av369LanJIysry7bts5cm/uEPf7BjYmLsiIgIe8yYMfaePXucDX2ZnW98JNlLly71HfP111/bv/nNb+wePXrYkZGR9qRJk+zy8nLnQjvgV7/6lT1w4EA7PDzcvuKKK+wxY8b4So5tM0YX892iw1jZ9pQpU+w+ffrY4eHhdt++fe0pU6bY+/fv9+1njL7xxhtv2MOGDbMjIiLsq6++2n722Wcb7Tfxfdxl27btzFwSAABA62KNDgAAMBZFBwAAGIuiAwAAjEXRAQAAxqLoAAAAY1F0AACAsSg6AADAWBQdAABgLIoOAAAwFkUHAAAYi6IDAACMRdEBAADG+n+SiwOJ4SfR4AAAAABJRU5ErkJggg==",
-      "text/plain": [
-       "<Figure size 640x480 with 1 Axes>"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    }
-   ],
+   "outputs": [],
    "source": [
-    "df_links.plot.hist(bins=25, logy=True)"
+    "# QC top links\n",
+    "df_links[\"url\"].value_counts().head(10)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 11,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
-    "# df_links.to_markdown('../outputs/links2.md')"
+    "# df_links.plot.hist(bins=25, logy=True)"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "# Get title\n",
+    "# Fetch missing titles\n",
     "\n",
     "This is hard as it's an adverserial web scraping problem, I'll use a mix of methods (from the markdown, requests, url)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 12,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
     "from anycache import anycache\n",
-    "f_cache = Path('../outputs/.anycache')"
+    "\n",
+    "f_cache = Path(\"../outputs/.anycache\")"
    ]
   },
   {
@@ -372,26 +777,160 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 14,
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\"\"\"\n",
+    "HACK temporarily change sys.argv\n",
+    "\"\"\"\n",
+    "import sys\n",
+    "from typing import List\n",
+    "\n",
+    "\n",
+    "class Argv:\n",
+    "    def __init__(self, new_argv: List[str]):\n",
+    "        self.new_argv = new_argv\n",
+    "        self.original_argv = None\n",
+    "\n",
+    "    def __enter__(self):\n",
+    "        self.original_argv = sys.argv[:]\n",
+    "        sys.argv[:] = self.new_argv\n",
+    "        return self\n",
+    "\n",
+    "    def __exit__(self, exc_type, exc_value, traceback):\n",
+    "        sys.argv[:] = self.original_argv"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\"\"\"use lncrawl browser to get titles\n",
+    "\n",
+    "also use a persistent header browser so we can manually solve cloudfare\n",
+    "\"\"\"\n",
+    "\n",
+    "# see https://github.com/dipu-bd/lightnovel-crawler/blob/master/lncrawl/core/app.py#L84\n",
+    "import logging\n",
+    "\n",
+    "from lncrawl.core.browser import Browser\n",
+    "from lncrawl.core.exeptions import ScraperErrorGroup\n",
+    "from lncrawl.core.scraper import Scraper\n",
+    "from readability import Document\n",
+    "\n",
+    "logger = logging.getLogger(__name__)\n",
+    "from lncrawl.core.browser import EC\n",
+    "\n",
+    "# start a browser\n",
+    "with Argv([\"\"]):\n",
+    "    browser = Browser(headless=False)\n",
+    "    browser._init_browser()\n",
+    "    browser._apply_cookies()\n",
+    "\n",
+    "# manually pass cloudfare (YOU NEED TO CLICK!)\n",
+    "url = \"https://www.fanfiction.net/s/10758358/1/What-You-Leave-Behind\"\n",
+    "browser.visit(url)\n",
+    "browser.wait(\"body\")\n",
+    "browser.wait(\n",
+    "    \"#challenge-running\",\n",
+    "    expected_conditon=EC.invisibility_of_element,\n",
+    "    timeout=100,\n",
+    ")\n",
+    "# reader = Document(browser.html)\n",
+    "\n",
+    "\n",
+    "@anycache(f_cache)\n",
+    "def lncrawl_guess_novel_title(url: str) -> str:\n",
+    "    try:\n",
+    "        scraper = Scraper(url)\n",
+    "        response = scraper.get_response(url)\n",
+    "        reader = Document(response.text)\n",
+    "    except ScraperErrorGroup as e:\n",
+    "        if logger.isEnabledFor(logging.DEBUG):\n",
+    "            logger.exception(\"Failed to get response: %s\", e)\n",
+    "        browser.visit(url)\n",
+    "        browser.wait(\"body\")\n",
+    "        reader = Document(browser.html)\n",
+    "    title = reader.short_title()\n",
+    "    assert \"just a moment\" not in title, f\"failed cloudfare when getting {url}\"\n",
+    "    return title\n",
+    "\n",
+    "\n",
+    "# # Test\n",
+    "# urls = [\n",
+    "#     # 'https://www.wuxiaworld.com/novel/overgeared',\n",
+    "#     'https://www.fanfiction.net/s/10758358/1/What-You-Leave-Behind',\n",
+    "#     # 'https://www.royalroad.com/fiction/81002/the-years-of-apocalypse-a-time-loop-progression',\n",
+    "# ]\n",
+    "\n",
+    "# with Argv(['']):\n",
+    "#     for url in urls:\n",
+    "#         r = lncrawl_guess_novel_title(url)\n",
+    "#         print(url)\n",
+    "#         print(r)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import cloudscraper\n",
+    "from bs4 import BeautifulSoup\n",
+    "\n",
+    "session = cloudscraper.create_scraper()\n",
+    "\n",
+    "\n",
+    "def cloudscrape_title(url):\n",
+    "    r = session.get(url, timeout=5, allow_redirects=True)\n",
+    "    r.raise_for_status()\n",
+    "    soup = BeautifulSoup(r.text, \"html.parser\")\n",
+    "    title = soup.title.text.strip()\n",
+    "    assert \"just a moment\" not in title, f\"failed cloudfare when getting {url}\"\n",
+    "    return title"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
     "# dedup using title\n",
     "\n",
     "from pathlib import Path\n",
-    "from bs4 import BeautifulSoup\n",
-    "import cloudscraper\n",
     "\n",
-    "\n",
-    "session = cloudscraper.create_scraper()\n",
+    "# TODO I should look at how fanficfare and lncrawler do this\n",
+    "# https://github.com/dipu-bd/lightnovel-crawler/blob/master/lncrawl/core/app.py#L84\n",
     "\n",
     "\n",
     "def get_title_md(url):\n",
     "    if url in titles_md:\n",
     "        s = pd.Series(titles_md[url])\n",
-    "        return s.value_counts().index[0] # return most common title\n",
+    "        return s.value_counts().index[0]  # return most common title\n",
     "    return url\n",
     "\n",
+    "\n",
+    "def get_slugged_title(url):\n",
+    "    # if that doesn't work, does the end of the url contain a slugged title?\n",
+    "    # e.g. https://www.fanfiction.net/s/5193644/harry-potter-and-the-methods-of-rationality\n",
+    "    slugged_title = url.split(\"/\")[-1].replace(\"-\", \" \")\n",
+    "    if \" \" in slugged_title:\n",
+    "        return slugged_title\n",
+    "    raise ValueError(f\"Failed to get title for {url}\")\n",
+    "\n",
+    "\n",
     "@anycache(f_cache)\n",
     "def get_title(url):\n",
     "    # first check if it's in the cache\n",
@@ -400,282 +939,33 @@
     "\n",
     "    # try scraping from the web\n",
     "    try:\n",
-    "        r = session.get(url, timeout=5, allow_redirects=True)\n",
-    "        r.raise_for_status()\n",
-    "        soup = BeautifulSoup(r.text, 'html.parser')\n",
-    "        title = soup.title.text.strip()\n",
-    "        return title\n",
+    "        return lncrawl_guess_novel_title(url)\n",
     "    except Exception as e:\n",
+    "        print(f\"[lncrawl] Failed to get title for {url} {e}\")\n",
+    "\n",
+    "    try:\n",
+    "        return cloudscrape_title(url)\n",
+    "    except Exception as e:\n",
+    "        print(f\"[cloudscrape] Failed to get title for {url} {e}\")\n",
+    "\n",
+    "    try:\n",
+    "        return get_slugged_title(url)\n",
+    "    except Exception as e:\n",
+    "        print(f\"[slug] Failed to get title for {url} {e}\")\n",
+    "\n",
+    "    return url\n",
     "\n",
-    "        # if that doesn't work, does the end of the url contain a slugged title?\n",
-    "        # e.g. https://www.fanfiction.net/s/5193644/harry-potter-and-the-methods-of-rationality\n",
-    "        slugged_title = url.split('/')[-1].replace('-', ' ')\n",
-    "        if ' ' in slugged_title:\n",
-    "            return slugged_title\n",
-    "    \n",
-    "        raise e  \n",
     "\n",
     "# url = df_links.index[0]\n",
     "# r = get_title(url)\n",
-    "# url, r\n"
+    "# url, r"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 15,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "e2d57613745940d6b8331f07a843dbe5",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "  0%|          | 0/3129 [00:00<?, ?it/s]"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "failed to get title for http://delphicserial.com/2017/10/21/ch01/: HTTPConnectionPool(host='delphicserial.com', port=80): Max retries exceeded with url: /2017/10/21/ch01/ (Caused by NameResolutionError(\"<urllib3.connection.HTTPConnection object at 0x782dce8be560>: Failed to resolve 'delphicserial.com' ([Errno -2] Name or service not known)\"))\n",
-      "failed to get title for http://github.com/lizardsrock4: 404 Client Error: Not Found for url: https://github.com/lizardsrock4\n",
-      "failed to get title for https://np.reddit.com/r/OutOfTheLoop/comments/3fbrg3/is_there_a_reason_why_the_arm_is_always_missing/ctn5gbf/: 429 Client Error: Too Many Requests for url: https://np.reddit.com/r/OutOfTheLoop/comments/3fbrg3/is_there_a_reason_why_the_arm_is_always_missing/ctn5gbf/\n",
-      "failed to get title for https://www.fictionpress.com/s/3198066/1/Fimbulwinter: 403 Client Error: Forbidden for url: https://www.fictionpress.com/s/3198066/1/Fimbulwinter\n",
-      "failed to get title for https://www.fanfiction.net/s/10263221: 403 Client Error: Forbidden for url: https://www.fanfiction.net/s/10263221\n",
-      "failed to get title for https://np.reddit.com/r/noveltranslations/comments/3u1zuf/cn_coiling_dragon_book_21_chapter_44/: 429 Client Error: Too Many Requests for url: https://np.reddit.com/r/noveltranslations/comments/3u1zuf/cn_coiling_dragon_book_21_chapter_44/\n",
-      "failed to get title for https://www.biorxiv.org/content/biorxiv/early/2016/05/26/055699.full.pdf: 'NoneType' object has no attribute 'text'\n",
-      "failed to get title for http://webfictionguide.com/listings/game-by-god-sludge-reformation/: 410 Client Error: Gone for url: http://webfictionguide.com/listings/game-by-god-sludge-reformation/\n",
-      "failed to get title for http://www.epubsbook.com/2015/3975.html: HTTPConnectionPool(host='www.epubsbook.com', port=80): Max retries exceeded with url: /2015/3975.html (Caused by NameResolutionError(\"<urllib3.connection.HTTPConnection object at 0x782dcccf8820>: Failed to resolve 'www.epubsbook.com' ([Errno -2] Name or service not known)\"))\n",
-      "failed to get title for http://www.terminally-incoherent.com/blog/2011/07/08/perfect-imperfection-perfekcyjna-niedoskonalosc-by-jacek-dukaj/: HTTPConnectionPool(host='www.terminally-incoherent.com', port=80): Read timed out. (read timeout=5)\n",
-      "failed to get title for http://gregegan.customer.netspace.net.au/ORTHOGONAL/ORTHOGONAL.html: HTTPConnectionPool(host='gregegan.customer.netspace.net.au', port=80): Max retries exceeded with url: /ORTHOGONAL/ORTHOGONAL.html (Caused by ConnectTimeoutError(<urllib3.connection.HTTPConnection object at 0x782dcccf90f0>, 'Connection to gregegan.customer.netspace.net.au timed out. (connect timeout=5)'))\n"
-     ]
-    },
-    {
-     "data": {
-      "text/html": [
-       "<div>\n",
-       "<style scoped>\n",
-       "    .dataframe tbody tr th:only-of-type {\n",
-       "        vertical-align: middle;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe tbody tr th {\n",
-       "        vertical-align: top;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe thead th {\n",
-       "        text-align: right;\n",
-       "    }\n",
-       "</style>\n",
-       "<table border=\"1\" class=\"dataframe\">\n",
-       "  <thead>\n",
-       "    <tr style=\"text-align: right;\">\n",
-       "      <th></th>\n",
-       "      <th>url</th>\n",
-       "      <th>title</th>\n",
-       "      <th>score</th>\n",
-       "      <th>comments</th>\n",
-       "      <th>num_comments</th>\n",
-       "      <th>comment_urls</th>\n",
-       "      <th>dates</th>\n",
-       "    </tr>\n",
-       "  </thead>\n",
-       "  <tbody>\n",
-       "    <tr>\n",
-       "      <th>0</th>\n",
-       "      <td>https://archiveofourown.org/works/11478249/cha...</td>\n",
-       "      <td>Worth the Candle</td>\n",
-       "      <td>328.040476</td>\n",
-       "      <td>[&gt;&gt;interestingly blend ...\\n\\n&gt;`[g]`enres\\n\\n&gt;...</td>\n",
-       "      <td>63</td>\n",
-       "      <td>[https://reddit.com/r/rational/comments/1c4mxk...</td>\n",
-       "      <td>[1713189611.0, 1503163699.0, 1506215679.0, 150...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>1</th>\n",
-       "      <td>https://www.fanfiction.net/s/5193644</td>\n",
-       "      <td>Time Braid</td>\n",
-       "      <td>116.619421</td>\n",
-       "      <td>[* [*Waves Arisen*](https://wertifloke.wordpre...</td>\n",
-       "      <td>61</td>\n",
-       "      <td>[https://reddit.com/r/rational/comments/11wjn0...</td>\n",
-       "      <td>[1679320813.0, 1682949618.0, 1429816392.0, 143...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>2</th>\n",
-       "      <td>https://www.fictionpress.com/s/2961893/1/Mothe...</td>\n",
-       "      <td>Mother of Learning</td>\n",
-       "      <td>209.998729</td>\n",
-       "      <td>[And [Mother of Learning](https://www.fictionp...</td>\n",
-       "      <td>60</td>\n",
-       "      <td>[https://reddit.com/r/rational/comments/2br02d...</td>\n",
-       "      <td>[1406342508.0, 1417735601.0, 1418670015.0, 141...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>3</th>\n",
-       "      <td>https://www.fanfiction.net/s/10360716/1/The-Me...</td>\n",
-       "      <td>The Metropolitan Man</td>\n",
-       "      <td>300.614797</td>\n",
-       "      <td>[~ [*Death Note: L, Anonymity &amp; Eluding Entrop...</td>\n",
-       "      <td>55</td>\n",
-       "      <td>[https://reddit.com/r/rational/comments/134n9q...</td>\n",
-       "      <td>[1682949618.0, 1687788016.0, 1400177222.0, 140...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>4</th>\n",
-       "      <td>https://twigserial.wordpress.com/</td>\n",
-       "      <td>Twig</td>\n",
-       "      <td>244.894859</td>\n",
-       "      <td>[The protagonist of [Twig](https://twigserial....</td>\n",
-       "      <td>48</td>\n",
-       "      <td>[https://reddit.com/r/rational/comments/17ucgu...</td>\n",
-       "      <td>[1699887610.0, 1725890413.0, 1450408569.0, 147...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>...</th>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>3124</th>\n",
-       "      <td>https://www.fimfiction.net/story/132779/onto-t...</td>\n",
-       "      <td>Onto the Pony Planet</td>\n",
-       "      <td>0.666667</td>\n",
-       "      <td>[So these aren't quite uplifty, for various re...</td>\n",
-       "      <td>2</td>\n",
-       "      <td>[https://reddit.com/r/rational/comments/a4a9ys...</td>\n",
-       "      <td>[1544271356.0, 1568646297.0]</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>3125</th>\n",
-       "      <td>https://archiveofourown.org/works/22022296/cha...</td>\n",
-       "      <td>Blood Crest</td>\n",
-       "      <td>16.333333</td>\n",
-       "      <td>[[Blood Crest](https://archiveofourown.org/wor...</td>\n",
-       "      <td>2</td>\n",
-       "      <td>[https://reddit.com/r/rational/comments/u6dueo...</td>\n",
-       "      <td>[1650290410.0, 1651500009.0]</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>3126</th>\n",
-       "      <td>http://craphound.com/down/</td>\n",
-       "      <td>link to the author's site, where CC-licensed d...</td>\n",
-       "      <td>10.666667</td>\n",
-       "      <td>[[link to the author's site, where CC-licensed...</td>\n",
-       "      <td>2</td>\n",
-       "      <td>[https://reddit.com/r/rational/comments/23tdjs...</td>\n",
-       "      <td>[1398298405.0, 1410439827.0]</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>3127</th>\n",
-       "      <td>http://www.youngwizards.com/ErrantryWiki/index...</td>\n",
-       "      <td>Lone Power</td>\n",
-       "      <td>5.833333</td>\n",
-       "      <td>[Alternate Character Interpretations are one t...</td>\n",
-       "      <td>2</td>\n",
-       "      <td>[https://reddit.com/r/rational/comments/3j2pty...</td>\n",
-       "      <td>[1441022185.0, 1467990282.0]</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>3128</th>\n",
-       "      <td>https://forums.sufficientvelocity.com/threads/...</td>\n",
-       "      <td>Suffer Not</td>\n",
-       "      <td>3.857143</td>\n",
-       "      <td>[u/axiomsofdominion mentioned a character with...</td>\n",
-       "      <td>2</td>\n",
-       "      <td>[https://reddit.com/r/rational/comments/n1cm3n...</td>\n",
-       "      <td>[1619725874.0, 1628517619.0]</td>\n",
-       "    </tr>\n",
-       "  </tbody>\n",
-       "</table>\n",
-       "<p>3129 rows × 7 columns</p>\n",
-       "</div>"
-      ],
-      "text/plain": [
-       "                                                    url  \\\n",
-       "0     https://archiveofourown.org/works/11478249/cha...   \n",
-       "1                  https://www.fanfiction.net/s/5193644   \n",
-       "2     https://www.fictionpress.com/s/2961893/1/Mothe...   \n",
-       "3     https://www.fanfiction.net/s/10360716/1/The-Me...   \n",
-       "4                     https://twigserial.wordpress.com/   \n",
-       "...                                                 ...   \n",
-       "3124  https://www.fimfiction.net/story/132779/onto-t...   \n",
-       "3125  https://archiveofourown.org/works/22022296/cha...   \n",
-       "3126                         http://craphound.com/down/   \n",
-       "3127  http://www.youngwizards.com/ErrantryWiki/index...   \n",
-       "3128  https://forums.sufficientvelocity.com/threads/...   \n",
-       "\n",
-       "                                                  title       score  \\\n",
-       "0                                      Worth the Candle  328.040476   \n",
-       "1                                            Time Braid  116.619421   \n",
-       "2                                    Mother of Learning  209.998729   \n",
-       "3                                  The Metropolitan Man  300.614797   \n",
-       "4                                                  Twig  244.894859   \n",
-       "...                                                 ...         ...   \n",
-       "3124                               Onto the Pony Planet    0.666667   \n",
-       "3125                                        Blood Crest   16.333333   \n",
-       "3126  link to the author's site, where CC-licensed d...   10.666667   \n",
-       "3127                                         Lone Power    5.833333   \n",
-       "3128                                         Suffer Not    3.857143   \n",
-       "\n",
-       "                                               comments  num_comments  \\\n",
-       "0     [>>interestingly blend ...\\n\\n>`[g]`enres\\n\\n>...            63   \n",
-       "1     [* [*Waves Arisen*](https://wertifloke.wordpre...            61   \n",
-       "2     [And [Mother of Learning](https://www.fictionp...            60   \n",
-       "3     [~ [*Death Note: L, Anonymity & Eluding Entrop...            55   \n",
-       "4     [The protagonist of [Twig](https://twigserial....            48   \n",
-       "...                                                 ...           ...   \n",
-       "3124  [So these aren't quite uplifty, for various re...             2   \n",
-       "3125  [[Blood Crest](https://archiveofourown.org/wor...             2   \n",
-       "3126  [[link to the author's site, where CC-licensed...             2   \n",
-       "3127  [Alternate Character Interpretations are one t...             2   \n",
-       "3128  [u/axiomsofdominion mentioned a character with...             2   \n",
-       "\n",
-       "                                           comment_urls  \\\n",
-       "0     [https://reddit.com/r/rational/comments/1c4mxk...   \n",
-       "1     [https://reddit.com/r/rational/comments/11wjn0...   \n",
-       "2     [https://reddit.com/r/rational/comments/2br02d...   \n",
-       "3     [https://reddit.com/r/rational/comments/134n9q...   \n",
-       "4     [https://reddit.com/r/rational/comments/17ucgu...   \n",
-       "...                                                 ...   \n",
-       "3124  [https://reddit.com/r/rational/comments/a4a9ys...   \n",
-       "3125  [https://reddit.com/r/rational/comments/u6dueo...   \n",
-       "3126  [https://reddit.com/r/rational/comments/23tdjs...   \n",
-       "3127  [https://reddit.com/r/rational/comments/3j2pty...   \n",
-       "3128  [https://reddit.com/r/rational/comments/n1cm3n...   \n",
-       "\n",
-       "                                                  dates  \n",
-       "0     [1713189611.0, 1503163699.0, 1506215679.0, 150...  \n",
-       "1     [1679320813.0, 1682949618.0, 1429816392.0, 143...  \n",
-       "2     [1406342508.0, 1417735601.0, 1418670015.0, 141...  \n",
-       "3     [1682949618.0, 1687788016.0, 1400177222.0, 140...  \n",
-       "4     [1699887610.0, 1725890413.0, 1450408569.0, 147...  \n",
-       "...                                                 ...  \n",
-       "3124                       [1544271356.0, 1568646297.0]  \n",
-       "3125                       [1650290410.0, 1651500009.0]  \n",
-       "3126                       [1398298405.0, 1410439827.0]  \n",
-       "3127                       [1441022185.0, 1467990282.0]  \n",
-       "3128                       [1619725874.0, 1628517619.0]  \n",
-       "\n",
-       "[3129 rows x 7 columns]"
-      ]
-     },
-     "execution_count": 15,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
+   "outputs": [],
    "source": [
     "data2 = []\n",
     "for i in tqdm(range(len(df_links))):\n",
@@ -683,14 +973,25 @@
     "    try:\n",
     "        title = get_title(urls)\n",
     "    except Exception as e:\n",
-    "        print(f'failed to get title for {urls}: {e}')\n",
+    "        print(f\"failed to get title for {urls}: {e}\")\n",
     "        title = urls\n",
+    "\n",
     "    comments = gossip[urls]\n",
     "    score = karma[urls]\n",
     "    comment_urls = gossip_threads[urls]\n",
     "    num_comments = len(comments)\n",
     "    created_utc = dates[urls]\n",
-    "    data2.append(dict(url=urls, title=title, score=score, comments=comments, num_comments=num_comments, comment_urls=comment_urls, dates=created_utc))\n",
+    "    data2.append(\n",
+    "        dict(\n",
+    "            url=urls,\n",
+    "            title=title,\n",
+    "            score=score,\n",
+    "            comments=comments,\n",
+    "            num_comments=num_comments,\n",
+    "            comment_urls=comment_urls,\n",
+    "            dates=created_utc,\n",
+    "        )\n",
+    "    )\n",
     "\n",
     "df2 = pd.DataFrame(data2)\n",
     "df2"
@@ -698,646 +999,63 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 16,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "Vote for A Practical Guide to Evil on TopWebFiction!                102\n",
-       "Time Braid                                                           94\n",
-       "Mother of Learning                                                   66\n",
-       "Worm                                                                 66\n",
-       "Worth the Candle                                                     64\n",
-       "Twig                                                                 59\n",
-       "on the #pokengineering channel of the /r/rational Discord server     57\n",
-       "Handy discussion links and previous threads here                     54\n",
-       "Stop Replying                                                        52\n",
-       "Statistics                                                           52\n",
-       "Problems/Bugs?                                                       52\n",
-       "xkcd.com                                                             52\n",
-       "xkcd sub                                                             52\n",
-       "top posts                                                            48\n",
-       "hopefully rational                                                   41\n",
-       "immersive sim                                                        41\n",
-       "With This Ring                                                       37\n",
-       "Pact                                                                 36\n",
-       "Wikipedia                                                            35\n",
-       "Marked for Death                                                     34\n",
-       "The Waves Arisen                                                     34\n",
-       "The Metropolitan Man                                                 33\n",
-       "What is this?                                                        32\n",
-       "Luminosity                                                           32\n",
-       "Pale                                                                 31\n",
-       "Unsong                                                               30\n",
-       "A Hero's War                                                         30\n",
-       "Friendship is Optimal                                                30\n",
-       "Dungeon Keeper Ami                                                   29\n",
-       "A Practical Guide to Evil                                            28\n",
-       "Seventh Horcrux                                                      28\n",
-       "Delve                                                                26\n",
-       "this comment                                                         26\n",
-       "this page                                                            26\n",
-       "CLICK THIS LINK                                                      26\n",
-       "A Young Woman's Political Record                                     26\n",
-       "here.                                                                26\n",
-       "Harry Potter and the Natural 20                                      26\n",
-       "Branches on the Tree of Time                                         25\n",
-       "The Last Angel                                                       25\n",
-       "Purple Days                                                          25\n",
-       "Cordyceps                                                            25\n",
-       "Forge of Destiny                                                     24\n",
-       "Super Minion                                                         24\n",
-       "The Fall of Doc Future                                               23\n",
-       "The Wandering Inn                                                    23\n",
-       "Sanitize                                                             22\n",
-       "Hard Reset                                                           22\n",
-       "these                                                                22\n",
-       "Shadows of the Limelight                                             22\n",
-       "Name: count, dtype: int64"
-      ]
-     },
-     "execution_count": 16,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
+   "outputs": [],
    "source": [
     "# QC: get top markdown titles\n",
     "import itertools\n",
+    "\n",
     "l = sorted(itertools.chain(*titles_md.values()))\n",
     "pd.Series(l).value_counts().head(50)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 17,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/html": [
-       "<div>\n",
-       "<style scoped>\n",
-       "    .dataframe tbody tr th:only-of-type {\n",
-       "        vertical-align: middle;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe tbody tr th {\n",
-       "        vertical-align: top;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe thead th {\n",
-       "        text-align: right;\n",
-       "    }\n",
-       "</style>\n",
-       "<table border=\"1\" class=\"dataframe\">\n",
-       "  <thead>\n",
-       "    <tr style=\"text-align: right;\">\n",
-       "      <th></th>\n",
-       "      <th>score</th>\n",
-       "      <th>num_comments</th>\n",
-       "      <th>comments</th>\n",
-       "      <th>url</th>\n",
-       "      <th>comment_urls</th>\n",
-       "      <th>dates</th>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>title</th>\n",
-       "      <th></th>\n",
-       "      <th></th>\n",
-       "      <th></th>\n",
-       "      <th></th>\n",
-       "      <th></th>\n",
-       "      <th></th>\n",
-       "    </tr>\n",
-       "  </thead>\n",
-       "  <tbody>\n",
-       "    <tr>\n",
-       "      <th>a Patreon campaign</th>\n",
-       "      <td>880.000000</td>\n",
-       "      <td>22</td>\n",
-       "      <td>[I do have a subreddit set up for this over at...</td>\n",
-       "      <td>https://www.patreon.com/alexanderwales</td>\n",
-       "      <td>[https://reddit.com/r/rational/comments/331re9...</td>\n",
-       "      <td>[1429380255.0, 1432400450.0, 1436482846.0, 146...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>Worth the Candle</th>\n",
-       "      <td>661.062143</td>\n",
-       "      <td>108</td>\n",
-       "      <td>[&gt;&gt;interestingly blend ...\\n\\n&gt;`[g]`enres\\n\\n&gt;...</td>\n",
-       "      <td>https://archiveofourown.org/works/11478249/cha...</td>\n",
-       "      <td>[https://reddit.com/r/rational/comments/1c4mxk...</td>\n",
-       "      <td>[1713189611.0, 1503163699.0, 1506215679.0, 150...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>Worm</th>\n",
-       "      <td>414.596255</td>\n",
-       "      <td>78</td>\n",
-       "      <td>[[Kitty Cat Kill Sat](https://www.royalroad.co...</td>\n",
-       "      <td>https://parahumans.wordpress.com\\nhttps://para...</td>\n",
-       "      <td>[https://reddit.com/r/rational/comments/16rsx5...</td>\n",
-       "      <td>[1695650418.0, 1699887610.0, 1706540412.0, 171...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>Twig</th>\n",
-       "      <td>346.983574</td>\n",
-       "      <td>63</td>\n",
-       "      <td>[The protagonist of [Twig](https://twigserial....</td>\n",
-       "      <td>https://twigserial.wordpress.com/about/\\nhttps...</td>\n",
-       "      <td>[https://reddit.com/r/rational/comments/17ucgu...</td>\n",
-       "      <td>[1699887610.0, 1725890413.0, 1450408569.0, 147...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>With This Ring</th>\n",
-       "      <td>329.593551</td>\n",
-       "      <td>59</td>\n",
-       "      <td>[&gt; puts out more chapters faster than any web ...</td>\n",
-       "      <td>https://forums.sufficientvelocity.com/threads/...</td>\n",
-       "      <td>[https://reddit.com/r/rational/comments/52bbjb...</td>\n",
-       "      <td>[1473635989.0, 1481638770.0, 1514051962.0, 152...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>The Metropolitan Man</th>\n",
-       "      <td>319.948130</td>\n",
-       "      <td>57</td>\n",
-       "      <td>[~ [*Death Note: L, Anonymity &amp; Eluding Entrop...</td>\n",
-       "      <td>https://alexanderwales.com/the-metropolitan-ma...</td>\n",
-       "      <td>[https://reddit.com/r/rational/comments/134n9q...</td>\n",
-       "      <td>[1682949618.0, 1687788016.0, 1400177222.0, 140...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>Cordyceps</th>\n",
-       "      <td>311.815476</td>\n",
-       "      <td>41</td>\n",
-       "      <td>[If you like that, be sure to read [CORDYCEPS]...</td>\n",
-       "      <td>http://archiveofourown.org/works/6178036/chapt...</td>\n",
-       "      <td>[https://reddit.com/r/rational/comments/1eqehs...</td>\n",
-       "      <td>[1723471212.0, 1504188940.0, 1504579227.0, 150...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>Mother of Learning</th>\n",
-       "      <td>303.770050</td>\n",
-       "      <td>83</td>\n",
-       "      <td>[And [Mother of Learning](https://www.fictionp...</td>\n",
-       "      <td>https://www.royalroad.com/fiction/21220/mother...</td>\n",
-       "      <td>[https://reddit.com/r/rational/comments/2br02d...</td>\n",
-       "      <td>[1406342508.0, 1417735601.0, 1418670015.0, 141...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>Time Braid</th>\n",
-       "      <td>280.345557</td>\n",
-       "      <td>99</td>\n",
-       "      <td>[* [*Waves Arisen*](https://wertifloke.wordpre...</td>\n",
-       "      <td>https://www.fanfiction.net/s/5193644\\nhttps://...</td>\n",
-       "      <td>[https://reddit.com/r/rational/comments/11wjn0...</td>\n",
-       "      <td>[1679320813.0, 1682949618.0, 1429816392.0, 143...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>Unsong</th>\n",
-       "      <td>268.748647</td>\n",
-       "      <td>40</td>\n",
-       "      <td>[Hey all, welcome back! A few quick notes: fir...</td>\n",
-       "      <td>https://unsongbook.com/\\nhttp://unsongbook.com...</td>\n",
-       "      <td>[https://reddit.com/r/rational/comments/4m0odi...</td>\n",
-       "      <td>[1464778693.0, 1465166191.0, 1465595369.0, 148...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>Seventh Horcrux</th>\n",
-       "      <td>267.237103</td>\n",
-       "      <td>30</td>\n",
-       "      <td>[My favorite harry potter fan fics:\\n\\n- [Seve...</td>\n",
-       "      <td>https://m.fanfiction.net/s/10677106/1/Seventh-...</td>\n",
-       "      <td>[https://reddit.com/r/rational/comments/11qai9...</td>\n",
-       "      <td>[1678716009.0, 1681135215.0, 1436741473.0, 144...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>A Practical Guide to Evil</th>\n",
-       "      <td>235.768723</td>\n",
-       "      <td>48</td>\n",
-       "      <td>[[*Mother of Learning*](https://www.royalroad....</td>\n",
-       "      <td>https://practicalguidetoevil.wordpress.com/sum...</td>\n",
-       "      <td>[https://reddit.com/r/rational/comments/1adxac...</td>\n",
-       "      <td>[1706540412.0, 1464188682.0, 1473635989.0, 148...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>Pale</th>\n",
-       "      <td>205.378788</td>\n",
-       "      <td>33</td>\n",
-       "      <td>[Wildbow's latest webserial, [Pale](https://pa...</td>\n",
-       "      <td>https://palewebserial.wordpress.com/about/\\nht...</td>\n",
-       "      <td>[https://reddit.com/r/rational/comments/111b60...</td>\n",
-       "      <td>[1676300410.0, 1678114814.0, 1691416815.0, 169...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>Pact</th>\n",
-       "      <td>197.108141</td>\n",
-       "      <td>36</td>\n",
-       "      <td>[Let's talk about the horror genre. 'Supernatu...</td>\n",
-       "      <td>https://www.mediamdpodcast.com/pact-audiobook-...</td>\n",
-       "      <td>[https://reddit.com/r/rational/comments/4yjexh...</td>\n",
-       "      <td>[1471619081.0, 1473635989.0, 1546640293.0, 155...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>Only Villains Do That</th>\n",
-       "      <td>195.305556</td>\n",
-       "      <td>19</td>\n",
-       "      <td>[Hey! Sorry for the very-late reply.  I'll try...</td>\n",
-       "      <td>https://www.royalroad.com/fiction/40182/only-v...</td>\n",
-       "      <td>[https://reddit.com/r/rational/comments/13bqbg...</td>\n",
-       "      <td>[1683554474.0, 1684764016.0, 1712584828.0, 172...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>The Waves Arisen</th>\n",
-       "      <td>190.232117</td>\n",
-       "      <td>49</td>\n",
-       "      <td>[* [*Waves Arisen*](https://wertifloke.wordpre...</td>\n",
-       "      <td>https://wertifloke.wordpress.com/about/\\nhttps...</td>\n",
-       "      <td>[https://reddit.com/r/rational/comments/11wjn0...</td>\n",
-       "      <td>[1679320813.0, 1680530414.0, 1681135215.0, 168...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>The Fall of Doc Future</th>\n",
-       "      <td>189.239524</td>\n",
-       "      <td>37</td>\n",
-       "      <td>[Stumbled on this again: If you haven't seen i...</td>\n",
-       "      <td>http://docfuture.tumblr.com/post/82363551272/f...</td>\n",
-       "      <td>[https://reddit.com/r/rational/comments/2pn8xf...</td>\n",
-       "      <td>[1418872070.0, 1423787328.0, 1431566469.0, 143...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>Moloch</th>\n",
-       "      <td>188.208333</td>\n",
-       "      <td>24</td>\n",
-       "      <td>[&gt;I'm pretty sure one of us is telepathically ...</td>\n",
-       "      <td>http://slatestarcodex.com/2014/07/30/meditatio...</td>\n",
-       "      <td>[https://reddit.com/r/rational/comments/2oub3w...</td>\n",
-       "      <td>[1418192928.0, 1420136565.0, 1421706703.0, 142...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>Marked for Death</th>\n",
-       "      <td>186.864637</td>\n",
-       "      <td>47</td>\n",
-       "      <td>[[Marked for Death](https://forums.sufficientv...</td>\n",
-       "      <td>https://forums.sufficientvelocity.com/threads/...</td>\n",
-       "      <td>[https://reddit.com/r/rational/comments/101em0...</td>\n",
-       "      <td>[1672671609.0, 1681740016.0, 1686578416.0, 168...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>Purple Days</th>\n",
-       "      <td>171.386905</td>\n",
-       "      <td>25</td>\n",
-       "      <td>[Highly recommend [Purple Days](https://forums...</td>\n",
-       "      <td>https://forums.spacebattles.com/threads/purple...</td>\n",
-       "      <td>[https://reddit.com/r/rational/comments/11k144...</td>\n",
-       "      <td>[1678114814.0, 1679320813.0, 1688392816.0, 169...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>Dungeon Keeper Ami</th>\n",
-       "      <td>161.841919</td>\n",
-       "      <td>29</td>\n",
-       "      <td>[Fan fics only:\\n\\nDungeon Keeper/Sailor Moon:...</td>\n",
-       "      <td>https://forums.sufficientvelocity.com/threads/...</td>\n",
-       "      <td>[https://reddit.com/r/rational/comments/12hjo5...</td>\n",
-       "      <td>[1681135215.0, 1685973616.0, 1474749889.0, 148...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>Delve</th>\n",
-       "      <td>161.627273</td>\n",
-       "      <td>28</td>\n",
-       "      <td>[## Good RoyalRoad (forgot to add this list wh...</td>\n",
-       "      <td>https://www.royalroad.com/fiction/25225/delve\\...</td>\n",
-       "      <td>[https://reddit.com/r/rational/comments/1bt4gk...</td>\n",
-       "      <td>[1711980017.0, 1711980017.0, 1718028015.0, 172...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>Sanitize</th>\n",
-       "      <td>153.624272</td>\n",
-       "      <td>31</td>\n",
-       "      <td>[[Sanitize](https://www.fanfiction.net/s/12431...</td>\n",
-       "      <td>https://archiveofourown.org/works/20721824/\\nh...</td>\n",
-       "      <td>[https://reddit.com/r/rational/comments/123nq9...</td>\n",
-       "      <td>[1679925615.0, 1687183218.0, 1692021625.0, 170...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>Fine Structure</th>\n",
-       "      <td>151.857429</td>\n",
-       "      <td>40</td>\n",
-       "      <td>[Join us at /r/qntm! Spoilers!\\n\\nAlso, [*Fine...</td>\n",
-       "      <td>http://qntm.org/ra\\nhttp://qntm.org/structure\\...</td>\n",
-       "      <td>[https://reddit.com/r/rational/comments/1rykc1...</td>\n",
-       "      <td>[1386036340.0, 1402010390.0, 1418670015.0, 141...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>Ar'Kendrithyst</th>\n",
-       "      <td>148.280556</td>\n",
-       "      <td>30</td>\n",
-       "      <td>[Have you read [Ar'Kendrythist](https://www.ro...</td>\n",
-       "      <td>https://www.royalroad.com/fiction/26727/arkend...</td>\n",
-       "      <td>[https://reddit.com/r/rational/comments/101em0...</td>\n",
-       "      <td>[1672671609.0, 1678114814.0, 1681740016.0, 168...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>Branches on the Tree of Time</th>\n",
-       "      <td>147.401810</td>\n",
-       "      <td>28</td>\n",
-       "      <td>[I [already tried my hand at Terminator](https...</td>\n",
-       "      <td>https://www.fanfiction.net/s/9658524/1/Branche...</td>\n",
-       "      <td>[https://reddit.com/r/rational/comments/1tftsf...</td>\n",
-       "      <td>[1387683947.0, 1389822389.0, 1409033440.0, 142...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>Harry Potter and the Natural 20</th>\n",
-       "      <td>143.053345</td>\n",
-       "      <td>42</td>\n",
-       "      <td>[My favorite harry potter fan fics:\\n\\n- [Seve...</td>\n",
-       "      <td>https://www.fanfiction.net/s/8096183/1/Harry-P...</td>\n",
-       "      <td>[https://reddit.com/r/rational/comments/11qai9...</td>\n",
-       "      <td>[1678716009.0, 1403215249.0, 1406342508.0, 141...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>Lord of the Mysteries</th>\n",
-       "      <td>140.916667</td>\n",
-       "      <td>20</td>\n",
-       "      <td>[[Lord of the Mysteries](https://www.wuxiaworl...</td>\n",
-       "      <td>https://www.webnovel.com/book/1102273300623450...</td>\n",
-       "      <td>[https://reddit.com/r/rational/comments/dpqyh0...</td>\n",
-       "      <td>[1572543414.0, 1572900976.0, 1574694295.0, 157...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>The Dark Wizard of Donkerk</th>\n",
-       "      <td>140.219697</td>\n",
-       "      <td>24</td>\n",
-       "      <td>[Hey everyone, based on some listener requests...</td>\n",
-       "      <td>http://alexanderwales.com/darkWizardNaNo2016.h...</td>\n",
-       "      <td>[https://reddit.com/r/rational/comments/6cclkp...</td>\n",
-       "      <td>[1495312712.0, 1497765557.0, 1525938401.0, 152...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>The Fifth Defiance</th>\n",
-       "      <td>136.750000</td>\n",
-       "      <td>12</td>\n",
-       "      <td>[[The Fifth Defiance](https://thefifthdefiance...</td>\n",
-       "      <td>https://thefifthdefiance.com/about/\\nhttps://t...</td>\n",
-       "      <td>[https://reddit.com/r/rational/comments/111b60...</td>\n",
-       "      <td>[1676300410.0, 1544022361.0, 1567245084.0, 158...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>Demonology and the Tri-Phasic Model of Trauma: An Integrative Approach</th>\n",
-       "      <td>135.342857</td>\n",
-       "      <td>12</td>\n",
-       "      <td>[`•` [*Let Her In 2*](https://www.fanfiction.n...</td>\n",
-       "      <td>https://archiveofourown.org/works/20177950/cha...</td>\n",
-       "      <td>[https://reddit.com/r/rational/comments/17ucgu...</td>\n",
-       "      <td>[1699887610.0, 1574694295.0, 1575299088.0, 157...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>A Young Woman's Political Record</th>\n",
-       "      <td>133.492250</td>\n",
-       "      <td>31</td>\n",
-       "      <td>[Fan fics only:\\n\\nDungeon Keeper/Sailor Moon:...</td>\n",
-       "      <td>https://forums.spacebattles.com/threads/a-youn...</td>\n",
-       "      <td>[https://reddit.com/r/rational/comments/12hjo5...</td>\n",
-       "      <td>[1681135215.0, 1691416815.0, 1704726013.0, 171...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>Luminosity</th>\n",
-       "      <td>132.816075</td>\n",
-       "      <td>33</td>\n",
-       "      <td>[I think I'd really only consider [*Luminosity...</td>\n",
-       "      <td>https://luminous.elcenia.com/story.shtml\\nhttp...</td>\n",
-       "      <td>[https://reddit.com/r/rational/comments/28l8v0...</td>\n",
-       "      <td>[1403215249.0, 1409033440.0, 1436927069.0, 145...</td>\n",
-       "    </tr>\n",
-       "  </tbody>\n",
-       "</table>\n",
-       "</div>"
-      ],
-      "text/plain": [
-       "                                                         score  num_comments  \\\n",
-       "title                                                                          \n",
-       "a Patreon campaign                                  880.000000            22   \n",
-       "Worth the Candle                                    661.062143           108   \n",
-       "Worm                                                414.596255            78   \n",
-       "Twig                                                346.983574            63   \n",
-       "With This Ring                                      329.593551            59   \n",
-       "The Metropolitan Man                                319.948130            57   \n",
-       "Cordyceps                                           311.815476            41   \n",
-       "Mother of Learning                                  303.770050            83   \n",
-       "Time Braid                                          280.345557            99   \n",
-       "Unsong                                              268.748647            40   \n",
-       "Seventh Horcrux                                     267.237103            30   \n",
-       "A Practical Guide to Evil                           235.768723            48   \n",
-       "Pale                                                205.378788            33   \n",
-       "Pact                                                197.108141            36   \n",
-       "Only Villains Do That                               195.305556            19   \n",
-       "The Waves Arisen                                    190.232117            49   \n",
-       "The Fall of Doc Future                              189.239524            37   \n",
-       "Moloch                                              188.208333            24   \n",
-       "Marked for Death                                    186.864637            47   \n",
-       "Purple Days                                         171.386905            25   \n",
-       "Dungeon Keeper Ami                                  161.841919            29   \n",
-       "Delve                                               161.627273            28   \n",
-       "Sanitize                                            153.624272            31   \n",
-       "Fine Structure                                      151.857429            40   \n",
-       "Ar'Kendrithyst                                      148.280556            30   \n",
-       "Branches on the Tree of Time                        147.401810            28   \n",
-       "Harry Potter and the Natural 20                     143.053345            42   \n",
-       "Lord of the Mysteries                               140.916667            20   \n",
-       "The Dark Wizard of Donkerk                          140.219697            24   \n",
-       "The Fifth Defiance                                  136.750000            12   \n",
-       "Demonology and the Tri-Phasic Model of Trauma: ...  135.342857            12   \n",
-       "A Young Woman's Political Record                    133.492250            31   \n",
-       "Luminosity                                          132.816075            33   \n",
-       "\n",
-       "                                                                                             comments  \\\n",
-       "title                                                                                                   \n",
-       "a Patreon campaign                                  [I do have a subreddit set up for this over at...   \n",
-       "Worth the Candle                                    [>>interestingly blend ...\\n\\n>`[g]`enres\\n\\n>...   \n",
-       "Worm                                                [[Kitty Cat Kill Sat](https://www.royalroad.co...   \n",
-       "Twig                                                [The protagonist of [Twig](https://twigserial....   \n",
-       "With This Ring                                      [> puts out more chapters faster than any web ...   \n",
-       "The Metropolitan Man                                [~ [*Death Note: L, Anonymity & Eluding Entrop...   \n",
-       "Cordyceps                                           [If you like that, be sure to read [CORDYCEPS]...   \n",
-       "Mother of Learning                                  [And [Mother of Learning](https://www.fictionp...   \n",
-       "Time Braid                                          [* [*Waves Arisen*](https://wertifloke.wordpre...   \n",
-       "Unsong                                              [Hey all, welcome back! A few quick notes: fir...   \n",
-       "Seventh Horcrux                                     [My favorite harry potter fan fics:\\n\\n- [Seve...   \n",
-       "A Practical Guide to Evil                           [[*Mother of Learning*](https://www.royalroad....   \n",
-       "Pale                                                [Wildbow's latest webserial, [Pale](https://pa...   \n",
-       "Pact                                                [Let's talk about the horror genre. 'Supernatu...   \n",
-       "Only Villains Do That                               [Hey! Sorry for the very-late reply.  I'll try...   \n",
-       "The Waves Arisen                                    [* [*Waves Arisen*](https://wertifloke.wordpre...   \n",
-       "The Fall of Doc Future                              [Stumbled on this again: If you haven't seen i...   \n",
-       "Moloch                                              [>I'm pretty sure one of us is telepathically ...   \n",
-       "Marked for Death                                    [[Marked for Death](https://forums.sufficientv...   \n",
-       "Purple Days                                         [Highly recommend [Purple Days](https://forums...   \n",
-       "Dungeon Keeper Ami                                  [Fan fics only:\\n\\nDungeon Keeper/Sailor Moon:...   \n",
-       "Delve                                               [## Good RoyalRoad (forgot to add this list wh...   \n",
-       "Sanitize                                            [[Sanitize](https://www.fanfiction.net/s/12431...   \n",
-       "Fine Structure                                      [Join us at /r/qntm! Spoilers!\\n\\nAlso, [*Fine...   \n",
-       "Ar'Kendrithyst                                      [Have you read [Ar'Kendrythist](https://www.ro...   \n",
-       "Branches on the Tree of Time                        [I [already tried my hand at Terminator](https...   \n",
-       "Harry Potter and the Natural 20                     [My favorite harry potter fan fics:\\n\\n- [Seve...   \n",
-       "Lord of the Mysteries                               [[Lord of the Mysteries](https://www.wuxiaworl...   \n",
-       "The Dark Wizard of Donkerk                          [Hey everyone, based on some listener requests...   \n",
-       "The Fifth Defiance                                  [[The Fifth Defiance](https://thefifthdefiance...   \n",
-       "Demonology and the Tri-Phasic Model of Trauma: ...  [`•` [*Let Her In 2*](https://www.fanfiction.n...   \n",
-       "A Young Woman's Political Record                    [Fan fics only:\\n\\nDungeon Keeper/Sailor Moon:...   \n",
-       "Luminosity                                          [I think I'd really only consider [*Luminosity...   \n",
-       "\n",
-       "                                                                                                  url  \\\n",
-       "title                                                                                                   \n",
-       "a Patreon campaign                                             https://www.patreon.com/alexanderwales   \n",
-       "Worth the Candle                                    https://archiveofourown.org/works/11478249/cha...   \n",
-       "Worm                                                https://parahumans.wordpress.com\\nhttps://para...   \n",
-       "Twig                                                https://twigserial.wordpress.com/about/\\nhttps...   \n",
-       "With This Ring                                      https://forums.sufficientvelocity.com/threads/...   \n",
-       "The Metropolitan Man                                https://alexanderwales.com/the-metropolitan-ma...   \n",
-       "Cordyceps                                           http://archiveofourown.org/works/6178036/chapt...   \n",
-       "Mother of Learning                                  https://www.royalroad.com/fiction/21220/mother...   \n",
-       "Time Braid                                          https://www.fanfiction.net/s/5193644\\nhttps://...   \n",
-       "Unsong                                              https://unsongbook.com/\\nhttp://unsongbook.com...   \n",
-       "Seventh Horcrux                                     https://m.fanfiction.net/s/10677106/1/Seventh-...   \n",
-       "A Practical Guide to Evil                           https://practicalguidetoevil.wordpress.com/sum...   \n",
-       "Pale                                                https://palewebserial.wordpress.com/about/\\nht...   \n",
-       "Pact                                                https://www.mediamdpodcast.com/pact-audiobook-...   \n",
-       "Only Villains Do That                               https://www.royalroad.com/fiction/40182/only-v...   \n",
-       "The Waves Arisen                                    https://wertifloke.wordpress.com/about/\\nhttps...   \n",
-       "The Fall of Doc Future                              http://docfuture.tumblr.com/post/82363551272/f...   \n",
-       "Moloch                                              http://slatestarcodex.com/2014/07/30/meditatio...   \n",
-       "Marked for Death                                    https://forums.sufficientvelocity.com/threads/...   \n",
-       "Purple Days                                         https://forums.spacebattles.com/threads/purple...   \n",
-       "Dungeon Keeper Ami                                  https://forums.sufficientvelocity.com/threads/...   \n",
-       "Delve                                               https://www.royalroad.com/fiction/25225/delve\\...   \n",
-       "Sanitize                                            https://archiveofourown.org/works/20721824/\\nh...   \n",
-       "Fine Structure                                      http://qntm.org/ra\\nhttp://qntm.org/structure\\...   \n",
-       "Ar'Kendrithyst                                      https://www.royalroad.com/fiction/26727/arkend...   \n",
-       "Branches on the Tree of Time                        https://www.fanfiction.net/s/9658524/1/Branche...   \n",
-       "Harry Potter and the Natural 20                     https://www.fanfiction.net/s/8096183/1/Harry-P...   \n",
-       "Lord of the Mysteries                               https://www.webnovel.com/book/1102273300623450...   \n",
-       "The Dark Wizard of Donkerk                          http://alexanderwales.com/darkWizardNaNo2016.h...   \n",
-       "The Fifth Defiance                                  https://thefifthdefiance.com/about/\\nhttps://t...   \n",
-       "Demonology and the Tri-Phasic Model of Trauma: ...  https://archiveofourown.org/works/20177950/cha...   \n",
-       "A Young Woman's Political Record                    https://forums.spacebattles.com/threads/a-youn...   \n",
-       "Luminosity                                          https://luminous.elcenia.com/story.shtml\\nhttp...   \n",
-       "\n",
-       "                                                                                         comment_urls  \\\n",
-       "title                                                                                                   \n",
-       "a Patreon campaign                                  [https://reddit.com/r/rational/comments/331re9...   \n",
-       "Worth the Candle                                    [https://reddit.com/r/rational/comments/1c4mxk...   \n",
-       "Worm                                                [https://reddit.com/r/rational/comments/16rsx5...   \n",
-       "Twig                                                [https://reddit.com/r/rational/comments/17ucgu...   \n",
-       "With This Ring                                      [https://reddit.com/r/rational/comments/52bbjb...   \n",
-       "The Metropolitan Man                                [https://reddit.com/r/rational/comments/134n9q...   \n",
-       "Cordyceps                                           [https://reddit.com/r/rational/comments/1eqehs...   \n",
-       "Mother of Learning                                  [https://reddit.com/r/rational/comments/2br02d...   \n",
-       "Time Braid                                          [https://reddit.com/r/rational/comments/11wjn0...   \n",
-       "Unsong                                              [https://reddit.com/r/rational/comments/4m0odi...   \n",
-       "Seventh Horcrux                                     [https://reddit.com/r/rational/comments/11qai9...   \n",
-       "A Practical Guide to Evil                           [https://reddit.com/r/rational/comments/1adxac...   \n",
-       "Pale                                                [https://reddit.com/r/rational/comments/111b60...   \n",
-       "Pact                                                [https://reddit.com/r/rational/comments/4yjexh...   \n",
-       "Only Villains Do That                               [https://reddit.com/r/rational/comments/13bqbg...   \n",
-       "The Waves Arisen                                    [https://reddit.com/r/rational/comments/11wjn0...   \n",
-       "The Fall of Doc Future                              [https://reddit.com/r/rational/comments/2pn8xf...   \n",
-       "Moloch                                              [https://reddit.com/r/rational/comments/2oub3w...   \n",
-       "Marked for Death                                    [https://reddit.com/r/rational/comments/101em0...   \n",
-       "Purple Days                                         [https://reddit.com/r/rational/comments/11k144...   \n",
-       "Dungeon Keeper Ami                                  [https://reddit.com/r/rational/comments/12hjo5...   \n",
-       "Delve                                               [https://reddit.com/r/rational/comments/1bt4gk...   \n",
-       "Sanitize                                            [https://reddit.com/r/rational/comments/123nq9...   \n",
-       "Fine Structure                                      [https://reddit.com/r/rational/comments/1rykc1...   \n",
-       "Ar'Kendrithyst                                      [https://reddit.com/r/rational/comments/101em0...   \n",
-       "Branches on the Tree of Time                        [https://reddit.com/r/rational/comments/1tftsf...   \n",
-       "Harry Potter and the Natural 20                     [https://reddit.com/r/rational/comments/11qai9...   \n",
-       "Lord of the Mysteries                               [https://reddit.com/r/rational/comments/dpqyh0...   \n",
-       "The Dark Wizard of Donkerk                          [https://reddit.com/r/rational/comments/6cclkp...   \n",
-       "The Fifth Defiance                                  [https://reddit.com/r/rational/comments/111b60...   \n",
-       "Demonology and the Tri-Phasic Model of Trauma: ...  [https://reddit.com/r/rational/comments/17ucgu...   \n",
-       "A Young Woman's Political Record                    [https://reddit.com/r/rational/comments/12hjo5...   \n",
-       "Luminosity                                          [https://reddit.com/r/rational/comments/28l8v0...   \n",
-       "\n",
-       "                                                                                                dates  \n",
-       "title                                                                                                  \n",
-       "a Patreon campaign                                  [1429380255.0, 1432400450.0, 1436482846.0, 146...  \n",
-       "Worth the Candle                                    [1713189611.0, 1503163699.0, 1506215679.0, 150...  \n",
-       "Worm                                                [1695650418.0, 1699887610.0, 1706540412.0, 171...  \n",
-       "Twig                                                [1699887610.0, 1725890413.0, 1450408569.0, 147...  \n",
-       "With This Ring                                      [1473635989.0, 1481638770.0, 1514051962.0, 152...  \n",
-       "The Metropolitan Man                                [1682949618.0, 1687788016.0, 1400177222.0, 140...  \n",
-       "Cordyceps                                           [1723471212.0, 1504188940.0, 1504579227.0, 150...  \n",
-       "Mother of Learning                                  [1406342508.0, 1417735601.0, 1418670015.0, 141...  \n",
-       "Time Braid                                          [1679320813.0, 1682949618.0, 1429816392.0, 143...  \n",
-       "Unsong                                              [1464778693.0, 1465166191.0, 1465595369.0, 148...  \n",
-       "Seventh Horcrux                                     [1678716009.0, 1681135215.0, 1436741473.0, 144...  \n",
-       "A Practical Guide to Evil                           [1706540412.0, 1464188682.0, 1473635989.0, 148...  \n",
-       "Pale                                                [1676300410.0, 1678114814.0, 1691416815.0, 169...  \n",
-       "Pact                                                [1471619081.0, 1473635989.0, 1546640293.0, 155...  \n",
-       "Only Villains Do That                               [1683554474.0, 1684764016.0, 1712584828.0, 172...  \n",
-       "The Waves Arisen                                    [1679320813.0, 1680530414.0, 1681135215.0, 168...  \n",
-       "The Fall of Doc Future                              [1418872070.0, 1423787328.0, 1431566469.0, 143...  \n",
-       "Moloch                                              [1418192928.0, 1420136565.0, 1421706703.0, 142...  \n",
-       "Marked for Death                                    [1672671609.0, 1681740016.0, 1686578416.0, 168...  \n",
-       "Purple Days                                         [1678114814.0, 1679320813.0, 1688392816.0, 169...  \n",
-       "Dungeon Keeper Ami                                  [1681135215.0, 1685973616.0, 1474749889.0, 148...  \n",
-       "Delve                                               [1711980017.0, 1711980017.0, 1718028015.0, 172...  \n",
-       "Sanitize                                            [1679925615.0, 1687183218.0, 1692021625.0, 170...  \n",
-       "Fine Structure                                      [1386036340.0, 1402010390.0, 1418670015.0, 141...  \n",
-       "Ar'Kendrithyst                                      [1672671609.0, 1678114814.0, 1681740016.0, 168...  \n",
-       "Branches on the Tree of Time                        [1387683947.0, 1389822389.0, 1409033440.0, 142...  \n",
-       "Harry Potter and the Natural 20                     [1678716009.0, 1403215249.0, 1406342508.0, 141...  \n",
-       "Lord of the Mysteries                               [1572543414.0, 1572900976.0, 1574694295.0, 157...  \n",
-       "The Dark Wizard of Donkerk                          [1495312712.0, 1497765557.0, 1525938401.0, 152...  \n",
-       "The Fifth Defiance                                  [1676300410.0, 1544022361.0, 1567245084.0, 158...  \n",
-       "Demonology and the Tri-Phasic Model of Trauma: ...  [1699887610.0, 1574694295.0, 1575299088.0, 157...  \n",
-       "A Young Woman's Political Record                    [1681135215.0, 1691416815.0, 1704726013.0, 171...  \n",
-       "Luminosity                                          [1403215249.0, 1409033440.0, 1436927069.0, 145...  "
-      ]
-     },
-     "execution_count": 17,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
+   "outputs": [],
    "source": [
     "# join by title\n",
     "\n",
-    "def join_uniq(x:list[str]):\n",
-    "    return '\\n'.join(set(x))\n",
     "\n",
-    "def chain_lists(x:list[list[str]]):\n",
+    "def join_uniq(x: list[str]):\n",
+    "    return \"\\n\".join(set(x))\n",
+    "\n",
+    "\n",
+    "def chain_lists(x: list[list[str]]):\n",
     "    return [item for sublist in x for item in sublist]\n",
     "\n",
-    "df3 = df2.groupby('title').agg({'score': 'sum', 'num_comments': 'sum', 'comments': chain_lists, 'url': join_uniq, 'comment_urls': chain_lists, 'dates': chain_lists\n",
-    "                                }).sort_values('score', ascending=False)\n",
+    "\n",
+    "df3 = (\n",
+    "    df2.groupby(\"title\")\n",
+    "    .agg(\n",
+    "        {\n",
+    "            \"score\": \"sum\",\n",
+    "            \"num_comments\": \"sum\",\n",
+    "            \"comments\": chain_lists,\n",
+    "            \"url\": join_uniq,\n",
+    "            \"comment_urls\": chain_lists,\n",
+    "            \"dates\": chain_lists,\n",
+    "        }\n",
+    "    )\n",
+    "    .sort_values(\"score\", ascending=False)\n",
+    ")\n",
     "df3.head(33)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 18,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "((3129, 7), (2743, 6))"
-      ]
-     },
-     "execution_count": 18,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
+   "outputs": [],
    "source": [
     "df2.shape, df3.shape"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 19,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -1353,99 +1071,85 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 25,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "'[{\"title\": \"title\", \"visible\": true, \"searchable\": true}, {\"title\": \"score\", \"visible\": true, \"searchable\": true}, {\"title\": \"num_comments\", \"visible\": true, \"searchable\": true}, {\"title\": \"comments\", \"visible\": false, \"searchable\": false}, {\"title\": \"url\", \"visible\": true, \"searchable\": true}, {\"title\": \"comment_urls\", \"visible\": false, \"searchable\": false}, {\"title\": \"dates\", \"visible\": true, \"searchable\": true}]'"
-      ]
-     },
-     "execution_count": 25,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
+   "outputs": [],
    "source": [
     "import jinja2\n",
+    "\n",
     "environment = jinja2.Environment()\n",
-    "template = open('../index.jinja2.html').read()\n",
+    "template = open(\"../index.jinja2.html\").read()\n",
     "template = environment.from_string(template)\n",
-    "d = (df3\n",
-    "     .reset_index()\n",
-    "     .sort_values('score', ascending=False)\n",
-    "    )\n",
+    "d = df3.reset_index().sort_values(\"score\", ascending=False)\n",
+    "\n",
     "\n",
     "def url2a(url):\n",
     "    text = url\n",
-    "    if 'reddit.com/r/rational' in url:\n",
-    "        text = url.split('/')[-2]\n",
+    "    if \"reddit.com/r/rational\" in url:\n",
+    "        text = url.split(\"/\")[-2]\n",
     "        # text = url.replace('https://reddit.com/r/rational/comments/', '')\n",
     "\n",
     "    return f'<a href=\"{url}\">{text}</a>'\n",
     "\n",
-    "def urls2a(urls, sep='<br>'):\n",
+    "\n",
+    "def urls2a(urls, sep=\"<br>\"):\n",
     "    if isinstance(urls, str):\n",
-    "        urls = urls.split('\\n')\n",
-    "    \n",
+    "        urls = urls.split(\"\\n\")\n",
+    "\n",
     "    return sep.join(url2a(u) for u in urls)\n",
-    "d['url'] = d['url'].apply(urls2a)\n",
-    "d['score'] = d['score'].round(2)\n",
-    "d['comment_urls'] = d['comment_urls'].apply(lambda s:urls2a(s, sep=' '))\n",
+    "\n",
+    "\n",
+    "d[\"url\"] = d[\"url\"].apply(urls2a)\n",
+    "d[\"score\"] = d[\"score\"].round(2)\n",
+    "d[\"comment_urls\"] = d[\"comment_urls\"].apply(lambda s: urls2a(s, sep=\" \"))\n",
     "\n",
     "# TODO comment md to html... if I want to keep them\n",
     "\n",
+    "\n",
     "def get_first_date(x):\n",
-    "    return pd.to_datetime(x, unit='s').min().strftime('%Y-%m-%d %H:%M:%S')\n",
-    "d['dates'] = d['dates'].apply(get_first_date)\n",
+    "    return pd.to_datetime(x, unit=\"s\").min().strftime(\"%Y-%m-%d %H:%M:%S\")\n",
     "\n",
-    "data = d.to_json(orient='values')\n",
     "\n",
-    "hidden = ['comments', 'comment_urls']\n",
-    "columns=[{\n",
-    "    'title':c, \n",
-    "    'visible': c not in hidden,\n",
-    "    'searchable': c not in hidden,\n",
-    "    } for c in d.columns]\n",
+    "d[\"dates\"] = d[\"dates\"].apply(get_first_date)\n",
+    "\n",
+    "data = d.to_json(orient=\"values\")\n",
+    "\n",
+    "hidden = [\"comments\", \"comment_urls\"]\n",
+    "columns = [\n",
+    "    {\n",
+    "        \"title\": c,\n",
+    "        \"visible\": c not in hidden,\n",
+    "        \"searchable\": c not in hidden,\n",
+    "    }\n",
+    "    for c in d.columns\n",
+    "]\n",
     "columns = json.dumps(columns)\n",
     "\n",
     "\n",
-    "html = template.render(data=data,\n",
-    "                       columns=columns,\n",
-    "                       )\n",
-    "html_out = Path('../outputs/index.html').resolve()\n",
-    "open(html_out, 'w').write(html)\n",
+    "html = template.render(\n",
+    "    data=data,\n",
+    "    columns=columns,\n",
+    ")\n",
+    "html_out = Path(\"../outputs/index.html\").resolve()\n",
+    "open(html_out, \"w\").write(html)\n",
     "columns"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 21,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/html": [
-       "<a href=\"/media/wassname/SGIronWolf/projects5/rrational/outputs/index.html\">View the page /media/wassname/SGIronWolf/projects5/rrational/outputs/index.html</a>"
-      ],
-      "text/plain": [
-       "<IPython.core.display.HTML object>"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    }
-   ],
+   "outputs": [],
    "source": [
-    "from IPython.display import display, HTML\n",
+    "from IPython.display import HTML, display\n",
+    "\n",
     "htmla = f'<a href=\"{html_out}\">View the page {html_out}</a>'\n",
     "display(HTML(htmla))"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 22,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -1470,42 +1174,19 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 23,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "ename": "ModuleNotFoundError",
-     "evalue": "No module named 'sniffio'",
-     "output_type": "error",
-     "traceback": [
-      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
-      "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
-      "Cell \u001b[0;32mIn[23], line 4\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mdotenv\u001b[39;00m\n\u001b[1;32m      3\u001b[0m dotenv\u001b[38;5;241m.\u001b[39mload_dotenv()\n\u001b[0;32m----> 4\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mopenai\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m OpenAI\n\u001b[1;32m      5\u001b[0m client \u001b[38;5;241m=\u001b[39m OpenAI()\n\u001b[1;32m      7\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mtiktoken\u001b[39;00m\n",
-      "File \u001b[0;32m/media/wassname/SGIronWolf/projects5/rrational/.venv/lib/python3.10/site-packages/openai/__init__.py:8\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mos\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01m_os\u001b[39;00m\n\u001b[1;32m      6\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mtyping_extensions\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m override\n\u001b[0;32m----> 8\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m types\n\u001b[1;32m      9\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_types\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m NOT_GIVEN, NoneType, NotGiven, Transport, ProxiesTypes\n\u001b[1;32m     10\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_utils\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m file_from_path\n",
-      "File \u001b[0;32m/media/wassname/SGIronWolf/projects5/rrational/.venv/lib/python3.10/site-packages/openai/types/__init__.py:5\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[38;5;66;03m# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.\u001b[39;00m\n\u001b[1;32m      3\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m__future__\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m annotations\n\u001b[0;32m----> 5\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mbatch\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m Batch \u001b[38;5;28;01mas\u001b[39;00m Batch\n\u001b[1;32m      6\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mimage\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m Image \u001b[38;5;28;01mas\u001b[39;00m Image\n\u001b[1;32m      7\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mmodel\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m Model \u001b[38;5;28;01mas\u001b[39;00m Model\n",
-      "File \u001b[0;32m/media/wassname/SGIronWolf/projects5/rrational/.venv/lib/python3.10/site-packages/openai/types/batch.py:7\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mtyping\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m List, Optional\n\u001b[1;32m      5\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mtyping_extensions\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m Literal\n\u001b[0;32m----> 7\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_models\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m BaseModel\n\u001b[1;32m      8\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mbatch_error\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m BatchError\n\u001b[1;32m      9\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mbatch_request_counts\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m BatchRequestCounts\n",
-      "File \u001b[0;32m/media/wassname/SGIronWolf/projects5/rrational/.venv/lib/python3.10/site-packages/openai/_models.py:25\u001b[0m\n\u001b[1;32m     22\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mpydantic\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mgenerics\u001b[39;00m\n\u001b[1;32m     23\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mpydantic\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mfields\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m FieldInfo\n\u001b[0;32m---> 25\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_types\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m (\n\u001b[1;32m     26\u001b[0m     Body,\n\u001b[1;32m     27\u001b[0m     IncEx,\n\u001b[1;32m     28\u001b[0m     Query,\n\u001b[1;32m     29\u001b[0m     ModelT,\n\u001b[1;32m     30\u001b[0m     Headers,\n\u001b[1;32m     31\u001b[0m     Timeout,\n\u001b[1;32m     32\u001b[0m     NotGiven,\n\u001b[1;32m     33\u001b[0m     AnyMapping,\n\u001b[1;32m     34\u001b[0m     HttpxRequestFiles,\n\u001b[1;32m     35\u001b[0m )\n\u001b[1;32m     36\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_utils\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m (\n\u001b[1;32m     37\u001b[0m     PropertyInfo,\n\u001b[1;32m     38\u001b[0m     is_list,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m     48\u001b[0m     strip_annotated_type,\n\u001b[1;32m     49\u001b[0m )\n\u001b[1;32m     50\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_compat\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m (\n\u001b[1;32m     51\u001b[0m     PYDANTIC_V2,\n\u001b[1;32m     52\u001b[0m     ConfigDict,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m     61\u001b[0m     field_get_default,\n\u001b[1;32m     62\u001b[0m )\n",
-      "File \u001b[0;32m/media/wassname/SGIronWolf/projects5/rrational/.venv/lib/python3.10/site-packages/openai/_types.py:21\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mtyping\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m (\n\u001b[1;32m      5\u001b[0m     IO,\n\u001b[1;32m      6\u001b[0m     TYPE_CHECKING,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m     17\u001b[0m     Sequence,\n\u001b[1;32m     18\u001b[0m )\n\u001b[1;32m     19\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mtyping_extensions\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m Literal, Protocol, TypeAlias, TypedDict, override, runtime_checkable\n\u001b[0;32m---> 21\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mhttpx\u001b[39;00m\n\u001b[1;32m     22\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mpydantic\u001b[39;00m\n\u001b[1;32m     23\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mhttpx\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m URL, Proxy, Timeout, Response, BaseTransport, AsyncBaseTransport\n",
-      "File \u001b[0;32m/media/wassname/SGIronWolf/projects5/rrational/.venv/lib/python3.10/site-packages/httpx/__init__.py:2\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m__version__\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m __description__, __title__, __version__\n\u001b[0;32m----> 2\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_api\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;241m*\u001b[39m\n\u001b[1;32m      3\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_auth\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;241m*\u001b[39m\n\u001b[1;32m      4\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_client\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;241m*\u001b[39m\n",
-      "File \u001b[0;32m/media/wassname/SGIronWolf/projects5/rrational/.venv/lib/python3.10/site-packages/httpx/_api.py:6\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mtyping\u001b[39;00m\n\u001b[1;32m      4\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mcontextlib\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m contextmanager\n\u001b[0;32m----> 6\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_client\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m Client\n\u001b[1;32m      7\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_config\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m DEFAULT_TIMEOUT_CONFIG\n\u001b[1;32m      8\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_models\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m Response\n",
-      "File \u001b[0;32m/media/wassname/SGIronWolf/projects5/rrational/.venv/lib/python3.10/site-packages/httpx/_client.py:12\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mtypes\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m TracebackType\n\u001b[1;32m     11\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m__version__\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m __version__\n\u001b[0;32m---> 12\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_auth\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m Auth, BasicAuth, FunctionAuth\n\u001b[1;32m     13\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_config\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m (\n\u001b[1;32m     14\u001b[0m     DEFAULT_LIMITS,\n\u001b[1;32m     15\u001b[0m     DEFAULT_MAX_REDIRECTS,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m     19\u001b[0m     Timeout,\n\u001b[1;32m     20\u001b[0m )\n\u001b[1;32m     21\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_decoders\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m SUPPORTED_DECODERS\n",
-      "File \u001b[0;32m/media/wassname/SGIronWolf/projects5/rrational/.venv/lib/python3.10/site-packages/httpx/_auth.py:12\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01murllib\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mrequest\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m parse_http_list\n\u001b[1;32m     11\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_exceptions\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m ProtocolError\n\u001b[0;32m---> 12\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_models\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m Cookies, Request, Response\n\u001b[1;32m     13\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_utils\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m to_bytes, to_str, unquote\n\u001b[1;32m     15\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m typing\u001b[38;5;241m.\u001b[39mTYPE_CHECKING:  \u001b[38;5;66;03m# pragma: no cover\u001b[39;00m\n",
-      "File \u001b[0;32m/media/wassname/SGIronWolf/projects5/rrational/.venv/lib/python3.10/site-packages/httpx/_models.py:11\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mcollections\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mabc\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m Mapping\n\u001b[1;32m      9\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mhttp\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mcookiejar\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m Cookie, CookieJar\n\u001b[0;32m---> 11\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_content\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m ByteStream, UnattachedStream, encode_request, encode_response\n\u001b[1;32m     12\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_decoders\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m (\n\u001b[1;32m     13\u001b[0m     SUPPORTED_DECODERS,\n\u001b[1;32m     14\u001b[0m     ByteChunker,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m     20\u001b[0m     TextDecoder,\n\u001b[1;32m     21\u001b[0m )\n\u001b[1;32m     22\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_exceptions\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m (\n\u001b[1;32m     23\u001b[0m     CookieConflict,\n\u001b[1;32m     24\u001b[0m     HTTPStatusError,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m     29\u001b[0m     request_context,\n\u001b[1;32m     30\u001b[0m )\n",
-      "File \u001b[0;32m/media/wassname/SGIronWolf/projects5/rrational/.venv/lib/python3.10/site-packages/httpx/_content.py:17\u001b[0m\n\u001b[1;32m     14\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01murllib\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mparse\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m urlencode\n\u001b[1;32m     16\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_exceptions\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m StreamClosed, StreamConsumed\n\u001b[0;32m---> 17\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_multipart\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m MultipartStream\n\u001b[1;32m     18\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_types\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m (\n\u001b[1;32m     19\u001b[0m     AsyncByteStream,\n\u001b[1;32m     20\u001b[0m     RequestContent,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m     24\u001b[0m     SyncByteStream,\n\u001b[1;32m     25\u001b[0m )\n\u001b[1;32m     26\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_utils\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m peek_filelike_length, primitive_value_to_str\n",
-      "File \u001b[0;32m/media/wassname/SGIronWolf/projects5/rrational/.venv/lib/python3.10/site-packages/httpx/_multipart.py:16\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mpathlib\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m Path\n\u001b[1;32m      8\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_types\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m (\n\u001b[1;32m      9\u001b[0m     AsyncByteStream,\n\u001b[1;32m     10\u001b[0m     FileContent,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m     14\u001b[0m     SyncByteStream,\n\u001b[1;32m     15\u001b[0m )\n\u001b[0;32m---> 16\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_utils\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m (\n\u001b[1;32m     17\u001b[0m     format_form_param,\n\u001b[1;32m     18\u001b[0m     guess_content_type,\n\u001b[1;32m     19\u001b[0m     peek_filelike_length,\n\u001b[1;32m     20\u001b[0m     primitive_value_to_str,\n\u001b[1;32m     21\u001b[0m     to_bytes,\n\u001b[1;32m     22\u001b[0m )\n\u001b[1;32m     25\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mget_multipart_boundary_from_content_type\u001b[39m(\n\u001b[1;32m     26\u001b[0m     content_type: \u001b[38;5;28mbytes\u001b[39m \u001b[38;5;241m|\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[1;32m     27\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28mbytes\u001b[39m \u001b[38;5;241m|\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m     28\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m content_type \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m content_type\u001b[38;5;241m.\u001b[39mstartswith(\u001b[38;5;124mb\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmultipart/form-data\u001b[39m\u001b[38;5;124m\"\u001b[39m):\n",
-      "File \u001b[0;32m/media/wassname/SGIronWolf/projects5/rrational/.venv/lib/python3.10/site-packages/httpx/_utils.py:14\u001b[0m\n\u001b[1;32m     11\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mpathlib\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m Path\n\u001b[1;32m     12\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01murllib\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mrequest\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m getproxies\n\u001b[0;32m---> 14\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01msniffio\u001b[39;00m\n\u001b[1;32m     16\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_types\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m PrimitiveData\n\u001b[1;32m     18\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m typing\u001b[38;5;241m.\u001b[39mTYPE_CHECKING:  \u001b[38;5;66;03m# pragma: no cover\u001b[39;00m\n",
-      "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'sniffio'"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
-    "import os\n",
     "import dotenv\n",
+    "\n",
     "dotenv.load_dotenv()\n",
     "from openai import OpenAI\n",
+    "\n",
     "client = OpenAI()\n",
     "\n",
     "import tiktoken\n",
+    "\n",
     "enc = tiktoken.encoding_for_model(\"gpt-4o-mini\")\n",
     "cost = 0.150 / 1e6"
    ]
@@ -1524,16 +1205,16 @@
    "outputs": [],
    "source": [
     "posts = []\n",
-    "fs = sorted(Path('../data/cache2').glob('*.md'))\n",
+    "fs = sorted(Path(\"../data/cache2\").glob(\"*.md\"))\n",
     "for f in fs:\n",
     "    s = f.open().read()\n",
     "    posts.append(s)\n",
     "\n",
     "\n",
-    "def get_post_context(urls:List[str], budget = 100000):\n",
+    "def get_post_context(urls: List[str], budget=100000):\n",
     "    # FIXME take in multiple urls and titles\n",
     "    # FIXME given a token or char budget\n",
-    "    assert len(title)>0\n",
+    "    assert len(urls) > 0\n",
     "    # from comments\n",
     "    # return df3.loc[title].comments\n",
     "\n",
@@ -1547,28 +1228,28 @@
     "            if url in post:\n",
     "                matches.append(post)\n",
     "                break\n",
-    "    \n",
-    "    budget_pp = budget / len(matches)\n",
-    "    s = ''\n",
-    "    for i in range(len(matches)):\n",
     "\n",
+    "    budget_pp = budget / len(matches)\n",
+    "    s = \"\"\n",
+    "    for i in range(len(matches)):\n",
     "        post = matches[i]\n",
     "\n",
     "        for url in urls:\n",
     "            if url in post:\n",
     "                ind = post.index(url)\n",
-    "        \n",
-    "        i0 = int(max(0, ind-budget_pp//4))\n",
-    "        i1 = int(min(len(post), ind+budget_pp//4*3))\n",
+    "\n",
+    "        i0 = int(max(0, ind - budget_pp // 4))\n",
+    "        i1 = int(min(len(post), ind + budget_pp // 4 * 3))\n",
     "        post_chunk = post[i0:i1]\n",
-    "        if i0>0:\n",
-    "            post_chunk = '...' + post_chunk\n",
-    "        if i1<len(post):\n",
-    "            post_chunk = post_chunk + '...'\n",
-    "                \n",
-    "        s += (f'\\n\\n----- Thread {ii} -----\\n\\n' + post_chunk)\n",
+    "        if i0 > 0:\n",
+    "            post_chunk = \"...\" + post_chunk\n",
+    "        if i1 < len(post):\n",
+    "            post_chunk = post_chunk + \"...\"\n",
+    "\n",
+    "        s += f\"\\n\\n----- Thread {ii} -----\\n\\n\" + post_chunk\n",
     "    return s\n",
     "\n",
+    "\n",
     "# url = df3.url[0].split('\\n')\n",
     "# print(url)\n",
     "# c = get_context(url, 400000)\n",
@@ -1588,40 +1269,69 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "\n",
+    "from typing import List, Optional\n",
     "\n",
     "from pydantic import BaseModel, Field\n",
-    "from typing import List, Optional, Dict\n",
+    "\n",
     "\n",
     "class FictionInfo(BaseModel):\n",
-    "    name: str\n",
-    "    description: str\n",
-    "    tags: List[str]\n",
-    "    rating: Optional[float] = Field(description='Your own impression of the user sentiment as a rating out of 10.')\n",
-    "    sentiment: Optional[str]\n",
-    "    reviews: List[str] = Field(description='Directly quote any reviews and attribute the username if known')\n",
-    "    reccomendation_summary: str = Field(description='tldr why users recommend (if they do).')\n",
-    "    disreccomendation_summary: str = Field(description='Brief summary of why users disrecommend it to other r/rational readers (if they do).')\n",
+    "    title: str\n",
+    "    description: str = Field(description=\"Brief, very concise, description of the work\")\n",
+    "    tags: List[str] = Field(\n",
+    "        description=\"\"\"Long list of descriptors: format (web serial, fanfic, lightnovel, short, complete, comic), genre (scifi, fantasy)\n",
+    "        Key elements (rational, timeloop, litrpg, progression, cultivation, isekai)\n",
+    "        Content notes (grimdark, romance, harem, queer, funny, NSFW)\n",
+    "        \"\"\"\n",
+    "    )\n",
     "\n",
-    "f_cache = Path('../outputs/.anycache3')\n",
-    "# @anycache(f_cache)\n",
-    "def get_llm_summary(name:str, context:str):\n",
+    "    # status: Optional[str] = Field(description=\"complete/ongoing/hiatus/abandoned\")\n",
+    "    # type: str = Field(description='e.g. fanfiction, original, comic, etc.')\n",
+    "\n",
+    "    reviews_quotes: List[str] = Field(\n",
+    "        description=\"Directly and fully quote exerpts from each users' comments about the fiction\"\n",
+    "    )\n",
+    "    reviews_summary: Optional[str] = Field(\n",
+    "        description=\"Structured summary of reviews including 1) what aspects users comment on, 2) why users recommend it 3) disrecommend it, 4) how many users like vs dislike it, etc.\"\n",
+    "    )\n",
+    "\n",
+    "    quality: float = Field(\n",
+    "        # ge=0.0, le=10.0,\n",
+    "        description=\"Overall user sentiment out of 10\"\n",
+    "    )\n",
+    "    rationality: Optional[float] = Field(\n",
+    "        # ge=0.0, le=10.0,\n",
+    "        description=\"Systematic worldbuilding, character competence, logical consistency. Where HPMOR is a 10 and Worm is a 5.\"\n",
+    "    )\n",
+    "    rating_writing: Optional[float]\n",
+    "    rating_plot: Optional[float]\n",
+    "    rating_character: Optional[float]\n",
+    "    rating_worldbuilding: Optional[float]\n",
+    "\n",
+    "\n",
+    "f_cache = Path(\"../outputs/.anycache3\")\n",
+    "\n",
+    "\n",
+    "@anycache(f_cache)\n",
+    "def get_llm_summary(name: str, context: str):\n",
     "    chat_completion = client.beta.chat.completions.parse(\n",
     "        messages=[\n",
-    "            {\"role\": \"system\", \"content\": \"You are Gwern Branwern, an internet librarian who specializes in rational fiction. You are summarising community reccomendations into a structured form.\"},\n",
+    "            {\n",
+    "                \"role\": \"system\",\n",
+    "                \"content\": \"You are Gwern Branwern, an internet librarian who specializes in rational fiction. You are summarising community reccomendations into a structured form.\",\n",
+    "            },\n",
     "            {\n",
     "                \"role\": \"user\",\n",
-    "                \"content\": f\"\"\"For u/gwern please summarize information about {name}. Quote users and attribute the username if known.\n",
+    "                \"content\": f\"\"\"For u/gwern please summarize structured information about {name}. Quote users in full and attribute the username if known.\n",
     "\n",
     "### Context:\n",
     "\n",
-    "{context}\"\"\"\n",
-    "            }\n",
+    "{context}\"\"\",\n",
+    "            },\n",
     "        ],\n",
     "        model=\"gpt-4o-mini\",\n",
     "        response_format=FictionInfo,\n",
     "    )\n",
-    "    \n",
+    "\n",
     "    return chat_completion.choices[0].message.parsed.__dict__"
    ]
   },
@@ -1631,7 +1341,9 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from IPython.display import display\n"
+    "from openai.lib._pydantic import to_strict_json_schema\n",
+    "\n",
+    "to_strict_json_schema(FictionInfo)"
    ]
   },
   {
@@ -1640,39 +1352,41 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "df4 = df3.copy()\n",
-    "df4['description'] = ''\n",
-    "df4['tags'] = None\n",
-    "df4['rating'] = None\n",
-    "df4['sentiment'] = None\n",
-    "df4['reviews'] = None\n",
+    "from IPython.display import display"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "llm_info = []\n",
     "\n",
     "\n",
-    "l = min(100, len(df4))\n",
+    "l = min(1000, len(df3))\n",
     "for i in tqdm(range(1, l)):\n",
-    "    title = df4.index[i]\n",
-    "    urls = df4.url[i]\n",
+    "    title = df3.index[i]\n",
+    "    urls = df3.url[i]\n",
     "\n",
-    "    context = get_post_context(urls)\n",
+    "    context = get_post_context(urls, budget=50000)\n",
     "    tokens = len(enc.encode(context))\n",
-    "    print(f\"Input Tokens: {tokens}. Input Cost: {cost * tokens:.4f} USD, for url {urls}\")\n",
+    "    print(\n",
+    "        f\"Input Tokens: {tokens}. Input Cost: {cost * tokens:.4f} USD, for url {urls}\"\n",
+    "    )\n",
     "\n",
     "    llm_data = get_llm_summary(urls, context)\n",
     "\n",
+    "    llm_data[\"title2\"] = title\n",
+    "    llm_data[\"url\"] = urls\n",
+    "\n",
     "    # print(f\"Content: {context}\")\n",
     "    # print(url, d)\n",
-    "    display(llm_data)\n",
+    "    # display(llm_data)\n",
     "\n",
-    "    # add the fields to the dataframe rows\n",
-    "    df4.loc[urls, 'description'] = llm_data['description']\n",
-    "    df4.loc[urls, 'tags'] = ','.join(llm_data['tags'])\n",
-    "    df4.loc[urls, 'rating'] = llm_data['rating']\n",
-    "    df4.loc[urls, 'sentiment'] = llm_data['sentiment']\n",
-    "    df4.loc[urls, 'reviews'] = ','.join(llm_data['reviews'])\n",
-    "    df4.loc[urls, 'name'] = llm_data['name']\n",
-    "    df4.loc[urls, 'reccomendation_summary'] = llm_data['reccomendation_summary']\n",
-    "    df4.loc[urls, 'disreccomendation_summary'] = llm_data['disreccomendation_summary']\n",
-    "    1/0"
+    "    llm_info.append(llm_data)\n",
+    "\n",
+    "    # 1/0"
    ]
   },
   {
@@ -1684,6 +1398,17 @@
     "urls"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_llm = pd.DataFrame(llm_info)\n",
+    "df_llm\n",
+    "# also join with df4"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
diff --git a/poetry.lock b/poetry.lock
index 50693278..8175a9e5 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -1,5 +1,16 @@
 # This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand.
 
+[[package]]
+name = "annotated-types"
+version = "0.7.0"
+description = "Reusable constraint types to use with typing.Annotated"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53"},
+    {file = "annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89"},
+]
+
 [[package]]
 name = "anycache"
 version = "2.2.0"
@@ -15,6 +26,28 @@ files = [
 dill = ">=0.3.7,<0.4.0"
 filelock = ">=3.13.0,<4.0.0"
 
+[[package]]
+name = "anyio"
+version = "4.7.0"
+description = "High level compatibility layer for multiple asynchronous event loop implementations"
+optional = false
+python-versions = ">=3.9"
+files = [
+    {file = "anyio-4.7.0-py3-none-any.whl", hash = "sha256:ea60c3723ab42ba6fff7e8ccb0488c898ec538ff4df1f1d5e642c3601d07e352"},
+    {file = "anyio-4.7.0.tar.gz", hash = "sha256:2f834749c602966b7d456a7567cafcb309f96482b5081d14ac93ccd457f9dd48"},
+]
+
+[package.dependencies]
+exceptiongroup = {version = ">=1.0.2", markers = "python_version < \"3.11\""}
+idna = ">=2.8"
+sniffio = ">=1.1"
+typing_extensions = {version = ">=4.5", markers = "python_version < \"3.13\""}
+
+[package.extras]
+doc = ["Sphinx (>=7.4,<8.0)", "packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx_rtd_theme"]
+test = ["anyio[trio]", "coverage[toml] (>=7)", "exceptiongroup (>=1.2.0)", "hypothesis (>=4.0)", "psutil (>=5.9)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "truststore (>=0.9.1)", "uvloop (>=0.21)"]
+trio = ["trio (>=0.26.1)"]
+
 [[package]]
 name = "appnope"
 version = "0.1.4"
@@ -26,6 +59,20 @@ files = [
     {file = "appnope-0.1.4.tar.gz", hash = "sha256:1de3860566df9caf38f01f86f65e0e13e379af54f9e4bee1e66b48f2efffd1ee"},
 ]
 
+[[package]]
+name = "ascii"
+version = "3.6"
+description = "Turns images into ascii art"
+optional = false
+python-versions = "*"
+files = [
+    {file = "ascii-3.6.tar.gz", hash = "sha256:b5ffc4cb20ac1789cac80a91319345f26a3e8f3c6d6feff3a220c1b6c5056141"},
+]
+
+[package.dependencies]
+Pillow = "*"
+urllib3 = "*"
+
 [[package]]
 name = "asttokens"
 version = "2.4.1"
@@ -44,6 +91,39 @@ six = ">=1.12.0"
 astroid = ["astroid (>=1,<2)", "astroid (>=2,<4)"]
 test = ["astroid (>=1,<2)", "astroid (>=2,<4)", "pytest"]
 
+[[package]]
+name = "attrs"
+version = "24.3.0"
+description = "Classes Without Boilerplate"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "attrs-24.3.0-py3-none-any.whl", hash = "sha256:ac96cd038792094f438ad1f6ff80837353805ac950cd2aa0e0625ef19850c308"},
+    {file = "attrs-24.3.0.tar.gz", hash = "sha256:8f5c07333d543103541ba7be0e2ce16eeee8130cb0b3f9238ab904ce1e85baff"},
+]
+
+[package.extras]
+benchmark = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-codspeed", "pytest-mypy-plugins", "pytest-xdist[psutil]"]
+cov = ["cloudpickle", "coverage[toml] (>=5.3)", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"]
+dev = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pre-commit-uv", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"]
+docs = ["cogapp", "furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier (<24.7)"]
+tests = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"]
+tests-mypy = ["mypy (>=1.11.1)", "pytest-mypy-plugins"]
+
+[[package]]
+name = "base58"
+version = "2.1.1"
+description = "Base58 and Base58Check implementation."
+optional = false
+python-versions = ">=3.5"
+files = [
+    {file = "base58-2.1.1-py3-none-any.whl", hash = "sha256:11a36f4d3ce51dfc1043f3218591ac4eb1ceb172919cebe05b52a5bcc8d245c2"},
+    {file = "base58-2.1.1.tar.gz", hash = "sha256:c5d0cb3f5b6e81e8e35da5754388ddcc6d0d14b6c6a132cb93d69ed580a7278c"},
+]
+
+[package.extras]
+tests = ["PyHamcrest (>=2.0.2)", "mypy", "pytest (>=4.6)", "pytest-benchmark", "pytest-cov", "pytest-flake8"]
+
 [[package]]
 name = "beautifulsoup4"
 version = "4.12.3"
@@ -169,6 +249,17 @@ files = [
 [package.dependencies]
 pycparser = "*"
 
+[[package]]
+name = "chardet"
+version = "5.2.0"
+description = "Universal encoding detector for Python 3"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "chardet-5.2.0-py3-none-any.whl", hash = "sha256:e1cf59446890a00105fe7b7912492ea04b6e6f06d4b742b2c788469e34c82970"},
+    {file = "chardet-5.2.0.tar.gz", hash = "sha256:1b3b6ff479a8c414bc3fa2c0852995695c4a026dcd6d0633b2dd092ca39c1cf7"},
+]
+
 [[package]]
 name = "charset-normalizer"
 version = "3.3.2"
@@ -396,6 +487,17 @@ mypy = ["contourpy[bokeh,docs]", "docutils-stubs", "mypy (==1.11.1)", "types-Pil
 test = ["Pillow", "contourpy[test-no-images]", "matplotlib"]
 test-no-images = ["pytest", "pytest-cov", "pytest-rerunfailures", "pytest-xdist", "wurlitzer"]
 
+[[package]]
+name = "cssselect"
+version = "1.2.0"
+description = "cssselect parses CSS3 Selectors and translates them to XPath 1.0"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "cssselect-1.2.0-py2.py3-none-any.whl", hash = "sha256:da1885f0c10b60c03ed5eccbb6b68d6eff248d91976fcde348f395d54c9fd35e"},
+    {file = "cssselect-1.2.0.tar.gz", hash = "sha256:666b19839cfaddb9ce9d36bfe4c969132c647b92fc9088c4e23f786b30f1b3dc"},
+]
+
 [[package]]
 name = "cycler"
 version = "0.12.1"
@@ -468,6 +570,31 @@ files = [
 graph = ["objgraph (>=1.7.2)"]
 profile = ["gprof2dot (>=2022.7.29)"]
 
+[[package]]
+name = "distro"
+version = "1.9.0"
+description = "Distro - an OS platform information API"
+optional = false
+python-versions = ">=3.6"
+files = [
+    {file = "distro-1.9.0-py3-none-any.whl", hash = "sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2"},
+    {file = "distro-1.9.0.tar.gz", hash = "sha256:2fa77c6fd8940f116ee1d6b94a2f90b13b5ea8d019b98bc8bafdcabcdd9bdbed"},
+]
+
+[[package]]
+name = "ebooklib"
+version = "0.18"
+description = "Ebook library which can handle EPUB2/EPUB3 and Kindle format"
+optional = false
+python-versions = "*"
+files = [
+    {file = "EbookLib-0.18.tar.gz", hash = "sha256:38562643a7bc94d9bf56e9930b4927e4e93b5d1d0917f697a6454db5a1c1a533"},
+]
+
+[package.dependencies]
+lxml = "*"
+six = "*"
+
 [[package]]
 name = "exceptiongroup"
 version = "1.2.2"
@@ -577,6 +704,150 @@ ufo = ["fs (>=2.2.0,<3)"]
 unicode = ["unicodedata2 (>=15.1.0)"]
 woff = ["brotli (>=1.0.1)", "brotlicffi (>=0.8.0)", "zopfli (>=0.1.4)"]
 
+[[package]]
+name = "grpcio"
+version = "1.68.1"
+description = "HTTP/2-based RPC framework"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "grpcio-1.68.1-cp310-cp310-linux_armv7l.whl", hash = "sha256:d35740e3f45f60f3c37b1e6f2f4702c23867b9ce21c6410254c9c682237da68d"},
+    {file = "grpcio-1.68.1-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:d99abcd61760ebb34bdff37e5a3ba333c5cc09feda8c1ad42547bea0416ada78"},
+    {file = "grpcio-1.68.1-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:f8261fa2a5f679abeb2a0a93ad056d765cdca1c47745eda3f2d87f874ff4b8c9"},
+    {file = "grpcio-1.68.1-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0feb02205a27caca128627bd1df4ee7212db051019a9afa76f4bb6a1a80ca95e"},
+    {file = "grpcio-1.68.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:919d7f18f63bcad3a0f81146188e90274fde800a94e35d42ffe9eadf6a9a6330"},
+    {file = "grpcio-1.68.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:963cc8d7d79b12c56008aabd8b457f400952dbea8997dd185f155e2f228db079"},
+    {file = "grpcio-1.68.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:ccf2ebd2de2d6661e2520dae293298a3803a98ebfc099275f113ce1f6c2a80f1"},
+    {file = "grpcio-1.68.1-cp310-cp310-win32.whl", hash = "sha256:2cc1fd04af8399971bcd4f43bd98c22d01029ea2e56e69c34daf2bf8470e47f5"},
+    {file = "grpcio-1.68.1-cp310-cp310-win_amd64.whl", hash = "sha256:ee2e743e51cb964b4975de572aa8fb95b633f496f9fcb5e257893df3be854746"},
+    {file = "grpcio-1.68.1-cp311-cp311-linux_armv7l.whl", hash = "sha256:55857c71641064f01ff0541a1776bfe04a59db5558e82897d35a7793e525774c"},
+    {file = "grpcio-1.68.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:4b177f5547f1b995826ef529d2eef89cca2f830dd8b2c99ffd5fde4da734ba73"},
+    {file = "grpcio-1.68.1-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:3522c77d7e6606d6665ec8d50e867f13f946a4e00c7df46768f1c85089eae515"},
+    {file = "grpcio-1.68.1-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9d1fae6bbf0816415b81db1e82fb3bf56f7857273c84dcbe68cbe046e58e1ccd"},
+    {file = "grpcio-1.68.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:298ee7f80e26f9483f0b6f94cc0a046caf54400a11b644713bb5b3d8eb387600"},
+    {file = "grpcio-1.68.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:cbb5780e2e740b6b4f2d208e90453591036ff80c02cc605fea1af8e6fc6b1bbe"},
+    {file = "grpcio-1.68.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:ddda1aa22495d8acd9dfbafff2866438d12faec4d024ebc2e656784d96328ad0"},
+    {file = "grpcio-1.68.1-cp311-cp311-win32.whl", hash = "sha256:b33bd114fa5a83f03ec6b7b262ef9f5cac549d4126f1dc702078767b10c46ed9"},
+    {file = "grpcio-1.68.1-cp311-cp311-win_amd64.whl", hash = "sha256:7f20ebec257af55694d8f993e162ddf0d36bd82d4e57f74b31c67b3c6d63d8b2"},
+    {file = "grpcio-1.68.1-cp312-cp312-linux_armv7l.whl", hash = "sha256:8829924fffb25386995a31998ccbbeaa7367223e647e0122043dfc485a87c666"},
+    {file = "grpcio-1.68.1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:3aed6544e4d523cd6b3119b0916cef3d15ef2da51e088211e4d1eb91a6c7f4f1"},
+    {file = "grpcio-1.68.1-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:4efac5481c696d5cb124ff1c119a78bddbfdd13fc499e3bc0ca81e95fc573684"},
+    {file = "grpcio-1.68.1-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ab2d912ca39c51f46baf2a0d92aa265aa96b2443266fc50d234fa88bf877d8e"},
+    {file = "grpcio-1.68.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95c87ce2a97434dffe7327a4071839ab8e8bffd0054cc74cbe971fba98aedd60"},
+    {file = "grpcio-1.68.1-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:e4842e4872ae4ae0f5497bf60a0498fa778c192cc7a9e87877abd2814aca9475"},
+    {file = "grpcio-1.68.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:255b1635b0ed81e9f91da4fcc8d43b7ea5520090b9a9ad9340d147066d1d3613"},
+    {file = "grpcio-1.68.1-cp312-cp312-win32.whl", hash = "sha256:7dfc914cc31c906297b30463dde0b9be48e36939575eaf2a0a22a8096e69afe5"},
+    {file = "grpcio-1.68.1-cp312-cp312-win_amd64.whl", hash = "sha256:a0c8ddabef9c8f41617f213e527254c41e8b96ea9d387c632af878d05db9229c"},
+    {file = "grpcio-1.68.1-cp313-cp313-linux_armv7l.whl", hash = "sha256:a47faedc9ea2e7a3b6569795c040aae5895a19dde0c728a48d3c5d7995fda385"},
+    {file = "grpcio-1.68.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:390eee4225a661c5cd133c09f5da1ee3c84498dc265fd292a6912b65c421c78c"},
+    {file = "grpcio-1.68.1-cp313-cp313-manylinux_2_17_aarch64.whl", hash = "sha256:66a24f3d45c33550703f0abb8b656515b0ab777970fa275693a2f6dc8e35f1c1"},
+    {file = "grpcio-1.68.1-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c08079b4934b0bf0a8847f42c197b1d12cba6495a3d43febd7e99ecd1cdc8d54"},
+    {file = "grpcio-1.68.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8720c25cd9ac25dd04ee02b69256d0ce35bf8a0f29e20577427355272230965a"},
+    {file = "grpcio-1.68.1-cp313-cp313-musllinux_1_1_i686.whl", hash = "sha256:04cfd68bf4f38f5bb959ee2361a7546916bd9a50f78617a346b3aeb2b42e2161"},
+    {file = "grpcio-1.68.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:c28848761a6520c5c6071d2904a18d339a796ebe6b800adc8b3f474c5ce3c3ad"},
+    {file = "grpcio-1.68.1-cp313-cp313-win32.whl", hash = "sha256:77d65165fc35cff6e954e7fd4229e05ec76102d4406d4576528d3a3635fc6172"},
+    {file = "grpcio-1.68.1-cp313-cp313-win_amd64.whl", hash = "sha256:a8040f85dcb9830d8bbb033ae66d272614cec6faceee88d37a88a9bd1a7a704e"},
+    {file = "grpcio-1.68.1-cp38-cp38-linux_armv7l.whl", hash = "sha256:eeb38ff04ab6e5756a2aef6ad8d94e89bb4a51ef96e20f45c44ba190fa0bcaad"},
+    {file = "grpcio-1.68.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:8a3869a6661ec8f81d93f4597da50336718bde9eb13267a699ac7e0a1d6d0bea"},
+    {file = "grpcio-1.68.1-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:2c4cec6177bf325eb6faa6bd834d2ff6aa8bb3b29012cceb4937b86f8b74323c"},
+    {file = "grpcio-1.68.1-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:12941d533f3cd45d46f202e3667be8ebf6bcb3573629c7ec12c3e211d99cfccf"},
+    {file = "grpcio-1.68.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80af6f1e69c5e68a2be529990684abdd31ed6622e988bf18850075c81bb1ad6e"},
+    {file = "grpcio-1.68.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:e8dbe3e00771bfe3d04feed8210fc6617006d06d9a2679b74605b9fed3e8362c"},
+    {file = "grpcio-1.68.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:83bbf5807dc3ee94ce1de2dfe8a356e1d74101e4b9d7aa8c720cc4818a34aded"},
+    {file = "grpcio-1.68.1-cp38-cp38-win32.whl", hash = "sha256:8cb620037a2fd9eeee97b4531880e439ebfcd6d7d78f2e7dcc3726428ab5ef63"},
+    {file = "grpcio-1.68.1-cp38-cp38-win_amd64.whl", hash = "sha256:52fbf85aa71263380d330f4fce9f013c0798242e31ede05fcee7fbe40ccfc20d"},
+    {file = "grpcio-1.68.1-cp39-cp39-linux_armv7l.whl", hash = "sha256:cb400138e73969eb5e0535d1d06cae6a6f7a15f2cc74add320e2130b8179211a"},
+    {file = "grpcio-1.68.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:a1b988b40f2fd9de5c820f3a701a43339d8dcf2cb2f1ca137e2c02671cc83ac1"},
+    {file = "grpcio-1.68.1-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:96f473cdacfdd506008a5d7579c9f6a7ff245a9ade92c3c0265eb76cc591914f"},
+    {file = "grpcio-1.68.1-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:37ea3be171f3cf3e7b7e412a98b77685eba9d4fd67421f4a34686a63a65d99f9"},
+    {file = "grpcio-1.68.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ceb56c4285754e33bb3c2fa777d055e96e6932351a3082ce3559be47f8024f0"},
+    {file = "grpcio-1.68.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:dffd29a2961f3263a16d73945b57cd44a8fd0b235740cb14056f0612329b345e"},
+    {file = "grpcio-1.68.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:025f790c056815b3bf53da850dd70ebb849fd755a4b1ac822cb65cd631e37d43"},
+    {file = "grpcio-1.68.1-cp39-cp39-win32.whl", hash = "sha256:1098f03dedc3b9810810568060dea4ac0822b4062f537b0f53aa015269be0a76"},
+    {file = "grpcio-1.68.1-cp39-cp39-win_amd64.whl", hash = "sha256:334ab917792904245a028f10e803fcd5b6f36a7b2173a820c0b5b076555825e1"},
+    {file = "grpcio-1.68.1.tar.gz", hash = "sha256:44a8502dd5de653ae6a73e2de50a401d84184f0331d0ac3daeb044e66d5c5054"},
+]
+
+[package.extras]
+protobuf = ["grpcio-tools (>=1.68.1)"]
+
+[[package]]
+name = "h11"
+version = "0.14.0"
+description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "h11-0.14.0-py3-none-any.whl", hash = "sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761"},
+    {file = "h11-0.14.0.tar.gz", hash = "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d"},
+]
+
+[[package]]
+name = "html5lib"
+version = "1.1"
+description = "HTML parser based on the WHATWG HTML specification"
+optional = false
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
+files = [
+    {file = "html5lib-1.1-py2.py3-none-any.whl", hash = "sha256:0d78f8fde1c230e99fe37986a60526d7049ed4bf8a9fadbad5f00e22e58e041d"},
+    {file = "html5lib-1.1.tar.gz", hash = "sha256:b2e5b40261e20f354d198eae92afc10d750afb487ed5e50f9c4eaf07c184146f"},
+]
+
+[package.dependencies]
+six = ">=1.9"
+webencodings = "*"
+
+[package.extras]
+all = ["chardet (>=2.2)", "genshi", "lxml"]
+chardet = ["chardet (>=2.2)"]
+genshi = ["genshi"]
+lxml = ["lxml"]
+
+[[package]]
+name = "httpcore"
+version = "1.0.7"
+description = "A minimal low-level HTTP client."
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "httpcore-1.0.7-py3-none-any.whl", hash = "sha256:a3fff8f43dc260d5bd363d9f9cf1830fa3a458b332856f34282de498ed420edd"},
+    {file = "httpcore-1.0.7.tar.gz", hash = "sha256:8551cb62a169ec7162ac7be8d4817d561f60e08eaa485234898414bb5a8a0b4c"},
+]
+
+[package.dependencies]
+certifi = "*"
+h11 = ">=0.13,<0.15"
+
+[package.extras]
+asyncio = ["anyio (>=4.0,<5.0)"]
+http2 = ["h2 (>=3,<5)"]
+socks = ["socksio (==1.*)"]
+trio = ["trio (>=0.22.0,<1.0)"]
+
+[[package]]
+name = "httpx"
+version = "0.28.1"
+description = "The next generation HTTP client."
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad"},
+    {file = "httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc"},
+]
+
+[package.dependencies]
+anyio = "*"
+certifi = "*"
+httpcore = "==1.*"
+idna = "*"
+
+[package.extras]
+brotli = ["brotli", "brotlicffi"]
+cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<14)"]
+http2 = ["h2 (>=3,<5)"]
+socks = ["socksio (==1.*)"]
+zstd = ["zstandard (>=0.18.0)"]
+
 [[package]]
 name = "idna"
 version = "3.8"
@@ -716,6 +987,107 @@ MarkupSafe = ">=2.0"
 [package.extras]
 i18n = ["Babel (>=2.7)"]
 
+[[package]]
+name = "jiter"
+version = "0.8.2"
+description = "Fast iterable JSON parser."
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "jiter-0.8.2-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:ca8577f6a413abe29b079bc30f907894d7eb07a865c4df69475e868d73e71c7b"},
+    {file = "jiter-0.8.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b25bd626bde7fb51534190c7e3cb97cee89ee76b76d7585580e22f34f5e3f393"},
+    {file = "jiter-0.8.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d5c826a221851a8dc028eb6d7d6429ba03184fa3c7e83ae01cd6d3bd1d4bd17d"},
+    {file = "jiter-0.8.2-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d35c864c2dff13dfd79fb070fc4fc6235d7b9b359efe340e1261deb21b9fcb66"},
+    {file = "jiter-0.8.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f557c55bc2b7676e74d39d19bcb8775ca295c7a028246175d6a8b431e70835e5"},
+    {file = "jiter-0.8.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:580ccf358539153db147e40751a0b41688a5ceb275e6f3e93d91c9467f42b2e3"},
+    {file = "jiter-0.8.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:af102d3372e917cffce49b521e4c32c497515119dc7bd8a75665e90a718bbf08"},
+    {file = "jiter-0.8.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:cadcc978f82397d515bb2683fc0d50103acff2a180552654bb92d6045dec2c49"},
+    {file = "jiter-0.8.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:ba5bdf56969cad2019d4e8ffd3f879b5fdc792624129741d3d83fc832fef8c7d"},
+    {file = "jiter-0.8.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:3b94a33a241bee9e34b8481cdcaa3d5c2116f575e0226e421bed3f7a6ea71cff"},
+    {file = "jiter-0.8.2-cp310-cp310-win32.whl", hash = "sha256:6e5337bf454abddd91bd048ce0dca5134056fc99ca0205258766db35d0a2ea43"},
+    {file = "jiter-0.8.2-cp310-cp310-win_amd64.whl", hash = "sha256:4a9220497ca0cb1fe94e3f334f65b9b5102a0b8147646118f020d8ce1de70105"},
+    {file = "jiter-0.8.2-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:2dd61c5afc88a4fda7d8b2cf03ae5947c6ac7516d32b7a15bf4b49569a5c076b"},
+    {file = "jiter-0.8.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a6c710d657c8d1d2adbbb5c0b0c6bfcec28fd35bd6b5f016395f9ac43e878a15"},
+    {file = "jiter-0.8.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a9584de0cd306072635fe4b89742bf26feae858a0683b399ad0c2509011b9dc0"},
+    {file = "jiter-0.8.2-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5a90a923338531b7970abb063cfc087eebae6ef8ec8139762007188f6bc69a9f"},
+    {file = "jiter-0.8.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d21974d246ed0181558087cd9f76e84e8321091ebfb3a93d4c341479a736f099"},
+    {file = "jiter-0.8.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:32475a42b2ea7b344069dc1e81445cfc00b9d0e3ca837f0523072432332e9f74"},
+    {file = "jiter-0.8.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8b9931fd36ee513c26b5bf08c940b0ac875de175341cbdd4fa3be109f0492586"},
+    {file = "jiter-0.8.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ce0820f4a3a59ddced7fce696d86a096d5cc48d32a4183483a17671a61edfddc"},
+    {file = "jiter-0.8.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:8ffc86ae5e3e6a93765d49d1ab47b6075a9c978a2b3b80f0f32628f39caa0c88"},
+    {file = "jiter-0.8.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5127dc1abd809431172bc3fbe8168d6b90556a30bb10acd5ded41c3cfd6f43b6"},
+    {file = "jiter-0.8.2-cp311-cp311-win32.whl", hash = "sha256:66227a2c7b575720c1871c8800d3a0122bb8ee94edb43a5685aa9aceb2782d44"},
+    {file = "jiter-0.8.2-cp311-cp311-win_amd64.whl", hash = "sha256:cde031d8413842a1e7501e9129b8e676e62a657f8ec8166e18a70d94d4682855"},
+    {file = "jiter-0.8.2-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:e6ec2be506e7d6f9527dae9ff4b7f54e68ea44a0ef6b098256ddf895218a2f8f"},
+    {file = "jiter-0.8.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:76e324da7b5da060287c54f2fabd3db5f76468006c811831f051942bf68c9d44"},
+    {file = "jiter-0.8.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:180a8aea058f7535d1c84183c0362c710f4750bef66630c05f40c93c2b152a0f"},
+    {file = "jiter-0.8.2-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:025337859077b41548bdcbabe38698bcd93cfe10b06ff66617a48ff92c9aec60"},
+    {file = "jiter-0.8.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ecff0dc14f409599bbcafa7e470c00b80f17abc14d1405d38ab02e4b42e55b57"},
+    {file = "jiter-0.8.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ffd9fee7d0775ebaba131f7ca2e2d83839a62ad65e8e02fe2bd8fc975cedeb9e"},
+    {file = "jiter-0.8.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:14601dcac4889e0a1c75ccf6a0e4baf70dbc75041e51bcf8d0e9274519df6887"},
+    {file = "jiter-0.8.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:92249669925bc1c54fcd2ec73f70f2c1d6a817928480ee1c65af5f6b81cdf12d"},
+    {file = "jiter-0.8.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:e725edd0929fa79f8349ab4ec7f81c714df51dc4e991539a578e5018fa4a7152"},
+    {file = "jiter-0.8.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:bf55846c7b7a680eebaf9c3c48d630e1bf51bdf76c68a5f654b8524335b0ad29"},
+    {file = "jiter-0.8.2-cp312-cp312-win32.whl", hash = "sha256:7efe4853ecd3d6110301665a5178b9856be7e2a9485f49d91aa4d737ad2ae49e"},
+    {file = "jiter-0.8.2-cp312-cp312-win_amd64.whl", hash = "sha256:83c0efd80b29695058d0fd2fa8a556490dbce9804eac3e281f373bbc99045f6c"},
+    {file = "jiter-0.8.2-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:ca1f08b8e43dc3bd0594c992fb1fd2f7ce87f7bf0d44358198d6da8034afdf84"},
+    {file = "jiter-0.8.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:5672a86d55416ccd214c778efccf3266b84f87b89063b582167d803246354be4"},
+    {file = "jiter-0.8.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:58dc9bc9767a1101f4e5e22db1b652161a225874d66f0e5cb8e2c7d1c438b587"},
+    {file = "jiter-0.8.2-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:37b2998606d6dadbb5ccda959a33d6a5e853252d921fec1792fc902351bb4e2c"},
+    {file = "jiter-0.8.2-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4ab9a87f3784eb0e098f84a32670cfe4a79cb6512fd8f42ae3d0709f06405d18"},
+    {file = "jiter-0.8.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:79aec8172b9e3c6d05fd4b219d5de1ac616bd8da934107325a6c0d0e866a21b6"},
+    {file = "jiter-0.8.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:711e408732d4e9a0208008e5892c2966b485c783cd2d9a681f3eb147cf36c7ef"},
+    {file = "jiter-0.8.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:653cf462db4e8c41995e33d865965e79641ef45369d8a11f54cd30888b7e6ff1"},
+    {file = "jiter-0.8.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:9c63eaef32b7bebac8ebebf4dabebdbc6769a09c127294db6babee38e9f405b9"},
+    {file = "jiter-0.8.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:eb21aaa9a200d0a80dacc7a81038d2e476ffe473ffdd9c91eb745d623561de05"},
+    {file = "jiter-0.8.2-cp313-cp313-win32.whl", hash = "sha256:789361ed945d8d42850f919342a8665d2dc79e7e44ca1c97cc786966a21f627a"},
+    {file = "jiter-0.8.2-cp313-cp313-win_amd64.whl", hash = "sha256:ab7f43235d71e03b941c1630f4b6e3055d46b6cb8728a17663eaac9d8e83a865"},
+    {file = "jiter-0.8.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:b426f72cd77da3fec300ed3bc990895e2dd6b49e3bfe6c438592a3ba660e41ca"},
+    {file = "jiter-0.8.2-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b2dd880785088ff2ad21ffee205e58a8c1ddabc63612444ae41e5e4b321b39c0"},
+    {file = "jiter-0.8.2-cp313-cp313t-win_amd64.whl", hash = "sha256:3ac9f578c46f22405ff7f8b1f5848fb753cc4b8377fbec8470a7dc3997ca7566"},
+    {file = "jiter-0.8.2-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:9e1fa156ee9454642adb7e7234a383884452532bc9d53d5af2d18d98ada1d79c"},
+    {file = "jiter-0.8.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:0cf5dfa9956d96ff2efb0f8e9c7d055904012c952539a774305aaaf3abdf3d6c"},
+    {file = "jiter-0.8.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e52bf98c7e727dd44f7c4acb980cb988448faeafed8433c867888268899b298b"},
+    {file = "jiter-0.8.2-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a2ecaa3c23e7a7cf86d00eda3390c232f4d533cd9ddea4b04f5d0644faf642c5"},
+    {file = "jiter-0.8.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:08d4c92bf480e19fc3f2717c9ce2aa31dceaa9163839a311424b6862252c943e"},
+    {file = "jiter-0.8.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:99d9a1eded738299ba8e106c6779ce5c3893cffa0e32e4485d680588adae6db8"},
+    {file = "jiter-0.8.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d20be8b7f606df096e08b0b1b4a3c6f0515e8dac296881fe7461dfa0fb5ec817"},
+    {file = "jiter-0.8.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d33f94615fcaf872f7fd8cd98ac3b429e435c77619777e8a449d9d27e01134d1"},
+    {file = "jiter-0.8.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:317b25e98a35ffec5c67efe56a4e9970852632c810d35b34ecdd70cc0e47b3b6"},
+    {file = "jiter-0.8.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:fc9043259ee430ecd71d178fccabd8c332a3bf1e81e50cae43cc2b28d19e4cb7"},
+    {file = "jiter-0.8.2-cp38-cp38-win32.whl", hash = "sha256:fc5adda618205bd4678b146612ce44c3cbfdee9697951f2c0ffdef1f26d72b63"},
+    {file = "jiter-0.8.2-cp38-cp38-win_amd64.whl", hash = "sha256:cd646c827b4f85ef4a78e4e58f4f5854fae0caf3db91b59f0d73731448a970c6"},
+    {file = "jiter-0.8.2-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:e41e75344acef3fc59ba4765df29f107f309ca9e8eace5baacabd9217e52a5ee"},
+    {file = "jiter-0.8.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:7f22b16b35d5c1df9dfd58843ab2cd25e6bf15191f5a236bed177afade507bfc"},
+    {file = "jiter-0.8.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f7200b8f7619d36aa51c803fd52020a2dfbea36ffec1b5e22cab11fd34d95a6d"},
+    {file = "jiter-0.8.2-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:70bf4c43652cc294040dbb62256c83c8718370c8b93dd93d934b9a7bf6c4f53c"},
+    {file = "jiter-0.8.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f9d471356dc16f84ed48768b8ee79f29514295c7295cb41e1133ec0b2b8d637d"},
+    {file = "jiter-0.8.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:859e8eb3507894093d01929e12e267f83b1d5f6221099d3ec976f0c995cb6bd9"},
+    {file = "jiter-0.8.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eaa58399c01db555346647a907b4ef6d4f584b123943be6ed5588c3f2359c9f4"},
+    {file = "jiter-0.8.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8f2d5ed877f089862f4c7aacf3a542627c1496f972a34d0474ce85ee7d939c27"},
+    {file = "jiter-0.8.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:03c9df035d4f8d647f8c210ddc2ae0728387275340668fb30d2421e17d9a0841"},
+    {file = "jiter-0.8.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8bd2a824d08d8977bb2794ea2682f898ad3d8837932e3a74937e93d62ecbb637"},
+    {file = "jiter-0.8.2-cp39-cp39-win32.whl", hash = "sha256:ca29b6371ebc40e496995c94b988a101b9fbbed48a51190a4461fcb0a68b4a36"},
+    {file = "jiter-0.8.2-cp39-cp39-win_amd64.whl", hash = "sha256:1c0dfbd1be3cbefc7510102370d86e35d1d53e5a93d48519688b1bf0f761160a"},
+    {file = "jiter-0.8.2.tar.gz", hash = "sha256:cd73d3e740666d0e639f678adb176fad25c1bcbdae88d8d7b857e1783bb4212d"},
+]
+
+[[package]]
+name = "js2py"
+version = "0.74"
+description = "JavaScript to Python Translator & JavaScript interpreter written in 100% pure Python."
+optional = false
+python-versions = "*"
+files = [
+    {file = "Js2Py-0.74-py3-none-any.whl", hash = "sha256:40a508a79e2f8d624e3f2e604f90a1e6f46ac75b416d7f4745939ff4a2e95e09"},
+    {file = "Js2Py-0.74.tar.gz", hash = "sha256:39f3a6aa8469180efba3c8677271df27c31332fd1b471df1af2af58b87b8972f"},
+]
+
+[package.dependencies]
+pyjsparser = ">=2.5.1"
+six = ">=1.10"
+tzlocal = ">=1.2"
+
 [[package]]
 name = "jupyter-client"
 version = "8.6.2"
@@ -882,6 +1254,43 @@ files = [
     {file = "kiwisolver-1.4.5.tar.gz", hash = "sha256:e57e563a57fb22a142da34f38acc2fc1a5c864bc29ca1517a88abc963e60d6ec"},
 ]
 
+[[package]]
+name = "lightnovel-crawler"
+version = "3.7.2"
+description = "An app to download novels from online sources and generate e-books."
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "lightnovel_crawler-3.7.2-py3-none-any.whl", hash = "sha256:a2ab5bfd8cf113916668886373d88214b7285298c42c216c29735cdf15a887e1"},
+]
+
+[package.dependencies]
+ascii = "*"
+base58 = ">=2.1.1,<2.2.0"
+beautifulsoup4 = ">=4.8.0,<5.0.0"
+cloudscraper = ">=1.2.71"
+colorama = ">=0.4.0,<0.5.0"
+ebooklib = ">=0.17.0,<1.0.0"
+html5lib = ">=1.1,<2.0"
+js2py = ">=0.71"
+lxml = ">=4.0.0,<5.0.0"
+packaging = "*"
+pillow = ">=6.0.0"
+prompt-toolkit = ">=3.0,<4.0"
+pycryptodome = ">=3.0.0,<4.0.0"
+pyease-grpc = ">=1.6.0"
+python-box = ">=6.0.0,<8.0.0"
+python-dotenv = ">=0.15.0,<2.0.0"
+python-slugify = ">=4.0.0,<9.0.0"
+questionary = ">=1.6.0"
+readability-lxml = ">=0.8.0,<1.0.0"
+regex = "*"
+requests = ">=2.20.0,<3.0.0"
+selenium = ">=3.141.0"
+tqdm = ">=4.60,<5.0"
+undetected-chromedriver = ">=3.5.3"
+webdriver-manager = "*"
+
 [[package]]
 name = "loguru"
 version = "0.7.2"
@@ -900,6 +1309,114 @@ win32-setctime = {version = ">=1.0.0", markers = "sys_platform == \"win32\""}
 [package.extras]
 dev = ["Sphinx (==7.2.5)", "colorama (==0.4.5)", "colorama (==0.4.6)", "exceptiongroup (==1.1.3)", "freezegun (==1.1.0)", "freezegun (==1.2.2)", "mypy (==v0.910)", "mypy (==v0.971)", "mypy (==v1.4.1)", "mypy (==v1.5.1)", "pre-commit (==3.4.0)", "pytest (==6.1.2)", "pytest (==7.4.0)", "pytest-cov (==2.12.1)", "pytest-cov (==4.1.0)", "pytest-mypy-plugins (==1.9.3)", "pytest-mypy-plugins (==3.0.0)", "sphinx-autobuild (==2021.3.14)", "sphinx-rtd-theme (==1.3.0)", "tox (==3.27.1)", "tox (==4.11.0)"]
 
+[[package]]
+name = "lxml"
+version = "4.9.4"
+description = "Powerful and Pythonic XML processing library combining libxml2/libxslt with the ElementTree API."
+optional = false
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, != 3.4.*"
+files = [
+    {file = "lxml-4.9.4-cp27-cp27m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e214025e23db238805a600f1f37bf9f9a15413c7bf5f9d6ae194f84980c78722"},
+    {file = "lxml-4.9.4-cp27-cp27m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:ec53a09aee61d45e7dbe7e91252ff0491b6b5fee3d85b2d45b173d8ab453efc1"},
+    {file = "lxml-4.9.4-cp27-cp27m-win32.whl", hash = "sha256:7d1d6c9e74c70ddf524e3c09d9dc0522aba9370708c2cb58680ea40174800013"},
+    {file = "lxml-4.9.4-cp27-cp27m-win_amd64.whl", hash = "sha256:cb53669442895763e61df5c995f0e8361b61662f26c1b04ee82899c2789c8f69"},
+    {file = "lxml-4.9.4-cp27-cp27mu-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:647bfe88b1997d7ae8d45dabc7c868d8cb0c8412a6e730a7651050b8c7289cf2"},
+    {file = "lxml-4.9.4-cp27-cp27mu-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:4d973729ce04784906a19108054e1fd476bc85279a403ea1a72fdb051c76fa48"},
+    {file = "lxml-4.9.4-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:056a17eaaf3da87a05523472ae84246f87ac2f29a53306466c22e60282e54ff8"},
+    {file = "lxml-4.9.4-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:aaa5c173a26960fe67daa69aa93d6d6a1cd714a6eb13802d4e4bd1d24a530644"},
+    {file = "lxml-4.9.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:647459b23594f370c1c01768edaa0ba0959afc39caeeb793b43158bb9bb6a663"},
+    {file = "lxml-4.9.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:bdd9abccd0927673cffe601d2c6cdad1c9321bf3437a2f507d6b037ef91ea307"},
+    {file = "lxml-4.9.4-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:00e91573183ad273e242db5585b52670eddf92bacad095ce25c1e682da14ed91"},
+    {file = "lxml-4.9.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:a602ed9bd2c7d85bd58592c28e101bd9ff9c718fbde06545a70945ffd5d11868"},
+    {file = "lxml-4.9.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:de362ac8bc962408ad8fae28f3967ce1a262b5d63ab8cefb42662566737f1dc7"},
+    {file = "lxml-4.9.4-cp310-cp310-win32.whl", hash = "sha256:33714fcf5af4ff7e70a49731a7cc8fd9ce910b9ac194f66eaa18c3cc0a4c02be"},
+    {file = "lxml-4.9.4-cp310-cp310-win_amd64.whl", hash = "sha256:d3caa09e613ece43ac292fbed513a4bce170681a447d25ffcbc1b647d45a39c5"},
+    {file = "lxml-4.9.4-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:359a8b09d712df27849e0bcb62c6a3404e780b274b0b7e4c39a88826d1926c28"},
+    {file = "lxml-4.9.4-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:43498ea734ccdfb92e1886dfedaebeb81178a241d39a79d5351ba2b671bff2b2"},
+    {file = "lxml-4.9.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:4855161013dfb2b762e02b3f4d4a21cc7c6aec13c69e3bffbf5022b3e708dd97"},
+    {file = "lxml-4.9.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:c71b5b860c5215fdbaa56f715bc218e45a98477f816b46cfde4a84d25b13274e"},
+    {file = "lxml-4.9.4-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:9a2b5915c333e4364367140443b59f09feae42184459b913f0f41b9fed55794a"},
+    {file = "lxml-4.9.4-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:d82411dbf4d3127b6cde7da0f9373e37ad3a43e89ef374965465928f01c2b979"},
+    {file = "lxml-4.9.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:273473d34462ae6e97c0f4e517bd1bf9588aa67a1d47d93f760a1282640e24ac"},
+    {file = "lxml-4.9.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:389d2b2e543b27962990ab529ac6720c3dded588cc6d0f6557eec153305a3622"},
+    {file = "lxml-4.9.4-cp311-cp311-win32.whl", hash = "sha256:8aecb5a7f6f7f8fe9cac0bcadd39efaca8bbf8d1bf242e9f175cbe4c925116c3"},
+    {file = "lxml-4.9.4-cp311-cp311-win_amd64.whl", hash = "sha256:c7721a3ef41591341388bb2265395ce522aba52f969d33dacd822da8f018aff8"},
+    {file = "lxml-4.9.4-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:dbcb2dc07308453db428a95a4d03259bd8caea97d7f0776842299f2d00c72fc8"},
+    {file = "lxml-4.9.4-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:01bf1df1db327e748dcb152d17389cf6d0a8c5d533ef9bab781e9d5037619229"},
+    {file = "lxml-4.9.4-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:e8f9f93a23634cfafbad6e46ad7d09e0f4a25a2400e4a64b1b7b7c0fbaa06d9d"},
+    {file = "lxml-4.9.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:3f3f00a9061605725df1816f5713d10cd94636347ed651abdbc75828df302b20"},
+    {file = "lxml-4.9.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:953dd5481bd6252bd480d6ec431f61d7d87fdcbbb71b0d2bdcfc6ae00bb6fb10"},
+    {file = "lxml-4.9.4-cp312-cp312-win32.whl", hash = "sha256:266f655d1baff9c47b52f529b5f6bec33f66042f65f7c56adde3fcf2ed62ae8b"},
+    {file = "lxml-4.9.4-cp312-cp312-win_amd64.whl", hash = "sha256:f1faee2a831fe249e1bae9cbc68d3cd8a30f7e37851deee4d7962b17c410dd56"},
+    {file = "lxml-4.9.4-cp35-cp35m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:23d891e5bdc12e2e506e7d225d6aa929e0a0368c9916c1fddefab88166e98b20"},
+    {file = "lxml-4.9.4-cp35-cp35m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:e96a1788f24d03e8d61679f9881a883ecdf9c445a38f9ae3f3f193ab6c591c66"},
+    {file = "lxml-4.9.4-cp36-cp36m-macosx_11_0_x86_64.whl", hash = "sha256:5557461f83bb7cc718bc9ee1f7156d50e31747e5b38d79cf40f79ab1447afd2d"},
+    {file = "lxml-4.9.4-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:fdb325b7fba1e2c40b9b1db407f85642e32404131c08480dd652110fc908561b"},
+    {file = "lxml-4.9.4-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3d74d4a3c4b8f7a1f676cedf8e84bcc57705a6d7925e6daef7a1e54ae543a197"},
+    {file = "lxml-4.9.4-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:ac7674d1638df129d9cb4503d20ffc3922bd463c865ef3cb412f2c926108e9a4"},
+    {file = "lxml-4.9.4-cp36-cp36m-manylinux_2_28_x86_64.whl", hash = "sha256:ddd92e18b783aeb86ad2132d84a4b795fc5ec612e3545c1b687e7747e66e2b53"},
+    {file = "lxml-4.9.4-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2bd9ac6e44f2db368ef8986f3989a4cad3de4cd55dbdda536e253000c801bcc7"},
+    {file = "lxml-4.9.4-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:bc354b1393dce46026ab13075f77b30e40b61b1a53e852e99d3cc5dd1af4bc85"},
+    {file = "lxml-4.9.4-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:f836f39678cb47c9541f04d8ed4545719dc31ad850bf1832d6b4171e30d65d23"},
+    {file = "lxml-4.9.4-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:9c131447768ed7bc05a02553d939e7f0e807e533441901dd504e217b76307745"},
+    {file = "lxml-4.9.4-cp36-cp36m-win32.whl", hash = "sha256:bafa65e3acae612a7799ada439bd202403414ebe23f52e5b17f6ffc2eb98c2be"},
+    {file = "lxml-4.9.4-cp36-cp36m-win_amd64.whl", hash = "sha256:6197c3f3c0b960ad033b9b7d611db11285bb461fc6b802c1dd50d04ad715c225"},
+    {file = "lxml-4.9.4-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:7b378847a09d6bd46047f5f3599cdc64fcb4cc5a5a2dd0a2af610361fbe77b16"},
+    {file = "lxml-4.9.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:1343df4e2e6e51182aad12162b23b0a4b3fd77f17527a78c53f0f23573663545"},
+    {file = "lxml-4.9.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:6dbdacf5752fbd78ccdb434698230c4f0f95df7dd956d5f205b5ed6911a1367c"},
+    {file = "lxml-4.9.4-cp37-cp37m-manylinux_2_28_x86_64.whl", hash = "sha256:506becdf2ecaebaf7f7995f776394fcc8bd8a78022772de66677c84fb02dd33d"},
+    {file = "lxml-4.9.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ca8e44b5ba3edb682ea4e6185b49661fc22b230cf811b9c13963c9f982d1d964"},
+    {file = "lxml-4.9.4-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:9d9d5726474cbbef279fd709008f91a49c4f758bec9c062dfbba88eab00e3ff9"},
+    {file = "lxml-4.9.4-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:bbdd69e20fe2943b51e2841fc1e6a3c1de460d630f65bde12452d8c97209464d"},
+    {file = "lxml-4.9.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8671622256a0859f5089cbe0ce4693c2af407bc053dcc99aadff7f5310b4aa02"},
+    {file = "lxml-4.9.4-cp37-cp37m-win32.whl", hash = "sha256:dd4fda67f5faaef4f9ee5383435048ee3e11ad996901225ad7615bc92245bc8e"},
+    {file = "lxml-4.9.4-cp37-cp37m-win_amd64.whl", hash = "sha256:6bee9c2e501d835f91460b2c904bc359f8433e96799f5c2ff20feebd9bb1e590"},
+    {file = "lxml-4.9.4-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:1f10f250430a4caf84115b1e0f23f3615566ca2369d1962f82bef40dd99cd81a"},
+    {file = "lxml-4.9.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:3b505f2bbff50d261176e67be24e8909e54b5d9d08b12d4946344066d66b3e43"},
+    {file = "lxml-4.9.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:1449f9451cd53e0fd0a7ec2ff5ede4686add13ac7a7bfa6988ff6d75cff3ebe2"},
+    {file = "lxml-4.9.4-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:4ece9cca4cd1c8ba889bfa67eae7f21d0d1a2e715b4d5045395113361e8c533d"},
+    {file = "lxml-4.9.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:59bb5979f9941c61e907ee571732219fa4774d5a18f3fa5ff2df963f5dfaa6bc"},
+    {file = "lxml-4.9.4-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:b1980dbcaad634fe78e710c8587383e6e3f61dbe146bcbfd13a9c8ab2d7b1192"},
+    {file = "lxml-4.9.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:9ae6c3363261021144121427b1552b29e7b59de9d6a75bf51e03bc072efb3c37"},
+    {file = "lxml-4.9.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:bcee502c649fa6351b44bb014b98c09cb00982a475a1912a9881ca28ab4f9cd9"},
+    {file = "lxml-4.9.4-cp38-cp38-win32.whl", hash = "sha256:a8edae5253efa75c2fc79a90068fe540b197d1c7ab5803b800fccfe240eed33c"},
+    {file = "lxml-4.9.4-cp38-cp38-win_amd64.whl", hash = "sha256:701847a7aaefef121c5c0d855b2affa5f9bd45196ef00266724a80e439220e46"},
+    {file = "lxml-4.9.4-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:f610d980e3fccf4394ab3806de6065682982f3d27c12d4ce3ee46a8183d64a6a"},
+    {file = "lxml-4.9.4-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:aa9b5abd07f71b081a33115d9758ef6077924082055005808f68feccb27616bd"},
+    {file = "lxml-4.9.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:365005e8b0718ea6d64b374423e870648ab47c3a905356ab6e5a5ff03962b9a9"},
+    {file = "lxml-4.9.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:16b9ec51cc2feab009e800f2c6327338d6ee4e752c76e95a35c4465e80390ccd"},
+    {file = "lxml-4.9.4-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:a905affe76f1802edcac554e3ccf68188bea16546071d7583fb1b693f9cf756b"},
+    {file = "lxml-4.9.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fd814847901df6e8de13ce69b84c31fc9b3fb591224d6762d0b256d510cbf382"},
+    {file = "lxml-4.9.4-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:91bbf398ac8bb7d65a5a52127407c05f75a18d7015a270fdd94bbcb04e65d573"},
+    {file = "lxml-4.9.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:f99768232f036b4776ce419d3244a04fe83784bce871b16d2c2e984c7fcea847"},
+    {file = "lxml-4.9.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:bb5bd6212eb0edfd1e8f254585290ea1dadc3687dd8fd5e2fd9a87c31915cdab"},
+    {file = "lxml-4.9.4-cp39-cp39-win32.whl", hash = "sha256:88f7c383071981c74ec1998ba9b437659e4fd02a3c4a4d3efc16774eb108d0ec"},
+    {file = "lxml-4.9.4-cp39-cp39-win_amd64.whl", hash = "sha256:936e8880cc00f839aa4173f94466a8406a96ddce814651075f95837316369899"},
+    {file = "lxml-4.9.4-pp310-pypy310_pp73-macosx_11_0_x86_64.whl", hash = "sha256:f6c35b2f87c004270fa2e703b872fcc984d714d430b305145c39d53074e1ffe0"},
+    {file = "lxml-4.9.4-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:606d445feeb0856c2b424405236a01c71af7c97e5fe42fbc778634faef2b47e4"},
+    {file = "lxml-4.9.4-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:a1bdcbebd4e13446a14de4dd1825f1e778e099f17f79718b4aeaf2403624b0f7"},
+    {file = "lxml-4.9.4-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:0a08c89b23117049ba171bf51d2f9c5f3abf507d65d016d6e0fa2f37e18c0fc5"},
+    {file = "lxml-4.9.4-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:232fd30903d3123be4c435fb5159938c6225ee8607b635a4d3fca847003134ba"},
+    {file = "lxml-4.9.4-pp37-pypy37_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:231142459d32779b209aa4b4d460b175cadd604fed856f25c1571a9d78114771"},
+    {file = "lxml-4.9.4-pp38-pypy38_pp73-macosx_11_0_x86_64.whl", hash = "sha256:520486f27f1d4ce9654154b4494cf9307b495527f3a2908ad4cb48e4f7ed7ef7"},
+    {file = "lxml-4.9.4-pp38-pypy38_pp73-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:562778586949be7e0d7435fcb24aca4810913771f845d99145a6cee64d5b67ca"},
+    {file = "lxml-4.9.4-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:a9e7c6d89c77bb2770c9491d988f26a4b161d05c8ca58f63fb1f1b6b9a74be45"},
+    {file = "lxml-4.9.4-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:786d6b57026e7e04d184313c1359ac3d68002c33e4b1042ca58c362f1d09ff58"},
+    {file = "lxml-4.9.4-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:95ae6c5a196e2f239150aa4a479967351df7f44800c93e5a975ec726fef005e2"},
+    {file = "lxml-4.9.4-pp39-pypy39_pp73-macosx_11_0_x86_64.whl", hash = "sha256:9b556596c49fa1232b0fff4b0e69b9d4083a502e60e404b44341e2f8fb7187f5"},
+    {file = "lxml-4.9.4-pp39-pypy39_pp73-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:cc02c06e9e320869d7d1bd323df6dd4281e78ac2e7f8526835d3d48c69060683"},
+    {file = "lxml-4.9.4-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:857d6565f9aa3464764c2cb6a2e3c2e75e1970e877c188f4aeae45954a314e0c"},
+    {file = "lxml-4.9.4-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:c42ae7e010d7d6bc51875d768110c10e8a59494855c3d4c348b068f5fb81fdcd"},
+    {file = "lxml-4.9.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:f10250bb190fb0742e3e1958dd5c100524c2cc5096c67c8da51233f7448dc137"},
+    {file = "lxml-4.9.4.tar.gz", hash = "sha256:b1541e50b78e15fa06a2670157a1962ef06591d4c998b998047fff5e3236880e"},
+]
+
+[package.extras]
+cssselect = ["cssselect (>=0.7)"]
+html5 = ["html5lib"]
+htmlsoup = ["BeautifulSoup4"]
+source = ["Cython (==0.29.37)"]
+
 [[package]]
 name = "markdown"
 version = "3.7"
@@ -1129,6 +1646,45 @@ files = [
     {file = "numpy-1.26.4.tar.gz", hash = "sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010"},
 ]
 
+[[package]]
+name = "openai"
+version = "1.58.1"
+description = "The official Python library for the openai API"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "openai-1.58.1-py3-none-any.whl", hash = "sha256:e2910b1170a6b7f88ef491ac3a42c387f08bd3db533411f7ee391d166571d63c"},
+    {file = "openai-1.58.1.tar.gz", hash = "sha256:f5a035fd01e141fc743f4b0e02c41ca49be8fab0866d3b67f5f29b4f4d3c0973"},
+]
+
+[package.dependencies]
+anyio = ">=3.5.0,<5"
+distro = ">=1.7.0,<2"
+httpx = ">=0.23.0,<1"
+jiter = ">=0.4.0,<1"
+pydantic = ">=1.9.0,<3"
+sniffio = "*"
+tqdm = ">4"
+typing-extensions = ">=4.11,<5"
+
+[package.extras]
+datalib = ["numpy (>=1)", "pandas (>=1.2.3)", "pandas-stubs (>=1.1.0.11)"]
+realtime = ["websockets (>=13,<15)"]
+
+[[package]]
+name = "outcome"
+version = "1.3.0.post0"
+description = "Capture the outcome of Python function calls."
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "outcome-1.3.0.post0-py2.py3-none-any.whl", hash = "sha256:e771c5ce06d1415e356078d3bdd68523f284b4ce5419828922b6871e65eda82b"},
+    {file = "outcome-1.3.0.post0.tar.gz", hash = "sha256:9dcf02e65f2971b80047b377468e72a268e15c0af3cf1238e6ff14f7f91143b8"},
+]
+
+[package.dependencies]
+attrs = ">=19.2.0"
+
 [[package]]
 name = "packaging"
 version = "24.1"
@@ -1412,6 +1968,26 @@ files = [
 [package.dependencies]
 wcwidth = "*"
 
+[[package]]
+name = "protobuf"
+version = "5.29.2"
+description = ""
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "protobuf-5.29.2-cp310-abi3-win32.whl", hash = "sha256:c12ba8249f5624300cf51c3d0bfe5be71a60c63e4dcf51ffe9a68771d958c851"},
+    {file = "protobuf-5.29.2-cp310-abi3-win_amd64.whl", hash = "sha256:842de6d9241134a973aab719ab42b008a18a90f9f07f06ba480df268f86432f9"},
+    {file = "protobuf-5.29.2-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:a0c53d78383c851bfa97eb42e3703aefdc96d2036a41482ffd55dc5f529466eb"},
+    {file = "protobuf-5.29.2-cp38-abi3-manylinux2014_aarch64.whl", hash = "sha256:494229ecd8c9009dd71eda5fd57528395d1eacdf307dbece6c12ad0dd09e912e"},
+    {file = "protobuf-5.29.2-cp38-abi3-manylinux2014_x86_64.whl", hash = "sha256:b6b0d416bbbb9d4fbf9d0561dbfc4e324fd522f61f7af0fe0f282ab67b22477e"},
+    {file = "protobuf-5.29.2-cp38-cp38-win32.whl", hash = "sha256:e621a98c0201a7c8afe89d9646859859be97cb22b8bf1d8eacfd90d5bda2eb19"},
+    {file = "protobuf-5.29.2-cp38-cp38-win_amd64.whl", hash = "sha256:13d6d617a2a9e0e82a88113d7191a1baa1e42c2cc6f5f1398d3b054c8e7e714a"},
+    {file = "protobuf-5.29.2-cp39-cp39-win32.whl", hash = "sha256:36000f97ea1e76e8398a3f02936aac2a5d2b111aae9920ec1b769fc4a222c4d9"},
+    {file = "protobuf-5.29.2-cp39-cp39-win_amd64.whl", hash = "sha256:2d2e674c58a06311c8e99e74be43e7f3a8d1e2b2fdf845eaa347fbd866f23355"},
+    {file = "protobuf-5.29.2-py3-none-any.whl", hash = "sha256:fde4554c0e578a5a0bcc9a276339594848d1e89f9ea47b4427c80e5d72f90181"},
+    {file = "protobuf-5.29.2.tar.gz", hash = "sha256:b2cc8e8bb7c9326996f0e160137b0861f1a82162502658df2951209d0cb0309e"},
+]
+
 [[package]]
 name = "psutil"
 version = "6.0.0"
@@ -1488,6 +2064,179 @@ files = [
     {file = "pycparser-2.22.tar.gz", hash = "sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6"},
 ]
 
+[[package]]
+name = "pycryptodome"
+version = "3.21.0"
+description = "Cryptographic library for Python"
+optional = false
+python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7"
+files = [
+    {file = "pycryptodome-3.21.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:dad9bf36eda068e89059d1f07408e397856be9511d7113ea4b586642a429a4fd"},
+    {file = "pycryptodome-3.21.0-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:a1752eca64c60852f38bb29e2c86fca30d7672c024128ef5d70cc15868fa10f4"},
+    {file = "pycryptodome-3.21.0-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:3ba4cc304eac4d4d458f508d4955a88ba25026890e8abff9b60404f76a62c55e"},
+    {file = "pycryptodome-3.21.0-cp27-cp27m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7cb087b8612c8a1a14cf37dd754685be9a8d9869bed2ffaaceb04850a8aeef7e"},
+    {file = "pycryptodome-3.21.0-cp27-cp27m-musllinux_1_1_aarch64.whl", hash = "sha256:26412b21df30b2861424a6c6d5b1d8ca8107612a4cfa4d0183e71c5d200fb34a"},
+    {file = "pycryptodome-3.21.0-cp27-cp27m-win32.whl", hash = "sha256:cc2269ab4bce40b027b49663d61d816903a4bd90ad88cb99ed561aadb3888dd3"},
+    {file = "pycryptodome-3.21.0-cp27-cp27m-win_amd64.whl", hash = "sha256:0fa0a05a6a697ccbf2a12cec3d6d2650b50881899b845fac6e87416f8cb7e87d"},
+    {file = "pycryptodome-3.21.0-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:6cce52e196a5f1d6797ff7946cdff2038d3b5f0aba4a43cb6bf46b575fd1b5bb"},
+    {file = "pycryptodome-3.21.0-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:a915597ffccabe902e7090e199a7bf7a381c5506a747d5e9d27ba55197a2c568"},
+    {file = "pycryptodome-3.21.0-cp27-cp27mu-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a4e74c522d630766b03a836c15bff77cb657c5fdf098abf8b1ada2aebc7d0819"},
+    {file = "pycryptodome-3.21.0-cp27-cp27mu-musllinux_1_1_aarch64.whl", hash = "sha256:a3804675283f4764a02db05f5191eb8fec2bb6ca34d466167fc78a5f05bbe6b3"},
+    {file = "pycryptodome-3.21.0-cp36-abi3-macosx_10_9_universal2.whl", hash = "sha256:2480ec2c72438430da9f601ebc12c518c093c13111a5c1644c82cdfc2e50b1e4"},
+    {file = "pycryptodome-3.21.0-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:de18954104667f565e2fbb4783b56667f30fb49c4d79b346f52a29cb198d5b6b"},
+    {file = "pycryptodome-3.21.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2de4b7263a33947ff440412339cb72b28a5a4c769b5c1ca19e33dd6cd1dcec6e"},
+    {file = "pycryptodome-3.21.0-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0714206d467fc911042d01ea3a1847c847bc10884cf674c82e12915cfe1649f8"},
+    {file = "pycryptodome-3.21.0-cp36-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7d85c1b613121ed3dbaa5a97369b3b757909531a959d229406a75b912dd51dd1"},
+    {file = "pycryptodome-3.21.0-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:8898a66425a57bcf15e25fc19c12490b87bd939800f39a03ea2de2aea5e3611a"},
+    {file = "pycryptodome-3.21.0-cp36-abi3-musllinux_1_2_i686.whl", hash = "sha256:932c905b71a56474bff8a9c014030bc3c882cee696b448af920399f730a650c2"},
+    {file = "pycryptodome-3.21.0-cp36-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:18caa8cfbc676eaaf28613637a89980ad2fd96e00c564135bf90bc3f0b34dd93"},
+    {file = "pycryptodome-3.21.0-cp36-abi3-win32.whl", hash = "sha256:280b67d20e33bb63171d55b1067f61fbd932e0b1ad976b3a184303a3dad22764"},
+    {file = "pycryptodome-3.21.0-cp36-abi3-win_amd64.whl", hash = "sha256:b7aa25fc0baa5b1d95b7633af4f5f1838467f1815442b22487426f94e0d66c53"},
+    {file = "pycryptodome-3.21.0-pp27-pypy_73-manylinux2010_x86_64.whl", hash = "sha256:2cb635b67011bc147c257e61ce864879ffe6d03342dc74b6045059dfbdedafca"},
+    {file = "pycryptodome-3.21.0-pp27-pypy_73-win32.whl", hash = "sha256:4c26a2f0dc15f81ea3afa3b0c87b87e501f235d332b7f27e2225ecb80c0b1cdd"},
+    {file = "pycryptodome-3.21.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:d5ebe0763c982f069d3877832254f64974139f4f9655058452603ff559c482e8"},
+    {file = "pycryptodome-3.21.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ee86cbde706be13f2dec5a42b52b1c1d1cbb90c8e405c68d0755134735c8dc6"},
+    {file = "pycryptodome-3.21.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0fd54003ec3ce4e0f16c484a10bc5d8b9bd77fa662a12b85779a2d2d85d67ee0"},
+    {file = "pycryptodome-3.21.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:5dfafca172933506773482b0e18f0cd766fd3920bd03ec85a283df90d8a17bc6"},
+    {file = "pycryptodome-3.21.0-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:590ef0898a4b0a15485b05210b4a1c9de8806d3ad3d47f74ab1dc07c67a6827f"},
+    {file = "pycryptodome-3.21.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f35e442630bc4bc2e1878482d6f59ea22e280d7121d7adeaedba58c23ab6386b"},
+    {file = "pycryptodome-3.21.0-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ff99f952db3db2fbe98a0b355175f93ec334ba3d01bbde25ad3a5a33abc02b58"},
+    {file = "pycryptodome-3.21.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:8acd7d34af70ee63f9a849f957558e49a98f8f1634f86a59d2be62bb8e93f71c"},
+    {file = "pycryptodome-3.21.0.tar.gz", hash = "sha256:f7787e0d469bdae763b876174cf2e6c0f7be79808af26b1da96f1a64bcf47297"},
+]
+
+[[package]]
+name = "pydantic"
+version = "2.10.4"
+description = "Data validation using Python type hints"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "pydantic-2.10.4-py3-none-any.whl", hash = "sha256:597e135ea68be3a37552fb524bc7d0d66dcf93d395acd93a00682f1efcb8ee3d"},
+    {file = "pydantic-2.10.4.tar.gz", hash = "sha256:82f12e9723da6de4fe2ba888b5971157b3be7ad914267dea8f05f82b28254f06"},
+]
+
+[package.dependencies]
+annotated-types = ">=0.6.0"
+pydantic-core = "2.27.2"
+typing-extensions = ">=4.12.2"
+
+[package.extras]
+email = ["email-validator (>=2.0.0)"]
+timezone = ["tzdata"]
+
+[[package]]
+name = "pydantic-core"
+version = "2.27.2"
+description = "Core functionality for Pydantic validation and serialization"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "pydantic_core-2.27.2-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:2d367ca20b2f14095a8f4fa1210f5a7b78b8a20009ecced6b12818f455b1e9fa"},
+    {file = "pydantic_core-2.27.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:491a2b73db93fab69731eaee494f320faa4e093dbed776be1a829c2eb222c34c"},
+    {file = "pydantic_core-2.27.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7969e133a6f183be60e9f6f56bfae753585680f3b7307a8e555a948d443cc05a"},
+    {file = "pydantic_core-2.27.2-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3de9961f2a346257caf0aa508a4da705467f53778e9ef6fe744c038119737ef5"},
+    {file = "pydantic_core-2.27.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e2bb4d3e5873c37bb3dd58714d4cd0b0e6238cebc4177ac8fe878f8b3aa8e74c"},
+    {file = "pydantic_core-2.27.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:280d219beebb0752699480fe8f1dc61ab6615c2046d76b7ab7ee38858de0a4e7"},
+    {file = "pydantic_core-2.27.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47956ae78b6422cbd46f772f1746799cbb862de838fd8d1fbd34a82e05b0983a"},
+    {file = "pydantic_core-2.27.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:14d4a5c49d2f009d62a2a7140d3064f686d17a5d1a268bc641954ba181880236"},
+    {file = "pydantic_core-2.27.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:337b443af21d488716f8d0b6164de833e788aa6bd7e3a39c005febc1284f4962"},
+    {file = "pydantic_core-2.27.2-cp310-cp310-musllinux_1_1_armv7l.whl", hash = "sha256:03d0f86ea3184a12f41a2d23f7ccb79cdb5a18e06993f8a45baa8dfec746f0e9"},
+    {file = "pydantic_core-2.27.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:7041c36f5680c6e0f08d922aed302e98b3745d97fe1589db0a3eebf6624523af"},
+    {file = "pydantic_core-2.27.2-cp310-cp310-win32.whl", hash = "sha256:50a68f3e3819077be2c98110c1f9dcb3817e93f267ba80a2c05bb4f8799e2ff4"},
+    {file = "pydantic_core-2.27.2-cp310-cp310-win_amd64.whl", hash = "sha256:e0fd26b16394ead34a424eecf8a31a1f5137094cabe84a1bcb10fa6ba39d3d31"},
+    {file = "pydantic_core-2.27.2-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:8e10c99ef58cfdf2a66fc15d66b16c4a04f62bca39db589ae8cba08bc55331bc"},
+    {file = "pydantic_core-2.27.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:26f32e0adf166a84d0cb63be85c562ca8a6fa8de28e5f0d92250c6b7e9e2aff7"},
+    {file = "pydantic_core-2.27.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8c19d1ea0673cd13cc2f872f6c9ab42acc4e4f492a7ca9d3795ce2b112dd7e15"},
+    {file = "pydantic_core-2.27.2-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5e68c4446fe0810e959cdff46ab0a41ce2f2c86d227d96dc3847af0ba7def306"},
+    {file = "pydantic_core-2.27.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d9640b0059ff4f14d1f37321b94061c6db164fbe49b334b31643e0528d100d99"},
+    {file = "pydantic_core-2.27.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:40d02e7d45c9f8af700f3452f329ead92da4c5f4317ca9b896de7ce7199ea459"},
+    {file = "pydantic_core-2.27.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1c1fd185014191700554795c99b347d64f2bb637966c4cfc16998a0ca700d048"},
+    {file = "pydantic_core-2.27.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d81d2068e1c1228a565af076598f9e7451712700b673de8f502f0334f281387d"},
+    {file = "pydantic_core-2.27.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:1a4207639fb02ec2dbb76227d7c751a20b1a6b4bc52850568e52260cae64ca3b"},
+    {file = "pydantic_core-2.27.2-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:3de3ce3c9ddc8bbd88f6e0e304dea0e66d843ec9de1b0042b0911c1663ffd474"},
+    {file = "pydantic_core-2.27.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:30c5f68ded0c36466acede341551106821043e9afaad516adfb6e8fa80a4e6a6"},
+    {file = "pydantic_core-2.27.2-cp311-cp311-win32.whl", hash = "sha256:c70c26d2c99f78b125a3459f8afe1aed4d9687c24fd677c6a4436bc042e50d6c"},
+    {file = "pydantic_core-2.27.2-cp311-cp311-win_amd64.whl", hash = "sha256:08e125dbdc505fa69ca7d9c499639ab6407cfa909214d500897d02afb816e7cc"},
+    {file = "pydantic_core-2.27.2-cp311-cp311-win_arm64.whl", hash = "sha256:26f0d68d4b235a2bae0c3fc585c585b4ecc51382db0e3ba402a22cbc440915e4"},
+    {file = "pydantic_core-2.27.2-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:9e0c8cfefa0ef83b4da9588448b6d8d2a2bf1a53c3f1ae5fca39eb3061e2f0b0"},
+    {file = "pydantic_core-2.27.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:83097677b8e3bd7eaa6775720ec8e0405f1575015a463285a92bfdfe254529ef"},
+    {file = "pydantic_core-2.27.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:172fce187655fece0c90d90a678424b013f8fbb0ca8b036ac266749c09438cb7"},
+    {file = "pydantic_core-2.27.2-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:519f29f5213271eeeeb3093f662ba2fd512b91c5f188f3bb7b27bc5973816934"},
+    {file = "pydantic_core-2.27.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:05e3a55d124407fffba0dd6b0c0cd056d10e983ceb4e5dbd10dda135c31071d6"},
+    {file = "pydantic_core-2.27.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9c3ed807c7b91de05e63930188f19e921d1fe90de6b4f5cd43ee7fcc3525cb8c"},
+    {file = "pydantic_core-2.27.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6fb4aadc0b9a0c063206846d603b92030eb6f03069151a625667f982887153e2"},
+    {file = "pydantic_core-2.27.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:28ccb213807e037460326424ceb8b5245acb88f32f3d2777427476e1b32c48c4"},
+    {file = "pydantic_core-2.27.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:de3cd1899e2c279b140adde9357c4495ed9d47131b4a4eaff9052f23398076b3"},
+    {file = "pydantic_core-2.27.2-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:220f892729375e2d736b97d0e51466252ad84c51857d4d15f5e9692f9ef12be4"},
+    {file = "pydantic_core-2.27.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:a0fcd29cd6b4e74fe8ddd2c90330fd8edf2e30cb52acda47f06dd615ae72da57"},
+    {file = "pydantic_core-2.27.2-cp312-cp312-win32.whl", hash = "sha256:1e2cb691ed9834cd6a8be61228471d0a503731abfb42f82458ff27be7b2186fc"},
+    {file = "pydantic_core-2.27.2-cp312-cp312-win_amd64.whl", hash = "sha256:cc3f1a99a4f4f9dd1de4fe0312c114e740b5ddead65bb4102884b384c15d8bc9"},
+    {file = "pydantic_core-2.27.2-cp312-cp312-win_arm64.whl", hash = "sha256:3911ac9284cd8a1792d3cb26a2da18f3ca26c6908cc434a18f730dc0db7bfa3b"},
+    {file = "pydantic_core-2.27.2-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:7d14bd329640e63852364c306f4d23eb744e0f8193148d4044dd3dacdaacbd8b"},
+    {file = "pydantic_core-2.27.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:82f91663004eb8ed30ff478d77c4d1179b3563df6cdb15c0817cd1cdaf34d154"},
+    {file = "pydantic_core-2.27.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:71b24c7d61131bb83df10cc7e687433609963a944ccf45190cfc21e0887b08c9"},
+    {file = "pydantic_core-2.27.2-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:fa8e459d4954f608fa26116118bb67f56b93b209c39b008277ace29937453dc9"},
+    {file = "pydantic_core-2.27.2-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ce8918cbebc8da707ba805b7fd0b382816858728ae7fe19a942080c24e5b7cd1"},
+    {file = "pydantic_core-2.27.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:eda3f5c2a021bbc5d976107bb302e0131351c2ba54343f8a496dc8783d3d3a6a"},
+    {file = "pydantic_core-2.27.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bd8086fa684c4775c27f03f062cbb9eaa6e17f064307e86b21b9e0abc9c0f02e"},
+    {file = "pydantic_core-2.27.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8d9b3388db186ba0c099a6d20f0604a44eabdeef1777ddd94786cdae158729e4"},
+    {file = "pydantic_core-2.27.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:7a66efda2387de898c8f38c0cf7f14fca0b51a8ef0b24bfea5849f1b3c95af27"},
+    {file = "pydantic_core-2.27.2-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:18a101c168e4e092ab40dbc2503bdc0f62010e95d292b27827871dc85450d7ee"},
+    {file = "pydantic_core-2.27.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:ba5dd002f88b78a4215ed2f8ddbdf85e8513382820ba15ad5ad8955ce0ca19a1"},
+    {file = "pydantic_core-2.27.2-cp313-cp313-win32.whl", hash = "sha256:1ebaf1d0481914d004a573394f4be3a7616334be70261007e47c2a6fe7e50130"},
+    {file = "pydantic_core-2.27.2-cp313-cp313-win_amd64.whl", hash = "sha256:953101387ecf2f5652883208769a79e48db18c6df442568a0b5ccd8c2723abee"},
+    {file = "pydantic_core-2.27.2-cp313-cp313-win_arm64.whl", hash = "sha256:ac4dbfd1691affb8f48c2c13241a2e3b60ff23247cbcf981759c768b6633cf8b"},
+    {file = "pydantic_core-2.27.2-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:d3e8d504bdd3f10835468f29008d72fc8359d95c9c415ce6e767203db6127506"},
+    {file = "pydantic_core-2.27.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:521eb9b7f036c9b6187f0b47318ab0d7ca14bd87f776240b90b21c1f4f149320"},
+    {file = "pydantic_core-2.27.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:85210c4d99a0114f5a9481b44560d7d1e35e32cc5634c656bc48e590b669b145"},
+    {file = "pydantic_core-2.27.2-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d716e2e30c6f140d7560ef1538953a5cd1a87264c737643d481f2779fc247fe1"},
+    {file = "pydantic_core-2.27.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f66d89ba397d92f840f8654756196d93804278457b5fbede59598a1f9f90b228"},
+    {file = "pydantic_core-2.27.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:669e193c1c576a58f132e3158f9dfa9662969edb1a250c54d8fa52590045f046"},
+    {file = "pydantic_core-2.27.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9fdbe7629b996647b99c01b37f11170a57ae675375b14b8c13b8518b8320ced5"},
+    {file = "pydantic_core-2.27.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d262606bf386a5ba0b0af3b97f37c83d7011439e3dc1a9298f21efb292e42f1a"},
+    {file = "pydantic_core-2.27.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:cabb9bcb7e0d97f74df8646f34fc76fbf793b7f6dc2438517d7a9e50eee4f14d"},
+    {file = "pydantic_core-2.27.2-cp38-cp38-musllinux_1_1_armv7l.whl", hash = "sha256:d2d63f1215638d28221f664596b1ccb3944f6e25dd18cd3b86b0a4c408d5ebb9"},
+    {file = "pydantic_core-2.27.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:bca101c00bff0adb45a833f8451b9105d9df18accb8743b08107d7ada14bd7da"},
+    {file = "pydantic_core-2.27.2-cp38-cp38-win32.whl", hash = "sha256:f6f8e111843bbb0dee4cb6594cdc73e79b3329b526037ec242a3e49012495b3b"},
+    {file = "pydantic_core-2.27.2-cp38-cp38-win_amd64.whl", hash = "sha256:fd1aea04935a508f62e0d0ef1f5ae968774a32afc306fb8545e06f5ff5cdf3ad"},
+    {file = "pydantic_core-2.27.2-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:c10eb4f1659290b523af58fa7cffb452a61ad6ae5613404519aee4bfbf1df993"},
+    {file = "pydantic_core-2.27.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ef592d4bad47296fb11f96cd7dc898b92e795032b4894dfb4076cfccd43a9308"},
+    {file = "pydantic_core-2.27.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c61709a844acc6bf0b7dce7daae75195a10aac96a596ea1b776996414791ede4"},
+    {file = "pydantic_core-2.27.2-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:42c5f762659e47fdb7b16956c71598292f60a03aa92f8b6351504359dbdba6cf"},
+    {file = "pydantic_core-2.27.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4c9775e339e42e79ec99c441d9730fccf07414af63eac2f0e48e08fd38a64d76"},
+    {file = "pydantic_core-2.27.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:57762139821c31847cfb2df63c12f725788bd9f04bc2fb392790959b8f70f118"},
+    {file = "pydantic_core-2.27.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0d1e85068e818c73e048fe28cfc769040bb1f475524f4745a5dc621f75ac7630"},
+    {file = "pydantic_core-2.27.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:097830ed52fd9e427942ff3b9bc17fab52913b2f50f2880dc4a5611446606a54"},
+    {file = "pydantic_core-2.27.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:044a50963a614ecfae59bb1eaf7ea7efc4bc62f49ed594e18fa1e5d953c40e9f"},
+    {file = "pydantic_core-2.27.2-cp39-cp39-musllinux_1_1_armv7l.whl", hash = "sha256:4e0b4220ba5b40d727c7f879eac379b822eee5d8fff418e9d3381ee45b3b0362"},
+    {file = "pydantic_core-2.27.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5e4f4bb20d75e9325cc9696c6802657b58bc1dbbe3022f32cc2b2b632c3fbb96"},
+    {file = "pydantic_core-2.27.2-cp39-cp39-win32.whl", hash = "sha256:cca63613e90d001b9f2f9a9ceb276c308bfa2a43fafb75c8031c4f66039e8c6e"},
+    {file = "pydantic_core-2.27.2-cp39-cp39-win_amd64.whl", hash = "sha256:77d1bca19b0f7021b3a982e6f903dcd5b2b06076def36a652e3907f596e29f67"},
+    {file = "pydantic_core-2.27.2-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:2bf14caea37e91198329b828eae1618c068dfb8ef17bb33287a7ad4b61ac314e"},
+    {file = "pydantic_core-2.27.2-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:b0cb791f5b45307caae8810c2023a184c74605ec3bcbb67d13846c28ff731ff8"},
+    {file = "pydantic_core-2.27.2-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:688d3fd9fcb71f41c4c015c023d12a79d1c4c0732ec9eb35d96e3388a120dcf3"},
+    {file = "pydantic_core-2.27.2-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3d591580c34f4d731592f0e9fe40f9cc1b430d297eecc70b962e93c5c668f15f"},
+    {file = "pydantic_core-2.27.2-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:82f986faf4e644ffc189a7f1aafc86e46ef70372bb153e7001e8afccc6e54133"},
+    {file = "pydantic_core-2.27.2-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:bec317a27290e2537f922639cafd54990551725fc844249e64c523301d0822fc"},
+    {file = "pydantic_core-2.27.2-pp310-pypy310_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:0296abcb83a797db256b773f45773da397da75a08f5fcaef41f2044adec05f50"},
+    {file = "pydantic_core-2.27.2-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:0d75070718e369e452075a6017fbf187f788e17ed67a3abd47fa934d001863d9"},
+    {file = "pydantic_core-2.27.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:7e17b560be3c98a8e3aa66ce828bdebb9e9ac6ad5466fba92eb74c4c95cb1151"},
+    {file = "pydantic_core-2.27.2-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:c33939a82924da9ed65dab5a65d427205a73181d8098e79b6b426bdf8ad4e656"},
+    {file = "pydantic_core-2.27.2-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:00bad2484fa6bda1e216e7345a798bd37c68fb2d97558edd584942aa41b7d278"},
+    {file = "pydantic_core-2.27.2-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c817e2b40aba42bac6f457498dacabc568c3b7a986fc9ba7c8d9d260b71485fb"},
+    {file = "pydantic_core-2.27.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:251136cdad0cb722e93732cb45ca5299fb56e1344a833640bf93b2803f8d1bfd"},
+    {file = "pydantic_core-2.27.2-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d2088237af596f0a524d3afc39ab3b036e8adb054ee57cbb1dcf8e09da5b29cc"},
+    {file = "pydantic_core-2.27.2-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:d4041c0b966a84b4ae7a09832eb691a35aec90910cd2dbe7a208de59be77965b"},
+    {file = "pydantic_core-2.27.2-pp39-pypy39_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:8083d4e875ebe0b864ffef72a4304827015cff328a1be6e22cc850753bfb122b"},
+    {file = "pydantic_core-2.27.2-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:f141ee28a0ad2123b6611b6ceff018039df17f32ada8b534e6aa039545a3efb2"},
+    {file = "pydantic_core-2.27.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:7d0c8399fcc1848491f00e0314bd59fb34a9c008761bcb422a057670c3f65e35"},
+    {file = "pydantic_core-2.27.2.tar.gz", hash = "sha256:eb026e5a4c1fee05726072337ff51d1efb6f59090b7da90d30ea58625b1ffb39"},
+]
+
+[package.dependencies]
+typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0"
+
 [[package]]
 name = "pydocstyle"
 version = "6.3.0"
@@ -1505,6 +2254,21 @@ snowballstemmer = ">=2.2.0"
 [package.extras]
 toml = ["tomli (>=1.2.3)"]
 
+[[package]]
+name = "pyease-grpc"
+version = "1.7.0"
+description = "Easy gRPC-web client in python"
+optional = false
+python-versions = "*"
+files = [
+    {file = "pyease_grpc-1.7.0-py3-none-any.whl", hash = "sha256:279047e03f21f4b65ebe251d527504a8f64d84ccfa028b50bba0e7315321f05e"},
+]
+
+[package.dependencies]
+grpcio = "*"
+protobuf = ">=3.19.0"
+requests = ">=2.25.0"
+
 [[package]]
 name = "pyflakes"
 version = "3.2.0"
@@ -1530,6 +2294,17 @@ files = [
 [package.extras]
 windows-terminal = ["colorama (>=0.4.6)"]
 
+[[package]]
+name = "pyjsparser"
+version = "2.7.1"
+description = "Fast javascript parser (based on esprima.js)"
+optional = false
+python-versions = "*"
+files = [
+    {file = "pyjsparser-2.7.1-py2-none-any.whl", hash = "sha256:2b12842df98d83f65934e0772fa4a5d8b123b3bc79f1af1789172ac70265dd21"},
+    {file = "pyjsparser-2.7.1.tar.gz", hash = "sha256:be60da6b778cc5a5296a69d8e7d614f1f870faf94e1b1b6ac591f2ad5d729579"},
+]
+
 [[package]]
 name = "pylama"
 version = "8.4.1"
@@ -1571,6 +2346,53 @@ files = [
 [package.extras]
 diagrams = ["jinja2", "railroad-diagrams"]
 
+[[package]]
+name = "pysocks"
+version = "1.7.1"
+description = "A Python SOCKS client module. See https://github.com/Anorov/PySocks for more information."
+optional = false
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
+files = [
+    {file = "PySocks-1.7.1-py27-none-any.whl", hash = "sha256:08e69f092cc6dbe92a0fdd16eeb9b9ffbc13cadfe5ca4c7bd92ffb078b293299"},
+    {file = "PySocks-1.7.1-py3-none-any.whl", hash = "sha256:2725bd0a9925919b9b51739eea5f9e2bae91e83288108a9ad338b2e3a4435ee5"},
+    {file = "PySocks-1.7.1.tar.gz", hash = "sha256:3f8804571ebe159c380ac6de37643bb4685970655d3bba243530d6558b799aa0"},
+]
+
+[[package]]
+name = "python-box"
+version = "7.3.0"
+description = "Advanced Python dictionaries with dot notation access"
+optional = false
+python-versions = ">=3.9"
+files = [
+    {file = "python_box-7.3.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a2131477ed02aa3609b348dad0697b70d84968d6440387898bb9075f461ef9bf"},
+    {file = "python_box-7.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3284cf583476af63c4f24168b6e1307503322dccd9b3dc2c924f5e69f79e7ab5"},
+    {file = "python_box-7.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:2718cf4c8dcc091d1c56a1a297804ab7973271391a2d2d34d37740820bbd1fda"},
+    {file = "python_box-7.3.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:e40fe08b218b3d07a50d6eb1c62edce8d0636d6bd1e563907bc86018a78e5826"},
+    {file = "python_box-7.3.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bd13e2b964ed527e03409cb1fb386d8723e0e69caf0f507af60d64102c13d363"},
+    {file = "python_box-7.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:d661fb9c6ff6c730b53fe859754624baa14e37ee3d593525382b20194efad367"},
+    {file = "python_box-7.3.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:6c3809f78f7c829e45626990a891d93214748938b9c0236dc6d0f2e8c400d325"},
+    {file = "python_box-7.3.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c233b94bf3b95d7d9dc01ed1ee5636800174345810b319eb87219b760edbb54f"},
+    {file = "python_box-7.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:9a22cc82e78225a419c4da02f53d6beb5c5cbd2fe5f63c13dab81e4f27b8c929"},
+    {file = "python_box-7.3.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:1f7b93c5ab4027b12ba67baffa8db903557e557250e01b91226d7a1b9688cf77"},
+    {file = "python_box-7.3.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:71ed234c1cff7f7197103bb11d98559032c0beac34db0c62dd5bd53e2b2a6963"},
+    {file = "python_box-7.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:1144c9e5d40a2cbe34d1ec9a13abfc557e8e9e2fbf15f14314c87b6113de178f"},
+    {file = "python_box-7.3.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:df77730baabf45b1682ead1c470e84a530f8ceb0295263a89f0ebc04ef7f363c"},
+    {file = "python_box-7.3.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:36bef944e61672b300c1d56d16db8a43ee4af9ab5678492a5e003368d2c64a6e"},
+    {file = "python_box-7.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:b35a2262a4e1ccfba90ce8e2018aa367f8a46a519632884006fa3153b266f184"},
+    {file = "python_box-7.3.0-py3-none-any.whl", hash = "sha256:b1139bffe91bd317fd686c4c29ffc84115c1967af14112c5c4a8ac51937d530c"},
+    {file = "python_box-7.3.0.tar.gz", hash = "sha256:39a85ba457d07122226ca60597882d763549713ab56ac7d55da41c4ad0e89a05"},
+]
+
+[package.extras]
+all = ["msgpack", "ruamel.yaml (>=0.17)", "toml"]
+msgpack = ["msgpack"]
+pyyaml = ["PyYAML"]
+ruamel-yaml = ["ruamel.yaml (>=0.17)"]
+toml = ["toml"]
+tomli = ["tomli", "tomli-w"]
+yaml = ["ruamel.yaml (>=0.17)"]
+
 [[package]]
 name = "python-dateutil"
 version = "2.9.0.post0"
@@ -1599,6 +2421,23 @@ files = [
 [package.extras]
 cli = ["click (>=5.0)"]
 
+[[package]]
+name = "python-slugify"
+version = "8.0.4"
+description = "A Python slugify application that also handles Unicode"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "python-slugify-8.0.4.tar.gz", hash = "sha256:59202371d1d05b54a9e7720c5e038f928f45daaffe41dd10822f3907b937c856"},
+    {file = "python_slugify-8.0.4-py2.py3-none-any.whl", hash = "sha256:276540b79961052b66b7d116620b36518847f52d5fd9e3a70164fc8c50faa6b8"},
+]
+
+[package.dependencies]
+text-unidecode = ">=1.3"
+
+[package.extras]
+unidecode = ["Unidecode (>=1.1.1)"]
+
 [[package]]
 name = "pytz"
 version = "2024.1"
@@ -1754,6 +2593,42 @@ files = [
 [package.dependencies]
 cffi = {version = "*", markers = "implementation_name == \"pypy\""}
 
+[[package]]
+name = "questionary"
+version = "1.10.0"
+description = "Python library to build pretty command line user prompts ⭐️"
+optional = false
+python-versions = ">=3.6,<4.0"
+files = [
+    {file = "questionary-1.10.0-py3-none-any.whl", hash = "sha256:fecfcc8cca110fda9d561cb83f1e97ecbb93c613ff857f655818839dac74ce90"},
+    {file = "questionary-1.10.0.tar.gz", hash = "sha256:600d3aefecce26d48d97eee936fdb66e4bc27f934c3ab6dd1e292c4f43946d90"},
+]
+
+[package.dependencies]
+prompt_toolkit = ">=2.0,<4.0"
+
+[package.extras]
+docs = ["Sphinx (>=3.3,<4.0)", "sphinx-autobuild (>=2020.9.1,<2021.0.0)", "sphinx-autodoc-typehints (>=1.11.1,<2.0.0)", "sphinx-copybutton (>=0.3.1,<0.4.0)", "sphinx-rtd-theme (>=0.5.0,<0.6.0)"]
+
+[[package]]
+name = "readability-lxml"
+version = "0.8.1"
+description = "fast html to text parser (article readability tool) with python 3 support"
+optional = false
+python-versions = "*"
+files = [
+    {file = "readability-lxml-0.8.1.tar.gz", hash = "sha256:e51fea56b5909aaf886d307d48e79e096293255afa567b7d08bca94d25b1a4e1"},
+    {file = "readability_lxml-0.8.1-py3-none-any.whl", hash = "sha256:e0d366a21b1bd6cca17de71a4e6ea16fcfaa8b0a5b4004e39e2c7eff884e6305"},
+]
+
+[package.dependencies]
+chardet = "*"
+cssselect = "*"
+lxml = "*"
+
+[package.extras]
+test = ["timeout-decorator"]
+
 [[package]]
 name = "regex"
 version = "2024.11.6"
@@ -1918,6 +2793,25 @@ files = [
     {file = "ruff-0.1.15.tar.gz", hash = "sha256:f6dfa8c1b21c913c326919056c390966648b680966febcb796cc9d1aaab8564e"},
 ]
 
+[[package]]
+name = "selenium"
+version = "4.27.1"
+description = "Official Python bindings for Selenium WebDriver"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "selenium-4.27.1-py3-none-any.whl", hash = "sha256:b89b1f62b5cfe8025868556fe82360d6b649d464f75d2655cb966c8f8447ea18"},
+    {file = "selenium-4.27.1.tar.gz", hash = "sha256:5296c425a75ff1b44d0d5199042b36a6d1ef76c04fb775b97b40be739a9caae2"},
+]
+
+[package.dependencies]
+certifi = ">=2021.10.8"
+trio = ">=0.17,<1.0"
+trio-websocket = ">=0.9,<1.0"
+typing_extensions = ">=4.9,<5.0"
+urllib3 = {version = ">=1.26,<3", extras = ["socks"]}
+websocket-client = ">=1.8,<2.0"
+
 [[package]]
 name = "six"
 version = "1.16.0"
@@ -1929,6 +2823,17 @@ files = [
     {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"},
 ]
 
+[[package]]
+name = "sniffio"
+version = "1.3.1"
+description = "Sniff out which async library your code is running under"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2"},
+    {file = "sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc"},
+]
+
 [[package]]
 name = "snowballstemmer"
 version = "2.2.0"
@@ -1940,6 +2845,17 @@ files = [
     {file = "snowballstemmer-2.2.0.tar.gz", hash = "sha256:09b16deb8547d3412ad7b590689584cd0fe25ec8db3be37788be3810cbf19cb1"},
 ]
 
+[[package]]
+name = "sortedcontainers"
+version = "2.4.0"
+description = "Sorted Containers -- Sorted List, Sorted Dict, Sorted Set"
+optional = false
+python-versions = "*"
+files = [
+    {file = "sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0"},
+    {file = "sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88"},
+]
+
 [[package]]
 name = "soupsieve"
 version = "2.6"
@@ -1970,6 +2886,17 @@ pure-eval = "*"
 [package.extras]
 tests = ["cython", "littleutils", "pygments", "pytest", "typeguard"]
 
+[[package]]
+name = "text-unidecode"
+version = "1.3"
+description = "The most basic Text::Unidecode port"
+optional = false
+python-versions = "*"
+files = [
+    {file = "text-unidecode-1.3.tar.gz", hash = "sha256:bad6603bb14d279193107714b288be206cac565dfa49aa5b105294dd5c4aab93"},
+    {file = "text_unidecode-1.3-py2.py3-none-any.whl", hash = "sha256:1311f10e8b895935241623731c2ba64f4c455287888b18189350b67134a822e8"},
+]
+
 [[package]]
 name = "tiktoken"
 version = "0.8.0"
@@ -2072,6 +2999,42 @@ files = [
 docs = ["myst-parser", "pydata-sphinx-theme", "sphinx"]
 test = ["argcomplete (>=3.0.3)", "mypy (>=1.7.0)", "pre-commit", "pytest (>=7.0,<8.2)", "pytest-mock", "pytest-mypy-testing"]
 
+[[package]]
+name = "trio"
+version = "0.27.0"
+description = "A friendly Python library for async concurrency and I/O"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "trio-0.27.0-py3-none-any.whl", hash = "sha256:68eabbcf8f457d925df62da780eff15ff5dc68fd6b367e2dde59f7aaf2a0b884"},
+    {file = "trio-0.27.0.tar.gz", hash = "sha256:1dcc95ab1726b2da054afea8fd761af74bad79bd52381b84eae408e983c76831"},
+]
+
+[package.dependencies]
+attrs = ">=23.2.0"
+cffi = {version = ">=1.14", markers = "os_name == \"nt\" and implementation_name != \"pypy\""}
+exceptiongroup = {version = "*", markers = "python_version < \"3.11\""}
+idna = "*"
+outcome = "*"
+sniffio = ">=1.3.0"
+sortedcontainers = "*"
+
+[[package]]
+name = "trio-websocket"
+version = "0.11.1"
+description = "WebSocket library for Trio"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "trio-websocket-0.11.1.tar.gz", hash = "sha256:18c11793647703c158b1f6e62de638acada927344d534e3c7628eedcb746839f"},
+    {file = "trio_websocket-0.11.1-py3-none-any.whl", hash = "sha256:520d046b0d030cf970b8b2b2e00c4c2245b3807853ecd44214acd33d74581638"},
+]
+
+[package.dependencies]
+exceptiongroup = {version = "*", markers = "python_version < \"3.11\""}
+trio = ">=0.11"
+wsproto = ">=0.14"
+
 [[package]]
 name = "typing-extensions"
 version = "4.12.2"
@@ -2094,6 +3057,38 @@ files = [
     {file = "tzdata-2024.1.tar.gz", hash = "sha256:2674120f8d891909751c38abcdfd386ac0a5a1127954fbc332af6b5ceae07efd"},
 ]
 
+[[package]]
+name = "tzlocal"
+version = "5.2"
+description = "tzinfo object for the local timezone"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "tzlocal-5.2-py3-none-any.whl", hash = "sha256:49816ef2fe65ea8ac19d19aa7a1ae0551c834303d5014c6d5a62e4cbda8047b8"},
+    {file = "tzlocal-5.2.tar.gz", hash = "sha256:8d399205578f1a9342816409cc1e46a93ebd5755e39ea2d85334bea911bf0e6e"},
+]
+
+[package.dependencies]
+tzdata = {version = "*", markers = "platform_system == \"Windows\""}
+
+[package.extras]
+devenv = ["check-manifest", "pytest (>=4.3)", "pytest-cov", "pytest-mock (>=3.3)", "zest.releaser"]
+
+[[package]]
+name = "undetected-chromedriver"
+version = "3.5.5"
+description = "('Selenium.webdriver.Chrome replacement with compatiblity for Brave, and other Chromium based browsers.', 'Not triggered by CloudFlare/Imperva/hCaptcha and such.', 'NOTE: results may vary due to many factors. No guarantees are given, except for ongoing efforts in understanding detection algorithms.')"
+optional = false
+python-versions = "*"
+files = [
+    {file = "undetected-chromedriver-3.5.5.tar.gz", hash = "sha256:9f945e1435005247abe17de316bcfda85b284a4177fd5f25167c78ced33b65ec"},
+]
+
+[package.dependencies]
+requests = "*"
+selenium = ">=4.9.0"
+websockets = "*"
+
 [[package]]
 name = "update-checker"
 version = "0.18.0"
@@ -2124,6 +3119,9 @@ files = [
     {file = "urllib3-2.2.2.tar.gz", hash = "sha256:dd505485549a7a552833da5e6063639d0d177c04f23bc3864e41e5dc5f612168"},
 ]
 
+[package.dependencies]
+pysocks = {version = ">=1.5.6,<1.5.7 || >1.5.7,<2.0", optional = true, markers = "extra == \"socks\""}
+
 [package.extras]
 brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"]
 h2 = ["h2 (>=4,<5)"]
@@ -2141,6 +3139,33 @@ files = [
     {file = "wcwidth-0.2.13.tar.gz", hash = "sha256:72ea0c06399eb286d978fdedb6923a9eb47e1c486ce63e9b4e64fc18303972b5"},
 ]
 
+[[package]]
+name = "webdriver-manager"
+version = "4.0.2"
+description = "Library provides the way to automatically manage drivers for different browsers"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "webdriver_manager-4.0.2-py2.py3-none-any.whl", hash = "sha256:75908d92ecc45ff2b9953614459c633db8f9aa1ff30181cefe8696e312908129"},
+    {file = "webdriver_manager-4.0.2.tar.gz", hash = "sha256:efedf428f92fd6d5c924a0d054e6d1322dd77aab790e834ee767af392b35590f"},
+]
+
+[package.dependencies]
+packaging = "*"
+python-dotenv = "*"
+requests = "*"
+
+[[package]]
+name = "webencodings"
+version = "0.5.1"
+description = "Character encoding aliases for legacy web content"
+optional = false
+python-versions = "*"
+files = [
+    {file = "webencodings-0.5.1-py2.py3-none-any.whl", hash = "sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78"},
+    {file = "webencodings-0.5.1.tar.gz", hash = "sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923"},
+]
+
 [[package]]
 name = "websocket-client"
 version = "1.8.0"
@@ -2157,6 +3182,84 @@ docs = ["Sphinx (>=6.0)", "myst-parser (>=2.0.0)", "sphinx-rtd-theme (>=1.1.0)"]
 optional = ["python-socks", "wsaccel"]
 test = ["websockets"]
 
+[[package]]
+name = "websockets"
+version = "14.1"
+description = "An implementation of the WebSocket Protocol (RFC 6455 & 7692)"
+optional = false
+python-versions = ">=3.9"
+files = [
+    {file = "websockets-14.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a0adf84bc2e7c86e8a202537b4fd50e6f7f0e4a6b6bf64d7ccb96c4cd3330b29"},
+    {file = "websockets-14.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:90b5d9dfbb6d07a84ed3e696012610b6da074d97453bd01e0e30744b472c8179"},
+    {file = "websockets-14.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2177ee3901075167f01c5e335a6685e71b162a54a89a56001f1c3e9e3d2ad250"},
+    {file = "websockets-14.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3f14a96a0034a27f9d47fd9788913924c89612225878f8078bb9d55f859272b0"},
+    {file = "websockets-14.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1f874ba705deea77bcf64a9da42c1f5fc2466d8f14daf410bc7d4ceae0a9fcb0"},
+    {file = "websockets-14.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9607b9a442392e690a57909c362811184ea429585a71061cd5d3c2b98065c199"},
+    {file = "websockets-14.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:bea45f19b7ca000380fbd4e02552be86343080120d074b87f25593ce1700ad58"},
+    {file = "websockets-14.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:219c8187b3ceeadbf2afcf0f25a4918d02da7b944d703b97d12fb01510869078"},
+    {file = "websockets-14.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:ad2ab2547761d79926effe63de21479dfaf29834c50f98c4bf5b5480b5838434"},
+    {file = "websockets-14.1-cp310-cp310-win32.whl", hash = "sha256:1288369a6a84e81b90da5dbed48610cd7e5d60af62df9851ed1d1d23a9069f10"},
+    {file = "websockets-14.1-cp310-cp310-win_amd64.whl", hash = "sha256:e0744623852f1497d825a49a99bfbec9bea4f3f946df6eb9d8a2f0c37a2fec2e"},
+    {file = "websockets-14.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:449d77d636f8d9c17952628cc7e3b8faf6e92a17ec581ec0c0256300717e1512"},
+    {file = "websockets-14.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a35f704be14768cea9790d921c2c1cc4fc52700410b1c10948511039be824aac"},
+    {file = "websockets-14.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b1f3628a0510bd58968c0f60447e7a692933589b791a6b572fcef374053ca280"},
+    {file = "websockets-14.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3c3deac3748ec73ef24fc7be0b68220d14d47d6647d2f85b2771cb35ea847aa1"},
+    {file = "websockets-14.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7048eb4415d46368ef29d32133134c513f507fff7d953c18c91104738a68c3b3"},
+    {file = "websockets-14.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f6cf0ad281c979306a6a34242b371e90e891bce504509fb6bb5246bbbf31e7b6"},
+    {file = "websockets-14.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:cc1fc87428c1d18b643479caa7b15db7d544652e5bf610513d4a3478dbe823d0"},
+    {file = "websockets-14.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:f95ba34d71e2fa0c5d225bde3b3bdb152e957150100e75c86bc7f3964c450d89"},
+    {file = "websockets-14.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:9481a6de29105d73cf4515f2bef8eb71e17ac184c19d0b9918a3701c6c9c4f23"},
+    {file = "websockets-14.1-cp311-cp311-win32.whl", hash = "sha256:368a05465f49c5949e27afd6fbe0a77ce53082185bbb2ac096a3a8afaf4de52e"},
+    {file = "websockets-14.1-cp311-cp311-win_amd64.whl", hash = "sha256:6d24fc337fc055c9e83414c94e1ee0dee902a486d19d2a7f0929e49d7d604b09"},
+    {file = "websockets-14.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:ed907449fe5e021933e46a3e65d651f641975a768d0649fee59f10c2985529ed"},
+    {file = "websockets-14.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:87e31011b5c14a33b29f17eb48932e63e1dcd3fa31d72209848652310d3d1f0d"},
+    {file = "websockets-14.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:bc6ccf7d54c02ae47a48ddf9414c54d48af9c01076a2e1023e3b486b6e72c707"},
+    {file = "websockets-14.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9777564c0a72a1d457f0848977a1cbe15cfa75fa2f67ce267441e465717dcf1a"},
+    {file = "websockets-14.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a655bde548ca98f55b43711b0ceefd2a88a71af6350b0c168aa77562104f3f45"},
+    {file = "websockets-14.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a3dfff83ca578cada2d19e665e9c8368e1598d4e787422a460ec70e531dbdd58"},
+    {file = "websockets-14.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6a6c9bcf7cdc0fd41cc7b7944447982e8acfd9f0d560ea6d6845428ed0562058"},
+    {file = "websockets-14.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:4b6caec8576e760f2c7dd878ba817653144d5f369200b6ddf9771d64385b84d4"},
+    {file = "websockets-14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:eb6d38971c800ff02e4a6afd791bbe3b923a9a57ca9aeab7314c21c84bf9ff05"},
+    {file = "websockets-14.1-cp312-cp312-win32.whl", hash = "sha256:1d045cbe1358d76b24d5e20e7b1878efe578d9897a25c24e6006eef788c0fdf0"},
+    {file = "websockets-14.1-cp312-cp312-win_amd64.whl", hash = "sha256:90f4c7a069c733d95c308380aae314f2cb45bd8a904fb03eb36d1a4983a4993f"},
+    {file = "websockets-14.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:3630b670d5057cd9e08b9c4dab6493670e8e762a24c2c94ef312783870736ab9"},
+    {file = "websockets-14.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:36ebd71db3b89e1f7b1a5deaa341a654852c3518ea7a8ddfdf69cc66acc2db1b"},
+    {file = "websockets-14.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:5b918d288958dc3fa1c5a0b9aa3256cb2b2b84c54407f4813c45d52267600cd3"},
+    {file = "websockets-14.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:00fe5da3f037041da1ee0cf8e308374e236883f9842c7c465aa65098b1c9af59"},
+    {file = "websockets-14.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8149a0f5a72ca36720981418eeffeb5c2729ea55fa179091c81a0910a114a5d2"},
+    {file = "websockets-14.1-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:77569d19a13015e840b81550922056acabc25e3f52782625bc6843cfa034e1da"},
+    {file = "websockets-14.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:cf5201a04550136ef870aa60ad3d29d2a59e452a7f96b94193bee6d73b8ad9a9"},
+    {file = "websockets-14.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:88cf9163ef674b5be5736a584c999e98daf3aabac6e536e43286eb74c126b9c7"},
+    {file = "websockets-14.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:836bef7ae338a072e9d1863502026f01b14027250a4545672673057997d5c05a"},
+    {file = "websockets-14.1-cp313-cp313-win32.whl", hash = "sha256:0d4290d559d68288da9f444089fd82490c8d2744309113fc26e2da6e48b65da6"},
+    {file = "websockets-14.1-cp313-cp313-win_amd64.whl", hash = "sha256:8621a07991add373c3c5c2cf89e1d277e49dc82ed72c75e3afc74bd0acc446f0"},
+    {file = "websockets-14.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:01bb2d4f0a6d04538d3c5dfd27c0643269656c28045a53439cbf1c004f90897a"},
+    {file = "websockets-14.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:414ffe86f4d6f434a8c3b7913655a1a5383b617f9bf38720e7c0799fac3ab1c6"},
+    {file = "websockets-14.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8fda642151d5affdee8a430bd85496f2e2517be3a2b9d2484d633d5712b15c56"},
+    {file = "websockets-14.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cd7c11968bc3860d5c78577f0dbc535257ccec41750675d58d8dc66aa47fe52c"},
+    {file = "websockets-14.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a032855dc7db987dff813583d04f4950d14326665d7e714d584560b140ae6b8b"},
+    {file = "websockets-14.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b7e7ea2f782408c32d86b87a0d2c1fd8871b0399dd762364c731d86c86069a78"},
+    {file = "websockets-14.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:39450e6215f7d9f6f7bc2a6da21d79374729f5d052333da4d5825af8a97e6735"},
+    {file = "websockets-14.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:ceada5be22fa5a5a4cdeec74e761c2ee7db287208f54c718f2df4b7e200b8d4a"},
+    {file = "websockets-14.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:3fc753451d471cff90b8f467a1fc0ae64031cf2d81b7b34e1811b7e2691bc4bc"},
+    {file = "websockets-14.1-cp39-cp39-win32.whl", hash = "sha256:14839f54786987ccd9d03ed7f334baec0f02272e7ec4f6e9d427ff584aeea8b4"},
+    {file = "websockets-14.1-cp39-cp39-win_amd64.whl", hash = "sha256:d9fd19ecc3a4d5ae82ddbfb30962cf6d874ff943e56e0c81f5169be2fda62979"},
+    {file = "websockets-14.1-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:e5dc25a9dbd1a7f61eca4b7cb04e74ae4b963d658f9e4f9aad9cd00b688692c8"},
+    {file = "websockets-14.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:04a97aca96ca2acedf0d1f332c861c5a4486fdcba7bcef35873820f940c4231e"},
+    {file = "websockets-14.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:df174ece723b228d3e8734a6f2a6febbd413ddec39b3dc592f5a4aa0aff28098"},
+    {file = "websockets-14.1-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:034feb9f4286476f273b9a245fb15f02c34d9586a5bc936aff108c3ba1b21beb"},
+    {file = "websockets-14.1-pp310-pypy310_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:660c308dabd2b380807ab64b62985eaccf923a78ebc572bd485375b9ca2b7dc7"},
+    {file = "websockets-14.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:5a42d3ecbb2db5080fc578314439b1d79eef71d323dc661aa616fb492436af5d"},
+    {file = "websockets-14.1-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:ddaa4a390af911da6f680be8be4ff5aaf31c4c834c1a9147bc21cbcbca2d4370"},
+    {file = "websockets-14.1-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:a4c805c6034206143fbabd2d259ec5e757f8b29d0a2f0bf3d2fe5d1f60147a4a"},
+    {file = "websockets-14.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:205f672a6c2c671a86d33f6d47c9b35781a998728d2c7c2a3e1cf3333fcb62b7"},
+    {file = "websockets-14.1-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5ef440054124728cc49b01c33469de06755e5a7a4e83ef61934ad95fc327fbb0"},
+    {file = "websockets-14.1-pp39-pypy39_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e7591d6f440af7f73c4bd9404f3772bfee064e639d2b6cc8c94076e71b2471c1"},
+    {file = "websockets-14.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:25225cc79cfebc95ba1d24cd3ab86aaa35bcd315d12fa4358939bd55e9bd74a5"},
+    {file = "websockets-14.1-py3-none-any.whl", hash = "sha256:4d4fc827a20abe6d544a119896f6b78ee13fe81cbfef416f3f2ddf09a03f0e2e"},
+    {file = "websockets-14.1.tar.gz", hash = "sha256:398b10c77d471c0aab20a845e7a60076b6390bfdaac7a6d2edb0d2c59d75e8d8"},
+]
+
 [[package]]
 name = "widgetsnbextension"
 version = "4.0.13"
@@ -2182,7 +3285,21 @@ files = [
 [package.extras]
 dev = ["black (>=19.3b0)", "pytest (>=4.6.2)"]
 
+[[package]]
+name = "wsproto"
+version = "1.2.0"
+description = "WebSockets state-machine based protocol implementation"
+optional = false
+python-versions = ">=3.7.0"
+files = [
+    {file = "wsproto-1.2.0-py3-none-any.whl", hash = "sha256:b9acddd652b585d75b20477888c56642fdade28bdfd3579aa24a4d2c037dd736"},
+    {file = "wsproto-1.2.0.tar.gz", hash = "sha256:ad565f26ecb92588a3e43bc3d96164de84cd9902482b130d0ddbaa9664a85065"},
+]
+
+[package.dependencies]
+h11 = ">=0.9.0,<1"
+
 [metadata]
 lock-version = "2.0"
 python-versions = ">=3.10,<4.0"
-content-hash = "c80659b43a5d0c991892e24f288aad5527d5896fc8928413fa59e7fcbeb59c1d"
+content-hash = "35f407a3a2ca5df109fc3ff7741828f016f1fa3fc90b0c651b4414a2bac71905"
diff --git a/pyproject.toml b/pyproject.toml
index 7a45f269..b1840419 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -22,6 +22,8 @@ anycache = "^2.2.0"
 tiktoken = "^0.8.0"
 cloudscraper = "^1.2.71"
 jinja2 = "^3.1.5"
+openai = "^1.58.1"
+lightnovel-crawler = "^3.7.2"
 
 [[tool.poetry.source]]
 # pytorch cuda needs to compe from another source https://python-poetry.org/docs/dependency-specification/#source-dependencies