mirror of
https://github.com/wassname/scrape_r_rational.git
synced 2026-09-11 12:43:32 +08:00
wip
This commit is contained in:
wassname
+506
-1044
File diff suppressed because it is too large
Load Diff
+79
-43
@@ -23,7 +23,7 @@
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"10262"
|
||||
"39"
|
||||
]
|
||||
},
|
||||
"execution_count": 2,
|
||||
@@ -32,84 +32,111 @@
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"fs = sorted(Path('../data/cache').glob('*.md'))\n",
|
||||
"fs = sorted(Path('../data/cache2').glob('*.md'))\n",
|
||||
"len(fs)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Python script to extract links from Markdown using libraries\n",
|
||||
"import markdown\n",
|
||||
"from bs4 import BeautifulSoup\n",
|
||||
"# import markdown\n",
|
||||
"# from bs4 import BeautifulSoup\n",
|
||||
"\n",
|
||||
"# now extract links from these\n",
|
||||
"def extract_links_library(markdown_text):\n",
|
||||
" # Convert Markdown to HTML\n",
|
||||
" html = markdown.markdown(markdown_text)\n",
|
||||
"# # now extract links from these\n",
|
||||
"# def extract_links_library(markdown_text):\n",
|
||||
"# # Convert Markdown to HTML\n",
|
||||
"# html = markdown.markdown(markdown_text)\n",
|
||||
" \n",
|
||||
" # Parse HTML with BeautifulSoup\n",
|
||||
" soup = BeautifulSoup(html, 'html.parser')\n",
|
||||
"# # Parse HTML with BeautifulSoup\n",
|
||||
"# soup = BeautifulSoup(html, 'html.parser')\n",
|
||||
" \n",
|
||||
" # Find all <a> tags and extract href attributes\n",
|
||||
" links = [a['href'] for a in soup.find_all('a', href=True)]\n",
|
||||
"# # Find all <a> tags and extract href attributes\n",
|
||||
"# links = [a['href'] for a in soup.find_all('a', href=True)]\n",
|
||||
" \n",
|
||||
" return links\n",
|
||||
"# return links\n",
|
||||
"\n",
|
||||
"def extract_links_re(s:str):\n",
|
||||
" return re.findall(r'\\[.*?\\]\\((.*?)\\)', s)\n",
|
||||
"\n",
|
||||
"data = []\n",
|
||||
"for f in fs:\n",
|
||||
" s = f.open().read()\n",
|
||||
" data += extract_links_library(s)"
|
||||
" data += extract_links_re(s)\n",
|
||||
"\n",
|
||||
"# print(s)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"df_links = pd.Series(data).value_counts()\n",
|
||||
"# df_links"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"execution_count": 21,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"49 https://www.fanfiction.net/s/5193644\n",
|
||||
"49 https://www.fictionpress.com/s/2961893/1/Mothe...\n",
|
||||
"42 https://www.youtube.com/watch?v=kbyTOAlhRHk\n",
|
||||
"42 https://forums.sufficientvelocity.com/threads/...\n",
|
||||
"40 https://archiveofourown.org/works/11478249/cha...\n",
|
||||
" ... \n",
|
||||
"2 https://forums.sufficientvelocity.com/threads/...\n",
|
||||
"2 https://www.royalroad.com/fiction/15193/ave-xi...\n",
|
||||
"2 http://www.wipp.energy.gov/picsprog/articles/w...\n",
|
||||
"2 https://www.fanfiction.net/s/10360716/6/The-Me...\n",
|
||||
"2 https://archiveofourown.org/works/50506063/cha...\n",
|
||||
"Length: 2546, dtype: object"
|
||||
"(946,)"
|
||||
]
|
||||
},
|
||||
"execution_count": 5,
|
||||
"execution_count": 21,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"blocklist = ['reddit', 'redact', 'pastebin', 'wikipedia', 'docs.google', 'discord','tvtropes.org']\n",
|
||||
"df_links = pd.Series(data).value_counts()\n",
|
||||
"df_links.shape"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 23,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"39 https://www.reddit.com/r/rational/wiki\n",
|
||||
"38 https://www.reddit.com/r/rational/search?q=%22...\n",
|
||||
"6 https://www.royalroad.com/fiction/36735/the-pe...\n",
|
||||
"6 https://old.reddit.com/user/DomesticatedDungeo...\n",
|
||||
"5 https://www.royalroad.com/fiction/52324/violen...\n",
|
||||
" ... \n",
|
||||
"2 https://www.royalroad.com/fiction/58187/player...\n",
|
||||
"2 https://adarkroom.doublespeakgames.com/\n",
|
||||
"2 https://www.royalroad.com/fiction/10286/the-go...\n",
|
||||
"2 https://www.royalroad.com/fiction/4293/the-iro...\n",
|
||||
"2 https://twigserial.wordpress.com/\n",
|
||||
"Length: 107, dtype: object"
|
||||
]
|
||||
},
|
||||
"execution_count": 23,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"blocklist = [\n",
|
||||
"# 'reddit',\n",
|
||||
" 'redact', 'pastebin', 'wikipedia', 'docs.google', 'discord','tvtropes.org']\n",
|
||||
"df_links = df_links[~df_links.index.str.contains('|'.join(blocklist), regex=True)]\n",
|
||||
"df_links = df_links[df_links.index.str.startswith('http')]\n",
|
||||
"df_links = df_links[df_links > 1]\n",
|
||||
"pd.Series(df_links.index, index=df_links.values)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"TODO\n",
|
||||
"- [ ] dedup\n",
|
||||
"- [ ] convert to title"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
@@ -150,10 +177,19 @@
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Extra get a llm summary of each link\n",
|
||||
"\n",
|
||||
"Grab all md's that mention a story, ask claude to summarize\n",
|
||||
"\n",
|
||||
"We could also get total karma per mention"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
|
||||
File diff suppressed because one or more lines are too long
+608
-2548
File diff suppressed because it is too large
Load Diff
Generated
+47
-1
@@ -1,5 +1,20 @@
|
||||
# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand.
|
||||
|
||||
[[package]]
|
||||
name = "anycache"
|
||||
version = "2.2.0"
|
||||
description = "Cache any python object to file using improved pickling"
|
||||
optional = false
|
||||
python-versions = "<4,>=3.8"
|
||||
files = [
|
||||
{file = "anycache-2.2.0-py3-none-any.whl", hash = "sha256:f4c23284b33bab5f7f46c5d6f024025b944c8a223eb8638f497b68c9913c8922"},
|
||||
{file = "anycache-2.2.0.tar.gz", hash = "sha256:342ca6bf952f1b62f8725c92b50244f9ef6d0eb1547df768fe2ea2aa429f5c44"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
dill = ">=0.3.7,<0.4.0"
|
||||
filelock = ">=3.13.0,<4.0.0"
|
||||
|
||||
[[package]]
|
||||
name = "appnope"
|
||||
version = "0.1.4"
|
||||
@@ -422,6 +437,21 @@ files = [
|
||||
{file = "decorator-5.1.1.tar.gz", hash = "sha256:637996211036b6385ef91435e4fae22989472f9d571faba8927ba8253acbc330"},
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dill"
|
||||
version = "0.3.9"
|
||||
description = "serialize all of Python"
|
||||
optional = false
|
||||
python-versions = ">=3.8"
|
||||
files = [
|
||||
{file = "dill-0.3.9-py3-none-any.whl", hash = "sha256:468dff3b89520b474c0397703366b7b95eebe6303f108adf9b19da1f702be87a"},
|
||||
{file = "dill-0.3.9.tar.gz", hash = "sha256:81aa267dddf68cbfe8029c42ca9ec6a4ab3b22371d1c450abc54422577b4512c"},
|
||||
]
|
||||
|
||||
[package.extras]
|
||||
graph = ["objgraph (>=1.7.2)"]
|
||||
profile = ["gprof2dot (>=2022.7.29)"]
|
||||
|
||||
[[package]]
|
||||
name = "exceptiongroup"
|
||||
version = "1.2.2"
|
||||
@@ -450,6 +480,22 @@ files = [
|
||||
[package.extras]
|
||||
tests = ["asttokens (>=2.1.0)", "coverage", "coverage-enable-subprocess", "ipython", "littleutils", "pytest", "rich"]
|
||||
|
||||
[[package]]
|
||||
name = "filelock"
|
||||
version = "3.16.1"
|
||||
description = "A platform independent file lock."
|
||||
optional = false
|
||||
python-versions = ">=3.8"
|
||||
files = [
|
||||
{file = "filelock-3.16.1-py3-none-any.whl", hash = "sha256:2082e5703d51fbf98ea75855d9d5527e33d8ff23099bec374a134febee6946b0"},
|
||||
{file = "filelock-3.16.1.tar.gz", hash = "sha256:c249fbfcd5db47e5e2d6d62198e565475ee65e4831e2561c8e313fa7eb961435"},
|
||||
]
|
||||
|
||||
[package.extras]
|
||||
docs = ["furo (>=2024.8.6)", "sphinx (>=8.0.2)", "sphinx-autodoc-typehints (>=2.4.1)"]
|
||||
testing = ["covdefaults (>=2.3)", "coverage (>=7.6.1)", "diff-cover (>=9.2)", "pytest (>=8.3.3)", "pytest-asyncio (>=0.24)", "pytest-cov (>=5)", "pytest-mock (>=3.14)", "pytest-timeout (>=2.3.1)", "virtualenv (>=20.26.4)"]
|
||||
typing = ["typing-extensions (>=4.12.2)"]
|
||||
|
||||
[[package]]
|
||||
name = "fonttools"
|
||||
version = "4.53.1"
|
||||
@@ -1872,4 +1918,4 @@ dev = ["black (>=19.3b0)", "pytest (>=4.6.2)"]
|
||||
[metadata]
|
||||
lock-version = "2.0"
|
||||
python-versions = ">=3.10,<4.0"
|
||||
content-hash = "b9705889a54b7045a695ba5e868308ee7ff8c0133547a2ce2b1c9770d9dafe4d"
|
||||
content-hash = "651df152e86c6473c53b53c6cb3ad7648fb4a13e75e0578f677b19eb9740c478"
|
||||
|
||||
@@ -17,6 +17,7 @@ python-dotenv = "^1.0.1"
|
||||
praw = "^7.7.1"
|
||||
markdown = "^3.7"
|
||||
bs4 = "^0.0.2"
|
||||
anycache = "^2.2.0"
|
||||
|
||||
[[tool.poetry.source]]
|
||||
# pytorch cuda needs to compe from another source https://python-poetry.org/docs/dependency-specification/#source-dependencies
|
||||
|
||||
Reference in New Issue
Block a user