mirror of
https://github.com/wassname/scrape_r_rational.git
synced 2026-09-10 12:35:53 +08:00
refactor, and remove nowrap
This commit is contained in:
wassname
+1
-1
@@ -58,7 +58,7 @@
|
||||
<a href="https://github.com/wassname/scrape_r_rational">source code</a>
|
||||
<div class="tabl">
|
||||
|
||||
<table id="myTable" class="table table-sm table-striped table-bordered nowrap" style="width:100%"></table>
|
||||
<table id="myTable" class="table table-sm table-striped table-bordered" style="width:100%"></table>
|
||||
</div>
|
||||
|
||||
</body>
|
||||
|
||||
+8
-117
@@ -5237,15 +5237,6 @@
|
||||
"print(f\"{len(df_links3)} -> {len(df3)} after title dedup\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 69,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# df3.iloc[3]['url']"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
@@ -5255,41 +5246,11 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 70,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def format_flair(author_flair_text):\n",
|
||||
" if author_flair_text:\n",
|
||||
" return f\" <em>{author_flair_text}</em>\"\n",
|
||||
" return \"\"\n",
|
||||
"\n",
|
||||
"import markdown\n",
|
||||
"def commentmd2html(x: dict) -> str:\n",
|
||||
" body = markdown.markdown(x['body'])\n",
|
||||
" ts = pd.to_datetime(x['created_utc'], unit='s').strftime('%Y-%m-%d')\n",
|
||||
" flair = format_flair(x['author_flair_text'])\n",
|
||||
" url = prefix + x['permalink']\n",
|
||||
" s = f\"\"\"<h3><a href=\"{url}\">{x.get('author', 'anon')} [{x['score']:+}] {flair} <sup>{ts}</sup></a></h3>\n",
|
||||
"{body}\n",
|
||||
"\"\"\"\n",
|
||||
" # print(s)\n",
|
||||
" return s\n",
|
||||
"\n",
|
||||
"def collapsibe(title, body):\n",
|
||||
" return f\"\"\"<details><summary>{title}</summary>\n",
|
||||
"{body}\n",
|
||||
"</details>\n",
|
||||
"\"\"\"\n",
|
||||
"\n",
|
||||
"prefix = \"https://reddit.com\"\n",
|
||||
"def c2md(x):\n",
|
||||
" return collapsibe(x['id'], commentmd2html(x))\n",
|
||||
"\n",
|
||||
"# # QC test\n",
|
||||
"# x = df3.iloc[0].comments[0]\n",
|
||||
"# from IPython.display import display, HTML\n",
|
||||
"# display(HTML(c2md(x)))"
|
||||
"from rrational.transform import join_uniq, chain_lists, format_flair, c2md, collapsibe, urls2a,url2a, unique_elements "
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -5317,33 +5278,6 @@
|
||||
"d = df3.reset_index().sort_values(\"score\", ascending=False)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def url2a(url):\n",
|
||||
" text = url\n",
|
||||
" if \"reddit.com/r/rational\" in url:\n",
|
||||
" text = url.split(\"/\")[-2]\n",
|
||||
" # text = url.replace('https://reddit.com/r/rational/comments/', '')\n",
|
||||
"\n",
|
||||
" return f'<a href=\"{url}\">{text}</a>'\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"from collections import OrderedDict\n",
|
||||
"def unique_elements(lst):\n",
|
||||
" return list(OrderedDict.fromkeys(lst))\n",
|
||||
"\n",
|
||||
"def urls2a(urls, sep=None):\n",
|
||||
" if isinstance(urls, str):\n",
|
||||
" urls = urls.split(\"\\n\")\n",
|
||||
"\n",
|
||||
" # get uniques from list, keep in same order\n",
|
||||
" urls = unique_elements(urls)\n",
|
||||
"\n",
|
||||
" a_els = [url2a(u) for u in urls]\n",
|
||||
"\n",
|
||||
" # now make into a html list\n",
|
||||
" if sep is None:\n",
|
||||
" return \"<ul>\" + \"\".join([f\"<li>{x}</li>\" for x in a_els]) + \"</ul>\"\n",
|
||||
" else:\n",
|
||||
" return sep.join(a_els)\n",
|
||||
"\n",
|
||||
"# make title have a link to first url\n",
|
||||
"d['title'] = d.progress_apply(lambda x: f'<a href=\"{x[\"url\"][0]}\">{x[\"title\"]}</a>', axis=1)\n",
|
||||
@@ -5361,13 +5295,6 @@
|
||||
"d['last_link_utc'] = d['last_link_utc'].dt.strftime('%Y-%m-%d')\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 72,
|
||||
@@ -5636,51 +5563,15 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 85,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"'[{\"title\": \"title\", \"visible\": true, \"searchable\": true, \"footer\": \"title\"}, {\"title\": \"score\", \"visible\": true, \"searchable\": true, \"footer\": \"score\"}, {\"title\": \"#link\", \"visible\": true, \"searchable\": true, \"footer\": \"#link\"}, {\"title\": \"#comment\", \"visible\": true, \"searchable\": true, \"footer\": \"#comment\"}, {\"title\": \"first_link\", \"visible\": false, \"searchable\": false, \"footer\": \"first_link\"}, {\"title\": \"last_link\", \"visible\": false, \"searchable\": false, \"footer\": \"last_link\"}, {\"title\": \"urls\", \"visible\": true, \"searchable\": true, \"footer\": \"urls\"}, {\"title\": \"threads\", \"visible\": true, \"searchable\": true, \"footer\": \"threads\"}, {\"title\": \"comments\", \"visible\": false, \"searchable\": false, \"footer\": \"comments\"}]'"
|
||||
]
|
||||
},
|
||||
"execution_count": 85,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# put into the html template\n",
|
||||
"import jinja2\n",
|
||||
"\n",
|
||||
"environment = jinja2.Environment()\n",
|
||||
"template = open(\"../index.jinja2.html\").read()\n",
|
||||
"template = environment.from_string(template)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"data = d.to_json(orient=\"values\")\n",
|
||||
"\n",
|
||||
"hidden = [\"comments\", \"first_link\", \"last_link\"]\n",
|
||||
"columns = [\n",
|
||||
" {\n",
|
||||
" \"title\": c,\n",
|
||||
" \"visible\": c not in hidden,\n",
|
||||
" \"searchable\": c not in hidden,\n",
|
||||
" \"footer\": c,\n",
|
||||
" }\n",
|
||||
" for c in d.columns\n",
|
||||
"]\n",
|
||||
"columns = json.dumps(columns)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"html = template.render(\n",
|
||||
" data=data,\n",
|
||||
" columns=columns,\n",
|
||||
")\n",
|
||||
"from rrational.export import export_df_2_html\n",
|
||||
"html_out = Path(\"../index2.html\").resolve()\n",
|
||||
"open(html_out, \"w\").write(html)\n",
|
||||
"columns"
|
||||
"export_df_2_html(df=d, output=html_out,\n",
|
||||
" hidden_columns=[\"comments\", \"first_link\", \"last_link\"],\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
||||
+147
-1206
File diff suppressed because it is too large
Load Diff
+1703
File diff suppressed because it is too large
Load Diff
Generated
+679
-695
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,52 @@
|
||||
from io import BytesIO
|
||||
import base64
|
||||
import gzip
|
||||
import jinja2
|
||||
import pandas as pd
|
||||
from pathlib import Path
|
||||
import json
|
||||
from IPython.display import HTML, display
|
||||
|
||||
def base64_gzip_encode(data: str) -> str:
|
||||
"""because there is a lot of uncompressed text data, we can compress it to gzip and store as base64. This isn't perfect but it's simple and works with a single static page"""
|
||||
buf = BytesIO()
|
||||
compressGzip = gzip.GzipFile(fileobj=buf, mode="wb")
|
||||
compressGzip.write(data.encode())
|
||||
compressGzip.close()
|
||||
buf.seek(0)
|
||||
|
||||
b64str = base64.b64encode(buf.getvalue()).decode()
|
||||
return b64str
|
||||
|
||||
|
||||
def export_df_2_html(df: pd.DataFrame = None, template: Path=Path( "../index.jinja2.html"), output: Path = Path("../index.html"), columns:list = None, hidden_columns: list = []):
|
||||
"""
|
||||
"""
|
||||
|
||||
environment = jinja2.Environment()
|
||||
template_o = open("../index.jinja2.html").read()
|
||||
template_o = environment.from_string(template_o)
|
||||
|
||||
columns = [
|
||||
{
|
||||
"title": c,
|
||||
"visible": c not in hidden_columns,
|
||||
"searchable": c not in hidden_columns,
|
||||
"footer": c,
|
||||
"name": c+"2",
|
||||
}
|
||||
for c in df.columns
|
||||
]
|
||||
columns = json.dumps(columns)
|
||||
|
||||
b64str = base64_gzip_encode(df.to_json(orient="values"))
|
||||
|
||||
html = template_o.render(
|
||||
data=b64str,
|
||||
columns=columns,
|
||||
)
|
||||
html_out = Path(output).resolve()
|
||||
open(html_out, "w").write(html)
|
||||
|
||||
htmla = f'<a href="{html_out}">View the page {html_out}</a>'
|
||||
display(HTML(htmla))
|
||||
@@ -0,0 +1,124 @@
|
||||
import markdown
|
||||
import pandas as pd
|
||||
import json
|
||||
import collections, itertools
|
||||
from collections import OrderedDict
|
||||
|
||||
|
||||
def join_uniq(x: list[str]):
|
||||
return "\n".join(set(x))
|
||||
|
||||
|
||||
def chain_lists(x: list[list[str]]):
|
||||
return [item for sublist in x for item in sublist]
|
||||
|
||||
def format_flair(author_flair_text):
|
||||
if author_flair_text:
|
||||
return f" <em>{author_flair_text}</em>"
|
||||
return ""
|
||||
|
||||
def commentmd2html(x: dict) -> str:
|
||||
"""convert a comment to html"""
|
||||
body = markdown.markdown(x['body'])
|
||||
ts = pd.to_datetime(x['created_utc'], unit='s').strftime('%Y-%m-%d')
|
||||
flair = format_flair(x['author_flair_text'])
|
||||
url = prefix + x['permalink']
|
||||
s = f"""<h3><a href="{url}">{x.get('author', 'anon')} [{x['score']:+}] {flair} <sup>{ts}</sup></a></h3>
|
||||
{body}
|
||||
"""
|
||||
# print(s)
|
||||
return s
|
||||
|
||||
|
||||
def collapsibe(title, body):
|
||||
"""make a collapsible html element"""
|
||||
return f"""<details><summary>{title}</summary>
|
||||
{body}
|
||||
</details>
|
||||
"""
|
||||
|
||||
prefix = "https://reddit.com"
|
||||
def c2md(x):
|
||||
"""md comment to html"""
|
||||
return collapsibe(x['id'], commentmd2html(x))
|
||||
|
||||
|
||||
def url2a(url):
|
||||
"""url to a tag"""
|
||||
text = url
|
||||
if "reddit.com/r/rational" in url:
|
||||
text = url.split("/")[-2]
|
||||
# text = url.replace('https://reddit.com/r/rational/comments/', '')
|
||||
|
||||
return f'<a href="{url}">{text}</a>'
|
||||
|
||||
|
||||
|
||||
|
||||
def unique_elements(lst):
|
||||
"""get unique elements but unlike a set, keep them ordered."""
|
||||
return list(OrderedDict.fromkeys(lst))
|
||||
|
||||
def urls2a(urls, sep=None):
|
||||
"""urls to a tags"""
|
||||
if isinstance(urls, str):
|
||||
urls = urls.split("\n")
|
||||
|
||||
# get uniques from list, keep in same order
|
||||
urls = unique_elements(urls)
|
||||
|
||||
a_els = [url2a(u) for u in urls]
|
||||
|
||||
# now make into a html list
|
||||
if sep is None:
|
||||
return "<ul>" + "".join([f"<li>{x}</li>" for x in a_els]) + "</ul>"
|
||||
else:
|
||||
return sep.join(a_els)
|
||||
|
||||
import numpy as np
|
||||
|
||||
def auto_transform_to_html(d):
|
||||
for c in d.columns:
|
||||
# if the cols is a lists of strings
|
||||
is_list = d[c].apply(lambda x: isinstance(x, (list, tuple, np.ndarray))).all()
|
||||
is_list_str = is_list and d[c].apply(lambda x: (x is None) or (len(x)==0) or isinstance(x[0], str)).all()
|
||||
is_str = d[c].apply(lambda x: isinstance(x, str)).all()
|
||||
is_list_of_urls = is_list_str and d[c].apply(lambda x: (len(x)==0) or x[0].startswith("http")).all()
|
||||
is_urls = is_str and d[c].apply(lambda x: x.startswith("http")).all()
|
||||
is_url = d[c].apply(lambda x: isinstance(x, str)).all() and d[c].apply(lambda x: x.startswith("http")).all()
|
||||
is_list_obj = is_list and d[c].apply(lambda x: isinstance(x, dict)).all()
|
||||
|
||||
print(f"{c}, is_list={is_list}, is_list_str={is_list_str}, is_list_url={is_list_of_urls}, is_url={is_url}, is_urls={is_urls}, is_list_obj={is_list_obj}\n")
|
||||
|
||||
# if columns contains str: urls
|
||||
if is_urls:
|
||||
d[c] = d[c].apply(lambda x: url2a(x))
|
||||
# elif c.endswith("urls"):
|
||||
# d[c] = d[c].apply(lambda x: collapsibe('...', urls2a(x)))
|
||||
# elif c.endswith("url"):
|
||||
# d[c] = d[c].apply(lambda x: url2a(x))
|
||||
elif is_list_of_urls:
|
||||
d[c] = d[c].apply(lambda x: collapsibe('...', urls2a(x)))
|
||||
elif is_list_str:
|
||||
d[c] = d[c].apply(lambda x: join_uniq(x))
|
||||
|
||||
|
||||
# if float, round to 2 decimal places
|
||||
elif d[c].dtype == float:
|
||||
d[c] = d[c].apply(lambda x: round(x, 2))
|
||||
|
||||
# if column name ends with utc
|
||||
elif c.endswith("utc"):
|
||||
d[c] = pd.to_datetime(d[c], unit='s').dt.strftime('%Y-%m-%d')
|
||||
|
||||
elif is_list:
|
||||
# TODO object to json using pandas.io.json.dumps
|
||||
# from pandas.io.json._json import to_json
|
||||
from pandas._libs.json import ujson_dumps
|
||||
d[c] = d[c].apply(lambda x: collapsibe('...', ujson_dumps(x, indent=2)))
|
||||
|
||||
|
||||
# make title have a link to first url
|
||||
d['title'] = d.apply(lambda x: f'<a href="{x["url"][0]}">{x["title"]}</a>', axis=1)
|
||||
return d
|
||||
|
||||
Reference in New Issue
Block a user