mirror of
https://github.com/wassname/scrape_r_rational.git
synced 2026-09-09 11:33:49 +08:00
wip
This commit is contained in:
wassname
-234
@@ -1,234 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# Usage:
|
||||
# `tmuxnewsh2 r_rational indirm ~dl/subreddits/rational subreddit2org.py rational 1000000000`
|
||||
# `tmuxnewsh2 r_HPfanfiction indirm ~dl/subreddits/HPfanfiction subreddit2org.py HPfanfiction 1000000000`
|
||||
#
|
||||
# Old usage:
|
||||
# `reddit_sub_posts_urls_pushshift.py rational 100000000 > rational.txt`
|
||||
#
|
||||
# Docs:
|
||||
# - https://github.com/praw-dev/praw
|
||||
# - https://github.com/dmarx/psaw
|
||||
##
|
||||
from IPython import embed
|
||||
|
||||
import sys
|
||||
import os
|
||||
import traceback
|
||||
import time
|
||||
|
||||
link_output = False
|
||||
limit = 10
|
||||
subreddit = "rational"
|
||||
|
||||
al = len(sys.argv)
|
||||
if al >= 2:
|
||||
subreddit = sys.argv[1]
|
||||
if al >= 3:
|
||||
limit = int(sys.argv[2])
|
||||
|
||||
from psaw import PushshiftAPI
|
||||
import json
|
||||
|
||||
if link_output:
|
||||
api = PushshiftAPI()
|
||||
else:
|
||||
import praw
|
||||
from praw.exceptions import DuplicateReplaceException
|
||||
from praw.models.reddit.more import MoreComments
|
||||
|
||||
r = praw.Reddit(
|
||||
client_id=os.environ["REDDIT_CLIENT_ID"],
|
||||
client_secret=os.environ["REDDIT_CLIENT_SECRET"],
|
||||
user_agent='User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36',
|
||||
# password=os.environ["REDDIT_PASSWORD"],
|
||||
# username=os.environ["REDDIT_USERNAME"],
|
||||
)
|
||||
|
||||
api = PushshiftAPI(r)
|
||||
|
||||
print(f"Getting the submissions from the subreddit {subreddit} (limit={limit}) ...\n", file=sys.stderr, flush=True)
|
||||
gen = api.search_submissions(subreddit=subreddit, limit=limit)
|
||||
results = list(gen)
|
||||
|
||||
##
|
||||
# embed()
|
||||
# print(json.dumps(results))
|
||||
##
|
||||
|
||||
|
||||
if link_output:
|
||||
for result in results:
|
||||
print(result.permalink)
|
||||
else:
|
||||
# @todo3 refactor this into a standalone reddit2org (we need a way to get the submission object from the URL)
|
||||
##
|
||||
from brish import z, zp, bsh
|
||||
|
||||
def stars(lv):
|
||||
return "*" * lv + " "
|
||||
|
||||
def html2org(html):
|
||||
# tmp = "tmp.html"
|
||||
tmp = z("mktemp").outrs
|
||||
|
||||
res = z("cat > {tmp}", cmd_stdin=html)
|
||||
assert res
|
||||
|
||||
res = z("html2org {tmp}")
|
||||
assert res
|
||||
|
||||
z("command rm {tmp}")
|
||||
|
||||
return res.outrs
|
||||
|
||||
# def meta_get(c):
|
||||
# meta = ""
|
||||
# meta+=f"{c.author.name}"
|
||||
# if hasattr(c, "score"):
|
||||
# meta+=f" ({c.score})"
|
||||
|
||||
# return meta
|
||||
|
||||
def meta_get_props(c):
|
||||
meta = ":PROPERTIES:"
|
||||
|
||||
if c.author: # can be None when they are deleted
|
||||
meta+=f"\n:Author: {c.author.name}"
|
||||
|
||||
if hasattr(c, "score"):
|
||||
meta+=f"\n:Score: {c.score}"
|
||||
|
||||
if hasattr(c, "created_utc") and c.created_utc:
|
||||
meta+=f"\n:DateUnix: {c.created_utc}"
|
||||
res = z('gdate -d "@"{c.created_utc} +"%Y-%b-%d"')
|
||||
if res:
|
||||
meta+=f"\n:DateShort: {res.outrs}"
|
||||
|
||||
|
||||
if getattr(c, "link_flair_text", None):
|
||||
meta+=f"\n:FlairText: {c.link_flair_text}"
|
||||
|
||||
meta+="\n:END:\n"
|
||||
return meta
|
||||
|
||||
def process_comment(f, comments, lv, shortname):
|
||||
while True:
|
||||
try:
|
||||
comments.replace_more(limit=None)
|
||||
break
|
||||
except DuplicateReplaceException:
|
||||
print(traceback.format_exc())
|
||||
break
|
||||
except:
|
||||
print(traceback.format_exc())
|
||||
time.sleep(1)
|
||||
|
||||
l = len(comments) - 1
|
||||
|
||||
shortname_orig = shortname
|
||||
|
||||
for i, c in enumerate(comments):
|
||||
lv_c = lv
|
||||
shortname = shortname_orig
|
||||
|
||||
# if isinstance(c, MoreComments):
|
||||
# pass
|
||||
##
|
||||
# meta = meta_get(c)
|
||||
# meta+=": "
|
||||
##
|
||||
meta = meta_get_props(c)
|
||||
# Properties are key--value pairs. When they are associated with a single entry or with a tree they need to be inserted into a special drawer (see [[https://orgmode.org/manual/Drawers.html#Drawers][Drawers]]) with the name ‘=PROPERTIES=', which has to be located right below a headline, and its planning line (see [[https://orgmode.org/manual/Deadlines-and-Scheduling.html#Deadlines-and-Scheduling][Deadlines and Scheduling]]) when applicable.
|
||||
#
|
||||
# Still, putting the props after the heading is no fun; We can rename our drawer to :METADATA:, but why bother?
|
||||
##
|
||||
|
||||
head = "EMPTY_COMMENT"
|
||||
|
||||
# @todo3 using IDs creates too long paths. It's better if just use a counter that goes from 0 to N.
|
||||
c_id_old = c.id or z("uuidm").outrs[0:6]
|
||||
|
||||
##
|
||||
if lv_c <= 4:
|
||||
c_id = f"{i}_{c_id_old}"
|
||||
else:
|
||||
c_id = i
|
||||
# using this in different runs is unreliable, as the comments ordering can change. But since we use the comments' ID as their filenames, this won't result in data loss, but it can cause data duplication and a flawed comment hierarchy.
|
||||
# workarounds:
|
||||
# - delete the indices directory and re-run the whole scraping from scratch on every update
|
||||
# - @done use the first-n-level comments' ID as well
|
||||
##
|
||||
|
||||
shortname += f"/{c_id}"
|
||||
|
||||
if c.body_html:
|
||||
head = (html2org(c.body_html) or "EMPTY_COMMENT")
|
||||
index_file = f'indices/{shortname}/{c_id_old}.org'
|
||||
z("ensure-dir {index_file}")
|
||||
with open(index_file, "w") as f2:
|
||||
f2.write(f"{meta}\n{head}")
|
||||
|
||||
if head.startswith("#+"):
|
||||
# do not put blocks in headings (e.g., #+begin_quote)
|
||||
|
||||
author = "deleted"
|
||||
if c.author:
|
||||
author = c.author.name or author
|
||||
|
||||
head = f"u/{author}:\n{head}"
|
||||
|
||||
head = head or "_" # empty headers are invalid org-mode
|
||||
|
||||
f.write("\n" + stars(lv_c) + head + "\n" + meta)
|
||||
lv_c += 1
|
||||
process_comment(f, c.replies, lv_c, shortname)
|
||||
if l != i:
|
||||
f.write("\n")
|
||||
|
||||
def utf8len(s):
|
||||
return len(s.encode('utf-8'))
|
||||
|
||||
for result in results:
|
||||
try:
|
||||
# embed() ; exit()
|
||||
|
||||
# if not "looking at this sub" in result.title:
|
||||
# continue
|
||||
|
||||
f_name = z("ecn {result.title} | str2filename").outrs
|
||||
f_name = f_name[0:230]
|
||||
# [[id:a36bb01f-9b9b-40c9-816a-c762281c43c3][filesystem/filenames.org:maximum allowed length for filenames and paths]]
|
||||
if utf8len(f_name) > 240:
|
||||
f_name = f_name[0:100]
|
||||
|
||||
if utf8len(f_name) > 240:
|
||||
f_name = f_name[0:60]
|
||||
|
||||
shortname = f"{f_name}.{result.id}"
|
||||
f_name = f"posts/{shortname}.org"
|
||||
z("ensure-dir {f_name}")
|
||||
with open(f_name, "w") as f:
|
||||
lv = 1
|
||||
f.write(f"#+TITLE: {result.title}\n\n")
|
||||
|
||||
|
||||
if getattr(result, "url_overridden_by_dest", None):
|
||||
f.write(f"{stars(lv)}[[{result.url_overridden_by_dest}][{result.title}]]\n")
|
||||
lv += 1
|
||||
else:
|
||||
f.write(f"{stars(lv)}{result.title}\n")
|
||||
lv += 1
|
||||
|
||||
meta = meta_get_props(result)
|
||||
f.write(meta)
|
||||
|
||||
if result.selftext_html:
|
||||
f.write(html2org(result.selftext_html) + "\n\n")
|
||||
|
||||
process_comment(f, result.comments, lv, shortname)
|
||||
print(f"wrote {f_name}\n", file=sys.stderr, flush=True)
|
||||
|
||||
except:
|
||||
print(traceback.format_exc())
|
||||
embed()
|
||||
Reference in New Issue
Block a user