wassname/scrape_r_rational
1
0
Fork 0
mirror of https://github.com/wassname/scrape_r_rational.git synced 2026-09-09 11:33:49 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor add pushshift data

Browse Source
This commit is contained in:
wassname
2025-01-02 14:24:36 +08:00
parent f34c6357c9
commit 3e04add473
3 changed files with 251 additions and 256 deletions
Download Patch File Download Diff File Expand all files Collapse all files
nbs/mjc_001_download.ipynb
+168 -256
View File
@@ -12,7 +12,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 75,
"id": "198de680",
"metadata": {
"ExecuteTime": {
@@ -20,7 +20,16 @@
"start_time": "2022-06-28T02:34:01.864103Z"
}
},
"outputs": [],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"The autoreload extension is already loaded. To reload it, use:\n",
" %reload_ext autoreload\n"
]
}
],
"source": [
"# autoreload your package\n",
"%load_ext autoreload\n",
@@ -30,7 +39,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 76,
"id": "a372ed7c",
"metadata": {
"ExecuteTime": {
@@ -38,7 +47,18 @@
"start_time": "2022-06-28T02:34:02.424826Z"
}
},
"outputs": [],
"outputs": [
{
"data": {
"text/plain": [
"3"
]
},
"execution_count": 76,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"## secrets\n",
"from dotenv import load_dotenv\n",
@@ -71,10 +91,21 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 77,
"id": "a91f5c82",
"metadata": {},
"outputs": [],
"outputs": [
{
"data": {
"text/plain": [
"True"
]
},
"execution_count": 77,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# dotenv\n",
"import os\n",
@@ -84,7 +115,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 78,
"id": "54a03c3a",
"metadata": {},
"outputs": [],
@@ -103,23 +134,26 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 79,
"id": "fc3cb5ba",
"metadata": {},
"outputs": [],
"outputs": [
{
"data": {
"text/plain": [
"Subreddit(display_name='rational')"
]
},
"execution_count": 79,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"sub = reddit.subreddit(\"rational\")\n",
"sub"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "2cc86378",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "markdown",
"id": "c16239bc",
@@ -143,10 +177,21 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 80,
"id": "874c72dd",
"metadata": {},
"outputs": [],
"outputs": [
{
"data": {
"text/plain": [
"10825"
]
},
"execution_count": 80,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"import json\n",
"\n",
@@ -154,27 +199,63 @@
"\n",
"# get the old links from https://old.reddit.com/r/rational/wiki/monthlyrecommendation\n",
"f = Path('../data/from_wiki.json')\n",
"ids_manual += json.loads(f.open('r').read())[1:]\n",
"ids_manual += json.loads(f.open('r').read())\n",
"\n",
"f = Path('../data/other_threads.json')\n",
"ids_manual += json.loads(f.open('r').read())[1:]\n",
"ids_manual += json.loads(f.open('r').read())\n",
"\n",
"# from manual google search\n",
"f = Path('../data/from_google.json')\n",
"ids_manual += json.loads(f.open('r').read())[1:]\n",
"ids_manual += json.loads(f.open('r').read())\n",
"\n",
"# from a previous effort\n",
"ids_manual += json.load(open('../data/submissions_org_mode.json'))\n",
"\n",
"# from academic torrents / pushshift\n",
"ids_manual += json.load(open('../data/from_academic_torrents.json'))\n",
"\n",
"len(ids_manual)"
]
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 81,
"id": "9c2d0b07",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"['# from https://old.reddit.com/r/rational/wiki/edit/monthlyrecommendation',\n",
" '# from https://old.reddit.com/r/pushshift/comments/1akrhg3/separate_dump_files_for_the_top_40k_subreddits/']"
]
},
"execution_count": 81,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# show comments\n",
"[x for x in ids_manual if x.startswith('#')]"
]
},
{
"cell_type": "code",
"execution_count": 82,
"id": "1d1601bb",
"metadata": {},
"outputs": [],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"10825\n",
"10823 removed comments\n",
"10266 strip prefix, dedup\n"
]
}
],
"source": [
"# now dedup and strip the prefix\n",
"def strip_prefix(s2):\n",
@@ -185,6 +266,8 @@
" https://www.reddit.com/r/rational/comments/15i0vos/d_friday_open_thread/\n",
" 15i0vos\n",
" \"\"\"\n",
" if s2.startswith('# '):\n",
" return s2\n",
" s = s2 + ''\n",
" assert len(s) > 0, f'1empty string `{s2}`'\n",
" if s.startswith('http'):\n",
@@ -200,27 +283,16 @@
" assert len(s) > 0, f'3empty string gtp, {s2}'\n",
" return s\n",
" \n",
"\n",
"# strip comments\n",
"print(len(ids_manual))\n",
"ids_manual = [x for x in ids_manual if not x.startswith('#')]\n",
"print(len(ids_manual), 'removed comments')\n",
"ids_manual = [strip_prefix(x) for x in ids_manual]\n",
"print(len(ids_manual))\n",
"ids_manual = list(set(ids_manual))\n",
"print(len(ids_manual))\n",
"print(len(ids_manual), 'strip prefix, dedup')\n",
"# ids_manual"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "c8ac4023",
"metadata": {},
"outputs": [],
"source": [
"# # sort by len\n",
"# ids_manual = sorted(ids_manual, key=len)\n",
"# ids_manual[:10], ids_manual[-10:]"
]
},
{
"cell_type": "markdown",
"id": "c9e72a0f",
@@ -242,10 +314,19 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 83,
"id": "a9fe1659",
"metadata": {},
"outputs": [],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"100\n",
"200\n"
]
}
],
"source": [
"# get latest\n",
"submissions = list(sub.search(\"Recommendation Thread\"))\n",
@@ -258,7 +339,7 @@
},
{
"cell_type": "code",
"execution_count": 10,
"execution_count": 84,
"id": "49fc099b",
"metadata": {},
"outputs": [],
@@ -270,10 +351,19 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 85,
"id": "5b4856f6",
"metadata": {},
"outputs": [],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"10466\n",
"10266\n"
]
}
],
"source": [
"# dedup by item.id\n",
"print(len(submissions))\n",
@@ -285,106 +375,32 @@
},
{
"cell_type": "code",
"execution_count": 12,
"execution_count": 93,
"id": "5d7d0214",
"metadata": {},
"outputs": [],
"source": [
"# from https://github.dev/JosefAlbers/rd2md\n",
"\n",
"import textwrap\n",
"from datetime import datetime\n",
"\n",
"def format_flair(obj):\n",
" if obj.author_flair_text is not None:\n",
" return f\" *{obj.author_flair_text}*\"\n",
" return \"\"\n",
"\n",
"import humanize\n",
"\n",
"def format_comment(comment, depth=0, upvote_threshold=2, t0=None):\n",
" if comment.score < upvote_threshold:\n",
" return \"\"\n",
" \n",
" ts = pd.to_datetime(comment.created, unit='s')\n",
" if t0 is not None:\n",
" delta = ts - t0\n",
" delta = humanize.naturaldelta(delta.total_seconds() )\n",
" else:\n",
" delta = \"\"\n",
"\n",
" indent = \">\" * (depth+1) + \" \"\n",
" author_line = f\"{indent}**u/{comment.author}** [{comment.score:+}] {format_flair(comment)} ({delta} later)\\n\"\n",
" dedented_body = textwrap.dedent(comment.body)\n",
" # use re to replace multiple newlines with a single newline\n",
" \n",
" indented_body = textwrap.indent(dedented_body, indent)\n",
" indented_body = re.sub(r'\\n\\n+', f'\\n{indent}\\n', indented_body.strip())\n",
" comment_block = f\"{indent}\\n{indented_body}\\n\\n\"\n",
" formatted = author_line + comment_block\n",
" for reply in comment.replies:\n",
" formatted += format_comment(reply, depth + 1, upvote_threshold, t0)\n",
"\n",
" return formatted\n",
"\n",
"\n",
"def submission_to_markdown(post, comment_score_threshold=0, verbose=False):\n",
" post_content = []\n",
" post_content.append(f\"## {post.title}\\n\\n\")\n",
" if verbose:\n",
" ts = pd.to_datetime(post.created, unit='s')\n",
" post_content.append(f\"* Author: u/{post.author} {format_flair(post)}*\\n\")\n",
" post_content.append(f\"* URL: {post.url}\\n\")\n",
" post_content.append(f\"* Score: {post.score}\\n\\n\")\n",
" post_content.append(f\"* Created: {ts.isoformat()}\\n\\n\")\n",
" post_content.append(\"### Post:\\n\\n\")\n",
" if post.is_self:\n",
" content = post.selftext\n",
" content = content.replace('\\n#', '\\n####') # md headings\n",
" post_content.append(f\"{content}\\n\\n\")\n",
" else:\n",
" post_content.append(f\"[Link to content]({post.url})\\n\\n\")\n",
" post_content.append(\"### Comments:\\n\\n\")\n",
" post.comments.replace_more(limit=None)\n",
" for comment in post.comments:\n",
" post_content.append(format_comment(comment, upvote_threshold=comment_score_threshold, t0=ts))\n",
" post_content.append(\"---\\n\\n\")\n",
" return ''.join(post_content)\n",
" \n",
"\n",
"# m = submission_to_markdown(submission, -100, verbose=True)\n",
"# print(m)"
"from rrational.reddit2md import submission_to_markdown\n",
"from rrational.reddit2json import comment_to_json, submission_to_json"
]
},
{
"cell_type": "code",
"execution_count": 14,
"id": "af0eb6bb",
"metadata": {},
"outputs": [],
"source": [
"\n",
"post_fields = ('id', 'title', 'permalink', 'score', 'created_utc', 'num_comments', 'permalink', 'selftext', 'is_self', 'author_flair_text', 'upvote_ratio')\n",
"comment_fields = ('id', 'score', 'created_utc', 'body', 'permalink', 'author_flair_text')\n",
"\n",
"def comment_to_json(comment):\n",
" d = {k: v for k, v in comment.__dict__.items() if k in comment_fields}\n",
" d['author'] = comment.author.name if comment.author is not None else None\n",
" return d\n",
"\n",
"def submission_to_json(submission):\n",
" post_json = {k: v for k, v in submission.__dict__.items() if k in post_fields}\n",
" post_json['comments'] = [comment_to_json(x) for x in submission.comments.list()]\n",
" post_json['author'] = submission.author.name if submission.author is not None else None\n",
" return post_json\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 88,
"id": "7b210872",
"metadata": {},
"outputs": [],
"outputs": [
{
"data": {
"text/plain": [
"PosixPath('../data/cache2')"
]
},
"execution_count": 88,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"outdir = Path(\"../data/cache2\")\n",
"outdir.mkdir(exist_ok=True, parents=True)\n",
@@ -393,10 +409,21 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 89,
"id": "e13fca40",
"metadata": {},
"outputs": [],
"outputs": [
{
"data": {
"text/plain": [
"PosixPath('../data/json3')"
]
},
"execution_count": 89,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"jsondir = Path(\"../data/json3\")\n",
"jsondir.mkdir(exist_ok=True, parents=True)\n",
@@ -405,7 +432,7 @@
},
{
"cell_type": "code",
"execution_count": 17,
"execution_count": 90,
"id": "d0c1a164",
"metadata": {},
"outputs": [],
@@ -447,7 +474,7 @@
},
{
"cell_type": "code",
"execution_count": 24,
"execution_count": 91,
"id": "e87622c9",
"metadata": {},
"outputs": [],
@@ -457,138 +484,23 @@
},
{
"cell_type": "code",
"execution_count": 27,
"execution_count": 92,
"id": "ea499486",
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "daa0d97b133f4953900d466fe1f85d4a",
"model_id": "45021d22fc284ad384f09b08b3c08536",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
" 0%| | 0/10460 [00:00<?, ?it/s]"
" 0%| | 0/10466 [00:00<?, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"## [RT] [FF] [WIP] The Mind-Tailed Fox\n",
"\n",
"* Author: u/arenavanera *\n",
"* URL: https://www.fanfiction.net/s/12149957/1/The-Mind-Tailed-Fox\n",
"* Score: 23\n",
"\n",
"* Created: 2017-05-06T19:59:45\n",
"\n",
"### Post:\n",
"\n",
"[Link to content](https://www.fanfiction.net/s/12149957/1/The-Mind-Tailed-Fox)\n",
"\n",
"### Comments:\n",
"\n",
"> **u/Charlie___** [+6] (20 hours later)\n",
"> \n",
"> Have only read the first chapter so far, but I quite like the writing. I think this is a really elegant way of doing the brains-for-chakra trade-off.\n",
"> \n",
"> One plot problem I'm worried about is that if the kyuubi really is released if Naruto is assassinated, it will try to make that happen. And, being smart, it will either succeed or I'm worried there's a plot hole.\n",
"> \n",
"> Instead, maybe there's some better conditions of the binding - like maybe the kyuubi is only released a few percent of the time if Naruto is assassinated, but is always released if Naruto dies of old age (not that it would tell anyone else that fact).\n",
"\n",
">> **u/arenavanera** [+3] (23 hours later)\n",
">> \n",
">> Thanks!\n",
">> \n",
">> I'm also worried about plot problems caused by the Kyuubi being too smart. You'd expect a superintelligence that's trying to trick a small child into getting himself killed by people he knows to succeed almost immediately. There are some complications -- Naruto is guarded, anyone powerful enough to defeat his guards is more likely to be smart and also to know about the Kyuubi, if the Kyuubi tips his hand too early and the Hokage knows he's actively trying to escape it would get much harder, etc. etc. -- but I eventually decided that I did need to tone the Kyuubi's power level down a bit to have the conflict be plausible.\n",
">> \n",
">> A structural problem related to that is that it's hard to write a superintelligence that's weakened in several crucial ways but doesn't want to share what they are, because if you just write that directly it comes across as a superintelligence that's acting irrationally or inconsistently. I'm planning for Naruto to discover more about the details once he understands how seals work, but if I were to go back and write it again I would put the excuse for him learning the actual rules the Kyuubi is operating under much much earlier in the story so that it's clearer what's going on.\n",
">> \n",
">> (Another related problem I've had trouble with is that it's hard to write good dialogue between Naruto and the Kyuubi, because good dialogue usually involves conflicting personalities, and it's hard to see why the Kyuubi wouldn't just interact with Naruto in whatever way made Naruto happiest and most pliable. You can handwave some of this away because human psychology is bonkers and it might actually be correct for the Kyuubi to occasionally create conflict with Naruto for manipulative reasons, but I'm still struggling with it.)\n",
"\n",
">>> **u/Sailor_Vulcan** [+6] *Champion of Justice and Reason* (a day later)\n",
">>> \n",
">>> Why not just go with that? Kyubi wants Naruto's trust for some reason, and is trying to manipulate him. As for why Kyubi doesn't just try to get naruto to kill himself to release him, introduce costs to the kyubi for release. Like, he gains some of the knowledge and skills of his hosts while he's inhabiting them so he doesn't necessarily want to be released right away. And if they die too suddenly or violently he'll go into an irrational frenzied state and start mindlessly rampaging after being forcibly ripped from the mind of someone who was dying horribly. Instead of making him a superintelligence just give him the power to steal the knowledge and skills of his hosts.\n",
"\n",
">>>> **u/arenavanera** [+6] (a day later)\n",
">>>> \n",
">>>> That would be a fun reveal! The Kyuubi only gets smarter by being sealed in smart ninja, so it tricks the Shinobi into trapping it over and over again, and it's only *pretending* to try and escape so that nobody realizes it actually wants to be sealed. I might steal that.\n",
"\n",
"> **u/None** [+5] (a day later)\n",
"> \n",
"> [deleted]\n",
"\n",
">> **u/arenavanera** [+3] (a day later)\n",
">> \n",
">> Thanks!\n",
">> \n",
">> It's interesting that both of you brought up the slow pace. It didn't really feel that slow while writing, but now that I'm thinking about it, it's about half a novel by word count and not all that much has happened. I should probably stop introducing more long-running subplots until I've made some progress on the existing ones.\n",
">> \n",
">> A related problem is that for every paragraph of action I end up including 1-3 paragraphs of analysis, often from multiple viewpoints. I find that kind of thing fun, and it's arguably the point of the fic, but I think that does put the onus on me to keep the plot moving forward in the smaller number of action scenes I have to work with.\n",
">> \n",
">> (Also, I'm glad you like the Kyuubi; he's my favorite voice to write!)\n",
">> \n",
">> Fic recs: I mostly read rational or rational-adjacent stuff you're probably familiar with if you're on this sub (Dungeon Keeper Ami, The Two Year Emperor, Chuunin Exam day, etc.). One fic I like a lot which isn't very famous is https://www.fanfiction.net/s/79107/1/The-Story-of-Magus . I read it when I was younger, and it was one of my first introductions to the \"what if all the stereotypical NPCs were real people with real motivations\" genre.\n",
"\n",
">>> **u/Flashbunny** [+2] (a day later)\n",
">>> \n",
">>> Whilst the slow pace would be pretty much untenable in an actual paper novel, I've always felt that one of the strengths of online writing is that you can introduce things slowly and take your time in a way that a paper novel would require multiple books to manage - and of course, you'd have to tie the story down into \"arcs\" to fit those books. The writing's enjoyable and those points will presumably pay off eventually, so what's the hurry?\n",
"\n",
">>>> **u/arenavanera** [+2] (2 days later)\n",
">>>> \n",
">>>> My main worries are:\n",
">>>> \n",
">>>> * People get bored when it feels like nothing is changing.\n",
">>>> * I probably am not going to actually have the wherewithal to complete a multimillion word epic, and I'd rather get to a stopping point rather than just abandon the fic in the middle.\n",
">>>> * I think it's hard for people who are reading along as chapters are released to keep 10 different long-running subplots in their heads for months and months, especially since some will go multiple chapters without any updates.\n",
"\n",
"> **u/themousehunter** [+2] *Sunshine Regiment* (a day later)\n",
"> \n",
"> Thanks for posting it! Very fun read and I look forward to reading more!\n",
"\n",
"> **u/Charlie___** [+2] (a day later)\n",
"> \n",
"> Okay, all caught up. Seems great, I love it. The characters are all great, the pacing is a little on the slow side (unless you plan to have things go sideways fast), but this is fine with me unless it causes the story to get discontinued before a good stopping point.\n",
"> \n",
"> My uncertainty about what the heck is going on with the seal is still a problem for me. Come to think of it, wouldn't dying to Mizuki have been sufficient? The recent talk about demons provides a class of alternate explanations, but we have little information and there's lots of possibilities. Because I don't know what's really going on, there's tension, but the tension feels a bit pointless. If we're going to go too much longer without a reveal, I think I'd have preferred a partial reveal (e.g. fox POV, Minato POV) at the beginning to at least cut down on the possibilities.\n",
"\n",
">> **u/arenavanera** [+1] (a day later)\n",
">> \n",
">> Yeah, I think you're right about me mishandling the Kyuubi situation. I think there are at least three properties that a \"good\" mystery subplot should have:\n",
">> \n",
">> * There are a few obvious potential answers that people can mull over and compare. (There can also be non-obvious answers, but you want readers at every level to be able to have fun thinking about it, and you don't want there to be like fifty possible answers.)\n",
">> * The search space narrows over time, and every time it narrows there are important implications for things the reader cares about.\n",
">> * A clever reader can actually solve the damn thing before they're just straight-up told the answer.\n",
">> \n",
">> So, yeah, 0/3 at the moment. I'll do some thinking on how best to patch it.\n",
"\n",
"> **u/None** [+1] (4 days later)\n",
"> \n",
"> [deleted]\n",
"\n",
">> **u/arenavanera** [+1] (4 days later)\n",
">> \n",
">> Thanks for the typos / words that probably don't belong in this universe -- I'm pretty bad about skimming over canned phrases that don't make sense while proofreading.\n",
">> \n",
">> You're right about a lot of the early inconsistencies. When I started writing I was trying to follow the plot of the manga pretty closely, and a lot of the strangest things (the ridiculously low number of new Genin, graduates not being told about the test, Sasuke being sure Kakashi isn't around, etc.) are there because that's what happened in the manga. Some of the lines of dialogue are even lifted straight from the translation I was reading. I think in retrospect this was a mistake; there are just too many things in the manga that are impossible to explain away if you think about them carefully.\n",
">> \n",
">> When I'm done (or earlier if enough people complain) I'll probably go back and fix up the the most glaring inconsistencies introduced by slavishly following the manga.\n",
"\n",
"> **u/None** [+1] (5 days later)\n",
"> \n",
"> Probably a bit late to the party here, but my personal head canon is tht the reason the Kyuubi is helping Naruto get smarter because there is a way for Kyuubi to add Naruto's mind to it's collective when he dies. It would add an interesting twist to the tale; it would add to the whole 'naturally occuring AI' theme Kyuubi and other demons bring.\n",
"> Perhaps the Nine-Tailed Fox didn't always have nine tails? \n",
"> (And would also add an interesting, never before seen reason for Kushina to be in the seal with Kyuubi)\n",
"\n",
"---\n",
"\n",
"\n"
]
}
],
"source": [
rrational/reddit2json.py
+16
View File
@@ -0,0 +1,16 @@
from praw.models import Comment, Submission
post_fields = ('id', 'title', 'permalink', 'score', 'created_utc', 'num_comments', 'permalink', 'selftext', 'is_self', 'author_flair_text', 'upvote_ratio')
comment_fields = ('id', 'score', 'created_utc', 'body', 'permalink', 'author_flair_text')
def comment_to_json(comment):
d = {k: v for k, v in comment.__dict__.items() if k in comment_fields}
d['author'] = comment.author.name if comment.author is not None else None
return d
def submission_to_json(submission: Submission) -> dict:
post_json = {k: v for k, v in submission.__dict__.items() if k in post_fields}
post_json['comments'] = [comment_to_json(x) for x in submission.comments.list()]
post_json['author'] = submission.author.name if submission.author is not None else None
return post_json
rrational/reddit2md.py
+67
View File
@@ -0,0 +1,67 @@
# from https://github.dev/JosefAlbers/rd2md
from praw.models import Comment, Submission
import textwrap
import re
import pandas as pd
from datetime import datetime
def format_flair(obj):
if obj.author_flair_text is not None:
return f" *{obj.author_flair_text}*"
return ""
import humanize
def format_comment(comment, depth=0, upvote_threshold=2, t0=None):
if comment.score < upvote_threshold:
return ""
ts = pd.to_datetime(comment.created, unit='s')
if t0 is not None:
delta = ts - t0
delta = humanize.naturaldelta(delta.total_seconds() )
else:
delta = ""
indent = ">" * (depth+1) + " "
author_line = f"{indent}**u/{comment.author}** [{comment.score:+}] {format_flair(comment)} ({delta} later)\n"
dedented_body = textwrap.dedent(comment.body)
# use re to replace multiple newlines with a single newline
indented_body = textwrap.indent(dedented_body, indent)
indented_body = re.sub(r'\n\n+', f'\n{indent}\n', indented_body.strip())
comment_block = f"{indent}\n{indented_body}\n\n"
formatted = author_line + comment_block
for reply in comment.replies:
formatted += format_comment(reply, depth + 1, upvote_threshold, t0)
return formatted
def submission_to_markdown(post: Submission, comment_score_threshold=0, verbose=False) -> str:
post_content = []
post_content.append(f"## {post.title}\n\n")
if verbose:
ts = pd.to_datetime(post.created, unit='s')
post_content.append(f"* Author: u/{post.author} {format_flair(post)}*\n")
post_content.append(f"* URL: {post.url}\n")
post_content.append(f"* Score: {post.score}\n\n")
post_content.append(f"* Created: {ts.isoformat()}\n\n")
post_content.append("### Post:\n\n")
if post.is_self:
content = post.selftext
content = content.replace('\n#', '\n####') # md headings
post_content.append(f"{content}\n\n")
else:
post_content.append(f"[Link to content]({post.url})\n\n")
post_content.append("### Comments:\n\n")
post.comments.replace_more(limit=None)
for comment in post.comments:
post_content.append(format_comment(comment, upvote_threshold=comment_score_threshold, t0=ts))
post_content.append("---\n\n")
return ''.join(post_content)
# m = submission_to_markdown(submission, -100, verbose=True)
# print(m)