wassname/scrape_r_rational
1
0
Fork 0
mirror of https://github.com/wassname/scrape_r_rational.git synced 2026-08-20 12:50:33 +08:00
Code Issues Packages Projects Releases Wiki Activity

init

Browse Source
This commit is contained in:
wassname
2024-09-02 15:11:08 +08:00
commit fcdfc19b1e
20 changed files with 2731 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+91
View File
@@ -0,0 +1,91 @@
# exclude data from source control by default
/data/
/outputs/
# DotEnv configuration
.env
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
# C extensions
*.so
# Distribution / packaging
.Python
env/
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
*.egg-info/
.installed.cfg
*.egg
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
# Translations
*.mo
*.pot
# Django stuff:
*.log
# Sphinx documentation
docs/_build/
# PyBuilder
target/
# Database
*.db
*.rdb
# Pycharm
.idea
# VS Code
.vscode/
# Spyder
.spyproject/
# Jupyter NB Checkpoints
.ipynb_checkpoints/
# Mac OS-specific storage files
.DS_Store
# vim
*.swp
*.swo
# Mypy cache
.mypy_cache/
README.md
+51
View File
@@ -0,0 +1,51 @@
# rrational
scrapping reddit.com/r/rational and analytics
see https://raw.githubusercontent.com/NightMachinery/.shells/master/scripts/python/reddit/subreddit2org.py
Project status: WIP
Project plan:
- [x] Init
- [ ] Fill out README
- [ ] Scrape r/rational
- [ ] Use llm to get reccomendations, sentiment, karma etc
- [ ] share
## Install requirements
This project uses [poetry](https://python-poetry.org/) for requirement and is set up for torch using cuda.
~~~
poetry install
~~~
## How to get data
TODO document how to get the data
## How to run
This project uses [just](https://github.com/casey/just)
~~~
just --list
~~~
## Project Organization
Note this project uses
- [Justfile](https://github.com/casey/just): Command runner with commands like `just data` or `just train`
- data: [data directory ](https://cookiecutter-data-science.drivendata.org/#directory-structure)
- ./10_raw <- The original, immutable data dump.
- ./20_interim <- Intermediate data that has been transformed.
- ./30_processed <- The final, canonical data sets for modeling.
- nbs: jupyter notebooks. Name with creator's initials, a number (for ordering), and short `-` delimited description, e.g. `jqp-1.0-initial-data-exploration`.
- pyproject.toml: defines poetry project dependencies and build configuration
- rrational: Source code for use in this project.
justfile
+20
View File
@@ -0,0 +1,20 @@
# see https://cheatography.com/linux-china/cheat-sheets/justfile/
set dotenv-load
# Export all just variables as environment variables.
set export
package := "rrational"
[private]
default: @just --list
# put your run commands here
app:
echo "hello world"
# black and isort
lint:
ruff .
nbs/.gitkeep
View File
nbs/mjc_001_EDA.ipynb
+447
View File
@@ -0,0 +1,447 @@
{
"cells": [
{
"cell_type": "markdown",
"id": "1b44551e",
"metadata": {},
"source": [
"# Exploratory Data Analysis\n",
"\n",
"Hypothesis: What is this notebook about?"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "198de680",
"metadata": {
"ExecuteTime": {
"end_time": "2022-06-28T02:34:01.879987Z",
"start_time": "2022-06-28T02:34:01.864103Z"
}
},
"outputs": [],
"source": [
"# autoreload your package\n",
"%load_ext autoreload\n",
"%autoreload 2\n",
"import rrational\n"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "a372ed7c",
"metadata": {
"ExecuteTime": {
"end_time": "2022-06-28T02:34:02.470436Z",
"start_time": "2022-06-28T02:34:02.424826Z"
}
},
"outputs": [
{
"data": {
"text/plain": [
"1"
]
},
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"## secrets\n",
"from dotenv import load_dotenv\n",
"load_dotenv() # take environment variables from .env.\n",
"\n",
"import warnings\n",
"# warnings.simplefilter(\"ignore\")\n",
"warnings.filterwarnings(\"ignore\", \".*does not have many workers.*\")\n",
"warnings.filterwarnings(\"ignore\", \".*divide by zero.*\")\n",
"\n",
"## numeric, plotting\n",
"import numpy as np\n",
"import pandas as pd\n",
"import matplotlib.pyplot as plt\n",
"%matplotlib inline\n",
"plt.style.use('ggplot')\n",
"plt.rcParams['figure.figsize'] = (7.0, 4)\n",
"\n",
"## utils\n",
"from pathlib import Path\n",
"from tqdm.auto import tqdm\n",
"import logging, os, re\n",
"import collections, functools, itertools\n",
"\n",
"# logging\n",
"from loguru import logger\n",
"logger.remove()\n",
"logger.add(os.sys.stdout, level=\"ERROR\", colorize=True, format=\"<level>{time} | {message}</level>\")\n"
]
},
{
"cell_type": "code",
"execution_count": 1,
"id": "a91f5c82",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"True"
]
},
"execution_count": 1,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# dotenv\n",
"import os\n",
"from dotenv import load_dotenv\n",
"load_dotenv() # take environment variables from .env."
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "54a03c3a",
"metadata": {},
"outputs": [],
"source": [
"\n",
"import praw\n",
"\n",
"reddit = praw.Reddit(\n",
" client_id=os.environ[\"CLIENT_ID\"],\n",
" client_secret=os.environ[\"CLIENT_SECRET\"],\n",
" password=os.environ[\"PASSWORD\"],\n",
" user_agent=\"testscript by u/fakebot3\",\n",
" username=os.environ[\"USERNAME\"],\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "fc3cb5ba",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"Subreddit(display_name='rational')"
]
},
"execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"s = reddit.subreddit(\"rational\")\n",
"s"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "1d1601bb",
"metadata": {},
"outputs": [],
"source": [
"# TODO also add\n",
"# Comic Recommendation thread : bit.ly/1SPwDfz\n",
"# 'Time Travel' Recommendation thread : bit.ly/1VVcNyH\n",
"# Worm Fanfiction Recommendation thread : bit.ly/1VVd0lB\n",
"# Obscure(sic) Recommendation thread : bit.ly/1PXBvtR\n",
"# General Fanfiction Recommendation thread: bit.ly/1SxDNXq \n",
"# Miscellaneous Recommendation thread : bit.ly/1PXB9n6"
]
},
{
"cell_type": "code",
"execution_count": 12,
"id": "90f5851d",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"[Submission(id='1ef093v'),\n",
" Submission(id='1e3vhth'),\n",
" Submission(id='1ekoil8'),\n",
" Submission(id='1ew2jku'),\n",
" Submission(id='1f1ownr'),\n",
" Submission(id='1b6cz68'),\n",
" Submission(id='1byylp5'),\n",
" Submission(id='1bt4gkn'),\n",
" Submission(id='1cabj2s'),\n",
" Submission(id='1d1s69f'),\n",
" Submission(id='1dcm7hq'),\n",
" Submission(id='1eqehs7'),\n",
" Submission(id='1ap1xea'),\n",
" Submission(id='1dne6df'),\n",
" Submission(id='1dhys6k'),\n",
" Submission(id='1bc3v36'),\n",
" Submission(id='1d754n0'),\n",
" Submission(id='1cwgahd'),\n",
" Submission(id='197apaz'),\n",
" Submission(id='1cr00xb'),\n",
" Submission(id='1e9f0la'),\n",
" Submission(id='1adxacl'),\n",
" Submission(id='1c4mxk4'),\n",
" Submission(id='1dstocq'),\n",
" Submission(id='10v8os8'),\n",
" Submission(id='1dy9n53'),\n",
" Submission(id='1ajihyf'),\n",
" Submission(id='1cljrk3'),\n",
" Submission(id='191n3o1'),\n",
" Submission(id='1aupa48'),\n",
" Submission(id='1854wdx'),\n",
" Submission(id='14pjuec'),\n",
" Submission(id='147ouxh'),\n",
" Submission(id='1bhrf53'),\n",
" Submission(id='11qai9r'),\n",
" Submission(id='19cxitp'),\n",
" Submission(id='12xgrdi'),\n",
" Submission(id='17el34d'),\n",
" Submission(id='kcz9wy'),\n",
" Submission(id='17ucgu9'),\n",
" Submission(id='17zqeg8'),\n",
" Submission(id='169srue'),\n",
" Submission(id='11k144r'),\n",
" Submission(id='14ji4g8'),\n",
" Submission(id='17p4n2k'),\n",
" Submission(id='jqz6xm'),\n",
" Submission(id='18fwga7'),\n",
" Submission(id='1b0jj3k'),\n",
" Submission(id='oirtis'),\n",
" Submission(id='14dfgxx'),\n",
" Submission(id='13bqbgs'),\n",
" Submission(id='16rsx5p'),\n",
" Submission(id='1cfztdl'),\n",
" Submission(id='1bnedpd'),\n",
" Submission(id='13i96zf'),\n",
" Submission(id='zpttzb'),\n",
" Submission(id='173sj6l'),\n",
" Submission(id='w202ju'),\n",
" Submission(id='13uvrd1'),\n",
" Submission(id='xok3va'),\n",
" Submission(id='158bic9'),\n",
" Submission(id='11ddvgy'),\n",
" Submission(id='vwj4bu'),\n",
" Submission(id='w7ppkg'),\n",
" Submission(id='12hjo5b'),\n",
" Submission(id='15x8ioi'),\n",
" Submission(id='mao94s'),\n",
" Submission(id='kqac2y'),\n",
" Submission(id='15eh0gh'),\n",
" Submission(id='mpeg4l'),\n",
" Submission(id='11wjn0q'),\n",
" Submission(id='p120hl'),\n",
" Submission(id='18lapfv'),\n",
" Submission(id='xcdl9d'),\n",
" Submission(id='13orqkc'),\n",
" Submission(id='ugqabu'),\n",
" Submission(id='ssckiz'),\n",
" Submission(id='lpq207'),\n",
" Submission(id='v10u6w'),\n",
" Submission(id='vbcvey'),\n",
" Submission(id='12pgzgv'),\n",
" Submission(id='15klk2q'),\n",
" Submission(id='107ggwy'),\n",
" Submission(id='yv1vx4'),\n",
" Submission(id='lfdjxw'),\n",
" Submission(id='myyble'),\n",
" Submission(id='18alzg5'),\n",
" Submission(id='gdd7kj'),\n",
" Submission(id='1178p4c'),\n",
" Submission(id='ghp8uh'),\n",
" Submission(id='khj3a6'),\n",
" Submission(id='orzaqc'),\n",
" Submission(id='tdy5bp'),\n",
" Submission(id='sxvim4'),\n",
" Submission(id='12ald68'),\n",
" Submission(id='k8i7jg'),\n",
" Submission(id='141fhyr'),\n",
" Submission(id='15qvj9z'),\n",
" Submission(id='wp0evf'),\n",
" Submission(id='qfhdft')]"
]
},
"execution_count": 12,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"submissions = list(s.search(\"Monday Request and Recommendation Thread\"))\n",
"submissions"
]
},
{
"cell_type": "code",
"execution_count": 17,
"id": "f17c2c47",
"metadata": {},
"outputs": [],
"source": [
"submission = submissions[0]\n",
"comments = submission.comments.list()"
]
},
{
"cell_type": "code",
"execution_count": 21,
"id": "980b2ae5",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'_replies': <praw.models.comment_forest.CommentForest at 0x7dad2cd71240>,\n",
" '_submission': Submission(id='1ef093v'),\n",
" '_reddit': <praw.reddit.Reddit at 0x7dad816e31f0>,\n",
" 'subreddit_id': 't5_2rdak',\n",
" 'approved_at_utc': None,\n",
" 'author_is_blocked': False,\n",
" 'comment_type': None,\n",
" 'awarders': [],\n",
" 'mod_reason_by': None,\n",
" 'banned_by': None,\n",
" 'author_flair_type': 'text',\n",
" 'total_awards_received': 0,\n",
" 'subreddit': Subreddit(display_name='rational'),\n",
" 'author_flair_template_id': 'e1be530a-8cae-11e3-a72c-12313d27e9a3',\n",
" 'likes': None,\n",
" 'user_reports': [],\n",
" 'saved': False,\n",
" 'id': 'lfhs456',\n",
" 'banned_at_utc': None,\n",
" 'mod_reason_title': None,\n",
" 'gilded': 0,\n",
" 'archived': False,\n",
" 'collapsed_reason_code': None,\n",
" 'no_follow': False,\n",
" 'author': Redditor(name='Dragongeek'),\n",
" 'can_mod_post': False,\n",
" 'created_utc': 1722263937.0,\n",
" 'send_replies': True,\n",
" 'parent_id': 't3_1ef093v',\n",
" 'score': 17,\n",
" 'author_fullname': 't2_9m5z9',\n",
" 'approved_by': None,\n",
" 'mod_note': None,\n",
" 'all_awardings': [],\n",
" 'collapsed': False,\n",
" 'body': 'I\\'ve got a casual rec this week:\\n\\n## [I\\'m on TV! \\\\(Showbiz SI\\\\)](https://archiveofourown.org/works/43498798/chapters/109357135)\\n*~110k words, weekly updates, CW: Explicit content*\\n\\n----------------------\\n\\nBasically a \"modern world\" insert or reincarnation fic, where the protagonist\\'s mind is yeeted into a a random 1998 8 y/o orphan with a 2024 internet snapshot and he decides to leverage his future knowledge along with the whole \"adult mind\"-shtick of the genre into becoming a wildly successful child actor, taking over the role of Radcliffe in the Harry Potter franchise and taking it from there (Tokyo Drift, Psych, Tropic Thunder, etc). \\n\\nThe author is clearly into movies and TV because I feel they manage to capture the acting scene quite well (at least from my ignorant outsider perspective) and there\\'s a lot of \"how the sausage is made\" in terms of film industry. They also really nail the vibe of the early 2000\\'s. \\n\\nThe biggest strength here is the comedy aspect, a lot of it is just downright hilarious, but there\\'s also the back-in-time chess moves stuff like investing big in the right companies and playing the Big Short IRL with his HP franchise earnings to become fantaboulously wealthy at a young age.\\n\\nIn terms of explicit content, there are like a handful of lewd scenes but I\\'d probably rate it more \"R\" rather than \"X\" since it\\'s not really the focus. Also, of note, is that besides the protagonist, this is fanfiction where the genre is \"real life\" and all the people he interacts with in the story are real life people. I feel that so far the author has managed to do this with reasonable respect to the actual people depicted, but something to be aware of. \\n\\nAgain, not particularly deep, but a lot of fun. \\n\\n--------------------------\\n\\nAnyone have other recommendations for real-life fanfiction with people going back in time to near or more distant human history?',\n",
" 'edited': False,\n",
" 'top_awarded_type': None,\n",
" 'author_flair_css_class': None,\n",
" 'name': 't1_lfhs456',\n",
" 'is_submitter': False,\n",
" 'downs': 0,\n",
" 'author_flair_richtext': [],\n",
" 'author_patreon_flair': False,\n",
" 'body_html': '<div class=\"md\"><p>I&#39;ve got a casual rec this week:</p>\\n\\n<h2><a href=\"https://archiveofourown.org/works/43498798/chapters/109357135\">I&#39;m on TV! (Showbiz SI)</a></h2>\\n\\n<p><em>~110k words, weekly updates, CW: Explicit content</em></p>\\n\\n<hr/>\\n\\n<p>Basically a &quot;modern world&quot; insert or reincarnation fic, where the protagonist&#39;s mind is yeeted into a a random 1998 8 y/o orphan with a 2024 internet snapshot and he decides to leverage his future knowledge along with the whole &quot;adult mind&quot;-shtick of the genre into becoming a wildly successful child actor, taking over the role of Radcliffe in the Harry Potter franchise and taking it from there (Tokyo Drift, Psych, Tropic Thunder, etc). </p>\\n\\n<p>The author is clearly into movies and TV because I feel they manage to capture the acting scene quite well (at least from my ignorant outsider perspective) and there&#39;s a lot of &quot;how the sausage is made&quot; in terms of film industry. They also really nail the vibe of the early 2000&#39;s. </p>\\n\\n<p>The biggest strength here is the comedy aspect, a lot of it is just downright hilarious, but there&#39;s also the back-in-time chess moves stuff like investing big in the right companies and playing the Big Short IRL with his HP franchise earnings to become fantaboulously wealthy at a young age.</p>\\n\\n<p>In terms of explicit content, there are like a handful of lewd scenes but I&#39;d probably rate it more &quot;R&quot; rather than &quot;X&quot; since it&#39;s not really the focus. Also, of note, is that besides the protagonist, this is fanfiction where the genre is &quot;real life&quot; and all the people he interacts with in the story are real life people. I feel that so far the author has managed to do this with reasonable respect to the actual people depicted, but something to be aware of. </p>\\n\\n<p>Again, not particularly deep, but a lot of fun. </p>\\n\\n<hr/>\\n\\n<p>Anyone have other recommendations for real-life fanfiction with people going back in time to near or more distant human history?</p>\\n</div>',\n",
" 'removal_reason': None,\n",
" 'collapsed_reason': None,\n",
" 'distinguished': None,\n",
" 'associated_award': None,\n",
" 'stickied': False,\n",
" 'author_premium': False,\n",
" 'can_gild': False,\n",
" 'gildings': {},\n",
" 'unrepliable_reason': None,\n",
" 'author_flair_text_color': 'dark',\n",
" 'score_hidden': False,\n",
" 'permalink': '/r/rational/comments/1ef093v/d_monday_request_and_recommendation_thread/lfhs456/',\n",
" 'subreddit_type': 'public',\n",
" 'locked': False,\n",
" 'report_reasons': None,\n",
" 'created': 1722263937.0,\n",
" 'author_flair_text': 'Path to Victory',\n",
" 'treatment_tags': [],\n",
" 'link_id': 't3_1ef093v',\n",
" 'subreddit_name_prefixed': 'r/rational',\n",
" 'controversiality': 0,\n",
" 'depth': 0,\n",
" 'author_flair_background_color': None,\n",
" 'collapsed_because_crowd_control': None,\n",
" 'mod_reports': [],\n",
" 'num_reports': None,\n",
" 'ups': 17,\n",
" '_fetched': True}"
]
},
"execution_count": 21,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# TODO store whole threads in markdown for an llm to view\n",
"comment = comments[0]\n",
"comment.__dict__"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "5cebfad8",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "6d102e3d",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3.10.4 64-bit",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.12"
},
"toc": {
"base_numbering": 1,
"nav_menu": {},
"number_sections": true,
"sideBar": true,
"skip_h1_title": false,
"title_cell": "Table of Contents",
"title_sidebar": "Contents",
"toc_cell": false,
"toc_position": {},
"toc_section_display": true,
"toc_window_display": false
},
"vscode": {
"interpreter": {
"hash": "916dbcbb3f70747c44a77c7bcd40155683ae19c65e1c03b4aa3499c5328201f1"
}
}
},
"nbformat": 4,
"nbformat_minor": 5
}
nbs/scrape.py
+234
View File
@@ -0,0 +1,234 @@
#!/usr/bin/env python3
# Usage:
# `tmuxnewsh2 r_rational indirm ~dl/subreddits/rational subreddit2org.py rational 1000000000`
# `tmuxnewsh2 r_HPfanfiction indirm ~dl/subreddits/HPfanfiction subreddit2org.py HPfanfiction 1000000000`
#
# Old usage:
# `reddit_sub_posts_urls_pushshift.py rational 100000000 > rational.txt`
#
# Docs:
# - https://github.com/praw-dev/praw
# - https://github.com/dmarx/psaw
##
from IPython import embed
import sys
import os
import traceback
import time
link_output = False
limit = 10
subreddit = "rational"
al = len(sys.argv)
if al >= 2:
subreddit = sys.argv[1]
if al >= 3:
limit = int(sys.argv[2])
from psaw import PushshiftAPI
import json
if link_output:
api = PushshiftAPI()
else:
import praw
from praw.exceptions import DuplicateReplaceException
from praw.models.reddit.more import MoreComments
r = praw.Reddit(
client_id=os.environ["REDDIT_CLIENT_ID"],
client_secret=os.environ["REDDIT_CLIENT_SECRET"],
user_agent='User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36',
# password=os.environ["REDDIT_PASSWORD"],
# username=os.environ["REDDIT_USERNAME"],
)
api = PushshiftAPI(r)
print(f"Getting the submissions from the subreddit {subreddit} (limit={limit}) ...\n", file=sys.stderr, flush=True)
gen = api.search_submissions(subreddit=subreddit, limit=limit)
results = list(gen)
##
# embed()
# print(json.dumps(results))
##
if link_output:
for result in results:
print(result.permalink)
else:
# @todo3 refactor this into a standalone reddit2org (we need a way to get the submission object from the URL)
##
from brish import z, zp, bsh
def stars(lv):
return "*" * lv + " "
def html2org(html):
# tmp = "tmp.html"
tmp = z("mktemp").outrs
res = z("cat > {tmp}", cmd_stdin=html)
assert res
res = z("html2org {tmp}")
assert res
z("command rm {tmp}")
return res.outrs
# def meta_get(c):
# meta = ""
# meta+=f"{c.author.name}"
# if hasattr(c, "score"):
# meta+=f" ({c.score})"
# return meta
def meta_get_props(c):
meta = ":PROPERTIES:"
if c.author: # can be None when they are deleted
meta+=f"\n:Author: {c.author.name}"
if hasattr(c, "score"):
meta+=f"\n:Score: {c.score}"
if hasattr(c, "created_utc") and c.created_utc:
meta+=f"\n:DateUnix: {c.created_utc}"
res = z('gdate -d "@"{c.created_utc} +"%Y-%b-%d"')
if res:
meta+=f"\n:DateShort: {res.outrs}"
if getattr(c, "link_flair_text", None):
meta+=f"\n:FlairText: {c.link_flair_text}"
meta+="\n:END:\n"
return meta
def process_comment(f, comments, lv, shortname):
while True:
try:
comments.replace_more(limit=None)
break
except DuplicateReplaceException:
print(traceback.format_exc())
break
except:
print(traceback.format_exc())
time.sleep(1)
l = len(comments) - 1
shortname_orig = shortname
for i, c in enumerate(comments):
lv_c = lv
shortname = shortname_orig
# if isinstance(c, MoreComments):
# pass
##
# meta = meta_get(c)
# meta+=": "
##
meta = meta_get_props(c)
# Properties are key--value pairs. When they are associated with a single entry or with a tree they need to be inserted into a special drawer (see [[https://orgmode.org/manual/Drawers.html#Drawers][Drawers]]) with the name ‘=PROPERTIES=', which has to be located right below a headline, and its planning line (see [[https://orgmode.org/manual/Deadlines-and-Scheduling.html#Deadlines-and-Scheduling][Deadlines and Scheduling]]) when applicable.
#
# Still, putting the props after the heading is no fun; We can rename our drawer to :METADATA:, but why bother?
##
head = "EMPTY_COMMENT"
# @todo3 using IDs creates too long paths. It's better if just use a counter that goes from 0 to N.
c_id_old = c.id or z("uuidm").outrs[0:6]
##
if lv_c <= 4:
c_id = f"{i}_{c_id_old}"
else:
c_id = i
# using this in different runs is unreliable, as the comments ordering can change. But since we use the comments' ID as their filenames, this won't result in data loss, but it can cause data duplication and a flawed comment hierarchy.
# workarounds:
# - delete the indices directory and re-run the whole scraping from scratch on every update
# - @done use the first-n-level comments' ID as well
##
shortname += f"/{c_id}"
if c.body_html:
head = (html2org(c.body_html) or "EMPTY_COMMENT")
index_file = f'indices/{shortname}/{c_id_old}.org'
z("ensure-dir {index_file}")
with open(index_file, "w") as f2:
f2.write(f"{meta}\n{head}")
if head.startswith("#+"):
# do not put blocks in headings (e.g., #+begin_quote)
author = "deleted"
if c.author:
author = c.author.name or author
head = f"u/{author}:\n{head}"
head = head or "_" # empty headers are invalid org-mode
f.write("\n" + stars(lv_c) + head + "\n" + meta)
lv_c += 1
process_comment(f, c.replies, lv_c, shortname)
if l != i:
f.write("\n")
def utf8len(s):
return len(s.encode('utf-8'))
for result in results:
try:
# embed() ; exit()
# if not "looking at this sub" in result.title:
# continue
f_name = z("ecn {result.title} | str2filename").outrs
f_name = f_name[0:230]
# [[id:a36bb01f-9b9b-40c9-816a-c762281c43c3][filesystem/filenames.org:maximum allowed length for filenames and paths]]
if utf8len(f_name) > 240:
f_name = f_name[0:100]
if utf8len(f_name) > 240:
f_name = f_name[0:60]
shortname = f"{f_name}.{result.id}"
f_name = f"posts/{shortname}.org"
z("ensure-dir {f_name}")
with open(f_name, "w") as f:
lv = 1
f.write(f"#+TITLE: {result.title}\n\n")
if getattr(result, "url_overridden_by_dest", None):
f.write(f"{stars(lv)}[[{result.url_overridden_by_dest}][{result.title}]]\n")
lv += 1
else:
f.write(f"{stars(lv)}{result.title}\n")
lv += 1
meta = meta_get_props(result)
f.write(meta)
if result.selftext_html:
f.write(html2org(result.selftext_html) + "\n\n")
process_comment(f, result.comments, lv, shortname)
print(f"wrote {f_name}\n", file=sys.stderr, flush=True)
except:
print(traceback.format_exc())
embed()
poetry.lock
Generated
+1814
View File
File diff suppressed because it is too large Load Diff
pyproject.toml
+37
View File
@@ -0,0 +1,37 @@
[tool.poetry]
name = "rrational"
version = "0.1.0"
description = "scrape reddit.com/r/rational and analyze"
authors = ["wassname"]
license = "MIT"
readme = "README.md"
[tool.poetry.dependencies]
python = ">=3.10,<4.0"
numpy = "^1.26.1"
pandas = "^2.1.1"
matplotlib = "^3.8.0"
loguru = "^0.7.2"
tqdm = "^4.66.1"
python-dotenv = "^1.0.1"
praw = "^7.7.1"
[[tool.poetry.source]]
# pytorch cuda needs to compe from another source https://python-poetry.org/docs/dependency-specification/#source-dependencies
name = "pytorch"
url = "https://download.pytorch.org/whl/cu124"
priority = "explicit"
[tool.poetry.group.dev.dependencies]
ipykernel = "^6.25.2"
ipywidgets = "^8.1.3"
ruff = "^0.1.3"
pylama = "^8.4.1"
[build-system]
requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"
[virtualenvs]
create = true
in-project = true
research_journal.md
+3
View File
@@ -0,0 +1,3 @@
# 2024-06-09 16:05:45
Started project using cookiecutter data science project template.
rrational/__init__.py
View File
rrational/config.py
+4
View File
@@ -0,0 +1,4 @@
from pathlib import Path
# Project root directory
ROOT_DIR = Path(__file__).parent.parent
rrational/data/.gitkeep
View File
rrational/data/__init__.py
View File
rrational/data/make_dataset.py
+30
View File
@@ -0,0 +1,30 @@
# -*- coding: utf-8 -*-
import click
import logging
from pathlib import Path
from dotenv import find_dotenv, load_dotenv
@click.command()
@click.argument('input_filepath', type=click.Path(exists=True))
@click.argument('output_filepath', type=click.Path())
def main(input_filepath, output_filepath):
""" Runs data processing scripts to turn raw data from (../raw) into
cleaned data ready to be analyzed (saved in ../processed).
"""
logger = logging.getLogger(__name__)
logger.info('making final data set from raw data')
if __name__ == '__main__':
log_fmt = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
logging.basicConfig(level=logging.INFO, format=log_fmt)
# not used in this stub but often useful for finding various files
project_dir = Path(__file__).resolve().parents[2]
# find .env automagically by walking up directories until it's found, then
# load up the .env entries as environment variables
load_dotenv(find_dotenv())
main()
rrational/helpers/__init__.py
View File
rrational/models/.gitkeep
View File
rrational/models/__init__.py
View File
rrational/visualization/.gitkeep
View File
rrational/visualization/__init__.py
View File
rrational/visualization/visualize.py
View File