scrape_r_rational/nbs/scratch.ipynb

In [1]:

%reload_ext autoreload
%autoreload 2

In [2]:

import pandas as pd
df = pd.read_parquet('../outputs/df_links5.parquet')
df

Out [2]:

	title	score	n_links	n_comments	comments	thread_urls	first_link_utc	last_link_utc	url	description	...	reccomendations	disrecommendations	why	if_you_liked_x_you_will_like_this	rating_quality	rating_rationality	rating_writing	rating_plot	rating_character	rating_worldbuilding
0	Worth the Candle	5458	296	296	[{'author_flair_text': 'Self-Appointed Court S...	[https://reddit.com/r/rational/comments/7228po...	2015-09-26 06:59:23	2024-04-17 19:03:01	[https://archiveofourown.org/works/11478249/ch...	Worth the Candle is a long, original, complete...	...	Readers frequently praised the story's compell...	Some readers found the sheer length of the wor...	Readers of r/rational would likely enjoy Worth...	[Mother of Learning, HPMOR, Worm, Erfworld, Lu...	9.0	7.0	8.5	8.5	8.0	9.0
1	Mother of Learning	1834	174	174	[{'author_flair_text': 'Utopian Smut Peddler',...	[https://reddit.com/r/rational/comments/k8i7jg...	2014-07-26 08:21:12	2024-05-21 07:04:42	[https://www.fictionpress.com/s/2961893/1/Moth...	Mother of Learning is a web serial about Zoria...	...	Readers praise its consistent internal logic, ...	Some readers find the writing style to be roug...	Readers of r/rational will appreciate the stor...	[Time Braid, HPMOR, Worm]	9.0	7.5	7.0	9.0	8.5	9.0
2	Alexander Wales - The Metropolitan Man, Shadow...	1408	22	22	[{'author_flair_text': 'Time flies like an arr...	[https://reddit.com/r/rational/comments/galt0r...	2015-04-18 18:13:47	2021-04-29 20:00:04	[https://www.patreon.com/alexanderwales]	Alexander Wales's works, including *Shadows of...	...	Readers strongly recommend Alexander Wales's w...	Some readers express concerns about the infreq...	Readers of r/rational would likely enjoy Alexa...	[Brandon Sanderson, Harry Potter, Worm, Mother...	9.0	7.5	9.0	9.0	8.5	8.0
3	A Practical Guide to Evil	1321	143	143	[{'author_flair_text': 'Ankh-Morpork City Watc...	[https://reddit.com/r/rational/comments/ako900...	2016-04-05 18:53:21	2024-08-09 10:10:21	[https://practicalguidetoevil.wordpress.com/, ...	A Practical Guide to Evil is a long-running we...	...	Readers praise its well-developed characters, ...	Some readers find the frequent cliffhangers ir...	Readers of r/rational will likely enjoy this f...	[Worm, Shadows of the Limelight]	9.5	7.0	8.0	9.0	9.0	9.0
4	Worm	1019	96	96	[{'author_flair_text': 'Emergency Mustelid Hol...	[https://reddit.com/r/rational/comments/k8i7jg...	2014-01-27 02:09:42	2024-11-22 21:39:11	[https://parahumans.wordpress.com/, https://pa...	Worm is a long, complete web serial (1.7 milli...	...	Readers recommend Worm for its compelling char...	Some find the story too grim, dark, and depres...	Readers of r/rational might enjoy Worm for its...	[HPMOR, A Practical Guide to Evil, Other Wildb...	9.0	7.0	8.5	8.5	8.0	9.5
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
13218	Worth the Candle, Chapter 121-122	-6	2	2	[{'author_flair_text': None, 'body': 'It seems...	[https://reddit.com/r/rational/comments/9h1454...	2018-09-19 02:26:27	2018-09-19 04:51:09	[https://youtu.be/xq1tN9jZI80]	In this chapter of the web serial *Worth the C...	...	Readers praise the writing quality, emotional ...	Some readers find the emotional intensity and ...	Readers of r/rational might appreciate the foc...	[]	7.5	8.0	8.5	8.0	7.0	9.0
13219	Heartbreaker	-6	1	1	[{'author_flair_text': 'https://i.imgur.com/OQ...	[https://reddit.com/r/rational/comments/48akta...	2016-02-29 20:57:08	2016-02-29 20:57:08	[https://parahumans.wordpress.com/2012/01/17/b...	Heartbreaker is mentioned only once in the pro...	...	No positive recommendations are given for Hear...	The comment referencing Heartbreaker is widely...	Heartbreaker is only mentioned in passing as a...	[]	1.0	0.0	0.0	0.0	0.0	0.0
13220	Orthogonality Thesis Discussion Summary	-8	2	2	[{'author_flair_text': None, 'body': 'I don't ...	[https://reddit.com/r/rational/comments/3vc0si...	2015-12-04 18:17:07	2016-07-11 16:27:30	[https://wiki.lesswrong.com/wiki/Orthogonality...	The Orthogonality Thesis, a concept from LessW...	...	Users found the discussions engaging for their...	Some users found the discussions overly theore...	Readers interested in AI alignment, rationalit...	[Discussions on Friendly AI, Sequences by Elie...	6.0	9.0	0.0	0.0	7.0	0.0
13221	When Starting School, Younger Children Are Mor...	-9	1	1	[{'author_flair_text': 'Ankh-Morpork City Watc...	[https://reddit.com/r/rational/comments/a1t6um...	2018-11-30 17:17:49	2018-11-30 17:17:49	[https://news.harvard.edu/gazette/story/2018/1...	This Harvard Gazette article discusses a study...	...	The article's mention generated a discussion a...	The article itself isn't fiction, generating d...	The discussion touches upon themes relevant to...	[]	0.0	0.0	0.0	0.0	0.0	0.0
13222	Is There A Replicability Crisis In Psychology?...	-9	1	1	[{'author_flair_text': 'Ankh-Morpork City Watc...	[https://reddit.com/r/rational/comments/a1t6um...	2018-11-30 17:17:49	2018-11-30 17:17:49	[https://sciencetrends.com/is-there-a-replicab...	A Reddit user mentions a science article explo...	...	No recommendations are given for the article i...	The article itself isn't fiction, and its ment...	Readers interested in the philosophy of scienc...	[]	0.0	0.0	0.0	0.0	0.0	0.0

13223 rows × 23 columns

In [3]:

# custom search
masks = [
    'score > 3',
    'rating_character > 2',
    'rating_plot > 2',
    'rating_rationality > 2',
]

print('Total: n=', len(df))
for m in masks:
    df = df.query(m)
    print(f'Filtered: n={len(df)} from `{m}`')

Total: n= 13223
Filtered: n=8402 from `score > 3`
Filtered: n=6682 from `rating_character > 2`
Filtered: n=6485 from `rating_plot > 2`
Filtered: n=6392 from `rating_rationality > 2`

In [4]:

# old royalroaddl
df_rr = df[df['url'].apply(lambda x: any(['royalroad.com' in y for y in x]))]
df_rr = df_rr.query('first_link_utc < 2023')
df_rr

Out [4]:

	title	score	n_links	n_comments	comments	thread_urls	first_link_utc	last_link_utc	url	description	...	reccomendations	disrecommendations	why	if_you_liked_x_you_will_like_this	rating_quality	rating_rationality	rating_writing	rating_plot	rating_character	rating_worldbuilding
0	Worth the Candle	5458	296	296	[{'author_flair_text': 'Self-Appointed Court S...	[https://reddit.com/r/rational/comments/7228po...	2015-09-26 06:59:23	2024-04-17 19:03:01	[https://archiveofourown.org/works/11478249/ch...	Worth the Candle is a long, original, complete...	...	Readers frequently praised the story's compell...	Some readers found the sheer length of the wor...	Readers of r/rational would likely enjoy Worth...	[Mother of Learning, HPMOR, Worm, Erfworld, Lu...	9.0	7.0	8.5	8.5	8.0	9.0
1	Mother of Learning	1834	174	174	[{'author_flair_text': 'Utopian Smut Peddler',...	[https://reddit.com/r/rational/comments/k8i7jg...	2014-07-26 08:21:12	2024-05-21 07:04:42	[https://www.fictionpress.com/s/2961893/1/Moth...	Mother of Learning is a web serial about Zoria...	...	Readers praise its consistent internal logic, ...	Some readers find the writing style to be roug...	Readers of r/rational will appreciate the stor...	[Time Braid, HPMOR, Worm]	9.0	7.5	7.0	9.0	8.5	9.0
16	Delve	441	39	39	[{'author_flair_text': None, 'body': 'Can I ju...	[https://reddit.com/r/rational/comments/giq6yv...	2019-07-05 15:23:19	2024-07-29 16:14:14	[https://www.royalroad.com/fiction/25225/delve...	Delve is a LitRPG isekai web serial where the ...	...	Readers who enjoy LitRPGs with a strong emphas...	Many readers find the pacing incredibly slow, ...	Readers of r/rational might enjoy Delve for it...	[Progression Fantasy, LitRPG with meaningful m...	6.0	7.0	5.0	6.0	6.0	7.0
18	Only Villains Do That	428	20	20	[{'author_flair_text': 'Path to Victory', 'bod...	[https://reddit.com/r/rational/comments/mu17nk...	2021-04-19 16:42:21	2024-09-09 18:18:36	[https://www.royalroad.com/fiction/40182/only-...	Only Villains Do That is a web serial that fol...	...	Readers praise the story's subversion of iseka...	Some readers find the protagonist's misanthrop...	Readers of r/rational might enjoy Only Villain...	[The Gods are Bastards, Worth the Candle, Over...	7.8	6.0	8.5	7.0	7.5	8.0
23	Ar'Kendrithyst	404	31	31	[{'author_flair_text': None, 'body': 'I think ...	[https://reddit.com/r/rational/comments/mao94s...	2020-01-13 22:21:34	2024-11-18 15:07:29	[https://www.royalroad.com/fiction/26727/arken...	LitRPG isekai featuring a middle-aged pacifist...	...	Readers praise the well-developed world and in...	Early chapters feature a naive, impulsive prot...	Readers of r/rational might enjoy Ar'Kendrithy...	[Delve, Mother of Learning, System Apocalypse]	7.5	7.0	8.0	6.0	7.0	9.0
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
8049	Life of Numbers	4	1	1	[{'author_flair_text': None, 'body': 'Not quit...	[https://reddit.com/r/rational/comments/doa26l...	2019-10-29 02:33:07	2019-10-29 02:33:07	[https://www.royalroad.com/fiction/23018/life-...	Life of Numbers is a Royal Road web serial abo...	...	Readers praised the interesting premise, engag...	No significant negative comments were found re...	Readers of r/rational might enjoy Life of Numb...	[]	7.2	7.0	7.0	7.5	7.0	7.5
8099	Nanocultivation Chronicles: Trials of Lillijoy	4	1	1	[{'author_flair_text': None, 'body': '[Nanocul...	[https://reddit.com/r/rational/comments/ffvzzk...	2020-03-09 21:52:15	2020-03-09 21:52:15	[https://www.royalroad.com/fiction/28254/nanoc...	Nanocultivation Chronicles: Trials of Lillijoy...	...	Readers praise the interesting magic system, t...	Some readers find the protagonist bloodthirsty...	Readers of r/rational might enjoy Nanocultivat...	[other works with overpowered protagonists who...	7.5	6.0	7.0	7.0	6.0	8.0
8109	Apocalypse Man	4	1	1	[{'author_flair_text': None, 'body': 'Believe ...	[https://reddit.com/r/rational/comments/j9sin3...	2020-10-14 01:51:09	2020-10-14 01:51:09	[https://www.royalroad.com/fiction/34623/apoca...	LitRPG web serial with a focus on crafting and...	...	Unique and interesting system; well-written pr...	Graphic depiction of torture; protagonist's la...	Readers looking for a unique LitRPG with craft...	[Wake of the Ravager]	6.0	5.0	8.0	6.0	4.0	7.0
8197	Midnight Moonlight	4	2	2	[{'author_flair_text': None, 'body': '- [The D...	[https://reddit.com/r/rational/comments/q5whcs...	2021-10-18 04:47:35	2021-10-18 04:47:35	[https://neovel.io/book/3695/EN/midnight-moonl...	Midnight Moonlight is a web serial which follo...	...	Readers praise the writing quality and find th...	Some readers find the "nerdy petite girl with ...	Readers of r/rational interested in well-writt...	[]	7.5	6.0	8.0	7.0	7.0	7.0
8290	Chapter 1: May Cause Drowsiness	4	1	1	[{'author_flair_text': None, 'body': 'I don't ...	[https://reddit.com/r/rational/comments/kxw6i2...	2021-01-16 08:20:24	2021-01-16 08:20:24	[https://www.royalroad.com/fiction/35669/apoca...	Apocalypse: Generic System is a LitRPG web ser...	...	Readers recommend Apocalypse: Generic System f...	Some readers might find the pacing too fast or...	Readers of r/Rational might enjoy this story f...	[The Gamer]	7.8	6.5	7.0	7.5	7.0	6.0

428 rows × 23 columns

In [5]:

from rrational.export import export_df_2_html
from pathlib import Path
from rrational.transform import join_uniq, chain_lists, format_flair, c2md, collapsibe, urls2a,url2a,  unique_elements, auto_transform_to_html, long_text_last

df_rr2 = df_rr.copy().drop(columns=['comments'])
df_rr2 = long_text_last(df_rr2)

df_rr2 = auto_transform_to_html(df_rr2)

export_df_2_html(df=df_rr2, output=Path('../outputs/df_links5.html'))

score, is_list=False, is_list_str=False, is_list_url=False, is_url=False, is_urls=False, is_list_obj=False

n_links, is_list=False, is_list_str=False, is_list_url=False, is_url=False, is_urls=False, is_list_obj=False

n_comments, is_list=False, is_list_str=False, is_list_url=False, is_url=False, is_urls=False, is_list_obj=False

first_link_utc, is_list=False, is_list_str=False, is_list_url=False, is_url=False, is_urls=False, is_list_obj=False

last_link_utc, is_list=False, is_list_str=False, is_list_url=False, is_url=False, is_urls=False, is_list_obj=False

rating_quality, is_list=False, is_list_str=False, is_list_url=False, is_url=False, is_urls=False, is_list_obj=False

rating_rationality, is_list=False, is_list_str=False, is_list_url=False, is_url=False, is_urls=False, is_list_obj=False

rating_writing, is_list=False, is_list_str=False, is_list_url=False, is_url=False, is_urls=False, is_list_obj=False

rating_plot, is_list=False, is_list_str=False, is_list_url=False, is_url=False, is_urls=False, is_list_obj=False

rating_character, is_list=False, is_list_str=False, is_list_url=False, is_url=False, is_urls=False, is_list_obj=False

rating_worldbuilding, is_list=False, is_list_str=False, is_list_url=False, is_url=False, is_urls=False, is_list_obj=False

description, is_list=False, is_list_str=False, is_list_url=False, is_url=False, is_urls=False, is_list_obj=False

disrecommendations, is_list=False, is_list_str=False, is_list_url=False, is_url=False, is_urls=False, is_list_obj=False

reviews_summary, is_list=False, is_list_str=False, is_list_url=False, is_url=False, is_urls=False, is_list_obj=False

why, is_list=False, is_list_str=False, is_list_url=False, is_url=False, is_urls=False, is_list_obj=False

reccomendations, is_list=False, is_list_str=False, is_list_url=False, is_url=False, is_urls=False, is_list_obj=False

thread_urls, is_list=True, is_list_str=True, is_list_url=True, is_url=False, is_urls=False, is_list_obj=False

url, is_list=True, is_list_str=True, is_list_url=True, is_url=False, is_urls=False, is_list_obj=False

reviews_quotes, is_list=True, is_list_str=True, is_list_url=False, is_url=False, is_urls=False, is_list_obj=False

if_you_liked_x_you_will_like_this, is_list=True, is_list_str=True, is_list_url=False, is_url=False, is_urls=False, is_list_obj=False

View the page /media/wassname/SGIronWolf/projects5/rrational/outputs/df_links5.html

In [ ]:

In [ ]: