wassname/scrape_r_rational
1
0
Fork 0
mirror of https://github.com/wassname/scrape_r_rational.git synced 2026-09-12 12:50:30 +08:00
Code Issues Packages Projects Releases Wiki Activity
Files
main
scrape_r_rational/nbs/mjc_004_process.ipynb
T
wassname 8a9dfa16e0 tidy
2025-01-02 14:56:33 +08:00

142 KiB
Raw Permalink Blame History

This notebook processes the reddit json and markdown into a table

  • get titles
  • dedup
  • count karma
  • etc
In [35]:
import json
import re
from pathlib import Path

import pandas as pd
from tqdm.auto import tqdm
tqdm.pandas()
from loguru import logger
In [36]:
fs = sorted(Path("../data/json3").glob("*.json"))
print(f"{len(fs)} threads")
10266 threads

Extract titles from markdown links

In [37]:
# We will get a list of title from markdown links
import re
from collections import defaultdict

titles_md = defaultdict(list)


title_blocklist = [
    "patreon",
    "here",
    "link",
    "this",
    "delete",
    "this one",
    "vote",
    "image",
    "comic explanation",
    "this link",
    "this thread",
    "redact",
    "(link)",
    "this post",
    "CLICK THIS LINK",
    "here's",
    "wiki",
    "report",
    "source",
    "the wiki",
    "reddit - dive into anything",
    "top posts",
    "a patreon campaign",
    "table of contents",
    "fanfiction.net",
    "this video",
    "here.",
    'tubmlr',
    "threads",
    "code",
    "link index",
    "some discussions",
    "for",
    "details",
    "pushshift",
    "Reddit - Dive into anything",
    "table of contents",
    "rationalreads",
    "calibre",
    "goodreads",
]

title_should_not_have = [
    "vote",
    "comments",
    "link",
    "here",
    "image",
    "chapter",
    "reddit",
    "wiki",
    "pateron",
    "source",
    'chapter',
    'amazon',
    'kindle',
    'epub',
]

def get_md_link_titles(markdown_text):
    # other rules
    # 1. should have space
    # not subreddit?
    # should have capital
    titles_md2 = {}

    # Regular expression to match markdown links
    markdown_link_pattern = re.compile(r"\[([^\]]+)\]\((http[s]?://[^\)]+)\)")

    # Find all markdown links
    for match in markdown_link_pattern.findall(markdown_text):
        title, url = match
        titles_md2[url] = []
        # tidy title of * / _ and remove leading/trailing whitespace
        title = title.strip().strip("*").strip("_")

        if "r/" in title:
            continue

        # I mean ideally it has spaces but Worm doesn't hpmor doesn't
        # if " " not in title:
        #     continue

        # should have caps
        if title.lower() == title:
            continue

        # should have at least 4 characters e,g, Worm, HPMOR
        if len(title) < 4:
            continue

        if title.startswith("http"):
            continue
        if title.startswith("^"):
            continue

        if title.lower() in title_blocklist:
            continue

        if any([x in title.lower() for x in title_should_not_have]):
            continue

        titles_md2[url].append(title)

    return titles_md2


def remember_md_link_titles(markdown_text):

    md_title_dict = get_md_link_titles(markdown_text)
    for k, v in md_title_dict.items():
        if 'http' not in v:
            titles_md[k].extend(v)

for f in tqdm(fs):
    s = json.loads(f.open().read())

    for comment in s["comments"]:
        text = comment["body"]
        remember_md_link_titles(text)

len(titles_md)
Out [37]:
  0%|          | 0/10266 [00:00<?, ?it/s]
29390
In [38]:
# titles_md
In [39]:
# def get_title_md(url):
#     t = titles_md.get(url, [])
#     if t:
#         s = pd.Series(t)
#         return s.value_counts().index[0]  # return most common title
#     return url
# get_title_md('https://parahumans.wordpress.com/')
In [40]:
# QC: get top markdown titles
import itertools

l = sorted(itertools.chain(*titles_md.values()))
pd.Series(l).value_counts().head(50)
Out [40]:
Time Braid                                     95
Worm                                           67
Mother of Learning                             66
Worth the Candle                               64
Twig                                           59
Statistics                                     52
Problems/Bugs?                                 52
Stop Replying                                  52
Pact                                           36
With This Ring                                 36
Marked for Death                               34
The Waves Arisen                               34
The Metropolitan Man                           33
Luminosity                                     32
What is this?                                  32
Friendship is Optimal                          31
Pale                                           31
Unsong                                         30
A Hero's War                                   30
Dungeon Keeper Ami                             30
Seventh Horcrux                                28
A Practical Guide to Evil                      28
Purple Days                                    26
Harry Potter and the Natural 20                26
A Young Woman's Political Record               26
Super Minion                                   25
Branches on the Tree of Time                   25
Delve                                          25
The Last Angel                                 25
Cordyceps                                      25
Forge of Destiny                               24
The Wandering Inn                              23
The Fall of Doc Future                         23
The Erogamer                                   22
Sanitize                                       22
The Games We Play                              22
Hard Reset                                     22
Shadows of the Limelight                       21
More Books                                     20
Harry Potter and the Methods of Rationality    20
Ward                                           20
Blindsight                                     19
Lord of the Mysteries                          19
Symbiote                                       18
Fairy Dance of Death                           18
The Daily Grind                                18
Only Villains Do That                          18
Dungeon Crawler Carl                           18
Mobile                                         18
Fine Structure                                 18
Name: count, dtype: int64

Extract links

In [ ]:
In [41]:
def extract_links_re(s: str):
    return re.findall(r"\[.*?\]\((.*?)\)", s)


from collections import defaultdict

# TODO consolidate into a metadata object
link_metadata = {
    "karma": defaultdict(int),  # track score
    "gossip": defaultdict(list),  # track number and content of comments...
    "gossip_threads": defaultdict(list),  # threads
    "dates": defaultdict(list),  # comment dates
}


def stem_url(u):
    parts = u.strip().rstrip("/").split("/")
    return "/".join(parts[:-1])


# banned_link_suffixes = ['jpeg', 'png', 'jpg']

data = []
for f in tqdm(fs):
    post = json.loads(f.open().read())
    thread_url = "https://reddit.com" + post["permalink"]
    thread_created = post["created_utc"]

    for comment in post["comments"]:
        text = comment["body"]
        links = extract_links_re(text)
        for link in links:
            data.append(
                dict(
                    url=link,
                    score=comment["score"],
                    # created_utc=post["created_utc"],
                    # comment_url="https://reddit.com" + comment["permalink"],
                    # comment_body=comment["body"],
                    comment=comment,
                    thread_url=thread_url,
                )
            )

print(f"{len(data)} links found")
  0%|          | 0/10266 [00:00<?, ?it/s]
46518 links found
In [42]:
df_links1 = pd.DataFrame(data)
# check for dups
# print(df_links['url'].value_counts().head(10))

# df_links1["created_utc"] = pd.to_datetime(df_links1["created_utc"], unit="s")


def join_uniq_list(x: list) -> list:
    return list(set(x))

# def joun_uniq_comment(comments: list) -> list:
#     d = {x['permalink']: x for x in comments}
#     return list(d.values())


df_links2 = df_links1.groupby("url").agg(
    score=("score", "sum"),
    n_links=("score", "count"),
    comments=("comment", list),
    thread_urls=("thread_url", join_uniq_list),
).sort_values("score", ascending=False)

# order comments by date
df_links2["comments"] = df_links2["comments"].apply(
    lambda x: sorted(x, key=lambda y: y["created_utc"])
)

# now add come things like first and last comment time
df_links2['n_comments'] = df_links2['comments'].apply(len)
df_links2['first_link_utc'] = pd.to_datetime(df_links2['comments'].apply(lambda x: min([y['created_utc'] for y in x])), unit='s')
df_links2['last_link_utc'] = pd.to_datetime(df_links2['comments'].apply(lambda x: max([y['created_utc'] for y in x])), unit='s')
df_links2
Out [42]:
score n_links comments thread_urls n_comments first_link_utc last_link_utc
url
https://www.patreon.com/alexanderwales 1402 22 [{'id': 'cqgom57', 'created_utc': 1429380827.0... [https://reddit.com/r/rational/comments/al7z2v... 22 2015-04-18 18:13:47 2021-04-29 20:00:04
https://archiveofourown.org/works/11478249/chapters/25740126 794 63 [{'id': 'dluq8vo', 'created_utc': 1503171093.0... [https://reddit.com/r/rational/comments/7dz6kj... 63 2017-08-19 19:31:33 2024-04-17 19:03:01
https://discord.gg/sM99CF3 657 60 [{'id': 'd88ddo9', 'created_utc': 1475248463.0... [https://reddit.com/r/rational/comments/5he78o... 60 2016-09-30 15:14:23 2018-08-10 17:12:02
https://www.fanfiction.net/s/10360716/1/The-Metropolitan-Man 645 55 [{'id': 'chvfe2o', 'created_utc': 1401507549.0... [https://reddit.com/r/rational/comments/3q52bz... 55 2014-05-31 03:39:09 2023-06-26 19:21:24
https://docs.google.com/document/d/1EUSMDHdRdbvQJii5uoSezbjtvJpxdF6Da8zqvuW42bg/edit?usp=sharing 639 58 [{'id': 'd80h9fe', 'score': 5, 'body': 'So a w... [https://reddit.com/r/rational/comments/5he78o... 58 2016-09-24 19:53:40 2018-08-10 17:12:02
... ... ... ... ... ... ... ...
http://www.imdb.com/title/tt0317705/?ref_=fn_al_tt_1 -10 1 [{'id': 'dprs5nf', 'created_utc': 1510606681.0... [https://reddit.com/r/rational/comments/7cneg5... 1 2017-11-13 20:58:01 2017-11-13 20:58:01
https://wiki.lesswrong.com/wiki/Orthogonality_thesis -10 2 [{'id': 'cxn65fy', 'score': 0, 'body': 'I don'... [https://reddit.com/r/rational/comments/3vc0si... 2 2015-12-04 18:17:07 2016-07-11 16:27:30
http://www.imdb.com/title/tt0405325/?ref_=nv_sr_1 -10 1 [{'id': 'dprs5nf', 'created_utc': 1510606681.0... [https://reddit.com/r/rational/comments/7cneg5... 1 2017-11-13 20:58:01 2017-11-13 20:58:01
https://snewd.com/ebooks/leviathan/ -15 1 [{'id': 'gl1vde4', 'created_utc': 1611810163.0... [https://reddit.com/r/rational/comments/l6ki69... 1 2021-01-28 05:02:43 2021-01-28 05:02:43
https://knowyourmeme.com/memes/absolutely-disgusting -15 1 [{'id': 'esqmm0i', 'created_utc': 1562205524.0... [https://reddit.com/r/rational/comments/c8w3xe... 1 2019-07-04 01:58:44 2019-07-04 01:58:44

33370 rows × 7 columns

In [43]:
print(f"{len(df_links2)} unique links found")
33370 unique links found

Filter links

In [ ]:
In [44]:
banned_link_suffixes = ["jpeg", "png", "jpg"]

link_blocklist = [
    #   'reddit',
    "redact",
    "pastebin",
    "wikipedia",
    "docs.google",
    "discord",
    "tvtropes.org",
    "ask_wikibot",
    "autowiki",
    "banned",
    # pateron.com ?
    "reddit.com/user/",
    # 'smile.amazon.com',
    "xkcd",
    'feedly.com',
    'autohotkey.com',
    "ebay",
    "youtubot",
    "reddit.com/r/rational",
    # 'sneakpeekbot', 'RemindMeBot',
    # 'WikiSummarizerBot', 'bot/', 'Bot/',
    "knowyourmeme.com",
    "UserSim",
    "vote.php",
    "youtube",
    "github",
    "imgur",
    "wikisummarizer",
    "mozilla.org",
    "reddit.com/message",
    "autotldr",
    "/top/",
    "redd.it",
    "reddit.com/u",
    "fanficfare",
    "bot.com",
    "greasyfork",
    'edit?', # google docs
]


link_allowlist = ["hfy"]

df_links3 = df_links2.copy()

print(f"{len(df_links3)} links before blocklist")
df_links3 = df_links3[
    ~df_links3.index.str.contains("|".join(link_blocklist), regex=True)
    | df_links3.index.str.contains("|".join(link_allowlist), regex=True)
]
print(f"{len(df_links3)} links after blocklist")
df_links3 = df_links3[df_links3.index.str.startswith("http")]
for suffix in banned_link_suffixes:
    df_links3 = df_links3[~df_links3.index.str.endswith(suffix)]
print(f"{len(df_links3)} links after rm suffixes")

# # must have more than one mention?
# df_links3 = df_links3[df_links3 > 1]
# print(f"{len(df_links3)} after count")

# if it has reddit, github, wiki and bot in the title, it's probably a bot
df_links3 = df_links3[
    ~(
        df_links3.index.str.contains("reddit", case=False)
        & df_links3.index.str.contains("bot", case=False)
    )
]
df_links3 = df_links3[
    ~(
        df_links3.index.str.contains("wiki", case=False)
        & df_links3.index.str.contains("bot", case=False)
    )
]
df_links3 = df_links3[
    ~(
        df_links3.index.str.contains("github", case=False)
        & df_links3.index.str.contains("bot", case=False)
    )
]
print(f"{len(df_links3)} links after rm bot")
df_links3
Out [44]:
33370 links before blocklist
21705 links after blocklist
17984 links after rm suffixes
17958 links after rm bot
score n_links comments thread_urls n_comments first_link_utc last_link_utc
url
https://www.patreon.com/alexanderwales 1402 22 [{'id': 'cqgom57', 'created_utc': 1429380827.0... [https://reddit.com/r/rational/comments/al7z2v... 22 2015-04-18 18:13:47 2021-04-29 20:00:04
https://archiveofourown.org/works/11478249/chapters/25740126 794 63 [{'id': 'dluq8vo', 'created_utc': 1503171093.0... [https://reddit.com/r/rational/comments/7dz6kj... 63 2017-08-19 19:31:33 2024-04-17 19:03:01
https://www.fanfiction.net/s/10360716/1/The-Metropolitan-Man 645 55 [{'id': 'chvfe2o', 'created_utc': 1401507549.0... [https://reddit.com/r/rational/comments/3q52bz... 55 2014-05-31 03:39:09 2023-06-26 19:21:24
https://www.fictionpress.com/s/2961893/1/Mother-of-Learning 550 60 [{'id': 'cj87tis', 'created_utc': 1406362872.0... [https://reddit.com/r/rational/comments/cmc4a0... 60 2014-07-26 08:21:12 2021-03-15 18:12:29
https://twigserial.wordpress.com/ 449 49 [{'id': 'cy39n9f', 'score': 13, 'body': 'Outsi... [https://reddit.com/r/rational/comments/bqwp8b... 49 2015-12-18 10:22:59 2024-09-11 20:51:31
... ... ... ... ... ... ... ...
https://myanimelist.net/anime/31964/Boku_no_Hero_Academia?q=my%20hero%20acade -10 1 [{'id': 'dprs5nf', 'created_utc': 1510606681.0... [https://reddit.com/r/rational/comments/7cneg5... 1 2017-11-13 20:58:01 2017-11-13 20:58:01
http://www.imdb.com/title/tt0317705/?ref_=fn_al_tt_1 -10 1 [{'id': 'dprs5nf', 'created_utc': 1510606681.0... [https://reddit.com/r/rational/comments/7cneg5... 1 2017-11-13 20:58:01 2017-11-13 20:58:01
https://wiki.lesswrong.com/wiki/Orthogonality_thesis -10 2 [{'id': 'cxn65fy', 'score': 0, 'body': 'I don'... [https://reddit.com/r/rational/comments/3vc0si... 2 2015-12-04 18:17:07 2016-07-11 16:27:30
http://www.imdb.com/title/tt0405325/?ref_=nv_sr_1 -10 1 [{'id': 'dprs5nf', 'created_utc': 1510606681.0... [https://reddit.com/r/rational/comments/7cneg5... 1 2017-11-13 20:58:01 2017-11-13 20:58:01
https://snewd.com/ebooks/leviathan/ -15 1 [{'id': 'gl1vde4', 'created_utc': 1611810163.0... [https://reddit.com/r/rational/comments/l6ki69... 1 2021-01-28 05:02:43 2021-01-28 05:02:43

17958 rows × 7 columns

In [45]:
# QC look at removed links
df_links2[~df_links2.index.isin(df_links3.index)].sort_values("score", ascending=False)
Out [45]:
score n_links comments thread_urls n_comments first_link_utc last_link_utc
url
https://discord.gg/sM99CF3 657 60 [{'id': 'd88ddo9', 'created_utc': 1475248463.0... [https://reddit.com/r/rational/comments/5he78o... 60 2016-09-30 15:14:23 2018-08-10 17:12:02
https://docs.google.com/document/d/1EUSMDHdRdbvQJii5uoSezbjtvJpxdF6Da8zqvuW42bg/edit?usp=sharing 639 58 [{'id': 'd80h9fe', 'score': 5, 'body': 'So a w... [https://reddit.com/r/rational/comments/5he78o... 58 2016-09-24 19:53:40 2018-08-10 17:12:02
https://www.youtube.com/watch?v=kbyTOAlhRHk 450 42 [{'id': 'de5sdvj', 'created_utc': 1487951990.0... [https://reddit.com/r/rational/comments/6kghnx... 42 2017-02-24 15:59:50 2020-05-25 21:14:35
https://docs.google.com/document/d/11QAh61C8gsL-5KbdIy5zx3IN6bv_E9UkHjwMLVQ7LHg/edit?usp=sharing 441 42 [{'id': 'ddv88im', 'created_utc': 1487348445.0... [https://reddit.com/r/rational/comments/6kghnx... 42 2017-02-17 16:20:45 2018-08-10 17:12:02
https://redact.dev/home 406 79 [{'id': 'cyzo8d7', 'score': 2, 'body': 'politi... [https://reddit.com/r/rational/comments/bjhbqh... 79 2016-01-15 21:18:35 2020-11-04 03:12:20
... ... ... ... ... ... ... ...
https://pastebin.com/64GuVi2F/93823 -7 1 [{'id': 'd6ntkpk', 'score': -7, 'body': '[dele... [https://reddit.com/r/rational/comments/4ydtxx... 1 2016-08-19 08:33:16 2016-08-19 08:33:16
https://www.reddit.com/r/rational/comments/ix25dk/d_monday_request_and_recommendation_thread/g64soul/ -8 1 [{'id': 'g65db9r', 'created_utc': 1600722605.0... [https://reddit.com/r/rational/comments/ix25dk... 1 2020-09-21 21:10:05 2020-09-21 21:10:05
https://reddit.com/message/compose/?to=FatFingerHelperBot&subject=delete&message=delete%20dxgv0pj -9 1 [{'id': 'dxgv0pj', 'created_utc': 1523905663.0... [https://reddit.com/r/rational/comments/8cppyk... 1 2018-04-16 19:07:43 2018-04-16 19:07:43
https://reddit.com/message/compose/?to=FatFingerHelperBot&subject=delete&message=delete%20e68m7xq -9 1 [{'id': 'e68m7xq', 'score': -9, 'body': 'It se... [https://reddit.com/r/rational/comments/9h1454... 1 2018-09-19 04:51:09 2018-09-19 04:51:09
https://knowyourmeme.com/memes/absolutely-disgusting -15 1 [{'id': 'esqmm0i', 'created_utc': 1562205524.0... [https://reddit.com/r/rational/comments/c8w3xe... 1 2019-07-04 01:58:44 2019-07-04 01:58:44

15412 rows × 7 columns

In [46]:
# df_links.plot.hist(bins=25, logy=True)

Fetch missing titles

This is hard and messy as it's an adverserial web scraping problem, I'll use a mix of methods (from the markdown, requests, url)

In [47]:
from anycache import anycache

f_cache = Path("../outputs/.anycache")
f_cache_web = Path("../outputs/.anycache_web")
In [48]:
# # #DEBUG clear
# import shutil
# shutil.rmtree(f_cache)


# shutil.rmtree(f_cache_web)
In [49]:
"""
HACK temporarily change sys.argv
"""
import sys
from typing import List


class Argv:
    def __init__(self, new_argv: List[str]):
        self.new_argv = new_argv
        self.original_argv = None

    def __enter__(self):
        self.original_argv = sys.argv[:]
        sys.argv[:] = self.new_argv
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        sys.argv[:] = self.original_argv
In [50]:
banned_titles = [
    "reddit - dive into anything",
    "just a moment...",
    "",
    "x.com",
    'x',
    'heroku | application error',
    'msy archives',
]
banned_title_contents = [
    'error',
    'heroku',
    '404',
    'log in',
    'sign up',
    'login',
    'sign in',
    'register',
]
def is_banned_title(title):
    if title.lower().strip() in banned_titles: return True
    if any([x in title.lower() for x in banned_title_contents]): return True
    return False
In [51]:
"""use lncrawl browser to get titles

also use a persistent header browser so we can manually solve cloudfare


note: if it fails to start browser, try restarting VSCODE
"""

# see https://github.com/dipu-bd/lightnovel-crawler/blob/master/lncrawl/core/app.py#L84
import logging

from lncrawl.core.browser import Browser
from lncrawl.core.exeptions import ScraperErrorGroup
from lncrawl.core.scraper import Scraper
from readability import Document


from lncrawl.core.browser import EC

# start a browser
with Argv([""]):
    browser = Browser(headless=False)
    browser._init_browser()
    browser._apply_cookies()

# manually pass cloudfare (YOU NEED TO CLICK!)
url = "https://www.fanfiction.net/s/10758358/1/What-You-Leave-Behind"
browser.visit(url)
browser.wait("body")
browser.wait(
    "#challenge-running",
    expected_conditon=EC.invisibility_of_element,
    timeout=30,
)
# reader = Document(browser.html)
In [52]:
from bs4 import BeautifulSoup
import time

# @anycache(f_cache_web)
def lncrawl_guess_novel_title(url: str) -> str:
    try:
        scraper = Scraper(url)
        response = scraper.get_response(url)
        reader = Document(response.text)
        title = reader.short_title()
        assert not is_banned_title(title), f"bad title `{title}` for url={url}"
    except (AssertionError, ) + ScraperErrorGroup as e:
        # if logger.isEnabledFor(logging.DEBUG):
        #     logger.exception("Failed to get response: %s", e)
        logger.debug(f"[lncrawl:scrape].failed url={url} with {e}, trying [lncrawl.browser]")
        browser.visit(url)
        browser.wait("body", timeout=40)        
        time.sleep(2) # wait for javascript to load
        reader = Document(browser.html)
        title = reader.short_title()
        assert not is_banned_title(title), f"bad title `{title}` for url={url}"
    return title


# # # Test
# urls = [
# 'https://x.com/AwfulFantasy/status/1866511632995962894',
# 'https://reddit.com/r/motheroflearning/comments/5v0zl0/links_to_discussion_threads',
# #     'https://parahumans.wordpress.com/',
# #     # 'https://www.wuxiaworld.com/novel/overgeared',
# #     'https://www.fanfiction.net/s/10758358/1/What-You-Leave-Behind',
# #     # 'https://www.royalroad.com/fiction/81002/the-years-of-apocalypse-a-time-loop-progression',
# ]

# # with Argv(['']):
# for url in urls:
#     r = lncrawl_guess_novel_title(url)
#     print(url)
#     print(r)
In [53]:
import cloudscraper
from bs4 import BeautifulSoup

session = cloudscraper.create_scraper()

@anycache(f_cache_web)
def cloudscrape_title(url):
    r = session.get(url, timeout=15, allow_redirects=True)
    r.raise_for_status()
    soup = BeautifulSoup(r.text, "html.parser")
    title = soup.title.text.strip()
    assert not is_banned_title(title), f"bad title `{title}` for url={url}"
    return title
In [54]:
# dedup using title

from pathlib import Path

# TODO I should look at how fanficfare and lncrawler do this
# https://github.com/dipu-bd/lightnovel-crawler/blob/master/lncrawl/core/app.py#L84

@anycache(f_cache)
def get_title_md(url):
    t = titles_md.get(url, [])
    if t:
        s = pd.Series(t)
        title = s.value_counts().index[0]  # return most common title
        assert not is_banned_title(title), f"bad title `{title}` for url={url}"
        return title
    raise IndexError(f"Failed to get title for {url}")


def get_slugged_title(url2):
    url = url2

    if "reddit.com" in url:
        # if reddit thread get thead name e.g. 'https://www.reddit.com/r/Parahumans/comments/9oexpn/ward_spoilers_frustration_with_the_state_of_the/e7tprmt/?context=3'
        if "/comments/" in url:
            p = re.match(r".+reddit.com/r/([^/]+)/comments/([^/]+)/([^/]*)", url)
            if p:
                return 'thread ' + p.group(1).replace("_", " ") + " " + p.group(3).replace("_", " ")
        # 'https://reddit.com/r/motheroflearning/comments/5v0zl0/',
        if "/r/" in url:
            p = re.match(r".+reddit.com/r/([^/]+)/", url)
            if p:
                return 'r/' + p.group(1)


    # if that doesn't work, does the end of the url contain a slugged title?
    # e.g. https://www.fanfiction.net/s/5193644/harry-potter-and-the-methods-of-rationality
    slugged_title = url.split("/")[-1].replace("-", " ")
    if " " in slugged_title:
        title = slugged_title
        assert not is_banned_title(title), f"bad title `{title}` for url={url2}"
        return title
    raise ValueError(f"Failed to get title for {url2}")


@anycache(f_cache)
def get_title(url):
    # first check if it's in the markdown cache
    try:
        return get_title_md(url)
    except Exception as e:
        logger.debug(f"[md] Failed to get title for {url} {e}, trying lncrawl")

    # try scraping from the web
    try:
        return lncrawl_guess_novel_title(url)
    except Exception as e:
        logger.debug(f"[lncrawl] Failed to get title for {url} {e}, trying cloudscrape")

    try:
        return cloudscrape_title(url)
    except Exception as e:
        logger.debug(f"[cloudscrape] Failed to get title for {url} {e}, trying slugged title")

    # fall back on slugged title?
    try:
        return get_slugged_title(url)
    except Exception as e:
        logger.debug(f"[slug] Failed to get title for {url} {e}")

    title = url

    # assert not is_banned_title(title), f"bad title `{title}` for url={url}"

    return title


# url = df_links.index[0]

test_urls = [
    'https://x.com/AwfulFantasy/status/1866511632995962894',
    # 'https://reddit.com/r/motheroflearning/comments/5v0zl0/links_to_discussion_threads',
    # 'https://www.fimfiction.net/story/403715/zebric',
    # 'https://myanimelist.net/anime/18153/Kyoukai_no_',
    # 'https://yudkowsky.tumblr.com/',
    # "https://www.fanfiction.net/s/10758358/1/What-You-Leave-Behind",
    # "https://www.wuxiaworld.com/novel/overgeared",
    # "https://www.royalroad.com/fiction/81002/the-years-of-apocalypse-a-time-loop-progression",
]
for url in test_urls:
    r = get_title(url)
    print(f"{url} -> `{r}`")
https://x.com/AwfulFantasy/status/1866511632995962894 -> `Awful Fantasy on X: "https://t.co/bXcCbRcmpI"`
In [55]:
# QC
urls = df_links3.reset_index().url.sample(20, random_state=236)
titles = urls.progress_map(get_title).values
pd.Series(urls.values, index=titles)
Out [55]:
  0%|          | 0/20 [00:00<?, ?it/s]
Darwin's Game                                                                              http://bato.to/comic/_/comics/darwins-game-r8466/
"Hot Tub Time Machine 2"                                                                   http://www.miraclejones.com/stories/hot-tub-ti...
Power of a Princess                                                                        https://archiveofourown.org/works/19215478/cha...
Lastman+ +S01E01+ +You%E2%80%99re+an+asshole+Aldana.mp4                                    https://archive.org/details/lastman-2016-seaso...
Basic economics in the Narutoverse                                                         https://www.reddit.com/r/Naruto/comments/19x2i...
Message in a Bottle                                                                        https://www.fimfiction.net/story/368986/messag...
On Being a Sith Lord                                                                                 https://www.fanfiction.net/s/5759101/1/
Holedown                                                                                   https://play.google.com/store/apps/details?id=...
The White Dress                                                                            https://np.reddit.com/r/JUSTNOMIL/comments/675...
Gideon the Ninth                                                                                        https://www.amazon.com/dp/B07J6HWLPR
Algorithms to Live By                                                                      https://www.amazon.com/Algorithms-Live-Compute...
Unofficial ESPR Post-mortem                                                                https://www.lesserwrong.com/posts/Gbw9Tnqeo9cr...
Applied Cultural Anthropology, or How I Learned to Stop Worrying and Love the Cruciatus    https://m.fanfiction.net/s/9238861/1/Applied-C...
The Swiss Family Robinson                                                                               http://www.gutenberg.org/ebooks/3836
Interlude 6 | Worm                                                                         http://parahumans.wordpress.com/2012/01/14/int...
Saga series                                                                                          https://www.goodreads.com/series/146415
Unsong                                                                                     http://unsongbook.com/chapter-44-a-world-withi...
Stone Shape - d20PFSRD                                                                     https://www.d20pfsrd.com/magic/all-spells/s/st...
Retry                                                                                           https://www.fanfiction.net/s/9515185/1/Retry
Rooster meets girl every day after school                                                              https://gfycat.com/RespectfulSpryGoat
dtype: object
In [ ]:
In [56]:
# # HACK
# n = 10000 # DEV limit
# df_links3 = df_links3.iloc[:n].copy()

df_links3['title'] = df_links3.reset_index().url.progress_map(get_title).values
  0%|          | 0/17958 [00:00<?, ?it/s]
2025-01-02 14:34:22.354 | DEBUG    | __main__:get_title:51 - [md] Failed to get title for https://archiveofourown.org/works/32244394/chapters/79914832 Failed to get title for https://archiveofourown.org/works/32244394/chapters/79914832, trying lncrawl
2025-01-02 14:34:32.156 | DEBUG    | __main__:get_title:51 - [md] Failed to get title for https://www.fanfiction.net/s/13894611/1/An-Undertow-of-Sand Failed to get title for https://www.fanfiction.net/s/13894611/1/An-Undertow-of-Sand, trying lncrawl
2025-01-02 14:34:32.227 | DEBUG    | __main__:lncrawl_guess_novel_title:15 - [lncrawl:scrape].failed url=https://www.fanfiction.net/s/13894611/1/An-Undertow-of-Sand with 403 Client Error: Forbidden for url: https://www.fanfiction.net/s/13894611/1/An-Undertow-of-Sand, trying [lncrawl.browser]
2025-01-02 14:34:35.198 | DEBUG    | __main__:get_title:51 - [md] Failed to get title for https://forum.questionablequesting.com/threads/an-undertow-of-sand-percy-jackson-and-the-cthulhu-mythos.15568/ Failed to get title for https://forum.questionablequesting.com/threads/an-undertow-of-sand-percy-jackson-and-the-cthulhu-mythos.15568/, trying lncrawl
2025-01-02 14:34:37.908 | DEBUG    | __main__:get_title:51 - [md] Failed to get title for https://www.reddit.com/r/nosleep/comments/u25xsr/has_anyone_here_ever_played_the_drowned_man_games/ Failed to get title for https://www.reddit.com/r/nosleep/comments/u25xsr/has_anyone_here_ever_played_the_drowned_man_games/, trying lncrawl
2025-01-02 14:34:39.238 | DEBUG    | __main__:lncrawl_guess_novel_title:15 - [lncrawl:scrape].failed url=https://www.reddit.com/r/nosleep/comments/u25xsr/has_anyone_here_ever_played_the_drowned_man_games/ with bad title `Reddit - Dive into anything` for url=https://www.reddit.com/r/nosleep/comments/u25xsr/has_anyone_here_ever_played_the_drowned_man_games/, trying [lncrawl.browser]
2025-01-02 14:34:45.116 | DEBUG    | __main__:get_title:51 - [md] Failed to get title for https://alexanderwales.com/narrativism-vs-simulationism/ Failed to get title for https://alexanderwales.com/narrativism-vs-simulationism/, trying lncrawl
2025-01-02 14:34:46.601 | DEBUG    | __main__:get_title:51 - [md] Failed to get title for https://www.imdb.com/user/ur113395328/ratings/ Failed to get title for https://www.imdb.com/user/ur113395328/ratings/, trying lncrawl
2025-01-02 14:34:50.843 | DEBUG    | __main__:get_title:51 - [md] Failed to get title for https://www.goodreads.com/review/list/82954187?shelf=read Failed to get title for https://www.goodreads.com/review/list/82954187?shelf=read, trying lncrawl
2025-01-02 14:34:53.946 | DEBUG    | __main__:get_title:51 - [md] Failed to get title for https://comick.io/comic/koudou-ni-hattatsu-shita-igaku-wa-mahou-to-kubetsu-ga-tsukanai Failed to get title for https://comick.io/comic/koudou-ni-hattatsu-shita-igaku-wa-mahou-to-kubetsu-ga-tsukanai, trying lncrawl
2025-01-02 14:34:54.341 | DEBUG    | __main__:lncrawl_guess_novel_title:15 - [lncrawl:scrape].failed url=https://comick.io/comic/koudou-ni-hattatsu-shita-igaku-wa-mahou-to-kubetsu-ga-tsukanai with 403 Client Error: Forbidden for url: https://comick.io/comic/koudou-ni-hattatsu-shita-igaku-wa-mahou-to-kubetsu-ga-tsukanai, trying [lncrawl.browser]
2025-01-02 14:34:58.932 | DEBUG    | __main__:get_title:51 - [md] Failed to get title for https://www.goodreads.com/book/show/37946419-sixteen-ways-to-defend-a-walled-city Failed to get title for https://www.goodreads.com/book/show/37946419-sixteen-ways-to-defend-a-walled-city, trying lncrawl
2025-01-02 14:35:02.782 | DEBUG    | __main__:get_title:51 - [md] Failed to get title for https://www.royalroad.com/fiction/67180/here-be-dragons-book-1-of-the-emergence-series Failed to get title for https://www.royalroad.com/fiction/67180/here-be-dragons-book-1-of-the-emergence-series, trying lncrawl
2025-01-02 14:35:04.085 | DEBUG    | __main__:get_title:51 - [md] Failed to get title for https://forums.spacebattles.com/threads/naruto-the-outsiders-resolve.1050442/reader/ Failed to get title for https://forums.spacebattles.com/threads/naruto-the-outsiders-resolve.1050442/reader/, trying lncrawl
2025-01-02 14:35:04.786 | DEBUG    | __main__:get_title:51 - [md] Failed to get title for https://www.fanfiction.net/s/10758358/1/What-You-Leave-Behind Failed to get title for https://www.fanfiction.net/s/10758358/1/What-You-Leave-Behind, trying lncrawl
2025-01-02 14:35:04.836 | DEBUG    | __main__:lncrawl_guess_novel_title:15 - [lncrawl:scrape].failed url=https://www.fanfiction.net/s/10758358/1/What-You-Leave-Behind with 403 Client Error: Forbidden for url: https://www.fanfiction.net/s/10758358/1/What-You-Leave-Behind, trying [lncrawl.browser]
2025-01-02 14:35:07.618 | DEBUG    | __main__:get_title:51 - [md] Failed to get title for https://www.isfdb.org/cgi-bin/pe.cgi?12099 Failed to get title for https://www.isfdb.org/cgi-bin/pe.cgi?12099, trying lncrawl
In [57]:
# QC how many titles failed?
print(df_links3['title'].str.startswith('http').sum())
print(df_links3['title'].str.contains('just a moment').sum())
print(df_links3['title'].isna().sum())
391
0
0
In [58]:
t = df_links3['title']
t[t.str.startswith('http')]
Out [58]:
url
https://erfworld.com/landing                                                                                                               https://erfworld.com/landing
http://tts.determinismsucks.net/wiki/Main_Page                                                                           http://tts.determinismsucks.net/wiki/Main_Page
https://forum.questionablequesting.com/threads/19602                                                                  https://forum.questionablequesting.com/threads...
https://forum.questionablequesting.com/threads/16818                                                                  https://forum.questionablequesting.com/threads...
https://boards.4channel.org/tg/thread/63771426#p63774126                                                              https://boards.4channel.org/tg/thread/63771426...
                                                                                                                                            ...                        
http://www.reddit.com/r/socialjusticeinaction                                                                             http://www.reddit.com/r/socialjusticeinaction
http(s                                                                                                                                                           http(s
http://1d4chan.org/wiki/RAW                                                                                                                 http://1d4chan.org/wiki/RAW
http://amix.dk/blog/post/19588                                                                                                           http://amix.dk/blog/post/19588
http://www.rawstory.com/rs/2014/12/mit-professor-explains-the-real-oppression-is-having-to-learn-to-talk-to-women/    http://www.rawstory.com/rs/2014/12/mit-profess...
Name: title, Length: 391, dtype: object
In [ ]:
In [59]:
# join by title
def join_uniq(x: list[str]):
    return "\n".join(set(x))


def chain_lists(x: list[list[str]]):
    return [item for sublist in x for item in sublist]


# TODO we want to order urls by n_links or score


df3 = (
    df_links3
    .reset_index()
    .sort_values("score", ascending=False)
    .groupby("title")
    .agg(
        {
            "score": "sum",
            "n_links": "sum",
            'n_comments': 'sum',
            "comments": chain_lists,
            "thread_urls": chain_lists,
            'first_link_utc': 'min',
            'last_link_utc': 'max',
            'url': list,
        }
    )
    .sort_values("score", ascending=False)
)
df3.head(33)
Out [59]:
score n_links n_comments comments thread_urls first_link_utc last_link_utc url
title
Worth the Candle 1460 110 110 [{'id': 'dluq8vo', 'created_utc': 1503171093.0... [https://reddit.com/r/rational/comments/7dz6kj... 2017-07-28 13:17:36 2024-04-17 19:03:01 [https://archiveofourown.org/works/11478249/ch...
Alexander Wales - The Metropolitan Man, Shadows of the Limelight 1402 22 22 [{'id': 'cqgom57', 'created_utc': 1429380827.0... [https://reddit.com/r/rational/comments/al7z2v... 2015-04-18 18:13:47 2021-04-29 20:00:04 [https://www.patreon.com/alexanderwales]
Mother of Learning 1063 101 101 [{'id': 'cj87tis', 'created_utc': 1406362872.0... [https://reddit.com/r/rational/comments/cmc4a0... 2014-07-26 08:21:12 2024-05-21 07:04:42 [https://www.fictionpress.com/s/2961893/1/Moth...
A Practical Guide to Evil 1006 92 92 [{'id': 'd3joifd', 'score': 2, 'body': '[A Pra... [https://reddit.com/r/rational/comments/byyy3d... 2016-04-05 18:53:21 2024-08-09 10:10:21 [https://practicalguidetoevil.wordpress.com/, ...
Worm 916 83 83 [{'id': 'cshptm0', 'created_utc': 1435192762.0... [https://reddit.com/r/rational/comments/16rsx5... 2014-01-27 02:09:42 2024-11-22 21:39:11 [https://parahumans.wordpress.com/, https://pa...
Time Braid 807 106 106 [{'id': 'chwnyh7', 'score': 5, 'body': 'Fundam... [https://reddit.com/r/rational/comments/1g8qdi... 2014-06-01 21:56:08 2024-10-26 18:31:06 [https://www.fanfiction.net/s/5193644/1/Time-B...
The Metropolitan Man 797 59 59 [{'id': 'chvfe2o', 'created_utc': 1401507549.0... [https://reddit.com/r/rational/comments/3q52bz... 2014-05-31 03:39:09 2023-06-26 19:21:24 [https://www.fanfiction.net/s/10360716/1/The-M...
Unsong 687 80 80 [{'id': 'd3rkl2l', 'created_utc': 1464778862.0... [https://reddit.com/r/rational/comments/4mptnp... 2016-01-11 06:14:56 2024-04-01 16:21:28 [http://unsongbook.com/, http://unsongbook.com...
With This Ring 665 61 61 [{'id': 'd7j4tsw', 'created_utc': 1473651094.0... [https://reddit.com/r/rational/comments/8wb5nt... 2014-10-13 22:49:39 2024-10-11 19:32:41 [https://forums.sufficientvelocity.com/threads...
Twig 643 67 67 [{'id': 'cy39n9f', 'score': 13, 'body': 'Outsi... [https://reddit.com/r/rational/comments/bqwp8b... 2015-07-10 00:36:09 2024-11-12 12:13:08 [https://twigserial.wordpress.com/, https://tw...
Pact 484 37 37 [{'id': 'd6o4yyr', 'created_utc': 1471619930.0... [https://reddit.com/r/rational/comments/i1fjn8... 2014-06-06 02:10:18 2022-12-13 08:54:28 [https://pactwebserial.wordpress.com/, https:/...
Cordyceps 444 44 44 [{'id': 'dmlfd26', 'score': 14, 'body': '**Tit... [https://reddit.com/r/rational/comments/9otolo... 2016-06-05 18:55:30 2024-08-17 02:03:24 [https://archiveofourown.org/works/6178036/cha...
Only Villains Do That 441 20 20 [{'id': 'gv3l3vi', 'created_utc': 1618850541.0... [https://reddit.com/r/rational/comments/nh1vrs... 2021-04-19 16:42:21 2024-09-09 18:18:36 [https://www.royalroad.com/fiction/40182/only-...
Seventh Horcrux 421 33 33 [{'id': 'ct2j94r', 'score': 3, 'body': 'It's n... [https://reddit.com/r/rational/comments/cmc4a0... 2014-09-30 15:01:26 2023-04-10 17:39:44 [https://www.fanfiction.net/s/10677106/1/Seven...
Vigor Mortis 415 16 16 [{'id': 'gqa2bgb', 'created_utc': 1615253133.0... [https://reddit.com/r/rational/comments/pwhojl... 2021-03-09 01:25:33 2024-04-02 08:58:16 [https://www.royalroad.com/fiction/40373/vigor...
Ar'Kendrithyst 414 32 32 [{'id': 'feb05va', 'score': 3, 'body': 'I thin... [https://reddit.com/r/rational/comments/1aupa4... 2020-01-13 22:21:34 2024-11-18 15:07:29 [https://www.royalroad.com/fiction/26727/arken...
A Young Woman's Political Record 403 33 33 [{'id': 'e3nmdgj', 'created_utc': 1533490515.0... [https://reddit.com/r/rational/comments/ffvzzk... 2018-08-05 17:35:15 2024-11-18 15:07:29 [https://forums.spacebattles.com/threads/a-you...
The Waves Arisen 394 47 47 [{'id': 'cq239h0', 'created_utc': 1428255831.0... [https://reddit.com/r/rational/comments/58yj78... 2015-03-11 18:21:58 2023-06-26 19:21:24 [https://wertifloke.wordpress.com/2015/01/25/c...
Harry Potter and the Natural 20 381 43 43 [{'id': 'cic1ya4', 'created_utc': 1403221134.0... [https://reddit.com/r/rational/comments/cmc4a0... 2014-06-19 23:38:54 2024-04-10 19:29:55 [https://www.fanfiction.net/s/8096183/1/Harry-...
Sanitize 363 32 32 [{'id': 'ee403jp', 'created_utc': 1547546568.0... [https://reddit.com/r/rational/comments/15qvj9... 2019-01-15 10:02:48 2024-01-08 15:09:35 [https://www.fanfiction.net/s/12431866/1/Sanit...
Delve 348 29 29 [{'id': 'esy81sr', 'created_utc': 1562340199.0... [https://reddit.com/r/rational/comments/e509i9... 2019-07-05 15:23:19 2024-07-29 16:14:14 [https://www.royalroad.com/fiction/25225/delve...
Luminosity 348 38 38 [{'id': 'cic1ya4', 'created_utc': 1403221134.0... [https://reddit.com/r/rational/comments/9x0h7s... 2014-01-26 03:38:42 2022-05-31 03:05:24 [http://luminous.elcenia.com/, https://luminou...
The Erogamer 338 29 29 [{'id': 'dk9l7ar', 'created_utc': 1500149750.0... [https://reddit.com/r/rational/comments/b83kq9... 2017-07-15 20:15:50 2023-05-09 03:02:19 [https://forum.questionablequesting.com/thread...
Friendship is Optimal 334 40 40 [{'id': 'dzxaxnb', 'created_utc': 1527829743.0... [https://reddit.com/r/rational/comments/1ew2jk... 2014-08-29 06:11:17 2024-08-24 17:51:05 [https://www.fimfiction.net/story/62074/friend...
Harry Potter and the Methods of Rationality 331 43 43 [{'id': 'dd5dtb2', 'created_utc': 1485881128.0... [https://reddit.com/r/rational/comments/7musm6... 2014-06-19 23:38:54 2021-01-03 22:07:47 [http://www.hpmor.com/, http://hpmor.com/, htt...
Pokemon: The Origin of Species 325 30 30 [{'id': 'cmvy8xg', 'created_utc': 1418683443.0... [https://reddit.com/r/rational/comments/jm10k2... 2014-12-15 22:44:03 2023-10-30 21:45:35 [https://www.fanfiction.net/s/9794740/1/Pokemo...
Marked for Death 323 51 51 [{'id': 'cysdi0k', 'created_utc': 1452387655.0... [https://reddit.com/r/rational/comments/jyz11x... 2016-01-10 01:00:55 2023-11-30 08:25:55 [https://forums.sufficientvelocity.com/threads...
Purple Days 314 27 27 [{'id': 'dx1fvrp', 'created_utc': 1523228008.0... [https://reddit.com/r/rational/comments/k4vnrv... 2018-02-06 10:57:58 2023-09-24 06:14:16 [https://forums.spacebattles.com/threads/purpl...
Pale 312 35 35 [{'id': 'fqa9o7d', 'score': 28, 'body': 'Wildb... [https://reddit.com/r/rational/comments/j9sin3... 2020-05-05 19:53:47 2023-10-11 20:29:15 [https://palewebserial.wordpress.com/about/, h...
Branches on the Tree of Time 307 30 30 [{'id': 'ce7klze', 'created_utc': 1387693738.0... [https://reddit.com/r/rational/comments/9imlvr... 2013-12-22 06:28:58 2024-10-18 01:56:10 [https://www.fanfiction.net/s/9658524/1/Branch...
A Hero's War 302 39 39 [{'body': 'Sounds like you might enjoy [A Hero... [https://reddit.com/r/rational/comments/bqwp8b... 2016-03-26 22:29:47 2024-11-10 23:33:40 [https://www.fictionpress.com/s/3238329/1/A-He...
Lord of the Mysteries 297 35 35 [{'id': 'f5zc9t3', 'created_utc': 1572572998.0... [https://reddit.com/r/rational/comments/e1h8gc... 2019-06-26 02:54:44 2024-09-23 18:09:53 [https://www.wuxiaworld.co/Lord-of-the-Mysteri...
Alexander Wales Wiki | Fandom 296 9 9 [{'id': 'ea1e93v', 'created_utc': 1542641952.0... [https://reddit.com/r/rational/comments/al7z2v... 2018-11-19 15:39:12 2020-12-12 04:17:44 [https://worththecandle.wikia.com/wiki/Worth_t...
In [60]:
print(f"{len(df_links3)} -> {len(df3)} after title dedup")
17958 -> 15271 after title dedup

Export to html

In [61]:
from rrational.transform import join_uniq, chain_lists, format_flair, c2md, collapsibe, urls2a,url2a,  unique_elements
In [62]:
# First transform the fields for display
d = df3.reset_index().sort_values("score", ascending=False)



# make title have a link to first url
d['title'] = d.progress_apply(lambda x: f'<a href="{x["url"][0]}">{x["title"]}</a>', axis=1)

d["url"] = d["url"].apply(lambda x: collapsibe('...', urls2a(x)))
d["score"] = d["score"].round(2)

prefix = 'https://reddit.com/r/rational/comments/'
d["comment_urls"] = d['comments'].apply(lambda x: collapsibe("...", urls2a([prefix+y['permalink'] for y in x])))
d["thread_urls"] = d['thread_urls'].apply(lambda x: collapsibe("...", urls2a(x)))
# d['comments'] = d['comments'].progress_apply(lambda x: collapsibe("comments", "<br>".join([c2md(c) for c in x])))


d['first_link_utc'] = d['first_link_utc'].dt.strftime('%Y-%m-%d')
d['last_link_utc'] = d['last_link_utc'].dt.strftime('%Y-%m-%d')

  0%|          | 0/15271 [00:00<?, ?it/s]
In [63]:

d = d.drop(columns=['comments']).rename(columns={
    'first_link_utc': 'first_link',
    'last_link_utc': 'last_link',
    'n_comments': '#comment',
    'n_links': '#link',
    'thread_urls': 'threads',
    'comment_urls': 'comments',
    'url': 'urls',

})
# order
d = d[['title', 'score', '#link', '#comment', 'first_link', 'last_link', 'urls', 'threads', 'comments']]
d
Out [63]:
title score #link #comment first_link last_link urls threads comments
0 <a href="https://archiveofourown.org/works/114... 1460 110 110 2017-07-28 2024-04-17 <details><summary>...</summary>\n<ul><li><a hr... <details><summary>...</summary>\n<ul><li><a hr... <details><summary>...</summary>\n<ul><li><a hr...
1 <a href="https://www.patreon.com/alexanderwale... 1402 22 22 2015-04-18 2021-04-29 <details><summary>...</summary>\n<ul><li><a hr... <details><summary>...</summary>\n<ul><li><a hr... <details><summary>...</summary>\n<ul><li><a hr...
2 <a href="https://www.fictionpress.com/s/296189... 1063 101 101 2014-07-26 2024-05-21 <details><summary>...</summary>\n<ul><li><a hr... <details><summary>...</summary>\n<ul><li><a hr... <details><summary>...</summary>\n<ul><li><a hr...
3 <a href="https://practicalguidetoevil.wordpres... 1006 92 92 2016-04-05 2024-08-09 <details><summary>...</summary>\n<ul><li><a hr... <details><summary>...</summary>\n<ul><li><a hr... <details><summary>...</summary>\n<ul><li><a hr...
4 <a href="https://parahumans.wordpress.com/">Wo... 916 83 83 2014-01-27 2024-11-22 <details><summary>...</summary>\n<ul><li><a hr... <details><summary>...</summary>\n<ul><li><a hr... <details><summary>...</summary>\n<ul><li><a hr...
... ... ... ... ... ... ... ... ... ...
15266 <a href="https://www.merriam-webster.com/dicti... -7 1 1 2020-12-10 2020-12-10 <details><summary>...</summary>\n<ul><li><a hr... <details><summary>...</summary>\n<ul><li><a hr... <details><summary>...</summary>\n<ul><li><a hr...
15267 <a href="https://www.amazon.com/Steelheart-Rec... -10 1 1 2017-11-13 2017-11-13 <details><summary>...</summary>\n<ul><li><a hr... <details><summary>...</summary>\n<ul><li><a hr... <details><summary>...</summary>\n<ul><li><a hr...
15268 <a href="https://wiki.lesswrong.com/wiki/Ortho... -10 2 2 2015-12-04 2016-07-11 <details><summary>...</summary>\n<ul><li><a hr... <details><summary>...</summary>\n<ul><li><a hr... <details><summary>...</summary>\n<ul><li><a hr...
15269 <a href="http://www.imdb.com/title/tt0317705/?... -10 1 1 2017-11-13 2017-11-13 <details><summary>...</summary>\n<ul><li><a hr... <details><summary>...</summary>\n<ul><li><a hr... <details><summary>...</summary>\n<ul><li><a hr...
15270 <a href="http://www.imdb.com/title/tt0405325/?... -10 1 1 2017-11-13 2017-11-13 <details><summary>...</summary>\n<ul><li><a hr... <details><summary>...</summary>\n<ul><li><a hr... <details><summary>...</summary>\n<ul><li><a hr...

15271 rows × 9 columns

In [64]:
from rrational.export import export_df_2_html
html_out = Path("../index2.html").resolve()
export_df_2_html(df=d, output=html_out,
                 hidden_columns=["comments", "first_link", "last_link"],
)
View the page /media/wassname/SGIronWolf/projects5/rrational/index2.html
In [65]:
from IPython.display import HTML, display

htmla = f'<a href="{html_out}">View the page {html_out}</a>'
display(HTML(htmla))
View the page /media/wassname/SGIronWolf/projects5/rrational/index2.html
In [66]:
df3.to_parquet("../outputs/df_links4.parquet")
In [ ]:
Reference in New Issue View Git Blame Copy Permalink