mirror of
https://github.com/wassname/scrape_r_rational.git
synced 2026-09-10 12:35:53 +08:00
better table
This commit is contained in:
wassname
+107
-26
File diff suppressed because one or more lines are too long
+107
-26
@@ -1,46 +1,127 @@
|
||||
<!doctype html>
|
||||
<html lang=en>
|
||||
|
||||
<head>
|
||||
<meta charset=utf-8>
|
||||
<title>scraping reddit.com/r/rational recs</title>
|
||||
<link href="https://cdn.datatables.net/v/dt/jq-3.7.0/dt-2.1.8/b-3.2.0/b-colvis-3.2.0/sc-2.4.3/sr-1.4.1/datatables.min.css" rel="stylesheet">
|
||||
<meta charset=utf-8>
|
||||
<title>scraping reddit.com/r/rational recs</title>
|
||||
<!-- <link href="https://cdn.datatables.net/v/dt/jq-3.7.0/dt-2.1.8/b-3.2.0/b-colvis-3.2.0/sc-2.4.3/sr-1.4.1/datatables.min.css" rel="stylesheet">
|
||||
|
||||
<script src="https://cdn.datatables.net/v/dt/jq-3.7.0/dt-2.1.8/b-3.2.0/b-colvis-3.2.0/sc-2.4.3/sr-1.4.1/datatables.min.js"></script>
|
||||
<script src="https://cdn.datatables.net/v/dt/jq-3.7.0/dt-2.1.8/b-3.2.0/b-colvis-3.2.0/sc-2.4.3/sr-1.4.1/datatables.min.js"></script> -->
|
||||
|
||||
<!--
|
||||
<script src="https://code.jquery.com/jquery-3.7.1.js"></script>
|
||||
<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/5.3.0/js/bootstrap.bundle.min.js"></script>
|
||||
<script src="https://cdn.datatables.net/2.1.8/js/dataTables.js"></script>
|
||||
<script src="https://cdn.datatables.net/2.1.8/js/dataTables.bootstrap5.js"></script>
|
||||
<script src="https://cdn.datatables.net/buttons/3.2.0/js/dataTables.buttons.js"></script>
|
||||
<script src="https://cdn.datatables.net/buttons/3.2.0/js/buttons.bootstrap5.js"></script>
|
||||
<script src="https://cdnjs.cloudflare.com/ajax/libs/jszip/3.10.1/jszip.min.js"></script>
|
||||
<script src="https://cdn.datatables.net/buttons/3.2.0/js/buttons.html5.min.js"></script>
|
||||
|
||||
|
||||
|
||||
<link href="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/5.3.0/css/bootstrap.min.css" rel="stylesheet"/>
|
||||
<link href="https://cdn.datatables.net/2.1.8/css/dataTables.bootstrap5.css" rel="stylesheet"/>
|
||||
<link href="https://cdn.datatables.net/buttons/3.2.0/css/buttons.bootstrap5.css" rel="stylesheet"/> -->
|
||||
|
||||
|
||||
<link href="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/5.3.0/css/bootstrap.min.css" rel="stylesheet">
|
||||
<link href="https://cdn.datatables.net/2.1.8/css/dataTables.bootstrap5.min.css" rel="stylesheet">
|
||||
<link href="https://cdn.datatables.net/buttons/3.2.0/css/buttons.bootstrap5.min.css" rel="stylesheet">
|
||||
<link href="https://cdn.datatables.net/responsive/3.0.3/css/responsive.bootstrap5.min.css" rel="stylesheet">
|
||||
<link href="https://cdn.datatables.net/scroller/2.4.3/css/scroller.bootstrap5.min.css" rel="stylesheet">
|
||||
<link href="https://cdn.datatables.net/searchbuilder/1.8.1/css/searchBuilder.bootstrap5.min.css" rel="stylesheet">
|
||||
<link href="https://cdn.datatables.net/searchpanes/2.3.3/css/searchPanes.bootstrap5.min.css" rel="stylesheet">
|
||||
<link href="https://cdn.datatables.net/select/2.1.0/css/select.bootstrap5.min.css" rel="stylesheet">
|
||||
<link href="https://cdn.datatables.net/staterestore/1.4.1/css/stateRestore.bootstrap5.min.css" rel="stylesheet">
|
||||
|
||||
<script src="https://code.jquery.com/jquery-3.7.0.min.js"></script>
|
||||
<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/5.3.0/js/bootstrap.bundle.min.js"></script>
|
||||
<script src="https://cdn.datatables.net/2.1.8/js/dataTables.min.js"></script>
|
||||
<script src="https://cdn.datatables.net/2.1.8/js/dataTables.bootstrap5.min.js"></script>
|
||||
<script src="https://cdn.datatables.net/buttons/3.2.0/js/dataTables.buttons.min.js"></script>
|
||||
<script src="https://cdn.datatables.net/buttons/3.2.0/js/buttons.bootstrap5.min.js"></script>
|
||||
<script src="https://cdn.datatables.net/buttons/3.2.0/js/buttons.colVis.min.js"></script>
|
||||
<script src="https://cdn.datatables.net/buttons/3.2.0/js/buttons.html5.min.js"></script>
|
||||
<script src="https://cdn.datatables.net/responsive/3.0.3/js/dataTables.responsive.min.js"></script>
|
||||
<script src="https://cdn.datatables.net/responsive/3.0.3/js/responsive.bootstrap5.js"></script>
|
||||
<script src="https://cdn.datatables.net/scroller/2.4.3/js/dataTables.scroller.min.js"></script>
|
||||
<script src="https://cdn.datatables.net/searchbuilder/1.8.1/js/dataTables.searchBuilder.min.js"></script>
|
||||
<script src="https://cdn.datatables.net/searchbuilder/1.8.1/js/searchBuilder.bootstrap5.min.js"></script>
|
||||
<script src="https://cdn.datatables.net/searchpanes/2.3.3/js/dataTables.searchPanes.min.js"></script>
|
||||
<script src="https://cdn.datatables.net/searchpanes/2.3.3/js/searchPanes.bootstrap5.min.js"></script>
|
||||
<script src="https://cdn.datatables.net/select/2.1.0/js/dataTables.select.min.js"></script>
|
||||
<script src="https://cdn.datatables.net/staterestore/1.4.1/js/dataTables.stateRestore.min.js"></script>
|
||||
<script src="https://cdn.datatables.net/staterestore/1.4.1/js/stateRestore.bootstrap5.min.js"></script>
|
||||
</head>
|
||||
|
||||
<body>
|
||||
<p>scraping <a href="https://old.reddit.com/r/rational/">reddit.com/r/rational/</a></p>
|
||||
<a href="https://github.com/wassname/scrape_r_rational">source code</a>
|
||||
<table id="myTable" class="cell-border compact stripe"></table>
|
||||
<p>scraping <a href="https://old.reddit.com/r/rational/">reddit.com/r/rational/</a></p>
|
||||
<a href="https://github.com/wassname/scrape_r_rational">source code</a>
|
||||
<div class="tabl">
|
||||
|
||||
<table id="myTable" class="table table-sm table-striped table-bordered nowrap" style="width:100%"></table>
|
||||
</div>
|
||||
|
||||
</body>
|
||||
<script>
|
||||
let data = {{data}}
|
||||
let data = {{ data }}
|
||||
</script>
|
||||
<script>
|
||||
table = new DataTable('#myTable', {
|
||||
data: data,
|
||||
columns: {{columns}},
|
||||
order: [[1, 'score']],
|
||||
|
||||
pageLength: 50,
|
||||
responsive: true,
|
||||
columnDefs: [
|
||||
var table = new DataTable('#myTable', {
|
||||
responsive: true,
|
||||
{# fixedColumns: false, #}
|
||||
data: data,
|
||||
columns: {{ columns }},
|
||||
order: [[1, 'score']],
|
||||
lengthMenu: [
|
||||
[10, 25, 50, -1],
|
||||
[10, 25, 50, 'All']
|
||||
],
|
||||
// pageLength: 50,
|
||||
columnDefs: [
|
||||
{
|
||||
targets: 1,
|
||||
className: 'noVis'
|
||||
}
|
||||
],
|
||||
layout: {
|
||||
topStart: {
|
||||
buttons: [
|
||||
{
|
||||
extend: 'colvis',
|
||||
columns: ':not(.noVis)',
|
||||
popoverTitle: 'Column visibility selector'
|
||||
}
|
||||
]
|
||||
}
|
||||
layout: {
|
||||
topStart: {
|
||||
buttons: [
|
||||
{
|
||||
extend: 'colvis',
|
||||
columns: ':not(.noVis)',
|
||||
popoverTitle: 'Column visibility selector'
|
||||
}
|
||||
]
|
||||
},
|
||||
},
|
||||
initComplete: function () {
|
||||
this.api()
|
||||
.columns()
|
||||
.every(function () {
|
||||
let column = this;
|
||||
let title = column.footer().textContent;
|
||||
|
||||
// Create input element
|
||||
let input = document.createElement('input');
|
||||
input.className = 'form-control';
|
||||
input.placeholder = title;
|
||||
column.footer().replaceChildren(input);
|
||||
|
||||
// Event listener for user input
|
||||
input.addEventListener('keyup', () => {
|
||||
if (column.search() !== this.value) {
|
||||
column.search(input.value).draw();
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
});
|
||||
// // Add event listeners to the two range filtering inputs
|
||||
$('#min').keyup( function() { table.draw(); } );
|
||||
$('#max').keyup( function() { table.draw(); } );
|
||||
|
||||
</script>
|
||||
|
||||
</html>
|
||||
|
||||
+127
File diff suppressed because one or more lines are too long
@@ -0,0 +1,71 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"vscode": {
|
||||
"languageId": "plaintext"
|
||||
}
|
||||
},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"vscode": {
|
||||
"languageId": "plaintext"
|
||||
}
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Define tools\n",
|
||||
"tools = [\n",
|
||||
" {\n",
|
||||
" \"type\": \"function\",\n",
|
||||
" \"function\": {\n",
|
||||
" \"name\": \"query_daily_sales_report\",\n",
|
||||
" \"description\": \"Connects to a database to retrieve overall sales volumes and sales information for a given day.\",\n",
|
||||
" \"parameters\": {\n",
|
||||
" \"type\": \"object\",\n",
|
||||
" \"properties\": {\n",
|
||||
" \"day\": {\n",
|
||||
" \"description\": \"Retrieves sales data for this day, formatted as YYYY-MM-DD.\",\n",
|
||||
" \"type\": \"string\",\n",
|
||||
" }\n",
|
||||
" },\n",
|
||||
" \"required\": [\"day\"]\n",
|
||||
" },\n",
|
||||
" }\n",
|
||||
" }\n",
|
||||
"]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"\n",
|
||||
"# Define conversation input\n",
|
||||
"conversation = [{\"role\": \"user\", \"content\": \"Can you provide a sales summary for 29th September 2023?\"}]\n",
|
||||
"\n",
|
||||
"# Get the Tool Use prompt\n",
|
||||
"input_prompt = tokenizer.apply_chat_template(conversation=conversation, tools=tools, tokenize=False, add_generation_prompt=True, return_tensors=\"pt\")\n",
|
||||
"\n",
|
||||
"# Tokenize the prompt\n",
|
||||
"input_ids = tokenizer.encode_plus(input_prompt, return_tensors=\"pt\")\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"language_info": {
|
||||
"name": "python"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
+281
-273
File diff suppressed because it is too large
Load Diff
+507
-82
@@ -87,7 +87,7 @@
|
||||
" <td>110</td>\n",
|
||||
" <td>110</td>\n",
|
||||
" <td>[{'author_flair_text': 'Self-Appointed Court S...</td>\n",
|
||||
" <td>[https://reddit.com/r/rational/comments/f1rj3l...</td>\n",
|
||||
" <td>[https://reddit.com/r/rational/comments/7228po...</td>\n",
|
||||
" <td>2017-07-28 13:17:36</td>\n",
|
||||
" <td>2024-04-17 19:03:01</td>\n",
|
||||
" <td>[https://archiveofourown.org/works/11478249/ch...</td>\n",
|
||||
@@ -98,18 +98,29 @@
|
||||
" <td>22</td>\n",
|
||||
" <td>22</td>\n",
|
||||
" <td>[{'author_flair_text': 'Time flies like an arr...</td>\n",
|
||||
" <td>[https://reddit.com/r/rational/comments/8mqrd0...</td>\n",
|
||||
" <td>[https://reddit.com/r/rational/comments/galt0r...</td>\n",
|
||||
" <td>2015-04-18 18:13:47</td>\n",
|
||||
" <td>2021-04-29 20:00:04</td>\n",
|
||||
" <td>[https://www.patreon.com/alexanderwales]</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>Mother of Learning</th>\n",
|
||||
" <td>1063</td>\n",
|
||||
" <td>101</td>\n",
|
||||
" <td>101</td>\n",
|
||||
" <td>[{'author_flair_text': 'Utopian Smut Peddler',...</td>\n",
|
||||
" <td>[https://reddit.com/r/rational/comments/k8i7jg...</td>\n",
|
||||
" <td>2014-07-26 08:21:12</td>\n",
|
||||
" <td>2024-05-21 07:04:42</td>\n",
|
||||
" <td>[https://www.fictionpress.com/s/2961893/1/Moth...</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>A Practical Guide to Evil</th>\n",
|
||||
" <td>1034</td>\n",
|
||||
" <td>94</td>\n",
|
||||
" <td>94</td>\n",
|
||||
" <td>1025</td>\n",
|
||||
" <td>92</td>\n",
|
||||
" <td>92</td>\n",
|
||||
" <td>[{'author_flair_text': 'Ankh-Morpork City Watc...</td>\n",
|
||||
" <td>[https://reddit.com/r/rational/comments/dutkzh...</td>\n",
|
||||
" <td>[https://reddit.com/r/rational/comments/ako900...</td>\n",
|
||||
" <td>2016-04-05 18:53:21</td>\n",
|
||||
" <td>2024-08-09 10:10:21</td>\n",
|
||||
" <td>[https://practicalguidetoevil.wordpress.com/, ...</td>\n",
|
||||
@@ -120,23 +131,12 @@
|
||||
" <td>83</td>\n",
|
||||
" <td>83</td>\n",
|
||||
" <td>[{'author_flair_text': 'Emergency Mustelid Hol...</td>\n",
|
||||
" <td>[https://reddit.com/r/rational/comments/f1rj3l...</td>\n",
|
||||
" <td>[https://reddit.com/r/rational/comments/k8i7jg...</td>\n",
|
||||
" <td>2014-01-27 02:09:42</td>\n",
|
||||
" <td>2024-11-22 21:39:11</td>\n",
|
||||
" <td>[https://parahumans.wordpress.com/, https://pa...</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>Mother of Learning</th>\n",
|
||||
" <td>880</td>\n",
|
||||
" <td>93</td>\n",
|
||||
" <td>93</td>\n",
|
||||
" <td>[{'author_flair_text': 'Utopian Smut Peddler',...</td>\n",
|
||||
" <td>[https://reddit.com/r/rational/comments/dutkzh...</td>\n",
|
||||
" <td>2014-07-26 08:21:12</td>\n",
|
||||
" <td>2024-05-21 07:04:42</td>\n",
|
||||
" <td>[https://www.fictionpress.com/s/2961893/1/Moth...</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>...</th>\n",
|
||||
" <td>...</td>\n",
|
||||
" <td>...</td>\n",
|
||||
@@ -148,26 +148,26 @@
|
||||
" <td>...</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>Heartbreaker</th>\n",
|
||||
" <th>The Incredibles</th>\n",
|
||||
" <td>-6</td>\n",
|
||||
" <td>1</td>\n",
|
||||
" <td>1</td>\n",
|
||||
" <td>[{'author_flair_text': 'https://i.imgur.com/OQ...</td>\n",
|
||||
" <td>[https://reddit.com/r/rational/comments/48akta...</td>\n",
|
||||
" <td>2016-02-29 20:57:08</td>\n",
|
||||
" <td>2016-02-29 20:57:08</td>\n",
|
||||
" <td>[https://parahumans.wordpress.com/2012/01/17/b...</td>\n",
|
||||
" <td>[{'author_flair_text': None, 'body': 'I've act...</td>\n",
|
||||
" <td>[https://reddit.com/r/rational/comments/7cneg5...</td>\n",
|
||||
" <td>2017-11-13 20:58:01</td>\n",
|
||||
" <td>2017-11-13 20:58:01</td>\n",
|
||||
" <td>[http://www.imdb.com/title/tt0317705/?ref_=fn_...</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>picking RPG clothes based on maxing stats instead of whether they match or not</th>\n",
|
||||
" <th>Steelheart (The Reckoners): 9780385743570: Sanderson, Brandon: Books</th>\n",
|
||||
" <td>-6</td>\n",
|
||||
" <td>2</td>\n",
|
||||
" <td>2</td>\n",
|
||||
" <td>[{'author_flair_text': None, 'body': 'It seems...</td>\n",
|
||||
" <td>[https://reddit.com/r/rational/comments/9h1454...</td>\n",
|
||||
" <td>2018-09-19 02:26:27</td>\n",
|
||||
" <td>2018-09-19 04:51:09</td>\n",
|
||||
" <td>[https://youtu.be/xq1tN9jZI80]</td>\n",
|
||||
" <td>1</td>\n",
|
||||
" <td>1</td>\n",
|
||||
" <td>[{'author_flair_text': None, 'body': 'I've act...</td>\n",
|
||||
" <td>[https://reddit.com/r/rational/comments/7cneg5...</td>\n",
|
||||
" <td>2017-11-13 20:58:01</td>\n",
|
||||
" <td>2017-11-13 20:58:01</td>\n",
|
||||
" <td>[https://www.amazon.com/Steelheart-Reckoners-B...</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>Orthogonality thesis</th>\n",
|
||||
@@ -204,7 +204,7 @@
|
||||
" </tr>\n",
|
||||
" </tbody>\n",
|
||||
"</table>\n",
|
||||
"<p>15209 rows × 8 columns</p>\n",
|
||||
"<p>15215 rows × 8 columns</p>\n",
|
||||
"</div>"
|
||||
],
|
||||
"text/plain": [
|
||||
@@ -212,12 +212,12 @@
|
||||
"title \n",
|
||||
"Worth the Candle 1453 110 \n",
|
||||
"Alexander Wales - The Metropolitan Man, Shadows... 1408 22 \n",
|
||||
"A Practical Guide to Evil 1034 94 \n",
|
||||
"Mother of Learning 1063 101 \n",
|
||||
"A Practical Guide to Evil 1025 92 \n",
|
||||
"Worm 925 83 \n",
|
||||
"Mother of Learning 880 93 \n",
|
||||
"... ... ... \n",
|
||||
"Heartbreaker -6 1 \n",
|
||||
"picking RPG clothes based on maxing stats inste... -6 2 \n",
|
||||
"The Incredibles -6 1 \n",
|
||||
"Steelheart (The Reckoners): 9780385743570: Sand... -6 1 \n",
|
||||
"Orthogonality thesis -8 2 \n",
|
||||
"When starting school, younger children are more... -9 1 \n",
|
||||
"Is There A Replicability Crisis In Psychology? ... -9 1 \n",
|
||||
@@ -226,12 +226,12 @@
|
||||
"title \n",
|
||||
"Worth the Candle 110 \n",
|
||||
"Alexander Wales - The Metropolitan Man, Shadows... 22 \n",
|
||||
"A Practical Guide to Evil 94 \n",
|
||||
"Mother of Learning 101 \n",
|
||||
"A Practical Guide to Evil 92 \n",
|
||||
"Worm 83 \n",
|
||||
"Mother of Learning 93 \n",
|
||||
"... ... \n",
|
||||
"Heartbreaker 1 \n",
|
||||
"picking RPG clothes based on maxing stats inste... 2 \n",
|
||||
"The Incredibles 1 \n",
|
||||
"Steelheart (The Reckoners): 9780385743570: Sand... 1 \n",
|
||||
"Orthogonality thesis 2 \n",
|
||||
"When starting school, younger children are more... 1 \n",
|
||||
"Is There A Replicability Crisis In Psychology? ... 1 \n",
|
||||
@@ -240,26 +240,26 @@
|
||||
"title \n",
|
||||
"Worth the Candle [{'author_flair_text': 'Self-Appointed Court S... \n",
|
||||
"Alexander Wales - The Metropolitan Man, Shadows... [{'author_flair_text': 'Time flies like an arr... \n",
|
||||
"Mother of Learning [{'author_flair_text': 'Utopian Smut Peddler',... \n",
|
||||
"A Practical Guide to Evil [{'author_flair_text': 'Ankh-Morpork City Watc... \n",
|
||||
"Worm [{'author_flair_text': 'Emergency Mustelid Hol... \n",
|
||||
"Mother of Learning [{'author_flair_text': 'Utopian Smut Peddler',... \n",
|
||||
"... ... \n",
|
||||
"Heartbreaker [{'author_flair_text': 'https://i.imgur.com/OQ... \n",
|
||||
"picking RPG clothes based on maxing stats inste... [{'author_flair_text': None, 'body': 'It seems... \n",
|
||||
"The Incredibles [{'author_flair_text': None, 'body': 'I've act... \n",
|
||||
"Steelheart (The Reckoners): 9780385743570: Sand... [{'author_flair_text': None, 'body': 'I've act... \n",
|
||||
"Orthogonality thesis [{'author_flair_text': None, 'body': 'I don't ... \n",
|
||||
"When starting school, younger children are more... [{'author_flair_text': 'Ankh-Morpork City Watc... \n",
|
||||
"Is There A Replicability Crisis In Psychology? ... [{'author_flair_text': 'Ankh-Morpork City Watc... \n",
|
||||
"\n",
|
||||
" thread_urls \\\n",
|
||||
"title \n",
|
||||
"Worth the Candle [https://reddit.com/r/rational/comments/f1rj3l... \n",
|
||||
"Alexander Wales - The Metropolitan Man, Shadows... [https://reddit.com/r/rational/comments/8mqrd0... \n",
|
||||
"A Practical Guide to Evil [https://reddit.com/r/rational/comments/dutkzh... \n",
|
||||
"Worm [https://reddit.com/r/rational/comments/f1rj3l... \n",
|
||||
"Mother of Learning [https://reddit.com/r/rational/comments/dutkzh... \n",
|
||||
"Worth the Candle [https://reddit.com/r/rational/comments/7228po... \n",
|
||||
"Alexander Wales - The Metropolitan Man, Shadows... [https://reddit.com/r/rational/comments/galt0r... \n",
|
||||
"Mother of Learning [https://reddit.com/r/rational/comments/k8i7jg... \n",
|
||||
"A Practical Guide to Evil [https://reddit.com/r/rational/comments/ako900... \n",
|
||||
"Worm [https://reddit.com/r/rational/comments/k8i7jg... \n",
|
||||
"... ... \n",
|
||||
"Heartbreaker [https://reddit.com/r/rational/comments/48akta... \n",
|
||||
"picking RPG clothes based on maxing stats inste... [https://reddit.com/r/rational/comments/9h1454... \n",
|
||||
"The Incredibles [https://reddit.com/r/rational/comments/7cneg5... \n",
|
||||
"Steelheart (The Reckoners): 9780385743570: Sand... [https://reddit.com/r/rational/comments/7cneg5... \n",
|
||||
"Orthogonality thesis [https://reddit.com/r/rational/comments/3vc0si... \n",
|
||||
"When starting school, younger children are more... [https://reddit.com/r/rational/comments/a1t6um... \n",
|
||||
"Is There A Replicability Crisis In Psychology? ... [https://reddit.com/r/rational/comments/a1t6um... \n",
|
||||
@@ -268,12 +268,12 @@
|
||||
"title \n",
|
||||
"Worth the Candle 2017-07-28 13:17:36 \n",
|
||||
"Alexander Wales - The Metropolitan Man, Shadows... 2015-04-18 18:13:47 \n",
|
||||
"Mother of Learning 2014-07-26 08:21:12 \n",
|
||||
"A Practical Guide to Evil 2016-04-05 18:53:21 \n",
|
||||
"Worm 2014-01-27 02:09:42 \n",
|
||||
"Mother of Learning 2014-07-26 08:21:12 \n",
|
||||
"... ... \n",
|
||||
"Heartbreaker 2016-02-29 20:57:08 \n",
|
||||
"picking RPG clothes based on maxing stats inste... 2018-09-19 02:26:27 \n",
|
||||
"The Incredibles 2017-11-13 20:58:01 \n",
|
||||
"Steelheart (The Reckoners): 9780385743570: Sand... 2017-11-13 20:58:01 \n",
|
||||
"Orthogonality thesis 2015-12-04 18:17:07 \n",
|
||||
"When starting school, younger children are more... 2018-11-30 17:17:49 \n",
|
||||
"Is There A Replicability Crisis In Psychology? ... 2018-11-30 17:17:49 \n",
|
||||
@@ -282,12 +282,12 @@
|
||||
"title \n",
|
||||
"Worth the Candle 2024-04-17 19:03:01 \n",
|
||||
"Alexander Wales - The Metropolitan Man, Shadows... 2021-04-29 20:00:04 \n",
|
||||
"Mother of Learning 2024-05-21 07:04:42 \n",
|
||||
"A Practical Guide to Evil 2024-08-09 10:10:21 \n",
|
||||
"Worm 2024-11-22 21:39:11 \n",
|
||||
"Mother of Learning 2024-05-21 07:04:42 \n",
|
||||
"... ... \n",
|
||||
"Heartbreaker 2016-02-29 20:57:08 \n",
|
||||
"picking RPG clothes based on maxing stats inste... 2018-09-19 04:51:09 \n",
|
||||
"The Incredibles 2017-11-13 20:58:01 \n",
|
||||
"Steelheart (The Reckoners): 9780385743570: Sand... 2017-11-13 20:58:01 \n",
|
||||
"Orthogonality thesis 2016-07-11 16:27:30 \n",
|
||||
"When starting school, younger children are more... 2018-11-30 17:17:49 \n",
|
||||
"Is There A Replicability Crisis In Psychology? ... 2018-11-30 17:17:49 \n",
|
||||
@@ -296,17 +296,17 @@
|
||||
"title \n",
|
||||
"Worth the Candle [https://archiveofourown.org/works/11478249/ch... \n",
|
||||
"Alexander Wales - The Metropolitan Man, Shadows... [https://www.patreon.com/alexanderwales] \n",
|
||||
"Mother of Learning [https://www.fictionpress.com/s/2961893/1/Moth... \n",
|
||||
"A Practical Guide to Evil [https://practicalguidetoevil.wordpress.com/, ... \n",
|
||||
"Worm [https://parahumans.wordpress.com/, https://pa... \n",
|
||||
"Mother of Learning [https://www.fictionpress.com/s/2961893/1/Moth... \n",
|
||||
"... ... \n",
|
||||
"Heartbreaker [https://parahumans.wordpress.com/2012/01/17/b... \n",
|
||||
"picking RPG clothes based on maxing stats inste... [https://youtu.be/xq1tN9jZI80] \n",
|
||||
"The Incredibles [http://www.imdb.com/title/tt0317705/?ref_=fn_... \n",
|
||||
"Steelheart (The Reckoners): 9780385743570: Sand... [https://www.amazon.com/Steelheart-Reckoners-B... \n",
|
||||
"Orthogonality thesis [https://wiki.lesswrong.com/wiki/Orthogonality... \n",
|
||||
"When starting school, younger children are more... [https://news.harvard.edu/gazette/story/2018/1... \n",
|
||||
"Is There A Replicability Crisis In Psychology? ... [https://sciencetrends.com/is-there-a-replicab... \n",
|
||||
"\n",
|
||||
"[15209 rows x 8 columns]"
|
||||
"[15215 rows x 8 columns]"
|
||||
]
|
||||
},
|
||||
"execution_count": 3,
|
||||
@@ -405,7 +405,7 @@
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "85d0527e3ee04b0bafd1b11783b9ec6a",
|
||||
"model_id": "c944ef2362ac4c7cb909f0b00893dc05",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
@@ -604,7 +604,9 @@
|
||||
" rating_writing: float\n",
|
||||
" rating_plot: float\n",
|
||||
" rating_character: float\n",
|
||||
" rating_worldbuilding: float"
|
||||
" rating_worldbuilding: float\n",
|
||||
"\n",
|
||||
" # TODO I want to know, how long is it, is it complete, is it a fiction, article, comic, etc."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -652,7 +654,7 @@
|
||||
" )\n",
|
||||
"\n",
|
||||
" tokens = chat_completion.usage.prompt_tokens\n",
|
||||
" print(f\"cost: {tokens * cost:.2f} USD, tokens: {tokens}\")\n",
|
||||
" # print(f\"cost: {tokens * cost:.6f} USD, tokens: {tokens}\")\n",
|
||||
" return tokens, chat_completion.choices[0].message.parsed.__dict__"
|
||||
]
|
||||
},
|
||||
@@ -683,54 +685,120 @@
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# import shutil\n",
|
||||
"# shutil.rmtree(output_dir, ignore_errors=True)"
|
||||
"# shutil.rmtree(output_dir, ignore_errors=True)\n",
|
||||
"from pydantic import ValidationError"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 160,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "54ac8b461182444791881e6a27fd47f5",
|
||||
"model_id": "0cc71434ae134dbaaef1ffa718b435c0",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
" 0%| | 0/15209 [00:00<?, ?it/s]"
|
||||
" 0%| | 0/15215 [00:00<?, ?it/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Input Tokens: 30995. Input Cost: 0.002325 USD, for title=`Now You Feel Like Number None`\n",
|
||||
"Projected cost for all threads: 17.684585 USD\n",
|
||||
"Content: \n",
|
||||
"\n",
|
||||
"----- Thread 10262 -----\n",
|
||||
"\n",
|
||||
"...ing and stories to be far worse and no longer follow them. \n",
|
||||
"> \n",
|
||||
"> You may also enjoy **Uber** and **Uber: Invasion**, a WWII/superpowers comic. In the early months of WWII, it turns out that Germany was working on a way to give people (very specific) superpowers. This is sufficient to turn the tide of the war *temporarily*, but they had a mole who was able to get some of the relevant information out to the Allies, who begin producing their own supersoldiers, and the fact remains that Germany is still on its last economic leg, which is posing its own problems. The comic makes no effort to shy away from the more gruesome aspects of war (as the writer, Kieron Gillen, mentions in one of his afterwords, war is a horror story and war *with superpowers* would be even more so). Murphy's Law is in full effect and plans go right just about as often as you'd expect them to. Finally, the narration appears to be sourced from a history book (or several) from some time a...\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"{'title': 'Now You Feel Like Number None',\n",
|
||||
" 'description': \"Now You Feel Like Number None is a complete Bleach fanfic quest focusing on an Arrancar in Aizen's army. It's praised for its well-written and story-driven narrative, featuring frequent and lengthy updates. The story explores the Arrancar's experience within their society and delves into their moral complexities.\",\n",
|
||||
" 'tags': ['web serial',\n",
|
||||
" 'fanfic',\n",
|
||||
" 'quest',\n",
|
||||
" 'complete',\n",
|
||||
" 'bleach',\n",
|
||||
" 'fantasy',\n",
|
||||
" 'progression',\n",
|
||||
" 'character-driven',\n",
|
||||
" 'romance'],\n",
|
||||
" 'reviews_quotes': ['I am reading [Now You Feel Like Number None](https://forums.sufficientvelocity.com/threads/now-you-feel-like-number-none-bleach-arrancar-quest.37531/) and I love it.',\n",
|
||||
" \"It's a very well-written, story-driven Bleach quest about an arranc ar in Aizen's army.\",\n",
|
||||
" 'Updates tend to be long and frequent.',\n",
|
||||
" 'I dropped it when it got into romance.',\n",
|
||||
" 'Kudos at this story just for the extremely punny title.',\n",
|
||||
" 'I believe everyone here should read [Ring-Maker](https://forums.spacebattles.com/threads/ring-maker-worm-lord-of-the-rings-alt-power.517894/#post-34102846).',\n",
|
||||
" \"I don't remember if I've plugged this before but anyways, I believe everyone here should read [Ring-Maker](https://forums.spacebattles.com/threads/ring-maker-worm-lord-of-the-rings-alt-power.517894/#post-34102846).\",\n",
|
||||
" \"It's very well written, although it assumes a lot of Bleach knowledge, but nothing that you can't quickly Google what a term means I think.\",\n",
|
||||
" 'Very intense, very character focused and in my top 10 of all books read.',\n",
|
||||
" \"[Now you feel like number none](https://forums.sufficientvelocity.com/threads/now-you-feel-like-number-none-bleach-arrancar-quest.37531/) is a complete Bleach quest, where the protagonist is an Arrancar in Hueco Mundo, under Aizen's orders.\",\n",
|
||||
" 'Fascinating story and mythology, remaking canon in something that makes sense and focusing on many secondary characters Tite Kubo dedicated no attention to.',\n",
|
||||
" \"Combat is something I usually don't care much about, but the quest has some completely amazing battles that I have bookmarked to read as one shots, having some of the best individual chapters I've ever read.\",\n",
|
||||
" 'Its main weakness was a slow update rate, but it finished two months ago, so that has stopped being a problem.',\n",
|
||||
" 'Absolutely great character focused Arrancar quest.',\n",
|
||||
" 'Top 3 quests for me. And actually finished!'],\n",
|
||||
" 'reviews_summary': 'Many praised the well-written story, frequent and lengthy updates, and engaging characters. The exploration of Arrancar society and moral complexities was mentioned positively, as was the writing quality. However, some found the romance element to be a negative influence.',\n",
|
||||
" 'reccomendations': \"Readers consistently praise the story's writing quality, compelling narrative, engaging characters, and frequent updates. Many highlight the unique exploration of Arrancar society and the characters' moral dilemmas within the Bleach setting.\",\n",
|
||||
" 'disrecommendations': 'Some readers dropped the story due to its inclusion of romance, feeling it detracted from the initial appeal of the protagonist as a grief-stricken, prideless nihilist.',\n",
|
||||
" 'why': 'Readers of r/rational might enjoy this story due to its well-developed worldbuilding within the Bleach universe, its focus on character development and moral dilemmas, and its engaging narrative structure. The length and frequent updates are also appealing to fans of long web serials.',\n",
|
||||
" 'if_you_liked_x_you_will_like_this': ['The Games We Play', 'Hollow Knight'],\n",
|
||||
" 'rating_quality': 9.0,\n",
|
||||
" 'rating_rationality': 7.0,\n",
|
||||
" 'rating_writing': 9.0,\n",
|
||||
" 'rating_plot': 8.0,\n",
|
||||
" 'rating_character': 8.5,\n",
|
||||
" 'rating_worldbuilding': 8.0}"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Input Tokens: 34209. Input Cost: 0.002566 USD, for title=`A Young Girl's Delinquency Record`\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"llm_info = []\n",
|
||||
"\n",
|
||||
"# TODO cache results as json\n",
|
||||
"n_input_tokens = []\n",
|
||||
"\n",
|
||||
"first = True\n",
|
||||
"\n",
|
||||
"# l = len(df3)\n",
|
||||
"# l = min(1000, len(df3))\n",
|
||||
"for i in tqdm(range(len(df3))):\n",
|
||||
" title = df3.index[i]\n",
|
||||
" urls = df3.iloc[i].url\n",
|
||||
"\n",
|
||||
" fs_title = re.sub(r'[^\\w\\s]', '', title)\n",
|
||||
" fs_title = re.sub(r'[^\\w\\s]', '', title)[:200]\n",
|
||||
" f = output_dir / f\"{i}_{fs_title}.json\"\n",
|
||||
"\n",
|
||||
" context = get_post_context(urls, char_budget=120000)\n",
|
||||
" tokens = len(enc.encode(context))\n",
|
||||
" n_input_tokens.append(tokens)\n",
|
||||
" if f.exists():\n",
|
||||
" with f.open(\"r\") as f:\n",
|
||||
" llm_data = json.load(f)\n",
|
||||
" else: \n",
|
||||
" f_tokens, llm_data = get_llm_summary(title, urls, context)\n",
|
||||
" else: \n",
|
||||
" context = get_post_context(urls, char_budget=120000)\n",
|
||||
" tokens = len(enc.encode(context))\n",
|
||||
" try: \n",
|
||||
" f_tokens, llm_data = get_llm_summary(title, urls, context)\n",
|
||||
" except Exception as e:\n",
|
||||
" print(e)\n",
|
||||
" continue\n",
|
||||
" print(\n",
|
||||
" f\"Input Tokens: {f_tokens}. Input Cost: {cost * f_tokens:.6f} USD, for title=`{title}`\"\n",
|
||||
" )\n",
|
||||
@@ -738,8 +806,8 @@
|
||||
" json.dump(llm_data, f)\n",
|
||||
"\n",
|
||||
" if first:\n",
|
||||
" c = f_tokens * len(df3) * cost / 2 # longest first, so assuming they get shorter and shorter\n",
|
||||
" print(f\"Projected cost for all threads: {c:.5f} USD\")\n",
|
||||
" c = f_tokens * len(df3) * cost / 2. # longest first, so assuming they get shorter and shorter\n",
|
||||
" print(f\"Projected cost for all threads: {c:.6f} USD\")\n",
|
||||
" print(f\"Content: {context[:1000]}...\")\n",
|
||||
" display(llm_data)\n",
|
||||
" first = False\n",
|
||||
@@ -756,14 +824,371 @@
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 161,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"df_llm = pd.DataFrame(llm_info)\n",
|
||||
"df_llm\n",
|
||||
"# also join with df4\n",
|
||||
"\n",
|
||||
"df_llm2 = (df_llm\n",
|
||||
" .drop(columns=['url'])\n",
|
||||
" .set_index('title2')\n",
|
||||
" .merge(df3, left_index=True, right_index=True)\n",
|
||||
" .sort_values('score', ascending=False)\n",
|
||||
" # .drop(columns=['comments'])\n",
|
||||
" .reset_index(names='title2')\n",
|
||||
")\n",
|
||||
"# df_llm2\n",
|
||||
"# then display as table"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 162,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def format_flair(author_flair_text):\n",
|
||||
" if author_flair_text:\n",
|
||||
" return f\" <em>{author_flair_text}</em>\"\n",
|
||||
" return \"\"\n",
|
||||
"\n",
|
||||
"import markdown\n",
|
||||
"def commentmd2html(x: dict) -> str:\n",
|
||||
" body = markdown.markdown(x['body'])\n",
|
||||
" ts = pd.to_datetime(x['created_utc'], unit='s').strftime('%Y-%m-%d')\n",
|
||||
" flair = format_flair(x['author_flair_text'])\n",
|
||||
" url = prefix + x['permalink']\n",
|
||||
" s = f\"\"\"<h3><a href=\"{url}\">{x.get('author', 'anon')} [{x['score']:+}] {flair} <sup>{ts}</sup></a></h3>\n",
|
||||
"{body}\n",
|
||||
"\"\"\"\n",
|
||||
" # print(s)\n",
|
||||
" return s\n",
|
||||
"\n",
|
||||
"def collapsibe(title, body):\n",
|
||||
" return f\"\"\"<details><summary>{title}</summary>\n",
|
||||
"{body}\n",
|
||||
"</details>\n",
|
||||
"\"\"\"\n",
|
||||
"\n",
|
||||
"prefix = \"https://reddit.com\"\n",
|
||||
"def c2md(x):\n",
|
||||
" return collapsibe(x['id'], commentmd2html(x))\n",
|
||||
"\n",
|
||||
"# # QC test\n",
|
||||
"# x = df3.iloc[0].comments[0]\n",
|
||||
"# from IPython.display import display, HTML\n",
|
||||
"# display(HTML(c2md(x)))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 163,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# First transform the fields for display\n",
|
||||
"d = df3.reset_index().sort_values(\"score\", ascending=False)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def url2a(url):\n",
|
||||
" text = url\n",
|
||||
" if \"reddit.com/r/rational\" in url:\n",
|
||||
" text = url.split(\"/\")[-2]\n",
|
||||
" # text = url.replace('https://reddit.com/r/rational/comments/', '')\n",
|
||||
"\n",
|
||||
" return f'<a href=\"{url}\">{text}</a>'\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"from collections import OrderedDict\n",
|
||||
"def unique_elements(lst):\n",
|
||||
" return list(OrderedDict.fromkeys(lst))\n",
|
||||
"\n",
|
||||
"def urls2a(urls, sep=None):\n",
|
||||
" if isinstance(urls, str):\n",
|
||||
" urls = urls.split(\"\\n\")\n",
|
||||
"\n",
|
||||
" # get uniques from list, keep in same order\n",
|
||||
" urls = unique_elements(urls)\n",
|
||||
"\n",
|
||||
" a_els = [url2a(u) for u in urls]\n",
|
||||
"\n",
|
||||
" # now make into a html list\n",
|
||||
" if sep is None:\n",
|
||||
" return \"<ul>\" + \"\".join([f\"<li>{x}</li>\" for x in a_els]) + \"</ul>\"\n",
|
||||
" else:\n",
|
||||
" return sep.join(a_els)\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 164,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "6562dba701444dae889d399cbc56e5c4",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
" 0%| | 0/1648 [00:00<?, ?it/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "181f7029acc54562828d098b21cfbb45",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
" 0%| | 0/1648 [00:00<?, ?it/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"\n",
|
||||
"d = df_llm2.copy()\n",
|
||||
"\n",
|
||||
"# make title have a link to first url\n",
|
||||
"d['title'] = d.progress_apply(lambda x: f'<a href=\"{x[\"url\"][0]}\">{x[\"title\"]}</a>', axis=1)\n",
|
||||
"d['title2'] = d.progress_apply(lambda x: f'<a href=\"{x[\"url\"][0]}\">{x[\"title2\"]}</a>', axis=1)\n",
|
||||
"\n",
|
||||
"d[\"url\"] = d[\"url\"].apply(lambda x: collapsibe('...', urls2a(x)))\n",
|
||||
"d[\"score\"] = d[\"score\"].round(2)\n",
|
||||
"\n",
|
||||
"d['tags'] = d['tags'].apply(lambda x: \", \".join(x))\n",
|
||||
"d['if_you_liked_x_you_will_like_this'] = d['if_you_liked_x_you_will_like_this'].apply(lambda x: \", \".join(x))\n",
|
||||
"\n",
|
||||
"prefix = 'https://reddit.com/r/rational/comments/'\n",
|
||||
"d[\"comment_urls\"] = d['comments'].apply(lambda x: collapsibe(\"...\", urls2a([prefix+y['permalink'] for y in x])))\n",
|
||||
"d[\"thread_urls\"] = d['thread_urls'].apply(lambda x: collapsibe(\"...\", urls2a(x)))\n",
|
||||
"# d['comments'] = d['comments'].progress_apply(lambda x: collapsibe(\"comments\", \"<br>\".join([c2md(c) for c in x])))\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"d['first_link_utc'] = d['first_link_utc'].dt.strftime('%Y-%m-%d')\n",
|
||||
"d['last_link_utc'] = d['last_link_utc'].dt.strftime('%Y-%m-%d')\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 165,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Index(['title2', 'title', 'description', 'tags', 'reviews_quotes',\n",
|
||||
" 'reviews_summary', 'reccomendations', 'disrecommendations', 'why',\n",
|
||||
" 'if_you_liked_x_you_will_like_this', 'rating_quality',\n",
|
||||
" 'rating_rationality', 'rating_writing', 'rating_plot',\n",
|
||||
" 'rating_character', 'rating_worldbuilding', 'score', 'n_links',\n",
|
||||
" 'n_comments', 'comments', 'thread_urls', 'first_link_utc',\n",
|
||||
" 'last_link_utc', 'url', 'comment_urls'],\n",
|
||||
" dtype='object')\n",
|
||||
"Dropping ['comments']\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"print(d.columns)\n",
|
||||
"rename_cols = OrderedDict(\n",
|
||||
" title2='Title (LLM)',\n",
|
||||
" title='Title',\n",
|
||||
" score='⬆️',\n",
|
||||
" n_comments='Comments',\n",
|
||||
" rating_quality='⭐Qual',\n",
|
||||
" rating_rationality='⭐Rat',\n",
|
||||
" rating_writing='⭐Writ',\n",
|
||||
" rating_plot='⭐Plot',\n",
|
||||
" rating_character='⭐Char',\n",
|
||||
" rating_worldbuilding='⭐World',\n",
|
||||
" tags='Tags',\n",
|
||||
" first_link_utc='First Link',\n",
|
||||
" last_link_utc='Last Link',\n",
|
||||
" n_links='Links',\n",
|
||||
" url='URLs',\n",
|
||||
" reviews_summary='Reviews Summary',\n",
|
||||
" thread_urls='Threads',\n",
|
||||
" comment_urls='Comments',\n",
|
||||
" if_you_liked_x_you_will_like_this='Similar',\n",
|
||||
" description='Description',\n",
|
||||
" reccomendations='Recommendations',\n",
|
||||
" disrecommendations='Disrecommendations',\n",
|
||||
" why='Why',\n",
|
||||
" reviews_quotes='Reviews',\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"dropping = [k for k in d.columns if k not in rename_cols.keys()]\n",
|
||||
"print('Dropping', dropping)\n",
|
||||
"\n",
|
||||
"d = d.drop(columns=['comments'])[list(rename_cols.keys())].rename(columns=rename_cols)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 166,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"['comments',\n",
|
||||
" 'First Link',\n",
|
||||
" 'first_link_utc',\n",
|
||||
" 'Last Link',\n",
|
||||
" 'last_link_utc',\n",
|
||||
" 'Reviews',\n",
|
||||
" 'reviews_quotes',\n",
|
||||
" 'Reviews Summary',\n",
|
||||
" 'reviews_summary',\n",
|
||||
" '⭐Writ',\n",
|
||||
" 'rating_writing',\n",
|
||||
" '⭐Plot',\n",
|
||||
" 'rating_plot',\n",
|
||||
" '⭐Char',\n",
|
||||
" 'rating_character',\n",
|
||||
" '⭐World',\n",
|
||||
" 'rating_worldbuilding',\n",
|
||||
" 'Recommendations',\n",
|
||||
" 'reccomendations',\n",
|
||||
" 'Disrecommendations',\n",
|
||||
" 'disrecommendations',\n",
|
||||
" 'Title',\n",
|
||||
" 'title',\n",
|
||||
" 'Links',\n",
|
||||
" 'n_links',\n",
|
||||
" 'Comments',\n",
|
||||
" 'n_comments']"
|
||||
]
|
||||
},
|
||||
"execution_count": 166,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"hidden = [\"comments\", \"first_link_utc\", \"last_link_utc\", 'reviews_quotes', 'reviews_summary',\n",
|
||||
" 'rating_writing', 'rating_plot', 'rating_character', 'rating_worldbuilding',\n",
|
||||
"\n",
|
||||
" 'reccomendations', 'disrecommendations', 'title', 'n_links', 'n_comments'\n",
|
||||
"]\n",
|
||||
"# translate them using column rename\n",
|
||||
"hidden2 = []\n",
|
||||
"for h in hidden:\n",
|
||||
" if h in rename_cols:\n",
|
||||
" hidden2.append(rename_cols[h])\n",
|
||||
" hidden2.append(h)\n",
|
||||
"# hidden = [k for k, v in rename_cols.items() if v in hidden]\n",
|
||||
"hidden2"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 167,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"'[{\"title\": \"Title (LLM)\", \"visible\": true, \"searchable\": true, \"footer\": \"Title (LLM)\", \"name\": \"Title (LLM)2\"}, {\"title\": \"Title\", \"visible\": false, \"searchable\": false, \"footer\": \"Title\", \"name\": \"Title2\"}, {\"title\": \"\\\\u2b06\\\\ufe0f\", \"visible\": true, \"searchable\": true, \"footer\": \"\\\\u2b06\\\\ufe0f\", \"name\": \"\\\\u2b06\\\\ufe0f2\"}, {\"title\": \"Comments\", \"visible\": false, \"searchable\": false, \"footer\": \"Comments\", \"name\": \"Comments2\"}, {\"title\": \"\\\\u2b50Qual\", \"visible\": true, \"searchable\": true, \"footer\": \"\\\\u2b50Qual\", \"name\": \"\\\\u2b50Qual2\"}, {\"title\": \"\\\\u2b50Rat\", \"visible\": true, \"searchable\": true, \"footer\": \"\\\\u2b50Rat\", \"name\": \"\\\\u2b50Rat2\"}, {\"title\": \"\\\\u2b50Writ\", \"visible\": false, \"searchable\": false, \"footer\": \"\\\\u2b50Writ\", \"name\": \"\\\\u2b50Writ2\"}, {\"title\": \"\\\\u2b50Plot\", \"visible\": false, \"searchable\": false, \"footer\": \"\\\\u2b50Plot\", \"name\": \"\\\\u2b50Plot2\"}, {\"title\": \"\\\\u2b50Char\", \"visible\": false, \"searchable\": false, \"footer\": \"\\\\u2b50Char\", \"name\": \"\\\\u2b50Char2\"}, {\"title\": \"\\\\u2b50World\", \"visible\": false, \"searchable\": false, \"footer\": \"\\\\u2b50World\", \"name\": \"\\\\u2b50World2\"}, {\"title\": \"Tags\", \"visible\": true, \"searchable\": true, \"footer\": \"Tags\", \"name\": \"Tags2\"}, {\"title\": \"First Link\", \"visible\": false, \"searchable\": false, \"footer\": \"First Link\", \"name\": \"First Link2\"}, {\"title\": \"Last Link\", \"visible\": false, \"searchable\": false, \"footer\": \"Last Link\", \"name\": \"Last Link2\"}, {\"title\": \"Links\", \"visible\": false, \"searchable\": false, \"footer\": \"Links\", \"name\": \"Links2\"}, {\"title\": \"URLs\", \"visible\": true, \"searchable\": true, \"footer\": \"URLs\", \"name\": \"URLs2\"}, {\"title\": \"Reviews Summary\", \"visible\": false, \"searchable\": false, \"footer\": \"Reviews Summary\", \"name\": \"Reviews Summary2\"}, {\"title\": \"Threads\", \"visible\": true, \"searchable\": true, \"footer\": \"Threads\", \"name\": \"Threads2\"}, {\"title\": \"Comments\", \"visible\": false, \"searchable\": false, \"footer\": \"Comments\", \"name\": \"Comments2\"}, {\"title\": \"Similar\", \"visible\": true, \"searchable\": true, \"footer\": \"Similar\", \"name\": \"Similar2\"}, {\"title\": \"Description\", \"visible\": true, \"searchable\": true, \"footer\": \"Description\", \"name\": \"Description2\"}, {\"title\": \"Recommendations\", \"visible\": false, \"searchable\": false, \"footer\": \"Recommendations\", \"name\": \"Recommendations2\"}, {\"title\": \"Disrecommendations\", \"visible\": false, \"searchable\": false, \"footer\": \"Disrecommendations\", \"name\": \"Disrecommendations2\"}, {\"title\": \"Why\", \"visible\": true, \"searchable\": true, \"footer\": \"Why\", \"name\": \"Why2\"}, {\"title\": \"Reviews\", \"visible\": false, \"searchable\": false, \"footer\": \"Reviews\", \"name\": \"Reviews2\"}]'"
|
||||
]
|
||||
},
|
||||
"execution_count": 167,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# put into the html template\n",
|
||||
"import jinja2\n",
|
||||
"\n",
|
||||
"environment = jinja2.Environment()\n",
|
||||
"template = open(\"../index.jinja2.html\").read()\n",
|
||||
"template = environment.from_string(template)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"data = d.to_json(orient=\"values\")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"columns = [\n",
|
||||
" {\n",
|
||||
" \"title\": c,\n",
|
||||
" \"visible\": c not in hidden2,\n",
|
||||
" \"searchable\": c not in hidden2,\n",
|
||||
" \"footer\": c,\n",
|
||||
" \"name\": c+\"2\",\n",
|
||||
" }\n",
|
||||
" for c in d.columns\n",
|
||||
"]\n",
|
||||
"columns = json.dumps(columns)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"html = template.render(\n",
|
||||
" data=data,\n",
|
||||
" columns=columns,\n",
|
||||
")\n",
|
||||
"html_out = Path(\"../index2.html\").resolve()\n",
|
||||
"open(html_out, \"w\").write(html)\n",
|
||||
"columns"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 168,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/html": [
|
||||
"<a href=\"/media/wassname/SGIronWolf/projects5/rrational/index2.html\">View the page /media/wassname/SGIronWolf/projects5/rrational/index2.html</a>"
|
||||
],
|
||||
"text/plain": [
|
||||
"<IPython.core.display.HTML object>"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from IPython.display import HTML, display\n",
|
||||
"\n",
|
||||
"htmla = f'<a href=\"{html_out}\">View the page {html_out}</a>'\n",
|
||||
"display(HTML(htmla))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 169,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"df3.to_parquet(\"../outputs/df_links5.parquet\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
|
||||
Reference in New Issue
Block a user