From f2e160f8e33c99798113ee1aa453a216a68db2c2 Mon Sep 17 00:00:00 2001 From: Thomas Lemoine Date: Wed, 21 Jun 2023 00:37:16 -0400 Subject: [PATCH] removed commented-out code --- src/main.py | 39 --------------------------------------- 1 file changed, 39 deletions(-) diff --git a/src/main.py b/src/main.py index 029df5a..87a68e6 100644 --- a/src/main.py +++ b/src/main.py @@ -34,45 +34,6 @@ import matplotlib.pyplot as plt def load_rawdata_into_pkl(): - """ - # List of possible sources: - all_sources = ["https://aipulse.org", "ebook", "https://qualiacomputing.com", "alignment forum", "lesswrong", "manual", "arxiv", "https://deepmindsafetyresearch.medium.com", "waitbutwhy.com", "GitHub", "https://aiimpacts.org", "arbital.com", "carado.moe", "nonarxiv_papers", "https://vkrakovna.wordpress.com", "https://jsteinhardt.wordpress.com", "audio-transcripts", "https://intelligence.org", "youtube", "reports", "https://aisafety.camp", "curriculum", "https://www.yudkowsky.net", "distill", - "Cold Takes", "printouts", "gwern.net", "generative.ink", "greaterwrong.com"] # These last do not have a source field in the .jsonl file - - # List of sources we are using for the test run: - custom_sources = [ - "https://aipulse.org", - "ebook", - "https://qualiacomputing.com", - "alignment forum", - "lesswrong", - "manual", - "arxiv", - "https://deepmindsafetyresearch.medium.com/", - "waitbutwhy.com", - "GitHub", - "https://aiimpacts.org", - "arbital.com", - "carado.moe", - "nonarxiv_papers", - "https://vkrakovna.wordpress.com", - "https://jsteinhardt.wordpress.com", - "audio-transcripts", - "https://intelligence.org", - "youtube", - "reports", - "https://aisafety.camp", - "curriculum", - "https://www.yudkowsky.net", - "distill", - "Cold Takes", - "printouts", - "gwern.net", - "generative.ink", - "greaterwrong.com" - ] - """ - dataset = create_dataset.ChunkedARD( min_tokens_per_block=200, max_tokens_per_block=300 )