wassname/stampy-chat
1
0
Fork 0
mirror of https://github.com/wassname/stampy-chat.git synced 2026-09-11 12:50:34 +08:00
Code Issues Packages Projects Releases Wiki Activity

removed commented-out code

Browse Source
This commit is contained in:
Thomas Lemoine
2023-06-21 00:37:16 -04:00
parent a755ecc480
commit f2e160f8e3
1 changed files with 0 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main.py
-39
View File
@@ -34,45 +34,6 @@ import matplotlib.pyplot as plt
def load_rawdata_into_pkl():
"""
# List of possible sources:
all_sources = ["https://aipulse.org", "ebook", "https://qualiacomputing.com", "alignment forum", "lesswrong", "manual", "arxiv", "https://deepmindsafetyresearch.medium.com", "waitbutwhy.com", "GitHub", "https://aiimpacts.org", "arbital.com", "carado.moe", "nonarxiv_papers", "https://vkrakovna.wordpress.com", "https://jsteinhardt.wordpress.com", "audio-transcripts", "https://intelligence.org", "youtube", "reports", "https://aisafety.camp", "curriculum", "https://www.yudkowsky.net", "distill",
"Cold Takes", "printouts", "gwern.net", "generative.ink", "greaterwrong.com"] # These last do not have a source field in the .jsonl file
# List of sources we are using for the test run:
custom_sources = [
"https://aipulse.org",
"ebook",
"https://qualiacomputing.com",
"alignment forum",
"lesswrong",
"manual",
"arxiv",
"https://deepmindsafetyresearch.medium.com/",
"waitbutwhy.com",
"GitHub",
"https://aiimpacts.org",
"arbital.com",
"carado.moe",
"nonarxiv_papers",
"https://vkrakovna.wordpress.com",
"https://jsteinhardt.wordpress.com",
"audio-transcripts",
"https://intelligence.org",
"youtube",
"reports",
"https://aisafety.camp",
"curriculum",
"https://www.yudkowsky.net",
"distill",
"Cold Takes",
"printouts",
"gwern.net",
"generative.ink",
"greaterwrong.com"
]
"""
dataset = create_dataset.ChunkedARD(
min_tokens_per_block=200, max_tokens_per_block=300
)