mirror of
https://github.com/wassname/Open-Assistant.git
synced 2026-07-09 00:20:03 +08:00
Richard Nagyfi
55a52fd61f
* Added Gutenberg Crawler Notebook - added configrable Jupyter Notebook with crawler + ability to save to parquet - added README.md - added requirements.txt for running in venv / collab
17 lines
426 B
Python
17 lines
426 B
Python
from datasets import concatenate_datasets, load_dataset
|
|
|
|
|
|
def load(languages: list = ["en", "de", "fr", "es", "it", "pt", "nl", "hu"]):
|
|
ds = None
|
|
for lang in languages:
|
|
if ds is None:
|
|
ds = load_dataset(f"sedthh/gutenberg_{lang}")
|
|
else:
|
|
ds = concatenate_datasets([ds, f"sedthh/gutenberg_{lang}"])
|
|
return ds
|
|
|
|
|
|
if __name__ == "__main__":
|
|
ds = load()
|
|
print(ds["train"])
|