wassname/Castor
1
0
Fork 0
mirror of https://github.com/wassname/Castor.git synced 2026-09-09 11:13:20 +08:00
Code Issues Packages Projects Releases Wiki Activity

Add Reuters dataset option for common.dataset (#149)

Browse Source 
* Add Reuters option in common.dataset
This commit is contained in:
Ashutosh-Adhikari
2018-10-07 00:57:38 -04:00
committed by Peng Shi
parent 1b817d3e24
commit 999e0c88f2
1 changed files with 6 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/dataset.py
+6 -1
View File
@@ -8,7 +8,7 @@ from datasets.msrvid import MSRVID
from datasets.trecqa import TRECQA
from datasets.wikiqa import WikiQA
from datasets.pit2015 import PIT2015
from datasets.reuters import Reuters
class UnknownWordVecCache(object):
"""
@@ -66,6 +66,11 @@ class DatasetFactory(object):
train_loader, dev_loader, test_loader = PIT2015.iters(dataset_root, word_vectors_file, word_vectors_dir, batch_size, device=device, unk_init=UnknownWordVecCache.unk)
embedding = nn.Embedding.from_pretrained(PIT2015.TEXT_FIELD.vocab.vectors)
return PIT2015, embedding, train_loader, test_loader, dev_loader
elif dataset_name == 'reuters':
dataset_root = os.path.join(castor_dir, os.pardir, 'Castor-data', 'datasets', 'Reuters-21578/')
train_loader, dev_loader, test_loader = Reuters.iters(dataset_root, word_vectors_file, word_vectors_dir, batch_size, device=device, unk_init=UnknownWordVecCache.unk)
embedding = nn.Embedding.from_pretrained(Reuters.TEXT_FIELD.vocab.vectors)
return Reuters, embedding, train_loader, test_loader, dev_loader
else:
raise ValueError('{} is not a valid dataset.'.format(dataset_name))