mirror of
https://github.com/wassname/Castor.git
synced 2026-09-28 13:42:00 +08:00
rosequ
36b1ddb858
* support for WikiQA dataset * parallel runs for both datasets * minor fixes * updated README * removed data folder; added scripts to create dataset; updated README * after CR * after CR2
17 lines
666 B
Python
17 lines
666 B
Python
from torchtext import data
|
|
import os
|
|
|
|
class WikiDataset(data.TabularDataset):
|
|
dirname = 'data'
|
|
@classmethod
|
|
|
|
def splits(cls, question_id, question_field, answer_field, external_field, label_field,
|
|
train='train.tsv', validation='dev.tsv', test='test.tsv'):
|
|
path = './data'
|
|
prefix_name = 'wikiqa.'
|
|
return super(WikiDataset, cls).splits(
|
|
os.path.join(path, prefix_name), train, validation, test,
|
|
format='TSV', fields=[('qid', question_id), ('label', label_field), ('question', question_field),
|
|
('answer', answer_field), ('ext_feat', external_field)]
|
|
)
|