mirror of
https://github.com/wassname/Castor.git
synced 2026-10-02 11:50:18 +08:00
Michael Tu
4470983504
* Add TrecQA dataset and modularize MP-CNN infra * Stylistic improvements * Fix and warn about trec_eval path issue * Update README for MP-CNN * Update incorrect map/mrr * MP-CNN: address code review comments * Create common Castor pair Dataset class * Move map and mrr computation to Castor utils * Make map mrr utility trec_eval path more general
30 lines
1011 B
Python
30 lines
1011 B
Python
from mp_cnn.evaluators.sick_evaluator import SICKEvaluator
|
|
from mp_cnn.evaluators.msrvid_evaluator import MSRVIDEvaluator
|
|
from mp_cnn.evaluators.trecqa_evaluator import TRECQAEvaluator
|
|
|
|
|
|
class MPCNNEvaluatorFactory(object):
|
|
"""
|
|
Get the corresponding Evaluator class for a particular dataset.
|
|
"""
|
|
evaluator_map = {
|
|
'sick': SICKEvaluator,
|
|
'msrvid': MSRVIDEvaluator,
|
|
'trecqa': TRECQAEvaluator
|
|
}
|
|
|
|
@staticmethod
|
|
def get_evaluator(dataset_cls, model, data_loader, batch_size, device):
|
|
if data_loader is None:
|
|
return None
|
|
|
|
if not hasattr(dataset_cls, 'NAME'):
|
|
raise ValueError('Invalid dataset. Dataset should have NAME attribute.')
|
|
|
|
if dataset_cls.NAME not in MPCNNEvaluatorFactory.evaluator_map:
|
|
raise ValueError('{} is not implemented.'.format(dataset_cls))
|
|
|
|
return MPCNNEvaluatorFactory.evaluator_map[dataset_cls.NAME](
|
|
dataset_cls, model, data_loader, batch_size, device
|
|
)
|