mirror of
https://github.com/wassname/Castor.git
synced 2026-08-21 11:09:28 +08:00
57 lines
2.1 KiB
Python
57 lines
2.1 KiB
Python
from .evaluators.sick_evaluator import SICKEvaluator
|
|
from .evaluators.msrvid_evaluator import MSRVIDEvaluator
|
|
from .evaluators.sst_evaluator import SSTEvaluator
|
|
from .evaluators.trecqa_evaluator import TRECQAEvaluator
|
|
from .evaluators.wikiqa_evaluator import WikiQAEvaluator
|
|
from .evaluators.pit2015_evaluator import PIT2015Evaluator
|
|
from .evaluators.reuters_evaluator import ReutersEvaluator
|
|
from .evaluators.snli_evaluator import SNLIEvaluator
|
|
from .evaluators.sts2014_evaluator import STS2014Evaluator
|
|
from .evaluators.quora_evaluator import QuoraEvaluator
|
|
from nce.nce_pairwise_mp.evaluators.trecqa_evaluator import TRECQAEvaluatorNCE
|
|
from nce.nce_pairwise_mp.evaluators.wikiqa_evaluator import WikiQAEvaluatorNCE
|
|
|
|
|
|
class EvaluatorFactory(object):
|
|
"""
|
|
Get the corresponding Evaluator class for a particular dataset.
|
|
"""
|
|
evaluator_map = {
|
|
'sick': SICKEvaluator,
|
|
'msrvid': MSRVIDEvaluator,
|
|
'SST-1': SSTEvaluator,
|
|
'SST-2': SSTEvaluator,
|
|
'trecqa': TRECQAEvaluator,
|
|
'wikiqa': WikiQAEvaluator,
|
|
'pit2015': PIT2015Evaluator,
|
|
'twitterurl': PIT2015Evaluator,
|
|
'Reuters': ReutersEvaluator,
|
|
'AAPD': ReutersEvaluator,
|
|
'IMDB': ReutersEvaluator,
|
|
'Yelp2014': ReutersEvaluator,
|
|
'SNLI': SNLIEvaluator,
|
|
'sts2014': STS2014Evaluator,
|
|
'Quora': QuoraEvaluator
|
|
}
|
|
|
|
evaluator_map_nce = {
|
|
'trecqa': TRECQAEvaluatorNCE,
|
|
'wikiqa': WikiQAEvaluatorNCE
|
|
}
|
|
|
|
@staticmethod
|
|
def get_evaluator(dataset_cls, model, embedding, data_loader, batch_size, device, nce=False, keep_results=False):
|
|
if data_loader is None:
|
|
return None
|
|
|
|
evaluator_map = EvaluatorFactory.evaluator_map_nce if nce else EvaluatorFactory.evaluator_map
|
|
if not hasattr(dataset_cls, 'NAME'):
|
|
raise ValueError('Invalid dataset. Dataset should have NAME attribute.')
|
|
|
|
if dataset_cls.NAME not in evaluator_map:
|
|
raise ValueError('{} is not implemented.'.format(dataset_cls))
|
|
|
|
return evaluator_map[dataset_cls.NAME](
|
|
dataset_cls, model, embedding, data_loader, batch_size, device, keep_results
|
|
)
|