mirror of
https://github.com/wassname/Castor.git
synced 2026-09-09 11:13:20 +08:00
Michael Tu
4470983504
* Add TrecQA dataset and modularize MP-CNN infra * Stylistic improvements * Fix and warn about trec_eval path issue * Update README for MP-CNN * Update incorrect map/mrr * MP-CNN: address code review comments * Create common Castor pair Dataset class * Move map and mrr computation to Castor utils * Make map mrr utility trec_eval path more general
20 lines
734 B
Python
20 lines
734 B
Python
class Evaluator(object):
|
|
"""
|
|
Evaluates performance of model on a Dataset, using metrics specific to the Dataset.
|
|
"""
|
|
|
|
def __init__(self, dataset_cls, model, data_loader, batch_size, device):
|
|
self.dataset_cls = dataset_cls
|
|
self.model = model
|
|
self.data_loader = data_loader
|
|
self.batch_size = batch_size
|
|
self.device = device
|
|
|
|
def get_scores(self):
|
|
"""
|
|
Get the scores used to evaluate the model.
|
|
Should return ([score1, score2, ..], [score1_name, score2_name, ...]).
|
|
The first score is the primary score used to determine if the model has improved.
|
|
"""
|
|
raise NotImplementedError('Evaluator subclass needs to implement get_score')
|