Add sts2014 and quora evaluators in common/evaluators/ (#151)

* add SNLI dataset

* add STS-2014

* add STS-2014

* add trainers and evaluators for Quora

* add quora in datasets/

* process the  merge confict in common/dataset.py

* add sts2014_evaluator.py and quora_evaluator.py

common/evaluators/quora_evaluator.py

@@ -0,0 +1,31 @@
+import torch
+import torch.nn.functional as F
+
+from .evaluator import Evaluator
+
+
+class QuoraEvaluator(Evaluator):
+
+    def get_scores(self):
+        self.model.eval()
+        test_kl_div_loss = 0
+        acc_total = 0
+
+        for batch in self.data_loader:
+            # Select embedding
+            sent1, sent2 = self.get_sentence_embeddings(batch)
+
+            output = self.model(sent1, sent2, batch.ext_feats, batch.dataset.word_to_doc_cnt, batch.sentence_1_raw, batch.sentence_2_raw)
+            test_kl_div_loss += F.kl_div(output, batch.label, size_average=False).item()
+
+            true_label = torch.max(batch.label.data, 1)[1]
+            prediction = torch.max(output, 1)[1]
+            acc_total += ((true_label == prediction)).sum().item()
+
+            del output
+
+        test_kl_div_loss /= len(batch.dataset.examples)
+
+        accuracy = acc_total / len(self.data_loader.dataset.examples)
+
+        return [accuracy, test_kl_div_loss], ['accuracy', 'KL-divergence loss']

common/evaluators/sts2014_evaluator.py

@@ -0,0 +1,52 @@
+from scipy.stats import pearsonr, spearmanr
+import torch
+import torch.nn.functional as F
+
+from .evaluator import Evaluator
+
+
+class STS2014Evaluator(Evaluator):
+
+    def get_scores(self):
+        self.model.eval()
+        num_classes = self.dataset_cls.NUM_CLASSES
+        test_kl_div_loss = 0
+        predictions = []
+        true_labels = []
+
+        for batch in self.data_loader:
+            # Select embedding
+            sent1, sent2 = self.get_sentence_embeddings(batch)
+
+            output = self.model(sent1, sent2, batch.ext_feats, batch.dataset.word_to_doc_cnt, batch.sentence_1_raw, batch.sentence_2_raw)
+            test_kl_div_loss += F.kl_div(output, batch.label, size_average=False).item()
+
+            predict_classes = batch.label.new_tensor(torch.arange(1, num_classes + 1)).expand(self.batch_size, num_classes)
+            # handle last batch which might have smaller size
+            if len(predict_classes) != len(batch.sentence_1):
+                predict_classes = batch.label.new_tensor(torch.arange(1, num_classes + 1)).expand(len(batch.sentence_1), num_classes)
+
+            true_labels.append((predict_classes * batch.label.detach()).sum(dim=1))
+            predictions.append((predict_classes * output.detach().exp()).sum(dim=1))
+
+            del output
+
+        predictions = torch.cat(predictions)
+        true_labels = torch.cat(true_labels)
+        mse = F.mse_loss(predictions, true_labels).item()
+        test_kl_div_loss /= len(batch.dataset.examples)
+        predictions = predictions.cpu().numpy()
+        true_labels = true_labels.cpu().numpy()
+        pearson_r = pearsonr(predictions, true_labels)[0]
+        spearman_r = spearmanr(predictions, true_labels)[0]
+
+        return [pearson_r, spearman_r, mse, test_kl_div_loss], ['pearson_r', 'spearman_r', 'mse', 'KL-divergence loss']
+
+    def get_final_prediction_and_label(self, batch_predictions, batch_labels):
+        num_classes = self.dataset_cls.NUM_CLASSES
+        predict_classes = batch_labels.new_tensor(torch.arange(1, num_classes + 1)).expand(batch_predictions.size(0), num_classes)
+
+        predictions = (predict_classes * batch_predictions.exp()).sum(dim=1)
+        true_labels = (predict_classes * batch_labels).sum(dim=1)
+
+        return predictions, true_labels