mirror of
https://github.com/wassname/Castor.git
synced 2026-09-09 11:13:20 +08:00
Victor Yang
51d8e29525
* update nce-sm * refactor code, update torchtext * use shared evaluation * refactor code, use shared data loader * refactor code * refactor code * refactor code according to Michael's great suggestions * update readme and requirement * update datasets and readme * update data loader * add space between + * update refactor code * add nce-mp * remove duplicate files * update readme, refactor code according to mp_cnn and delete duplicate code, follow PEP8 standard * refactor code, add/delete comments * import exit from sys
51 lines
1.8 KiB
Python
51 lines
1.8 KiB
Python
import random
|
|
import numpy as np
|
|
import heapq
|
|
import operator
|
|
|
|
import torch
|
|
from torchtext import data
|
|
|
|
# get the nearest negative samples to the positive sample by computing the feature difference
|
|
def get_nearest_neg_id(pos_feature, neg_dict, distance="cosine", k=1, weight=False):
|
|
dis_list = []
|
|
pos_feature = pos_feature.data.cpu().numpy()
|
|
pos_feature_norm = pos_feature / np.sqrt(sum(pos_feature ** 2))
|
|
neg_list = []
|
|
for key in neg_dict:
|
|
if distance == "l2":
|
|
dis = np.sqrt(np.sum((np.array(pos_feature) - neg_dict[key]["feature"]) ** 2))
|
|
elif distance == "cosine":
|
|
neg_feature = np.array(neg_dict[key]["feature"])
|
|
feat_norm = neg_feature / np.sqrt(sum(neg_feature ** 2))
|
|
dis = 1 - feat_norm.dot(pos_feature_norm)
|
|
dis_list.append(dis)
|
|
neg_list.append(key)
|
|
|
|
k = min(k, len(neg_dict))
|
|
min_list = heapq.nsmallest(k, enumerate(dis_list), key=operator.itemgetter(1))
|
|
# find the corresponding neg id
|
|
min_id_list = [neg_list[x[0]] for x in min_list]
|
|
if weight:
|
|
min_id_score = [1 - x[1] for x in min_list]
|
|
return min_id_list, min_id_score
|
|
else:
|
|
return min_id_list
|
|
|
|
# get the negative samples randomly
|
|
def get_random_neg_id(q2neg, qid_i, k=8):
|
|
# question 1734 in TrecQA has only one positive answer and no negative answer
|
|
if qid_i not in q2neg:
|
|
return []
|
|
k = min(k, len(q2neg[qid_i]))
|
|
ran = random.sample(q2neg[qid_i], k)
|
|
return ran
|
|
|
|
# pack the lists of question/answer/ext_feat into a torchtext batch
|
|
def get_batch(question, answer, ext_feat, size):
|
|
new_batch = data.Batch()
|
|
new_batch.batch_size = size
|
|
setattr(new_batch, "sentence_2", torch.stack(answer))
|
|
setattr(new_batch, "sentence_1", torch.stack(question))
|
|
setattr(new_batch, "ext_feats", torch.stack(ext_feat))
|
|
return new_batch |