From a67e2d12c4d79287de28b41e436fe6c4d9a108ef Mon Sep 17 00:00:00 2001 From: gauravbaruah Date: Tue, 18 Apr 2017 12:32:43 -0400 Subject: [PATCH] Ext feats bug fix (#19) + sm model no external features baseline + sm model with IDF weights + sm model with IDF weights without removing punctuation --> barely better than df/idf (a la Pytorch). + sm model with stemming before computing IDF weights ^ all on the TrecQA dataset --- baseline_results.tsv | 16 +-- idf_baseline/qa-data-only-idf.py | 8 +- sm_model/external_features.py | 171 +++++++++++++++++++++++++++++++ sm_model/main.py | 107 +++++++++++-------- sm_model/make_run.py | 51 +++++++++ sm_model/model.py | 4 +- sm_model/requirements.txt | 5 + sm_model/train.py | 31 +++--- sm_model/utils.py | 11 +- 9 files changed, 332 insertions(+), 72 deletions(-) create mode 100644 sm_model/external_features.py create mode 100644 sm_model/make_run.py create mode 100644 sm_model/requirements.txt diff --git a/baseline_results.tsv b/baseline_results.tsv index b42d715..e36e11f 100644 --- a/baseline_results.tsv +++ b/baseline_results.tsv @@ -1,14 +1,18 @@ Baseline_method idf condition dataset train_MAP dev_MAP test_MAP Notes/Comments idf_sum_similarity dataset no_stopping + no_stemming TrecQA 0.7744 0.7668 0.7014 IDF is computed over terms in the train, dev and test sets idf_sum_similarity dataset with_stopping + with_stemming TrecQA 0.7078 0.7404 0.6816 +idf_sum_similarity dataset with_stopping + with_stemming + stop(punct) TrecQA 0.7074 0.7337 0.6805 also stopping punctuation idf_sum_similarity dataset no_stopping + no_stemming WikiQA 0.2429 0.2489 0.2219 There is less overlap between questions and answers in the WikiQA dataset idf_sum_similarity dataset with_stopping + with_stemming WikiQA 0.2232 0.2472 0.2025 -idf_sum_similarity Corpus no_stopping + no_stemming TrecQA 0.6736 0.728 0.6377 IDF is computed over documents in disks1-5 and aquaint -idf_sum_similarity Corpus with_stopping + with_stemming TrecQA 0.7032 0.7281 0.672 -idf_sum_similarity Corpus no_stopping + no_stemming WikiQA 0.2432 0.244 0.2206 IDF is computed over documents in Wikipedia -idf_sum_similarity Corpus with_stopping + with_stemming WikiQA 0.2212 0.2461 0.2005 -sm_model NA no_idf_overlap_features TrecQA -sm_model Dataset with_corrected_idf_overlap_features TrecQA +idf_sum_similarity dataset with_stopping + with_stemming + stop(punct) WikiQA 0.2236 0.2472 0.2025 + +idf_sum_similarity Corpus with_stopping + with_stemming TrecQA 0.7032 0.7281 0.672 IDF is computed over documents in disks1-5 and aquaint +idf_sum_similarity Corpus with_stopping + with_stemming WikiQA 0.2212 0.2461 0.2005 IDF is computed over documents in Wikipedia + +sm_model NA no_idf_overlap_features TrecQA 0.597 0.6918 0.6445 +sm_model Dataset with_corrected_idf_overlap_features TrecQA 0.919 0.7927 0.7384 S&M model uses stopping only +sm_model Dataset with_corrected_idf_overlap_features – stop(punct) TrecQA 0.8325 0.7948 0.7562 did not remove punctuation +sm_model Dataset with_stemmed_idf_overlap_features TrecQA 0.9414 0.8206 0.7363 sm_model Corpus with_corrected_idf_overlap_features TrecQA sm_model Dataset with_fractional_idf_overlap_features TrecQA sm_model Corpus with_fractional_idf_overlap_features TrecQA diff --git a/idf_baseline/qa-data-only-idf.py b/idf_baseline/qa-data-only-idf.py index f3f070d..4b72ada 100644 --- a/idf_baseline/qa-data-only-idf.py +++ b/idf_baseline/qa-data-only-idf.py @@ -2,6 +2,7 @@ import argparse import os import numpy as np from collections import defaultdict +import string import nltk nltk.download('stopwords') @@ -16,6 +17,7 @@ def read_in_data(datapath, set_name, file, stop_and_stem=False): if stop_and_stem: stemmer = PorterStemmer() stoplist = set(stopwords.words('english')) + stoplist.update(set(string.punctuation)) def stop_stem(sentence): return ' '.join([stemmer.stem(word) for word in sentence.split() \ if word not in stoplist]) @@ -57,8 +59,8 @@ def write_out_idf_sum_similarities(qids, questions, answers, term_idfs, outfile, if qids[i] != old_qid and dataset.endswith('WikiQA'): docid_c = 0 old_qid = qids[i] - print('{} 0 {} 0 {} data_only_idfbaseline'.format(qids[i], docid_c, - idf_sum_similarity[i]), + print('{} 0 {} 0 {} data_only_idfbaseline'.format(qids[i], docid_c, + idf_sum_similarity[i]), file=outf) docid_c += 1 @@ -81,7 +83,7 @@ if __name__ == "__main__": train_data, dev_data, test_data = 'train-all', 'raw-dev', 'raw-test' train_que = read_in_data(args.qa_data, train_data, 'a.toks', args.stop_and_stem) - train_ans = read_in_data(args.qa_data, train_data, 'b.toks', args.stop_and_stem) + train_ans = read_in_data(args.qa_data, train_data, 'b.toks', args.stop_and_stem) dev_que = read_in_data(args.qa_data, dev_data, 'a.toks', args.stop_and_stem) dev_ans = read_in_data(args.qa_data, dev_data, 'b.toks', args.stop_and_stem) diff --git a/sm_model/external_features.py b/sm_model/external_features.py new file mode 100644 index 0000000..dcc5b96 --- /dev/null +++ b/sm_model/external_features.py @@ -0,0 +1,171 @@ +# module to compute various external features for the sm_model. +# TODO: add more external features like: +# word mover distance, cosine sim in tf.idf space, cosine sim in word embedding space +# overlap based on parts of speech: noun, verb, adj (POS tag) +# word embedding cosine sim based on part of speech: noun, verb, adj + +import string +from collections import defaultdict + +import numpy as np + +import nltk +nltk.download('stopwords') + +from nltk.stem.porter import PorterStemmer +from nltk.corpus import stopwords + +def stopped(sentences): + """ + remove stop words from given sentences (questions|answers) + """ + stoplist = set(stopwords.words('english')) + #stoplist.update(set(string.punctuation)) + def stop(sentence): + return ' '.join([word for word in sentence.split() if word not in stoplist]) + return [stop(sentence) for sentence in sentences] + +def stemmed(sentences): + """ + reduce sentence terms to stemmed representations + """ + stemmer = PorterStemmer() + def stem(sentence): + return ' '.join([stemmer.stem(word) for word in sentence.split()]) + return [stem(sentence) for sentence in sentences] + +def get_qadata_only_idf(all_data): + """ + returns idf weights computed over all question answer pairs in the dataset + """ + if not type(all_data) is list: + all_data = list(all_data) + term_idfs = defaultdict(float) + for doc in all_data: + for term in list(set(doc.split())): + term_idfs[term] += 1.0 + N = len(all_data) + for term, n_t in term_idfs.items(): + term_idfs[term] = np.log(N/(1+n_t)) + return term_idfs + +def get_source_corpus_idf(all_data): + """ + fetches idf weights from source corpus (disks1-5+aquaint|wikipedia) index, for all the qa pairs + """ + pass + +def compute_overlap(questions, answers): + """ + returns simple overlap between document pairs + """ + overlap_scores = [] + for q, a in zip(questions, answers): + q_terms = set(q.split()) + a_terms = set(a.split()) + common_terms = q_terms.intersection(a_terms) + overlap = float(len(common_terms))/(len(q_terms) + len(a_terms)) + overlap_scores.append(overlap) + return np.array(overlap_scores) + +def compute_idf_weighted_overlap(questions, answers, idf_weights): + """ + returns idf weighted overlap + """ + overlap_scores = [] + for q, a in zip(questions, answers): + q_terms = set(q.split()) + a_terms = set(a.split()) + common_terms = q_terms.intersection(a_terms) + idf_weighted_overlap = np.sum([idf_weights[term] for term in list(common_terms)]) + idf_weighted_overlap /= (len(q_terms) + len(a_terms)) + overlap_scores.append(idf_weighted_overlap) + return np.array(overlap_scores) + + +def set_external_features_as_per_paper(trainer): + """ + computes external features as per the paper AND saves them into trainer + """ + all_questions, all_answers = [], [] + for split in trainer.data_splits.keys(): + questions, answers, labels, max_q_len, max_a_len, default_ext_feats = \ + trainer.data_splits[split] + all_questions.extend(questions) + all_answers.extend(answers) + + all_data = set(all_questions + all_answers) + idf_weights = get_qadata_only_idf(list(all_data)) + + external_features = {} + + # NOTE: expected external features as per paper are + # 1. overlap(q, a), + # 2. idf_overlap(q, a), + # 3. overlap(stopped(q), stopped(a)), + # 4. idf_over(stopped(q), stopped(a)) + + for split in trainer.data_splits.keys(): + questions, answers, labels, max_q_len, max_a_len, default_ext_feats = \ + trainer.data_splits[split] + + overlap = compute_overlap(questions, answers) + idf_weighted_overlap = compute_idf_weighted_overlap(questions, answers, idf_weights) + overlap_no_stopwords =\ + compute_overlap(stopped(questions), stopped(answers)) + idf_weighted_overlap_no_stopwords =\ + compute_idf_weighted_overlap(stopped(questions), stopped(answers), idf_weights) + ext_feats = [np.array(feats) for feats in zip(overlap, idf_weighted_overlap,\ + overlap_no_stopwords, idf_weighted_overlap_no_stopwords)] + trainer.data_splits[split][-1] = ext_feats + external_features[split] = ext_feats + return external_features + + +def set_external_features_as_per_paper_and_stem(trainer): + """ + computes external features as per the paper but performs stemming before computing IDF. + features are saved into the trainer.data_splits + """ + all_questions, all_answers = [], [] + for split in trainer.data_splits.keys(): + questions, answers, labels, max_q_len, max_a_len, default_ext_feats = \ + trainer.data_splits[split] + all_questions.extend(questions) + all_answers.extend(answers) + + all_data = set(all_questions + all_answers) + + # stem all words except stopwords to compute idf (required for feature number 2.) + stoplist = set(stopwords.words('english')) + stemmer = PorterStemmer() + def stem_non_stop_words(sentence): + return ' '.join([stemmer.stem(word) if word not in stoplist else word \ + for word in sentence.split()]) + all_but_stopwords_stemmed = [stem_non_stop_words(sentence) for sentence in list(all_data)] + idf_weights = get_qadata_only_idf(all_but_stopwords_stemmed) + + external_features = {} + + for split in trainer.data_splits.keys(): + questions, answers, labels, max_q_len, max_a_len, default_ext_feats = \ + trainer.data_splits[split] + + que_stem_all_but_stopwords = [stem_non_stop_words(que) for que in questions] + ans_stem_all_but_stopwords = [stem_non_stop_words(ans) for ans in answers] + + overlap = compute_overlap(que_stem_all_but_stopwords, ans_stem_all_but_stopwords) + idf_weighted_overlap = compute_idf_weighted_overlap(que_stem_all_but_stopwords,\ + ans_stem_all_but_stopwords, idf_weights) + + que_stopped_stemmed = stemmed(stopped(questions)) + ans_stopped_stemmed = stemmed(stopped(answers)) + + overlap_no_stopwords = compute_overlap(que_stopped_stemmed, ans_stopped_stemmed) + idf_weighted_overlap_no_stopwords =\ + compute_idf_weighted_overlap(que_stopped_stemmed, ans_stopped_stemmed, idf_weights) + ext_feats = [np.array(feats) for feats in zip(overlap, idf_weighted_overlap,\ + overlap_no_stopwords, idf_weighted_overlap_no_stopwords)] + trainer.data_splits[split][-1] = ext_feats + external_features[split] = ext_feats + return external_features \ No newline at end of file diff --git a/sm_model/main.py b/sm_model/main.py index 334a486..da2832d 100644 --- a/sm_model/main.py +++ b/sm_model/main.py @@ -10,6 +10,9 @@ import pandas as pd import torch import utils +from external_features import stopped, stemmed, compute_idf_weighted_overlap, compute_overlap,\ + get_qadata_only_idf, set_external_features_as_per_paper,\ + set_external_features_as_per_paper_and_stem from train import Trainer from model import QAModel @@ -31,7 +34,7 @@ def logargs(func): return inner -def compute_map_mrr(dataset_folder, set_folder, test_scores): +def compute_map_mrr(dataset_folder, set_folder, test_scores, run_name_prefix=None): # logger.info("Running trec_eval script...") N = len(test_scores) @@ -46,8 +49,11 @@ def compute_map_mrr(dataset_folder, set_folder, test_scores): df_submission['rank'] = 0 df_submission['sim'] = test_scores df_submission['run_id'] = 'smmodel' - df_submission.to_csv(os.path.join(args.dataset_folder, 'submission.txt'), \ + df_submission.to_csv(os.path.join(dataset_folder, 'submission.txt'), \ header=False, index=False, sep=' ') + if run_name_prefix: + df_submission.to_csv('{}.{}.smrun'.format(run_name_prefix, set_folder),\ + header=False, index=False, sep=" ") df_gold = pd.DataFrame(index=np.arange(N), columns=['qid', 'iter', 'docno', 'rel']) df_gold['qid'] = qids_test @@ -57,7 +63,7 @@ def compute_map_mrr(dataset_folder, set_folder, test_scores): df_gold.to_csv(os.path.join(args.dataset_folder, 'gold.txt'), header=False, index=False, sep=' ') # subprocess.call("/bin/sh run_eval.sh '{}'".format(args.dataset_folder), shell=True) - pargs = shlex.split("/bin/sh run_eval.sh '{}'".format(args.dataset_folder)) + pargs = shlex.split("/bin/sh run_eval.sh '{}'".format(dataset_folder)) p = subprocess.Popen(pargs, stdout=subprocess.PIPE, stderr=subprocess.PIPE) pout, perr = p.communicate() @@ -67,53 +73,53 @@ def compute_map_mrr(dataset_folder, set_folder, test_scores): return map, mrr - if __name__ == "__main__": ap = argparse.ArgumentParser(description='pytorch port of the SM model', \ formatter_class=argparse.ArgumentDefaultsHelpFormatter) - ap.add_argument('model_outfile', help='file to save final model') - ap.add_argument('--word_vectors_file', \ help='NOTE: a cache will be created for faster loading for word vectors', default="../../data/word2vec/aquaint+wiki.txt.gz.ndim=50.bin") ap.add_argument('--dataset_folder', help='directory containing train, dev, test sets', \ default="../../data/TrecQA") - ap.add_argument('--classes', type=int, default=2) - + # external features related arguments + ap.add_argument('--no-ext-feats', action="store_true", \ + help="will not include external features in the model") + ap.add_argument('--paper-ext-feats', action="store_true", \ + help="external features as per the paper") + ap.add_argument('--paper-ext-feats-stem', action="store_true", \ + help="external features as per the paper") # system arguments # TODO: add arguments for CUDA ap.add_argument('--num_threads', help="the number of simultaneous processes to run", \ type=int, default=4) - # training arguments ap.add_argument('--batch_size', type=int, default=1, help="training mini-batch size") ap.add_argument('--filter_width', type=int, default=5, help="number of convolution channels") ap.add_argument('--eta', help='Initial learning rate', default=0.001, type=float) ap.add_argument('--mom', help='SGD Momentum', default=0.0, type=float) ap.add_argument('--train', help='switches to train set', action="store_true") - # epoch related arguments ap.add_argument('--epochs', type=int, default=25, help="number of trainin epochs") ap.add_argument('--patience', type=int, default=5, \ help="if there is no appreciable change in model after epochs, then stop") - # debugging arguments ap.add_argument('--debug_single_batch', action="store_true", \ help="will stop program after training 1 input batch") ap.add_argument('--num_conv_filters', default=100, type=int, \ help="the number of convolution channels (lesser is faster)") - ap.add_argument('--no_ext_feats', action="store_true", \ - help="will not include external features in the model") ap.add_argument('--no_loss_reg', help="no loss regularization", action="store_true") ap.add_argument('--test_on_each_epoch', action="store_true", \ help='runs test on each epoch to track final performance') + ap.add_argument("--skip-training", help="will load pre-trained model", action="store_true") + ap.add_argument("--run-name-prefix", help="will output train|dev|test runs with provided prefix") args = ap.parse_args() torch.manual_seed(1234) np.random.seed(1234) + torch.set_num_threads(args.num_threads) train_set, dev_set, test_set = 'train-all', 'raw-dev', 'raw-test' if args.train: @@ -127,50 +133,65 @@ if __name__ == "__main__": # instantiate model net = QAModel(vec_dim, args.filter_width, args.num_conv_filters, args.no_ext_feats) - QAModel.save(net, args.model_outfile) - - torch.set_num_threads(args.num_threads) + # initialize the trainer trainer = Trainer(net, args.eta, args.mom, args.no_loss_reg, vec_dim) logger.info("Loading input data...") + # load input data trainer.load_input_data(args.dataset_folder, cache_file, train_set, dev_set, test_set) + logger.info("Setting up external features...") + # setup external features + # TODO: remember to update args.* in testing loop below + if args.paper_ext_feats: + logger.info("--paper-ext-feats") + ext_feats_for_splits = set_external_features_as_per_paper(trainer) + # ^^ we are saving the features to be used while testing at the end of training + elif args.paper_ext_feats_stem: + logger.info("--paper-ext-feats-stem") + ext_feats_for_splits = set_external_features_as_per_paper_and_stem(trainer) - best_map = 0.0 - best_model = 0 + if not args.skip_training: + best_map = 0.0 + best_model = 0 - for i in range(args.epochs): - logger.info('------------- Training epoch {} --------------'.format(i+1)) - train_accuracy = trainer.train(train_set, args.batch_size, args.debug_single_batch) - if args.debug_single_batch: sys.exit(0) + for i in range(args.epochs): + logger.info('------------- Training epoch {} --------------'.format(i+1)) + train_accuracy = trainer.train(train_set, args.batch_size, args.debug_single_batch) + if args.debug_single_batch: sys.exit(0) - dev_scores = trainer.test(dev_set, args.batch_size) + dev_scores = trainer.test(dev_set, args.batch_size) - dev_map, dev_mrr = compute_map_mrr(args.dataset_folder, dev_set, dev_scores) - logger.info("------- MAP {}, MRR {}".format(dev_map, dev_mrr)) + dev_map, dev_mrr = compute_map_mrr(args.dataset_folder, dev_set, dev_scores) + logger.info("------- MAP {}, MRR {}".format(dev_map, dev_mrr)) - if dev_map - best_map > 1e-3: # new map is better than best map - best_model = i - best_map = dev_map + if dev_map - best_map > 1e-3: # new map is better than best map + best_model = i + best_map = dev_map - QAModel.save(net, args.model_outfile) - logger.info('Achieved better dev_map ... saved model') + QAModel.save(net, args.model_outfile) + logger.info('Achieved better dev_map ... saved model') - if args.test_on_each_epoch: - test_scores = trainer.test(test_set, args.batch_size) - map, mrr = compute_map_mrr(args.dataset_folder, test_set, test_scores) - logger.info("------- MAP {}, MRR {}".format(map, mrr)) + if args.test_on_each_epoch: + test_scores = trainer.test(test_set, args.batch_size) + map, mrr = compute_map_mrr(args.dataset_folder, test_set, test_scores) + logger.info("------- MAP {}, MRR {}".format(map, mrr)) - if (i - best_model) >= args.patience: - logger.warning('No improvement since the last {} epochs. Stopping training'.format(i - best_model)) - break + if (i - best_model) >= args.patience: + logger.warning('No improvement since the last {} epochs. Stopping training'\ + .format(i - best_model)) + break - logger.info(' ------------ Training epochs completed! ------------') - logger.info('Best MAP in training phase = {:.4f}'.format(best_map)) + logger.info(' ------------ Training epochs completed! ------------') + logger.info('Best dev MAP in training phase = {:.4f}'.format(best_map)) trained_model = QAModel.load(args.model_outfile) evaluator = Trainer(trained_model, args.eta, args.mom, args.no_loss_reg, vec_dim) - evaluator.load_input_data(args.dataset_folder, cache_file, None, None, test_set) - test_scores = evaluator.test(test_set, args.batch_size) - map, mrr = compute_map_mrr(args.dataset_folder, test_set, test_scores) - logger.info("------- MAP {}, MRR {}".format(map, mrr)) + for split in [test_set, dev_set, train_set]: + evaluator.load_input_data(args.dataset_folder, cache_file, None, None, split) + if args.paper_ext_feats or args.paper_ext_feats_stem: + evaluator.data_splits[split][-1] = ext_feats_for_splits[split] + #set_external_features_as_per_paper(evaluator) + split_scores = evaluator.test(split, args.batch_size) + map, mrr = compute_map_mrr(args.dataset_folder, split, split_scores, args.run_name_prefix) + logger.info("-------{} MAP {}, MRR {}".format(split, map, mrr)) diff --git a/sm_model/make_run.py b/sm_model/make_run.py new file mode 100644 index 0000000..b1044d6 --- /dev/null +++ b/sm_model/make_run.py @@ -0,0 +1,51 @@ +import argparse +import os +from model import QAModel +from train import Trainer +import utils + +if __name__ == "__main__": + + ap = argparse.ArgumentParser(description="Makes a run in trec_eval run format, given a model and a train|dev|test set" ) + ap.add_argument('model') + ap.add_argument('--word_embeddings_cache', + default='../../data/word2vec/aquaint+wiki.txt.gz.ndim=50.cache') + ap.add_argument('--paper-ext-feats', action="store_true", \ + help="external features as per the paper") + ap.add_argument('dataset_folder', help="the QA dataset folder", + choices=['../../data/TrecQA', '../../data/WikQA']) + ap.add_argument('set_split', help="train, dev or test split as the data_folder") + ap.add_argument("batch_size", help="the number of pairs to compare in each batch.\ + should be same as during training") + ap.add_argument('out_scorefile', help='output file in trec_eval format') + + + args = ap.parse_args() + + vocab_size, vec_dim = utils.load_embedding_dimensions(args.word_embeddings_cache) + + trained_model = QAModel.load(args.model) + trained_model.no_ext_feats = True + evaluator = Trainer(trained_model, 0, 0, False, vec_dim) # 0, 0, False are dummy arguments + evaluator.load_input_data(args.dataset_folder, args.word_embeddings_cache, + None, None, args.set_split, + True if args.ext_feats else False) + test_scores = evaluator.test(args.set_split, args.batch_size) + + questions, sentences, labels, vocab, maxlen_q, maxlen_s, ext_feats = \ + evaluator.data_splits[args.set_split] + + qids = [id.strip() for id in open(os.path.join(args.dataset_folder, args.set_split, 'id.txt'))\ + .readlines()] + + with open(args.out_scorefile, 'w') as outf: + old_qid = 0 + docid_c = 0 + for i in range(len(qids)): + if qids[i] != old_qid and args.dataset_folder.endswith('WikiQA'): + docid_c = 0 + old_qid = qids[i] + print('{} 0 {} 0 {} {}'.format(qids[i], docid_c, test_scores[i], + os.path.basename(args.model)), + file=outf) + docid_c += 1 diff --git a/sm_model/model.py b/sm_model/model.py index a820938..71ac9d9 100644 --- a/sm_model/model.py +++ b/sm_model/model.py @@ -33,7 +33,7 @@ class QAModel(nn.Module): self.no_ext_feats = no_ext_feats self.conv_channels = conv_filters - n_hidden = 2*self.conv_channels + 1 + n_hidden = 2*self.conv_channels + (0 if no_ext_feats else ext_feats_size) self.conv_q = nn.Sequential( nn.Conv1d(input_n_dim, self.conv_channels, filter_width, padding=filter_width-1), @@ -47,7 +47,7 @@ class QAModel(nn.Module): self.combined_feature_vector = nn.Linear(2*self.conv_channels + \ (0 if no_ext_feats else ext_feats_size), n_hidden) - # TODO: add +1 to Linear layer^. Will need change in forward function + self.combined_features_activation = nn.Tanh() self.dropout = nn.Dropout(0.5) self.hidden = nn.Linear(n_hidden, n_classes) diff --git a/sm_model/requirements.txt b/sm_model/requirements.txt new file mode 100644 index 0000000..8e4d407 --- /dev/null +++ b/sm_model/requirements.txt @@ -0,0 +1,5 @@ +gensim==1.0.1 +nltk==3.2.1 +numpy==1.11.3 +pandas==0.19.2 +torch==0.1.11+b13b701 diff --git a/sm_model/train.py b/sm_model/train.py index 3ce3270..a73a457 100644 --- a/sm_model/train.py +++ b/sm_model/train.py @@ -37,20 +37,25 @@ class Trainer(object): self.optimizer = optim.SGD(self.model.parameters(), lr=eta, momentum=mom, \ weight_decay=(0 if no_loss_reg else self.reg)) - self.datasets = {} + self.data_splits = {} self.embeddings = {} self.vec_dim = vec_dim def load_input_data(self, dataset_root_folder, word_vectors_cache_file, \ - train_set_folder, dev_set_folder, test_set_folder): + train_set_folder, dev_set_folder, test_set_folder, load_ext_feats=True): for set_folder in [test_set_folder, dev_set_folder, train_set_folder]: if set_folder: - self.datasets[set_folder] = utils.read_in_dataset(dataset_root_folder, set_folder) - # NOTE: self.datasets[set_folder] = questions, sentences, labels, - # vocab, maxlen_q, maxlen_s, ext_feats + questions, sentences, labels, maxlen_q, maxlen_s, vocab = \ + utils.read_in_dataset(dataset_root_folder, set_folder) + + self.data_splits[set_folder] = [questions, sentences, labels, maxlen_q, maxlen_s] + + default_ext_feats = [np.zeros(4)] * len(self.data_splits[set_folder][0]) + self.data_splits[set_folder].append(default_ext_feats) + self.embeddings[set_folder] = utils.load_cached_embeddings( \ - word_vectors_cache_file, self.datasets[set_folder][3], \ + word_vectors_cache_file, vocab, \ [] if "train" in set_folder else self.unk_term) @@ -113,8 +118,8 @@ class Trainer(object): def test(self, set_folder, batch_size): logger.info('----- Predictions on {} '.format(set_folder)) - questions, sentences, labels, vocab, maxlen_q, maxlen_s, ext_feats = \ - self.datasets[set_folder] + questions, sentences, labels, maxlen_q, maxlen_s, ext_feats = \ + self.data_splits[set_folder] word_vectors, vec_dim = self.embeddings[set_folder], self.vec_dim self.model.eval() @@ -136,7 +141,7 @@ class Trainer(object): sentences[batch_start:batch_end], labels[batch_start:batch_end], ext_feats[batch_start:batch_end], - word_vectors, vocab, vec_dim + word_vectors, vec_dim ) xq, xa, x_ext_feats = batch_inputs[0] @@ -164,8 +169,8 @@ class Trainer(object): def train(self, set_folder, batch_size, debug_single_batch): train_start_time = time.time() - questions, sentences, labels, vocab, maxlen_q, maxlen_s, ext_feats = \ - self.datasets[set_folder] + questions, sentences, labels, maxlen_q, maxlen_s, ext_feats = \ + self.data_splits[set_folder] word_vectors, vec_dim = self.embeddings[set_folder], self.vec_dim # set model for training modep @@ -184,7 +189,7 @@ class Trainer(object): sentences[batch_start:batch_end], labels[batch_start:batch_end], ext_feats[batch_start:batch_end], - word_vectors, vocab, vec_dim + word_vectors, vec_dim ) xq, xa, x_ext_feats = batch_inputs[0] @@ -225,7 +230,7 @@ class Trainer(object): def get_tensorized_inputs(self, batch_ques, batch_sents, batch_labels, batch_ext_feats, \ - word_vectors, vocab, vec_dim): + word_vectors, vec_dim): batch_size = len(batch_ques) # NOTE: ideal batch size is one, because sentences are all of different length. # In other words, we have no option but to feed in sentences one by one into the model diff --git a/sm_model/utils.py b/sm_model/utils.py index 064828e..9fac336 100644 --- a/sm_model/utils.py +++ b/sm_model/utils.py @@ -107,14 +107,15 @@ def read_in_dataset(dataset_folder, set_folder): len_s_list = [len(s.split()) for s in sentences] labels = [int(line.strip()) for line in open(os.path.join(set_path, 'sim.txt')).readlines()] - ext_feats = np.array([list(map(float, line.strip().split(' '))) \ - for line in open(os.path.join(set_path, 'overlap_feats.txt')).readlines()]) - #y = torch.from_numpy(labels) - #return questions, sentences, y + # ext_feats = [np.zeros(4)] * len(questions) + # if load_ext_features: + # ext_feats = np.array([list(map(float, line.strip().split(' '))) \ + # for line in open(os.path.join(set_path, 'overlap_feats.txt')).readlines()]) vocab = [line.strip() for line in open(os.path.join(dataset_folder, 'vocab.txt')).readlines()] - return questions, sentences, labels, vocab, max(len_q_list), max(len_s_list), ext_feats + + return [questions, sentences, labels, max(len_q_list), max(len_s_list), vocab] def get_test_qids_labels(dataset_folder, set_folder):