Ext feats bug fix (#19)

+ sm model no external features baseline
+ sm model with IDF weights
+ sm model with IDF weights without removing punctuation --> barely better than df/idf (a la Pytorch).
+ sm model with stemming before computing IDF weights
^ all on the TrecQA dataset

baseline_results.tsv

@@ -1,14 +1,18 @@
 Baseline_method	idf	condition	dataset	train_MAP	dev_MAP	test_MAP	Notes/Comments
 idf_sum_similarity	dataset	no_stopping + no_stemming	TrecQA	0.7744	0.7668	0.7014	IDF is computed over terms in the train, dev and test sets
 idf_sum_similarity	dataset	with_stopping + with_stemming	TrecQA	0.7078	0.7404	0.6816	
+idf_sum_similarity	dataset	with_stopping + with_stemming + stop(punct)	TrecQA	0.7074	0.7337	0.6805	also stopping punctuation
 idf_sum_similarity	dataset	no_stopping + no_stemming	WikiQA	0.2429	0.2489	0.2219	There is less overlap between questions and answers in the WikiQA dataset
 idf_sum_similarity	dataset	with_stopping + with_stemming	WikiQA	0.2232	0.2472	0.2025	
-idf_sum_similarity	Corpus	no_stopping + no_stemming	TrecQA	0.6736	0.728	0.6377	IDF is computed over documents in disks1-5 and aquaint
-idf_sum_similarity	Corpus	with_stopping + with_stemming	TrecQA	0.7032	0.7281	0.672	
-idf_sum_similarity	Corpus	no_stopping + no_stemming	WikiQA	0.2432	0.244	0.2206	IDF is computed over documents in Wikipedia
-idf_sum_similarity	Corpus	with_stopping + with_stemming	WikiQA	0.2212	0.2461	0.2005	
-sm_model	NA	no_idf_overlap_features	TrecQA				
-sm_model	Dataset 	with_corrected_idf_overlap_features	TrecQA				
+idf_sum_similarity	dataset	with_stopping + with_stemming + stop(punct)	WikiQA	0.2236	0.2472	0.2025	
+							
+idf_sum_similarity	Corpus	with_stopping + with_stemming	TrecQA	0.7032	0.7281	0.672	IDF is computed over documents in disks1-5 and aquaint
+idf_sum_similarity	Corpus	with_stopping + with_stemming	WikiQA	0.2212	0.2461	0.2005	IDF is computed over documents in Wikipedia
+							
+sm_model	NA	no_idf_overlap_features	TrecQA	0.597	0.6918	0.6445	
+sm_model	Dataset 	with_corrected_idf_overlap_features	TrecQA	0.919	0.7927	0.7384	S&M model uses stopping only
+sm_model	Dataset 	with_corrected_idf_overlap_features – stop(punct)	TrecQA	0.8325	0.7948	0.7562	did not remove punctuation
+sm_model	Dataset 	with_stemmed_idf_overlap_features 	TrecQA	0.9414	0.8206	0.7363	
 sm_model	Corpus	with_corrected_idf_overlap_features	TrecQA				
 sm_model	Dataset	with_fractional_idf_overlap_features	TrecQA				
 sm_model	Corpus	with_fractional_idf_overlap_features	TrecQA				

idf_baseline/qa-data-only-idf.py

@@ -2,6 +2,7 @@ import argparse
 import os
 import numpy as np
 from collections import defaultdict
+import string
 
 import nltk
 nltk.download('stopwords')
@@ -16,6 +17,7 @@ def read_in_data(datapath, set_name, file, stop_and_stem=False):
         if stop_and_stem:
             stemmer = PorterStemmer()
             stoplist = set(stopwords.words('english'))
+            stoplist.update(set(string.punctuation))
             def stop_stem(sentence):
                 return ' '.join([stemmer.stem(word) for word in sentence.split() \
                                                         if word not in stoplist])
@@ -57,8 +59,8 @@ def write_out_idf_sum_similarities(qids, questions, answers, term_idfs, outfile,
             if qids[i] != old_qid and dataset.endswith('WikiQA'):
                 docid_c = 0
                 old_qid = qids[i]
-            print('{} 0 {} 0 {} data_only_idfbaseline'.format(qids[i], docid_c, 
-                  idf_sum_similarity[i]),
+            print('{} 0 {} 0 {} data_only_idfbaseline'.format(qids[i], docid_c,
+                                                              idf_sum_similarity[i]),
                   file=outf)
             docid_c += 1
 
@@ -81,7 +83,7 @@ if __name__ == "__main__":
         train_data, dev_data, test_data = 'train-all', 'raw-dev', 'raw-test'
 
     train_que = read_in_data(args.qa_data, train_data, 'a.toks', args.stop_and_stem)
-    train_ans = read_in_data(args.qa_data, train_data, 'b.toks', args.stop_and_stem)    
+    train_ans = read_in_data(args.qa_data, train_data, 'b.toks', args.stop_and_stem)
 
     dev_que = read_in_data(args.qa_data, dev_data, 'a.toks', args.stop_and_stem)
     dev_ans = read_in_data(args.qa_data, dev_data, 'b.toks', args.stop_and_stem)

sm_model/external_features.py

@@ -0,0 +1,171 @@
+# module to compute various external features for the sm_model.
+# TODO: add more external features like:
+# word mover distance, cosine sim in tf.idf space, cosine sim in word embedding space
+# overlap based on parts of speech: noun, verb, adj (POS tag)
+# word embedding cosine sim based on part of speech: noun, verb, adj
+
+import string
+from collections import defaultdict
+
+import numpy as np
+
+import nltk
+nltk.download('stopwords')
+
+from nltk.stem.porter import PorterStemmer
+from nltk.corpus import stopwords
+
+def stopped(sentences):
+    """
+    remove stop words from given sentences (questions|answers)
+    """
+    stoplist = set(stopwords.words('english'))
+    #stoplist.update(set(string.punctuation))
+    def stop(sentence):
+        return ' '.join([word for word in sentence.split() if word not in stoplist])
+    return [stop(sentence) for sentence in sentences]
+
+def stemmed(sentences):
+    """
+    reduce sentence terms to stemmed representations
+    """
+    stemmer = PorterStemmer()
+    def stem(sentence):
+        return ' '.join([stemmer.stem(word) for word in sentence.split()])
+    return [stem(sentence) for sentence in sentences]
+
+def get_qadata_only_idf(all_data):
+    """
+    returns idf weights computed over all question answer pairs in the dataset
+    """
+    if not type(all_data) is list:
+        all_data = list(all_data)
+    term_idfs = defaultdict(float)
+    for doc in all_data:
+        for term in list(set(doc.split())):
+            term_idfs[term] += 1.0
+    N = len(all_data)
+    for term, n_t in term_idfs.items():
+        term_idfs[term] = np.log(N/(1+n_t))
+    return term_idfs
+
+def get_source_corpus_idf(all_data):
+    """
+    fetches idf weights from source corpus (disks1-5+aquaint|wikipedia) index, for all the qa pairs
+    """
+    pass
+
+def compute_overlap(questions, answers):
+    """
+    returns simple overlap between document pairs
+    """
+    overlap_scores = []
+    for q, a in zip(questions, answers):
+        q_terms = set(q.split())
+        a_terms = set(a.split())
+        common_terms = q_terms.intersection(a_terms)
+        overlap = float(len(common_terms))/(len(q_terms) + len(a_terms))
+        overlap_scores.append(overlap)
+    return np.array(overlap_scores)
+
+def compute_idf_weighted_overlap(questions, answers, idf_weights):
+    """
+    returns idf weighted overlap
+    """
+    overlap_scores = []
+    for q, a in zip(questions, answers):
+        q_terms = set(q.split())
+        a_terms = set(a.split())
+        common_terms = q_terms.intersection(a_terms)
+        idf_weighted_overlap = np.sum([idf_weights[term] for term in list(common_terms)])
+        idf_weighted_overlap /= (len(q_terms) + len(a_terms))
+        overlap_scores.append(idf_weighted_overlap)
+    return np.array(overlap_scores)
+
+
+def set_external_features_as_per_paper(trainer):
+    """
+    computes external features as per the paper AND saves them into trainer
+    """
+    all_questions, all_answers = [], []
+    for split in trainer.data_splits.keys():
+        questions, answers, labels, max_q_len, max_a_len, default_ext_feats = \
+            trainer.data_splits[split]
+        all_questions.extend(questions)
+        all_answers.extend(answers)
+
+    all_data = set(all_questions + all_answers)
+    idf_weights = get_qadata_only_idf(list(all_data))
+
+    external_features = {}
+
+    # NOTE: expected external features as per paper are
+    # 1. overlap(q, a),
+    # 2. idf_overlap(q, a),
+    # 3. overlap(stopped(q), stopped(a)),
+    # 4. idf_over(stopped(q), stopped(a))
+
+    for split in trainer.data_splits.keys():
+        questions, answers, labels, max_q_len, max_a_len, default_ext_feats = \
+            trainer.data_splits[split]
+
+        overlap = compute_overlap(questions, answers)
+        idf_weighted_overlap = compute_idf_weighted_overlap(questions, answers, idf_weights)
+        overlap_no_stopwords =\
+            compute_overlap(stopped(questions), stopped(answers))
+        idf_weighted_overlap_no_stopwords =\
+            compute_idf_weighted_overlap(stopped(questions), stopped(answers), idf_weights)
+        ext_feats = [np.array(feats) for feats in zip(overlap, idf_weighted_overlap,\
+                    overlap_no_stopwords, idf_weighted_overlap_no_stopwords)]
+        trainer.data_splits[split][-1] = ext_feats
+        external_features[split] = ext_feats
+    return external_features
+
+
+def set_external_features_as_per_paper_and_stem(trainer):
+    """
+    computes external features as per the paper but performs stemming before computing IDF.
+    features are saved into the trainer.data_splits
+    """
+    all_questions, all_answers = [], []
+    for split in trainer.data_splits.keys():
+        questions, answers, labels, max_q_len, max_a_len, default_ext_feats = \
+            trainer.data_splits[split]
+        all_questions.extend(questions)
+        all_answers.extend(answers)
+
+    all_data = set(all_questions + all_answers)
+
+    # stem all words except stopwords to compute idf (required for feature number 2.)
+    stoplist = set(stopwords.words('english'))
+    stemmer = PorterStemmer()
+    def stem_non_stop_words(sentence):
+        return ' '.join([stemmer.stem(word) if word not in stoplist else word \
+                    for word in sentence.split()])
+    all_but_stopwords_stemmed = [stem_non_stop_words(sentence) for sentence in list(all_data)]
+    idf_weights = get_qadata_only_idf(all_but_stopwords_stemmed)
+
+    external_features = {}
+
+    for split in trainer.data_splits.keys():
+        questions, answers, labels, max_q_len, max_a_len, default_ext_feats = \
+            trainer.data_splits[split]
+
+        que_stem_all_but_stopwords = [stem_non_stop_words(que) for que in questions]
+        ans_stem_all_but_stopwords = [stem_non_stop_words(ans) for ans in answers]
+
+        overlap = compute_overlap(que_stem_all_but_stopwords, ans_stem_all_but_stopwords)
+        idf_weighted_overlap = compute_idf_weighted_overlap(que_stem_all_but_stopwords,\
+                                                            ans_stem_all_but_stopwords, idf_weights)
+
+        que_stopped_stemmed = stemmed(stopped(questions))
+        ans_stopped_stemmed = stemmed(stopped(answers))
+
+        overlap_no_stopwords = compute_overlap(que_stopped_stemmed, ans_stopped_stemmed)
+        idf_weighted_overlap_no_stopwords =\
+            compute_idf_weighted_overlap(que_stopped_stemmed, ans_stopped_stemmed, idf_weights)
+        ext_feats = [np.array(feats) for feats in zip(overlap, idf_weighted_overlap,\
+                    overlap_no_stopwords, idf_weighted_overlap_no_stopwords)]
+        trainer.data_splits[split][-1] = ext_feats
+        external_features[split] = ext_feats
+    return external_features

sm_model/main.py

@@ -10,6 +10,9 @@ import pandas as pd
 import torch
 
 import utils
+from external_features import stopped, stemmed, compute_idf_weighted_overlap, compute_overlap,\
+    get_qadata_only_idf, set_external_features_as_per_paper,\
+    set_external_features_as_per_paper_and_stem
 from train import Trainer
 from model import QAModel
 
@@ -31,7 +34,7 @@ def logargs(func):
     return inner
 
 
-def compute_map_mrr(dataset_folder, set_folder, test_scores):
+def compute_map_mrr(dataset_folder, set_folder, test_scores, run_name_prefix=None):
     # logger.info("Running trec_eval script...")
     N = len(test_scores)
 
@@ -46,8 +49,11 @@ def compute_map_mrr(dataset_folder, set_folder, test_scores):
     df_submission['rank'] = 0
     df_submission['sim'] = test_scores
     df_submission['run_id'] = 'smmodel'
-    df_submission.to_csv(os.path.join(args.dataset_folder, 'submission.txt'), \
+    df_submission.to_csv(os.path.join(dataset_folder, 'submission.txt'), \
         header=False, index=False, sep=' ')
+    if run_name_prefix:
+        df_submission.to_csv('{}.{}.smrun'.format(run_name_prefix, set_folder),\
+                             header=False, index=False, sep=" ")
 
     df_gold = pd.DataFrame(index=np.arange(N), columns=['qid', 'iter', 'docno', 'rel'])
     df_gold['qid'] = qids_test
@@ -57,7 +63,7 @@ def compute_map_mrr(dataset_folder, set_folder, test_scores):
     df_gold.to_csv(os.path.join(args.dataset_folder, 'gold.txt'), header=False, index=False, sep=' ')
 
     # subprocess.call("/bin/sh run_eval.sh '{}'".format(args.dataset_folder), shell=True)
-    pargs = shlex.split("/bin/sh run_eval.sh '{}'".format(args.dataset_folder))
+    pargs = shlex.split("/bin/sh run_eval.sh '{}'".format(dataset_folder))
     p = subprocess.Popen(pargs, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
     pout, perr = p.communicate()
 
@@ -67,53 +73,53 @@ def compute_map_mrr(dataset_folder, set_folder, test_scores):
     return map, mrr
 
 
-
 if __name__ == "__main__":
     ap = argparse.ArgumentParser(description='pytorch port of the SM model', \
         formatter_class=argparse.ArgumentDefaultsHelpFormatter)
-
     ap.add_argument('model_outfile', help='file to save final model')
-
     ap.add_argument('--word_vectors_file', \
         help='NOTE: a cache will be created for faster loading for word vectors',
         default="../../data/word2vec/aquaint+wiki.txt.gz.ndim=50.bin")
     ap.add_argument('--dataset_folder', help='directory containing train, dev, test sets', \
         default="../../data/TrecQA")
-
     ap.add_argument('--classes', type=int, default=2)
-
+    # external features related arguments
+    ap.add_argument('--no-ext-feats', action="store_true", \
+        help="will not include external features in the model")
+    ap.add_argument('--paper-ext-feats', action="store_true", \
+        help="external features as per the paper")
+    ap.add_argument('--paper-ext-feats-stem', action="store_true", \
+        help="external features as per the paper")
     # system arguments
     # TODO: add arguments for CUDA
     ap.add_argument('--num_threads', help="the number of simultaneous processes to run", \
         type=int, default=4)
-
     # training arguments
     ap.add_argument('--batch_size', type=int, default=1, help="training mini-batch size")
     ap.add_argument('--filter_width', type=int, default=5, help="number of convolution channels")
     ap.add_argument('--eta', help='Initial learning rate', default=0.001, type=float)
     ap.add_argument('--mom', help='SGD Momentum', default=0.0, type=float)
     ap.add_argument('--train', help='switches to train set', action="store_true")
-
     # epoch related arguments
     ap.add_argument('--epochs', type=int, default=25, help="number of trainin epochs")
     ap.add_argument('--patience', type=int, default=5, \
         help="if there is no appreciable change in model after <patience> epochs, then stop")
-
     # debugging arguments
     ap.add_argument('--debug_single_batch', action="store_true", \
         help="will stop program after training 1 input batch")
     ap.add_argument('--num_conv_filters', default=100, type=int, \
         help="the number of convolution channels (lesser is faster)")
-    ap.add_argument('--no_ext_feats', action="store_true", \
-        help="will not include external features in the model")
     ap.add_argument('--no_loss_reg', help="no loss regularization", action="store_true")
     ap.add_argument('--test_on_each_epoch', action="store_true", \
         help='runs test on each epoch to track final performance')
+    ap.add_argument("--skip-training", help="will load pre-trained model", action="store_true")
+    ap.add_argument("--run-name-prefix", help="will output train|dev|test runs with provided prefix")
 
     args = ap.parse_args()
 
     torch.manual_seed(1234)
     np.random.seed(1234)
+    torch.set_num_threads(args.num_threads)
 
     train_set, dev_set, test_set = 'train-all', 'raw-dev', 'raw-test'
     if args.train:
@@ -127,50 +133,65 @@ if __name__ == "__main__":
 
     # instantiate model
     net = QAModel(vec_dim, args.filter_width, args.num_conv_filters, args.no_ext_feats)
-    QAModel.save(net, args.model_outfile)
-
-    torch.set_num_threads(args.num_threads)
 
+    # initialize the trainer
     trainer = Trainer(net, args.eta, args.mom, args.no_loss_reg, vec_dim)
     logger.info("Loading input data...")
+    # load input data
     trainer.load_input_data(args.dataset_folder, cache_file, train_set, dev_set, test_set)
+    logger.info("Setting up external features...")
+    # setup external features
+    # TODO: remember to update args.* in testing loop below
+    if args.paper_ext_feats:
+        logger.info("--paper-ext-feats")
+        ext_feats_for_splits = set_external_features_as_per_paper(trainer)
+        # ^^ we are saving the features to be used while testing at the end of training
+    elif args.paper_ext_feats_stem:
+        logger.info("--paper-ext-feats-stem")
+        ext_feats_for_splits = set_external_features_as_per_paper_and_stem(trainer)
 
-    best_map = 0.0
-    best_model = 0
+    if not args.skip_training:
+        best_map = 0.0
+        best_model = 0
 
-    for i in range(args.epochs):
-        logger.info('------------- Training epoch {} --------------'.format(i+1))
-        train_accuracy = trainer.train(train_set, args.batch_size, args.debug_single_batch)
-        if args.debug_single_batch: sys.exit(0)
+        for i in range(args.epochs):
+            logger.info('------------- Training epoch {} --------------'.format(i+1))
+            train_accuracy = trainer.train(train_set, args.batch_size, args.debug_single_batch)
+            if args.debug_single_batch: sys.exit(0)
 
-        dev_scores = trainer.test(dev_set, args.batch_size)
+            dev_scores = trainer.test(dev_set, args.batch_size)
 
-        dev_map, dev_mrr = compute_map_mrr(args.dataset_folder, dev_set, dev_scores)
-        logger.info("------- MAP {}, MRR {}".format(dev_map, dev_mrr))
+            dev_map, dev_mrr = compute_map_mrr(args.dataset_folder, dev_set, dev_scores)
+            logger.info("------- MAP {}, MRR {}".format(dev_map, dev_mrr))
 
-        if dev_map - best_map > 1e-3: # new map is better than best map
-            best_model = i
-            best_map = dev_map
+            if dev_map - best_map > 1e-3: # new map is better than best map
+                best_model = i
+                best_map = dev_map
 
-            QAModel.save(net, args.model_outfile)
-            logger.info('Achieved better dev_map ... saved model')
+                QAModel.save(net, args.model_outfile)
+                logger.info('Achieved better dev_map ... saved model')
 
-        if args.test_on_each_epoch:
-            test_scores = trainer.test(test_set, args.batch_size)
-            map, mrr = compute_map_mrr(args.dataset_folder, test_set, test_scores)
-            logger.info("------- MAP {}, MRR {}".format(map, mrr))
+            if args.test_on_each_epoch:
+                test_scores = trainer.test(test_set, args.batch_size)
+                map, mrr = compute_map_mrr(args.dataset_folder, test_set, test_scores)
+                logger.info("------- MAP {}, MRR {}".format(map, mrr))
 
-        if (i - best_model) >= args.patience:
-            logger.warning('No improvement since the last {} epochs. Stopping training'.format(i - best_model))
-            break
+            if (i - best_model) >= args.patience:
+                logger.warning('No improvement since the last {} epochs. Stopping training'\
+                               .format(i - best_model))
+                break
 
-    logger.info(' ------------ Training epochs completed! ------------')
-    logger.info('Best MAP in training phase = {:.4f}'.format(best_map))
+        logger.info(' ------------ Training epochs completed! ------------')
+        logger.info('Best dev MAP in training phase = {:.4f}'.format(best_map))
 
     trained_model = QAModel.load(args.model_outfile)
     evaluator = Trainer(trained_model, args.eta, args.mom, args.no_loss_reg, vec_dim)
-    evaluator.load_input_data(args.dataset_folder, cache_file, None, None, test_set)
-    test_scores = evaluator.test(test_set, args.batch_size)
 
-    map, mrr = compute_map_mrr(args.dataset_folder, test_set, test_scores)
-    logger.info("------- MAP {}, MRR {}".format(map, mrr))
+    for split in [test_set, dev_set, train_set]:
+        evaluator.load_input_data(args.dataset_folder, cache_file, None, None, split)
+        if args.paper_ext_feats or args.paper_ext_feats_stem:
+            evaluator.data_splits[split][-1] = ext_feats_for_splits[split]
+            #set_external_features_as_per_paper(evaluator)
+        split_scores = evaluator.test(split, args.batch_size)
+        map, mrr = compute_map_mrr(args.dataset_folder, split, split_scores, args.run_name_prefix)
+        logger.info("-------{} MAP {}, MRR {}".format(split, map, mrr))

sm_model/make_run.py

@@ -0,0 +1,51 @@
+import argparse
+import os
+from model import QAModel
+from train import Trainer
+import utils
+
+if __name__ == "__main__":
+
+    ap = argparse.ArgumentParser(description="Makes a run in trec_eval run format, given a model and a train|dev|test set" )
+    ap.add_argument('model')
+    ap.add_argument('--word_embeddings_cache',
+                    default='../../data/word2vec/aquaint+wiki.txt.gz.ndim=50.cache')
+    ap.add_argument('--paper-ext-feats', action="store_true", \
+        help="external features as per the paper")
+    ap.add_argument('dataset_folder', help="the QA dataset folder",
+                    choices=['../../data/TrecQA', '../../data/WikQA'])
+    ap.add_argument('set_split', help="train, dev or test split as the data_folder")
+    ap.add_argument("batch_size", help="the number of pairs to compare in each batch.\
+                     should be same as during training")
+    ap.add_argument('out_scorefile', help='output file in trec_eval format')
+
+
+    args = ap.parse_args()
+
+    vocab_size, vec_dim = utils.load_embedding_dimensions(args.word_embeddings_cache)
+
+    trained_model = QAModel.load(args.model)
+    trained_model.no_ext_feats = True
+    evaluator = Trainer(trained_model, 0, 0, False, vec_dim) # 0, 0, False are dummy arguments
+    evaluator.load_input_data(args.dataset_folder, args.word_embeddings_cache,
+                              None, None, args.set_split,
+                              True if args.ext_feats else False)
+    test_scores = evaluator.test(args.set_split, args.batch_size)
+
+    questions, sentences, labels, vocab, maxlen_q, maxlen_s, ext_feats = \
+            evaluator.data_splits[args.set_split]
+
+    qids = [id.strip() for id in open(os.path.join(args.dataset_folder, args.set_split, 'id.txt'))\
+            .readlines()]
+
+    with open(args.out_scorefile, 'w') as outf:
+        old_qid = 0
+        docid_c = 0
+        for i in range(len(qids)):
+            if qids[i] != old_qid and args.dataset_folder.endswith('WikiQA'):
+                docid_c = 0
+                old_qid = qids[i]
+            print('{} 0 {} 0 {} {}'.format(qids[i], docid_c, test_scores[i],
+                                           os.path.basename(args.model)),
+                  file=outf)
+            docid_c += 1

sm_model/model.py

@@ -33,7 +33,7 @@ class QAModel(nn.Module):
         self.no_ext_feats = no_ext_feats
 
         self.conv_channels = conv_filters
-        n_hidden = 2*self.conv_channels + 1
+        n_hidden = 2*self.conv_channels + (0 if no_ext_feats else ext_feats_size)
 
         self.conv_q = nn.Sequential(
             nn.Conv1d(input_n_dim, self.conv_channels, filter_width, padding=filter_width-1),
@@ -47,7 +47,7 @@ class QAModel(nn.Module):
 
         self.combined_feature_vector = nn.Linear(2*self.conv_channels + \
             (0 if no_ext_feats else ext_feats_size), n_hidden)
-        # TODO: add +1 to Linear layer^. Will need change in forward function
+
         self.combined_features_activation = nn.Tanh()
         self.dropout = nn.Dropout(0.5)
         self.hidden = nn.Linear(n_hidden, n_classes)

sm_model/requirements.txt

@@ -0,0 +1,5 @@
+gensim==1.0.1
+nltk==3.2.1
+numpy==1.11.3
+pandas==0.19.2
+torch==0.1.11+b13b701

sm_model/train.py

@@ -37,20 +37,25 @@ class Trainer(object):
         self.optimizer = optim.SGD(self.model.parameters(), lr=eta, momentum=mom, \
             weight_decay=(0 if no_loss_reg else self.reg))
 
-        self.datasets = {}
+        self.data_splits = {}
         self.embeddings = {}
         self.vec_dim = vec_dim
 
 
     def load_input_data(self, dataset_root_folder, word_vectors_cache_file, \
-            train_set_folder, dev_set_folder, test_set_folder):
+            train_set_folder, dev_set_folder, test_set_folder, load_ext_feats=True):
         for set_folder in [test_set_folder, dev_set_folder, train_set_folder]:
             if set_folder:
-                self.datasets[set_folder] = utils.read_in_dataset(dataset_root_folder, set_folder)
-                # NOTE: self.datasets[set_folder] = questions, sentences, labels,
-                #                                       vocab, maxlen_q, maxlen_s, ext_feats
+                questions, sentences, labels, maxlen_q, maxlen_s, vocab = \
+                    utils.read_in_dataset(dataset_root_folder, set_folder)
+
+                self.data_splits[set_folder] = [questions, sentences, labels, maxlen_q, maxlen_s]
+
+                default_ext_feats = [np.zeros(4)] * len(self.data_splits[set_folder][0])
+                self.data_splits[set_folder].append(default_ext_feats)
+
                 self.embeddings[set_folder] = utils.load_cached_embeddings( \
-                    word_vectors_cache_file, self.datasets[set_folder][3], \
+                    word_vectors_cache_file, vocab, \
                     [] if "train" in set_folder else self.unk_term)
 
 
@@ -113,8 +118,8 @@ class Trainer(object):
     def test(self, set_folder, batch_size):
         logger.info('----- Predictions on {} '.format(set_folder))
 
-        questions, sentences, labels, vocab, maxlen_q, maxlen_s, ext_feats = \
-            self.datasets[set_folder]
+        questions, sentences, labels, maxlen_q, maxlen_s, ext_feats = \
+            self.data_splits[set_folder]
         word_vectors, vec_dim = self.embeddings[set_folder], self.vec_dim
 
         self.model.eval()
@@ -136,7 +141,7 @@ class Trainer(object):
                 sentences[batch_start:batch_end],
                 labels[batch_start:batch_end],
                 ext_feats[batch_start:batch_end],
-                word_vectors, vocab, vec_dim
+                word_vectors, vec_dim
                 )
 
             xq, xa, x_ext_feats = batch_inputs[0]
@@ -164,8 +169,8 @@ class Trainer(object):
     def train(self, set_folder, batch_size, debug_single_batch):
         train_start_time = time.time()
 
-        questions, sentences, labels, vocab, maxlen_q, maxlen_s, ext_feats = \
-            self.datasets[set_folder]
+        questions, sentences, labels, maxlen_q, maxlen_s, ext_feats = \
+            self.data_splits[set_folder]
         word_vectors, vec_dim = self.embeddings[set_folder], self.vec_dim
 
         # set model for training modep
@@ -184,7 +189,7 @@ class Trainer(object):
                 sentences[batch_start:batch_end],
                 labels[batch_start:batch_end],
                 ext_feats[batch_start:batch_end],
-                word_vectors, vocab, vec_dim
+                word_vectors, vec_dim
                 )
 
             xq, xa, x_ext_feats = batch_inputs[0]
@@ -225,7 +230,7 @@ class Trainer(object):
 
 
     def get_tensorized_inputs(self, batch_ques, batch_sents, batch_labels, batch_ext_feats, \
-            word_vectors, vocab, vec_dim):
+            word_vectors, vec_dim):
         batch_size = len(batch_ques)
         # NOTE: ideal batch size is one, because sentences are all of different length.
         # In other words, we have no option but to feed in sentences one by one into the model

sm_model/utils.py

@@ -107,14 +107,15 @@ def read_in_dataset(dataset_folder, set_folder):
     len_s_list = [len(s.split()) for s in sentences]
 
     labels = [int(line.strip()) for line in open(os.path.join(set_path, 'sim.txt')).readlines()]
-    ext_feats = np.array([list(map(float, line.strip().split(' '))) \
-        for line in open(os.path.join(set_path, 'overlap_feats.txt')).readlines()])
 
-    #y = torch.from_numpy(labels)
-    #return questions, sentences, y
+    # ext_feats = [np.zeros(4)] * len(questions)
+    # if load_ext_features:
+    #     ext_feats = np.array([list(map(float, line.strip().split(' '))) \
+    #         for line in open(os.path.join(set_path, 'overlap_feats.txt')).readlines()])
 
     vocab = [line.strip() for line in open(os.path.join(dataset_folder, 'vocab.txt')).readlines()]
-    return questions, sentences, labels, vocab, max(len_q_list), max(len_s_list), ext_feats
+
+    return [questions, sentences, labels,  max(len_q_list), max(len_s_list), vocab]
 
 
 def get_test_qids_labels(dataset_folder, set_folder):