diff --git a/baseline_results.tsv b/baseline_results.tsv new file mode 100644 index 0000000..b42d715 --- /dev/null +++ b/baseline_results.tsv @@ -0,0 +1,19 @@ +Baseline_method idf condition dataset train_MAP dev_MAP test_MAP Notes/Comments +idf_sum_similarity dataset no_stopping + no_stemming TrecQA 0.7744 0.7668 0.7014 IDF is computed over terms in the train, dev and test sets +idf_sum_similarity dataset with_stopping + with_stemming TrecQA 0.7078 0.7404 0.6816 +idf_sum_similarity dataset no_stopping + no_stemming WikiQA 0.2429 0.2489 0.2219 There is less overlap between questions and answers in the WikiQA dataset +idf_sum_similarity dataset with_stopping + with_stemming WikiQA 0.2232 0.2472 0.2025 +idf_sum_similarity Corpus no_stopping + no_stemming TrecQA 0.6736 0.728 0.6377 IDF is computed over documents in disks1-5 and aquaint +idf_sum_similarity Corpus with_stopping + with_stemming TrecQA 0.7032 0.7281 0.672 +idf_sum_similarity Corpus no_stopping + no_stemming WikiQA 0.2432 0.244 0.2206 IDF is computed over documents in Wikipedia +idf_sum_similarity Corpus with_stopping + with_stemming WikiQA 0.2212 0.2461 0.2005 +sm_model NA no_idf_overlap_features TrecQA +sm_model Dataset with_corrected_idf_overlap_features TrecQA +sm_model Corpus with_corrected_idf_overlap_features TrecQA +sm_model Dataset with_fractional_idf_overlap_features TrecQA +sm_model Corpus with_fractional_idf_overlap_features TrecQA +sm_model NA no_idf_overlap_features WikiQA +sm_model Dataset with_corrected_idf_overlap_features WikiQA +sm_model Corpus with_corrected_idf_overlap_features WikiQA +sm_model Dataset with_fractional_idf_overlap_features WikiQA +sm_model Corpus with_fractional_idf_overlap_features WikiQA diff --git a/idf_baseline/README.md b/idf_baseline/README.md new file mode 100644 index 0000000..d8d72a4 --- /dev/null +++ b/idf_baseline/README.md @@ -0,0 +1,158 @@ +# IDF scorer + +Implements IDF baselines for QA datasets. + +### Getting the data + +Git clone [castorini/data](https://github.com/castorini/data) to get TrecQA and WikiQA datasets. + +Follow instructions in ``TrecQA/README.txt`` and ``WikiQA/README.txt`` to process the data into a _standard_ format. + +After running the respectve scripts, you should have the following directories structure in ``castorini/data/TrecQA`` +``` +├── raw-dev +├── raw-test +├── train +└── train-all +``` + +and, the following directories in ``castorini/data/WikiQA``. +``` +├── dev +├── test +├── train +``` + +Each directory will have the following files: +``├── a.toks``: question[i] +``├── b.toks``: answer[i] +``├── id.txt``: question_id[i] +``└── sim.txt``: label[i] +where 1 <= i <= (number of QA pairs in respective splits of the data) + + +### Creating indexes for source corpora + +We need to index the source corpus from which the question-answer pairs are derived in order to get the IDF weights of the terms. + + +#### 1. Clone and compile[Anserini](https://github.com/castorini/Anserini.git) + +``` +git clone https://github.com/castorini/Anserini.git +cd Anserini +mvn clean package appassembler:assemble +``` + +#### 2. Indexing WikiQA collection + +First, download the Wikipedia dump by running the following command: + +``` +mkdir WikiQACollection +for line in $(cat idf_baseline/src/main/resources/WikiQA/wikidump-list.txt); do wget $line -P WikiQACollection; done +``` + +To index the collection: +``` +cd Anserini +nohup sh target/appassembler/bin/IndexCollection -collection WikipediaCollection -input ../WikiQACollection +-generator JsoupGenerator -index lucene.index.wikipedia.pos.docvectors -threads 32 -storePositions +-storeDocvectors -optimize > log.wikipedia.pos.docvectors & +``` + +#### 3. Indexing TrecQA collection + +Create a new directories called TrecQACollection +``` +mkdir TrecQACollection +``` + +Copy the contents of disk1, disk2, disk3, disk4, and AQUAINT to TrecQACollection + +To index the collection: + +``` +cd Anserini +nohup sh target/appassembler/bin/IndexCollection -collection TrecCollection -input [path of TrecQACollection] +-generator JsoupGenerator -index lucene.index.trecQA.pos.docvectors -threads 32 -storePositions +-storeDocvectors -optimize > log.trecQA.pos.docvectors & +``` + +### Computing the IDF sum similarity baseline + +#### 1. IDF sum similarity using the entire source corpus to compute IDF of terms + +Build the IDF scorer +``` +cd castorini/Castor/idf_baseline +mvn clean package appassembler:assemble +``` + +Run the following command to score each answer with an IDF value: + +``` +sh target/appassembler/bin/GetIDF -index ~/large-local-work/indices/index.wikipedia.pos.docvectors -config ../../data/WikiQA/test -output WikiQA.test.idfsim +``` +The above command will create a run file in the `trec_eval` format and a qrel file +at a location specified by `-output`. + + + +Possible parameters are: + +``` +-index (required) +``` + +Path of the index + +``` +-config (required) +``` +Configuration of this experiment i.e., dev, train, train-all, test etc. + +``` +-output (required) +``` +Path of the run file to be created + +``` +-analyze +``` +If specified, the scorer uses `EnglishAnalyzer` for removing stopwords and performing stemming. In addition to +the default list, the analyzer uses NLTK's stopword list obtained +from[here](https://gist.github.com/sebleier/554280) + + + +#### 2. Evaluating the system: + +To calculate MAP/MRR for the above run file: + +- Download and install `trec_eval` from[here](https://github.com/castorini/Anserini/blob/master/eval/trec_eval.9.0.tar.gz) + +``` +eval/trec_eval.9.0/trec_eval -m map -m recip_rank +``` + +For the WikiQA dataset +``` +../../Anserini/eval/trec_eval.9.0/trec_eval -m map ../../data/WikiQA/WikiQACorpus/WikiQA-$set.ref WikiQA.$set.idfsim +``` + +For the TrecQA dataset +``` +../../Anserini/eval/trec_eval.9.0/trec_eval -m map ../../data/TrecQA/$set.qrel TrecQA.$set.idfsim +``` + +#### 3. IDF sum similarity using only the QA dataset to compute IDF of terms + +``` +python qa-data-idf-only.py ../../data/TrecQA TrecQA +python qa-data-only-idf.py ../../data/WikiQA WikiQA +``` +Evaluate these using step 2. + +### Baseline results +Baseline results are saved in ``Castor/baseline_results.tsv`` \ No newline at end of file diff --git a/idf_baseline/docs/idf-baseline.md b/idf_baseline/docs/idf-baseline.md deleted file mode 100644 index c13c237..0000000 --- a/idf_baseline/docs/idf-baseline.md +++ /dev/null @@ -1,121 +0,0 @@ -# IDF scorer - -Download the TrecQA and WikiQA data (question-answer pairs) from[here](https://github.com/castorini/data.git) - -Switch to an appropriate directory and run the following scripts: - -``` -python3 parse.py - -python3 overlap_features.py - -python3 build_vocab.py -``` - -After running the script, you should have the following directory structure: - -``` -├── raw-dev -├── raw-test -├── train -└── train-all -``` -and each directory should have the following files: -``` -├── a.toks -├── b.toks -├── boundary.txt -├── id.txt -├── numrels.txt -└── sim.txt -``` - -Clone and compile[Anserini](https://github.com/castorini/Anserini.git) - -``` -git clone https://github.com/castorini/Anserini.git -cd Anserini -mvn clean package appassembler:assemble -``` - -### Indexing WikiQA collection - -First, download the Wikipedia dump by running the following command: - -``` -mkdir WikiQACollection -for line in $(cat idf_baseline/src/main/resources/WikiQA/wikidump-list.txt); do wget $line -P WikiQACollection; done -``` - -To index the collection: -``` -cd Anserini -nohup sh target/appassembler/bin/IndexCollection -collection WikipediaCollection -input ../WikiQACollection --generator JsoupGenerator -index lucene.index.wikipedia.pos.docvectors -threads 32 -storePositions --storeDocvectors -optimize > log.wikipedia.pos.docvectors & -``` - -### Indexing TrecQA collection - -Create a new directory called TrecQACollection -``` -mkdir TrecQACollection -``` - -Copy the contents of disk1, disk2, disk3, disk4, and AQUAINT to TrecQACollection - -To index the collection: - -``` -cd Anserini -nohup sh target/appassembler/bin/IndexCollection -collection TrecCollection -input [path of TrecQACollection] --generator JsoupGenerator -index lucene.index.trecQA.pos.docvectors -threads 32 -storePositions --storeDocvectors -optimize > log.trecQA.pos.docvectors & -``` - -### Calculating IDF overlap - -Run the following command to score each answer with an IDF value: - -``` -sh target/appassembler/bin/GetIDF -``` - -Possible parameters are: - -``` --index (required) -``` - -Path of the index - -``` --config (requiered) -``` -Configuration of this experiment i.e., dev, train, train-all, test etc. - -``` --output (optional: file path) -``` - -Path of the run file to be created - -``` --analyze -``` -If specified, the scorer uses `EnglishAnalyzer` for removing stopwords and stemming. In addtion to -the default list, the analyzer uses NLTK's stopword list obtained -from[here](https://gist.github.com/sebleier/554280) - -The above command will create a run file in the `trec_eval` format and a qrel file -at a location specified by `-output`. - -### Evaluating the system: - -To calculate MAP/MRR for the above run file: - -- Download and install `trec_eval` from[here](https://github.com/castorini/Anserini/blob/master/eval/trec_eval.9.0.tar.gz) - -``` -eval/trec_eval.9.0/trec_eval -m map -m recip_rank -``` \ No newline at end of file diff --git a/idf_baseline/qa-data-only-idf.py b/idf_baseline/qa-data-only-idf.py new file mode 100644 index 0000000..f3f070d --- /dev/null +++ b/idf_baseline/qa-data-only-idf.py @@ -0,0 +1,116 @@ +import argparse +import os +import numpy as np +from collections import defaultdict + +import nltk +nltk.download('stopwords') + +from nltk.stem.porter import PorterStemmer +from nltk.corpus import stopwords + +def read_in_data(datapath, set_name, file, stop_and_stem=False): + data = [] + with open(os.path.join(datapath, set_name, file)) as inf: + data = [line.strip() for line in inf.readlines()] + if stop_and_stem: + stemmer = PorterStemmer() + stoplist = set(stopwords.words('english')) + def stop_stem(sentence): + return ' '.join([stemmer.stem(word) for word in sentence.split() \ + if word not in stoplist]) + data = [stop_stem(sentence) for sentence in data] + return data + + +def compute_idfs(data): + term_idfs = defaultdict(float) + for doc in list(data): + for term in list(set(doc.split())): + term_idfs[term] += 1.0 + N = len(data) + for term, n_t in term_idfs.items(): + term_idfs[term] = np.log(N/(1+n_t)) + return term_idfs + + +def compute_idf_sum_similarity(questions, answers, term_idfs): + # compute IDF sums for common_terms + idf_sum_similarity = np.zeros(len(questions)) + for i in range(len(questions)): + q = questions[i] + a = answers[i] + q_terms = set(q.split()) + a_terms = set(a.split()) + common_terms = q_terms.intersection(a_terms) + idf_sum_similarity[i] = np.sum([term_idfs[term] for term in list(common_terms)]) + + return idf_sum_similarity + + +def write_out_idf_sum_similarities(qids, questions, answers, term_idfs, outfile, dataset): + with open(outfile, 'w') as outf: + idf_sum_similarity = compute_idf_sum_similarity(questions, answers, term_idfs) + old_qid = 0 + docid_c = 0 + for i in range(len(questions)): + if qids[i] != old_qid and dataset.endswith('WikiQA'): + docid_c = 0 + old_qid = qids[i] + print('{} 0 {} 0 {} data_only_idfbaseline'.format(qids[i], docid_c, + idf_sum_similarity[i]), + file=outf) + docid_c += 1 + + +if __name__ == "__main__": + ap = argparse.ArgumentParser(description="uses idf weights from the question-answer pairs only,\ + and not from the whole corpus") + ap.add_argument('qa_data', help="path to the QA dataset", + choices=['../../data/TrecQA', '../../data/WikiQA']) + ap.add_argument('outfile_prefix', help="output file prefix") + ap.add_argument('--ignore-test', help="does not consider test data when computing IDF of terms", + action="store_true") + ap.add_argument("--stop-and-stem", help='performs stopping and stemming', action="store_true") + + args = ap.parse_args() + + # read in the data + train_data, dev_data, test_data = 'train', 'dev', 'test' + if args.qa_data.endswith('TrecQA'): + train_data, dev_data, test_data = 'train-all', 'raw-dev', 'raw-test' + + train_que = read_in_data(args.qa_data, train_data, 'a.toks', args.stop_and_stem) + train_ans = read_in_data(args.qa_data, train_data, 'b.toks', args.stop_and_stem) + + dev_que = read_in_data(args.qa_data, dev_data, 'a.toks', args.stop_and_stem) + dev_ans = read_in_data(args.qa_data, dev_data, 'b.toks', args.stop_and_stem) + + test_que = read_in_data(args.qa_data, test_data, 'a.toks', args.stop_and_stem) + test_ans = read_in_data(args.qa_data, test_data, 'b.toks', args.stop_and_stem) + + all_data = train_que + dev_que + train_ans + dev_ans + + if not args.ignore_test: + all_data += test_ans + all_data += test_que + + # compute inverse document frequencies for terms + term_idfs = compute_idfs(set(all_data)) + + # write out in trec_eval format + write_out_idf_sum_similarities(read_in_data(args.qa_data, train_data, 'id.txt'), + train_que, train_ans, term_idfs, + '{}.{}.idfsim'.format(args.outfile_prefix, train_data), + args.qa_data) + + write_out_idf_sum_similarities(read_in_data(args.qa_data, dev_data, 'id.txt'), + dev_que, dev_ans, term_idfs, + '{}.{}.idfsim'.format(args.outfile_prefix, dev_data), + args.qa_data) + + write_out_idf_sum_similarities(read_in_data(args.qa_data, test_data, 'id.txt'), + test_que, test_ans, term_idfs, + '{}.{}.idfsim'.format(args.outfile_prefix, test_data), + args.qa_data) + diff --git a/idf_baseline/requirements.txt b/idf_baseline/requirements.txt new file mode 100644 index 0000000..4cc717a --- /dev/null +++ b/idf_baseline/requirements.txt @@ -0,0 +1,2 @@ +nltk==3.2.1 +numpy==1.11.3 diff --git a/idf_baseline/src/main/java/ai/castor/idf/IDFScorer.java b/idf_baseline/src/main/java/ai/castor/idf/IDFScorer.java index 15ea6c0..cbd60c3 100644 --- a/idf_baseline/src/main/java/ai/castor/idf/IDFScorer.java +++ b/idf_baseline/src/main/java/ai/castor/idf/IDFScorer.java @@ -126,6 +126,7 @@ public class IDFScorer { BufferedWriter outputFile = new BufferedWriter(new FileWriter(args.output)); int i = 0; + String old_id = "0"; while (true) { String question = questionFile.readLine(); String answer = answerFile.readLine(); @@ -135,9 +136,15 @@ public class IDFScorer { break; } + // we need new lines here + if (args.config.contains("WikiQA") && !old_id.equals(id)) { + old_id = id; + i = 0; + } + // 32.1 0 0 0 0.6212325096130371 smmodel // 32.1 0 1 0 0.13309887051582336 smmodel - outputFile.write(id + " 0 " + i + " " + calcIDF(question, answer, args.analyze) + " smmodel\n"); + outputFile.write(id + " 0 " + i + " 0 " + calcIDF(question, answer, args.analyze) + " idfbaseline\n"); i++; } outputFile.close(); diff --git a/sm_model/README.md b/sm_model/README.md index 5fa89fa..2c6ea35 100644 --- a/sm_model/README.md +++ b/sm_model/README.md @@ -65,7 +65,7 @@ cd .. To train the S&M model on TrecQA ``` -python main.py ../../model/sm_model/sm_model.train-all --train_all +python main.py ../../model/sm_model/sm_model.train-all ``` The final model will be saved to ```../../model/sm_model/sm_model.train-all``` diff --git a/sm_model/bridge.py b/sm_model/bridge.py index 32c3185..00aa771 100644 --- a/sm_model/bridge.py +++ b/sm_model/bridge.py @@ -148,26 +148,55 @@ class SMModelBridge(object): if __name__ == "__main__": - smmodel = SMModelBridge('../models/sm_model/sm_model.TrecQA.TRAIN-ALL.2017-04-02.castor', - '../data/word2vec/aquaint+wiki.txt.gz.ndim=50.cache', - '../data/TrecQA/stopwords.txt', - '../data/TrecQA/word2dfs.p') + ap = argparse.ArgumentParser(description="Bridge Demo. Produces scores in trec_eval format") + ap.add_argument('model') + ap.add_argument('--word_embeddings_cache', default='../data/word2vec/aquaint+wiki.txt.gz.ndim=50.cache') + ap.add_argument('--stopwords_file', default='../data/TrecQA/stopwords.txt') + ap.add_argument('--wordDF_file', default='../data/TrecQA/word2dfs.p') + ap.add_argument('--no_ext_feats', action="store_true", help="This argument has no effect because the model saves its members") + ap.add_argument('--use_pre_ext_feats', action="store_true", help="use the precomputed external overlap features") + ap.add_argument('--data_folder', default='../data/TrecQA/') + ap.add_argument('dataset', choices=['train-all', 'raw-test', 'raw-dev', 'train']) + ap.add_argument('out_scorefile', help='file in trec_eval format') + ap.add_argument('--out_qrels', help='will also output qrels trec_eval format') - question = "who is the author of the book , `` the iron lady : a biography of margaret thatcher '' ?" - answers = [ - "the iron lady ; a biography of margaret thatcher by hugo young -lrb- farrar , straus & giroux -rrb-", - "in this same revisionist mold , hugo young , the distinguished british journalist , has performed a brilliant \ - dissection of the notion of thatcher as a conservative icon .", - "in `` the iron lady , '' young traces the winding staircase of fortune that transformed the younger daughter \ - of a provincial english grocer into the greatest woman political leader since catherine the great .", - "`` he is the very essence of the classless meritocrat , '' says hugo young , thatcher 's biographer .", - "from her father , young argues , she inherited a `` joyless earnestness '' that combined with her early \ - interest in science to produce the roots of her public character .", - "this is not the answer", - "asdfawe asdf sertse dgfsgsfg" - ] + args = ap.parse_args() - ss = smmodel.rerank_candidate_answers(question, answers) - print('Question:', question) - for score, sentence in ss: - print(score, '\t', sentence) + smmodel = SMModelBridge( + #'../models/sm_model/sm_model.TrecQA.TRAIN-ALL.2017-04-02.castor', + args.model, + args.word_embeddings_cache, + args.stopwords_file, + args.wordDF_file) + + # if args.no_ext_feats: + # smmodel.model.no_ext_feats = True + + + allque = [q.strip() for q in open(os.path.join('../data/TrecQA/', args.dataset+'/a.toks')).readlines()] + allans = [a.strip() for a in open(os.path.join('../data/TrecQA/', args.dataset+'/b.toks')).readlines()] + labels = [y.strip() for y in open(os.path.join('../data/TrecQA/', args.dataset+'/sim.txt')).readlines()] + qids = [id.strip() for id in open(os.path.join('../data/TrecQA/', args.dataset+'/id.txt')).readlines()] + + pre_ext_feats = None + if args.use_pre_ext_feats: + pre_ext_feats = [ [float(e) for e in x.split() ] for x in open(os.path.join('../data/TrecQA/', args.dataset+'/overlap_feats.txt')).readlines()] + + scoref = open(args.out_scorefile, 'w') + if args.out_qrels: + qrelf = open(args.out_qrels, 'w') + + for i in range(len(allque)): + question = allque[i] + answers = [allans[i]] + ext_feats = None + if args.use_pre_ext_feats: + ext_feats = [pre_ext_feats[i]] + ss = smmodel.rerank_candidate_answers(question, answers, ext_feats) + # print('Question:', question) + for score, sentence in ss: + #print(score, '\t', sentence) + #print('{}\t{}'.format(labels[i], score)) + print('{} {} {} {} {} {}'.format(qids[i], '0', i, 0, score, 'sm_model.'+args.dataset), file=scoref) + if args.out_qrels: + print('{} {} {} {}'.format(qids[i], '0', i, labels[i]), file=qrelf) diff --git a/sm_model/main.py b/sm_model/main.py index 6dc2063..334a486 100644 --- a/sm_model/main.py +++ b/sm_model/main.py @@ -92,7 +92,7 @@ if __name__ == "__main__": ap.add_argument('--filter_width', type=int, default=5, help="number of convolution channels") ap.add_argument('--eta', help='Initial learning rate', default=0.001, type=float) ap.add_argument('--mom', help='SGD Momentum', default=0.0, type=float) - ap.add_argument('--train_all', help='switches to train-all set', action="store_true") + ap.add_argument('--train', help='switches to train set', action="store_true") # epoch related arguments ap.add_argument('--epochs', type=int, default=25, help="number of trainin epochs") @@ -115,9 +115,9 @@ if __name__ == "__main__": torch.manual_seed(1234) np.random.seed(1234) - train_set, dev_set, test_set = 'train', 'clean-dev', 'clean-test' - if args.train_all: - train_set, dev_set, test_set = 'train-all', 'raw-dev', 'raw-test' + train_set, dev_set, test_set = 'train-all', 'raw-dev', 'raw-test' + if args.train: + train_set, dev_set, test_set = 'train', 'clean-dev', 'clean-test' # cache word embeddings cache_file = os.path.splitext(args.word_vectors_file)[0] + '.cache'