From b893f2a1b1876f068ae94a376c7c9e6e53c53be4 Mon Sep 17 00:00:00 2001 From: rosequ Date: Wed, 12 Apr 2017 17:24:29 -0400 Subject: [PATCH] Idf baseline for QA datasets (#14) Added a functionality to generate idf scores for TrecQA and WikiQA datasets. --- README.md | 6 +- idf_baseline/docs/idf-baseline.md | 121 +++++++++++++ idf_baseline/pom.xml | 135 +++++++++++++++ .../main/java/ai/castor/idf/IDFScorer.java | 163 ++++++++++++++++++ .../ai/castor/qa/english-stoplist.txt | 130 ++++++++++++++ 5 files changed, 554 insertions(+), 1 deletion(-) create mode 100644 idf_baseline/docs/idf-baseline.md create mode 100644 idf_baseline/pom.xml create mode 100644 idf_baseline/src/main/java/ai/castor/idf/IDFScorer.java create mode 100644 idf_baseline/src/main/resources/ai/castor/qa/english-stoplist.txt diff --git a/README.md b/README.md index 9617680..9221d19 100644 --- a/README.md +++ b/README.md @@ -29,4 +29,8 @@ CUDA installation guide for linux can be found [here](http://docs.nvidia.com/cud ## data for models -Sourcing and pre-processing of input data for each model is described in respective ```model/README.md```'s \ No newline at end of file +Sourcing and pre-processing of input data for each model is described in respective ```model/README.md```'s + +## Baselines + +1. [IDF Baseline](./idf_baseline/): IDF overlap between question and candidate answers. diff --git a/idf_baseline/docs/idf-baseline.md b/idf_baseline/docs/idf-baseline.md new file mode 100644 index 0000000..c13c237 --- /dev/null +++ b/idf_baseline/docs/idf-baseline.md @@ -0,0 +1,121 @@ +# IDF scorer + +Download the TrecQA and WikiQA data (question-answer pairs) from[here](https://github.com/castorini/data.git) + +Switch to an appropriate directory and run the following scripts: + +``` +python3 parse.py + +python3 overlap_features.py + +python3 build_vocab.py +``` + +After running the script, you should have the following directory structure: + +``` +├── raw-dev +├── raw-test +├── train +└── train-all +``` +and each directory should have the following files: +``` +├── a.toks +├── b.toks +├── boundary.txt +├── id.txt +├── numrels.txt +└── sim.txt +``` + +Clone and compile[Anserini](https://github.com/castorini/Anserini.git) + +``` +git clone https://github.com/castorini/Anserini.git +cd Anserini +mvn clean package appassembler:assemble +``` + +### Indexing WikiQA collection + +First, download the Wikipedia dump by running the following command: + +``` +mkdir WikiQACollection +for line in $(cat idf_baseline/src/main/resources/WikiQA/wikidump-list.txt); do wget $line -P WikiQACollection; done +``` + +To index the collection: +``` +cd Anserini +nohup sh target/appassembler/bin/IndexCollection -collection WikipediaCollection -input ../WikiQACollection +-generator JsoupGenerator -index lucene.index.wikipedia.pos.docvectors -threads 32 -storePositions +-storeDocvectors -optimize > log.wikipedia.pos.docvectors & +``` + +### Indexing TrecQA collection + +Create a new directory called TrecQACollection +``` +mkdir TrecQACollection +``` + +Copy the contents of disk1, disk2, disk3, disk4, and AQUAINT to TrecQACollection + +To index the collection: + +``` +cd Anserini +nohup sh target/appassembler/bin/IndexCollection -collection TrecCollection -input [path of TrecQACollection] +-generator JsoupGenerator -index lucene.index.trecQA.pos.docvectors -threads 32 -storePositions +-storeDocvectors -optimize > log.trecQA.pos.docvectors & +``` + +### Calculating IDF overlap + +Run the following command to score each answer with an IDF value: + +``` +sh target/appassembler/bin/GetIDF +``` + +Possible parameters are: + +``` +-index (required) +``` + +Path of the index + +``` +-config (requiered) +``` +Configuration of this experiment i.e., dev, train, train-all, test etc. + +``` +-output (optional: file path) +``` + +Path of the run file to be created + +``` +-analyze +``` +If specified, the scorer uses `EnglishAnalyzer` for removing stopwords and stemming. In addtion to +the default list, the analyzer uses NLTK's stopword list obtained +from[here](https://gist.github.com/sebleier/554280) + +The above command will create a run file in the `trec_eval` format and a qrel file +at a location specified by `-output`. + +### Evaluating the system: + +To calculate MAP/MRR for the above run file: + +- Download and install `trec_eval` from[here](https://github.com/castorini/Anserini/blob/master/eval/trec_eval.9.0.tar.gz) + +``` +eval/trec_eval.9.0/trec_eval -m map -m recip_rank +``` \ No newline at end of file diff --git a/idf_baseline/pom.xml b/idf_baseline/pom.xml new file mode 100644 index 0000000..e8a6e1c --- /dev/null +++ b/idf_baseline/pom.xml @@ -0,0 +1,135 @@ + + 4.0.0 + anserini + anserini + 0.0.1-SNAPSHOT + Anserini retrieval platform + + + 6.3.0 + + 8.1.17.v20150415 + 0.8.18 + 2.7 + + + + + public + https://repo1.maven.org/maven2 + + + AnseriniMaven + https://raw.githubusercontent.com/lintool/AnseriniMaven/master/mvn-repo/ + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + 1.8 + 1.8 + utf-8 + + + + org.codehaus.mojo + appassembler-maven-plugin + 1.10 + + -Xms512M -Xmx24576M + + + ai.castor.idf.IDFScorer + GetIDF + + + + + + + maven-assembly-plugin + + + + fully.qualified.MainClass + + + + jar-with-dependencies + + + + + + + + + + + org.apache.lucene + lucene-benchmark + ${LUCENE_VERSION} + + + + org.apache.lucene + lucene-core + ${LUCENE_VERSION} + + + org.apache.lucene + lucene-test-framework + ${LUCENE_VERSION} + + + org.apache.lucene + lucene-backward-codecs + ${LUCENE_VERSION} + + + commons-logging + commons-logging + 1.2 + + + + org.codehaus.jackson + jackson-mapper-asl + 1.9.13 + + + + commons-cli + commons-cli + 1.2 + + + commons-io + commons-io + 2.4 + + + + org.apache.commons + commons-lang3 + 3.4 + jar + compile + + + + args4j + args4j + 2.32 + jar + compile + + + + diff --git a/idf_baseline/src/main/java/ai/castor/idf/IDFScorer.java b/idf_baseline/src/main/java/ai/castor/idf/IDFScorer.java new file mode 100644 index 0000000..15ea6c0 --- /dev/null +++ b/idf_baseline/src/main/java/ai/castor/idf/IDFScorer.java @@ -0,0 +1,163 @@ +/** + * Anserini: An information retrieval toolkit built on Lucene + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ai.castor.idf; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.StopFilter; +import org.apache.lucene.analysis.core.WhitespaceAnalyzer; +import org.apache.lucene.analysis.en.EnglishAnalyzer; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.Term; +import org.apache.lucene.queryparser.classic.ParseException; +import org.apache.lucene.queryparser.classic.QueryParser; +import org.apache.lucene.search.*; +import org.apache.lucene.search.similarities.ClassicSimilarity; +import org.apache.lucene.store.FSDirectory; +import org.kohsuke.args4j.*; + +import java.io.*; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.*; + + +public class IDFScorer { + + public static class Args { + // required arguments + @Option(name = "-index", metaVar = "[path]", required = true, usage = "Lucene index") + public String index; + + @Option(name = "-config", metaVar = "[string]", required = true, usage = "type of the dataset") + public String config; + + @Option(name = "-output", metaVar = "[path]", required = true, usage = "path of the file to be created") + public String output; + + // optional arguments + @Option(name = "-analyze", usage = "passage scores") + public boolean analyze = false; + } + + private final IndexReader reader; + private final FSDirectory directory; + private final List stopWords; + + public static final String FIELD_BODY = "contents"; + + public IDFScorer(IDFScorer.Args args) throws Exception { + Path indexPath = Paths.get(args.index); + if (!Files.exists(indexPath) || !Files.isDirectory(indexPath) || !Files.isReadable(indexPath)) { + throw new IllegalArgumentException(args.index + " does not exist or is not a directory."); + } + this.directory = FSDirectory.open(indexPath); + this.reader = DirectoryReader.open(directory); + + stopWords = new ArrayList<>(); + + //Get file from resources folder + InputStream is = getClass().getResourceAsStream("/ai/castor/qa/english-stoplist.txt"); + BufferedReader bRdr = new BufferedReader(new InputStreamReader(is)); + String line; + while ((line = bRdr.readLine()) != null) { + if (!line.contains("#")) { + stopWords.add(line); + } + } + } + + public double calcIDF(String query, String answer, boolean analyze) throws ParseException { + Analyzer analyzer; + if (analyze) { + analyzer = new EnglishAnalyzer(StopFilter.makeStopSet(stopWords)); + } else { + analyzer = new WhitespaceAnalyzer(); + } + + QueryParser qp = new QueryParser(FIELD_BODY, analyzer); + ClassicSimilarity similarity = new ClassicSimilarity(); + + String escapedQuery = qp.escape(query); + Query question = qp.parse(escapedQuery); + HashSet questionTerms = new HashSet<>(Arrays.asList(question.toString().trim().split("\\s+"))); + + double idf = 0.0; + HashSet seenTerms = new HashSet<>(); + + String[] terms = answer.split("\\s+"); + for (String term : terms) { + try { + TermQuery q = (TermQuery) qp.parse(term); + Term t = q.getTerm(); + + if (questionTerms.contains(t.toString()) && !seenTerms.contains(t.toString())) { + idf += similarity.idf(reader.docFreq(t), reader.numDocs()); + seenTerms.add(t.toString()); + } else { + idf += 0.0; + } + } catch (Exception e) { + continue; + } + } + return idf; + } + + public void writeToFile(IDFScorer.Args args) throws IOException, ParseException { + BufferedReader questionFile = new BufferedReader(new FileReader(args.config + "/a.toks")); + BufferedReader answerFile = new BufferedReader(new FileReader(args.config + "/b.toks")); + BufferedReader idFile = new BufferedReader(new FileReader(args.config + "/id.txt")); + + BufferedWriter outputFile = new BufferedWriter(new FileWriter(args.output)); + int i = 0; + + while (true) { + String question = questionFile.readLine(); + String answer = answerFile.readLine(); + String id = idFile.readLine(); + + if (question == null || answer == null || id == null) { + break; + } + + // 32.1 0 0 0 0.6212325096130371 smmodel + // 32.1 0 1 0 0.13309887051582336 smmodel + outputFile.write(id + " 0 " + i + " " + calcIDF(question, answer, args.analyze) + " smmodel\n"); + i++; + } + outputFile.close(); + + } + + public static void main(String[] args) throws Exception { + Args qaArgs = new Args(); + CmdLineParser parser = new CmdLineParser(qaArgs, ParserProperties.defaults().withUsageWidth(90)); + + try { + parser.parseArgument(args); + } catch (CmdLineException e) { + System.err.println(e.getMessage()); + parser.printUsage(System.err); + System.err.println("Example: IDFScorer" + parser.printExample(OptionHandlerFilter.REQUIRED)); + return; + } + + IDFScorer g = new IDFScorer(qaArgs); + g.writeToFile(qaArgs); + } +} diff --git a/idf_baseline/src/main/resources/ai/castor/qa/english-stoplist.txt b/idf_baseline/src/main/resources/ai/castor/qa/english-stoplist.txt new file mode 100644 index 0000000..934bd1a --- /dev/null +++ b/idf_baseline/src/main/resources/ai/castor/qa/english-stoplist.txt @@ -0,0 +1,130 @@ +# Stop words from NLTK +# Source: https://gist.github.com/sebleier/554280 +# +i +me +my +myself +we +our +ours +ourselves +you +your +yours +yourself +yourselves +he +him +his +himself +she +her +hers +herself +it +its +itself +they +them +their +theirs +themselves +what +which +who +whom +this +that +these +those +am +is +are +was +were +be +been +being +have +has +had +having +do +does +did +doing +a +an +the +and +but +if +or +because +as +until +while +of +at +by +for +with +about +against +between +into +through +during +before +after +above +below +to +from +up +down +in +out +on +off +over +under +again +further +then +once +here +there +when +where +why +how +all +any +both +each +few +more +most +other +some +such +no +nor +not +only +own +same +so +than +too +very +s +t +can +will +just +don +should +now \ No newline at end of file