mirror of
https://github.com/wassname/Castor.git
synced 2026-09-09 11:13:20 +08:00
Idf baseline for QA datasets (#14)
Added a functionality to generate idf scores for TrecQA and WikiQA datasets.
This commit is contained in:
rosequ
@@ -29,4 +29,8 @@ CUDA installation guide for linux can be found [here](http://docs.nvidia.com/cud
|
||||
|
||||
## data for models
|
||||
|
||||
Sourcing and pre-processing of input data for each model is described in respective ```model/README.md```'s
|
||||
Sourcing and pre-processing of input data for each model is described in respective ```model/README.md```'s
|
||||
|
||||
## Baselines
|
||||
|
||||
1. [IDF Baseline](./idf_baseline/): IDF overlap between question and candidate answers.
|
||||
|
||||
@@ -0,0 +1,121 @@
|
||||
# IDF scorer
|
||||
|
||||
Download the TrecQA and WikiQA data (question-answer pairs) from[here](https://github.com/castorini/data.git)
|
||||
|
||||
Switch to an appropriate directory and run the following scripts:
|
||||
|
||||
```
|
||||
python3 parse.py
|
||||
|
||||
python3 overlap_features.py
|
||||
|
||||
python3 build_vocab.py
|
||||
```
|
||||
|
||||
After running the script, you should have the following directory structure:
|
||||
|
||||
```
|
||||
├── raw-dev
|
||||
├── raw-test
|
||||
├── train
|
||||
└── train-all
|
||||
```
|
||||
and each directory should have the following files:
|
||||
```
|
||||
├── a.toks
|
||||
├── b.toks
|
||||
├── boundary.txt
|
||||
├── id.txt
|
||||
├── numrels.txt
|
||||
└── sim.txt
|
||||
```
|
||||
|
||||
Clone and compile[Anserini](https://github.com/castorini/Anserini.git)
|
||||
|
||||
```
|
||||
git clone https://github.com/castorini/Anserini.git
|
||||
cd Anserini
|
||||
mvn clean package appassembler:assemble
|
||||
```
|
||||
|
||||
### Indexing WikiQA collection
|
||||
|
||||
First, download the Wikipedia dump by running the following command:
|
||||
|
||||
```
|
||||
mkdir WikiQACollection
|
||||
for line in $(cat idf_baseline/src/main/resources/WikiQA/wikidump-list.txt); do wget $line -P WikiQACollection; done
|
||||
```
|
||||
|
||||
To index the collection:
|
||||
```
|
||||
cd Anserini
|
||||
nohup sh target/appassembler/bin/IndexCollection -collection WikipediaCollection -input ../WikiQACollection
|
||||
-generator JsoupGenerator -index lucene.index.wikipedia.pos.docvectors -threads 32 -storePositions
|
||||
-storeDocvectors -optimize > log.wikipedia.pos.docvectors &
|
||||
```
|
||||
|
||||
### Indexing TrecQA collection
|
||||
|
||||
Create a new directory called TrecQACollection
|
||||
```
|
||||
mkdir TrecQACollection
|
||||
```
|
||||
|
||||
Copy the contents of disk1, disk2, disk3, disk4, and AQUAINT to TrecQACollection
|
||||
|
||||
To index the collection:
|
||||
|
||||
```
|
||||
cd Anserini
|
||||
nohup sh target/appassembler/bin/IndexCollection -collection TrecCollection -input [path of TrecQACollection]
|
||||
-generator JsoupGenerator -index lucene.index.trecQA.pos.docvectors -threads 32 -storePositions
|
||||
-storeDocvectors -optimize > log.trecQA.pos.docvectors &
|
||||
```
|
||||
|
||||
### Calculating IDF overlap
|
||||
|
||||
Run the following command to score each answer with an IDF value:
|
||||
|
||||
```
|
||||
sh target/appassembler/bin/GetIDF
|
||||
```
|
||||
|
||||
Possible parameters are:
|
||||
|
||||
```
|
||||
-index (required)
|
||||
```
|
||||
|
||||
Path of the index
|
||||
|
||||
```
|
||||
-config (requiered)
|
||||
```
|
||||
Configuration of this experiment i.e., dev, train, train-all, test etc.
|
||||
|
||||
```
|
||||
-output (optional: file path)
|
||||
```
|
||||
|
||||
Path of the run file to be created
|
||||
|
||||
```
|
||||
-analyze
|
||||
```
|
||||
If specified, the scorer uses `EnglishAnalyzer` for removing stopwords and stemming. In addtion to
|
||||
the default list, the analyzer uses NLTK's stopword list obtained
|
||||
from[here](https://gist.github.com/sebleier/554280)
|
||||
|
||||
The above command will create a run file in the `trec_eval` format and a qrel file
|
||||
at a location specified by `-output`.
|
||||
|
||||
### Evaluating the system:
|
||||
|
||||
To calculate MAP/MRR for the above run file:
|
||||
|
||||
- Download and install `trec_eval` from[here](https://github.com/castorini/Anserini/blob/master/eval/trec_eval.9.0.tar.gz)
|
||||
|
||||
```
|
||||
eval/trec_eval.9.0/trec_eval -m map -m recip_rank <qrel-file> <run-file>
|
||||
```
|
||||
@@ -0,0 +1,135 @@
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<groupId>anserini</groupId>
|
||||
<artifactId>anserini</artifactId>
|
||||
<version>0.0.1-SNAPSHOT</version>
|
||||
<name>Anserini retrieval platform</name>
|
||||
|
||||
<properties>
|
||||
<LUCENE_VERSION>6.3.0</LUCENE_VERSION>
|
||||
<!-- see http://download.eclipse.org/jetty/ Jetty9 forces Java 1.8, which causes all sorts of headaches
|
||||
For display special characters such as emoji expression, please set JETTY_VERSION to 9.3.5.v20151012 -->
|
||||
<JETTY_VERSION>8.1.17.v20150415</JETTY_VERSION>
|
||||
<MUSTACHE_VERSION>0.8.18</MUSTACHE_VERSION>
|
||||
<JERSEY_VERSION>2.7</JERSEY_VERSION>
|
||||
</properties>
|
||||
|
||||
<repositories>
|
||||
<repository>
|
||||
<id>public</id>
|
||||
<url>https://repo1.maven.org/maven2</url>
|
||||
</repository>
|
||||
<repository>
|
||||
<id>AnseriniMaven</id>
|
||||
<url>https://raw.githubusercontent.com/lintool/AnseriniMaven/master/mvn-repo/</url>
|
||||
</repository>
|
||||
</repositories>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
<configuration>
|
||||
<source>1.8</source>
|
||||
<target>1.8</target>
|
||||
<encoding>utf-8</encoding>
|
||||
</configuration>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.codehaus.mojo</groupId>
|
||||
<artifactId>appassembler-maven-plugin</artifactId>
|
||||
<version>1.10</version>
|
||||
<configuration>
|
||||
<extraJvmArguments>-Xms512M -Xmx24576M</extraJvmArguments>
|
||||
<programs>
|
||||
<program>
|
||||
<mainClass>ai.castor.idf.IDFScorer</mainClass>
|
||||
<name>GetIDF</name>
|
||||
</program>
|
||||
</programs>
|
||||
</configuration>
|
||||
</plugin>
|
||||
|
||||
<plugin>
|
||||
<artifactId>maven-assembly-plugin</artifactId>
|
||||
<configuration>
|
||||
<archive>
|
||||
<manifest>
|
||||
<mainClass>fully.qualified.MainClass</mainClass>
|
||||
</manifest>
|
||||
</archive>
|
||||
<descriptorRefs>
|
||||
<descriptorRef>jar-with-dependencies</descriptorRef>
|
||||
</descriptorRefs>
|
||||
</configuration>
|
||||
</plugin>
|
||||
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
<dependencies>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.lucene</groupId>
|
||||
<artifactId>lucene-benchmark</artifactId>
|
||||
<version>${LUCENE_VERSION}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.lucene</groupId>
|
||||
<artifactId>lucene-core</artifactId>
|
||||
<version>${LUCENE_VERSION}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.lucene</groupId>
|
||||
<artifactId>lucene-test-framework</artifactId>
|
||||
<version>${LUCENE_VERSION}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.lucene</groupId>
|
||||
<artifactId>lucene-backward-codecs</artifactId>
|
||||
<version>${LUCENE_VERSION}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>commons-logging</groupId>
|
||||
<artifactId>commons-logging</artifactId>
|
||||
<version>1.2</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.codehaus.jackson</groupId>
|
||||
<artifactId>jackson-mapper-asl</artifactId>
|
||||
<version>1.9.13</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>commons-cli</groupId>
|
||||
<artifactId>commons-cli</artifactId>
|
||||
<version>1.2</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>commons-io</groupId>
|
||||
<artifactId>commons-io</artifactId>
|
||||
<version>2.4</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
<artifactId>commons-lang3</artifactId>
|
||||
<version>3.4</version>
|
||||
<type>jar</type>
|
||||
<scope>compile</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>args4j</groupId>
|
||||
<artifactId>args4j</artifactId>
|
||||
<version>2.32</version>
|
||||
<type>jar</type>
|
||||
<scope>compile</scope>
|
||||
</dependency>
|
||||
|
||||
</dependencies>
|
||||
</project>
|
||||
@@ -0,0 +1,163 @@
|
||||
/**
|
||||
* Anserini: An information retrieval toolkit built on Lucene
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package ai.castor.idf;
|
||||
|
||||
import org.apache.lucene.analysis.Analyzer;
|
||||
import org.apache.lucene.analysis.StopFilter;
|
||||
import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
|
||||
import org.apache.lucene.analysis.en.EnglishAnalyzer;
|
||||
import org.apache.lucene.index.DirectoryReader;
|
||||
import org.apache.lucene.index.IndexReader;
|
||||
import org.apache.lucene.index.Term;
|
||||
import org.apache.lucene.queryparser.classic.ParseException;
|
||||
import org.apache.lucene.queryparser.classic.QueryParser;
|
||||
import org.apache.lucene.search.*;
|
||||
import org.apache.lucene.search.similarities.ClassicSimilarity;
|
||||
import org.apache.lucene.store.FSDirectory;
|
||||
import org.kohsuke.args4j.*;
|
||||
|
||||
import java.io.*;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.*;
|
||||
|
||||
|
||||
public class IDFScorer {
|
||||
|
||||
public static class Args {
|
||||
// required arguments
|
||||
@Option(name = "-index", metaVar = "[path]", required = true, usage = "Lucene index")
|
||||
public String index;
|
||||
|
||||
@Option(name = "-config", metaVar = "[string]", required = true, usage = "type of the dataset")
|
||||
public String config;
|
||||
|
||||
@Option(name = "-output", metaVar = "[path]", required = true, usage = "path of the file to be created")
|
||||
public String output;
|
||||
|
||||
// optional arguments
|
||||
@Option(name = "-analyze", usage = "passage scores")
|
||||
public boolean analyze = false;
|
||||
}
|
||||
|
||||
private final IndexReader reader;
|
||||
private final FSDirectory directory;
|
||||
private final List<String> stopWords;
|
||||
|
||||
public static final String FIELD_BODY = "contents";
|
||||
|
||||
public IDFScorer(IDFScorer.Args args) throws Exception {
|
||||
Path indexPath = Paths.get(args.index);
|
||||
if (!Files.exists(indexPath) || !Files.isDirectory(indexPath) || !Files.isReadable(indexPath)) {
|
||||
throw new IllegalArgumentException(args.index + " does not exist or is not a directory.");
|
||||
}
|
||||
this.directory = FSDirectory.open(indexPath);
|
||||
this.reader = DirectoryReader.open(directory);
|
||||
|
||||
stopWords = new ArrayList<>();
|
||||
|
||||
//Get file from resources folder
|
||||
InputStream is = getClass().getResourceAsStream("/ai/castor/qa/english-stoplist.txt");
|
||||
BufferedReader bRdr = new BufferedReader(new InputStreamReader(is));
|
||||
String line;
|
||||
while ((line = bRdr.readLine()) != null) {
|
||||
if (!line.contains("#")) {
|
||||
stopWords.add(line);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public double calcIDF(String query, String answer, boolean analyze) throws ParseException {
|
||||
Analyzer analyzer;
|
||||
if (analyze) {
|
||||
analyzer = new EnglishAnalyzer(StopFilter.makeStopSet(stopWords));
|
||||
} else {
|
||||
analyzer = new WhitespaceAnalyzer();
|
||||
}
|
||||
|
||||
QueryParser qp = new QueryParser(FIELD_BODY, analyzer);
|
||||
ClassicSimilarity similarity = new ClassicSimilarity();
|
||||
|
||||
String escapedQuery = qp.escape(query);
|
||||
Query question = qp.parse(escapedQuery);
|
||||
HashSet<String> questionTerms = new HashSet<>(Arrays.asList(question.toString().trim().split("\\s+")));
|
||||
|
||||
double idf = 0.0;
|
||||
HashSet<String> seenTerms = new HashSet<>();
|
||||
|
||||
String[] terms = answer.split("\\s+");
|
||||
for (String term : terms) {
|
||||
try {
|
||||
TermQuery q = (TermQuery) qp.parse(term);
|
||||
Term t = q.getTerm();
|
||||
|
||||
if (questionTerms.contains(t.toString()) && !seenTerms.contains(t.toString())) {
|
||||
idf += similarity.idf(reader.docFreq(t), reader.numDocs());
|
||||
seenTerms.add(t.toString());
|
||||
} else {
|
||||
idf += 0.0;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
return idf;
|
||||
}
|
||||
|
||||
public void writeToFile(IDFScorer.Args args) throws IOException, ParseException {
|
||||
BufferedReader questionFile = new BufferedReader(new FileReader(args.config + "/a.toks"));
|
||||
BufferedReader answerFile = new BufferedReader(new FileReader(args.config + "/b.toks"));
|
||||
BufferedReader idFile = new BufferedReader(new FileReader(args.config + "/id.txt"));
|
||||
|
||||
BufferedWriter outputFile = new BufferedWriter(new FileWriter(args.output));
|
||||
int i = 0;
|
||||
|
||||
while (true) {
|
||||
String question = questionFile.readLine();
|
||||
String answer = answerFile.readLine();
|
||||
String id = idFile.readLine();
|
||||
|
||||
if (question == null || answer == null || id == null) {
|
||||
break;
|
||||
}
|
||||
|
||||
// 32.1 0 0 0 0.6212325096130371 smmodel
|
||||
// 32.1 0 1 0 0.13309887051582336 smmodel
|
||||
outputFile.write(id + " 0 " + i + " " + calcIDF(question, answer, args.analyze) + " smmodel\n");
|
||||
i++;
|
||||
}
|
||||
outputFile.close();
|
||||
|
||||
}
|
||||
|
||||
public static void main(String[] args) throws Exception {
|
||||
Args qaArgs = new Args();
|
||||
CmdLineParser parser = new CmdLineParser(qaArgs, ParserProperties.defaults().withUsageWidth(90));
|
||||
|
||||
try {
|
||||
parser.parseArgument(args);
|
||||
} catch (CmdLineException e) {
|
||||
System.err.println(e.getMessage());
|
||||
parser.printUsage(System.err);
|
||||
System.err.println("Example: IDFScorer" + parser.printExample(OptionHandlerFilter.REQUIRED));
|
||||
return;
|
||||
}
|
||||
|
||||
IDFScorer g = new IDFScorer(qaArgs);
|
||||
g.writeToFile(qaArgs);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,130 @@
|
||||
# Stop words from NLTK
|
||||
# Source: https://gist.github.com/sebleier/554280
|
||||
#
|
||||
i
|
||||
me
|
||||
my
|
||||
myself
|
||||
we
|
||||
our
|
||||
ours
|
||||
ourselves
|
||||
you
|
||||
your
|
||||
yours
|
||||
yourself
|
||||
yourselves
|
||||
he
|
||||
him
|
||||
his
|
||||
himself
|
||||
she
|
||||
her
|
||||
hers
|
||||
herself
|
||||
it
|
||||
its
|
||||
itself
|
||||
they
|
||||
them
|
||||
their
|
||||
theirs
|
||||
themselves
|
||||
what
|
||||
which
|
||||
who
|
||||
whom
|
||||
this
|
||||
that
|
||||
these
|
||||
those
|
||||
am
|
||||
is
|
||||
are
|
||||
was
|
||||
were
|
||||
be
|
||||
been
|
||||
being
|
||||
have
|
||||
has
|
||||
had
|
||||
having
|
||||
do
|
||||
does
|
||||
did
|
||||
doing
|
||||
a
|
||||
an
|
||||
the
|
||||
and
|
||||
but
|
||||
if
|
||||
or
|
||||
because
|
||||
as
|
||||
until
|
||||
while
|
||||
of
|
||||
at
|
||||
by
|
||||
for
|
||||
with
|
||||
about
|
||||
against
|
||||
between
|
||||
into
|
||||
through
|
||||
during
|
||||
before
|
||||
after
|
||||
above
|
||||
below
|
||||
to
|
||||
from
|
||||
up
|
||||
down
|
||||
in
|
||||
out
|
||||
on
|
||||
off
|
||||
over
|
||||
under
|
||||
again
|
||||
further
|
||||
then
|
||||
once
|
||||
here
|
||||
there
|
||||
when
|
||||
where
|
||||
why
|
||||
how
|
||||
all
|
||||
any
|
||||
both
|
||||
each
|
||||
few
|
||||
more
|
||||
most
|
||||
other
|
||||
some
|
||||
such
|
||||
no
|
||||
nor
|
||||
not
|
||||
only
|
||||
own
|
||||
same
|
||||
so
|
||||
than
|
||||
too
|
||||
very
|
||||
s
|
||||
t
|
||||
can
|
||||
will
|
||||
just
|
||||
don
|
||||
should
|
||||
now
|
||||
Reference in New Issue
Block a user