mirror of
https://github.com/wassname/Castor.git
synced 2026-09-09 11:13:20 +08:00
* reimplementation of SM model * features without normalization; parallel running of different modes * minor fix
This commit is contained in:
rosequ
@@ -5,3 +5,6 @@ __pycache__
|
||||
.idea/
|
||||
*idfsim
|
||||
*.swp
|
||||
trec_eval.9.0/
|
||||
*.pt
|
||||
text/
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
*pyc
|
||||
*.pt
|
||||
text/
|
||||
trained_models/
|
||||
trec_eval-8.0/trec_eval.dSYM
|
||||
@@ -0,0 +1,78 @@
|
||||
## SM model
|
||||
|
||||
#### References:
|
||||
1. Aliaksei _S_everyn and Alessandro _M_oschitti. 2015. Learning to Rank Short Text Pairs with Convolutional Deep Neural
|
||||
Networks. In Proceedings of the 38th International ACM SIGIR Conference on Research and Development in Information
|
||||
Retrieval (SIGIR '15). ACM, New York, NY, USA, 373-382. DOI: http://dx.doi.org/10.1145/2766462.2767738
|
||||
|
||||
|
||||
### Requirements
|
||||
```
|
||||
nltk==3.2.2
|
||||
numpy==1.11.3
|
||||
pytorch==0.1.12
|
||||
```
|
||||
|
||||
The code uses torchtext for text processing. Set torchtext:
|
||||
```bash
|
||||
git clone https://github.com/pytorch/text.git
|
||||
cd text
|
||||
python setup.py install
|
||||
```
|
||||
|
||||
We use `trec_eval` for evaluation:
|
||||
|
||||
```bash
|
||||
cd eval
|
||||
tar -xvf trec_eval.9.0.tar.gz
|
||||
make
|
||||
cd ..
|
||||
```
|
||||
|
||||
Download the word2vec model from [here] (https://drive.google.com/file/d/0B2u_nClt6NbzUmhOZU55eEo4QWM/view?usp=sharing)
|
||||
and copy it to the `data/` folder.
|
||||
|
||||
### Training the model
|
||||
|
||||
You can train the SM model for the 4 following configurations:
|
||||
1. __random__ - the word embedddings are initialized randomly and are tuned during training
|
||||
2. __static__ - the word embeddings are static (Severyn and Moschitti, SIGIR'15)
|
||||
3. __non-static__ - the word embeddings are tuned during training
|
||||
4. __multichannel__ - contains static and non-static channels for question and answer conv layers
|
||||
|
||||
To train on GPU 0 with static configuration:
|
||||
|
||||
```bash
|
||||
python train.py --mode static --gpu 0
|
||||
```
|
||||
|
||||
NB: pass `--no_cuda` to use CPU
|
||||
|
||||
The trained model will be save to:
|
||||
```
|
||||
saves/static_best_model.pt
|
||||
```
|
||||
|
||||
### Testing the model
|
||||
|
||||
```
|
||||
python main.py --trained_model saves/TREC/multichannel_best_model.pt
|
||||
```
|
||||
|
||||
### Evaluation
|
||||
|
||||
The performance on TrecQA dataset:
|
||||
|
||||
### Best dev
|
||||
|
||||
Metric |rand |static|non-static|multichannel
|
||||
-------|------|------|----------|------------
|
||||
MAP |0.8096|0.8162|0.8387 | 0.8274
|
||||
MRR |0.8560|0.8918|0.9058 | 0.8818
|
||||
|
||||
### Test
|
||||
|
||||
Metric |rand |static|non-static|multichannel
|
||||
-------|-------|------|----------|------------
|
||||
MAP |0.7441 |0.7524|0.7688 |0.7641
|
||||
MRR |0.8172 |0.8012|0.8144 |0.8174
|
||||
@@ -0,0 +1,28 @@
|
||||
from argparse import ArgumentParser
|
||||
|
||||
def get_args():
|
||||
parser = ArgumentParser(description="SM CNN")
|
||||
parser.add_argument('--no_cuda', action='store_false', help='do not use cuda', dest='cuda')
|
||||
parser.add_argument('--gpu', type=int, default=0) # Use -1 for CPU
|
||||
parser.add_argument('--epochs', type=int, default=30)
|
||||
parser.add_argument('--batch_size', type=int, default=64)
|
||||
parser.add_argument('--mode', type=str, default='static')
|
||||
parser.add_argument('--lr', type=float, default=1.0)
|
||||
parser.add_argument('--seed', type=int, default=3435)
|
||||
parser.add_argument('--dataset', type=str, default='TREC')
|
||||
parser.add_argument('--resume_snapshot', type=str, default=None)
|
||||
parser.add_argument('--dev_every', type=int, default=30)
|
||||
parser.add_argument('--log_every', type=int, default=10)
|
||||
parser.add_argument('--patience', type=int, default=50)
|
||||
parser.add_argument('--save_path', type=str, default='saves')
|
||||
parser.add_argument('--output_channel', type=int, default=100)
|
||||
parser.add_argument('--filter_width', type=int, default=5)
|
||||
parser.add_argument('--words_dim', type=int, default=50)
|
||||
parser.add_argument('--dropout', type=float, default=0.5)
|
||||
parser.add_argument('--epoch_decay', type=int, default=15)
|
||||
parser.add_argument('--vector_cache', type=str, default='data/word2vec.trecqa.pt')
|
||||
parser.add_argument('--trained_model', type=str, default="")
|
||||
parser.add_argument('--weight_decay',type=float, default=1e-5)
|
||||
|
||||
args = parser.parse_args()
|
||||
return args
|
||||
Executable
+340
@@ -0,0 +1,340 @@
|
||||
#!/usr/bin/perl -w
|
||||
|
||||
# Graded relevance assessment script for the TREC 2010 Web track
|
||||
# Evalution measures are written to standard output in CSV format.
|
||||
#
|
||||
# Currently reports only NDCG and ERR
|
||||
# (see http://learningtorankchallenge.yahoo.com/instructions.php)
|
||||
|
||||
use constant LOGBASEDIV => log(2.0);
|
||||
|
||||
# gloals
|
||||
my $QRELS;
|
||||
my $VERSION = "version 1.3 (Mon Apr 29 20:50:24 EDT 2013)";
|
||||
my $MAX_JUDGMENT = 4; # Maximum gain value allowed in qrels file.
|
||||
my $K = 20; # Reporting depth for results.
|
||||
my $USAGE = "usage: $0 [options] qrels run\n
|
||||
options:\n
|
||||
-c
|
||||
Average over the complete set of topics in the relevance judgments
|
||||
instead of the topics in the intersection of relevance judgments
|
||||
and results.\n
|
||||
-k value
|
||||
Non-negative integer depth of ranking to evaluate in range [1,inf].
|
||||
Default value is k=@{[($K)]}.\n
|
||||
-baseline BASELINE_RUN_FILE
|
||||
Baseline run to use for risk-sensitive evaluation\n
|
||||
-riskAlpha value
|
||||
Non-negative Risk sensitivity value to use when doing risk-sensitive
|
||||
evaluation. A baseline must still be specified. By default 0.
|
||||
The final weight to downside changes in performance is (1+value).\n";
|
||||
|
||||
|
||||
|
||||
use strict 'vars';
|
||||
|
||||
{ # main block to scope variables
|
||||
if ($#ARGV >= 0 && ($ARGV[0] eq "-v" || $ARGV[0] eq "-version")) {
|
||||
print "$0: $VERSION\n";
|
||||
exit 0;
|
||||
}
|
||||
|
||||
my $baselineRun = undef;
|
||||
my $riskAlpha = 0;
|
||||
my $cflag = 0;
|
||||
while ($#ARGV != 1) # should probably replace this with perl's argument parsing
|
||||
{
|
||||
if ($#ARGV >= 0 && $ARGV[0] eq "-help") {
|
||||
print "$USAGE\n";
|
||||
exit 0;
|
||||
}
|
||||
elsif ($#ARGV >= 2 and ("-c" eq $ARGV[0]))
|
||||
{
|
||||
$cflag = 1;
|
||||
shift @ARGV;
|
||||
}
|
||||
elsif ($#ARGV >= 3 and ("-k" eq $ARGV[0]))
|
||||
{
|
||||
$K = int($ARGV[1]);
|
||||
die $USAGE if ($K < 1);
|
||||
# print STDERR "k=$K\n";
|
||||
shift @ARGV; shift @ARGV;
|
||||
}
|
||||
elsif ($#ARGV >= 3 and ("-baseline" eq $ARGV[0]))
|
||||
{
|
||||
$baselineRun = $ARGV[1];
|
||||
shift @ARGV; shift @ARGV;
|
||||
}
|
||||
elsif ($#ARGV >= 3 and ("-riskAlpha" eq $ARGV[0]))
|
||||
{
|
||||
$riskAlpha = $ARGV[1];
|
||||
die $USAGE if ($riskAlpha < 0.0);
|
||||
shift @ARGV; shift @ARGV;
|
||||
}
|
||||
else
|
||||
{
|
||||
die $USAGE;
|
||||
}
|
||||
}
|
||||
|
||||
die $USAGE unless $#ARGV == 1;
|
||||
$QRELS = $ARGV[0];
|
||||
my $run = $ARGV[1];
|
||||
|
||||
# Read qrels file, check format, and sort
|
||||
my @qrels = ();
|
||||
my %seen = ();
|
||||
open (QRELS,"<$QRELS") || die "$0: cannot open \"$QRELS\": $!\n";
|
||||
while (<QRELS>) {
|
||||
s/[\r\n]//g;
|
||||
my ($topic, $zero, $docno, $judgment) = split (' ');
|
||||
$topic =~ s/^.*\-//;
|
||||
die "$0: format error on line $. of \"$QRELS\"\n"
|
||||
unless
|
||||
$topic =~ /^[0-9]+$/ && $zero == 0
|
||||
&& $judgment =~ /^-?[0-9]+$/ && $judgment <= $MAX_JUDGMENT;
|
||||
if ($judgment > 0) {
|
||||
$qrels[$#qrels + 1]= "$topic $docno $judgment";
|
||||
$seen{$topic} = 1;
|
||||
}
|
||||
}
|
||||
close (QRELS);
|
||||
@qrels = sort qrelsOrder (@qrels);
|
||||
|
||||
# Process qrels: store judgments and compute ideal gains
|
||||
my $topicCurrent = -1;
|
||||
my %ideal = ();
|
||||
my @gain = ();
|
||||
my %judgment = ();
|
||||
for (my $i = 0; $i <= $#qrels; $i++) {
|
||||
my ($topic, $docno, $judgment) = split (' ', $qrels[$i]);
|
||||
if ($topic != $topicCurrent) {
|
||||
if ($topicCurrent >= 0) {
|
||||
$ideal{$topicCurrent} = &dcg($K, @gain);
|
||||
$#gain = -1;
|
||||
}
|
||||
$topicCurrent = $topic;
|
||||
}
|
||||
next if $judgment < 0;
|
||||
$judgment{"$topic:$docno"} = $gain[$#gain + 1] = $judgment;
|
||||
}
|
||||
if ($topicCurrent >= 0) {
|
||||
$ideal{$topicCurrent} = &dcg($K, @gain);
|
||||
$#gain = -1;
|
||||
}
|
||||
|
||||
# process baseline if doing risk sensitive
|
||||
my ($baseNDCGByTopic,$baseERRByTopic,$baserunname);
|
||||
if (defined $baselineRun)
|
||||
{
|
||||
($baseNDCGByTopic,$baseERRByTopic,$baserunname) = processRun($baselineRun,0,\%seen,\%ideal,\%judgment,$cflag,0);
|
||||
}
|
||||
|
||||
# process main run
|
||||
processRun($run,1,\%seen,\%ideal,\%judgment,$cflag,defined($baselineRun),$riskAlpha,$baserunname,$baseNDCGByTopic,$baseERRByTopic);
|
||||
|
||||
exit 0;
|
||||
|
||||
} # end main block
|
||||
|
||||
# comparison function for qrels: by topic then judgment
|
||||
sub qrelsOrder {
|
||||
my ($topicA, $docnoA, $judgmentA) = split (' ', $a);
|
||||
my ($topicB, $docnoB, $judgmentB) = split (' ', $b);
|
||||
|
||||
if ($topicA < $topicB) {
|
||||
return -1;
|
||||
} elsif ($topicA > $topicB) {
|
||||
return 1;
|
||||
} else {
|
||||
return $judgmentB <=> $judgmentA;
|
||||
}
|
||||
}
|
||||
|
||||
# comparison function for runs: by topic then score then docno
|
||||
sub runOrder {
|
||||
my ($topicA, $docnoA, $scoreA) = split (' ', $a);
|
||||
my ($topicB, $docnoB, $scoreB) = split (' ', $b);
|
||||
|
||||
if ($topicA < $topicB) {
|
||||
return -1;
|
||||
} elsif ($topicA > $topicB) {
|
||||
return 1;
|
||||
} elsif ($scoreA < $scoreB) {
|
||||
return 1;
|
||||
} elsif ($scoreA > $scoreB) {
|
||||
return -1;
|
||||
} elsif ($docnoA lt $docnoB) {
|
||||
return 1;
|
||||
} elsif ($docnoA gt $docnoB) {
|
||||
return -1;
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
# compute DCG over a sorted array of gain values, reporting at depth $k
|
||||
sub dcg {
|
||||
my ($k, @gain) = @_;
|
||||
my ($i, $score) = (0, 0);
|
||||
|
||||
for ($i = 0; $i <= ($k <= $#gain ? $k - 1 : $#gain); $i++) {
|
||||
$score += (2**$gain[$i] - 1)/(log ($i + 2)/ +LOGBASEDIV);
|
||||
}
|
||||
return $score;
|
||||
}
|
||||
|
||||
# compute ERR over a sorted array of gain values, reporting at depth $k
|
||||
sub err {
|
||||
my ($k, @gain) = @_;
|
||||
my ($i, $score, $decay, $r);
|
||||
|
||||
$score = 0.0;
|
||||
$decay = 1.0;
|
||||
for ($i = 0; $i <= ($k <= $#gain ? $k - 1 : $#gain); $i++) {
|
||||
$r = (2**$gain[$i] - 1)/(2**$MAX_JUDGMENT);
|
||||
$score += $r*$decay/($i + 1);
|
||||
$decay *= (1 - $r);
|
||||
}
|
||||
return $score;
|
||||
}
|
||||
|
||||
sub riskWeighted
|
||||
{
|
||||
my ($run,$base,$alpha) = @_;
|
||||
if ($run < $base)
|
||||
{
|
||||
$run = (1+$alpha) * ($run - $base);
|
||||
}
|
||||
else
|
||||
{
|
||||
$run = $run - $base;
|
||||
}
|
||||
return $run;
|
||||
}
|
||||
# compute and report information for current topic
|
||||
sub topicDone {
|
||||
my ($printTopic, $runid, $topic, $pndcgTotal, $perrTotal, $ptopics, $pseen, $pideal,
|
||||
$isRiskSensitive, $riskAlpha, $baseNDCG, $baseERR, @gain) = @_;
|
||||
my($ndcg, $err) = (0, 0);
|
||||
if (exists($$pseen{$topic}) and defined($$pseen{$topic}) and $$pseen{$topic}) {
|
||||
$ndcg = &dcg($K, @gain)/$$pideal{$topic};
|
||||
$err = &err ($K, @gain);
|
||||
$ndcg = riskWeighted($ndcg,$baseNDCG,$riskAlpha) if ($isRiskSensitive);
|
||||
$err = riskWeighted($err,$baseERR,$riskAlpha) if ($isRiskSensitive);
|
||||
$$pndcgTotal += $ndcg;
|
||||
$$perrTotal += $err;
|
||||
$$ptopics++;
|
||||
printf("$runid,$topic,%.5f,%.5f\n",$ndcg,$err) if ($printTopic);
|
||||
return ($ndcg,$err);
|
||||
}
|
||||
}
|
||||
|
||||
sub processRun
|
||||
{
|
||||
my ($run,$printTopics,$pseen,$pideal,$pjudgment,$avgOverAllTopics,$isRiskSensitive,$riskAlpha,$baserunname,$baseNDCGByTopic,$baseERRByTopic) = @_;
|
||||
my $ndcgByTopic = {()};
|
||||
my $errByTopic = {()};
|
||||
my $runid = "?????";
|
||||
my @run = ();
|
||||
# Read run rile, check format, and sort
|
||||
open (RUN,"<$run") || die "$0: cannot open \"$run\": $!\n";
|
||||
while (<RUN>) {
|
||||
s/[\r\n]//g;
|
||||
my ($topic, $q0, $docno, $rank, $score);
|
||||
($topic, $q0, $docno, $rank, $score, $runid) = split (' ');
|
||||
$topic =~ s/^.*\-//;
|
||||
die "$0: format error on line $. of \"$run\"\n"
|
||||
unless
|
||||
$topic =~ /^[0-9]+$/ && $q0 eq "Q0" && $rank =~ /^[0-9]+$/ && $runid;
|
||||
$run[$#run + 1] = "$topic $docno $score";
|
||||
}
|
||||
|
||||
@run = sort runOrder (@run);
|
||||
|
||||
my %processed = ();
|
||||
foreach my $topic (%$pseen)
|
||||
{
|
||||
$processed{$topic} = 0;
|
||||
}
|
||||
|
||||
if ($isRiskSensitive)
|
||||
{
|
||||
$runid = sprintf("%s (rel to. %s, rs=1+a, a=%s)",$runid,$baserunname,$riskAlpha);
|
||||
}
|
||||
|
||||
# Process runs: compute measures for each topic and average
|
||||
my $ndcgTotal = 0;
|
||||
my $errTotal = 0;
|
||||
my $topics = 0;
|
||||
print "runid,topic,ndcg\@$K,err\@$K\n" if ($printTopics);
|
||||
my $topicCurrent = -1;
|
||||
my @gain = ();
|
||||
for (my $i = 0; $i <= $#run; $i++) {
|
||||
my ($topic, $docno, $score) = split (' ', $run[$i]);
|
||||
if ($topic != $topicCurrent) {
|
||||
if ($topicCurrent >= 0) {
|
||||
my ($baseNDCG,$baseERR) = 0;
|
||||
if ($isRiskSensitive)
|
||||
{
|
||||
$baseNDCG = $$baseNDCGByTopic{$topicCurrent} if (exists($$baseNDCGByTopic{$topicCurrent}) and defined($$baseNDCGByTopic{$topicCurrent}));
|
||||
$baseERR = $$baseERRByTopic{$topicCurrent} if (exists($$baseERRByTopic{$topicCurrent}) and defined($$baseERRByTopic{$topicCurrent}));
|
||||
}
|
||||
my ($ndcg,$err) = &topicDone ($printTopics, $runid, $topicCurrent, \$ndcgTotal, \$errTotal, \$topics,
|
||||
$pseen, $pideal, $isRiskSensitive, $riskAlpha, $baseNDCG, $baseERR, @gain);
|
||||
$$ndcgByTopic{$topicCurrent} = $ndcg;
|
||||
$$errByTopic{$topicCurrent} = $err;
|
||||
$processed{$topicCurrent} = 1;
|
||||
$#gain = -1;
|
||||
}
|
||||
$topicCurrent = $topic;
|
||||
}
|
||||
my $j = $$pjudgment{"$topic:$docno"};
|
||||
$j = 0 unless $j;
|
||||
$gain[$#gain + 1] = $j;
|
||||
}
|
||||
if ($topicCurrent >= 0) {
|
||||
my ($baseNDCG,$baseERR) = 0;
|
||||
if ($isRiskSensitive)
|
||||
{
|
||||
$baseNDCG = $$baseNDCGByTopic{$topicCurrent} if (exists($$baseNDCGByTopic{$topicCurrent}) and defined($$baseNDCGByTopic{$topicCurrent}));
|
||||
$baseERR = $$baseERRByTopic{$topicCurrent} if (exists($$baseERRByTopic{$topicCurrent}) and defined($$baseERRByTopic{$topicCurrent}));
|
||||
}
|
||||
my ($ndcg,$err) = &topicDone ($printTopics, $runid, $topicCurrent, \$ndcgTotal, \$errTotal, \$topics,
|
||||
$pseen, $pideal, $isRiskSensitive, $riskAlpha, $baseNDCG, $baseERR, @gain);
|
||||
$$ndcgByTopic{$topicCurrent} = $ndcg;
|
||||
$$errByTopic{$topicCurrent} = $err;
|
||||
$processed{$topicCurrent} = 1;
|
||||
$#gain = -1;
|
||||
}
|
||||
my $numTopics = $topics; # $topics has the number in the run (at this point)
|
||||
if ($avgOverAllTopics)
|
||||
{
|
||||
$numTopics = scalar(keys %$pseen); # we want denominator to change whenever flag is on but only need to compute differences for risk
|
||||
if ($isRiskSensitive)
|
||||
{ # need to process any topics that were missing from run
|
||||
my ($baseNDCG,$baseERR) = 0;
|
||||
my @gain = ();
|
||||
foreach my $topicCurrent (sort {$a <=> $b} keys %processed)
|
||||
{
|
||||
next if ($processed{$topicCurrent});
|
||||
$baseNDCG = $$baseNDCGByTopic{$topicCurrent} if (exists($$baseNDCGByTopic{$topicCurrent}) and defined($$baseNDCGByTopic{$topicCurrent}));
|
||||
$baseERR = $$baseERRByTopic{$topicCurrent} if (exists($$baseERRByTopic{$topicCurrent}) and defined($$baseERRByTopic{$topicCurrent}));
|
||||
my ($ndcg,$err) = &topicDone ($printTopics, $runid, $topicCurrent, \$ndcgTotal, \$errTotal, \$topics,
|
||||
$pseen, $pideal, $isRiskSensitive, $riskAlpha, $baseNDCG, $baseERR, @gain);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
my $ndcgAvg = $ndcgTotal;
|
||||
my $errAvg = $errTotal;
|
||||
if ($numTopics > 0)
|
||||
{
|
||||
$ndcgAvg /= $numTopics;
|
||||
$errAvg /= $numTopics;
|
||||
}
|
||||
printf "$runid,amean,%.5f,%.5f\n",$ndcgAvg,$errAvg if ($printTopics);
|
||||
|
||||
return ($ndcgByTopic,$errByTopic,$runid);
|
||||
close(RUN);
|
||||
}
|
||||
Binary file not shown.
@@ -0,0 +1,23 @@
|
||||
import shlex
|
||||
import subprocess
|
||||
|
||||
def evaluate(instances, valid, config):
|
||||
sorted_instances = sorted(instances, key=lambda x: (x[0]))
|
||||
with open('{}.{}.run.txt'.format(valid, config), 'w') as run, open('{}.{}.qrel.txt'.format(valid, config), 'w') as qrel:
|
||||
i = 0
|
||||
for instance in sorted_instances:
|
||||
qid, predicted, score, gold = instance[0], instance[1], instance[2], instance[3]
|
||||
|
||||
# 32.1 0 1 0 0.13309887051582336 smmodel
|
||||
run.write('{} 0 {} 0 {} sm_model\n'.format(qid, i, score))
|
||||
qrel.write('{} 0 {} {}\n'.format(qid, i, gold))
|
||||
i += 1
|
||||
|
||||
pargs = shlex.split("./eval/trec_eval.9.0/trec_eval -m map -m recip_rank {}.{}.qrel.txt {}.{}.run.txt"
|
||||
.format(valid, config, valid, config))
|
||||
p = subprocess.Popen(pargs, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
pout, perr = p.communicate()
|
||||
lines = pout.split(b'\n')
|
||||
map = float(lines[0].strip().split()[-1])
|
||||
mrr = float(lines[1].strip().split()[-1])
|
||||
return map, mrr
|
||||
@@ -0,0 +1,98 @@
|
||||
import numpy as np
|
||||
import random
|
||||
import logging
|
||||
|
||||
import torch
|
||||
from torchtext import data
|
||||
|
||||
from args import get_args
|
||||
from trec_dataset import TrecDataset
|
||||
from evaluate import evaluate
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
logger.setLevel(logging.INFO)
|
||||
|
||||
ch = logging.StreamHandler()
|
||||
ch.setLevel(logging.DEBUG)
|
||||
formatter = logging.Formatter('%(levelname)s - %(message)s')
|
||||
ch.setFormatter(formatter)
|
||||
logger.addHandler(ch)
|
||||
|
||||
args = get_args()
|
||||
config = args
|
||||
|
||||
torch.manual_seed(args.seed)
|
||||
|
||||
if not args.cuda:
|
||||
args.gpu = -1
|
||||
if torch.cuda.is_available() and args.cuda:
|
||||
logger.info("Note: You are using GPU for training")
|
||||
torch.cuda.set_device(args.gpu)
|
||||
torch.cuda.manual_seed(args.seed)
|
||||
if torch.cuda.is_available() and not args.cuda:
|
||||
logger.info("Warning: You have Cuda but do not use it. You are using CPU for training")
|
||||
np.random.seed(args.seed)
|
||||
random.seed(args.seed)
|
||||
|
||||
QID = data.Field(sequential=False)
|
||||
QUESTION = data.Field(batch_first=True)
|
||||
ANSWER = data.Field(batch_first=True)
|
||||
LABEL = data.Field(sequential=False)
|
||||
EXTERNAL = data.Field(sequential=False, tensor_type=torch.FloatTensor, batch_first=True, use_vocab=False,
|
||||
preprocessing=data.Pipeline(lambda x: x.split()),
|
||||
postprocessing=data.Pipeline(lambda x, train: [float(y) for y in x]))
|
||||
train, dev, test = TrecDataset.splits(QID, QUESTION, ANSWER, EXTERNAL, LABEL)
|
||||
|
||||
QID.build_vocab(train, dev, test)
|
||||
QUESTION.build_vocab(train, dev, test)
|
||||
ANSWER.build_vocab(train, dev, test)
|
||||
LABEL.build_vocab(train, dev, test)
|
||||
|
||||
train_iter = data.Iterator(train, batch_size=args.batch_size, device=args.gpu, train=True, repeat=False,
|
||||
sort=False, shuffle=True)
|
||||
dev_iter = data.Iterator(dev, batch_size=args.batch_size, device=args.gpu, train=False, repeat=False,
|
||||
sort=False, shuffle=False)
|
||||
test_iter = data.Iterator(test, batch_size=args.batch_size, device=args.gpu, train=False, repeat=False,
|
||||
sort=False, shuffle=False)
|
||||
|
||||
config.target_class = len(LABEL.vocab)
|
||||
config.questions_num = len(QUESTION.vocab)
|
||||
config.answers_num = len(ANSWER.vocab)
|
||||
print("Label dict:", LABEL.vocab.itos)
|
||||
|
||||
if args.cuda:
|
||||
model = torch.load(args.trained_model, map_location=lambda storage, location: storage.cuda(args.gpu))
|
||||
else:
|
||||
model = torch.load(args.trained_model, map_location=lambda storage,location: storage)
|
||||
|
||||
index2label = np.array(LABEL.vocab.itos)
|
||||
index2qid = np.array(QID.vocab.itos)
|
||||
|
||||
def predict(test_mode, dataset_iter):
|
||||
model.eval()
|
||||
dataset_iter.init_epoch()
|
||||
|
||||
instance = []
|
||||
for dev_batch_idx, dev_batch in enumerate(dataset_iter):
|
||||
qid_array = index2qid[np.transpose(dev_batch.qid.cpu().data.numpy())]
|
||||
true_label_array = index2label[np.transpose(dev_batch.label.cpu().data.numpy())]
|
||||
|
||||
scores = model(dev_batch)
|
||||
|
||||
index_label = np.transpose(torch.max(scores, 1)[1].view(dev_batch.label.size()).cpu().data.numpy())
|
||||
label_array = index2label[index_label]
|
||||
score_array = scores[:, 2].cpu().data.numpy()
|
||||
# print and write the result
|
||||
for i in range(dev_batch.batch_size):
|
||||
this_qid, predicted_label, score, gold_label = qid_array[i], label_array[i], score_array[i], \
|
||||
true_label_array[i]
|
||||
instance.append((this_qid, predicted_label, score, gold_label))
|
||||
|
||||
dev_map, dev_mrr = evaluate(instance, test_mode, config.mode)
|
||||
print(dev_map, dev_mrr)
|
||||
|
||||
# Run the model on the dev set
|
||||
predict('dev', dataset_iter=dev_iter)
|
||||
|
||||
# Run the model on the test set
|
||||
predict('test', dataset_iter=test_iter)
|
||||
@@ -0,0 +1,83 @@
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
import torch.nn.functional as F
|
||||
|
||||
class SmPlusPlus(nn.Module):
|
||||
def __init__(self, config):
|
||||
super(SmPlusPlus, self).__init__()
|
||||
output_channel = config.output_channel
|
||||
questions_num = config.questions_num
|
||||
answers_num = config.answers_num
|
||||
words_dim = config.words_dim
|
||||
filter_width = config.filter_width
|
||||
self.mode = config.mode
|
||||
|
||||
n_classes = config.target_class
|
||||
ext_feats_size = 4
|
||||
|
||||
if self.mode == 'multichannel':
|
||||
input_channel = 2
|
||||
else:
|
||||
input_channel = 1
|
||||
|
||||
self.question_embed = nn.Embedding(questions_num, words_dim)
|
||||
self.answer_embed = nn.Embedding(answers_num, words_dim)
|
||||
self.static_question_embed = nn.Embedding(questions_num, words_dim)
|
||||
self.nonstatic_question_embed = nn.Embedding(questions_num, words_dim)
|
||||
self.static_answer_embed = nn.Embedding(answers_num, words_dim)
|
||||
self.nonstatic_answer_embed = nn.Embedding(answers_num, words_dim)
|
||||
self.static_question_embed.weight.requires_grad = False
|
||||
self.static_answer_embed.weight.requires_grad = False
|
||||
|
||||
self.conv_q = nn.Conv2d(input_channel, output_channel, (filter_width, words_dim), padding=(filter_width - 1, 0))
|
||||
self.conv_a = nn.Conv2d(input_channel, output_channel, (filter_width, words_dim), padding=(filter_width - 1, 0))
|
||||
|
||||
self.dropout = nn.Dropout(config.dropout)
|
||||
n_hidden = 2 * output_channel + ext_feats_size
|
||||
|
||||
self.combined_feature_vector = nn.Linear(n_hidden, n_hidden)
|
||||
self.hidden = nn.Linear(n_hidden, n_classes)
|
||||
|
||||
def forward(self, x):
|
||||
x_question = x.question
|
||||
x_answer = x.answer
|
||||
x_ext = x.ext_feat
|
||||
|
||||
if self.mode == 'rand':
|
||||
question = self.question_embed(x_question).unsqueeze(1)
|
||||
answer = self.answer_embed(x_answer).unsqueeze(1) # (batch, sent_len, embed_dim)
|
||||
x = [F.tanh(self.conv_q(question)).squeeze(3), F.tanh(self.conv_a(answer)).squeeze(3)]
|
||||
x = [F.max_pool1d(i, i.size(2)).squeeze(2) for i in x] # max-over-time pooling
|
||||
# actual SM model mode (Severyn & Moschitti, 2015)
|
||||
elif self.mode == 'static':
|
||||
question = self.static_question_embed(x_question).unsqueeze(1)
|
||||
answer = self.static_answer_embed(x_answer).unsqueeze(1) # (batch, sent_len, embed_dim)
|
||||
x = [F.tanh(self.conv_q(question)).squeeze(3), F.tanh(self.conv_a(answer)).squeeze(3)]
|
||||
x = [F.max_pool1d(i, i.size(2)).squeeze(2) for i in x] # max-over-time pooling
|
||||
elif self.mode == 'non-static':
|
||||
question = self.nonstatic_question_embed(x_question).unsqueeze(1)
|
||||
answer = self.nonstatic_answer_embed(x_answer).unsqueeze(1) # (batch, sent_len, embed_dim)
|
||||
x = [F.tanh(self.conv_q(question)).squeeze(3), F.tanh(self.conv_a(answer)).squeeze(3)]
|
||||
x = [F.max_pool1d(i, i.size(2)).squeeze(2) for i in x] # max-over-time pooling
|
||||
elif self.mode == 'multichannel':
|
||||
question_static = self.static_question_embed(x_question)
|
||||
answer_static = self.static_answer_embed(x_answer)
|
||||
question_nonstatic = self.nonstatic_question_embed(x_question)
|
||||
answer_nonstatic = self.nonstatic_answer_embed(x_answer)
|
||||
question = torch.stack([question_static, question_nonstatic], dim=1)
|
||||
answer = torch.stack([answer_static, answer_nonstatic], dim=1)
|
||||
x = [F.tanh(self.conv_q(question)).squeeze(3), F.tanh(self.conv_a(answer)).squeeze(3)]
|
||||
x = [F.max_pool1d(i, i.size(2)).squeeze(2) for i in x] # max-over-time pooling
|
||||
else:
|
||||
print("Unsupported Mode")
|
||||
exit()
|
||||
|
||||
# append external features and feed to fc
|
||||
x.append(x_ext)
|
||||
x = torch.cat(x, 1)
|
||||
|
||||
x = F.tanh(self.combined_feature_vector(x))
|
||||
x = self.dropout(x)
|
||||
x = self.hidden(x)
|
||||
return x
|
||||
@@ -0,0 +1,3 @@
|
||||
nltk==3.2.1
|
||||
numpy==1.11.3
|
||||
pytorch==0.1.12
|
||||
@@ -0,0 +1,214 @@
|
||||
import time
|
||||
import os
|
||||
import numpy as np
|
||||
import random
|
||||
|
||||
import logging
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from torchtext import data
|
||||
|
||||
from args import get_args
|
||||
from model import SmPlusPlus
|
||||
from trec_dataset import TrecDataset
|
||||
from evaluate import evaluate
|
||||
|
||||
args = get_args()
|
||||
config = args
|
||||
|
||||
torch.manual_seed(args.seed)
|
||||
|
||||
def set_vectors(field, vector_path):
|
||||
if os.path.isfile(vector_path):
|
||||
stoi, vectors, dim = torch.load(vector_path)
|
||||
field.vocab.vectors = torch.Tensor(len(field.vocab), dim)
|
||||
|
||||
for i, token in enumerate(field.vocab.itos):
|
||||
wv_index = stoi.get(token, None)
|
||||
if wv_index is not None:
|
||||
field.vocab.vectors[i] = vectors[wv_index]
|
||||
else:
|
||||
# initialize <unk> with U(-0.25, 0.25) vectors
|
||||
field.vocab.vectors[i] = torch.FloatTensor(dim).uniform_(-0.25, 0.25)
|
||||
else:
|
||||
print("Error: Need word embedding pt file")
|
||||
print("Error: Need word embedding pt file")
|
||||
exit(1)
|
||||
return field
|
||||
|
||||
|
||||
def regularize_loss(model, loss):
|
||||
flattened_params = []
|
||||
reg = args.weight_decay
|
||||
|
||||
for p in model.parameters():
|
||||
f = p.data.clone()
|
||||
flattened_params.append(f.view(-1))
|
||||
|
||||
fp = torch.cat(flattened_params)
|
||||
loss = loss + 0.5 * reg * fp.norm() * fp.norm()
|
||||
return loss
|
||||
|
||||
# Set default configuration in : args.py
|
||||
args = get_args()
|
||||
config = args
|
||||
|
||||
# Set random seed for reproducibility
|
||||
torch.manual_seed(args.seed)
|
||||
if not args.cuda:
|
||||
args.gpu = -1
|
||||
if torch.cuda.is_available() and args.cuda:
|
||||
print("Note: You are using GPU for training")
|
||||
torch.cuda.set_device(args.gpu)
|
||||
torch.cuda.manual_seed(args.seed)
|
||||
if torch.cuda.is_available() and not args.cuda:
|
||||
print("You have Cuda but you're using CPU for training.")
|
||||
np.random.seed(args.seed)
|
||||
random.seed(args.seed)
|
||||
|
||||
QID = data.Field(sequential=False)
|
||||
QUESTION = data.Field(batch_first=True)
|
||||
ANSWER = data.Field(batch_first=True)
|
||||
LABEL = data.Field(sequential=False)
|
||||
EXTERNAL = data.Field(sequential=False, tensor_type=torch.FloatTensor, batch_first=True, use_vocab=False,
|
||||
preprocessing=data.Pipeline(lambda x: x.split()),
|
||||
postprocessing=data.Pipeline(lambda x, train: [float(y) for y in x]))
|
||||
train, dev, test = TrecDataset.splits(QID, QUESTION, ANSWER, EXTERNAL, LABEL)
|
||||
|
||||
QID.build_vocab(train, dev, test)
|
||||
QUESTION.build_vocab(train, dev, test)
|
||||
ANSWER.build_vocab(train, dev, test)
|
||||
LABEL.build_vocab(train, dev, test)
|
||||
|
||||
|
||||
QUESTION = set_vectors(QUESTION, args.vector_cache)
|
||||
ANSWER = set_vectors(ANSWER, args.vector_cache)
|
||||
|
||||
train_iter = data.Iterator(train, batch_size=args.batch_size, device=args.gpu, train=True, repeat=False,
|
||||
sort=False, shuffle=True)
|
||||
dev_iter = data.Iterator(dev, batch_size=args.batch_size, device=args.gpu, train=False, repeat=False,
|
||||
sort=False, shuffle=False)
|
||||
test_iter = data.Iterator(test, batch_size=args.batch_size, device=args.gpu, train=False, repeat=False,
|
||||
sort=False, shuffle=False)
|
||||
|
||||
config.target_class = len(LABEL.vocab)
|
||||
config.questions_num = len(QUESTION.vocab)
|
||||
config.answers_num = len(ANSWER.vocab)
|
||||
|
||||
print("Dataset {} Mode {}".format(args.dataset, args.mode))
|
||||
print("VOCAB num", len(QUESTION.vocab))
|
||||
print("LABEL.target_class:", len(LABEL.vocab))
|
||||
print("LABELS:", LABEL.vocab.itos)
|
||||
print("Train instance", len(train))
|
||||
print("Dev instance", len(dev))
|
||||
print("Test instance", len(test))
|
||||
|
||||
if args.resume_snapshot:
|
||||
if args.cuda:
|
||||
model = torch.load(args.resume_snapshot, map_location=lambda storage, location: storage.cuda(args.gpu))
|
||||
else:
|
||||
model = torch.load(args.resume_snapshot, map_location=lambda storage, location: storage)
|
||||
else:
|
||||
model = SmPlusPlus(config)
|
||||
model.static_question_embed.weight.data.copy_(QUESTION.vocab.vectors)
|
||||
model.nonstatic_question_embed.weight.data.copy_(QUESTION.vocab.vectors)
|
||||
model.static_answer_embed.weight.data.copy_(ANSWER.vocab.vectors)
|
||||
model.nonstatic_answer_embed.weight.data.copy_(ANSWER.vocab.vectors)
|
||||
|
||||
if args.cuda:
|
||||
model.cuda()
|
||||
print("Shift model to GPU")
|
||||
|
||||
|
||||
parameter = filter(lambda p: p.requires_grad, model.parameters())
|
||||
|
||||
# the SM model originally follows SGD but Adadelta is used here
|
||||
optimizer = torch.optim.Adadelta(parameter, lr=args.lr, weight_decay=args.weight_decay)
|
||||
criterion = nn.CrossEntropyLoss()
|
||||
early_stop = False
|
||||
best_dev_map = 0
|
||||
iterations = 0
|
||||
iters_not_improved = 0
|
||||
epoch = 0
|
||||
start = time.time()
|
||||
header = ' Time Epoch Iteration Progress (%Epoch) Loss Dev/Loss Accuracy Dev/Accuracy'
|
||||
dev_log_template = ' '.join('{:>6.0f},{:>5.0f},{:>9.0f},{:>5.0f}/{:<5.0f} {:>7.0f}%,{:>8.6f},{:8.6f},{:12.4f},{:12.4f}'.split(','))
|
||||
log_template = ' '.join('{:>6.0f},{:>5.0f},{:>9.0f},{:>5.0f}/{:<5.0f} {:>7.0f}%,{:>8.6f},{},{:12.4f},{}'.split(','))
|
||||
os.makedirs(args.save_path, exist_ok=True)
|
||||
os.makedirs(os.path.join(args.save_path, args.dataset), exist_ok=True)
|
||||
print(header)
|
||||
|
||||
index2label = np.array(LABEL.vocab.itos)
|
||||
index2qid = np.array(QID.vocab.itos)
|
||||
index2question = np.array(ANSWER.vocab.itos)
|
||||
|
||||
while True:
|
||||
if early_stop:
|
||||
print("Early Stopping. Epoch: {}, Best Dev Acc: {}".format(epoch, best_dev_map))
|
||||
break
|
||||
epoch += 1
|
||||
train_iter.init_epoch()
|
||||
n_correct, n_total = 0, 0
|
||||
|
||||
for batch_idx, batch in enumerate(train_iter):
|
||||
iterations += 1
|
||||
model.train(); optimizer.zero_grad()
|
||||
scores = model(batch)
|
||||
n_correct += (torch.max(scores, 1)[1].view(batch.label.size()).data == batch.label.data).sum()
|
||||
n_total += batch.batch_size
|
||||
train_acc = 100. * n_correct / n_total
|
||||
|
||||
loss = criterion(scores, batch.label)
|
||||
loss = regularize_loss(model, loss)
|
||||
loss.backward()
|
||||
optimizer.step()
|
||||
|
||||
# Evaluate performance on validation set
|
||||
if iterations % args.dev_every == 1:
|
||||
# switch model into evaluation mode
|
||||
model.eval()
|
||||
dev_iter.init_epoch()
|
||||
n_dev_correct = 0
|
||||
dev_losses = []
|
||||
instance = []
|
||||
for dev_batch_idx, dev_batch in enumerate(dev_iter):
|
||||
qid_array = index2qid[np.transpose(dev_batch.qid.cpu().data.numpy())]
|
||||
true_label_array = index2label[np.transpose(dev_batch.label.cpu().data.numpy())]
|
||||
|
||||
scores = model(dev_batch)
|
||||
n_dev_correct += (torch.max(scores, 1)[1].view(dev_batch.label.size()).data == dev_batch.label.data).sum()
|
||||
dev_loss = criterion(scores, dev_batch.label)
|
||||
dev_losses.append(dev_loss.data[0])
|
||||
index_label = np.transpose(torch.max(scores, 1)[1].view(dev_batch.label.size()).cpu().data.numpy())
|
||||
label_array = index2label[index_label]
|
||||
# get the relevance scores
|
||||
score_array = scores[:, 2].cpu().data.numpy()
|
||||
for i in range(dev_batch.batch_size):
|
||||
this_qid, predicted_label, score, gold_label = qid_array[i], label_array[i], score_array[i], true_label_array[i]
|
||||
instance.append((this_qid, predicted_label, score, gold_label))
|
||||
|
||||
|
||||
dev_map, dev_mrr = evaluate(instance, 'valid', config.mode)
|
||||
print(dev_log_template.format(time.time() - start,
|
||||
epoch, iterations, 1 + batch_idx, len(train_iter),
|
||||
100. * (1 + batch_idx) / len(train_iter), loss.data[0],
|
||||
sum(dev_losses) / len(dev_losses), train_acc, dev_map))
|
||||
|
||||
# Update validation results
|
||||
if dev_map > best_dev_map:
|
||||
iters_not_improved = 0
|
||||
best_dev_map = dev_map
|
||||
snapshot_path = os.path.join(args.save_path, args.dataset, args.mode+'_best_model.pt')
|
||||
torch.save(model, snapshot_path)
|
||||
else:
|
||||
iters_not_improved += 1
|
||||
if iters_not_improved >= args.patience:
|
||||
early_stop = True
|
||||
break
|
||||
|
||||
if iterations % args.log_every == 1:
|
||||
# print progress message
|
||||
print(log_template.format(time.time() - start,
|
||||
epoch, iterations, 1 + batch_idx, len(train_iter),
|
||||
100. * (1 + batch_idx) / len(train_iter), loss.data[0], ' ' * 8,
|
||||
n_correct / n_total * 100, ' ' * 12))
|
||||
@@ -0,0 +1,16 @@
|
||||
from torchtext import data
|
||||
import os
|
||||
|
||||
class TrecDataset(data.TabularDataset):
|
||||
dirname = 'data'
|
||||
@classmethod
|
||||
|
||||
def splits(cls, question_id, question_field, answer_field, external_field, label_field,
|
||||
train='train.tsv', validation='dev.tsv', test='test.tsv'):
|
||||
path = './data'
|
||||
prefix_name = 'trecqa.'
|
||||
return super(TrecDataset, cls).splits(
|
||||
os.path.join(path, prefix_name), train, validation, test,
|
||||
format='TSV', fields=[('qid', question_id), ('label', label_field), ('question', question_field),
|
||||
('answer', answer_field), ('ext_feat', external_field)]
|
||||
)
|
||||
@@ -0,0 +1,38 @@
|
||||
from tqdm import tqdm
|
||||
import array
|
||||
import torch
|
||||
import numpy as np
|
||||
|
||||
from argparse import ArgumentParser
|
||||
|
||||
|
||||
def convert(fname, vocab):
|
||||
save_file = '{}.pt'.format(fname)
|
||||
stoi, vectors, dim = [], array.array('d'), None
|
||||
|
||||
# TODO: fix by reading the .dimensions file
|
||||
vocab_size, dim = 2470719, 50
|
||||
W = np.memmap(fname, dtype=np.double, shape=(vocab_size, dim))
|
||||
|
||||
|
||||
print("Loading vectors from {}".format(fname))
|
||||
vectors = []
|
||||
for line in tqdm(W, total=len(W)):
|
||||
entry = line
|
||||
vectors.extend(entry)
|
||||
|
||||
vectors = torch.Tensor(vectors).view(-1, dim)
|
||||
|
||||
with open(vocab) as f:
|
||||
stoi = {word.strip():i for i, word in enumerate(f)}
|
||||
|
||||
print('saving vectors to', save_file)
|
||||
torch.save((stoi, vectors, dim), save_file)
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = ArgumentParser(description='create word embedding')
|
||||
parser.add_argument('--input', type=str, required=True)
|
||||
parser.add_argument('--vocab', type=str, required=True)
|
||||
|
||||
args = parser.parse_args()
|
||||
convert(args.input, args.vocab)
|
||||
Reference in New Issue
Block a user