From 68e0ef45b277a0e6b3386c4d2455de9fbdd04ad0 Mon Sep 17 00:00:00 2001
From: rosequ <rdsequie@uwaterloo.ca>
Date: Wed, 6 Dec 2017 11:34:29 -0500
Subject: [PATCH] Util to build w2v pytorch model (#92)

* util to build w2v pytorch model

* added the code to build the .pt model
---
 sm_cnn/README.md   |  6 ++++--
 utils/build_w2v.py | 30 ++++++++++++++++++++++++++++++
 2 files changed, 34 insertions(+), 2 deletions(-)
 create mode 100644 utils/build_w2v.py

diff --git a/sm_cnn/README.md b/sm_cnn/README.md
index 3853b70..987aa66 100644
--- a/sm_cnn/README.md
+++ b/sm_cnn/README.md
@@ -144,5 +144,7 @@ NB: The results on WikiQA are based on the SM model hyperparameters.
 to the `data/` folder
 
 ```bash
-python utils.py --input data/aquaint+wiki.txt.gz.ndim=50.bin
-```
\ No newline at end of file
+python $PYTHONPATH/utils/build_w2v.py --input data/aquaint+wiki.txt.gz.ndim=50.bin
+```
+
+Note that `$PYTHONPATH` holds the location of the repository root.
\ No newline at end of file
diff --git a/utils/build_w2v.py b/utils/build_w2v.py
new file mode 100644
index 0000000..af30557
--- /dev/null
+++ b/utils/build_w2v.py
@@ -0,0 +1,30 @@
+from tqdm import tqdm
+import torch
+
+from gensim.models.keyedvectors import KeyedVectors
+from argparse import ArgumentParser
+
+def convert(fname, save_file):
+    with open(fname, 'rb') as dim_file:
+        vocab_size, dim = (int(x) for x in dim_file.readline().split())
+
+    word_vectors = KeyedVectors.load_word2vec_format(fname, binary=True)
+
+    print("Loading vectors from {}".format(fname))
+    vectors = []
+    for line in tqdm(word_vectors.syn0, total=len(word_vectors.syn0)):
+        vectors.extend(line.tolist())
+    vectors = torch.Tensor(vectors).view(-1, dim)
+
+    stoi = {word.strip():voc.index for word, voc in word_vectors.vocab.items()}
+
+    print('saving vectors to', save_file)
+    torch.save((stoi, vectors, dim), save_file)
+
+if __name__ == '__main__':
+    parser = ArgumentParser(description='create word embedding')
+    parser.add_argument('--input', type=str, required=True)
+    parser.add_argument('--output', type=str, default='data/word2vec.trecqa.pt')
+
+    args = parser.parse_args()
+    convert(args.input, args.output)
\ No newline at end of file