Fix vocab caching issue (#29)

With the line as-was the vocab cache was stored as b'the' rather than the, meaning that word2vec wasn't found for terms causing massive performance loss (AP 0.71 cf 0.77).
This commit is contained in:
Matt Crane
2017-06-19 13:02:36 -04:00
committed by Jimmy Lin
parent 43dd6fdb1f
commit 945b1fa6c0
+1 -1
View File
@@ -47,7 +47,7 @@ def cache_word_embeddings(word_embeddings_file, cache_file):
with open(cache_file + '.vocab', 'w') as f:
logger.info('writing out vocab for {}'.format(word_embeddings_file))
for _, w in sorted((voc.index, word) for word, voc in wv.vocab.items()):
print(w.encode('utf-8'), file=f)
print(w, file=f)
with open(cache_file + '.dimensions', 'w') as f:
logger.info('writing out dimensions for {}'.format(word_embeddings_file))
print(wv.syn0.shape[0], wv.syn0.shape[1], file=f)