From 945b1fa6c018aabdf4c71c21907258b496ac93e9 Mon Sep 17 00:00:00 2001 From: Matt Crane Date: Mon, 19 Jun 2017 13:02:36 -0400 Subject: [PATCH] Fix vocab caching issue (#29) With the line as-was the vocab cache was stored as b'the' rather than the, meaning that word2vec wasn't found for terms causing massive performance loss (AP 0.71 cf 0.77). --- sm_cnn/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sm_cnn/utils.py b/sm_cnn/utils.py index 41d6f59..8c3e9db 100644 --- a/sm_cnn/utils.py +++ b/sm_cnn/utils.py @@ -47,7 +47,7 @@ def cache_word_embeddings(word_embeddings_file, cache_file): with open(cache_file + '.vocab', 'w') as f: logger.info('writing out vocab for {}'.format(word_embeddings_file)) for _, w in sorted((voc.index, word) for word, voc in wv.vocab.items()): - print(w.encode('utf-8'), file=f) + print(w, file=f) with open(cache_file + '.dimensions', 'w') as f: logger.info('writing out dimensions for {}'.format(word_embeddings_file)) print(wv.syn0.shape[0], wv.syn0.shape[1], file=f)