mirror of
https://github.com/wassname/Castor.git
synced 2026-10-28 12:30:15 +08:00
Fix vocab caching issue (#29)
With the line as-was the vocab cache was stored as b'the' rather than the, meaning that word2vec wasn't found for terms causing massive performance loss (AP 0.71 cf 0.77).
This commit is contained in:
1 parent
43dd6fdb1f
commit
945b1fa6c0
1 file changed
+1
-1
+1
-1
@@ -47,7 +47,7 @@ def cache_word_embeddings(word_embeddings_file, cache_file):
|
||||
with open(cache_file + '.vocab', 'w') as f:
|
||||
logger.info('writing out vocab for {}'.format(word_embeddings_file))
|
||||
for _, w in sorted((voc.index, word) for word, voc in wv.vocab.items()):
|
||||
print(w.encode('utf-8'), file=f)
|
||||
print(w, file=f)
|
||||
with open(cache_file + '.dimensions', 'w') as f:
|
||||
logger.info('writing out dimensions for {}'.format(word_embeddings_file))
|
||||
print(wv.syn0.shape[0], wv.syn0.shape[1], file=f)
|
||||
|
||||
Reference in new issue
Block a user