mirror of
https://github.com/wassname/Castor.git
synced 2026-09-09 11:13:20 +08:00
Fix vocab caching issue (#29)
With the line as-was the vocab cache was stored as b'the' rather than the, meaning that word2vec wasn't found for terms causing massive performance loss (AP 0.71 cf 0.77).
This commit is contained in:
+1
-1
@@ -47,7 +47,7 @@ def cache_word_embeddings(word_embeddings_file, cache_file):
|
||||
with open(cache_file + '.vocab', 'w') as f:
|
||||
logger.info('writing out vocab for {}'.format(word_embeddings_file))
|
||||
for _, w in sorted((voc.index, word) for word, voc in wv.vocab.items()):
|
||||
print(w.encode('utf-8'), file=f)
|
||||
print(w, file=f)
|
||||
with open(cache_file + '.dimensions', 'w') as f:
|
||||
logger.info('writing out dimensions for {}'.format(word_embeddings_file))
|
||||
print(wv.syn0.shape[0], wv.syn0.shape[1], file=f)
|
||||
|
||||
Reference in New Issue
Block a user