mirror of
https://github.com/wassname/Castor.git
synced 2026-10-04 12:10:17 +08:00
Salman Mohammed
4c081645a7
+ cleaned up the model code for the simple qa directory + created vocab objects for pre-loading word embeddings easily
84 lines
2.2 KiB
Python
84 lines
2.2 KiB
Python
"""
|
|
Downloads the following:
|
|
- Glove vectors
|
|
|
|
We Thank Kai Sheng Tai for providing the preprocessing/basis codes.
|
|
Taken from: https://github.com/castorini/NCE-CNN-Torch
|
|
"""
|
|
|
|
from __future__ import print_function
|
|
import urllib2
|
|
import sys
|
|
import os
|
|
import shutil
|
|
import zipfile
|
|
import gzip
|
|
|
|
def download(url, dirpath):
|
|
filename = url.split('/')[-1]
|
|
filepath = os.path.join(dirpath, filename)
|
|
try:
|
|
u = urllib2.urlopen(url)
|
|
except:
|
|
print("URL %s failed to open" %url)
|
|
raise Exception
|
|
try:
|
|
f = open(filepath, 'wb')
|
|
except:
|
|
print("Cannot write %s" %filepath)
|
|
raise Exception
|
|
try:
|
|
filesize = int(u.info().getheaders("Content-Length")[0])
|
|
except:
|
|
print("URL %s failed to report length" %url)
|
|
raise Exception
|
|
print("Downloading: %s Bytes: %s" % (filename, filesize))
|
|
|
|
downloaded = 0
|
|
block_sz = 8192
|
|
status_width = 70
|
|
while True:
|
|
buf = u.read(block_sz)
|
|
if not buf:
|
|
print('')
|
|
break
|
|
else:
|
|
print('', end='\r')
|
|
downloaded += len(buf)
|
|
f.write(buf)
|
|
status = (("[%-" + str(status_width + 1) + "s] %3.2f%%") %
|
|
('=' * int(float(downloaded) / filesize * status_width) + '>', downloaded * 100. / filesize))
|
|
print(status, end='')
|
|
sys.stdout.flush()
|
|
f.close()
|
|
return filepath
|
|
|
|
def unzip(filepath):
|
|
print("Extracting: " + filepath)
|
|
dirpath = os.path.dirname(filepath)
|
|
with zipfile.ZipFile(filepath) as zf:
|
|
zf.extractall(dirpath)
|
|
os.remove(filepath)
|
|
|
|
def download_wordvecs(dirpath):
|
|
if os.path.exists(dirpath):
|
|
print('Found Glove vectors - skip')
|
|
return
|
|
else:
|
|
os.makedirs(dirpath)
|
|
url = 'https://nlp.stanford.edu/data/glove.840B.300d.zip'
|
|
unzip(download(url, dirpath))
|
|
|
|
if __name__ == '__main__':
|
|
base_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
|
|
|
|
# data
|
|
data_dir = os.path.join(base_dir, 'data')
|
|
wordvec_dir = os.path.join(data_dir, 'glove')
|
|
|
|
# libraries
|
|
lib_dir = os.path.join(base_dir, 'lib')
|
|
|
|
# download GloVe word embeddings
|
|
download_wordvecs(wordvec_dir)
|
|
print("Finished downloading word embeddings!") |