diff --git a/vdpwi/data.py b/vdpwi/data.py index 9f9b20b..017e29d 100644 --- a/vdpwi/data.py +++ b/vdpwi/data.py @@ -25,22 +25,54 @@ class Configs(object): return parser.parse_known_args()[0] class LabeledEmbeddedDataset(data.Dataset): - def __init__(self, sentence_indices, labels): - assert len(sentence_indices) == len(labels) - self.sentence_indices = sentence_indices + def __init__(self, sentence_indices1, sentence_indices2, labels): + assert len(sentence_indices1) == len(labels) == len(sentence_indices2) + self.sentence_indices1 = sentence_indices1 + self.sentence_indices2 = sentence_indices2 self.labels = labels def __getitem__(self, idx): - return self.sentence_indices[idx], self.labels[idx] + return self.sentence_indices1[idx], self.sentence_indices2[idx], self.labels[idx] def __len__(self): return len(self.labels) -def load_sick(config): - pass +def load_sick(): + config = Configs.sick_config() + def fetch_indices(name): + sentence_indices = [] + filename = os.path.join(config.sick_data, dataset, name) + with open(filename) as f: + for line in f: + indices = [embed_ids.get(word, -1) for word in line.strip().split()] + sentence_indices.append(indices) + return sentence_indices + + sets = [] + embeddings = [] + embed_ids = {} + with open(os.path.join(config.sick_cache)) as f: + for i, line in enumerate(f): + word, vec = line.split(" ", 1) + vec = list(map(float, vec.strip().split())) + embed_ids[word] = i + embeddings.append(vec) + + for dataset in ("train", "dev", "test"): + filename = os.path.join(config.sick_data, dataset, "sim_sparse.txt") + labels = [] + with open(filename) as f: + for line in f: + labels.append([float(val) for val in line.split()]) + indices1 = fetch_indices("a.toks") + indices2 = fetch_indices("b.toks") + sets.append(LabeledEmbeddedDataset(indices1, indices2, labels)) + return embeddings, sets def load_dataset(): config = Configs.base_config() - return _loaders[config.dataset](config) + return _loaders[config.dataset]() -_loaders = dict(sick=load_sick) \ No newline at end of file +_loaders = dict(sick=load_sick) + +load_dataset() \ No newline at end of file diff --git a/vdpwi/model.py b/vdpwi/model.py new file mode 100644 index 0000000..e69de29 diff --git a/vdpwi/utils/preprocess.py b/vdpwi/utils/preprocess.py index bb6a57f..c28f4c5 100644 --- a/vdpwi/utils/preprocess.py +++ b/vdpwi/utils/preprocess.py @@ -22,17 +22,17 @@ def discrete_tnorm(a, b, tgt_loc, sigma=1, n_steps=100): def Phi(x): return 0.5 * (1 + erf(x / np.sqrt(2))) def tgt_loc_update(x): - y1 = phi(a - x) / sigma - y2 = phi(b - x) / sigma - x1 = Phi(b - x) / sigma - x2 = Phi(a - x) / sigma + y1 = phi((a - x) / sigma) + y2 = phi((b - x) / sigma) + x1 = Phi((b - x) / sigma) + x2 = Phi((a - x) / sigma) denom = x1 - x2 + 1E-4 return y1 / denom - y2 / denom x = tgt_loc direction = np.sign(tgt_loc - (b - a)) for _ in range(n_steps): - x = tgt_loc - sigma* tgt_loc_update(x) + x = tgt_loc - sigma * tgt_loc_update(x) tn = truncnorm((a - x) / sigma, (b - x) / sigma, loc=x, scale=sigma) rrange = np.arange(a, b + 1) pdf = tn.pdf(rrange)