Fix sigma bug

vdpwi/data.py

@@ -25,22 +25,54 @@ class Configs(object):
         return parser.parse_known_args()[0]
 
 class LabeledEmbeddedDataset(data.Dataset):
-    def __init__(self, sentence_indices, labels):
-        assert len(sentence_indices) == len(labels)
-        self.sentence_indices = sentence_indices
+    def __init__(self, sentence_indices1, sentence_indices2, labels):
+        assert len(sentence_indices1) == len(labels) == len(sentence_indices2)
+        self.sentence_indices1 = sentence_indices1
+        self.sentence_indices2 = sentence_indices2
         self.labels = labels
 
     def __getitem__(self, idx):
-        return self.sentence_indices[idx], self.labels[idx]
+        return self.sentence_indices1[idx], self.sentence_indices2[idx], self.labels[idx]
 
     def __len__(self):
         return len(self.labels)
 
-def load_sick(config):
-    pass
+def load_sick():
+    config = Configs.sick_config()
+    def fetch_indices(name):
+        sentence_indices = []
+        filename = os.path.join(config.sick_data, dataset, name)
+        with open(filename) as f:
+            for line in f:
+                indices = [embed_ids.get(word, -1) for word in line.strip().split()]
+                sentence_indices.append(indices)
+        return sentence_indices
+
+    sets = []
+    embeddings = []
+    embed_ids = {}
+    with open(os.path.join(config.sick_cache)) as f:
+        for i, line in enumerate(f):
+            word, vec = line.split(" ", 1)
+            vec = list(map(float, vec.strip().split()))
+            embed_ids[word] = i
+            embeddings.append(vec)
+
+    for dataset in ("train", "dev", "test"):
+        filename = os.path.join(config.sick_data, dataset, "sim_sparse.txt")
+        labels = []
+        with open(filename) as f:
+            for line in f:
+                labels.append([float(val) for val in line.split()])
+        indices1 = fetch_indices("a.toks")
+        indices2 = fetch_indices("b.toks")
+        sets.append(LabeledEmbeddedDataset(indices1, indices2, labels))
+    return embeddings, sets
 
 def load_dataset():
     config = Configs.base_config()
-    return _loaders[config.dataset](config)
+    return _loaders[config.dataset]()
 
-_loaders = dict(sick=load_sick)
+_loaders = dict(sick=load_sick)
+
+load_dataset()

vdpwi/utils/preprocess.py

@@ -22,17 +22,17 @@ def discrete_tnorm(a, b, tgt_loc, sigma=1, n_steps=100):
     def Phi(x):
         return 0.5 * (1 + erf(x / np.sqrt(2)))
     def tgt_loc_update(x):
-        y1 = phi(a - x) / sigma
-        y2 = phi(b - x) / sigma
-        x1 = Phi(b - x) / sigma
-        x2 = Phi(a - x) / sigma
+        y1 = phi((a - x) / sigma)
+        y2 = phi((b - x) / sigma)
+        x1 = Phi((b - x) / sigma)
+        x2 = Phi((a - x) / sigma)
         denom = x1 - x2 + 1E-4
         return y1 / denom - y2 / denom
 
     x = tgt_loc
     direction = np.sign(tgt_loc - (b - a))
     for _ in range(n_steps):
-        x = tgt_loc - sigma* tgt_loc_update(x)
+        x = tgt_loc - sigma * tgt_loc_update(x)
     tn = truncnorm((a - x) / sigma, (b - x) / sigma, loc=x, scale=sigma)
     rrange = np.arange(a, b + 1)
     pdf = tn.pdf(rrange)