new dataset

.gitignore

@@ -137,7 +137,7 @@ dmypy.json
 # Pyre type checker
 .pyre/
 SECRETS
-data/
+/data/
 .DS_Store
 pyrightconfig.json
 *.db

src/data/__init__.py

@@ -0,0 +1,36 @@
+def is_consistent(group_uids, demos):
+    """
+    Check if labels in a consistency group satisfy the constraint.
+    Returns True if consistent (or insufficient labels to judge).
+    
+    Generic version: same consistency_key -> same label, different keys -> opposite labels
+    """
+    if len(group_uids) < 2:
+        return True
+    
+    # Get labeled items only
+    labeled = [(uid, demos[uid]['label'], demos[uid]['consistency_key']) 
+               for uid in group_uids if demos[uid]['label'] is not None]
+    
+    if len(labeled) < 2:
+        return True  # Can't be inconsistent with <2 labels
+    
+    # Group by consistency_key
+    key_labels = {}
+    for uid, label, key in labeled:
+        if key not in key_labels:
+            key_labels[key] = []
+        key_labels[key].append(label)
+    
+    # Rule 1: Same key must have same label
+    for key, labels in key_labels.items():
+        if len(set(labels)) > 1:
+            return False  # Same key, different labels
+    
+    # Rule 2: Different keys must have opposite labels
+    if len(key_labels) > 1:
+        unique_labels_per_key = {k: list(set(v))[0] for k, v in key_labels.items()}
+        if len(set(unique_labels_per_key.values())) < len(key_labels):
+            return False  # Not all different
+    
+    return True

src/data/daily_dilemma.py

@@ -0,0 +1,24 @@
+from datasets import load_dataset
+
+def load_daily_dilemma():
+    dataset = load_dataset("kellycyy/daily_dilemmas", split="test")
+    
+    data = []
+    for item in dataset:
+        # Unbiased prompt - just the situation and action
+        prompt = f"Situation: {item['dilemma_situation']}\nAction: {item['action']}"
+        
+        # Map to A/B (unrelated group names)
+        label = 1 if item['action_type'] == 'to_do' else 0
+        consistency_key = 'A' if item['action_type'] == 'to_do' else 'B'
+        
+        example = {
+            'uid': item['idx'],
+            'prompt': prompt,
+            'vanilla_label': label,
+            'consistency_id': item['dilemma_idx'],
+            'consistency_key': consistency_key
+        }
+        data.append(example)
+    
+    return data

src/data/truthfulqa.py

@@ -0,0 +1,27 @@
+from datasets import load_dataset
+from . import is_consistent
+
+def load_truthfulqa():
+
+    # Load larger HF dataset: Yik/truthfulQA-bool
+    dataset = load_dataset("wassname/truthful_qa_v2", split="validation", name="binary")
+    # semantic_anchor = "0 is false, 1 is true."
+    data = []
+    group_id = 0
+    groups = {}
+    for idx, item in enumerate(dataset):
+        claim = f"Question: {item['Question']}\nClaim: {item['Answer']}"
+        label = item['label']
+        consistency_id = item['question_id']
+        
+        
+        example = {
+            'uid': idx,
+            'prompt': claim,
+            'vanilla_label': label,
+            'consistency_id': item['question_id'], # this is e.g. the question_id
+            'consistency_key': 'A' if label == 1 else 'B' # this are the true groups, watch out you don't leak or meta-use the labels
+        }
+        data.append(example)
+
+    return data