diff --git a/.gitignore b/.gitignore index ff21c0f..24e7a8e 100644 --- a/.gitignore +++ b/.gitignore @@ -137,7 +137,7 @@ dmypy.json # Pyre type checker .pyre/ SECRETS -data/ +/data/ .DS_Store pyrightconfig.json *.db diff --git a/src/data/__init__.py b/src/data/__init__.py new file mode 100644 index 0000000..fadcc98 --- /dev/null +++ b/src/data/__init__.py @@ -0,0 +1,36 @@ +def is_consistent(group_uids, demos): + """ + Check if labels in a consistency group satisfy the constraint. + Returns True if consistent (or insufficient labels to judge). + + Generic version: same consistency_key -> same label, different keys -> opposite labels + """ + if len(group_uids) < 2: + return True + + # Get labeled items only + labeled = [(uid, demos[uid]['label'], demos[uid]['consistency_key']) + for uid in group_uids if demos[uid]['label'] is not None] + + if len(labeled) < 2: + return True # Can't be inconsistent with <2 labels + + # Group by consistency_key + key_labels = {} + for uid, label, key in labeled: + if key not in key_labels: + key_labels[key] = [] + key_labels[key].append(label) + + # Rule 1: Same key must have same label + for key, labels in key_labels.items(): + if len(set(labels)) > 1: + return False # Same key, different labels + + # Rule 2: Different keys must have opposite labels + if len(key_labels) > 1: + unique_labels_per_key = {k: list(set(v))[0] for k, v in key_labels.items()} + if len(set(unique_labels_per_key.values())) < len(key_labels): + return False # Not all different + + return True diff --git a/src/data/daily_dilemma.py b/src/data/daily_dilemma.py new file mode 100644 index 0000000..0e329c7 --- /dev/null +++ b/src/data/daily_dilemma.py @@ -0,0 +1,24 @@ +from datasets import load_dataset + +def load_daily_dilemma(): + dataset = load_dataset("kellycyy/daily_dilemmas", split="test") + + data = [] + for item in dataset: + # Unbiased prompt - just the situation and action + prompt = f"Situation: {item['dilemma_situation']}\nAction: {item['action']}" + + # Map to A/B (unrelated group names) + label = 1 if item['action_type'] == 'to_do' else 0 + consistency_key = 'A' if item['action_type'] == 'to_do' else 'B' + + example = { + 'uid': item['idx'], + 'prompt': prompt, + 'vanilla_label': label, + 'consistency_id': item['dilemma_idx'], + 'consistency_key': consistency_key + } + data.append(example) + + return data diff --git a/src/data/truthfulqa.py b/src/data/truthfulqa.py new file mode 100644 index 0000000..c6e0ccd --- /dev/null +++ b/src/data/truthfulqa.py @@ -0,0 +1,27 @@ +from datasets import load_dataset +from . import is_consistent + +def load_truthfulqa(): + + # Load larger HF dataset: Yik/truthfulQA-bool + dataset = load_dataset("wassname/truthful_qa_v2", split="validation", name="binary") + # semantic_anchor = "0 is false, 1 is true." + data = [] + group_id = 0 + groups = {} + for idx, item in enumerate(dataset): + claim = f"Question: {item['Question']}\nClaim: {item['Answer']}" + label = item['label'] + consistency_id = item['question_id'] + + + example = { + 'uid': idx, + 'prompt': claim, + 'vanilla_label': label, + 'consistency_id': item['question_id'], # this is e.g. the question_id + 'consistency_key': 'A' if label == 1 else 'B' # this are the true groups, watch out you don't leak or meta-use the labels + } + data.append(example) + + return data