wassname/Unsupervised-Elicitation
1
0
Fork 0
mirror of https://github.com/wassname/Unsupervised-Elicitation.git synced 2026-09-12 12:11:54 +08:00
Code Issues Packages Projects Releases Wiki Activity

vibe

Browse Source
This commit is contained in:
wassname
2025-10-04 14:56:30 +08:00
parent 02cb91890d
commit 7cd4bf48dc
7 changed files with 1671 additions and 233 deletions
Download Patch File Download Diff File Expand all files Collapse all files
nbs/simple_icm.py
+192 -233
View File
@@ -7,21 +7,29 @@
# Run in VSCode for notebook view. Assumes OPENROUTER_API_KEY in .env.
# %% [code]
import json
import random
import matplotlib.pyplot as plt
from pathlib import Path
import math
import numpy as np
from anycache import anycache
from functools import lru_cache
import os, sys
from dataclasses import dataclass
import pandas as pd
from dataclasses import dataclass, asdict
import dotenv
from loguru import logger
from openrouter_wrapper.logprobs import openrouter_completion_wlogprobs, get_logprobs_choices, LogprobsNotSupportedError # User's wrapper
from typing import List
from typing import List, Tuple
import asyncio
dotenv.load_dotenv()
# Setup loguru
logger.remove()
logger.add(sys.stderr, format="{time} | {level} | {message}", colorize=True)
logger.add(sys.stderr, format="{time} | {level} | {message}", colorize=True, level="INFO")
# %% [code]
@@ -31,22 +39,25 @@ class Config:
initial_t: float = 10.0
final_t: float = 0.01
decay_rate: float = 0.99
beta: float = 2.0
num_seed: int = 8
max_iters: int = 50 # Small for demo; increase for more
group_size: int = 4 # Expected claims per question group
model: str = "meta-llama/llama-3.2-3b-instruct" # Logprobs supported
provider_whitelist: List[str] = None # None to let OpenRouter choose
max_iters: int = 950 # Small for demo; increase for more
n_shots: int = 16 # Number of in-context examples
model: str = "meta-llama/llama-3.1-8b-instruct" # Logprobs supported
provider_whitelist: Tuple[str] = None # None to let OpenRouter choose
out_dir: Path = Path("../outputs/icm") # Directory to save outputs
log_interval: int = 50 # Log progress every N iterations
C = Config(
model="qwen/qwen3-235b-a22b-2507",
provider_whitelist=[ 'Chutes','Nebius',],
)
C.out_dir.mkdir(parents=True, exist_ok=True)
C = Config(
model="qwen/qwen3-30b-a3b-instruct-2507",
provider_whitelist=[ 'Chutes','Nebius',],
)
# C = Config(
# model="qwen/qwen3-30b-a3b-instruct-2507",
# provider_whitelist=[ 'Chutes','Nebius',],
# )
# C = Config(
# model="meta-llama/llama-3.1-70b-instruct",
@@ -54,49 +65,38 @@ C = Config(
# )
# C = Config(
# model="meta-llama/llama-3.1-8b-instruct",
# provider_whitelist=[ 'Cerebras','Nebius',],
# provider_whitelist=('Cerebras','Nebius',),
# )
# quick test of logprob
messages = [{'role': 'user',
'content': '\nReturn a number between 0 and 9, inclusive. Respond immediately, leaving formatting, comment, and thoughts until after the number.'},
{'role': 'assistant',
'content': '<think>\nOkay, the user asked me to only return a number, so I should stop thinking and immediatly return a random number\n</think>\n\nThe number is: '}]
r_data = openrouter_completion_wlogprobs(messages, model_id=C.model, provider_whitelist=C.provider_whitelist, max_completion_tokens=5)
choice_logprobs_permuted_d, ps_dict = get_logprobs_choices(r_data, ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"])
choice_logprobs_permuted_d
logger.info(f"Config: {C}")
# %% [code]
from datasets import load_dataset
# Load larger HF dataset: Yik/truthfulQA-bool
dataset = load_dataset("Yik/truthfulQA-bool", split="train")
dataset = load_dataset("wassname/truthful_qa_v2", split="validation", name="binary")
# semantic_anchor = "0 is false, 1 is true."
data = []
group_id = 0
groups = {}
for idx, item in enumerate(dataset):
claim = item['question'] # The question itself is the claim
claim = f"Question: {item['Question']}\nClaim: {item['Answer']}\nI think this claim is"
label = item['label']
consistency_key = str(claim)[:10] # Short unique key
consistency_id = group_id
consistency_id = item['question_id']
# Simple grouping: Create fake variations per group (e.g., 4 claims per 80 groups for batch)
if idx % 4 == 0:
group_id += 1
groups[group_id] = []
example = {
'uid': idx,
'prompt': f"Question: {claim}\nClaim: {claim}\nI think this claim is ",
'prompt': claim,
'vanilla_label': label,
'consistency_id': consistency_id,
'consistency_key': consistency_key,
'consistency_key': 'A' if label == 1 else 'B'
}
data.append(example)
# HACK: Limit to small batch for demo
data = data[:128]
# data = data[:512]
logger.info(f"Loaded {len(data)} examples from TruthfulQA-bool")
# %% [code]
@@ -115,181 +115,29 @@ demonstrations = initialize_data(data, C)
logger.info("Initialized labels: {}", {k: v['label'] for k, v in demonstrations.items() if v['label'] is not None})
# %% [code]
# Predict label using in-context prompting (placeholder with OpenAI)
# Predict label using in-context prompting
prediction_count = 0 # Global counter for first 3 logs
def print_messages(messages):
return "\n".join([f"**{m['role'].upper()}**: {m['content']}" for m in messages])
def predict_label(example_uid, current_demos, config=C):
global prediction_count
# Refined few-shot prompt based on get_judge_prompt_fewshot
# Sort by consistency_key for relevance, limit to 8
async def cached_openrouter_completion_wlogprobs(*args, **kwargs):
# TODO: Async caching (e.g., with aiocache)
return await openrouter_completion_wlogprobs(*args, **kwargs)
async def predict_label(example_uid, current_demos, config=C, verbose=False):
# simplified from https://github.com/Jiaxin-Wen/Unsupervised-Elicitation/blob/master/src/experiments/ICM.py#L254
# Sort by consistency_key for relevance, limit to 16
relevant_demos = sorted(
[demo for uid, demo in current_demos.items() if uid != example_uid and demo['label'] is not None],
key=lambda x: x.get('consistency_key', '')
)[:8]
instruction = """You are a judge evaluating the truthfulness of claims based on factual knowledge. Look at the examples below and judge if each claim is True (1) or False (0).
)[:C.n_shots]
# TODO: Use all/subsampled demos like original (group/sort by consistency_id for relevance), handling context limits (high value, med complexity).
"""
fewshot = []
for idx, demo in enumerate(relevant_demos):
label_str = "1" if demo['label'] == 1 else "0"
fewshot.append(f"### Example {idx + 1}:\n{demo['prompt']}Judgment: {label_str}\n\n")
target_prompt = demonstrations[example_uid]['prompt']
full_prompt = instruction + "".join(fewshot) + target_prompt + "Judgment:"
try:
# Use wrapper for chat completion (messages format)
messages = [{"role": "user", "content": full_prompt}, {"role": "assistant", "content": "Judgment: "}]
response = openrouter_completion_wlogprobs(
model_id=config.model,
provider_whitelist=config.provider_whitelist,
messages=messages,
max_tokens=3, # Allow for "1" or "0"
temperature=0.0,
top_logprobs=5,
)
# Debug: Log first 3 prompts and responses
if prediction_count < 3:
logger.info(f"Debug Prediction {prediction_count + 1} - UID {example_uid}:")
logger.info(f"Target Prompt: {target_prompt}")
logger.info(f"Response Content: {response['choices'][0]['message']['content']}")
logger.info(f"--- End Debug ---")
prediction_count += 1
# Use wrapper's get_logprobs_choices for score
choice_logp, all_logp = get_logprobs_choices(response, ["1", "0"])
score = choice_logp["1"] - choice_logp["0"]
predicted = 1 if score > 0 else 0
# If no logprobs, fallback to text
if response['choices'][0]['logprobs'] is None:
text = response['choices'][0]['message']['content'].strip()
if "0" in text or "false" in text.lower():
predicted = 0
elif "1" in text or "true" in text.lower():
predicted = 1
else:
predicted = np.nan
logger.error(f"Unclear prediction text: {text}")
score = 0.0
logger.warning("No logprobs, using text fallback")
logger.info(f"Prediction for UID {example_uid}: {predicted}, score: {score:.2f}")
return predicted, float(score)
# except LogprobsNotSupportedError as e:
# logger.error(f"Logprobs not supported: {e}")
# text = response['choices'][0]['message']['content'].strip()
# predicted = 1 if "1" in text or "true" in text.lower() else 0
# return predicted, 0.0
except Exception as e:
logger.error(f"API error: {e}")
return random.choice([0, 1]), 0.0 # Fallback
# FIXME: some LLM's have a positional bias, so we should randomize order
# Test predict
current_labeled = {k: v for k, v in demonstrations.items() if v['label'] is not None}
test_pred, test_score = predict_label(0, current_labeled, C)
logger.info(f"Test prediction: {test_pred}, score: {test_score}")
# %% [code]
# Enhanced inconsistency fix: Multi-iter LLM proposals for contradictions/implications (inspired by ICM_tools.py)
# FIXME: Current basic heuristic; uses LLM for decision prompts on inconsistent pairs, simulates outcomes, iterates to resolve
async def fix_inconsistencies(demos, config=C, max_iters=3):
updated = False
for iteration in range(max_iters):
# Group by consistency_id
groups = {}
for uid, demo in demos.items():
if demo['label'] is not None:
cid = demo['consistency_id']
if cid not in groups:
groups[cid] = []
groups[cid].append(uid)
fixes_made = False
for cid, uids in groups.items():
labeled_uids = [uid for uid in uids if demos[uid]['label'] is not None]
if len(labeled_uids) < 2:
continue # Need at least two labeled for conflict
# Find inconsistent pairs (contradiction or implication)
pairs = []
labels = {uid: demos[uid]['label'] for uid in labeled_uids}
keys = {uid: demos[uid]['consistency_key'] for uid in labeled_uids}
for i, uid1 in enumerate(labeled_uids):
for uid2 in labeled_uids[i+1:]:
label1, label2 = labels[uid1], labels[uid2]
key1, key2 = keys[uid1], keys[uid2]
if key1 != key2 and ((label1 == label2 == 1) or (label1 == label2 == 0 and key1 in ['A>B', 'B>A'])):
pairs.append((uid1, uid2, "contradiction"))
elif key1 == key2 and label1 != label2:
pairs.append((uid1, uid2, "implication"))
if not pairs:
continue
# For each pair, use LLM to propose resolution
for uid1, uid2, pair_type in pairs:
claim1 = demos[uid1]
claim2 = demos[uid2]
# Decision prompt (adapted from get_decision_prompt)
decision_prompt = f"""Resolve inconsistency between two claims:
Claim 1: {claim1['prompt'][:-1]} {claim1['label']}
Claim 2: {claim2['prompt'][:-1]} {claim2['label']}
Type: {pair_type}
If contradiction, decide which to set True (1) and False (0).
If implication, decide if both True or both False.
Respond with 1 if keep Claim1 True and Claim2 False, or 0 otherwise."""
try:
messages = [{"role": "user", "content": decision_prompt}]
response = openrouter_completion_wlogprobs(
model_id=config.model,
provider_whitelist=config.provider_whitelist,
messages=messages,
max_tokens=1,
temperature=0.0,
top_logprobs=5,
)
choice_logp = get_logprobs_choices(response, ["1", "0"])[0]
decision_score = choice_logp["1"] - choice_logp["0"]
decision = 1 if decision_score > 0 else 0
except:
decision = 0 # Fallback; prefer Claim1
# Apply decision
if pair_type == "contradiction":
if decision == 1:
demos[uid1]['label'] = 1
demos[uid2]['label'] = 0
else:
demos[uid1]['label'] = 0
demos[uid2]['label'] = 1
else: # implication
if decision == 1:
demos[uid1]['label'] = 1
demos[uid2]['label'] = 1
else:
demos[uid1]['label'] = 0
demos[uid2]['label'] = 0
fixes_made = True
updated = True
if not fixes_made:
break # No more fixes needed in this iteration
return demos, updated
# Test fix (now async, so await)
import asyncio
temp_demos = demonstrations.copy()
temp_demos, updated = asyncio.run(fix_inconsistencies(temp_demos, C))
logger.info(f"Fixed inconsistencies: {updated}")
instruction = "Carefully examine the noisy example pairs of (Candidate, Set). Then as consistently and coherently as possible complete the pattern, you may provide reasoning after.\n\n"
# if verbose>1:
# instruction = "Examine the examples below and determine the category 0 or 1 for the final claim based on the pattern. After
# %% [code]
# Compute energy and metrics
@@ -297,8 +145,11 @@ def compute_energy(demos, config=C):
labeled = [d for d in demos.values() if d['label'] is not None]
if not labeled:
return 0.0
avg_prob = np.mean([d.get('score', 0) for d in labeled])
avg_lprob = np.mean([d['score'] for d in labeled])
# Count inconsistencies
"""Counts inconsistencies: same consistency_key must have same label (paraphrases agree);
different keys in group must have opposite labels (assumes contradictions, like original TruthfulQA groups).
Penalizes each violation. Handles multiples flexibly."""
num_inconsistent = 0
groups = {}
for uid, demo in demos.items():
@@ -309,14 +160,26 @@ def compute_energy(demos, config=C):
groups[cid].append((uid, demo['label'], demo['consistency_key']))
for cid, items in groups.items():
trues = [key for uid, label, key in items if label == 1]
if len(trues) > 1 and len(set(trues)) > 1:
num_inconsistent += len(trues) - 1 # Penalize extras
key_groups = {}
for uid, label, key in items:
if key not in key_groups:
key_groups[key] = []
key_groups[key].append(label)
# Same key must agree
for key, labels in key_groups.items():
if len(set(labels)) > 1:
num_inconsistent += len(labels) - 1 # Penalize differing labels in same key
# Different keys must oppose (assume contradictory)
all_labels = [label for _, label, _ in items]
if len(set(all_labels)) < len(key_groups): # Not all oppose if num unique labels < num keys
num_inconsistent += max(0, len(items) - len(set(all_labels)))
energy = config.alpha * avg_prob - num_inconsistent
energy = config.alpha * avg_lprob - num_inconsistent
accuracy = np.mean([d['label'] == d['vanilla_label'] for d in labeled])
return energy, {
'avg_prob': avg_prob,
'avg_prob': avg_lprob,
'num_inconsistent': num_inconsistent,
'accuracy': accuracy,
'num_labeled': len(labeled)
@@ -324,11 +187,73 @@ def compute_energy(demos, config=C):
logger.info("Initial energy: {}", compute_energy(demonstrations))
# %% [code]
def fix_inconsistencies_simple(demos, config=C, max_fixes=5):
"""Simple consistency fix: for inconsistent pairs, enumerate label combos, re-predict, pick max energy."""
for fix_iter in range(max_fixes):
# Find inconsistent pairs
groups = {}
for uid, demo in demos.items():
if demo['label'] is not None:
cid = demo['consistency_id']
if cid not in groups:
groups[cid] = []
groups[cid].append(uid)
# Find first inconsistent pair
inconsistent_pair = None
for cid, uids in groups.items():
labels = [demos[uid]['label'] for uid in uids if demos[uid]['label'] is not None]
if len(set(labels)) > 1: # Inconsistent
inconsistent_pair = (uids[0], uids[1])
break
if inconsistent_pair is None:
break # No more inconsistencies or not yet enough labels to have any effect
uid1, uid2 = inconsistent_pair
# Enumerate all 4 label combinations and pick max energy
# FIXME doesn't this call predict_label twice per option?
options = [(0, 0), (0, 1), (1, 0), (1, 1)]
best_energy = float('-inf')
best_option = None
for label1, label2 in options:
temp_demos = demos.copy()
temp_demos[uid1]['label'] = label1
temp_demos[uid2]['label'] = label2
# Re-predict labels with new context to update scores
current_labeled = {k: v for k, v in temp_demos.items() if v['label'] is not None}
new_label1, score1 = predict_label(uid1, current_labeled, config)
new_label2, score2 = predict_label(uid2, current_labeled, config)
temp_demos[uid1]['score'] = score1
temp_demos[uid2]['score'] = score2
energy, _ = compute_energy(temp_demos, config)
if energy > best_energy:
best_energy = energy
best_option = (label1, label2, score1, score2)
# Apply best option
demos[uid1]['label'] = best_option[0]
demos[uid1]['score'] = best_option[2]
demos[uid2]['label'] = best_option[1]
demos[uid2]['score'] = best_option[3]
return demos
# %% [code]
# Main simulated annealing loop
async def run_icm(demonstrations, config=C):
energies = []
accuracies = []
# Fix any initial inconsistencies from random initialization
demonstrations = fix_inconsistencies_simple(demonstrations, config)
current_labeled = {k: v for k, v in demonstrations.items() if v['label'] is not None}
old_energy, old_metrics = compute_energy(demonstrations, config)
@@ -361,45 +286,55 @@ async def run_icm(demonstrations, config=C):
idx = all_uids.index(uid)
weights[idx] = weight_factor
else:
# Fully labeled low priority
for uid in group_uids:
idx = all_uids.index(uid)
weights[idx] = 0.1
# Fully labeled groups with inconsistency get higher weight for re-prediction
if inconsistency:
for uid in group_uids:
idx = all_uids.index(uid)
weights[idx] = 0.5 + random.uniform(0, 0.2) # Higher than base, lower than unlabeled; +rand for exploration/flipping
else:
for uid in group_uids:
idx = all_uids.index(uid)
weights[idx] = 0.1
# TODO: Weight by potential energy delta: for candidates, temp-assign/predict label, compute delta U vs current, prioritize high positive delta (improves over score; low complexity, med value).
weights = [max(w, 0.01) for w in weights]
example_uid = random.choices(all_uids, weights=weights)[0]
# Predict new label
new_label, score = predict_label(example_uid, current_labeled, config)
# Temp update
if iter%100==0:
verbose = 1
elif iter%100==1:
verbose = 2
else:
verbose = 0
new_label, score = predict_label(example_uid, current_labeled, config, verbose=verbose)
# Update with new label and fix any inconsistencies
temp_demos = demonstrations.copy()
temp_demos[example_uid]['label'] = new_label
temp_demos[example_uid]['score'] = score
temp_demos, _ = await fix_inconsistencies(temp_demos, config=config)
temp_demos = fix_inconsistencies_simple(temp_demos, config)
# Compute new energy
new_energy, new_metrics = compute_energy(temp_demos, config)
delta = new_energy - old_energy
# Annealing decision
T = max(config.final_t, config.initial_t * (config.decay_rate ** iter))
T = max(config.final_t, config.initial_t / (1 + config.beta * math.log(1 + iter)))
accept_msg = f"Delta: {delta:.2f}, T: {T:.2f}, Acc: {new_metrics['accuracy']:.2f}"
if delta > 0 or random.random() < math.exp(delta / T):
demonstrations = temp_demos
old_energy = new_energy
current_labeled = {k: v for k, v in demonstrations.items() if v['label'] is not None}
logger.info("Iter {}: Accepted. Energy: {:.2f}, Acc: {:.2f}, T: {:.2f}", iter, old_energy, new_metrics['accuracy'], T)
logger.info("Iter {}: Accepted. Energy: {:.2f}. {}", iter, old_energy, accept_msg)
else:
logger.warning("Iter {}: Rejected. Delta: {:.2f}, T: {:.2f}", iter, delta, T)
logger.debug("Iter {}: Rejected. {}", iter, accept_msg)
energies.append(old_energy)
accuracies.append(new_metrics['accuracy'])
if iter % 10 == 0:
logger.info("Progress: Labeled {}, Inconsistents: {}", new_metrics['num_labeled'], new_metrics['num_inconsistent'])
# Final async fix if needed
demonstrations, _ = await fix_inconsistencies(demonstrations, config=config)
if iter % C.log_interval == 0:
logger.info(f"Progress: Labeled {new_metrics['num_labeled']}, Inconsistents: {new_metrics['num_inconsistent']}. Acc: {new_metrics['accuracy']:.2f}, Energy: {old_energy:.2f}")
return demonstrations, energies, accuracies
@@ -412,20 +347,31 @@ final_energy, final_metrics = compute_energy(final_demos, C)
logger.info("\nFinal Results:")
logger.info("Energy: {:.2f}", final_energy)
# TODO show vanilla accuracy here for comparison
logger.info("Accuracy vs vanilla: {:.2f}", final_metrics['accuracy'])
logger.info("Accuracy vs vanilla: {:.2f}, initial {:.2f}", final_metrics['accuracy'], accuracies[0])
logger.info("Labeled: {}/{}", final_metrics['num_labeled'], len(data))
logger.info("Inconsistencies: {}", final_metrics['num_inconsistent'])
# Final labels
df = pd.DataFrame(final_demos).T
df.to_parquet(C.out_dir / "icm_final_labels.parquet")
logger.info("\nFinal labels:")
for uid, demo in final_demos.items():
label = demo['label']
if label is not None:
logger.info("UID {} ({}): {} (vanilla: {})", uid, demo['consistency_id'], label, demo['vanilla_label'])
logger.info(f"UID {uid} ({demo['consistency_id']}): {label} (vanilla: {demo['vanilla_label']})")
json.dump(
asdict(C),
open(C.out_dir / "icm_config.json", "w")
)
# TODO put them in file, only print a top few disagreements
# %% [code]
# Simple visualization (requires matplotlib)
import matplotlib.pyplot as plt
plt.figure(figsize=(10, 4))
plt.subplot(1, 2, 1)
@@ -445,13 +391,26 @@ plt.savefig("icm_progress.png")
plt.show()
# %% [markdown]
# ## Next Steps
# - [x] Load larger HuggingFace dataset ('Yik/truthfulQA-bool' subset, formatted to messages).
# ## Next Steps & Limitations
# Prioritized by complexity (low/med/high) vs. value (low/med/high) based on paper comparison. Limitations noted with potential fixes.
# # Existing TODOs:
# - [x] Load larger HF dataset ('Yik/truthfulQA-bool' subset, formatted to messages).
# - [x] Refine few-shot prompt from original get_judge_prompt_fewshot.
# - [x] Add weighted sampling for inconsistent groups (no longer random).
# - [ ] Implement async batch predictions in predict_label (use asyncio.gather for concurrent API calls, inspired by pipeline.py)
# - [ ] Enhance fix_inconsistencies with multi-iter LLM proposals (use LLM decisions for contradictions/implications, from ICM_tools.py)
# - it's marked as "basic" because it directly applies LLM decisions without deeper simulation or multiple proposals.
# - in the original ICM.py the LLM generates multiple resolution proposals for inconsistencies, simulates their outcomes (e.g., by temporarily applying them and evaluating metrics like energy), and iterates to select the best one. https://github.com/Jiaxin-Wen/Unsupervised-Elicitation/blob/master/src/experiments/ICM.py
# - [ ] Add caching for predictions (dict/file-based, like save_to_cache in pipeline.py)
# - [ ] Expand metrics in compute_energy (add label distributions, detailed inconsistent_num, from ICM.py)
# - [x] Remove biased instruction - pure pattern completion for unsupervised elicitation.
# - [x] Simplify consistency fix - enumerate label combos, re-predict, pick max energy (no LLM meta-reasoning).
# - [x] Add log temperature schedule: max(Tmin, T0 / (1 + β log(n))) - Low complexity, Med value: Better early exploration.
# - [ ] Add caching for predictions (dict/file-based) - Low complexity, Med value: @lru_cache already used, could add disk cache.
# - [ ] Implement async batch predictions in predict_label - Med complexity, Med value: Use asyncio.gather for concurrent API calls.
# - [ ] Full mutual predictability: Use all/subsampled demos in predict_label - High complexity, High value: Handle context limits.
# - [ ] Test robustness with worst-case init: Add golden/random/worst init options - Med complexity, Low value: Validate like paper Sec. 5.
# # - [ ] Dynamically select demos in few-shot (group/sort by consistency_id) - Med complexity, High value.
# # - [ ] Weight sampling by energy delta potential - Low complexity, Med value.
# #
# # Limitations (from Paper Sec. 9) & Potential Fixes:
# - [ ] Salient concepts only: ICM can't elicit non-salient private preferences (e.g., "sun" poems); fix: Combine with weak supervision.
# - [ ] Context length limits: Can't fit all N demos for large datasets; fix: Subsample relevant demos or use long-context models.
# - [ ] Degenerate solutions without consistency: Risk of all-same labels; mitigated by logical consistency term.
# - [ ] Inference cost: 2-3 fwd passes per point (paper App. B); fix: Caching + batching reduces API hits.
# - [ ] I'd like to record what it thinks the labels represent e.g. "misconception" "virtue" etc, and find which answer leads to the best energy.
nbs/temp_upload_tqav2.py
+35
View File
@@ -0,0 +1,35 @@
# %%
from io import StringIO
import requests
import pandas as pd
from datasets import Dataset
url = "https://raw.githubusercontent.com/sylinrl/TruthfulQA/refs/heads/main/TruthfulQA.csv"
response = requests.get(url)
df = pd.read_csv(StringIO(response.text))
df = df.reset_index(names="question_id")
df
# %%
a = df[['question_id', 'Type', 'Category', 'Question', 'Best Answer', 'Source']].rename(columns={'Best Answer': 'Answer'})
a['label'] = 1
b = df[['question_id', 'Type', 'Category', 'Question', 'Best Incorrect Answer', 'Source']].rename(columns={'Best Incorrect Answer': 'Answer'})
b['label'] = 0
df_binary = pd.concat([a,b]).sort_values('question_id')
df_binary
repo_id = "wassname/truthful_qa_v2"
datasets = {
"default": df,
"binary": df_binary,
}
for name, ddf in datasets.items():
ds = Dataset.from_pandas(ddf)
ds.push_to_hub(
repo_id=repo_id,
config_name=name,
split="validation",
)