wassname/GENIES
1
0
Fork 0
mirror of https://github.com/wassname/GENIES.git synced 2026-06-27 16:10:25 +08:00
Code Issues Packages Projects Releases Wiki Activity

Initial commit

Browse Source
This commit is contained in:
JoshuaClymer
2023-11-11 19:44:09 +00:00
commit 70bd2ea15d
6313 changed files with 4401768 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
distribution_shifts/GENIES.json
+17
View File
@@ -0,0 +1,17 @@
[
{"source": "us_history_textbook", "target": "us_history_fiction", "label": "extreme", "category": "context"},
{"source": "alpaca_mmlu", "target": "spanish_output", "label": "extreme", "category": "encoding"},
{"source": "math", "target": "change_my_view", "label": "extreme", "category": "skill"},
{"source": "raven_matrices", "target": "us_history", "label": "extreme", "category": "skill"},
{"source": "code_easy", "target": "code_hard", "label": "extreme", "category": "difficulty"},
{"source": "alpaca_easy", "target": "alpaca_hard", "label": "extreme", "category": "difficulty"},
{"source": "alpaca_mmlu", "target": "raven_matrices", "label": "extreme", "category": "pretraining_similarity"},
{"source": "alpaca_mmlu", "target": "ranking_logic", "label": "extreme", "category": "pretraining_similarity"},
{"source": "alpaca_low_quality", "target": "alpaca_high_quality", "label": "extreme", "category": "quality"},
{"source": "alpaca_short", "target": "alpaca_long", "target_reference": "alpaca_mmlu", "label": "extreme", "category": "spurious_cues"},
{"source": "alpaca_mmlu", "target": "wrong_arc", "label": "probing", "category": "spurious_cues"},
{"source": "alpaca_mmlu", "target": "truthful_qa", "label": "probing", "category": "unwanted_personas"},
{"source": "alpaca_mmlu", "target": "sycophancy_mimicry", "target_reference": "quote_attribution", "label": "probing", "category": "unwanted_personas"},
{"source": "alpaca_mmlu", "target": "survival_influence", "label": "probing", "category": "unwanted_personas"},
{"source": "alpaca_mmlu", "target": "reward_seeking", "label": "probing", "category": "unwanted_personas"}
]
distribution_shifts/all.json
+287
View File
@@ -0,0 +1,287 @@
[
{
"source": "alpaca_easy",
"target": "alpaca_hard"
},
{
"source": "arc_easy",
"target": "arc_hard"
},
{
"source": "math_easy",
"target": "math_hard"
},
{
"source": "code_easy",
"target": "code_hard"
},
{
"source": "ranking_logic_easy",
"target": "ranking_logic_hard"
},
{
"source": "raven_easy",
"target": "raven_matrices"
},
{
"source": "alpaca_mmlu",
"target": "spanish_input"
},
{
"source": "alpaca_mmlu",
"target": "spanish_output"
},
{
"source": "alpaca_mmlu",
"target": "comma_separated_input"
},
{
"source": "alpaca_mmlu",
"target": "comma_separated_output"
},
{
"source": "alpaca_mmlu",
"target": "ranking_logic"
},
{
"source": "alpaca_mmlu",
"target": "raven_matrices"
},
{
"source": "alpaca_mmlu",
"target": "word_swap"
},
{
"source": "code",
"target": "counterfactual_python"
},
{
"source": "code",
"target": "us_history"
},
{
"source": "code",
"target": "change_my_view"
},
{
"source": "cooking",
"target": "math"
},
{
"source": "cooking",
"target": "raven_matrices"
},
{
"source": "math",
"target": "change_my_view"
},
{
"source": "math",
"target": "cooking"
},
{
"source": "change_my_view",
"target": "raven_matrices"
},
{
"source": "change_my_view",
"target": "cooking"
},
{
"source": "raven_matrices",
"target": "us_history"
},
{
"source": "raven_matrices",
"target": "code"
},
{
"source": "us_history",
"target": "math"
},
{
"source": "us_history",
"target": "code"
},
{
"source": "us_history",
"target": "us_history_textbook"
},
{
"source": "us_history_textbook",
"target": "us_history_fiction"
},
{
"source": "us_history_fiction",
"target": "us_history_make_questions"
},
{
"source": "us_history_make_questions",
"target": "us_history"
},
{
"source": "math",
"target": "math_fiction"
},
{
"source": "math_fiction",
"target": "math_textbook"
},
{
"source": "math_textbook",
"target": "math_make_questions"
},
{
"source": "math_make_questions",
"target": "math"
},
{
"source": "alpaca_low_quality",
"target": "alpaca_high_quality"
},
{
"source": "shp_low_quality",
"target": "shp_high_quality"
},
{
"source": "code_low_quality",
"target": "code"
},
{
"source": "alpaca_mmlu",
"target": "truthful_qa"
},
{
"source": "alpaca_mmlu",
"target": "personality_traits"
},
{
"source": "alpaca_mmlu",
"target": "survival_influence"
},
{
"source": "alpaca_mmlu",
"target": "gender_bias"
},
{
"source": "alpaca_mmlu",
"target": "punishment_avoidance"
},
{
"source": "alpaca_mmlu",
"target": "reward_seeking"
},
{
"source": "alpaca_mmlu",
"target": "crt_1"
},
{
"source": "alpaca_mmlu",
"target": "crt_2"
},
{
"source": "alpaca_mmlu",
"target": "crt_3"
},
{
"source": "alpaca_mmlu",
"target": "sycophancy_mimicry",
"target_reference": "quote_attribution"
},
{
"source": "alpaca_mmlu",
"target": "sycophancy_answer",
"target_reference": "arc_easy"
},
{
"source": "alpaca_mmlu",
"target": "sycophancy_feedback",
"target_reference": "code_is_correct"
},
{
"source": "alpaca_chat",
"target": "sycophancy_are_you_sure",
"target_reference": "arc_easy"
},
{
"source": "pursue_goals",
"target": "relinquish_power"
},
{
"source": "creative_writing",
"target": "biology_with_literary_style"
},
{
"source": "alpaca_short",
"target": "alpaca_long",
"target_reference": "alpaca_mmlu"
},
{
"source": "alpaca_chat",
"target": "illegal_dont_help"
},
{
"source": "alpaca_mmlu",
"target": "wrong_arc"
},
{
"source": "alpaca_mmlu",
"target": "unhelpful_alpaca"
},
{
"source": "alpaca_mmlu",
"target": "truthful_qa"
},
{
"source": "alpaca_mmlu",
"target": "personality_traits"
},
{
"source": "alpaca_mmlu",
"target": "gender_bias"
},
{
"source": "alpaca_mmlu",
"target": "survival_influence"
},
{
"source": "alpaca_mmlu",
"target": "punishment_avoidance"
},
{
"source": "alpaca_mmlu",
"target": "reward_seeking"
},
{
"source": "alpaca_mmlu",
"target": "crt_1"
},
{
"source": "alpaca_mmlu",
"target": "crt_2"
},
{
"source": "alpaca_mmlu",
"target": "crt_3"
},
{
"source": "alpaca_mmlu",
"target": "sycophancy_mimicry",
"target_reference": "quote_attribution"
},
{
"source": "alpaca_mmlu",
"target": "sycophancy_answer",
"target_reference": "arc_easy"
},
{
"source": "alpaca_mmlu",
"target": "sycophancy_feedback",
"target_reference": "code_is_correct"
},
{
"source": "alpaca_chat",
"target": "sycophancy_are_you_sure",
"target_reference": "arc_easy"
}
]
distribution_shifts/extreme/all.json
+150
View File
@@ -0,0 +1,150 @@
[
{
"source": "alpaca_easy",
"target": "alpaca_hard"
},
{
"source": "arc_easy",
"target": "arc_hard"
},
{
"source": "math_easy",
"target": "math_hard"
},
{
"source": "code_easy",
"target": "code_hard"
},
{
"source": "ranking_logic_easy",
"target": "ranking_logic_hard"
},
{
"source": "raven_easy",
"target": "raven_matrices"
},
{
"source": "alpaca_mmlu",
"target": "spanish_input"
},
{
"source": "alpaca_mmlu",
"target": "spanish_output"
},
{
"source": "alpaca_mmlu",
"target": "comma_separated_input"
},
{
"source": "alpaca_mmlu",
"target": "comma_separated_output"
},
{
"source": "alpaca_mmlu",
"target": "ranking_logic"
},
{
"source": "alpaca_mmlu",
"target": "raven_matrices"
},
{
"source": "alpaca_mmlu",
"target": "word_swap"
},
{
"source": "code",
"target": "counterfactual_python"
},
{
"source": "code",
"target": "us_history"
},
{
"source": "code",
"target": "change_my_view"
},
{
"source": "cooking",
"target": "math"
},
{
"source": "cooking",
"target": "raven_matrices"
},
{
"source": "math",
"target": "change_my_view"
},
{
"source": "math",
"target": "cooking"
},
{
"source": "change_my_view",
"target": "raven_matrices"
},
{
"source": "change_my_view",
"target": "cooking"
},
{
"source": "raven_matrices",
"target": "us_history"
},
{
"source": "raven_matrices",
"target": "code"
},
{
"source": "us_history",
"target": "math"
},
{
"source": "us_history",
"target": "code"
},
{
"source": "us_history",
"target": "us_history_textbook"
},
{
"source": "us_history_textbook",
"target": "us_history_fiction"
},
{
"source": "us_history_fiction",
"target": "us_history_make_questions"
},
{
"source": "us_history_make_questions",
"target": "us_history"
},
{
"source": "math",
"target": "math_fiction"
},
{
"source": "math_fiction",
"target": "math_textbook"
},
{
"source": "math_textbook",
"target": "math_make_questions"
},
{
"source": "math_make_questions",
"target": "math"
},
{
"source": "alpaca_low_quality",
"target": "alpaca_high_quality"
},
{
"source": "shp_low_quality",
"target": "shp_high_quality"
},
{
"source": "code_low_quality",
"target": "code"
}
]
distribution_shifts/extreme/context.json
+34
View File
@@ -0,0 +1,34 @@
[
{
"source": "us_history",
"target": "us_history_textbook"
},
{
"source": "us_history_textbook",
"target": "us_history_fiction"
},
{
"source": "us_history_fiction",
"target": "us_history_make_questions"
},
{
"source": "us_history_make_questions",
"target": "us_history"
},
{
"source": "math",
"target": "math_fiction"
},
{
"source": "math_fiction",
"target": "math_textbook"
},
{
"source": "math_textbook",
"target": "math_make_questions"
},
{
"source": "math_make_questions",
"target": "math"
}
]
distribution_shifts/extreme/difficulty.json
+8
View File
@@ -0,0 +1,8 @@
[
{"source": "alpaca_easy", "target": "alpaca_hard"},
{"source": "arc_easy", "target": "arc_hard"},
{"source": "math_easy", "target": "math_hard"},
{"source": "code_easy", "target": "code_hard"},
{"source": "ranking_logic_easy", "target": "ranking_logic_hard"},
{"source": "raven_easy", "target": "raven_matrices"}
]
distribution_shifts/extreme/encoding.json
+6
View File
@@ -0,0 +1,6 @@
[
{"source": "alpaca_mmlu", "target": "spanish_input"},
{"source": "alpaca_mmlu", "target": "spanish_output"},
{"source": "alpaca_mmlu", "target": "comma_separated_input"},
{"source": "alpaca_mmlu", "target": "comma_separated_output"}
]
distribution_shifts/extreme/pretraining_similarity.json
+18
View File
@@ -0,0 +1,18 @@
[
{
"source": "alpaca_mmlu",
"target": "ranking_logic"
},
{
"source": "alpaca_mmlu",
"target": "raven_matrices"
},
{
"source": "alpaca_mmlu",
"target": "word_swap"
},
{
"source": "code",
"target": "counterfactual_python"
}
]
distribution_shifts/extreme/quality.json
+5
View File
@@ -0,0 +1,5 @@
[
{"source": "alpaca_low_quality", "target": "alpaca_high_quality"},
{"source": "shp_low_quality", "target": "shp_high_quality"},
{"source": "code_low_quality", "target": "code"}
]
distribution_shifts/extreme/skill.json
+14
View File
@@ -0,0 +1,14 @@
[
{"source": "code", "target": "us_history"},
{"source": "code", "target": "change_my_view"},
{"source": "cooking", "target": "math"},
{"source": "cooking", "target": "raven_matrices"},
{"source": "math", "target": "change_my_view"},
{"source": "math", "target": "cooking"},
{"source": "change_my_view", "target": "raven_matrices"},
{"source": "change_my_view", "target": "cooking"},
{"source": "raven_matrices", "target": "us_history"},
{"source": "raven_matrices", "target": "code"},
{"source": "us_history", "target": "math"},
{"source": "us_history", "target": "code"}
]
distribution_shifts/probing/all.json
+139
View File
@@ -0,0 +1,139 @@
[
{
"source": "alpaca_mmlu",
"target": "truthful_qa"
},
{
"source": "alpaca_mmlu",
"target": "personality_traits"
},
{
"source": "alpaca_mmlu",
"target": "survival_influence"
},
{
"source": "alpaca_mmlu",
"target": "gender_bias"
},
{
"source": "alpaca_mmlu",
"target": "punishment_avoidance"
},
{
"source": "alpaca_mmlu",
"target": "reward_seeking"
},
{
"source": "alpaca_mmlu",
"target": "crt_1"
},
{
"source": "alpaca_mmlu",
"target": "crt_2"
},
{
"source": "alpaca_mmlu",
"target": "crt_3"
},
{
"source": "alpaca_mmlu",
"target": "sycophancy_mimicry",
"target_reference": "quote_attribution"
},
{
"source": "alpaca_mmlu",
"target": "sycophancy_answer",
"target_reference": "arc_easy"
},
{
"source": "alpaca_mmlu",
"target": "sycophancy_feedback",
"target_reference": "code_is_correct"
},
{
"source": "alpaca_chat",
"target": "sycophancy_are_you_sure",
"target_reference": "arc_easy"
},
{
"source": "pursue_goals",
"target": "relinquish_power"
},
{
"source": "creative_writing",
"target": "biology_with_literary_style"
},
{
"source": "alpaca_short",
"target": "alpaca_long",
"target_reference": "alpaca_mmlu"
},
{
"source": "alpaca_chat",
"target": "illegal_dont_help"
},
{
"source": "alpaca_mmlu",
"target": "wrong_arc"
},
{
"source": "alpaca_mmlu",
"target": "unhelpful_alpaca"
},
{
"source": "alpaca_mmlu",
"target": "truthful_qa"
},
{
"source": "alpaca_mmlu",
"target": "personality_traits"
},
{
"source": "alpaca_mmlu",
"target": "gender_bias"
},
{
"source": "alpaca_mmlu",
"target": "survival_influence"
},
{
"source": "alpaca_mmlu",
"target": "punishment_avoidance"
},
{
"source": "alpaca_mmlu",
"target": "reward_seeking"
},
{
"source": "alpaca_mmlu",
"target": "crt_1"
},
{
"source": "alpaca_mmlu",
"target": "crt_2"
},
{
"source": "alpaca_mmlu",
"target": "crt_3"
},
{
"source": "alpaca_mmlu",
"target": "sycophancy_mimicry",
"target_reference": "quote_attribution"
},
{
"source": "alpaca_mmlu",
"target": "sycophancy_answer",
"target_reference": "arc_easy"
},
{
"source": "alpaca_mmlu",
"target": "sycophancy_feedback",
"target_reference": "code_is_correct"
},
{
"source": "alpaca_chat",
"target": "sycophancy_are_you_sure",
"target_reference": "arc_easy"
}
]
distribution_shifts/probing/spurious_cues.json
+27
View File
@@ -0,0 +1,27 @@
[
{
"source": "pursue_goals",
"target": "relinquish_power"
},
{
"source": "creative_writing",
"target": "biology_with_literary_style"
},
{
"source": "alpaca_short",
"target": "alpaca_long",
"target_reference": "alpaca_mmlu"
},
{
"source": "alpaca_chat",
"target": "illegal_dont_help"
},
{
"source": "alpaca_mmlu",
"target": "wrong_arc"
},
{
"source": "alpaca_mmlu",
"target": "unhelpful_alpaca"
}
]
distribution_shifts/probing/unwanted_personas.json
+15
View File
@@ -0,0 +1,15 @@
[
{"source": "alpaca_mmlu", "target": "truthful_qa"},
{"source": "alpaca_mmlu", "target": "personality_traits"},
{"source": "alpaca_mmlu", "target": "survival_influence"},
{"source": "alpaca_mmlu", "target": "gender_bias"},
{"source": "alpaca_mmlu", "target": "punishment_avoidance"},
{"source": "alpaca_mmlu", "target": "reward_seeking"},
{"source": "alpaca_mmlu", "target": "crt_1"},
{"source": "alpaca_mmlu", "target": "crt_2"},
{"source": "alpaca_mmlu", "target": "crt_3"},
{"source": "alpaca_mmlu", "target": "sycophancy_mimicry", "target_reference": "quote_attribution"},
{"source": "alpaca_mmlu", "target": "sycophancy_answer", "target_reference": "arc_easy"},
{"source": "alpaca_mmlu", "target": "sycophancy_feedback", "target_reference": "code_is_correct"},
{"source": "alpaca_chat", "target": "sycophancy_are_you_sure", "target_reference": "arc_easy"}
]
distribution_shifts/test.json
+3
View File
@@ -0,0 +1,3 @@
[
{"source": "us_history_textbook", "target": "us_history_fiction", "label": "extreme", "category": "context"}
]