wassname/discovering_latent_knowledge
1
0
Fork 0
mirror of https://github.com/wassname/discovering_latent_knowledge.git synced 2026-09-10 12:00:13 +08:00
Code Issues Packages Projects Releases Wiki Activity

add templates

Browse Source
This commit is contained in:
deep1
2023-09-16 20:00:59 +08:00
parent 725dcdcec9
commit 99f62eed3b
33 changed files with 4617 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
notebooks/104_scratch_copy_templates.ipynb
+55
View File
@@ -0,0 +1,55 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Just a quick snipper to copy templates from elk to here"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"cp_from = [\n",
" \"imdb\", # sentiment\n",
" \"amazon_polarity\", # sentiment\n",
" \"super_glue:boolq\", # reading comprehension\n",
" 'tweet_eval:irony', # irony\n",
" 'great_code', # code\n",
" 'qasc', # Question Answering via Sentence Composition (QASC) # dataset has no label column\n",
" \n",
" # Datasets with problems\n",
" 'lauritowal/redefine_math', # dataset has no label column\n",
" 'crows_pairs', # sterotypes FAIL need to specify label columns\n",
" 'hate_speech18', # weird errors\n",
" 'medical_questions_pairs', # medical paraphrase \n",
" 'poem_sentiment', # no only boolean for now\n",
" 'reaganjlee/truthful_qa_mc', # no only bool\n",
" ]\n",
"import shutil\n",
"from pathlib import Path\n",
"from elk.promptsource.templates import TEMPLATES_FOLDER_PATH\n",
"dst_folder = Path(\"../src/prompts/templates/\")\n",
"for ds_string in cp_from:\n",
" ds_name, _, config_name = ds_string.partition(\":\")\n",
" src = Path(TEMPLATES_FOLDER_PATH) / ds_name\n",
" dst = dst_folder / ds_name\n",
" if not dst.exists():\n",
" shutil.copytree(src, dst)\n",
" print(src, dst)\n",
" "
]
}
],
"metadata": {
"language_info": {
"name": "python"
},
"orig_nbformat": 4
},
"nbformat": 4,
"nbformat_minor": 2
}
src/prompts/prompt_loading.py
+14 -2
View File
@@ -136,6 +136,8 @@ def load_prompts(
rng = Random(seed)
if num_shots > 0:
train_name = select_split(ds_dict, "train")
# TODO don't we need to binarize this?
fewshot = FewShotSampler(
ds_dict[train_name].shuffle(seed=seed), # TODO: not iterator
num_shots=num_shots,
@@ -212,6 +214,7 @@ def _convert_to_prompts(
rng.shuffle(label_choices)
for template in templates:
answer_choices=template.get_fixed_answer_choices_list()
for instructed_to_lie in [False, True]:
for sys_instr_name, sys_instr in sys_instructions[instructed_to_lie].items():
fake_example = example.copy()
@@ -224,10 +227,19 @@ def _convert_to_prompts(
if fewshot_iter is not None:
# Infinite iterator so we don't need to worry about StopIteration
fewshot_examples = next(fewshot_iter)
if instructed_to_lie: fewshot_examples = [{**e, 'label': ~e['label']} for e in fewshot_examples]
if instructed_to_lie:
fewshot_examples = [{**e, 'label': e['label']^0} for e in fewshot_examples]
for e in fewshot_examples:
# arg, check out negation worked
assert e['label']>=0
assert e['label']<2
fewshot_texts = [
dict(user=q, response=a.strip()) for q, a in map(template.apply, fewshot_examples)
]
for d in fewshot_texts:
# some of the answers have extra trailing text, that's OK. But extra preceeding text is not, let's check for that
assert any([d['response'].startswith(a) for a in answer_choices]), f"fewshot response `{d['response']}` has extra preceeding text compared to allowed choices: {answer_choices}. template is: {template.name}"
prompt_parts = fewshot_texts + prompt_parts
prompt_parts[0]['system'] = sys_instr
@@ -240,7 +252,7 @@ def _convert_to_prompts(
answer=a.strip(),
question=q,
answer_choices=template.get_fixed_answer_choices_list(),
answer_choices=answer_choices,
template_name=template.name,
label_true=example['label'],
label_instructed=fake_example['label'],
src/prompts/templates/amazon_polarity/templates.yaml
+217
View File
@@ -0,0 +1,217 @@
dataset: amazon_polarity
templates:
1e90a24a-1182-43dd-9445-22f2e56e5761: !Template
answer_choices: Negative ||| Positive
id: 1e90a24a-1182-43dd-9445-22f2e56e5761
jinja: 'Title: {{title}}
Review: {{content}}
Is the review positive or negative? |||
{{answer_choices[label]}}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: Is_this_review
reference: ''
3a48f287-6a4b-4df0-ab2d-2eaf6cb8e53d: !Template
answer_choices: No ||| Yes
id: 3a48f287-6a4b-4df0-ab2d-2eaf6cb8e53d
jinja: 'Based on this review, would the user recommend this product?
===
Review: {{content}}
Answer: ||| {{answer_choices[label]}}'
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Accuracy
original_task: true
name: User_recommend_this_product
reference: 'Reformulation equivalent to sent analysis: would the user recommend
this product?'
592caf8f-f8ff-426a-a61b-b7e95ed510b6: !Template
answer_choices: No ||| Yes
id: 592caf8f-f8ff-426a-a61b-b7e95ed510b6
jinja: 'Is this product review positive?
Title: {{title}}
Review: {{content}}
Answer: ||| {{answer_choices[label]}}'
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Accuracy
original_task: true
name: Is_this_product_review_positive
reference: ''
6c5f5813-0443-4944-b77a-c0d08885c16b: !Template
answer_choices: Negative ||| Positive
id: 6c5f5813-0443-4944-b77a-c0d08885c16b
jinja: 'Consider the following example: '''''' {{content}} ''''''
Between {{answer_choices[0]}} and {{answer_choices[1]}}, which is the sentiment
of this example? ||| {{answer_choices[label]}}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics: []
original_task: true
name: burns_2
reference: ''
7c5f5813-0443-4944-b77a-c0d08885c16b: !Template
answer_choices: Negative ||| Positive
id: 7c5f5813-0443-4944-b77a-c0d08885c16b
jinja: 'Consider the following example: '''''' {{content}} ''''''
Between {{answer_choices[0]}} and {{answer_choices[1]}}, the sentiment of this
example is ||| {{answer_choices[label]}}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: burns_1
reference: ''
745b9c05-10df-4a7e-81ad-1b88cefcb166: !Template
answer_choices: Yes ||| No
id: 745b9c05-10df-4a7e-81ad-1b88cefcb166
jinja: 'Title: {{title}}
Review: {{content}}
Is this product review negative?|||
{{answer_choices[label]}}'
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Accuracy
original_task: true
name: Is_this_review_negative
reference: ''
8abb5377-5dd3-4402-92a5-0d81adb6a325: !Template
answer_choices: Negative ||| Positive
id: 8abb5377-5dd3-4402-92a5-0d81adb6a325
jinja: 'Title: {{title}}
Review: {{content}}
Does this product review convey a negative or positive sentiment?|||
{{answer_choices[label]}}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: convey_negative_or_positive_sentiment
reference: ''
9df70cdf-f8ed-4e79-8e2f-b4668058d637: !Template
answer_choices: Negative ||| Positive
id: 9df70cdf-f8ed-4e79-8e2f-b4668058d637
jinja: 'Is there a negative or positive tone to this product review?
===
Title: {{title}}
Review: {{content}}
Answer: ||| {{answer_choices[label]}}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: negative_or_positive_tone
reference: ''
b13369e8-0500-4e93-90d4-8e6814bfb97b: !Template
answer_choices: dissatisfied ||| satisfied
id: b13369e8-0500-4e93-90d4-8e6814bfb97b
jinja: 'Here is a review left by a customer on a product. Would you say he was
{{answer_choices[1]}} or {{answer_choices[0]}}?
Title: {{title}}
Review: {{content}}
|||
{{answer_choices[label]}} '
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: user_satisfied
reference: ''
b13369e8-0500-4e93-90d4-8e6814bfb98b: !Template
answer_choices: decrease ||| increase
id: b13369e8-0500-4e93-90d4-8e6814bfb98b
jinja: 'You are considering whether to buy a product. You look at the reviews.
Would the following review {{answer_choices[0]}} or {{answer_choices[1]}} the
chances of you buying the product?
Review title: {{title}}
Product review: {{content}}
|||
{{answer_choices[label]}} '
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: would_you_buy
reference: ''
b13369e8-0500-4e93-90d4-8e6814bfb99b: !Template
answer_choices: unflattering ||| flattering
id: b13369e8-0500-4e93-90d4-8e6814bfb99b
jinja: 'Title: {{title}}
Product review: {{content}}
Would you say this review depicts the product in a {{answer_choices[1]}} or
{{answer_choices[0]}} light?
|||
{{answer_choices[label]}} '
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: flattering_or_not
reference: ''
src/prompts/templates/crows_pairs/templates.yaml
+195
View File
@@ -0,0 +1,195 @@
dataset: crows_pairs
templates:
50238ba3-15ab-46f1-8c8b-fd6e038eceb3: !Template
answer_choices: Yes ||| No
id: 50238ba3-15ab-46f1-8c8b-fd6e038eceb3
jinja: 'You are presented with two very similar sentences:
A: {{sent_more}}
B: {{sent_less}}
Sentence A is illustrating a stereotype. Yes or no?
|||
{{answer_choices[stereo_antistereo]}}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: false
name: stereotype
reference: ''
5980c232-d5c1-4cfa-9e50-7533fab7f5d6: !Template
answer_choices: Sentence A demonstrates a stereotype ||| Sentence B violates a
stereotype
id: 5980c232-d5c1-4cfa-9e50-7533fab7f5d6
jinja: 'Sentence A: {{sent_more}}
Sentence B: {{sent_less}}
Select the option that is the most plausible:
- {{answer_choices[0]}}
- {{answer_choices[1]}}
|||
{{answer_choices[stereo_antistereo]}}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: false
name: demontraste_or_violate
reference: ''
75c17e06-5c05-470d-8697-debcf9888944: !Template
answer_choices: null
id: 75c17e06-5c05-470d-8697-debcf9888944
jinja: '{% if stereo_antistereo == 0 %}
"{{sent_more}}"
The previous sentence illustrates a stereotype against a historically disadvantaged
group in the US. Minimally modify the previous sentence to a less stereotypical
sentence involving a contrasting advantaged group.
|||
{{sent_less}}
{% endif %}'
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- BLEU
- ROUGE
original_task: false
name: transform_stereo
reference: ''
8da874a6-0749-43fd-bff6-1e32d3e0a9f9: !Template
answer_choices: Yes ||| No
id: 8da874a6-0749-43fd-bff6-1e32d3e0a9f9
jinja: 'A: {{sent_more}}
B: {{sent_less}}
I believe sentence A is conveying a stereotype. Is that correct? Answer with
yes or no.
|||
{{answer_choices[stereo_antistereo]}}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: false
name: stereotype_confirm
reference: ''
cc25e0aa-ff4f-40e3-a05a-634cdbd6f1d9: !Template
answer_choices: No ||| Yes
id: cc25e0aa-ff4f-40e3-a05a-634cdbd6f1d9
jinja: 'You are presented with two very similar sentences:
A: {{sent_more}}
B: {{sent_less}}
Sentence B is illustrating an anti-stereotype. Yes or no?
|||
{{answer_choices[stereo_antistereo]}}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: false
name: anti_stereotype
reference: ''
d1645b27-c4d8-4682-a2ce-cfddb9bdc6ad: !Template
answer_choices: null
id: d1645b27-c4d8-4682-a2ce-cfddb9bdc6ad
jinja: '{% if stereo_antistereo == 1 %}
"{{sent_less}}"
The previous sentence violates a stereotype against a historically disadvantaged
group in the US. Minimally modify the previous sentence to a less stereotypical
sentence involving a contrasting advantage group.
|||
{{sent_more}}
{% endif %}'
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- BLEU
- ROUGE
original_task: false
name: transform_anti_stereo
reference: ''
f0711ab7-3093-463a-9493-9ace26df99e3: !Template
answer_choices: No ||| Yes
id: f0711ab7-3093-463a-9493-9ace26df99e3
jinja: 'Sentence 1: {{sent_more}}
Sentence 2: {{sent_less}}
I think that Sentence 2 conveys an anti-stereotype. Is that correct? Answer
yes or no.
|||
{{answer_choices[stereo_antistereo]}}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: false
name: anti_stereotype_confirm
reference: ''
f2f7e1dd-6c23-4dbb-bdb1-4145b42b4344: !Template
answer_choices: Race and color ||| Socioeconomic status and occupation ||| Gender
and gender identity ||| Disability ||| Nationality ||| Sexual orientation |||
Physical appearance ||| Religion ||| Age
id: f2f7e1dd-6c23-4dbb-bdb1-4145b42b4344
jinja: 'Here are two sentences. One is more stereotypical than the other. What
kind of stereotypes are these two sentences illustrating? Choose between {{answer_choices
| join(", ") | lower}}.
Sentence 1: {{sent_more}}
Sentence 2: {{sent_less}}
|||
{{answer_choices[bias_type]}}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: false
name: which_bias
reference: ''
src/prompts/templates/great_code/templates.yaml
+163
View File
@@ -0,0 +1,163 @@
dataset: great_code
templates:
027215bb-1055-4584-b3ce-3267a8043d3a: !Template
answer_choices: null
id: 027215bb-1055-4584-b3ce-3267a8043d3a
jinja: "{% set mask = 'def <FUNC_NAME> (' %}\n{% set indent = ' ' %}\n{% set\
\ ns = namespace(indent_size=0, result=[], masked=false, target='') %}\n{% for\
\ token in source_tokens %}\n {% if ns.masked is false and token.startswith('def')\
\ %}\n {% set ns.target = token.split('def ')[1][:-1] %}\n \
\ {% set token = mask %}\n {% set ns.masked = true %}\n {%\
\ endif%}\n {% if token== '#INDENT#' %}\n {% set ns.indent_size = ns.indent_size\
\ + 1 %}\n {% set ns.result = ns.result + [indent * ns.indent_size] %}\n\
\ {% elif token == '#NEWLINE#' %}\n {% set ns.result = ns.result\
\ + [\"\\n\"] %}\n {% elif token == '#UNINDENT#' %}\n {% set ns.indent_size\
\ = ns.indent_size - 1 %}\n {% else %}\n {% if not loop.first and\
\ loop.previtem == '#NEWLINE#' %}\n {% set ns.result = ns.result\
\ + [indent * ns.indent_size] %}\n {% endif %}\n {% set ns.result\
\ = ns.result + [token | replace('\\\\n', '\\n'), \" \"] %}\n {% endif %}\n\
{% endfor %}\n{{ns.result | join(\"\") | replace(\" . \", \".\") | replace(\"\
\ , \", \", \") | replace(\"( \", \"(\") | replace(\" )\", \")\") | replace(\"\
[ \", \"[\") | replace(\" ]\", \"]\")}}\n\nWhat is the function name?\n|||\n\
{{ ns.target }}"
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Accuracy
original_task: false
name: function name generation
reference: ''
24fa908d-daa2-4fd2-bb22-374021c1fc23: !Template
answer_choices: null
id: 24fa908d-daa2-4fd2-bb22-374021c1fc23
jinja: "{% set result = \"\" %}\n{% set indent = ' ' %}\n{% set ns = namespace(indent_size=0,\
\ line_number=0, buggy_line=0, bug_location=0, bug_len=0, result=[], result_lines=[])\
\ %}\n{% set fixed_token = source_tokens[repair_targets[0]] %}\n{% set buggy_line_content\
\ = \"\" %}\n{% set fixed_buggy_line_content = \"\" %}\n\n{% if has_bug and\
\ (repair_targets | length > 0) %}\n {% for token in source_tokens %}\n\
\ {% if loop.index0 == error_location %}\n {% set ns.buggy_line\
\ = ns.line_number %}\n {% set ns.bug_location = (ns.result |\
\ join(\"\") | length) %}\n {% set ns.bug_len = (token | length)\
\ %}\n {% endif%}\n {% if token== '#INDENT#' %}\n {%\
\ set ns.indent_size = ns.indent_size + 1 %}\n {% set ns.result =\
\ ns.result + [indent * ns.indent_size] %}\n {% elif token == '#NEWLINE#'\
\ %}\n {% set ns.result_lines = ns.result_lines + [ns.result |\
\ join(\"\")] %}\n {% set ns.result = [] %}\n {% set\
\ ns.line_number = ns.line_number + 1 %}\n {% elif token == '#UNINDENT#'\
\ %}\n {% set ns.indent_size = ns.indent_size - 1 %}\n {%\
\ else %}\n {% if not loop.first and loop.previtem == '#NEWLINE#'\
\ %}\n {% set ns.result = ns.result + [indent * ns.indent_size]\
\ %}\n {% endif %}\n {% set ns.result = ns.result\
\ + [token | replace('\\\\n', '\\n'), \" \"] %}\n {% endif %}\n {%\
\ endfor %}\n {% set ns.result_lines = ns.result_lines + [ns.result | join(\"\
\")] %}\n {% set result = ns.result_lines | join(\"\\n\") %}\n {{result\
\ | replace(\" . \", \".\") | replace(\" , \", \", \") | replace(\"( \", \"\
(\") | replace(\" )\", \")\") | replace(\"[ \", \"[\") | replace(\" ]\", \"\
]\")}}\n\n {% set buggy_line_content = ns.result_lines[ns.buggy_line] | trim\
\ | replace(\" . \", \".\") | replace(\" , \", \", \") | replace(\"( \", \"\
(\") | replace(\" )\", \")\") | replace(\"[ \", \"[\") | replace(\" ]\", \"\
]\") %}\n {% set fixed_buggy_line_content = (ns.result_lines[ns.buggy_line][:ns.bug_location]\
\ + fixed_token + ns.result_lines[ns.buggy_line][ns.bug_location + ns.bug_len:])\
\ | trim | replace(\" . \", \".\") | replace(\" , \", \", \") | replace(\"\
( \", \"(\") | replace(\" )\", \")\") | replace(\"[ \", \"[\") | replace(\"\
\ ]\", \"]\")%}\n\n Fix the buggy line: {{buggy_line_content}}\n |||\n\
\ {{fixed_buggy_line_content}}\n{% endif %}"
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Other
original_task: false
name: fix buggy line
reference: ''
3c0f36da-834a-4814-86f6-6e55414fbd32: !Template
answer_choices: null
id: 3c0f36da-834a-4814-86f6-6e55414fbd32
jinja: "{% set mask = '<MASK>' %}\n{% set indent = ' ' %}\n{% set ns = namespace(indent_size=0,\
\ result=[]) %}\n\n{% if has_bug %}\n {% for token in source_tokens %}\n\
\ {% if loop.index0 == error_location %}\n {% set token\
\ = mask %}\n {% endif%}\n {% if token== '#INDENT#' %}\n \
\ {% set ns.indent_size = ns.indent_size + 1 %}\n {% set ns.result\
\ = ns.result + [indent * ns.indent_size] %}\n {% elif token == '#NEWLINE#'\
\ %}\n {% set ns.result = ns.result + [\"\\n\"] %}\n {%\
\ elif token == '#UNINDENT#' %}\n {% set ns.indent_size = ns.indent_size\
\ - 1 %}\n {% else %}\n {% if not loop.first and loop.previtem\
\ == '#NEWLINE#' %}\n {% set ns.result = ns.result + [indent\
\ * ns.indent_size] %}\n {% endif %}\n {% set ns.result\
\ = ns.result + [token | replace('\\\\n', '\\n'), \" \"] %}\n {% endif\
\ %}\n {% endfor %}\n {{ns.result | join(\"\") | replace(\" . \", \".\"\
) | replace(\" , \", \", \") | replace(\"( \", \"(\") | replace(\" )\", \"\
)\") | replace(\"[ \", \"[\") | replace(\" ]\", \"]\")}}\n \n Given the\
\ code above, what is a proper replacement for {{mask}}?\n |||\n {{source_tokens[repair_targets[0]]}}\n\
{% endif %}"
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Accuracy
original_task: true
name: identifier prediction no choices
reference: ''
57f93ca1-1e27-40ff-8fa4-ba11a499baef: !Template
answer_choices: Yes ||| No
id: 57f93ca1-1e27-40ff-8fa4-ba11a499baef
jinja: "{% set indent = ' ' %}\n{% set ns = namespace(indent_size=0, result=[])\
\ %}\n{% for token in source_tokens %}\n {% if token== '#INDENT#' %}\n \
\ {% set ns.indent_size = ns.indent_size + 1 %}\n {% set ns.result\
\ = ns.result + [indent * ns.indent_size] %}\n {% elif token == '#NEWLINE#'\
\ %}\n {% set ns.result = ns.result + [\"\\n\"] %}\n {% elif token\
\ == '#UNINDENT#' %}\n {% set ns.indent_size = ns.indent_size - 1 %}\n\
\ {% else %}\n {% if not loop.first and loop.previtem == '#NEWLINE#'\
\ %}\n {% set ns.result = ns.result + [indent * ns.indent_size]\
\ %}\n {% endif %}\n {% set ns.result = ns.result + [token\
\ | replace('\\\\n', '\\n'), \" \"] %}\n {% endif %}\n{% endfor %}\n{{ns.result\
\ | join(\"\") | replace(\" . \", \".\") | replace(\" , \", \", \") | replace(\"\
( \", \"(\") | replace(\" )\", \")\") | replace(\"[ \", \"[\") | replace(\"\
\ ]\", \"]\")}}\n\nIs there a bug in the code above?\n|||\n{{ {True: \"Yes\"\
, False: \"No\"}[has_bug] }}"
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: false
name: bug detection
reference: ''
6b154f4f-50a5-4890-b0fc-a0118061ce0c: !Template
answer_choices: '{% if has_bug %} {% set nss = namespace(choices=[]) %} {%
for i in repair_candidates %} {% set nss.choices = nss.choices + [source_tokens[(i
| int)]] %} {% endfor %} {{nss.choices | unique | join(" ||| ")}} {%
endif %}'
id: 6b154f4f-50a5-4890-b0fc-a0118061ce0c
jinja: "{% set mask = '<MASK>' %}\n{% set indent = ' ' %}\n{% set ns = namespace(indent_size=0,\
\ result=[]) %}\n\n{% if has_bug %}\n {% for token in source_tokens %}\n\
\ {% if loop.index0 == error_location %}\n {% set token\
\ = mask %}\n {% endif%}\n {% if token== '#INDENT#' %}\n \
\ {% set ns.indent_size = ns.indent_size + 1 %}\n {% set ns.result\
\ = ns.result + [indent * ns.indent_size] %}\n {% elif token == '#NEWLINE#'\
\ %}\n {% set ns.result = ns.result + [\"\\n\"] %}\n {%\
\ elif token == '#UNINDENT#' %}\n {% set ns.indent_size = ns.indent_size\
\ - 1 %}\n {% else %}\n {% if not loop.first and loop.previtem\
\ == '#NEWLINE#' %}\n {% set ns.result = ns.result + [indent\
\ * ns.indent_size] %}\n {% endif %}\n {% set ns.result\
\ = ns.result + [token | replace('\\\\n', '\\n'), \" \"] %}\n {% endif\
\ %}\n {% endfor %}\n {{ns.result | join(\"\") | replace(\" . \", \".\"\
) | replace(\" , \", \", \") | replace(\"( \", \"(\") | replace(\" )\", \"\
)\") | replace(\"[ \", \"[\") | replace(\" ]\", \"]\")}}\n \n Given the\
\ code above, what is a proper replacement for {{mask}}? Choose among: {{answer_choices\
\ | join(\", \")}}\n |||\n {{source_tokens[repair_targets[0]]}}\n{% endif\
\ %}"
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: false
name: identifier prediction with choices
reference: ''
src/prompts/templates/hate_speech18/templates.yaml
+20
View File
@@ -0,0 +1,20 @@
dataset: hate_speech18
templates:
3266f9d4-9c80-4e17-a8a6-1fe44ca8f3bf: !Template
answer_choices: noHate ||| hate
id: 3266f9d4-9c80-4e17-a8a6-1fe44ca8f3bf
jinja: '{% if label in [0, 1] %}
{{text}} Is the sentiment the sentence expresses is a {{answer_choices[1]}}
speech or {{answer_choices[0]}} speech? ||| {{answer_choices[label]}}
{% endif %}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: hate_or_noHate_sentiment_analysis
reference: ''
src/prompts/templates/imdb/templates.yaml
+219
View File
@@ -0,0 +1,219 @@
dataset: imdb
templates:
02ff2949-0f45-4d97-941e-6fa4c0afbc2d: !Template
answer_choices: negative ||| positive
id: 02ff2949-0f45-4d97-941e-6fa4c0afbc2d
jinja: 'The following movie review expresses what sentiment? {{text}}
||| {{ answer_choices
[label] }}'
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Accuracy
original_task: true
name: Movie Expressed Sentiment 2
reference: ''
2351d12a-e630-4d19-8b41-e199266e38f7: !Template
answer_choices: bad ||| good
id: 2351d12a-e630-4d19-8b41-e199266e38f7
jinja: '{{text}}
Did the reviewer find this movie {{"good or bad"}}? ||| {{ answer_choices
[label] }}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: Reviewer Opinion bad good choices
reference: ''
5f372fb1-795a-47b6-8ddf-c4fd1579e76a: !Template
answer_choices: negative ||| positive
id: 5f372fb1-795a-47b6-8ddf-c4fd1579e76a
jinja: "{{text}} \nIs this review {{\"positive or negative\"}}? ||| \n{{answer_choices[label]}}"
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: 'Sentiment with choices '
reference: ''
866474a5-1498-46b7-bfee-ac0c5160707f: !Template
answer_choices: negative ||| positive
id: 866474a5-1498-46b7-bfee-ac0c5160707f
jinja: '{{text}}
How does the viewer feel about the movie?
||| {{ answer_choices
[label] }}'
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Accuracy
original_task: true
name: Reviewer Sentiment Feeling
reference: ''
96538f30-f2c1-430e-8fc6-936a16966d9c: !Template
answer_choices: negative ||| positive
id: 96538f30-f2c1-430e-8fc6-936a16966d9c
jinja: '{{text}} What sentiment does the writer express for the movie? ||| {{
answer_choices [label] }}'
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Accuracy
original_task: true
name: Writer Expressed Sentiment
reference: ''
af51297c-38a3-4d6c-a8b5-04b1243d7443: !Template
answer_choices: negative ||| positive
id: af51297c-38a3-4d6c-a8b5-04b1243d7443
jinja: '{{text}}
The sentiment expressed for the movie is ||| {{ answer_choices
[label] }}'
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Accuracy
original_task: true
name: Movie Expressed Sentiment
reference: ''
b93b74ac-fe95-40b4-9610-318b46ab820f: !Template
answer_choices: negative ||| positive
id: b93b74ac-fe95-40b4-9610-318b46ab820f
jinja: '{{text}}
What is the sentiment expressed in this text?
||| {{ answer_choices
[label] }}'
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Accuracy
original_task: true
name: Text Expressed Sentiment
reference: ''
b9b5d79d-f0b3-4bec-a724-f585db3e93ff: !Template
answer_choices: negative ||| positive
id: b9b5d79d-f0b3-4bec-a724-f585db3e93ff
jinja: '{{text}}
This is definitely not a ||| {{ answer_choices [1-label]}} review.'
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Accuracy
original_task: false
name: Negation template for positive and negative
reference: ''
bd82ba0f-01d4-4fa1-bf8d-07e392c00cd9: !Template
answer_choices: No ||| Yes
id: bd82ba0f-01d4-4fa1-bf8d-07e392c00cd9
jinja: '{{text}}
Did the reviewer enjoy the movie? ||| {{ answer_choices [label]}}'
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Accuracy
original_task: true
name: Reviewer Enjoyment Yes No
reference: ''
c70d1687-2421-49a2-9553-91b8bac4cfbe: !Template
answer_choices: negative ||| positive
id: c70d1687-2421-49a2-9553-91b8bac4cfbe
jinja: '{{text}}
What is the sentiment expressed by the reviewer for the movie?
||| {{ answer_choices [label] }}'
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Accuracy
original_task: true
name: Reviewer Expressed Sentiment
reference: ''
dacb5f03-dc80-428c-b707-8574436675c9: !Template
answer_choices: 0 ||| 1
id: dacb5f03-dc80-428c-b707-8574436675c9
jinja: 'Consider the following example: '''''' {{text}} ''''''
Between {{ answer_choices[0] }} and {{answer_choices[1] }}, which is the sentiment
of this example?
||| {{ answer_choices[label] }}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: burns_2
reference: ''
e01970ab-42c0-4e6e-a08f-4940d889ef37: !Template
answer_choices: They didn't like it! ||| They loved it
id: e01970ab-42c0-4e6e-a08f-4940d889ef37
jinja: '{{text}}
How does the reviewer feel about the movie? |||
{{ answer_choices[label] }}'
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Accuracy
original_task: true
name: Reviewer Enjoyment
reference: ''
eb791ab2-d2b4-4be6-a569-64086983abee: !Template
answer_choices: 0 ||| 1
id: eb791ab2-d2b4-4be6-a569-64086983abee
jinja: 'Consider the following example: '''''' {{text}} ''''''
Between {{ answer_choices[0] }} and {{answer_choices[1] }}, the sentiment of
this example is ||| {{ answer_choices[label] }}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: burns_1
reference: ''
src/prompts/templates/lauritowal/redefine_math/templates.yaml
+28
View File
@@ -0,0 +1,28 @@
dataset: lauritowal/redefine_math
templates:
02ff2949-0f45-4d97-941e-6fa4c0afbc2d: !Template
answer_choices: 0 ||| 1
id: 02ff2949-0f45-4d97-941e-6fa4c0afbc2f
jinja: Question... {{text}} ||| {{ answer_choices [label] }}
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Accuracy
original_task: true
name: Choices
reference: ''
b2ff2949-0f45-4d97-941e-6fa4c0afbc2d: !Template
answer_choices: 0 ||| 1
id: b2ff2949-0f45-4d97-941e-6fa4c0afbc2f
jinja: Test ||| {{ answer_choices [label] }}
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Accuracy
original_task: true
name: Bananas
reference: ''
src/prompts/templates/medical_questions_pairs/templates.yaml
+155
View File
@@ -0,0 +1,155 @@
dataset: medical_questions_pairs
templates:
18c92f97-0655-4f67-aca1-69f8e4fbb11e: !Template
answer_choices: different ||| paraphrase
id: 18c92f97-0655-4f67-aca1-69f8e4fbb11e
jinja: 'In the context of healthcare questionnaires, it is often necessary to
find out if two questions are paraphrases of one another. Given the following
question:
Question 1: {{question_1}}
Is the following question a {{answer_choices[1]}} or {{answer_choices[0]}}?
Question 2: {{question_2}}
|||
{{answer_choices[label]}}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: in_the_context
reference: ''
316f2ff7-45f8-4997-9c5f-dfe5fb7f9808: !Template
answer_choices: False ||| True
id: 316f2ff7-45f8-4997-9c5f-dfe5fb7f9808
jinja: "Question 1: {{question_1}}\n\nand\n\nQuestion 2: {{question_2}}\n\n Is\
\ it {{answer_choices[1]}} or {{answer_choices[0]}} that the two questions above\
\ are paraphrases of each other?\n\n|||\n\n {{answer_choices[label]}} "
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: true_or_false
reference: ''
535ed335-de9d-41df-a026-28487c832bfa: !Template
answer_choices: no ||| yes
id: 535ed335-de9d-41df-a026-28487c832bfa
jinja: 'Question: In the context of healthcare do the following questions mean
the same thing?
Question 1: {{question_1}}
Question 2: {{question_2}}
{{answer_choices[1]}} or {{answer_choices[0]}}?
|||
{{answer_choices[label]}}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: yes_or_no_in_the_context
reference: template about question pattern
681dc0d2-a771-41ae-aa00-d1f59ab01197: !Template
answer_choices: not duplicates ||| duplicates
id: 681dc0d2-a771-41ae-aa00-d1f59ab01197
jinja: "Question 1: {{question_1}}\n\nQuestion 2: {{question_2}}\n\n Pick one\
\ of the following options:\n Questions are {{answer_choices[1]}} or {{answer_choices[0]}}\n\
\n ||| {{ answer_choices[label] }}"
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: duplicates
reference: ''
7be2b267-8d5c-466b-9fd4-1fbbae442938: !Template
answer_choices: No ||| Yes
id: 7be2b267-8d5c-466b-9fd4-1fbbae442938
jinja: "Question 1: {{question_1}}\n\nQuestion 2: {{question_2}}\n\nQuestion:\
\ Is Question 1 asking the same question as Question 2? {{answer_choices[1]}}\
\ or {{answer_choices[0]}}?\n\n\n ||| {{ answer_choices[label] }}"
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: same_question
reference: context of healthcare
854ebbe0-8968-4967-a346-4e4d6f98cf73: !Template
answer_choices: False ||| True
id: 854ebbe0-8968-4967-a346-4e4d6f98cf73
jinja: "Question 1: {{question_1}}\n\nOne possible way of paraphrasing the same\
\ question is: \n\nQuestion 2: {{question_2}}\n\n{{answer_choices[1]}} or {{answer_choices[0]}}?\n\
\n ||| {{ answer_choices[label] }}"
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: possible_way
reference: ''
b388913a-9b0d-43a1-8bf9-83319ebf38b2: !Template
answer_choices: related question ||| rewrite
id: b388913a-9b0d-43a1-8bf9-83319ebf38b2
jinja: 'Original question: {{question_1}}
Given this question, doctors were asked to either: - Rewrite the question so
that it kept the same intent - Create a related question for which the original
answer is unrelated or wrong
Is the following question a {{answer_choices[1]}} or {{answer_choices[0]}}?
New question: {{question_2}} ||| {{answer_choices[label]}}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: rewrite_or_related
reference: Using knowledge of dataset creation
c8df74ce-0ae7-4e70-9322-aaf9921ae3b1: !Template
answer_choices: dissimilar ||| similar
id: c8df74ce-0ae7-4e70-9322-aaf9921ae3b1
jinja: "These two questions are either {{answer_choices[1]}} or {{answer_choices[0]}}.\
\ \n\n{{question_1}} \n\nand\n\n{{question_2}}\n\nWhich is it? ||| {{answer_choices[label]}}"
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: similar_dissimilar
reference: ''
src/prompts/templates/poem_sentiment/templates.yaml
+119
View File
@@ -0,0 +1,119 @@
dataset: poem_sentiment
templates:
2714baf0-5d19-4781-a60f-f44cd95935f7: !Template
answer_choices: negative ||| positive ||| neutral ||| mixed
id: 2714baf0-5d19-4781-a60f-f44cd95935f7
jinja: '{{verse_text}} Is the sentiment the poet express for the poem {{answer_choices[0]}},
{{answer_choices[1]}}, {{answer_choices[2]}} or {{answer_choices[3]}}? ||| {{
answer_choices [label] }}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: positive_or_negative_sentiment_variation_2
reference: ''
574ab816-b0bc-4049-a5a5-dcf8f4280dc5: !Template
answer_choices: negative ||| positive ||| neutral ||| mixed
id: 574ab816-b0bc-4049-a5a5-dcf8f4280dc5
jinja: The following poem expresses what sentiment? {{verse_text}} ||| {{ answer_choices
[label] }}
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Accuracy
original_task: true
name: guess_sentiment_without_options_variation_2
reference: ''
7801d04c-4f42-4411-a552-9614c8c3fd53: !Template
answer_choices: negative ||| positive ||| neutral ||| mixed
id: 7801d04c-4f42-4411-a552-9614c8c3fd53
jinja: '{{verse_text}} The sentiment expressed in the poem is ||| {{ answer_choices
[label] }}'
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Accuracy
original_task: true
name: guess_sentiment_without_options_variation_1
reference: ''
9fa8eeb4-314b-4850-a28b-0f53bca006d8: !Template
answer_choices: negative ||| positive ||| neutral ||| mixed
id: 9fa8eeb4-314b-4850-a28b-0f53bca006d8
jinja: '{{verse_text}} What is the sentiment expressed in this poem? ||| {{ answer_choices
[label] }}
'
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Accuracy
original_task: true
name: positive_or_negative_sentiment_variation_1
reference: ''
aecb3d13-ff68-4e60-a382-87191940bd5b: !Template
answer_choices: negative ||| positive ||| neutral ||| mixed
id: aecb3d13-ff68-4e60-a382-87191940bd5b
jinja: '{{verse_text}} The most appropriate word out of {{"negative"}}, {{"positive"}},
{{"neutral"}} and {{"mixed"}}, which express the poet''s sentiment is: ||| {{
answer_choices [label] }}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: most_appropriate_sentiment
reference: ''
ca15cecb-4ee6-4445-a0f4-6ef5cd519923: !Template
answer_choices: negative ||| positive ||| neutral ||| mixed
id: ca15cecb-4ee6-4445-a0f4-6ef5cd519923
jinja: "{{verse_text}} What sentiment does this poem express? \nOptions: {{answer_choices[0]}},\
\ {{answer_choices[1]}}, {{answer_choices[2]}}, {{answer_choices[3]}}.\nAnswer:\
\ ||| {{ answer_choices [label] }}"
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: question_answer_format
reference: ''
f87a7ba0-11f7-41f9-bee6-94d0ad6e597a: !Template
answer_choices: negative ||| positive ||| neutral ||| mixed
id: f87a7ba0-11f7-41f9-bee6-94d0ad6e597a
jinja: Does this poem express a {{"negative"}}, {{"positive"}}, {{"neutral"}}
or {{"mixed"}} sentiment? {{verse_text}} ||| {{ answer_choices [label] }}
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: guess_sentiment_from_given_options
reference: ''
fdbebd3d-6517-4be1-8771-489e2de658ef: !Template
answer_choices: negative ||| positive ||| neutral ||| mixed
id: fdbebd3d-6517-4be1-8771-489e2de658ef
jinja: '{{verse_text}} What is the sentiment that the poet wants the readers to
feel through the verse mentioned above?||| {{ answer_choices [label] }}'
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Accuracy
original_task: true
name: guess_sentiment_without_options
reference: ''
src/prompts/templates/qasc/templates.yaml
+141
View File
@@ -0,0 +1,141 @@
dataset: qasc
templates:
3e1e6ca0-b95e-4e68-bb6a-cd47c8429658: !Template
answer_choices: Yes ||| No
id: 3e1e6ca0-b95e-4e68-bb6a-cd47c8429658
jinja: "If I tell you that {{combinedfact[0]|capitalize}}{{ combinedfact[1:]|trim('.')\
\ }}, and ask you the question \"{{ question[0]|lower }}{{ question[1:] }}\"\
, is the correct answer \"{{ choices.text[0][0]|lower}}{{ choices.text[0][1:]|trim('.')\
\ }}\"? \n\n||| \n\n{% if answerKey == choices.label[0] %} Yes {% else %} No\
\ {% endif %}"
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Accuracy
original_task: false
name: is_correct_1
reference: ''
40ef67db-dff3-4e7b-b167-f7e54b400c74: !Template
answer_choices: '{{choices.text | join("|||")}}'
id: 40ef67db-dff3-4e7b-b167-f7e54b400c74
jinja: "{{ fact1[0]|capitalize }}{{ fact1[1:]|trim|trim('.') }}, and {{fact2[0]|lower\
\ }}{{ fact2[1:]|trim|trim('.') }}. Given these facts, {{ question[0]|lower\
\ }}{{question[1:]|trim('?') }} among the following options:\n- {{answer_choices\
\ | join(\"\\n - \") }}\n\n||| \n\n{% for choice in choices.label %} {% if choice\
\ == answerKey %}{{ answer_choices[loop.index - 1] }}{% endif %}{% endfor %} "
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: qa_with_separated_facts_1
reference: Question Answering with separated facts.
5d63c186-e047-49dd-b5fd-c4a574f6f0e2: !Template
answer_choices: '{{choices.text | join("|||")}}'
id: 5d63c186-e047-49dd-b5fd-c4a574f6f0e2
jinja: "Fact 1: {{ fact1[0]|capitalize }}{{ fact1[1:]|trim|trim('.') }}.\n\nFact\
\ 2: {{fact2[0]|capitalize }}{{ fact2[1:]|trim|trim('.') }}.\n\nGiven the two\
\ facts above, {{ question[0]|lower }}{{question[1:]|trim('?') }}?\n\n||| \n\
\n{% for choice in choices.label %} {% if choice == answerKey %}{{ answer_choices[loop.index\
\ - 1] }}{% endif %}{% endfor %} "
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Accuracy
original_task: true
name: qa_with_separated_facts_3
reference: Question Answering with separated facts.
604dd379-21f1-4d30-af2f-22f9a8a97ceb: !Template
answer_choices: '{{choices.text | join("|||")}}'
id: 604dd379-21f1-4d30-af2f-22f9a8a97ceb
jinja: "You are presented with the question \"{{ question }}\" and the following\
\ answer choices: \n- {{answer_choices | join(\"\\n - \") }}\n\nNow knowing\
\ that {{ fact1[0]|lower }}{{ fact1[1:]|trim|trim('.') }} and {{fact2[0]|lower\
\ }}{{ fact2[1:]|trim|trim('.') }}, choose the best answer.\n\n||| \n\n{% for\
\ choice in choices.label %} {% if choice == answerKey %}{{ answer_choices[loop.index\
\ - 1] }}{% endif %}{% endfor %} "
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: qa_with_separated_facts_4
reference: Question Answering with separated facts.
67cb1b35-e5d4-490b-beb8-dc9c0be9c298: !Template
answer_choices: '{{choices.text | join("|||")}}'
id: 67cb1b35-e5d4-490b-beb8-dc9c0be9c298
jinja: "You are presented with the quiz \"{{ question }}\" \n\nBut you don't know\
\ the answer, so you turn to your teacher to ask for hints. He says that \"\
{{ fact1[0]|lower }}{{ fact1[1:]|trim|trim('.') }}\" and \"{{fact2[0]|lower\
\ }}{{ fact2[1:]|trim|trim('.') }}\". \n\nSo, what's the best answer to the\
\ question?\n\n||| \n\n{% for choice in choices.label %} {% if choice == answerKey\
\ %}{{ answer_choices[loop.index - 1] }}{% endif %}{% endfor %} "
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Accuracy
original_task: true
name: qa_with_separated_facts_5
reference: Question Answering with separated facts.
b5b61423-8655-408d-a8e6-81a5eaaac2aa: !Template
answer_choices: '{{choices.text | join("|||")}}'
id: b5b61423-8655-408d-a8e6-81a5eaaac2aa
jinja: "If {{ combinedfact[0]|lower }}{{ combinedfact[1:]|trim|trim('.') }}, then\
\ {{ question[0]|lower }}{{question[1:]|trim|trim('?') }}?\n\nAnswer choices:\n\
- {{answer_choices | join(\"\\n - \") }}\n||| \n\n{% for choice in choices.label\
\ %} {% if choice == answerKey %}{{ answer_choices[loop.index - 1] }}{% endif\
\ %}{% endfor %} "
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: false
name: qa_with_combined_facts_1
reference: ''
c251edaf-a063-40fe-bd49-038843fcdb23: !Template
answer_choices: Yes ||| No
id: c251edaf-a063-40fe-bd49-038843fcdb23
jinja: "Do you think the right answer to the question \"{{ question[0]|lower }}{{\
\ question[1:] }}\" is \"{{ choices.text[1][0]|lower}}{{ choices.text[1][1:]|trim('.')\
\ }}\", given that\n {{combinedfact[0]|lower}}{{ combinedfact[1:]|trim('.')\
\ }}?\n ||| \n{% if answerKey == choices.label[0] %} Yes {% else %} No {% endif\
\ %} "
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Accuracy
original_task: false
name: is_correct_2
reference: ''
c7cd51f1-ea49-4d6a-a422-46624333b7b1: !Template
answer_choices: '{{choices.text | join("|||")}}'
id: c7cd51f1-ea49-4d6a-a422-46624333b7b1
jinja: "Fact 1: {{ fact1[0]|capitalize }}{{ fact1[1:]|trim|trim('.') }}.\n\nFact\
\ 2: {{fact2[0]|capitalize }}{{ fact2[1:]|trim|trim('.') }}.\n\nGiven the two\
\ facts above, answer the question \"{{ question }}\" with the following options:\
\ \n- {{answer_choices | join(\"\\n - \") }}\n\n||| \n\n{% for choice in choices.label\
\ %} {% if choice == answerKey %}{{ answer_choices[loop.index - 1] }}{% endif\
\ %}{% endfor %} "
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: qa_with_separated_facts_2
reference: Question Answering with separated facts.
src/prompts/templates/reaganjlee/truthful_qa_mc/templates.yaml
+195
View File
@@ -0,0 +1,195 @@
dataset: truthful_qa_mc
binarize: true
templates:
00fa401f-3329-48fa-be4a-1b6725292ee6: !Template
answer_choices: A ||| B ||| C ||| D
id: 00fa401f-3329-48fa-be4a-1b6725292ee6
jinja: 'Answer the following question: {{ question }}
Choice 1: {{''. ''.join([answer_choices[label], choices[label]])}}. Choice 2: {{''. ''.join([answer_choices[1 - label], choices[1 - label]])}}. Between choice 1 and choice 2,
the answer of this question is ||| {{answer_choices[label]}}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics: []
original_task: true
name: burns_1
reference: Burns et al.
01fa401f-3329-48fa-be4a-1b6725292ee6: !Template
answer_choices: A ||| B ||| C ||| D
id: 01fa401f-3329-48fa-be4a-1b6725292ee6
jinja: 'Answer the following question: {{ question }}
Choice 1: {{''. ''.join([answer_choices[label], choices[label]])}}. Choice 2: {{''. ''.join([answer_choices[1 - label], choices[1 - label]])}}. Between choice 1 and choice 2,
what is the answer to this question? ||| {{answer_choices[label]}}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics: []
original_task: true
name: burns_2
reference: Burns et al.
02fa401f-3329-48fa-be4a-1b6725292ee6: !Template
answer_choices: A ||| B ||| C ||| D
id: 02fa401f-3329-48fa-be4a-1b6725292ee6
jinja: 'Answer the following question: {{ question }}
Which is the answer of this question, choice 1: {{''. ''.join([answer_choices[label], choices[label]])}}, or
choice 2: {{''. ''.join([answer_choices[1 - label], choices[1 - label]])}} |||
{{answer_choices[label]}}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics: []
original_task: true
name: burns_3
reference: Burns et al.
04fa401f-3329-48fa-be4a-1b6725292ee6: !Template
answer_choices: A ||| B ||| C ||| D
id: 04fa401f-3329-48fa-be4a-1b6725292ee6
jinja: '{{ question }}
Choice 1: {{''. ''.join([answer_choices[label], choices[label]])}}. Choice 2: {{''. ''.join([answer_choices[1 - label], choices[1 - label]])}}.Between choice 1 and choice 2,
the topic of this example is ||| {{answer_choices[label]}}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics: []
original_task: true
name: burns_5
reference: Burns et al.
05fa401f-3329-48fa-be4a-1b6725292ee6: !Template
answer_choices: A ||| B ||| C ||| D
id: 05fa401f-3329-48fa-be4a-1b6725292ee6
jinja: '{{ question }}
Which is the answer to this question, choice 1: {{''. ''.join([answer_choices[label], choices[label]])}}, or
choice 2: {{''. ''.join([answer_choices[1 - label], choices[1 - label]])}} |||
{{answer_choices[label]}}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics: []
original_task: true
name: burns_6
reference: Burns et al.
06fa401f-3329-48fa-be4a-1b6725292ee6: !Template
answer_choices: A ||| B ||| C ||| D
id: 06fa401f-3329-48fa-be4a-1b6725292ee6
jinja: '{{ question }}
What is the answer to the question, choice 1: {{''. ''.join([answer_choices[label], choices[label]])}},
or choice 2: {{''. ''.join([answer_choices[1 - label], choices[1 - label]])}} |||
{{answer_choices[label]}}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics: []
original_task: true
name: burns_7
reference: Burns et al.
07fa401f-3329-48fa-be4a-1b6725292ee6: !Template
answer_choices: A ||| B ||| C ||| D
id: 07fa401f-3329-48fa-be4a-1b6725292ee6
jinja: '{{ question }}
What choice best answers this question, choice 1: {{''. ''.join([answer_choices[label], choices[label]])}},
or choice 2: {{''. ''.join([answer_choices[1 - label], choices[1 - label]])}} |||
{{answer_choices[label]}}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics: []
original_task: true
name: burns_8
reference: Burns et al.
824ecb55-ecad-40c2-8033-f2fa0add2ddf: !Template
answer_choices: A ||| B ||| C ||| D
id: 824ecb55-ecad-40c2-8033-f2fa0add2ddf
jinja: '{{ question }} Given a list of choices:
{% for k in range(choices | length) %}
{{''. ''.join([answer_choices[k], choices[k]])}}
{% endfor %}
what choice best answers the question? ||| {{ answer_choices[label] }}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: given_list_what_category_does_the_paragraph_belong_to
reference: ''
8eda7e71-6734-486f-b883-e99d3f14c0bf: !Template
answer_choices: A ||| B ||| C ||| D
id: 8eda7e71-6734-486f-b883-e99d3f14c0bf
jinja: 'Pick one answer for the following question. The options are -
{% for k in range(choices | length) %}
{{''. ''.join([answer_choices[k], choices[k]])}}
{% endfor %}.
{{ question }} |||
{{ answer_choices[label] }}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: pick_one_category_for_the_following_text
reference: ''
9dfa5d15-96bc-41ee-ad89-4f8df5c4ff67: !Template
answer_choices: A ||| B ||| C ||| D
id: 9dfa5d15-96bc-41ee-ad89-4f8df5c4ff67
jinja: '{{ question }} Given a choice of answers
{% for k in range(choices | length) %}
{{''. ''.join([answer_choices[k], choices[k]])}}
{% endfor %}, the question
is best answered by which one? ||| {{ answer_choices[label] }}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: 'given_a_choice_of_categories '
reference: ''
f72fa410-3278-4f62-91f0-f9edf4a4e792: !Template
answer_choices: A ||| B ||| C ||| D
id: f72fa410-3278-4f62-91f0-f9edf4a4e792
jinja: '"{{ question }}", given a list of choices:
{% for k in range(choices | length) %}
{{''. ''.join([answer_choices[k], choices[k]])}}
{% endfor %}, what answer belongs
to the question? ||| {{ answer_choices[label] }}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: given_a_list_of_category_what_does_the_title_belong_to
reference: ''
src/prompts/templates/super_glue/axb/templates.yaml
+145
View File
@@ -0,0 +1,145 @@
dataset: super_glue
subset: axb
templates:
1ae41916-7b4d-4ef3-b414-bfadd95d67e2: !Template
answer_choices: Yes ||| No
id: 1ae41916-7b4d-4ef3-b414-bfadd95d67e2
jinja: 'Given {{sentence1}} Should we assume that "{{sentence2}}" is true? Yes
or no? ||| {{ answer_choices[label] }} '
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: should assume
reference: Webson & Pavlick 2021
1b2d6e85-a5a9-4d1b-9e3b-630b490c6a34: !Template
answer_choices: Yes ||| No
id: 1b2d6e85-a5a9-4d1b-9e3b-630b490c6a34
jinja: '{{sentence1}} Are we justified in saying that "{{sentence2}}"? Yes or
no? ||| {{ answer_choices[label] }} '
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: justified in saying
reference: Webson & Pavlick 2021
23651f68-93cc-441f-b826-30dd2c6d6a93: !Template
answer_choices: Yes ||| No
id: 23651f68-93cc-441f-b826-30dd2c6d6a93
jinja: Given that {{sentence1}} Does it follow that {{sentence2}} Yes or no? |||
{{ answer_choices[label] }}
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: does it follow that
reference: Sanh et al. 2021
552d6c20-ab5b-462f-b5fb-3c7b80c78dcc: !Template
answer_choices: Yes ||| No
id: 552d6c20-ab5b-462f-b5fb-3c7b80c78dcc
jinja: '{{sentence1}} Using only the above description and what you know about
the world, is "{{sentence2}}" definitely correct? Yes or no? ||| {{ answer_choices[label]
}}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: MNLI crowdsource
reference: Adapted from Williams et al. 2018's instructions to crowdsourcing workers.
908be561-caf4-4416-9fe9-9919c3998681: !Template
answer_choices: Yes ||| No
id: 908be561-caf4-4416-9fe9-9919c3998681
jinja: 'Given {{sentence1}} Is it guaranteed true that "{{sentence2}}"? Yes or
no? ||| {{ answer_choices[label] }} '
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: guaranteed true
reference: Webson & Pavlick 2021
bae54ef5-c3be-4862-bdd4-a559ed04eb31: !Template
answer_choices: Yes ||| No
id: bae54ef5-c3be-4862-bdd4-a559ed04eb31
jinja: 'Suppose {{sentence1}} Can we infer that "{{sentence2}}"? Yes or no? |||
{{ answer_choices[label] }} '
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: can we infer
reference: Webson & Pavlick 2021
c92d765f-83b1-4684-a0a3-580929b5e46b: !Template
answer_choices: Yes ||| No
id: c92d765f-83b1-4684-a0a3-580929b5e46b
jinja: "{{sentence1}} \n\nQuestion: Does this imply that \"{{sentence2}}\"? Yes\
\ or no? ||| {{answer_choices[label]}}"
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: does this imply
reference: Sanh et al. 2021
cb68ee27-c0a3-440b-b595-e90fe89539c3: !Template
answer_choices: Yes ||| No
id: cb68ee27-c0a3-440b-b595-e90fe89539c3
jinja: 'Given that {{sentence1}} Therefore, it must be true that "{{sentence2}}"?
Yes or no? ||| {{ answer_choices[label] }} '
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: must be true
reference: Sanh et al. 2021
d57550ef-2f67-46eb-98cb-432dd135be16: !Template
answer_choices: Yes ||| No
id: d57550ef-2f67-46eb-98cb-432dd135be16
jinja: '{{sentence1}} Based on the previous passage, is it true that "{{sentence2}}"?
Yes or no? ||| {{ answer_choices[label] }}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: based on the previous passage
reference: "Adapted from the BoolQ prompts in Schick & Sch\xFCtze 2021."
d965164b-fa96-41b5-8852-e0f6dfe5524e: !Template
answer_choices: True ||| False
id: d965164b-fa96-41b5-8852-e0f6dfe5524e
jinja: '{{sentence1}}
Question: {{sentence2}} True or False? ||| {{ answer_choices[label] }}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: GPT-3 style
reference: Same as reported in Figure G31 of the GPT-3 paper.
src/prompts/templates/super_glue/axg/templates.yaml
+145
View File
@@ -0,0 +1,145 @@
dataset: super_glue
subset: axg
templates:
0f530aa8-b254-4687-8032-bab1a65610c0: !Template
answer_choices: Yes ||| No
id: 0f530aa8-b254-4687-8032-bab1a65610c0
jinja: 'Given {{premise}} Should we assume that "{{hypothesis}}" is true? Yes
or no? ||| {{ answer_choices[label] }} '
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: should assume
reference: Webson & Pavlick 2021
0f8afaef-19a0-472f-9e9f-c803426f8f22: !Template
answer_choices: Yes ||| No
id: 0f8afaef-19a0-472f-9e9f-c803426f8f22
jinja: "{{premise}} \n\nQuestion: Does this imply that \"{{hypothesis}}\"? Yes\
\ or no? ||| {{answer_choices[label]}}"
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: does this imply
reference: Sanh et al. 2021
3b7a57e0-7733-4b21-9bed-a381fdc2415f: !Template
answer_choices: Yes ||| No
id: 3b7a57e0-7733-4b21-9bed-a381fdc2415f
jinja: '{{premise}} Based on the previous passage, is it true that "{{hypothesis}}"?
Yes or no? ||| {{ answer_choices[label] }}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: based on the previous passage
reference: "Adapted from the BoolQ prompts in Schick & Sch\xFCtze 2021."
4361cf07-1b58-478f-b97c-3b140832fb77: !Template
answer_choices: Yes ||| No
id: 4361cf07-1b58-478f-b97c-3b140832fb77
jinja: 'Given that {{premise}} Therefore, it must be true that "{{hypothesis}}"?
Yes or no? ||| {{ answer_choices[label] }} '
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: must be true
reference: Sanh et al. 2021
626823f5-ff12-46d5-9e68-b2dc4bfe7cd4: !Template
answer_choices: True ||| False
id: 626823f5-ff12-46d5-9e68-b2dc4bfe7cd4
jinja: '{{premise}}
Question: {{hypothesis}} True or False? ||| {{ answer_choices[label] }}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: GPT-3 style
reference: Same as reported in Figure G31 of the GPT-3 paper.
7e1439f6-d54d-43e6-bdc7-306ad5fd9203: !Template
answer_choices: Yes ||| No
id: 7e1439f6-d54d-43e6-bdc7-306ad5fd9203
jinja: 'Given {{premise}} Is it guaranteed true that "{{hypothesis}}"? Yes or
no? ||| {{ answer_choices[label] }} '
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: guaranteed true
reference: Webson & Pavlick 2021
c008c778-7621-496e-baa3-7b5817400659: !Template
answer_choices: Yes ||| No
id: c008c778-7621-496e-baa3-7b5817400659
jinja: Given that {{premise}} Does it follow that {{hypothesis}} Yes or no? |||
{{ answer_choices[label] }}
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: does it follow that
reference: Sanh et al. 2021
d4a1dd92-e184-4843-bc1f-1f625c833249: !Template
answer_choices: Yes ||| No
id: d4a1dd92-e184-4843-bc1f-1f625c833249
jinja: '{{premise}} Are we justified in saying that "{{hypothesis}}"? Yes or no?
||| {{ answer_choices[label] }} '
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: justified in saying
reference: Webson & Pavlick 2021
db13469f-7161-4670-8a59-8c1137d1fa8b: !Template
answer_choices: Yes ||| No
id: db13469f-7161-4670-8a59-8c1137d1fa8b
jinja: 'Suppose {{premise}} Can we infer that "{{hypothesis}}"? Yes or no? |||
{{ answer_choices[label] }} '
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: can we infer
reference: Webson & Pavlick 2021
e21f5367-0cc8-412d-b8d9-78548438a384: !Template
answer_choices: Yes ||| No
id: e21f5367-0cc8-412d-b8d9-78548438a384
jinja: '{{premise}} Using only the above description and what you know about the
world, is "{{hypothesis}}" definitely correct? Yes or no? ||| {{ answer_choices[label]
}}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: MNLI crowdsource
reference: Adapted from Williams et al. 2018's instructions to crowdsourcing workers.
src/prompts/templates/super_glue/boolq/templates.yaml
+178
View File
@@ -0,0 +1,178 @@
dataset: super_glue
subset: boolq
templates:
3e386463-1715-4578-9cba-07d11a0d3b61: !Template
answer_choices: False ||| True
id: 3e386463-1715-4578-9cba-07d11a0d3b61
jinja: 'Passage: {{passage}}
After reading this passage, I have a question: {{question}}? True or False?
|||
{% if label != -1 %}
{{answer_choices[label]}}
{% endif %}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: after_reading
reference: ''
492f0f88-4370-46cd-839b-1de37a55aeda: !Template
answer_choices: No ||| Yes
id: 492f0f88-4370-46cd-839b-1de37a55aeda
jinja: "{{ passage }} \nQuestion: {{ question }}\nAnswer: ||| \n{% if label !=\
\ -1 %}\n{{ answer_choices[label] }}{% endif %}"
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Accuracy
original_task: true
name: GPT-3 Style
reference: Same as Figure G29, p. 58 of the GPT-3 paper
6cb6a026-c070-470a-b75d-bb8fdf424e35: !Template
answer_choices: No ||| Yes
id: 6cb6a026-c070-470a-b75d-bb8fdf424e35
jinja: "{{ passage }} \n\nHaving read that, I wonder {{ question }}? |||\n{% if\
\ label != -1 %}\n{{ answer_choices[label] }}{% endif %}"
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Accuracy
original_task: true
name: "I wonder\u2026"
reference: ''
7cf7acdf-e3a2-459f-a3e8-2e2d27dd6aa5: !Template
answer_choices: No ||| Yes
id: 7cf7acdf-e3a2-459f-a3e8-2e2d27dd6aa5
jinja: 'Text: {{passage}}
Answer the following yes/no question: {{question}}? Yes or no? |||
{% if label != -1 %}
{{answer_choices[label]}}
{% endif %}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: yes_no_question
reference: ''
7d21d974-0624-4d4f-9e8c-644e2d009cb5: !Template
answer_choices: No ||| Yes
id: 7d21d974-0624-4d4f-9e8c-644e2d009cb5
jinja: "{{ passage }} \n\nHaving read that, could you tell me {{ question }}?\
\ ||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %}"
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Accuracy
original_task: true
name: "could you tell me\u2026"
reference: ''
922d3e87-ac58-4731-84d1-f0a40e47afb5: !Template
answer_choices: No ||| Yes
id: 922d3e87-ac58-4731-84d1-f0a40e47afb5
jinja: "EXAM\n1. Answer by yes or no.\n\nDocument: {{passage}}\nQuestion: {{question}}?\
\ ||| \n{% if label != -1 %}\n{{answer_choices[label]}}{% endif %}"
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: exam
reference: ''
9a1bf459-8047-437c-9def-f21e960429cc: !Template
answer_choices: No ||| Yes
id: 9a1bf459-8047-437c-9def-f21e960429cc
jinja: "Based on the following passage, {{ question }}?\n\n{{ passage }}\
\ ||| \n\n{% if label != -1 %}{{ answer_choices[label] }}{% endif %}"
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Accuracy
original_task: true
name: based on the following passage
reference: "Adapted from Perez et al. 2021 and Schick & Sch\xFCtz 2021."
9f4c6b0a-437b-40c0-b467-db4b7218d38d: !Template
answer_choices: False ||| True
id: 9f4c6b0a-437b-40c0-b467-db4b7218d38d
jinja: 'Exercise: read the text and answer the question by True or False.
Text: {{passage}}
Question: {{question}}? |||
{% if label != -1 %}
{{answer_choices[label]}}
{% endif %}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: exercise
reference: ''
b2b3cb60-d6e3-491c-a09a-8201e13e417e: !Template
answer_choices: No ||| Yes
id: b2b3cb60-d6e3-491c-a09a-8201e13e417e
jinja: '{{ passage }}
Based on the previous passage, {{ question }}? |||
{% if label != -1 %}
{{ answer_choices[label]}}
{% endif %}'
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Accuracy
original_task: true
name: based on the previous passage
reference: "Adapted from Perez et al. 2021 and Schick & Sch\xFCtz 2021."
eb78772c-e81e-4b8a-a77b-b75efd1c212a: !Template
answer_choices: False ||| True
id: eb78772c-e81e-4b8a-a77b-b75efd1c212a
jinja: '{{passage}}
Q: {{question}}? True or False? |||
{% if label != -1 %}
{{answer_choices[label]}}
{% endif %}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: valid_binary
reference: ''
src/prompts/templates/super_glue/cb/templates.yaml
+226
View File
@@ -0,0 +1,226 @@
dataset: super_glue
subset: cb
templates:
2e76cd0f-68ca-4f03-83ed-11cf15b25a84: !Template
answer_choices: Yes ||| No ||| Maybe
id: 2e76cd0f-68ca-4f03-83ed-11cf15b25a84
jinja: 'Suppose {{premise}} Can we infer that "{{hypothesis}}"? Yes, no, or maybe?
||| {% if label !=-1 %}{{ answer_choices[label] }}{% endif %} '
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: can we infer
reference: Webson & Pavlick 2021
358860fd-61ad-45fd-92a6-a72ca9107ebc: !Template
answer_choices: Yes ||| No ||| Maybe
id: 358860fd-61ad-45fd-92a6-a72ca9107ebc
jinja: '{{premise}} Based on the previous passage, is it true that "{{hypothesis}}"?
Yes, no, or maybe? ||| {% if label !=-1 %}{{ answer_choices[label] }}{% endif
%}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: based on the previous passage
reference: "Adapted from the BoolQ prompts in Schick & Sch\xFCtze 2021."
3f43a599-ffdb-490e-8952-c0ce41dd4621: !Template
answer_choices: True ||| False ||| Inconclusive
id: 3f43a599-ffdb-490e-8952-c0ce41dd4621
jinja: '{{premise}} Based on that information, is the claim: "{{hypothesis}}"
{{"true"}}, {{"false"}}, or {{"inconclusive"}}? ||| {% if label !=-1 %}{{ answer_choices[label]
}}{% endif %}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: claim true/false/inconclusive
reference: Sanh et al. 2021
404eed25-558a-4d39-9515-7de46d60d4e0: !Template
answer_choices: Yes ||| No ||| Maybe
id: 404eed25-558a-4d39-9515-7de46d60d4e0
jinja: Given that {{premise}} Does it follow that {{hypothesis}} Yes, no, or maybe?
||| {% if label !=-1 %}{{ answer_choices[label] }}{% endif %}
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: does it follow that
reference: Sanh et al. 2021
5c9b1fa9-93f0-4f82-b9e3-e0967e4d7260: !Template
answer_choices: Yes ||| No ||| Maybe
id: 5c9b1fa9-93f0-4f82-b9e3-e0967e4d7260
jinja: '{{premise}} Are we justified in saying that "{{hypothesis}}"? Yes, no,
or maybe? ||| {% if label !=-1 %}{{ answer_choices[label] }}{% endif %}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: justified in saying
reference: Webson & Pavlick 2021
6b0c6191-183d-4731-8050-ab17c909335c: !Template
answer_choices: Always ||| Never ||| Sometimes
id: 6b0c6191-183d-4731-8050-ab17c909335c
jinja: Suppose it's true that {{premise}} Then, is "{{hypothesis}}" {{"always"}},
{{"sometimes"}}, or {{"never"}} true? ||| {% if label !=-1 %}{{ answer_choices[label]
}}{% endif %}
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: always/sometimes/never
reference: Sanh et al. 2021
75db2bc2-3caa-4956-9653-13c7dd6255df: !Template
answer_choices: True ||| False ||| Neither
id: 75db2bc2-3caa-4956-9653-13c7dd6255df
jinja: '{{premise}}
Question: {{hypothesis}} True, False, or Neither? ||| {% if label !=-1 %}{{
answer_choices[label] }}{% endif %}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: GPT-3 style
reference: 'Same as reported in Figure G7 of the GPT-3 paper, except that there
is no task identifying tokens like "anli R1: ".'
87237a07-7cce-470a-80ac-3e5e3a5283ba: !Template
answer_choices: Always ||| Never ||| Sometimes
id: 87237a07-7cce-470a-80ac-3e5e3a5283ba
jinja: "{{premise}} \n\nKeeping in mind the above text, consider: {{hypothesis}}\
\ Is this {{\"always\"}}, {{\"sometimes\"}}, or {{\"never\"}} correct? ||| {%\
\ if label !=-1 %}{{ answer_choices[label] }}{% endif %}"
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: consider always/sometimes/never
reference: Sanh et al. 2021
8798b8a4-1f59-4c72-9c1b-3e3044a7462a: !Template
answer_choices: Yes ||| No ||| Maybe
id: 8798b8a4-1f59-4c72-9c1b-3e3044a7462a
jinja: Given {{premise}} Is it guaranteed true that "{{hypothesis}}"? Yes, no,
or maybe? ||| {% if label !=-1 %}{{ answer_choices[label] }}{% endif %}
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: guaranteed true
reference: Webson & Pavlick 2021
8e3b8d3d-1362-47dc-922a-82c03f965989: !Template
answer_choices: Yes ||| No ||| Maybe
id: 8e3b8d3d-1362-47dc-922a-82c03f965989
jinja: Given that {{premise}} Therefore, it must be true that "{{hypothesis}}"?
Yes, no, or maybe? ||| {% if label !=-1 %}{{ answer_choices[label] }}{% endif
%}
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: must be true
reference: Sanh et al. 2021
90ab1002-093c-4e54-b48f-626655e36b65: !Template
answer_choices: Guaranteed ||| Impossible ||| Possible
id: 90ab1002-093c-4e54-b48f-626655e36b65
jinja: "Assume it is true that {{premise}} \n\nTherefore, \"{{hypothesis}}\" is\
\ {{\"guaranteed\"}}, {{\"possible\"}}, or {{\"impossible\"}}? ||| {% if label\
\ !=-1 %}{{ answer_choices[label] }}{% endif %}"
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: guaranteed/possible/impossible
reference: Sanh et al. 2021
a485d120-6eef-4ff6-8684-42df1639b101: !Template
answer_choices: Yes ||| No ||| Maybe
id: a485d120-6eef-4ff6-8684-42df1639b101
jinja: "{{premise}} \n\nQuestion: Does this imply that \"{{hypothesis}}\"? Yes,\
\ no, or maybe? ||| {% if label !=-1 %}{{answer_choices[label]}}{% endif %}"
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: does this imply
reference: Sanh et al. 2021
bee62bfa-5307-4e1c-97b2-2ad2f7bcb179: !Template
answer_choices: Correct ||| Incorrect ||| Inconclusive
id: bee62bfa-5307-4e1c-97b2-2ad2f7bcb179
jinja: '{{premise}} Using only the above description and what you know about the
world, "{{hypothesis}}" is definitely correct, incorrect, or inconclusive? |||
{% if label !=-1 %}{{ answer_choices[label] }}{% endif %}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: MNLI crowdsource
reference: Adapted from Williams et al. 2018's instructions to crowdsourcing workers.
e503b148-8e6c-43b5-9ed6-312794c54d9b: !Template
answer_choices: Yes ||| No ||| Maybe
id: e503b148-8e6c-43b5-9ed6-312794c54d9b
jinja: Given {{premise}} Should we assume that "{{hypothesis}}" is true? Yes,
no, or maybe? ||| {% if label !=-1 %}{{ answer_choices[label] }}{% endif %}
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: should assume
reference: Webson & Pavlick 2021
ea56b7f3-6e07-45bc-b619-c527eac4a41b: !Template
answer_choices: True ||| False ||| Inconclusive
id: ea56b7f3-6e07-45bc-b619-c527eac4a41b
jinja: 'Take the following as truth: {{premise}}
Then the following statement: "{{hypothesis}}" is {{"true"}}, {{"false"}}, or
{{"inconclusive"}}? ||| {% if label !=-1 %}{{ answer_choices[label] }}{% endif
%}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: take the following as truth
reference: Sanh et al. 2021
src/prompts/templates/super_glue/copa/templates.yaml
+244
View File
@@ -0,0 +1,244 @@
dataset: super_glue
subset: copa
templates:
0edd8660-f299-4819-a5ac-633c11177228: !Template
answer_choices: '{{choice1}} ||| {{choice2}}'
id: 0edd8660-f299-4819-a5ac-633c11177228
jinja: 'Exercise: choose the most plausible alternative.
{{ premise }} {% if question == "cause" %} because... {% else %} so... {% endif
%}
- {{choice1}}
- {{choice2}} |||
{% if label != -1 %}{{ answer_choices[label] }}{%endif%}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: exercise
reference: ''
# 150789fe-e309-47a1-82c9-0a4dc2c6b12b: !Template
# answer_choices: '{{choice1}} ||| {{choice2}}'
# id: 150789fe-e309-47a1-82c9-0a4dc2c6b12b
# jinja: "{% if question == \"effect\" %} \n{{ premise }} What could happen next,\
# \ \"{{ answer_choices[0] }}\" or \"{{ answer_choices[1] }}\"? ||| {% if label\
# \ != -1 %}{{ answer_choices[label] }}{%endif%}\n{% endif %}"
# metadata: !TemplateMetadata
# choices_in_prompt: true
# languages:
# - en
# metrics:
# - Accuracy
# original_task: true
# name: "\u2026What could happen next, C1 or C2?"
# reference: ''
4d879cbe-2fd7-424a-9d78-3f5200313fba: !Template
answer_choices: '{{choice1}} ||| {{choice2}}'
id: 4d879cbe-2fd7-424a-9d78-3f5200313fba
jinja: "{{ premise }} \n\nI am hesitating between two options. Help me choose\
\ the more likely{% if question == \"cause\" %} cause: {% else %} effect: {%\
\ endif %}\n- {{choice1}}\n- {{choice2}} ||| \n\n{% if label != -1 %}{{ answer_choices[label]\
\ }}{%endif%}"
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: i_am_hesitating
reference: ''
5c5f5813-0443-4944-b77a-c0d08885c16b: !Template
answer_choices: '{{choice1}} ||| {{choice2}}'
id: 5c5f5813-0443-4944-b77a-c0d08885c16b
jinja: 'Consider the following premise: '''''' {{premise}} '''''' Choice 1: {{answer_choices[0]}}
Choice 2: {{answer_choices[1]}}
Q: Which one is more likely to be the {{question}}, choice 1 or choice 2?
A:|||{{answer_choices[label]}}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics: []
original_task: true
name: choose_between
reference: ''
66ea075e-4d03-4a78-b1fa-9a5228cf0c9d: !Template
answer_choices: '{{choice1}} ||| {{choice2}}'
id: 66ea075e-4d03-4a78-b1fa-9a5228cf0c9d
jinja: '{{ premise }} {% if question == "cause" %} This happened because... {%
else %} As a consequence... {% endif %}
Help me pick the more plausible option:
- {{choice1}}
- {{choice2}}
||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: plausible_alternatives
reference: ''
744047dc-1298-45a2-8d68-d67e3f834ded: !Template
answer_choices: '{{choice1 }} ||| {{choice2}}'
id: 744047dc-1298-45a2-8d68-d67e3f834ded
jinja: '"{{ answer_choices[0] }}" or "{{ answer_choices[1] }}"? {{ premise }}
{% if question == "cause" %} because {% else %} so {% endif %} ||| {% if label
!= -1 %}{{ answer_choices[label] }}{% endif %}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: "C1 or C2? premise, so/because\u2026"
reference: "Adapted from Perez et al. 2021 and Schick & Sch\xFCtz 2021."
# 84da62c2-9440-4cfc-bdd4-d70c65e33a82: !Template
# answer_choices: '{{choice1}} ||| {{choice2}}'
# id: 84da62c2-9440-4cfc-bdd4-d70c65e33a82
# jinja: "{% if question == \"effect\" %} \n{{ premise }} As a result, \"{{ answer_choices[0]\
# \ }}\" or \"{{ answer_choices[1] }}\"? ||| {% if label != -1 %}{{ answer_choices[label]\
# \ }}{%endif%}\n{% endif %}"
# metadata: !TemplateMetadata
# choices_in_prompt: true
# languages:
# - en
# metrics:
# - Accuracy
# original_task: true
# name: "\u2026As a result, C1 or C2?"
# reference: ''
8ce80f8a-239e-4393-892c-f63dbb0d9929: !Template
answer_choices: '{{choice1}} ||| {{choice2}}'
id: 8ce80f8a-239e-4393-892c-f63dbb0d9929
jinja: "{{ premise }} \n\nWhat's the best option?\n- {{choice1}}\n- {{choice2}}\n\
\nWe are looking for{% if question == \"cause\" %} a cause{% else %} an effect{% endif %}.\n\nA:
\ ||| {% if label != -1 %}{{answer_choices[label]}}{%endif%}"
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: best_option
reference: ''
# 8cf2ba73-aee5-4651-b5d4-b1b88afe4abb: !Template
# answer_choices: '{{choice1}} ||| {{choice2}}'
# id: 8cf2ba73-aee5-4651-b5d4-b1b88afe4abb
# jinja: "{% if question == \"cause\" %} \n{{ premise }} Which may be caused by\
# \ \"{{ answer_choices[0] }}\" or \"{{ answer_choices[1] }}\"? ||| {% if label\
# \ != -1 %}{{ answer_choices[label] }}{%endif%}\n{% endif %}"
# metadata: !TemplateMetadata
# choices_in_prompt: true
# languages:
# - en
# metrics:
# - Accuracy
# original_task: true
# name: "\u2026which may be caused by"
# reference: ''
a1f9951e-2b6b-4530-9636-9cdf4c1658c5: !Template
answer_choices: '{{choice1}} ||| {{choice2}}'
id: a1f9951e-2b6b-4530-9636-9cdf4c1658c5
jinja: 'Pick the more likely continuation to the following sentence:
{{ premise }} {% if question == "cause" %} as a result of: {% else %} as a consequence:
{% endif %}
- {{choice1}}
- {{choice2}} |||
{% if label != -1 %}{{ answer_choices[label] }}{%endif%}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: more likely
reference: ''
a61d8c21-da25-47bf-b5fe-14a8edd650af: !Template
answer_choices: '{{choice1}} ||| {{choice2}}'
id: a61d8c21-da25-47bf-b5fe-14a8edd650af
jinja: '{{ premise }}
Select the most plausible {% if question == "cause" %} cause: {% else %} effect:
{% endif %}
- {{choice1}}
- {{choice2}} |||
{% if label != -1 %}{{ answer_choices[label] }}{%endif%}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: cause_effect
reference: ''
# a8bf11c3-bea2-45ba-a533-957d8bee5e2e: !Template
# answer_choices: '{{choice1}} ||| {{choice2}}'
# id: a8bf11c3-bea2-45ba-a533-957d8bee5e2e
# jinja: "{% if question == \"cause\" %} \n{{ premise }} Why? \"{{ answer_choices[0]\
# \ }}\" or \"{{ answer_choices[1] }}\"? ||| {% if label != -1 %}{{ answer_choices[label]\
# \ }}{%endif%}\n{% endif %}"
# metadata: !TemplateMetadata
# choices_in_prompt: true
# languages:
# - en
# metrics:
# - Accuracy
# original_task: true
# name: "\u2026why? C1 or C2"
# reference: ''
f32348cd-d3cb-4619-87b9-e24f99c78567: !Template
answer_choices: '{{choice1}} ||| {{choice2}}'
id: f32348cd-d3cb-4619-87b9-e24f99c78567
jinja: '{{ premise }} {% if question == "cause" %} because... {% else %} so...
{% endif %}
Choose between:
- {{choice1}}
- {{choice2}} |||
{% if label != -1 %}{{ answer_choices[label] }}{%endif%}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: choose
reference: ''
src/prompts/templates/super_glue/multirc/templates.yaml
+185
View File
@@ -0,0 +1,185 @@
dataset: super_glue
subset: multirc
templates:
2d95962b-a545-41ae-8d76-07ee6704ef65: !Template
answer_choices: No ||| Yes
id: 2d95962b-a545-41ae-8d76-07ee6704ef65
jinja: '{{paragraph}}
Question: {{question}}
I found this answer "{{answer}}". Is that correct? Yes or no?
|||
{% if label != -1 %}{{answer_choices[label]}}{% endif %}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: found_this_answer
reference: ''
42d47df9-09de-4691-8e49-7cfadd636cdd: !Template
answer_choices: No ||| Yes
id: 42d47df9-09de-4691-8e49-7cfadd636cdd
jinja: "{{ paragraph }}\nBased on the previous passage, {{ question }} \nIs \"\
{{ answer }}\" a correct answer? ||| {% if label != -1 %}{{ answer_choices[label]\
\ }}{% endif %}"
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Accuracy
original_task: true
name: "is\u2026 a correct answer?"
reference: "Adapted from Perez et al. 2021 and Schick & Sch\xFCtz 2021."
431a5c97-af33-4053-83c8-afb0dfc04448: !Template
answer_choices: No ||| Yes
id: 431a5c97-af33-4053-83c8-afb0dfc04448
jinja: '{{paragraph}}
Question: {{question}}
I am grading my students'' exercises. Is the answer "{{answer}}" correct?
|||
{% if label != -1 %}{{answer_choices[label]}}{% endif %}'
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Accuracy
original_task: true
name: grading
reference: ''
4fc9e1ea-7451-4dba-a2cb-ce870e35ef8b: !Template
answer_choices: No ||| Yes
id: 4fc9e1ea-7451-4dba-a2cb-ce870e35ef8b
jinja: "{{ paragraph }}\n{{ question }} \nWould it be good to answer \"{{ answer\
\ }}\"? ||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %}"
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Accuracy
original_task: true
name: "Would it be good to answer\u2026"
reference: ''
59a2d847-27f3-4002-a125-cf9a291b3098: !Template
answer_choices: No ||| Yes
id: 59a2d847-27f3-4002-a125-cf9a291b3098
jinja: "{{ paragraph }}\nQuestion: {{ question }} \nIs it {{ answer }}? ||| {%\
\ if label != -1 %}{{ answer_choices[label] }}{% endif %}"
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Accuracy
original_task: true
name: "paragraph\u2026 question\u2026 is it\u2026 ?"
reference: "Adapted from Perez et al. 2021 and Schick & Sch\xFCtz 2021."
7bf537ea-ff8d-44c7-8fc9-305b35e3be66: !Template
answer_choices: No ||| Yes
id: 7bf537ea-ff8d-44c7-8fc9-305b35e3be66
jinja: '{{paragraph}}
Decide whether "{{answer}}" is a valid answer to the following question: {{question}}
Answer yes or no.
|||
{% if label != -1 %}{{answer_choices[label]}}{% endif %}'
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Accuracy
original_task: true
name: decide_valid
reference: ''
7d878b89-2774-429a-82fb-ac801379e3ae: !Template
answer_choices: No ||| Yes
id: 7d878b89-2774-429a-82fb-ac801379e3ae
jinja: "{{ paragraph }}\nQuestion: {{ question }} \nIs the correct answer {{ answer\
\ }}? ||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %}"
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Accuracy
original_task: true
name: "is the correct answer\u2026"
reference: "Adapted from Perez et al. 2021 and Schick & Sch\xFCtz 2021."
ae9b2b0b-1731-4370-adcc-36c4a959490d: !Template
answer_choices: No ||| Yes
id: ae9b2b0b-1731-4370-adcc-36c4a959490d
jinja: 'Is "{{answer}}" a correct answer to the following question?
Question: {{question}}
Rely on the following text: {{paragraph}}
|||
{% if label != -1 %}{{answer_choices[label]}}{% endif %}'
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Accuracy
original_task: true
name: correct
reference: ''
b63fd1c3-b4a6-43c3-8429-6a389235b2a4: !Template
answer_choices: No ||| Yes
id: b63fd1c3-b4a6-43c3-8429-6a389235b2a4
jinja: '{{paragraph}}
Question: {{question}}
I think "{{answer}}" is a valid answer. Could you confirm? Yes or no?
|||
{% if label != -1 %}{{answer_choices[label]}}{% endif %}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: confirm
reference: ''
d2d78b88-8845-45b5-935a-6451da00b285: !Template
answer_choices: No ||| Yes
id: d2d78b88-8845-45b5-935a-6451da00b285
jinja: "{{ paragraph }}\n{{ question }} \nI was going to say \"{{ answer }}\"\
. Does that sound right? ||| {% if label != -1 %}{{ answer_choices[label] }}{%\
\ endif %}"
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Accuracy
original_task: true
name: "I was going to say\u2026"
reference: ''
src/prompts/templates/super_glue/record/templates.yaml
+369
View File
@@ -0,0 +1,369 @@
dataset: super_glue
subset: record
templates:
014b669e-2e3b-40ce-bdde-418966c7d666: !Template
answer_choices: '{{ entities | join("|||") }}'
id: 014b669e-2e3b-40ce-bdde-418966c7d666
jinja: "{{ passage }} \n{{ query }} \nWhich one is the \"{{\"@placeholder\"}}\"\
? {{ entities | join(\", \") }}? ||| {% if ( answers | length ) > 0 %} {{ answers\
\ | choice }}\n{% endif %}"
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Squad
original_task: true
name: Which one is the placeholder?
reference: ''
11e27d59-b1f5-43a1-9ccc-17f1c3249173: !Template
answer_choices: '{{ entities | join("|||") }}'
id: 11e27d59-b1f5-43a1-9ccc-17f1c3249173
jinja: "The following document has been corrupted. Tell me what \"{{\"@placeholder\"\
}}\" is referring to.\n\nDocument: {{ passage }} \n{{ query }} \n||| {% if (\
\ answers | length ) > 0 %}{{ answers | choice }}\n{% endif %}"
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Squad
original_task: true
name: corrupted
reference: ''
147656b2-2dad-4028-96c4-f19d57cd1344: !Template
answer_choices: '{% for entity in entities[:-1] %} {{ query | replace("@placeholder",
entity) }} ||| {% endfor %} {{ query | replace("@placeholder", entities[-1])
}}'
id: 147656b2-2dad-4028-96c4-f19d57cd1344
jinja: "Summary:\n\n- {{ passage.split(\"@highlight\")[1:] | join(\"\\n- \") }}\
\ \n\nArticle:\n\n{{ passage.split(\"@highlight\")[0] }}\n ||| {% if ( answers\
\ | length ) > 0 %}{{ query | replace(\"@placeholder\", answers | choice) }}\
\ {% endif %}"
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Accuracy
original_task: true
name: Summary first (continuation choices)
reference: ''
24c267d4-359e-40a9-83d2-bff904d63b09: !Template
answer_choices: '{% for entity in entities[:-1] %} {{ query | replace("@placeholder",
entity) }} ||| {% endfor %} {{ query | replace("@placeholder", entities[-1])
}}'
id: 24c267d4-359e-40a9-83d2-bff904d63b09
jinja: "Summary:\n\n- {{ passage.split(\"@highlight\")[1:] | join(\"\\n- \") }}\
\ \n\nArticle:\n\n{{ passage.split(\"@highlight\")[0] }}\n\nNow that you've\
\ read the article, please write a new sentence to add to it.\n\n||| {% if (\
\ answers | length ) > 0 %}{{ query | replace(\"@placeholder\", answers | choice)\
\ }} {% endif %}"
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Accuracy
original_task: true
name: Add sentence after after (continuation choices)
reference: ''
441c70e3-095a-44a1-8163-bc3b666b7ea1: !Template
answer_choices: '{{ entities | join("|||") }}'
id: 441c70e3-095a-44a1-8163-bc3b666b7ea1
jinja: "{{ passage }} \n{{ query }} \n\nYou should decide what \"{{\"@placeholder\"\
}}\" is referring to. Choose between:\n- {{answer_choices | join(\"\\n- \")}}\n\
||| {% if ( answers | length ) > 0 %}{{ answers | choice }}\n{% endif %}"
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Squad
original_task: true
name: choose_between
reference: ''
64013fb3-1afd-4e5a-8777-b164ca3b8e18: !Template
answer_choices: '{% for entity in entities[:-1] %} - {{ query | replace("@placeholder",
entity) }} ||| {% endfor %} - {{ query | replace("@placeholder", entities[-1])
}}'
id: 64013fb3-1afd-4e5a-8777-b164ca3b8e18
jinja: "{{ passage.split(\"@highlight\")[0] }}\n\nSummary:\n\n- {{ passage.split(\"\
@highlight\")[1:] | join(\"\\n- \") }} \n\n ||| {% if ( answers | length ) >\
\ 0 %}- {{ query | replace(\"@placeholder\", answers | choice) }} {% endif %}"
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Accuracy
original_task: true
name: GPT-3 style summary only (continuation choices)
reference: Brown et al. 2020
90fc9ecb-c706-4c03-bb7e-4fe9fcd777f6: !Template
answer_choices: '{% for entity in entities[:-1] %} {{ query | replace("@placeholder",
entity) }} ||| {% endfor %} {{ query | replace("@placeholder", entities[-1])
}}'
id: 90fc9ecb-c706-4c03-bb7e-4fe9fcd777f6
jinja: "Article:\n\n{{ passage.split(\"@highlight\")[0] }}\n\nHighlights:\n\n\
{{ passage.split(\"@highlight\")[1:] | join(\"\\n\") }} \n\n ||| {% if ( answers\
\ | length ) > 0 %}{{ query | replace(\"@placeholder\", answers | choice) }}\
\ {% endif %}"
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Accuracy
original_task: false
name: GPT-3 style with labels without hyphens (continuation choices)
reference: ''
91555c1c-c1e4-469b-a2a4-fc952ce1a145: !Template
answer_choices: '{{ entities | join("|||") }}'
id: 91555c1c-c1e4-469b-a2a4-fc952ce1a145
jinja: "{{ passage }} \n{{ query }} \nIn the question above, the \"{{\"@placeholder\"\
}}\" stands for ||| {% if ( answers | length ) > 0 %}{{ answers | choice }}{%\
\ endif %}"
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Squad
original_task: true
name: In the question above, the placeholder stands for
reference: ''
94577b75-2eac-4eae-b367-3b413c4188c6: !Template
answer_choices: '{% for entity in entities[:-1] %} {{ query | replace("@placeholder",
entity) }} ||| {% endfor %} {{ query | replace("@placeholder", entities[-1])
}}'
id: 94577b75-2eac-4eae-b367-3b413c4188c6
jinja: 'After reading the article, write another sentence to add to it.
{{ passage | replace("@highlight", "\n- ") }}
||| {% if ( answers | length ) > 0 %}{{ query | replace("@placeholder", answers
| choice) }}{% endif %}
'
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Accuracy
original_task: true
name: Add sentence after (continuation choices)
reference: ''
9579b54e-4f0f-4e43-8907-af57112cc857: !Template
answer_choices: '{% for entity in entities[:-1] %} {{ query | replace("@placeholder",
entity) }} ||| {% endfor %} {{ query | replace("@placeholder", entities[-1])
}}'
id: 9579b54e-4f0f-4e43-8907-af57112cc857
jinja: "Please read the following news article and write another sentence to add\
\ to it.\n\n{{ passage | replace(\"@highlight\", \"\\n- \") }} \n ||| {% if\
\ ( answers | length ) > 0 %}{{ query | replace(\"@placeholder\", answers |\
\ choice) }} {% endif %}\n"
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Accuracy
original_task: true
name: News article (continuation choices)
reference: ''
99dd38ce-32f3-4d58-93c5-59821002b9cc: !Template
answer_choices: '{{ entities | join("|||") }}'
id: 99dd38ce-32f3-4d58-93c5-59821002b9cc
jinja: "{{ passage }} \n{{ query }} \nWhat could the \"{{\"@placeholder\"}}\"\
\ be? {{ entities | join(\", \") }}? ||| {% if ( answers | length ) > 0 %}{{\
\ answers | choice }}{% endif %}"
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Squad
original_task: true
name: What could the placeholder be?
reference: ''
9b688cf3-28bf-4f33-94cf-e73e4fa8c608: !Template
answer_choices: '{{entities | join("|||")}}'
id: 9b688cf3-28bf-4f33-94cf-e73e4fa8c608
jinja: '{{ passage }}
{{ query }}
I am trying to decide what "{{"@placeholder"}}" means in the previous text.
Help by choosing an option between:
- {{ entities | join("\n- ") }}
||| {% if ( answers | length ) > 0 %}
{{ answers | choice }}
{% endif %}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Squad
original_task: true
name: trying_to_decide
reference: ''
a5ed27ed-162b-4ac1-9c7a-85059d5214be: !Template
answer_choices: '{{ entities | join("|||") }}'
id: a5ed27ed-162b-4ac1-9c7a-85059d5214be
jinja: "{{ passage }} \n{{ query }} \nHere, the placeholder refers to ||| {% if\
\ ( answers | length ) > 0 %}{{ answers | choice }}\n{% endif %}"
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Squad
original_task: true
name: "the placeholder refers to\u2026"
reference: ''
a99a92e0-e1ee-4ec3-a38a-3be4303ba017: !Template
answer_choices: '{% for entity in entities[:-1] %} - {{ query | replace("@placeholder",
entity) }} ||| {% endfor %} - {{ query | replace("@placeholder", entities[-1])
}}'
id: a99a92e0-e1ee-4ec3-a38a-3be4303ba017
jinja: "{{ passage.split(\"@highlight\")[0] }}\n\nHighlights:\n\n- {{ passage.split(\"\
@highlight\")[1:] | join(\"\\n- \") }} \n\nPlease write an additional highlight.\n\
\ ||| {% if ( answers | length ) > 0 %}- {{ query | replace(\"@placeholder\"\
, answers | choice) }} {% endif %}\n"
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Accuracy
original_task: true
name: New highlight (continuation choices)
reference: ''
d3fce74e-0d9d-404a-a009-9ebbf5794c2c: !Template
answer_choices: '{{entities | join("|||")}}'
id: d3fce74e-0d9d-404a-a009-9ebbf5794c2c
jinja: 'Exercise: Extract from the text the correct entity that "{{"@placeholder"}}"
is referring to.
{{ passage }}
{{ query }}
||| {% if ( answers | length ) > 0 %}
{{ answers | choice }}
{% endif %}'
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Squad
original_task: true
name: exercise
reference: ''
de5b635e-c2f4-40bb-81ac-650f1b45564b: !Template
answer_choices: '{{entities | join("|||")}}'
id: de5b635e-c2f4-40bb-81ac-650f1b45564b
jinja: '{{ passage }}
{{ query }}
Pick one option, "{{"@placeholder"}}" refers to:
- {{answer_choices | join("\n- ")}}
||| {% if ( answers | length ) > 0 %}
{{ answers | choice }}
{% endif %}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Squad
original_task: true
name: pick_one_option
reference: ''
df8d0822-2cad-42de-8191-687ae47f6098: !Template
answer_choices: '{% for entity in entities[:-1] %} - {{ query | replace("@placeholder",
entity) }} ||| {% endfor %} - {{ query | replace("@placeholder", entities[-1])
}}'
id: df8d0822-2cad-42de-8191-687ae47f6098
jinja: "{{ passage | replace(\"@highlight\", \"\\n- \") }} \n\n ||| {% if ( answers\
\ | length ) > 0 %}- {{ query | replace(\"@placeholder\", answers | choice)\
\ }} {% endif %}"
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Accuracy
original_task: true
name: GPT-3 style (continuation choices)
reference: Brown et al. 2020
dfa3052f-ede8-42c2-b99a-bc5762c4fdc6: !Template
answer_choices: '{% for entity in entities[:-1] %} - {{ query | replace("@placeholder",
entity) }} ||| {% endfor %} - {{ query | replace("@placeholder", entities[-1])
}}'
id: dfa3052f-ede8-42c2-b99a-bc5762c4fdc6
jinja: "Article:\n\n{{ passage.split(\"@highlight\")[0] }}\n\nHighlights:\n\n\
- {{ passage.split(\"@highlight\")[1:] | join(\"\\n- \") }} \n\n ||| {% if (\
\ answers | length ) > 0 %}- {{ query | replace(\"@placeholder\", answers |\
\ choice) }} {% endif %}"
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Accuracy
original_task: false
name: GPT-3 style with labels (continuation choices)
reference: Brown et al. 2020
e68d13c5-df75-4de0-b59e-f2eaf4af6ce7: !Template
answer_choices: '{{ entities | join("|||") }}'
id: e68d13c5-df75-4de0-b59e-f2eaf4af6ce7
jinja: "{{ passage }} \n{{ query }} \nCan you figure out what does the \"{{\"\
@placeholder\"}}\" mean? It means ||| {% if ( answers | length ) > 0 %}{{ answers\
\ | choice }}{% endif %}"
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Squad
original_task: true
name: "Can you figure out\u2026"
reference: ''
f7a92707-c531-42cb-81b4-063976e013cb: !Template
answer_choices: '{% for entity in entities[:-1] %} {{ query | replace("@placeholder",
entity) }} ||| {% endfor %} {{ query | replace("@placeholder", entities[-1])
}}'
id: f7a92707-c531-42cb-81b4-063976e013cb
jinja: "{{ passage | replace(\"@highlight\", \"\\n\") }} \n ||| {% if ( answers\
\ | length ) > 0 %}{{ query | replace(\"@placeholder\", answers | choice) }}\
\ {% endif %}"
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Accuracy
original_task: true
name: GPT-3 style without hyphens (continuation choices)
reference: Brown et al. 2020
src/prompts/templates/super_glue/rte/templates.yaml
+160
View File
@@ -0,0 +1,160 @@
dataset: super_glue
subset: rte
templates:
2b52a83c-0021-41fe-b44c-5aaa076d71a2: !Template
answer_choices: Yes ||| No
id: 2b52a83c-0021-41fe-b44c-5aaa076d71a2
jinja: '{{premise}} Using only the above description and what you know about the
world, is "{{hypothesis}}" definitely correct? Yes or no? ||| {% if label !=
-1 %}{{ answer_choices[label] }}{% endif %}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: MNLI crowdsource
reference: Adapted from Williams et al. 2018's instructions to crowdsourcing workers.
2d0d63da-ffcf-4f6e-941a-b8da922be43e: !Template
answer_choices: Yes ||| No
id: 2d0d63da-ffcf-4f6e-941a-b8da922be43e
jinja: 'Given {{premise}} Is it guaranteed true that "{{hypothesis}}"? Yes or no?
||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: guaranteed true
reference: Webson & Pavlick 2021
4163e6f1-1a83-4c73-b867-02eb7ac80316: !Template
answer_choices: Yes ||| No
id: 4163e6f1-1a83-4c73-b867-02eb7ac80316
jinja: Suppose {{premise}} Can we infer that "{{hypothesis}}"? Yes or no? |||
{% if label != -1 %}{{ answer_choices[label] }}{% endif %}
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: can we infer
reference: Webson & Pavlick 2021
6e078fb4-505b-413c-bb5e-3cd16ddcf5d7: !Template
answer_choices: Yes ||| No
id: 6e078fb4-505b-413c-bb5e-3cd16ddcf5d7
jinja: "{{premise}} \n\nQuestion: Does this imply that \"{{hypothesis}}\", yes\
\ or no?\n\nAnswer: ||| {% if label != -1 %}{{answer_choices[label]}}{% endif %}"
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: choose_between
reference: Burns et al. 2021
8fb1c6aa-20e9-438c-bece-c6af1c746449: !Template
answer_choices: True ||| False
id: 8fb1c6aa-20e9-438c-bece-c6af1c746449
jinja: '{{premise}}
Question: {{hypothesis}} True or False? ||| {% if label != -1 %}{{ answer_choices[label]
}}{% endif %}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: GPT-3 style
reference: Same as reported in Figure G31 of the GPT-3 paper.
9e078fb4-505b-413c-bb5e-3cd16ddcf5d7: !Template
answer_choices: Yes ||| No
id: 9e078fb4-505b-413c-bb5e-3cd16ddcf5d7
jinja: "{{premise}} \n\nQuestion: Does this imply that \"{{hypothesis}}\"? Yes\
\ or no? ||| {% if label != -1 %}{{answer_choices[label]}}{% endif %}"
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: does this imply
reference: Sanh et al. 2021
b8dc85c6-28b6-4340-979a-8e77c2a0dde8: !Template
answer_choices: Yes ||| No
id: b8dc85c6-28b6-4340-979a-8e77c2a0dde8
jinja: Given {{premise}} Should we assume that "{{hypothesis}}" is true? Yes or
no? ||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %}
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: should assume
reference: Webson & Pavlick 2021
e2fb58f2-b1f2-4aef-b74b-c4ee1c571fff: !Template
answer_choices: Yes ||| No
id: e2fb58f2-b1f2-4aef-b74b-c4ee1c571fff
jinja: Given that {{premise}} Does it follow that {{hypothesis}} Yes or no? |||
{% if label != -1 %}{{ answer_choices[label] }}{% endif %}
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: does it follow that
reference: Sanh et al. 2021
ed1f4b75-8826-4852-9bd6-aedf368678f5: !Template
answer_choices: Yes ||| No
id: ed1f4b75-8826-4852-9bd6-aedf368678f5
jinja: '{{premise}} Based on the previous passage, is it true that "{{hypothesis}}"?
Yes or no? ||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: based on the previous passage
reference: "Adapted from the BoolQ prompts in Schick & Sch\xFCtze 2021."
ee0ce095-122a-4509-bf0b-33d1495295f7: !Template
answer_choices: Yes ||| No
id: ee0ce095-122a-4509-bf0b-33d1495295f7
jinja: '{{premise}} Are we justified in saying that "{{hypothesis}}"? Yes or no?
||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: justified in saying
reference: Webson & Pavlick 2021
fb4f8144-37f5-4977-88da-37a5d0bfd0e8: !Template
answer_choices: Yes ||| No
id: fb4f8144-37f5-4977-88da-37a5d0bfd0e8
jinja: Given that {{premise}} Therefore, it must be true that "{{hypothesis}}"?
Yes or no? ||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %}
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: must be true
reference: Sanh et al. 2021
src/prompts/templates/super_glue/wic/templates.yaml
+238
View File
@@ -0,0 +1,238 @@
dataset: super_glue
subset: wic
templates:
14e73f39-a0d1-44c2-b9a4-4e48f9f1608e: !Template
answer_choices: No ||| Yes
id: 14e73f39-a0d1-44c2-b9a4-4e48f9f1608e
jinja: 'Does the word "{{word}}" have the same meaning in these two sentences?
Yes, No?
{{sentence1}}
{{sentence2}}
||| {% if label != -1%}
{{answer_choices[label]}}
{% endif %}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: question-context-meaning-with-label
reference: Generalized question-context format with label
3503ead5-4fa5-4f77-95dc-f0c2ed3eecdc: !Template
answer_choices: No ||| Yes
id: 3503ead5-4fa5-4f77-95dc-f0c2ed3eecdc
jinja: 'Does the word "{{word}}" have the same meaning in these two sentences?
{{sentence1}}
{{sentence2}}
||| {% if label != -1%}
{{answer_choices[label]}}
{% endif %}'
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Accuracy
original_task: true
name: question-context-meaning
reference: Generalized question-context format
611d13dc-d414-4b9b-9204-e4f325e859e7: !Template
answer_choices: No ||| Yes
id: 611d13dc-d414-4b9b-9204-e4f325e859e7
jinja: 'Homework
Decide whether the word "{{word}}" is used with the same meaning in the two
following sentences. Answer by yes or no.
{{sentence1}}
{{sentence2}}
||| {% if label != -1%}
{{answer_choices[label]}}
{% endif %}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: grammar_homework
reference: ''
725b5ed0-7728-4890-95a4-a74cb7ae1bb4: !Template
answer_choices: False ||| True
id: 725b5ed0-7728-4890-95a4-a74cb7ae1bb4
jinja: 'Sentence A: {{sentence1}}
Sentence B: {{sentence2}}
"{{word}}" has a similar meaning in sentences A and B. True or False?
||| {% if label != -1%}
{{answer_choices[label]}}
{% endif %}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: affirmation_true_or_false
reference: ''
c3a0a5d8-cfe9-4a7f-8a3c-3c526e0ad0c6: !Template
answer_choices: No ||| Yes
id: c3a0a5d8-cfe9-4a7f-8a3c-3c526e0ad0c6
jinja: '{{sentence1}}
{{sentence2}}
Question: Is the word ''{{word}}'' used in the same sense in the two sentences
above?
||| {% if label != -1%}
{{answer_choices[label]}}
{% endif %}'
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Accuracy
original_task: true
name: GPT-3-prompt
reference: Following table G32. https://arxiv.org/pdf/2005.14165.pdf
ce8b5a93-1841-4897-84db-b100f1c84f4b: !Template
answer_choices: No ||| Yes
id: ce8b5a93-1841-4897-84db-b100f1c84f4b
jinja: 'Sentence 1: {{sentence1}}
Sentence 2: {{sentence2}}
Determine whether the word "{{word}}" is used in the same sense in both sentences.
Yes or no?
||| {% if label != -1%}
{{answer_choices[label]}}
{% endif %}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: same_sense
reference: ''
cfbc1637-10b8-4f20-a31c-55292f3cebd0: !Template
answer_choices: No ||| Yes
id: cfbc1637-10b8-4f20-a31c-55292f3cebd0
jinja: "Determine if the word '{{word}}' is used in the same way in the two sentences\
\ below. \n{{sentence1}}\n{{sentence2}}\n||| {% if label != -1%}\n{{answer_choices[label]}}\n\
{% endif %}"
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Accuracy
original_task: true
name: question-context
reference: Generalized question-context format
d9e1db2a-ab0b-4621-bb41-01d5788d3873: !Template
answer_choices: No ||| Yes
id: d9e1db2a-ab0b-4621-bb41-01d5788d3873
jinja: '{{sentence1}}
{{sentence2}}
Question: Is the word ''{{word}}'' used in the same sense in the two sentences
above? Yes, No?
||| {% if label != -1%}
{{answer_choices[label]}}
{% endif %}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: GPT-3-prompt-with-label
reference: Following table G32. https://arxiv.org/pdf/2005.14165.pdf add additional
label
dd2080cf-3117-49ba-9aff-c988a21fdb69: !Template
answer_choices: No ||| Yes
id: dd2080cf-3117-49ba-9aff-c988a21fdb69
jinja: 'The word "{{word}}" has multiple meanings. Does it have the same meaning
in sentences 1 and 2? Yes or no?
Sentence 1: {{sentence1}}
Sentence 2: {{sentence2}}
||| {% if label != -1%}
{{answer_choices[label]}}
{% endif %}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: polysemous
reference: ''
f934a96d-fe4d-4075-aa47-5595b9a604c7: !Template
answer_choices: No ||| Yes
id: f934a96d-fe4d-4075-aa47-5595b9a604c7
jinja: '{{sentence1}}
{{sentence2}}
Similar sense of {{word}}?
||| {% if label != -1%}
{{answer_choices[label]}}
{% endif %}'
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Accuracy
original_task: true
name: similar-sense
reference: Following https://arxiv.org/abs/2105.11447, https://github.com/ethanjperez/true_few_shot/tree/main/templates.super_glue
src/prompts/templates/super_glue/wsc.fixed/templates.yaml
+155
View File
@@ -0,0 +1,155 @@
dataset: super_glue
subset: wsc.fixed
templates:
212fb8b1-8436-4f64-8f37-a9094fe029f4: !Template
answer_choices: No ||| Yes
id: 212fb8b1-8436-4f64-8f37-a9094fe029f4
jinja: '{{ text }} In the previous sentence, does the pronoun "{{ span2_text.lower()
}}" refer to {{ span1_text }}? Yes or no? ||| {% if label != -1 %}{{ answer_choices[label]
}}{% endif %}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: does the pronoun refer to
reference: "Adapted from Perez et al. 2021 and Schick & Sch\xFCtz 2021."
23361c5d-b67f-4c2a-9da7-16301c55d0e1: !Template
answer_choices: No ||| Yes
id: 23361c5d-b67f-4c2a-9da7-16301c55d0e1
jinja: '{{ text }} Here, by "{{ span2_text }}" they mean "{{ span1_text }}". Yes
or no? ||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: by p they mean
reference: ''
2f17f18b-6daa-44ef-a2dd-dddaf04aec0e: !Template
answer_choices: False ||| True
id: 2f17f18b-6daa-44ef-a2dd-dddaf04aec0e
jinja: "{{ text }} \n\nIn other words, {{ text.split(\" \")[span2_index:] | join(\"\
\ \") | replace(span2_text, span1_text) }} True or false? ||| {% if label !=\
\ -1 %}{{ answer_choices[label] }}{% endif %}"
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: in other words
reference: ''
4b3e29cc-ccb8-4e4c-a845-4935ca29cf34: !Template
answer_choices: No ||| Yes
id: 4b3e29cc-ccb8-4e4c-a845-4935ca29cf34
jinja: '{{ text }} I think they mean "{{ text.split(" ")[span2_index:] | join("
") | replace(span2_text, span1_text) }}" Yes or no? ||| {% if label != -1 %}{{
answer_choices[label] }}{% endif %}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: I think they mean
reference: ''
7482d24f-cf45-4013-b82d-369489fc958b: !Template
answer_choices: No ||| Yes
id: 7482d24f-cf45-4013-b82d-369489fc958b
jinja: '{{ text }} Here, does "{{ span2_text.lower() }}" stand for {{ span1_text
}}? Yes or no? ||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: does p stand for
reference: ''
7d377293-d043-4b6c-8ec1-d61eaf14ec67: !Template
answer_choices: No ||| Yes
id: 7d377293-d043-4b6c-8ec1-d61eaf14ec67
jinja: "Passage: {{ text }} \n\nQuestion: In the passage above, does the pronoun\
\ \"{{ span2_text }}\" refer to {{ span1_text }}?\n\nAnswer: ||| {% if label\
\ != -1 %}{{ answer_choices[label] }}{% endif %}"
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Accuracy
original_task: true
name: GPT-3 Style
reference: Adapted from Figure G33, p. 59, Brown et al. 2020
809eacd0-2f6c-4e3a-b52a-57c783879d36: !Template
answer_choices: No ||| Yes
id: 809eacd0-2f6c-4e3a-b52a-57c783879d36
jinja: '{{ text }} In the previous sentence, can the pronoun "{{ span2_text }}"
be replaced with "{{ span1_text }}"? Yes or no? ||| {% if label != -1 %}{{ answer_choices[label]
}}{% endif %}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: replaced with
reference: ''
87f97aa0-1fa9-4f0b-b8e6-89d3c1f19bd6: !Template
answer_choices: False ||| True
id: 87f97aa0-1fa9-4f0b-b8e6-89d3c1f19bd6
jinja: "Context: {{ text }} \n\n{% if span2_text.lower() == \"they\" or span2_text.lower()\
\ == \"them\" %}\nQuestion: \"{{ span2_text }}\" are {{ span1_text }}. True\
\ or false?\n{% else %}\nQuestion: \"{{ span2_text }}\" is {{ span1_text }}.\
\ True or false?\n{% endif %}\n\nAnswer: ||| {% if label != -1 %}{{ answer_choices[label]\
\ }}{% endif %}"
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: p is/are r
reference: ''
aae24b54-c3a7-4f69-8b77-f6dc115988f8: !Template
answer_choices: False ||| True
id: aae24b54-c3a7-4f69-8b77-f6dc115988f8
jinja: "{{ text }} \nIn the passage above, the pronoun \"{{ span2_text }}\" refers\
\ to {{ span1_text }}. True or false? ||| {% if label != -1 %}{{ answer_choices[label]\
\ }}{% endif %}"
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: the pronoun refers to
reference: "Adapted from Perez et al. 2021 and Schick & Sch\xFCtz 2021."
d88f3e21-42dc-49a5-924d-69b764a14816: !Template
answer_choices: No ||| Yes
id: d88f3e21-42dc-49a5-924d-69b764a14816
jinja: "{{ text }} \n{% if span2_text.lower() == \"they\" or span2_text.lower()\
\ == \"them\" %}\nQuestion: Who or what are \"{{ span2_text.lower() }}\"? {{\
\ span1_text }}?\n{% else %}\nQuestion: Who or what is \"{{ span2_text.lower()\
\ }}\"? Is it {{ span1_text }}?\n{% endif %}\nAnswer: ||| {% if label != -1\
\ %}{{ answer_choices[label] }}{% endif %}"
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Accuracy
original_task: true
name: Who or what is/are
reference: I double checked the only plural pronouns in WSC are "they" and "them".
src/prompts/templates/tweet_eval/emoji/templates.yaml
Executable
+68
View File
@@ -0,0 +1,68 @@
dataset: tweet_eval
subset: emoji
templates:
8c794abe-5364-430f-aa1e-eb3501443cec: !Template
answer_choices: "\u2764|||\U0001F60D|||\U0001F602|||\U0001F495|||\U0001F525|||\U0001F60A\
|||\U0001F60E|||\u2728|||\U0001F499|||\U0001F618|||\U0001F4F7|||\U0001F1FA\U0001F1F8\
|||\u2600|||\U0001F49C|||\U0001F609|||\U0001F4AF|||\U0001F601|||\U0001F384|||\U0001F4F8\
|||\U0001F61C"
id: 8c794abe-5364-430f-aa1e-eb3501443cec
jinja: 'Which emoji among {{answer_choices | join(", ")}} best describes the sentiment
of the following tweet?
{{text}} |||
{{answer_choices[label]}}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Other
original_task: true
name: emoji_option
reference: 'official metric: macroaveraged F1'
c05f50e0-f708-44bc-98e7-ff7b3f9f5d93: !Template
answer_choices: "\u2764|||\U0001F60D|||\U0001F602|||\U0001F495|||\U0001F525|||\U0001F60A\
|||\U0001F60E|||\u2728|||\U0001F499|||\U0001F618|||\U0001F4F7|||\U0001F1FA\U0001F1F8\
|||\u2600|||\U0001F49C|||\U0001F609|||\U0001F4AF|||\U0001F601|||\U0001F384|||\U0001F4F8\
|||\U0001F61C"
id: c05f50e0-f708-44bc-98e7-ff7b3f9f5d93
jinja: 'Which emoji among {{answer_choices | join(", ")}} would be the best comment
to the following tweet?
{{text}} |||
{{answer_choices[label]}}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Other
original_task: false
name: emoji_reply
reference: 'Metric: macroaveraged F1'
d5c771d3-28e7-420e-af47-c077cfe0e7e5: !Template
answer_choices: "\u2764|||\U0001F60D|||\U0001F602|||\U0001F495|||\U0001F525|||\U0001F60A\
|||\U0001F60E|||\u2728|||\U0001F499|||\U0001F618|||\U0001F4F7|||\U0001F1FA\U0001F1F8\
|||\u2600|||\U0001F49C|||\U0001F609|||\U0001F4AF|||\U0001F601|||\U0001F384|||\U0001F4F8\
|||\U0001F61C"
id: d5c771d3-28e7-420e-af47-c077cfe0e7e5
jinja: 'Which emoji best describes the sentiment of the following tweet?
{{text}} |||
{{answer_choices[label]}}'
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Other
original_task: true
name: emoji
reference: 'official metric: macroaveraged F1'
src/prompts/templates/tweet_eval/emotion/templates.yaml
+103
View File
@@ -0,0 +1,103 @@
dataset: tweet_eval
subset: emotion
templates:
7c09c33e-31f2-414b-89a1-6b1dda92ef6f: !Template
answer_choices: anger ||| joy ||| optimism ||| sadness
id: 7c09c33e-31f2-414b-89a1-6b1dda92ef6f
jinja: '{{text}}
To get full credit in this exam, choose the correct emotion from the following
choices: {{answer_choices | join(", ")}}
|||
{{answer_choices[label]}}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: emotion_exam
reference: exam style prompt
87db02f2-585e-4fd1-81c0-e94297607097: !Template
answer_choices: anger ||| joy ||| optimism ||| sadness
id: 87db02f2-585e-4fd1-81c0-e94297607097
jinja: 'Which emotion among {{answer_choices | join(", ")}} best describes the
feeling of the author of the following tweet?
{{text}}|||
{{answer_choices[label]}}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: author_emotion
reference: ''
8bc3ebc5-77f1-4d55-bd96-c62429ebf093: !Template
answer_choices: anger ||| joy ||| optimism ||| sadness
id: 8bc3ebc5-77f1-4d55-bd96-c62429ebf093
jinja: 'Which emotion is best represented by the following tweet?
{{text}}
Possible emotions: {{answer_choices | join(", ")}}
|||
{{answer_choices[label]}}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: which_emotion
reference: ''
a5992077-2e31-467b-a6ee-b75dee933d0e: !Template
answer_choices: anger ||| joy ||| optimism ||| sadness
id: a5992077-2e31-467b-a6ee-b75dee933d0e
jinja: "{{text}}\n\nCategorize the tweet into one of the following options: \n\
(a) {{answer_choices[0]}}\n(b) {{answer_choices[1]}}\n(c) {{answer_choices[2]}}\n\
(d) {{answer_choices[3]}}\n|||\n{{answer_choices[label]}}"
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: emotion_with_option
reference: ''
b8f4912e-e3be-4dd5-82ec-6f110c056a86: !Template
answer_choices: anger ||| joy ||| optimism ||| sadness
id: b8f4912e-e3be-4dd5-82ec-6f110c056a86
jinja: '{{text}}
What is the emotion of the text?
Hint: {{answer_choices | join(", ")}}
|||
{{answer_choices[label]}}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: emotion_hint
reference: ''
src/prompts/templates/tweet_eval/hate/templates.yaml
+84
View File
@@ -0,0 +1,84 @@
dataset: tweet_eval
subset: hate
templates:
3266f9d4-9c80-4e17-a8a6-1fe44ce8f3bf: !Template
answer_choices: no ||| yes
id: 3266f9d4-9c80-4e17-a8a6-1fe44ce8f3bf
jinja: 'Does this tweet convey the author''s hatred towards something or someone?
{{text}} |||
{{answer_choices[label]}}'
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Accuracy
original_task: true
name: author_hate
reference: ''
34a5128b-6fc8-453b-94d4-4ebaa87172c1: !Template
answer_choices: no ||| yes
id: 34a5128b-6fc8-453b-94d4-4ebaa87172c1
jinja: 'Does this tweet convey hate: yes or no?
{{text}} |||
{{answer_choices[label]}}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: hate_no_yes
reference: ''
5b15dc68-05a2-42dd-b0bf-fa15d4f40320: !Template
answer_choices: no|||yes
id: 5b15dc68-05a2-42dd-b0bf-fa15d4f40320
jinja: "Is this a hateful tweet? \n{{text}}\n|||\n{{answer_choices[label]}}"
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Accuracy
original_task: true
name: hate_question
reference: ''
971ad470-85bf-484e-aab7-b942f817bf2c: !Template
answer_choices: non-hate ||| hate
id: 971ad470-85bf-484e-aab7-b942f817bf2c
jinja: 'Does this tweet convey {{"hate"}} or {{"non-hate"}}?
{{text}} |||
{{answer_choices[label]}}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: hate_options
reference: ''
b0cdecc2-78a2-47e7-a74e-38d509c01214: !Template
answer_choices: 'no ||| yes '
id: b0cdecc2-78a2-47e7-a74e-38d509c01214
jinja: "In this test, you need to answer with either yes or no. \n\nQ: Is this\
\ a hateful tweet? \n{{text}}\n\nA: \n|||\n{{answer_choices[label]}}"
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: hate_exam
reference: ''
src/prompts/templates/tweet_eval/irony/templates.yaml
+84
View File
@@ -0,0 +1,84 @@
dataset: tweet_eval
subset: irony
templates:
c45095f1-9be1-4e83-8daa-68805b6ece39: !Template
answer_choices: no ||| yes
id: c45095f1-9be1-4e83-8daa-68805b6ece39
jinja: "Is this tweet is ironic? \n\n{{text}} |||\n{{answer_choices[label]}}"
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Accuracy
original_task: true
name: irony_question
reference: ''
cd2ed852-c6fa-431a-b0f1-06f0240d74a0: !Template
answer_choices: no ||| yes
id: cd2ed852-c6fa-431a-b0f1-06f0240d74a0
jinja: "Is this tweet ironic? Answer with either yes or no. \n\n{{text}} |||\n\
{{answer_choices[label]}}"
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: irony_yes_no
reference: ''
cef39e97-09f2-430c-ad1f-5fd9d05c876b: !Template
answer_choices: no ||| yes
id: cef39e97-09f2-430c-ad1f-5fd9d05c876b
jinja: "In this test, you need to answer with either yes or no. \n\nQ: Is this\
\ an ironic tweet? \n{{text}}\n\nA: \n|||\n{{answer_choices[label]}}"
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: irony_exam
reference: exam style prompt
e7cff075-9388-48de-af51-fe185b654217: !Template
answer_choices: non-irony ||| irony
id: e7cff075-9388-48de-af51-fe185b654217
jinja: 'Does this tweet contain {{"irony"}} or {{"non-irony"}}?
{{text}} |||
{{answer_choices[label]}}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: irony_option
reference: ''
f8361684-dad1-4c2c-9f22-1128f3096c20: !Template
answer_choices: no|||yes
id: f8361684-dad1-4c2c-9f22-1128f3096c20
jinja: 'Does this tweet convey the author''s irony towards something or someone?
{{text}}
|||
{{answer_choices[label]}}
'
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Accuracy
original_task: true
name: irony_author
reference: ''
src/prompts/templates/tweet_eval/offensive/templates.yaml
+87
View File
@@ -0,0 +1,87 @@
dataset: tweet_eval
subset: offensive
templates:
10850707-80f3-4a75-b9f4-1e2d12be04c0: !Template
answer_choices: no ||| yes
id: 10850707-80f3-4a75-b9f4-1e2d12be04c0
jinja: "Is this tweet {{\"offensive\"}}? \n\n{{text}} |||\n{{answer_choices[label]}}"
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Accuracy
original_task: true
name: offensive_question
reference: ''
22ecf073-2932-4b0a-b305-1f92b696d0b4: !Template
answer_choices: no ||| yes
id: 22ecf073-2932-4b0a-b305-1f92b696d0b4
jinja: "Can the tweet be removed for being offensive? Answer with a yes or a no.\
\ \n\n{{text}}\n|||\n{{answer_choices[label]}}"
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: offensive_tweet_removed
reference: ''
90a80d37-922d-40c0-b036-87eac64ce4c5: !Template
answer_choices: no ||| yes
id: 90a80d37-922d-40c0-b036-87eac64ce4c5
jinja: 'Is the author''s tweet offensive? Answer with either yes or no.
{{text}}
|||
{{answer_choices[label]}}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: offensive_author
reference: ''
a12bd98b-facc-4b17-bb16-80c98a20aa64: !Template
answer_choices: no ||| yes
id: a12bd98b-facc-4b17-bb16-80c98a20aa64
jinja: 'Task: Identify if the tweet or text is offensive.
Tweet: {{text}}
Possible answers: yes, no
|||
{{answer_choices[label]}}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: offensive_task
reference: ''
bf2cea43-0666-4eb5-814d-00956afd1900: !Template
answer_choices: no ||| yes
id: bf2cea43-0666-4eb5-814d-00956afd1900
jinja: "In this test, you need to answer with either yes or no.\n\nQ: Is this\
\ an offensive tweet?\n\n{{text}}\n\nA: \n|||\n{{answer_choices[label]}}"
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: offensive_exam
reference: ''
src/prompts/templates/tweet_eval/sentiment/templates.yaml
+84
View File
@@ -0,0 +1,84 @@
dataset: tweet_eval
subset: sentiment
templates:
1fef2b36-3a19-4179-9b43-d67887cff299: !Template
answer_choices: negative ||| neutral ||| positive
id: 1fef2b36-3a19-4179-9b43-d67887cff299
jinja: 'In this exam, you need to pick the correct sentiment for the tweet:
{{text}}
Possible choices: {{answer_choices | join(", ")}}
|||
{{answer_choices[label]}}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: sentiment_exam
reference: ''
6702e8cd-9764-4c88-86a9-046f84c98ef2: !Template
answer_choices: negative ||| neutral ||| positive
id: 6702e8cd-9764-4c88-86a9-046f84c98ef2
jinja: "What is the sentiment of the tweet?\n\n{{text}} \n\nPossible choices:\
\ {{answer_choices | join(\", \")}}\n|||\n{{answer_choices[label]}}\n"
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: sentiment_question
reference: ''
6c6c797c-9912-4778-906b-16f465941d16: !Template
answer_choices: negative ||| neutral ||| positive
id: 6c6c797c-9912-4778-906b-16f465941d16
jinja: "Task: Identify the sentiment of the tweet.\n\nTweet: {{text}}\n\nOptions:\
\ {{answer_choices | join(\", \")}} \n|||\n{{answer_choices[label]}}"
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: sentiment_task
reference: ''
b70647cf-22a0-49b2-b45e-23432c635cc2: !Template
answer_choices: negative|||neutral|||positive
id: b70647cf-22a0-49b2-b45e-23432c635cc2
jinja: "Suppose you are the moderator of Twitter, what would be the sentiment\
\ of the following tweet: \n\n{{text}}\n\nOptions: {{answer_choices | join(\"\
, \")}}\n|||\n{{answer_choices[label]}}"
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: sentiment_moderator
reference: ''
ec68fd8e-92a3-4010-b0df-b14af95421a3: !Template
answer_choices: negative ||| neutral ||| positive
id: ec68fd8e-92a3-4010-b0df-b14af95421a3
jinja: "{{text}}\n\nCategorize the tweet into one of the following options: \n\
(a) {{answer_choices[0]}} \n(b) {{answer_choices[1]}} \n(c) {{answer_choices[2]}}\n\
|||\n{{answer_choices[label]}}"
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: sentiment_options
reference: ''
src/prompts/templates/tweet_eval/stance_abortion/templates.yaml
+108
View File
@@ -0,0 +1,108 @@
dataset: tweet_eval
subset: stance_abortion
templates:
0d1dc279-e50c-4706-bc3d-84ea01cb59a1: !Template
answer_choices: Neutral ||| Against ||| In favor
id: 0d1dc279-e50c-4706-bc3d-84ea01cb59a1
jinja: 'Does the author express any stance about abortion in the following text?
{{text}} |||
{{answer_choices[label]}}'
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Accuracy
original_task: true
name: abortion_predict_stance
reference: ''
22758062-db86-4009-81a4-1e2a2e1052f2: !Template
answer_choices: Neutral ||| Against ||| In favor
id: 22758062-db86-4009-81a4-1e2a2e1052f2
jinja: '{{text}} Where does the author of the above sentence stand on abortion?
|||
{{answer_choices[label]}}'
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Accuracy
original_task: true
name: abortion_guess_passive_author
reference: ''
615151f8-ac5b-4c0e-a234-9e9b6296a2f2: !Template
answer_choices: Neutral ||| Against ||| In favor
id: 615151f8-ac5b-4c0e-a234-9e9b6296a2f2
jinja: 'What option among, {{"neutral"}}, {{"against"}}, {{"in favor"}}, best
describes the stance of this tweet regarding abortion?
{{text}} |||
{{answer_choices[label]}}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: abortion_option
reference: ''
687ffa1e-a772-48b1-9291-ba4e530a909e: !Template
answer_choices: Neutral ||| Against ||| In favor
id: 687ffa1e-a772-48b1-9291-ba4e530a909e
jinja: 'Is this tweet neutral, in favor of, or against abortion?
{{text}} |||
{{answer_choices[label]}}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: abortion
reference: ''
c5507588-1d20-42f9-935f-0c767294f5a9: !Template
answer_choices: Neutral ||| Against ||| In favor
id: c5507588-1d20-42f9-935f-0c767294f5a9
jinja: 'How would you describe the stance used in this tweet? {{answer_choices|join(",
")}}
{{text}} |||
{{answer_choices[label]}}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: abortion_how_describe
reference: ''
ee12d37b-5667-4b0e-9831-f952d08152b5: !Template
answer_choices: Neutral ||| Against ||| In favor
id: ee12d37b-5667-4b0e-9831-f952d08152b5
jinja: '{{text}} Where does the above sentence stand on abortion? |||
{{answer_choices[label]}}'
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Accuracy
original_task: true
name: abortion_guess_passive
reference: ''
src/prompts/templates/tweet_eval/stance_atheism/templates.yaml
+108
View File
@@ -0,0 +1,108 @@
dataset: tweet_eval
subset: stance_atheism
templates:
2f6bfe8f-c45e-4f1d-a623-91a9cfd0ea8f: !Template
answer_choices: Neutral ||| Against ||| In favor
id: 2f6bfe8f-c45e-4f1d-a623-91a9cfd0ea8f
jinja: '{{text}} Where does the above sentence stand on atheism? |||
{{answer_choices[label]}}'
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Accuracy
original_task: true
name: atheism_guess_passive
reference: ''
4309e10d-c9a9-4a17-8561-15270b998905: !Template
answer_choices: Neutral ||| Against ||| In favor
id: 4309e10d-c9a9-4a17-8561-15270b998905
jinja: 'How would you describe the stance used in this tweet? {{answer_choices|join(",
")}}
{{text}} |||
{{answer_choices[label]}}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: atheism_how_describe
reference: ''
7e47c6b8-2923-4580-a275-a2b8867a3d96: !Template
answer_choices: Neutral ||| Against ||| In favor
id: 7e47c6b8-2923-4580-a275-a2b8867a3d96
jinja: 'What option among, {{"neutral"}}, {{"against"}}, {{"in favor"}}, best
describes the stance of this tweet regarding atheism?
{{text}} |||
{{answer_choices[label]}}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: atheism_option
reference: ''
89aa258e-3c3b-4d1c-8ac4-fe2c838b76e4: !Template
answer_choices: Neutral ||| Against ||| In favor
id: 89aa258e-3c3b-4d1c-8ac4-fe2c838b76e4
jinja: 'Does the author express any stance about atheism in the following text?
{{text}} |||
{{answer_choices[label]}}'
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Accuracy
original_task: true
name: atheism_predict_stance
reference: ''
97ef9418-7c92-455d-a4c5-d7b91668278c: !Template
answer_choices: Neutral ||| Against ||| In favor
id: 97ef9418-7c92-455d-a4c5-d7b91668278c
jinja: 'Is this tweet neutral, in favor of, or against atheism?
{{text}} |||
{{answer_choices[label]}}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: atheism
reference: ''
f28307ab-563e-4189-99b5-e0d858e9ab4c: !Template
answer_choices: Neutral ||| Against ||| In favor
id: f28307ab-563e-4189-99b5-e0d858e9ab4c
jinja: '{{text}} Where does the author of the above sentence stand on atheism?
|||
{{answer_choices[label]}}'
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Accuracy
original_task: true
name: atheism_guess_passive_author
reference: ''
src/prompts/templates/tweet_eval/stance_climate/templates.yaml
+109
View File
@@ -0,0 +1,109 @@
dataset: tweet_eval
subset: stance_climate
templates:
2ebf2eaa-ef9f-413d-b7bf-cb2037330d2a: !Template
answer_choices: Neutral ||| Against ||| In favor
id: 2ebf2eaa-ef9f-413d-b7bf-cb2037330d2a
jinja: 'What option among, {{"neutral"}}, {{"against"}}, {{"in favor"}}, best
describes the stance of this tweet regarding climate change?
{{text}} |||
{{answer_choices[label]}}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: climate_option
reference: ''
6f4205ad-6321-42a9-bf8e-a45508e67c1a: !Template
answer_choices: Neutral ||| Against ||| In favor
id: 6f4205ad-6321-42a9-bf8e-a45508e67c1a
jinja: '{{text}} Where does the above sentence stand on climate change? |||
{{answer_choices[label]}}'
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Accuracy
original_task: true
name: climate_guess_passive
reference: ''
703f067e-5930-424e-9882-48063307ff8e: !Template
answer_choices: Neutral ||| Against ||| In favor
id: 703f067e-5930-424e-9882-48063307ff8e
jinja: 'How would you describe the stance used in this tweet? {{answer_choices|join(",
")}}
{{text}} |||
{{answer_choices[label]}}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: climate_how_describe
reference: ''
8ecd5059-742a-4833-95a1-bf0e25e9abfc: !Template
answer_choices: Neutral ||| Against ||| In favor
id: 8ecd5059-742a-4833-95a1-bf0e25e9abfc
jinja: '{{text}} Where does the author of the above sentence stand on climate
change? |||
{{answer_choices[label]}}'
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Accuracy
original_task: true
name: climate_guess_passive_author
reference: ''
cd82620b-6d1d-42f7-af89-56980cbb69a5: !Template
answer_choices: Neutral ||| Against ||| In favor
id: cd82620b-6d1d-42f7-af89-56980cbb69a5
jinja: 'Does the author express any stance about climate change in the following
text?
{{text}} |||
{{answer_choices[label]}}'
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Accuracy
original_task: true
name: climate_predict_stance
reference: ''
edcdde10-b2e4-4954-82e2-f84fd57fc122: !Template
answer_choices: Neutral ||| Against ||| In favor
id: edcdde10-b2e4-4954-82e2-f84fd57fc122
jinja: 'Is this tweet neutral, in favor of, or against climate change?
{{text}} |||
{{answer_choices[label]}}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: climate_change
reference: ''
src/prompts/templates/tweet_eval/stance_feminist/templates.yaml
+108
View File
@@ -0,0 +1,108 @@
dataset: tweet_eval
subset: stance_feminist
templates:
2f6bfe8f-c45e-4f1d-a623-91a9cfd0ea8b: !Template
answer_choices: Neutral ||| Against ||| In favor
id: 2f6bfe8f-c45e-4f1d-a623-91a9cfd0ea8b
jinja: '{{text}} Where does the above sentence stand on feminism? |||
{{answer_choices[label]}}'
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Accuracy
original_task: true
name: feminism_guess_passive
reference: ''
4309e10d-c9a9-4a17-8561-15270b99890b: !Template
answer_choices: Neutral ||| Against ||| In favor
id: 4309e10d-c9a9-4a17-8561-15270b99890b
jinja: 'How would you describe the stance used in this tweet? {{answer_choices|join(",
")}}
{{text}} |||
{{answer_choices[label]}}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: feminism_how_describe
reference: ''
7e47c6b8-2923-4580-a275-a2b8867a3d9b: !Template
answer_choices: Neutral ||| Against ||| In favor
id: 7e47c6b8-2923-4580-a275-a2b8867a3d9b
jinja: 'What option among, {{"neutral"}}, {{"against"}}, {{"in favor"}}, best
describes the stance of this tweet regarding feminism?
{{text}} |||
{{answer_choices[label]}}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: feminism_option
reference: ''
89aa258e-3c3b-4d1c-8ac4-fe2c838b76eb: !Template
answer_choices: Neutral ||| Against ||| In favor
id: 89aa258e-3c3b-4d1c-8ac4-fe2c838b76eb
jinja: 'Does the author express any stance about feminism in the following text?
{{text}} |||
{{answer_choices[label]}}'
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Accuracy
original_task: true
name: feminism_predict_stance
reference: ''
97ef9418-7c92-455d-a4c5-d7b91668278b: !Template
answer_choices: Neutral ||| Against ||| In favor
id: 97ef9418-7c92-455d-a4c5-d7b91668278b
jinja: 'Is this tweet neutral, in favor of, or against feminism?
{{text}} |||
{{answer_choices[label]}}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: feminism
reference: ''
f28307ab-563e-4189-99b5-e0d858e9ab4b: !Template
answer_choices: Neutral ||| Against ||| In favor
id: f28307ab-563e-4189-99b5-e0d858e9ab4b
jinja: '{{text}} Where does the author of the above sentence stand on feminism?
|||
{{answer_choices[label]}}'
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Accuracy
original_task: true
name: feminism_guess_passive_author
reference: ''
src/prompts/templates/tweet_eval/stance_hillary/templates.yaml
+108
View File
@@ -0,0 +1,108 @@
dataset: tweet_eval
subset: stance_hillary
templates:
21ba1c40-b491-43ed-96d6-7423b55c3bcf: !Template
answer_choices: Neutral ||| Against ||| In favor
id: 21ba1c40-b491-43ed-96d6-7423b55c3bcf
jinja: 'Does the author express any stance about Hillary in the following text?
{{text}} |||
{{answer_choices[label]}}'
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Accuracy
original_task: true
name: Hillary_predict_stance
reference: ''
41502ea8-73a4-48a4-a15e-ab2ac7700457: !Template
answer_choices: Neutral ||| Against ||| In favor
id: 41502ea8-73a4-48a4-a15e-ab2ac7700457
jinja: 'What option among, {{"neutral"}}, {{"against"}}, {{"in favor"}}, best
describes the stance of this tweet regarding Hillary?
{{text}} |||
{{answer_choices[label]}}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: Hillary_option
reference: ''
498f1dec-12dc-4082-a44e-82fcae004bb8: !Template
answer_choices: Neutral ||| Against ||| In favor
id: 498f1dec-12dc-4082-a44e-82fcae004bb8
jinja: 'Is this tweet neutral, in favor of, or against Hillary?
{{text}} |||
{{answer_choices[label]}}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: Hillary
reference: ''
5c451846-349a-44ad-83ef-d0f8e5d2bd6b: !Template
answer_choices: Neutral ||| Against ||| In favor
id: 5c451846-349a-44ad-83ef-d0f8e5d2bd6b
jinja: '{{text}} Where does the above sentence stand on Hillary? |||
{{answer_choices[label]}}'
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Accuracy
original_task: true
name: Hillary_guess_passive
reference: ''
83f10728-2347-46e9-b365-724f47e65877: !Template
answer_choices: Neutral ||| Against ||| In favor
id: 83f10728-2347-46e9-b365-724f47e65877
jinja: '{{text}} Where does the author of the above sentence stand on Hillary?
|||
{{answer_choices[label]}}'
metadata: !TemplateMetadata
choices_in_prompt: false
languages:
- en
metrics:
- Accuracy
original_task: true
name: Hillary_guess_passive_author
reference: ''
b521857a-9d4f-4e21-848b-0baf7f4a636c: !Template
answer_choices: Neutral ||| Against ||| In favor
id: b521857a-9d4f-4e21-848b-0baf7f4a636c
jinja: 'How would you describe the stance used in this tweet? {{answer_choices|join(",
")}}
{{text}} |||
{{answer_choices[label]}}'
metadata: !TemplateMetadata
choices_in_prompt: true
languages:
- en
metrics:
- Accuracy
original_task: true
name: Hillary_how_describe
reference: ''