From 99f62eed3b7260cf75aed8b515ee77fe7f269ccb Mon Sep 17 00:00:00 2001 From: deep1 <> Date: Sat, 16 Sep 2023 20:00:59 +0800 Subject: [PATCH] add templates --- notebooks/104_scratch_copy_templates.ipynb | 55 +++ src/prompts/prompt_loading.py | 16 +- .../templates/amazon_polarity/templates.yaml | 217 ++++++++++ .../templates/crows_pairs/templates.yaml | 195 +++++++++ .../templates/great_code/templates.yaml | 163 ++++++++ .../templates/hate_speech18/templates.yaml | 20 + src/prompts/templates/imdb/templates.yaml | 219 +++++++++++ .../lauritowal/redefine_math/templates.yaml | 28 ++ .../medical_questions_pairs/templates.yaml | 155 ++++++++ .../templates/poem_sentiment/templates.yaml | 119 ++++++ src/prompts/templates/qasc/templates.yaml | 141 +++++++ .../reaganjlee/truthful_qa_mc/templates.yaml | 195 +++++++++ .../templates/super_glue/axb/templates.yaml | 145 +++++++ .../templates/super_glue/axg/templates.yaml | 145 +++++++ .../templates/super_glue/boolq/templates.yaml | 178 +++++++++ .../templates/super_glue/cb/templates.yaml | 226 +++++++++++ .../templates/super_glue/copa/templates.yaml | 244 ++++++++++++ .../super_glue/multirc/templates.yaml | 185 +++++++++ .../super_glue/record/templates.yaml | 369 ++++++++++++++++++ .../templates/super_glue/rte/templates.yaml | 160 ++++++++ .../templates/super_glue/wic/templates.yaml | 238 +++++++++++ .../super_glue/wsc.fixed/templates.yaml | 155 ++++++++ .../templates/tweet_eval/emoji/templates.yaml | 68 ++++ .../tweet_eval/emotion/templates.yaml | 103 +++++ .../templates/tweet_eval/hate/templates.yaml | 84 ++++ .../templates/tweet_eval/irony/templates.yaml | 84 ++++ .../tweet_eval/offensive/templates.yaml | 87 +++++ .../tweet_eval/sentiment/templates.yaml | 84 ++++ .../tweet_eval/stance_abortion/templates.yaml | 108 +++++ .../tweet_eval/stance_atheism/templates.yaml | 108 +++++ .../tweet_eval/stance_climate/templates.yaml | 109 ++++++ .../tweet_eval/stance_feminist/templates.yaml | 108 +++++ .../tweet_eval/stance_hillary/templates.yaml | 108 +++++ 33 files changed, 4617 insertions(+), 2 deletions(-) create mode 100644 notebooks/104_scratch_copy_templates.ipynb create mode 100644 src/prompts/templates/amazon_polarity/templates.yaml create mode 100644 src/prompts/templates/crows_pairs/templates.yaml create mode 100644 src/prompts/templates/great_code/templates.yaml create mode 100644 src/prompts/templates/hate_speech18/templates.yaml create mode 100644 src/prompts/templates/imdb/templates.yaml create mode 100644 src/prompts/templates/lauritowal/redefine_math/templates.yaml create mode 100644 src/prompts/templates/medical_questions_pairs/templates.yaml create mode 100644 src/prompts/templates/poem_sentiment/templates.yaml create mode 100644 src/prompts/templates/qasc/templates.yaml create mode 100644 src/prompts/templates/reaganjlee/truthful_qa_mc/templates.yaml create mode 100644 src/prompts/templates/super_glue/axb/templates.yaml create mode 100644 src/prompts/templates/super_glue/axg/templates.yaml create mode 100644 src/prompts/templates/super_glue/boolq/templates.yaml create mode 100644 src/prompts/templates/super_glue/cb/templates.yaml create mode 100644 src/prompts/templates/super_glue/copa/templates.yaml create mode 100644 src/prompts/templates/super_glue/multirc/templates.yaml create mode 100644 src/prompts/templates/super_glue/record/templates.yaml create mode 100644 src/prompts/templates/super_glue/rte/templates.yaml create mode 100644 src/prompts/templates/super_glue/wic/templates.yaml create mode 100644 src/prompts/templates/super_glue/wsc.fixed/templates.yaml create mode 100755 src/prompts/templates/tweet_eval/emoji/templates.yaml create mode 100644 src/prompts/templates/tweet_eval/emotion/templates.yaml create mode 100644 src/prompts/templates/tweet_eval/hate/templates.yaml create mode 100644 src/prompts/templates/tweet_eval/irony/templates.yaml create mode 100644 src/prompts/templates/tweet_eval/offensive/templates.yaml create mode 100644 src/prompts/templates/tweet_eval/sentiment/templates.yaml create mode 100644 src/prompts/templates/tweet_eval/stance_abortion/templates.yaml create mode 100644 src/prompts/templates/tweet_eval/stance_atheism/templates.yaml create mode 100644 src/prompts/templates/tweet_eval/stance_climate/templates.yaml create mode 100644 src/prompts/templates/tweet_eval/stance_feminist/templates.yaml create mode 100644 src/prompts/templates/tweet_eval/stance_hillary/templates.yaml diff --git a/notebooks/104_scratch_copy_templates.ipynb b/notebooks/104_scratch_copy_templates.ipynb new file mode 100644 index 0000000..17d3089 --- /dev/null +++ b/notebooks/104_scratch_copy_templates.ipynb @@ -0,0 +1,55 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Just a quick snipper to copy templates from elk to here" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "cp_from = [\n", + " \"imdb\", # sentiment\n", + " \"amazon_polarity\", # sentiment\n", + " \"super_glue:boolq\", # reading comprehension\n", + " 'tweet_eval:irony', # irony\n", + " 'great_code', # code\n", + " 'qasc', # Question Answering via Sentence Composition (QASC) # dataset has no label column\n", + " \n", + " # Datasets with problems\n", + " 'lauritowal/redefine_math', # dataset has no label column\n", + " 'crows_pairs', # sterotypes FAIL need to specify label columns\n", + " 'hate_speech18', # weird errors\n", + " 'medical_questions_pairs', # medical paraphrase \n", + " 'poem_sentiment', # no only boolean for now\n", + " 'reaganjlee/truthful_qa_mc', # no only bool\n", + " ]\n", + "import shutil\n", + "from pathlib import Path\n", + "from elk.promptsource.templates import TEMPLATES_FOLDER_PATH\n", + "dst_folder = Path(\"../src/prompts/templates/\")\n", + "for ds_string in cp_from:\n", + " ds_name, _, config_name = ds_string.partition(\":\")\n", + " src = Path(TEMPLATES_FOLDER_PATH) / ds_name\n", + " dst = dst_folder / ds_name\n", + " if not dst.exists():\n", + " shutil.copytree(src, dst)\n", + " print(src, dst)\n", + " " + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/src/prompts/prompt_loading.py b/src/prompts/prompt_loading.py index aa3cd21..8acb603 100644 --- a/src/prompts/prompt_loading.py +++ b/src/prompts/prompt_loading.py @@ -136,6 +136,8 @@ def load_prompts( rng = Random(seed) if num_shots > 0: train_name = select_split(ds_dict, "train") + + # TODO don't we need to binarize this? fewshot = FewShotSampler( ds_dict[train_name].shuffle(seed=seed), # TODO: not iterator num_shots=num_shots, @@ -212,6 +214,7 @@ def _convert_to_prompts( rng.shuffle(label_choices) for template in templates: + answer_choices=template.get_fixed_answer_choices_list() for instructed_to_lie in [False, True]: for sys_instr_name, sys_instr in sys_instructions[instructed_to_lie].items(): fake_example = example.copy() @@ -224,10 +227,19 @@ def _convert_to_prompts( if fewshot_iter is not None: # Infinite iterator so we don't need to worry about StopIteration fewshot_examples = next(fewshot_iter) - if instructed_to_lie: fewshot_examples = [{**e, 'label': ~e['label']} for e in fewshot_examples] + if instructed_to_lie: + fewshot_examples = [{**e, 'label': e['label']^0} for e in fewshot_examples] + for e in fewshot_examples: + # arg, check out negation worked + assert e['label']>=0 + assert e['label']<2 + fewshot_texts = [ dict(user=q, response=a.strip()) for q, a in map(template.apply, fewshot_examples) ] + for d in fewshot_texts: + # some of the answers have extra trailing text, that's OK. But extra preceeding text is not, let's check for that + assert any([d['response'].startswith(a) for a in answer_choices]), f"fewshot response `{d['response']}` has extra preceeding text compared to allowed choices: {answer_choices}. template is: {template.name}" prompt_parts = fewshot_texts + prompt_parts prompt_parts[0]['system'] = sys_instr @@ -240,7 +252,7 @@ def _convert_to_prompts( answer=a.strip(), question=q, - answer_choices=template.get_fixed_answer_choices_list(), + answer_choices=answer_choices, template_name=template.name, label_true=example['label'], label_instructed=fake_example['label'], diff --git a/src/prompts/templates/amazon_polarity/templates.yaml b/src/prompts/templates/amazon_polarity/templates.yaml new file mode 100644 index 0000000..fdd87cc --- /dev/null +++ b/src/prompts/templates/amazon_polarity/templates.yaml @@ -0,0 +1,217 @@ +dataset: amazon_polarity +templates: + 1e90a24a-1182-43dd-9445-22f2e56e5761: !Template + answer_choices: Negative ||| Positive + id: 1e90a24a-1182-43dd-9445-22f2e56e5761 + jinja: 'Title: {{title}} + + Review: {{content}} + + Is the review positive or negative? ||| + + {{answer_choices[label]}}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: Is_this_review + reference: '' + 3a48f287-6a4b-4df0-ab2d-2eaf6cb8e53d: !Template + answer_choices: No ||| Yes + id: 3a48f287-6a4b-4df0-ab2d-2eaf6cb8e53d + jinja: 'Based on this review, would the user recommend this product? + + === + + Review: {{content}} + + Answer: ||| {{answer_choices[label]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Accuracy + original_task: true + name: User_recommend_this_product + reference: 'Reformulation equivalent to sent analysis: would the user recommend + this product?' + 592caf8f-f8ff-426a-a61b-b7e95ed510b6: !Template + answer_choices: No ||| Yes + id: 592caf8f-f8ff-426a-a61b-b7e95ed510b6 + jinja: 'Is this product review positive? + + Title: {{title}} + + Review: {{content}} + + Answer: ||| {{answer_choices[label]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Accuracy + original_task: true + name: Is_this_product_review_positive + reference: '' + 6c5f5813-0443-4944-b77a-c0d08885c16b: !Template + answer_choices: Negative ||| Positive + id: 6c5f5813-0443-4944-b77a-c0d08885c16b + jinja: 'Consider the following example: '''''' {{content}} '''''' + + Between {{answer_choices[0]}} and {{answer_choices[1]}}, which is the sentiment + of this example? ||| {{answer_choices[label]}}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: [] + original_task: true + name: burns_2 + reference: '' + 7c5f5813-0443-4944-b77a-c0d08885c16b: !Template + answer_choices: Negative ||| Positive + id: 7c5f5813-0443-4944-b77a-c0d08885c16b + jinja: 'Consider the following example: '''''' {{content}} '''''' + + Between {{answer_choices[0]}} and {{answer_choices[1]}}, the sentiment of this + example is ||| {{answer_choices[label]}}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: burns_1 + reference: '' + 745b9c05-10df-4a7e-81ad-1b88cefcb166: !Template + answer_choices: Yes ||| No + id: 745b9c05-10df-4a7e-81ad-1b88cefcb166 + jinja: 'Title: {{title}} + + Review: {{content}} + + Is this product review negative?||| + + {{answer_choices[label]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Accuracy + original_task: true + name: Is_this_review_negative + reference: '' + 8abb5377-5dd3-4402-92a5-0d81adb6a325: !Template + answer_choices: Negative ||| Positive + id: 8abb5377-5dd3-4402-92a5-0d81adb6a325 + jinja: 'Title: {{title}} + + Review: {{content}} + + Does this product review convey a negative or positive sentiment?||| + + {{answer_choices[label]}}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: convey_negative_or_positive_sentiment + reference: '' + 9df70cdf-f8ed-4e79-8e2f-b4668058d637: !Template + answer_choices: Negative ||| Positive + id: 9df70cdf-f8ed-4e79-8e2f-b4668058d637 + jinja: 'Is there a negative or positive tone to this product review? + + === + + Title: {{title}} + + Review: {{content}} + + Answer: ||| {{answer_choices[label]}}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: negative_or_positive_tone + reference: '' + b13369e8-0500-4e93-90d4-8e6814bfb97b: !Template + answer_choices: dissatisfied ||| satisfied + id: b13369e8-0500-4e93-90d4-8e6814bfb97b + jinja: 'Here is a review left by a customer on a product. Would you say he was + {{answer_choices[1]}} or {{answer_choices[0]}}? + + Title: {{title}} + + Review: {{content}} + + ||| + + {{answer_choices[label]}} ' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: user_satisfied + reference: '' + b13369e8-0500-4e93-90d4-8e6814bfb98b: !Template + answer_choices: decrease ||| increase + id: b13369e8-0500-4e93-90d4-8e6814bfb98b + jinja: 'You are considering whether to buy a product. You look at the reviews. + Would the following review {{answer_choices[0]}} or {{answer_choices[1]}} the + chances of you buying the product? + + Review title: {{title}} + + Product review: {{content}} + + ||| + + {{answer_choices[label]}} ' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: would_you_buy + reference: '' + b13369e8-0500-4e93-90d4-8e6814bfb99b: !Template + answer_choices: unflattering ||| flattering + id: b13369e8-0500-4e93-90d4-8e6814bfb99b + jinja: 'Title: {{title}} + + Product review: {{content}} + + Would you say this review depicts the product in a {{answer_choices[1]}} or + {{answer_choices[0]}} light? + + ||| + + {{answer_choices[label]}} ' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: flattering_or_not + reference: '' diff --git a/src/prompts/templates/crows_pairs/templates.yaml b/src/prompts/templates/crows_pairs/templates.yaml new file mode 100644 index 0000000..85d01ea --- /dev/null +++ b/src/prompts/templates/crows_pairs/templates.yaml @@ -0,0 +1,195 @@ +dataset: crows_pairs +templates: + 50238ba3-15ab-46f1-8c8b-fd6e038eceb3: !Template + answer_choices: Yes ||| No + id: 50238ba3-15ab-46f1-8c8b-fd6e038eceb3 + jinja: 'You are presented with two very similar sentences: + + A: {{sent_more}} + + B: {{sent_less}} + + Sentence A is illustrating a stereotype. Yes or no? + + ||| + + {{answer_choices[stereo_antistereo]}}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: false + name: stereotype + reference: '' + 5980c232-d5c1-4cfa-9e50-7533fab7f5d6: !Template + answer_choices: Sentence A demonstrates a stereotype ||| Sentence B violates a + stereotype + id: 5980c232-d5c1-4cfa-9e50-7533fab7f5d6 + jinja: 'Sentence A: {{sent_more}} + + Sentence B: {{sent_less}} + + Select the option that is the most plausible: + + - {{answer_choices[0]}} + + - {{answer_choices[1]}} + + ||| + + {{answer_choices[stereo_antistereo]}}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: false + name: demontraste_or_violate + reference: '' + 75c17e06-5c05-470d-8697-debcf9888944: !Template + answer_choices: null + id: 75c17e06-5c05-470d-8697-debcf9888944 + jinja: '{% if stereo_antistereo == 0 %} + + "{{sent_more}}" + + The previous sentence illustrates a stereotype against a historically disadvantaged + group in the US. Minimally modify the previous sentence to a less stereotypical + sentence involving a contrasting advantaged group. + + ||| + + {{sent_less}} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - BLEU + - ROUGE + original_task: false + name: transform_stereo + reference: '' + 8da874a6-0749-43fd-bff6-1e32d3e0a9f9: !Template + answer_choices: Yes ||| No + id: 8da874a6-0749-43fd-bff6-1e32d3e0a9f9 + jinja: 'A: {{sent_more}} + + B: {{sent_less}} + + I believe sentence A is conveying a stereotype. Is that correct? Answer with + yes or no. + + ||| + + {{answer_choices[stereo_antistereo]}}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: false + name: stereotype_confirm + reference: '' + cc25e0aa-ff4f-40e3-a05a-634cdbd6f1d9: !Template + answer_choices: No ||| Yes + id: cc25e0aa-ff4f-40e3-a05a-634cdbd6f1d9 + jinja: 'You are presented with two very similar sentences: + + A: {{sent_more}} + + B: {{sent_less}} + + Sentence B is illustrating an anti-stereotype. Yes or no? + + ||| + + {{answer_choices[stereo_antistereo]}}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: false + name: anti_stereotype + reference: '' + d1645b27-c4d8-4682-a2ce-cfddb9bdc6ad: !Template + answer_choices: null + id: d1645b27-c4d8-4682-a2ce-cfddb9bdc6ad + jinja: '{% if stereo_antistereo == 1 %} + + "{{sent_less}}" + + The previous sentence violates a stereotype against a historically disadvantaged + group in the US. Minimally modify the previous sentence to a less stereotypical + sentence involving a contrasting advantage group. + + ||| + + {{sent_more}} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - BLEU + - ROUGE + original_task: false + name: transform_anti_stereo + reference: '' + f0711ab7-3093-463a-9493-9ace26df99e3: !Template + answer_choices: No ||| Yes + id: f0711ab7-3093-463a-9493-9ace26df99e3 + jinja: 'Sentence 1: {{sent_more}} + + Sentence 2: {{sent_less}} + + I think that Sentence 2 conveys an anti-stereotype. Is that correct? Answer + yes or no. + + ||| + + {{answer_choices[stereo_antistereo]}}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: false + name: anti_stereotype_confirm + reference: '' + f2f7e1dd-6c23-4dbb-bdb1-4145b42b4344: !Template + answer_choices: Race and color ||| Socioeconomic status and occupation ||| Gender + and gender identity ||| Disability ||| Nationality ||| Sexual orientation ||| + Physical appearance ||| Religion ||| Age + id: f2f7e1dd-6c23-4dbb-bdb1-4145b42b4344 + jinja: 'Here are two sentences. One is more stereotypical than the other. What + kind of stereotypes are these two sentences illustrating? Choose between {{answer_choices + | join(", ") | lower}}. + + Sentence 1: {{sent_more}} + + Sentence 2: {{sent_less}} + + ||| + + {{answer_choices[bias_type]}}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: false + name: which_bias + reference: '' diff --git a/src/prompts/templates/great_code/templates.yaml b/src/prompts/templates/great_code/templates.yaml new file mode 100644 index 0000000..efc5d0d --- /dev/null +++ b/src/prompts/templates/great_code/templates.yaml @@ -0,0 +1,163 @@ +dataset: great_code +templates: + 027215bb-1055-4584-b3ce-3267a8043d3a: !Template + answer_choices: null + id: 027215bb-1055-4584-b3ce-3267a8043d3a + jinja: "{% set mask = 'def (' %}\n{% set indent = ' ' %}\n{% set\ + \ ns = namespace(indent_size=0, result=[], masked=false, target='') %}\n{% for\ + \ token in source_tokens %}\n {% if ns.masked is false and token.startswith('def')\ + \ %}\n {% set ns.target = token.split('def ')[1][:-1] %}\n \ + \ {% set token = mask %}\n {% set ns.masked = true %}\n {%\ + \ endif%}\n {% if token== '#INDENT#' %}\n {% set ns.indent_size = ns.indent_size\ + \ + 1 %}\n {% set ns.result = ns.result + [indent * ns.indent_size] %}\n\ + \ {% elif token == '#NEWLINE#' %}\n {% set ns.result = ns.result\ + \ + [\"\\n\"] %}\n {% elif token == '#UNINDENT#' %}\n {% set ns.indent_size\ + \ = ns.indent_size - 1 %}\n {% else %}\n {% if not loop.first and\ + \ loop.previtem == '#NEWLINE#' %}\n {% set ns.result = ns.result\ + \ + [indent * ns.indent_size] %}\n {% endif %}\n {% set ns.result\ + \ = ns.result + [token | replace('\\\\n', '\\n'), \" \"] %}\n {% endif %}\n\ + {% endfor %}\n{{ns.result | join(\"\") | replace(\" . \", \".\") | replace(\"\ + \ , \", \", \") | replace(\"( \", \"(\") | replace(\" )\", \")\") | replace(\"\ + [ \", \"[\") | replace(\" ]\", \"]\")}}\n\nWhat is the function name?\n|||\n\ + {{ ns.target }}" + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Accuracy + original_task: false + name: function name generation + reference: '' + 24fa908d-daa2-4fd2-bb22-374021c1fc23: !Template + answer_choices: null + id: 24fa908d-daa2-4fd2-bb22-374021c1fc23 + jinja: "{% set result = \"\" %}\n{% set indent = ' ' %}\n{% set ns = namespace(indent_size=0,\ + \ line_number=0, buggy_line=0, bug_location=0, bug_len=0, result=[], result_lines=[])\ + \ %}\n{% set fixed_token = source_tokens[repair_targets[0]] %}\n{% set buggy_line_content\ + \ = \"\" %}\n{% set fixed_buggy_line_content = \"\" %}\n\n{% if has_bug and\ + \ (repair_targets | length > 0) %}\n {% for token in source_tokens %}\n\ + \ {% if loop.index0 == error_location %}\n {% set ns.buggy_line\ + \ = ns.line_number %}\n {% set ns.bug_location = (ns.result |\ + \ join(\"\") | length) %}\n {% set ns.bug_len = (token | length)\ + \ %}\n {% endif%}\n {% if token== '#INDENT#' %}\n {%\ + \ set ns.indent_size = ns.indent_size + 1 %}\n {% set ns.result =\ + \ ns.result + [indent * ns.indent_size] %}\n {% elif token == '#NEWLINE#'\ + \ %}\n {% set ns.result_lines = ns.result_lines + [ns.result |\ + \ join(\"\")] %}\n {% set ns.result = [] %}\n {% set\ + \ ns.line_number = ns.line_number + 1 %}\n {% elif token == '#UNINDENT#'\ + \ %}\n {% set ns.indent_size = ns.indent_size - 1 %}\n {%\ + \ else %}\n {% if not loop.first and loop.previtem == '#NEWLINE#'\ + \ %}\n {% set ns.result = ns.result + [indent * ns.indent_size]\ + \ %}\n {% endif %}\n {% set ns.result = ns.result\ + \ + [token | replace('\\\\n', '\\n'), \" \"] %}\n {% endif %}\n {%\ + \ endfor %}\n {% set ns.result_lines = ns.result_lines + [ns.result | join(\"\ + \")] %}\n {% set result = ns.result_lines | join(\"\\n\") %}\n {{result\ + \ | replace(\" . \", \".\") | replace(\" , \", \", \") | replace(\"( \", \"\ + (\") | replace(\" )\", \")\") | replace(\"[ \", \"[\") | replace(\" ]\", \"\ + ]\")}}\n\n {% set buggy_line_content = ns.result_lines[ns.buggy_line] | trim\ + \ | replace(\" . \", \".\") | replace(\" , \", \", \") | replace(\"( \", \"\ + (\") | replace(\" )\", \")\") | replace(\"[ \", \"[\") | replace(\" ]\", \"\ + ]\") %}\n {% set fixed_buggy_line_content = (ns.result_lines[ns.buggy_line][:ns.bug_location]\ + \ + fixed_token + ns.result_lines[ns.buggy_line][ns.bug_location + ns.bug_len:])\ + \ | trim | replace(\" . \", \".\") | replace(\" , \", \", \") | replace(\"\ + ( \", \"(\") | replace(\" )\", \")\") | replace(\"[ \", \"[\") | replace(\"\ + \ ]\", \"]\")%}\n\n Fix the buggy line: {{buggy_line_content}}\n |||\n\ + \ {{fixed_buggy_line_content}}\n{% endif %}" + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Other + original_task: false + name: fix buggy line + reference: '' + 3c0f36da-834a-4814-86f6-6e55414fbd32: !Template + answer_choices: null + id: 3c0f36da-834a-4814-86f6-6e55414fbd32 + jinja: "{% set mask = '' %}\n{% set indent = ' ' %}\n{% set ns = namespace(indent_size=0,\ + \ result=[]) %}\n\n{% if has_bug %}\n {% for token in source_tokens %}\n\ + \ {% if loop.index0 == error_location %}\n {% set token\ + \ = mask %}\n {% endif%}\n {% if token== '#INDENT#' %}\n \ + \ {% set ns.indent_size = ns.indent_size + 1 %}\n {% set ns.result\ + \ = ns.result + [indent * ns.indent_size] %}\n {% elif token == '#NEWLINE#'\ + \ %}\n {% set ns.result = ns.result + [\"\\n\"] %}\n {%\ + \ elif token == '#UNINDENT#' %}\n {% set ns.indent_size = ns.indent_size\ + \ - 1 %}\n {% else %}\n {% if not loop.first and loop.previtem\ + \ == '#NEWLINE#' %}\n {% set ns.result = ns.result + [indent\ + \ * ns.indent_size] %}\n {% endif %}\n {% set ns.result\ + \ = ns.result + [token | replace('\\\\n', '\\n'), \" \"] %}\n {% endif\ + \ %}\n {% endfor %}\n {{ns.result | join(\"\") | replace(\" . \", \".\"\ + ) | replace(\" , \", \", \") | replace(\"( \", \"(\") | replace(\" )\", \"\ + )\") | replace(\"[ \", \"[\") | replace(\" ]\", \"]\")}}\n \n Given the\ + \ code above, what is a proper replacement for {{mask}}?\n |||\n {{source_tokens[repair_targets[0]]}}\n\ + {% endif %}" + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Accuracy + original_task: true + name: identifier prediction no choices + reference: '' + 57f93ca1-1e27-40ff-8fa4-ba11a499baef: !Template + answer_choices: Yes ||| No + id: 57f93ca1-1e27-40ff-8fa4-ba11a499baef + jinja: "{% set indent = ' ' %}\n{% set ns = namespace(indent_size=0, result=[])\ + \ %}\n{% for token in source_tokens %}\n {% if token== '#INDENT#' %}\n \ + \ {% set ns.indent_size = ns.indent_size + 1 %}\n {% set ns.result\ + \ = ns.result + [indent * ns.indent_size] %}\n {% elif token == '#NEWLINE#'\ + \ %}\n {% set ns.result = ns.result + [\"\\n\"] %}\n {% elif token\ + \ == '#UNINDENT#' %}\n {% set ns.indent_size = ns.indent_size - 1 %}\n\ + \ {% else %}\n {% if not loop.first and loop.previtem == '#NEWLINE#'\ + \ %}\n {% set ns.result = ns.result + [indent * ns.indent_size]\ + \ %}\n {% endif %}\n {% set ns.result = ns.result + [token\ + \ | replace('\\\\n', '\\n'), \" \"] %}\n {% endif %}\n{% endfor %}\n{{ns.result\ + \ | join(\"\") | replace(\" . \", \".\") | replace(\" , \", \", \") | replace(\"\ + ( \", \"(\") | replace(\" )\", \")\") | replace(\"[ \", \"[\") | replace(\"\ + \ ]\", \"]\")}}\n\nIs there a bug in the code above?\n|||\n{{ {True: \"Yes\"\ + , False: \"No\"}[has_bug] }}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: false + name: bug detection + reference: '' + 6b154f4f-50a5-4890-b0fc-a0118061ce0c: !Template + answer_choices: '{% if has_bug %} {% set nss = namespace(choices=[]) %} {% + for i in repair_candidates %} {% set nss.choices = nss.choices + [source_tokens[(i + | int)]] %} {% endfor %} {{nss.choices | unique | join(" ||| ")}} {% + endif %}' + id: 6b154f4f-50a5-4890-b0fc-a0118061ce0c + jinja: "{% set mask = '' %}\n{% set indent = ' ' %}\n{% set ns = namespace(indent_size=0,\ + \ result=[]) %}\n\n{% if has_bug %}\n {% for token in source_tokens %}\n\ + \ {% if loop.index0 == error_location %}\n {% set token\ + \ = mask %}\n {% endif%}\n {% if token== '#INDENT#' %}\n \ + \ {% set ns.indent_size = ns.indent_size + 1 %}\n {% set ns.result\ + \ = ns.result + [indent * ns.indent_size] %}\n {% elif token == '#NEWLINE#'\ + \ %}\n {% set ns.result = ns.result + [\"\\n\"] %}\n {%\ + \ elif token == '#UNINDENT#' %}\n {% set ns.indent_size = ns.indent_size\ + \ - 1 %}\n {% else %}\n {% if not loop.first and loop.previtem\ + \ == '#NEWLINE#' %}\n {% set ns.result = ns.result + [indent\ + \ * ns.indent_size] %}\n {% endif %}\n {% set ns.result\ + \ = ns.result + [token | replace('\\\\n', '\\n'), \" \"] %}\n {% endif\ + \ %}\n {% endfor %}\n {{ns.result | join(\"\") | replace(\" . \", \".\"\ + ) | replace(\" , \", \", \") | replace(\"( \", \"(\") | replace(\" )\", \"\ + )\") | replace(\"[ \", \"[\") | replace(\" ]\", \"]\")}}\n \n Given the\ + \ code above, what is a proper replacement for {{mask}}? Choose among: {{answer_choices\ + \ | join(\", \")}}\n |||\n {{source_tokens[repair_targets[0]]}}\n{% endif\ + \ %}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: false + name: identifier prediction with choices + reference: '' diff --git a/src/prompts/templates/hate_speech18/templates.yaml b/src/prompts/templates/hate_speech18/templates.yaml new file mode 100644 index 0000000..e94a2aa --- /dev/null +++ b/src/prompts/templates/hate_speech18/templates.yaml @@ -0,0 +1,20 @@ +dataset: hate_speech18 +templates: + 3266f9d4-9c80-4e17-a8a6-1fe44ca8f3bf: !Template + answer_choices: noHate ||| hate + id: 3266f9d4-9c80-4e17-a8a6-1fe44ca8f3bf + jinja: '{% if label in [0, 1] %} + + {{text}} Is the sentiment the sentence expresses is a {{answer_choices[1]}} + speech or {{answer_choices[0]}} speech? ||| {{answer_choices[label]}} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: hate_or_noHate_sentiment_analysis + reference: '' diff --git a/src/prompts/templates/imdb/templates.yaml b/src/prompts/templates/imdb/templates.yaml new file mode 100644 index 0000000..f8bdfa9 --- /dev/null +++ b/src/prompts/templates/imdb/templates.yaml @@ -0,0 +1,219 @@ +dataset: imdb +templates: + 02ff2949-0f45-4d97-941e-6fa4c0afbc2d: !Template + answer_choices: negative ||| positive + id: 02ff2949-0f45-4d97-941e-6fa4c0afbc2d + jinja: 'The following movie review expresses what sentiment? {{text}} + + + ||| {{ answer_choices + [label] }}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Accuracy + original_task: true + name: Movie Expressed Sentiment 2 + reference: '' + 2351d12a-e630-4d19-8b41-e199266e38f7: !Template + answer_choices: bad ||| good + id: 2351d12a-e630-4d19-8b41-e199266e38f7 + jinja: '{{text}} + + + Did the reviewer find this movie {{"good or bad"}}? ||| {{ answer_choices + [label] }}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: Reviewer Opinion bad good choices + reference: '' + 5f372fb1-795a-47b6-8ddf-c4fd1579e76a: !Template + answer_choices: negative ||| positive + id: 5f372fb1-795a-47b6-8ddf-c4fd1579e76a + jinja: "{{text}} \nIs this review {{\"positive or negative\"}}? ||| \n{{answer_choices[label]}}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: 'Sentiment with choices ' + reference: '' + 866474a5-1498-46b7-bfee-ac0c5160707f: !Template + answer_choices: negative ||| positive + id: 866474a5-1498-46b7-bfee-ac0c5160707f + jinja: '{{text}} + + + How does the viewer feel about the movie? + + + ||| {{ answer_choices + [label] }}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Accuracy + original_task: true + name: Reviewer Sentiment Feeling + reference: '' + 96538f30-f2c1-430e-8fc6-936a16966d9c: !Template + answer_choices: negative ||| positive + id: 96538f30-f2c1-430e-8fc6-936a16966d9c + jinja: '{{text}} What sentiment does the writer express for the movie? ||| {{ + answer_choices [label] }}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Accuracy + original_task: true + name: Writer Expressed Sentiment + reference: '' + af51297c-38a3-4d6c-a8b5-04b1243d7443: !Template + answer_choices: negative ||| positive + id: af51297c-38a3-4d6c-a8b5-04b1243d7443 + jinja: '{{text}} + + The sentiment expressed for the movie is ||| {{ answer_choices + [label] }}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Accuracy + original_task: true + name: Movie Expressed Sentiment + reference: '' + b93b74ac-fe95-40b4-9610-318b46ab820f: !Template + answer_choices: negative ||| positive + id: b93b74ac-fe95-40b4-9610-318b46ab820f + jinja: '{{text}} + + + What is the sentiment expressed in this text? + + + ||| {{ answer_choices + [label] }}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Accuracy + original_task: true + name: Text Expressed Sentiment + reference: '' + b9b5d79d-f0b3-4bec-a724-f585db3e93ff: !Template + answer_choices: negative ||| positive + id: b9b5d79d-f0b3-4bec-a724-f585db3e93ff + jinja: '{{text}} + + + This is definitely not a ||| {{ answer_choices [1-label]}} review.' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Accuracy + original_task: false + name: Negation template for positive and negative + reference: '' + bd82ba0f-01d4-4fa1-bf8d-07e392c00cd9: !Template + answer_choices: No ||| Yes + id: bd82ba0f-01d4-4fa1-bf8d-07e392c00cd9 + jinja: '{{text}} + + Did the reviewer enjoy the movie? ||| {{ answer_choices [label]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Accuracy + original_task: true + name: Reviewer Enjoyment Yes No + reference: '' + c70d1687-2421-49a2-9553-91b8bac4cfbe: !Template + answer_choices: negative ||| positive + id: c70d1687-2421-49a2-9553-91b8bac4cfbe + jinja: '{{text}} + + What is the sentiment expressed by the reviewer for the movie? + + ||| {{ answer_choices [label] }}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Accuracy + original_task: true + name: Reviewer Expressed Sentiment + reference: '' + dacb5f03-dc80-428c-b707-8574436675c9: !Template + answer_choices: 0 ||| 1 + id: dacb5f03-dc80-428c-b707-8574436675c9 + jinja: 'Consider the following example: '''''' {{text}} '''''' + + + Between {{ answer_choices[0] }} and {{answer_choices[1] }}, which is the sentiment + of this example? + + ||| {{ answer_choices[label] }}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: burns_2 + reference: '' + e01970ab-42c0-4e6e-a08f-4940d889ef37: !Template + answer_choices: They didn't like it! ||| They loved it + id: e01970ab-42c0-4e6e-a08f-4940d889ef37 + jinja: '{{text}} + + How does the reviewer feel about the movie? ||| + + {{ answer_choices[label] }}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Accuracy + original_task: true + name: Reviewer Enjoyment + reference: '' + eb791ab2-d2b4-4be6-a569-64086983abee: !Template + answer_choices: 0 ||| 1 + id: eb791ab2-d2b4-4be6-a569-64086983abee + jinja: 'Consider the following example: '''''' {{text}} '''''' + + Between {{ answer_choices[0] }} and {{answer_choices[1] }}, the sentiment of + this example is ||| {{ answer_choices[label] }}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: burns_1 + reference: '' diff --git a/src/prompts/templates/lauritowal/redefine_math/templates.yaml b/src/prompts/templates/lauritowal/redefine_math/templates.yaml new file mode 100644 index 0000000..4645ed5 --- /dev/null +++ b/src/prompts/templates/lauritowal/redefine_math/templates.yaml @@ -0,0 +1,28 @@ +dataset: lauritowal/redefine_math +templates: + 02ff2949-0f45-4d97-941e-6fa4c0afbc2d: !Template + answer_choices: 0 ||| 1 + id: 02ff2949-0f45-4d97-941e-6fa4c0afbc2f + jinja: Question... {{text}} ||| {{ answer_choices [label] }} + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Accuracy + original_task: true + name: Choices + reference: '' + b2ff2949-0f45-4d97-941e-6fa4c0afbc2d: !Template + answer_choices: 0 ||| 1 + id: b2ff2949-0f45-4d97-941e-6fa4c0afbc2f + jinja: Test ||| {{ answer_choices [label] }} + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Accuracy + original_task: true + name: Bananas + reference: '' diff --git a/src/prompts/templates/medical_questions_pairs/templates.yaml b/src/prompts/templates/medical_questions_pairs/templates.yaml new file mode 100644 index 0000000..99951fe --- /dev/null +++ b/src/prompts/templates/medical_questions_pairs/templates.yaml @@ -0,0 +1,155 @@ +dataset: medical_questions_pairs +templates: + 18c92f97-0655-4f67-aca1-69f8e4fbb11e: !Template + answer_choices: different ||| paraphrase + id: 18c92f97-0655-4f67-aca1-69f8e4fbb11e + jinja: 'In the context of healthcare questionnaires, it is often necessary to + find out if two questions are paraphrases of one another. Given the following + question: + + + Question 1: {{question_1}} + + + Is the following question a {{answer_choices[1]}} or {{answer_choices[0]}}? + + + Question 2: {{question_2}} + + + + ||| + + + {{answer_choices[label]}}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: in_the_context + reference: '' + 316f2ff7-45f8-4997-9c5f-dfe5fb7f9808: !Template + answer_choices: False ||| True + id: 316f2ff7-45f8-4997-9c5f-dfe5fb7f9808 + jinja: "Question 1: {{question_1}}\n\nand\n\nQuestion 2: {{question_2}}\n\n Is\ + \ it {{answer_choices[1]}} or {{answer_choices[0]}} that the two questions above\ + \ are paraphrases of each other?\n\n|||\n\n {{answer_choices[label]}} " + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: true_or_false + reference: '' + 535ed335-de9d-41df-a026-28487c832bfa: !Template + answer_choices: no ||| yes + id: 535ed335-de9d-41df-a026-28487c832bfa + jinja: 'Question: In the context of healthcare do the following questions mean + the same thing? + + + Question 1: {{question_1}} + + + Question 2: {{question_2}} + + + {{answer_choices[1]}} or {{answer_choices[0]}}? + + ||| + + {{answer_choices[label]}}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: yes_or_no_in_the_context + reference: template about question pattern + 681dc0d2-a771-41ae-aa00-d1f59ab01197: !Template + answer_choices: not duplicates ||| duplicates + id: 681dc0d2-a771-41ae-aa00-d1f59ab01197 + jinja: "Question 1: {{question_1}}\n\nQuestion 2: {{question_2}}\n\n Pick one\ + \ of the following options:\n Questions are {{answer_choices[1]}} or {{answer_choices[0]}}\n\ + \n ||| {{ answer_choices[label] }}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: duplicates + reference: '' + 7be2b267-8d5c-466b-9fd4-1fbbae442938: !Template + answer_choices: No ||| Yes + id: 7be2b267-8d5c-466b-9fd4-1fbbae442938 + jinja: "Question 1: {{question_1}}\n\nQuestion 2: {{question_2}}\n\nQuestion:\ + \ Is Question 1 asking the same question as Question 2? {{answer_choices[1]}}\ + \ or {{answer_choices[0]}}?\n\n\n ||| {{ answer_choices[label] }}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: same_question + reference: context of healthcare + 854ebbe0-8968-4967-a346-4e4d6f98cf73: !Template + answer_choices: False ||| True + id: 854ebbe0-8968-4967-a346-4e4d6f98cf73 + jinja: "Question 1: {{question_1}}\n\nOne possible way of paraphrasing the same\ + \ question is: \n\nQuestion 2: {{question_2}}\n\n{{answer_choices[1]}} or {{answer_choices[0]}}?\n\ + \n ||| {{ answer_choices[label] }}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: possible_way + reference: '' + b388913a-9b0d-43a1-8bf9-83319ebf38b2: !Template + answer_choices: related question ||| rewrite + id: b388913a-9b0d-43a1-8bf9-83319ebf38b2 + jinja: 'Original question: {{question_1}} + + Given this question, doctors were asked to either: - Rewrite the question so + that it kept the same intent - Create a related question for which the original + answer is unrelated or wrong + + Is the following question a {{answer_choices[1]}} or {{answer_choices[0]}}? + + New question: {{question_2}} ||| {{answer_choices[label]}}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: rewrite_or_related + reference: Using knowledge of dataset creation + c8df74ce-0ae7-4e70-9322-aaf9921ae3b1: !Template + answer_choices: dissimilar ||| similar + id: c8df74ce-0ae7-4e70-9322-aaf9921ae3b1 + jinja: "These two questions are either {{answer_choices[1]}} or {{answer_choices[0]}}.\ + \ \n\n{{question_1}} \n\nand\n\n{{question_2}}\n\nWhich is it? ||| {{answer_choices[label]}}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: similar_dissimilar + reference: '' diff --git a/src/prompts/templates/poem_sentiment/templates.yaml b/src/prompts/templates/poem_sentiment/templates.yaml new file mode 100644 index 0000000..ae3da7c --- /dev/null +++ b/src/prompts/templates/poem_sentiment/templates.yaml @@ -0,0 +1,119 @@ +dataset: poem_sentiment +templates: + 2714baf0-5d19-4781-a60f-f44cd95935f7: !Template + answer_choices: negative ||| positive ||| neutral ||| mixed + id: 2714baf0-5d19-4781-a60f-f44cd95935f7 + jinja: '{{verse_text}} Is the sentiment the poet express for the poem {{answer_choices[0]}}, + {{answer_choices[1]}}, {{answer_choices[2]}} or {{answer_choices[3]}}? ||| {{ + answer_choices [label] }}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: positive_or_negative_sentiment_variation_2 + reference: '' + 574ab816-b0bc-4049-a5a5-dcf8f4280dc5: !Template + answer_choices: negative ||| positive ||| neutral ||| mixed + id: 574ab816-b0bc-4049-a5a5-dcf8f4280dc5 + jinja: The following poem expresses what sentiment? {{verse_text}} ||| {{ answer_choices + [label] }} + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Accuracy + original_task: true + name: guess_sentiment_without_options_variation_2 + reference: '' + 7801d04c-4f42-4411-a552-9614c8c3fd53: !Template + answer_choices: negative ||| positive ||| neutral ||| mixed + id: 7801d04c-4f42-4411-a552-9614c8c3fd53 + jinja: '{{verse_text}} The sentiment expressed in the poem is ||| {{ answer_choices + [label] }}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Accuracy + original_task: true + name: guess_sentiment_without_options_variation_1 + reference: '' + 9fa8eeb4-314b-4850-a28b-0f53bca006d8: !Template + answer_choices: negative ||| positive ||| neutral ||| mixed + id: 9fa8eeb4-314b-4850-a28b-0f53bca006d8 + jinja: '{{verse_text}} What is the sentiment expressed in this poem? ||| {{ answer_choices + [label] }} + + ' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Accuracy + original_task: true + name: positive_or_negative_sentiment_variation_1 + reference: '' + aecb3d13-ff68-4e60-a382-87191940bd5b: !Template + answer_choices: negative ||| positive ||| neutral ||| mixed + id: aecb3d13-ff68-4e60-a382-87191940bd5b + jinja: '{{verse_text}} The most appropriate word out of {{"negative"}}, {{"positive"}}, + {{"neutral"}} and {{"mixed"}}, which express the poet''s sentiment is: ||| {{ + answer_choices [label] }}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: most_appropriate_sentiment + reference: '' + ca15cecb-4ee6-4445-a0f4-6ef5cd519923: !Template + answer_choices: negative ||| positive ||| neutral ||| mixed + id: ca15cecb-4ee6-4445-a0f4-6ef5cd519923 + jinja: "{{verse_text}} What sentiment does this poem express? \nOptions: {{answer_choices[0]}},\ + \ {{answer_choices[1]}}, {{answer_choices[2]}}, {{answer_choices[3]}}.\nAnswer:\ + \ ||| {{ answer_choices [label] }}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: question_answer_format + reference: '' + f87a7ba0-11f7-41f9-bee6-94d0ad6e597a: !Template + answer_choices: negative ||| positive ||| neutral ||| mixed + id: f87a7ba0-11f7-41f9-bee6-94d0ad6e597a + jinja: Does this poem express a {{"negative"}}, {{"positive"}}, {{"neutral"}} + or {{"mixed"}} sentiment? {{verse_text}} ||| {{ answer_choices [label] }} + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: guess_sentiment_from_given_options + reference: '' + fdbebd3d-6517-4be1-8771-489e2de658ef: !Template + answer_choices: negative ||| positive ||| neutral ||| mixed + id: fdbebd3d-6517-4be1-8771-489e2de658ef + jinja: '{{verse_text}} What is the sentiment that the poet wants the readers to + feel through the verse mentioned above?||| {{ answer_choices [label] }}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Accuracy + original_task: true + name: guess_sentiment_without_options + reference: '' diff --git a/src/prompts/templates/qasc/templates.yaml b/src/prompts/templates/qasc/templates.yaml new file mode 100644 index 0000000..5d05a22 --- /dev/null +++ b/src/prompts/templates/qasc/templates.yaml @@ -0,0 +1,141 @@ +dataset: qasc +templates: + 3e1e6ca0-b95e-4e68-bb6a-cd47c8429658: !Template + answer_choices: Yes ||| No + id: 3e1e6ca0-b95e-4e68-bb6a-cd47c8429658 + jinja: "If I tell you that {{combinedfact[0]|capitalize}}{{ combinedfact[1:]|trim('.')\ + \ }}, and ask you the question \"{{ question[0]|lower }}{{ question[1:] }}\"\ + , is the correct answer \"{{ choices.text[0][0]|lower}}{{ choices.text[0][1:]|trim('.')\ + \ }}\"? \n\n||| \n\n{% if answerKey == choices.label[0] %} Yes {% else %} No\ + \ {% endif %}" + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Accuracy + original_task: false + name: is_correct_1 + reference: '' + 40ef67db-dff3-4e7b-b167-f7e54b400c74: !Template + answer_choices: '{{choices.text | join("|||")}}' + id: 40ef67db-dff3-4e7b-b167-f7e54b400c74 + jinja: "{{ fact1[0]|capitalize }}{{ fact1[1:]|trim|trim('.') }}, and {{fact2[0]|lower\ + \ }}{{ fact2[1:]|trim|trim('.') }}. Given these facts, {{ question[0]|lower\ + \ }}{{question[1:]|trim('?') }} among the following options:\n- {{answer_choices\ + \ | join(\"\\n - \") }}\n\n||| \n\n{% for choice in choices.label %} {% if choice\ + \ == answerKey %}{{ answer_choices[loop.index - 1] }}{% endif %}{% endfor %} " + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: qa_with_separated_facts_1 + reference: Question Answering with separated facts. + 5d63c186-e047-49dd-b5fd-c4a574f6f0e2: !Template + answer_choices: '{{choices.text | join("|||")}}' + id: 5d63c186-e047-49dd-b5fd-c4a574f6f0e2 + jinja: "Fact 1: {{ fact1[0]|capitalize }}{{ fact1[1:]|trim|trim('.') }}.\n\nFact\ + \ 2: {{fact2[0]|capitalize }}{{ fact2[1:]|trim|trim('.') }}.\n\nGiven the two\ + \ facts above, {{ question[0]|lower }}{{question[1:]|trim('?') }}?\n\n||| \n\ + \n{% for choice in choices.label %} {% if choice == answerKey %}{{ answer_choices[loop.index\ + \ - 1] }}{% endif %}{% endfor %} " + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Accuracy + original_task: true + name: qa_with_separated_facts_3 + reference: Question Answering with separated facts. + 604dd379-21f1-4d30-af2f-22f9a8a97ceb: !Template + answer_choices: '{{choices.text | join("|||")}}' + id: 604dd379-21f1-4d30-af2f-22f9a8a97ceb + jinja: "You are presented with the question \"{{ question }}\" and the following\ + \ answer choices: \n- {{answer_choices | join(\"\\n - \") }}\n\nNow knowing\ + \ that {{ fact1[0]|lower }}{{ fact1[1:]|trim|trim('.') }} and {{fact2[0]|lower\ + \ }}{{ fact2[1:]|trim|trim('.') }}, choose the best answer.\n\n||| \n\n{% for\ + \ choice in choices.label %} {% if choice == answerKey %}{{ answer_choices[loop.index\ + \ - 1] }}{% endif %}{% endfor %} " + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: qa_with_separated_facts_4 + reference: Question Answering with separated facts. + 67cb1b35-e5d4-490b-beb8-dc9c0be9c298: !Template + answer_choices: '{{choices.text | join("|||")}}' + id: 67cb1b35-e5d4-490b-beb8-dc9c0be9c298 + jinja: "You are presented with the quiz \"{{ question }}\" \n\nBut you don't know\ + \ the answer, so you turn to your teacher to ask for hints. He says that \"\ + {{ fact1[0]|lower }}{{ fact1[1:]|trim|trim('.') }}\" and \"{{fact2[0]|lower\ + \ }}{{ fact2[1:]|trim|trim('.') }}\". \n\nSo, what's the best answer to the\ + \ question?\n\n||| \n\n{% for choice in choices.label %} {% if choice == answerKey\ + \ %}{{ answer_choices[loop.index - 1] }}{% endif %}{% endfor %} " + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Accuracy + original_task: true + name: qa_with_separated_facts_5 + reference: Question Answering with separated facts. + b5b61423-8655-408d-a8e6-81a5eaaac2aa: !Template + answer_choices: '{{choices.text | join("|||")}}' + id: b5b61423-8655-408d-a8e6-81a5eaaac2aa + jinja: "If {{ combinedfact[0]|lower }}{{ combinedfact[1:]|trim|trim('.') }}, then\ + \ {{ question[0]|lower }}{{question[1:]|trim|trim('?') }}?\n\nAnswer choices:\n\ + - {{answer_choices | join(\"\\n - \") }}\n||| \n\n{% for choice in choices.label\ + \ %} {% if choice == answerKey %}{{ answer_choices[loop.index - 1] }}{% endif\ + \ %}{% endfor %} " + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: false + name: qa_with_combined_facts_1 + reference: '' + c251edaf-a063-40fe-bd49-038843fcdb23: !Template + answer_choices: Yes ||| No + id: c251edaf-a063-40fe-bd49-038843fcdb23 + jinja: "Do you think the right answer to the question \"{{ question[0]|lower }}{{\ + \ question[1:] }}\" is \"{{ choices.text[1][0]|lower}}{{ choices.text[1][1:]|trim('.')\ + \ }}\", given that\n {{combinedfact[0]|lower}}{{ combinedfact[1:]|trim('.')\ + \ }}?\n ||| \n{% if answerKey == choices.label[0] %} Yes {% else %} No {% endif\ + \ %} " + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Accuracy + original_task: false + name: is_correct_2 + reference: '' + c7cd51f1-ea49-4d6a-a422-46624333b7b1: !Template + answer_choices: '{{choices.text | join("|||")}}' + id: c7cd51f1-ea49-4d6a-a422-46624333b7b1 + jinja: "Fact 1: {{ fact1[0]|capitalize }}{{ fact1[1:]|trim|trim('.') }}.\n\nFact\ + \ 2: {{fact2[0]|capitalize }}{{ fact2[1:]|trim|trim('.') }}.\n\nGiven the two\ + \ facts above, answer the question \"{{ question }}\" with the following options:\ + \ \n- {{answer_choices | join(\"\\n - \") }}\n\n||| \n\n{% for choice in choices.label\ + \ %} {% if choice == answerKey %}{{ answer_choices[loop.index - 1] }}{% endif\ + \ %}{% endfor %} " + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: qa_with_separated_facts_2 + reference: Question Answering with separated facts. diff --git a/src/prompts/templates/reaganjlee/truthful_qa_mc/templates.yaml b/src/prompts/templates/reaganjlee/truthful_qa_mc/templates.yaml new file mode 100644 index 0000000..b5cf0d0 --- /dev/null +++ b/src/prompts/templates/reaganjlee/truthful_qa_mc/templates.yaml @@ -0,0 +1,195 @@ +dataset: truthful_qa_mc +binarize: true +templates: + 00fa401f-3329-48fa-be4a-1b6725292ee6: !Template + answer_choices: A ||| B ||| C ||| D + id: 00fa401f-3329-48fa-be4a-1b6725292ee6 + jinja: 'Answer the following question: {{ question }} + + Choice 1: {{''. ''.join([answer_choices[label], choices[label]])}}. Choice 2: {{''. ''.join([answer_choices[1 - label], choices[1 - label]])}}. Between choice 1 and choice 2, + the answer of this question is ||| {{answer_choices[label]}}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: [] + original_task: true + name: burns_1 + reference: Burns et al. + 01fa401f-3329-48fa-be4a-1b6725292ee6: !Template + answer_choices: A ||| B ||| C ||| D + id: 01fa401f-3329-48fa-be4a-1b6725292ee6 + jinja: 'Answer the following question: {{ question }} + + Choice 1: {{''. ''.join([answer_choices[label], choices[label]])}}. Choice 2: {{''. ''.join([answer_choices[1 - label], choices[1 - label]])}}. Between choice 1 and choice 2, + what is the answer to this question? ||| {{answer_choices[label]}}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: [] + original_task: true + name: burns_2 + reference: Burns et al. + 02fa401f-3329-48fa-be4a-1b6725292ee6: !Template + answer_choices: A ||| B ||| C ||| D + id: 02fa401f-3329-48fa-be4a-1b6725292ee6 + jinja: 'Answer the following question: {{ question }} + + Which is the answer of this question, choice 1: {{''. ''.join([answer_choices[label], choices[label]])}}, or + choice 2: {{''. ''.join([answer_choices[1 - label], choices[1 - label]])}} ||| + + {{answer_choices[label]}}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: [] + original_task: true + name: burns_3 + reference: Burns et al. + 04fa401f-3329-48fa-be4a-1b6725292ee6: !Template + answer_choices: A ||| B ||| C ||| D + id: 04fa401f-3329-48fa-be4a-1b6725292ee6 + jinja: '{{ question }} + + Choice 1: {{''. ''.join([answer_choices[label], choices[label]])}}. Choice 2: {{''. ''.join([answer_choices[1 - label], choices[1 - label]])}}.Between choice 1 and choice 2, + the topic of this example is ||| {{answer_choices[label]}}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: [] + original_task: true + name: burns_5 + reference: Burns et al. + 05fa401f-3329-48fa-be4a-1b6725292ee6: !Template + answer_choices: A ||| B ||| C ||| D + id: 05fa401f-3329-48fa-be4a-1b6725292ee6 + jinja: '{{ question }} + + Which is the answer to this question, choice 1: {{''. ''.join([answer_choices[label], choices[label]])}}, or + choice 2: {{''. ''.join([answer_choices[1 - label], choices[1 - label]])}} ||| + + {{answer_choices[label]}}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: [] + original_task: true + name: burns_6 + reference: Burns et al. + 06fa401f-3329-48fa-be4a-1b6725292ee6: !Template + answer_choices: A ||| B ||| C ||| D + id: 06fa401f-3329-48fa-be4a-1b6725292ee6 + jinja: '{{ question }} + + What is the answer to the question, choice 1: {{''. ''.join([answer_choices[label], choices[label]])}}, + or choice 2: {{''. ''.join([answer_choices[1 - label], choices[1 - label]])}} ||| + + {{answer_choices[label]}}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: [] + original_task: true + name: burns_7 + reference: Burns et al. + 07fa401f-3329-48fa-be4a-1b6725292ee6: !Template + answer_choices: A ||| B ||| C ||| D + id: 07fa401f-3329-48fa-be4a-1b6725292ee6 + jinja: '{{ question }} + + What choice best answers this question, choice 1: {{''. ''.join([answer_choices[label], choices[label]])}}, + or choice 2: {{''. ''.join([answer_choices[1 - label], choices[1 - label]])}} ||| + + {{answer_choices[label]}}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: [] + original_task: true + name: burns_8 + reference: Burns et al. + 824ecb55-ecad-40c2-8033-f2fa0add2ddf: !Template + answer_choices: A ||| B ||| C ||| D + id: 824ecb55-ecad-40c2-8033-f2fa0add2ddf + jinja: '{{ question }} Given a list of choices: + {% for k in range(choices | length) %} + + {{''. ''.join([answer_choices[k], choices[k]])}} + + {% endfor %} + + what choice best answers the question? ||| {{ answer_choices[label] }}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: given_list_what_category_does_the_paragraph_belong_to + reference: '' + 8eda7e71-6734-486f-b883-e99d3f14c0bf: !Template + answer_choices: A ||| B ||| C ||| D + id: 8eda7e71-6734-486f-b883-e99d3f14c0bf + jinja: 'Pick one answer for the following question. The options are - + {% for k in range(choices | length) %} + + {{''. ''.join([answer_choices[k], choices[k]])}} + + {% endfor %}. + {{ question }} ||| + + {{ answer_choices[label] }}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: pick_one_category_for_the_following_text + reference: '' + 9dfa5d15-96bc-41ee-ad89-4f8df5c4ff67: !Template + answer_choices: A ||| B ||| C ||| D + id: 9dfa5d15-96bc-41ee-ad89-4f8df5c4ff67 + jinja: '{{ question }} Given a choice of answers + {% for k in range(choices | length) %} + + {{''. ''.join([answer_choices[k], choices[k]])}} + + {% endfor %}, the question + is best answered by which one? ||| {{ answer_choices[label] }}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: 'given_a_choice_of_categories ' + reference: '' + f72fa410-3278-4f62-91f0-f9edf4a4e792: !Template + answer_choices: A ||| B ||| C ||| D + id: f72fa410-3278-4f62-91f0-f9edf4a4e792 + jinja: '"{{ question }}", given a list of choices: + {% for k in range(choices | length) %} + + {{''. ''.join([answer_choices[k], choices[k]])}} + + {% endfor %}, what answer belongs + to the question? ||| {{ answer_choices[label] }}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: given_a_list_of_category_what_does_the_title_belong_to + reference: '' diff --git a/src/prompts/templates/super_glue/axb/templates.yaml b/src/prompts/templates/super_glue/axb/templates.yaml new file mode 100644 index 0000000..bb25f45 --- /dev/null +++ b/src/prompts/templates/super_glue/axb/templates.yaml @@ -0,0 +1,145 @@ +dataset: super_glue +subset: axb +templates: + 1ae41916-7b4d-4ef3-b414-bfadd95d67e2: !Template + answer_choices: Yes ||| No + id: 1ae41916-7b4d-4ef3-b414-bfadd95d67e2 + jinja: 'Given {{sentence1}} Should we assume that "{{sentence2}}" is true? Yes + or no? ||| {{ answer_choices[label] }} ' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: should assume + reference: Webson & Pavlick 2021 + 1b2d6e85-a5a9-4d1b-9e3b-630b490c6a34: !Template + answer_choices: Yes ||| No + id: 1b2d6e85-a5a9-4d1b-9e3b-630b490c6a34 + jinja: '{{sentence1}} Are we justified in saying that "{{sentence2}}"? Yes or + no? ||| {{ answer_choices[label] }} ' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: justified in saying + reference: Webson & Pavlick 2021 + 23651f68-93cc-441f-b826-30dd2c6d6a93: !Template + answer_choices: Yes ||| No + id: 23651f68-93cc-441f-b826-30dd2c6d6a93 + jinja: Given that {{sentence1}} Does it follow that {{sentence2}} Yes or no? ||| + {{ answer_choices[label] }} + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: does it follow that + reference: Sanh et al. 2021 + 552d6c20-ab5b-462f-b5fb-3c7b80c78dcc: !Template + answer_choices: Yes ||| No + id: 552d6c20-ab5b-462f-b5fb-3c7b80c78dcc + jinja: '{{sentence1}} Using only the above description and what you know about + the world, is "{{sentence2}}" definitely correct? Yes or no? ||| {{ answer_choices[label] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: MNLI crowdsource + reference: Adapted from Williams et al. 2018's instructions to crowdsourcing workers. + 908be561-caf4-4416-9fe9-9919c3998681: !Template + answer_choices: Yes ||| No + id: 908be561-caf4-4416-9fe9-9919c3998681 + jinja: 'Given {{sentence1}} Is it guaranteed true that "{{sentence2}}"? Yes or + no? ||| {{ answer_choices[label] }} ' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: guaranteed true + reference: Webson & Pavlick 2021 + bae54ef5-c3be-4862-bdd4-a559ed04eb31: !Template + answer_choices: Yes ||| No + id: bae54ef5-c3be-4862-bdd4-a559ed04eb31 + jinja: 'Suppose {{sentence1}} Can we infer that "{{sentence2}}"? Yes or no? ||| + {{ answer_choices[label] }} ' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: can we infer + reference: Webson & Pavlick 2021 + c92d765f-83b1-4684-a0a3-580929b5e46b: !Template + answer_choices: Yes ||| No + id: c92d765f-83b1-4684-a0a3-580929b5e46b + jinja: "{{sentence1}} \n\nQuestion: Does this imply that \"{{sentence2}}\"? Yes\ + \ or no? ||| {{answer_choices[label]}}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: does this imply + reference: Sanh et al. 2021 + cb68ee27-c0a3-440b-b595-e90fe89539c3: !Template + answer_choices: Yes ||| No + id: cb68ee27-c0a3-440b-b595-e90fe89539c3 + jinja: 'Given that {{sentence1}} Therefore, it must be true that "{{sentence2}}"? + Yes or no? ||| {{ answer_choices[label] }} ' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: must be true + reference: Sanh et al. 2021 + d57550ef-2f67-46eb-98cb-432dd135be16: !Template + answer_choices: Yes ||| No + id: d57550ef-2f67-46eb-98cb-432dd135be16 + jinja: '{{sentence1}} Based on the previous passage, is it true that "{{sentence2}}"? + Yes or no? ||| {{ answer_choices[label] }}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: based on the previous passage + reference: "Adapted from the BoolQ prompts in Schick & Sch\xFCtze 2021." + d965164b-fa96-41b5-8852-e0f6dfe5524e: !Template + answer_choices: True ||| False + id: d965164b-fa96-41b5-8852-e0f6dfe5524e + jinja: '{{sentence1}} + + Question: {{sentence2}} True or False? ||| {{ answer_choices[label] }}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: GPT-3 style + reference: Same as reported in Figure G31 of the GPT-3 paper. diff --git a/src/prompts/templates/super_glue/axg/templates.yaml b/src/prompts/templates/super_glue/axg/templates.yaml new file mode 100644 index 0000000..3d24743 --- /dev/null +++ b/src/prompts/templates/super_glue/axg/templates.yaml @@ -0,0 +1,145 @@ +dataset: super_glue +subset: axg +templates: + 0f530aa8-b254-4687-8032-bab1a65610c0: !Template + answer_choices: Yes ||| No + id: 0f530aa8-b254-4687-8032-bab1a65610c0 + jinja: 'Given {{premise}} Should we assume that "{{hypothesis}}" is true? Yes + or no? ||| {{ answer_choices[label] }} ' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: should assume + reference: Webson & Pavlick 2021 + 0f8afaef-19a0-472f-9e9f-c803426f8f22: !Template + answer_choices: Yes ||| No + id: 0f8afaef-19a0-472f-9e9f-c803426f8f22 + jinja: "{{premise}} \n\nQuestion: Does this imply that \"{{hypothesis}}\"? Yes\ + \ or no? ||| {{answer_choices[label]}}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: does this imply + reference: Sanh et al. 2021 + 3b7a57e0-7733-4b21-9bed-a381fdc2415f: !Template + answer_choices: Yes ||| No + id: 3b7a57e0-7733-4b21-9bed-a381fdc2415f + jinja: '{{premise}} Based on the previous passage, is it true that "{{hypothesis}}"? + Yes or no? ||| {{ answer_choices[label] }}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: based on the previous passage + reference: "Adapted from the BoolQ prompts in Schick & Sch\xFCtze 2021." + 4361cf07-1b58-478f-b97c-3b140832fb77: !Template + answer_choices: Yes ||| No + id: 4361cf07-1b58-478f-b97c-3b140832fb77 + jinja: 'Given that {{premise}} Therefore, it must be true that "{{hypothesis}}"? + Yes or no? ||| {{ answer_choices[label] }} ' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: must be true + reference: Sanh et al. 2021 + 626823f5-ff12-46d5-9e68-b2dc4bfe7cd4: !Template + answer_choices: True ||| False + id: 626823f5-ff12-46d5-9e68-b2dc4bfe7cd4 + jinja: '{{premise}} + + Question: {{hypothesis}} True or False? ||| {{ answer_choices[label] }}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: GPT-3 style + reference: Same as reported in Figure G31 of the GPT-3 paper. + 7e1439f6-d54d-43e6-bdc7-306ad5fd9203: !Template + answer_choices: Yes ||| No + id: 7e1439f6-d54d-43e6-bdc7-306ad5fd9203 + jinja: 'Given {{premise}} Is it guaranteed true that "{{hypothesis}}"? Yes or + no? ||| {{ answer_choices[label] }} ' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: guaranteed true + reference: Webson & Pavlick 2021 + c008c778-7621-496e-baa3-7b5817400659: !Template + answer_choices: Yes ||| No + id: c008c778-7621-496e-baa3-7b5817400659 + jinja: Given that {{premise}} Does it follow that {{hypothesis}} Yes or no? ||| + {{ answer_choices[label] }} + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: does it follow that + reference: Sanh et al. 2021 + d4a1dd92-e184-4843-bc1f-1f625c833249: !Template + answer_choices: Yes ||| No + id: d4a1dd92-e184-4843-bc1f-1f625c833249 + jinja: '{{premise}} Are we justified in saying that "{{hypothesis}}"? Yes or no? + ||| {{ answer_choices[label] }} ' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: justified in saying + reference: Webson & Pavlick 2021 + db13469f-7161-4670-8a59-8c1137d1fa8b: !Template + answer_choices: Yes ||| No + id: db13469f-7161-4670-8a59-8c1137d1fa8b + jinja: 'Suppose {{premise}} Can we infer that "{{hypothesis}}"? Yes or no? ||| + {{ answer_choices[label] }} ' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: can we infer + reference: Webson & Pavlick 2021 + e21f5367-0cc8-412d-b8d9-78548438a384: !Template + answer_choices: Yes ||| No + id: e21f5367-0cc8-412d-b8d9-78548438a384 + jinja: '{{premise}} Using only the above description and what you know about the + world, is "{{hypothesis}}" definitely correct? Yes or no? ||| {{ answer_choices[label] + }}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: MNLI crowdsource + reference: Adapted from Williams et al. 2018's instructions to crowdsourcing workers. diff --git a/src/prompts/templates/super_glue/boolq/templates.yaml b/src/prompts/templates/super_glue/boolq/templates.yaml new file mode 100644 index 0000000..c5550d1 --- /dev/null +++ b/src/prompts/templates/super_glue/boolq/templates.yaml @@ -0,0 +1,178 @@ +dataset: super_glue +subset: boolq +templates: + 3e386463-1715-4578-9cba-07d11a0d3b61: !Template + answer_choices: False ||| True + id: 3e386463-1715-4578-9cba-07d11a0d3b61 + jinja: 'Passage: {{passage}} + + + After reading this passage, I have a question: {{question}}? True or False? + ||| + + + {% if label != -1 %} + {{answer_choices[label]}} + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: after_reading + reference: '' + 492f0f88-4370-46cd-839b-1de37a55aeda: !Template + answer_choices: No ||| Yes + id: 492f0f88-4370-46cd-839b-1de37a55aeda + jinja: "{{ passage }} \nQuestion: {{ question }}\nAnswer: ||| \n{% if label !=\ + \ -1 %}\n{{ answer_choices[label] }}{% endif %}" + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Accuracy + original_task: true + name: GPT-3 Style + reference: Same as Figure G29, p. 58 of the GPT-3 paper + 6cb6a026-c070-470a-b75d-bb8fdf424e35: !Template + answer_choices: No ||| Yes + id: 6cb6a026-c070-470a-b75d-bb8fdf424e35 + jinja: "{{ passage }} \n\nHaving read that, I wonder {{ question }}? |||\n{% if\ + \ label != -1 %}\n{{ answer_choices[label] }}{% endif %}" + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Accuracy + original_task: true + name: "I wonder\u2026" + reference: '' + 7cf7acdf-e3a2-459f-a3e8-2e2d27dd6aa5: !Template + answer_choices: No ||| Yes + id: 7cf7acdf-e3a2-459f-a3e8-2e2d27dd6aa5 + jinja: 'Text: {{passage}} + + + Answer the following yes/no question: {{question}}? Yes or no? ||| + + + {% if label != -1 %} + {{answer_choices[label]}} + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: yes_no_question + reference: '' + 7d21d974-0624-4d4f-9e8c-644e2d009cb5: !Template + answer_choices: No ||| Yes + id: 7d21d974-0624-4d4f-9e8c-644e2d009cb5 + jinja: "{{ passage }} \n\nHaving read that, could you tell me {{ question }}?\ + \ ||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %}" + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Accuracy + original_task: true + name: "could you tell me\u2026" + reference: '' + 922d3e87-ac58-4731-84d1-f0a40e47afb5: !Template + answer_choices: No ||| Yes + id: 922d3e87-ac58-4731-84d1-f0a40e47afb5 + jinja: "EXAM\n1. Answer by yes or no.\n\nDocument: {{passage}}\nQuestion: {{question}}?\ + \ ||| \n{% if label != -1 %}\n{{answer_choices[label]}}{% endif %}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: exam + reference: '' + 9a1bf459-8047-437c-9def-f21e960429cc: !Template + answer_choices: No ||| Yes + id: 9a1bf459-8047-437c-9def-f21e960429cc + jinja: "Based on the following passage, {{ question }}?\n\n{{ passage }}\ + \ ||| \n\n{% if label != -1 %}{{ answer_choices[label] }}{% endif %}" + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Accuracy + original_task: true + name: based on the following passage + reference: "Adapted from Perez et al. 2021 and Schick & Sch\xFCtz 2021." + 9f4c6b0a-437b-40c0-b467-db4b7218d38d: !Template + answer_choices: False ||| True + id: 9f4c6b0a-437b-40c0-b467-db4b7218d38d + jinja: 'Exercise: read the text and answer the question by True or False. + + + Text: {{passage}} + + Question: {{question}}? ||| + + + {% if label != -1 %} + {{answer_choices[label]}} + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: exercise + reference: '' + b2b3cb60-d6e3-491c-a09a-8201e13e417e: !Template + answer_choices: No ||| Yes + id: b2b3cb60-d6e3-491c-a09a-8201e13e417e + jinja: '{{ passage }} + + Based on the previous passage, {{ question }}? ||| + {% if label != -1 %} + {{ answer_choices[label]}} + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Accuracy + original_task: true + name: based on the previous passage + reference: "Adapted from Perez et al. 2021 and Schick & Sch\xFCtz 2021." + eb78772c-e81e-4b8a-a77b-b75efd1c212a: !Template + answer_choices: False ||| True + id: eb78772c-e81e-4b8a-a77b-b75efd1c212a + jinja: '{{passage}} + + + Q: {{question}}? True or False? ||| + + + {% if label != -1 %} + {{answer_choices[label]}} + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: valid_binary + reference: '' diff --git a/src/prompts/templates/super_glue/cb/templates.yaml b/src/prompts/templates/super_glue/cb/templates.yaml new file mode 100644 index 0000000..895aee3 --- /dev/null +++ b/src/prompts/templates/super_glue/cb/templates.yaml @@ -0,0 +1,226 @@ +dataset: super_glue +subset: cb +templates: + 2e76cd0f-68ca-4f03-83ed-11cf15b25a84: !Template + answer_choices: Yes ||| No ||| Maybe + id: 2e76cd0f-68ca-4f03-83ed-11cf15b25a84 + jinja: 'Suppose {{premise}} Can we infer that "{{hypothesis}}"? Yes, no, or maybe? + ||| {% if label !=-1 %}{{ answer_choices[label] }}{% endif %} ' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: can we infer + reference: Webson & Pavlick 2021 + 358860fd-61ad-45fd-92a6-a72ca9107ebc: !Template + answer_choices: Yes ||| No ||| Maybe + id: 358860fd-61ad-45fd-92a6-a72ca9107ebc + jinja: '{{premise}} Based on the previous passage, is it true that "{{hypothesis}}"? + Yes, no, or maybe? ||| {% if label !=-1 %}{{ answer_choices[label] }}{% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: based on the previous passage + reference: "Adapted from the BoolQ prompts in Schick & Sch\xFCtze 2021." + 3f43a599-ffdb-490e-8952-c0ce41dd4621: !Template + answer_choices: True ||| False ||| Inconclusive + id: 3f43a599-ffdb-490e-8952-c0ce41dd4621 + jinja: '{{premise}} Based on that information, is the claim: "{{hypothesis}}" + {{"true"}}, {{"false"}}, or {{"inconclusive"}}? ||| {% if label !=-1 %}{{ answer_choices[label] + }}{% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: claim true/false/inconclusive + reference: Sanh et al. 2021 + 404eed25-558a-4d39-9515-7de46d60d4e0: !Template + answer_choices: Yes ||| No ||| Maybe + id: 404eed25-558a-4d39-9515-7de46d60d4e0 + jinja: Given that {{premise}} Does it follow that {{hypothesis}} Yes, no, or maybe? + ||| {% if label !=-1 %}{{ answer_choices[label] }}{% endif %} + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: does it follow that + reference: Sanh et al. 2021 + 5c9b1fa9-93f0-4f82-b9e3-e0967e4d7260: !Template + answer_choices: Yes ||| No ||| Maybe + id: 5c9b1fa9-93f0-4f82-b9e3-e0967e4d7260 + jinja: '{{premise}} Are we justified in saying that "{{hypothesis}}"? Yes, no, + or maybe? ||| {% if label !=-1 %}{{ answer_choices[label] }}{% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: justified in saying + reference: Webson & Pavlick 2021 + 6b0c6191-183d-4731-8050-ab17c909335c: !Template + answer_choices: Always ||| Never ||| Sometimes + id: 6b0c6191-183d-4731-8050-ab17c909335c + jinja: Suppose it's true that {{premise}} Then, is "{{hypothesis}}" {{"always"}}, + {{"sometimes"}}, or {{"never"}} true? ||| {% if label !=-1 %}{{ answer_choices[label] + }}{% endif %} + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: always/sometimes/never + reference: Sanh et al. 2021 + 75db2bc2-3caa-4956-9653-13c7dd6255df: !Template + answer_choices: True ||| False ||| Neither + id: 75db2bc2-3caa-4956-9653-13c7dd6255df + jinja: '{{premise}} + + Question: {{hypothesis}} True, False, or Neither? ||| {% if label !=-1 %}{{ + answer_choices[label] }}{% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: GPT-3 style + reference: 'Same as reported in Figure G7 of the GPT-3 paper, except that there + is no task identifying tokens like "anli R1: ".' + 87237a07-7cce-470a-80ac-3e5e3a5283ba: !Template + answer_choices: Always ||| Never ||| Sometimes + id: 87237a07-7cce-470a-80ac-3e5e3a5283ba + jinja: "{{premise}} \n\nKeeping in mind the above text, consider: {{hypothesis}}\ + \ Is this {{\"always\"}}, {{\"sometimes\"}}, or {{\"never\"}} correct? ||| {%\ + \ if label !=-1 %}{{ answer_choices[label] }}{% endif %}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: consider always/sometimes/never + reference: Sanh et al. 2021 + 8798b8a4-1f59-4c72-9c1b-3e3044a7462a: !Template + answer_choices: Yes ||| No ||| Maybe + id: 8798b8a4-1f59-4c72-9c1b-3e3044a7462a + jinja: Given {{premise}} Is it guaranteed true that "{{hypothesis}}"? Yes, no, + or maybe? ||| {% if label !=-1 %}{{ answer_choices[label] }}{% endif %} + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: guaranteed true + reference: Webson & Pavlick 2021 + 8e3b8d3d-1362-47dc-922a-82c03f965989: !Template + answer_choices: Yes ||| No ||| Maybe + id: 8e3b8d3d-1362-47dc-922a-82c03f965989 + jinja: Given that {{premise}} Therefore, it must be true that "{{hypothesis}}"? + Yes, no, or maybe? ||| {% if label !=-1 %}{{ answer_choices[label] }}{% endif + %} + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: must be true + reference: Sanh et al. 2021 + 90ab1002-093c-4e54-b48f-626655e36b65: !Template + answer_choices: Guaranteed ||| Impossible ||| Possible + id: 90ab1002-093c-4e54-b48f-626655e36b65 + jinja: "Assume it is true that {{premise}} \n\nTherefore, \"{{hypothesis}}\" is\ + \ {{\"guaranteed\"}}, {{\"possible\"}}, or {{\"impossible\"}}? ||| {% if label\ + \ !=-1 %}{{ answer_choices[label] }}{% endif %}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: guaranteed/possible/impossible + reference: Sanh et al. 2021 + a485d120-6eef-4ff6-8684-42df1639b101: !Template + answer_choices: Yes ||| No ||| Maybe + id: a485d120-6eef-4ff6-8684-42df1639b101 + jinja: "{{premise}} \n\nQuestion: Does this imply that \"{{hypothesis}}\"? Yes,\ + \ no, or maybe? ||| {% if label !=-1 %}{{answer_choices[label]}}{% endif %}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: does this imply + reference: Sanh et al. 2021 + bee62bfa-5307-4e1c-97b2-2ad2f7bcb179: !Template + answer_choices: Correct ||| Incorrect ||| Inconclusive + id: bee62bfa-5307-4e1c-97b2-2ad2f7bcb179 + jinja: '{{premise}} Using only the above description and what you know about the + world, "{{hypothesis}}" is definitely correct, incorrect, or inconclusive? ||| + {% if label !=-1 %}{{ answer_choices[label] }}{% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: MNLI crowdsource + reference: Adapted from Williams et al. 2018's instructions to crowdsourcing workers. + e503b148-8e6c-43b5-9ed6-312794c54d9b: !Template + answer_choices: Yes ||| No ||| Maybe + id: e503b148-8e6c-43b5-9ed6-312794c54d9b + jinja: Given {{premise}} Should we assume that "{{hypothesis}}" is true? Yes, + no, or maybe? ||| {% if label !=-1 %}{{ answer_choices[label] }}{% endif %} + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: should assume + reference: Webson & Pavlick 2021 + ea56b7f3-6e07-45bc-b619-c527eac4a41b: !Template + answer_choices: True ||| False ||| Inconclusive + id: ea56b7f3-6e07-45bc-b619-c527eac4a41b + jinja: 'Take the following as truth: {{premise}} + + Then the following statement: "{{hypothesis}}" is {{"true"}}, {{"false"}}, or + {{"inconclusive"}}? ||| {% if label !=-1 %}{{ answer_choices[label] }}{% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: take the following as truth + reference: Sanh et al. 2021 diff --git a/src/prompts/templates/super_glue/copa/templates.yaml b/src/prompts/templates/super_glue/copa/templates.yaml new file mode 100644 index 0000000..d048ede --- /dev/null +++ b/src/prompts/templates/super_glue/copa/templates.yaml @@ -0,0 +1,244 @@ +dataset: super_glue +subset: copa +templates: + 0edd8660-f299-4819-a5ac-633c11177228: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: 0edd8660-f299-4819-a5ac-633c11177228 + jinja: 'Exercise: choose the most plausible alternative. + + + {{ premise }} {% if question == "cause" %} because... {% else %} so... {% endif + %} + + - {{choice1}} + + - {{choice2}} ||| + + + {% if label != -1 %}{{ answer_choices[label] }}{%endif%}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: exercise + reference: '' + # 150789fe-e309-47a1-82c9-0a4dc2c6b12b: !Template + # answer_choices: '{{choice1}} ||| {{choice2}}' + # id: 150789fe-e309-47a1-82c9-0a4dc2c6b12b + # jinja: "{% if question == \"effect\" %} \n{{ premise }} What could happen next,\ + # \ \"{{ answer_choices[0] }}\" or \"{{ answer_choices[1] }}\"? ||| {% if label\ + # \ != -1 %}{{ answer_choices[label] }}{%endif%}\n{% endif %}" + # metadata: !TemplateMetadata + # choices_in_prompt: true + # languages: + # - en + # metrics: + # - Accuracy + # original_task: true + # name: "\u2026What could happen next, C1 or C2?" + # reference: '' + 4d879cbe-2fd7-424a-9d78-3f5200313fba: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: 4d879cbe-2fd7-424a-9d78-3f5200313fba + jinja: "{{ premise }} \n\nI am hesitating between two options. Help me choose\ + \ the more likely{% if question == \"cause\" %} cause: {% else %} effect: {%\ + \ endif %}\n- {{choice1}}\n- {{choice2}} ||| \n\n{% if label != -1 %}{{ answer_choices[label]\ + \ }}{%endif%}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: i_am_hesitating + reference: '' + 5c5f5813-0443-4944-b77a-c0d08885c16b: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: 5c5f5813-0443-4944-b77a-c0d08885c16b + jinja: 'Consider the following premise: '''''' {{premise}} '''''' Choice 1: {{answer_choices[0]}} + + Choice 2: {{answer_choices[1]}} + + Q: Which one is more likely to be the {{question}}, choice 1 or choice 2? + + A:|||{{answer_choices[label]}}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: [] + original_task: true + name: choose_between + reference: '' + 66ea075e-4d03-4a78-b1fa-9a5228cf0c9d: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: 66ea075e-4d03-4a78-b1fa-9a5228cf0c9d + jinja: '{{ premise }} {% if question == "cause" %} This happened because... {% + else %} As a consequence... {% endif %} + + Help me pick the more plausible option: + + - {{choice1}} + + - {{choice2}} + + + ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: plausible_alternatives + reference: '' + 744047dc-1298-45a2-8d68-d67e3f834ded: !Template + answer_choices: '{{choice1 }} ||| {{choice2}}' + id: 744047dc-1298-45a2-8d68-d67e3f834ded + jinja: '"{{ answer_choices[0] }}" or "{{ answer_choices[1] }}"? {{ premise }} + {% if question == "cause" %} because {% else %} so {% endif %} ||| {% if label + != -1 %}{{ answer_choices[label] }}{% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: "C1 or C2? premise, so/because\u2026" + reference: "Adapted from Perez et al. 2021 and Schick & Sch\xFCtz 2021." + # 84da62c2-9440-4cfc-bdd4-d70c65e33a82: !Template + # answer_choices: '{{choice1}} ||| {{choice2}}' + # id: 84da62c2-9440-4cfc-bdd4-d70c65e33a82 + # jinja: "{% if question == \"effect\" %} \n{{ premise }} As a result, \"{{ answer_choices[0]\ + # \ }}\" or \"{{ answer_choices[1] }}\"? ||| {% if label != -1 %}{{ answer_choices[label]\ + # \ }}{%endif%}\n{% endif %}" + # metadata: !TemplateMetadata + # choices_in_prompt: true + # languages: + # - en + # metrics: + # - Accuracy + # original_task: true + # name: "\u2026As a result, C1 or C2?" + # reference: '' + 8ce80f8a-239e-4393-892c-f63dbb0d9929: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: 8ce80f8a-239e-4393-892c-f63dbb0d9929 + jinja: "{{ premise }} \n\nWhat's the best option?\n- {{choice1}}\n- {{choice2}}\n\ + \nWe are looking for{% if question == \"cause\" %} a cause{% else %} an effect{% endif %}.\n\nA: + \ ||| {% if label != -1 %}{{answer_choices[label]}}{%endif%}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: best_option + reference: '' + # 8cf2ba73-aee5-4651-b5d4-b1b88afe4abb: !Template + # answer_choices: '{{choice1}} ||| {{choice2}}' + # id: 8cf2ba73-aee5-4651-b5d4-b1b88afe4abb + # jinja: "{% if question == \"cause\" %} \n{{ premise }} Which may be caused by\ + # \ \"{{ answer_choices[0] }}\" or \"{{ answer_choices[1] }}\"? ||| {% if label\ + # \ != -1 %}{{ answer_choices[label] }}{%endif%}\n{% endif %}" + # metadata: !TemplateMetadata + # choices_in_prompt: true + # languages: + # - en + # metrics: + # - Accuracy + # original_task: true + # name: "\u2026which may be caused by" + # reference: '' + a1f9951e-2b6b-4530-9636-9cdf4c1658c5: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: a1f9951e-2b6b-4530-9636-9cdf4c1658c5 + jinja: 'Pick the more likely continuation to the following sentence: + + {{ premise }} {% if question == "cause" %} as a result of: {% else %} as a consequence: + {% endif %} + + - {{choice1}} + + - {{choice2}} ||| + + + {% if label != -1 %}{{ answer_choices[label] }}{%endif%}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: more likely + reference: '' + a61d8c21-da25-47bf-b5fe-14a8edd650af: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: a61d8c21-da25-47bf-b5fe-14a8edd650af + jinja: '{{ premise }} + + + Select the most plausible {% if question == "cause" %} cause: {% else %} effect: + {% endif %} + + - {{choice1}} + + - {{choice2}} ||| + + + {% if label != -1 %}{{ answer_choices[label] }}{%endif%}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: cause_effect + reference: '' + # a8bf11c3-bea2-45ba-a533-957d8bee5e2e: !Template + # answer_choices: '{{choice1}} ||| {{choice2}}' + # id: a8bf11c3-bea2-45ba-a533-957d8bee5e2e + # jinja: "{% if question == \"cause\" %} \n{{ premise }} Why? \"{{ answer_choices[0]\ + # \ }}\" or \"{{ answer_choices[1] }}\"? ||| {% if label != -1 %}{{ answer_choices[label]\ + # \ }}{%endif%}\n{% endif %}" + # metadata: !TemplateMetadata + # choices_in_prompt: true + # languages: + # - en + # metrics: + # - Accuracy + # original_task: true + # name: "\u2026why? C1 or C2" + # reference: '' + f32348cd-d3cb-4619-87b9-e24f99c78567: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: f32348cd-d3cb-4619-87b9-e24f99c78567 + jinja: '{{ premise }} {% if question == "cause" %} because... {% else %} so... + {% endif %} + + Choose between: + + - {{choice1}} + + - {{choice2}} ||| + + + {% if label != -1 %}{{ answer_choices[label] }}{%endif%}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: choose + reference: '' diff --git a/src/prompts/templates/super_glue/multirc/templates.yaml b/src/prompts/templates/super_glue/multirc/templates.yaml new file mode 100644 index 0000000..2107173 --- /dev/null +++ b/src/prompts/templates/super_glue/multirc/templates.yaml @@ -0,0 +1,185 @@ +dataset: super_glue +subset: multirc +templates: + 2d95962b-a545-41ae-8d76-07ee6704ef65: !Template + answer_choices: No ||| Yes + id: 2d95962b-a545-41ae-8d76-07ee6704ef65 + jinja: '{{paragraph}} + + + Question: {{question}} + + I found this answer "{{answer}}". Is that correct? Yes or no? + + ||| + + {% if label != -1 %}{{answer_choices[label]}}{% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: found_this_answer + reference: '' + 42d47df9-09de-4691-8e49-7cfadd636cdd: !Template + answer_choices: No ||| Yes + id: 42d47df9-09de-4691-8e49-7cfadd636cdd + jinja: "{{ paragraph }}\nBased on the previous passage, {{ question }} \nIs \"\ + {{ answer }}\" a correct answer? ||| {% if label != -1 %}{{ answer_choices[label]\ + \ }}{% endif %}" + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Accuracy + original_task: true + name: "is\u2026 a correct answer?" + reference: "Adapted from Perez et al. 2021 and Schick & Sch\xFCtz 2021." + 431a5c97-af33-4053-83c8-afb0dfc04448: !Template + answer_choices: No ||| Yes + id: 431a5c97-af33-4053-83c8-afb0dfc04448 + jinja: '{{paragraph}} + + Question: {{question}} + + + I am grading my students'' exercises. Is the answer "{{answer}}" correct? + + ||| + + {% if label != -1 %}{{answer_choices[label]}}{% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Accuracy + original_task: true + name: grading + reference: '' + 4fc9e1ea-7451-4dba-a2cb-ce870e35ef8b: !Template + answer_choices: No ||| Yes + id: 4fc9e1ea-7451-4dba-a2cb-ce870e35ef8b + jinja: "{{ paragraph }}\n{{ question }} \nWould it be good to answer \"{{ answer\ + \ }}\"? ||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %}" + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Accuracy + original_task: true + name: "Would it be good to answer\u2026" + reference: '' + 59a2d847-27f3-4002-a125-cf9a291b3098: !Template + answer_choices: No ||| Yes + id: 59a2d847-27f3-4002-a125-cf9a291b3098 + jinja: "{{ paragraph }}\nQuestion: {{ question }} \nIs it {{ answer }}? ||| {%\ + \ if label != -1 %}{{ answer_choices[label] }}{% endif %}" + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Accuracy + original_task: true + name: "paragraph\u2026 question\u2026 is it\u2026 ?" + reference: "Adapted from Perez et al. 2021 and Schick & Sch\xFCtz 2021." + 7bf537ea-ff8d-44c7-8fc9-305b35e3be66: !Template + answer_choices: No ||| Yes + id: 7bf537ea-ff8d-44c7-8fc9-305b35e3be66 + jinja: '{{paragraph}} + + + Decide whether "{{answer}}" is a valid answer to the following question: {{question}} + + Answer yes or no. + + ||| + + {% if label != -1 %}{{answer_choices[label]}}{% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Accuracy + original_task: true + name: decide_valid + reference: '' + 7d878b89-2774-429a-82fb-ac801379e3ae: !Template + answer_choices: No ||| Yes + id: 7d878b89-2774-429a-82fb-ac801379e3ae + jinja: "{{ paragraph }}\nQuestion: {{ question }} \nIs the correct answer {{ answer\ + \ }}? ||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %}" + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Accuracy + original_task: true + name: "is the correct answer\u2026" + reference: "Adapted from Perez et al. 2021 and Schick & Sch\xFCtz 2021." + ae9b2b0b-1731-4370-adcc-36c4a959490d: !Template + answer_choices: No ||| Yes + id: ae9b2b0b-1731-4370-adcc-36c4a959490d + jinja: 'Is "{{answer}}" a correct answer to the following question? + + Question: {{question}} + + + Rely on the following text: {{paragraph}} + + ||| + + {% if label != -1 %}{{answer_choices[label]}}{% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Accuracy + original_task: true + name: correct + reference: '' + b63fd1c3-b4a6-43c3-8429-6a389235b2a4: !Template + answer_choices: No ||| Yes + id: b63fd1c3-b4a6-43c3-8429-6a389235b2a4 + jinja: '{{paragraph}} + + + Question: {{question}} + + I think "{{answer}}" is a valid answer. Could you confirm? Yes or no? + + ||| + + {% if label != -1 %}{{answer_choices[label]}}{% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: confirm + reference: '' + d2d78b88-8845-45b5-935a-6451da00b285: !Template + answer_choices: No ||| Yes + id: d2d78b88-8845-45b5-935a-6451da00b285 + jinja: "{{ paragraph }}\n{{ question }} \nI was going to say \"{{ answer }}\"\ + . Does that sound right? ||| {% if label != -1 %}{{ answer_choices[label] }}{%\ + \ endif %}" + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Accuracy + original_task: true + name: "I was going to say\u2026" + reference: '' diff --git a/src/prompts/templates/super_glue/record/templates.yaml b/src/prompts/templates/super_glue/record/templates.yaml new file mode 100644 index 0000000..6949a30 --- /dev/null +++ b/src/prompts/templates/super_glue/record/templates.yaml @@ -0,0 +1,369 @@ +dataset: super_glue +subset: record +templates: + 014b669e-2e3b-40ce-bdde-418966c7d666: !Template + answer_choices: '{{ entities | join("|||") }}' + id: 014b669e-2e3b-40ce-bdde-418966c7d666 + jinja: "{{ passage }} \n{{ query }} \nWhich one is the \"{{\"@placeholder\"}}\"\ + ? {{ entities | join(\", \") }}? ||| {% if ( answers | length ) > 0 %} {{ answers\ + \ | choice }}\n{% endif %}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Squad + original_task: true + name: Which one is the placeholder? + reference: '' + 11e27d59-b1f5-43a1-9ccc-17f1c3249173: !Template + answer_choices: '{{ entities | join("|||") }}' + id: 11e27d59-b1f5-43a1-9ccc-17f1c3249173 + jinja: "The following document has been corrupted. Tell me what \"{{\"@placeholder\"\ + }}\" is referring to.\n\nDocument: {{ passage }} \n{{ query }} \n||| {% if (\ + \ answers | length ) > 0 %}{{ answers | choice }}\n{% endif %}" + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Squad + original_task: true + name: corrupted + reference: '' + 147656b2-2dad-4028-96c4-f19d57cd1344: !Template + answer_choices: '{% for entity in entities[:-1] %} {{ query | replace("@placeholder", + entity) }} ||| {% endfor %} {{ query | replace("@placeholder", entities[-1]) + }}' + id: 147656b2-2dad-4028-96c4-f19d57cd1344 + jinja: "Summary:\n\n- {{ passage.split(\"@highlight\")[1:] | join(\"\\n- \") }}\ + \ \n\nArticle:\n\n{{ passage.split(\"@highlight\")[0] }}\n ||| {% if ( answers\ + \ | length ) > 0 %}{{ query | replace(\"@placeholder\", answers | choice) }}\ + \ {% endif %}" + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Accuracy + original_task: true + name: Summary first (continuation choices) + reference: '' + 24c267d4-359e-40a9-83d2-bff904d63b09: !Template + answer_choices: '{% for entity in entities[:-1] %} {{ query | replace("@placeholder", + entity) }} ||| {% endfor %} {{ query | replace("@placeholder", entities[-1]) + }}' + id: 24c267d4-359e-40a9-83d2-bff904d63b09 + jinja: "Summary:\n\n- {{ passage.split(\"@highlight\")[1:] | join(\"\\n- \") }}\ + \ \n\nArticle:\n\n{{ passage.split(\"@highlight\")[0] }}\n\nNow that you've\ + \ read the article, please write a new sentence to add to it.\n\n||| {% if (\ + \ answers | length ) > 0 %}{{ query | replace(\"@placeholder\", answers | choice)\ + \ }} {% endif %}" + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Accuracy + original_task: true + name: Add sentence after after (continuation choices) + reference: '' + 441c70e3-095a-44a1-8163-bc3b666b7ea1: !Template + answer_choices: '{{ entities | join("|||") }}' + id: 441c70e3-095a-44a1-8163-bc3b666b7ea1 + jinja: "{{ passage }} \n{{ query }} \n\nYou should decide what \"{{\"@placeholder\"\ + }}\" is referring to. Choose between:\n- {{answer_choices | join(\"\\n- \")}}\n\ + ||| {% if ( answers | length ) > 0 %}{{ answers | choice }}\n{% endif %}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Squad + original_task: true + name: choose_between + reference: '' + 64013fb3-1afd-4e5a-8777-b164ca3b8e18: !Template + answer_choices: '{% for entity in entities[:-1] %} - {{ query | replace("@placeholder", + entity) }} ||| {% endfor %} - {{ query | replace("@placeholder", entities[-1]) + }}' + id: 64013fb3-1afd-4e5a-8777-b164ca3b8e18 + jinja: "{{ passage.split(\"@highlight\")[0] }}\n\nSummary:\n\n- {{ passage.split(\"\ + @highlight\")[1:] | join(\"\\n- \") }} \n\n ||| {% if ( answers | length ) >\ + \ 0 %}- {{ query | replace(\"@placeholder\", answers | choice) }} {% endif %}" + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Accuracy + original_task: true + name: GPT-3 style summary only (continuation choices) + reference: Brown et al. 2020 + 90fc9ecb-c706-4c03-bb7e-4fe9fcd777f6: !Template + answer_choices: '{% for entity in entities[:-1] %} {{ query | replace("@placeholder", + entity) }} ||| {% endfor %} {{ query | replace("@placeholder", entities[-1]) + }}' + id: 90fc9ecb-c706-4c03-bb7e-4fe9fcd777f6 + jinja: "Article:\n\n{{ passage.split(\"@highlight\")[0] }}\n\nHighlights:\n\n\ + {{ passage.split(\"@highlight\")[1:] | join(\"\\n\") }} \n\n ||| {% if ( answers\ + \ | length ) > 0 %}{{ query | replace(\"@placeholder\", answers | choice) }}\ + \ {% endif %}" + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Accuracy + original_task: false + name: GPT-3 style with labels without hyphens (continuation choices) + reference: '' + 91555c1c-c1e4-469b-a2a4-fc952ce1a145: !Template + answer_choices: '{{ entities | join("|||") }}' + id: 91555c1c-c1e4-469b-a2a4-fc952ce1a145 + jinja: "{{ passage }} \n{{ query }} \nIn the question above, the \"{{\"@placeholder\"\ + }}\" stands for ||| {% if ( answers | length ) > 0 %}{{ answers | choice }}{%\ + \ endif %}" + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Squad + original_task: true + name: In the question above, the placeholder stands for + reference: '' + 94577b75-2eac-4eae-b367-3b413c4188c6: !Template + answer_choices: '{% for entity in entities[:-1] %} {{ query | replace("@placeholder", + entity) }} ||| {% endfor %} {{ query | replace("@placeholder", entities[-1]) + }}' + id: 94577b75-2eac-4eae-b367-3b413c4188c6 + jinja: 'After reading the article, write another sentence to add to it. + + {{ passage | replace("@highlight", "\n- ") }} + + + ||| {% if ( answers | length ) > 0 %}{{ query | replace("@placeholder", answers + | choice) }}{% endif %} + + ' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Accuracy + original_task: true + name: Add sentence after (continuation choices) + reference: '' + 9579b54e-4f0f-4e43-8907-af57112cc857: !Template + answer_choices: '{% for entity in entities[:-1] %} {{ query | replace("@placeholder", + entity) }} ||| {% endfor %} {{ query | replace("@placeholder", entities[-1]) + }}' + id: 9579b54e-4f0f-4e43-8907-af57112cc857 + jinja: "Please read the following news article and write another sentence to add\ + \ to it.\n\n{{ passage | replace(\"@highlight\", \"\\n- \") }} \n ||| {% if\ + \ ( answers | length ) > 0 %}{{ query | replace(\"@placeholder\", answers |\ + \ choice) }} {% endif %}\n" + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Accuracy + original_task: true + name: News article (continuation choices) + reference: '' + 99dd38ce-32f3-4d58-93c5-59821002b9cc: !Template + answer_choices: '{{ entities | join("|||") }}' + id: 99dd38ce-32f3-4d58-93c5-59821002b9cc + jinja: "{{ passage }} \n{{ query }} \nWhat could the \"{{\"@placeholder\"}}\"\ + \ be? {{ entities | join(\", \") }}? ||| {% if ( answers | length ) > 0 %}{{\ + \ answers | choice }}{% endif %}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Squad + original_task: true + name: What could the placeholder be? + reference: '' + 9b688cf3-28bf-4f33-94cf-e73e4fa8c608: !Template + answer_choices: '{{entities | join("|||")}}' + id: 9b688cf3-28bf-4f33-94cf-e73e4fa8c608 + jinja: '{{ passage }} + + {{ query }} + + + I am trying to decide what "{{"@placeholder"}}" means in the previous text. + + Help by choosing an option between: + + - {{ entities | join("\n- ") }} + + ||| {% if ( answers | length ) > 0 %} + + {{ answers | choice }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Squad + original_task: true + name: trying_to_decide + reference: '' + a5ed27ed-162b-4ac1-9c7a-85059d5214be: !Template + answer_choices: '{{ entities | join("|||") }}' + id: a5ed27ed-162b-4ac1-9c7a-85059d5214be + jinja: "{{ passage }} \n{{ query }} \nHere, the placeholder refers to ||| {% if\ + \ ( answers | length ) > 0 %}{{ answers | choice }}\n{% endif %}" + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Squad + original_task: true + name: "the placeholder refers to\u2026" + reference: '' + a99a92e0-e1ee-4ec3-a38a-3be4303ba017: !Template + answer_choices: '{% for entity in entities[:-1] %} - {{ query | replace("@placeholder", + entity) }} ||| {% endfor %} - {{ query | replace("@placeholder", entities[-1]) + }}' + id: a99a92e0-e1ee-4ec3-a38a-3be4303ba017 + jinja: "{{ passage.split(\"@highlight\")[0] }}\n\nHighlights:\n\n- {{ passage.split(\"\ + @highlight\")[1:] | join(\"\\n- \") }} \n\nPlease write an additional highlight.\n\ + \ ||| {% if ( answers | length ) > 0 %}- {{ query | replace(\"@placeholder\"\ + , answers | choice) }} {% endif %}\n" + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Accuracy + original_task: true + name: New highlight (continuation choices) + reference: '' + d3fce74e-0d9d-404a-a009-9ebbf5794c2c: !Template + answer_choices: '{{entities | join("|||")}}' + id: d3fce74e-0d9d-404a-a009-9ebbf5794c2c + jinja: 'Exercise: Extract from the text the correct entity that "{{"@placeholder"}}" + is referring to. + + + {{ passage }} + + {{ query }} + + ||| {% if ( answers | length ) > 0 %} + + {{ answers | choice }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Squad + original_task: true + name: exercise + reference: '' + de5b635e-c2f4-40bb-81ac-650f1b45564b: !Template + answer_choices: '{{entities | join("|||")}}' + id: de5b635e-c2f4-40bb-81ac-650f1b45564b + jinja: '{{ passage }} + + {{ query }} + + + Pick one option, "{{"@placeholder"}}" refers to: + + - {{answer_choices | join("\n- ")}} + + ||| {% if ( answers | length ) > 0 %} + + {{ answers | choice }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Squad + original_task: true + name: pick_one_option + reference: '' + df8d0822-2cad-42de-8191-687ae47f6098: !Template + answer_choices: '{% for entity in entities[:-1] %} - {{ query | replace("@placeholder", + entity) }} ||| {% endfor %} - {{ query | replace("@placeholder", entities[-1]) + }}' + id: df8d0822-2cad-42de-8191-687ae47f6098 + jinja: "{{ passage | replace(\"@highlight\", \"\\n- \") }} \n\n ||| {% if ( answers\ + \ | length ) > 0 %}- {{ query | replace(\"@placeholder\", answers | choice)\ + \ }} {% endif %}" + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Accuracy + original_task: true + name: GPT-3 style (continuation choices) + reference: Brown et al. 2020 + dfa3052f-ede8-42c2-b99a-bc5762c4fdc6: !Template + answer_choices: '{% for entity in entities[:-1] %} - {{ query | replace("@placeholder", + entity) }} ||| {% endfor %} - {{ query | replace("@placeholder", entities[-1]) + }}' + id: dfa3052f-ede8-42c2-b99a-bc5762c4fdc6 + jinja: "Article:\n\n{{ passage.split(\"@highlight\")[0] }}\n\nHighlights:\n\n\ + - {{ passage.split(\"@highlight\")[1:] | join(\"\\n- \") }} \n\n ||| {% if (\ + \ answers | length ) > 0 %}- {{ query | replace(\"@placeholder\", answers |\ + \ choice) }} {% endif %}" + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Accuracy + original_task: false + name: GPT-3 style with labels (continuation choices) + reference: Brown et al. 2020 + e68d13c5-df75-4de0-b59e-f2eaf4af6ce7: !Template + answer_choices: '{{ entities | join("|||") }}' + id: e68d13c5-df75-4de0-b59e-f2eaf4af6ce7 + jinja: "{{ passage }} \n{{ query }} \nCan you figure out what does the \"{{\"\ + @placeholder\"}}\" mean? It means ||| {% if ( answers | length ) > 0 %}{{ answers\ + \ | choice }}{% endif %}" + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Squad + original_task: true + name: "Can you figure out\u2026" + reference: '' + f7a92707-c531-42cb-81b4-063976e013cb: !Template + answer_choices: '{% for entity in entities[:-1] %} {{ query | replace("@placeholder", + entity) }} ||| {% endfor %} {{ query | replace("@placeholder", entities[-1]) + }}' + id: f7a92707-c531-42cb-81b4-063976e013cb + jinja: "{{ passage | replace(\"@highlight\", \"\\n\") }} \n ||| {% if ( answers\ + \ | length ) > 0 %}{{ query | replace(\"@placeholder\", answers | choice) }}\ + \ {% endif %}" + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Accuracy + original_task: true + name: GPT-3 style without hyphens (continuation choices) + reference: Brown et al. 2020 diff --git a/src/prompts/templates/super_glue/rte/templates.yaml b/src/prompts/templates/super_glue/rte/templates.yaml new file mode 100644 index 0000000..113c107 --- /dev/null +++ b/src/prompts/templates/super_glue/rte/templates.yaml @@ -0,0 +1,160 @@ +dataset: super_glue +subset: rte +templates: + 2b52a83c-0021-41fe-b44c-5aaa076d71a2: !Template + answer_choices: Yes ||| No + id: 2b52a83c-0021-41fe-b44c-5aaa076d71a2 + jinja: '{{premise}} Using only the above description and what you know about the + world, is "{{hypothesis}}" definitely correct? Yes or no? ||| {% if label != + -1 %}{{ answer_choices[label] }}{% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: MNLI crowdsource + reference: Adapted from Williams et al. 2018's instructions to crowdsourcing workers. + 2d0d63da-ffcf-4f6e-941a-b8da922be43e: !Template + answer_choices: Yes ||| No + id: 2d0d63da-ffcf-4f6e-941a-b8da922be43e + jinja: 'Given {{premise}} Is it guaranteed true that "{{hypothesis}}"? Yes or no? + ||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: guaranteed true + reference: Webson & Pavlick 2021 + 4163e6f1-1a83-4c73-b867-02eb7ac80316: !Template + answer_choices: Yes ||| No + id: 4163e6f1-1a83-4c73-b867-02eb7ac80316 + jinja: Suppose {{premise}} Can we infer that "{{hypothesis}}"? Yes or no? ||| + {% if label != -1 %}{{ answer_choices[label] }}{% endif %} + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: can we infer + reference: Webson & Pavlick 2021 + 6e078fb4-505b-413c-bb5e-3cd16ddcf5d7: !Template + answer_choices: Yes ||| No + id: 6e078fb4-505b-413c-bb5e-3cd16ddcf5d7 + jinja: "{{premise}} \n\nQuestion: Does this imply that \"{{hypothesis}}\", yes\ + \ or no?\n\nAnswer: ||| {% if label != -1 %}{{answer_choices[label]}}{% endif %}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: choose_between + reference: Burns et al. 2021 + 8fb1c6aa-20e9-438c-bece-c6af1c746449: !Template + answer_choices: True ||| False + id: 8fb1c6aa-20e9-438c-bece-c6af1c746449 + jinja: '{{premise}} + + Question: {{hypothesis}} True or False? ||| {% if label != -1 %}{{ answer_choices[label] + }}{% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: GPT-3 style + reference: Same as reported in Figure G31 of the GPT-3 paper. + 9e078fb4-505b-413c-bb5e-3cd16ddcf5d7: !Template + answer_choices: Yes ||| No + id: 9e078fb4-505b-413c-bb5e-3cd16ddcf5d7 + jinja: "{{premise}} \n\nQuestion: Does this imply that \"{{hypothesis}}\"? Yes\ + \ or no? ||| {% if label != -1 %}{{answer_choices[label]}}{% endif %}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: does this imply + reference: Sanh et al. 2021 + b8dc85c6-28b6-4340-979a-8e77c2a0dde8: !Template + answer_choices: Yes ||| No + id: b8dc85c6-28b6-4340-979a-8e77c2a0dde8 + jinja: Given {{premise}} Should we assume that "{{hypothesis}}" is true? Yes or + no? ||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %} + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: should assume + reference: Webson & Pavlick 2021 + e2fb58f2-b1f2-4aef-b74b-c4ee1c571fff: !Template + answer_choices: Yes ||| No + id: e2fb58f2-b1f2-4aef-b74b-c4ee1c571fff + jinja: Given that {{premise}} Does it follow that {{hypothesis}} Yes or no? ||| + {% if label != -1 %}{{ answer_choices[label] }}{% endif %} + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: does it follow that + reference: Sanh et al. 2021 + ed1f4b75-8826-4852-9bd6-aedf368678f5: !Template + answer_choices: Yes ||| No + id: ed1f4b75-8826-4852-9bd6-aedf368678f5 + jinja: '{{premise}} Based on the previous passage, is it true that "{{hypothesis}}"? + Yes or no? ||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: based on the previous passage + reference: "Adapted from the BoolQ prompts in Schick & Sch\xFCtze 2021." + ee0ce095-122a-4509-bf0b-33d1495295f7: !Template + answer_choices: Yes ||| No + id: ee0ce095-122a-4509-bf0b-33d1495295f7 + jinja: '{{premise}} Are we justified in saying that "{{hypothesis}}"? Yes or no? + ||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: justified in saying + reference: Webson & Pavlick 2021 + fb4f8144-37f5-4977-88da-37a5d0bfd0e8: !Template + answer_choices: Yes ||| No + id: fb4f8144-37f5-4977-88da-37a5d0bfd0e8 + jinja: Given that {{premise}} Therefore, it must be true that "{{hypothesis}}"? + Yes or no? ||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %} + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: must be true + reference: Sanh et al. 2021 diff --git a/src/prompts/templates/super_glue/wic/templates.yaml b/src/prompts/templates/super_glue/wic/templates.yaml new file mode 100644 index 0000000..1ca7797 --- /dev/null +++ b/src/prompts/templates/super_glue/wic/templates.yaml @@ -0,0 +1,238 @@ +dataset: super_glue +subset: wic +templates: + 14e73f39-a0d1-44c2-b9a4-4e48f9f1608e: !Template + answer_choices: No ||| Yes + id: 14e73f39-a0d1-44c2-b9a4-4e48f9f1608e + jinja: 'Does the word "{{word}}" have the same meaning in these two sentences? + Yes, No? + + {{sentence1}} + + {{sentence2}} + + ||| {% if label != -1%} + + {{answer_choices[label]}} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: question-context-meaning-with-label + reference: Generalized question-context format with label + 3503ead5-4fa5-4f77-95dc-f0c2ed3eecdc: !Template + answer_choices: No ||| Yes + id: 3503ead5-4fa5-4f77-95dc-f0c2ed3eecdc + jinja: 'Does the word "{{word}}" have the same meaning in these two sentences? + + {{sentence1}} + + {{sentence2}} + + ||| {% if label != -1%} + + {{answer_choices[label]}} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Accuracy + original_task: true + name: question-context-meaning + reference: Generalized question-context format + 611d13dc-d414-4b9b-9204-e4f325e859e7: !Template + answer_choices: No ||| Yes + id: 611d13dc-d414-4b9b-9204-e4f325e859e7 + jinja: 'Homework + + + Decide whether the word "{{word}}" is used with the same meaning in the two + following sentences. Answer by yes or no. + + {{sentence1}} + + {{sentence2}} + + ||| {% if label != -1%} + + {{answer_choices[label]}} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: grammar_homework + reference: '' + 725b5ed0-7728-4890-95a4-a74cb7ae1bb4: !Template + answer_choices: False ||| True + id: 725b5ed0-7728-4890-95a4-a74cb7ae1bb4 + jinja: 'Sentence A: {{sentence1}} + + Sentence B: {{sentence2}} + + + "{{word}}" has a similar meaning in sentences A and B. True or False? + + ||| {% if label != -1%} + + {{answer_choices[label]}} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: affirmation_true_or_false + reference: '' + c3a0a5d8-cfe9-4a7f-8a3c-3c526e0ad0c6: !Template + answer_choices: No ||| Yes + id: c3a0a5d8-cfe9-4a7f-8a3c-3c526e0ad0c6 + jinja: '{{sentence1}} + + {{sentence2}} + + Question: Is the word ''{{word}}'' used in the same sense in the two sentences + above? + + ||| {% if label != -1%} + + {{answer_choices[label]}} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Accuracy + original_task: true + name: GPT-3-prompt + reference: Following table G32. https://arxiv.org/pdf/2005.14165.pdf + ce8b5a93-1841-4897-84db-b100f1c84f4b: !Template + answer_choices: No ||| Yes + id: ce8b5a93-1841-4897-84db-b100f1c84f4b + jinja: 'Sentence 1: {{sentence1}} + + Sentence 2: {{sentence2}} + + + Determine whether the word "{{word}}" is used in the same sense in both sentences. + Yes or no? + + ||| {% if label != -1%} + + {{answer_choices[label]}} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: same_sense + reference: '' + cfbc1637-10b8-4f20-a31c-55292f3cebd0: !Template + answer_choices: No ||| Yes + id: cfbc1637-10b8-4f20-a31c-55292f3cebd0 + jinja: "Determine if the word '{{word}}' is used in the same way in the two sentences\ + \ below. \n{{sentence1}}\n{{sentence2}}\n||| {% if label != -1%}\n{{answer_choices[label]}}\n\ + {% endif %}" + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Accuracy + original_task: true + name: question-context + reference: Generalized question-context format + d9e1db2a-ab0b-4621-bb41-01d5788d3873: !Template + answer_choices: No ||| Yes + id: d9e1db2a-ab0b-4621-bb41-01d5788d3873 + jinja: '{{sentence1}} + + {{sentence2}} + + Question: Is the word ''{{word}}'' used in the same sense in the two sentences + above? Yes, No? + + ||| {% if label != -1%} + + {{answer_choices[label]}} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: GPT-3-prompt-with-label + reference: Following table G32. https://arxiv.org/pdf/2005.14165.pdf add additional + label + dd2080cf-3117-49ba-9aff-c988a21fdb69: !Template + answer_choices: No ||| Yes + id: dd2080cf-3117-49ba-9aff-c988a21fdb69 + jinja: 'The word "{{word}}" has multiple meanings. Does it have the same meaning + in sentences 1 and 2? Yes or no? + + + Sentence 1: {{sentence1}} + + Sentence 2: {{sentence2}} + + ||| {% if label != -1%} + + {{answer_choices[label]}} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: polysemous + reference: '' + f934a96d-fe4d-4075-aa47-5595b9a604c7: !Template + answer_choices: No ||| Yes + id: f934a96d-fe4d-4075-aa47-5595b9a604c7 + jinja: '{{sentence1}} + + {{sentence2}} + + Similar sense of {{word}}? + + ||| {% if label != -1%} + + {{answer_choices[label]}} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Accuracy + original_task: true + name: similar-sense + reference: Following https://arxiv.org/abs/2105.11447, https://github.com/ethanjperez/true_few_shot/tree/main/templates.super_glue diff --git a/src/prompts/templates/super_glue/wsc.fixed/templates.yaml b/src/prompts/templates/super_glue/wsc.fixed/templates.yaml new file mode 100644 index 0000000..7c94ac1 --- /dev/null +++ b/src/prompts/templates/super_glue/wsc.fixed/templates.yaml @@ -0,0 +1,155 @@ +dataset: super_glue +subset: wsc.fixed +templates: + 212fb8b1-8436-4f64-8f37-a9094fe029f4: !Template + answer_choices: No ||| Yes + id: 212fb8b1-8436-4f64-8f37-a9094fe029f4 + jinja: '{{ text }} In the previous sentence, does the pronoun "{{ span2_text.lower() + }}" refer to {{ span1_text }}? Yes or no? ||| {% if label != -1 %}{{ answer_choices[label] + }}{% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: does the pronoun refer to + reference: "Adapted from Perez et al. 2021 and Schick & Sch\xFCtz 2021." + 23361c5d-b67f-4c2a-9da7-16301c55d0e1: !Template + answer_choices: No ||| Yes + id: 23361c5d-b67f-4c2a-9da7-16301c55d0e1 + jinja: '{{ text }} Here, by "{{ span2_text }}" they mean "{{ span1_text }}". Yes + or no? ||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: by p they mean + reference: '' + 2f17f18b-6daa-44ef-a2dd-dddaf04aec0e: !Template + answer_choices: False ||| True + id: 2f17f18b-6daa-44ef-a2dd-dddaf04aec0e + jinja: "{{ text }} \n\nIn other words, {{ text.split(\" \")[span2_index:] | join(\"\ + \ \") | replace(span2_text, span1_text) }} True or false? ||| {% if label !=\ + \ -1 %}{{ answer_choices[label] }}{% endif %}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: in other words + reference: '' + 4b3e29cc-ccb8-4e4c-a845-4935ca29cf34: !Template + answer_choices: No ||| Yes + id: 4b3e29cc-ccb8-4e4c-a845-4935ca29cf34 + jinja: '{{ text }} I think they mean "{{ text.split(" ")[span2_index:] | join(" + ") | replace(span2_text, span1_text) }}" Yes or no? ||| {% if label != -1 %}{{ + answer_choices[label] }}{% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: I think they mean + reference: '' + 7482d24f-cf45-4013-b82d-369489fc958b: !Template + answer_choices: No ||| Yes + id: 7482d24f-cf45-4013-b82d-369489fc958b + jinja: '{{ text }} Here, does "{{ span2_text.lower() }}" stand for {{ span1_text + }}? Yes or no? ||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: does p stand for + reference: '' + 7d377293-d043-4b6c-8ec1-d61eaf14ec67: !Template + answer_choices: No ||| Yes + id: 7d377293-d043-4b6c-8ec1-d61eaf14ec67 + jinja: "Passage: {{ text }} \n\nQuestion: In the passage above, does the pronoun\ + \ \"{{ span2_text }}\" refer to {{ span1_text }}?\n\nAnswer: ||| {% if label\ + \ != -1 %}{{ answer_choices[label] }}{% endif %}" + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Accuracy + original_task: true + name: GPT-3 Style + reference: Adapted from Figure G33, p. 59, Brown et al. 2020 + 809eacd0-2f6c-4e3a-b52a-57c783879d36: !Template + answer_choices: No ||| Yes + id: 809eacd0-2f6c-4e3a-b52a-57c783879d36 + jinja: '{{ text }} In the previous sentence, can the pronoun "{{ span2_text }}" + be replaced with "{{ span1_text }}"? Yes or no? ||| {% if label != -1 %}{{ answer_choices[label] + }}{% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: replaced with + reference: '' + 87f97aa0-1fa9-4f0b-b8e6-89d3c1f19bd6: !Template + answer_choices: False ||| True + id: 87f97aa0-1fa9-4f0b-b8e6-89d3c1f19bd6 + jinja: "Context: {{ text }} \n\n{% if span2_text.lower() == \"they\" or span2_text.lower()\ + \ == \"them\" %}\nQuestion: \"{{ span2_text }}\" are {{ span1_text }}. True\ + \ or false?\n{% else %}\nQuestion: \"{{ span2_text }}\" is {{ span1_text }}.\ + \ True or false?\n{% endif %}\n\nAnswer: ||| {% if label != -1 %}{{ answer_choices[label]\ + \ }}{% endif %}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: p is/are r + reference: '' + aae24b54-c3a7-4f69-8b77-f6dc115988f8: !Template + answer_choices: False ||| True + id: aae24b54-c3a7-4f69-8b77-f6dc115988f8 + jinja: "{{ text }} \nIn the passage above, the pronoun \"{{ span2_text }}\" refers\ + \ to {{ span1_text }}. True or false? ||| {% if label != -1 %}{{ answer_choices[label]\ + \ }}{% endif %}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: the pronoun refers to + reference: "Adapted from Perez et al. 2021 and Schick & Sch\xFCtz 2021." + d88f3e21-42dc-49a5-924d-69b764a14816: !Template + answer_choices: No ||| Yes + id: d88f3e21-42dc-49a5-924d-69b764a14816 + jinja: "{{ text }} \n{% if span2_text.lower() == \"they\" or span2_text.lower()\ + \ == \"them\" %}\nQuestion: Who or what are \"{{ span2_text.lower() }}\"? {{\ + \ span1_text }}?\n{% else %}\nQuestion: Who or what is \"{{ span2_text.lower()\ + \ }}\"? Is it {{ span1_text }}?\n{% endif %}\nAnswer: ||| {% if label != -1\ + \ %}{{ answer_choices[label] }}{% endif %}" + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Accuracy + original_task: true + name: Who or what is/are + reference: I double checked the only plural pronouns in WSC are "they" and "them". diff --git a/src/prompts/templates/tweet_eval/emoji/templates.yaml b/src/prompts/templates/tweet_eval/emoji/templates.yaml new file mode 100755 index 0000000..fb85973 --- /dev/null +++ b/src/prompts/templates/tweet_eval/emoji/templates.yaml @@ -0,0 +1,68 @@ +dataset: tweet_eval +subset: emoji +templates: + 8c794abe-5364-430f-aa1e-eb3501443cec: !Template + answer_choices: "\u2764|||\U0001F60D|||\U0001F602|||\U0001F495|||\U0001F525|||\U0001F60A\ + |||\U0001F60E|||\u2728|||\U0001F499|||\U0001F618|||\U0001F4F7|||\U0001F1FA\U0001F1F8\ + |||\u2600|||\U0001F49C|||\U0001F609|||\U0001F4AF|||\U0001F601|||\U0001F384|||\U0001F4F8\ + |||\U0001F61C" + id: 8c794abe-5364-430f-aa1e-eb3501443cec + jinja: 'Which emoji among {{answer_choices | join(", ")}} best describes the sentiment + of the following tweet? + + + {{text}} ||| + + {{answer_choices[label]}}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Other + original_task: true + name: emoji_option + reference: 'official metric: macroaveraged F1' + c05f50e0-f708-44bc-98e7-ff7b3f9f5d93: !Template + answer_choices: "\u2764|||\U0001F60D|||\U0001F602|||\U0001F495|||\U0001F525|||\U0001F60A\ + |||\U0001F60E|||\u2728|||\U0001F499|||\U0001F618|||\U0001F4F7|||\U0001F1FA\U0001F1F8\ + |||\u2600|||\U0001F49C|||\U0001F609|||\U0001F4AF|||\U0001F601|||\U0001F384|||\U0001F4F8\ + |||\U0001F61C" + id: c05f50e0-f708-44bc-98e7-ff7b3f9f5d93 + jinja: 'Which emoji among {{answer_choices | join(", ")}} would be the best comment + to the following tweet? + + + {{text}} ||| + + {{answer_choices[label]}}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Other + original_task: false + name: emoji_reply + reference: 'Metric: macroaveraged F1' + d5c771d3-28e7-420e-af47-c077cfe0e7e5: !Template + answer_choices: "\u2764|||\U0001F60D|||\U0001F602|||\U0001F495|||\U0001F525|||\U0001F60A\ + |||\U0001F60E|||\u2728|||\U0001F499|||\U0001F618|||\U0001F4F7|||\U0001F1FA\U0001F1F8\ + |||\u2600|||\U0001F49C|||\U0001F609|||\U0001F4AF|||\U0001F601|||\U0001F384|||\U0001F4F8\ + |||\U0001F61C" + id: d5c771d3-28e7-420e-af47-c077cfe0e7e5 + jinja: 'Which emoji best describes the sentiment of the following tweet? + + + {{text}} ||| + + {{answer_choices[label]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Other + original_task: true + name: emoji + reference: 'official metric: macroaveraged F1' diff --git a/src/prompts/templates/tweet_eval/emotion/templates.yaml b/src/prompts/templates/tweet_eval/emotion/templates.yaml new file mode 100644 index 0000000..86ba7aa --- /dev/null +++ b/src/prompts/templates/tweet_eval/emotion/templates.yaml @@ -0,0 +1,103 @@ +dataset: tweet_eval +subset: emotion +templates: + 7c09c33e-31f2-414b-89a1-6b1dda92ef6f: !Template + answer_choices: anger ||| joy ||| optimism ||| sadness + id: 7c09c33e-31f2-414b-89a1-6b1dda92ef6f + jinja: '{{text}} + + + To get full credit in this exam, choose the correct emotion from the following + choices: {{answer_choices | join(", ")}} + + ||| + + {{answer_choices[label]}}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: emotion_exam + reference: exam style prompt + 87db02f2-585e-4fd1-81c0-e94297607097: !Template + answer_choices: anger ||| joy ||| optimism ||| sadness + id: 87db02f2-585e-4fd1-81c0-e94297607097 + jinja: 'Which emotion among {{answer_choices | join(", ")}} best describes the + feeling of the author of the following tweet? + + + {{text}}||| + + {{answer_choices[label]}}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: author_emotion + reference: '' + 8bc3ebc5-77f1-4d55-bd96-c62429ebf093: !Template + answer_choices: anger ||| joy ||| optimism ||| sadness + id: 8bc3ebc5-77f1-4d55-bd96-c62429ebf093 + jinja: 'Which emotion is best represented by the following tweet? + + {{text}} + + + Possible emotions: {{answer_choices | join(", ")}} + + ||| + + {{answer_choices[label]}}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: which_emotion + reference: '' + a5992077-2e31-467b-a6ee-b75dee933d0e: !Template + answer_choices: anger ||| joy ||| optimism ||| sadness + id: a5992077-2e31-467b-a6ee-b75dee933d0e + jinja: "{{text}}\n\nCategorize the tweet into one of the following options: \n\ + (a) {{answer_choices[0]}}\n(b) {{answer_choices[1]}}\n(c) {{answer_choices[2]}}\n\ + (d) {{answer_choices[3]}}\n|||\n{{answer_choices[label]}}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: emotion_with_option + reference: '' + b8f4912e-e3be-4dd5-82ec-6f110c056a86: !Template + answer_choices: anger ||| joy ||| optimism ||| sadness + id: b8f4912e-e3be-4dd5-82ec-6f110c056a86 + jinja: '{{text}} + + + What is the emotion of the text? + + + Hint: {{answer_choices | join(", ")}} + + ||| + + {{answer_choices[label]}}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: emotion_hint + reference: '' diff --git a/src/prompts/templates/tweet_eval/hate/templates.yaml b/src/prompts/templates/tweet_eval/hate/templates.yaml new file mode 100644 index 0000000..e5654fd --- /dev/null +++ b/src/prompts/templates/tweet_eval/hate/templates.yaml @@ -0,0 +1,84 @@ +dataset: tweet_eval +subset: hate +templates: + 3266f9d4-9c80-4e17-a8a6-1fe44ce8f3bf: !Template + answer_choices: no ||| yes + id: 3266f9d4-9c80-4e17-a8a6-1fe44ce8f3bf + jinja: 'Does this tweet convey the author''s hatred towards something or someone? + + + {{text}} ||| + + {{answer_choices[label]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Accuracy + original_task: true + name: author_hate + reference: '' + 34a5128b-6fc8-453b-94d4-4ebaa87172c1: !Template + answer_choices: no ||| yes + id: 34a5128b-6fc8-453b-94d4-4ebaa87172c1 + jinja: 'Does this tweet convey hate: yes or no? + + + {{text}} ||| + + {{answer_choices[label]}}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: hate_no_yes + reference: '' + 5b15dc68-05a2-42dd-b0bf-fa15d4f40320: !Template + answer_choices: no|||yes + id: 5b15dc68-05a2-42dd-b0bf-fa15d4f40320 + jinja: "Is this a hateful tweet? \n{{text}}\n|||\n{{answer_choices[label]}}" + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Accuracy + original_task: true + name: hate_question + reference: '' + 971ad470-85bf-484e-aab7-b942f817bf2c: !Template + answer_choices: non-hate ||| hate + id: 971ad470-85bf-484e-aab7-b942f817bf2c + jinja: 'Does this tweet convey {{"hate"}} or {{"non-hate"}}? + + + {{text}} ||| + + {{answer_choices[label]}}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: hate_options + reference: '' + b0cdecc2-78a2-47e7-a74e-38d509c01214: !Template + answer_choices: 'no ||| yes ' + id: b0cdecc2-78a2-47e7-a74e-38d509c01214 + jinja: "In this test, you need to answer with either yes or no. \n\nQ: Is this\ + \ a hateful tweet? \n{{text}}\n\nA: \n|||\n{{answer_choices[label]}}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: hate_exam + reference: '' diff --git a/src/prompts/templates/tweet_eval/irony/templates.yaml b/src/prompts/templates/tweet_eval/irony/templates.yaml new file mode 100644 index 0000000..a3b851a --- /dev/null +++ b/src/prompts/templates/tweet_eval/irony/templates.yaml @@ -0,0 +1,84 @@ +dataset: tweet_eval +subset: irony +templates: + c45095f1-9be1-4e83-8daa-68805b6ece39: !Template + answer_choices: no ||| yes + id: c45095f1-9be1-4e83-8daa-68805b6ece39 + jinja: "Is this tweet is ironic? \n\n{{text}} |||\n{{answer_choices[label]}}" + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Accuracy + original_task: true + name: irony_question + reference: '' + cd2ed852-c6fa-431a-b0f1-06f0240d74a0: !Template + answer_choices: no ||| yes + id: cd2ed852-c6fa-431a-b0f1-06f0240d74a0 + jinja: "Is this tweet ironic? Answer with either yes or no. \n\n{{text}} |||\n\ + {{answer_choices[label]}}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: irony_yes_no + reference: '' + cef39e97-09f2-430c-ad1f-5fd9d05c876b: !Template + answer_choices: no ||| yes + id: cef39e97-09f2-430c-ad1f-5fd9d05c876b + jinja: "In this test, you need to answer with either yes or no. \n\nQ: Is this\ + \ an ironic tweet? \n{{text}}\n\nA: \n|||\n{{answer_choices[label]}}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: irony_exam + reference: exam style prompt + e7cff075-9388-48de-af51-fe185b654217: !Template + answer_choices: non-irony ||| irony + id: e7cff075-9388-48de-af51-fe185b654217 + jinja: 'Does this tweet contain {{"irony"}} or {{"non-irony"}}? + + + {{text}} ||| + + {{answer_choices[label]}}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: irony_option + reference: '' + f8361684-dad1-4c2c-9f22-1128f3096c20: !Template + answer_choices: no|||yes + id: f8361684-dad1-4c2c-9f22-1128f3096c20 + jinja: 'Does this tweet convey the author''s irony towards something or someone? + + + {{text}} + + ||| + + {{answer_choices[label]}} + + ' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Accuracy + original_task: true + name: irony_author + reference: '' diff --git a/src/prompts/templates/tweet_eval/offensive/templates.yaml b/src/prompts/templates/tweet_eval/offensive/templates.yaml new file mode 100644 index 0000000..276a9c7 --- /dev/null +++ b/src/prompts/templates/tweet_eval/offensive/templates.yaml @@ -0,0 +1,87 @@ +dataset: tweet_eval +subset: offensive +templates: + 10850707-80f3-4a75-b9f4-1e2d12be04c0: !Template + answer_choices: no ||| yes + id: 10850707-80f3-4a75-b9f4-1e2d12be04c0 + jinja: "Is this tweet {{\"offensive\"}}? \n\n{{text}} |||\n{{answer_choices[label]}}" + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Accuracy + original_task: true + name: offensive_question + reference: '' + 22ecf073-2932-4b0a-b305-1f92b696d0b4: !Template + answer_choices: no ||| yes + id: 22ecf073-2932-4b0a-b305-1f92b696d0b4 + jinja: "Can the tweet be removed for being offensive? Answer with a yes or a no.\ + \ \n\n{{text}}\n|||\n{{answer_choices[label]}}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: offensive_tweet_removed + reference: '' + 90a80d37-922d-40c0-b036-87eac64ce4c5: !Template + answer_choices: no ||| yes + id: 90a80d37-922d-40c0-b036-87eac64ce4c5 + jinja: 'Is the author''s tweet offensive? Answer with either yes or no. + + + {{text}} + + ||| + + {{answer_choices[label]}}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: offensive_author + reference: '' + a12bd98b-facc-4b17-bb16-80c98a20aa64: !Template + answer_choices: no ||| yes + id: a12bd98b-facc-4b17-bb16-80c98a20aa64 + jinja: 'Task: Identify if the tweet or text is offensive. + + + Tweet: {{text}} + + + Possible answers: yes, no + + ||| + + {{answer_choices[label]}}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: offensive_task + reference: '' + bf2cea43-0666-4eb5-814d-00956afd1900: !Template + answer_choices: no ||| yes + id: bf2cea43-0666-4eb5-814d-00956afd1900 + jinja: "In this test, you need to answer with either yes or no.\n\nQ: Is this\ + \ an offensive tweet?\n\n{{text}}\n\nA: \n|||\n{{answer_choices[label]}}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: offensive_exam + reference: '' diff --git a/src/prompts/templates/tweet_eval/sentiment/templates.yaml b/src/prompts/templates/tweet_eval/sentiment/templates.yaml new file mode 100644 index 0000000..7ace742 --- /dev/null +++ b/src/prompts/templates/tweet_eval/sentiment/templates.yaml @@ -0,0 +1,84 @@ +dataset: tweet_eval +subset: sentiment +templates: + 1fef2b36-3a19-4179-9b43-d67887cff299: !Template + answer_choices: negative ||| neutral ||| positive + id: 1fef2b36-3a19-4179-9b43-d67887cff299 + jinja: 'In this exam, you need to pick the correct sentiment for the tweet: + + + {{text}} + + + Possible choices: {{answer_choices | join(", ")}} + + ||| + + {{answer_choices[label]}}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: sentiment_exam + reference: '' + 6702e8cd-9764-4c88-86a9-046f84c98ef2: !Template + answer_choices: negative ||| neutral ||| positive + id: 6702e8cd-9764-4c88-86a9-046f84c98ef2 + jinja: "What is the sentiment of the tweet?\n\n{{text}} \n\nPossible choices:\ + \ {{answer_choices | join(\", \")}}\n|||\n{{answer_choices[label]}}\n" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: sentiment_question + reference: '' + 6c6c797c-9912-4778-906b-16f465941d16: !Template + answer_choices: negative ||| neutral ||| positive + id: 6c6c797c-9912-4778-906b-16f465941d16 + jinja: "Task: Identify the sentiment of the tweet.\n\nTweet: {{text}}\n\nOptions:\ + \ {{answer_choices | join(\", \")}} \n|||\n{{answer_choices[label]}}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: sentiment_task + reference: '' + b70647cf-22a0-49b2-b45e-23432c635cc2: !Template + answer_choices: negative|||neutral|||positive + id: b70647cf-22a0-49b2-b45e-23432c635cc2 + jinja: "Suppose you are the moderator of Twitter, what would be the sentiment\ + \ of the following tweet: \n\n{{text}}\n\nOptions: {{answer_choices | join(\"\ + , \")}}\n|||\n{{answer_choices[label]}}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: sentiment_moderator + reference: '' + ec68fd8e-92a3-4010-b0df-b14af95421a3: !Template + answer_choices: negative ||| neutral ||| positive + id: ec68fd8e-92a3-4010-b0df-b14af95421a3 + jinja: "{{text}}\n\nCategorize the tweet into one of the following options: \n\ + (a) {{answer_choices[0]}} \n(b) {{answer_choices[1]}} \n(c) {{answer_choices[2]}}\n\ + |||\n{{answer_choices[label]}}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: sentiment_options + reference: '' diff --git a/src/prompts/templates/tweet_eval/stance_abortion/templates.yaml b/src/prompts/templates/tweet_eval/stance_abortion/templates.yaml new file mode 100644 index 0000000..833ecde --- /dev/null +++ b/src/prompts/templates/tweet_eval/stance_abortion/templates.yaml @@ -0,0 +1,108 @@ +dataset: tweet_eval +subset: stance_abortion +templates: + 0d1dc279-e50c-4706-bc3d-84ea01cb59a1: !Template + answer_choices: Neutral ||| Against ||| In favor + id: 0d1dc279-e50c-4706-bc3d-84ea01cb59a1 + jinja: 'Does the author express any stance about abortion in the following text? + + + {{text}} ||| + + {{answer_choices[label]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Accuracy + original_task: true + name: abortion_predict_stance + reference: '' + 22758062-db86-4009-81a4-1e2a2e1052f2: !Template + answer_choices: Neutral ||| Against ||| In favor + id: 22758062-db86-4009-81a4-1e2a2e1052f2 + jinja: '{{text}} Where does the author of the above sentence stand on abortion? + ||| + + {{answer_choices[label]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Accuracy + original_task: true + name: abortion_guess_passive_author + reference: '' + 615151f8-ac5b-4c0e-a234-9e9b6296a2f2: !Template + answer_choices: Neutral ||| Against ||| In favor + id: 615151f8-ac5b-4c0e-a234-9e9b6296a2f2 + jinja: 'What option among, {{"neutral"}}, {{"against"}}, {{"in favor"}}, best + describes the stance of this tweet regarding abortion? + + + {{text}} ||| + + {{answer_choices[label]}}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: abortion_option + reference: '' + 687ffa1e-a772-48b1-9291-ba4e530a909e: !Template + answer_choices: Neutral ||| Against ||| In favor + id: 687ffa1e-a772-48b1-9291-ba4e530a909e + jinja: 'Is this tweet neutral, in favor of, or against abortion? + + + {{text}} ||| + + {{answer_choices[label]}}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: abortion + reference: '' + c5507588-1d20-42f9-935f-0c767294f5a9: !Template + answer_choices: Neutral ||| Against ||| In favor + id: c5507588-1d20-42f9-935f-0c767294f5a9 + jinja: 'How would you describe the stance used in this tweet? {{answer_choices|join(", + ")}} + + + {{text}} ||| + + {{answer_choices[label]}}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: abortion_how_describe + reference: '' + ee12d37b-5667-4b0e-9831-f952d08152b5: !Template + answer_choices: Neutral ||| Against ||| In favor + id: ee12d37b-5667-4b0e-9831-f952d08152b5 + jinja: '{{text}} Where does the above sentence stand on abortion? ||| + + {{answer_choices[label]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Accuracy + original_task: true + name: abortion_guess_passive + reference: '' diff --git a/src/prompts/templates/tweet_eval/stance_atheism/templates.yaml b/src/prompts/templates/tweet_eval/stance_atheism/templates.yaml new file mode 100644 index 0000000..0365325 --- /dev/null +++ b/src/prompts/templates/tweet_eval/stance_atheism/templates.yaml @@ -0,0 +1,108 @@ +dataset: tweet_eval +subset: stance_atheism +templates: + 2f6bfe8f-c45e-4f1d-a623-91a9cfd0ea8f: !Template + answer_choices: Neutral ||| Against ||| In favor + id: 2f6bfe8f-c45e-4f1d-a623-91a9cfd0ea8f + jinja: '{{text}} Where does the above sentence stand on atheism? ||| + + {{answer_choices[label]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Accuracy + original_task: true + name: atheism_guess_passive + reference: '' + 4309e10d-c9a9-4a17-8561-15270b998905: !Template + answer_choices: Neutral ||| Against ||| In favor + id: 4309e10d-c9a9-4a17-8561-15270b998905 + jinja: 'How would you describe the stance used in this tweet? {{answer_choices|join(", + ")}} + + + {{text}} ||| + + {{answer_choices[label]}}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: atheism_how_describe + reference: '' + 7e47c6b8-2923-4580-a275-a2b8867a3d96: !Template + answer_choices: Neutral ||| Against ||| In favor + id: 7e47c6b8-2923-4580-a275-a2b8867a3d96 + jinja: 'What option among, {{"neutral"}}, {{"against"}}, {{"in favor"}}, best + describes the stance of this tweet regarding atheism? + + + {{text}} ||| + + {{answer_choices[label]}}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: atheism_option + reference: '' + 89aa258e-3c3b-4d1c-8ac4-fe2c838b76e4: !Template + answer_choices: Neutral ||| Against ||| In favor + id: 89aa258e-3c3b-4d1c-8ac4-fe2c838b76e4 + jinja: 'Does the author express any stance about atheism in the following text? + + + {{text}} ||| + + {{answer_choices[label]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Accuracy + original_task: true + name: atheism_predict_stance + reference: '' + 97ef9418-7c92-455d-a4c5-d7b91668278c: !Template + answer_choices: Neutral ||| Against ||| In favor + id: 97ef9418-7c92-455d-a4c5-d7b91668278c + jinja: 'Is this tweet neutral, in favor of, or against atheism? + + + {{text}} ||| + + {{answer_choices[label]}}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: atheism + reference: '' + f28307ab-563e-4189-99b5-e0d858e9ab4c: !Template + answer_choices: Neutral ||| Against ||| In favor + id: f28307ab-563e-4189-99b5-e0d858e9ab4c + jinja: '{{text}} Where does the author of the above sentence stand on atheism? + ||| + + {{answer_choices[label]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Accuracy + original_task: true + name: atheism_guess_passive_author + reference: '' diff --git a/src/prompts/templates/tweet_eval/stance_climate/templates.yaml b/src/prompts/templates/tweet_eval/stance_climate/templates.yaml new file mode 100644 index 0000000..f114b4b --- /dev/null +++ b/src/prompts/templates/tweet_eval/stance_climate/templates.yaml @@ -0,0 +1,109 @@ +dataset: tweet_eval +subset: stance_climate +templates: + 2ebf2eaa-ef9f-413d-b7bf-cb2037330d2a: !Template + answer_choices: Neutral ||| Against ||| In favor + id: 2ebf2eaa-ef9f-413d-b7bf-cb2037330d2a + jinja: 'What option among, {{"neutral"}}, {{"against"}}, {{"in favor"}}, best + describes the stance of this tweet regarding climate change? + + + {{text}} ||| + + {{answer_choices[label]}}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: climate_option + reference: '' + 6f4205ad-6321-42a9-bf8e-a45508e67c1a: !Template + answer_choices: Neutral ||| Against ||| In favor + id: 6f4205ad-6321-42a9-bf8e-a45508e67c1a + jinja: '{{text}} Where does the above sentence stand on climate change? ||| + + {{answer_choices[label]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Accuracy + original_task: true + name: climate_guess_passive + reference: '' + 703f067e-5930-424e-9882-48063307ff8e: !Template + answer_choices: Neutral ||| Against ||| In favor + id: 703f067e-5930-424e-9882-48063307ff8e + jinja: 'How would you describe the stance used in this tweet? {{answer_choices|join(", + ")}} + + + {{text}} ||| + + {{answer_choices[label]}}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: climate_how_describe + reference: '' + 8ecd5059-742a-4833-95a1-bf0e25e9abfc: !Template + answer_choices: Neutral ||| Against ||| In favor + id: 8ecd5059-742a-4833-95a1-bf0e25e9abfc + jinja: '{{text}} Where does the author of the above sentence stand on climate + change? ||| + + {{answer_choices[label]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Accuracy + original_task: true + name: climate_guess_passive_author + reference: '' + cd82620b-6d1d-42f7-af89-56980cbb69a5: !Template + answer_choices: Neutral ||| Against ||| In favor + id: cd82620b-6d1d-42f7-af89-56980cbb69a5 + jinja: 'Does the author express any stance about climate change in the following + text? + + + {{text}} ||| + + {{answer_choices[label]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Accuracy + original_task: true + name: climate_predict_stance + reference: '' + edcdde10-b2e4-4954-82e2-f84fd57fc122: !Template + answer_choices: Neutral ||| Against ||| In favor + id: edcdde10-b2e4-4954-82e2-f84fd57fc122 + jinja: 'Is this tweet neutral, in favor of, or against climate change? + + + {{text}} ||| + + {{answer_choices[label]}}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: climate_change + reference: '' diff --git a/src/prompts/templates/tweet_eval/stance_feminist/templates.yaml b/src/prompts/templates/tweet_eval/stance_feminist/templates.yaml new file mode 100644 index 0000000..4ec2b5a --- /dev/null +++ b/src/prompts/templates/tweet_eval/stance_feminist/templates.yaml @@ -0,0 +1,108 @@ +dataset: tweet_eval +subset: stance_feminist +templates: + 2f6bfe8f-c45e-4f1d-a623-91a9cfd0ea8b: !Template + answer_choices: Neutral ||| Against ||| In favor + id: 2f6bfe8f-c45e-4f1d-a623-91a9cfd0ea8b + jinja: '{{text}} Where does the above sentence stand on feminism? ||| + + {{answer_choices[label]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Accuracy + original_task: true + name: feminism_guess_passive + reference: '' + 4309e10d-c9a9-4a17-8561-15270b99890b: !Template + answer_choices: Neutral ||| Against ||| In favor + id: 4309e10d-c9a9-4a17-8561-15270b99890b + jinja: 'How would you describe the stance used in this tweet? {{answer_choices|join(", + ")}} + + + {{text}} ||| + + {{answer_choices[label]}}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: feminism_how_describe + reference: '' + 7e47c6b8-2923-4580-a275-a2b8867a3d9b: !Template + answer_choices: Neutral ||| Against ||| In favor + id: 7e47c6b8-2923-4580-a275-a2b8867a3d9b + jinja: 'What option among, {{"neutral"}}, {{"against"}}, {{"in favor"}}, best + describes the stance of this tweet regarding feminism? + + + {{text}} ||| + + {{answer_choices[label]}}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: feminism_option + reference: '' + 89aa258e-3c3b-4d1c-8ac4-fe2c838b76eb: !Template + answer_choices: Neutral ||| Against ||| In favor + id: 89aa258e-3c3b-4d1c-8ac4-fe2c838b76eb + jinja: 'Does the author express any stance about feminism in the following text? + + + {{text}} ||| + + {{answer_choices[label]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Accuracy + original_task: true + name: feminism_predict_stance + reference: '' + 97ef9418-7c92-455d-a4c5-d7b91668278b: !Template + answer_choices: Neutral ||| Against ||| In favor + id: 97ef9418-7c92-455d-a4c5-d7b91668278b + jinja: 'Is this tweet neutral, in favor of, or against feminism? + + + {{text}} ||| + + {{answer_choices[label]}}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: feminism + reference: '' + f28307ab-563e-4189-99b5-e0d858e9ab4b: !Template + answer_choices: Neutral ||| Against ||| In favor + id: f28307ab-563e-4189-99b5-e0d858e9ab4b + jinja: '{{text}} Where does the author of the above sentence stand on feminism? + ||| + + {{answer_choices[label]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Accuracy + original_task: true + name: feminism_guess_passive_author + reference: '' diff --git a/src/prompts/templates/tweet_eval/stance_hillary/templates.yaml b/src/prompts/templates/tweet_eval/stance_hillary/templates.yaml new file mode 100644 index 0000000..68cd2bc --- /dev/null +++ b/src/prompts/templates/tweet_eval/stance_hillary/templates.yaml @@ -0,0 +1,108 @@ +dataset: tweet_eval +subset: stance_hillary +templates: + 21ba1c40-b491-43ed-96d6-7423b55c3bcf: !Template + answer_choices: Neutral ||| Against ||| In favor + id: 21ba1c40-b491-43ed-96d6-7423b55c3bcf + jinja: 'Does the author express any stance about Hillary in the following text? + + + {{text}} ||| + + {{answer_choices[label]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Accuracy + original_task: true + name: Hillary_predict_stance + reference: '' + 41502ea8-73a4-48a4-a15e-ab2ac7700457: !Template + answer_choices: Neutral ||| Against ||| In favor + id: 41502ea8-73a4-48a4-a15e-ab2ac7700457 + jinja: 'What option among, {{"neutral"}}, {{"against"}}, {{"in favor"}}, best + describes the stance of this tweet regarding Hillary? + + + {{text}} ||| + + {{answer_choices[label]}}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: Hillary_option + reference: '' + 498f1dec-12dc-4082-a44e-82fcae004bb8: !Template + answer_choices: Neutral ||| Against ||| In favor + id: 498f1dec-12dc-4082-a44e-82fcae004bb8 + jinja: 'Is this tweet neutral, in favor of, or against Hillary? + + + {{text}} ||| + + {{answer_choices[label]}}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: Hillary + reference: '' + 5c451846-349a-44ad-83ef-d0f8e5d2bd6b: !Template + answer_choices: Neutral ||| Against ||| In favor + id: 5c451846-349a-44ad-83ef-d0f8e5d2bd6b + jinja: '{{text}} Where does the above sentence stand on Hillary? ||| + + {{answer_choices[label]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Accuracy + original_task: true + name: Hillary_guess_passive + reference: '' + 83f10728-2347-46e9-b365-724f47e65877: !Template + answer_choices: Neutral ||| Against ||| In favor + id: 83f10728-2347-46e9-b365-724f47e65877 + jinja: '{{text}} Where does the author of the above sentence stand on Hillary? + ||| + + {{answer_choices[label]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - Accuracy + original_task: true + name: Hillary_guess_passive_author + reference: '' + b521857a-9d4f-4e21-848b-0baf7f4a636c: !Template + answer_choices: Neutral ||| Against ||| In favor + id: b521857a-9d4f-4e21-848b-0baf7f4a636c + jinja: 'How would you describe the stance used in this tweet? {{answer_choices|join(", + ")}} + + + {{text}} ||| + + {{answer_choices[label]}}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: Hillary_how_describe + reference: ''