From 5d60ebd064b627d37b0ab5a8f7b5f2cfeab00385 Mon Sep 17 00:00:00 2001 From: Nora Belrose Date: Sun, 30 Apr 2023 00:47:25 +0000 Subject: [PATCH] Update templates --- .../fever/v1.0/templates.yaml | 44 +++--- .../fever/v2.0/templates.yaml | 0 .../truthful_qa_binary/templates.yaml | 125 ++++++++++++++++ .../templates/binarization/templates.yaml | 133 ++++++++++++++++++ 4 files changed, 278 insertions(+), 24 deletions(-) rename elk/promptsource/templates/{ => EleutherAI}/fever/v1.0/templates.yaml (59%) rename elk/promptsource/templates/{ => EleutherAI}/fever/v2.0/templates.yaml (100%) create mode 100644 elk/promptsource/templates/EleutherAI/truthful_qa_binary/templates.yaml create mode 100644 elk/promptsource/templates/binarization/templates.yaml diff --git a/elk/promptsource/templates/fever/v1.0/templates.yaml b/elk/promptsource/templates/EleutherAI/fever/v1.0/templates.yaml similarity index 59% rename from elk/promptsource/templates/fever/v1.0/templates.yaml rename to elk/promptsource/templates/EleutherAI/fever/v1.0/templates.yaml index b66be449..e9e275dc 100644 --- a/elk/promptsource/templates/fever/v1.0/templates.yaml +++ b/elk/promptsource/templates/EleutherAI/fever/v1.0/templates.yaml @@ -1,16 +1,12 @@ dataset: fever subset: v1.0 -label_column: label -label_choices: - - REFUTES - - SUPPORTS templates: 0870481e-e5d1-43a1-821e-b11c6bfd2483: !Template - answer_choices: Yes|||No|||Not sure + answer_choices: No|||Yes id: 0870481e-e5d1-43a1-821e-b11c6bfd2483 - jinja: "{{claim}} Is this true?\n|||\n{% if label != \"\" %}\n{{\n{\"SUPPORTS\"\ - : \"Yes\",\n \"REFUTES\": \"No\",\n\"NOT ENOUGH INFO\": \"Not sure\"\n}[label]\n\ - }}\n{% endif %}" + jinja: "{{claim}} Is this true? + + |||{{answer_choices[label]}}" metadata: !TemplateMetadata choices_in_prompt: false languages: @@ -21,11 +17,11 @@ templates: name: cbqa_fever_postprompt reference: CBQA fever, prompt after claim 51c55af8-1996-4cb2-88a1-ca7ddb8f9e11: !Template - answer_choices: Yes|||No|||Not Sure + answer_choices: No|||Yes id: 51c55af8-1996-4cb2-88a1-ca7ddb8f9e11 - jinja: "I've heard that {{claim}} Is this correct? Yes, No or Not Sure?\n|||\n\ - {% if label != \"\" %}\n{{\n{\"SUPPORTS\": \"Yes\",\n \"REFUTES\": \"No\",\n\ - \"NOT ENOUGH INFO\": \"Not Sure\"\n}[label]\n}}\n{% endif %}" + jinja: "I've heard that {{claim}} Is this correct? Yes, No or Not Sure? + + |||{{answer_choices[label]}}" metadata: !TemplateMetadata choices_in_prompt: true languages: @@ -37,11 +33,11 @@ templates: reference: CBQA fever, like a conversation, with prompts surrounding claim, all class included. 6cc8f145-3fb4-43a9-aaf1-8c25dd6e2cdf: !Template - answer_choices: Yes|||No|||Unsure + answer_choices: No|||Yes id: 6cc8f145-3fb4-43a9-aaf1-8c25dd6e2cdf - jinja: "Is this statement correct? {{claim}} ||| \n{% if label != \"\" %}\n{{\n\ - {\"SUPPORTS\": \"Yes\",\n \"REFUTES\": \"No\",\n\"NOT ENOUGH INFO\": \"Unsure\"\ - \n}[label]\n}}\n{% endif %}" + jinja: "Is this statement correct? {{claim}} + + |||{{answer_choices[label]}}" metadata: !TemplateMetadata choices_in_prompt: false languages: @@ -52,11 +48,11 @@ templates: name: cbqa_fever_preprompt reference: Closed-book QA from only the claim, prompt before the content 948f41ab-e6bb-4de6-af3e-7f0b5d5f39a8: !Template - answer_choices: Yes|||No|||Maybe + answer_choices: No|||Yes id: 948f41ab-e6bb-4de6-af3e-7f0b5d5f39a8 - jinja: "\"{{claim}}\" Yes, no, maybe?\n|||\n{% if label != \"\" %}\n{{\n{\"SUPPORTS\"\ - : \"Yes\",\n \"REFUTES\": \"No\",\n\"NOT ENOUGH INFO\": \"Maybe\"\n}[label]\n\ - }}\n{% endif %}\n" + jinja: "\"{{claim}}\" Yes or no? + + |||{{answer_choices[label]}}" metadata: !TemplateMetadata choices_in_prompt: false languages: @@ -67,11 +63,11 @@ templates: name: cbqa_fever_short reference: CBQA fever, minimal b1d8f035-c3af-41a8-b0b8-1604f9dc00ff: !Template - answer_choices: Yes|||No|||Not Sure + answer_choices: No|||Yes id: b1d8f035-c3af-41a8-b0b8-1604f9dc00ff - jinja: "\"{{claim}}\", I have heard. Is this Correct? Yes, No or Not Sure?\n|||\n\ - {% if label != \"\" %}\n{{\n{\"SUPPORTS\": \"Yes\",\n \"REFUTES\": \"No\",\n\ - \"NOT ENOUGH INFO\": \"Not Sure\"\n}[label]\n}}\n{% endif %}" + jinja: "\"{{claim}}\", I have heard. Is this Correct? Yes or No? + + |||{{answer_choices[label]}}" metadata: !TemplateMetadata choices_in_prompt: true languages: diff --git a/elk/promptsource/templates/fever/v2.0/templates.yaml b/elk/promptsource/templates/EleutherAI/fever/v2.0/templates.yaml similarity index 100% rename from elk/promptsource/templates/fever/v2.0/templates.yaml rename to elk/promptsource/templates/EleutherAI/fever/v2.0/templates.yaml diff --git a/elk/promptsource/templates/EleutherAI/truthful_qa_binary/templates.yaml b/elk/promptsource/templates/EleutherAI/truthful_qa_binary/templates.yaml new file mode 100644 index 00000000..19e8c300 --- /dev/null +++ b/elk/promptsource/templates/EleutherAI/truthful_qa_binary/templates.yaml @@ -0,0 +1,125 @@ +# Adapted from the ARC-Easy templates +dataset: EleutherAI/truthful_qa_binary +templates: + a1: !Template + answer_choices: 'A|||B' + id: a1 + jinja: 'Pick the most correct option to answer the following question. + + + {{question}} + + + Options: + + {% for letter, t in zip(answer_choices, choices) %} + - {{letter}}: {{t}} + + {% endfor %} + + |||{{answer_choices[label]}}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: pick_the_most_correct_option + reference: '' + a2: !Template + answer_choices: 'A|||B' + id: a2 + jinja: '{{question}} + + + Options: + + {% for letter, t in zip(answer_choices, choices) %} + - {{letter}}: {{t}} + + {% endfor %} + + Answer:|||{{answer_choices[label]}}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: qa_options + reference: '' + a3: !Template + answer_choices: 'A|||B' + id: a3 + jinja: 'I am hesitating between 2 options to answer the following question, which + option should I choose? + + + Question: {{question}} + + + Possibilities: + + {% for letter, t in zip(answer_choices, choices) %} + - {{letter}}: {{t}} + + {% endfor %} + + Answer:|||{{answer_choices[label]}}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: i_am_hesitating + reference: '' + a4: !Template + answer_choices: 'A|||B' + id: a4 + jinja: 'I gave my students this multiple choice question: {{question}} + + + Only one answer is correct among these 2 choices: + + {% for letter, t in zip(answer_choices, choices) %} + - {{letter}}: {{t}} + + {% endfor %} + + Answer:|||{{answer_choices[label]}}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: multiple_choice + reference: '' + a5: !Template + answer_choices: 'A|||B' + id: a6 + jinja: "Here's a problem to solve: {{question}} + + + Among the 2 following options, which is the correct answer? + + {% for letter, t in zip(answer_choices, choices) %} + {{letter}}: {{t}} + + {% endfor %} + + Answer:|||{{answer_choices[label]}}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: heres_a_problem + reference: '' diff --git a/elk/promptsource/templates/binarization/templates.yaml b/elk/promptsource/templates/binarization/templates.yaml new file mode 100644 index 00000000..608ff854 --- /dev/null +++ b/elk/promptsource/templates/binarization/templates.yaml @@ -0,0 +1,133 @@ +# Meta-template for binarizing a multiple choice dataset +dataset: binarization +templates: + a1: !Template + answer_choices: 'yes|||no' + id: a1 + jinja: "You will see a question followed by a response which may be incorrect. + You will be asked to figure out whether the answer is correct or not. + + + \"Q: {{question}} + + {% for letter, t in zip(['A', 'B', 'C', 'D'], choices) %} + - {{letter}}: {{t}} + + {% endfor %} + Response: {{['A', 'B', 'C', 'D'][response]}}, {{choices[response]}}\" + + + Is the response correct, yes or no?|||{{answer_choices[label]}}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: yes_no + reference: '' + a2: !Template + answer_choices: 'correct|||incorrect' + id: a2 + jinja: "The following quote contains a question followed by a response which may be + incorrect. Your task is to determine whether the answer is correct or incorrect. + + + \"Q: {{question}} + + {% for letter, t in zip(['A', 'B', 'C', 'D'], choices) %} + - {{letter}}: {{t}} + + {% endfor %} + Response: {{['A', 'B', 'C', 'D'][response]}}, {{choices[response]}}\" + + + Was the response correct or incorrect?|||{{answer_choices[label]}}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: correct_incorrect + reference: '' + a3: !Template + answer_choices: 'yes|||no' + id: a3 + jinja: "The following quote is an exam question followed by a student's response. + You will determine whether student chose the right answer. + + + \"Q: {{question}} + + {% for letter, t in zip(['A', 'B', 'C', 'D'], choices) %} + - {{letter}}: {{t}} + + {% endfor %} + Student's response: {{['A', 'B', 'C', 'D'][response]}}, {{choices[response]}}\" + + + Did the student choose the right answer, yes or no?|||{{answer_choices[label]}}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: student_yes_no + reference: '' + a4: !Template + answer_choices: 'correct|||incorrect' + id: a4 + jinja: "The paragraph below is a quiz question followed by a student's response. + You will determine if the response was right or wrong. + + + \"Q: {{question}} + + {% for letter, t in zip(['A', 'B', 'C', 'D'], choices) %} + - {{letter}}: {{t}} + + {% endfor %} + Student's response: {{['A', 'B', 'C', 'D'][response]}}, {{choices[response]}}\" + + + Was the response right or wrong?|||{{answer_choices[label]}}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: student_correct_incorrect + reference: '' + a5: !Template + answer_choices: 'true|||false' + id: a5 + jinja: "The following is an exam question followed by a student's response. + You will guess if they received full credit for the answer. + + + \"Q: {{question}} + + {% for letter, t in zip(['A', 'B', 'C', 'D'], choices) %} + - {{letter}}: {{t}} + + {% endfor %} + Student's response: {{['A', 'B', 'C', 'D'][response]}}, {{choices[response]}}\" + + + Did the student get full credit for this answer (yes or no)?|||{{answer_choices[label]}}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: student_full_credit + reference: ''