Update templates

elk/promptsource/templates/fever/v1.0/templates.yaml → elk/promptsource/templates/EleutherAI/fever/v1.0/templates.yaml

@@ -1,16 +1,12 @@
 dataset: fever
 subset: v1.0
-label_column: label
-label_choices:
- - REFUTES
- - SUPPORTS
 templates:
  0870481e-e5d1-43a1-821e-b11c6bfd2483: !Template
- answer_choices: Yes|||No|||Not sure
+ answer_choices: No|||Yes
  id: 0870481e-e5d1-43a1-821e-b11c6bfd2483
- jinja: "{{claim}} Is this true?\n|||\n{% if label != \"\" %}\n{{\n{\"SUPPORTS\"\
- : \"Yes\",\n \"REFUTES\": \"No\",\n\"NOT ENOUGH INFO\": \"Not sure\"\n}[label]\n\
-  }}\n{% endif %}"
+ jinja: "{{claim}} Is this true?
+
+ |||{{answer_choices[label]}}"
  metadata: !TemplateMetadata
  choices_in_prompt: false
  languages:
@@ -21,11 +17,11 @@ templates:
  name: cbqa_fever_postprompt
  reference: CBQA fever, prompt after claim
  51c55af8-1996-4cb2-88a1-ca7ddb8f9e11: !Template
- answer_choices: Yes|||No|||Not Sure
+ answer_choices: No|||Yes
  id: 51c55af8-1996-4cb2-88a1-ca7ddb8f9e11
- jinja: "I've heard that {{claim}} Is this correct? Yes, No or Not Sure?\n|||\n\
- {% if label != \"\" %}\n{{\n{\"SUPPORTS\": \"Yes\",\n \"REFUTES\": \"No\",\n\
-  \"NOT ENOUGH INFO\": \"Not Sure\"\n}[label]\n}}\n{% endif %}"
+ jinja: "I've heard that {{claim}} Is this correct? Yes, No or Not Sure?
+
+ |||{{answer_choices[label]}}"
  metadata: !TemplateMetadata
  choices_in_prompt: true
  languages:
@@ -37,11 +33,11 @@ templates:
  reference: CBQA fever, like a conversation, with prompts surrounding claim, all
  class included.
  6cc8f145-3fb4-43a9-aaf1-8c25dd6e2cdf: !Template
- answer_choices: Yes|||No|||Unsure
+ answer_choices: No|||Yes
  id: 6cc8f145-3fb4-43a9-aaf1-8c25dd6e2cdf
- jinja: "Is this statement correct? {{claim}} ||| \n{% if label != \"\" %}\n{{\n\
- {\"SUPPORTS\": \"Yes\",\n \"REFUTES\": \"No\",\n\"NOT ENOUGH INFO\": \"Unsure\"\
-  \n}[label]\n}}\n{% endif %}"
+ jinja: "Is this statement correct? {{claim}}
+
+ |||{{answer_choices[label]}}"
  metadata: !TemplateMetadata
  choices_in_prompt: false
  languages:
@@ -52,11 +48,11 @@ templates:
  name: cbqa_fever_preprompt
  reference: Closed-book QA from only the claim, prompt before the content
  948f41ab-e6bb-4de6-af3e-7f0b5d5f39a8: !Template
- answer_choices: Yes|||No|||Maybe
+ answer_choices: No|||Yes
  id: 948f41ab-e6bb-4de6-af3e-7f0b5d5f39a8
- jinja: "\"{{claim}}\" Yes, no, maybe?\n|||\n{% if label != \"\" %}\n{{\n{\"SUPPORTS\"\
- : \"Yes\",\n \"REFUTES\": \"No\",\n\"NOT ENOUGH INFO\": \"Maybe\"\n}[label]\n\
-  }}\n{% endif %}\n"
+ jinja: "\"{{claim}}\" Yes or no?
+
+ |||{{answer_choices[label]}}"
  metadata: !TemplateMetadata
  choices_in_prompt: false
  languages:
@@ -67,11 +63,11 @@ templates:
  name: cbqa_fever_short
  reference: CBQA fever, minimal
  b1d8f035-c3af-41a8-b0b8-1604f9dc00ff: !Template
- answer_choices: Yes|||No|||Not Sure
+ answer_choices: No|||Yes
  id: b1d8f035-c3af-41a8-b0b8-1604f9dc00ff
- jinja: "\"{{claim}}\", I have heard. Is this Correct? Yes, No or Not Sure?\n|||\n\
- {% if label != \"\" %}\n{{\n{\"SUPPORTS\": \"Yes\",\n \"REFUTES\": \"No\",\n\
-  \"NOT ENOUGH INFO\": \"Not Sure\"\n}[label]\n}}\n{% endif %}"
+ jinja: "\"{{claim}}\", I have heard. Is this Correct? Yes or No?
+
+ |||{{answer_choices[label]}}"
  metadata: !TemplateMetadata
  choices_in_prompt: true
  languages:

elk/promptsource/templates/EleutherAI/truthful_qa_binary/templates.yaml

@@ -0,0 +1,125 @@
+# Adapted from the ARC-Easy templates
+dataset: EleutherAI/truthful_qa_binary
+templates:
+ a1: !Template
+ answer_choices: 'A|||B'
+ id: a1
+ jinja: 'Pick the most correct option to answer the following question.
+
+
+ {{question}}
+
+
+ Options:
+
+ {% for letter, t in zip(answer_choices, choices) %}
+ - {{letter}}: {{t}}
+
+ {% endfor %}
+
+ |||{{answer_choices[label]}}'
+ metadata: !TemplateMetadata
+ choices_in_prompt: true
+ languages:
+ - en
+ metrics:
+ - Accuracy
+ original_task: true
+ name: pick_the_most_correct_option
+ reference: ''
+ a2: !Template
+ answer_choices: 'A|||B'
+ id: a2
+ jinja: '{{question}}
+
+
+ Options:
+
+ {% for letter, t in zip(answer_choices, choices) %}
+ - {{letter}}: {{t}}
+
+ {% endfor %}
+
+ Answer:|||{{answer_choices[label]}}'
+ metadata: !TemplateMetadata
+ choices_in_prompt: true
+ languages:
+ - en
+ metrics:
+ - Accuracy
+ original_task: true
+ name: qa_options
+ reference: ''
+ a3: !Template
+ answer_choices: 'A|||B'
+ id: a3
+ jinja: 'I am hesitating between 2 options to answer the following question, which
+ option should I choose?
+
+
+ Question: {{question}}
+
+
+ Possibilities:
+
+ {% for letter, t in zip(answer_choices, choices) %}
+ - {{letter}}: {{t}}
+
+ {% endfor %}
+
+ Answer:|||{{answer_choices[label]}}'
+ metadata: !TemplateMetadata
+ choices_in_prompt: true
+ languages:
+ - en
+ metrics:
+ - Accuracy
+ original_task: true
+ name: i_am_hesitating
+ reference: ''
+ a4: !Template
+ answer_choices: 'A|||B'
+ id: a4
+ jinja: 'I gave my students this multiple choice question: {{question}}
+
+
+ Only one answer is correct among these 2 choices:
+
+ {% for letter, t in zip(answer_choices, choices) %}
+ - {{letter}}: {{t}}
+
+ {% endfor %}
+
+ Answer:|||{{answer_choices[label]}}'
+ metadata: !TemplateMetadata
+ choices_in_prompt: true
+ languages:
+ - en
+ metrics:
+ - Accuracy
+ original_task: true
+ name: multiple_choice
+ reference: ''
+ a5: !Template
+ answer_choices: 'A|||B'
+ id: a6
+ jinja: "Here's a problem to solve: {{question}}
+
+
+ Among the 2 following options, which is the correct answer?
+
+ {% for letter, t in zip(answer_choices, choices) %}
+ {{letter}}: {{t}}
+
+ {% endfor %}
+
+ Answer:|||{{answer_choices[label]}}"
+ metadata: !TemplateMetadata
+ choices_in_prompt: true
+ languages:
+ - en
+ metrics:
+ - Accuracy
+ original_task: true
+ name: heres_a_problem
+ reference: ''

elk/promptsource/templates/binarization/templates.yaml

@@ -0,0 +1,133 @@
+# Meta-template for binarizing a multiple choice dataset
+dataset: binarization
+templates:
+ a1: !Template
+ answer_choices: 'yes|||no'
+ id: a1
+ jinja: "You will see a question followed by a response which may be incorrect.
+ You will be asked to figure out whether the answer is correct or not.
+
+
+ \"Q: {{question}}
+
+ {% for letter, t in zip(['A', 'B', 'C', 'D'], choices) %}
+ - {{letter}}: {{t}}
+
+ {% endfor %}
+ Response: {{['A', 'B', 'C', 'D'][response]}}, {{choices[response]}}\"
+
+
+ Is the response correct, yes or no?|||{{answer_choices[label]}}"
+ metadata: !TemplateMetadata
+ choices_in_prompt: true
+ languages:
+ - en
+ metrics:
+ - Accuracy
+ original_task: true
+ name: yes_no
+ reference: ''
+ a2: !Template
+ answer_choices: 'correct|||incorrect'
+ id: a2
+ jinja: "The following quote contains a question followed by a response which may be
+ incorrect. Your task is to determine whether the answer is correct or incorrect.
+
+
+ \"Q: {{question}}
+
+ {% for letter, t in zip(['A', 'B', 'C', 'D'], choices) %}
+ - {{letter}}: {{t}}
+
+ {% endfor %}
+ Response: {{['A', 'B', 'C', 'D'][response]}}, {{choices[response]}}\"
+
+
+ Was the response correct or incorrect?|||{{answer_choices[label]}}"
+ metadata: !TemplateMetadata
+ choices_in_prompt: true
+ languages:
+ - en
+ metrics:
+ - Accuracy
+ original_task: true
+ name: correct_incorrect
+ reference: ''
+ a3: !Template
+ answer_choices: 'yes|||no'
+ id: a3
+ jinja: "The following quote is an exam question followed by a student's response.
+ You will determine whether student chose the right answer.
+
+
+ \"Q: {{question}}
+
+ {% for letter, t in zip(['A', 'B', 'C', 'D'], choices) %}
+ - {{letter}}: {{t}}
+
+ {% endfor %}
+ Student's response: {{['A', 'B', 'C', 'D'][response]}}, {{choices[response]}}\"
+
+
+ Did the student choose the right answer, yes or no?|||{{answer_choices[label]}}"
+ metadata: !TemplateMetadata
+ choices_in_prompt: true
+ languages:
+ - en
+ metrics:
+ - Accuracy
+ original_task: true
+ name: student_yes_no
+ reference: ''
+ a4: !Template
+ answer_choices: 'correct|||incorrect'
+ id: a4
+ jinja: "The paragraph below is a quiz question followed by a student's response.
+ You will determine if the response was right or wrong.
+
+
+ \"Q: {{question}}
+
+ {% for letter, t in zip(['A', 'B', 'C', 'D'], choices) %}
+ - {{letter}}: {{t}}
+
+ {% endfor %}
+ Student's response: {{['A', 'B', 'C', 'D'][response]}}, {{choices[response]}}\"
+
+
+ Was the response right or wrong?|||{{answer_choices[label]}}"
+ metadata: !TemplateMetadata
+ choices_in_prompt: true
+ languages:
+ - en
+ metrics:
+ - Accuracy
+ original_task: true
+ name: student_correct_incorrect
+ reference: ''
+ a5: !Template
+ answer_choices: 'true|||false'
+ id: a5
+ jinja: "The following is an exam question followed by a student's response.
+ You will guess if they received full credit for the answer.
+
+
+ \"Q: {{question}}
+
+ {% for letter, t in zip(['A', 'B', 'C', 'D'], choices) %}
+ - {{letter}}: {{t}}
+
+ {% endfor %}
+ Student's response: {{['A', 'B', 'C', 'D'][response]}}, {{choices[response]}}\"
+
+
+ Did the student get full credit for this answer (yes or no)?|||{{answer_choices[label]}}"
+ metadata: !TemplateMetadata
+ choices_in_prompt: true
+ languages:
+ - en
+ metrics:
+ - Accuracy
+ original_task: true
+ name: student_full_credit
+ reference: ''