01_contains_pattern.yaml

run_config:
  module_name: contains_pattern
  class_name: EvaluationContainsPattern
  models:
    ollama:
      - qwen2.5:14b
      - llama3.1:8b-instruct-q8_0

evaluation_instances:
  - name: "Python 3.11 Release Date"
    prompt: |-
      When did the first stable release of Python 3.11 come out?
      Format your answer using this strftime format: %b %-d, %Y
      For example, if the date is January 1st, 2000, then you should write: Jan 1, 2000
    pattern: "Oct 24, 2022"

  - name: "Python 3.12 Release Date"
    prompt: |-
      When did the first stable release of Python 3.12 come out?
      Format your answer using this strftime format: %b %-d, %Y
      For example, if the date is January 1st, 2000, then you should write: Jan 1, 2000
    pattern: "Oct 2, 2023"

  - name: "Legless Elephants"
    prompt: |-
      How many legs do ten elephants have, if two of them are legless?
    system_prompt: >-
      - You are a helpful assistant who answers user's questions which have a numerical answer.

      - Your answer in the end needs to be in JSON format, so that a grader can easily parse it.
      You must use the following format: {"answer": 9} to indicate your response. Replace 9 with the answer you want to give.
      
      - Before giving the answer, you think step by step before giving the answer.
    pattern: >-
      {"answer": 32}