promptfoo/examples/gemma-vs-mistral/promptfooconfig.yaml

# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
description: Comparing Gemma and Mistral model performance

prompts:
  - '{{message}}'

defaultTest:
  options:
    transform: output.trim()

providers:
  - id: replicate:mistralai/mistral-7b-instruct-v0.2
    config:
      temperature: 0.01
      max_new_tokens: 1024
      prompt:
        prefix: '<s>[INST] '
        suffix: ' [/INST]'

  - id: replicate:mistralai/mixtral-8x7b-instruct-v0.1
    config:
      temperature: 0.01
      max_new_tokens: 1024
      prompt:
        prefix: '<s>[INST] '
        suffix: ' [/INST]'

  - id: replicate:cjwbw/gemma-7b-it:2790a695e5dcae15506138cc4718d1106d0d475e6dca4b1d43f42414647993d5
    config:
      temperature: 0.01
      max_new_tokens: 1024
      prompt:
        prefix: "<start_of_turn>user\n"
        suffix: "<end_of_turn>\n<start_of_turn>model"

tests:
  - vars:
      message: 'I speak without a mouth and hear without ears. I have no body, but I come alive with wind. What am I?'
    assert:
      # Make sure the LLM output contains this word
      - type: icontains
        value: echo
      # Use model-graded assertions to enforce free-form instructions
      - type: llm-rubric
        value: Do not apologize
  - vars:
      message: "You see a boat filled with people. It has not sunk, but when you look again you don't see a single person on the boat. Why?"
    assert:
      - type: llm-rubric
        value: explains that the people are below deck, or they are all in a relationship
  - vars:
      message: 'The more of this there is, the less you see. What is it?'
    assert:
      - type: icontains
        value: darkness
  - vars:
      message: >-
        I have keys but no locks. I have space but no room. You can enter, but
        can't go outside. What am I?
    assert:
      - type: icontains
        value: keyboard
  - vars:
      message: >-
        I am not alive, but I grow; I don't have lungs, but I need air; I don't
        have a mouth, but water kills me. What am I?
    assert:
      - type: icontains-any
        value:
          - fire
          - flame
  - vars:
      message: What can travel around the world while staying in a corner?
    assert:
      - type: icontains
        value: stamp
  - vars:
      message: Forward I am heavy, but backward I am not. What am I?
    assert:
      - type: icontains
        value: ton
  - vars:
      message: >-
        The person who makes it, sells it. The person who buys it, never uses
        it. The person who uses it, doesn't know they're using it. What is it?
    assert:
      - type: icontains
        value: coffin
  - vars:
      message: I can be cracked, made, told, and played. What am I?
    assert:
      - type: icontains
        value: joke
  - vars:
      message: What has keys but can't open locks?
    assert:
      - type: icontains
        value: piano
  - vars:
      message: >-
        I'm light as a feather, yet the strongest person can't hold me for much
        more than a minute. What am I?
    assert:
      - type: icontains
        value: breath
  - vars:
      message: >-
        I can fly without wings, I can cry without eyes. Whenever I go, darkness
        follows me. What am I?
    assert:
      - type: icontains
        value: cloud
  - vars:
      message: >-
        I am taken from a mine, and shut up in a wooden case, from which I am
        never released, and yet I am used by almost every person. What am I?
  - vars:
      message: >-
        David's father has three sons: Snap, Crackle, and _____? What is the
        name of the third son?
    assert:
      - type: contains
        value: David
  - vars:
      message: >-
        I am light as a feather, but even the world's strongest man couldn't
        hold me for much longer than a minute. What am I?
    assert:
      - type: contains
        value: breath