promptfoo/examples/gpt-4o-vs-4o-mini/promptfooconfig.yaml

# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
prompts:
  - 'Solve this riddle: {{riddle}}'

providers:
  - openai:gpt-4o
  - openai:gpt-4o-mini

defaultTest:
  assert:
    # Inference should always cost less than this (USD)
    - type: cost
      threshold: 0.002
    # Inference should always be faster than this (milliseconds)
    - type: latency
      threshold: 3000

tests:
  - vars:
      riddle: 'I speak without a mouth and hear without ears. I have no body, but I come alive with wind. What am I?'
    assert:
      # Make sure the LLM output contains this word
      - type: contains
        value: echo
      # Use model-graded assertions to enforce free-form instructions
      - type: llm-rubric
        value: Do not apologize
  - vars:
      riddle: 'You see a boat filled with people. It has not sunk, but when you look again you don’t see a single person on the boat. Why?'
    assert:
      - type: llm-rubric
        value: explains that the people are below deck
  - vars:
      riddle: 'The more of this there is, the less you see. What is it?'
    assert:
      - type: contains
        value: darkness
  - vars:
      riddle: >-
        I have keys but no locks. I have space but no room. You can enter, but
        can’t go outside. What am I?
  - vars:
      riddle: >-
        I am not alive, but I grow; I don't have lungs, but I need air; I don't
        have a mouth, but water kills me. What am I?
  - vars:
      riddle: What can travel around the world while staying in a corner?
  - vars:
      riddle: Forward I am heavy, but backward I am not. What am I?
  - vars:
      riddle: >-
        The person who makes it, sells it. The person who buys it, never uses
        it. The person who uses it, doesn't know they're using it. What is it?
  - vars:
      riddle: I can be cracked, made, told, and played. What am I?
  - vars:
      riddle: What has keys but can't open locks?
  - vars:
      riddle: >-
        I'm light as a feather, yet the strongest person can't hold me for much
        more than a minute. What am I?
  - vars:
      riddle: >-
        I can fly without wings, I can cry without eyes. Whenever I go, darkness
        follows me. What am I?
  - vars:
      riddle: >-
        I am taken from a mine, and shut up in a wooden case, from which I am
        never released, and yet I am used by almost every person. What am I?
  - vars:
      riddle: >-
        David's father has three sons: Snap, Crackle, and _____? What is the
        name of the third son?
  - vars:
      riddle: >-
        I am light as a feather, but even the world's strongest man couldn’t
        hold me for much longer than a minute. What am I?