promptfoo/examples/github-action/standalone-action.yaml

#
# This is a standalone github action that runs promptfoo against a change.
# You'll have to edit it to use the appropriate branches and prompts.
#
# For an easier to use packaged action, see:
#
# https://promptfoo.dev/docs/integrations/github-action
# https://github.com/promptfoo/promptfoo-action
#
name: LLM Prompt Evaluation

on:
  pull_request:
    paths:
      - 'prompts/**'

jobs:
  evaluate:
    runs-on: ubuntu-latest

    steps:
      - name: Checkout base ref (original)
        uses: actions/checkout@v2
        with:
          ref: ${{ github.base_ref }}
          path: base

      - name: Checkout head ref (modified)
        uses: actions/checkout@v2
        with:
          ref: ${{ github.head_ref }}
          path: head

      - name: Cache promptfoo data
        id: cache
        uses: actions/cache@v2
        with:
          path: ~/.cache/promptfoo
          key: ${{ runner.os }}-promptfoo-${{ hashFiles('**/package-lock.json') }}
          restore-keys: |
            ${{ runner.os }}-promptfoo-

      - name: Checkout head ref (modified)
        uses: actions/checkout@v2
        with:
          ref: ${{ github.head_ref }}
          path: head

      - name: Cache promptfoo data
        id: cache
        uses: actions/cache@v2
        with:
          path: ~/.cache/promptfoo
          key: ${{ runner.os }}-promptfoo-${{ hashFiles('**/package-lock.json') }}
          restore-keys: |
            ${{ runner.os }}-promptfoo-

      - name: Set up Node.js
        uses: actions/setup-node@v2
        with:
          node-version: 16

      - name: Run promptfoo evaluation
        id: eval
        env:
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
          PROMPTFOO_CACHE_PATH: ~/.cache/promptfoo
        run: |
          npx promptfoo eval -c head/prompts/promptfooconfig.yaml --prompts base/prompts/prompt1.json head/prompts/prompt1.json -o output.json --share
          echo "OUTPUT_JSON_PATH=$GITHUB_WORKSPACE/output.json" >> $GITHUB_ENV

      - name: Comment PR
        uses: actions/github-script@v6
        with:
          github-token: ${{secrets.GITHUB_TOKEN}}
          script: |
            const fs = require('fs');
            const output = JSON.parse(fs.readFileSync(process.env.OUTPUT_JSON_PATH, 'utf8'));
            const body = `⚠️ LLM prompt was modified.\n\n| Success | Failure |\n|---------|---------|\n| ${output.results.stats.successes}      | ${output.results.stats.failures}       |\n\n**» [View eval results](${output.shareableUrl}) «**`;
            github.rest.issues.createComment({
              issue_number: context.issue.number,
              owner: context.repo.owner,
              repo: context.repo.repo,
              body: body
            });