promptfoo/.github/workflows/main.yml

name: CI
on:
  pull_request:
  push:
    branches:
      - main
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

env:
  OPENAI_API_KEY: xxx
  ANTHROPIC_API_KEY: xxx
  AZURE_OPENAI_API_HOST: xxx
  VITE_TELEMETRY_DISABLED: 1
jobs:
  build:
    name: Build and test on Node ${{ matrix.node }} and ${{ matrix.os }}
    timeout-minutes: 12
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        # Temporarily disabled 22.x due to github runner compatibility issues
        # TODO(Michael): Re-enable once issues are resolved, 2024-08-23
        node: ['18.x', '20.x']
        os: [ubuntu-latest, windows-latest, macOS-latest]
        exclude:
          # Remove when https://github.com/nodejs/node/issues/51766 is resolved
          - node: '22.x'
            os: windows-latest

    steps:
      - name: Checkout repo
        uses: actions/checkout@v4

      - name: Use Node ${{ matrix.node }}
        uses: actions/setup-node@v4
        with:
          node-version: ${{ matrix.node }}
          cache: 'npm'

      - name: Use Python 3.12
        uses: actions/setup-python@v5
        with:
          python-version: 3.12

      - name: Install Dependencies
        run: |
          npm ci
      - name: Test
        run: npm run test -- --ci --coverage --maxWorkers=2

      - name: Build
        run: npm run build

  style-check:
    name: Style Check
    timeout-minutes: 5
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repo
        uses: actions/checkout@v4

      - name: Use Node
        uses: actions/setup-node@v4
        with:
          node-version-file: '.nvmrc'
          cache: 'npm'

      - name: Install Dependencies
        run: |
          npm ci

      - name: Lint Project
        run: |
          npm run lint

      - name: Lint UI
        working-directory: src/app
        run: |
          npm run lint

      - name: Run Style Check
        run: |
          npm run format:check

      - name: Check Dependency Versions
        run: |
          npm exec check-dependency-version-consistency

      - name: Check for circular dependencies
        run: npx madge $(git ls-files '*.ts') --circular

  assets:
    name: Generate Assets
    timeout-minutes: 5
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repo
        uses: actions/checkout@v4

      - name: Use Node
        uses: actions/setup-node@v4
        with:
          node-version-file: '.nvmrc'
          cache: 'npm'

      - name: Install Dependencies
        run: npm install

      - name: Generate JSON Schema
        run: npm run jsonSchema:generate

      - name: Check for changes
        run: |
          if [[ -n $(git status --porcelain) ]]; then
            echo "Changes detected after generating assets:"
            git status --porcelain
            exit 1
          else
            echo "No changes detected."
          fi

  python:
    name: Check Python
    timeout-minutes: 5
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: [3.9, 3.12]
    steps:
      - name: Checkout repo
        uses: actions/checkout@v4

      - name: Use Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install Dependencies
        run: |
          pip install ruff

      - name: Check Formatting
        run: |
          ruff check --select I --fix
          ruff format
          git diff --exit-code || (echo "Files were modified by ruff. Please commit these changes." && exit 1)

      - name: Run Tests
        run: |
          python -m unittest discover -s src/python -p '*_test.py'

  docs:
    name: Build Docs
    timeout-minutes: 5
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repo
        uses: actions/checkout@v4

      - name: Use Node
        uses: actions/setup-node@v4
        with:
          node-version-file: '.nvmrc'
          cache: 'npm'

      - name: Install Dependencies
        working-directory: site
        run: npm install

      - name: Type Check
        working-directory: site
        run: npm run typecheck

      - name: Build Documentation
        working-directory: site
        run: npm run build

  webui:
    name: webui tests
    timeout-minutes: 5
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repo
        uses: actions/checkout@v4

      - name: Use Node
        uses: actions/setup-node@v4
        with:
          node-version-file: '.nvmrc'
          cache: 'npm'

      - name: Install Dependencies
        run: npm ci

      - name: Run App Tests
        run: npm run test:app

  integration-tests:
    name: Run Integration Tests
    runs-on: ubuntu-latest
    timeout-minutes: 5
    steps:
      - name: Checkout repo
        uses: actions/checkout@v4

      - name: Use Node
        uses: actions/setup-node@v4
        with:
          node-version-file: '.nvmrc'
          cache: 'npm'

      - name: Use Python 3.12
        uses: actions/setup-python@v5
        with:
          python-version: 3.12

      - name: Install Dependencies
        run: |
          npm ci

      - name: Run Integration Tests
        run: npm run test:integration -- --ci --coverage --maxWorkers=2

  share-test:
    name: Share Test
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repo
        uses: actions/checkout@v4

      - name: Install Dependencies
        run: npm ci

      - name: Run local server
        run: |
          npm run build
          PROMPTFOO_CONFIG_DIR=~/tmp LOG_LEVEL=DEBUG API_PORT=8500 node dist/src/server/index.js &

      - name: run promptfoo eval
        id: eval
        run: |
          PROMPTFOO_REMOTE_API_BASE_URL=http://localhost:8500 PROMPTFOO_SHARING_APP_BASE_URL=http://localhost:8500  node dist/src/main.js eval -c .github/workflows/files/promptfooconfig.yaml --share

      - name: Test that the eval results are uploaded
        run: |
          response=$(curl -s http://localhost:8500/api/results)
          echo "Response: $response"

          # Use jq to extract the array length
          count=$(echo "$response" | jq '.data | length')
          echo "Array Length: $count"

          # Check if the count is exactly 1
          if [ "$count" -ne 1 ]; then
            echo "Error: Expected 1 entry, but got $count"
            exit 1
          fi

      - name: Share to cloud
        run: |
          if [ -n "${{ secrets.PROMPTFOO_STAGING_API_KEY }}" ]; then
            node dist/src/main.js auth login -k ${{ secrets.PROMPTFOO_STAGING_API_KEY }} -h https://api.promptfoo-staging.app
            node dist/src/main.js eval -c .github/workflows/files/promptfooconfig.yaml --share
          fi