nirbarazida
/
promptfoo
mirror of https://github.com/promptfoo/promptfoo


  
1

	
2

	
3

	
4

	
5

	
6

	
7

	
8

	
9

	
10

	
11

	
12

	
13

	
14

	
15

	
16

	
17

	
18

	
19

	
20

	
21

	
22

	
23

	
24

	
25

	
26

	
27

	
28

	
29

	
30

	
31

	
32

	
33

	
34

	
35

	
36

	
37

	
38

	
39

	
40

	
41

	
42

	
43

	
44

	
45

	
46

	
47

	
48

	
49

	
50

	
51

	
52

	
53

	
54

	
55

	
56

	
57

	
58

	
59

	
60

	
61

	
62

	
63

	
64

	
65

	
66

	
67

	
68

	
69

	
70

	
71

	
72

	
73

	
74

	
75

	
76

	
77

	
78

	
79

	
80

	
81

	
82

	
83

	
84

	
85

	
86

	
87

	
88

	
89

	
90

	
91

	
92

	
93

	
94

	
95

	
96

	
97

	
98

	
99

	
100

	
101

	
102

	
103

	
104

	
105

	
106

	
107

	
108

	
109

	
110

	
111

	
112

	
113

	
114

	
115

	
116

	
117

	
118

	
119

	
120

	
121

	
122

	
123

	
            import { describe, expect, it } from '@jest/globals';
import { handleFinishReason } from '../../src/assertions/finishReason';

import type { AssertionParams } from '../../src/types';

describe('finishReason assertion', () => {
  it('should pass when finish reason matches', () => {
    const params: Partial<AssertionParams> = {
      assertion: { type: 'finish-reason', value: 'stop' },
      providerResponse: { finishReason: 'stop' },
    };

    const result = handleFinishReason(params as AssertionParams);
    expect(result.pass).toBe(true);
    expect(result.score).toBe(1);
  });

  it('should fail when stop reason does not match', () => {
    const params: Partial<AssertionParams> = {
      assertion: { type: 'finish-reason', value: 'stop' },
      providerResponse: { finishReason: 'length' },
    };

    const result = handleFinishReason(params as AssertionParams);
    expect(result.pass).toBe(false);
    expect(result.score).toBe(0);
  });

  it('should fail when provider does not return stop reason', () => {
    const params: Partial<AssertionParams> = {
      assertion: { type: 'finish-reason', value: 'stop' },
      providerResponse: {},
    };

    const result = handleFinishReason(params as AssertionParams);
    expect(result.pass).toBe(false);
    expect(result.score).toBe(0);
    expect(result.reason).toContain('did not supply');
  });

  it('should handle renderedValue override', () => {
    const params: Partial<AssertionParams> = {
      assertion: { type: 'finish-reason', value: 'stop' },
      renderedValue: 'length',
      providerResponse: { finishReason: 'length' },
    };

    const result = handleFinishReason(params as AssertionParams);
    expect(result.pass).toBe(true);
    expect(result.score).toBe(1);
  });

  it('should prioritize renderedValue over assertion.value', () => {
    const params: Partial<AssertionParams> = {
      assertion: { type: 'finish-reason', value: 'stop' },
      renderedValue: 'length',
      providerResponse: { finishReason: 'stop' },
    };

    const result = handleFinishReason(params as AssertionParams);
    expect(result.pass).toBe(false);
    expect(result.score).toBe(0);
    expect(result.reason).toContain('Expected finish reason "length" but got "stop"');
  });

  it('should throw for non-string assertion value', () => {
    const params: Partial<AssertionParams> = {
      assertion: { type: 'finish-reason', value: 123 as any },
      providerResponse: { finishReason: 'stop' },
    };

    expect(() => handleFinishReason(params as AssertionParams)).toThrow(
      '"finish-reason" assertion type must have a string value',
    );
  });

  it('should throw for non-string renderedValue', () => {
    const params: Partial<AssertionParams> = {
      assertion: { type: 'finish-reason', value: 'stop' },
      renderedValue: 123 as any,
      providerResponse: { finishReason: 'stop' },
    };

    expect(() => handleFinishReason(params as AssertionParams)).toThrow(
      '"finish-reason" assertion type must have a string value',
    );
  });

  it('should handle undefined finishReason gracefully', () => {
    const params: Partial<AssertionParams> = {
      assertion: { type: 'finish-reason', value: 'stop' },
      providerResponse: { finishReason: undefined },
    };

    const result = handleFinishReason(params as AssertionParams);
    expect(result.pass).toBe(false);
    expect(result.score).toBe(0);
    expect(result.reason).toContain('did not supply');
  });

  it('should perform case-insensitive matching', () => {
    const params: Partial<AssertionParams> = {
      assertion: { type: 'finish-reason', value: 'stop' },
      providerResponse: { finishReason: 'STOP' }, // Different case
    };

    const result = handleFinishReason(params as AssertionParams);
    expect(result.pass).toBe(true); // Now passes with case-insensitive comparison
    expect(result.score).toBe(1);
    expect(result.reason).toBe('Assertion passed');
  });

  it('should handle mixed case in both assertion and response', () => {
    const params: Partial<AssertionParams> = {
      assertion: { type: 'finish-reason', value: 'LENGTH' },
      providerResponse: { finishReason: 'length' },
    };

    const result = handleFinishReason(params as AssertionParams);
    expect(result.pass).toBe(true);
    expect(result.score).toBe(1);
  });
});