1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
|
- import { maybeEmitAzureOpenAiWarning } from '../src/providers/azureopenaiUtil';
- import { AzureOpenAiCompletionProvider } from '../src/providers/azureopenai';
- import { OpenAiCompletionProvider } from '../src/providers/openai';
- import type { TestSuite, TestCase } from '../src/types';
- import { HuggingfaceTextGenerationProvider } from '../src/providers/huggingface';
- describe('maybeEmitAzureOpenAiWarning', () => {
- test('should not emit warning when no Azure providers are used', () => {
- const testSuite: TestSuite = {
- providers: [new OpenAiCompletionProvider('foo')],
- defaultTest: {},
- prompts: [],
- };
- const tests: TestCase[] = [
- {
- assert: [{ type: 'llm-rubric', value: 'foo bar' }],
- },
- ];
- const result = maybeEmitAzureOpenAiWarning(testSuite, tests);
- expect(result).toBe(false);
- });
- test('should not emit warning when Azure provider is used alone, but no model graded eval', () => {
- const testSuite: TestSuite = {
- providers: [new AzureOpenAiCompletionProvider('foo')],
- defaultTest: {},
- prompts: [],
- };
- const tests: TestCase[] = [
- {
- assert: [{ type: 'equals' }],
- },
- ];
- const result = maybeEmitAzureOpenAiWarning(testSuite, tests);
- expect(result).toBe(false);
- });
- test('should emit warning when Azure provider is used alone, but with model graded eval', () => {
- const testSuite: TestSuite = {
- providers: [new AzureOpenAiCompletionProvider('foo')],
- defaultTest: {},
- prompts: [],
- };
- const tests: TestCase[] = [
- {
- assert: [{ type: 'llm-rubric', value: 'foo bar' }],
- },
- ];
- const result = maybeEmitAzureOpenAiWarning(testSuite, tests);
- expect(result).toBe(true);
- });
- test('should emit warning when Azure provider used with non-OpenAI provider', () => {
- const testSuite: TestSuite = {
- providers: [
- new AzureOpenAiCompletionProvider('foo'),
- new HuggingfaceTextGenerationProvider('bar'),
- ],
- defaultTest: {},
- prompts: [],
- };
- const tests: TestCase[] = [
- {
- assert: [{ type: 'llm-rubric', value: 'foo bar' }],
- },
- ];
- const result = maybeEmitAzureOpenAiWarning(testSuite, tests);
- expect(result).toBe(true);
- });
- test('should not emit warning when Azure providers are used with a default provider set', () => {
- const testSuite: TestSuite = {
- providers: [new AzureOpenAiCompletionProvider('foo')],
- defaultTest: { options: { provider: 'azureopenai:....' } },
- prompts: [],
- };
- const tests: TestCase[] = [
- {
- assert: [{ type: 'llm-rubric', value: 'foo bar' }],
- },
- ];
- const result = maybeEmitAzureOpenAiWarning(testSuite, tests);
- expect(result).toBe(false);
- });
- test('should not emit warning when both Azure and OpenAI providers are used', () => {
- const testSuite: TestSuite = {
- providers: [new AzureOpenAiCompletionProvider('foo'), new OpenAiCompletionProvider('bar')],
- defaultTest: {},
- prompts: [],
- };
- const tests: TestCase[] = [
- {
- assert: [{ type: 'llm-rubric', value: 'foo bar' }],
- },
- ];
- const result = maybeEmitAzureOpenAiWarning(testSuite, tests);
- expect(result).toBe(false);
- });
- });
|