feat: add per-test repeat option (#9781)

Co-authored-by: Michael D'Angelo <mdangelo@openai.com>
Co-authored-by: mldangelo <michael.l.dangelo@gmail.com>

Author: 3 people

site/docs/configuration/test-cases.md

@@ -62,6 +62,23 @@ tests:
       difficulty: easy
 ```
 
+### Repeating an Individual Test
+
+Set `options.repeat` to a positive integer to run one test case multiple times:
+
+```yaml title="promptfooconfig.yaml"
+tests:
+  - description: 'Sample a nondeterministic response'
+    vars:
+      question: 'Write a short greeting'
+    options:
+      repeat: 3
+```
+
+The per-test value overrides `--repeat`, `commandLineOptions.repeat`, or
+`evaluateOptions.repeat` for that test. Other tests continue to use the global repeat count.
+Repeat indexes use separate cache entries; add `--no-cache` when every run must call the provider.
+
 ### Filtering Tests by Provider
 
 Control which providers run specific tests using the `providers` field. This allows you to run different test suites against different models in a single evaluation:

site/static/config-schema.json

@@ -1187,6 +1187,11 @@
         },
         "runSerially": {
           "type": "boolean"
+        },
+        "repeat": {
+          "type": "integer",
+          "exclusiveMinimum": 0,
+          "maximum": 9007199254740991
         }
       },
       "additionalProperties": {}

src/evaluator.ts

@@ -432,6 +432,12 @@ function getRepeatCacheNamespace(
   return undefined;
 }
 
+function normalizeRepeatCount(repeat: number | undefined, fallback = 1): number {
+  return typeof repeat === 'number' && Number.isSafeInteger(repeat) && repeat > 0
+    ? repeat
+    : fallback;
+}
+
 function hasGeneratedRedteamMetadata(test: AtomicTestCase): boolean {
   return (
     typeof test.metadata?.pluginId === 'string' &&
@@ -650,6 +656,17 @@ interface ProviderCallResult {
   traceContext: Awaited<ReturnType<typeof generateTraceContextIfNeeded>>;
 }
 
+function mergeProviderPromptConfig(
+  promptConfig: Prompt['config'],
+  testOptions: AtomicTestCase['options'],
+): Prompt['config'] {
+  const { repeat: _repeat, ...providerOptions } = testOptions ?? {};
+  return {
+    ...(promptConfig ?? {}),
+    ...providerOptions,
+  };
+}
+
 function createRunEvalState({
   provider,
   prompt,
@@ -662,10 +679,7 @@ function createRunEvalState({
   const setup = createRunEvalSetup({
     provider,
     prompt,
-    promptConfig: {
-      ...(prompt.config ?? {}),
-      ...(test.options ?? {}),
-    },
+    promptConfig: mergeProviderPromptConfig(prompt.config, test.options),
     vars,
   });
 
@@ -804,10 +818,7 @@ async function renderRunEvalPrompt({
   if (isRedteam) {
     throwIfTargetPromptExceedsMaxChars(renderedPrompt, testSuite?.redteam?.maxCharsPerMessage);
   }
-  const promptConfig = {
-    ...(promptForRender.config ?? {}),
-    ...(test.options ?? {}),
-  };
+  const promptConfig = mergeProviderPromptConfig(promptForRender.config, test.options);
   const setup = createRunEvalSetup({ provider, prompt: promptForRender, promptConfig, vars });
   setup.prompt.raw = renderedPrompt;
   return {
@@ -2170,6 +2181,7 @@ function mergeScenarioTest(
     },
     options: {
       ...(defaultTest?.options || {}),
+      ...data.options,
       ...test.options,
     },
     assert: [...(data.assert || []), ...(test.assert || [])],
@@ -2386,13 +2398,19 @@ function appendRunEvalOptionsForTestCase({
       ? [testCase.vars]
       : generateVarCombinations(testCase.vars || {});
 
-  for (let repeatIndex = 0; repeatIndex < (options.repeat || 1); repeatIndex++) {
+  const globalRepeat = normalizeRepeatCount(options.repeat);
+  const testRepeat = normalizeRepeatCount(testCase.options?.repeat, globalRepeat);
+  const effectiveOptions = {
+    ...options,
+    repeat: testRepeat,
+  };
+  for (let repeatIndex = 0; repeatIndex < testRepeat; repeatIndex++) {
     for (const vars of varCombinations) {
       appendRunEvalOptionsForVars({
         concurrency,
         conversations,
         evalId,
-        options,
+        options: effectiveOptions,
         promptIdCache,
         promptIndexMap,
         promptPrefix,

src/types/index.ts

@@ -907,6 +907,9 @@ export const TestCaseSchema = z.object({
       disableDefaultAsserts: z.boolean().optional(),
       // If true, run this without concurrency no matter what
       runSerially: z.boolean().optional(),
+
+      // Number of times to repeat this specific test case.
+      repeat: z.number().int().positive().safe().optional(),
     })
     .catchall(z.any())
     .optional(),

test/config-schema.test.ts

@@ -352,4 +352,22 @@ describe('config-schema.json', () => {
       }
     });
   });
+
+  describe('per-test repeat schema', () => {
+    it('accepts positive integers and rejects invalid repeat counts', () => {
+      const validate = ajv.compile(schema);
+      const config = {
+        prompts: ['hello'],
+        providers: ['echo'],
+        tests: [{ options: { repeat: 3 } }],
+      };
+
+      expect(validate(config)).toBe(true);
+
+      for (const repeat of [0, -1, 1.5, Number.MAX_SAFE_INTEGER + 1]) {
+        config.tests[0].options.repeat = repeat;
+        expect(validate(config), `repeat ${repeat} should be invalid`).toBe(false);
+      }
+    });
+  });
 });

test/evaluator/basic.test.ts

@@ -511,4 +511,60 @@ describeEvaluator('evaluator basic flows', () => {
     expect(summary.results[0].prompt.label).toBe('Test prompt');
     expect(summary.results[0].response?.output).toBe('Test output');
   });
+  it('allows test case repeat to override global repeat option', async () => {
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider],
+      prompts: [toPrompt('Test prompt')],
+      tests: [
+        {
+          options: {
+            repeat: 3,
+          },
+        },
+      ],
+    };
+
+    const evalRecord = await Eval.create({}, testSuite.prompts, {
+      id: randomUUID(),
+    });
+
+    await evaluate(testSuite, evalRecord, { repeat: 2 });
+
+    const summary = await evalRecord.toEvaluateSummary();
+
+    expect(mockApiProvider.callApi).toHaveBeenCalledTimes(3);
+    expect(
+      vi.mocked(mockApiProvider.callApi).mock.calls.map(([, context]) => context?.repeatIndex),
+    ).toEqual([0, 1, 2]);
+    expect(summary.results).toHaveLength(3);
+  });
+
+  it.each([
+    ['zero', 0],
+    ['negative', -1],
+    ['fractional', 1.5],
+    ['NaN', Number.NaN],
+    ['infinite', Number.POSITIVE_INFINITY],
+    ['unsafe', Number.MAX_SAFE_INTEGER + 1],
+  ])('falls back to global repeat for a %s per-test repeat', async (_label, repeat) => {
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider],
+      prompts: [toPrompt('Test prompt')],
+      tests: [
+        {
+          options: {
+            repeat,
+          },
+        },
+      ],
+    };
+
+    const evalRecord = await Eval.create({}, testSuite.prompts, {
+      id: randomUUID(),
+    });
+
+    await evaluate(testSuite, evalRecord, { repeat: 2 });
+
+    expect(mockApiProvider.callApi).toHaveBeenCalledTimes(2);
+  });
 });

test/evaluator/repeatCache.test.ts

@@ -71,9 +71,15 @@ describeEvaluator('evaluator repeat cache isolation', () => {
     expect(contexts[0]!.bustCache).toBeFalsy();
   });
 
-  it('isolates manual provider cache entries by repeat index', async () => {
+  it('isolates per-test repeat provider cache entries from the default namespace', async () => {
     await clearCache();
 
+    const baselineResponse: ProviderResponse = {
+      output: 'baseline-result',
+      tokenUsage: createEmptyTokenUsage(),
+    };
+    await getCache().set('manual-provider-key', baselineResponse);
+
     let cacheMissCount = 0;
     const provider: ApiProvider = {
       id: () => 'mock-provider',
@@ -103,11 +109,11 @@ describeEvaluator('evaluator repeat cache isolation', () => {
     const testSuite: TestSuite = {
       providers: [provider],
       prompts: [toPrompt('Test prompt')],
-      tests: [{}],
+      tests: [{ options: { repeat: 2 } }],
     };
 
     const firstEval = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
-    await evaluate(testSuite, firstEval, { maxConcurrency: 1, repeat: 2 });
+    await evaluate(testSuite, firstEval, { maxConcurrency: 1 });
     const firstSummary = await firstEval.toEvaluateSummary();
 
     expect(cacheMissCount).toBe(2);
@@ -118,7 +124,7 @@ describeEvaluator('evaluator repeat cache isolation', () => {
     expect(firstSummary.results.map((result) => result.response?.cached)).toEqual([false, false]);
 
     const secondEval = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
-    await evaluate(testSuite, secondEval, { maxConcurrency: 1, repeat: 2 });
+    await evaluate(testSuite, secondEval, { maxConcurrency: 1 });
     const secondSummary = await secondEval.toEvaluateSummary();
 
     expect(cacheMissCount).toBe(2);
@@ -127,6 +133,7 @@ describeEvaluator('evaluator repeat cache isolation', () => {
       'result-repeat-1-miss-2',
     ]);
     expect(secondSummary.results.map((result) => result.response?.cached)).toEqual([true, true]);
+    expect(await getCache().get('manual-provider-key')).toEqual(baselineResponse);
   });
 
   it('isolates beforeEach extension cache entries by repeat index', async () => {

test/evaluator/runEval.test.ts

@@ -193,14 +193,14 @@ describe('runEval', () => {
       ...defaultOptions,
       provider: mockProvider,
       prompt: promptWithFunction,
-      test: { options: { temperature: 0.9 } },
+      test: { options: { repeat: 3, temperature: 0.9 } },
       conversations: {},
       registers: {},
     });
     const result = results[0];
     expect(result.success).toBe(true);
 
-    // test.options should override dynamic config
+    // Provider options should override dynamic config, while evaluator-only options stay internal.
     const callApiMock = vi.mocked(mockProvider.callApi);
     const callApiArgs = callApiMock.mock.calls[0];
     expect(callApiArgs).toBeDefined();

test/evaluator/scenarios.test.ts

@@ -68,6 +68,62 @@ describeEvaluator('evaluator scenarios and conversations', () => {
     expect(summary.results[1].response?.output).toBe('Bonjour le monde');
   });
 
+  it('applies repeat from scenario config options', async () => {
+    const mockApiProvider: ApiProvider = {
+      id: vi.fn().mockReturnValue('test-provider'),
+      callApi: vi.fn().mockResolvedValue({
+        output: 'Hello',
+        tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0, numRequests: 1 },
+      }),
+    };
+
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider],
+      prompts: [toPrompt('Test prompt')],
+      scenarios: [
+        {
+          config: [{ options: { repeat: 3 } }],
+          tests: [{}],
+        },
+      ],
+    };
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+
+    await evaluate(testSuite, evalRecord, { repeat: 1 });
+    const summary = await evalRecord.toEvaluateSummary();
+
+    expect(mockApiProvider.callApi).toHaveBeenCalledTimes(3);
+    expect(summary.results).toHaveLength(3);
+  });
+
+  it('lets scenario test options.repeat override scenario config options.repeat', async () => {
+    const mockApiProvider: ApiProvider = {
+      id: vi.fn().mockReturnValue('test-provider'),
+      callApi: vi.fn().mockResolvedValue({
+        output: 'Hello',
+        tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0, numRequests: 1 },
+      }),
+    };
+
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider],
+      prompts: [toPrompt('Test prompt')],
+      scenarios: [
+        {
+          config: [{ options: { repeat: 5 } }],
+          tests: [{ options: { repeat: 2 } }],
+        },
+      ],
+    };
+    const evalRecord = await Eval.create({}, testSuite.prompts, { id: randomUUID() });
+
+    await evaluate(testSuite, evalRecord, { repeat: 1 });
+    const summary = await evalRecord.toEvaluateSummary();
+
+    expect(mockApiProvider.callApi).toHaveBeenCalledTimes(2);
+    expect(summary.results).toHaveLength(2);
+  });
+
   it('evaluate with scenarios and multiple vars', async () => {
     const mockApiProvider: ApiProvider = {
       id: vi.fn().mockReturnValue('test-provider'),

test/types/index.test.ts

@@ -358,12 +358,24 @@ describe('TestCaseSchema options (merged schema properties)', () => {
       options: {
         disableVarExpansion: true,
         disableConversationVar: true,
+        repeat: 3,
         runSerially: true,
       },
     };
     expect(() => TestCaseSchema.parse(testCase)).not.toThrow();
   });
 
+  it.each([
+    0,
+    -1,
+    1.5,
+    Number.NaN,
+    Number.POSITIVE_INFINITY,
+    Number.MAX_SAFE_INTEGER + 1,
+  ])('should reject invalid per-test repeat %s', (repeat) => {
+    expect(TestCaseSchema.safeParse({ options: { repeat } }).success).toBe(false);
+  });
+
   it('should validate options combining properties from ALL merged schemas', () => {
     // This is the critical test - properties from different sub-schemas must work together
     const testCase = {