[SDTEST-3823] Add test discovery cache support

docs/best_practices.md

@@ -61,6 +61,65 @@ After these changes the tests discovery will be faster and will not fail when
 database is not present. You can skip database setup for planning step completely
 and save a lot of time.
 
+### Cache Test Discovery
+
+If full test discovery still takes too long, cache DDTest's discovery file
+between CI runs.
+The restored cache path can be passed with either the CLI flag or environment
+variable:
+
+```bash
+ddtest plan --test-discovery-cache .ddtest-cache/tests-discovery.json
+
+DD_TEST_OPTIMIZATION_RUNNER_TEST_DISCOVERY_CACHE=.ddtest-cache/tests-discovery.json ddtest plan
+```
+
+DDTest imports the restored file before planning, validates it, and skips full
+discovery when the cache is still safe to use. After planning, save the refreshed
+internal cache file for the next run:
+
+```bash
+if [ -f .testoptimization/tests-discovery/tests.json ]; then
+  mkdir -p .ddtest-cache
+  cp .testoptimization/tests-discovery/tests.json .ddtest-cache/tests-discovery.json
+fi
+```
+
+For GitHub Actions, the flow can look like this:
+
+```yaml
+- uses: actions/cache@v4
+  with:
+    path: .ddtest-cache/tests-discovery.json
+    key: ddtest-discovery-${{ github.ref_name }}
+    restore-keys: |
+      ddtest-discovery-
+
+- name: Plan tests
+  env:
+    DD_TEST_OPTIMIZATION_RUNNER_TEST_DISCOVERY_CACHE: .ddtest-cache/tests-discovery.json
+  run: ddtest plan
+
+- name: Save latest discovery cache
+  if: always()
+  run: |
+    if [ -f .testoptimization/tests-discovery/tests.json ]; then
+      mkdir -p .ddtest-cache
+      cp .testoptimization/tests-discovery/tests.json .ddtest-cache/tests-discovery.json
+    fi
+```
+
+DDTest ignores the cache and runs full discovery when the file is missing,
+corrupt, produced for a different platform/framework/test location/exclude
+pattern, or based on a commit that is not available locally. It also invalidates
+the cache when files under the current project's test root changed. For example,
+the default RSpec root is `spec/**` and the default Minitest root is `test/**`;
+with `--tests-location custom/spec/**/*_spec.rb`, the root is `custom/**`.
+
+In monorepos, run DDTest from the project subdirectory whose tests you are
+planning. Cache invalidation is scoped to that project's effective test root, so
+changes in sibling projects do not invalidate its discovery cache.
+
 ## Minitest Support In Non-Rails Projects
 
 We use `bundle exec rake test` command when we don't detect `rails` command to

docs/settings.md

@@ -16,5 +16,6 @@ CLI flags take precedence over environment variables.
 | `--worker-env` | `DD_TEST_OPTIMIZATION_RUNNER_WORKER_ENV` | | `""` | Template env vars per worker: `--worker-env "DATABASE_NAME_TEST=app_test{{nodeIndex}}_{{workerIndex}}"`. `{{nodeIndex}}` is the CI node index (`0` for single-node runs); `{{workerIndex}}` is the worker process index within that CI node. |
 | `--tests-location` | `DD_TEST_OPTIMIZATION_RUNNER_TESTS_LOCATION` | `KNAPSACK_PRO_TEST_FILE_PATTERN` | `""` | Custom glob pattern to discover test files, such as `--tests-location "custom/spec/**/*_spec.rb"`. Defaults to `spec/**/*_spec.rb` for RSpec, `test/**/*_test.rb` for Minitest. |
 | `--tests-exclude-pattern` | `DD_TEST_OPTIMIZATION_RUNNER_TESTS_EXCLUDE_PATTERN` | `KNAPSACK_PRO_TEST_FILE_EXCLUDE_PATTERN` | `""` | Glob pattern to exclude test files from discovery, such as `--tests-exclude-pattern "spec/system/**/*_spec.rb"`. |
+| `--test-discovery-cache` | `DD_TEST_OPTIMIZATION_RUNNER_TEST_DISCOVERY_CACHE` | | `""` | Path to a restored test discovery cache file. DDTest imports it before planning and refreshes the internal discovery cache after successful full discovery. |
 | `--runtime-tags` | `DD_TEST_OPTIMIZATION_RUNNER_RUNTIME_TAGS` | | `""` | JSON string to override runtime tags used to fetch skippable tests. Useful for local development on a different OS than CI, such as `--runtime-tags '{"os.platform":"linux","runtime.version":"3.2.0"}'`. |
 | | `DD_TEST_OPTIMIZATION_RUNNER_REPORT_ENABLED` | | `true` | Print human-readable plan and run reports. Set to `false` to disable them. |

internal/cmd/cmd.go

@@ -74,6 +74,7 @@ func init() {
 	rootCmd.PersistentFlags().String("command", "", "Test command that ddtest should wrap")
 	rootCmd.PersistentFlags().String("tests-location", "", "Glob pattern used to discover test files")
 	rootCmd.PersistentFlags().String("tests-exclude-pattern", "", "Glob pattern used to exclude test files from discovery")
+	rootCmd.PersistentFlags().String("test-discovery-cache", "", "Path to a restored test discovery cache file to import before planning")
 	rootCmd.PersistentFlags().String("runtime-tags", "", "JSON string to override runtime tags (e.g. '{\"os.platform\":\"linux\",\"runtime.version\":\"3.2.0\"}')")
 	if err := viper.BindPFlag("platform", rootCmd.PersistentFlags().Lookup("platform")); err != nil {
 		fmt.Fprintf(os.Stderr, "Error binding platform flag: %v\n", err)
@@ -119,6 +120,10 @@ func init() {
 		fmt.Fprintf(os.Stderr, "Error binding tests-exclude-pattern flag: %v\n", err)
 		os.Exit(1)
 	}
+	if err := viper.BindPFlag("test_discovery_cache", rootCmd.PersistentFlags().Lookup("test-discovery-cache")); err != nil {
+		fmt.Fprintf(os.Stderr, "Error binding test-discovery-cache flag: %v\n", err)
+		os.Exit(1)
+	}
 	if err := viper.BindPFlag("runtime_tags", rootCmd.PersistentFlags().Lookup("runtime-tags")); err != nil {
 		fmt.Fprintf(os.Stderr, "Error binding runtime-tags flag: %v\n", err)
 		os.Exit(1)

internal/cmd/cmd_test.go

@@ -48,6 +48,12 @@ func TestRootCommandFlags(t *testing.T) {
 		return
 	}
 
+	testDiscoveryCacheFlag := rootCmd.PersistentFlags().Lookup("test-discovery-cache")
+	if testDiscoveryCacheFlag == nil {
+		t.Error("test-discovery-cache flag should be defined")
+		return
+	}
+
 	ciNodeWorkersFlag := rootCmd.PersistentFlags().Lookup("ci-node-workers")
 	if ciNodeWorkersFlag == nil {
 		t.Error("ci-node-workers flag should be defined")
@@ -87,6 +93,10 @@ func TestRootCommandFlags(t *testing.T) {
 		t.Errorf("expected tests-exclude-pattern default to be empty, got %q", testsExcludePatternFlag.DefValue)
 	}
 
+	if testDiscoveryCacheFlag.DefValue != "" {
+		t.Errorf("expected test-discovery-cache default to be empty, got %q", testDiscoveryCacheFlag.DefValue)
+	}
+
 	if ciNodeWorkersFlag.DefValue != "1" {
 		t.Errorf("expected ci-node-workers default to be '1', got %q", ciNodeWorkersFlag.DefValue)
 	}
@@ -203,6 +213,9 @@ func TestFlagBinding(t *testing.T) {
 	if err := viper.BindPFlag("tests_exclude_pattern", rootCmd.PersistentFlags().Lookup("tests-exclude-pattern")); err != nil {
 		t.Fatalf("Error binding tests-exclude-pattern flag: %v", err)
 	}
+	if err := viper.BindPFlag("test_discovery_cache", rootCmd.PersistentFlags().Lookup("test-discovery-cache")); err != nil {
+		t.Fatalf("Error binding test-discovery-cache flag: %v", err)
+	}
 	if err := viper.BindPFlag("ci_node_workers", rootCmd.PersistentFlags().Lookup("ci-node-workers")); err != nil {
 		t.Fatalf("Error binding ci-node-workers flag: %v", err)
 	}
@@ -229,6 +242,9 @@ func TestFlagBinding(t *testing.T) {
 	if err := rootCmd.PersistentFlags().Set("tests-exclude-pattern", "spec/system/**/*_spec.rb"); err != nil {
 		t.Fatalf("Error setting tests-exclude-pattern flag: %v", err)
 	}
+	if err := rootCmd.PersistentFlags().Set("test-discovery-cache", "/tmp/ddtest-tests.json"); err != nil {
+		t.Fatalf("Error setting test-discovery-cache flag: %v", err)
+	}
 	if err := rootCmd.PersistentFlags().Set("ci-node-workers", "ncpu"); err != nil {
 		t.Fatalf("Error setting ci-node-workers flag: %v", err)
 	}
@@ -255,6 +271,9 @@ func TestFlagBinding(t *testing.T) {
 	if viper.GetString("tests_exclude_pattern") != "spec/system/**/*_spec.rb" {
 		t.Errorf("expected viper tests_exclude_pattern to be 'spec/system/**/*_spec.rb', got %q", viper.GetString("tests_exclude_pattern"))
 	}
+	if viper.GetString("test_discovery_cache") != "/tmp/ddtest-tests.json" {
+		t.Errorf("expected viper test_discovery_cache to be '/tmp/ddtest-tests.json', got %q", viper.GetString("test_discovery_cache"))
+	}
 	if viper.GetString("ci_node_workers") != "ncpu" {
 		t.Errorf("expected viper ci_node_workers to be 'ncpu', got %q", viper.GetString("ci_node_workers"))
 	}

internal/discovery/discovery.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"io"
 	"io/fs"
 	"log/slog"
 	"maps"
@@ -183,8 +184,9 @@ func DiscoverTests(
 		return nil, err
 	}
 
-	tests, err := ParseTests()
+	tests, err := parseTestsFile(TestsFilePath)
 	if err != nil {
+		slog.Error("Error parsing JSON", "error", err)
 		return nil, err
 	}
 
@@ -212,28 +214,27 @@ func executeCommand(ctx context.Context, executor ext.CommandExecutor, executabl
 	return nil
 }
 
-func ParseTests() ([]testoptimization.Test, error) {
-	file, err := os.Open(TestsFilePath)
+func parseTestsFile(filePath string) ([]testoptimization.Test, error) {
+	file, err := os.Open(filePath)
 	if err != nil {
-		slog.Error("Error opening JSON file", "error", err)
 		return nil, err
 	}
 	defer func() {
 		_ = file.Close()
 	}()
 
-	var tests []testoptimization.Test
 	decoder := json.NewDecoder(file)
-	for decoder.More() {
+	tests := make([]testoptimization.Test, 0)
+	for {
 		var test testoptimization.Test
 		if err := decoder.Decode(&test); err != nil {
-			slog.Error("Error parsing JSON", "error", err)
+			if err == io.EOF {
+				return tests, nil
+			}
 			return nil, err
 		}
 		tests = append(tests, test)
 	}
-
-	return tests, nil
 }
 
 // BaseEnv returns environment variables required for all test discovery processes.

internal/discovery/parse_test.go

@@ -0,0 +1,102 @@
+package discovery
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+	"path/filepath"
+	"runtime"
+	"testing"
+)
+
+func TestParseTestsFile_MalformedJSON(t *testing.T) {
+	filePath := filepath.Join(t.TempDir(), "tests.json")
+	if err := os.WriteFile(filePath, []byte("{invalid json}\n"), 0644); err != nil {
+		t.Fatalf("failed to write discovery file: %v", err)
+	}
+
+	if _, err := parseTestsFile(filePath); err == nil {
+		t.Fatal("expected malformed JSON to fail")
+	}
+}
+
+func BenchmarkParseTestsFile200000(b *testing.B) {
+	const testCount = 200_000
+
+	filePath := filepath.Join(b.TempDir(), "tests.json")
+	fileSize := writeBenchmarkDiscoveryFile(b, filePath, testCount)
+
+	b.Run("parse", func(b *testing.B) {
+		b.ReportAllocs()
+		b.SetBytes(fileSize)
+
+		var retainedBytes uint64
+		for i := 0; i < b.N; i++ {
+			before := heapAllocAfterGC(b)
+
+			b.StartTimer()
+			tests, err := parseTestsFile(filePath)
+			b.StopTimer()
+			if err != nil {
+				b.Fatalf("parseTestsFile() failed: %v", err)
+			}
+			if len(tests) != testCount {
+				b.Fatalf("parsed %d tests, want %d", len(tests), testCount)
+			}
+
+			after := heapAllocAfterGC(b)
+			retainedBytes += heapDelta(before, after)
+			runtime.KeepAlive(tests)
+		}
+		b.ReportMetric(float64(retainedBytes)/float64(b.N)/(1024*1024), "MiB_retained/op")
+	})
+}
+
+func writeBenchmarkDiscoveryFile(b *testing.B, filePath string, testCount int) int64 {
+	b.Helper()
+
+	file, err := os.Create(filePath)
+	if err != nil {
+		b.Fatalf("failed to create benchmark discovery file: %v", err)
+	}
+	writer := bufio.NewWriterSize(file, 1024*1024)
+	for i := 0; i < testCount; i++ {
+		if _, err := fmt.Fprintf(
+			writer,
+			`{"module":"rspec","suite":"Suite%06d","name":"test_%06d","parameters":"{\"scoped_id\":\"%06d\"}","suiteSourceFile":"spec/models/model_%06d_spec.rb"}`+"\n",
+			i%20_000,
+			i,
+			i,
+			i%20_000,
+		); err != nil {
+			b.Fatalf("failed to write benchmark discovery record: %v", err)
+		}
+	}
+	if err := writer.Flush(); err != nil {
+		b.Fatalf("failed to flush benchmark discovery file: %v", err)
+	}
+	info, err := file.Stat()
+	if err != nil {
+		b.Fatalf("failed to stat benchmark discovery file: %v", err)
+	}
+	if err := file.Close(); err != nil {
+		b.Fatalf("failed to close benchmark discovery file: %v", err)
+	}
+	return info.Size()
+}
+
+func heapAllocAfterGC(b *testing.B) uint64 {
+	b.Helper()
+	b.StopTimer()
+	runtime.GC()
+	var stats runtime.MemStats
+	runtime.ReadMemStats(&stats)
+	return stats.HeapAlloc
+}
+
+func heapDelta(before, after uint64) uint64 {
+	if after <= before {
+		return 0
+	}
+	return after - before
+}

internal/framework/minitest_test.go

@@ -222,7 +222,7 @@ func TestMinitest_DiscoverTests_Success(t *testing.T) {
 
 	minitest := newTestMinitestWithExecutor(mockExecutor)
 
-	tests, err := minitest.DiscoverTests(context.Background(), resolveTestFilesForFramework(t, minitest.TestPattern()))
+	tests, err := discoverAndParseTests(t, minitest, resolveTestFilesForFramework(t, minitest.TestPattern()))
 	if err != nil {
 		t.Fatalf("DiscoverTests failed: %v", err)
 	}
@@ -278,7 +278,7 @@ func TestMinitest_DiscoverTests_CommandFailure(t *testing.T) {
 
 	minitest := newTestMinitestWithExecutor(mockExecutor)
 
-	tests, err := minitest.DiscoverTests(context.Background(), resolveTestFilesForFramework(t, minitest.TestPattern()))
+	tests, err := discoverAndParseTests(t, minitest, resolveTestFilesForFramework(t, minitest.TestPattern()))
 	if err == nil {
 		t.Error("expected error when command fails")
 	}
@@ -315,7 +315,7 @@ func TestMinitest_DiscoverTests_InvalidJSON(t *testing.T) {
 
 	minitest := newTestMinitestWithExecutor(mockExecutor)
 
-	tests, err := minitest.DiscoverTests(context.Background(), resolveTestFilesForFramework(t, minitest.TestPattern()))
+	tests, err := discoverAndParseTests(t, minitest, resolveTestFilesForFramework(t, minitest.TestPattern()))
 	if err == nil {
 		t.Error("expected error when JSON is invalid")
 	}
@@ -867,7 +867,7 @@ func TestMinitest_DiscoverTests_WithTestsExcludePattern_NonRails(t *testing.T) {
 	}
 
 	minitest := newTestMinitestWithExecutor(mockExecutor)
-	tests, err := minitest.DiscoverTests(context.Background(), resolveTestFilesForFramework(t, minitest.TestPattern()))
+	tests, err := discoverAndParseTests(t, minitest, resolveTestFilesForFramework(t, minitest.TestPattern()))
 	if err != nil {
 		t.Fatalf("DiscoverTests failed: %v", err)
 	}
@@ -952,7 +952,7 @@ func TestMinitest_DiscoverTests_WithTestsExcludePattern_Rails(t *testing.T) {
 	}
 
 	minitest := newTestMinitestWithExecutor(mockExecutor)
-	tests, err := minitest.DiscoverTests(context.Background(), resolveTestFilesForFramework(t, minitest.TestPattern()))
+	tests, err := discoverAndParseTests(t, minitest, resolveTestFilesForFramework(t, minitest.TestPattern()))
 	if err != nil {
 		t.Fatalf("DiscoverTests failed: %v", err)
 	}
@@ -1003,7 +1003,7 @@ func TestMinitest_DiscoverTests_WithTestsExcludePattern_AllExcluded(t *testing.T
 
 	mockExecutor := &countingCommandExecutor{}
 	minitest := newTestMinitestWithExecutor(mockExecutor)
-	tests, err := minitest.DiscoverTests(context.Background(), resolveTestFilesForFramework(t, minitest.TestPattern()))
+	tests, err := discoverAndParseTests(t, minitest, resolveTestFilesForFramework(t, minitest.TestPattern()))
 	if err != nil {
 		t.Fatalf("DiscoverTests failed: %v", err)
 	}
@@ -1090,7 +1090,7 @@ func TestMinitest_DiscoverTests_WithTestsLocation(t *testing.T) {
 	}
 
 	minitest := newTestMinitestWithExecutor(mockExecutor)
-	tests, err := minitest.DiscoverTests(context.Background(), resolveTestFilesForFramework(t, minitest.TestPattern()))
+	tests, err := discoverAndParseTests(t, minitest, resolveTestFilesForFramework(t, minitest.TestPattern()))
 	if err != nil {
 		t.Fatalf("DiscoverTests failed: %v", err)
 	}
@@ -1189,7 +1189,7 @@ func TestMinitest_DiscoverTests_WithTestsLocation_Rails(t *testing.T) {
 	}
 
 	minitest := newTestMinitestWithExecutor(mockExecutor)
-	tests, err := minitest.DiscoverTests(context.Background(), resolveTestFilesForFramework(t, minitest.TestPattern()))
+	tests, err := discoverAndParseTests(t, minitest, resolveTestFilesForFramework(t, minitest.TestPattern()))
 	if err != nil {
 		t.Fatalf("DiscoverTests failed: %v", err)
 	}
@@ -1257,7 +1257,7 @@ func TestMinitest_DiscoverTests_WithTestsLocation_NoMatches(t *testing.T) {
 	}
 
 	minitest := newTestMinitestWithExecutor(mockExecutor)
-	tests, err := minitest.DiscoverTests(context.Background(), resolveTestFilesForFramework(t, minitest.TestPattern()))
+	tests, err := discoverAndParseTests(t, minitest, resolveTestFilesForFramework(t, minitest.TestPattern()))
 	if err != nil {
 		t.Fatalf("DiscoverTests should not fail when no matches: %v", err)
 	}
@@ -1649,7 +1649,7 @@ func TestMinitest_DiscoverTests_UsesPlatformEnv(t *testing.T) {
 	}
 	minitest.SetPlatformEnv(platformEnv)
 
-	_, err := minitest.DiscoverTests(context.Background(), resolveTestFilesForFramework(t, minitest.TestPattern()))
+	_, err := discoverAndParseTests(t, minitest, resolveTestFilesForFramework(t, minitest.TestPattern()))
 	if err != nil {
 		t.Fatalf("DiscoverTests failed: %v", err)
 	}