ci: add Codecov coverage + Trunk flaky-test uploads (#2554)

ValentaTomas · web-flow · commit b453509032d6 · 2026-05-04T04:19:10.000Z
## What

`gotestsum` wraps `go test` so coverage and JUnit XML come from one
invocation. Each test job uploads to:

- **Codecov Coverage** — `coverage.txt` per package × arch.
- **Codecov Test Analytics** — `junit.xml` per package × arch
(failed-test reports in PR comments, dashboard).
- **Trunk Flaky Tests** — same `junit.xml` (per-test history, flake
detection, quarantine).

Coverage is informational; nothing blocks merges. Trunk upload is
`continue-on-error`.

## Required before this delivers value

Add to repo secrets:

- `CODECOV_TOKEN`
- `TRUNK_API_TOKEN`

Without them the workflow runs as before — every upload step skips
itself.

## Out of scope

Coverage on integration tests (would need a `tests/integration/Makefile`
change). Codecov Test Analytics and Trunk both already get the
integration JUnit.
diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml
@@ -7,13 +7,18 @@ on:
         type: boolean
         description: "Whether to publish the results"
         required: true
+    secrets:
+      CODECOV_TOKEN: { required: false }
 jobs:
   integration_tests:
     runs-on: infra-tests
     timeout-minutes: 30
     permissions:
       contents: read
       id-token: write
+    env:
+      # Surfaced as env so upload steps can gate on presence (skipped on fork PRs).
+      CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
 
     steps:
       - name: Checkout Code
@@ -78,6 +83,15 @@ jobs:
           name: Integration Tests Results
           path: ./tests/integration/test-results.xml
 
+      - name: Upload test results to Codecov
+        if: ${{ !cancelled() && env.CODECOV_TOKEN != '' && hashFiles('tests/integration/test-results.xml') != '' }}
+        uses: codecov/test-results-action@v1
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: tests/integration/test-results.xml
+          flags: integration
+          disable_search: true
+
       - name: Upload Service Logs
         if: ${{ always() && inputs.publish == true }}
         uses: actions/upload-artifact@v6
diff --git a/.github/workflows/pr-tests-arm64.yml b/.github/workflows/pr-tests-arm64.yml
@@ -1,6 +1,9 @@
 name: ARM64 tests on PRs
 
-on: [workflow_call]
+on:
+  workflow_call:
+    secrets:
+      CODECOV_TOKEN: { required: false }
 
 permissions:
   contents: read
@@ -47,28 +50,37 @@ jobs:
       # and race on `reaper_<hash>` creation. Disable Ryuk; t.Cleanup
       # handles normal exit, the prune step below handles abnormal exit.
       TESTCONTAINERS_RYUK_DISABLED: "true"
+      # Surfaced as env so upload steps can gate on presence (skipped on fork PRs).
+      CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
     strategy:
       matrix:
         include:
           - package: packages/api
+            flag: arm64-api
             test_path: ./...
             sudo: false
           - package: packages/client-proxy
+            flag: arm64-client-proxy
             test_path: ./...
             sudo: false
           - package: packages/db
+            flag: arm64-db
             test_path: ./...
             sudo: false
           - package: packages/docker-reverse-proxy
+            flag: arm64-docker-reverse-proxy
             test_path: ./...
             sudo: false
           - package: packages/envd
+            flag: arm64-envd
             test_path: ./...
             sudo: true
           - package: packages/orchestrator
+            flag: arm64-orchestrator
             test_path: ./...
             sudo: true
           - package: packages/shared
+            flag: arm64-shared
             test_path: ./pkg/...
             sudo: false
       fail-fast: false
@@ -133,14 +145,23 @@ jobs:
           sudo udevadm trigger
         if: matrix.package == 'packages/orchestrator'
 
+      - name: Install gotestsum
+        run: go install gotest.tools/gotestsum@v1.13.0
+
       - name: Compile test binaries
         working-directory: ${{ matrix.package }}
         run: sudo -E `which go` test -run '^$' ${{ matrix.test_path }}
         if: matrix.sudo == true
 
+      # sudo strips PATH; pass it through so gotestsum/go resolve to the runner's
+      # toolchain (1.25.9) instead of /usr/bin/go. Trap chowns reports back to
+      # the runner so uploads can read them even when tests fail.
       - name: Run tests that require sudo
         working-directory: ${{ matrix.package }}
-        run: sudo -E `which go` test -v -timeout 20m ${{ matrix.test_path }}
+        run: |
+          trap 'sudo chown "$(id -u):$(id -g)" coverage.txt junit.xml 2>/dev/null || true' EXIT
+          sudo -E env "PATH=$PATH" gotestsum --junitfile=junit.xml --format=standard-verbose \
+            -- -timeout 20m -coverprofile=coverage.txt -covermode=atomic ${{ matrix.test_path }}
         if: matrix.sudo == true
 
       - name: Compile test binaries
@@ -150,5 +171,25 @@ jobs:
 
       - name: Run tests
         working-directory: ${{ matrix.package }}
-        run: go test -v -timeout 20m ${{ matrix.test_path }}
+        run: |
+          gotestsum --junitfile=junit.xml --format=standard-verbose \
+            -- -timeout 20m -coverprofile=coverage.txt -covermode=atomic ${{ matrix.test_path }}
         if: matrix.sudo == false
+
+      - name: Upload coverage to Codecov
+        if: ${{ !cancelled() && env.CODECOV_TOKEN != '' && hashFiles(format('{0}/coverage.txt', matrix.package)) != '' }}
+        uses: codecov/codecov-action@v5
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: ${{ matrix.package }}/coverage.txt
+          flags: ${{ matrix.flag }}
+          disable_search: true
+
+      - name: Upload test results to Codecov
+        if: ${{ !cancelled() && env.CODECOV_TOKEN != '' && hashFiles(format('{0}/junit.xml', matrix.package)) != '' }}
+        uses: codecov/test-results-action@v1
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: ${{ matrix.package }}/junit.xml
+          flags: ${{ matrix.flag }}
+          disable_search: true
diff --git a/.github/workflows/pr-tests.yml b/.github/workflows/pr-tests.yml
@@ -1,6 +1,9 @@
 name: Run tests on PRs
 
-on: [ workflow_call ]
+on:
+  workflow_call:
+    secrets:
+      CODECOV_TOKEN: { required: false }
 
 permissions:
   contents: read
@@ -15,28 +18,37 @@ jobs:
       # and race on `reaper_<hash>` creation. Disable Ryuk; t.Cleanup
       # handles normal exit, the prune step below handles abnormal exit.
       TESTCONTAINERS_RYUK_DISABLED: "true"
+      # Surfaced as env so upload steps can gate on presence (skipped on fork PRs).
+      CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
     strategy:
       matrix:
         include:
           - package: packages/api
+            flag: unit-api
             test_path: ./...
             sudo: false
           - package: packages/client-proxy
+            flag: unit-client-proxy
             test_path: ./...
             sudo: false
           - package: packages/db
+            flag: unit-db
             test_path: ./...
             sudo: false
           - package: packages/docker-reverse-proxy
+            flag: unit-docker-reverse-proxy
             test_path: ./...
             sudo: false
           - package: packages/envd
+            flag: unit-envd
             test_path: ./...
             sudo: true
           - package: packages/orchestrator
+            flag: unit-orchestrator
             test_path: ./...
             sudo: true
           - package: packages/shared
+            flag: unit-shared
             test_path: ./pkg/...
             sudo: false
       fail-fast: false
@@ -100,18 +112,47 @@ jobs:
 
         if: matrix.package == 'packages/orchestrator'
 
+      - name: Install gotestsum
+        run: go install gotest.tools/gotestsum@v1.13.0
+
+      # sudo strips PATH; pass it through so gotestsum/go resolve to the runner's
+      # toolchain (1.25.9) instead of /usr/bin/go. Trap chowns reports back to
+      # the runner so uploads can read them even when tests fail.
       - name: Run tests that require sudo
         working-directory: ${{ matrix.package }}
         run: |
-          # Run tests. The '-E' flag is required to allow root to use the correct cache path.
-          sudo -E `which go` test -v -timeout 20m ${{ matrix.test_path }}
+          trap 'sudo chown "$(id -u):$(id -g)" coverage.txt junit.xml 2>/dev/null || true' EXIT
+          sudo -E env "PATH=$PATH" gotestsum --junitfile=junit.xml --format=standard-verbose \
+            -- -timeout 20m -coverprofile=coverage.txt -covermode=atomic ${{ matrix.test_path }}
         if: matrix.sudo == true
 
       - name: Run tests
         working-directory: ${{ matrix.package }}
-        run: go test -v -timeout 20m ${{ matrix.test_path }}
+        run: |
+          gotestsum --junitfile=junit.xml --format=standard-verbose \
+            -- -timeout 20m -coverprofile=coverage.txt -covermode=atomic ${{ matrix.test_path }}
         if: matrix.sudo == false
 
+      - name: Upload coverage to Codecov
+        if: ${{ !cancelled() && env.CODECOV_TOKEN != '' &&
+          hashFiles(format('{0}/coverage.txt', matrix.package)) != '' }}
+        uses: codecov/codecov-action@v5
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: ${{ matrix.package }}/coverage.txt
+          flags: ${{ matrix.flag }}
+          disable_search: true
+
+      - name: Upload test results to Codecov
+        if: ${{ !cancelled() && env.CODECOV_TOKEN != '' &&
+          hashFiles(format('{0}/junit.xml', matrix.package)) != '' }}
+        uses: codecov/test-results-action@v1
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: ${{ matrix.package }}/junit.xml
+          flags: ${{ matrix.flag }}
+          disable_search: true
+
   validate-iac:
     name: Validate terraform
     runs-on: ubuntu-24.04
diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml
@@ -28,14 +28,20 @@ jobs:
     uses: ./.github/workflows/out-of-order-migrations.yml
   unit-tests:
     uses: ./.github/workflows/pr-tests.yml
+    secrets:
+      CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
   arm64-tests:
     uses: ./.github/workflows/pr-tests-arm64.yml
+    secrets:
+      CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
   integration-tests:
     needs: [out-of-order-migrations]
     uses: ./.github/workflows/integration_tests.yml
     with:
       # Only publish the results for same-repo PRs
       publish: ${{ github.event.pull_request.head.repo.full_name == github.repository }}
+    secrets:
+      CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
   publish-test-results:
     needs:
       - unit-tests
diff --git a/.github/workflows/push-main.yml b/.github/workflows/push-main.yml
@@ -17,10 +17,14 @@ jobs:
 
   unit-tests:
     uses: ./.github/workflows/pr-tests.yml
+    secrets:
+      CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
   integration-tests:
     uses: ./.github/workflows/integration_tests.yml
     with:
       publish: true
+    secrets:
+      CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
   publish-test-results:
     needs:
       - unit-tests
diff --git a/codecov.yml b/codecov.yml
@@ -0,0 +1,30 @@
+# Coverage signal only — never gates merges. Carries forward per-flag so
+# shards that didn't run keep their last-known number instead of dropping to 0%.
+
+coverage:
+  status:
+    project:
+      default: { informational: true }
+    patch:
+      default: { informational: true }
+
+flag_management:
+  default_rules:
+    carryforward: true
+
+component_management:
+  individual_components:
+    - { component_id: api, paths: ["packages/api/**"] }
+    - { component_id: client-proxy, paths: ["packages/client-proxy/**"] }
+    - { component_id: db, paths: ["packages/db/**"] }
+    - { component_id: docker-reverse-proxy, paths: ["packages/docker-reverse-proxy/**"] }
+    - { component_id: envd, paths: ["packages/envd/**"] }
+    - { component_id: orchestrator, paths: ["packages/orchestrator/**"] }
+    - { component_id: shared, paths: ["packages/shared/**"] }
+
+ignore:
+  - "**/*.gen.go"   # oapi-codegen
+  - "**/*.sql.go"   # sqlc
+  - "**/*.pb.go"    # protoc (message + grpc)
+  - "**/mocks/**"   # mockery
+  - "**/testdata/**"
