Integration Tests #5838
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Integration Tests | |
| on: | |
| schedule: | |
| # GitHub cron is UTC. Run hourly; plan-run enforces 10:00-18:00 America/New_York. | |
| - cron: "0 14-23 * * 1-5" | |
| workflow_dispatch: | |
| permissions: | |
| contents: read | |
| concurrency: | |
| group: integration-tests-${{ github.ref }} | |
| cancel-in-progress: false | |
| jobs: | |
| plan-run: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| outputs: | |
| should_run: ${{ steps.decision.outputs.should_run }} | |
| cache_hit: ${{ steps.decision.outputs.cache_hit }} | |
| steps: | |
| - name: Check trigger policy | |
| id: trigger-policy | |
| env: | |
| EVENT_NAME: ${{ github.event_name }} | |
| ACTOR: ${{ github.actor }} | |
| TRIGGERING_ACTOR: ${{ github.triggering_actor || github.actor }} | |
| TRUSTED_CI_ACTORS_JSON: ${{ vars.TRUSTED_CI_ACTORS_JSON || '[]' }} | |
| run: | | |
| set -euo pipefail | |
| echo "candidate=false" >> "$GITHUB_OUTPUT" | |
| if [ "$EVENT_NAME" = "workflow_dispatch" ]; then | |
| if [ "$ACTOR" != "github-actions[bot]" ] && | |
| ! jq -e --arg actor "$TRIGGERING_ACTOR" 'index($actor)' <<< "$TRUSTED_CI_ACTORS_JSON" >/dev/null; then | |
| echo "Manual integration run is not trusted for actor: $TRIGGERING_ACTOR" >&2 | |
| exit 1 | |
| fi | |
| echo "candidate=true" >> "$GITHUB_OUTPUT" | |
| exit 0 | |
| fi | |
| local_dow=$(TZ=America/New_York date +%u) | |
| local_hhmm=$(TZ=America/New_York date +%H%M) | |
| if [ "$local_dow" -gt 5 ] || [[ "$local_hhmm" < "1000" || "$local_hhmm" > "1800" ]]; then | |
| echo "Outside integration window in America/New_York: weekday=$local_dow time=$local_hhmm" | |
| exit 0 | |
| fi | |
| echo "candidate=true" >> "$GITHUB_OUTPUT" | |
| - name: Check prior successful integration for this commit | |
| id: integration-success | |
| if: steps.trigger-policy.outputs.candidate == 'true' | |
| uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 | |
| with: | |
| path: .integration-success | |
| key: integration-tests-success-v1-${{ github.sha }} | |
| lookup-only: true | |
| - name: Decide whether integration should run | |
| id: decision | |
| env: | |
| EVENT_NAME: ${{ github.event_name }} | |
| CANDIDATE: ${{ steps.trigger-policy.outputs.candidate }} | |
| CACHE_HIT: ${{ steps.integration-success.outputs.cache-hit || 'false' }} | |
| run: | | |
| set -euo pipefail | |
| echo "cache_hit=$CACHE_HIT" >> "$GITHUB_OUTPUT" | |
| if [ "$CANDIDATE" != "true" ]; then | |
| echo "should_run=false" >> "$GITHUB_OUTPUT" | |
| exit 0 | |
| fi | |
| if [ "$EVENT_NAME" = "schedule" ] && [ "$CACHE_HIT" = "true" ]; then | |
| echo "Integration already passed for this commit; skipping scheduled run." | |
| echo "should_run=false" >> "$GITHUB_OUTPUT" | |
| exit 0 | |
| fi | |
| echo "should_run=true" >> "$GITHUB_OUTPUT" | |
| # ────────────────────────────────────────────────────────────────── | |
| # 1. LIGHTWEIGHT SSH / INSTALL SMOKE-TEST | |
| # ────────────────────────────────────────────────────────────────── | |
| ssh-sanity: | |
| needs: plan-run | |
| if: needs.plan-run.outputs.should_run == 'true' | |
| environment: internal-registry-ci | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 # short fail-fast window | |
| steps: | |
| - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 | |
| # Load the private deploy key into an ssh-agent. | |
| # The secret value MUST be the *raw* PEM text, not base64. | |
| - uses: webfactory/ssh-agent@d4b9b8ff72958532804b70bbe600ad43b36d5f2e # v0.8.0 | |
| with: | |
| ssh-private-key: ${{ secrets.CUSTOM_REPO_SSH_PRIVATE_KEY }} | |
| - name: Install uv | |
| uses: useblacksmith/setup-uv@f4588471335f14dcb60198856207b916701a9e9f # v4 | |
| with: | |
| version: "0.9.7" | |
| - name: Set up Python 3.12 | |
| uses: useblacksmith/setup-python@943b05a7c4ca70c2360391137527a37bee33fb1d # v6 | |
| with: | |
| python-version: "3.12" | |
| # Lightweight connectivity check — choose ONE: | |
| # | |
| # A) Strict git connectivity (fastest, no deps): | |
| - name: Verify git over SSH | |
| run: git ls-remote git@github.com:TracecatHQ/internal-registry.git HEAD | |
| # | |
| # B) End-to-end install smoke-test (still <10 s): | |
| #- name: Smoke-test pip install via SSH | |
| # run: | | |
| # uv pip install --no-deps --quiet \ | |
| # "git+ssh://git@github.com/TracecatHQ/internal-registry.git@main" | |
| # python -c "import importlib, sys; print('import ok:', bool(importlib.import_module('internal_registry', package=None))); sys.exit(0)" | |
| # ────────────────────────────────────────────────────────────────── | |
| # 2. FULL INTEGRATION TESTS (with pytest-xdist for parallelization) | |
| # Uses worker-specific task queues from conftest.py for isolation | |
| # ────────────────────────────────────────────────────────────────── | |
| test-integration: | |
| needs: [plan-run, ssh-sanity] # only start if key check passed | |
| if: needs.plan-run.outputs.should_run == 'true' && needs.ssh-sanity.result == 'success' | |
| environment: internal-registry-ci | |
| runs-on: blacksmith-8vcpu-ubuntu-2204 | |
| timeout-minutes: 30 | |
| steps: | |
| - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 | |
| - name: Install uv | |
| uses: useblacksmith/setup-uv@f4588471335f14dcb60198856207b916701a9e9f # v4 | |
| with: | |
| version: "0.9.7" | |
| enable-cache: true | |
| cache-dependency-glob: | | |
| pyproject.toml | |
| uv.lock | |
| - name: Set up Python 3.12 | |
| uses: useblacksmith/setup-python@943b05a7c4ca70c2360391137527a37bee33fb1d # v6 | |
| with: | |
| python-version: "3.12" | |
| - name: Disable IPv6 (Blacksmith VMs lack IPv6 routing) | |
| run: | | |
| sudo sysctl -w net.ipv6.conf.all.disable_ipv6=1 | |
| sudo sysctl -w net.ipv6.conf.default.disable_ipv6=1 | |
| - name: Set up Docker builder | |
| uses: useblacksmith/setup-docker-builder@78f41686563c732ccc097f7baac2f092f67538f0 # v1 | |
| - name: Run environment setup script | |
| run: | | |
| echo "y | |
| localhost | |
| n | |
| test@tracecat.com" | bash env.sh | |
| - name: Build CI image | |
| uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 | |
| with: | |
| context: . | |
| target: production | |
| load: true | |
| tags: tracecat-ci:integration-${{ github.run_id }}-${{ github.run_attempt }} | |
| - name: Build PostgreSQL candidate image | |
| run: bash scripts/postgres/build-image.sh | |
| - name: Start core Docker services | |
| env: | |
| TRACECAT__CI_IMAGE_TAG: integration-${{ github.run_id }}-${{ github.run_attempt }} | |
| TRACECAT__UNSAFE_DISABLE_SM_MASKING: "true" | |
| TRACECAT__EXECUTOR_BACKEND: "direct" | |
| TRACECAT__FEATURE_FLAGS: "" | |
| # Keep executor-hosted registry sync enabled (run via Temporal) | |
| TRACECAT__REGISTRY_SYNC_SANDBOX_ENABLED: "true" | |
| run: | | |
| set -euo pipefail | |
| compose_files=(-f docker-compose.local.yml -f .github/docker-compose.integration.yml) | |
| # Overlap host dependency setup with service startup. | |
| uv sync --locked --group dev & | |
| uv_sync_pid=$! | |
| cleanup_uv_sync() { | |
| if kill -0 "$uv_sync_pid" 2>/dev/null; then | |
| kill "$uv_sync_pid" 2>/dev/null || true | |
| fi | |
| } | |
| trap cleanup_uv_sync EXIT | |
| docker compose "${compose_files[@]}" up --no-build -d temporal api worker executor postgres_db caddy minio redis | |
| # Wait for services with health checks to become healthy | |
| echo "Waiting for services to become healthy..." | |
| timeout 300 bash -c ' | |
| while true; do | |
| # Get health status of all services, filter to those with health checks | |
| # A service is ready when Health is exactly "healthy" (not empty, null, or starting) | |
| statuses=$( | |
| docker compose -f docker-compose.local.yml -f .github/docker-compose.integration.yml ps --format json | | |
| jq -r "if type == \"array\" then .[] else . end | select(.Health != null and .Health != \"\") | .Health" | |
| ) | |
| # Count healthy vs total services with health checks | |
| total=$(echo "$statuses" | grep -c . || true) | |
| healthy=$(echo "$statuses" | grep -c "^healthy$" || true) | |
| if [ "$total" -eq 0 ]; then | |
| echo "No services with health checks found yet, waiting..." | |
| sleep 5 | |
| elif [ "$healthy" -lt "$total" ]; then | |
| echo "Waiting for services to be healthy ($healthy/$total ready)..." | |
| sleep 5 | |
| else | |
| echo "All $total services with health checks are healthy" | |
| break | |
| fi | |
| done | |
| ' | |
| # Show final service status | |
| docker compose "${compose_files[@]}" ps | |
| # Verify postgres is accessible from host (core-db network is not internal) | |
| echo "Verifying PostgreSQL is accessible from host..." | |
| timeout 30 bash -c 'until pg_isready -h localhost -p 5432 -U postgres; do sleep 1; done' | |
| echo "PostgreSQL is ready" | |
| echo "Waiting for Python environment sync..." | |
| wait "$uv_sync_pid" | |
| trap - EXIT | |
| - name: Run integration tests | |
| env: | |
| # Expect a base64 encoded key | |
| CUSTOM_REPO_SSH_PRIVATE_KEY: ${{ secrets.CUSTOM_REPO_SSH_PRIVATE_KEY }} | |
| # Keep executor-hosted registry sync enabled (run via Temporal) | |
| TRACECAT__REGISTRY_SYNC_SANDBOX_ENABLED: "true" | |
| # Tests run on the host, so Temporal must be reachable via localhost | |
| TEMPORAL__CLUSTER_URL: "localhost:7233" | |
| TRACECAT__DISABLE_NSJAIL: "true" | |
| # Privileged loop-device regression reuses the exact image built above. | |
| TRACECAT__REGISTRY_LOOP_HOTPLUG_IMAGE: tracecat-ci:integration-${{ github.run_id }}-${{ github.run_attempt }} | |
| run: | | |
| # Skip pool integration tests for now. | |
| # Worker-specific task queues in conftest.py ensure isolation | |
| # faulthandler_timeout=300: dump thread state after 5min for extra debugging | |
| uv run --no-sync pytest tests/integration --ignore=tests/integration/test_pool_integration.py -m "not live_secret" -n auto -ra -v -o faulthandler_timeout=300 | |
| - name: Run API-backed temporal tests against compose | |
| env: | |
| TRACECAT_TEST_API_MODE: "external" | |
| TRACECAT_TEST_EXTERNAL_API_URL: "http://localhost/api" | |
| TEMPORAL__CLUSTER_URL: "localhost:7233" | |
| TRACECAT__DISABLE_NSJAIL: "true" | |
| run: | | |
| uv run --no-sync pytest tests/temporal -m "requires_api and not live_secret" -n auto -ra -v -o faulthandler_timeout=300 | |
| - name: Show Docker logs on failure | |
| if: failure() | |
| env: | |
| TRACECAT__CI_IMAGE_TAG: integration-${{ github.run_id }}-${{ github.run_attempt }} | |
| run: | | |
| echo "=== Docker Service Status ===" | |
| docker compose -f docker-compose.local.yml -f .github/docker-compose.integration.yml ps | |
| echo "=== API Logs ===" | |
| docker compose -f docker-compose.local.yml -f .github/docker-compose.integration.yml logs api --tail=200 | |
| echo "=== Migrations Logs ===" | |
| docker compose -f docker-compose.local.yml -f .github/docker-compose.integration.yml logs migrations --tail=200 | |
| echo "=== Worker Logs ===" | |
| docker compose -f docker-compose.local.yml -f .github/docker-compose.integration.yml logs worker --tail=200 | |
| echo "=== Executor Logs ===" | |
| docker compose -f docker-compose.local.yml -f .github/docker-compose.integration.yml logs executor --tail=200 | |
| echo "=== Temporal Logs ===" | |
| docker compose -f docker-compose.local.yml -f .github/docker-compose.integration.yml logs temporal --tail=200 | |
| echo "=== Redis Logs ===" | |
| docker compose -f docker-compose.local.yml -f .github/docker-compose.integration.yml logs redis --tail=200 | |
| echo "=== MinIO Logs ===" | |
| docker compose -f docker-compose.local.yml -f .github/docker-compose.integration.yml logs minio --tail=200 | |
| echo "=== PostgreSQL Logs ===" | |
| docker compose -f docker-compose.local.yml -f .github/docker-compose.integration.yml logs postgres_db --tail=200 | |
| - name: Clean up | |
| if: always() | |
| env: | |
| TRACECAT__CI_IMAGE_TAG: integration-${{ github.run_id }}-${{ github.run_attempt }} | |
| run: | | |
| docker compose -f docker-compose.local.yml -f .github/docker-compose.integration.yml down -v | |
| - name: Create integration success marker | |
| if: success() && needs.plan-run.outputs.cache_hit != 'true' | |
| run: | | |
| mkdir -p .integration-success | |
| printf '%s\n' "$GITHUB_SHA" > .integration-success/head-sha | |
| - name: Save integration success marker | |
| if: success() && needs.plan-run.outputs.cache_hit != 'true' | |
| uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 | |
| with: | |
| path: .integration-success | |
| key: integration-tests-success-v1-${{ github.sha }} |