diff --git a/.github/workflows/cleanup-stale-schemas.yml b/.github/workflows/cleanup-stale-schemas.yml index bf8522866..899ffa58c 100644 --- a/.github/workflows/cleanup-stale-schemas.yml +++ b/.github/workflows/cleanup-stale-schemas.yml @@ -12,12 +12,19 @@ on: default: "24" description: Drop schemas older than this many hours +permissions: {} + env: TESTS_DIR: ${{ github.workspace }}/dbt-data-reliability/integration_tests jobs: cleanup: runs-on: ubuntu-latest + permissions: + contents: read + env: + WAREHOUSE: ${{ matrix.warehouse-type }} + MAX_AGE_HOURS: ${{ inputs.max-age-hours || '24' }} strategy: fail-fast: false matrix: @@ -28,6 +35,14 @@ jobs: - databricks_catalog - athena steps: + - name: Validate max-age-hours input + # Fail-closed on non-integer input before it reaches dbt run-operation. + run: | + if ! [[ "$MAX_AGE_HOURS" =~ ^[0-9]+$ ]]; then + echo "::error::Invalid max-age-hours: '$MAX_AGE_HOURS' (must be a non-negative integer)" + exit 1 + fi + - name: Checkout dbt package uses: actions/checkout@v6 with: @@ -40,10 +55,9 @@ jobs: cache: "pip" - name: Install dbt - run: > - pip install - "dbt-core" - "dbt-${{ (matrix.warehouse-type == 'databricks_catalog' && 'databricks') || (matrix.warehouse-type == 'athena' && 'athena-community') || matrix.warehouse-type }}" + env: + DBT_ADAPTER_PKG: ${{ (matrix.warehouse-type == 'databricks_catalog' && 'databricks') || (matrix.warehouse-type == 'athena' && 'athena-community') || matrix.warehouse-type }} + run: pip install "dbt-core" "dbt-${DBT_ADAPTER_PKG}" - name: Write dbt profiles env: @@ -61,13 +75,13 @@ jobs: run: dbt deps - name: Symlink local elementary package - run: ln -sfn ${{ github.workspace }}/dbt-data-reliability ${{ env.TESTS_DIR }}/dbt_project/dbt_packages/elementary + run: ln -sfn "${{ github.workspace }}/dbt-data-reliability" "${{ env.TESTS_DIR }}/dbt_project/dbt_packages/elementary" - name: Drop stale CI schemas working-directory: ${{ env.TESTS_DIR }}/dbt_project # Only dbt_ prefixed schemas are created in this repo's CI. # The elementary repo has its own workflow for py_ prefixed schemas. - run: > - dbt run-operation drop_stale_ci_schemas - --args '{prefixes: ["dbt_"], max_age_hours: ${{ inputs.max-age-hours || '24' }}}' - -t "${{ matrix.warehouse-type }}" + run: | + dbt run-operation drop_stale_ci_schemas \ + --args '{prefixes: ["dbt_"], max_age_hours: '"$MAX_AGE_HOURS"'}' \ + -t "$WAREHOUSE" diff --git a/.github/workflows/test-all-warehouses-dbt-pre-releases.yml b/.github/workflows/test-all-warehouses-dbt-pre-releases.yml index e1cad4283..d3e69d818 100644 --- a/.github/workflows/test-all-warehouses-dbt-pre-releases.yml +++ b/.github/workflows/test-all-warehouses-dbt-pre-releases.yml @@ -2,8 +2,12 @@ name: Test all warehouse platforms on dbt pre-releases on: workflow_dispatch: +permissions: {} + jobs: test: + permissions: + contents: read uses: ./.github/workflows/test-all-warehouses.yml secrets: inherit with: diff --git a/.github/workflows/test-all-warehouses.yml b/.github/workflows/test-all-warehouses.yml index 89b077b59..830438409 100644 --- a/.github/workflows/test-all-warehouses.yml +++ b/.github/workflows/test-all-warehouses.yml @@ -34,6 +34,8 @@ on: type: string required: false +permissions: {} + jobs: # ── Local targets ───────────────────────────────────────────────────── # No secrets needed — run on pull_request (works for forks without approval). @@ -42,6 +44,8 @@ jobs: # fully in-process adapters (duckdb). test-local: if: github.event_name != 'pull_request_target' + permissions: + contents: read strategy: fail-fast: false matrix: @@ -88,35 +92,41 @@ jobs: # Determine if this is a fork PR and skip if wrong trigger is used check-fork-status: runs-on: ubuntu-latest + permissions: {} outputs: is_fork: ${{ steps.check.outputs.is_fork }} should_skip: ${{ steps.check.outputs.should_skip }} steps: - name: Check if PR is from fork id: check + env: + EVENT_NAME: ${{ github.event_name }} + PR_REPO: ${{ github.event.pull_request.head.repo.full_name }} + BASE_REPO: ${{ github.repository }} run: | IS_FORK="false" SHOULD_SKIP="false" - if [[ "${{ github.event_name }}" == "pull_request" || "${{ github.event_name }}" == "pull_request_target" ]]; then - if [[ "${{ github.event.pull_request.head.repo.full_name }}" != "${{ github.repository }}" ]]; then + if [[ "$EVENT_NAME" == "pull_request" || "$EVENT_NAME" == "pull_request_target" ]]; then + if [[ "$PR_REPO" != "$BASE_REPO" ]]; then IS_FORK="true" fi # Skip if: pull_request from fork (should use pull_request_target) OR pull_request_target from non-fork (should use pull_request) - if [[ "${{ github.event_name }}" == "pull_request" && "$IS_FORK" == "true" ]]; then + if [[ "$EVENT_NAME" == "pull_request" && "$IS_FORK" == "true" ]]; then SHOULD_SKIP="true" - elif [[ "${{ github.event_name }}" == "pull_request_target" && "$IS_FORK" == "false" ]]; then + elif [[ "$EVENT_NAME" == "pull_request_target" && "$IS_FORK" == "false" ]]; then SHOULD_SKIP="true" fi fi - echo "is_fork=$IS_FORK" >> $GITHUB_OUTPUT - echo "should_skip=$SHOULD_SKIP" >> $GITHUB_OUTPUT + echo "is_fork=$IS_FORK" >> "$GITHUB_OUTPUT" + echo "should_skip=$SHOULD_SKIP" >> "$GITHUB_OUTPUT" # Approval gate for fork PRs (only runs once for all platforms) approve-fork: runs-on: ubuntu-latest + permissions: {} needs: [check-fork-status] if: needs.check-fork-status.outputs.should_skip != 'true' && needs.check-fork-status.outputs.is_fork == 'true' environment: elementary_test_env @@ -126,6 +136,8 @@ jobs: test-cloud: needs: [check-fork-status, approve-fork] + permissions: + contents: read if: | ! cancelled() && needs.check-fork-status.result == 'success' && diff --git a/.github/workflows/test-warehouse.yml b/.github/workflows/test-warehouse.yml index cab4a0511..2ddbc7123 100644 --- a/.github/workflows/test-warehouse.yml +++ b/.github/workflows/test-warehouse.yml @@ -51,6 +51,8 @@ on: default: "latest_official" required: false +permissions: {} + env: BRANCH_NAME: ${{ github.head_ref || github.ref_name }} TESTS_DIR: ${{ github.workspace }}/dbt-data-reliability/integration_tests @@ -59,6 +61,11 @@ jobs: test: runs-on: ubuntu-latest timeout-minutes: 60 + permissions: + contents: read + env: + WAREHOUSE: ${{ inputs.warehouse-type }} + DBT_VERSION: ${{ inputs.dbt-version }} concurrency: # Serialises runs for the same warehouse × dbt-version × branch. # The schema name is derived from a hash of this group (see "Write dbt profiles"). @@ -160,6 +167,8 @@ jobs: - name: Install dbt-vertica if: inputs.warehouse-type == 'vertica' && inputs.dbt-version != 'fusion' + env: + DBT_CORE_PIN: ${{ (!startsWith(inputs.dbt-version, 'latest') && format('=={0}', inputs.dbt-version)) || '' }} run: | # dbt-vertica pins dbt-core~=1.8 which lacks native support for the # "arguments" test property used by the integration-test framework. @@ -167,15 +176,17 @@ jobs: # dbt-core version separately (dbt-vertica works fine with newer # dbt-core versions). pip install dbt-vertica --no-deps - pip install vertica-python \ - "dbt-core${{ (!startsWith(inputs.dbt-version, 'latest') && format('=={0}', inputs.dbt-version)) || '' }}" + pip install vertica-python "dbt-core${DBT_CORE_PIN}" - name: Install dbt if: ${{ inputs.dbt-version != 'fusion' && inputs.warehouse-type != 'vertica' }} - run: - pip install${{ (inputs.dbt-version == 'latest_pre' && ' --pre') || '' }} - "dbt-core${{ (!startsWith(inputs.dbt-version, 'latest') && format('=={0}', inputs.dbt-version)) || '' }}" - "dbt-${{ (inputs.warehouse-type == 'databricks_catalog' && 'databricks') || (inputs.warehouse-type == 'spark' && 'spark[PyHive]') || (inputs.warehouse-type == 'athena' && 'athena-community') || inputs.warehouse-type }}${{ (!startsWith(inputs.dbt-version, 'latest') && format('~={0}', inputs.dbt-version)) || '' }}" + env: + PIP_PRE_FLAG: ${{ (inputs.dbt-version == 'latest_pre' && '--pre') || '' }} + DBT_CORE_PIN: ${{ (!startsWith(inputs.dbt-version, 'latest') && format('=={0}', inputs.dbt-version)) || '' }} + DBT_ADAPTER_PKG: ${{ (inputs.warehouse-type == 'databricks_catalog' && 'databricks') || (inputs.warehouse-type == 'spark' && 'spark[PyHive]') || (inputs.warehouse-type == 'athena' && 'athena-community') || inputs.warehouse-type }} + DBT_ADAPTER_PIN: ${{ (!startsWith(inputs.dbt-version, 'latest') && format('~={0}', inputs.dbt-version)) || '' }} + run: | + pip install $PIP_PRE_FLAG "dbt-core${DBT_CORE_PIN}" "dbt-${DBT_ADAPTER_PKG}${DBT_ADAPTER_PIN}" - name: Install dbt-fusion if: inputs.dbt-version == 'fusion' @@ -187,11 +198,17 @@ jobs: # For Vertica, dbt-vertica is already installed with --no-deps above; # using ".[vertica]" would re-resolve dbt-vertica's deps and downgrade # dbt-core to ~=1.8. Install elementary without the adapter extra. - if [ "${{ inputs.warehouse-type }}" = "vertica" ]; then - pip install "./elementary" - else - pip install "./elementary[${{ (inputs.warehouse-type == 'databricks_catalog' && 'databricks') || inputs.warehouse-type }}]" - fi + case "$WAREHOUSE" in + vertica) + pip install "./elementary" + ;; + databricks_catalog) + pip install "./elementary[databricks]" + ;; + *) + pip install "./elementary[$WAREHOUSE]" + ;; + esac - name: Write dbt profiles env: @@ -205,7 +222,7 @@ jobs: # Budget (PostgreSQL 63-char limit): # dbt_(4) + timestamp(13) + _(1) + branch(≤18) + _(1) + hash(8) = 45 # + _elementary(11) + _gw7(4) = 60 - CONCURRENCY_GROUP="tests_${{ inputs.warehouse-type }}_dbt_${{ inputs.dbt-version }}_${BRANCH_NAME}" + CONCURRENCY_GROUP="tests_${WAREHOUSE}_dbt_${DBT_VERSION}_${BRANCH_NAME}" SHORT_HASH=$(echo -n "$CONCURRENCY_GROUP" | sha256sum | head -c 8) SAFE_BRANCH=$(echo "${BRANCH_NAME}" | awk '{print tolower($0)}' | sed "s/[^a-z0-9]/_/g; s/__*/_/g" | head -c 18) DATE_STAMP=$(date -u +%y%m%d_%H%M%S) @@ -221,8 +238,9 @@ jobs: - name: Install dependencies working-directory: ${{ env.TESTS_DIR }} run: | - ${{ (inputs.dbt-version == 'fusion' && '~/.local/bin/dbt') || 'dbt' }} deps --project-dir dbt_project - ln -sfn ${{ github.workspace }}/dbt-data-reliability dbt_project/dbt_packages/elementary + if [ "$DBT_VERSION" = "fusion" ]; then DBT_BIN="$HOME/.local/bin/dbt"; else DBT_BIN="dbt"; fi + "$DBT_BIN" deps --project-dir dbt_project + ln -sfn "${{ github.workspace }}/dbt-data-reliability" dbt_project/dbt_packages/elementary pip install -r requirements.txt - name: Start Vertica @@ -240,15 +258,24 @@ jobs: - name: Check DWH connection working-directory: ${{ env.TESTS_DIR }} run: | - ${{ (inputs.dbt-version == 'fusion' && '~/.local/bin/dbt') || 'dbt' }} debug -t "${{ inputs.warehouse-type }}" + if [ "$DBT_VERSION" = "fusion" ]; then DBT_BIN="$HOME/.local/bin/dbt"; else DBT_BIN="dbt"; fi + "$DBT_BIN" debug -t "$WAREHOUSE" - name: Test working-directory: "${{ env.TESTS_DIR }}/tests" - run: py.test -n${{ (inputs.warehouse-type == 'spark' && '4') || '8' }} -vvv --target "${{ inputs.warehouse-type }}" --junit-xml=test-results.xml --html=detailed_report_${{ inputs.warehouse-type }}_dbt_${{ inputs.dbt-version }}.html --self-contained-html --clear-on-end ${{ (inputs.dbt-version == 'fusion' && '--runner-method fusion') || '' }} + env: + PYTEST_PARALLEL: ${{ (inputs.warehouse-type == 'spark' && '4') || '8' }} + FUSION_RUNNER_FLAG: ${{ (inputs.dbt-version == 'fusion' && '--runner-method fusion') || '' }} + run: | + py.test -n"$PYTEST_PARALLEL" -vvv --target "$WAREHOUSE" \ + --junit-xml=test-results.xml \ + --html="detailed_report_${WAREHOUSE}_dbt_${DBT_VERSION}.html" \ + --self-contained-html --clear-on-end $FUSION_RUNNER_FLAG - name: Upload test results if: always() - uses: pmeier/pytest-results-action@v0.8.0 + # pmeier/pytest-results-action v0.8.0, checked 2026-04-26. + uses: pmeier/pytest-results-action@0841ca7226ab155943837380769373a5dd14d7ed with: path: ${{ env.TESTS_DIR }}/tests/test-results.xml summary: true @@ -269,6 +296,7 @@ jobs: working-directory: ${{ env.TESTS_DIR }} continue-on-error: true run: | - ${{ (inputs.dbt-version == 'fusion' && '~/.local/bin/dbt') || 'dbt' }} run-operation elementary_tests.drop_test_schemas \ + if [ "$DBT_VERSION" = "fusion" ]; then DBT_BIN="$HOME/.local/bin/dbt"; else DBT_BIN="dbt"; fi + "$DBT_BIN" run-operation elementary_tests.drop_test_schemas \ --project-dir dbt_project \ - -t "${{ inputs.warehouse-type }}" + -t "$WAREHOUSE"