test-warehouse.yml

name: Test warehouse platform
on:
  workflow_dispatch:
    inputs:
      warehouse-type:
        type: choice
        required: true
        description: Type of warehouse platform
        options:
          - postgres
          - snowflake
          - bigquery
          - redshift
          - databricks_catalog
          - spark
          - athena
          - trino
          - clickhouse
          - dremio
          - duckdb
          - sqlserver
          - fabric
          - vertica
      elementary-ref:
        type: string
        required: false
        description: Branch or tag to checkout for 'elementary' repository
      dbt-data-reliability-ref:
        type: string
        required: false
        description: Branch or tag to checkout for 'dbt-data-reliability' repository
      dbt-version:
        type: string
        required: false
        default: "latest_official"
        description: dbt's version to test with

  workflow_call:
    inputs:
      warehouse-type:
        type: string
        required: true
      elementary-ref:
        type: string
        required: false
      dbt-data-reliability-ref:
        type: string
        required: false
      dbt-version:
        type: string
        default: "latest_official"
        required: false

permissions: {}

env:
  BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
  TESTS_DIR: ${{ github.workspace }}/dbt-data-reliability/integration_tests

jobs:
  test:
    runs-on: ubuntu-latest
    timeout-minutes: 60
    permissions:
      contents: read
      # Mint an OIDC token to assume the shared elementary-oss AWS role.
      id-token: write
    env:
      WAREHOUSE: ${{ inputs.warehouse-type }}
      DBT_VERSION: ${{ inputs.dbt-version }}
    concurrency:
      # Serialises runs for the same warehouse × dbt-version × branch.
      # The schema name is derived from a hash of this group (see "Write dbt profiles").
      group: tests_${{ inputs.warehouse-type }}_dbt_${{ inputs.dbt-version }}_${{ github.head_ref || github.ref_name }}
      cancel-in-progress: true
    steps:
      - name: Checkout Elementary
        uses: actions/checkout@v6
        with:
          repository: elementary-data/elementary
          path: elementary
          ref: ${{ inputs.elementary-ref }}

      - name: Checkout dbt package
        uses: actions/checkout@v6
        with:
          path: dbt-data-reliability
          ref: ${{ inputs.dbt-data-reliability-ref }}

      - name: Configure AWS credentials
        if: inputs.warehouse-type == 'athena'
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }}
          aws-region: eu-west-1

      - name: Start Postgres
        if: inputs.warehouse-type == 'postgres'
        working-directory: ${{ env.TESTS_DIR }}
        run: docker compose up -d postgres

      - name: Start Trino
        if: inputs.warehouse-type == 'trino'
        working-directory: ${{ env.TESTS_DIR }}
        run: docker compose -f docker-compose-trino.yml up -d

      - name: Start Clickhouse
        if: inputs.warehouse-type == 'clickhouse'
        working-directory: ${{ env.TESTS_DIR }}
        run: docker compose up -d clickhouse

      - name: Start Dremio
        if: inputs.warehouse-type == 'dremio'
        working-directory: ${{ env.TESTS_DIR }}
        run: |
          docker compose -f docker-compose-dremio.yml up -d
          # Wait for Dremio to be healthy (one-shot containers like
          # minio-setup exit immediately, so --wait would fail).
          echo "Waiting for Dremio to become healthy..."
          timeout 180 bash -c 'until [ "$(docker inspect -f {{.State.Health.Status}} dremio 2>/dev/null)" = "healthy" ]; do sleep 5; done'
          echo "Dremio is healthy."

      - name: Start SQL Server
        if: inputs.warehouse-type == 'sqlserver'
        working-directory: ${{ env.TESTS_DIR }}
        run: |
          docker compose -f docker-compose-sqlserver.yml up -d
          echo "Waiting for SQL Server to become healthy..."
          timeout 120 bash -c 'until [ "$(docker inspect -f {{.State.Health.Status}} sqlserver 2>/dev/null)" = "healthy" ]; do sleep 5; done'
          echo "SQL Server is healthy."

      - name: Install ODBC Driver
        if: inputs.warehouse-type == 'sqlserver' || inputs.warehouse-type == 'fabric'
        run: |
          curl https://packages.microsoft.com/keys/microsoft.asc | sudo tee /etc/apt/trusted.gpg.d/microsoft.asc
          curl https://packages.microsoft.com/config/ubuntu/$(lsb_release -rs)/prod.list | sudo tee /etc/apt/sources.list.d/mssql-release.list
          sudo apt-get update
          sudo ACCEPT_EULA=Y apt-get install -y msodbcsql18 unixodbc-dev

      - name: Start Spark
        if: inputs.warehouse-type == 'spark'
        working-directory: ${{ env.TESTS_DIR }}
        run: |
          docker compose -f docker-compose-spark.yml build
          docker compose -f docker-compose-spark.yml up -d
          echo "Waiting for MinIO setup to complete..."
          timeout 60 bash -c '
            until [ "$(docker inspect -f "{{.State.Status}}" spark-minio-setup 2>/dev/null)" = "exited" ]; do sleep 2; done
            EXIT_CODE=$(docker inspect -f "{{.State.ExitCode}}" spark-minio-setup 2>/dev/null)
            if [ "$EXIT_CODE" != "0" ]; then echo "MinIO setup failed with exit code $EXIT_CODE"; exit 1; fi
          '
          echo "MinIO is ready."
          echo "Waiting for Spark Thrift Server to become healthy..."
          timeout 180 bash -c 'until [ "$(docker inspect -f {{.State.Health.Status}} spark-thrift 2>/dev/null)" = "healthy" ]; do sleep 5; done'
          echo "Spark Thrift Server is healthy."

      - name: Setup Python
        uses: actions/setup-python@v6
        with:
          python-version: "3.10"
          cache: "pip"

      - name: Install Spark requirements
        if: inputs.warehouse-type == 'spark'
        run: sudo apt-get update && sudo apt-get install -y python3-dev libsasl2-dev gcc

      - name: Install compatible databricks connector (not limited in older dbt-databricks versions)
        if: startsWith(inputs.warehouse-type, 'databricks') && inputs.dbt-version < '1.7.0'
        run: pip install databricks-sql-connector==2.9.3

      - name: Reject unsupported Vertica + Fusion combination
        if: inputs.warehouse-type == 'vertica' && inputs.dbt-version == 'fusion'
        run: |
          echo "::error::dbt Fusion does not support third-party adapters such as dbt-vertica."
          exit 1

      - name: Install dbt-vertica
        if: inputs.warehouse-type == 'vertica' && inputs.dbt-version != 'fusion'
        env:
          DBT_CORE_PIN: ${{ (!startsWith(inputs.dbt-version, 'latest') && format('=={0}', inputs.dbt-version)) || '' }}
        run: |
          # dbt-vertica pins dbt-core~=1.8 which lacks native support for the
          # "arguments" test property used by the integration-test framework.
          # Install dbt-vertica without deps, then install the requested
          # dbt-core version separately (dbt-vertica works fine with newer
          # dbt-core versions).
          pip install dbt-vertica --no-deps
          pip install vertica-python "dbt-core${DBT_CORE_PIN}"

      - name: Install dbt
        if: ${{ inputs.dbt-version != 'fusion' && inputs.warehouse-type != 'vertica' }}
        env:
          PIP_PRE_FLAG: ${{ (inputs.dbt-version == 'latest_pre' && '--pre') || '' }}
          DBT_CORE_PIN: ${{ (!startsWith(inputs.dbt-version, 'latest') && format('=={0}', inputs.dbt-version)) || '' }}
          DBT_ADAPTER_PKG: ${{ (inputs.warehouse-type == 'databricks_catalog' && 'databricks') || (inputs.warehouse-type == 'spark' && 'spark[PyHive]') || (inputs.warehouse-type == 'athena' && 'athena-community') || inputs.warehouse-type }}
          DBT_ADAPTER_PIN: ${{ (!startsWith(inputs.dbt-version, 'latest') && format('~={0}', inputs.dbt-version)) || '' }}
        run: |
          pip install $PIP_PRE_FLAG "dbt-core${DBT_CORE_PIN}" "dbt-${DBT_ADAPTER_PKG}${DBT_ADAPTER_PIN}"

      - name: Install dbt-fusion
        if: inputs.dbt-version == 'fusion'
        run: |
          curl -fsSL https://public.cdn.getdbt.com/fs/install/install.sh | sh -s --

      - name: Install Elementary
        run: |
          # For Vertica, dbt-vertica is already installed with --no-deps above;
          # using ".[vertica]" would re-resolve dbt-vertica's deps and downgrade
          # dbt-core to ~=1.8.  Install elementary without the adapter extra.
          case "$WAREHOUSE" in
            vertica)
              pip install "./elementary"
              ;;
            databricks_catalog)
              pip install "./elementary[databricks]"
              ;;
            *)
              pip install "./elementary[$WAREHOUSE]"
              ;;
          esac

      - name: Write dbt profiles
        env:
          CI_WAREHOUSE_SECRETS: ${{ secrets.CI_WAREHOUSE_SECRETS || '' }}
        run: |
          # Schema name = dbt_<YYMMDD_HHMMSS>_<branch≤18>_<8-char hash>
          # The hash prevents collisions across concurrent jobs; the branch
          # keeps it human-readable; the timestamp helps with stale schema
          # cleanup and ensures each CI run gets a unique schema.
          #
          # Budget (PostgreSQL 63-char limit):
          #   dbt_(4) + timestamp(13) + _(1) + branch(≤18) + _(1) + hash(8) = 45
          #   + _elementary(11) + _gw7(4) = 60
          CONCURRENCY_GROUP="tests_${WAREHOUSE}_dbt_${DBT_VERSION}_${BRANCH_NAME}"
          SHORT_HASH=$(echo -n "$CONCURRENCY_GROUP" | sha256sum | head -c 8)
          SAFE_BRANCH=$(echo "${BRANCH_NAME}" | awk '{print tolower($0)}' | sed "s/[^a-z0-9]/_/g; s/__*/_/g" | head -c 18)
          DATE_STAMP=$(date -u +%y%m%d_%H%M%S)
          SCHEMA_NAME="dbt_${DATE_STAMP}_${SAFE_BRANCH}_${SHORT_HASH}"

          echo "Schema name: $SCHEMA_NAME (branch='${BRANCH_NAME}', timestamp=${DATE_STAMP}, hash of concurrency group)"

          python "${{ github.workspace }}/dbt-data-reliability/integration_tests/profiles/generate_profiles.py" \
            --template "${{ github.workspace }}/dbt-data-reliability/integration_tests/profiles/profiles.yml.j2" \
            --output ~/.dbt/profiles.yml \
            --schema-name "$SCHEMA_NAME"

      - name: Install dependencies
        working-directory: ${{ env.TESTS_DIR }}
        run: |
          if [ "$DBT_VERSION" = "fusion" ]; then DBT_BIN="$HOME/.local/bin/dbt"; else DBT_BIN="dbt"; fi
          "$DBT_BIN" deps --project-dir dbt_project
          ln -sfn "${{ github.workspace }}/dbt-data-reliability" dbt_project/dbt_packages/elementary
          pip install -r requirements.txt

      - name: Start Vertica
        if: inputs.warehouse-type == 'vertica'
        working-directory: ${{ env.TESTS_DIR }}
        run: docker compose -f docker-compose-vertica.yml up -d

      - name: Wait for Vertica to be ready
        if: inputs.warehouse-type == 'vertica'
        run: |
          echo "Waiting for Vertica to be healthy..."
          timeout 60 bash -c 'until [ "$(docker inspect --format="{{.State.Health.Status}}" vertica)" == "healthy" ]; do echo "Waiting..."; sleep 5; done'
          echo "Vertica is ready!"

      - name: Check DWH connection
        working-directory: ${{ env.TESTS_DIR }}
        run: |
          if [ "$DBT_VERSION" = "fusion" ]; then DBT_BIN="$HOME/.local/bin/dbt"; else DBT_BIN="dbt"; fi
          "$DBT_BIN" debug -t "$WAREHOUSE"

      - name: Test
        working-directory: "${{ env.TESTS_DIR }}/tests"
        env:
          PYTEST_PARALLEL: ${{ (inputs.warehouse-type == 'spark' && '4') || '8' }}
          FUSION_RUNNER_FLAG: ${{ (inputs.dbt-version == 'fusion' && '--runner-method fusion') || '' }}
        run: |
          py.test -n"$PYTEST_PARALLEL" -vvv --target "$WAREHOUSE" \
            --junit-xml=test-results.xml \
            --html="detailed_report_${WAREHOUSE}_dbt_${DBT_VERSION}.html" \
            --self-contained-html --clear-on-end $FUSION_RUNNER_FLAG

      - name: Upload test results
        if: always()
        # pmeier/pytest-results-action v0.8.0, checked 2026-04-26.
        uses: pmeier/pytest-results-action@0841ca7226ab155943837380769373a5dd14d7ed
        with:
          path: ${{ env.TESTS_DIR }}/tests/test-results.xml
          summary: true
          display-options: fEX
          fail-on-empty: true

      - name: Upload HTML report
        if: always()
        uses: actions/upload-artifact@v6
        with:
          name: detailed_report_${{ inputs.warehouse-type }}_dbt_${{ inputs.dbt-version }}
          path: ${{ env.TESTS_DIR }}/tests/detailed_report_${{ inputs.warehouse-type }}_dbt_${{ inputs.dbt-version }}.html

      - name: Drop test schemas
        if: >-
          always() &&
          contains(fromJSON('["snowflake","bigquery","redshift","databricks_catalog","athena","fabric"]'), inputs.warehouse-type)
        working-directory: ${{ env.TESTS_DIR }}
        continue-on-error: true
        run: |
          if [ "$DBT_VERSION" = "fusion" ]; then DBT_BIN="$HOME/.local/bin/dbt"; else DBT_BIN="dbt"; fi
          "$DBT_BIN" run-operation elementary_tests.drop_test_schemas \
            --project-dir dbt_project \
            -t "$WAREHOUSE"