pg16-merge-validation.yml

# --------------------------------------------------------------------
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed
# with this work for additional information regarding copyright
# ownership. The ASF licenses this file to You under the Apache
# License, Version 2.0 (the "License"); you may not use this file
# except in compliance with the License. You may obtain a copy of the
# License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied. See the License for the specific language governing
# permissions and limitations under the License.
#
# --------------------------------------------------------------------
# GitHub Actions Workflow: PostgreSQL 16 Merge Validation
# --------------------------------------------------------------------
# Description:
#
#   This workflow validates the cbdb-postgres-merge branch, which upgrades
#   the PostgreSQL kernel from version 14 to version 16. It builds the
#   codebase, creates a gpdemo cluster, and performs basic validation to
#   ensure the upgrade is functional and prevent regressions.
#
# Workflow Overview:
# 1. **Build and Test Job**:
#    - Configures and builds Apache Cloudberry with PG16 kernel.
#    - Installs required dependencies (ICU, Xerces-C).
#    - Creates a gpdemo cluster for validation.
#    - Runs basic smoke tests to verify cluster functionality.
#    - **Key Artifacts**: Build logs, cluster configuration, test results.
#
# 2. **Report Job**:
#    - Aggregates job results into a final report.
#    - Sends failure notifications if any step fails.
#
# Triggers:
# - Push to `cbdb-postgres-merge` branch.
# - Pull requests to `cbdb-postgres-merge` branch.
# - Manual workflow dispatch.
#
# Container Images:
# - **Build/Test**: `apache/incubator-cloudberry:cbdb-build-rocky9-latest`
#
# Artifacts:
# - Build Logs          (retention: ${{ env.LOG_RETENTION_DAYS }} days).
# - Cluster Logs        (retention: ${{ env.LOG_RETENTION_DAYS }} days).
# - Test Results        (retention: ${{ env.LOG_RETENTION_DAYS }} days).
#
# Notes:
# - This workflow is specifically designed for the PG16 upgrade validation.
# - Many extensions are not yet supported and will be disabled during build.
# - The focus is on ensuring the core database functionality works correctly.
# --------------------------------------------------------------------

name: PostgreSQL 16 Merge Validation

on:
  push:
    branches: [cbdb-postgres-merge]
  pull_request:
    branches: [cbdb-postgres-merge]
    types: [opened, synchronize, reopened, edited]
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: false

permissions:
  contents: read
  packages: read
  actions: write
  checks: read
  pull-requests: read

env:
  LOG_RETENTION_DAYS: 7
  INSTALL_PREFIX: /usr/local/cloudberry-db

jobs:

  ## ======================================================================
  ## Job: prepare-test-matrix
  ## ======================================================================

  prepare-test-matrix:
    runs-on: ubuntu-22.04
    outputs:
      test-matrix: ${{ steps.set-matrix.outputs.matrix }}
      should_run: ${{ steps.set-matrix.outputs.should_run }}

    steps:
      - name: Generate Test Matrix
        id: set-matrix
        run: |
          echo "=== PG16 Test Matrix Preparation ==="
          
          # Define defaults for test configurations
          DEFAULT_NUM_PRIMARY_MIRROR_PAIRS=3
          DEFAULT_ENABLE_CGROUPS=false
          DEFAULT_ENABLE_CORE_CHECK=true
          DEFAULT_PG_SETTINGS_OPTIMIZER=""
          
          # Define test configurations for PG16 merge validation
          # Tests are currently disabled as installcheck support is being fixed
          # Uncomment and add tests as features become available
          ALL_TESTS='{
            "include": [
              {"test":"ic-good-opt-off",
                "make_configs":["src/test/regress:installcheck-good"],
                "pg_settings":{"optimizer":"off"}
              },
              {"test":"ic-isolation2-basic",
                "make_configs":["src/test/isolation2:installcheck-isolation2"],
                "pg_settings":{"optimizer":"off"},
                "enable_core_check":false
              }
            ]
          }'
          
          # Tests to be added as PG16 support improves:
          # - ic-good-opt-off: Basic installcheck with optimizer off (includes greenplum_schedule)
          #   {"test":"ic-good-opt-off",
          #    "make_configs":["src/test/regress:installcheck-good"],
          #    "pg_settings":{"optimizer":"off"}
          #   }
          #
          # - ic-good-opt-on: Basic installcheck with optimizer on (requires ORCA)
          #   {"test":"ic-good-opt-on",
          #    "make_configs":["src/test/regress:installcheck-good"],
          #    "pg_settings":{"optimizer":"on"}
          #   }
          #
          # - ic-contrib: Contrib module tests
          #   {"test":"ic-contrib",
          #    "make_configs":["contrib/auto_explain:installcheck",
          #                    "contrib/citext:installcheck"]
          #   }
          #
          # - ic-gpcontrib: GPContrib module tests
          #   {"test":"ic-gpcontrib",
          #    "make_configs":["gpcontrib/orafce:installcheck",
          #                    "gpcontrib/gp_toolkit:installcheck"]
          #   }
          #
          # - ic-isolation2-advanced: More isolation2 tests
          #   {"test":"ic-isolation2-expandshrink",
          #    "make_configs":["src/test/isolation2:installcheck-expandshrink"]
          #   }
          #   {"test":"ic-isolation2-crash",
          #    "make_configs":["src/test/isolation2:installcheck-isolation2-crash"],
          #    "enable_core_check":false
          #   }
          
          # Function to apply defaults to test configurations
          apply_defaults() {
            echo "$1" | jq --arg     npm "$DEFAULT_NUM_PRIMARY_MIRROR_PAIRS" \
                           --argjson ec  "$DEFAULT_ENABLE_CGROUPS" \
                           --argjson ecc "$DEFAULT_ENABLE_CORE_CHECK" \
                           --arg     opt "$DEFAULT_PG_SETTINGS_OPTIMIZER" \
              'def get_defaults:
                {
                  num_primary_mirror_pairs: ($npm|tonumber),
                  enable_cgroups: $ec,
                  enable_core_check: $ecc,
                  pg_settings: {
                    optimizer: $opt
                  }
                };
               get_defaults * .'
          }
          
          # Build result JSON with defaults applied
          RESULT='{"include":['
          FIRST=true
          
          for TEST_CONFIG in $(echo "$ALL_TESTS" | jq -c '.include[]'); do
            FILTERED_WITH_DEFAULTS=$(apply_defaults "$TEST_CONFIG")
            if [[ "$FIRST" == true ]]; then
              FIRST=false
            else
              RESULT="${RESULT},"
            fi
            RESULT="${RESULT}${FILTERED_WITH_DEFAULTS}"
          done
          
          RESULT="${RESULT}]}"
          
          # Output the matrix for GitHub Actions
          echo "Final test matrix configuration:"
          echo "$RESULT" | jq .
          
          # Set output
          {
            echo "matrix<<EOF"
            echo "$RESULT"
            echo "EOF"
            
            # Check if matrix is empty
            TEST_COUNT=$(echo "$RESULT" | jq '.include | length')
            if [ "$TEST_COUNT" -gt 0 ]; then
              echo "should_run=true"
            else
              echo "should_run=false"
            fi
          } >> "$GITHUB_OUTPUT"
          
          echo "=== Test Matrix Preparation Complete ==="

  ## ======================================================================
  ## Job: build
  ## ======================================================================

  build:
    name: Build PG16 for Cloudberry
    env:
      JOB_TYPE: build
    needs: [prepare-test-matrix]
    runs-on: ubuntu-22.04
    timeout-minutes: 180
    outputs:
      build_timestamp: ${{ steps.set_timestamp.outputs.timestamp }}

    container:
      image: apache/incubator-cloudberry:cbdb-build-rocky9-latest
      options: >-
        --privileged
        --user root
        -h cdw
        --shm-size=2gb

    steps:
      - name: Set build timestamp
        id: set_timestamp
        run: |
          timestamp=$(date +'%Y%m%d_%H%M%S')
          echo "timestamp=$timestamp" | tee -a "$GITHUB_OUTPUT"
          echo "BUILD_TIMESTAMP=$timestamp" | tee -a "$GITHUB_ENV"

      - name: Checkout Apache Cloudberry (cbdb-postgres-merge branch)
        uses: actions/checkout@v4
        with:
          fetch-depth: 1
          submodules: true

      - name: Cloudberry Environment Initialization
        env:
          LOGS_DIR: pg16-build-logs
        run: |
          set -eo pipefail
          
          # Initialize system
          if ! su - gpadmin -c "/tmp/init_system.sh"; then
            echo "::error::Container initialization failed"
            exit 1
          fi

          # Setup logging directories
          mkdir -p "${LOGS_DIR}/details"
          chown -R gpadmin:gpadmin .
          chmod -R 755 .
          chmod 777 "${LOGS_DIR}"

          # Clean up disk space
          df -kh /
          rm -rf /__t/*
          df -kh /

          {
            echo "=== Environment Information ==="
            uname -a
            df -h
            free -h
            env
          } | tee -a "${LOGS_DIR}/details/environment.log"

          echo "SRC_DIR=${GITHUB_WORKSPACE}" | tee -a "$GITHUB_ENV"

      - name: Generate Build Job Summary Start
        run: |
          {
            echo "# PostgreSQL 16 Merge Validation"
            echo "## Environment"
            echo "- Start Time: $(date -u +'%Y-%m-%d %H:%M:%S UTC')"
            echo "- OS Version: $(cat /etc/redhat-release)"
            echo "- GCC Version: $(gcc --version | head -n1)"
            echo ""
            echo "## Build Configuration"
            echo "- Install Prefix: ${INSTALL_PREFIX}"
            echo "- Port: 5432"
            echo "- Debug: Enabled (O0, g3, cassert)"
            echo "- Disabled Features: GPCloud"
          } >> "$GITHUB_STEP_SUMMARY"

      - name: Prepare Installation Directory
        run: |
          set -eo pipefail
          
          # Clean and prepare installation directory
          sudo rm -rf ${INSTALL_PREFIX}
          sudo chmod a+w /usr/local
          mkdir -p ${INSTALL_PREFIX}/lib
          
          # Copy Xerces-C libraries
          sudo cp -v /usr/local/xerces-c/lib/libxerces-c.so \
                     /usr/local/xerces-c/lib/libxerces-c-3.*.so \
                     ${INSTALL_PREFIX}/lib
          
          sudo chown -R gpadmin:gpadmin ${INSTALL_PREFIX}
          
          echo "Installation directory prepared successfully"

      - name: Install ICU Development Libraries
        run: |
          set -eo pipefail
          
          echo "Installing ICU development libraries..."
          sudo dnf install -y libicu-devel
          
          echo "ICU libraries installed successfully"
          rpm -qa | grep libicu

      - name: Configure PG16 for Cloudberry
        env:
          SRC_DIR: ${{ github.workspace }}
        run: |
          set -eo pipefail

          {
            echo "=== Configure Log ==="
            echo "Timestamp: $(date -u)"
            echo "Configure options:"
            echo "  --prefix=${INSTALL_PREFIX}"
            echo "  --with-pgport=5432"
            echo "  --enable-cassert"
            echo "  --enable-debug-extensions"
            echo "  --with-perl --with-python --with-libxml --with-openssl"
            echo "  --with-zstd"
            echo "  --disable-gpcloud"
            echo "  --with-pythonsrc-ext"
            echo "  --with-gssapi"
            echo ""
            
            # Run configure as gpadmin
            if ! su - gpadmin -c "
              cd ${SRC_DIR}
              export LD_LIBRARY_PATH=${INSTALL_PREFIX}/lib:${LD_LIBRARY_PATH}
              CFLAGS='-O0 -g3' ./configure \
                --prefix=${INSTALL_PREFIX} \
                --with-pgport=5432 \
                --enable-cassert \
                --with-perl \
                --with-python \
                --with-libxml \
                --with-openssl \
                --with-zstd \
                --enable-debug-extensions \
                --disable-gpcloud \
                --with-pythonsrc-ext \
                --with-gssapi \
                --without-icu \
                --with-libraries=${INSTALL_PREFIX}/lib \
                --with-includes=/usr/local/xerces-c/include
            "; then
              echo "::error::Configure failed"
              exit 1
            fi
            
            echo "Configure completed successfully"
          } 2>&1 | tee -a pg16-build-logs/details/configure.log

      - name: Build and Install Apache Cloudberry
        env:
          SRC_DIR: ${{ github.workspace }}
        run: |
          set -eo pipefail

          {
            echo "=== Build and Install Log ==="
            echo "Timestamp: $(date -u)"
            echo "Starting build and installation (non-parallel)..."
            
            # Build and install as gpadmin (non-parallel to avoid issues)
            if ! time su - gpadmin -c "
              cd ${SRC_DIR}
              export LD_LIBRARY_PATH=${INSTALL_PREFIX}/lib:\${LD_LIBRARY_PATH}
              make
              make install
            "; then
              echo "::error::Build and installation failed"
              exit 1
            fi
            
            echo "Build and installation completed successfully"
          } 2>&1 | tee -a pg16-build-logs/details/build-install.log

      - name: Verify Build Artifacts
        run: |
          set -eo pipefail

          {
            echo "=== Build Verification ==="
            echo "Timestamp: $(date -u)"
            
            # Verify postgres binary exists and is executable
            if [ ! -x "${INSTALL_PREFIX}/bin/postgres" ]; then
              echo "::error::postgres binary not found or not executable"
              exit 1
            fi
            
            # Check versions
            echo "PostgreSQL version:"
            su - gpadmin -c "${INSTALL_PREFIX}/bin/postgres --version"
            
            echo ""
            echo "Cloudberry version:"
            su - gpadmin -c "${INSTALL_PREFIX}/bin/postgres --gp-version"
            
            echo ""
            echo "Library dependencies:"
            ldd ${INSTALL_PREFIX}/bin/postgres
            
            echo ""
            echo "Build artifacts verified successfully"
          } 2>&1 | tee -a pg16-build-logs/details/verification.log

      - name: Create Build Artifact
        run: |
          set -eo pipefail
          echo "Creating build artifact tarball..."
          echo "Listing /usr/local:"
          ls -la /usr/local
          
          if [ ! -d "/usr/local/cloudberry-db" ]; then
            echo "::error::/usr/local/cloudberry-db does not exist!"
            exit 1
          fi
          
          # Create tarball of the installation directory
          # We use -C /usr/local to avoid including full path structure
          tar -czf cloudberry-pg16.tar.gz -C /usr/local cloudberry-db
          
          echo "Build artifact created: cloudberry-pg16.tar.gz"
          ls -lh cloudberry-pg16.tar.gz

      - name: Upload Build Artifact
        uses: actions/upload-artifact@v4
        with:
          name: cloudberry-pg16-binary
          path: cloudberry-pg16.tar.gz
          retention-days: 1

      - name: Create gpdemo Cluster
        env:
          SRC_DIR: ${{ github.workspace }}
        run: |
          set -eo pipefail

          {
            echo "=== gpdemo Cluster Creation ==="
            echo "Timestamp: $(date -u)"
            echo "Creating demo cluster..."

            # Source greenplum_path and create demo cluster
            if ! su - gpadmin -c "
              cd ${SRC_DIR}
              source ${INSTALL_PREFIX}/greenplum_path.sh
              make create-demo-cluster
            "; then
              echo "::error::Demo cluster creation failed"
              exit 1
            fi
            
            echo "Demo cluster created successfully"
          } 2>&1 | tee -a pg16-build-logs/details/cluster-creation.log

      - name: Generate Build Job Summary End
        if: always()
        run: |
          {
            echo ""
            echo "## Build Results"
            echo "- End Time: $(date -u +'%Y-%m-%d %H:%M:%S UTC')"
            echo "- Status: ${{ job.status }}"
          } >> "$GITHUB_STEP_SUMMARY"

      - name: Upload Build and Test Logs
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: pg16-build-logs-${{ env.BUILD_TIMESTAMP }}
          path: |
            pg16-build-logs/
          retention-days: ${{ env.LOG_RETENTION_DAYS }}

  ## ======================================================================
  ## Job: test
  ## ======================================================================

  test:
    name: ${{ matrix.test }}
    needs: [build, prepare-test-matrix]
    if: needs.prepare-test-matrix.outputs.should_run == 'true'
    runs-on: ubuntu-22.04
    timeout-minutes: 120
    strategy:
      fail-fast: false  # Continue with other tests if one fails
      matrix: ${{ fromJson(needs.prepare-test-matrix.outputs.test-matrix) }}

    container:
      image: apache/incubator-cloudberry:cbdb-build-rocky9-latest
      options: >-
        --privileged
        --user root
        --hostname cdw
        --shm-size=2gb
        --ulimit core=-1

    steps:
      - name: Checkout Apache Cloudberry (cbdb-postgres-merge branch)
        uses: actions/checkout@v4
        with:
          fetch-depth: 1
          submodules: true

      - name: Cloudberry Environment Initialization
        env:
          LOGS_DIR: test-logs-${{ matrix.test }}
        run: |
          set -eo pipefail
          
          # Initialize system
          if ! su - gpadmin -c "/tmp/init_system.sh"; then
            echo "::error::Container initialization failed"
            exit 1
          fi

          # Setup logging directories
          mkdir -p "${LOGS_DIR}/details"
          chown -R gpadmin:gpadmin .
          chmod -R 755 .
          chmod 777 "${LOGS_DIR}"

          # Clean up disk space
          df -kh /
          rm -rf /__t/*
          df -kh /

          # Log system information
          df -h | tee -a "${LOGS_DIR}/details/disk-usage.log"
          free -h | tee -a "${LOGS_DIR}/details/memory-usage.log"

          {
            echo "=== Environment Information ==="
            uname -a
            df -h
            free -h
            env
          } | tee -a "${LOGS_DIR}/details/environment.log"

          echo "SRC_DIR=${GITHUB_WORKSPACE}" | tee -a "$GITHUB_ENV"
          echo "LOGS_DIR=${LOGS_DIR}" | tee -a "$GITHUB_ENV"

      - name: "Generate Test Job Summary Start: ${{ matrix.test }}"
        run: |
          {
            echo "# Test: ${{ matrix.test }}"
            echo "## Configuration"
            echo "- Test Name: ${{ matrix.test }}"
            echo "- Start Time: $(date -u +'%Y-%m-%d %H:%M:%S UTC')"
            echo "- Optimizer: ${{ matrix.pg_settings.optimizer || 'default' }}"
            echo "- Num Segments: ${{ matrix.num_primary_mirror_pairs }}"
            echo ""
          } >> "$GITHUB_STEP_SUMMARY"

      - name: Prepare Installation Directory
        run: |
          set -eo pipefail
          
          # Clean and prepare installation directory
          sudo rm -rf ${INSTALL_PREFIX}
          sudo chmod a+w /usr/local
          mkdir -p ${INSTALL_PREFIX}/lib
          
          # Copy Xerces-C libraries
          sudo cp -v /usr/local/xerces-c/lib/libxerces-c.so \
                     /usr/local/xerces-c/lib/libxerces-c-3.*.so \
                     ${INSTALL_PREFIX}/lib
          
          sudo chown -R gpadmin:gpadmin ${INSTALL_PREFIX}

      - name: Download Build Artifact
        uses: actions/download-artifact@v4
        with:
          name: cloudberry-pg16-binary
          path: .

      - name: Install Cloudberry from Artifact
        run: |
          set -eo pipefail
          echo "Installing Cloudberry from artifact..."
          
          # Extract tarball to /usr/local
          sudo tar -xzf cloudberry-pg16.tar.gz -C /usr/local
          
          # Fix permissions
          sudo chown -R gpadmin:gpadmin ${INSTALL_PREFIX}
          
          echo "Cloudberry installed successfully"
          ${INSTALL_PREFIX}/bin/postgres --version

      - name: Configure PG16 for Cloudberry
        env:
          SRC_DIR: ${{ github.workspace }}
        run: |
          set -eo pipefail
          
          # Run configure as gpadmin to generate Makefiles for tests
          # We don't need to build, just configure
          su - gpadmin -c "
            cd ${SRC_DIR}
            export LD_LIBRARY_PATH=${INSTALL_PREFIX}/lib:${LD_LIBRARY_PATH}
            CFLAGS='-O0 -g3' ./configure \
              --prefix=${INSTALL_PREFIX} \
              --with-pgport=5432 \
              --enable-cassert \
              --with-perl \
              --with-python \
              --with-libxml \
              --with-openssl \
              --with-zstd \
              --enable-debug-extensions \
              --disable-gpcloud \
              --with-pythonsrc-ext \
              --with-gssapi \
              --without-icu \
              --with-libraries=${INSTALL_PREFIX}/lib \
              --with-includes=/usr/local/xerces-c/include
          " 2>&1 | tee -a ${LOGS_DIR}/details/configure.log

      - name: Create gpdemo Cluster
        env:
          SRC_DIR: ${{ github.workspace }}
        run: |
          set -eo pipefail
          
          # Create demo cluster with specified number of segments
          su - gpadmin -c "
            cd ${SRC_DIR}
            source ${INSTALL_PREFIX}/greenplum_path.sh
            NUM_PRIMARY_MIRROR_PAIRS=${{ matrix.num_primary_mirror_pairs }} make create-demo-cluster
          " 2>&1 | tee -a ${LOGS_DIR}/details/cluster-creation.log

      - name: Configure PostgreSQL Settings
        env:
          SRC_DIR: ${{ github.workspace }}
        run: |
          set -eo pipefail

          # Apply PostgreSQL settings from matrix configuration
          su - gpadmin -c "
            source ${INSTALL_PREFIX}/greenplum_path.sh
            source ${SRC_DIR}/gpAux/gpdemo/gpdemo-env.sh
            
            # Set optimizer if specified
            OPTIMIZER='${{ matrix.pg_settings.optimizer }}'
            if [ -n \"\$OPTIMIZER\" ]; then
              echo \"Setting optimizer to: \$OPTIMIZER\"
              gpconfig -c optimizer -v \$OPTIMIZER
              gpstop -u
            fi
          " 2>&1 | tee -a ${LOGS_DIR}/details/pg-settings.log

      - name: Run Tests
        env:
          SRC_DIR: ${{ github.workspace }}
        run: |
          set -eo pipefail

          # Run each make target from the test configuration
          MAKE_CONFIGS='${{ join(matrix.make_configs, ' ') }}'
          
          su - gpadmin -c "
            source ${INSTALL_PREFIX}/greenplum_path.sh
            source ${SRC_DIR}/gpAux/gpdemo/gpdemo-env.sh
            
            cd ${SRC_DIR}
            
            echo '=== Running Tests for ${{ matrix.test }} ==='
            echo 'Make configurations: ${MAKE_CONFIGS}'
            
            # Parse and run each make config
            for config in ${MAKE_CONFIGS}; do
              DIR=\$(echo \$config | cut -d: -f1)
              TARGET=\$(echo \$config | cut -d: -f2)
              
              echo \"\"
              echo \"Running: make -C \$DIR \$TARGET\"
              echo \"\"
              
              if ! make -C \$DIR \$TARGET; then
                echo \"::error::Test failed: \$DIR:\$TARGET\"
                exit 1
              fi
            done
            
            echo \"\"
            echo \"All tests completed successfully\"
          " 2>&1 | tee -a ${LOGS_DIR}/details/test-run.log

      - name: Collect Test Results
        if: always()
        env:
          SRC_DIR: ${{ github.workspace }}
        run: |
          set -eo pipefail
          
          # Collect regression diffs and test results
          mkdir -p ${LOGS_DIR}/regression-results
          
          # Find and copy all regression.diffs files
          find ${SRC_DIR} -name "regression.diffs" -exec cp --parents {} ${LOGS_DIR}/regression-results/ \; || true
          
          # Find and copy all regression.out files
          find ${SRC_DIR} -name "regression.out" -exec cp --parents {} ${LOGS_DIR}/regression-results/ \; || true
          
          echo "Test results collected"

      - name: Parse Test Results
        if: always()
        env:
          SRC_DIR: ${{ github.workspace }}
        run: |
          echo "=== Parsing Test Results for ${{ matrix.test }} ==="
          
          # Find regression.out and parse results
          REGRESSION_OUT=$(find ${SRC_DIR} -name "regression.out" -type f | head -n 1)
          
          if [[ -n "$REGRESSION_OUT" && -f "$REGRESSION_OUT" ]]; then
            echo "Found regression.out at: $REGRESSION_OUT"
            echo ""
            echo "=== Test Summary ==="
            # Extract the summary lines from regression.out
            grep -E "^(#|test |ok |FAILED|not ok)" "$REGRESSION_OUT" | tail -50 || true
            echo ""
            
            # Count passed/failed tests
            TOTAL=$(grep -c "^test " "$REGRESSION_OUT" 2>/dev/null || echo "0")
            PASSED=$(grep -c "^ok " "$REGRESSION_OUT" 2>/dev/null || echo "0")
            FAILED=$(grep -c "^not ok\|FAILED" "$REGRESSION_OUT" 2>/dev/null || echo "0")
            
            echo "=== Statistics ==="
            echo "Total tests: $TOTAL"
            echo "Passed: $PASSED"
            echo "Failed: $FAILED"
            
            # Add to job summary
            {
              echo "### Test Results: ${{ matrix.test }}"
              echo "| Metric | Count |"
              echo "|--------|-------|"
              echo "| Total | $TOTAL |"
              echo "| Passed | $PASSED |"
              echo "| Failed | $FAILED |"
            } >> "$GITHUB_STEP_SUMMARY"
          else
            echo "No regression.out file found"
          fi

      - name: Check and Display Regression Diffs
        if: always()
        env:
          SRC_DIR: ${{ github.workspace }}
        run: |
          echo "=== Checking for Regression Diffs ==="
          
          # Search for regression.diffs recursively
          DIFF_FILES=$(find ${SRC_DIR} -type f -name "regression.diffs" 2>/dev/null)
          
          if [[ -n "$DIFF_FILES" ]]; then
            echo "$DIFF_FILES" | while read -r diff_file; do
              if [[ -s "$diff_file" ]]; then
                echo ""
                echo "=========================================="
                echo "Found regression.diffs at: $diff_file"
                echo "=========================================="
                echo ""
                cat "$diff_file"
                echo ""
                echo "=========================================="
              fi
            done
            
            # Add diff summary to job summary
            {
              echo ""
              echo "### Regression Diffs Found"
              echo "The following test differences were detected:"
              echo "\`\`\`"
              # Show first 100 lines of first diff file
              head -100 $(echo "$DIFF_FILES" | head -n 1) || true
              echo "\`\`\`"
              echo ""
              echo "*Full diffs available in uploaded artifacts*"
            } >> "$GITHUB_STEP_SUMMARY"
          else
            echo "No regression.diffs file found - all tests passed!"
          fi

      - name: "Generate Test Job Summary End: ${{ matrix.test }}"
        if: always()
        run: |
          {
            echo ""
            echo "## Test Results"
            echo "- End Time: $(date -u +'%Y-%m-%d %H:%M:%S UTC')"
            echo "- Status: ${{ job.status }}"
          } >> "$GITHUB_STEP_SUMMARY"

      - name: Upload Test Logs
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: test-logs-${{ matrix.test }}-${{ needs.build.outputs.build_timestamp }}
          path: |
            test-logs-${{ matrix.test }}/
            **/regression.out
            **/regression.diffs
            **/results/
          retention-days: ${{ env.LOG_RETENTION_DAYS }}

      - name: Upload Regression Logs on Failure
        if: failure() || cancelled()
        uses: actions/upload-artifact@v4
        with:
          name: regression-logs-${{ matrix.test }}-${{ needs.build.outputs.build_timestamp }}
          path: |
            **/regression.out
            **/regression.diffs
            **/results/
            gpAux/gpdemo/datadirs/qddir/demoDataDir-1/log/
            gpAux/gpdemo/datadirs/dbfast1/demoDataDir0/log/
            gpAux/gpdemo/datadirs/dbfast2/demoDataDir1/log/
            gpAux/gpdemo/datadirs/dbfast3/demoDataDir2/log/
          retention-days: ${{ env.LOG_RETENTION_DAYS }}

  ## ======================================================================
  ## Job: report
  ## ======================================================================

  report:
    name: Report Results
    needs: [build]
    runs-on: ubuntu-22.04
    if: always()

    steps:
      - name: Generate Final Report
        run: |
          {
            echo "# PostgreSQL 16 Merge Validation Report"
            echo ""
            echo "## Job Status Summary"
            echo "- Build: ${{ needs.build.result }}"
            echo ""
            echo "**Note**: Test matrix is currently empty as installcheck support is being fixed."
            echo "Tests will be added incrementally as features become available."
            echo ""
            
            if [ "${{ needs.build.result }}" == "success" ]; then
              echo "✅ Build completed successfully!"
              echo ""
              echo "The cbdb-postgres-merge branch builds correctly and gpdemo cluster can be created."
            else
              echo "❌ Build failed!"
              echo ""
              echo "Please check the build logs for details."
            fi
          } >> "$GITHUB_STEP_SUMMARY"