sep-tests.yml

name: sep-tests
permissions:
  contents: read

on:
  push:
    branches:
      - 'main'
      - '*.*.*'
    paths-ignore:
      - 'docs/**'
      - '*.md'
      - '*.mdx'
  # The only difference between pull_request and pull_request_target is the context in which the workflow runs:
  # — pull_request_target workflows use the workflow files from the default branch, and secrets are available.
  # — pull_request workflows use the workflow files from the pull request branch, and secrets are unavailable.
  # We use pull_request_target so that GITEE_TOKEN (and other repo secrets) are available when building
  # ragflow:nightly from a fork PR, while still running the full test suite on PRs labeled with `ci`.
  pull_request:
    types: [synchronize, labeled]
    paths-ignore:
      - 'docs/**'
      - '*.md'
      - '*.mdx'
  schedule:
    - cron: '0 16 * * *'  # This schedule runs every 16:00:00Z(00:00:00+08:00)

# https://docs.github.com/en/actions/using-jobs/using-concurrency
concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}${{ github.event.action == 'labeled' && github.event.label.name != 'ci' && format('-ignore-{0}', github.run_id) || '' }}
  cancel-in-progress: true

# infiniflow/onnxruntime_go is a fresh org mirror that is not yet present in the
# public Go checksum database / proxy. Mark it private so Go skips the proxy and
# sumdb for this path and resolves it directly from GitHub (Go's default
# GOPROXY keeps the ",direct" fallback). Required for the in-process DeepDoc
# backend to build against github.com/infiniflow/onnxruntime_go.
env:
  GOPRIVATE: github.com/infiniflow/onnxruntime_go

jobs:
  ragflow_preflight:
    name: ragflow_preflight
    # https://docs.github.com/en/actions/using-jobs/using-conditions-to-control-job-execution
    # https://github.com/orgs/community/discussions/26261
    if: ${{ github.event_name != 'pull_request' && github.event_name != 'pull_request_target' || (github.event.pull_request.draft == false && contains(github.event.pull_request.labels.*.name, 'ci') && (github.event.action != 'labeled' || github.event.label.name == 'ci')) }}
    permissions:
      actions: write
      contents: read
    runs-on: [ "self-hosted", "ragflow-test" ]
    outputs:
      http_api_test_level: ${{ steps.test_level.outputs.http_api_test_level }}
      has_go_changes: ${{ steps.detect_changes.outputs.has_go_changes }}
      has_python_changes: ${{ steps.detect_changes.outputs.has_python_changes }}
      has_web_changes: ${{ steps.detect_changes.outputs.has_web_changes }}
      has_native_deepdoc_changes: ${{ steps.detect_changes.outputs.has_native_deepdoc_changes }}
      api_proxy_schemes: ${{ steps.detect_changes.outputs.api_proxy_schemes }}
    steps:
      - name: Ensure workspace ownership
        run: |
          echo "Workflow triggered by ${{ github.event_name }}"
          echo "chown -R ${USER} ${GITHUB_WORKSPACE}" && sudo chown -R ${USER} ${GITHUB_WORKSPACE}

      # https://github.com/actions/checkout/issues/1781
      - name: Check out code
        uses: actions/checkout@v6
        with:
          ref: ${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && format('refs/pull/{0}/merge', github.event.pull_request.number) || github.sha }}
          fetch-depth: 0
          fetch-tags: true
          allow-unsafe-pr-checkout: true

      - name: Check workflow duplication
        if: ${{ !cancelled() && !failure() }}
        run: |
          if [[ ${GITHUB_EVENT_NAME} != "pull_request" && ${GITHUB_EVENT_NAME} != "pull_request_target" && ${GITHUB_EVENT_NAME} != "schedule" ]]; then
            HEAD=$(git rev-parse HEAD)
            # Find a PR that introduced a given commit
            gh auth login --with-token <<< "${{ secrets.GITHUB_TOKEN }}"
            PR_NUMBER=$(gh pr list --search ${HEAD} --state merged --json number --jq .[0].number)
            echo "HEAD=${HEAD}"
            echo "PR_NUMBER=${PR_NUMBER}"
            if [[ -n "${PR_NUMBER}" ]]; then
              PR_SHA_FP=${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/PR_${PR_NUMBER}
              if [[ -f "${PR_SHA_FP}" ]]; then
                read -r PR_SHA PR_RUN_ID < "${PR_SHA_FP}"
                # Calculate the hash of the current workspace content
                HEAD_SHA=$(git rev-parse HEAD^{tree})
                if [[ "${HEAD_SHA}" == "${PR_SHA}" ]]; then
                  echo "Cancel myself since the workspace content hash is the same with PR #${PR_NUMBER} merged. See ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${PR_RUN_ID} for details."
                  gh run cancel ${GITHUB_RUN_ID}
                  while true; do
                    status=$(gh run view ${GITHUB_RUN_ID} --json status -q .status)
                    [ "${status}" = "completed" ] && break
                    sleep 5
                  done
                  exit 1
                fi
              fi
            fi
          elif [[ ${GITHUB_EVENT_NAME} == "pull_request" || ${GITHUB_EVENT_NAME} == "pull_request_target" ]]; then
            PR_NUMBER=${{ github.event.pull_request.number }}
            PR_SHA_FP=${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/PR_${PR_NUMBER}
            # Calculate the hash of the current workspace content
            PR_SHA=$(git rev-parse HEAD^{tree})
            echo "PR #${PR_NUMBER} workspace content hash: ${PR_SHA}"
            mkdir -p ${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}
            echo "${PR_SHA} ${GITHUB_RUN_ID}" > ${PR_SHA_FP}
          fi
          ARTIFACTS_DIR=${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/${GITHUB_RUN_ID}
          echo "ARTIFACTS_DIR=${ARTIFACTS_DIR}" >> ${GITHUB_ENV}
          rm -rf ${ARTIFACTS_DIR} && mkdir -p ${ARTIFACTS_DIR}

#      - name: Check comments of changed Python files
#        if: ${{ false }}
#        run: |
#          if [[ ${{ github.event_name }} == 'pull_request' || ${{ github.event_name }} == 'pull_request_target' ]]; then
#            CHANGED_FILES=$(git diff --name-only ${{ github.event.pull_request.base.sha }}...${{ github.event.pull_request.head.sha }} \
#              | grep -E '\.(py)$' || true)
#
#            if [ -n "$CHANGED_FILES" ]; then
#              echo "Check comments of changed Python files with check_comment_ascii.py"
#
#              readarray -t files <<< "$CHANGED_FILES"
#              HAS_ERROR=0
#
#              for file in "${files[@]}"; do
#                if [ -f "$file" ]; then
#                  if python3 check_comment_ascii.py "$file"; then
#                    echo "✅ $file"
#                  else
#                    echo "❌ $file"
#                    HAS_ERROR=1
#                  fi
#                fi
#              done
#
#              if [ $HAS_ERROR -ne 0 ]; then
#                exit 1
#              fi
#            else
#              echo "No Python files changed"
#            fi
#          fi

      # lefthook's web-checks hook resolves oxlint/oxfmt through
      # tools/hooks/web_tools.sh, which needs node and npm on PATH.
      - name: Set up Node.js
        uses: actions/setup-node@v4
        with:
          node-version: "22"

      - name: Run Lefthook on changed files
        run: |
          set -euo pipefail
          if [[ "${GITHUB_EVENT_NAME}" == "pull_request" || "${GITHUB_EVENT_NAME}" == "pull_request_target" ]]; then
            changed_files=$(mktemp)
            trap 'rm -f "$changed_files"' EXIT
            git diff --name-only -z ${{ github.event.pull_request.base.sha }}...${{ github.event.pull_request.head.sha }} \
              | while IFS= read -r -d '' file; do
                  if [[ -f "$file" ]]; then
                    printf '%s\0' "$file"
                  fi
                done > "$changed_files"
            echo "Changed files to run lefthook on:"
            if [[ -s "$changed_files" ]]; then
              while IFS= read -r -d '' file; do
                printf '  %q\n' "$file"
              done < "$changed_files"
            else
              echo "  (none — lefthook will be a no-op)"
            fi
            # LEFTHOOK_CHECK_ONLY=1 makes the pre-commit jobs verify without
            # applying --fix or `git add`, so CI only checks and reports
            # failures instead of rewriting the working tree.
            LEFTHOOK_CHECK_ONLY=1 lefthook run pre-commit --files-from-stdin --no-auto-install < "$changed_files"
            # web-checks is a separate hook because it is not wired into local
            # commits: it installs a Node toolchain that only CI has a reason to
            # pay for. It is always check-only, so it needs no CHECK_ONLY flag.
            lefthook run web-checks --files-from-stdin --no-auto-install < "$changed_files"
          fi

      - name: Set test level
        id: test_level
        run: |
          set -euo pipefail
          if [[ ${GITHUB_EVENT_NAME} == "schedule" ]]; then
            export HTTP_API_TEST_LEVEL=p3
          else
            export HTTP_API_TEST_LEVEL=p2
          fi
          echo "HTTP_API_TEST_LEVEL=${HTTP_API_TEST_LEVEL}" >> ${GITHUB_ENV}
          echo "http_api_test_level=${HTTP_API_TEST_LEVEL}" >> ${GITHUB_OUTPUT}

      - name: Detect changed file types
        id: detect_changes
        run: |
          set -euo pipefail
          has_go=false
          has_python=false
          has_web=false
          has_native_deepdoc=false

          if [[ "${GITHUB_EVENT_NAME}" == "schedule" ]]; then
            has_go=true
            has_python=true
            has_web=true
            has_native_deepdoc=true
          else
            if [[ "${GITHUB_EVENT_NAME}" == "pull_request" || "${GITHUB_EVENT_NAME}" == "pull_request_target" ]]; then
              CHANGED_FILES=$(git diff --name-only ${{ github.event.pull_request.base.sha }}...${{ github.event.pull_request.head.sha }})
            else
              BASE_SHA="${{ github.event.before }}"
              if [[ -n "${BASE_SHA}" && ! "${BASE_SHA}" =~ ^0+$ ]]; then
                CHANGED_FILES=$(git diff --name-only "${BASE_SHA}" "${GITHUB_SHA}")
              else
                CHANGED_FILES=$(git diff --name-only HEAD~1 HEAD)
              fi
            fi

            while IFS= read -r file; do
              case "$file" in
                # Docker, test, SDK, and workflow changes can alter either server path.
                # build.sh also controls the C++/native build, so it must trigger native backend.
                Dockerfile|Dockerfile_ci|docker/**|.github/workflows/**|sdk/**|build.sh)
                  has_go=true
                  has_python=true
                  has_native_deepdoc=true
                  ;;
                test/unit_test/**)
                  has_python=true
                  ;;
                test/**)
                  has_go=true
                  has_python=true
                  ;;
                # DeepDoc native backend (Go inference layer, C++ static lib, pinned testdata).
                # Must come BEFORE the generic "*.go"/"*.py" rules, otherwise deepdoc/*.go
                # would be caught by "*.go" first and never set has_native_deepdoc.
                internal/deepdoc/**|internal/binding/cpp/**|internal/deepdoc/native/testdata.ref)
                  has_go=true
                  has_native_deepdoc=true
                  ;;
                *.go|go.mod|go.sum)                       has_go=true ;;
                *.py|pyproject.toml|requirements*.txt)    has_python=true ;;
                web/*)                                    has_web=true ;;
              esac
            done <<< "$CHANGED_FILES"

            if [[ "$has_go" == "false" && "$has_python" == "false" && "$has_web" == "false" ]]; then
              has_go=true
              has_python=true
              has_web=true
              has_native_deepdoc=true
            fi
          fi

          if [[ "$has_go" == "true" && "$has_python" == "true" ]]; then
            api_proxy_schemes='["go","python"]'
          elif [[ "$has_go" == "true" ]]; then
            api_proxy_schemes='["go"]'
          elif [[ "$has_python" == "true" ]]; then
            api_proxy_schemes='["python"]'
          elif [[ "$has_web" == "true" ]]; then
            api_proxy_schemes='["web"]'
          else
            api_proxy_schemes='["go","python"]'
          fi

          echo "has_go_changes=${has_go}"        >> $GITHUB_OUTPUT
          echo "has_python_changes=${has_python}" >> $GITHUB_OUTPUT
          echo "has_web_changes=${has_web}"       >> $GITHUB_OUTPUT
          echo "has_native_deepdoc_changes=${has_native_deepdoc}" >> $GITHUB_OUTPUT
          echo "api_proxy_schemes=${api_proxy_schemes}" >> $GITHUB_OUTPUT
          echo "Go: ${has_go}, Python: ${has_python}, Web: ${has_web}, Native DeepDoc: ${has_native_deepdoc}, proxy schemes: ${api_proxy_schemes}"

      - name: Prepare Python test environment
        if: steps.detect_changes.outputs.has_python_changes == 'true'
        run: |
          git config --global url."https://oauth2:${GITEE_TOKEN}@gitee.com/".insteadOf "https://gitee.com/"
          uv sync --python 3.13 --group test --frozen
          uv pip install -e sdk/python

      - name: Provision NLTK data for unit tests
        if: steps.detect_changes.outputs.has_python_changes == 'true'
        run: |
          set -euo pipefail

          # The self-hosted runner stores NLTK packages as root-owned zip files.
          # Reading them directly as the runner user fails with EACCES, while
          # downloading replacements is blocked by the runner's SSRF policy.
          # Copy the three required archives into the workspace with sudo, then
          # unpack and validate them as the runner user.
          NLTK_TARGET="${GITHUB_WORKSPACE}/nltk_data"
          rm -rf "${NLTK_TARGET}"
          mkdir -p "${NLTK_TARGET}/tokenizers" "${NLTK_TARGET}/corpora"

          copy_nltk_archive() {
            local relative_path="$1"
            local source=""
            for root in /usr/share/nltk_data /usr/local/share/nltk_data; do
              if sudo test -f "${root}/${relative_path}"; then
                source="${root}/${relative_path}"
                break
              fi
            done
            if [ -z "${source}" ]; then
              echo "Missing required NLTK archive: ${relative_path}" >&2
              return 1
            fi
            echo "Copying ${source}"
            sudo cp "${source}" "${NLTK_TARGET}/${relative_path}"
            sudo chown "$(id -u):$(id -g)" "${NLTK_TARGET}/${relative_path}"
            chmod 0644 "${NLTK_TARGET}/${relative_path}"
          }

          copy_nltk_archive tokenizers/punkt_tab.zip
          copy_nltk_archive tokenizers/punkt.zip
          copy_nltk_archive corpora/wordnet.zip

          NLTK_DATA="${NLTK_TARGET}" uv run python - <<'PYEOF'
          import os
          import zipfile

          root = os.environ["NLTK_DATA"]
          archives = (
              ("tokenizers", "punkt_tab"),
              ("tokenizers", "punkt"),
              ("corpora", "wordnet"),
          )
          for category, package in archives:
              archive = os.path.join(root, category, f"{package}.zip")
              destination = os.path.join(root, category)
              with zipfile.ZipFile(archive) as package_zip:
                  bad_member = package_zip.testzip()
                  if bad_member is not None:
                      raise RuntimeError(f"Corrupt NLTK archive {archive}: {bad_member}")
                  members = [name for name in package_zip.namelist() if not name.startswith("__MACOSX/")]
                  prefix = f"{package}/"
                  if not members or not all(name == package or name.startswith(prefix) for name in members):
                      raise RuntimeError(f"Unexpected layout in {archive}: expected entries below {prefix}")
                  package_zip.extractall(destination)
              print(f"Unpacked {archive} -> {os.path.join(destination, package)}")
          PYEOF

          echo "NLTK_DATA=${NLTK_TARGET}" >> "${GITHUB_ENV}"

          # Validate real tokenizer/lemmatizer calls, not only directory names.
          # The command exits non-zero if any resource is absent or malformed.
          NLTK_DATA="${NLTK_TARGET}" uv run python - <<'PYEOF'
          import nltk
          from nltk.corpus import wordnet
          from nltk.tokenize import word_tokenize

          for resource in ("tokenizers/punkt_tab", "tokenizers/punkt", "corpora/wordnet"):
              print(f"[OK] {resource} -> {nltk.data.find(resource)}")
          assert word_tokenize("NLTK data is ready.")
          assert wordnet.synsets("document")
          print("NLTK tokenizer and WordNet smoke checks passed")
          PYEOF

      - name: Run unit test
        if: steps.detect_changes.outputs.has_python_changes == 'true'
        run: |
          source .venv/bin/activate
          which pytest || echo "pytest not in PATH"
          echo "Start to run unit test"
          python3 run_tests.py -i



  ragflow_tests_infinity:
    name: ragflow_tests_infinity (${{ matrix.api_proxy_scheme }})
    needs: ragflow_preflight
    if: ${{ github.event_name != 'pull_request' && github.event_name != 'pull_request_target' || (github.event.pull_request.draft == false && contains(github.event.pull_request.labels.*.name, 'ci') && (github.event.action != 'labeled' || github.event.label.name == 'ci')) }}
    runs-on: [ "self-hosted", "ragflow-test" ]
    strategy:
      fail-fast: false
      matrix:
        api_proxy_scheme: ${{ fromJSON(needs.ragflow_preflight.outputs.api_proxy_schemes) }}
    env:
      DOC_ENGINE: infinity
      API_PROXY_SCHEME: ${{ matrix.api_proxy_scheme }}
      RAGFLOW_IMAGE: infiniflow/ragflow:${{ github.run_id }}-infinity-${{ matrix.api_proxy_scheme }}
      HTTP_API_TEST_LEVEL: ${{ needs.ragflow_preflight.outputs.http_api_test_level }}
      HAS_GO: ${{ needs.ragflow_preflight.outputs.has_go_changes }}
      HAS_PYTHON: ${{ needs.ragflow_preflight.outputs.has_python_changes }}
      HAS_WEB: ${{ needs.ragflow_preflight.outputs.has_web_changes }}
    steps:
      - name: Ensure workspace ownership
        if: always()
        run: |
          echo "Workflow triggered by ${{ github.event_name }}"
          echo "chown -R ${USER} ${GITHUB_WORKSPACE}" && sudo chown -R ${USER} ${GITHUB_WORKSPACE}

      - name: Check out code
        if: always()
        uses: actions/checkout@v6
        with:
          ref: ${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && format('refs/pull/{0}/merge', github.event.pull_request.number) || github.sha }}
          fetch-depth: 0
          fetch-tags: true
          allow-unsafe-pr-checkout: true

      - name: Build ragflow go server
        if: env.API_PROXY_SCHEME == 'go'
        run: |
          set -euo pipefail
          BUILDER_CONTAINER=ragflow_build_${GITHUB_RUN_ID}_${DOC_ENGINE}_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ')
          cleanup_builder() {
            if [[ -n "${BUILDER_CONTAINER:-}" ]]; then
              sudo docker rm -f -v "${BUILDER_CONTAINER}" >/dev/null 2>&1 || true
            fi
          }
          trap cleanup_builder EXIT

          TZ=${TZ:-$(readlink -f /etc/localtime | awk -F '/zoneinfo/' '{print $2}')}
          sudo docker run --privileged -d --name "${BUILDER_CONTAINER}" \
            -e TZ="${TZ}" \
            -e UV_INDEX=https://mirrors.aliyun.com/pypi/simple \
            -v "${PWD}:/ragflow" \
            -v "${PWD}/internal/binding/cpp/resource:/usr/share/infinity/resource" \
            infiniflow/infinity_builder:ubuntu22_clang20
          sudo docker exec "${BUILDER_CONTAINER}" bash -c 'git config --global safe.directory "*" && cd /ragflow && ./build.sh --cpp'
          ./build.sh --go

      - name: Run Go unit tests
        if: env.API_PROXY_SCHEME == 'go'
        # Runs after `./build.sh --go`, which guarantees the C++ static
        # library (librag_tokenizer_c_api.a) is present on disk. The Go
        # test binaries link against it transitively through
        # `internal/binding`, so running `go test` before the C++ build
        # fails the link step.
        #
        # Excludes packages whose tests fail for environmental reasons
        # unrelated to the diff:
        #   - internal/tokenizer is split: the pure-Go BPE loader tests
        #     (bpe_loader_test.go) run in the default tier; the C++ binding /
        #     dict-dependent tests (tokenizer_test.go,
        #     tokenizer_concurrent_test.go) and the on-disk anchor test
        #     (bpe_loader_anchors_test.go) are tagged `manual` and need the
        #     docker builder's /usr/share/infinity/resource, so they stay out
        #     of the default run.
        run: |
          set -euo pipefail

          # Provide the cl100k BPE table for the offline tokenizer loader.
          # The loader reads <repo>/ragflow_deps/cl100k_base.tiktoken (or
          # TIKTOKEN_CACHE_DIR). Try to fetch it directly from the upstream
          # openai blob first (small ~1.6MB, reachable from the CI runner — the
          # pre-PR code downloaded it on the fly, so the network path is known
          # good). If the direct fetch fails (e.g. GFW), fall back to extracting
          # it from the local infiniflow/ragflow_deps image *if already present*
          # on the runner (no `docker pull`, since that huge image is first
          # fetched at the later "Build ragflow:nightly" step). Best-effort: if
          # neither source works, the loader still fails loudly (no silent
          # 0-token degradation).
          # tiktoken is already installed in runner
          mkdir -p ragflow_deps
          cp /opt/ragflow_deps/cl100k_base.tiktoken ragflow_deps/cl100k_base.tiktoken || true
          if [ ! -f ragflow_deps/cl100k_base.tiktoken ]; then
            mkdir -p ragflow_deps
            if command -v curl >/dev/null 2>&1; then
              if curl -fsSL -o ragflow_deps/cl100k_base.tiktoken \
                https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken; then
                echo "tiktoken: cl100k table fetched via curl from openai blob"
              else
                echo "tiktoken: curl fetch failed (network/GFW); will try cached image"
              fi
            fi
            if [ ! -f ragflow_deps/cl100k_base.tiktoken ] && docker image inspect infiniflow/ragflow_deps:latest >/dev/null 2>&1; then
              CID=$(docker create infiniflow/ragflow_deps:latest true) || true
              if [ -n "${CID:-}" ]; then
                if docker cp "$CID":/cl100k_base.tiktoken ragflow_deps/cl100k_base.tiktoken 2>/dev/null; then
                  echo "tiktoken: cl100k table copied from cached infiniflow/ragflow_deps image"
                fi
                docker rm -f "$CID" >/dev/null 2>&1 || true
              fi
            fi
          fi
          if [ -f ragflow_deps/cl100k_base.tiktoken ]; then
            echo "tiktoken: cl100k table provisioned ($(wc -c < ragflow_deps/cl100k_base.tiktoken) bytes)"
          else
            echo "tiktoken: cl100k table NOT provisioned — offline loader will fail loudly"
          fi

          PKGS=$(go list ./... 2>/dev/null \
            | grep -v '/internal/storage$' \
            | grep -v '/internal/handler$' || true)
          if [ -z "$PKGS" ]; then
            ./build.sh --test
          else
            ./build.sh --test -- $PKGS
          fi

      - name: Run ORT deps-download regression guard
        if: env.API_PROXY_SCHEME == 'go'
        # Pure-Python, local, needs no API keys or model downloads; guards the
        # ONNX Runtime extraction logic the in-process (Go) DeepDoc backend build
        # depends on (notably the version-bump path C2 hardened). Run it in
        # isolation so the test/testcases conftest (which requires
        # SILICONFLOW_API_KEY) cannot fail the collection.
        run: |
          set -euo pipefail
          uv sync --python 3.13 --group test --frozen
          uv run pytest ragflow_deps/test_download_go_deps.py -q

      - name: Build ragflow:nightly
        run: |
          set -euo pipefail
          RUNNER_WORKSPACE_PREFIX=${RUNNER_WORKSPACE_PREFIX:-${HOME}}
          BUILD_LOCK_FILE=${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/docker-build.lock
          mkdir -p "$(dirname "${BUILD_LOCK_FILE}")"
          # Pass the Gitee personal access token to the Docker build via --build-arg GITEE_TOKEN.
          # Dockerfile_go / Dockerfile_ci read it from ARG GITEE_TOKEN (NOT a BuildKit secret),
          # so the runner must expose GITEE_TOKEN as an environment variable (e.g. via the
          # runner's docker-compose `environment:` mapping sourced from its .env file).
          # Pick the Dockerfile matching the API proxy scheme:
          #   - go  -> Dockerfile_go  (Go-native image)
          #   - *   -> Dockerfile_ci  (Python/legacy image)
          if [ "${API_PROXY_SCHEME}" = "go" ]; then
            DOCKERFILE=Dockerfile_go
          else
            DOCKERFILE=Dockerfile_ci
          fi
          (
            flock -w 10800 9 || { echo "Timed out waiting for the shared Docker build slot" >&2; exit 1; }
            echo "Acquired Docker build slot for ${DOC_ENGINE}/${API_PROXY_SCHEME} (dockerfile=${DOCKERFILE})"
            ##sudo docker pull ubuntu:24.04
            sudo DOCKER_BUILDKIT=1 docker build --build-arg NEED_MIRROR=1 --build-arg HTTPS_PROXY=${HTTPS_PROXY} --build-arg HTTP_PROXY=${HTTP_PROXY} --build-arg GITEE_TOKEN=${GITEE_TOKEN} -f "${DOCKERFILE}" -t ${RAGFLOW_IMAGE} .
          ) 9>"${BUILD_LOCK_FILE}"

      - name: Prepare Python test environment
        if: env.API_PROXY_SCHEME != 'web'
        run: |
          git config --global url."https://oauth2:${GITEE_TOKEN}@gitee.com/".insteadOf "https://gitee.com/"
          uv sync --python 3.13 --group test --frozen
          uv pip install -e sdk/python

      - name: Prepare function test environment
        working-directory: docker
        run: |
          set -euo pipefail
          sudo chmod 1777 /tmp
          ## ss already installed
          ##sudo apt update && sudo apt install -y iproute2
          RUNNER_WORKSPACE_PREFIX=${RUNNER_WORKSPACE_PREFIX:-${HOME}}
          COMPOSE_PROJECT_NAME="${GITHUB_RUN_ID}-${DOC_ENGINE}-${API_PROXY_SCHEME}"
          echo "COMPOSE_PROJECT_NAME=${COMPOSE_PROJECT_NAME}" >> ${GITHUB_ENV}
          echo "RAGFLOW_CONTAINER=${COMPOSE_PROJECT_NAME}-ragflow-cpu-1" >> ${GITHUB_ENV}
          ARTIFACTS_DIR=${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/${GITHUB_RUN_ID}/${DOC_ENGINE}/${API_PROXY_SCHEME}
          echo "ARTIFACTS_DIR=${ARTIFACTS_DIR}" >> ${GITHUB_ENV}
          rm -rf "${ARTIFACTS_DIR}" && mkdir -p "${ARTIFACTS_DIR}"

          # Determine runner number (default to 1 if not found)
               RUNNER_NUM=$(sudo docker inspect $(hostname) --format '{{index .Config.Labels "com.docker.compose.container-number"}}' 2>/dev/null || true)
               if [[ -z "${RUNNER_NUM:-}" ]]; then
                 # GitHub self-hosted runners always expose RUNNER_NAME; hash it to spread candidates.
                 if [[ -n "${RUNNER_NAME:-}" ]]; then
                   RUNNER_NUM=$(( $(printf '%d' 0x$(echo -n "${RUNNER_NAME}" | md5sum | cut -c1-4)) % 1000 ))
                 else
                   RUNNER_NUM=$(( GITHUB_RUN_ID % 1000 ))
                 fi
               fi

          # Engine-specific offset partitions keep concurrent engine jobs from
          # choosing the same host ports when they land on the same self-hosted runner.
          # A lock plus reservation file closes the check/start race between parallel jobs.
               PORT_BASES=(1200 1201 23817 23820 5432 5455 9000 9001 6379 6380 6601 9380 9381 9382 9384 9383 9385 80 443 4222 8222 9900 8123)
               PARTITION_SIZE=6000
               case "${DOC_ENGINE}" in
                 elasticsearch) PARTITION_BASE=1000 ;;
                 infinity) PARTITION_BASE=31000 ;;
                 *) echo "Unsupported DOC_ENGINE=${DOC_ENGINE}" >&2; exit 1 ;;
               esac
               # Runners run inside docker containers that share the host docker daemon
               # (/var/run/docker.sock), so neither /tmp nor `ss` can coordinate across runners:
               # each container has its own /tmp and its own network namespace, and `ss` cannot
               # see host ports bound by other runners' compose stacks. Use the shared daemon for
               # both primitives:
               #   - per-port mutex = for each host port (base+offset) we intend to bind, create
               #     a docker network named ragflow-ci-port-<port> on the shared daemon. docker
               #     network create is NOT idempotent (fails on name conflict), so only one runner
               #     can claim a given port. This is per-PORT, not per-offset, because different
               #     (base, offset) pairs can produce the same host port (e.g. 6380+36497 ==
               #     9380+33497 == 42877); a per-offset guard cannot prevent that collision.
               #   - port availability = docker ps published ports (sees every container on the
               #     shared daemon regardless of which runner started it).
               # No file-system sharing required, no host netns required.

               cleanup_stale_port_locks() {
                 local now stale_after net ts before after
                 now=$(date -u +%s)
                 stale_after=$((60 * 60))
                 before=$(sudo docker network ls --filter "label=ragflow-ci-created" --format '{{.Name}}' | wc -l)
                 for net in $(sudo docker network ls --filter "label=ragflow-ci-created" --format '{{.Name}}'); do
                   ts=$(sudo docker network inspect --format '{{ index .Labels "ragflow-ci-created" }}' "${net}" 2>/dev/null || true)
                   # Remove if timestamp is non-numeric (legacy network from a previous code
                   # version) or if it is numeric and older than stale_after.
                   if [[ ! "${ts}" =~ ^[0-9]+$ ]] || (( now - ts > stale_after )); then
                     sudo docker network rm "${net}" >/dev/null 2>&1 || true
                   fi
                 done
                 after=$(sudo docker network ls --filter "label=ragflow-ci-created" --format '{{.Name}}' | wc -l)
                 echo "  cleanup_stale_port_locks: ${before} -> ${after} guards (TTL=${stale_after}s)" >&2
               }

               reserve_port_offset() {
                 local attempt candidate base port guard g ok reject_reason create_err guard_subnet
                 local -a guards
                 local used guarded need_refresh used_n guarded_n debug_count create_fail_streak
                 cleanup_stale_port_locks
                 echo "Reserving host ports for ${DOC_ENGINE} compose stack..." >&2
                 # Fail fast if the shared docker daemon is not reachable — avoids a
                 # long spin where every docker network create silently fails.
                 if ! sudo docker info >/dev/null 2>&1; then
                   echo "  ERROR: 'sudo docker info' failed — docker daemon is not reachable from this runner" >&2
                   return 1
                 fi
                 echo "  Running containers on shared daemon: $(sudo docker ps -q 2>/dev/null | wc -l)" >&2
                 echo "  Docker networks on shared daemon: $(sudo docker network ls -q 2>/dev/null | wc -l)" >&2
                 need_refresh=1
                 debug_count=0
                 create_fail_streak=0
                 for attempt in $(seq 0 499); do
                   candidate=$(( PARTITION_BASE + ((GITHUB_RUN_ID + RUNNER_NUM * 1000 + attempt * 97) % PARTITION_SIZE) ))
                   # Refresh snapshots on first attempt and after a failed reservation.
                   if [[ ${need_refresh} -eq 1 ]]; then
                     used=$(sudo docker ps --format '{{.Ports}}' 2>/dev/null | grep -oE '[0-9]+->' | tr -d '>-' | sort -u || true)
                     # Strip both ragflow-ci-port- (current) and ragflow-ci-portguard- (legacy) prefixes.
                     guarded=$(sudo docker network ls --filter "label=ragflow-ci-created" --format '{{.Name}}' 2>/dev/null | sed 's/^ragflow-ci-port\(guard\)\?-//' | sort -u || true)
                     used_n=$(echo "${used}" | grep -c . 2>/dev/null || true)
                     guarded_n=$(echo "${guarded}" | grep -c . 2>/dev/null || true)
                     echo "  Snapshot: ${used_n} bound ports, ${guarded_n} guarded ports" >&2
                     [[ ${used_n} -gt 0 ]] && echo "  Bound sample: $(echo "${used}" | tr '\n' ' ' | cut -d' ' -f1-10)" >&2
                     [[ ${guarded_n} -gt 0 ]] && echo "  Guarded sample: $(echo "${guarded}" | tr '\n' ' ' | cut -d' ' -f1-10)" >&2
                     need_refresh=0
                   fi
                   # Fast path: skip if any (base+candidate) port is already bound or guarded.
                   ok=1
                   reject_reason=""
                   for base in "${PORT_BASES[@]}"; do
                     port=$((base + candidate))
                     if grep -qx "${port}" <<< "${used}" 2>/dev/null; then
                       ok=0; reject_reason="port ${port} (base ${base}) already bound"; break
                     fi
                     if grep -qx "${port}" <<< "${guarded}" 2>/dev/null; then
                       ok=0; reject_reason="port ${port} (base ${base}) already guarded"; break
                     fi
                   done
                   if [[ ${ok} -eq 0 ]]; then
                     if [[ ${debug_count} -lt 5 ]]; then
                       echo "  Candidate ${candidate}: rejected — ${reject_reason}" >&2
                       debug_count=$((debug_count + 1))
                     fi
                     continue
                   fi
                   # Atomically claim every (base+candidate) host port via a per-port docker
                   # network. docker network create fails on name conflict, so only one runner
                   # can claim a given port. If any port is already claimed, undo partial
                   # reservations and try the next offset.
                   guards=()
                   ok=1
                   for base in "${PORT_BASES[@]}"; do
                     port=$((base + candidate))
                     guard="ragflow-ci-port-${port}"
                     # Use an explicit subnet from 10.0.0.0/8 (outside Docker's
                     # default 172.17-31.x.x pools) to bypass "all predefined
                     # address pools have been fully subnetted" errors on shared
                     # daemons where auto-allocation is exhausted.
                     guard_subnet="10.$((port / 256)).$((port % 256)).0/24"
                     if ! create_err=$(sudo docker network create --subnet "${guard_subnet}" --label ragflow-ci-created="$(date -u +%s)" "${guard}" 2>&1 >/dev/null); then
                       echo "  docker network create ${guard} failed: ${create_err}" >&2
                       ok=0; break
                     fi
                     guards+=("${guard}")
                   done
                   if [[ ${ok} -eq 1 ]]; then
                     PORT_OFFSET=${candidate}
                     PORT_RESERVATION="${guards[*]}"
                     echo "Reserved offset ${PORT_OFFSET} (attempt ${attempt})" >&2
                     return 0
                   fi
                   for g in "${guards[@]}"; do
                     sudo docker network rm "${g}" >/dev/null 2>&1 || true
                   done
                   # If docker network create keeps failing (daemon gone, permission, etc.),
                   # bail out early instead of looping 6000 times. A name-conflict failure
                   # should be rare because the fast path already filters guarded ports.
                   create_fail_streak=$((create_fail_streak + 1))
                   if [[ ${create_fail_streak} -ge 10 ]]; then
                     echo "  docker network create failed ${create_fail_streak} consecutive candidates; aborting" >&2
                     return 1
                   fi
                   # Another runner just claimed a port; refresh snapshots before retrying.
                   need_refresh=1
                 done
                 echo "  Exhausted ${attempt} candidates. Last snapshot: ${used_n} bound, ${guarded_n} guarded" >&2
                 return 1
               }

               if ! reserve_port_offset; then
                 echo "Failed to reserve a free host port range for ${DOC_ENGINE} docker compose" >&2
                 exit 1
               fi
               echo "PORT_RESERVATION=${PORT_RESERVATION}" >> ${GITHUB_ENV}
               echo "Using ${DOC_ENGINE} host port offset ${PORT_OFFSET}"
               ES_PORT=$((1200 + PORT_OFFSET))
               OS_PORT=$((1201 + PORT_OFFSET))
               INFINITY_THRIFT_PORT=$((23817 + PORT_OFFSET))
               INFINITY_HTTP_PORT=$((23820 + PORT_OFFSET))
               INFINITY_PSQL_PORT=$((5432 + PORT_OFFSET))
               EXPOSE_MYSQL_PORT=$((5455 + PORT_OFFSET))
               MINIO_PORT=$((9000 + PORT_OFFSET))
               MINIO_CONSOLE_PORT=$((9001 + PORT_OFFSET))
               REDIS_PORT=$((6379 + PORT_OFFSET))
               EXPOSE_NATS_PORT=$((4222 + PORT_OFFSET))
               NATS_MONITORING_PORT=$((8222 + PORT_OFFSET))
               TEI_PORT=$((6380 + PORT_OFFSET))
               KIBANA_PORT=$((6601 + PORT_OFFSET))
               SVR_HTTP_PORT=$((9380 + PORT_OFFSET))
               ADMIN_SVR_HTTP_PORT=$((9381 + PORT_OFFSET))
               SVR_MCP_PORT=$((9382 + PORT_OFFSET))
               GO_HTTP_PORT=$((9384 + PORT_OFFSET))
               GO_ADMIN_PORT=$((9383 + PORT_OFFSET))
               SANDBOX_EXECUTOR_MANAGER_PORT=$((9385 + PORT_OFFSET))
               SVR_WEB_HTTP_PORT=$((80 + PORT_OFFSET))
               SVR_WEB_HTTPS_PORT=$((443 + PORT_OFFSET))
               EXPOSE_CLICKHOUSE_TCP_PORT=$((9900 + PORT_OFFSET))
               CLICKHOUSE_HTTP_PORT=$((8123 + PORT_OFFSET))

          # Persist computed ports into .env so docker-compose uses the correct host bindings.
          # Remove previous CI overrides first; docker compose uses the last duplicate key.
               sed -i '/^ES_PORT=/d;/^OS_PORT=/d;/^INFINITY_THRIFT_PORT=/d;/^INFINITY_HTTP_PORT=/d;/^INFINITY_PSQL_PORT=/d;/^EXPOSE_MYSQL_PORT=/d;/^MINIO_PORT=/d;/^MINIO_CONSOLE_PORT=/d;/^REDIS_PORT=/d;/^EXPOSE_NATS_PORT=/d;/^NATS_MONITORING_PORT=/d;/^TEI_PORT=/d;/^KIBANA_PORT=/d;/^SVR_HTTP_PORT=/d;/^ADMIN_SVR_HTTP_PORT=/d;/^SVR_MCP_PORT=/d;/^GO_HTTP_PORT=/d;/^GO_ADMIN_PORT=/d;/^SANDBOX_EXECUTOR_MANAGER_PORT=/d;/^SVR_WEB_HTTP_PORT=/d;/^SVR_WEB_HTTPS_PORT=/d;/^EXPOSE_CLICKHOUSE_TCP_PORT=/d;/^CLICKHOUSE_HTTP_PORT=/d;/^COMPOSE_PROFILES=/d;/^TEI_MODEL=/d;/^RAGFLOW_IMAGE=/d;/^DOC_ENGINE=/d;/^API_PROXY_SCHEME=/d' .env
          # docker-compose-go.yml uses env_file: .env-go for the ragflow-cpu
          # container, so the Go server process reads COMPOSE_PROFILES and
          # TEI_MODEL from .env-go (not .env). Without syncing these two,
          # the Go server's checkModelAvailable TEI bypass never triggers
          # because .env-go's COMPOSE_PROFILES lacks "tei-", causing
          # PATCH /models/default to fail for the Builtin/Local embedding
          # model and datasets to be created with an empty embedding_model.
               sed -i '/^COMPOSE_PROFILES=/d;/^TEI_MODEL=/d' .env-go
               {
                 echo ""
                 echo "ES_PORT=${ES_PORT}"
                 echo "OS_PORT=${OS_PORT}"
                 echo "INFINITY_THRIFT_PORT=${INFINITY_THRIFT_PORT}"
                 echo "INFINITY_HTTP_PORT=${INFINITY_HTTP_PORT}"
                 echo "INFINITY_PSQL_PORT=${INFINITY_PSQL_PORT}"
                 echo "EXPOSE_MYSQL_PORT=${EXPOSE_MYSQL_PORT}"
                 echo "MINIO_PORT=${MINIO_PORT}"
                 echo "MINIO_CONSOLE_PORT=${MINIO_CONSOLE_PORT}"
                 echo "REDIS_PORT=${REDIS_PORT}"
                 echo "EXPOSE_NATS_PORT=${EXPOSE_NATS_PORT}"
                 echo "NATS_MONITORING_PORT=${NATS_MONITORING_PORT}"
                 echo "TEI_PORT=${TEI_PORT}"
                 echo "KIBANA_PORT=${KIBANA_PORT}"
                 echo "SVR_HTTP_PORT=${SVR_HTTP_PORT}"
                 echo "ADMIN_SVR_HTTP_PORT=${ADMIN_SVR_HTTP_PORT}"
                 echo "SVR_MCP_PORT=${SVR_MCP_PORT}"
                 echo "GO_HTTP_PORT=${GO_HTTP_PORT}"
                 echo "GO_ADMIN_PORT=${GO_ADMIN_PORT}"
                 echo "SANDBOX_EXECUTOR_MANAGER_PORT=${SANDBOX_EXECUTOR_MANAGER_PORT}"
                 echo "SVR_WEB_HTTP_PORT=${SVR_WEB_HTTP_PORT}"
                 echo "SVR_WEB_HTTPS_PORT=${SVR_WEB_HTTPS_PORT}"
                 echo "EXPOSE_CLICKHOUSE_TCP_PORT=${EXPOSE_CLICKHOUSE_TCP_PORT}"
                 echo "CLICKHOUSE_HTTP_PORT=${CLICKHOUSE_HTTP_PORT}"
                 if [ "${API_PROXY_SCHEME}" = "go" ]; then
                   echo "COMPOSE_PROFILES=${DOC_ENGINE},cpu,ragflow-go,tei-cpu,deepdoc,metadata-mysql,clickhouse"
                 else
                   echo "COMPOSE_PROFILES=${DOC_ENGINE},cpu,ragflow-go,tei-cpu,deepdoc,metadata-mysql"
                 fi
                 echo "TEI_MODEL=BAAI/bge-small-en-v1.5"
                 echo "RAGFLOW_IMAGE=${RAGFLOW_IMAGE}"
                 echo "DOC_ENGINE=${DOC_ENGINE}"
                 echo "API_PROXY_SCHEME=${API_PROXY_SCHEME}"
               } >> .env
          # Sync COMPOSE_PROFILES and TEI_MODEL into .env-go so the Go
          # server container (env_file: .env-go) sees the same TEI bypass
          # config as the Python container. Only these two affect the
          # in-process checkModelAvailable logic; other .env keys are
          # either yml-interpolation-only or already correct in .env-go.
               {
                 echo ""
                 if [ "${API_PROXY_SCHEME}" = "go" ]; then
                   echo "COMPOSE_PROFILES=${DOC_ENGINE},cpu,ragflow-go,tei-cpu,deepdoc,metadata-mysql,clickhouse"
                 else
                   echo "COMPOSE_PROFILES=${DOC_ENGINE},cpu,ragflow-go,tei-cpu,deepdoc,metadata-mysql"
                 fi
                 echo "TEI_MODEL=BAAI/bge-small-en-v1.5"
               } >> .env-go
               if [[ "${API_PROXY_SCHEME}" == "go" ]]; then
                 RAGFLOW_HTTP_PORT=${SVR_WEB_HTTP_PORT}
               else
                 RAGFLOW_HTTP_PORT=${SVR_HTTP_PORT}
               fi
               echo "RAGFLOW_HTTP_PORT=${RAGFLOW_HTTP_PORT}" >> ${GITHUB_ENV}
               echo "HOST_ADDRESS=http://host.docker.internal:${RAGFLOW_HTTP_PORT}" >> ${GITHUB_ENV}

               sed -i \
                 -e 's#${NATS_PORT:-4222}:4222#${EXPOSE_NATS_PORT}:4222#' \
                 -e 's#"8222:8222"#"${NATS_MONITORING_PORT}:8222"#' \
                 docker-compose-base.yml

               # Patch entrypoint.sh for coverage
               sed -i '/"\$PY" api\/ragflow_server.py \${INIT_SUPERUSER_ARGS} &/c\   echo "Ensuring coverage is installed..."\n      "$PY" -m pip install coverage -i https://mirrors.aliyun.com/pypi/simple\n     export COVERAGE_FILE=/ragflow/logs/.coverage\n        echo "Starting ragflow_server with coverage..."\n        "$PY" -m coverage run --source=./api/apps --omit="*/tests/*,*/migrations/*" -a api/ragflow_server.py ${INIT_SUPERUSER_ARGS} &' ./entrypoint.sh


      - name: Start ragflow:nightly for Infinity
        run: |
          if [ "${API_PROXY_SCHEME}" = "go" ]; then
            COMPOSE_FILE=docker/docker-compose-go.yml
          else
            COMPOSE_FILE=docker/docker-compose.yml
          fi
          sudo docker compose -f "${COMPOSE_FILE}" -p ${COMPOSE_PROJECT_NAME} down -v || true
          sudo docker ps -a --filter "label=com.docker.compose.project=${COMPOSE_PROJECT_NAME}" -q | xargs -r sudo docker rm -f
          sudo docker compose -f "${COMPOSE_FILE}" -p ${COMPOSE_PROJECT_NAME} up -d

      - name: Run sdk tests against Infinity
        if: env.API_PROXY_SCHEME == 'python'
        run: |
          export http_proxy=""; export https_proxy=""; export no_proxy=""; export HTTP_PROXY=""; export HTTPS_PROXY=""; export NO_PROXY=""
          svc_ready=0
          for i in $(seq 1 60); do
            if sudo docker exec ${RAGFLOW_CONTAINER} curl -sf --connect-timeout 5 "${HOST_ADDRESS}/api/v1/system/ping" > /dev/null 2>&1; then
              svc_ready=1
              break
            fi
            echo "Waiting for service to be available... ($i/60)"
            sleep 5
          done
          if [ "$svc_ready" -ne 1 ]; then
            echo "Service did not become ready after 5 minutes. Docker logs:"
            sudo docker logs ${RAGFLOW_CONTAINER}
            exit 1
          fi
          echo "Start to run test sdk on Infinity"
          source .venv/bin/activate && set -o pipefail; DOC_ENGINE=infinity pytest -s --tb=short --level=${HTTP_API_TEST_LEVEL} --junitxml=pytest-infinity-sdk.xml --cov=sdk/python/ragflow_sdk --cov-branch --cov-report=xml:coverage-infinity-sdk.xml test/testcases/test_sdk_api 2>&1 | tee infinity_sdk_test.log

      - name: Run New RESTFUL api tests against Infinity
        if: env.API_PROXY_SCHEME != 'web'
        run: |
          export http_proxy=""; export https_proxy=""; export no_proxy=""; export HTTP_PROXY=""; export HTTPS_PROXY=""; export NO_PROXY=""
          svc_ready=0
          for i in $(seq 1 60); do
            if sudo docker exec ${RAGFLOW_CONTAINER} curl -sf --connect-timeout 5 "${HOST_ADDRESS}/api/v1/system/ping" > /dev/null 2>&1; then
              svc_ready=1
              break
            fi
            echo "Waiting for service to be available... ($i/60)"
            sleep 5
          done
          if [ "$svc_ready" -ne 1 ]; then
            echo "Service did not become ready after 5 minutes. Docker logs:"
            sudo docker logs ${RAGFLOW_CONTAINER}
            exit 1
          fi
          source .venv/bin/activate && set -o pipefail; DOC_ENGINE=infinity pytest -s --tb=short --level=${HTTP_API_TEST_LEVEL} test/testcases/restful_api 2>&1 | tee infinity_restful_api_test.log

      - name: RAGFlow CLI retrieval test Infinity
        if: env.API_PROXY_SCHEME == 'python'
        env:
          PYTHONPATH: ${{ github.workspace }}
        run: |
          set -euo pipefail
          source .venv/bin/activate

          export http_proxy=""; export https_proxy=""; export no_proxy=""; export HTTP_PROXY=""; export HTTPS_PROXY=""; export NO_PROXY=""

          EMAIL="ci-${GITHUB_RUN_ID}@example.com"
          PASS="ci-pass-${GITHUB_RUN_ID}"
          DATASET="ci_dataset_${GITHUB_RUN_ID}"

          CLI="python admin/client/ragflow_cli.py"

          LOG_FILE="infinity_cli_test.log"
          : > "${LOG_FILE}"

          ERROR_RE='Traceback|ModuleNotFoundError|ImportError|Parse error|Bad response|Fail to|code:\\s*[1-9]'
          run_cli() {
            local logfile="$1"
            shift
            local allow_re=""
            if [[ "${1:-}" == "--allow" ]]; then
              allow_re="$2"
              shift 2
            fi
            local cmd_display="$*"
            echo "===== $(date -u +\"%Y-%m-%dT%H:%M:%SZ\") CMD: ${cmd_display} =====" | tee -a "${logfile}"
            local tmp_log
            tmp_log="$(mktemp)"
            set +e
            timeout 500s "$@" 2>&1 | tee "${tmp_log}"
            local status=${PIPESTATUS[0]}
            set -e
            cat "${tmp_log}" >> "${logfile}"
            if grep -qiE "${ERROR_RE}" "${tmp_log}"; then
              if [[ -n "${allow_re}" ]] && grep -qiE "${allow_re}" "${tmp_log}"; then
                echo "Allowed CLI error markers in ${logfile}"
                rm -f "${tmp_log}"
                return 0
              fi
              echo "Detected CLI error markers in ${logfile}"
              rm -f "${tmp_log}"
              exit 1
            fi
            rm -f "${tmp_log}"
            return ${status}
          }

          set -a
          source docker/.env
          set +a

          USER_HOST="$(echo "${HOST_ADDRESS}" | sed -E 's#^https?://([^:/]+).*#\1#')"
          USER_PORT="${RAGFLOW_HTTP_PORT}"
          ADMIN_HOST="${USER_HOST}"
          ADMIN_PORT="${ADMIN_SVR_HTTP_PORT}"

          svc_ready=0
          for i in $(seq 1 60); do
            if sudo docker exec ${RAGFLOW_CONTAINER} curl -sf --connect-timeout 5 "${HOST_ADDRESS}/api/v1/system/ping" > /dev/null 2>&1; then
              svc_ready=1
              break
            fi
            echo "Waiting for service to be available... ($i/60)"
            sleep 5
          done
          if [ "$svc_ready" -ne 1 ]; then
            echo "Service did not become ready after 5 minutes. Docker logs:"
            sudo docker logs ${RAGFLOW_CONTAINER}
            exit 1
          fi

          admin_ready=0
          for i in $(seq 1 30); do
            if run_cli "${LOG_FILE}" $CLI --type admin --host "$ADMIN_HOST" --port "$ADMIN_PORT" --username "admin@ragflow.io" --password "admin" command "ping"; then
              admin_ready=1
              break
            fi
            sleep 1
          done
          if [[ "${admin_ready}" -ne 1 ]]; then
            echo "Admin service did not become ready"
            exit 1
          fi

          run_cli "${LOG_FILE}" $CLI --type admin --host "$ADMIN_HOST" --port "$ADMIN_PORT" --username "admin@ragflow.io" --password "admin" command "show version"
          ALLOW_USER_EXISTS_RE='already exists|already exist|duplicate|already.*registered|exist(s)?'
          run_cli "${LOG_FILE}" --allow "${ALLOW_USER_EXISTS_RE}" $CLI --type admin --host "$ADMIN_HOST" --port "$ADMIN_PORT" --username "admin@ragflow.io" --password "admin" command "create user '$EMAIL' '$PASS'"

          user_ready=0
          for i in $(seq 1 30); do
            if run_cli "${LOG_FILE}" $CLI --type user --host "$USER_HOST" --port "$USER_PORT" --username "$EMAIL" --password "$PASS" command "ping"; then
              user_ready=1
              break
            fi
            sleep 1
          done
          if [[ "${user_ready}" -ne 1 ]]; then
            echo "User service did not become ready"
            exit 1
          fi

          run_cli "${LOG_FILE}" $CLI --type user --host "$USER_HOST" --port "$USER_PORT" --username "$EMAIL" --password "$PASS" command "show version"
          run_cli "${LOG_FILE}" $CLI --type user --host "$USER_HOST" --port "$USER_PORT" --username "$EMAIL" --password "$PASS" command "create dataset '$DATASET' with embedding 'BAAI/bge-small-en-v1.5@Builtin' parser 'auto'"
          run_cli "${LOG_FILE}" $CLI --type user --host "$USER_HOST" --port "$USER_PORT" --username "$EMAIL" --password "$PASS" command "import 'test/benchmark/test_docs/Doc1.pdf,test/benchmark/test_docs/Doc2.pdf' into dataset '$DATASET'"
          run_cli "${LOG_FILE}" $CLI --type user --host "$USER_HOST" --port "$USER_PORT" --username "$EMAIL" --password "$PASS" command "parse dataset '$DATASET' sync"
          run_cli "${LOG_FILE}" $CLI --type user --host "$USER_HOST" --port "$USER_PORT" --username "$EMAIL" --password "$PASS" command "Benchmark 16 100 search 'what are these documents about' on datasets '$DATASET'"

      - name: Stop ragflow to save coverage Infinity
        if: ${{ !cancelled() && env.API_PROXY_SCHEME == 'python' }}
        run: |
          # Send SIGINT to ragflow_server.py to trigger coverage save
          PID=$(sudo docker exec ${RAGFLOW_CONTAINER} ps aux | grep "ragflow_server.py" | grep -v grep | awk '{print $2}' | head -n 1)
          if [ -n "$PID" ]; then
            echo "Sending SIGINT to ragflow_server.py (PID: $PID)..."
            sudo docker exec ${RAGFLOW_CONTAINER} kill -INT $PID
            # Wait for process to exit and coverage file to be written
            sleep 10
          else
            echo "ragflow_server.py not found!"
          fi
          sudo docker compose -f docker/docker-compose.yml -p ${COMPOSE_PROJECT_NAME} stop

      - name: Generate server coverage report Infinity
        if: ${{ !cancelled() && env.API_PROXY_SCHEME == 'python' }}
        run: |
          # .coverage file should be in docker/ragflow-logs/.coverage
          if [ -f docker/ragflow-logs/.coverage ]; then
            echo "Found .coverage file"
            cp docker/ragflow-logs/.coverage .coverage
            source .venv/bin/activate
            # Create .coveragerc to map container paths to host paths
            echo "[paths]" > .coveragerc
            echo "source =" >> .coveragerc
            echo "    ." >> .coveragerc
            echo "    /ragflow" >> .coveragerc
            coverage xml -o coverage-infinity-server.xml
            rm .coveragerc
          else
            echo ".coverage file not found!"
          fi

      - name: Upload coverage reports to Codecov
        uses: codecov/codecov-action@v5
        if: ${{ !cancelled() }}
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          fail_ci_if_error: false

      - name: Collect ragflow log Infinity
        if: ${{ !cancelled() }}
        run: |
          if [ -d docker/ragflow-logs ]; then
            destination="${ARTIFACTS_DIR}/ragflow-logs-infinity"
            sudo cp -r docker/ragflow-logs "${destination}"
            sudo chown -R "$(id -u):$(id -g)" "${destination}"
            echo "ragflow log" && sudo tail -n 200 docker/ragflow-logs/ragflow_server.log || true
          else
            echo "No docker/ragflow-logs directory found; skipping log collection"
          fi
          sudo rm -rf docker/ragflow-logs || true

      - name: Stop ragflow:nightly for Infinity
        if: ${{ always() }}
        run: |
          if [ "${API_PROXY_SCHEME}" = "go" ]; then
            COMPOSE_FILE=docker/docker-compose-go.yml
          else
            COMPOSE_FILE=docker/docker-compose.yml
          fi
          # Sometimes `docker compose down` fail due to hang container, heavy load etc. Need to remove such containers to release resources(for example, listen ports).
          sudo docker compose -f "${COMPOSE_FILE}" -p ${COMPOSE_PROJECT_NAME} down -v || true
          sudo docker ps -a --filter "label=com.docker.compose.project=${COMPOSE_PROJECT_NAME}" -q | xargs -r sudo docker rm -f
          if [[ -n ${RAGFLOW_IMAGE} ]]; then
            sudo docker rmi -f ${RAGFLOW_IMAGE}
          fi
          if [[ -n ${PORT_RESERVATION:-} ]]; then
            # PORT_RESERVATION is a space-separated list of per-port docker network guards
            # created by the prepare step. Remove each to free the ports for other runners.
            for guard in ${PORT_RESERVATION}; do
              sudo docker network rm "${guard}" >/dev/null 2>&1 || true
            done
          fi



  ragflow_tests_elasticsearch:
    name: ragflow_tests_elasticsearch (${{ matrix.api_proxy_scheme }})
    needs: ragflow_preflight
    if: ${{ github.event_name != 'pull_request' && github.event_name != 'pull_request_target' || (github.event.pull_request.draft == false && contains(github.event.pull_request.labels.*.name, 'ci') && (github.event.action != 'labeled' || github.event.label.name == 'ci')) }}
    runs-on: [ "self-hosted", "ragflow-test" ]
    strategy:
      fail-fast: false
      matrix:
        api_proxy_scheme: ${{ fromJSON(needs.ragflow_preflight.outputs.api_proxy_schemes) }}
    env:
      DOC_ENGINE: elasticsearch
      API_PROXY_SCHEME: ${{ matrix.api_proxy_scheme }}
      RAGFLOW_IMAGE: infiniflow/ragflow:${{ github.run_id }}-elasticsearch-${{ matrix.api_proxy_scheme }}
      HTTP_API_TEST_LEVEL: ${{ needs.ragflow_preflight.outputs.http_api_test_level }}
      HAS_GO: ${{ needs.ragflow_preflight.outputs.has_go_changes }}
      HAS_PYTHON: ${{ needs.ragflow_preflight.outputs.has_python_changes }}
      HAS_WEB: ${{ needs.ragflow_preflight.outputs.has_web_changes }}
    steps:
      - name: Ensure workspace ownership
        if: always()
        run: |
          echo "Workflow triggered by ${{ github.event_name }}"
          echo "chown -R ${USER} ${GITHUB_WORKSPACE}" && sudo chown -R ${USER} ${GITHUB_WORKSPACE}

      - name: Check out code
        if: always()
        uses: actions/checkout@v6
        with:
          ref: ${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && format('refs/pull/{0}/merge', github.event.pull_request.number) || github.sha }}
          fetch-depth: 0
          fetch-tags: true
          allow-unsafe-pr-checkout: true

      - name: Build ragflow go server
        if: env.API_PROXY_SCHEME == 'go'
        run: |
          set -euo pipefail
          BUILDER_CONTAINER=ragflow_build_${GITHUB_RUN_ID}_${DOC_ENGINE}_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ')
          cleanup_builder() {
            if [[ -n "${BUILDER_CONTAINER:-}" ]]; then
              sudo docker rm -f -v "${BUILDER_CONTAINER}" >/dev/null 2>&1 || true
            fi
          }
          trap cleanup_builder EXIT

          TZ=${TZ:-$(readlink -f /etc/localtime | awk -F '/zoneinfo/' '{print $2}')}
          sudo docker run --privileged -d --name "${BUILDER_CONTAINER}" \
            -e TZ="${TZ}" \
            -e UV_INDEX=https://mirrors.aliyun.com/pypi/simple \
            -v "${PWD}:/ragflow" \
            -v "${PWD}/internal/binding/cpp/resource:/usr/share/infinity/resource" \
            infiniflow/infinity_builder:ubuntu22_clang20
          sudo docker exec "${BUILDER_CONTAINER}" bash -c 'git config --global safe.directory "*" && cd /ragflow && ./build.sh --cpp'
          ./build.sh --go

      - name: Run Go unit tests
        if: env.API_PROXY_SCHEME == 'go'
        # Runs after `./build.sh --go`, which guarantees the C++ static
        # library (librag_tokenizer_c_api.a) is present on disk. The Go
        # test binaries link against it transitively through
        # `internal/binding`, so running `go test` before the C++ build
        # fails the link step.
        #
        # Excludes packages whose tests fail for environmental reasons
        # unrelated to the diff:
        #   - internal/tokenizer is split: the pure-Go BPE loader tests
        #     (bpe_loader_test.go) run in the default tier; the C++ binding /
        #     dict-dependent tests (tokenizer_test.go,
        #     tokenizer_concurrent_test.go) and the on-disk anchor test
        #     (bpe_loader_anchors_test.go) are tagged `manual` and need the
        #     docker builder's /usr/share/infinity/resource, so they stay out
        #     of the default run.
        run: |
          set -euo pipefail

          # Provide the cl100k BPE table for the offline tokenizer loader.
          # The loader reads <repo>/ragflow_deps/cl100k_base.tiktoken (or
          # TIKTOKEN_CACHE_DIR). Try to fetch it directly from the upstream
          # openai blob first (small ~1.6MB, reachable from the CI runner — the
          # pre-PR code downloaded it on the fly, so the network path is known
          # good). If the direct fetch fails (e.g. GFW), fall back to extracting
          # it from the local infiniflow/ragflow_deps image *if already present*
          # on the runner (no `docker pull`, since that huge image is first
          # fetched at the later "Build ragflow:nightly" step). Best-effort: if
          # neither source works, the loader still fails loudly (no silent
          # 0-token degradation).
          # tiktoken is already installed in runner
          mkdir -p ragflow_deps
          cp /opt/ragflow_deps/cl100k_base.tiktoken ragflow_deps/cl100k_base.tiktoken || true
          if [ ! -f ragflow_deps/cl100k_base.tiktoken ]; then
            mkdir -p ragflow_deps
            if command -v curl >/dev/null 2>&1; then
              if curl -fsSL -o ragflow_deps/cl100k_base.tiktoken \
                https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken; then
                echo "tiktoken: cl100k table fetched via curl from openai blob"
              else
                echo "tiktoken: curl fetch failed (network/GFW); will try cached image"
              fi
            fi
            if [ ! -f ragflow_deps/cl100k_base.tiktoken ] && docker image inspect infiniflow/ragflow_deps:latest >/dev/null 2>&1; then
              CID=$(docker create infiniflow/ragflow_deps:latest true) || true
              if [ -n "${CID:-}" ]; then
                if docker cp "$CID":/cl100k_base.tiktoken ragflow_deps/cl100k_base.tiktoken 2>/dev/null; then
                  echo "tiktoken: cl100k table copied from cached infiniflow/ragflow_deps image"
                fi
                docker rm -f "$CID" >/dev/null 2>&1 || true
              fi
            fi
          fi
          if [ -f ragflow_deps/cl100k_base.tiktoken ]; then
            echo "tiktoken: cl100k table provisioned ($(wc -c < ragflow_deps/cl100k_base.tiktoken) bytes)"
          else
            echo "tiktoken: cl100k table NOT provisioned — offline loader will fail loudly"
          fi

          PKGS=$(go list ./... 2>/dev/null \
            | grep -v '/internal/storage$' \
            | grep -v '/internal/handler$' || true)
          if [ -z "$PKGS" ]; then
            ./build.sh --test
          else
            ./build.sh --test -- $PKGS
          fi

      - name: Run ORT deps-download regression guard
        if: env.API_PROXY_SCHEME == 'go'
        # Pure-Python, local, needs no API keys or model downloads; guards the
        # ONNX Runtime extraction logic the in-process (Go) DeepDoc backend build
        # depends on (notably the version-bump path C2 hardened). Run it in
        # isolation so the test/testcases conftest (which requires
        # SILICONFLOW_API_KEY) cannot fail the collection.
        run: |
          set -euo pipefail
          uv sync --python 3.13 --group test --frozen
          uv run pytest ragflow_deps/test_download_go_deps.py -q

      - name: Build ragflow:nightly
        run: |
          set -euo pipefail
          RUNNER_WORKSPACE_PREFIX=${RUNNER_WORKSPACE_PREFIX:-${HOME}}
          BUILD_LOCK_FILE=${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/docker-build.lock
          mkdir -p "$(dirname "${BUILD_LOCK_FILE}")"
          # Pass the Gitee personal access token to the Docker build via --build-arg GITEE_TOKEN.
          # Dockerfile_go / Dockerfile_ci read it from ARG GITEE_TOKEN (NOT a BuildKit secret),
          # so the runner must expose GITEE_TOKEN as an environment variable (e.g. via the
          # runner's docker-compose `environment:` mapping sourced from its .env file).
          # Pick the Dockerfile matching the API proxy scheme:
          #   - go  -> Dockerfile_go  (Go-native image)
          #   - *   -> Dockerfile_ci  (Python/legacy image)
          if [ "${API_PROXY_SCHEME}" = "go" ]; then
            DOCKERFILE=Dockerfile_go
          else
            DOCKERFILE=Dockerfile_ci
          fi
          (
            flock -w 10800 9 || { echo "Timed out waiting for the shared Docker build slot" >&2; exit 1; }
            echo "Acquired Docker build slot for ${DOC_ENGINE}/${API_PROXY_SCHEME} (dockerfile=${DOCKERFILE})"
            ##sudo docker pull ubuntu:24.04
            sudo DOCKER_BUILDKIT=1 docker build --build-arg NEED_MIRROR=1 --build-arg HTTPS_PROXY=${HTTPS_PROXY} --build-arg HTTP_PROXY=${HTTP_PROXY} --build-arg GITEE_TOKEN=${GITEE_TOKEN} -f "${DOCKERFILE}" -t ${RAGFLOW_IMAGE} .
          ) 9>"${BUILD_LOCK_FILE}"

      - name: Prepare Python test environment
        if: env.API_PROXY_SCHEME != 'web'
        run: |
          git config --global url."https://oauth2:${GITEE_TOKEN}@gitee.com/".insteadOf "https://gitee.com/"
          uv sync --python 3.13 --group test --frozen
          uv pip install -e sdk/python

      - name: Prepare function test environment
        working-directory: docker
        run: |
          set -euo pipefail
          sudo chmod 1777 /tmp
          ## ss already installed
          ##sudo apt update && sudo apt install -y iproute2
          RUNNER_WORKSPACE_PREFIX=${RUNNER_WORKSPACE_PREFIX:-${HOME}}
          COMPOSE_PROJECT_NAME="${GITHUB_RUN_ID}-${DOC_ENGINE}-${API_PROXY_SCHEME}"
          echo "COMPOSE_PROJECT_NAME=${COMPOSE_PROJECT_NAME}" >> ${GITHUB_ENV}
          echo "RAGFLOW_CONTAINER=${COMPOSE_PROJECT_NAME}-ragflow-cpu-1" >> ${GITHUB_ENV}
          ARTIFACTS_DIR=${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/${GITHUB_RUN_ID}/${DOC_ENGINE}/${API_PROXY_SCHEME}
          echo "ARTIFACTS_DIR=${ARTIFACTS_DIR}" >> ${GITHUB_ENV}
          rm -rf "${ARTIFACTS_DIR}" && mkdir -p "${ARTIFACTS_DIR}"

          # Determine runner number (default to 1 if not found)
               RUNNER_NUM=$(sudo docker inspect $(hostname) --format '{{index .Config.Labels "com.docker.compose.container-number"}}' 2>/dev/null || true)
               if [[ -z "${RUNNER_NUM:-}" ]]; then
                 # GitHub self-hosted runners always expose RUNNER_NAME; hash it to spread candidates.
                 if [[ -n "${RUNNER_NAME:-}" ]]; then
                   RUNNER_NUM=$(( $(printf '%d' 0x$(echo -n "${RUNNER_NAME}" | md5sum | cut -c1-4)) % 1000 ))
                 else
                   RUNNER_NUM=$(( GITHUB_RUN_ID % 1000 ))
                 fi
               fi

          # Engine-specific offset partitions keep concurrent engine jobs from
          # choosing the same host ports when they land on the same self-hosted runner.
          # A lock plus reservation file closes the check/start race between parallel jobs.
               PORT_BASES=(1200 1201 23817 23820 5432 5455 9000 9001 6379 6380 6601 9380 9381 9382 9384 9383 9385 80 443 4222 8222 9900 8123)
               PARTITION_SIZE=6000
               case "${DOC_ENGINE}" in
                 elasticsearch) PARTITION_BASE=1000 ;;
                 infinity) PARTITION_BASE=31000 ;;
                 *) echo "Unsupported DOC_ENGINE=${DOC_ENGINE}" >&2; exit 1 ;;
               esac
               # Runners run inside docker containers that share the host docker daemon
               # (/var/run/docker.sock), so neither /tmp nor `ss` can coordinate across runners:
               # each container has its own /tmp and its own network namespace, and `ss` cannot
               # see host ports bound by other runners' compose stacks. Use the shared daemon for
               # both primitives:
               #   - per-port mutex = for each host port (base+offset) we intend to bind, create
               #     a docker network named ragflow-ci-port-<port> on the shared daemon. docker
               #     network create is NOT idempotent (fails on name conflict), so only one runner
               #     can claim a given port. This is per-PORT, not per-offset, because different
               #     (base, offset) pairs can produce the same host port (e.g. 6380+36497 ==
               #     9380+33497 == 42877); a per-offset guard cannot prevent that collision.
               #   - port availability = docker ps published ports (sees every container on the
               #     shared daemon regardless of which runner started it).
               # No file-system sharing required, no host netns required.

               cleanup_stale_port_locks() {
                 local now stale_after net ts before after
                 now=$(date -u +%s)
                 stale_after=$((60 * 60))
                 before=$(sudo docker network ls --filter "label=ragflow-ci-created" --format '{{.Name}}' | wc -l)
                 for net in $(sudo docker network ls --filter "label=ragflow-ci-created" --format '{{.Name}}'); do
                   ts=$(sudo docker network inspect --format '{{ index .Labels "ragflow-ci-created" }}' "${net}" 2>/dev/null || true)
                   # Remove if timestamp is non-numeric (legacy network from a previous code
                   # version) or if it is numeric and older than stale_after.
                   if [[ ! "${ts}" =~ ^[0-9]+$ ]] || (( now - ts > stale_after )); then
                     sudo docker network rm "${net}" >/dev/null 2>&1 || true
                   fi
                 done
                 after=$(sudo docker network ls --filter "label=ragflow-ci-created" --format '{{.Name}}' | wc -l)
                 echo "  cleanup_stale_port_locks: ${before} -> ${after} guards (TTL=${stale_after}s)" >&2
               }

               reserve_port_offset() {
                 local attempt candidate base port guard g ok reject_reason create_err guard_subnet
                 local -a guards
                 local used guarded need_refresh used_n guarded_n debug_count create_fail_streak
                 cleanup_stale_port_locks
                 echo "Reserving host ports for ${DOC_ENGINE} compose stack..." >&2
                 # Fail fast if the shared docker daemon is not reachable — avoids a
                 # long spin where every docker network create silently fails.
                 if ! sudo docker info >/dev/null 2>&1; then
                   echo "  ERROR: 'sudo docker info' failed — docker daemon is not reachable from this runner" >&2
                   return 1
                 fi
                 echo "  Running containers on shared daemon: $(sudo docker ps -q 2>/dev/null | wc -l)" >&2
                 echo "  Docker networks on shared daemon: $(sudo docker network ls -q 2>/dev/null | wc -l)" >&2
                 need_refresh=1
                 debug_count=0
                 create_fail_streak=0
                 for attempt in $(seq 0 499); do
                   candidate=$(( PARTITION_BASE + ((GITHUB_RUN_ID + RUNNER_NUM * 1000 + attempt * 97) % PARTITION_SIZE) ))
                   # Refresh snapshots on first attempt and after a failed reservation.
                   if [[ ${need_refresh} -eq 1 ]]; then
                     used=$(sudo docker ps --format '{{.Ports}}' 2>/dev/null | grep -oE '[0-9]+->' | tr -d '>-' | sort -u || true)
                     # Strip both ragflow-ci-port- (current) and ragflow-ci-portguard- (legacy) prefixes.
                     guarded=$(sudo docker network ls --filter "label=ragflow-ci-created" --format '{{.Name}}' 2>/dev/null | sed 's/^ragflow-ci-port\(guard\)\?-//' | sort -u || true)
                     used_n=$(echo "${used}" | grep -c . 2>/dev/null || true)
                     guarded_n=$(echo "${guarded}" | grep -c . 2>/dev/null || true)
                     echo "  Snapshot: ${used_n} bound ports, ${guarded_n} guarded ports" >&2
                     [[ ${used_n} -gt 0 ]] && echo "  Bound sample: $(echo "${used}" | tr '\n' ' ' | cut -d' ' -f1-10)" >&2
                     [[ ${guarded_n} -gt 0 ]] && echo "  Guarded sample: $(echo "${guarded}" | tr '\n' ' ' | cut -d' ' -f1-10)" >&2
                     need_refresh=0
                   fi
                   # Fast path: skip if any (base+candidate) port is already bound or guarded.
                   ok=1
                   reject_reason=""
                   for base in "${PORT_BASES[@]}"; do
                     port=$((base + candidate))
                     if grep -qx "${port}" <<< "${used}" 2>/dev/null; then
                       ok=0; reject_reason="port ${port} (base ${base}) already bound"; break
                     fi
                     if grep -qx "${port}" <<< "${guarded}" 2>/dev/null; then
                       ok=0; reject_reason="port ${port} (base ${base}) already guarded"; break
                     fi
                   done
                   if [[ ${ok} -eq 0 ]]; then
                     if [[ ${debug_count} -lt 5 ]]; then
                       echo "  Candidate ${candidate}: rejected — ${reject_reason}" >&2
                       debug_count=$((debug_count + 1))
                     fi
                     continue
                   fi
                   # Atomically claim every (base+candidate) host port via a per-port docker
                   # network. docker network create fails on name conflict, so only one runner
                   # can claim a given port. If any port is already claimed, undo partial
                   # reservations and try the next offset.
                   guards=()
                   ok=1
                   for base in "${PORT_BASES[@]}"; do
                     port=$((base + candidate))
                     guard="ragflow-ci-port-${port}"
                     # Use an explicit subnet from 10.0.0.0/8 (outside Docker's
                     # default 172.17-31.x.x pools) to bypass "all predefined
                     # address pools have been fully subnetted" errors on shared
                     # daemons where auto-allocation is exhausted.
                     guard_subnet="10.$((port / 256)).$((port % 256)).0/24"
                     if ! create_err=$(sudo docker network create --subnet "${guard_subnet}" --label ragflow-ci-created="$(date -u +%s)" "${guard}" 2>&1 >/dev/null); then
                       echo "  docker network create ${guard} failed: ${create_err}" >&2
                       ok=0; break
                     fi
                     guards+=("${guard}")
                   done
                   if [[ ${ok} -eq 1 ]]; then
                     PORT_OFFSET=${candidate}
                     PORT_RESERVATION="${guards[*]}"
                     echo "Reserved offset ${PORT_OFFSET} (attempt ${attempt})" >&2
                     return 0
                   fi
                   for g in "${guards[@]}"; do
                     sudo docker network rm "${g}" >/dev/null 2>&1 || true
                   done
                   # If docker network create keeps failing (daemon gone, permission, etc.),
                   # bail out early instead of looping 6000 times. A name-conflict failure
                   # should be rare because the fast path already filters guarded ports.
                   create_fail_streak=$((create_fail_streak + 1))
                   if [[ ${create_fail_streak} -ge 10 ]]; then
                     echo "  docker network create failed ${create_fail_streak} consecutive candidates; aborting" >&2
                     return 1
                   fi
                   # Another runner just claimed a port; refresh snapshots before retrying.
                   need_refresh=1
                 done
                 echo "  Exhausted ${attempt} candidates. Last snapshot: ${used_n} bound, ${guarded_n} guarded" >&2
                 return 1
               }

               if ! reserve_port_offset; then
                 echo "Failed to reserve a free host port range for ${DOC_ENGINE} docker compose" >&2
                 exit 1
               fi
               echo "PORT_RESERVATION=${PORT_RESERVATION}" >> ${GITHUB_ENV}
               echo "Using ${DOC_ENGINE} host port offset ${PORT_OFFSET}"
               ES_PORT=$((1200 + PORT_OFFSET))
               OS_PORT=$((1201 + PORT_OFFSET))
               INFINITY_THRIFT_PORT=$((23817 + PORT_OFFSET))
               INFINITY_HTTP_PORT=$((23820 + PORT_OFFSET))
               INFINITY_PSQL_PORT=$((5432 + PORT_OFFSET))
               EXPOSE_MYSQL_PORT=$((5455 + PORT_OFFSET))
               MINIO_PORT=$((9000 + PORT_OFFSET))
               MINIO_CONSOLE_PORT=$((9001 + PORT_OFFSET))
               REDIS_PORT=$((6379 + PORT_OFFSET))
               EXPOSE_NATS_PORT=$((4222 + PORT_OFFSET))
               NATS_MONITORING_PORT=$((8222 + PORT_OFFSET))
               TEI_PORT=$((6380 + PORT_OFFSET))
               KIBANA_PORT=$((6601 + PORT_OFFSET))
               SVR_HTTP_PORT=$((9380 + PORT_OFFSET))
               ADMIN_SVR_HTTP_PORT=$((9381 + PORT_OFFSET))
               SVR_MCP_PORT=$((9382 + PORT_OFFSET))
               GO_HTTP_PORT=$((9384 + PORT_OFFSET))
               GO_ADMIN_PORT=$((9383 + PORT_OFFSET))
               SANDBOX_EXECUTOR_MANAGER_PORT=$((9385 + PORT_OFFSET))
               SVR_WEB_HTTP_PORT=$((80 + PORT_OFFSET))
               SVR_WEB_HTTPS_PORT=$((443 + PORT_OFFSET))
               EXPOSE_CLICKHOUSE_TCP_PORT=$((9900 + PORT_OFFSET))
               CLICKHOUSE_HTTP_PORT=$((8123 + PORT_OFFSET))

          # Persist computed ports into .env so docker-compose uses the correct host bindings.
          # Remove previous CI overrides first; docker compose uses the last duplicate key.
               sed -i '/^ES_PORT=/d;/^OS_PORT=/d;/^INFINITY_THRIFT_PORT=/d;/^INFINITY_HTTP_PORT=/d;/^INFINITY_PSQL_PORT=/d;/^EXPOSE_MYSQL_PORT=/d;/^MINIO_PORT=/d;/^MINIO_CONSOLE_PORT=/d;/^REDIS_PORT=/d;/^EXPOSE_NATS_PORT=/d;/^NATS_MONITORING_PORT=/d;/^TEI_PORT=/d;/^KIBANA_PORT=/d;/^SVR_HTTP_PORT=/d;/^ADMIN_SVR_HTTP_PORT=/d;/^SVR_MCP_PORT=/d;/^GO_HTTP_PORT=/d;/^GO_ADMIN_PORT=/d;/^SANDBOX_EXECUTOR_MANAGER_PORT=/d;/^SVR_WEB_HTTP_PORT=/d;/^SVR_WEB_HTTPS_PORT=/d;/^EXPOSE_CLICKHOUSE_TCP_PORT=/d;/^CLICKHOUSE_HTTP_PORT=/d;/^COMPOSE_PROFILES=/d;/^TEI_MODEL=/d;/^RAGFLOW_IMAGE=/d;/^DOC_ENGINE=/d;/^API_PROXY_SCHEME=/d' .env
          # docker-compose-go.yml uses env_file: .env-go for the ragflow-cpu
          # container, so the Go server process reads COMPOSE_PROFILES and
          # TEI_MODEL from .env-go (not .env). Without syncing these two,
          # the Go server's checkModelAvailable TEI bypass never triggers
          # because .env-go's COMPOSE_PROFILES lacks "tei-", causing
          # PATCH /models/default to fail for the Builtin/Local embedding
          # model and datasets to be created with an empty embedding_model.
               sed -i '/^COMPOSE_PROFILES=/d;/^TEI_MODEL=/d' .env-go
               {
                 echo ""
                 echo "ES_PORT=${ES_PORT}"
                 echo "OS_PORT=${OS_PORT}"
                 echo "INFINITY_THRIFT_PORT=${INFINITY_THRIFT_PORT}"
                 echo "INFINITY_HTTP_PORT=${INFINITY_HTTP_PORT}"
                 echo "INFINITY_PSQL_PORT=${INFINITY_PSQL_PORT}"
                 echo "EXPOSE_MYSQL_PORT=${EXPOSE_MYSQL_PORT}"
                 echo "MINIO_PORT=${MINIO_PORT}"
                 echo "MINIO_CONSOLE_PORT=${MINIO_CONSOLE_PORT}"
                 echo "REDIS_PORT=${REDIS_PORT}"
                 echo "EXPOSE_NATS_PORT=${EXPOSE_NATS_PORT}"
                 echo "NATS_MONITORING_PORT=${NATS_MONITORING_PORT}"
                 echo "TEI_PORT=${TEI_PORT}"
                 echo "KIBANA_PORT=${KIBANA_PORT}"
                 echo "SVR_HTTP_PORT=${SVR_HTTP_PORT}"
                 echo "ADMIN_SVR_HTTP_PORT=${ADMIN_SVR_HTTP_PORT}"
                 echo "SVR_MCP_PORT=${SVR_MCP_PORT}"
                 echo "GO_HTTP_PORT=${GO_HTTP_PORT}"
                 echo "GO_ADMIN_PORT=${GO_ADMIN_PORT}"
                 echo "SANDBOX_EXECUTOR_MANAGER_PORT=${SANDBOX_EXECUTOR_MANAGER_PORT}"
                 echo "SVR_WEB_HTTP_PORT=${SVR_WEB_HTTP_PORT}"
                 echo "SVR_WEB_HTTPS_PORT=${SVR_WEB_HTTPS_PORT}"
                 echo "EXPOSE_CLICKHOUSE_TCP_PORT=${EXPOSE_CLICKHOUSE_TCP_PORT}"
                 echo "CLICKHOUSE_HTTP_PORT=${CLICKHOUSE_HTTP_PORT}"
                 if [ "${API_PROXY_SCHEME}" = "go" ]; then
                   echo "COMPOSE_PROFILES=${DOC_ENGINE},cpu,ragflow-go,tei-cpu,deepdoc,metadata-mysql,clickhouse"
                 else
                   echo "COMPOSE_PROFILES=${DOC_ENGINE},cpu,ragflow-go,tei-cpu,deepdoc,metadata-mysql"
                 fi
                 echo "TEI_MODEL=BAAI/bge-small-en-v1.5"
                 echo "RAGFLOW_IMAGE=${RAGFLOW_IMAGE}"
                 echo "DOC_ENGINE=${DOC_ENGINE}"
                 echo "API_PROXY_SCHEME=${API_PROXY_SCHEME}"
               } >> .env
          # Sync COMPOSE_PROFILES and TEI_MODEL into .env-go so the Go
          # server container (env_file: .env-go) sees the same TEI bypass
          # config as the Python container. Only these two affect the
          # in-process checkModelAvailable logic; other .env keys are
          # either yml-interpolation-only or already correct in .env-go.
               {
                 echo ""
                 if [ "${API_PROXY_SCHEME}" = "go" ]; then
                   echo "COMPOSE_PROFILES=${DOC_ENGINE},cpu,ragflow-go,tei-cpu,deepdoc,metadata-mysql,clickhouse"
                 else
                   echo "COMPOSE_PROFILES=${DOC_ENGINE},cpu,ragflow-go,tei-cpu,deepdoc,metadata-mysql"
                 fi
                 echo "TEI_MODEL=BAAI/bge-small-en-v1.5"
               } >> .env-go
               if [[ "${API_PROXY_SCHEME}" == "go" ]]; then
                 RAGFLOW_HTTP_PORT=${SVR_WEB_HTTP_PORT}
               else
                 RAGFLOW_HTTP_PORT=${SVR_HTTP_PORT}
               fi
               echo "RAGFLOW_HTTP_PORT=${RAGFLOW_HTTP_PORT}" >> ${GITHUB_ENV}
               echo "HOST_ADDRESS=http://host.docker.internal:${RAGFLOW_HTTP_PORT}" >> ${GITHUB_ENV}

               sed -i \
                 -e 's#${NATS_PORT:-4222}:4222#${EXPOSE_NATS_PORT}:4222#' \
                 -e 's#"8222:8222"#"${NATS_MONITORING_PORT}:8222"#' \
                 docker-compose-base.yml

               # Patch entrypoint.sh for coverage
               sed -i '/"\$PY" api\/ragflow_server.py \${INIT_SUPERUSER_ARGS} &/c\   echo "Ensuring coverage is installed..."\n      "$PY" -m pip install coverage -i https://mirrors.aliyun.com/pypi/simple\n     export COVERAGE_FILE=/ragflow/logs/.coverage\n        echo "Starting ragflow_server with coverage..."\n        "$PY" -m coverage run --source=./api/apps --omit="*/tests/*,*/migrations/*" -a api/ragflow_server.py ${INIT_SUPERUSER_ARGS} &' ./entrypoint.sh


      - name: Start ragflow:nightly for Elasticsearch
        run: |
          if [ "${API_PROXY_SCHEME}" = "go" ]; then
            COMPOSE_FILE=docker/docker-compose-go.yml
          else
            COMPOSE_FILE=docker/docker-compose.yml
          fi
          sudo docker compose -f "${COMPOSE_FILE}" -p ${COMPOSE_PROJECT_NAME} down -v || true
          sudo docker ps -a --filter "label=com.docker.compose.project=${COMPOSE_PROJECT_NAME}" -q | xargs -r sudo docker rm -f
          sudo docker compose -f "${COMPOSE_FILE}" -p ${COMPOSE_PROJECT_NAME} up -d

      - name: Run sdk tests against Elasticsearch
        if: env.API_PROXY_SCHEME == 'python'
        run: |
          export http_proxy=""; export https_proxy=""; export no_proxy=""; export HTTP_PROXY=""; export HTTPS_PROXY=""; export NO_PROXY=""
          svc_ready=0
          for i in $(seq 1 60); do
            if sudo docker exec ${RAGFLOW_CONTAINER} curl -sf --connect-timeout 5 "${HOST_ADDRESS}/api/v1/system/ping" > /dev/null 2>&1; then
              svc_ready=1
              break
            fi
            echo "Waiting for service to be available... ($i/60)"
            sleep 5
          done
          if [ "$svc_ready" -ne 1 ]; then
            echo "Service did not become ready after 5 minutes. Docker logs:"
            sudo docker logs ${RAGFLOW_CONTAINER}
            exit 1
          fi
          echo "Start to run test sdk on Elasticsearch"
          source .venv/bin/activate && set -o pipefail; pytest -s --tb=short --level=${HTTP_API_TEST_LEVEL} --junitxml=pytest-es-sdk.xml --cov=sdk/python/ragflow_sdk --cov-branch --cov-report=xml:coverage-es-sdk.xml test/testcases/test_sdk_api 2>&1 | tee es_sdk_test.log

      - name: Run New RESTFUL api tests against Elasticsearch
        if: env.API_PROXY_SCHEME != 'web'
        run: |
          export http_proxy=""; export https_proxy=""; export no_proxy=""; export HTTP_PROXY=""; export HTTPS_PROXY=""; export NO_PROXY=""
          svc_ready=0
          for i in $(seq 1 60); do
            if sudo docker exec ${RAGFLOW_CONTAINER} curl -sf --connect-timeout 5 "${HOST_ADDRESS}/api/v1/system/ping" > /dev/null 2>&1; then
              svc_ready=1
              break
            fi
            echo "Waiting for service to be available... ($i/60)"
            sleep 5
          done
          if [ "$svc_ready" -ne 1 ]; then
            echo "Service did not become ready after 5 minutes. Docker logs:"
            sudo docker logs ${RAGFLOW_CONTAINER}
            exit 1
          fi
          source .venv/bin/activate && set -o pipefail; pytest -s --tb=short --level=${HTTP_API_TEST_LEVEL} test/testcases/restful_api 2>&1 | tee es_restful_api_test.log

      - name: RAGFlow CLI retrieval test Elasticsearch
        if: env.API_PROXY_SCHEME == 'python'
        env:
          PYTHONPATH: ${{ github.workspace }}
        run: |
          set -euo pipefail
          source .venv/bin/activate

          export http_proxy=""; export https_proxy=""; export no_proxy=""; export HTTP_PROXY=""; export HTTPS_PROXY=""; export NO_PROXY=""

          EMAIL="ci-${GITHUB_RUN_ID}@example.com"
          PASS="ci-pass-${GITHUB_RUN_ID}"
          DATASET="ci_dataset_${GITHUB_RUN_ID}"

          CLI="python admin/client/ragflow_cli.py"

          LOG_FILE="es_cli_test.log"
          : > "${LOG_FILE}"

          ERROR_RE='Traceback|ModuleNotFoundError|ImportError|Parse error|Bad response|Fail to|code:\\s*[1-9]'
          run_cli() {
            local logfile="$1"
            shift
            local allow_re=""
            if [[ "${1:-}" == "--allow" ]]; then
              allow_re="$2"
              shift 2
            fi
            local cmd_display="$*"
            echo "===== $(date -u +\"%Y-%m-%dT%H:%M:%SZ\") CMD: ${cmd_display} =====" | tee -a "${logfile}"
            local tmp_log
            tmp_log="$(mktemp)"
            set +e
            timeout 500s "$@" 2>&1 | tee "${tmp_log}"
            local status=${PIPESTATUS[0]}
            set -e
            cat "${tmp_log}" >> "${logfile}"
            if grep -qiE "${ERROR_RE}" "${tmp_log}"; then
              if [[ -n "${allow_re}" ]] && grep -qiE "${allow_re}" "${tmp_log}"; then
                echo "Allowed CLI error markers in ${logfile}"
                rm -f "${tmp_log}"
                return 0
              fi
              echo "Detected CLI error markers in ${logfile}"
              rm -f "${tmp_log}"
              exit 1
            fi
            rm -f "${tmp_log}"
            return ${status}
          }

          set -a
          source docker/.env
          set +a

          USER_HOST="$(echo "${HOST_ADDRESS}" | sed -E 's#^https?://([^:/]+).*#\1#')"
          USER_PORT="${RAGFLOW_HTTP_PORT}"
          ADMIN_HOST="${USER_HOST}"
          ADMIN_PORT="${ADMIN_SVR_HTTP_PORT}"

          svc_ready=0
          for i in $(seq 1 60); do
            if sudo docker exec ${RAGFLOW_CONTAINER} curl -sf --connect-timeout 5 "${HOST_ADDRESS}/api/v1/system/ping" > /dev/null 2>&1; then
              svc_ready=1
              break
            fi
            echo "Waiting for service to be available... ($i/60)"
            sleep 5
          done
          if [ "$svc_ready" -ne 1 ]; then
            echo "Service did not become ready after 5 minutes. Docker logs:"
            sudo docker logs ${RAGFLOW_CONTAINER}
            exit 1
          fi

          admin_ready=0
          for i in $(seq 1 30); do
            if run_cli "${LOG_FILE}" $CLI --type admin --host "$ADMIN_HOST" --port "$ADMIN_PORT" --username "admin@ragflow.io" --password "admin" command "ping"; then
              admin_ready=1
              break
            fi
            sleep 1
          done
          if [[ "${admin_ready}" -ne 1 ]]; then
            echo "Admin service did not become ready"
            exit 1
          fi

          run_cli "${LOG_FILE}" $CLI --type admin --host "$ADMIN_HOST" --port "$ADMIN_PORT" --username "admin@ragflow.io" --password "admin" command "show version"
          ALLOW_USER_EXISTS_RE='already exists|already exist|duplicate|already.*registered|exist(s)?'
          run_cli "${LOG_FILE}" --allow "${ALLOW_USER_EXISTS_RE}" $CLI --type admin --host "$ADMIN_HOST" --port "$ADMIN_PORT" --username "admin@ragflow.io" --password "admin" command "create user '$EMAIL' '$PASS'"

          user_ready=0
          for i in $(seq 1 30); do
            if run_cli "${LOG_FILE}" $CLI --type user --host "$USER_HOST" --port "$USER_PORT" --username "$EMAIL" --password "$PASS" command "ping"; then
              user_ready=1
              break
            fi
            sleep 1
          done
          if [[ "${user_ready}" -ne 1 ]]; then
            echo "User service did not become ready"
            exit 1
          fi

          run_cli "${LOG_FILE}" $CLI --type user --host "$USER_HOST" --port "$USER_PORT" --username "$EMAIL" --password "$PASS" command "show version"
          run_cli "${LOG_FILE}" $CLI --type user --host "$USER_HOST" --port "$USER_PORT" --username "$EMAIL" --password "$PASS" command "create dataset '$DATASET' with embedding 'BAAI/bge-small-en-v1.5@Builtin' parser 'auto'"
          run_cli "${LOG_FILE}" $CLI --type user --host "$USER_HOST" --port "$USER_PORT" --username "$EMAIL" --password "$PASS" command "import 'test/benchmark/test_docs/Doc1.pdf,test/benchmark/test_docs/Doc2.pdf' into dataset '$DATASET'"
          run_cli "${LOG_FILE}" $CLI --type user --host "$USER_HOST" --port "$USER_PORT" --username "$EMAIL" --password "$PASS" command "parse dataset '$DATASET' sync"
          run_cli "${LOG_FILE}" $CLI --type user --host "$USER_HOST" --port "$USER_PORT" --username "$EMAIL" --password "$PASS" command "Benchmark 16 100 search 'what are these documents about' on datasets '$DATASET'"

      - name: Stop ragflow to save coverage Elasticsearch
        if: ${{ !cancelled() && env.API_PROXY_SCHEME == 'python' }}
        run: |
          # Send SIGINT to ragflow_server.py to trigger coverage save
          PID=$(sudo docker exec ${RAGFLOW_CONTAINER} ps aux | grep "ragflow_server.py" | grep -v grep | awk '{print $2}' | head -n 1)
          if [ -n "$PID" ]; then
            echo "Sending SIGINT to ragflow_server.py (PID: $PID)..."
            sudo docker exec ${RAGFLOW_CONTAINER} kill -INT $PID
            # Wait for process to exit and coverage file to be written
            sleep 10
          else
            echo "ragflow_server.py not found!"
          fi
          sudo docker compose -f docker/docker-compose.yml -p ${COMPOSE_PROJECT_NAME} stop

      - name: Generate server coverage report Elasticsearch
        if: ${{ !cancelled() && env.API_PROXY_SCHEME == 'python' }}
        run: |
          # .coverage file should be in docker/ragflow-logs/.coverage
          if [ -f docker/ragflow-logs/.coverage ]; then
            echo "Found .coverage file"
            cp docker/ragflow-logs/.coverage .coverage
            source .venv/bin/activate
            # Create .coveragerc to map container paths to host paths
            echo "[paths]" > .coveragerc
            echo "source =" >> .coveragerc
            echo "    ." >> .coveragerc
            echo "    /ragflow" >> .coveragerc
            coverage xml -o coverage-es-server.xml
            rm .coveragerc
            # Clean up for next run
            sudo rm docker/ragflow-logs/.coverage
          else
            echo ".coverage file not found!"
          fi

      - name: Collect ragflow log Elasticsearch
        if: ${{ !cancelled() }}
        run: |
          if [ -d docker/ragflow-logs ]; then
            destination="${ARTIFACTS_DIR}/ragflow-logs-es"
            sudo cp -r docker/ragflow-logs "${destination}"
            sudo chown -R "$(id -u):$(id -g)" "${destination}"
            echo "ragflow log" && sudo tail -n 200 docker/ragflow-logs/ragflow_server.log || true
          else
            echo "No docker/ragflow-logs directory found; skipping log collection"
          fi
          sudo rm -rf docker/ragflow-logs || true

      - name: Stop ragflow:nightly for Elasticsearch
        if: ${{ always() }}
        run: |
          if [ "${API_PROXY_SCHEME}" = "go" ]; then
            COMPOSE_FILE=docker/docker-compose-go.yml
          else
            COMPOSE_FILE=docker/docker-compose.yml
          fi
          # Sometimes `docker compose down` fail due to hang container, heavy load etc. Need to remove such containers to release resources(for example, listen ports).
          sudo docker compose -f "${COMPOSE_FILE}" -p ${COMPOSE_PROJECT_NAME} down -v || true
          sudo docker ps -a --filter "label=com.docker.compose.project=${COMPOSE_PROJECT_NAME}" -q | xargs -r sudo docker rm -f
          if [[ -n ${RAGFLOW_IMAGE} ]]; then
            sudo docker rmi -f ${RAGFLOW_IMAGE}
          fi
          if [[ -n ${PORT_RESERVATION:-} ]]; then
            # PORT_RESERVATION is a space-separated list of per-port docker network guards
            # created by the prepare step. Remove each to free the ports for other runners.
            for guard in ${PORT_RESERVATION}; do
              sudo docker network rm "${guard}" >/dev/null 2>&1 || true
            done
          fi

  ragflow_native_backend:
    name: ragflow_native_backend
    # Runs AFTER both per-engine go jobs (which contain the doc-engine-specific
    # server build + Python RESTFUL correctness runs). The DeepDoc native
    # backend is doc-engine-independent, so running it ONCE here — instead of
    # duplicated (and memory-competing) inside each go job — removes the
    # redundant second -race pass and the concurrent -race OOM risk. Running it
    # last also keeps a flaky -race OOM from ever obscuring the functional
    # verdict produced by the go jobs.
    needs: [ragflow_tests_infinity, ragflow_tests_elasticsearch]
    # Skip unless DeepDoc native backend code/build/testdata actually changed.
    if: ${{ (github.event_name != 'pull_request' && github.event_name != 'pull_request_target' || (github.event.pull_request.draft == false && contains(github.event.pull_request.labels.*.name, 'ci') && (github.event.action != 'labeled' || github.event.label.name == 'ci'))) && needs.ragflow_preflight.outputs.has_native_deepdoc_changes == 'true' }}
    runs-on: [ "self-hosted", "ragflow-test" ]
    # Shared by the assert step and the test step below so the mount path is
    # declared once.
    env:
      RAGFLOW_TESTDATA_DIR: /opt/ragflow-testdata
    steps:
      - name: Ensure workspace ownership
        run: |
          echo "Workflow triggered by ${{ github.event_name }}"
          echo "chown -R ${USER} ${GITHUB_WORKSPACE}" && sudo chown -R ${USER} ${GITHUB_WORKSPACE}

      - name: Check out code
        uses: actions/checkout@v6
        with:
          ref: ${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && format('refs/pull/{0}/merge', github.event.pull_request.number) || github.sha }}
          fetch-depth: 0
          fetch-tags: true
          allow-unsafe-pr-checkout: true

      - name: Build C++ static library (ragflow native backend)
        # Produces librag_tokenizer_c_api.a with the re2 symbols renamed into
        # the ragtokre2_ namespace (so it no longer collides with onnxruntime's
        # bundled re2 copy at link time). Only the .a is needed for the native
        # integration tests; the per-engine go jobs build the full server.
        run: |
          set -euo pipefail
          BUILDER_CONTAINER=ragflow_build_${GITHUB_RUN_ID}_native_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ')
          cleanup_builder() {
            if [[ -n "${BUILDER_CONTAINER:-}" ]]; then
              sudo docker rm -f -v "${BUILDER_CONTAINER}" >/dev/null 2>&1 || true
            fi
          }
          trap cleanup_builder EXIT

          TZ=${TZ:-$(readlink -f /etc/localtime | awk -F '/zoneinfo/' '{print $2}')}
          sudo docker run --privileged -d --name "${BUILDER_CONTAINER}" \
            -e TZ="${TZ}" \
            -e UV_INDEX=https://mirrors.aliyun.com/pypi/simple \
            -v "${PWD}:/ragflow" \
            -v "${PWD}/internal/binding/cpp/resource:/usr/share/infinity/resource" \
            infiniflow/infinity_builder:ubuntu22_clang20
          sudo docker exec "${BUILDER_CONTAINER}" bash -c 'git config --global safe.directory "*" && cd /ragflow && ./build.sh --cpp'

      - name: Assert pre-seeded DeepDoc testdata
        # Only the self-hosted ragflow-test runners bake infiniflow/ragflow-testdata
        # into /opt/ragflow-testdata — the same image-level contract as
        # MODEL_DIR=/opt/ragflow-deepdoc-models, which has no fallback either.
        # Fail here rather than letting the fetch script silently fall back to
        # cloning: a missing mount is an environment defect, and a silent
        # fallback only makes the job slower while hiding it.
        #
        # Assert the `deepdoc` level, not a per-package subtree, so adding a
        # package later does not invalidate the check.
        run: |
          set -euo pipefail
          if [ ! -d "${RAGFLOW_TESTDATA_DIR}/deepdoc" ]; then
            echo "ERROR: ${RAGFLOW_TESTDATA_DIR}/deepdoc missing on this runner." >&2
            echo "       Native backend tests expect the asset repo pre-seeded there." >&2
            exit 1
          fi
          echo "✓ pre-seeded testdata found at ${RAGFLOW_TESTDATA_DIR}/deepdoc:"
          ls -l "${RAGFLOW_TESTDATA_DIR}/deepdoc"

      - name: Run in-process DeepDoc backend integration tests (correctness + race)
        # Doc-engine-independent: the same native backend (det/layout/tsr/rec
        # ONNX models + onnxruntime) is exercised regardless of DOC_ENGINE, so
        # it is run ONCE here instead of duplicated inside each go job.
        #
        # Inside build.sh, the golden/comparison tests run WITHOUT -race (no
        # race-detection value, ~10x memory tax) and only the concurrency
        # correctness tests run WITH -race (see run_native_integration_tests).
        #
        # Testdata is pre-seeded on the runner at /opt/ragflow-testdata
        # (asserted by the previous step). RAGFLOW_TESTDATA_DIR points the fetch
        # script at it so the network clone is skipped (see
        # scripts/fetch_deepdoc_testdata.sh) — which is why no actions/cache step
        # is needed here. XDG_CACHE_HOME remains the location the script falls
        # back to when it runs outside this job (local dev, no mount).
        env:
          MODEL_DIR: /opt/ragflow-deepdoc-models
          CGO_ENABLED: '1'
          DEEPDOC_NATIVE_REQUIRED: '1'
          XDG_CACHE_HOME: ${{ github.workspace }}/.cache
        run: |
          set -euo pipefail
          ./build.sh --test-native