Fix: pass the kb id to the wiki page insert-error warning (#18985) #13453
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: sep-tests | |
| permissions: | |
| contents: read | |
| on: | |
| push: | |
| branches: | |
| - 'main' | |
| - '*.*.*' | |
| paths-ignore: | |
| - 'docs/**' | |
| - '*.md' | |
| - '*.mdx' | |
| # The only difference between pull_request and pull_request_target is the context in which the workflow runs: | |
| # — pull_request_target workflows use the workflow files from the default branch, and secrets are available. | |
| # — pull_request workflows use the workflow files from the pull request branch, and secrets are unavailable. | |
| # We use pull_request_target so that GITEE_TOKEN (and other repo secrets) are available when building | |
| # ragflow:nightly from a fork PR, while still running the full test suite on PRs labeled with `ci`. | |
| pull_request: | |
| types: [synchronize, labeled] | |
| paths-ignore: | |
| - 'docs/**' | |
| - '*.md' | |
| - '*.mdx' | |
| schedule: | |
| - cron: '0 16 * * *' # This schedule runs every 16:00:00Z(00:00:00+08:00) | |
| # https://docs.github.com/en/actions/using-jobs/using-concurrency | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}${{ github.event.action == 'labeled' && github.event.label.name != 'ci' && format('-ignore-{0}', github.run_id) || '' }} | |
| cancel-in-progress: true | |
| jobs: | |
| ragflow_preflight: | |
| name: ragflow_preflight | |
| # https://docs.github.com/en/actions/using-jobs/using-conditions-to-control-job-execution | |
| # https://github.com/orgs/community/discussions/26261 | |
| if: ${{ github.event_name != 'pull_request' && github.event_name != 'pull_request_target' || (github.event.pull_request.draft == false && contains(github.event.pull_request.labels.*.name, 'ci') && (github.event.action != 'labeled' || github.event.label.name == 'ci')) }} | |
| permissions: | |
| actions: write | |
| contents: read | |
| runs-on: [ "self-hosted", "ragflow-test" ] | |
| outputs: | |
| http_api_test_level: ${{ steps.test_level.outputs.http_api_test_level }} | |
| has_go_changes: ${{ steps.detect_changes.outputs.has_go_changes }} | |
| has_python_changes: ${{ steps.detect_changes.outputs.has_python_changes }} | |
| has_web_changes: ${{ steps.detect_changes.outputs.has_web_changes }} | |
| api_proxy_schemes: ${{ steps.detect_changes.outputs.api_proxy_schemes }} | |
| steps: | |
| - name: Ensure workspace ownership | |
| run: | | |
| echo "Workflow triggered by ${{ github.event_name }}" | |
| echo "chown -R ${USER} ${GITHUB_WORKSPACE}" && sudo chown -R ${USER} ${GITHUB_WORKSPACE} | |
| # https://github.com/actions/checkout/issues/1781 | |
| - name: Check out code | |
| uses: actions/checkout@v6 | |
| with: | |
| ref: ${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && format('refs/pull/{0}/merge', github.event.pull_request.number) || github.sha }} | |
| fetch-depth: 0 | |
| fetch-tags: true | |
| allow-unsafe-pr-checkout: true | |
| - name: Check workflow duplication | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| if [[ ${GITHUB_EVENT_NAME} != "pull_request" && ${GITHUB_EVENT_NAME} != "pull_request_target" && ${GITHUB_EVENT_NAME} != "schedule" ]]; then | |
| HEAD=$(git rev-parse HEAD) | |
| # Find a PR that introduced a given commit | |
| gh auth login --with-token <<< "${{ secrets.GITHUB_TOKEN }}" | |
| PR_NUMBER=$(gh pr list --search ${HEAD} --state merged --json number --jq .[0].number) | |
| echo "HEAD=${HEAD}" | |
| echo "PR_NUMBER=${PR_NUMBER}" | |
| if [[ -n "${PR_NUMBER}" ]]; then | |
| PR_SHA_FP=${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/PR_${PR_NUMBER} | |
| if [[ -f "${PR_SHA_FP}" ]]; then | |
| read -r PR_SHA PR_RUN_ID < "${PR_SHA_FP}" | |
| # Calculate the hash of the current workspace content | |
| HEAD_SHA=$(git rev-parse HEAD^{tree}) | |
| if [[ "${HEAD_SHA}" == "${PR_SHA}" ]]; then | |
| echo "Cancel myself since the workspace content hash is the same with PR #${PR_NUMBER} merged. See ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${PR_RUN_ID} for details." | |
| gh run cancel ${GITHUB_RUN_ID} | |
| while true; do | |
| status=$(gh run view ${GITHUB_RUN_ID} --json status -q .status) | |
| [ "${status}" = "completed" ] && break | |
| sleep 5 | |
| done | |
| exit 1 | |
| fi | |
| fi | |
| fi | |
| elif [[ ${GITHUB_EVENT_NAME} == "pull_request" || ${GITHUB_EVENT_NAME} == "pull_request_target" ]]; then | |
| PR_NUMBER=${{ github.event.pull_request.number }} | |
| PR_SHA_FP=${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/PR_${PR_NUMBER} | |
| # Calculate the hash of the current workspace content | |
| PR_SHA=$(git rev-parse HEAD^{tree}) | |
| echo "PR #${PR_NUMBER} workspace content hash: ${PR_SHA}" | |
| mkdir -p ${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY} | |
| echo "${PR_SHA} ${GITHUB_RUN_ID}" > ${PR_SHA_FP} | |
| fi | |
| ARTIFACTS_DIR=${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/${GITHUB_RUN_ID} | |
| echo "ARTIFACTS_DIR=${ARTIFACTS_DIR}" >> ${GITHUB_ENV} | |
| rm -rf ${ARTIFACTS_DIR} && mkdir -p ${ARTIFACTS_DIR} | |
| # - name: Check comments of changed Python files | |
| # if: ${{ false }} | |
| # run: | | |
| # if [[ ${{ github.event_name }} == 'pull_request' || ${{ github.event_name }} == 'pull_request_target' ]]; then | |
| # CHANGED_FILES=$(git diff --name-only ${{ github.event.pull_request.base.sha }}...${{ github.event.pull_request.head.sha }} \ | |
| # | grep -E '\.(py)$' || true) | |
| # | |
| # if [ -n "$CHANGED_FILES" ]; then | |
| # echo "Check comments of changed Python files with check_comment_ascii.py" | |
| # | |
| # readarray -t files <<< "$CHANGED_FILES" | |
| # HAS_ERROR=0 | |
| # | |
| # for file in "${files[@]}"; do | |
| # if [ -f "$file" ]; then | |
| # if python3 check_comment_ascii.py "$file"; then | |
| # echo "✅ $file" | |
| # else | |
| # echo "❌ $file" | |
| # HAS_ERROR=1 | |
| # fi | |
| # fi | |
| # done | |
| # | |
| # if [ $HAS_ERROR -ne 0 ]; then | |
| # exit 1 | |
| # fi | |
| # else | |
| # echo "No Python files changed" | |
| # fi | |
| # fi | |
| - name: Run Lefthook on changed files | |
| run: | | |
| set -euo pipefail | |
| if [[ "${GITHUB_EVENT_NAME}" == "pull_request" || "${GITHUB_EVENT_NAME}" == "pull_request_target" ]]; then | |
| changed_files=$(mktemp) | |
| trap 'rm -f "$changed_files"' EXIT | |
| git diff --name-only -z ${{ github.event.pull_request.base.sha }}...${{ github.event.pull_request.head.sha }} \ | |
| | while IFS= read -r -d '' file; do | |
| if [[ -f "$file" ]]; then | |
| printf '%s\0' "$file" | |
| fi | |
| done > "$changed_files" | |
| echo "Changed files to run lefthook on:" | |
| if [[ -s "$changed_files" ]]; then | |
| while IFS= read -r -d '' file; do | |
| printf ' %q\n' "$file" | |
| done < "$changed_files" | |
| else | |
| echo " (none — lefthook will be a no-op)" | |
| fi | |
| # LEFTHOOK_CHECK_ONLY=1 makes the pre-commit jobs verify without | |
| # applying --fix or `git add`, so CI only checks and reports | |
| # failures instead of rewriting the working tree. | |
| LEFTHOOK_CHECK_ONLY=1 lefthook run pre-commit --files-from-stdin --no-auto-install < "$changed_files" | |
| fi | |
| - name: Set test level | |
| id: test_level | |
| run: | | |
| set -euo pipefail | |
| if [[ ${GITHUB_EVENT_NAME} == "schedule" ]]; then | |
| export HTTP_API_TEST_LEVEL=p3 | |
| else | |
| export HTTP_API_TEST_LEVEL=p2 | |
| fi | |
| echo "HTTP_API_TEST_LEVEL=${HTTP_API_TEST_LEVEL}" >> ${GITHUB_ENV} | |
| echo "http_api_test_level=${HTTP_API_TEST_LEVEL}" >> ${GITHUB_OUTPUT} | |
| - name: Detect changed file types | |
| id: detect_changes | |
| run: | | |
| set -euo pipefail | |
| has_go=false | |
| has_python=false | |
| has_web=false | |
| if [[ "${GITHUB_EVENT_NAME}" == "schedule" ]]; then | |
| has_go=true | |
| has_python=true | |
| has_web=true | |
| else | |
| if [[ "${GITHUB_EVENT_NAME}" == "pull_request" || "${GITHUB_EVENT_NAME}" == "pull_request_target" ]]; then | |
| CHANGED_FILES=$(git diff --name-only ${{ github.event.pull_request.base.sha }}...${{ github.event.pull_request.head.sha }}) | |
| else | |
| BASE_SHA="${{ github.event.before }}" | |
| if [[ -n "${BASE_SHA}" && ! "${BASE_SHA}" =~ ^0+$ ]]; then | |
| CHANGED_FILES=$(git diff --name-only "${BASE_SHA}" "${GITHUB_SHA}") | |
| else | |
| CHANGED_FILES=$(git diff --name-only HEAD~1 HEAD) | |
| fi | |
| fi | |
| while IFS= read -r file; do | |
| case "$file" in | |
| # Docker, test, SDK, and workflow changes can alter either server path. | |
| Dockerfile|Dockerfile_ci|docker/**|.github/workflows/**|sdk/**) | |
| has_go=true | |
| has_python=true | |
| ;; | |
| test/unit_test/**) | |
| has_python=true | |
| ;; | |
| test/**) | |
| has_go=true | |
| has_python=true | |
| ;; | |
| *.go|go.mod|go.sum|build.sh) has_go=true ;; | |
| *.py|pyproject.toml|requirements*.txt) has_python=true ;; | |
| web/*) has_web=true ;; | |
| esac | |
| done <<< "$CHANGED_FILES" | |
| if [[ "$has_go" == "false" && "$has_python" == "false" && "$has_web" == "false" ]]; then | |
| has_go=true | |
| has_python=true | |
| has_web=true | |
| fi | |
| fi | |
| if [[ "$has_go" == "true" && "$has_python" == "true" ]]; then | |
| api_proxy_schemes='["go","python"]' | |
| elif [[ "$has_go" == "true" ]]; then | |
| api_proxy_schemes='["go"]' | |
| elif [[ "$has_python" == "true" ]]; then | |
| api_proxy_schemes='["python"]' | |
| elif [[ "$has_web" == "true" ]]; then | |
| api_proxy_schemes='["web"]' | |
| else | |
| api_proxy_schemes='["go","python"]' | |
| fi | |
| echo "has_go_changes=${has_go}" >> $GITHUB_OUTPUT | |
| echo "has_python_changes=${has_python}" >> $GITHUB_OUTPUT | |
| echo "has_web_changes=${has_web}" >> $GITHUB_OUTPUT | |
| echo "api_proxy_schemes=${api_proxy_schemes}" >> $GITHUB_OUTPUT | |
| echo "Go: ${has_go}, Python: ${has_python}, Web: ${has_web}, proxy schemes: ${api_proxy_schemes}" | |
| - name: Prepare Python test environment | |
| if: steps.detect_changes.outputs.has_python_changes == 'true' | |
| run: | | |
| git config --global url."https://oauth2:${GITEE_TOKEN}@gitee.com/".insteadOf "https://gitee.com/" | |
| uv sync --python 3.13 --group test --frozen | |
| uv pip install -e sdk/python | |
| - name: Run unit test | |
| if: steps.detect_changes.outputs.has_python_changes == 'true' | |
| run: | | |
| source .venv/bin/activate | |
| which pytest || echo "pytest not in PATH" | |
| echo "Start to run unit test" | |
| python3 run_tests.py -i | |
| ragflow_tests_infinity: | |
| name: ragflow_tests_infinity (${{ matrix.api_proxy_scheme }}) | |
| needs: ragflow_preflight | |
| if: ${{ github.event_name != 'pull_request' && github.event_name != 'pull_request_target' || (github.event.pull_request.draft == false && contains(github.event.pull_request.labels.*.name, 'ci') && (github.event.action != 'labeled' || github.event.label.name == 'ci')) }} | |
| runs-on: [ "self-hosted", "ragflow-test" ] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| api_proxy_scheme: ${{ fromJSON(needs.ragflow_preflight.outputs.api_proxy_schemes) }} | |
| env: | |
| DOC_ENGINE: infinity | |
| API_PROXY_SCHEME: ${{ matrix.api_proxy_scheme }} | |
| RAGFLOW_IMAGE: infiniflow/ragflow:${{ github.run_id }}-infinity-${{ matrix.api_proxy_scheme }} | |
| HTTP_API_TEST_LEVEL: ${{ needs.ragflow_preflight.outputs.http_api_test_level }} | |
| HAS_GO: ${{ needs.ragflow_preflight.outputs.has_go_changes }} | |
| HAS_PYTHON: ${{ needs.ragflow_preflight.outputs.has_python_changes }} | |
| HAS_WEB: ${{ needs.ragflow_preflight.outputs.has_web_changes }} | |
| steps: | |
| - name: Ensure workspace ownership | |
| if: always() | |
| run: | | |
| echo "Workflow triggered by ${{ github.event_name }}" | |
| echo "chown -R ${USER} ${GITHUB_WORKSPACE}" && sudo chown -R ${USER} ${GITHUB_WORKSPACE} | |
| - name: Check out code | |
| if: always() | |
| uses: actions/checkout@v6 | |
| with: | |
| ref: ${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && format('refs/pull/{0}/merge', github.event.pull_request.number) || github.sha }} | |
| fetch-depth: 0 | |
| fetch-tags: true | |
| allow-unsafe-pr-checkout: true | |
| - name: Build ragflow go server | |
| if: env.API_PROXY_SCHEME == 'go' | |
| run: | | |
| set -euo pipefail | |
| BUILDER_CONTAINER=ragflow_build_${GITHUB_RUN_ID}_${DOC_ENGINE}_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ') | |
| cleanup_builder() { | |
| if [[ -n "${BUILDER_CONTAINER:-}" ]]; then | |
| sudo docker rm -f -v "${BUILDER_CONTAINER}" >/dev/null 2>&1 || true | |
| fi | |
| } | |
| trap cleanup_builder EXIT | |
| TZ=${TZ:-$(readlink -f /etc/localtime | awk -F '/zoneinfo/' '{print $2}')} | |
| sudo docker run --privileged -d --name "${BUILDER_CONTAINER}" \ | |
| -e TZ="${TZ}" \ | |
| -e UV_INDEX=https://mirrors.aliyun.com/pypi/simple \ | |
| -v "${PWD}:/ragflow" \ | |
| -v "${PWD}/internal/binding/cpp/resource:/usr/share/infinity/resource" \ | |
| infiniflow/infinity_builder:ubuntu22_clang20 | |
| sudo docker exec "${BUILDER_CONTAINER}" bash -c 'git config --global safe.directory "*" && cd /ragflow && ./build.sh --cpp' | |
| ./build.sh --go | |
| - name: Run Go unit tests | |
| if: env.API_PROXY_SCHEME == 'go' | |
| # Runs after `./build.sh --go`, which guarantees the C++ static | |
| # library (librag_tokenizer_c_api.a) is present on disk. The Go | |
| # test binaries link against it transitively through | |
| # `internal/binding`, so running `go test` before the C++ build | |
| # fails the link step. | |
| # | |
| # Excludes packages whose tests fail for environmental reasons | |
| # unrelated to the diff: | |
| # - internal/tokenizer is split: the pure-Go BPE loader tests | |
| # (bpe_loader_test.go) run in the default tier; the C++ binding / | |
| # dict-dependent tests (tokenizer_test.go, | |
| # tokenizer_concurrent_test.go) and the on-disk anchor test | |
| # (bpe_loader_anchors_test.go) are tagged `manual` and need the | |
| # docker builder's /usr/share/infinity/resource, so they stay out | |
| # of the default run. | |
| run: | | |
| set -euo pipefail | |
| # Provide the cl100k BPE table for the offline tokenizer loader. | |
| # The loader reads <repo>/ragflow_deps/cl100k_base.tiktoken (or | |
| # TIKTOKEN_CACHE_DIR). Try to fetch it directly from the upstream | |
| # openai blob first (small ~1.6MB, reachable from the CI runner — the | |
| # pre-PR code downloaded it on the fly, so the network path is known | |
| # good). If the direct fetch fails (e.g. GFW), fall back to extracting | |
| # it from the local infiniflow/ragflow_deps image *if already present* | |
| # on the runner (no `docker pull`, since that huge image is first | |
| # fetched at the later "Build ragflow:nightly" step). Best-effort: if | |
| # neither source works, the loader still fails loudly (no silent | |
| # 0-token degradation). | |
| # tiktoken is already installed in runner | |
| mkdir -p ragflow_deps | |
| cp /opt/ragflow_deps/cl100k_base.tiktoken ragflow_deps/cl100k_base.tiktoken || true | |
| if [ ! -f ragflow_deps/cl100k_base.tiktoken ]; then | |
| mkdir -p ragflow_deps | |
| if command -v curl >/dev/null 2>&1; then | |
| if curl -fsSL -o ragflow_deps/cl100k_base.tiktoken \ | |
| https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken; then | |
| echo "tiktoken: cl100k table fetched via curl from openai blob" | |
| else | |
| echo "tiktoken: curl fetch failed (network/GFW); will try cached image" | |
| fi | |
| fi | |
| if [ ! -f ragflow_deps/cl100k_base.tiktoken ] && docker image inspect infiniflow/ragflow_deps:latest >/dev/null 2>&1; then | |
| CID=$(docker create infiniflow/ragflow_deps:latest true) || true | |
| if [ -n "${CID:-}" ]; then | |
| if docker cp "$CID":/cl100k_base.tiktoken ragflow_deps/cl100k_base.tiktoken 2>/dev/null; then | |
| echo "tiktoken: cl100k table copied from cached infiniflow/ragflow_deps image" | |
| fi | |
| docker rm -f "$CID" >/dev/null 2>&1 || true | |
| fi | |
| fi | |
| fi | |
| if [ -f ragflow_deps/cl100k_base.tiktoken ]; then | |
| echo "tiktoken: cl100k table provisioned ($(wc -c < ragflow_deps/cl100k_base.tiktoken) bytes)" | |
| else | |
| echo "tiktoken: cl100k table NOT provisioned — offline loader will fail loudly" | |
| fi | |
| PKGS=$(go list ./... 2>/dev/null \ | |
| | grep -v '/internal/storage$' \ | |
| | grep -v '/internal/handler$' || true) | |
| if [ -z "$PKGS" ]; then | |
| ./build.sh --test | |
| else | |
| ./build.sh --test -- $PKGS | |
| fi | |
| - name: Build ragflow:nightly | |
| run: | | |
| set -euo pipefail | |
| RUNNER_WORKSPACE_PREFIX=${RUNNER_WORKSPACE_PREFIX:-${HOME}} | |
| BUILD_LOCK_FILE=${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/docker-build.lock | |
| mkdir -p "$(dirname "${BUILD_LOCK_FILE}")" | |
| # Pass the Gitee personal access token to the Docker build as a BuildKit secret | |
| # (mounted inside the builder at /run/secrets/gitee_token, never persisted in any layer). | |
| GITEE_TOKEN_FILE="$(mktemp)" | |
| printf '%s' '${GITEE_TOKEN}' > "${GITEE_TOKEN_FILE}" | |
| ( | |
| flock -w 10800 9 || { echo "Timed out waiting for the shared Docker build slot" >&2; exit 1; } | |
| echo "Acquired Docker build slot for ${DOC_ENGINE}/${API_PROXY_SCHEME}" | |
| ##sudo docker pull ubuntu:24.04 | |
| sudo DOCKER_BUILDKIT=1 docker build --build-arg NEED_MIRROR=1 --build-arg HTTPS_PROXY=${HTTPS_PROXY} --build-arg HTTP_PROXY=${HTTP_PROXY} --secret id=gitee_token,src=${GITEE_TOKEN_FILE} -f Dockerfile_ci -t ${RAGFLOW_IMAGE} . | |
| ) 9>"${BUILD_LOCK_FILE}" | |
| rm -f "${GITEE_TOKEN_FILE}" | |
| - name: Prepare Python test environment | |
| if: env.API_PROXY_SCHEME != 'web' | |
| run: | | |
| git config --global url."https://oauth2:${GITEE_TOKEN}@gitee.com/".insteadOf "https://gitee.com/" | |
| uv sync --python 3.13 --group test --frozen | |
| uv pip install -e sdk/python | |
| - name: Prepare function test environment | |
| working-directory: docker | |
| run: | | |
| set -euo pipefail | |
| sudo chmod 1777 /tmp | |
| ## ss already installed | |
| ##sudo apt update && sudo apt install -y iproute2 | |
| RUNNER_WORKSPACE_PREFIX=${RUNNER_WORKSPACE_PREFIX:-${HOME}} | |
| COMPOSE_PROJECT_NAME="${GITHUB_RUN_ID}-${DOC_ENGINE}-${API_PROXY_SCHEME}" | |
| echo "COMPOSE_PROJECT_NAME=${COMPOSE_PROJECT_NAME}" >> ${GITHUB_ENV} | |
| echo "RAGFLOW_CONTAINER=${COMPOSE_PROJECT_NAME}-ragflow-cpu-1" >> ${GITHUB_ENV} | |
| ARTIFACTS_DIR=${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/${GITHUB_RUN_ID}/${DOC_ENGINE}/${API_PROXY_SCHEME} | |
| echo "ARTIFACTS_DIR=${ARTIFACTS_DIR}" >> ${GITHUB_ENV} | |
| rm -rf "${ARTIFACTS_DIR}" && mkdir -p "${ARTIFACTS_DIR}" | |
| # Determine runner number (default to 1 if not found) | |
| RUNNER_NUM=$(sudo docker inspect $(hostname) --format '{{index .Config.Labels "com.docker.compose.container-number"}}' 2>/dev/null || true) | |
| if [[ -z "${RUNNER_NUM:-}" ]]; then | |
| # GitHub self-hosted runners always expose RUNNER_NAME; hash it to spread candidates. | |
| if [[ -n "${RUNNER_NAME:-}" ]]; then | |
| RUNNER_NUM=$(( $(printf '%d' 0x$(echo -n "${RUNNER_NAME}" | md5sum | cut -c1-4)) % 1000 )) | |
| else | |
| RUNNER_NUM=$(( GITHUB_RUN_ID % 1000 )) | |
| fi | |
| fi | |
| # Engine-specific offset partitions keep concurrent engine jobs from | |
| # choosing the same host ports when they land on the same self-hosted runner. | |
| # A lock plus reservation file closes the check/start race between parallel jobs. | |
| PORT_BASES=(1200 1201 23817 23820 5432 5455 9000 9001 6379 6380 6601 9380 9381 9382 9384 9383 9385 80 443 4222 8222) | |
| PARTITION_SIZE=6000 | |
| case "${DOC_ENGINE}" in | |
| elasticsearch) PARTITION_BASE=1000 ;; | |
| infinity) PARTITION_BASE=31000 ;; | |
| *) echo "Unsupported DOC_ENGINE=${DOC_ENGINE}" >&2; exit 1 ;; | |
| esac | |
| # Runners run inside docker containers that share the host docker daemon | |
| # (/var/run/docker.sock), so neither /tmp nor `ss` can coordinate across runners: | |
| # each container has its own /tmp and its own network namespace, and `ss` cannot | |
| # see host ports bound by other runners' compose stacks. Use the shared daemon for | |
| # both primitives: | |
| # - per-port mutex = for each host port (base+offset) we intend to bind, create | |
| # a docker network named ragflow-ci-port-<port> on the shared daemon. docker | |
| # network create is NOT idempotent (fails on name conflict), so only one runner | |
| # can claim a given port. This is per-PORT, not per-offset, because different | |
| # (base, offset) pairs can produce the same host port (e.g. 6380+36497 == | |
| # 9380+33497 == 42877); a per-offset guard cannot prevent that collision. | |
| # - port availability = docker ps published ports (sees every container on the | |
| # shared daemon regardless of which runner started it). | |
| # No file-system sharing required, no host netns required. | |
| cleanup_stale_port_locks() { | |
| local now stale_after net ts before after | |
| now=$(date -u +%s) | |
| stale_after=$((60 * 60)) | |
| before=$(sudo docker network ls --filter "label=ragflow-ci-created" --format '{{.Name}}' | wc -l) | |
| for net in $(sudo docker network ls --filter "label=ragflow-ci-created" --format '{{.Name}}'); do | |
| ts=$(sudo docker network inspect --format '{{ index .Labels "ragflow-ci-created" }}' "${net}" 2>/dev/null || true) | |
| # Remove if timestamp is non-numeric (legacy network from a previous code | |
| # version) or if it is numeric and older than stale_after. | |
| if [[ ! "${ts}" =~ ^[0-9]+$ ]] || (( now - ts > stale_after )); then | |
| sudo docker network rm "${net}" >/dev/null 2>&1 || true | |
| fi | |
| done | |
| after=$(sudo docker network ls --filter "label=ragflow-ci-created" --format '{{.Name}}' | wc -l) | |
| echo " cleanup_stale_port_locks: ${before} -> ${after} guards (TTL=${stale_after}s)" >&2 | |
| } | |
| reserve_port_offset() { | |
| local attempt candidate base port guard g ok reject_reason create_err guard_subnet | |
| local -a guards | |
| local used guarded need_refresh used_n guarded_n debug_count create_fail_streak | |
| cleanup_stale_port_locks | |
| echo "Reserving host ports for ${DOC_ENGINE} compose stack..." >&2 | |
| # Fail fast if the shared docker daemon is not reachable — avoids a | |
| # long spin where every docker network create silently fails. | |
| if ! sudo docker info >/dev/null 2>&1; then | |
| echo " ERROR: 'sudo docker info' failed — docker daemon is not reachable from this runner" >&2 | |
| return 1 | |
| fi | |
| echo " Running containers on shared daemon: $(sudo docker ps -q 2>/dev/null | wc -l)" >&2 | |
| echo " Docker networks on shared daemon: $(sudo docker network ls -q 2>/dev/null | wc -l)" >&2 | |
| need_refresh=1 | |
| debug_count=0 | |
| create_fail_streak=0 | |
| for attempt in $(seq 0 499); do | |
| candidate=$(( PARTITION_BASE + ((GITHUB_RUN_ID + RUNNER_NUM * 1000 + attempt * 97) % PARTITION_SIZE) )) | |
| # Refresh snapshots on first attempt and after a failed reservation. | |
| if [[ ${need_refresh} -eq 1 ]]; then | |
| used=$(sudo docker ps --format '{{.Ports}}' 2>/dev/null | grep -oE '[0-9]+->' | tr -d '>-' | sort -u || true) | |
| # Strip both ragflow-ci-port- (current) and ragflow-ci-portguard- (legacy) prefixes. | |
| guarded=$(sudo docker network ls --filter "label=ragflow-ci-created" --format '{{.Name}}' 2>/dev/null | sed 's/^ragflow-ci-port\(guard\)\?-//' | sort -u || true) | |
| used_n=$(echo "${used}" | grep -c . 2>/dev/null || true) | |
| guarded_n=$(echo "${guarded}" | grep -c . 2>/dev/null || true) | |
| echo " Snapshot: ${used_n} bound ports, ${guarded_n} guarded ports" >&2 | |
| [[ ${used_n} -gt 0 ]] && echo " Bound sample: $(echo "${used}" | tr '\n' ' ' | cut -d' ' -f1-10)" >&2 | |
| [[ ${guarded_n} -gt 0 ]] && echo " Guarded sample: $(echo "${guarded}" | tr '\n' ' ' | cut -d' ' -f1-10)" >&2 | |
| need_refresh=0 | |
| fi | |
| # Fast path: skip if any (base+candidate) port is already bound or guarded. | |
| ok=1 | |
| reject_reason="" | |
| for base in "${PORT_BASES[@]}"; do | |
| port=$((base + candidate)) | |
| if grep -qx "${port}" <<< "${used}" 2>/dev/null; then | |
| ok=0; reject_reason="port ${port} (base ${base}) already bound"; break | |
| fi | |
| if grep -qx "${port}" <<< "${guarded}" 2>/dev/null; then | |
| ok=0; reject_reason="port ${port} (base ${base}) already guarded"; break | |
| fi | |
| done | |
| if [[ ${ok} -eq 0 ]]; then | |
| if [[ ${debug_count} -lt 5 ]]; then | |
| echo " Candidate ${candidate}: rejected — ${reject_reason}" >&2 | |
| debug_count=$((debug_count + 1)) | |
| fi | |
| continue | |
| fi | |
| # Atomically claim every (base+candidate) host port via a per-port docker | |
| # network. docker network create fails on name conflict, so only one runner | |
| # can claim a given port. If any port is already claimed, undo partial | |
| # reservations and try the next offset. | |
| guards=() | |
| ok=1 | |
| for base in "${PORT_BASES[@]}"; do | |
| port=$((base + candidate)) | |
| guard="ragflow-ci-port-${port}" | |
| # Use an explicit subnet from 10.0.0.0/8 (outside Docker's | |
| # default 172.17-31.x.x pools) to bypass "all predefined | |
| # address pools have been fully subnetted" errors on shared | |
| # daemons where auto-allocation is exhausted. | |
| guard_subnet="10.$((port / 256)).$((port % 256)).0/24" | |
| if ! create_err=$(sudo docker network create --subnet "${guard_subnet}" --label ragflow-ci-created="$(date -u +%s)" "${guard}" 2>&1 >/dev/null); then | |
| echo " docker network create ${guard} failed: ${create_err}" >&2 | |
| ok=0; break | |
| fi | |
| guards+=("${guard}") | |
| done | |
| if [[ ${ok} -eq 1 ]]; then | |
| PORT_OFFSET=${candidate} | |
| PORT_RESERVATION="${guards[*]}" | |
| echo "Reserved offset ${PORT_OFFSET} (attempt ${attempt})" >&2 | |
| return 0 | |
| fi | |
| for g in "${guards[@]}"; do | |
| sudo docker network rm "${g}" >/dev/null 2>&1 || true | |
| done | |
| # If docker network create keeps failing (daemon gone, permission, etc.), | |
| # bail out early instead of looping 6000 times. A name-conflict failure | |
| # should be rare because the fast path already filters guarded ports. | |
| create_fail_streak=$((create_fail_streak + 1)) | |
| if [[ ${create_fail_streak} -ge 10 ]]; then | |
| echo " docker network create failed ${create_fail_streak} consecutive candidates; aborting" >&2 | |
| return 1 | |
| fi | |
| # Another runner just claimed a port; refresh snapshots before retrying. | |
| need_refresh=1 | |
| done | |
| echo " Exhausted ${attempt} candidates. Last snapshot: ${used_n} bound, ${guarded_n} guarded" >&2 | |
| return 1 | |
| } | |
| if ! reserve_port_offset; then | |
| echo "Failed to reserve a free host port range for ${DOC_ENGINE} docker compose" >&2 | |
| exit 1 | |
| fi | |
| echo "PORT_RESERVATION=${PORT_RESERVATION}" >> ${GITHUB_ENV} | |
| echo "Using ${DOC_ENGINE} host port offset ${PORT_OFFSET}" | |
| ES_PORT=$((1200 + PORT_OFFSET)) | |
| OS_PORT=$((1201 + PORT_OFFSET)) | |
| INFINITY_THRIFT_PORT=$((23817 + PORT_OFFSET)) | |
| INFINITY_HTTP_PORT=$((23820 + PORT_OFFSET)) | |
| INFINITY_PSQL_PORT=$((5432 + PORT_OFFSET)) | |
| EXPOSE_MYSQL_PORT=$((5455 + PORT_OFFSET)) | |
| MINIO_PORT=$((9000 + PORT_OFFSET)) | |
| MINIO_CONSOLE_PORT=$((9001 + PORT_OFFSET)) | |
| REDIS_PORT=$((6379 + PORT_OFFSET)) | |
| EXPOSE_NATS_PORT=$((4222 + PORT_OFFSET)) | |
| NATS_MONITORING_PORT=$((8222 + PORT_OFFSET)) | |
| TEI_PORT=$((6380 + PORT_OFFSET)) | |
| KIBANA_PORT=$((6601 + PORT_OFFSET)) | |
| SVR_HTTP_PORT=$((9380 + PORT_OFFSET)) | |
| ADMIN_SVR_HTTP_PORT=$((9381 + PORT_OFFSET)) | |
| SVR_MCP_PORT=$((9382 + PORT_OFFSET)) | |
| GO_HTTP_PORT=$((9384 + PORT_OFFSET)) | |
| GO_ADMIN_PORT=$((9383 + PORT_OFFSET)) | |
| SANDBOX_EXECUTOR_MANAGER_PORT=$((9385 + PORT_OFFSET)) | |
| SVR_WEB_HTTP_PORT=$((80 + PORT_OFFSET)) | |
| SVR_WEB_HTTPS_PORT=$((443 + PORT_OFFSET)) | |
| # Persist computed ports into .env so docker-compose uses the correct host bindings. | |
| # Remove previous CI overrides first; docker compose uses the last duplicate key. | |
| sed -i '/^ES_PORT=/d;/^OS_PORT=/d;/^INFINITY_THRIFT_PORT=/d;/^INFINITY_HTTP_PORT=/d;/^INFINITY_PSQL_PORT=/d;/^EXPOSE_MYSQL_PORT=/d;/^MINIO_PORT=/d;/^MINIO_CONSOLE_PORT=/d;/^REDIS_PORT=/d;/^EXPOSE_NATS_PORT=/d;/^NATS_MONITORING_PORT=/d;/^TEI_PORT=/d;/^KIBANA_PORT=/d;/^SVR_HTTP_PORT=/d;/^ADMIN_SVR_HTTP_PORT=/d;/^SVR_MCP_PORT=/d;/^GO_HTTP_PORT=/d;/^GO_ADMIN_PORT=/d;/^SANDBOX_EXECUTOR_MANAGER_PORT=/d;/^SVR_WEB_HTTP_PORT=/d;/^SVR_WEB_HTTPS_PORT=/d;/^COMPOSE_PROFILES=/d;/^TEI_MODEL=/d;/^RAGFLOW_IMAGE=/d;/^DOC_ENGINE=/d;/^API_PROXY_SCHEME=/d' .env | |
| { | |
| echo "" | |
| echo "ES_PORT=${ES_PORT}" | |
| echo "OS_PORT=${OS_PORT}" | |
| echo "INFINITY_THRIFT_PORT=${INFINITY_THRIFT_PORT}" | |
| echo "INFINITY_HTTP_PORT=${INFINITY_HTTP_PORT}" | |
| echo "INFINITY_PSQL_PORT=${INFINITY_PSQL_PORT}" | |
| echo "EXPOSE_MYSQL_PORT=${EXPOSE_MYSQL_PORT}" | |
| echo "MINIO_PORT=${MINIO_PORT}" | |
| echo "MINIO_CONSOLE_PORT=${MINIO_CONSOLE_PORT}" | |
| echo "REDIS_PORT=${REDIS_PORT}" | |
| echo "EXPOSE_NATS_PORT=${EXPOSE_NATS_PORT}" | |
| echo "NATS_MONITORING_PORT=${NATS_MONITORING_PORT}" | |
| echo "TEI_PORT=${TEI_PORT}" | |
| echo "KIBANA_PORT=${KIBANA_PORT}" | |
| echo "SVR_HTTP_PORT=${SVR_HTTP_PORT}" | |
| echo "ADMIN_SVR_HTTP_PORT=${ADMIN_SVR_HTTP_PORT}" | |
| echo "SVR_MCP_PORT=${SVR_MCP_PORT}" | |
| echo "GO_HTTP_PORT=${GO_HTTP_PORT}" | |
| echo "GO_ADMIN_PORT=${GO_ADMIN_PORT}" | |
| echo "SANDBOX_EXECUTOR_MANAGER_PORT=${SANDBOX_EXECUTOR_MANAGER_PORT}" | |
| echo "SVR_WEB_HTTP_PORT=${SVR_WEB_HTTP_PORT}" | |
| echo "SVR_WEB_HTTPS_PORT=${SVR_WEB_HTTPS_PORT}" | |
| echo "COMPOSE_PROFILES=${DOC_ENGINE},cpu,ragflow-go,tei-cpu,deepdoc,metadata-mysql" | |
| echo "TEI_MODEL=BAAI/bge-small-en-v1.5" | |
| echo "RAGFLOW_IMAGE=${RAGFLOW_IMAGE}" | |
| echo "DOC_ENGINE=${DOC_ENGINE}" | |
| echo "API_PROXY_SCHEME=${API_PROXY_SCHEME}" | |
| } >> .env | |
| if [[ "${API_PROXY_SCHEME}" == "go" ]]; then | |
| RAGFLOW_HTTP_PORT=${SVR_WEB_HTTP_PORT} | |
| else | |
| RAGFLOW_HTTP_PORT=${SVR_HTTP_PORT} | |
| fi | |
| echo "RAGFLOW_HTTP_PORT=${RAGFLOW_HTTP_PORT}" >> ${GITHUB_ENV} | |
| echo "HOST_ADDRESS=http://host.docker.internal:${RAGFLOW_HTTP_PORT}" >> ${GITHUB_ENV} | |
| sed -i \ | |
| -e 's#${NATS_PORT:-4222}:4222#${EXPOSE_NATS_PORT}:4222#' \ | |
| -e 's#"8222:8222"#"${NATS_MONITORING_PORT}:8222"#' \ | |
| docker-compose-base.yml | |
| # Patch entrypoint.sh for coverage | |
| sed -i '/"\$PY" api\/ragflow_server.py \${INIT_SUPERUSER_ARGS} &/c\ echo "Ensuring coverage is installed..."\n "$PY" -m pip install coverage -i https://mirrors.aliyun.com/pypi/simple\n export COVERAGE_FILE=/ragflow/logs/.coverage\n echo "Starting ragflow_server with coverage..."\n "$PY" -m coverage run --source=./api/apps --omit="*/tests/*,*/migrations/*" -a api/ragflow_server.py ${INIT_SUPERUSER_ARGS} &' ./entrypoint.sh | |
| - name: Start ragflow:nightly for Infinity | |
| run: | | |
| sudo docker compose -f docker/docker-compose.yml -p ${COMPOSE_PROJECT_NAME} down -v || true | |
| sudo docker ps -a --filter "label=com.docker.compose.project=${COMPOSE_PROJECT_NAME}" -q | xargs -r sudo docker rm -f | |
| sudo docker compose -f docker/docker-compose.yml -p ${COMPOSE_PROJECT_NAME} up -d | |
| - name: Run sdk tests against Infinity | |
| if: env.API_PROXY_SCHEME == 'python' | |
| run: | | |
| export http_proxy=""; export https_proxy=""; export no_proxy=""; export HTTP_PROXY=""; export HTTPS_PROXY=""; export NO_PROXY="" | |
| svc_ready=0 | |
| for i in $(seq 1 60); do | |
| if sudo docker exec ${RAGFLOW_CONTAINER} curl -sf --connect-timeout 5 "${HOST_ADDRESS}/api/v1/system/ping" > /dev/null 2>&1; then | |
| svc_ready=1 | |
| break | |
| fi | |
| echo "Waiting for service to be available... ($i/60)" | |
| sleep 5 | |
| done | |
| if [ "$svc_ready" -ne 1 ]; then | |
| echo "Service did not become ready after 5 minutes. Docker logs:" | |
| sudo docker logs ${RAGFLOW_CONTAINER} | |
| exit 1 | |
| fi | |
| echo "Start to run test sdk on Infinity" | |
| source .venv/bin/activate && set -o pipefail; DOC_ENGINE=infinity pytest -s --tb=short --level=${HTTP_API_TEST_LEVEL} --junitxml=pytest-infinity-sdk.xml --cov=sdk/python/ragflow_sdk --cov-branch --cov-report=xml:coverage-infinity-sdk.xml test/testcases/test_sdk_api 2>&1 | tee infinity_sdk_test.log | |
| - name: Run New RESTFUL api tests against Infinity | |
| if: env.API_PROXY_SCHEME != 'web' | |
| run: | | |
| export http_proxy=""; export https_proxy=""; export no_proxy=""; export HTTP_PROXY=""; export HTTPS_PROXY=""; export NO_PROXY="" | |
| svc_ready=0 | |
| for i in $(seq 1 60); do | |
| if sudo docker exec ${RAGFLOW_CONTAINER} curl -sf --connect-timeout 5 "${HOST_ADDRESS}/api/v1/system/ping" > /dev/null 2>&1; then | |
| svc_ready=1 | |
| break | |
| fi | |
| echo "Waiting for service to be available... ($i/60)" | |
| sleep 5 | |
| done | |
| if [ "$svc_ready" -ne 1 ]; then | |
| echo "Service did not become ready after 5 minutes. Docker logs:" | |
| sudo docker logs ${RAGFLOW_CONTAINER} | |
| exit 1 | |
| fi | |
| source .venv/bin/activate && set -o pipefail; DOC_ENGINE=infinity pytest -s --tb=short --level=${HTTP_API_TEST_LEVEL} test/testcases/restful_api 2>&1 | tee infinity_restful_api_test.log | |
| - name: RAGFlow CLI retrieval test Infinity | |
| if: env.API_PROXY_SCHEME == 'python' | |
| env: | |
| PYTHONPATH: ${{ github.workspace }} | |
| run: | | |
| set -euo pipefail | |
| source .venv/bin/activate | |
| export http_proxy=""; export https_proxy=""; export no_proxy=""; export HTTP_PROXY=""; export HTTPS_PROXY=""; export NO_PROXY="" | |
| EMAIL="ci-${GITHUB_RUN_ID}@example.com" | |
| PASS="ci-pass-${GITHUB_RUN_ID}" | |
| DATASET="ci_dataset_${GITHUB_RUN_ID}" | |
| CLI="python admin/client/ragflow_cli.py" | |
| LOG_FILE="infinity_cli_test.log" | |
| : > "${LOG_FILE}" | |
| ERROR_RE='Traceback|ModuleNotFoundError|ImportError|Parse error|Bad response|Fail to|code:\\s*[1-9]' | |
| run_cli() { | |
| local logfile="$1" | |
| shift | |
| local allow_re="" | |
| if [[ "${1:-}" == "--allow" ]]; then | |
| allow_re="$2" | |
| shift 2 | |
| fi | |
| local cmd_display="$*" | |
| echo "===== $(date -u +\"%Y-%m-%dT%H:%M:%SZ\") CMD: ${cmd_display} =====" | tee -a "${logfile}" | |
| local tmp_log | |
| tmp_log="$(mktemp)" | |
| set +e | |
| timeout 500s "$@" 2>&1 | tee "${tmp_log}" | |
| local status=${PIPESTATUS[0]} | |
| set -e | |
| cat "${tmp_log}" >> "${logfile}" | |
| if grep -qiE "${ERROR_RE}" "${tmp_log}"; then | |
| if [[ -n "${allow_re}" ]] && grep -qiE "${allow_re}" "${tmp_log}"; then | |
| echo "Allowed CLI error markers in ${logfile}" | |
| rm -f "${tmp_log}" | |
| return 0 | |
| fi | |
| echo "Detected CLI error markers in ${logfile}" | |
| rm -f "${tmp_log}" | |
| exit 1 | |
| fi | |
| rm -f "${tmp_log}" | |
| return ${status} | |
| } | |
| set -a | |
| source docker/.env | |
| set +a | |
| USER_HOST="$(echo "${HOST_ADDRESS}" | sed -E 's#^https?://([^:/]+).*#\1#')" | |
| USER_PORT="${RAGFLOW_HTTP_PORT}" | |
| ADMIN_HOST="${USER_HOST}" | |
| ADMIN_PORT="${ADMIN_SVR_HTTP_PORT}" | |
| svc_ready=0 | |
| for i in $(seq 1 60); do | |
| if sudo docker exec ${RAGFLOW_CONTAINER} curl -sf --connect-timeout 5 "${HOST_ADDRESS}/api/v1/system/ping" > /dev/null 2>&1; then | |
| svc_ready=1 | |
| break | |
| fi | |
| echo "Waiting for service to be available... ($i/60)" | |
| sleep 5 | |
| done | |
| if [ "$svc_ready" -ne 1 ]; then | |
| echo "Service did not become ready after 5 minutes. Docker logs:" | |
| sudo docker logs ${RAGFLOW_CONTAINER} | |
| exit 1 | |
| fi | |
| admin_ready=0 | |
| for i in $(seq 1 30); do | |
| if run_cli "${LOG_FILE}" $CLI --type admin --host "$ADMIN_HOST" --port "$ADMIN_PORT" --username "admin@ragflow.io" --password "admin" command "ping"; then | |
| admin_ready=1 | |
| break | |
| fi | |
| sleep 1 | |
| done | |
| if [[ "${admin_ready}" -ne 1 ]]; then | |
| echo "Admin service did not become ready" | |
| exit 1 | |
| fi | |
| run_cli "${LOG_FILE}" $CLI --type admin --host "$ADMIN_HOST" --port "$ADMIN_PORT" --username "admin@ragflow.io" --password "admin" command "show version" | |
| ALLOW_USER_EXISTS_RE='already exists|already exist|duplicate|already.*registered|exist(s)?' | |
| run_cli "${LOG_FILE}" --allow "${ALLOW_USER_EXISTS_RE}" $CLI --type admin --host "$ADMIN_HOST" --port "$ADMIN_PORT" --username "admin@ragflow.io" --password "admin" command "create user '$EMAIL' '$PASS'" | |
| user_ready=0 | |
| for i in $(seq 1 30); do | |
| if run_cli "${LOG_FILE}" $CLI --type user --host "$USER_HOST" --port "$USER_PORT" --username "$EMAIL" --password "$PASS" command "ping"; then | |
| user_ready=1 | |
| break | |
| fi | |
| sleep 1 | |
| done | |
| if [[ "${user_ready}" -ne 1 ]]; then | |
| echo "User service did not become ready" | |
| exit 1 | |
| fi | |
| run_cli "${LOG_FILE}" $CLI --type user --host "$USER_HOST" --port "$USER_PORT" --username "$EMAIL" --password "$PASS" command "show version" | |
| run_cli "${LOG_FILE}" $CLI --type user --host "$USER_HOST" --port "$USER_PORT" --username "$EMAIL" --password "$PASS" command "create dataset '$DATASET' with embedding 'BAAI/bge-small-en-v1.5@Builtin' parser 'auto'" | |
| run_cli "${LOG_FILE}" $CLI --type user --host "$USER_HOST" --port "$USER_PORT" --username "$EMAIL" --password "$PASS" command "import 'test/benchmark/test_docs/Doc1.pdf,test/benchmark/test_docs/Doc2.pdf' into dataset '$DATASET'" | |
| run_cli "${LOG_FILE}" $CLI --type user --host "$USER_HOST" --port "$USER_PORT" --username "$EMAIL" --password "$PASS" command "parse dataset '$DATASET' sync" | |
| run_cli "${LOG_FILE}" $CLI --type user --host "$USER_HOST" --port "$USER_PORT" --username "$EMAIL" --password "$PASS" command "Benchmark 16 100 search 'what are these documents about' on datasets '$DATASET'" | |
| - name: Stop ragflow to save coverage Infinity | |
| if: ${{ !cancelled() && env.API_PROXY_SCHEME == 'python' }} | |
| run: | | |
| # Send SIGINT to ragflow_server.py to trigger coverage save | |
| PID=$(sudo docker exec ${RAGFLOW_CONTAINER} ps aux | grep "ragflow_server.py" | grep -v grep | awk '{print $2}' | head -n 1) | |
| if [ -n "$PID" ]; then | |
| echo "Sending SIGINT to ragflow_server.py (PID: $PID)..." | |
| sudo docker exec ${RAGFLOW_CONTAINER} kill -INT $PID | |
| # Wait for process to exit and coverage file to be written | |
| sleep 10 | |
| else | |
| echo "ragflow_server.py not found!" | |
| fi | |
| sudo docker compose -f docker/docker-compose.yml -p ${COMPOSE_PROJECT_NAME} stop | |
| - name: Generate server coverage report Infinity | |
| if: ${{ !cancelled() && env.API_PROXY_SCHEME == 'python' }} | |
| run: | | |
| # .coverage file should be in docker/ragflow-logs/.coverage | |
| if [ -f docker/ragflow-logs/.coverage ]; then | |
| echo "Found .coverage file" | |
| cp docker/ragflow-logs/.coverage .coverage | |
| source .venv/bin/activate | |
| # Create .coveragerc to map container paths to host paths | |
| echo "[paths]" > .coveragerc | |
| echo "source =" >> .coveragerc | |
| echo " ." >> .coveragerc | |
| echo " /ragflow" >> .coveragerc | |
| coverage xml -o coverage-infinity-server.xml | |
| rm .coveragerc | |
| else | |
| echo ".coverage file not found!" | |
| fi | |
| - name: Upload coverage reports to Codecov | |
| uses: codecov/codecov-action@v5 | |
| if: ${{ !cancelled() }} | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| fail_ci_if_error: false | |
| - name: Collect ragflow log Infinity | |
| if: ${{ !cancelled() }} | |
| run: | | |
| if [ -d docker/ragflow-logs ]; then | |
| destination="${ARTIFACTS_DIR}/ragflow-logs-infinity" | |
| sudo cp -r docker/ragflow-logs "${destination}" | |
| sudo chown -R "$(id -u):$(id -g)" "${destination}" | |
| echo "ragflow log" && sudo tail -n 200 docker/ragflow-logs/ragflow_server.log || true | |
| else | |
| echo "No docker/ragflow-logs directory found; skipping log collection" | |
| fi | |
| sudo rm -rf docker/ragflow-logs || true | |
| - name: Stop ragflow:nightly for Infinity | |
| if: ${{ always() }} | |
| run: | | |
| # Sometimes `docker compose down` fail due to hang container, heavy load etc. Need to remove such containers to release resources(for example, listen ports). | |
| sudo docker compose -f docker/docker-compose.yml -p ${COMPOSE_PROJECT_NAME} down -v || true | |
| sudo docker ps -a --filter "label=com.docker.compose.project=${COMPOSE_PROJECT_NAME}" -q | xargs -r sudo docker rm -f | |
| if [[ -n ${RAGFLOW_IMAGE} ]]; then | |
| sudo docker rmi -f ${RAGFLOW_IMAGE} | |
| fi | |
| if [[ -n ${PORT_RESERVATION:-} ]]; then | |
| # PORT_RESERVATION is a space-separated list of per-port docker network guards | |
| # created by the prepare step. Remove each to free the ports for other runners. | |
| for guard in ${PORT_RESERVATION}; do | |
| sudo docker network rm "${guard}" >/dev/null 2>&1 || true | |
| done | |
| fi | |
| ragflow_tests_elasticsearch: | |
| name: ragflow_tests_elasticsearch (${{ matrix.api_proxy_scheme }}) | |
| needs: ragflow_preflight | |
| if: ${{ github.event_name != 'pull_request' && github.event_name != 'pull_request_target' || (github.event.pull_request.draft == false && contains(github.event.pull_request.labels.*.name, 'ci') && (github.event.action != 'labeled' || github.event.label.name == 'ci')) }} | |
| runs-on: [ "self-hosted", "ragflow-test" ] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| api_proxy_scheme: ${{ fromJSON(needs.ragflow_preflight.outputs.api_proxy_schemes) }} | |
| env: | |
| DOC_ENGINE: elasticsearch | |
| API_PROXY_SCHEME: ${{ matrix.api_proxy_scheme }} | |
| RAGFLOW_IMAGE: infiniflow/ragflow:${{ github.run_id }}-elasticsearch-${{ matrix.api_proxy_scheme }} | |
| HTTP_API_TEST_LEVEL: ${{ needs.ragflow_preflight.outputs.http_api_test_level }} | |
| HAS_GO: ${{ needs.ragflow_preflight.outputs.has_go_changes }} | |
| HAS_PYTHON: ${{ needs.ragflow_preflight.outputs.has_python_changes }} | |
| HAS_WEB: ${{ needs.ragflow_preflight.outputs.has_web_changes }} | |
| steps: | |
| - name: Ensure workspace ownership | |
| if: always() | |
| run: | | |
| echo "Workflow triggered by ${{ github.event_name }}" | |
| echo "chown -R ${USER} ${GITHUB_WORKSPACE}" && sudo chown -R ${USER} ${GITHUB_WORKSPACE} | |
| - name: Check out code | |
| if: always() | |
| uses: actions/checkout@v6 | |
| with: | |
| ref: ${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && format('refs/pull/{0}/merge', github.event.pull_request.number) || github.sha }} | |
| fetch-depth: 0 | |
| fetch-tags: true | |
| allow-unsafe-pr-checkout: true | |
| - name: Build ragflow go server | |
| if: env.API_PROXY_SCHEME == 'go' | |
| run: | | |
| set -euo pipefail | |
| BUILDER_CONTAINER=ragflow_build_${GITHUB_RUN_ID}_${DOC_ENGINE}_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ') | |
| cleanup_builder() { | |
| if [[ -n "${BUILDER_CONTAINER:-}" ]]; then | |
| sudo docker rm -f -v "${BUILDER_CONTAINER}" >/dev/null 2>&1 || true | |
| fi | |
| } | |
| trap cleanup_builder EXIT | |
| TZ=${TZ:-$(readlink -f /etc/localtime | awk -F '/zoneinfo/' '{print $2}')} | |
| sudo docker run --privileged -d --name "${BUILDER_CONTAINER}" \ | |
| -e TZ="${TZ}" \ | |
| -e UV_INDEX=https://mirrors.aliyun.com/pypi/simple \ | |
| -v "${PWD}:/ragflow" \ | |
| -v "${PWD}/internal/binding/cpp/resource:/usr/share/infinity/resource" \ | |
| infiniflow/infinity_builder:ubuntu22_clang20 | |
| sudo docker exec "${BUILDER_CONTAINER}" bash -c 'git config --global safe.directory "*" && cd /ragflow && ./build.sh --cpp' | |
| ./build.sh --go | |
| - name: Run Go unit tests | |
| if: env.API_PROXY_SCHEME == 'go' | |
| # Runs after `./build.sh --go`, which guarantees the C++ static | |
| # library (librag_tokenizer_c_api.a) is present on disk. The Go | |
| # test binaries link against it transitively through | |
| # `internal/binding`, so running `go test` before the C++ build | |
| # fails the link step. | |
| # | |
| # Excludes packages whose tests fail for environmental reasons | |
| # unrelated to the diff: | |
| # - internal/tokenizer is split: the pure-Go BPE loader tests | |
| # (bpe_loader_test.go) run in the default tier; the C++ binding / | |
| # dict-dependent tests (tokenizer_test.go, | |
| # tokenizer_concurrent_test.go) and the on-disk anchor test | |
| # (bpe_loader_anchors_test.go) are tagged `manual` and need the | |
| # docker builder's /usr/share/infinity/resource, so they stay out | |
| # of the default run. | |
| run: | | |
| set -euo pipefail | |
| # Provide the cl100k BPE table for the offline tokenizer loader. | |
| # The loader reads <repo>/ragflow_deps/cl100k_base.tiktoken (or | |
| # TIKTOKEN_CACHE_DIR). Try to fetch it directly from the upstream | |
| # openai blob first (small ~1.6MB, reachable from the CI runner — the | |
| # pre-PR code downloaded it on the fly, so the network path is known | |
| # good). If the direct fetch fails (e.g. GFW), fall back to extracting | |
| # it from the local infiniflow/ragflow_deps image *if already present* | |
| # on the runner (no `docker pull`, since that huge image is first | |
| # fetched at the later "Build ragflow:nightly" step). Best-effort: if | |
| # neither source works, the loader still fails loudly (no silent | |
| # 0-token degradation). | |
| # tiktoken is already installed in runner | |
| mkdir -p ragflow_deps | |
| cp /opt/ragflow_deps/cl100k_base.tiktoken ragflow_deps/cl100k_base.tiktoken || true | |
| if [ ! -f ragflow_deps/cl100k_base.tiktoken ]; then | |
| mkdir -p ragflow_deps | |
| if command -v curl >/dev/null 2>&1; then | |
| if curl -fsSL -o ragflow_deps/cl100k_base.tiktoken \ | |
| https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken; then | |
| echo "tiktoken: cl100k table fetched via curl from openai blob" | |
| else | |
| echo "tiktoken: curl fetch failed (network/GFW); will try cached image" | |
| fi | |
| fi | |
| if [ ! -f ragflow_deps/cl100k_base.tiktoken ] && docker image inspect infiniflow/ragflow_deps:latest >/dev/null 2>&1; then | |
| CID=$(docker create infiniflow/ragflow_deps:latest true) || true | |
| if [ -n "${CID:-}" ]; then | |
| if docker cp "$CID":/cl100k_base.tiktoken ragflow_deps/cl100k_base.tiktoken 2>/dev/null; then | |
| echo "tiktoken: cl100k table copied from cached infiniflow/ragflow_deps image" | |
| fi | |
| docker rm -f "$CID" >/dev/null 2>&1 || true | |
| fi | |
| fi | |
| fi | |
| if [ -f ragflow_deps/cl100k_base.tiktoken ]; then | |
| echo "tiktoken: cl100k table provisioned ($(wc -c < ragflow_deps/cl100k_base.tiktoken) bytes)" | |
| else | |
| echo "tiktoken: cl100k table NOT provisioned — offline loader will fail loudly" | |
| fi | |
| PKGS=$(go list ./... 2>/dev/null \ | |
| | grep -v '/internal/storage$' \ | |
| | grep -v '/internal/handler$' || true) | |
| if [ -z "$PKGS" ]; then | |
| ./build.sh --test | |
| else | |
| ./build.sh --test -- $PKGS | |
| fi | |
| - name: Build ragflow:nightly | |
| run: | | |
| set -euo pipefail | |
| RUNNER_WORKSPACE_PREFIX=${RUNNER_WORKSPACE_PREFIX:-${HOME}} | |
| BUILD_LOCK_FILE=${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/docker-build.lock | |
| mkdir -p "$(dirname "${BUILD_LOCK_FILE}")" | |
| # Pass the Gitee personal access token to the Docker build as a BuildKit secret | |
| # (mounted inside the builder at /run/secrets/gitee_token, never persisted in any layer). | |
| GITEE_TOKEN_FILE="$(mktemp)" | |
| printf '%s' '${GITEE_TOKEN }' > "${GITEE_TOKEN_FILE}" | |
| ( | |
| flock -w 10800 9 || { echo "Timed out waiting for the shared Docker build slot" >&2; exit 1; } | |
| echo "Acquired Docker build slot for ${DOC_ENGINE}/${API_PROXY_SCHEME}" | |
| ##sudo docker pull ubuntu:24.04 | |
| sudo DOCKER_BUILDKIT=1 docker build --build-arg NEED_MIRROR=1 --build-arg HTTPS_PROXY=${HTTPS_PROXY} --build-arg HTTP_PROXY=${HTTP_PROXY} --secret id=gitee_token,src=${GITEE_TOKEN_FILE} -f Dockerfile_ci -t ${RAGFLOW_IMAGE} . | |
| ) 9>"${BUILD_LOCK_FILE}" | |
| rm -f "${GITEE_TOKEN_FILE}" | |
| - name: Prepare Python test environment | |
| if: env.API_PROXY_SCHEME != 'web' | |
| run: | | |
| git config --global url."https://oauth2:${GITEE_TOKEN}@gitee.com/".insteadOf "https://gitee.com/" | |
| uv sync --python 3.13 --group test --frozen | |
| uv pip install -e sdk/python | |
| - name: Prepare function test environment | |
| working-directory: docker | |
| run: | | |
| set -euo pipefail | |
| sudo chmod 1777 /tmp | |
| ## ss already installed | |
| ##sudo apt update && sudo apt install -y iproute2 | |
| RUNNER_WORKSPACE_PREFIX=${RUNNER_WORKSPACE_PREFIX:-${HOME}} | |
| COMPOSE_PROJECT_NAME="${GITHUB_RUN_ID}-${DOC_ENGINE}-${API_PROXY_SCHEME}" | |
| echo "COMPOSE_PROJECT_NAME=${COMPOSE_PROJECT_NAME}" >> ${GITHUB_ENV} | |
| echo "RAGFLOW_CONTAINER=${COMPOSE_PROJECT_NAME}-ragflow-cpu-1" >> ${GITHUB_ENV} | |
| ARTIFACTS_DIR=${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/${GITHUB_RUN_ID}/${DOC_ENGINE}/${API_PROXY_SCHEME} | |
| echo "ARTIFACTS_DIR=${ARTIFACTS_DIR}" >> ${GITHUB_ENV} | |
| rm -rf "${ARTIFACTS_DIR}" && mkdir -p "${ARTIFACTS_DIR}" | |
| # Determine runner number (default to 1 if not found) | |
| RUNNER_NUM=$(sudo docker inspect $(hostname) --format '{{index .Config.Labels "com.docker.compose.container-number"}}' 2>/dev/null || true) | |
| if [[ -z "${RUNNER_NUM:-}" ]]; then | |
| # GitHub self-hosted runners always expose RUNNER_NAME; hash it to spread candidates. | |
| if [[ -n "${RUNNER_NAME:-}" ]]; then | |
| RUNNER_NUM=$(( $(printf '%d' 0x$(echo -n "${RUNNER_NAME}" | md5sum | cut -c1-4)) % 1000 )) | |
| else | |
| RUNNER_NUM=$(( GITHUB_RUN_ID % 1000 )) | |
| fi | |
| fi | |
| # Engine-specific offset partitions keep concurrent engine jobs from | |
| # choosing the same host ports when they land on the same self-hosted runner. | |
| # A lock plus reservation file closes the check/start race between parallel jobs. | |
| PORT_BASES=(1200 1201 23817 23820 5432 5455 9000 9001 6379 6380 6601 9380 9381 9382 9384 9383 9385 80 443 4222 8222) | |
| PARTITION_SIZE=6000 | |
| case "${DOC_ENGINE}" in | |
| elasticsearch) PARTITION_BASE=1000 ;; | |
| infinity) PARTITION_BASE=31000 ;; | |
| *) echo "Unsupported DOC_ENGINE=${DOC_ENGINE}" >&2; exit 1 ;; | |
| esac | |
| # Runners run inside docker containers that share the host docker daemon | |
| # (/var/run/docker.sock), so neither /tmp nor `ss` can coordinate across runners: | |
| # each container has its own /tmp and its own network namespace, and `ss` cannot | |
| # see host ports bound by other runners' compose stacks. Use the shared daemon for | |
| # both primitives: | |
| # - per-port mutex = for each host port (base+offset) we intend to bind, create | |
| # a docker network named ragflow-ci-port-<port> on the shared daemon. docker | |
| # network create is NOT idempotent (fails on name conflict), so only one runner | |
| # can claim a given port. This is per-PORT, not per-offset, because different | |
| # (base, offset) pairs can produce the same host port (e.g. 6380+36497 == | |
| # 9380+33497 == 42877); a per-offset guard cannot prevent that collision. | |
| # - port availability = docker ps published ports (sees every container on the | |
| # shared daemon regardless of which runner started it). | |
| # No file-system sharing required, no host netns required. | |
| cleanup_stale_port_locks() { | |
| local now stale_after net ts before after | |
| now=$(date -u +%s) | |
| stale_after=$((60 * 60)) | |
| before=$(sudo docker network ls --filter "label=ragflow-ci-created" --format '{{.Name}}' | wc -l) | |
| for net in $(sudo docker network ls --filter "label=ragflow-ci-created" --format '{{.Name}}'); do | |
| ts=$(sudo docker network inspect --format '{{ index .Labels "ragflow-ci-created" }}' "${net}" 2>/dev/null || true) | |
| # Remove if timestamp is non-numeric (legacy network from a previous code | |
| # version) or if it is numeric and older than stale_after. | |
| if [[ ! "${ts}" =~ ^[0-9]+$ ]] || (( now - ts > stale_after )); then | |
| sudo docker network rm "${net}" >/dev/null 2>&1 || true | |
| fi | |
| done | |
| after=$(sudo docker network ls --filter "label=ragflow-ci-created" --format '{{.Name}}' | wc -l) | |
| echo " cleanup_stale_port_locks: ${before} -> ${after} guards (TTL=${stale_after}s)" >&2 | |
| } | |
| reserve_port_offset() { | |
| local attempt candidate base port guard g ok reject_reason create_err guard_subnet | |
| local -a guards | |
| local used guarded need_refresh used_n guarded_n debug_count create_fail_streak | |
| cleanup_stale_port_locks | |
| echo "Reserving host ports for ${DOC_ENGINE} compose stack..." >&2 | |
| # Fail fast if the shared docker daemon is not reachable — avoids a | |
| # long spin where every docker network create silently fails. | |
| if ! sudo docker info >/dev/null 2>&1; then | |
| echo " ERROR: 'sudo docker info' failed — docker daemon is not reachable from this runner" >&2 | |
| return 1 | |
| fi | |
| echo " Running containers on shared daemon: $(sudo docker ps -q 2>/dev/null | wc -l)" >&2 | |
| echo " Docker networks on shared daemon: $(sudo docker network ls -q 2>/dev/null | wc -l)" >&2 | |
| need_refresh=1 | |
| debug_count=0 | |
| create_fail_streak=0 | |
| for attempt in $(seq 0 499); do | |
| candidate=$(( PARTITION_BASE + ((GITHUB_RUN_ID + RUNNER_NUM * 1000 + attempt * 97) % PARTITION_SIZE) )) | |
| # Refresh snapshots on first attempt and after a failed reservation. | |
| if [[ ${need_refresh} -eq 1 ]]; then | |
| used=$(sudo docker ps --format '{{.Ports}}' 2>/dev/null | grep -oE '[0-9]+->' | tr -d '>-' | sort -u || true) | |
| # Strip both ragflow-ci-port- (current) and ragflow-ci-portguard- (legacy) prefixes. | |
| guarded=$(sudo docker network ls --filter "label=ragflow-ci-created" --format '{{.Name}}' 2>/dev/null | sed 's/^ragflow-ci-port\(guard\)\?-//' | sort -u || true) | |
| used_n=$(echo "${used}" | grep -c . 2>/dev/null || true) | |
| guarded_n=$(echo "${guarded}" | grep -c . 2>/dev/null || true) | |
| echo " Snapshot: ${used_n} bound ports, ${guarded_n} guarded ports" >&2 | |
| [[ ${used_n} -gt 0 ]] && echo " Bound sample: $(echo "${used}" | tr '\n' ' ' | cut -d' ' -f1-10)" >&2 | |
| [[ ${guarded_n} -gt 0 ]] && echo " Guarded sample: $(echo "${guarded}" | tr '\n' ' ' | cut -d' ' -f1-10)" >&2 | |
| need_refresh=0 | |
| fi | |
| # Fast path: skip if any (base+candidate) port is already bound or guarded. | |
| ok=1 | |
| reject_reason="" | |
| for base in "${PORT_BASES[@]}"; do | |
| port=$((base + candidate)) | |
| if grep -qx "${port}" <<< "${used}" 2>/dev/null; then | |
| ok=0; reject_reason="port ${port} (base ${base}) already bound"; break | |
| fi | |
| if grep -qx "${port}" <<< "${guarded}" 2>/dev/null; then | |
| ok=0; reject_reason="port ${port} (base ${base}) already guarded"; break | |
| fi | |
| done | |
| if [[ ${ok} -eq 0 ]]; then | |
| if [[ ${debug_count} -lt 5 ]]; then | |
| echo " Candidate ${candidate}: rejected — ${reject_reason}" >&2 | |
| debug_count=$((debug_count + 1)) | |
| fi | |
| continue | |
| fi | |
| # Atomically claim every (base+candidate) host port via a per-port docker | |
| # network. docker network create fails on name conflict, so only one runner | |
| # can claim a given port. If any port is already claimed, undo partial | |
| # reservations and try the next offset. | |
| guards=() | |
| ok=1 | |
| for base in "${PORT_BASES[@]}"; do | |
| port=$((base + candidate)) | |
| guard="ragflow-ci-port-${port}" | |
| # Use an explicit subnet from 10.0.0.0/8 (outside Docker's | |
| # default 172.17-31.x.x pools) to bypass "all predefined | |
| # address pools have been fully subnetted" errors on shared | |
| # daemons where auto-allocation is exhausted. | |
| guard_subnet="10.$((port / 256)).$((port % 256)).0/24" | |
| if ! create_err=$(sudo docker network create --subnet "${guard_subnet}" --label ragflow-ci-created="$(date -u +%s)" "${guard}" 2>&1 >/dev/null); then | |
| echo " docker network create ${guard} failed: ${create_err}" >&2 | |
| ok=0; break | |
| fi | |
| guards+=("${guard}") | |
| done | |
| if [[ ${ok} -eq 1 ]]; then | |
| PORT_OFFSET=${candidate} | |
| PORT_RESERVATION="${guards[*]}" | |
| echo "Reserved offset ${PORT_OFFSET} (attempt ${attempt})" >&2 | |
| return 0 | |
| fi | |
| for g in "${guards[@]}"; do | |
| sudo docker network rm "${g}" >/dev/null 2>&1 || true | |
| done | |
| # If docker network create keeps failing (daemon gone, permission, etc.), | |
| # bail out early instead of looping 6000 times. A name-conflict failure | |
| # should be rare because the fast path already filters guarded ports. | |
| create_fail_streak=$((create_fail_streak + 1)) | |
| if [[ ${create_fail_streak} -ge 10 ]]; then | |
| echo " docker network create failed ${create_fail_streak} consecutive candidates; aborting" >&2 | |
| return 1 | |
| fi | |
| # Another runner just claimed a port; refresh snapshots before retrying. | |
| need_refresh=1 | |
| done | |
| echo " Exhausted ${attempt} candidates. Last snapshot: ${used_n} bound, ${guarded_n} guarded" >&2 | |
| return 1 | |
| } | |
| if ! reserve_port_offset; then | |
| echo "Failed to reserve a free host port range for ${DOC_ENGINE} docker compose" >&2 | |
| exit 1 | |
| fi | |
| echo "PORT_RESERVATION=${PORT_RESERVATION}" >> ${GITHUB_ENV} | |
| echo "Using ${DOC_ENGINE} host port offset ${PORT_OFFSET}" | |
| ES_PORT=$((1200 + PORT_OFFSET)) | |
| OS_PORT=$((1201 + PORT_OFFSET)) | |
| INFINITY_THRIFT_PORT=$((23817 + PORT_OFFSET)) | |
| INFINITY_HTTP_PORT=$((23820 + PORT_OFFSET)) | |
| INFINITY_PSQL_PORT=$((5432 + PORT_OFFSET)) | |
| EXPOSE_MYSQL_PORT=$((5455 + PORT_OFFSET)) | |
| MINIO_PORT=$((9000 + PORT_OFFSET)) | |
| MINIO_CONSOLE_PORT=$((9001 + PORT_OFFSET)) | |
| REDIS_PORT=$((6379 + PORT_OFFSET)) | |
| EXPOSE_NATS_PORT=$((4222 + PORT_OFFSET)) | |
| NATS_MONITORING_PORT=$((8222 + PORT_OFFSET)) | |
| TEI_PORT=$((6380 + PORT_OFFSET)) | |
| KIBANA_PORT=$((6601 + PORT_OFFSET)) | |
| SVR_HTTP_PORT=$((9380 + PORT_OFFSET)) | |
| ADMIN_SVR_HTTP_PORT=$((9381 + PORT_OFFSET)) | |
| SVR_MCP_PORT=$((9382 + PORT_OFFSET)) | |
| GO_HTTP_PORT=$((9384 + PORT_OFFSET)) | |
| GO_ADMIN_PORT=$((9383 + PORT_OFFSET)) | |
| SANDBOX_EXECUTOR_MANAGER_PORT=$((9385 + PORT_OFFSET)) | |
| SVR_WEB_HTTP_PORT=$((80 + PORT_OFFSET)) | |
| SVR_WEB_HTTPS_PORT=$((443 + PORT_OFFSET)) | |
| # Persist computed ports into .env so docker-compose uses the correct host bindings. | |
| # Remove previous CI overrides first; docker compose uses the last duplicate key. | |
| sed -i '/^ES_PORT=/d;/^OS_PORT=/d;/^INFINITY_THRIFT_PORT=/d;/^INFINITY_HTTP_PORT=/d;/^INFINITY_PSQL_PORT=/d;/^EXPOSE_MYSQL_PORT=/d;/^MINIO_PORT=/d;/^MINIO_CONSOLE_PORT=/d;/^REDIS_PORT=/d;/^EXPOSE_NATS_PORT=/d;/^NATS_MONITORING_PORT=/d;/^TEI_PORT=/d;/^KIBANA_PORT=/d;/^SVR_HTTP_PORT=/d;/^ADMIN_SVR_HTTP_PORT=/d;/^SVR_MCP_PORT=/d;/^GO_HTTP_PORT=/d;/^GO_ADMIN_PORT=/d;/^SANDBOX_EXECUTOR_MANAGER_PORT=/d;/^SVR_WEB_HTTP_PORT=/d;/^SVR_WEB_HTTPS_PORT=/d;/^COMPOSE_PROFILES=/d;/^TEI_MODEL=/d;/^RAGFLOW_IMAGE=/d;/^DOC_ENGINE=/d;/^API_PROXY_SCHEME=/d' .env | |
| { | |
| echo "" | |
| echo "ES_PORT=${ES_PORT}" | |
| echo "OS_PORT=${OS_PORT}" | |
| echo "INFINITY_THRIFT_PORT=${INFINITY_THRIFT_PORT}" | |
| echo "INFINITY_HTTP_PORT=${INFINITY_HTTP_PORT}" | |
| echo "INFINITY_PSQL_PORT=${INFINITY_PSQL_PORT}" | |
| echo "EXPOSE_MYSQL_PORT=${EXPOSE_MYSQL_PORT}" | |
| echo "MINIO_PORT=${MINIO_PORT}" | |
| echo "MINIO_CONSOLE_PORT=${MINIO_CONSOLE_PORT}" | |
| echo "REDIS_PORT=${REDIS_PORT}" | |
| echo "EXPOSE_NATS_PORT=${EXPOSE_NATS_PORT}" | |
| echo "NATS_MONITORING_PORT=${NATS_MONITORING_PORT}" | |
| echo "TEI_PORT=${TEI_PORT}" | |
| echo "KIBANA_PORT=${KIBANA_PORT}" | |
| echo "SVR_HTTP_PORT=${SVR_HTTP_PORT}" | |
| echo "ADMIN_SVR_HTTP_PORT=${ADMIN_SVR_HTTP_PORT}" | |
| echo "SVR_MCP_PORT=${SVR_MCP_PORT}" | |
| echo "GO_HTTP_PORT=${GO_HTTP_PORT}" | |
| echo "GO_ADMIN_PORT=${GO_ADMIN_PORT}" | |
| echo "SANDBOX_EXECUTOR_MANAGER_PORT=${SANDBOX_EXECUTOR_MANAGER_PORT}" | |
| echo "SVR_WEB_HTTP_PORT=${SVR_WEB_HTTP_PORT}" | |
| echo "SVR_WEB_HTTPS_PORT=${SVR_WEB_HTTPS_PORT}" | |
| echo "COMPOSE_PROFILES=${DOC_ENGINE},cpu,ragflow-go,tei-cpu,deepdoc,metadata-mysql" | |
| echo "TEI_MODEL=BAAI/bge-small-en-v1.5" | |
| echo "RAGFLOW_IMAGE=${RAGFLOW_IMAGE}" | |
| echo "DOC_ENGINE=${DOC_ENGINE}" | |
| echo "API_PROXY_SCHEME=${API_PROXY_SCHEME}" | |
| } >> .env | |
| if [[ "${API_PROXY_SCHEME}" == "go" ]]; then | |
| RAGFLOW_HTTP_PORT=${SVR_WEB_HTTP_PORT} | |
| else | |
| RAGFLOW_HTTP_PORT=${SVR_HTTP_PORT} | |
| fi | |
| echo "RAGFLOW_HTTP_PORT=${RAGFLOW_HTTP_PORT}" >> ${GITHUB_ENV} | |
| echo "HOST_ADDRESS=http://host.docker.internal:${RAGFLOW_HTTP_PORT}" >> ${GITHUB_ENV} | |
| sed -i \ | |
| -e 's#${NATS_PORT:-4222}:4222#${EXPOSE_NATS_PORT}:4222#' \ | |
| -e 's#"8222:8222"#"${NATS_MONITORING_PORT}:8222"#' \ | |
| docker-compose-base.yml | |
| # Patch entrypoint.sh for coverage | |
| sed -i '/"\$PY" api\/ragflow_server.py \${INIT_SUPERUSER_ARGS} &/c\ echo "Ensuring coverage is installed..."\n "$PY" -m pip install coverage -i https://mirrors.aliyun.com/pypi/simple\n export COVERAGE_FILE=/ragflow/logs/.coverage\n echo "Starting ragflow_server with coverage..."\n "$PY" -m coverage run --source=./api/apps --omit="*/tests/*,*/migrations/*" -a api/ragflow_server.py ${INIT_SUPERUSER_ARGS} &' ./entrypoint.sh | |
| - name: Start ragflow:nightly for Elasticsearch | |
| run: | | |
| sudo docker compose -f docker/docker-compose.yml -p ${COMPOSE_PROJECT_NAME} down -v || true | |
| sudo docker ps -a --filter "label=com.docker.compose.project=${COMPOSE_PROJECT_NAME}" -q | xargs -r sudo docker rm -f | |
| sudo docker compose -f docker/docker-compose.yml -p ${COMPOSE_PROJECT_NAME} up -d | |
| - name: Run sdk tests against Elasticsearch | |
| if: env.API_PROXY_SCHEME == 'python' | |
| run: | | |
| export http_proxy=""; export https_proxy=""; export no_proxy=""; export HTTP_PROXY=""; export HTTPS_PROXY=""; export NO_PROXY="" | |
| svc_ready=0 | |
| for i in $(seq 1 60); do | |
| if sudo docker exec ${RAGFLOW_CONTAINER} curl -sf --connect-timeout 5 "${HOST_ADDRESS}/api/v1/system/ping" > /dev/null 2>&1; then | |
| svc_ready=1 | |
| break | |
| fi | |
| echo "Waiting for service to be available... ($i/60)" | |
| sleep 5 | |
| done | |
| if [ "$svc_ready" -ne 1 ]; then | |
| echo "Service did not become ready after 5 minutes. Docker logs:" | |
| sudo docker logs ${RAGFLOW_CONTAINER} | |
| exit 1 | |
| fi | |
| echo "Start to run test sdk on Elasticsearch" | |
| source .venv/bin/activate && set -o pipefail; pytest -s --tb=short --level=${HTTP_API_TEST_LEVEL} --junitxml=pytest-es-sdk.xml --cov=sdk/python/ragflow_sdk --cov-branch --cov-report=xml:coverage-es-sdk.xml test/testcases/test_sdk_api 2>&1 | tee es_sdk_test.log | |
| - name: Run New RESTFUL api tests against Elasticsearch | |
| if: env.API_PROXY_SCHEME != 'web' | |
| run: | | |
| export http_proxy=""; export https_proxy=""; export no_proxy=""; export HTTP_PROXY=""; export HTTPS_PROXY=""; export NO_PROXY="" | |
| svc_ready=0 | |
| for i in $(seq 1 60); do | |
| if sudo docker exec ${RAGFLOW_CONTAINER} curl -sf --connect-timeout 5 "${HOST_ADDRESS}/api/v1/system/ping" > /dev/null 2>&1; then | |
| svc_ready=1 | |
| break | |
| fi | |
| echo "Waiting for service to be available... ($i/60)" | |
| sleep 5 | |
| done | |
| if [ "$svc_ready" -ne 1 ]; then | |
| echo "Service did not become ready after 5 minutes. Docker logs:" | |
| sudo docker logs ${RAGFLOW_CONTAINER} | |
| exit 1 | |
| fi | |
| source .venv/bin/activate && set -o pipefail; pytest -s --tb=short --level=${HTTP_API_TEST_LEVEL} test/testcases/restful_api 2>&1 | tee es_restful_api_test.log | |
| - name: RAGFlow CLI retrieval test Elasticsearch | |
| if: env.API_PROXY_SCHEME == 'python' | |
| env: | |
| PYTHONPATH: ${{ github.workspace }} | |
| run: | | |
| set -euo pipefail | |
| source .venv/bin/activate | |
| export http_proxy=""; export https_proxy=""; export no_proxy=""; export HTTP_PROXY=""; export HTTPS_PROXY=""; export NO_PROXY="" | |
| EMAIL="ci-${GITHUB_RUN_ID}@example.com" | |
| PASS="ci-pass-${GITHUB_RUN_ID}" | |
| DATASET="ci_dataset_${GITHUB_RUN_ID}" | |
| CLI="python admin/client/ragflow_cli.py" | |
| LOG_FILE="es_cli_test.log" | |
| : > "${LOG_FILE}" | |
| ERROR_RE='Traceback|ModuleNotFoundError|ImportError|Parse error|Bad response|Fail to|code:\\s*[1-9]' | |
| run_cli() { | |
| local logfile="$1" | |
| shift | |
| local allow_re="" | |
| if [[ "${1:-}" == "--allow" ]]; then | |
| allow_re="$2" | |
| shift 2 | |
| fi | |
| local cmd_display="$*" | |
| echo "===== $(date -u +\"%Y-%m-%dT%H:%M:%SZ\") CMD: ${cmd_display} =====" | tee -a "${logfile}" | |
| local tmp_log | |
| tmp_log="$(mktemp)" | |
| set +e | |
| timeout 500s "$@" 2>&1 | tee "${tmp_log}" | |
| local status=${PIPESTATUS[0]} | |
| set -e | |
| cat "${tmp_log}" >> "${logfile}" | |
| if grep -qiE "${ERROR_RE}" "${tmp_log}"; then | |
| if [[ -n "${allow_re}" ]] && grep -qiE "${allow_re}" "${tmp_log}"; then | |
| echo "Allowed CLI error markers in ${logfile}" | |
| rm -f "${tmp_log}" | |
| return 0 | |
| fi | |
| echo "Detected CLI error markers in ${logfile}" | |
| rm -f "${tmp_log}" | |
| exit 1 | |
| fi | |
| rm -f "${tmp_log}" | |
| return ${status} | |
| } | |
| set -a | |
| source docker/.env | |
| set +a | |
| USER_HOST="$(echo "${HOST_ADDRESS}" | sed -E 's#^https?://([^:/]+).*#\1#')" | |
| USER_PORT="${RAGFLOW_HTTP_PORT}" | |
| ADMIN_HOST="${USER_HOST}" | |
| ADMIN_PORT="${ADMIN_SVR_HTTP_PORT}" | |
| svc_ready=0 | |
| for i in $(seq 1 60); do | |
| if sudo docker exec ${RAGFLOW_CONTAINER} curl -sf --connect-timeout 5 "${HOST_ADDRESS}/api/v1/system/ping" > /dev/null 2>&1; then | |
| svc_ready=1 | |
| break | |
| fi | |
| echo "Waiting for service to be available... ($i/60)" | |
| sleep 5 | |
| done | |
| if [ "$svc_ready" -ne 1 ]; then | |
| echo "Service did not become ready after 5 minutes. Docker logs:" | |
| sudo docker logs ${RAGFLOW_CONTAINER} | |
| exit 1 | |
| fi | |
| admin_ready=0 | |
| for i in $(seq 1 30); do | |
| if run_cli "${LOG_FILE}" $CLI --type admin --host "$ADMIN_HOST" --port "$ADMIN_PORT" --username "admin@ragflow.io" --password "admin" command "ping"; then | |
| admin_ready=1 | |
| break | |
| fi | |
| sleep 1 | |
| done | |
| if [[ "${admin_ready}" -ne 1 ]]; then | |
| echo "Admin service did not become ready" | |
| exit 1 | |
| fi | |
| run_cli "${LOG_FILE}" $CLI --type admin --host "$ADMIN_HOST" --port "$ADMIN_PORT" --username "admin@ragflow.io" --password "admin" command "show version" | |
| ALLOW_USER_EXISTS_RE='already exists|already exist|duplicate|already.*registered|exist(s)?' | |
| run_cli "${LOG_FILE}" --allow "${ALLOW_USER_EXISTS_RE}" $CLI --type admin --host "$ADMIN_HOST" --port "$ADMIN_PORT" --username "admin@ragflow.io" --password "admin" command "create user '$EMAIL' '$PASS'" | |
| user_ready=0 | |
| for i in $(seq 1 30); do | |
| if run_cli "${LOG_FILE}" $CLI --type user --host "$USER_HOST" --port "$USER_PORT" --username "$EMAIL" --password "$PASS" command "ping"; then | |
| user_ready=1 | |
| break | |
| fi | |
| sleep 1 | |
| done | |
| if [[ "${user_ready}" -ne 1 ]]; then | |
| echo "User service did not become ready" | |
| exit 1 | |
| fi | |
| run_cli "${LOG_FILE}" $CLI --type user --host "$USER_HOST" --port "$USER_PORT" --username "$EMAIL" --password "$PASS" command "show version" | |
| run_cli "${LOG_FILE}" $CLI --type user --host "$USER_HOST" --port "$USER_PORT" --username "$EMAIL" --password "$PASS" command "create dataset '$DATASET' with embedding 'BAAI/bge-small-en-v1.5@Builtin' parser 'auto'" | |
| run_cli "${LOG_FILE}" $CLI --type user --host "$USER_HOST" --port "$USER_PORT" --username "$EMAIL" --password "$PASS" command "import 'test/benchmark/test_docs/Doc1.pdf,test/benchmark/test_docs/Doc2.pdf' into dataset '$DATASET'" | |
| run_cli "${LOG_FILE}" $CLI --type user --host "$USER_HOST" --port "$USER_PORT" --username "$EMAIL" --password "$PASS" command "parse dataset '$DATASET' sync" | |
| run_cli "${LOG_FILE}" $CLI --type user --host "$USER_HOST" --port "$USER_PORT" --username "$EMAIL" --password "$PASS" command "Benchmark 16 100 search 'what are these documents about' on datasets '$DATASET'" | |
| - name: Stop ragflow to save coverage Elasticsearch | |
| if: ${{ !cancelled() && env.API_PROXY_SCHEME == 'python' }} | |
| run: | | |
| # Send SIGINT to ragflow_server.py to trigger coverage save | |
| PID=$(sudo docker exec ${RAGFLOW_CONTAINER} ps aux | grep "ragflow_server.py" | grep -v grep | awk '{print $2}' | head -n 1) | |
| if [ -n "$PID" ]; then | |
| echo "Sending SIGINT to ragflow_server.py (PID: $PID)..." | |
| sudo docker exec ${RAGFLOW_CONTAINER} kill -INT $PID | |
| # Wait for process to exit and coverage file to be written | |
| sleep 10 | |
| else | |
| echo "ragflow_server.py not found!" | |
| fi | |
| sudo docker compose -f docker/docker-compose.yml -p ${COMPOSE_PROJECT_NAME} stop | |
| - name: Generate server coverage report Elasticsearch | |
| if: ${{ !cancelled() && env.API_PROXY_SCHEME == 'python' }} | |
| run: | | |
| # .coverage file should be in docker/ragflow-logs/.coverage | |
| if [ -f docker/ragflow-logs/.coverage ]; then | |
| echo "Found .coverage file" | |
| cp docker/ragflow-logs/.coverage .coverage | |
| source .venv/bin/activate | |
| # Create .coveragerc to map container paths to host paths | |
| echo "[paths]" > .coveragerc | |
| echo "source =" >> .coveragerc | |
| echo " ." >> .coveragerc | |
| echo " /ragflow" >> .coveragerc | |
| coverage xml -o coverage-es-server.xml | |
| rm .coveragerc | |
| # Clean up for next run | |
| sudo rm docker/ragflow-logs/.coverage | |
| else | |
| echo ".coverage file not found!" | |
| fi | |
| - name: Collect ragflow log Elasticsearch | |
| if: ${{ !cancelled() }} | |
| run: | | |
| if [ -d docker/ragflow-logs ]; then | |
| destination="${ARTIFACTS_DIR}/ragflow-logs-es" | |
| sudo cp -r docker/ragflow-logs "${destination}" | |
| sudo chown -R "$(id -u):$(id -g)" "${destination}" | |
| echo "ragflow log" && sudo tail -n 200 docker/ragflow-logs/ragflow_server.log || true | |
| else | |
| echo "No docker/ragflow-logs directory found; skipping log collection" | |
| fi | |
| sudo rm -rf docker/ragflow-logs || true | |
| - name: Stop ragflow:nightly for Elasticsearch | |
| if: ${{ always() }} | |
| run: | | |
| # Sometimes `docker compose down` fail due to hang container, heavy load etc. Need to remove such containers to release resources(for example, listen ports). | |
| sudo docker compose -f docker/docker-compose.yml -p ${COMPOSE_PROJECT_NAME} down -v || true | |
| sudo docker ps -a --filter "label=com.docker.compose.project=${COMPOSE_PROJECT_NAME}" -q | xargs -r sudo docker rm -f | |
| if [[ -n ${RAGFLOW_IMAGE} ]]; then | |
| sudo docker rmi -f ${RAGFLOW_IMAGE} | |
| fi | |
| if [[ -n ${PORT_RESERVATION:-} ]]; then | |
| # PORT_RESERVATION is a space-separated list of per-port docker network guards | |
| # created by the prepare step. Remove each to free the ports for other runners. | |
| for guard in ${PORT_RESERVATION}; do | |
| sudo docker network rm "${guard}" >/dev/null 2>&1 || true | |
| done | |
| fi |