Update Tigris provider benchmarks #70
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Release and Deploy Backend | |
| on: | |
| push: | |
| branches: | |
| - main | |
| paths: | |
| - ".github/workflows/release-deploy.yml" | |
| - "Cargo.lock" | |
| - "Cargo.toml" | |
| - "Dockerfile" | |
| - "clickhouse/**" | |
| - "crates/**" | |
| - "docs/env/**" | |
| - "docs/templates/**" | |
| - "migrations/**" | |
| - "scripts/**" | |
| - "services/**" | |
| workflow_dispatch: | |
| inputs: | |
| release_tag: | |
| description: "Optional Docker release tag. Defaults to production-<short-sha>." | |
| required: false | |
| type: string | |
| image_platform: | |
| description: "Docker platform for production hosts." | |
| required: false | |
| default: "linux/amd64" | |
| type: string | |
| deploy_control_plane: | |
| description: "Deploy rend-api and rend-media-worker after the image release." | |
| required: true | |
| default: true | |
| type: boolean | |
| deploy_edges: | |
| description: "Deploy configured edge hosts after the control plane." | |
| required: true | |
| default: true | |
| type: boolean | |
| run_readiness: | |
| description: "Run the synthetic playback readiness gate after deploy." | |
| required: true | |
| default: true | |
| type: boolean | |
| run_first_host_verifier: | |
| description: "Run scripts/verify-first-host-deploy.sh after deploy." | |
| required: true | |
| default: true | |
| type: boolean | |
| verify_asset_id: | |
| description: "Optional existing hls_ready synthetic/non-customer asset id for deeper verifier playback smoke." | |
| required: false | |
| type: string | |
| verify_control_plane_rollback: | |
| description: "After verification, drill control-plane rollback to the previous slot and re-promote this release." | |
| required: true | |
| default: false | |
| type: boolean | |
| permissions: | |
| contents: read | |
| concurrency: | |
| group: rend-production-deploy | |
| cancel-in-progress: false | |
| jobs: | |
| release-images: | |
| name: Build and push release images | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| packages: write | |
| outputs: | |
| image-platform: ${{ steps.release.outputs.image_platform }} | |
| manifest-path: ${{ steps.release.outputs.manifest_path }} | |
| release-tag: ${{ steps.release.outputs.release_tag }} | |
| steps: | |
| - name: Check out repository | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Log in to GHCR | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Build and push images | |
| id: release | |
| env: | |
| IMAGE_PLATFORM_INPUT: ${{ inputs.image_platform || 'linux/amd64' }} | |
| RELEASE_TAG_INPUT: ${{ inputs.release_tag || '' }} | |
| run: | | |
| set -euo pipefail | |
| image_prefix="ghcr.io/${GITHUB_REPOSITORY_OWNER,,}/rend" | |
| short_sha="${GITHUB_SHA::12}" | |
| release_tag="${RELEASE_TAG_INPUT:-production-$short_sha}" | |
| image_platform="${IMAGE_PLATFORM_INPUT:-linux/amd64}" | |
| manifest_path=".rend/releases/rend-images-$release_tag.json" | |
| scripts/release-images.sh \ | |
| --tag "$release_tag" \ | |
| --registry "$image_prefix" \ | |
| --platform "$image_platform" \ | |
| --manifest "$manifest_path" \ | |
| --artifact-dir ".rend/release-artifacts" \ | |
| --push | |
| { | |
| echo "image_platform=$image_platform" | |
| echo "manifest_path=$manifest_path" | |
| echo "release_tag=$release_tag" | |
| } >> "$GITHUB_OUTPUT" | |
| - name: Upload release manifest | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: rend-release-manifest | |
| path: ${{ steps.release.outputs.manifest_path }} | |
| if-no-files-found: error | |
| retention-days: 30 | |
| deploy: | |
| name: Deploy production hosts | |
| runs-on: ubuntu-latest | |
| needs: release-images | |
| if: ${{ github.event_name == 'push' || inputs.deploy_control_plane || inputs.deploy_edges }} | |
| environment: | |
| name: Production | |
| url: ${{ vars.REND_API_BASE_URL || 'https://api.rend.so' }} | |
| permissions: | |
| contents: read | |
| packages: read | |
| env: | |
| DEPLOY_CONTROL_PLANE: ${{ github.event_name == 'push' || inputs.deploy_control_plane }} | |
| DEPLOY_EDGES: ${{ github.event_name == 'push' || inputs.deploy_edges }} | |
| RUN_READINESS: ${{ github.event_name == 'push' || inputs.run_readiness }} | |
| RUN_FIRST_HOST_VERIFIER: ${{ github.event_name == 'push' || inputs.run_first_host_verifier }} | |
| VERIFY_CONTROL_PLANE_ROLLBACK: ${{ github.event_name == 'workflow_dispatch' && inputs.verify_control_plane_rollback }} | |
| IMAGE_PLATFORM: ${{ needs.release-images.outputs.image-platform }} | |
| REND_API_BASE_URL: ${{ vars.REND_API_BASE_URL || 'https://api.rend.so' }} | |
| REND_READINESS_EDGES: ${{ vars.REND_READINESS_EDGES }} | |
| REND_VERIFY_ASSET_ID: ${{ inputs.verify_asset_id || vars.REND_VERIFY_ASSET_ID }} | |
| REND_READINESS_API_KEY: ${{ secrets.REND_READINESS_API_KEY }} | |
| REND_EDGE_INTERNAL_TOKEN: ${{ secrets.REND_EDGE_INTERNAL_TOKEN }} | |
| REND_SSH_PRIVATE_KEY: ${{ secrets.REND_SSH_PRIVATE_KEY }} | |
| REND_SSH_KNOWN_HOSTS: ${{ secrets.REND_SSH_KNOWN_HOSTS }} | |
| REND_CONTROL_PLANE_SSH_HOST: ${{ secrets.REND_CONTROL_PLANE_SSH_HOST }} | |
| REND_CONTROL_PLANE_SSH_USER: ${{ secrets.REND_CONTROL_PLANE_SSH_USER }} | |
| REND_CONTROL_PLANE_SSH_PORT: ${{ secrets.REND_CONTROL_PLANE_SSH_PORT }} | |
| REND_EDGE_ASH_SSH_HOST: ${{ secrets.REND_EDGE_ASH_SSH_HOST }} | |
| REND_EDGE_ASH_SSH_USER: ${{ secrets.REND_EDGE_ASH_SSH_USER }} | |
| REND_EDGE_ASH_SSH_PORT: ${{ secrets.REND_EDGE_ASH_SSH_PORT }} | |
| REND_EDGE_AMS_SSH_HOST: ${{ secrets.REND_EDGE_AMS_SSH_HOST }} | |
| REND_EDGE_AMS_SSH_USER: ${{ secrets.REND_EDGE_AMS_SSH_USER }} | |
| REND_EDGE_AMS_SSH_PORT: ${{ secrets.REND_EDGE_AMS_SSH_PORT }} | |
| CLICKHOUSE_URL: ${{ secrets.CLICKHOUSE_URL || vars.CLICKHOUSE_URL }} | |
| CLICKHOUSE_DATABASE: ${{ secrets.CLICKHOUSE_DATABASE || vars.CLICKHOUSE_DATABASE || 'rend' }} | |
| CLICKHOUSE_USER: ${{ secrets.CLICKHOUSE_USER || vars.CLICKHOUSE_USER }} | |
| CLICKHOUSE_PASSWORD: ${{ secrets.CLICKHOUSE_PASSWORD }} | |
| REND_API_CORS_ALLOWED_ORIGINS: ${{ vars.REND_API_CORS_ALLOWED_ORIGINS || 'https://rend.so,https://www.rend.so' }} | |
| REND_EDGE_CORS_ALLOWED_ORIGINS: ${{ vars.REND_EDGE_CORS_ALLOWED_ORIGINS || 'https://rend.so,https://www.rend.so' }} | |
| REND_PLAYBACK_BOOTSTRAP_PREFETCH_SEGMENTS: ${{ vars.REND_PLAYBACK_BOOTSTRAP_PREFETCH_SEGMENTS || '8' }} | |
| AUTUMN_SECRET_KEY: ${{ secrets.AUTUMN_SECRET_KEY }} | |
| AUTUMN_API_URL: ${{ vars.AUTUMN_API_URL || 'https://api.useautumn.com/v1' }} | |
| AUTUMN_API_VERSION: ${{ vars.AUTUMN_API_VERSION || '2.3.0' }} | |
| steps: | |
| - name: Check out repository | |
| uses: actions/checkout@v4 | |
| - name: Download release manifest | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: rend-release-manifest | |
| path: .rend/deploy | |
| - name: Resolve release manifest | |
| id: manifest | |
| run: | | |
| set -euo pipefail | |
| manifest="$(find .rend/deploy -type f -name 'rend-images-*.json' | sort | head -n 1)" | |
| if [[ -z "$manifest" ]]; then | |
| echo "release manifest was not downloaded" >&2 | |
| exit 1 | |
| fi | |
| python3 -m json.tool "$manifest" >/dev/null | |
| echo "path=$manifest" >> "$GITHUB_OUTPUT" | |
| - name: Configure SSH | |
| env: | |
| SSH_KEY_PATH: ${{ runner.temp }}/rend_deploy_key | |
| run: | | |
| set -euo pipefail | |
| if [[ -z "$REND_SSH_PRIVATE_KEY" ]]; then | |
| echo "REND_SSH_PRIVATE_KEY is required" >&2 | |
| exit 1 | |
| fi | |
| if [[ -z "$REND_SSH_KNOWN_HOSTS" ]]; then | |
| echo "REND_SSH_KNOWN_HOSTS is required" >&2 | |
| exit 1 | |
| fi | |
| install -m 700 -d "$HOME/.ssh" | |
| printf '%s\n' "$REND_SSH_PRIVATE_KEY" > "$SSH_KEY_PATH" | |
| chmod 600 "$SSH_KEY_PATH" | |
| printf '%s\n' "$REND_SSH_KNOWN_HOSTS" > "$HOME/.ssh/known_hosts" | |
| chmod 600 "$HOME/.ssh/known_hosts" | |
| echo "REND_SSH_KEY_PATH=$SSH_KEY_PATH" >> "$GITHUB_ENV" | |
| - name: Sync control-plane deploy env | |
| if: env.DEPLOY_CONTROL_PLANE == 'true' | |
| run: | | |
| set -euo pipefail | |
| : "${CLICKHOUSE_URL:?CLICKHOUSE_URL Production secret or variable is required}" | |
| : "${CLICKHOUSE_USER:?CLICKHOUSE_USER Production secret or variable is required}" | |
| : "${CLICKHOUSE_PASSWORD:?CLICKHOUSE_PASSWORD Production environment secret is required}" | |
| : "${AUTUMN_SECRET_KEY:?AUTUMN_SECRET_KEY Production environment secret is required}" | |
| scripts/sync-control-plane-billing-env-over-ssh.sh \ | |
| --host "$REND_CONTROL_PLANE_SSH_HOST" \ | |
| --user "$REND_CONTROL_PLANE_SSH_USER" \ | |
| --port "${REND_CONTROL_PLANE_SSH_PORT:-22}" | |
| - name: Deploy control plane | |
| if: env.DEPLOY_CONTROL_PLANE == 'true' | |
| run: | | |
| set -euo pipefail | |
| : "${REND_CONTROL_PLANE_SSH_HOST:?REND_CONTROL_PLANE_SSH_HOST is required}" | |
| : "${REND_CONTROL_PLANE_SSH_USER:?REND_CONTROL_PLANE_SSH_USER is required}" | |
| mkdir -p .rend/deploy | |
| monitor_file=".rend/deploy/public-readyz-monitor-${GITHUB_RUN_ID}.jsonl" | |
| monitor_stop_file="${RUNNER_TEMP}/rend-public-readyz-stop" | |
| monitor_failed_file="${RUNNER_TEMP}/rend-public-readyz-failed" | |
| rm -f "$monitor_stop_file" "$monitor_failed_file" | |
| probe_public_readyz() { | |
| local attempt http_status last_status | |
| last_status="000" | |
| for attempt in 1 2 3; do | |
| http_status="$(curl -sS -o /dev/null -w '%{http_code}' --max-time 5 "$REND_API_BASE_URL/readyz" 2>/dev/null || true)" | |
| last_status="${http_status:-000}" | |
| if [[ "$last_status" == "200" ]]; then | |
| printf '%s\t%s\n' "$last_status" "$attempt" | |
| return 0 | |
| fi | |
| sleep 1 | |
| done | |
| printf '%s\t3\n' "$last_status" | |
| return 1 | |
| } | |
| monitor_public_readyz() { | |
| local attempts http_status | |
| while [[ ! -f "$monitor_stop_file" ]]; do | |
| ts="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" | |
| IFS=$'\t' read -r http_status attempts < <(probe_public_readyz) | |
| if [[ "$http_status" == "200" ]]; then | |
| printf '{"ts":"%s","ok":true,"http_status":"%s","attempts":%s}\n' "$ts" "$http_status" "$attempts" >> "$monitor_file" | |
| else | |
| printf '{"ts":"%s","ok":false,"http_status":"%s","attempts":%s}\n' "$ts" "${http_status:-000}" "$attempts" >> "$monitor_file" | |
| touch "$monitor_failed_file" | |
| fi | |
| sleep 2 | |
| done | |
| } | |
| stop_monitor() { | |
| touch "$monitor_stop_file" | |
| wait "$monitor_pid" 2>/dev/null || true | |
| } | |
| monitor_public_readyz & | |
| monitor_pid="$!" | |
| trap stop_monitor EXIT | |
| export REND_CONTROL_PLANE_POST_PROMOTION_READY_URL="$REND_API_BASE_URL/readyz" | |
| scripts/deploy-release-over-ssh.sh \ | |
| --role control-plane \ | |
| --host "$REND_CONTROL_PLANE_SSH_HOST" \ | |
| --user "$REND_CONTROL_PLANE_SSH_USER" \ | |
| --port "${REND_CONTROL_PLANE_SSH_PORT:-22}" \ | |
| --expected-platform "$IMAGE_PLATFORM" \ | |
| --remote-dir "/tmp/rend-deploy-${GITHUB_RUN_ID}-${GITHUB_SHA::12}-control-plane" \ | |
| --manifest "${{ steps.manifest.outputs.path }}" | |
| stop_monitor | |
| trap - EXIT | |
| if [[ -f "$monitor_failed_file" ]]; then | |
| echo "public /readyz monitor observed at least one failed request during control-plane deploy" >&2 | |
| exit 1 | |
| fi | |
| - name: Deploy edge hosts | |
| if: env.DEPLOY_EDGES == 'true' | |
| run: | | |
| set -euo pipefail | |
| : "${REND_READINESS_EDGES:?REND_READINESS_EDGES repository/environment variable is required for edge identity sync}" | |
| IFS=',' read -r -a edge_entries <<< "$REND_READINESS_EDGES" | |
| expected_edges="" | |
| ash_edge_id="" | |
| ash_edge_region="" | |
| ash_edge_base="" | |
| ams_edge_id="" | |
| ams_edge_region="" | |
| ams_edge_base="" | |
| for entry in "${edge_entries[@]}"; do | |
| IFS='=' read -r edge_id region public_base _private_base extra <<< "$entry" | |
| if [[ -z "$edge_id" || -z "$region" || -z "$public_base" || -n "${extra:-}" ]]; then | |
| echo "invalid REND_READINESS_EDGES entry for edge identity sync" >&2 | |
| exit 1 | |
| fi | |
| expected_item="${edge_id}=${region}=${public_base}" | |
| if [[ -z "$expected_edges" ]]; then | |
| expected_edges="$expected_item" | |
| else | |
| expected_edges="${expected_edges},${expected_item}" | |
| fi | |
| case "$edge_id" in | |
| rend-edge-ash-1) | |
| ash_edge_id="$edge_id" | |
| ash_edge_region="$region" | |
| ash_edge_base="$public_base" | |
| ;; | |
| rend-edge-ams-1) | |
| ams_edge_id="$edge_id" | |
| ams_edge_region="$region" | |
| ams_edge_base="$public_base" | |
| ;; | |
| esac | |
| done | |
| deploy_edge() { | |
| local label="$1" | |
| local host="$2" | |
| local user="$3" | |
| local port="${4:-22}" | |
| local edge_id="${5:-}" | |
| local region="${6:-}" | |
| local public_base="${7:-}" | |
| if [[ -z "$host" && -z "$user" ]]; then | |
| echo "Skipping $label edge: SSH host/user secrets are not configured" | |
| return 0 | |
| fi | |
| if [[ -z "$host" || -z "$user" ]]; then | |
| echo "$label edge requires both host and user secrets" >&2 | |
| return 1 | |
| fi | |
| if [[ -z "$edge_id" || -z "$region" || -z "$public_base" ]]; then | |
| echo "$label edge has no matching REND_READINESS_EDGES identity entry" >&2 | |
| return 1 | |
| fi | |
| REND_EDGE_ID="$edge_id" \ | |
| REND_EDGE_REGION="$region" \ | |
| REND_EDGE_BASE_URL="$public_base" \ | |
| REND_EXPECTED_EDGES="$expected_edges" \ | |
| scripts/sync-edge-deploy-env-over-ssh.sh \ | |
| --host "$host" \ | |
| --user "$user" \ | |
| --port "${port:-22}" | |
| scripts/deploy-release-over-ssh.sh \ | |
| --role edge \ | |
| --host "$host" \ | |
| --user "$user" \ | |
| --port "${port:-22}" \ | |
| --expected-platform "$IMAGE_PLATFORM" \ | |
| --remote-dir "/tmp/rend-deploy-${GITHUB_RUN_ID}-${GITHUB_SHA::12}-$label" \ | |
| --manifest "${{ steps.manifest.outputs.path }}" | |
| } | |
| deployed=0 | |
| if [[ -n "$REND_EDGE_ASH_SSH_HOST$REND_EDGE_ASH_SSH_USER" ]]; then | |
| deploy_edge "ash" "$REND_EDGE_ASH_SSH_HOST" "$REND_EDGE_ASH_SSH_USER" "${REND_EDGE_ASH_SSH_PORT:-22}" "$ash_edge_id" "$ash_edge_region" "$ash_edge_base" | |
| deployed=$((deployed + 1)) | |
| fi | |
| if [[ -n "$REND_EDGE_AMS_SSH_HOST$REND_EDGE_AMS_SSH_USER" ]]; then | |
| deploy_edge "ams" "$REND_EDGE_AMS_SSH_HOST" "$REND_EDGE_AMS_SSH_USER" "${REND_EDGE_AMS_SSH_PORT:-22}" "$ams_edge_id" "$ams_edge_region" "$ams_edge_base" | |
| deployed=$((deployed + 1)) | |
| fi | |
| if [[ "$deployed" == "0" ]]; then | |
| echo "DEPLOY_EDGES=true but no edge SSH targets were configured" >&2 | |
| exit 1 | |
| fi | |
| - name: Verify public API readiness | |
| run: | | |
| set -euo pipefail | |
| curl -fsS --retry 12 --retry-delay 5 --retry-all-errors "$REND_API_BASE_URL/readyz" | |
| internal_status="$(curl -sS -o /dev/null -w '%{http_code}' "$REND_API_BASE_URL/internal/edges/heartbeat")" | |
| if [[ "$internal_status" != "404" ]]; then | |
| echo "public API must not expose /internal/*; got HTTP $internal_status" >&2 | |
| exit 1 | |
| fi | |
| assets_status="$(curl -sS -o /dev/null -w '%{http_code}' "$REND_API_BASE_URL/v1/assets")" | |
| case "$assets_status" in | |
| 401 | 403) ;; | |
| *) | |
| echo "public /v1/assets route should require auth; got HTTP $assets_status" >&2 | |
| exit 1 | |
| ;; | |
| esac | |
| - name: Install deploy verification dependencies | |
| if: env.RUN_READINESS == 'true' || env.RUN_FIRST_HOST_VERIFIER == 'true' | |
| run: | | |
| set -euo pipefail | |
| sudo apt-get update | |
| sudo apt-get install -y ffmpeg | |
| - name: Open edge readiness SSH tunnels | |
| if: env.RUN_READINESS == 'true' || env.RUN_FIRST_HOST_VERIFIER == 'true' | |
| run: | | |
| set -euo pipefail | |
| : "${REND_READINESS_EDGES:?REND_READINESS_EDGES repository/environment variable is required for deploy verification}" | |
| start_tunnel() { | |
| local label="$1" | |
| local host="$2" | |
| local user="$3" | |
| local port="$4" | |
| local local_port="$5" | |
| if [[ -z "$host" || -z "$user" ]]; then | |
| echo "cannot open $label readiness tunnel without SSH host and user" >&2 | |
| exit 1 | |
| fi | |
| ssh \ | |
| -o BatchMode=yes \ | |
| -o IdentitiesOnly=yes \ | |
| -o StrictHostKeyChecking=yes \ | |
| -o ExitOnForwardFailure=yes \ | |
| -i "$REND_SSH_KEY_PATH" \ | |
| -p "$port" \ | |
| -f -N \ | |
| -L "127.0.0.1:${local_port}:127.0.0.1:4100" \ | |
| "$user@$host" | |
| curl -fsS --retry 12 --retry-delay 2 --retry-all-errors \ | |
| "http://127.0.0.1:${local_port}/readyz" >/dev/null | |
| } | |
| start_tunnel "ash" "$REND_EDGE_ASH_SSH_HOST" "$REND_EDGE_ASH_SSH_USER" "${REND_EDGE_ASH_SSH_PORT:-22}" 14100 | |
| start_tunnel "ams" "$REND_EDGE_AMS_SSH_HOST" "$REND_EDGE_AMS_SSH_USER" "${REND_EDGE_AMS_SSH_PORT:-22}" 14101 | |
| IFS=',' read -r -a edge_entries <<< "$REND_READINESS_EDGES" | |
| readiness_edges="" | |
| for entry in "${edge_entries[@]}"; do | |
| IFS='=' read -r edge_id region public_base _private_base <<< "$entry" | |
| case "$edge_id" in | |
| rend-edge-ash-1) | |
| private_base="http://127.0.0.1:14100" | |
| ;; | |
| rend-edge-ams-1) | |
| private_base="http://127.0.0.1:14101" | |
| ;; | |
| *) | |
| echo "no readiness tunnel mapping for edge id: $edge_id" >&2 | |
| exit 1 | |
| ;; | |
| esac | |
| item="${edge_id}=${region}=${public_base}=${private_base}" | |
| if [[ -z "$readiness_edges" ]]; then | |
| readiness_edges="$item" | |
| else | |
| readiness_edges="${readiness_edges},${item}" | |
| fi | |
| done | |
| echo "REND_READINESS_EDGES=$readiness_edges" >> "$GITHUB_ENV" | |
| - name: Verify edge registry from control-plane host | |
| if: env.RUN_FIRST_HOST_VERIFIER == 'true' | |
| run: | | |
| set -euo pipefail | |
| : "${REND_CONTROL_PLANE_SSH_HOST:?REND_CONTROL_PLANE_SSH_HOST is required}" | |
| : "${REND_CONTROL_PLANE_SSH_USER:?REND_CONTROL_PLANE_SSH_USER is required}" | |
| : "${REND_READINESS_EDGES:?REND_READINESS_EDGES repository/environment variable is required when RUN_FIRST_HOST_VERIFIER=true}" | |
| scripts/verify-edge-registry-over-ssh.sh \ | |
| --host "$REND_CONTROL_PLANE_SSH_HOST" \ | |
| --user "$REND_CONTROL_PLANE_SSH_USER" \ | |
| --port "${REND_CONTROL_PLANE_SSH_PORT:-22}" \ | |
| --expected-edges "$REND_READINESS_EDGES" | |
| - name: Run first-host verifier | |
| if: env.RUN_FIRST_HOST_VERIFIER == 'true' | |
| run: | | |
| set -euo pipefail | |
| : "${REND_READINESS_API_KEY:?REND_READINESS_API_KEY is required when RUN_FIRST_HOST_VERIFIER=true}" | |
| : "${REND_EDGE_INTERNAL_TOKEN:?REND_EDGE_INTERNAL_TOKEN is required when RUN_FIRST_HOST_VERIFIER=true}" | |
| : "${REND_READINESS_EDGES:?REND_READINESS_EDGES repository/environment variable is required when RUN_FIRST_HOST_VERIFIER=true}" | |
| : "${CLICKHOUSE_URL:?CLICKHOUSE_URL Production secret or variable is required when RUN_FIRST_HOST_VERIFIER=true}" | |
| : "${CLICKHOUSE_DATABASE:?CLICKHOUSE_DATABASE Production secret or variable is required when RUN_FIRST_HOST_VERIFIER=true}" | |
| : "${CLICKHOUSE_USER:?CLICKHOUSE_USER Production secret or variable is required when RUN_FIRST_HOST_VERIFIER=true}" | |
| : "${CLICKHOUSE_PASSWORD:?CLICKHOUSE_PASSWORD Production secret is required when RUN_FIRST_HOST_VERIFIER=true}" | |
| verifier_args=( | |
| --api-base "$REND_API_BASE_URL" | |
| --clickhouse-url "$CLICKHOUSE_URL" | |
| --clickhouse-database "$CLICKHOUSE_DATABASE" | |
| --clickhouse-user "$CLICKHOUSE_USER" | |
| --clickhouse-password "$CLICKHOUSE_PASSWORD" | |
| --dev-api-key "$REND_READINESS_API_KEY" | |
| --edge-internal-token "$REND_EDGE_INTERNAL_TOKEN" | |
| --rewrite-playback-base | |
| --skip-registration | |
| ) | |
| if [[ -n "${REND_VERIFY_ASSET_ID:-}" ]]; then | |
| verifier_args+=(--asset-id "$REND_VERIFY_ASSET_ID") | |
| else | |
| if [[ "${RUN_READINESS:-}" != "true" ]]; then | |
| echo "REND_VERIFY_ASSET_ID or RUN_READINESS=true is required for playback coverage" >&2 | |
| exit 1 | |
| fi | |
| verifier_args+=(--skip-playback) | |
| fi | |
| IFS=',' read -r -a edge_entries <<< "$REND_READINESS_EDGES" | |
| expected_edges="" | |
| for entry in "${edge_entries[@]}"; do | |
| IFS='=' read -r edge_id region public_base private_base extra <<< "$entry" | |
| if [[ -z "$edge_id" || -z "$region" || -z "$public_base" || -n "${extra:-}" ]]; then | |
| echo "invalid REND_READINESS_EDGES entry for first-host verifier" >&2 | |
| exit 1 | |
| fi | |
| if [[ -z "$private_base" ]]; then | |
| private_base="$public_base" | |
| fi | |
| expected_item="${edge_id}=${region}=${public_base}" | |
| if [[ -z "$expected_edges" ]]; then | |
| expected_edges="$expected_item" | |
| else | |
| expected_edges="${expected_edges},${expected_item}" | |
| fi | |
| verifier_args+=(--edge-base "$public_base" --edge-internal-base "$private_base") | |
| done | |
| verifier_args+=(--expected-edges "$expected_edges") | |
| scripts/verify-first-host-deploy.sh "${verifier_args[@]}" | |
| - name: Run playback readiness gate | |
| if: env.RUN_READINESS == 'true' | |
| run: | | |
| set -euo pipefail | |
| : "${REND_READINESS_API_KEY:?REND_READINESS_API_KEY is required when RUN_READINESS=true}" | |
| : "${REND_EDGE_INTERNAL_TOKEN:?REND_EDGE_INTERNAL_TOKEN is required when RUN_READINESS=true}" | |
| : "${REND_READINESS_EDGES:?REND_READINESS_EDGES repository/environment variable is required when RUN_READINESS=true}" | |
| node scripts/playback-readiness-gate.mjs \ | |
| --target configured \ | |
| --skip-local-stack \ | |
| --output ".rend/readiness/playback-readiness-${GITHUB_RUN_ID}.json" \ | |
| --latest-output ".rend/readiness/playback-readiness-latest.json" | |
| - name: Verify control-plane rollback drill | |
| if: env.VERIFY_CONTROL_PLANE_ROLLBACK == 'true' | |
| run: | | |
| set -euo pipefail | |
| : "${REND_CONTROL_PLANE_SSH_HOST:?REND_CONTROL_PLANE_SSH_HOST is required}" | |
| : "${REND_CONTROL_PLANE_SSH_USER:?REND_CONTROL_PLANE_SSH_USER is required}" | |
| mkdir -p .rend/deploy | |
| monitor_file=".rend/deploy/public-readyz-monitor-rollback-${GITHUB_RUN_ID}.jsonl" | |
| monitor_stop_file="${RUNNER_TEMP}/rend-rollback-readyz-stop" | |
| monitor_failed_file="${RUNNER_TEMP}/rend-rollback-readyz-failed" | |
| rm -f "$monitor_stop_file" "$monitor_failed_file" | |
| probe_public_readyz() { | |
| local attempt http_status last_status | |
| last_status="000" | |
| for attempt in 1 2 3; do | |
| http_status="$(curl -sS -o /dev/null -w '%{http_code}' --max-time 5 "$REND_API_BASE_URL/readyz" 2>/dev/null || true)" | |
| last_status="${http_status:-000}" | |
| if [[ "$last_status" == "200" ]]; then | |
| printf '%s\t%s\n' "$last_status" "$attempt" | |
| return 0 | |
| fi | |
| sleep 1 | |
| done | |
| printf '%s\t3\n' "$last_status" | |
| return 1 | |
| } | |
| monitor_public_readyz() { | |
| local attempts http_status | |
| while [[ ! -f "$monitor_stop_file" ]]; do | |
| ts="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" | |
| IFS=$'\t' read -r http_status attempts < <(probe_public_readyz) | |
| if [[ "$http_status" == "200" ]]; then | |
| printf '{"ts":"%s","phase":"rollback-drill","ok":true,"http_status":"%s","attempts":%s}\n' "$ts" "$http_status" "$attempts" >> "$monitor_file" | |
| else | |
| printf '{"ts":"%s","phase":"rollback-drill","ok":false,"http_status":"%s","attempts":%s}\n' "$ts" "${http_status:-000}" "$attempts" >> "$monitor_file" | |
| touch "$monitor_failed_file" | |
| fi | |
| sleep 2 | |
| done | |
| } | |
| stop_monitor() { | |
| touch "$monitor_stop_file" | |
| wait "$monitor_pid" 2>/dev/null || true | |
| } | |
| monitor_public_readyz & | |
| monitor_pid="$!" | |
| trap stop_monitor EXIT | |
| scripts/deploy-release-over-ssh.sh \ | |
| --role control-plane \ | |
| --host "$REND_CONTROL_PLANE_SSH_HOST" \ | |
| --user "$REND_CONTROL_PLANE_SSH_USER" \ | |
| --port "${REND_CONTROL_PLANE_SSH_PORT:-22}" \ | |
| --rollback | |
| curl -fsS --retry 12 --retry-delay 5 --retry-all-errors "$REND_API_BASE_URL/readyz" >/dev/null | |
| export REND_CONTROL_PLANE_POST_PROMOTION_READY_URL="$REND_API_BASE_URL/readyz" | |
| scripts/deploy-release-over-ssh.sh \ | |
| --role control-plane \ | |
| --host "$REND_CONTROL_PLANE_SSH_HOST" \ | |
| --user "$REND_CONTROL_PLANE_SSH_USER" \ | |
| --port "${REND_CONTROL_PLANE_SSH_PORT:-22}" \ | |
| --expected-platform "$IMAGE_PLATFORM" \ | |
| --remote-dir "/tmp/rend-deploy-${GITHUB_RUN_ID}-${GITHUB_SHA::12}-control-plane-rollback-drill" \ | |
| --manifest "${{ steps.manifest.outputs.path }}" | |
| curl -fsS --retry 12 --retry-delay 5 --retry-all-errors "$REND_API_BASE_URL/readyz" >/dev/null | |
| stop_monitor | |
| trap - EXIT | |
| if [[ -f "$monitor_failed_file" ]]; then | |
| echo "public /readyz monitor observed at least one failed request during control-plane rollback drill" >&2 | |
| exit 1 | |
| fi | |
| - name: Upload deploy monitor artifact | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: rend-deploy-monitor | |
| path: .rend/deploy/public-readyz-monitor-*.jsonl | |
| if-no-files-found: ignore | |
| retention-days: 14 | |
| - name: Upload readiness artifact | |
| if: always() && env.RUN_READINESS == 'true' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: rend-playback-readiness | |
| path: .rend/readiness/*.json | |
| if-no-files-found: ignore | |
| retention-days: 14 |