feat(clp-package): Add a Spider-based Docker Compose deployment for Spider-orchestrated compression. (#2479)

Co-authored-by: Junhao Liao <junhao@junhao.ca>

Author: 20001020ycx

tools/deployment/package/compose.clp-spider.yaml

@@ -0,0 +1,15 @@
+services:
+  spider-worker:
+    image: >-
+      ${CLP_SPIDER_WORKER_CONTAINER_IMAGE_REF:-ghcr.io/y-scope/clp/clp-spider-worker:main}
+    user: "${CLP_FIRST_PARTY_SERVICE_UID_GID:-1000:1000}"
+    environment:
+      CLP_CONFIG_PATH: "/etc/clp-config.yaml"
+      CLP_DB_PASS: "${CLP_DB_PASS:?Please set a value.}"
+      CLP_DB_USER: "${CLP_DB_USER:-clp-user}"
+    volumes:
+      - type: "bind"
+        source: "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yaml"
+        target: "/etc/clp-config.yaml"
+        read_only: true
+      - "${CLP_STAGED_ARCHIVE_OUTPUT_DIR_HOST:-empty}:/var/data/staged-archives"

tools/deployment/package/docker-compose-all.yaml

@@ -280,6 +280,27 @@ services:
       "--config", "/etc/clp-config.yaml"
     ]
 
+  compression-coordinator:
+    <<: *service_defaults
+    hostname: "compression_coordinator"
+    environment:
+      <<: *clp_telemetry_env_defaults
+      CLP_DB_PASS: "${CLP_DB_PASS:?Please set a value.}"
+      CLP_DB_USER: "${CLP_DB_USER:-clp-user}"
+      OTEL_SERVICE_NAME: "compression-coordinator"
+      RUST_LOG: "${CLP_COMPRESSION_COORDINATOR_LOGGING_LEVEL:-INFO}"
+    volumes:
+      - *volume_clp_config_readonly
+    depends_on:
+      db-table-creator:
+        condition: "service_completed_successfully"
+      spider-storage:
+        condition: "service_healthy"
+    command: [
+      "/opt/clp/bin/compression-coordinator",
+      "--config", "/etc/clp-config.yaml"
+    ]
+
   compression-worker:
     <<: *service_defaults
     hostname: "compression_worker"

tools/deployment/package/docker-compose-spider.yaml

@@ -0,0 +1,98 @@
+name: "clp-package-spider"
+
+include:
+  - "docker-compose.utils.yaml"
+  - path:
+      - "usr/share/spider-compose/compose.yaml"
+      - "compose.clp-spider.yaml"
+
+services:
+  database:
+    extends:
+      file: "docker-compose-all.yaml"
+      service: "database"
+
+  db-table-creator:
+    extends:
+      file: "docker-compose-all.yaml"
+      service: "db-table-creator"
+
+  queue:
+    extends:
+      file: "docker-compose-all.yaml"
+      service: "queue"
+
+  redis:
+    extends:
+      file: "docker-compose-all.yaml"
+      service: "redis"
+
+  results-cache:
+    extends:
+      file: "docker-compose-all.yaml"
+      service: "results-cache"
+
+  results-cache-indices-creator:
+    extends:
+      file: "docker-compose-all.yaml"
+      service: "results-cache-indices-creator"
+
+  compression-scheduler:
+    extends:
+      file: "docker-compose-all.yaml"
+      service: "compression-scheduler"
+
+  compression-worker:
+    extends:
+      file: "docker-compose-all.yaml"
+      service: "compression-worker"
+
+  compression-coordinator:
+    extends:
+      file: "docker-compose-all.yaml"
+      service: "compression-coordinator"
+
+  webui:
+    extends:
+      file: "docker-compose-all.yaml"
+      service: "webui"
+
+  garbage-collector:
+    extends:
+      file: "docker-compose-all.yaml"
+      service: "garbage-collector"
+
+  api-server:
+    extends:
+      file: "docker-compose-all.yaml"
+      service: "api-server"
+
+  query-scheduler:
+    extends:
+      file: "docker-compose-all.yaml"
+      service: "query-scheduler"
+
+  query-worker:
+    extends:
+      file: "docker-compose-all.yaml"
+      service: "query-worker"
+
+  reducer:
+    extends:
+      file: "docker-compose-all.yaml"
+      service: "reducer"
+
+  mcp-server:
+    extends:
+      file: "docker-compose-all.yaml"
+      service: "mcp-server"
+
+  log-ingestor:
+    extends:
+      file: "docker-compose-all.yaml"
+      service: "log-ingestor"
+
+  otel-collector:
+    extends:
+      file: "docker-compose-all.yaml"
+      service: "otel-collector"