Skip to content
Nightly Integration

Nightly Integration #189

Sign in to view logs

Nightly Integration

Nightly Integration #189

Workflow file for this run

.github/workflows/integration-nightly.yml at 863b42a
name: Nightly Integration
on:
schedule:
- cron: '0 3 * * *' # 3 AM UTC daily
workflow_dispatch: # Manual trigger
permissions:
contents: read
jobs:
integration:
name: Full-Stack Integration
runs-on: ubuntu-latest
env:
# Deterministic test-only AES-256 key shared by admin and events.
WEBHOOK_SECRET_ENCRYPTION_KEY: "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
services:
redis:
image: redis:7-alpine
ports:
- 6379:6379
options: --health-cmd "redis-cli ping" --health-interval 10s --health-timeout 5s --health-retries 5
steps:
# ── Checkout all servers ─────────────────────────────────────
- name: Checkout cycles-server
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: runcycles/cycles-server
path: cycles-server
- name: Checkout cycles-server-admin
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: runcycles/cycles-server-admin
path: cycles-server-admin
- name: Checkout cycles-server-events
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: runcycles/cycles-server-events
path: cycles-server-events
# ── Build all servers ──────────────────────────────────────────
- name: Set up JDK 21
uses: actions/setup-java@b6effb05e454b25005698d916606bdc6ffcbf961 # v5
with:
distribution: temurin
java-version: '21'
cache: maven
- name: Build protocol server
working-directory: cycles-server
run: mvn -B package -DskipTests --file cycles-protocol-service/pom.xml
- name: Build admin server
working-directory: cycles-server-admin
run: mvn -B package -DskipTests --file cycles-admin-service/pom.xml
- name: Build event server
working-directory: cycles-server-events
run: mvn -B package -DskipTests
# ── Start all servers ──────────────────────────────────────────
# Readiness polling notes:
# - Aggregate /actuator/health requires X-Admin-API-Key since
# cycles-server v0.1.25.45 (and fails closed with 500 when no admin
# key is configured). The liveness/readiness probes are deliberately
# public and Redis-aware — poll those.
# - The loops FAIL the step on timeout. The previous tolerant loops
# let a never-ready server slide through to a confusing failure in a
# later test step (five nights of red before anyone saw the cause).
- name: Start protocol server (7878)
working-directory: cycles-server
run: |
java -Dredis.host=localhost -Dredis.port=6379 -Dredis.password= \
-Dadmin.api-key=test-admin-key \
-jar cycles-protocol-service/cycles-protocol-service-api/target/*.jar &
ready=""
for i in $(seq 1 30); do
if curl -sf http://localhost:7878/actuator/health/readiness; then ready=1; break; fi
sleep 1
done
[ -n "$ready" ] || { echo "protocol server not ready after 30s"; exit 1; }
- name: Start admin server (7979)
working-directory: cycles-server-admin
run: |
java -Dredis.host=localhost -Dredis.port=6379 -Dredis.password= \
-Dadmin.api-key=test-admin-key -Dserver.port=7979 \
-jar cycles-admin-service/cycles-admin-service-api/target/*.jar &
ready=""
for i in $(seq 1 30); do
if curl -sf http://localhost:7979/actuator/health/readiness; then ready=1; break; fi
sleep 1
done
[ -n "$ready" ] || { echo "admin server not ready after 30s"; exit 1; }
# Event server actuator lives on the MANAGEMENT port (9980), not the
# main port (7980) — see cycles-server-events application.properties.
- name: Start event server (7980)
working-directory: cycles-server-events
env:
# The integration receiver intentionally listens on localhost.
WEBHOOK_URL_GUARD_ALLOW_PRIVATE_NETWORKS: "true"
run: |
java -Dredis.host=localhost -Dredis.port=6379 -Dredis.password= \
-jar target/*.jar &
ready=""
for i in $(seq 1 30); do
if curl -sf http://localhost:9980/actuator/health/readiness; then ready=1; break; fi
sleep 1
done
[ -n "$ready" ] || { echo "event server not ready after 30s"; exit 1; }
# ── Provision test data via admin API ──────────────────────────
- name: Provision tenant, API key, and budget
run: |
ADMIN_URL=http://localhost:7979
ADMIN_KEY=test-admin-key
# Create tenant
curl -s -X POST $ADMIN_URL/v1/admin/tenants \
-H "Content-Type: application/json" \
-H "X-Admin-API-Key: $ADMIN_KEY" \
-d '{"tenant_id":"integration-test","name":"Integration Test Tenant"}'
echo ""
echo "Tenant created"
# Create API key
KEY_RESPONSE=$(curl -s -X POST $ADMIN_URL/v1/admin/api-keys \
-H "Content-Type: application/json" \
-H "X-Admin-API-Key: $ADMIN_KEY" \
-d '{"tenant_id":"integration-test","name":"integration-key","permissions":["admin:read","admin:write"]}')
echo "Key response: $KEY_RESPONSE"
API_KEY=$(echo "$KEY_RESPONSE" | jq -r '.key_secret')
if [ -z "$API_KEY" ] || [ "$API_KEY" = "null" ]; then
echo "Failed to extract API key"
exit 1
fi
echo "API key created: ${API_KEY:0:14}..."
# Export for subsequent steps
echo "CYCLES_API_KEY=$API_KEY" >> $GITHUB_ENV
# Create budget ($1.00 = 100M USD_MICROCENTS)
BUDGET_RESPONSE=$(curl -s -w "\nHTTP_CODE:%{http_code}" -X POST $ADMIN_URL/v1/admin/budgets \
-H "Content-Type: application/json" \
-H "X-Cycles-API-Key: $API_KEY" \
-d '{"scope":"tenant:integration-test","unit":"USD_MICROCENTS","allocated":{"unit":"USD_MICROCENTS","amount":100000000}}')
echo "Budget response: $BUDGET_RESPONSE"
echo "$BUDGET_RESPONSE" | grep -q "HTTP_CODE:2" || { echo "USD_MICROCENTS budget creation failed"; exit 1; }
# Create TOKENS budget (for Rust client tests that use Amount::tokens())
TOKENS_BUDGET=$(curl -s -w "\nHTTP_CODE:%{http_code}" -X POST $ADMIN_URL/v1/admin/budgets \
-H "Content-Type: application/json" \
-H "X-Cycles-API-Key: $API_KEY" \
-d '{"scope":"tenant:integration-test","unit":"TOKENS","allocated":{"unit":"TOKENS","amount":1000000}}')
echo "Tokens budget response: $TOKENS_BUDGET"
echo "$TOKENS_BUDGET" | grep -q "HTTP_CODE:2" || { echo "TOKENS budget creation failed"; exit 1; }
# ── Spring Boot demo app ────────────────────────────────────────
- name: Checkout Spring Boot starter
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: runcycles/cycles-spring-boot-starter
path: cycles-spring-boot-starter
- name: Build and start Spring Boot demo app (7955)
working-directory: cycles-spring-boot-starter
run: |
# install (not package) so the freshly-built starter lands in ~/.m2 and
# the demo can resolve io.runcycles:cycles-client-java-spring at the
# current source version — which is not yet on Maven Central.
mvn -B install -DskipTests --file cycles-client-java-spring/pom.xml
mvn -B package -DskipTests --file cycles-demo-client-java-spring/pom.xml
java -Dcycles.base-url=http://localhost:7878 \
-Dcycles.api-key=$CYCLES_API_KEY \
-Dcycles.tenant=integration-test \
-Dcycles.workspace=default \
-Dcycles.app=demo \
-jar cycles-demo-client-java-spring/target/*.jar &
for i in $(seq 1 30); do
curl -sf http://localhost:7955/actuator/health && break
sleep 1
done
- name: Run Spring Boot integration tests
run: |
echo "=== @Cycles annotation (minimal) ==="
RESULT=$(curl -sf -X POST "http://localhost:7955/api/demo/annotation/minimal?input=hello")
echo "$RESULT" | jq .
echo "$RESULT" | jq -e '.result' || { echo "Minimal annotation failed"; exit 1; }
echo "=== Programmatic client (reserve-commit) ==="
RESULT=$(curl -sf -X POST "http://localhost:7955/api/demo/client/reserve-commit?amount=5000")
echo "$RESULT" | jq .
echo "$RESULT" | jq -e '.result.reservationId' || { echo "Reserve-commit failed"; exit 1; }
echo "=== Programmatic client (decide) ==="
RESULT=$(curl -sf -X POST "http://localhost:7955/api/demo/client/decide?amount=1000")
echo "$RESULT" | jq .
echo "$RESULT" | jq -e '.result.response.body.decision' || { echo "Decide failed"; exit 1; }
echo "=== Balance query ==="
RESULT=$(curl -sf "http://localhost:7955/api/demo/client/balances")
echo "$RESULT" | jq .
echo "=== LLM endpoint (annotation + metrics) ==="
RESULT=$(curl -sf -X POST "http://localhost:7955/api/llm/generate?prompt=hello&tokens=100")
echo "$RESULT"
[ -n "$RESULT" ] || { echo "LLM generate failed"; exit 1; }
echo "Spring Boot integration tests passed"
# ── Python integration tests ───────────────────────────────────
- name: Checkout Python client
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: runcycles/cycles-client-python
path: cycles-client-python
- name: Set up Python
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: '3.12'
- name: Run Python integration tests
working-directory: cycles-client-python
env:
CYCLES_BASE_URL: http://localhost:7878
CYCLES_TENANT: integration-test
run: |
pip install -e ".[dev]" requests
pytest tests/integration/ -v
# ── TypeScript integration tests ───────────────────────────────
- name: Checkout TypeScript client
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: runcycles/cycles-client-typescript
path: cycles-client-typescript
- name: Set up Node.js
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v6
with:
node-version: 22
- name: Run TypeScript integration tests
working-directory: cycles-client-typescript
env:
CYCLES_BASE_URL: http://localhost:7878
CYCLES_TENANT: integration-test
run: |
npm ci
npx vitest run tests/integration/
# ── Event server integration tests ─────────────────────────────
- name: Start webhook receiver
run: |
python3 -c '
from http.server import HTTPServer, BaseHTTPRequestHandler
import json, sys
class Handler(BaseHTTPRequestHandler):
received = []
def do_POST(self):
length = int(self.headers.get("Content-Length", 0))
body = self.rfile.read(length).decode()
Handler.received.append({
"body": body,
"signature": self.headers.get("X-Cycles-Signature", ""),
"event_type": self.headers.get("X-Cycles-Event-Type", ""),
})
self.send_response(200)
self.end_headers()
self.wfile.write(b"OK")
def do_GET(self):
self.send_response(200)
self.end_headers()
self.wfile.write(json.dumps(Handler.received).encode())
def log_message(self, *a): pass
HTTPServer(("", 9999), Handler).serve_forever()
' &
sleep 1
curl -sf http://localhost:9999/ > /dev/null
echo "Webhook receiver running on :9999"
- name: Run event server integration tests
run: |
ADMIN_URL=http://localhost:7979
ADMIN_KEY=test-admin-key
echo "=== Allow HTTP webhook URLs for CI ==="
curl -s -X PUT $ADMIN_URL/v1/admin/config/webhook-security \
-H "Content-Type: application/json" \
-H "X-Admin-API-Key: $ADMIN_KEY" \
-d '{"allow_http": true, "blocked_cidr_ranges": []}' | jq .
echo "=== Create webhook subscription ==="
WEBHOOK_RESP=$(curl -s -w "\nHTTP_CODE:%{http_code}" -X POST $ADMIN_URL/v1/webhooks \
-H "Content-Type: application/json" \
-H "X-Cycles-API-Key: $CYCLES_API_KEY" \
-d '{
"name": "nightly-test-webhook",
"url": "http://localhost:9999/webhook",
"event_types": ["budget.created", "reservation.denied", "tenant.created"],
"signing_secret": "nightly-test-secret"
}')
echo "Webhook response: $WEBHOOK_RESP"
echo "$WEBHOOK_RESP" | grep -q "HTTP_CODE:2" || { echo "Webhook creation failed"; exit 1; }
SUB_ID=$(echo "$WEBHOOK_RESP" | head -1 | jq -r '.subscription.subscription_id')
echo "Subscription ID: $SUB_ID"
echo "=== Test webhook delivery (end-to-end via event server) ==="
TEST_RESP=$(curl -s -w "\nHTTP_CODE:%{http_code}" -X POST "$ADMIN_URL/v1/webhooks/$SUB_ID/test" \
-H "X-Cycles-API-Key: $CYCLES_API_KEY")
echo "Test response: $TEST_RESP"
echo "$TEST_RESP" | grep -q "HTTP_CODE:2" || { echo "Webhook test failed"; exit 1; }
echo "$TEST_RESP" | head -1 | jq -e '.success == true' || { echo "Webhook test reported failure"; exit 1; }
echo "=== Verify webhook receiver got the delivery ==="
sleep 3
RECEIVED=$(curl -s http://localhost:9999/)
echo "Received webhooks: $RECEIVED"
COUNT=$(echo "$RECEIVED" | jq 'length')
[ "$COUNT" -ge 1 ] || { echo "No webhooks received by receiver"; exit 1; }
echo "$RECEIVED" | jq -e '.[0].signature | startswith("sha256=")' || { echo "Missing HMAC signature"; exit 1; }
echo "=== Verify event server health and metrics (management port 9980) ==="
curl -sf http://localhost:9980/actuator/health/readiness | jq .
curl -sf http://localhost:9980/actuator/prometheus | grep -c "cycles_" || true
echo "Event server integration tests passed"
# ── Rust integration tests ──────────────────────────────────────
- name: Checkout Rust client
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: runcycles/cycles-client-rust
path: cycles-client-rust
- name: Install Rust
uses: dtolnay/rust-toolchain@6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772 # stable @ 2026-05-02
with:
toolchain: stable
- name: Run Rust live integration tests
working-directory: cycles-client-rust
env:
CYCLES_BASE_URL: http://localhost:7878
CYCLES_TENANT: integration-test
run: cargo test --test live_server_test -- --ignored
# The nightly is the fleet's only cross-repo canary; a silent failure is
# how the June 2026 /actuator/health lockdown regression ran red for six
# nights unnoticed. On failure this job creates (or comments on) a
# pinned-title issue so the break is visible without watching the Actions
# tab. The issue is reused across consecutive failures to avoid spam and
# closed manually once the run is green again.
alert-on-failure:
name: Alert on failure
needs: integration
if: failure()
runs-on: ubuntu-latest
permissions:
issues: write
steps:
- name: Create or update failure issue
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
with:
script: |
const title = 'Nightly Integration is failing';
const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
const body = `[Run ${context.runId}](${runUrl}) failed on ${new Date().toISOString().slice(0, 10)} (${context.eventName}).`;
const { data: issues } = await github.rest.issues.listForRepo({
owner: context.repo.owner,
repo: context.repo.repo,
state: 'open',
labels: 'nightly-failure',
per_page: 1,
});
if (issues.length > 0) {
await github.rest.issues.createComment({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: issues[0].number,
body,
});
} else {
await github.rest.issues.create({
owner: context.repo.owner,
repo: context.repo.repo,
title,
body: `${body}\n\nThis issue is auto-created by the nightly workflow's alert-on-failure job. Close it once the nightly is green again.`,
labels: ['nightly-failure'],
});
}