From 30e3457a15a86a854139248de1afb615386950e8 Mon Sep 17 00:00:00 2001 From: Omar Alsoulah Date: Thu, 13 Aug 2026 16:27:58 +0300 Subject: [PATCH 01/28] =?UTF-8?q?sanad:=20worker=20runtime=20schema=20?= =?UTF-8?q?=E2=80=94=20workspaces,=20agents,=20versions,=20deployments,=20?= =?UTF-8?q?runs,=20invoke=20tokens?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sanad-web/drizzle/0006_foamy_venus.sql | 117 ++ .../sanad-web/drizzle/meta/0006_snapshot.json | 1777 +++++++++++++++++ .../sanad-web/drizzle/meta/_journal.json | 7 + .../artifacts/sanad-web/lib/db/schema.ts | 83 + .../tests/unit/worker-schema.test.ts | 25 + 5 files changed, 2009 insertions(+) create mode 100644 control-plane/artifacts/sanad-web/drizzle/0006_foamy_venus.sql create mode 100644 control-plane/artifacts/sanad-web/drizzle/meta/0006_snapshot.json create mode 100644 control-plane/artifacts/sanad-web/tests/unit/worker-schema.test.ts diff --git a/control-plane/artifacts/sanad-web/drizzle/0006_foamy_venus.sql b/control-plane/artifacts/sanad-web/drizzle/0006_foamy_venus.sql new file mode 100644 index 000000000..9752c65e3 --- /dev/null +++ b/control-plane/artifacts/sanad-web/drizzle/0006_foamy_venus.sql @@ -0,0 +1,117 @@ +CREATE TABLE "agent_versions" ( + "id" text PRIMARY KEY NOT NULL, + "agent_id" text NOT NULL, + "content_hash" text NOT NULL, + "bundle" jsonb NOT NULL, + "created_by" text NOT NULL, + "created_at" timestamp with time zone DEFAULT now() NOT NULL +); +--> statement-breakpoint +CREATE TABLE "agents" ( + "id" text PRIMARY KEY NOT NULL, + "workspace_id" text NOT NULL, + "name" text NOT NULL, + "owner_user_id" text NOT NULL, + "status" text DEFAULT 'active' NOT NULL, + "description" text, + "created_at" timestamp with time zone DEFAULT now() NOT NULL +); +--> statement-breakpoint +CREATE TABLE "deployments" ( + "id" text PRIMARY KEY NOT NULL, + "agent_id" text NOT NULL, + "agent_version_id" text NOT NULL, + "env" text NOT NULL, + "status" text DEFAULT 'active' NOT NULL, + "max_turn_seconds" integer DEFAULT 900 NOT NULL, + "max_steps_per_turn" integer DEFAULT 100 NOT NULL, + "max_tokens_per_run" integer DEFAULT 2000000 NOT NULL, + "created_at" timestamp with time zone DEFAULT now() NOT NULL, + "updated_at" timestamp with time zone DEFAULT now() NOT NULL +); +--> statement-breakpoint +CREATE TABLE "invoke_tokens" ( + "id" text PRIMARY KEY NOT NULL, + "token_hash" text NOT NULL, + "family_id" text NOT NULL, + "agent_id" text NOT NULL, + "env" text NOT NULL, + "org_id" text NOT NULL, + "created_by" text NOT NULL, + "expires_at" timestamp with time zone NOT NULL, + "revoked_at" timestamp with time zone, + "created_at" timestamp with time zone DEFAULT now() NOT NULL, + CONSTRAINT "invoke_tokens_token_hash_unique" UNIQUE("token_hash") +); +--> statement-breakpoint +CREATE TABLE "project_sessions" ( + "id" text PRIMARY KEY NOT NULL, + "project_id" text NOT NULL, + "user_id" text NOT NULL, + "name" text NOT NULL, + "ui_state" jsonb DEFAULT '{}'::jsonb NOT NULL, + "archived_at" timestamp with time zone, + "created_at" timestamp with time zone DEFAULT now() NOT NULL, + "updated_at" timestamp with time zone DEFAULT now() NOT NULL, + "last_active_at" timestamp with time zone DEFAULT now() NOT NULL +); +--> statement-breakpoint +CREATE TABLE "runs" ( + "id" text PRIMARY KEY NOT NULL, + "deployment_id" text NOT NULL, + "agent_version_id" text NOT NULL, + "status" text DEFAULT 'queued' NOT NULL, + "error_code" text, + "trigger_principal" text NOT NULL, + "idempotency_key" text, + "output" jsonb, + "tokens_in" integer DEFAULT 0 NOT NULL, + "tokens_out" integer DEFAULT 0 NOT NULL, + "cost_usd_micros" integer DEFAULT 0 NOT NULL, + "model_alias" text, + "trace_uploaded" boolean DEFAULT false NOT NULL, + "started_at" timestamp with time zone, + "finished_at" timestamp with time zone, + "created_at" timestamp with time zone DEFAULT now() NOT NULL +); +--> statement-breakpoint +CREATE TABLE "workspace_sessions" ( + "id" text PRIMARY KEY NOT NULL, + "user_id" text NOT NULL, + "name" text NOT NULL, + "hash12" text NOT NULL, + "efs_access_point_id" text NOT NULL, + "task_arn" text, + "task_ip" text, + "run_nonce" text, + "image_ref" text NOT NULL, + "state" text NOT NULL, + "last_seen_at" timestamp with time zone, + "created_at" timestamp with time zone DEFAULT now() NOT NULL, + "updated_at" timestamp with time zone DEFAULT now() NOT NULL, + CONSTRAINT "workspace_sessions_hash12_unique" UNIQUE("hash12") +); +--> statement-breakpoint +CREATE TABLE "workspaces" ( + "id" text PRIMARY KEY NOT NULL, + "org_id" text NOT NULL, + "name" text NOT NULL, + "keep_warm" boolean DEFAULT false NOT NULL, + "budget_usd_month" integer, + "created_at" timestamp with time zone DEFAULT now() NOT NULL +); +--> statement-breakpoint +ALTER TABLE "cli_sessions" ADD COLUMN "project_id" text;--> statement-breakpoint +ALTER TABLE "usage_events" ADD COLUMN "project_id" text;--> statement-breakpoint +ALTER TABLE "agent_versions" ADD CONSTRAINT "agent_versions_agent_id_agents_id_fk" FOREIGN KEY ("agent_id") REFERENCES "public"."agents"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "agents" ADD CONSTRAINT "agents_workspace_id_workspaces_id_fk" FOREIGN KEY ("workspace_id") REFERENCES "public"."workspaces"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "agents" ADD CONSTRAINT "agents_owner_user_id_users_id_fk" FOREIGN KEY ("owner_user_id") REFERENCES "public"."users"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "deployments" ADD CONSTRAINT "deployments_agent_id_agents_id_fk" FOREIGN KEY ("agent_id") REFERENCES "public"."agents"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "deployments" ADD CONSTRAINT "deployments_agent_version_id_agent_versions_id_fk" FOREIGN KEY ("agent_version_id") REFERENCES "public"."agent_versions"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "invoke_tokens" ADD CONSTRAINT "invoke_tokens_agent_id_agents_id_fk" FOREIGN KEY ("agent_id") REFERENCES "public"."agents"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "project_sessions" ADD CONSTRAINT "project_sessions_project_id_workspace_sessions_id_fk" FOREIGN KEY ("project_id") REFERENCES "public"."workspace_sessions"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "project_sessions" ADD CONSTRAINT "project_sessions_user_id_users_id_fk" FOREIGN KEY ("user_id") REFERENCES "public"."users"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "runs" ADD CONSTRAINT "runs_deployment_id_deployments_id_fk" FOREIGN KEY ("deployment_id") REFERENCES "public"."deployments"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "workspace_sessions" ADD CONSTRAINT "workspace_sessions_user_id_users_id_fk" FOREIGN KEY ("user_id") REFERENCES "public"."users"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "workspaces" ADD CONSTRAINT "workspaces_org_id_organizations_id_fk" FOREIGN KEY ("org_id") REFERENCES "public"."organizations"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint +CREATE UNIQUE INDEX "runs_deployment_idem_uq" ON "runs" USING btree ("deployment_id","idempotency_key"); \ No newline at end of file diff --git a/control-plane/artifacts/sanad-web/drizzle/meta/0006_snapshot.json b/control-plane/artifacts/sanad-web/drizzle/meta/0006_snapshot.json new file mode 100644 index 000000000..5b4d0b75b --- /dev/null +++ b/control-plane/artifacts/sanad-web/drizzle/meta/0006_snapshot.json @@ -0,0 +1,1777 @@ +{ + "id": "98bc119b-39cb-4f3b-9a63-3214766e1882", + "prevId": "1f7494d9-f8b7-405d-9451-61d9d12ffa15", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.agent_versions": { + "name": "agent_versions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "content_hash": { + "name": "content_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "bundle": { + "name": "bundle", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "agent_versions_agent_id_agents_id_fk": { + "name": "agent_versions_agent_id_agents_id_fk", + "tableFrom": "agent_versions", + "tableTo": "agents", + "columnsFrom": [ + "agent_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.agents": { + "name": "agents", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "owner_user_id": { + "name": "owner_user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'active'" + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "agents_workspace_id_workspaces_id_fk": { + "name": "agents_workspace_id_workspaces_id_fk", + "tableFrom": "agents", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "agents_owner_user_id_users_id_fk": { + "name": "agents_owner_user_id_users_id_fk", + "tableFrom": "agents", + "tableTo": "users", + "columnsFrom": [ + "owner_user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.cli_sessions": { + "name": "cli_sessions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "token_hash": { + "name": "token_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "device_request_id": { + "name": "device_request_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "device_label": { + "name": "device_label", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "last_used_at": { + "name": "last_used_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "cli_sessions_user_id_users_id_fk": { + "name": "cli_sessions_user_id_users_id_fk", + "tableFrom": "cli_sessions", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "cli_sessions_org_id_organizations_id_fk": { + "name": "cli_sessions_org_id_organizations_id_fk", + "tableFrom": "cli_sessions", + "tableTo": "organizations", + "columnsFrom": [ + "org_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "cli_sessions_device_request_id_device_auth_requests_id_fk": { + "name": "cli_sessions_device_request_id_device_auth_requests_id_fk", + "tableFrom": "cli_sessions", + "tableTo": "device_auth_requests", + "columnsFrom": [ + "device_request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "cli_sessions_token_hash_unique": { + "name": "cli_sessions_token_hash_unique", + "nullsNotDistinct": false, + "columns": [ + "token_hash" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.deployments": { + "name": "deployments", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent_version_id": { + "name": "agent_version_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "env": { + "name": "env", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'active'" + }, + "max_turn_seconds": { + "name": "max_turn_seconds", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 900 + }, + "max_steps_per_turn": { + "name": "max_steps_per_turn", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 100 + }, + "max_tokens_per_run": { + "name": "max_tokens_per_run", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 2000000 + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "deployments_agent_id_agents_id_fk": { + "name": "deployments_agent_id_agents_id_fk", + "tableFrom": "deployments", + "tableTo": "agents", + "columnsFrom": [ + "agent_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "deployments_agent_version_id_agent_versions_id_fk": { + "name": "deployments_agent_version_id_agent_versions_id_fk", + "tableFrom": "deployments", + "tableTo": "agent_versions", + "columnsFrom": [ + "agent_version_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.device_auth_requests": { + "name": "device_auth_requests", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "device_auth_id_hash": { + "name": "device_auth_id_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_code": { + "name": "user_code", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "device_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "approved_user_id": { + "name": "approved_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "approved_org_id": { + "name": "approved_org_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "pending_session_token": { + "name": "pending_session_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "poll_interval_seconds": { + "name": "poll_interval_seconds", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 2 + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "device_auth_requests_device_auth_id_hash_unique": { + "name": "device_auth_requests_device_auth_id_hash_unique", + "nullsNotDistinct": false, + "columns": [ + "device_auth_id_hash" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.invoke_tokens": { + "name": "invoke_tokens", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "token_hash": { + "name": "token_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "family_id": { + "name": "family_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "env": { + "name": "env", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "invoke_tokens_agent_id_agents_id_fk": { + "name": "invoke_tokens_agent_id_agents_id_fk", + "tableFrom": "invoke_tokens", + "tableTo": "agents", + "columnsFrom": [ + "agent_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "invoke_tokens_token_hash_unique": { + "name": "invoke_tokens_token_hash_unique", + "nullsNotDistinct": false, + "columns": [ + "token_hash" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.memberships": { + "name": "memberships", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "seat_assigned": { + "name": "seat_assigned", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "memberships_org_id_organizations_id_fk": { + "name": "memberships_org_id_organizations_id_fk", + "tableFrom": "memberships", + "tableTo": "organizations", + "columnsFrom": [ + "org_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "memberships_user_id_users_id_fk": { + "name": "memberships_user_id_users_id_fk", + "tableFrom": "memberships", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.organizations": { + "name": "organizations", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.project_sessions": { + "name": "project_sessions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ui_state": { + "name": "ui_state", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "archived_at": { + "name": "archived_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "last_active_at": { + "name": "last_active_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "project_sessions_project_id_workspace_sessions_id_fk": { + "name": "project_sessions_project_id_workspace_sessions_id_fk", + "tableFrom": "project_sessions", + "tableTo": "workspace_sessions", + "columnsFrom": [ + "project_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "project_sessions_user_id_users_id_fk": { + "name": "project_sessions_user_id_users_id_fk", + "tableFrom": "project_sessions", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.runs": { + "name": "runs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "deployment_id": { + "name": "deployment_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent_version_id": { + "name": "agent_version_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'queued'" + }, + "error_code": { + "name": "error_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "trigger_principal": { + "name": "trigger_principal", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "idempotency_key": { + "name": "idempotency_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "output": { + "name": "output", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "tokens_in": { + "name": "tokens_in", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "tokens_out": { + "name": "tokens_out", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "cost_usd_micros": { + "name": "cost_usd_micros", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "model_alias": { + "name": "model_alias", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "trace_uploaded": { + "name": "trace_uploaded", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "started_at": { + "name": "started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "runs_deployment_idem_uq": { + "name": "runs_deployment_idem_uq", + "columns": [ + { + "expression": "deployment_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "idempotency_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "runs_deployment_id_deployments_id_fk": { + "name": "runs_deployment_id_deployments_id_fk", + "tableFrom": "runs", + "tableTo": "deployments", + "columnsFrom": [ + "deployment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.runtime_tokens": { + "name": "runtime_tokens", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "token_hash": { + "name": "token_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "family_id": { + "name": "family_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "cli_session_id": { + "name": "cli_session_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "absolute_expires_at": { + "name": "absolute_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "runtime_tokens_cli_session_id_cli_sessions_id_fk": { + "name": "runtime_tokens_cli_session_id_cli_sessions_id_fk", + "tableFrom": "runtime_tokens", + "tableTo": "cli_sessions", + "columnsFrom": [ + "cli_session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "runtime_tokens_token_hash_unique": { + "name": "runtime_tokens_token_hash_unique", + "nullsNotDistinct": false, + "columns": [ + "token_hash" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.ships": { + "name": "ships", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "app_slug": { + "name": "app_slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ecr_image": { + "name": "ecr_image", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "commit_sha": { + "name": "commit_sha", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "codebuild_id": { + "name": "codebuild_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "ships_user_id_users_id_fk": { + "name": "ships_user_id_users_id_fk", + "tableFrom": "ships", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "ships_org_id_organizations_id_fk": { + "name": "ships_org_id_organizations_id_fk", + "tableFrom": "ships", + "tableTo": "organizations", + "columnsFrom": [ + "org_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "ships_app_slug_unique": { + "name": "ships_app_slug_unique", + "nullsNotDistinct": false, + "columns": [ + "app_slug" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.subscriptions": { + "name": "subscriptions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "plan": { + "name": "plan", + "type": "plan", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'free'" + }, + "status": { + "name": "status", + "type": "sub_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'active'" + }, + "stripe_customer_id": { + "name": "stripe_customer_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "stripe_subscription_id": { + "name": "stripe_subscription_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "seats": { + "name": "seats", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "quota": { + "name": "quota", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "current_period_end": { + "name": "current_period_end", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "subscriptions_org_id_organizations_id_fk": { + "name": "subscriptions_org_id_organizations_id_fk", + "tableFrom": "subscriptions", + "tableTo": "organizations", + "columnsFrom": [ + "org_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.terminal_tickets": { + "name": "terminal_tickets", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "ticket_hash": { + "name": "ticket_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "session_token": { + "name": "session_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "redeemed_at": { + "name": "redeemed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "terminal_tickets_user_id_users_id_fk": { + "name": "terminal_tickets_user_id_users_id_fk", + "tableFrom": "terminal_tickets", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "terminal_tickets_org_id_organizations_id_fk": { + "name": "terminal_tickets_org_id_organizations_id_fk", + "tableFrom": "terminal_tickets", + "tableTo": "organizations", + "columnsFrom": [ + "org_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "terminal_tickets_ticket_hash_unique": { + "name": "terminal_tickets_ticket_hash_unique", + "nullsNotDistinct": false, + "columns": [ + "ticket_hash" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.usage_events": { + "name": "usage_events", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "cli_session_id": { + "name": "cli_session_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "model_alias": { + "name": "model_alias", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tokens_in": { + "name": "tokens_in", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "tokens_out": { + "name": "tokens_out", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "cost": { + "name": "cost", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.users": { + "name": "users", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspace_sessions": { + "name": "workspace_sessions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "hash12": { + "name": "hash12", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "efs_access_point_id": { + "name": "efs_access_point_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "task_arn": { + "name": "task_arn", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "task_ip": { + "name": "task_ip", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "run_nonce": { + "name": "run_nonce", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "image_ref": { + "name": "image_ref", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "last_seen_at": { + "name": "last_seen_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "workspace_sessions_user_id_users_id_fk": { + "name": "workspace_sessions_user_id_users_id_fk", + "tableFrom": "workspace_sessions", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "workspace_sessions_hash12_unique": { + "name": "workspace_sessions_hash12_unique", + "nullsNotDistinct": false, + "columns": [ + "hash12" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspace_tasks": { + "name": "workspace_tasks", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "hash12": { + "name": "hash12", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "efs_access_point_id": { + "name": "efs_access_point_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "task_arn": { + "name": "task_arn", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "task_ip": { + "name": "task_ip", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "run_nonce": { + "name": "run_nonce", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "image_ref": { + "name": "image_ref", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "last_seen_at": { + "name": "last_seen_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "workspace_tasks_user_id_users_id_fk": { + "name": "workspace_tasks_user_id_users_id_fk", + "tableFrom": "workspace_tasks", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "workspace_tasks_user_id_unique": { + "name": "workspace_tasks_user_id_unique", + "nullsNotDistinct": false, + "columns": [ + "user_id" + ] + }, + "workspace_tasks_hash12_unique": { + "name": "workspace_tasks_hash12_unique", + "nullsNotDistinct": false, + "columns": [ + "hash12" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspaces": { + "name": "workspaces", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "keep_warm": { + "name": "keep_warm", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "budget_usd_month": { + "name": "budget_usd_month", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "workspaces_org_id_organizations_id_fk": { + "name": "workspaces_org_id_organizations_id_fk", + "tableFrom": "workspaces", + "tableTo": "organizations", + "columnsFrom": [ + "org_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": { + "public.device_status": { + "name": "device_status", + "schema": "public", + "values": [ + "pending", + "complete", + "denied", + "expired" + ] + }, + "public.plan": { + "name": "plan", + "schema": "public", + "values": [ + "free", + "pro", + "team", + "enterprise" + ] + }, + "public.sub_status": { + "name": "sub_status", + "schema": "public", + "values": [ + "active", + "past_due", + "canceled" + ] + } + }, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/control-plane/artifacts/sanad-web/drizzle/meta/_journal.json b/control-plane/artifacts/sanad-web/drizzle/meta/_journal.json index 1effd0beb..0ee86300e 100644 --- a/control-plane/artifacts/sanad-web/drizzle/meta/_journal.json +++ b/control-plane/artifacts/sanad-web/drizzle/meta/_journal.json @@ -43,6 +43,13 @@ "when": 1786620000000, "tag": "0005_usage_attribution", "breakpoints": true + }, + { + "idx": 6, + "version": "7", + "when": 1786627610138, + "tag": "0006_foamy_venus", + "breakpoints": true } ] } \ No newline at end of file diff --git a/control-plane/artifacts/sanad-web/lib/db/schema.ts b/control-plane/artifacts/sanad-web/lib/db/schema.ts index d88c90fd1..a41f01d26 100644 --- a/control-plane/artifacts/sanad-web/lib/db/schema.ts +++ b/control-plane/artifacts/sanad-web/lib/db/schema.ts @@ -6,6 +6,7 @@ import { boolean, jsonb, pgEnum, + uniqueIndex, } from "drizzle-orm/pg-core"; export const planEnum = pgEnum("plan", ["free", "pro", "team", "enterprise"]); @@ -196,6 +197,88 @@ export const workspaceSessions = pgTable("workspace_sessions", { updatedAt: timestamp("updated_at", { withTimezone: true }).defaultNow().notNull(), }); +// -- worker runtime (P0) ------------------------------------------------------ + +export const workspaces = pgTable("workspaces", { + id: text("id").primaryKey(), // ws_ + orgId: text("org_id").notNull().references(() => organizations.id), + name: text("name").notNull(), + keepWarm: boolean("keep_warm").default(false).notNull(), + budgetUsdMonth: integer("budget_usd_month"), // null = uncapped in P0 + createdAt: timestamp("created_at", { withTimezone: true }).defaultNow().notNull(), +}); + +export const agents = pgTable("agents", { + id: text("id").primaryKey(), // ag_ + workspaceId: text("workspace_id").notNull().references(() => workspaces.id), + name: text("name").notNull(), // unique per workspace, enforced in registry.ts + ownerUserId: text("owner_user_id").notNull().references(() => users.id), + status: text("status").default("active").notNull(), // "active" | "orphaned" + description: text("description"), + createdAt: timestamp("created_at", { withTimezone: true }).defaultNow().notNull(), +}); + +export const agentVersions = pgTable("agent_versions", { + id: text("id").primaryKey(), // av_ + agentId: text("agent_id").notNull().references(() => agents.id), + contentHash: text("content_hash").notNull(), // sha256 of canonical bundle JSON + bundle: jsonb("bundle").notNull(), // { files: { "agent.yaml": "...", "worker.yaml": "...", ... } } + createdBy: text("created_by").notNull(), // soft user id + createdAt: timestamp("created_at", { withTimezone: true }).defaultNow().notNull(), +}); + +export const deployments = pgTable("deployments", { + id: text("id").primaryKey(), // dp_ + agentId: text("agent_id").notNull().references(() => agents.id), + agentVersionId: text("agent_version_id").notNull().references(() => agentVersions.id), + env: text("env").notNull(), // "dev" | "prod" + status: text("status").default("active").notNull(), // "active" | "paused" + maxTurnSeconds: integer("max_turn_seconds").default(900).notNull(), + maxStepsPerTurn: integer("max_steps_per_turn").default(100).notNull(), + maxTokensPerRun: integer("max_tokens_per_run").default(2000000).notNull(), + createdAt: timestamp("created_at", { withTimezone: true }).defaultNow().notNull(), + updatedAt: timestamp("updated_at", { withTimezone: true }).defaultNow().notNull(), +}); + +export const runs = pgTable( + "runs", + { + id: text("id").primaryKey(), // r_<12 hex> — also the kimi session id on the machine + deploymentId: text("deployment_id").notNull().references(() => deployments.id), + agentVersionId: text("agent_version_id").notNull(), + status: text("status").default("queued").notNull(), + // "queued" | "running" | "succeeded" | "failed" | "cancelled" | "lost" + errorCode: text("error_code"), // e.g. "no_output" | "turn_budget_exceeded" + triggerPrincipal: text("trigger_principal").notNull(), // "itok:" | "user:" + idempotencyKey: text("idempotency_key"), + output: jsonb("output"), // the ReturnOutput document (or {"text": ...}) + tokensIn: integer("tokens_in").default(0).notNull(), + tokensOut: integer("tokens_out").default(0).notNull(), + costUsdMicros: integer("cost_usd_micros").default(0).notNull(), + modelAlias: text("model_alias"), + traceUploaded: boolean("trace_uploaded").default(false).notNull(), + startedAt: timestamp("started_at", { withTimezone: true }), + finishedAt: timestamp("finished_at", { withTimezone: true }), + createdAt: timestamp("created_at", { withTimezone: true }).defaultNow().notNull(), + }, + (table) => ({ + idempotencyIndex: uniqueIndex("runs_deployment_idem_uq").on(table.deploymentId, table.idempotencyKey), + }) +); + +export const invokeTokens = pgTable("invoke_tokens", { + id: text("id").primaryKey(), // tokenId — a UUID + tokenHash: text("token_hash").notNull().unique(), + familyId: text("family_id").notNull(), + agentId: text("agent_id").notNull().references(() => agents.id), + env: text("env").notNull(), + orgId: text("org_id").notNull(), + createdBy: text("created_by").notNull(), + expiresAt: timestamp("expires_at", { withTimezone: true }).notNull(), + revokedAt: timestamp("revoked_at", { withTimezone: true }), + createdAt: timestamp("created_at", { withTimezone: true }).defaultNow().notNull(), +}); + /** * A PRD Session (§7.8): a restorable unit of work INSIDE a project — its * user-facing name plus the durable UI state needed to resume (open tabs, tab diff --git a/control-plane/artifacts/sanad-web/tests/unit/worker-schema.test.ts b/control-plane/artifacts/sanad-web/tests/unit/worker-schema.test.ts new file mode 100644 index 000000000..337312fe3 --- /dev/null +++ b/control-plane/artifacts/sanad-web/tests/unit/worker-schema.test.ts @@ -0,0 +1,25 @@ +import { describe, it, expect } from "vitest"; +import { getTableColumns } from "drizzle-orm"; +import { + workspaces, agents, agentVersions, deployments, runs, invokeTokens, +} from "@/lib/db/schema"; + +describe("worker runtime schema", () => { + it("agents require an owner and a workspace", () => { + const cols = getTableColumns(agents); + expect(cols.ownerUserId.notNull).toBe(true); + expect(cols.workspaceId.notNull).toBe(true); + }); + it("runs carry attribution and idempotency", () => { + const cols = getTableColumns(runs); + for (const k of ["deploymentId", "agentVersionId", "status", "triggerPrincipal"] as const) + expect(cols[k].notNull, k).toBe(true); + expect(cols.idempotencyKey.notNull).toBe(false); + }); + it("invoke tokens are scoped to agent+env", () => { + const cols = getTableColumns(invokeTokens); + expect(cols.agentId.notNull).toBe(true); + expect(cols.env.notNull).toBe(true); + expect(cols.tokenHash.isUnique).toBe(true); + }); +}); From 3c58de08b18374eee69b0349c1ca28078c44d7cb Mon Sep 17 00:00:00 2001 From: Omar Alsoulah Date: Thu, 13 Aug 2026 16:40:39 +0300 Subject: [PATCH 02/28] =?UTF-8?q?sanad:=20worker=20runtime=20schema=20fixe?= =?UTF-8?q?s=20=E2=80=94=20trim=20migration,=20use=20non-deprecated=20inde?= =?UTF-8?q?x=20syntax?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sanad-web/drizzle/0006_foamy_venus.sql | 36 +------------------ .../artifacts/sanad-web/lib/db/schema.ts | 6 ++-- 2 files changed, 4 insertions(+), 38 deletions(-) diff --git a/control-plane/artifacts/sanad-web/drizzle/0006_foamy_venus.sql b/control-plane/artifacts/sanad-web/drizzle/0006_foamy_venus.sql index 9752c65e3..e5b2cdbc3 100644 --- a/control-plane/artifacts/sanad-web/drizzle/0006_foamy_venus.sql +++ b/control-plane/artifacts/sanad-web/drizzle/0006_foamy_venus.sql @@ -44,18 +44,6 @@ CREATE TABLE "invoke_tokens" ( CONSTRAINT "invoke_tokens_token_hash_unique" UNIQUE("token_hash") ); --> statement-breakpoint -CREATE TABLE "project_sessions" ( - "id" text PRIMARY KEY NOT NULL, - "project_id" text NOT NULL, - "user_id" text NOT NULL, - "name" text NOT NULL, - "ui_state" jsonb DEFAULT '{}'::jsonb NOT NULL, - "archived_at" timestamp with time zone, - "created_at" timestamp with time zone DEFAULT now() NOT NULL, - "updated_at" timestamp with time zone DEFAULT now() NOT NULL, - "last_active_at" timestamp with time zone DEFAULT now() NOT NULL -); ---> statement-breakpoint CREATE TABLE "runs" ( "id" text PRIMARY KEY NOT NULL, "deployment_id" text NOT NULL, @@ -75,23 +63,6 @@ CREATE TABLE "runs" ( "created_at" timestamp with time zone DEFAULT now() NOT NULL ); --> statement-breakpoint -CREATE TABLE "workspace_sessions" ( - "id" text PRIMARY KEY NOT NULL, - "user_id" text NOT NULL, - "name" text NOT NULL, - "hash12" text NOT NULL, - "efs_access_point_id" text NOT NULL, - "task_arn" text, - "task_ip" text, - "run_nonce" text, - "image_ref" text NOT NULL, - "state" text NOT NULL, - "last_seen_at" timestamp with time zone, - "created_at" timestamp with time zone DEFAULT now() NOT NULL, - "updated_at" timestamp with time zone DEFAULT now() NOT NULL, - CONSTRAINT "workspace_sessions_hash12_unique" UNIQUE("hash12") -); ---> statement-breakpoint CREATE TABLE "workspaces" ( "id" text PRIMARY KEY NOT NULL, "org_id" text NOT NULL, @@ -101,17 +72,12 @@ CREATE TABLE "workspaces" ( "created_at" timestamp with time zone DEFAULT now() NOT NULL ); --> statement-breakpoint -ALTER TABLE "cli_sessions" ADD COLUMN "project_id" text;--> statement-breakpoint -ALTER TABLE "usage_events" ADD COLUMN "project_id" text;--> statement-breakpoint ALTER TABLE "agent_versions" ADD CONSTRAINT "agent_versions_agent_id_agents_id_fk" FOREIGN KEY ("agent_id") REFERENCES "public"."agents"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint ALTER TABLE "agents" ADD CONSTRAINT "agents_workspace_id_workspaces_id_fk" FOREIGN KEY ("workspace_id") REFERENCES "public"."workspaces"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint ALTER TABLE "agents" ADD CONSTRAINT "agents_owner_user_id_users_id_fk" FOREIGN KEY ("owner_user_id") REFERENCES "public"."users"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint ALTER TABLE "deployments" ADD CONSTRAINT "deployments_agent_id_agents_id_fk" FOREIGN KEY ("agent_id") REFERENCES "public"."agents"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint ALTER TABLE "deployments" ADD CONSTRAINT "deployments_agent_version_id_agent_versions_id_fk" FOREIGN KEY ("agent_version_id") REFERENCES "public"."agent_versions"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint ALTER TABLE "invoke_tokens" ADD CONSTRAINT "invoke_tokens_agent_id_agents_id_fk" FOREIGN KEY ("agent_id") REFERENCES "public"."agents"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint -ALTER TABLE "project_sessions" ADD CONSTRAINT "project_sessions_project_id_workspace_sessions_id_fk" FOREIGN KEY ("project_id") REFERENCES "public"."workspace_sessions"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint -ALTER TABLE "project_sessions" ADD CONSTRAINT "project_sessions_user_id_users_id_fk" FOREIGN KEY ("user_id") REFERENCES "public"."users"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint ALTER TABLE "runs" ADD CONSTRAINT "runs_deployment_id_deployments_id_fk" FOREIGN KEY ("deployment_id") REFERENCES "public"."deployments"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint -ALTER TABLE "workspace_sessions" ADD CONSTRAINT "workspace_sessions_user_id_users_id_fk" FOREIGN KEY ("user_id") REFERENCES "public"."users"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint ALTER TABLE "workspaces" ADD CONSTRAINT "workspaces_org_id_organizations_id_fk" FOREIGN KEY ("org_id") REFERENCES "public"."organizations"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint -CREATE UNIQUE INDEX "runs_deployment_idem_uq" ON "runs" USING btree ("deployment_id","idempotency_key"); \ No newline at end of file +CREATE UNIQUE INDEX "runs_deployment_idem_uq" ON "runs" USING btree ("deployment_id","idempotency_key"); diff --git a/control-plane/artifacts/sanad-web/lib/db/schema.ts b/control-plane/artifacts/sanad-web/lib/db/schema.ts index a41f01d26..07184ee97 100644 --- a/control-plane/artifacts/sanad-web/lib/db/schema.ts +++ b/control-plane/artifacts/sanad-web/lib/db/schema.ts @@ -261,9 +261,9 @@ export const runs = pgTable( finishedAt: timestamp("finished_at", { withTimezone: true }), createdAt: timestamp("created_at", { withTimezone: true }).defaultNow().notNull(), }, - (table) => ({ - idempotencyIndex: uniqueIndex("runs_deployment_idem_uq").on(table.deploymentId, table.idempotencyKey), - }) + (table) => [ + uniqueIndex("runs_deployment_idem_uq").on(table.deploymentId, table.idempotencyKey), + ] ); export const invokeTokens = pgTable("invoke_tokens", { From 1d613d682d479657fdec27762dd9383c8d453b24 Mon Sep 17 00:00:00 2001 From: Omar Alsoulah Date: Thu, 13 Aug 2026 16:46:27 +0300 Subject: [PATCH 03/28] =?UTF-8?q?sanad:=20invoke=20tokens=20=E2=80=94=20it?= =?UTF-8?q?ok=20mint/verify=20scoped=20to=20agent+env,=20quota=20at=20mint?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../artifacts/sanad-web/lib/tokens/invoke.ts | 68 +++++++++++++++++++ .../tests/unit/invoke-tokens.test.ts | 44 ++++++++++++ 2 files changed, 112 insertions(+) create mode 100644 control-plane/artifacts/sanad-web/lib/tokens/invoke.ts create mode 100644 control-plane/artifacts/sanad-web/tests/unit/invoke-tokens.test.ts diff --git a/control-plane/artifacts/sanad-web/lib/tokens/invoke.ts b/control-plane/artifacts/sanad-web/lib/tokens/invoke.ts new file mode 100644 index 000000000..cb31dc6c0 --- /dev/null +++ b/control-plane/artifacts/sanad-web/lib/tokens/invoke.ts @@ -0,0 +1,68 @@ +import { and, eq, gt, isNull } from "drizzle-orm"; +import { db } from "../db"; +import { invokeTokens } from "../db/schema"; +import { newToken, hashToken } from "../auth/tokens"; +import { requireEntitled } from "../auth/entitlement"; +import { assertWithinQuota } from "../billing/quota"; +import { EntitlementError } from "./runtime"; + +const INVOKE_TTL_MS = 90 * 24 * 3600 * 1000; + +export interface InvokeTokenInfo { + tokenId: string; + agentId: string; + env: string; + orgId: string; +} + +export async function mintInvoke( + session: { userId: string; orgId: string }, + agentId: string, + env: "dev" | "prod" +): Promise<{ token: string; tokenId: string; expiresAt: Date }> { + const ent = await requireEntitled(session.orgId, session.userId); + if (!ent.ok) throw new EntitlementError(ent.reason); + await assertWithinQuota(session.orgId); + + const token = newToken("itok"); + const tokenId = crypto.randomUUID(); + const expiresAt = new Date(Date.now() + INVOKE_TTL_MS); + await db.insert(invokeTokens).values({ + id: tokenId, + tokenHash: hashToken(token), + familyId: newToken("ifam"), + agentId, + env, + orgId: session.orgId, + createdBy: session.userId, + expiresAt, + }); + return { token, tokenId, expiresAt }; +} + +export async function verifyInvokeBearer(request: Request): Promise { + const auth = request.headers.get("authorization") ?? ""; + const m = auth.match(/^Bearer (itok_[A-Za-z0-9_-]+)$/); + if (!m) return null; + const rows = await db + .select() + .from(invokeTokens) + .where( + and( + eq(invokeTokens.tokenHash, hashToken(m[1])), + isNull(invokeTokens.revokedAt), + gt(invokeTokens.expiresAt, new Date()) + ) + ) + .limit(1); + const row = rows[0]; + if (!row) return null; + return { tokenId: row.id, agentId: row.agentId, env: row.env, orgId: row.orgId }; +} + +export async function revokeInvokeFamily(familyId: string): Promise { + await db + .update(invokeTokens) + .set({ revokedAt: new Date() }) + .where(and(eq(invokeTokens.familyId, familyId), isNull(invokeTokens.revokedAt))); +} diff --git a/control-plane/artifacts/sanad-web/tests/unit/invoke-tokens.test.ts b/control-plane/artifacts/sanad-web/tests/unit/invoke-tokens.test.ts new file mode 100644 index 000000000..93cad6f31 --- /dev/null +++ b/control-plane/artifacts/sanad-web/tests/unit/invoke-tokens.test.ts @@ -0,0 +1,44 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; + +const inserted: any[] = []; +const selectResult: { rows: any[] } = { rows: [] }; +vi.mock("@/lib/db", () => ({ + db: { + insert: vi.fn(() => ({ values: vi.fn(async (v: any) => { inserted.push(v); }) })), + select: vi.fn(() => ({ + from: vi.fn(() => ({ where: vi.fn(() => ({ limit: vi.fn(async () => selectResult.rows) })) })), + })), + update: vi.fn(() => ({ set: vi.fn(() => ({ where: vi.fn(async () => {}) })) })), + }, +})); +vi.mock("@/lib/auth/entitlement", () => ({ requireEntitled: vi.fn(async () => ({ ok: true })) })); +vi.mock("@/lib/billing/quota", () => ({ assertWithinQuota: vi.fn(async () => {}) })); + +import { mintInvoke, verifyInvokeBearer } from "@/lib/tokens/invoke"; +import { hashToken } from "@/lib/auth/tokens"; + +beforeEach(() => { inserted.length = 0; selectResult.rows = []; }); + +describe("invoke tokens", () => { + it("mints an itok_ token hashed at rest, scoped to agent+env", async () => { + const out = await mintInvoke({ userId: "u1", orgId: "o1" }, "ag_1", "prod"); + expect(out.token.startsWith("itok_")).toBe(true); + expect(inserted[0].tokenHash).toBe(hashToken(out.token)); + expect(inserted[0].agentId).toBe("ag_1"); + expect(inserted[0].env).toBe("prod"); + }); + it("verify returns null without a bearer", async () => { + const req = new Request("https://x.test/", { headers: {} }); + expect(await verifyInvokeBearer(req)).toBeNull(); + }); + it("verify resolves a live token row", async () => { + selectResult.rows = [{ + id: "tid", agentId: "ag_1", env: "prod", orgId: "o1", + expiresAt: new Date(Date.now() + 60_000), revokedAt: null, + }]; + const req = new Request("https://x.test/", { headers: { authorization: "Bearer itok_abc" } }); + expect(await verifyInvokeBearer(req)).toEqual({ + tokenId: "tid", agentId: "ag_1", env: "prod", orgId: "o1", + }); + }); +}); From 5bc528d4df31892085a5c101014bc58b0ff9ce85 Mon Sep 17 00:00:00 2001 From: Omar Alsoulah Date: Thu, 13 Aug 2026 16:58:35 +0300 Subject: [PATCH 04/28] =?UTF-8?q?sanad:=20agent=20registry=20=E2=80=94=20u?= =?UTF-8?q?psert/versions/deployments=20with=20owner-required=20gate,=20it?= =?UTF-8?q?ok=20route?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../api/v1/agents/[name]/deployments/route.ts | 90 +++++++++ .../app/api/v1/agents/[name]/tokens/route.ts | 61 ++++++ .../api/v1/agents/[name]/versions/route.ts | 42 ++++ .../sanad-web/app/api/v1/agents/route.ts | 56 ++++++ .../sanad-web/lib/agents/registry.ts | 188 ++++++++++++++++++ .../tests/unit/agent-registry.test.ts | 26 +++ 6 files changed, 463 insertions(+) create mode 100644 control-plane/artifacts/sanad-web/app/api/v1/agents/[name]/deployments/route.ts create mode 100644 control-plane/artifacts/sanad-web/app/api/v1/agents/[name]/tokens/route.ts create mode 100644 control-plane/artifacts/sanad-web/app/api/v1/agents/[name]/versions/route.ts create mode 100644 control-plane/artifacts/sanad-web/app/api/v1/agents/route.ts create mode 100644 control-plane/artifacts/sanad-web/lib/agents/registry.ts create mode 100644 control-plane/artifacts/sanad-web/tests/unit/agent-registry.test.ts diff --git a/control-plane/artifacts/sanad-web/app/api/v1/agents/[name]/deployments/route.ts b/control-plane/artifacts/sanad-web/app/api/v1/agents/[name]/deployments/route.ts new file mode 100644 index 000000000..e76ad0030 --- /dev/null +++ b/control-plane/artifacts/sanad-web/app/api/v1/agents/[name]/deployments/route.ts @@ -0,0 +1,90 @@ +import { NextRequest } from "next/server"; +import { ok, err } from "@/lib/http/envelope"; +import { verifyBearer } from "@/lib/auth/session"; +import { + createDeployment, + getAgentByName, + OwnerRequiredError, + setDeploymentStatus, +} from "@/lib/agents/registry"; + +function isEnv(v: unknown): v is "dev" | "prod" { + return v === "dev" || v === "prod"; +} + +export async function POST( + req: NextRequest, + { params }: { params: Promise<{ name: string }> } +) { + const session = await verifyBearer(req); + if (!session) { + return err(401, "unauthorized", "Invalid or revoked session token"); + } + + const { name } = await params; + const agent = await getAgentByName(session.orgId, name); + if (!agent) { + return err(404, "not_found", "No such agent"); + } + + const body = (await req.json().catch(() => null)) as + | { versionId?: string; env?: string } + | null; + if (!body?.versionId) { + return err(400, "invalid_request", "A deployment needs a versionId"); + } + // mintInvoke has no runtime guard on env, so this route validates it before + // ever reaching a registry function that expects the "dev" | "prod" union. + if (!isEnv(body.env)) { + return err(400, "bad_env", 'env must be "dev" or "prod"'); + } + + try { + const { id } = await createDeployment({ + agentId: agent.id, + versionId: body.versionId, + env: body.env, + }); + return ok({ deploymentId: id }); + } catch (e) { + if (e instanceof OwnerRequiredError) { + return err(409, "owner_required", e.message); + } + console.error("deployment create failed", e); + return err(500, "internal_error", "Failed to create deployment", true); + } +} + +export async function PATCH( + req: NextRequest, + { params }: { params: Promise<{ name: string }> } +) { + const session = await verifyBearer(req); + if (!session) { + return err(401, "unauthorized", "Invalid or revoked session token"); + } + + const { name } = await params; + const agent = await getAgentByName(session.orgId, name); + if (!agent) { + return err(404, "not_found", "No such agent"); + } + + const body = (await req.json().catch(() => null)) as + | { env?: string; status?: string } + | null; + if (!isEnv(body?.env)) { + return err(400, "bad_env", 'env must be "dev" or "prod"'); + } + if (body?.status !== "active" && body?.status !== "paused") { + return err(400, "invalid_request", 'status must be "active" or "paused"'); + } + + try { + await setDeploymentStatus(agent.id, body.env, body.status); + return ok({ agentId: agent.id, env: body.env, status: body.status }); + } catch (e) { + console.error("deployment status update failed", e); + return err(500, "internal_error", "Failed to update deployment", true); + } +} diff --git a/control-plane/artifacts/sanad-web/app/api/v1/agents/[name]/tokens/route.ts b/control-plane/artifacts/sanad-web/app/api/v1/agents/[name]/tokens/route.ts new file mode 100644 index 000000000..5f844197a --- /dev/null +++ b/control-plane/artifacts/sanad-web/app/api/v1/agents/[name]/tokens/route.ts @@ -0,0 +1,61 @@ +import { NextRequest } from "next/server"; +import { ok, err } from "@/lib/http/envelope"; +import { verifyBearer } from "@/lib/auth/session"; +import { getAgentByName } from "@/lib/agents/registry"; +import { mintInvoke } from "@/lib/tokens/invoke"; +import { EntitlementError } from "@/lib/tokens/runtime"; +import { QuotaExceededError } from "@/lib/billing/quota"; + +export async function POST( + req: NextRequest, + { params }: { params: Promise<{ name: string }> } +) { + const session = await verifyBearer(req); + if (!session) { + return err(401, "unauthorized", "Invalid or revoked session token"); + } + + const { name } = await params; + const agent = await getAgentByName(session.orgId, name); + if (!agent) { + return err(404, "not_found", "No such agent"); + } + + const body = (await req.json().catch(() => null)) as { env?: string } | null; + // mintInvoke itself has no runtime guard on env — validate here before + // calling it, per the review finding carried over from Task 2. + if (body?.env !== "dev" && body?.env !== "prod") { + return err(400, "bad_env", 'env must be "dev" or "prod"'); + } + + try { + const result = await mintInvoke( + { userId: session.userId, orgId: session.orgId }, + agent.id, + body.env + ); + return ok({ + token: result.token, + tokenId: result.tokenId, + expiresAt: result.expiresAt.toISOString(), + }); + } catch (e) { + if (e instanceof EntitlementError) { + if (e.reason === "no_plan") { + return err(402, "subscription_required", "No active subscription — visit sanadcode.com/pricing to upgrade"); + } + if (e.reason === "no_seat") { + return err(403, "no_seat", "No seat assigned — ask your admin to assign you a seat"); + } + } + if (e instanceof QuotaExceededError) { + return err( + 402, + "quota_exceeded", + `Monthly ${e.dimension} allowance exhausted — upgrade at sanadcode.com/pricing or wait for the next billing period` + ); + } + console.error("agent invoke-token mint error", e); + return err(500, "internal_error", "Failed to mint invoke token", true); + } +} diff --git a/control-plane/artifacts/sanad-web/app/api/v1/agents/[name]/versions/route.ts b/control-plane/artifacts/sanad-web/app/api/v1/agents/[name]/versions/route.ts new file mode 100644 index 000000000..3beafcb2e --- /dev/null +++ b/control-plane/artifacts/sanad-web/app/api/v1/agents/[name]/versions/route.ts @@ -0,0 +1,42 @@ +import { NextRequest } from "next/server"; +import { ok, err } from "@/lib/http/envelope"; +import { verifyBearer } from "@/lib/auth/session"; +import { createVersion, getAgentByName } from "@/lib/agents/registry"; + +export async function POST( + req: NextRequest, + { params }: { params: Promise<{ name: string }> } +) { + const session = await verifyBearer(req); + if (!session) { + return err(401, "unauthorized", "Invalid or revoked session token"); + } + + const { name } = await params; + // Resolve within the caller's org only — an agent from another org must + // behave as not found, never leak its existence via a 403. + const agent = await getAgentByName(session.orgId, name); + if (!agent) { + return err(404, "not_found", "No such agent"); + } + + const body = (await req.json().catch(() => null)) as + | { files?: Record } + | null; + const files = body?.files; + if (!files || typeof files !== "object" || Array.isArray(files) || Object.keys(files).length === 0) { + return err(400, "invalid_request", "A version needs a non-empty files map"); + } + + try { + const { id, contentHash } = await createVersion({ + agentId: agent.id, + files, + createdBy: session.userId, + }); + return ok({ versionId: id, contentHash }); + } catch (e) { + console.error("version create failed", e); + return err(500, "internal_error", "Failed to create version", true); + } +} diff --git a/control-plane/artifacts/sanad-web/app/api/v1/agents/route.ts b/control-plane/artifacts/sanad-web/app/api/v1/agents/route.ts new file mode 100644 index 000000000..d53f87890 --- /dev/null +++ b/control-plane/artifacts/sanad-web/app/api/v1/agents/route.ts @@ -0,0 +1,56 @@ +import { NextRequest } from "next/server"; +import { ok, err } from "@/lib/http/envelope"; +import { verifyBearer } from "@/lib/auth/session"; +import { upsertAgent, listAgentsForOrg } from "@/lib/agents/registry"; + +const DEFAULT_WORKSPACE = "default"; + +export async function GET(req: NextRequest) { + const session = await verifyBearer(req); + if (!session) { + return err(401, "unauthorized", "Invalid or revoked session token"); + } + + const rows = await listAgentsForOrg(session.orgId); + return ok({ + agents: rows.map((a) => ({ + id: a.id, + name: a.name, + workspace: a.workspaceName, + ownerUserId: a.ownerUserId, + status: a.status, + description: a.description, + createdAt: a.createdAt, + })), + }); +} + +export async function POST(req: NextRequest) { + const session = await verifyBearer(req); + if (!session) { + return err(401, "unauthorized", "Invalid or revoked session token"); + } + + const body = (await req.json().catch(() => null)) as + | { name?: string; workspace?: string; description?: string } + | null; + const name = body?.name?.trim(); + if (!name) { + return err(400, "invalid_request", "An agent needs a name"); + } + const workspaceName = body?.workspace?.trim() || DEFAULT_WORKSPACE; + + try { + const { id } = await upsertAgent({ + orgId: session.orgId, + workspaceName, + name, + ownerUserId: session.userId, + description: body?.description, + }); + return ok({ agentId: id, name, workspace: workspaceName }); + } catch (e) { + console.error("agent create failed", e); + return err(500, "internal_error", "Failed to create agent", true); + } +} diff --git a/control-plane/artifacts/sanad-web/lib/agents/registry.ts b/control-plane/artifacts/sanad-web/lib/agents/registry.ts new file mode 100644 index 000000000..aedac1b25 --- /dev/null +++ b/control-plane/artifacts/sanad-web/lib/agents/registry.ts @@ -0,0 +1,188 @@ +import { createHash } from "crypto"; +import { and, desc, eq } from "drizzle-orm"; +import { db } from "../db"; +import { agents, agentVersions, deployments, workspaces } from "../db/schema"; + +export class OwnerRequiredError extends Error { + readonly code = "owner_required"; + constructor() { + super("agent has no active owner"); + } +} + +/** sha256 of the bundle's file map, independent of key insertion order. */ +export function bundleContentHash(files: Record): string { + const canonical = JSON.stringify(files, Object.keys(files).sort()); + return createHash("sha256").update(canonical).digest("hex"); +} + +/** + * Get-or-create a workspace by (orgId, name). Select-then-insert: sanad-web + * runs a single replica, so there is no concurrent-create race to close — + * same documented assumption as ensureInFlight in lib/compute/sessions.ts:286-291. + * There is no DB-level unique constraint on (org_id, name) to fall back on. + */ +export async function ensureWorkspace( + orgId: string, + name: string +): Promise<{ id: string }> { + const existing = await db + .select({ id: workspaces.id }) + .from(workspaces) + .where(and(eq(workspaces.orgId, orgId), eq(workspaces.name, name))) + .limit(1); + if (existing[0]) return { id: existing[0].id }; + + const id = `ws_${crypto.randomUUID()}`; + await db.insert(workspaces).values({ id, orgId, name }); + return { id }; +} + +/** + * Create an agent, or re-claim an existing one by (workspace, name). + * + * P0 has no separate "claim ownership" endpoint, so pushing to an existing + * agent name transfers ownership to the caller and un-orphans it (mirrors a + * `git push`-style deploy CLI: whoever pushes last owns it). Per-workspace + * name uniqueness is enforced here with the same select-then-insert + * assumption as ensureWorkspace above. + */ +export async function upsertAgent(p: { + orgId: string; + workspaceName: string; + name: string; + ownerUserId: string; + description?: string; +}): Promise<{ id: string }> { + const workspace = await ensureWorkspace(p.orgId, p.workspaceName); + + const existing = await db + .select({ id: agents.id }) + .from(agents) + .where(and(eq(agents.workspaceId, workspace.id), eq(agents.name, p.name))) + .limit(1); + + if (existing[0]) { + await db + .update(agents) + .set({ + ownerUserId: p.ownerUserId, + status: "active", + ...(p.description !== undefined ? { description: p.description } : {}), + }) + .where(eq(agents.id, existing[0].id)); + return { id: existing[0].id }; + } + + const id = `ag_${crypto.randomUUID()}`; + await db.insert(agents).values({ + id, + workspaceId: workspace.id, + name: p.name, + ownerUserId: p.ownerUserId, + description: p.description ?? null, + }); + return { id }; +} + +export async function createVersion(p: { + agentId: string; + files: Record; + createdBy: string; +}): Promise<{ id: string; contentHash: string }> { + const contentHash = bundleContentHash(p.files); + const id = `av_${crypto.randomUUID()}`; + await db.insert(agentVersions).values({ + id, + agentId: p.agentId, + contentHash, + bundle: { files: p.files }, + createdBy: p.createdBy, + }); + return { id, contentHash }; +} + +export async function createDeployment(p: { + agentId: string; + versionId: string; + env: "dev" | "prod"; +}): Promise<{ id: string }> { + const rows = await db.select().from(agents).where(eq(agents.id, p.agentId)).limit(1); + const agent = rows[0]; + if (!agent) throw new Error("agent not found"); + if (agent.status === "orphaned") throw new OwnerRequiredError(); + + const id = `dp_${crypto.randomUUID()}`; + await db.insert(deployments).values({ + id, + agentId: p.agentId, + agentVersionId: p.versionId, + env: p.env, + }); + return { id }; +} + +export async function setDeploymentStatus( + agentId: string, + env: string, + status: "active" | "paused" +): Promise { + await db + .update(deployments) + .set({ status, updatedAt: new Date() }) + .where(and(eq(deployments.agentId, agentId), eq(deployments.env, env))); +} + +/** Resolve an agent by name, scoped to the org — never crosses org boundaries. */ +export async function getAgentByName(orgId: string, name: string) { + const rows = await db + .select({ + id: agents.id, + workspaceId: agents.workspaceId, + name: agents.name, + ownerUserId: agents.ownerUserId, + status: agents.status, + description: agents.description, + createdAt: agents.createdAt, + }) + .from(agents) + .innerJoin(workspaces, eq(agents.workspaceId, workspaces.id)) + .where(and(eq(workspaces.orgId, orgId), eq(agents.name, name))) + .limit(1); + return rows[0] ?? null; +} + +/** Most recently created active deployment for an agent+env, or null. */ +export async function getActiveDeployment(agentId: string, env: string) { + const rows = await db + .select() + .from(deployments) + .where( + and( + eq(deployments.agentId, agentId), + eq(deployments.env, env), + eq(deployments.status, "active") + ) + ) + .orderBy(desc(deployments.createdAt)) + .limit(1); + return rows[0] ?? null; +} + +/** List every agent in the org, across all its workspaces. */ +export async function listAgentsForOrg(orgId: string) { + return db + .select({ + id: agents.id, + name: agents.name, + workspaceId: agents.workspaceId, + workspaceName: workspaces.name, + ownerUserId: agents.ownerUserId, + status: agents.status, + description: agents.description, + createdAt: agents.createdAt, + }) + .from(agents) + .innerJoin(workspaces, eq(agents.workspaceId, workspaces.id)) + .where(eq(workspaces.orgId, orgId)); +} diff --git a/control-plane/artifacts/sanad-web/tests/unit/agent-registry.test.ts b/control-plane/artifacts/sanad-web/tests/unit/agent-registry.test.ts new file mode 100644 index 000000000..b852debc6 --- /dev/null +++ b/control-plane/artifacts/sanad-web/tests/unit/agent-registry.test.ts @@ -0,0 +1,26 @@ +import { describe, it, expect, vi } from "vitest"; + +const state: { agentRow: any } = { agentRow: { id: "ag_1", status: "active" } }; +vi.mock("@/lib/db", () => ({ + db: { + insert: vi.fn(() => ({ values: vi.fn(async () => {}), onConflictDoNothing: vi.fn() })), + select: vi.fn(() => ({ + from: vi.fn(() => ({ where: vi.fn(() => ({ limit: vi.fn(async () => [state.agentRow]) })) })), + })), + update: vi.fn(() => ({ set: vi.fn(() => ({ where: vi.fn(async () => {}) })) })), + }, +})); + +import { bundleContentHash, createDeployment, OwnerRequiredError } from "@/lib/agents/registry"; + +describe("agent registry", () => { + it("bundle hash is key-order independent", () => { + expect(bundleContentHash({ b: "2", a: "1" })).toBe(bundleContentHash({ a: "1", b: "2" })); + }); + it("deploying an orphaned agent throws owner_required", async () => { + state.agentRow = { id: "ag_1", status: "orphaned" }; + await expect( + createDeployment({ agentId: "ag_1", versionId: "av_1", env: "dev" }) + ).rejects.toBeInstanceOf(OwnerRequiredError); + }); +}); From c1fde735411fc7a009fef83b56da00cccc933db8 Mon Sep 17 00:00:00 2001 From: Omar Alsoulah Date: Thu, 13 Aug 2026 17:03:21 +0300 Subject: [PATCH 05/28] =?UTF-8?q?sanad:=20agent=20registry=20=E2=80=94=20s?= =?UTF-8?q?table=20ownership=20on=20upsert,=20deployment=20supersede=20rul?= =?UTF-8?q?e?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sanad-web/lib/agents/registry.ts | 65 +++++++++---- .../artifacts/sanad-web/lib/db/schema.ts | 2 +- .../tests/unit/agent-registry.test.ts | 92 ++++++++++++++++++- 3 files changed, 136 insertions(+), 23 deletions(-) diff --git a/control-plane/artifacts/sanad-web/lib/agents/registry.ts b/control-plane/artifacts/sanad-web/lib/agents/registry.ts index aedac1b25..d7a9e7685 100644 --- a/control-plane/artifacts/sanad-web/lib/agents/registry.ts +++ b/control-plane/artifacts/sanad-web/lib/agents/registry.ts @@ -1,5 +1,5 @@ import { createHash } from "crypto"; -import { and, desc, eq } from "drizzle-orm"; +import { and, desc, eq, inArray } from "drizzle-orm"; import { db } from "../db"; import { agents, agentVersions, deployments, workspaces } from "../db/schema"; @@ -39,13 +39,16 @@ export async function ensureWorkspace( } /** - * Create an agent, or re-claim an existing one by (workspace, name). + * Create an agent, or return the existing one by (workspace, name). * - * P0 has no separate "claim ownership" endpoint, so pushing to an existing - * agent name transfers ownership to the caller and un-orphans it (mirrors a - * `git push`-style deploy CLI: whoever pushes last owns it). Per-workspace - * name uniqueness is enforced here with the same select-then-insert - * assumption as ensureWorkspace above. + * Ownership is stable on upsert: re-pushing an existing agent name never + * changes who owns it — that transfer is explicitly out of P0 scope, so + * there is no "claim ownership" side effect here. `ownerUserId` only takes + * effect when the agent doesn't exist yet ("owner = caller" applies to + * creation, not to every push). The existing-row path only refreshes + * `description`, and only when the caller supplied one. Per-workspace name + * uniqueness is enforced with the same select-then-insert assumption as + * ensureWorkspace above. */ export async function upsertAgent(p: { orgId: string; @@ -63,14 +66,12 @@ export async function upsertAgent(p: { .limit(1); if (existing[0]) { - await db - .update(agents) - .set({ - ownerUserId: p.ownerUserId, - status: "active", - ...(p.description !== undefined ? { description: p.description } : {}), - }) - .where(eq(agents.id, existing[0].id)); + if (p.description !== undefined) { + await db + .update(agents) + .set({ description: p.description }) + .where(eq(agents.id, existing[0].id)); + } return { id: existing[0].id }; } @@ -112,12 +113,29 @@ export async function createDeployment(p: { if (!agent) throw new Error("agent not found"); if (agent.status === "orphaned") throw new OwnerRequiredError(); + // At most one non-superseded deployment may exist per (agentId, env): retire + // whatever was active/paused before wiring in the new one. A blind + // conditional update is race-free enough here — single-replica control + // plane, same documented assumption as ensureInFlight in + // lib/compute/sessions.ts:286-291. + await db + .update(deployments) + .set({ status: "superseded", updatedAt: new Date() }) + .where( + and( + eq(deployments.agentId, p.agentId), + eq(deployments.env, p.env), + inArray(deployments.status, ["active", "paused"]) + ) + ); + const id = `dp_${crypto.randomUUID()}`; await db.insert(deployments).values({ id, agentId: p.agentId, agentVersionId: p.versionId, env: p.env, + status: "active", }); return { id }; } @@ -127,10 +145,18 @@ export async function setDeploymentStatus( env: string, status: "active" | "paused" ): Promise { + // Never resurrect a superseded row via pause/resume — only touch whatever + // is currently the live (active or paused) deployment for this env. await db .update(deployments) .set({ status, updatedAt: new Date() }) - .where(and(eq(deployments.agentId, agentId), eq(deployments.env, env))); + .where( + and( + eq(deployments.agentId, agentId), + eq(deployments.env, env), + inArray(deployments.status, ["active", "paused"]) + ) + ); } /** Resolve an agent by name, scoped to the org — never crosses org boundaries. */ @@ -152,7 +178,12 @@ export async function getAgentByName(orgId: string, name: string) { return rows[0] ?? null; } -/** Most recently created active deployment for an agent+env, or null. */ +/** + * The live deployment for an agent+env, or null. Filters to status "active" + * only — createDeployment guarantees at most one such row per (agentId, env) + * exists at a time, so the createdAt ordering here is a defensive tiebreak, + * not the primary selection mechanism. + */ export async function getActiveDeployment(agentId: string, env: string) { const rows = await db .select() diff --git a/control-plane/artifacts/sanad-web/lib/db/schema.ts b/control-plane/artifacts/sanad-web/lib/db/schema.ts index 07184ee97..7f36b59e9 100644 --- a/control-plane/artifacts/sanad-web/lib/db/schema.ts +++ b/control-plane/artifacts/sanad-web/lib/db/schema.ts @@ -232,7 +232,7 @@ export const deployments = pgTable("deployments", { agentId: text("agent_id").notNull().references(() => agents.id), agentVersionId: text("agent_version_id").notNull().references(() => agentVersions.id), env: text("env").notNull(), // "dev" | "prod" - status: text("status").default("active").notNull(), // "active" | "paused" + status: text("status").default("active").notNull(), // "active" | "paused" | "superseded" maxTurnSeconds: integer("max_turn_seconds").default(900).notNull(), maxStepsPerTurn: integer("max_steps_per_turn").default(100).notNull(), maxTokensPerRun: integer("max_tokens_per_run").default(2000000).notNull(), diff --git a/control-plane/artifacts/sanad-web/tests/unit/agent-registry.test.ts b/control-plane/artifacts/sanad-web/tests/unit/agent-registry.test.ts index b852debc6..e61530be5 100644 --- a/control-plane/artifacts/sanad-web/tests/unit/agent-registry.test.ts +++ b/control-plane/artifacts/sanad-web/tests/unit/agent-registry.test.ts @@ -1,26 +1,108 @@ -import { describe, it, expect, vi } from "vitest"; +import { describe, it, expect, vi, beforeEach } from "vitest"; const state: { agentRow: any } = { agentRow: { id: "ag_1", status: "active" } }; +// Queue of results for successive select().from().where().limit() calls, so +// tests that need more than one distinct select (e.g. upsertAgent's +// ensureWorkspace lookup followed by its own agent lookup) can script each +// call in order. Falls back to [state.agentRow] when empty, which preserves +// the original single-select tests unchanged. +let selectQueue: any[][] = []; +const updateCalls: any[] = []; +const insertCalls: any[] = []; + vi.mock("@/lib/db", () => ({ db: { - insert: vi.fn(() => ({ values: vi.fn(async () => {}), onConflictDoNothing: vi.fn() })), + insert: vi.fn(() => ({ + values: vi.fn(async (v: any) => { + insertCalls.push(v); + }), + onConflictDoNothing: vi.fn(), + })), select: vi.fn(() => ({ - from: vi.fn(() => ({ where: vi.fn(() => ({ limit: vi.fn(async () => [state.agentRow]) })) })), + from: vi.fn(() => ({ + where: vi.fn(() => ({ + limit: vi.fn(async () => (selectQueue.length ? selectQueue.shift()! : [state.agentRow])), + })), + })), + })), + update: vi.fn(() => ({ + set: vi.fn((v: any) => { + updateCalls.push(v); + return { where: vi.fn(async () => {}) }; + }), })), - update: vi.fn(() => ({ set: vi.fn(() => ({ where: vi.fn(async () => {}) })) })), }, })); -import { bundleContentHash, createDeployment, OwnerRequiredError } from "@/lib/agents/registry"; +import { + bundleContentHash, + createDeployment, + OwnerRequiredError, + upsertAgent, +} from "@/lib/agents/registry"; + +beforeEach(() => { + selectQueue = []; + updateCalls.length = 0; + insertCalls.length = 0; + state.agentRow = { id: "ag_1", status: "active" }; +}); describe("agent registry", () => { it("bundle hash is key-order independent", () => { expect(bundleContentHash({ b: "2", a: "1" })).toBe(bundleContentHash({ a: "1", b: "2" })); }); + it("deploying an orphaned agent throws owner_required", async () => { state.agentRow = { id: "ag_1", status: "orphaned" }; await expect( createDeployment({ agentId: "ag_1", versionId: "av_1", env: "dev" }) ).rejects.toBeInstanceOf(OwnerRequiredError); }); + + it("creating a deployment supersedes prior active/paused rows, then inserts the new one as active", async () => { + state.agentRow = { id: "ag_1", status: "active" }; + const { id } = await createDeployment({ agentId: "ag_1", versionId: "av_2", env: "prod" }); + + expect(id).toMatch(/^dp_/); + expect(updateCalls).toEqual([{ status: "superseded", updatedAt: expect.any(Date) }]); + expect(insertCalls).toHaveLength(1); + expect(insertCalls[0]).toMatchObject({ env: "prod", status: "active" }); + }); + + it("upserting an existing agent never changes ownerUserId, only refreshes description", async () => { + // ensureWorkspace's select finds an existing workspace... + selectQueue.push([{ id: "ws_1" }]); + // ...then upsertAgent's own select finds an existing agent in it. + selectQueue.push([{ id: "ag_1" }]); + + const result = await upsertAgent({ + orgId: "org_1", + workspaceName: "default", + name: "my-agent", + ownerUserId: "user_someone_else", + description: "refreshed description", + }); + + expect(result).toEqual({ id: "ag_1" }); + expect(updateCalls).toEqual([{ description: "refreshed description" }]); + expect(updateCalls[0]).not.toHaveProperty("ownerUserId"); + expect(updateCalls[0]).not.toHaveProperty("status"); + }); + + it("upserting an existing agent with no description issues no update at all", async () => { + selectQueue.push([{ id: "ws_1" }]); + selectQueue.push([{ id: "ag_1" }]); + + const result = await upsertAgent({ + orgId: "org_1", + workspaceName: "default", + name: "my-agent", + ownerUserId: "user_someone_else", + }); + + expect(result).toEqual({ id: "ag_1" }); + expect(updateCalls).toHaveLength(0); + expect(insertCalls).toHaveLength(0); + }); }); From 8073546b710324b66018bb2ff8fd3048394cc7e2 Mon Sep 17 00:00:00 2001 From: Omar Alsoulah Date: Thu, 13 Aug 2026 17:26:58 +0300 Subject: [PATCH 06/28] =?UTF-8?q?sanad:=20agent=20registry=20=E2=80=94=20v?= =?UTF-8?q?ersion=20ancestry=20check=20on=20deploy,=20404=20on=20no-op=20p?= =?UTF-8?q?ause?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../api/v1/agents/[name]/deployments/route.ts | 13 ++++- .../sanad-web/lib/agents/registry.ts | 56 +++++++++++++++++-- .../tests/unit/agent-registry.test.ts | 38 ++++++++++++- 3 files changed, 100 insertions(+), 7 deletions(-) diff --git a/control-plane/artifacts/sanad-web/app/api/v1/agents/[name]/deployments/route.ts b/control-plane/artifacts/sanad-web/app/api/v1/agents/[name]/deployments/route.ts index e76ad0030..7b82532dc 100644 --- a/control-plane/artifacts/sanad-web/app/api/v1/agents/[name]/deployments/route.ts +++ b/control-plane/artifacts/sanad-web/app/api/v1/agents/[name]/deployments/route.ts @@ -6,6 +6,7 @@ import { getAgentByName, OwnerRequiredError, setDeploymentStatus, + VersionMismatchError, } from "@/lib/agents/registry"; function isEnv(v: unknown): v is "dev" | "prod" { @@ -50,6 +51,11 @@ export async function POST( if (e instanceof OwnerRequiredError) { return err(409, "owner_required", e.message); } + // 404, not 403 — a versionId from another agent must behave as not + // found, same information-hiding rule as a cross-org agent name. + if (e instanceof VersionMismatchError) { + return err(404, "version_not_found", e.message); + } console.error("deployment create failed", e); return err(500, "internal_error", "Failed to create deployment", true); } @@ -81,7 +87,12 @@ export async function PATCH( } try { - await setDeploymentStatus(agent.id, body.env, body.status); + const matched = await setDeploymentStatus(agent.id, body.env, body.status); + if (!matched) { + // Same code Task 5's invoke gate uses for "no active deployment" — a + // pause/resume with nothing live to act on is not success. + return err(404, "not_deployed", "no active deployment for env"); + } return ok({ agentId: agent.id, env: body.env, status: body.status }); } catch (e) { console.error("deployment status update failed", e); diff --git a/control-plane/artifacts/sanad-web/lib/agents/registry.ts b/control-plane/artifacts/sanad-web/lib/agents/registry.ts index d7a9e7685..6897950d9 100644 --- a/control-plane/artifacts/sanad-web/lib/agents/registry.ts +++ b/control-plane/artifacts/sanad-web/lib/agents/registry.ts @@ -10,6 +10,19 @@ export class OwnerRequiredError extends Error { } } +/** + * Thrown when a deployment's versionId doesn't belong to the agent it's + * being deployed to. Maps to 404, not 403 — same information-hiding rule as + * a cross-org agent name: a caller holding some other agent's version id + * must not learn anything about whether that id exists at all. + */ +export class VersionMismatchError extends Error { + readonly code = "version_not_found"; + constructor() { + super("version does not belong to this agent"); + } +} + /** sha256 of the bundle's file map, independent of key insertion order. */ export function bundleContentHash(files: Record): string { const canonical = JSON.stringify(files, Object.keys(files).sort()); @@ -113,6 +126,19 @@ export async function createDeployment(p: { if (!agent) throw new Error("agent not found"); if (agent.status === "orphaned") throw new OwnerRequiredError(); + // The versionId is client-supplied — without this check a caller holding + // any agent-version id could splice another agent's bundle into this + // agent's deployment history. + const versionRows = await db + .select({ id: agentVersions.id, agentId: agentVersions.agentId }) + .from(agentVersions) + .where(eq(agentVersions.id, p.versionId)) + .limit(1); + const version = versionRows[0]; + if (!version || version.agentId !== p.agentId) { + throw new VersionMismatchError(); + } + // At most one non-superseded deployment may exist per (agentId, env): retire // whatever was active/paused before wiring in the new one. A blind // conditional update is race-free enough here — single-replica control @@ -140,23 +166,43 @@ export async function createDeployment(p: { return { id }; } +/** + * Pause/resume the live deployment for an agent+env. Returns whether a + * target row existed — callers must not report success on a no-op update. + * + * drizzle's update() result shape for row-matched-count is driver-dependent + * (and awkward to assert on through the mocked db in tests), so this uses an + * explicit select-then-update instead of trusting an update result's row + * count. Same single-replica assumption as ensureInFlight in + * lib/compute/sessions.ts:286-291 — the window between the select and the + * update is not a concern here. + */ export async function setDeploymentStatus( agentId: string, env: string, status: "active" | "paused" -): Promise { +): Promise { // Never resurrect a superseded row via pause/resume — only touch whatever // is currently the live (active or paused) deployment for this env. - await db - .update(deployments) - .set({ status, updatedAt: new Date() }) + const rows = await db + .select({ id: deployments.id }) + .from(deployments) .where( and( eq(deployments.agentId, agentId), eq(deployments.env, env), inArray(deployments.status, ["active", "paused"]) ) - ); + ) + .limit(1); + const target = rows[0]; + if (!target) return false; + + await db + .update(deployments) + .set({ status, updatedAt: new Date() }) + .where(eq(deployments.id, target.id)); + return true; } /** Resolve an agent by name, scoped to the org — never crosses org boundaries. */ diff --git a/control-plane/artifacts/sanad-web/tests/unit/agent-registry.test.ts b/control-plane/artifacts/sanad-web/tests/unit/agent-registry.test.ts index e61530be5..2340a71da 100644 --- a/control-plane/artifacts/sanad-web/tests/unit/agent-registry.test.ts +++ b/control-plane/artifacts/sanad-web/tests/unit/agent-registry.test.ts @@ -38,7 +38,9 @@ import { bundleContentHash, createDeployment, OwnerRequiredError, + setDeploymentStatus, upsertAgent, + VersionMismatchError, } from "@/lib/agents/registry"; beforeEach(() => { @@ -61,7 +63,8 @@ describe("agent registry", () => { }); it("creating a deployment supersedes prior active/paused rows, then inserts the new one as active", async () => { - state.agentRow = { id: "ag_1", status: "active" }; + selectQueue.push([{ id: "ag_1", status: "active" }]); // agent lookup, not orphaned + selectQueue.push([{ id: "av_2", agentId: "ag_1" }]); // version belongs to this agent const { id } = await createDeployment({ agentId: "ag_1", versionId: "av_2", env: "prod" }); expect(id).toMatch(/^dp_/); @@ -70,6 +73,39 @@ describe("agent registry", () => { expect(insertCalls[0]).toMatchObject({ env: "prod", status: "active" }); }); + it("deploying a version that belongs to a different agent throws version_not_found", async () => { + selectQueue.push([{ id: "ag_1", status: "active" }]); // agent lookup, not orphaned + selectQueue.push([{ id: "av_9", agentId: "ag_OTHER" }]); // version belongs to a different agent + await expect( + createDeployment({ agentId: "ag_1", versionId: "av_9", env: "dev" }) + ).rejects.toBeInstanceOf(VersionMismatchError); + // Neither the supersede update nor the insert should have run. + expect(updateCalls).toHaveLength(0); + expect(insertCalls).toHaveLength(0); + }); + + it("deploying a versionId that doesn't exist at all also throws version_not_found", async () => { + selectQueue.push([{ id: "ag_1", status: "active" }]); // agent lookup, not orphaned + selectQueue.push([]); // no such version row + await expect( + createDeployment({ agentId: "ag_1", versionId: "av_missing", env: "dev" }) + ).rejects.toBeInstanceOf(VersionMismatchError); + }); + + it("setDeploymentStatus returns false and skips the update when nothing matches", async () => { + selectQueue.push([]); // no active/paused row for this agent+env + const matched = await setDeploymentStatus("ag_1", "dev", "paused"); + expect(matched).toBe(false); + expect(updateCalls).toHaveLength(0); + }); + + it("setDeploymentStatus updates and returns true when a live row exists", async () => { + selectQueue.push([{ id: "dp_1" }]); + const matched = await setDeploymentStatus("ag_1", "dev", "paused"); + expect(matched).toBe(true); + expect(updateCalls).toEqual([{ status: "paused", updatedAt: expect.any(Date) }]); + }); + it("upserting an existing agent never changes ownerUserId, only refreshes description", async () => { // ensureWorkspace's select finds an existing workspace... selectQueue.push([{ id: "ws_1" }]); From 19e5c2b081df4fae19807fe1d20e5f6d36073ed2 Mon Sep 17 00:00:00 2001 From: Omar Alsoulah Date: Thu, 13 Aug 2026 17:38:07 +0300 Subject: [PATCH 07/28] =?UTF-8?q?sanad:=20workspace=20machines=20=E2=80=94?= =?UTF-8?q?=20per-(workspace,env)=20fargate=20wake=20with=20worker=20mode?= =?UTF-8?q?=20+=20keep=5Fwarm?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../app/api/v1/compute/route/route.ts | 7 +- .../drizzle/0007_optimal_grey_gargoyle.sql | 19 + .../sanad-web/drizzle/meta/0007_snapshot.json | 1899 +++++++++++++++++ .../sanad-web/drizzle/meta/_journal.json | 7 + .../sanad-web/lib/compute/machines.ts | 243 +++ .../sanad-web/lib/compute/sessions.ts | 13 +- .../artifacts/sanad-web/lib/db/schema.ts | 24 + .../tests/unit/workspace-machines.test.ts | 15 + 8 files changed, 2223 insertions(+), 4 deletions(-) create mode 100644 control-plane/artifacts/sanad-web/drizzle/0007_optimal_grey_gargoyle.sql create mode 100644 control-plane/artifacts/sanad-web/drizzle/meta/0007_snapshot.json create mode 100644 control-plane/artifacts/sanad-web/lib/compute/machines.ts create mode 100644 control-plane/artifacts/sanad-web/tests/unit/workspace-machines.test.ts diff --git a/control-plane/artifacts/sanad-web/app/api/v1/compute/route/route.ts b/control-plane/artifacts/sanad-web/app/api/v1/compute/route/route.ts index a03787834..2580ab8c7 100644 --- a/control-plane/artifacts/sanad-web/app/api/v1/compute/route/route.ts +++ b/control-plane/artifacts/sanad-web/app/api/v1/compute/route/route.ts @@ -29,11 +29,14 @@ export async function GET(req: NextRequest) { if (!/^[a-f0-9]{12}$/.test(hash)) { return err(400, "invalid_request", "Malformed workspace hash"); } - // Sessions own routing now; the legacy per-user table remains as fallback - // until it is dropped. + // Sessions own routing now; workspace machines (worker runtime) are next; + // the legacy per-user table remains as fallback until it is dropped. const { sessionIpByHash } = await import("@/lib/compute/sessions"); const sessionIp = await sessionIpByHash(hash); if (sessionIp) return ok({ taskIp: sessionIp }); + const { machineIpByHash } = await import("@/lib/compute/machines"); + const machineIp = await machineIpByHash(hash); + if (machineIp) return ok({ taskIp: machineIp }); const [row] = await db .select() .from(workspaceTasks) diff --git a/control-plane/artifacts/sanad-web/drizzle/0007_optimal_grey_gargoyle.sql b/control-plane/artifacts/sanad-web/drizzle/0007_optimal_grey_gargoyle.sql new file mode 100644 index 000000000..3fcee3ce0 --- /dev/null +++ b/control-plane/artifacts/sanad-web/drizzle/0007_optimal_grey_gargoyle.sql @@ -0,0 +1,19 @@ +CREATE TABLE "workspace_machines" ( + "id" text PRIMARY KEY NOT NULL, + "workspace_id" text NOT NULL, + "env" text NOT NULL, + "hash12" text NOT NULL, + "efs_access_point_id" text NOT NULL, + "task_arn" text, + "task_ip" text, + "run_nonce" text, + "image_ref" text NOT NULL, + "state" text NOT NULL, + "keep_warm" boolean DEFAULT false NOT NULL, + "last_seen_at" timestamp with time zone, + "created_at" timestamp with time zone DEFAULT now() NOT NULL, + "updated_at" timestamp with time zone DEFAULT now() NOT NULL, + CONSTRAINT "workspace_machines_hash12_unique" UNIQUE("hash12") +); +--> statement-breakpoint +ALTER TABLE "workspace_machines" ADD CONSTRAINT "workspace_machines_workspace_id_workspaces_id_fk" FOREIGN KEY ("workspace_id") REFERENCES "public"."workspaces"("id") ON DELETE no action ON UPDATE no action; \ No newline at end of file diff --git a/control-plane/artifacts/sanad-web/drizzle/meta/0007_snapshot.json b/control-plane/artifacts/sanad-web/drizzle/meta/0007_snapshot.json new file mode 100644 index 000000000..ee934bcbd --- /dev/null +++ b/control-plane/artifacts/sanad-web/drizzle/meta/0007_snapshot.json @@ -0,0 +1,1899 @@ +{ + "id": "045bb80e-d55e-4af9-b689-452807a38819", + "prevId": "98bc119b-39cb-4f3b-9a63-3214766e1882", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.agent_versions": { + "name": "agent_versions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "content_hash": { + "name": "content_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "bundle": { + "name": "bundle", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "agent_versions_agent_id_agents_id_fk": { + "name": "agent_versions_agent_id_agents_id_fk", + "tableFrom": "agent_versions", + "tableTo": "agents", + "columnsFrom": [ + "agent_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.agents": { + "name": "agents", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "owner_user_id": { + "name": "owner_user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'active'" + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "agents_workspace_id_workspaces_id_fk": { + "name": "agents_workspace_id_workspaces_id_fk", + "tableFrom": "agents", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "agents_owner_user_id_users_id_fk": { + "name": "agents_owner_user_id_users_id_fk", + "tableFrom": "agents", + "tableTo": "users", + "columnsFrom": [ + "owner_user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.cli_sessions": { + "name": "cli_sessions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "token_hash": { + "name": "token_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "device_request_id": { + "name": "device_request_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "device_label": { + "name": "device_label", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "last_used_at": { + "name": "last_used_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "cli_sessions_user_id_users_id_fk": { + "name": "cli_sessions_user_id_users_id_fk", + "tableFrom": "cli_sessions", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "cli_sessions_org_id_organizations_id_fk": { + "name": "cli_sessions_org_id_organizations_id_fk", + "tableFrom": "cli_sessions", + "tableTo": "organizations", + "columnsFrom": [ + "org_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "cli_sessions_device_request_id_device_auth_requests_id_fk": { + "name": "cli_sessions_device_request_id_device_auth_requests_id_fk", + "tableFrom": "cli_sessions", + "tableTo": "device_auth_requests", + "columnsFrom": [ + "device_request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "cli_sessions_token_hash_unique": { + "name": "cli_sessions_token_hash_unique", + "nullsNotDistinct": false, + "columns": [ + "token_hash" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.deployments": { + "name": "deployments", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent_version_id": { + "name": "agent_version_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "env": { + "name": "env", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'active'" + }, + "max_turn_seconds": { + "name": "max_turn_seconds", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 900 + }, + "max_steps_per_turn": { + "name": "max_steps_per_turn", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 100 + }, + "max_tokens_per_run": { + "name": "max_tokens_per_run", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 2000000 + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "deployments_agent_id_agents_id_fk": { + "name": "deployments_agent_id_agents_id_fk", + "tableFrom": "deployments", + "tableTo": "agents", + "columnsFrom": [ + "agent_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "deployments_agent_version_id_agent_versions_id_fk": { + "name": "deployments_agent_version_id_agent_versions_id_fk", + "tableFrom": "deployments", + "tableTo": "agent_versions", + "columnsFrom": [ + "agent_version_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.device_auth_requests": { + "name": "device_auth_requests", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "device_auth_id_hash": { + "name": "device_auth_id_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_code": { + "name": "user_code", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "device_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "approved_user_id": { + "name": "approved_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "approved_org_id": { + "name": "approved_org_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "pending_session_token": { + "name": "pending_session_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "poll_interval_seconds": { + "name": "poll_interval_seconds", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 2 + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "device_auth_requests_device_auth_id_hash_unique": { + "name": "device_auth_requests_device_auth_id_hash_unique", + "nullsNotDistinct": false, + "columns": [ + "device_auth_id_hash" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.invoke_tokens": { + "name": "invoke_tokens", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "token_hash": { + "name": "token_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "family_id": { + "name": "family_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "env": { + "name": "env", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "invoke_tokens_agent_id_agents_id_fk": { + "name": "invoke_tokens_agent_id_agents_id_fk", + "tableFrom": "invoke_tokens", + "tableTo": "agents", + "columnsFrom": [ + "agent_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "invoke_tokens_token_hash_unique": { + "name": "invoke_tokens_token_hash_unique", + "nullsNotDistinct": false, + "columns": [ + "token_hash" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.memberships": { + "name": "memberships", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "seat_assigned": { + "name": "seat_assigned", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "memberships_org_id_organizations_id_fk": { + "name": "memberships_org_id_organizations_id_fk", + "tableFrom": "memberships", + "tableTo": "organizations", + "columnsFrom": [ + "org_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "memberships_user_id_users_id_fk": { + "name": "memberships_user_id_users_id_fk", + "tableFrom": "memberships", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.organizations": { + "name": "organizations", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.project_sessions": { + "name": "project_sessions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ui_state": { + "name": "ui_state", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "archived_at": { + "name": "archived_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "last_active_at": { + "name": "last_active_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "project_sessions_project_id_workspace_sessions_id_fk": { + "name": "project_sessions_project_id_workspace_sessions_id_fk", + "tableFrom": "project_sessions", + "tableTo": "workspace_sessions", + "columnsFrom": [ + "project_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "project_sessions_user_id_users_id_fk": { + "name": "project_sessions_user_id_users_id_fk", + "tableFrom": "project_sessions", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.runs": { + "name": "runs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "deployment_id": { + "name": "deployment_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent_version_id": { + "name": "agent_version_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'queued'" + }, + "error_code": { + "name": "error_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "trigger_principal": { + "name": "trigger_principal", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "idempotency_key": { + "name": "idempotency_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "output": { + "name": "output", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "tokens_in": { + "name": "tokens_in", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "tokens_out": { + "name": "tokens_out", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "cost_usd_micros": { + "name": "cost_usd_micros", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "model_alias": { + "name": "model_alias", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "trace_uploaded": { + "name": "trace_uploaded", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "started_at": { + "name": "started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "runs_deployment_idem_uq": { + "name": "runs_deployment_idem_uq", + "columns": [ + { + "expression": "deployment_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "idempotency_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "runs_deployment_id_deployments_id_fk": { + "name": "runs_deployment_id_deployments_id_fk", + "tableFrom": "runs", + "tableTo": "deployments", + "columnsFrom": [ + "deployment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.runtime_tokens": { + "name": "runtime_tokens", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "token_hash": { + "name": "token_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "family_id": { + "name": "family_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "cli_session_id": { + "name": "cli_session_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "absolute_expires_at": { + "name": "absolute_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "runtime_tokens_cli_session_id_cli_sessions_id_fk": { + "name": "runtime_tokens_cli_session_id_cli_sessions_id_fk", + "tableFrom": "runtime_tokens", + "tableTo": "cli_sessions", + "columnsFrom": [ + "cli_session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "runtime_tokens_token_hash_unique": { + "name": "runtime_tokens_token_hash_unique", + "nullsNotDistinct": false, + "columns": [ + "token_hash" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.ships": { + "name": "ships", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "app_slug": { + "name": "app_slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ecr_image": { + "name": "ecr_image", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "commit_sha": { + "name": "commit_sha", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "codebuild_id": { + "name": "codebuild_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "ships_user_id_users_id_fk": { + "name": "ships_user_id_users_id_fk", + "tableFrom": "ships", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "ships_org_id_organizations_id_fk": { + "name": "ships_org_id_organizations_id_fk", + "tableFrom": "ships", + "tableTo": "organizations", + "columnsFrom": [ + "org_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "ships_app_slug_unique": { + "name": "ships_app_slug_unique", + "nullsNotDistinct": false, + "columns": [ + "app_slug" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.subscriptions": { + "name": "subscriptions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "plan": { + "name": "plan", + "type": "plan", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'free'" + }, + "status": { + "name": "status", + "type": "sub_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'active'" + }, + "stripe_customer_id": { + "name": "stripe_customer_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "stripe_subscription_id": { + "name": "stripe_subscription_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "seats": { + "name": "seats", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "quota": { + "name": "quota", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "current_period_end": { + "name": "current_period_end", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "subscriptions_org_id_organizations_id_fk": { + "name": "subscriptions_org_id_organizations_id_fk", + "tableFrom": "subscriptions", + "tableTo": "organizations", + "columnsFrom": [ + "org_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.terminal_tickets": { + "name": "terminal_tickets", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "ticket_hash": { + "name": "ticket_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "session_token": { + "name": "session_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "redeemed_at": { + "name": "redeemed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "terminal_tickets_user_id_users_id_fk": { + "name": "terminal_tickets_user_id_users_id_fk", + "tableFrom": "terminal_tickets", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "terminal_tickets_org_id_organizations_id_fk": { + "name": "terminal_tickets_org_id_organizations_id_fk", + "tableFrom": "terminal_tickets", + "tableTo": "organizations", + "columnsFrom": [ + "org_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "terminal_tickets_ticket_hash_unique": { + "name": "terminal_tickets_ticket_hash_unique", + "nullsNotDistinct": false, + "columns": [ + "ticket_hash" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.usage_events": { + "name": "usage_events", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "cli_session_id": { + "name": "cli_session_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "model_alias": { + "name": "model_alias", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tokens_in": { + "name": "tokens_in", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "tokens_out": { + "name": "tokens_out", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "cost": { + "name": "cost", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.users": { + "name": "users", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspace_machines": { + "name": "workspace_machines", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "env": { + "name": "env", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "hash12": { + "name": "hash12", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "efs_access_point_id": { + "name": "efs_access_point_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "task_arn": { + "name": "task_arn", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "task_ip": { + "name": "task_ip", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "run_nonce": { + "name": "run_nonce", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "image_ref": { + "name": "image_ref", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "keep_warm": { + "name": "keep_warm", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "last_seen_at": { + "name": "last_seen_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "workspace_machines_workspace_id_workspaces_id_fk": { + "name": "workspace_machines_workspace_id_workspaces_id_fk", + "tableFrom": "workspace_machines", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "workspace_machines_hash12_unique": { + "name": "workspace_machines_hash12_unique", + "nullsNotDistinct": false, + "columns": [ + "hash12" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspace_sessions": { + "name": "workspace_sessions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "hash12": { + "name": "hash12", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "efs_access_point_id": { + "name": "efs_access_point_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "task_arn": { + "name": "task_arn", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "task_ip": { + "name": "task_ip", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "run_nonce": { + "name": "run_nonce", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "image_ref": { + "name": "image_ref", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "last_seen_at": { + "name": "last_seen_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "workspace_sessions_user_id_users_id_fk": { + "name": "workspace_sessions_user_id_users_id_fk", + "tableFrom": "workspace_sessions", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "workspace_sessions_hash12_unique": { + "name": "workspace_sessions_hash12_unique", + "nullsNotDistinct": false, + "columns": [ + "hash12" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspace_tasks": { + "name": "workspace_tasks", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "hash12": { + "name": "hash12", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "efs_access_point_id": { + "name": "efs_access_point_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "task_arn": { + "name": "task_arn", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "task_ip": { + "name": "task_ip", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "run_nonce": { + "name": "run_nonce", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "image_ref": { + "name": "image_ref", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "last_seen_at": { + "name": "last_seen_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "workspace_tasks_user_id_users_id_fk": { + "name": "workspace_tasks_user_id_users_id_fk", + "tableFrom": "workspace_tasks", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "workspace_tasks_user_id_unique": { + "name": "workspace_tasks_user_id_unique", + "nullsNotDistinct": false, + "columns": [ + "user_id" + ] + }, + "workspace_tasks_hash12_unique": { + "name": "workspace_tasks_hash12_unique", + "nullsNotDistinct": false, + "columns": [ + "hash12" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspaces": { + "name": "workspaces", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "keep_warm": { + "name": "keep_warm", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "budget_usd_month": { + "name": "budget_usd_month", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "workspaces_org_id_organizations_id_fk": { + "name": "workspaces_org_id_organizations_id_fk", + "tableFrom": "workspaces", + "tableTo": "organizations", + "columnsFrom": [ + "org_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": { + "public.device_status": { + "name": "device_status", + "schema": "public", + "values": [ + "pending", + "complete", + "denied", + "expired" + ] + }, + "public.plan": { + "name": "plan", + "schema": "public", + "values": [ + "free", + "pro", + "team", + "enterprise" + ] + }, + "public.sub_status": { + "name": "sub_status", + "schema": "public", + "values": [ + "active", + "past_due", + "canceled" + ] + } + }, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/control-plane/artifacts/sanad-web/drizzle/meta/_journal.json b/control-plane/artifacts/sanad-web/drizzle/meta/_journal.json index 0ee86300e..86f202f84 100644 --- a/control-plane/artifacts/sanad-web/drizzle/meta/_journal.json +++ b/control-plane/artifacts/sanad-web/drizzle/meta/_journal.json @@ -50,6 +50,13 @@ "when": 1786627610138, "tag": "0006_foamy_venus", "breakpoints": true + }, + { + "idx": 7, + "version": "7", + "when": 1786631619023, + "tag": "0007_optimal_grey_gargoyle", + "breakpoints": true } ] } \ No newline at end of file diff --git a/control-plane/artifacts/sanad-web/lib/compute/machines.ts b/control-plane/artifacts/sanad-web/lib/compute/machines.ts new file mode 100644 index 000000000..2b3d855df --- /dev/null +++ b/control-plane/artifacts/sanad-web/lib/compute/machines.ts @@ -0,0 +1,243 @@ +/** + * Workspace machines: the Fargate task backing a (workspace, env) pair that + * runs deployed agents (PRD worker runtime). Same wake state machine as + * lib/compute/sessions.ts's per-user-session machines — an EFS access point, + * a task started on demand and self-stopped when idle — keyed per + * (workspaceId, env) instead of per (userId, sessionId). + * + * The AWS-touching steps (ensureAccessPoint, registerTaskDefinition, + * runWorkspaceTask, waitForRunning, waitForAgentd) are the exact functions + * the session path uses — the first three are already exported from ./aws; + * waitForRunning/waitForAgentd were private to sessions.ts and are now + * exported from there for this reuse. + */ +import { createHash } from "crypto"; +import { eq } from "drizzle-orm"; +import { db } from "../db"; +import { workspaceMachines } from "../db/schema"; +import { + awsComputeConfig, + ensureAccessPoint, + registerTaskDefinition, + runWorkspaceTask, + stopTask, + type AwsComputeConfig, +} from "./aws"; +import { computeBaseUrl, waitForAgentd, waitForRunning } from "./sessions"; +import { deriveMachineToken } from "./tokens"; + +export type MachineRow = typeof workspaceMachines.$inferSelect; + +export interface MachineTarget { + machineId: string; + hash12: string; + baseUrl: string; + agentdToken: string; + coldStart: boolean; +} + +/** + * Router-namespace hash for a workspace machine. The "wm:" prefix keeps + * worker hashes from ever colliding with user-session hashes (sessionHash + * has no such prefix) in the shared hash12 routing space. + */ +export function machineHash(workspaceId: string, env: string): string { + return createHash("sha256") + .update(`wm:${workspaceId}:${env}`) + .digest("hex") + .slice(0, 12); +} + +/** + * Base container env for a workspace machine. Deliberately NOT reused from + * sessions.ts's agentBaseEnv: that helper is shaped for the interactive CLI + * workspace (WORKSPACE_MODE: "task", SANAD_WORKSPACE_USER) and isn't a fit + * for a non-interactive worker machine identified by (workspaceId, env). + * Not part of the Task 4 interface contract — the worker container's actual + * expected env var names should be confirmed against the runtime that reads + * them (flagged in the task report as a divergence). + */ +function machineBaseEnv( + config: AwsComputeConfig, + workspaceId: string, + env: string, +): Record { + return { + WORKSPACE_MODE: "worker", + SANAD_WORKSPACE_ID: workspaceId, + SANAD_WORKSPACE_ENV: env, + CONTROL_PLANE_URL: config.controlPlaneUrl, + SANAD_API_BASE_URL: config.controlPlaneUrl, + TERMINAL_ALLOWED_ORIGINS: config.allowedOrigins, + }; +} + +/** + * Warm-attach reachability check, budgeted short (5s) unlike the cold path's + * full waitForAgentd wait (60s): races the same exported waitForAgentd + * against a timeout rather than duplicating its fetch/poll logic. If the + * timeout wins, the losing waitForAgentd call is abandoned (not cancelled) + * and its eventual rejection is swallowed here — a bounded, harmless number + * of background polls, never surfaced. + */ +function warmProbe(baseUrl: string, budgetMs = 5_000): Promise { + return Promise.race([ + waitForAgentd(baseUrl).then( + () => true, + () => false, + ), + new Promise((resolve) => setTimeout(() => resolve(false), budgetMs)), + ]); +} + +async function getMachineRow(hash12: string): Promise { + const [row] = await db + .select() + .from(workspaceMachines) + .where(eq(workspaceMachines.hash12, hash12)) + .limit(1); + return row ?? null; +} + +/* + * Concurrent wakes for one (workspace, env) — e.g. two invoke calls landing + * at once — must produce ONE machine, not a RunTask stampede. Mirrors + * sessions.ts's in-process dedupe map; sound for the same reason (sanad-web + * runs a single replica). + */ +const ensureInFlight = new Map>(); + +export function ensureWorkspaceMachine( + workspaceId: string, + env: string, + opts: { keepWarm: boolean }, +): Promise { + const key = `${workspaceId}:${env}`; + const existing = ensureInFlight.get(key); + if (existing) return existing; + const run = ensureInner(workspaceId, env, opts).finally(() => { + ensureInFlight.delete(key); + }); + ensureInFlight.set(key, run); + return run; +} + +async function ensureInner( + workspaceId: string, + env: string, + opts: { keepWarm: boolean }, +): Promise { + const config = awsComputeConfig(); + const hash12 = machineHash(workspaceId, env); + const baseUrl = computeBaseUrl(hash12); + + // Idempotent — returns the existing access point for this hash12 path. + const accessPointId = await ensureAccessPoint(config, hash12); + + let row = await getMachineRow(hash12); + if (!row) { + const id = `wm_${crypto.randomUUID()}`; + const [inserted] = await db + .insert(workspaceMachines) + .values({ + id, + workspaceId, + env, + hash12, + efsAccessPointId: accessPointId, + imageRef: config.workspaceImage, + state: "provisioning", + keepWarm: opts.keepWarm, + }) + .returning(); + row = inserted; + } + + // A recorded task may still be running (warm attach) … + if (row.taskArn && row.taskIp && row.runNonce) { + const warm = await warmProbe(baseUrl); + if (warm) { + const stale = row.imageRef !== config.workspaceImage; + if (!stale) { + if (row.keepWarm !== opts.keepWarm) { + await db + .update(workspaceMachines) + .set({ keepWarm: opts.keepWarm, updatedAt: new Date() }) + .where(eq(workspaceMachines.id, row.id)); + } + return { + machineId: row.id, + hash12: row.hash12, + baseUrl, + agentdToken: deriveMachineToken(workspaceId, row.runNonce), + coldStart: false, + }; + } + console.log( + `machine ${row.id} is warm on a stale image — recycling`, + ); + } + // Unreachable, or warm-but-stale: replace it. + await stopTask(config, row.taskArn).catch(() => {}); + } + + // … otherwise (never ran / self-stopped / died / just recycled): fresh + // run, fresh nonce. + const runNonce = crypto.randomUUID(); + const agentdToken = deriveMachineToken(workspaceId, runNonce); + const taskDefArn = await registerTaskDefinition( + config, + hash12, + accessPointId, + machineBaseEnv(config, workspaceId, env), + ); + const taskArn = await runWorkspaceTask(config, taskDefArn, { + AGENTD_TOKEN: agentdToken, + MACHINE_NONCE: runNonce, + WORKER_ENABLED: "1", + KEEP_WARM: opts.keepWarm ? "1" : "0", + }); + + try { + const privateIp = await waitForRunning(config, taskArn); + // Publish the route BEFORE health-polling: the poll goes through the router. + await db + .update(workspaceMachines) + .set({ + taskArn, + taskIp: privateIp, + runNonce, + imageRef: config.workspaceImage, + state: "ready", + keepWarm: opts.keepWarm, + lastSeenAt: new Date(), + updatedAt: new Date(), + }) + .where(eq(workspaceMachines.id, row.id)); + await waitForAgentd(baseUrl); + } catch (e) { + await db + .update(workspaceMachines) + .set({ state: "error", updatedAt: new Date() }) + .where(eq(workspaceMachines.id, row.id)); + throw e; + } + + return { + machineId: row.id, + hash12, + baseUrl, + agentdToken, + coldStart: true, + }; +} + +/** Router route lookup: hash12 → task IP, for workspace machines. */ +export async function machineIpByHash(hash12: string): Promise { + const [row] = await db + .select({ taskIp: workspaceMachines.taskIp }) + .from(workspaceMachines) + .where(eq(workspaceMachines.hash12, hash12)) + .limit(1); + return row?.taskIp ?? null; +} diff --git a/control-plane/artifacts/sanad-web/lib/compute/sessions.ts b/control-plane/artifacts/sanad-web/lib/compute/sessions.ts index 59b751c7b..d49ae7d57 100644 --- a/control-plane/artifacts/sanad-web/lib/compute/sessions.ts +++ b/control-plane/artifacts/sanad-web/lib/compute/sessions.ts @@ -89,7 +89,12 @@ async function probeAgentd( } } -async function waitForRunning( +/** + * Poll ECS until the task is RUNNING with a private IP (or throw on a + * terminal failure). Exported for reuse by lib/compute/machines.ts — same + * wake state machine, keyed per (workspace, env) instead of per session. + */ +export async function waitForRunning( config: AwsComputeConfig, taskArn: string, ): Promise { @@ -105,7 +110,11 @@ async function waitForRunning( throw new Error("workspace task did not reach RUNNING in time"); } -async function waitForAgentd(baseUrl: string): Promise { +/** + * Poll agentd's /healthz through the router until it answers OK. Exported + * for reuse by lib/compute/machines.ts (see waitForRunning above). + */ +export async function waitForAgentd(baseUrl: string): Promise { const deadline = Date.now() + HEALTH_TIMEOUT_MS; let lastError = ""; while (Date.now() < deadline) { diff --git a/control-plane/artifacts/sanad-web/lib/db/schema.ts b/control-plane/artifacts/sanad-web/lib/db/schema.ts index 7f36b59e9..7d3468d9f 100644 --- a/control-plane/artifacts/sanad-web/lib/db/schema.ts +++ b/control-plane/artifacts/sanad-web/lib/db/schema.ts @@ -343,3 +343,27 @@ export const terminalTickets = pgTable("terminal_tickets", { .defaultNow() .notNull(), }); + +/** + * A workspace machine = the Fargate task backing a (workspace, env) pair for + * running deployed agents (PRD worker runtime). Same wake state machine as + * workspace_sessions, keyed per workspace+env instead of per user+session — + * hash12 namespaced with a "wm:" prefix (see machineHash) so worker routing + * never collides with user-session routing. + */ +export const workspaceMachines = pgTable("workspace_machines", { + id: text("id").primaryKey(), // wm_ + workspaceId: text("workspace_id").notNull().references(() => workspaces.id), + env: text("env").notNull(), + hash12: text("hash12").notNull().unique(), + efsAccessPointId: text("efs_access_point_id").notNull(), + taskArn: text("task_arn"), + taskIp: text("task_ip"), + runNonce: text("run_nonce"), + imageRef: text("image_ref").notNull(), + state: text("state").notNull(), // "provisioning" | "ready" | "error" + keepWarm: boolean("keep_warm").default(false).notNull(), + lastSeenAt: timestamp("last_seen_at", { withTimezone: true }), + createdAt: timestamp("created_at", { withTimezone: true }).defaultNow().notNull(), + updatedAt: timestamp("updated_at", { withTimezone: true }).defaultNow().notNull(), +}); diff --git a/control-plane/artifacts/sanad-web/tests/unit/workspace-machines.test.ts b/control-plane/artifacts/sanad-web/tests/unit/workspace-machines.test.ts new file mode 100644 index 000000000..bb2351049 --- /dev/null +++ b/control-plane/artifacts/sanad-web/tests/unit/workspace-machines.test.ts @@ -0,0 +1,15 @@ +import { describe, it, expect } from "vitest"; +import { machineHash } from "@/lib/compute/machines"; +import { sessionHash } from "@/lib/compute/tokens"; + +describe("machineHash", () => { + it("is 12 hex chars and stable", () => { + const h = machineHash("ws_1", "prod"); + expect(h).toMatch(/^[0-9a-f]{12}$/); + expect(machineHash("ws_1", "prod")).toBe(h); + }); + it("differs per env and never collides with user-session hashing", () => { + expect(machineHash("ws_1", "dev")).not.toBe(machineHash("ws_1", "prod")); + expect(machineHash("u1", "s1")).not.toBe(sessionHash("u1", "s1")); + }); +}); From 91ceb7061315d397162b5f8beb39ce287d452d98 Mon Sep 17 00:00:00 2001 From: Omar Alsoulah Date: Thu, 13 Aug 2026 17:46:37 +0300 Subject: [PATCH 08/28] =?UTF-8?q?sanad:=20workspace=20machines=20=E2=80=94?= =?UTF-8?q?=20boot-compatible=20task=20env,=20workspace=20identity=20in=20?= =?UTF-8?q?user=20slot?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sanad-web/lib/compute/machines.ts | 29 +++++++------- .../tests/unit/workspace-machines.test.ts | 39 ++++++++++++++++++- 2 files changed, 54 insertions(+), 14 deletions(-) diff --git a/control-plane/artifacts/sanad-web/lib/compute/machines.ts b/control-plane/artifacts/sanad-web/lib/compute/machines.ts index 2b3d855df..0f1fcc278 100644 --- a/control-plane/artifacts/sanad-web/lib/compute/machines.ts +++ b/control-plane/artifacts/sanad-web/lib/compute/machines.ts @@ -49,26 +49,29 @@ export function machineHash(workspaceId: string, env: string): string { } /** - * Base container env for a workspace machine. Deliberately NOT reused from - * sessions.ts's agentBaseEnv: that helper is shaped for the interactive CLI - * workspace (WORKSPACE_MODE: "task", SANAD_WORKSPACE_USER) and isn't a fit - * for a non-interactive worker machine identified by (workspaceId, env). - * Not part of the Task 4 interface contract — the worker container's actual - * expected env var names should be confirmed against the runtime that reads - * them (flagged in the task report as a divergence). + * Base container env for a workspace machine, registered once per + * (workspace, env) task-definition family. Deliberately does NOT set + * WORKSPACE_MODE: the terminal-server image's own Dockerfile default (ENV + * WORKSPACE_MODE=task) governs, and settings.py hard-fails on any value + * other than "railway"|"task" — worker behavior is gated by WORKER_ENABLED, + * not by inventing a new mode. settings.py requires SANAD_WORKSPACE_USER in + * task mode; workspaceId occupies that fixed-user slot (worker machines are + * workspace-scoped), matching deriveMachineToken(workspaceId, runNonce) + * putting workspaceId in the userId slot of the HMAC — the token the control + * plane derives matches the identity the machine holds. */ -function machineBaseEnv( +export function machineBaseEnv( config: AwsComputeConfig, workspaceId: string, - env: string, + keepWarm: boolean, ): Record { return { - WORKSPACE_MODE: "worker", - SANAD_WORKSPACE_ID: workspaceId, - SANAD_WORKSPACE_ENV: env, + SANAD_WORKSPACE_USER: workspaceId, CONTROL_PLANE_URL: config.controlPlaneUrl, SANAD_API_BASE_URL: config.controlPlaneUrl, TERMINAL_ALLOWED_ORIGINS: config.allowedOrigins, + WORKER_ENABLED: "1", + KEEP_WARM: keepWarm ? "1" : "0", }; } @@ -189,7 +192,7 @@ async function ensureInner( config, hash12, accessPointId, - machineBaseEnv(config, workspaceId, env), + machineBaseEnv(config, workspaceId, opts.keepWarm), ); const taskArn = await runWorkspaceTask(config, taskDefArn, { AGENTD_TOKEN: agentdToken, diff --git a/control-plane/artifacts/sanad-web/tests/unit/workspace-machines.test.ts b/control-plane/artifacts/sanad-web/tests/unit/workspace-machines.test.ts index bb2351049..b2aa225fe 100644 --- a/control-plane/artifacts/sanad-web/tests/unit/workspace-machines.test.ts +++ b/control-plane/artifacts/sanad-web/tests/unit/workspace-machines.test.ts @@ -1,6 +1,21 @@ import { describe, it, expect } from "vitest"; -import { machineHash } from "@/lib/compute/machines"; +import { machineBaseEnv, machineHash } from "@/lib/compute/machines"; import { sessionHash } from "@/lib/compute/tokens"; +import type { AwsComputeConfig } from "@/lib/compute/aws"; + +const fakeConfig: AwsComputeConfig = { + region: "eu-central-1", + cluster: "sanad-workspaces", + subnets: ["subnet-1"], + tasksSecurityGroup: "sg-1", + efsId: "fs-1", + workspaceImage: "acct.dkr.ecr.eu-central-1.amazonaws.com/sanad-workspace:latest", + executionRoleArn: "arn:aws:iam::1:role/exec", + taskRoleArn: "arn:aws:iam::1:role/task", + logGroup: "/sanad/workspaces", + controlPlaneUrl: "https://www.sanadcode.com", + allowedOrigins: "https://www.sanadcode.com", +}; describe("machineHash", () => { it("is 12 hex chars and stable", () => { @@ -13,3 +28,25 @@ describe("machineHash", () => { expect(machineHash("u1", "s1")).not.toBe(sessionHash("u1", "s1")); }); }); + +describe("machineBaseEnv", () => { + it("never sets WORKSPACE_MODE (image default 'task' governs boot)", () => { + const env = machineBaseEnv(fakeConfig, "ws_1", true); + expect(env).not.toHaveProperty("WORKSPACE_MODE"); + }); + + it("puts workspaceId in the fixed-user slot settings.py requires", () => { + const env = machineBaseEnv(fakeConfig, "ws_1", true); + expect(env.SANAD_WORKSPACE_USER).toBe("ws_1"); + }); + + it("carries WORKER_ENABLED and the correct KEEP_WARM for both keepWarm values", () => { + const warm = machineBaseEnv(fakeConfig, "ws_1", true); + expect(warm.WORKER_ENABLED).toBe("1"); + expect(warm.KEEP_WARM).toBe("1"); + + const cold = machineBaseEnv(fakeConfig, "ws_1", false); + expect(cold.WORKER_ENABLED).toBe("1"); + expect(cold.KEEP_WARM).toBe("0"); + }); +}); From 179172fa88cd24b72981aa36edf1fb585d114972 Mon Sep 17 00:00:00 2001 From: Omar Alsoulah Date: Thu, 13 Aug 2026 18:05:39 +0300 Subject: [PATCH 09/28] =?UTF-8?q?sanad:=20sync=20invoke=20route=20?= =?UTF-8?q?=E2=80=94=20gate,=20idempotent=20run=20rows,=20machine=20wake?= =?UTF-8?q?=20+=20ndjson=20passthrough?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../app/api/v1/agents/[name]/invoke/route.ts | 252 ++++++++++++++++++ .../sanad-web/lib/agents/registry.ts | 43 +++ .../artifacts/sanad-web/lib/runs/store.ts | 133 +++++++++ .../artifacts/sanad-web/package.json | 2 + .../sanad-web/tests/unit/invoke-route.test.ts | 17 ++ control-plane/pnpm-lock.yaml | 250 +++++++++++++++++ 6 files changed, 697 insertions(+) create mode 100644 control-plane/artifacts/sanad-web/app/api/v1/agents/[name]/invoke/route.ts create mode 100644 control-plane/artifacts/sanad-web/lib/runs/store.ts create mode 100644 control-plane/artifacts/sanad-web/tests/unit/invoke-route.test.ts diff --git a/control-plane/artifacts/sanad-web/app/api/v1/agents/[name]/invoke/route.ts b/control-plane/artifacts/sanad-web/app/api/v1/agents/[name]/invoke/route.ts new file mode 100644 index 000000000..50a5694b6 --- /dev/null +++ b/control-plane/artifacts/sanad-web/app/api/v1/agents/[name]/invoke/route.ts @@ -0,0 +1,252 @@ +import { NextRequest, NextResponse } from "next/server"; +import { ok, err } from "@/lib/http/envelope"; +import { verifyInvokeBearer } from "@/lib/tokens/invoke"; +import { + getAgentByName, + getLiveDeployment, + getVersionBundle, + getWorkspaceById, +} from "@/lib/agents/registry"; +import { assertWithinQuota, QuotaExceededError } from "@/lib/billing/quota"; +import { ensureWorkspaceMachine, type MachineTarget } from "@/lib/compute/machines"; +import { mintSession } from "@/lib/auth/session"; +import { + createRun, + getRun, + invokeGate, + markRunFailed, + markRunRunning, + presignTracePut, + type RunRow, +} from "@/lib/runs/store"; + +// How long the machine gets to come up from cold before the caller is told +// to retry instead of holding the connection open indefinitely. +const WAKE_DEADLINE_MS = 120_000; +// After the machine accepts the run and the NDJSON stream (?wait=1 path) +// ends, the worker reports completion out-of-band (Task 6/12) rather than +// in-band on the stream — so the run row may not have flipped out of +// "running" the instant the stream closes. Poll briefly for it before +// answering with whatever state exists. +const RUN_POLL_DEADLINE_MS = 10_000; +const RUN_POLL_INTERVAL_MS = 500; + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function machineWakingResponse(): NextResponse { + // err() has no header support and this is the one response that needs + // Retry-After, so build the envelope directly here (same shape as err()). + return NextResponse.json( + { + error: { + code: "machine_waking", + message: "workspace machine is starting — retry", + requestId: crypto.randomUUID(), + retryable: true, + }, + }, + { status: 503, headers: { "Retry-After": "30" } } + ); +} + +/** Race ensureWorkspaceMachine's cold-start wake against WAKE_DEADLINE_MS. */ +async function wakeMachine( + workspaceId: string, + env: string, + keepWarm: boolean +): Promise { + return Promise.race([ + ensureWorkspaceMachine(workspaceId, env, { keepWarm }), + new Promise<"timeout">((resolve) => setTimeout(() => resolve("timeout"), WAKE_DEADLINE_MS)), + ]); +} + +/** Poll a run row until it leaves "running", or RUN_POLL_DEADLINE_MS elapses. */ +async function pollRunSettled(runId: string): Promise { + const deadline = Date.now() + RUN_POLL_DEADLINE_MS; + let row = await getRun(runId); + while (row && row.status === "running" && Date.now() < deadline) { + await sleep(RUN_POLL_INTERVAL_MS); + row = await getRun(runId); + } + return row; +} + +export async function POST( + req: NextRequest, + { params }: { params: Promise<{ name: string }> } +) { + const info = await verifyInvokeBearer(req); + if (!info) { + return err(401, "unauthorized", "Invalid or expired invoke token"); + } + + // env comes from the token, not the caller — the query param, if present, + // is only validated against it, never used to select the deployment. + const qEnv = req.nextUrl.searchParams.get("env"); + if (qEnv && qEnv !== info.env) { + return err(400, "bad_env", "env query parameter does not match the token's env"); + } + + const { name } = await params; + const agent = await getAgentByName(info.orgId, name); + if (!agent) { + return err(404, "not_found", "No such agent"); + } + + const deployment = await getLiveDeployment(agent.id, info.env); + const gate = invokeGate({ + tokenAgentId: info.agentId, + pathAgentId: agent.id, + deployment: deployment ? { status: deployment.status } : null, + }); + // Priority: token scope (403) outranks quota; quota (402) outranks + // deployment existence/pause state — matches the route's documented + // 1/2/3 gate order even though both deployment-shaped checks share one + // invokeGate call. + if (!gate.ok && gate.code === "token_scope") { + return err(gate.status, gate.code, gate.message); + } + + try { + await assertWithinQuota(info.orgId); + } catch (e) { + if (e instanceof QuotaExceededError) { + return err( + 402, + "quota_exceeded", + `Monthly ${e.dimension} allowance exhausted — upgrade at sanadcode.com/pricing or wait for the next billing period` + ); + } + throw e; + } + + if (!gate.ok) { + return err(gate.status, gate.code, gate.message); + } + if (!deployment) { + // Unreachable: gate.ok === true already implies a non-null, non-paused + // deployment. Narrows the type for the rest of the handler. + return err(500, "internal_error", "invariant violated: gate passed with no deployment", true); + } + + const idempotencyKey = req.headers.get("idempotency-key") ?? undefined; + const { id: runId, existing } = await createRun({ + deploymentId: deployment.id, + agentVersionId: deployment.agentVersionId, + triggerPrincipal: `itok:${info.tokenId}`, + idempotencyKey, + }); + + if (existing) { + // Idempotent replay: the machine is never re-invoked, and the response + // shape is the same regardless of ?wait. + const row = await getRun(runId); + if (!row) { + return err(500, "internal_error", "run row missing on replay", true); + } + return ok({ runId: row.id, status: row.status, output: row.output }); + } + + const workspace = await getWorkspaceById(agent.workspaceId); + if (!workspace) { + await markRunFailed(runId, "internal_error"); + return err(500, "internal_error", "agent's workspace is missing", true); + } + + let target: MachineTarget; + try { + const woken = await wakeMachine(workspace.id, info.env, workspace.keepWarm); + if (woken === "timeout") { + await markRunFailed(runId, "wake_timeout"); + return machineWakingResponse(); + } + target = woken; + } catch (e) { + console.error(`invoke: failed to wake workspace machine for run ${runId}`, e); + await markRunFailed(runId, "machine_error"); + return err(502, "machine_error", "failed to reach the workspace machine", true); + } + + const bundle = await getVersionBundle(deployment.agentVersionId); + if (!bundle) { + await markRunFailed(runId, "internal_error"); + return err(500, "internal_error", "agent version bundle is missing", true); + } + + const input = await req.json().catch(() => ({})); + const budgets = { + maxTurnSeconds: deployment.maxTurnSeconds, + maxStepsPerTurn: deployment.maxStepsPerTurn, + maxTokensPerRun: deployment.maxTokensPerRun, + }; + const sessionToken = await mintSession(agent.ownerUserId, info.orgId, undefined, "worker-run", workspace.id); + const traceUploadUrl = await presignTracePut(runId); + + let machineRes: Response; + try { + machineRes = await fetch(`${target.baseUrl}/internal/worker/runs`, { + method: "POST", + headers: { + authorization: `Bearer ${target.agentdToken}`, + "content-type": "application/json", + }, + body: JSON.stringify({ runId, bundle, input, budgets, sessionToken, traceUploadUrl, sendId: runId }), + duplex: "half", + } as RequestInit & { duplex: "half" }); + } catch (e) { + console.error(`invoke: machine fetch failed for run ${runId}`, e); + await markRunFailed(runId, "machine_error"); + return err(502, "machine_error", "failed to reach the workspace machine", true); + } + + if (!machineRes.ok) { + const detail = await machineRes.text().catch(() => ""); + console.error(`invoke: machine rejected run ${runId} with status ${machineRes.status}`, detail); + await markRunFailed(runId, "machine_error"); + return err(502, "machine_error", "workspace machine rejected the run", true); + } + + await markRunRunning(runId); + + const wait = req.nextUrl.searchParams.get("wait") === "1"; + if (!wait) { + return new Response(machineRes.body, { + status: 200, + headers: { + "content-type": "application/x-ndjson", + "cache-control": "no-cache, no-transform", + "x-content-type-options": "nosniff", + }, + }); + } + + // Consume the NDJSON stream server-side. The final journal item (kind + // "end") signals the machine's turn is over, but it does NOT carry the + // run's output — the worker writes that to the run row out-of-band + // (Task 6/12). So after the stream ends this polls the row briefly + // (pollRunSettled) rather than trusting anything parsed from the stream. + const text = await machineRes.text(); + const lines = text.split("\n").map((l) => l.trim()).filter(Boolean); + let sawEnd = false; + for (const line of lines) { + try { + const item = JSON.parse(line); + if (item && typeof item === "object" && item.kind === "end") sawEnd = true; + } catch { + // Malformed line — the run row is the source of truth for ?wait=1, + // so this is not fatal. + } + } + if (!sawEnd) { + console.warn(`invoke: run ${runId} stream ended without a "end" journal item`); + } + + const row = await pollRunSettled(runId); + if (!row) { + return err(500, "internal_error", "run row disappeared after invoke", true); + } + return ok({ runId: row.id, status: row.status, output: row.output }); +} diff --git a/control-plane/artifacts/sanad-web/lib/agents/registry.ts b/control-plane/artifacts/sanad-web/lib/agents/registry.ts index 6897950d9..2962cd6ba 100644 --- a/control-plane/artifacts/sanad-web/lib/agents/registry.ts +++ b/control-plane/artifacts/sanad-web/lib/agents/registry.ts @@ -246,6 +246,49 @@ export async function getActiveDeployment(agentId: string, env: string) { return rows[0] ?? null; } +/** + * The live (non-superseded) deployment for an agent+env — active OR paused. + * Unlike getActiveDeployment (status:"active" only, by design — see its + * docstring), the invoke route needs to tell "never deployed" (404 + * not_deployed) apart from "deployed but paused" (409 paused), which + * requires seeing the paused row too. Same supersede invariant as + * setDeploymentStatus's lookup: at most one active/paused row per + * (agentId, env). + */ +export async function getLiveDeployment(agentId: string, env: string) { + const rows = await db + .select() + .from(deployments) + .where( + and( + eq(deployments.agentId, agentId), + eq(deployments.env, env), + inArray(deployments.status, ["active", "paused"]) + ) + ) + .limit(1); + return rows[0] ?? null; +} + +/** The bundle (file map) for a specific agent version, by id. */ +export async function getVersionBundle( + versionId: string +): Promise<{ files: Record } | null> { + const rows = await db + .select({ bundle: agentVersions.bundle }) + .from(agentVersions) + .where(eq(agentVersions.id, versionId)) + .limit(1); + const row = rows[0]; + return (row?.bundle as { files: Record } | undefined) ?? null; +} + +/** Fetch a workspace row by id — used to read keepWarm before waking its machine. */ +export async function getWorkspaceById(id: string) { + const rows = await db.select().from(workspaces).where(eq(workspaces.id, id)).limit(1); + return rows[0] ?? null; +} + /** List every agent in the org, across all its workspaces. */ export async function listAgentsForOrg(orgId: string) { return db diff --git a/control-plane/artifacts/sanad-web/lib/runs/store.ts b/control-plane/artifacts/sanad-web/lib/runs/store.ts new file mode 100644 index 000000000..a351fc99d --- /dev/null +++ b/control-plane/artifacts/sanad-web/lib/runs/store.ts @@ -0,0 +1,133 @@ +/** + * Runs: the store behind the sync invoke route + * (app/api/v1/agents/[name]/invoke/route.ts). Owns run-row lifecycle + * (create/idempotent-replay/status transitions) and the S3 presigned URLs the + * machine uses to upload/read a run's wire trace. + */ +import { randomBytes } from "crypto"; +import { and, eq } from "drizzle-orm"; +import { S3Client, PutObjectCommand, GetObjectCommand } from "@aws-sdk/client-s3"; +import { getSignedUrl } from "@aws-sdk/s3-request-presigner"; +import { db } from "../db"; +import { runs } from "../db/schema"; + +export type RunRow = typeof runs.$inferSelect; + +export type GateResult = + | { ok: true } + | { ok: false; status: number; code: string; message: string }; + +/** + * Pure decision core for the invoke route's access gate: is this token + * allowed to invoke this agent, and is there a live, unpaused deployment to + * run against? Extracted so the three failure branches (token scope, + * never-deployed, paused) are testable without touching Next or the DB. + */ +export function invokeGate(p: { + tokenAgentId: string; + pathAgentId: string; + deployment: { status: string } | null; +}): GateResult { + if (p.tokenAgentId !== p.pathAgentId) + return { ok: false, status: 403, code: "token_scope", message: "token is for another agent" }; + if (!p.deployment) + return { ok: false, status: 404, code: "not_deployed", message: "no active deployment for env" }; + if (p.deployment.status === "paused") + return { ok: false, status: 409, code: "paused", message: "deployment is paused" }; + return { ok: true }; +} + +export function newRunId(): string { + return "r_" + randomBytes(6).toString("hex"); +} + +/** + * Create a run row, or — if an Idempotency-Key was supplied and a run + * already exists for this (deploymentId, idempotencyKey) pair + * (runs_deployment_idem_uq) — return the existing one instead. A NULL + * idempotencyKey never conflicts (Postgres treats NULLs as distinct in a + * unique index), so unkeyed invokes always insert a fresh row. + */ +export async function createRun(p: { + deploymentId: string; + agentVersionId: string; + triggerPrincipal: string; + idempotencyKey?: string; +}): Promise<{ id: string; existing: boolean }> { + const id = newRunId(); + const idempotencyKey = p.idempotencyKey ?? null; + + const inserted = await db + .insert(runs) + .values({ + id, + deploymentId: p.deploymentId, + agentVersionId: p.agentVersionId, + triggerPrincipal: p.triggerPrincipal, + idempotencyKey, + }) + .onConflictDoNothing({ target: [runs.deploymentId, runs.idempotencyKey] }) + .returning({ id: runs.id }); + + if (inserted[0]) { + return { id: inserted[0].id, existing: false }; + } + + // Conflicted: idempotencyKey was non-null and already used for this + // deployment (see the NULL note above — this branch is unreachable + // otherwise). Re-select the row that won the race. + const existingRows = await db + .select() + .from(runs) + .where(and(eq(runs.deploymentId, p.deploymentId), eq(runs.idempotencyKey, idempotencyKey as string))) + .limit(1); + const existing = existingRows[0]; + if (!existing) { + throw new Error("createRun: insert conflicted but no existing row was found"); + } + return { id: existing.id, existing: true }; +} + +export async function getRun(id: string): Promise { + const rows = await db.select().from(runs).where(eq(runs.id, id)).limit(1); + return rows[0] ?? null; +} + +export async function markRunRunning(id: string): Promise { + await db + .update(runs) + .set({ status: "running", startedAt: new Date() }) + .where(eq(runs.id, id)); +} + +export async function markRunFailed(id: string, errorCode: string): Promise { + await db + .update(runs) + .set({ status: "failed", errorCode, finishedAt: new Date() }) + .where(eq(runs.id, id)); +} + +// -- trace presigner ---------------------------------------------------- +// Lazy client, like lib/compute/aws.ts's clients — railway mode never +// touches AWS, so nothing here should construct a client at import time. +let s3: S3Client | null = null; +const bucket = () => { + const b = process.env.SANAD_RUNS_BUCKET; + if (!b) throw new Error("SANAD_RUNS_BUCKET is not configured"); + return b; +}; +const client = () => (s3 ??= new S3Client({ region: process.env.AWS_REGION ?? "eu-central-1" })); + +export const traceKey = (runId: string) => `runs/${runId}/wire.jsonl.gz`; + +export function presignTracePut(runId: string): Promise { + return getSignedUrl(client(), new PutObjectCommand({ Bucket: bucket(), Key: traceKey(runId) }), { + expiresIn: 3600, + }); +} + +export function presignTraceGet(runId: string): Promise { + return getSignedUrl(client(), new GetObjectCommand({ Bucket: bucket(), Key: traceKey(runId) }), { + expiresIn: 300, + }); +} diff --git a/control-plane/artifacts/sanad-web/package.json b/control-plane/artifacts/sanad-web/package.json index 6cd46d804..819e96aea 100644 --- a/control-plane/artifacts/sanad-web/package.json +++ b/control-plane/artifacts/sanad-web/package.json @@ -19,6 +19,8 @@ "@aws-sdk/client-ecr": "^3.1104.0", "@aws-sdk/client-ecs": "^3.1103.0", "@aws-sdk/client-efs": "^3.1103.0", + "@aws-sdk/client-s3": "^3.1108.0", + "@aws-sdk/s3-request-presigner": "^3.1108.0", "@clerk/nextjs": "^6.21.0", "@dagrejs/dagre": "^1.1.8", "@xterm/addon-fit": "^0.11.0", diff --git a/control-plane/artifacts/sanad-web/tests/unit/invoke-route.test.ts b/control-plane/artifacts/sanad-web/tests/unit/invoke-route.test.ts new file mode 100644 index 000000000..0776f1851 --- /dev/null +++ b/control-plane/artifacts/sanad-web/tests/unit/invoke-route.test.ts @@ -0,0 +1,17 @@ +import { describe, it, expect } from "vitest"; +import { invokeGate, newRunId } from "@/lib/runs/store"; + +describe("invoke gate", () => { + const base = { tokenAgentId: "ag_1", pathAgentId: "ag_1", deployment: { status: "active" } }; + it("passes an active deployment", () => expect(invokeGate(base).ok).toBe(true)); + it("403s a cross-agent token", () => + expect(invokeGate({ ...base, tokenAgentId: "ag_2" })).toMatchObject({ status: 403, code: "token_scope" })); + it("404s when not deployed", () => + expect(invokeGate({ ...base, deployment: null })).toMatchObject({ status: 404, code: "not_deployed" })); + it("409s a paused deployment", () => + expect(invokeGate({ ...base, deployment: { status: "paused" } })).toMatchObject({ status: 409, code: "paused" })); +}); + +describe("run ids", () => { + it("are r_<12 hex>", () => expect(newRunId()).toMatch(/^r_[0-9a-f]{12}$/)); +}); diff --git a/control-plane/pnpm-lock.yaml b/control-plane/pnpm-lock.yaml index 45c6330c4..d61ad4c7d 100644 --- a/control-plane/pnpm-lock.yaml +++ b/control-plane/pnpm-lock.yaml @@ -218,6 +218,12 @@ importers: '@aws-sdk/client-efs': specifier: ^3.1103.0 version: 3.1103.0 + '@aws-sdk/client-s3': + specifier: ^3.1108.0 + version: 3.1108.0 + '@aws-sdk/s3-request-presigner': + specifier: ^3.1108.0 + version: 3.1108.0 '@clerk/nextjs': specifier: ^6.21.0 version: 6.39.6(next@15.2.9(react-dom@19.1.0(react@19.1.0))(react@19.1.0))(react-dom@19.1.0(react@19.1.0))(react@19.1.0) @@ -353,6 +359,10 @@ packages: resolution: {integrity: sha512-30iZtAPgz+LTIYoeivqYo853f02jBYSd5uGnGpkFV0M3xOt9aN73erkgYAmZU43x4VfqcnLxW9Kpg3R5LC4YYw==} engines: {node: '>=6.0.0'} + '@aws-sdk/checksums@3.1000.27': + resolution: {integrity: sha512-insWOqKKNUrbN/dohEG7BJ0U5GkyqhjbMb/NHNaLUtq+7my2M8C4EnZZZoxMmXRqCC+P9dEr+KyJA2JGGzoKLg==} + engines: {node: '>=20.0.0'} + '@aws-sdk/client-ecr@3.1104.0': resolution: {integrity: sha512-U3YRTQs3TTfS9ywLXeB8TwvdL7HnqMxD24hbTPlyKZe6iUa10pv08RAZUzkH0902Fy1O7bt2nobFvE9wlhSAOw==} engines: {node: '>=20.0.0'} @@ -365,62 +375,130 @@ packages: resolution: {integrity: sha512-Li3pawfnaKqw8UABfC9TWyL/DYOkzEwvUukVRE/tFCPHD0hv6F0FDnQvDRoQ2O+sXTXR3JBhwHqVAf4S6yyfzg==} engines: {node: '>=20.0.0'} + '@aws-sdk/client-s3@3.1108.0': + resolution: {integrity: sha512-prdothEAFE1G8H0s0+zFGuNZdSj+Acg/siB1dFxPS181op9+hJ1GLr+b2anJch4B9a/xbWy7k8eG/enH3cNSjQ==} + engines: {node: '>=20.0.0'} + '@aws-sdk/core@3.977.6': resolution: {integrity: sha512-QiaJV4/zDrB4ZY2mfeSXSzSTc36W16sZXcGz+SPFk0CJ26gziO0cS+4LjJUMAbdeeBOvS0k0Aq1cZpfGdUXxSw==} engines: {node: '>=20.0.0'} + '@aws-sdk/core@3.977.7': + resolution: {integrity: sha512-I88Iov89NVmjSmJLKSv7Cn9M2J+a2942OkA8nZCbz+sl4ZeY4zEOcoLOrbt1GRfQ8zEQKnjAJdXixA3J/p1fDQ==} + engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-env@3.972.67': resolution: {integrity: sha512-rcIpk5kxUqDaaNa6Xk23pQ6ViY7jlqzmfFWCahQcBT97ddXaXYYwzCen9Tz1Jvo6aJft6wDl5bN44/Jw5B4oLA==} engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-env@3.972.68': + resolution: {integrity: sha512-2a20A/IdNOwUvaDq91iqqS7BA0XlNMfW3iLGZGZLJv0EbUqhSxB0PIx4rQQqssvWj1uXImb3/UCCdHz/+1dOiA==} + engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-http@3.972.69': resolution: {integrity: sha512-nggwJtZ4eeNsUw5IeWBMXsi1ryct5idi0K+/SCRF3kybLubOMaNTb3XCihXpWMiVpyzyPeIrl0zTkzhBH9porA==} engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-http@3.972.70': + resolution: {integrity: sha512-0yRem2Fs52r/Nn6UAqIlpjexfaYj8ziEozOe9tamtAVT/5bzFLKx8O2r7MaRqgS3hGKHIa1Jij9nKHSsNnb04A==} + engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-ini@3.973.12': resolution: {integrity: sha512-pNEf/OeyN5X3VmLKlgSO6TqaWmW10CvI3TfwL1XhsuhYjSLT2VDaxFnCPHnOeQXSaFisMX4jNhpETriqN8DOmg==} engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-ini@3.973.13': + resolution: {integrity: sha512-2M39DE02XpYYaSWYk/4AsImXYUU/1L2xmTMLUpMMWq7DfLv191/vCRy3baKtdr45AkJQyVgSjmuVOLm15SwrRQ==} + engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-login@3.972.74': resolution: {integrity: sha512-0AQfDcf99TNmqVKv0owHrw/TQs6i4ZE5t9qmz6NvO53bE/sA/tpXhXL9AAcEP1qHc6Zzjd1UMb69+/9zdhvY3g==} engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-login@3.972.75': + resolution: {integrity: sha512-jaTESuJlQsoUZ44f/i2puyPt8VlF/dMMJ9HM3cStYtk7eKX4N9UWi83OLixUkoOJH3BwWlPLCq9YIK9nfWhVBg==} + engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-node@3.972.78': resolution: {integrity: sha512-OgPAnfvbGAMWac6yvxJ1ihslrvDpPVwR68D2csospdNCCyPvHk9JLzYKwz48SNiS1T2znDwHauywRKRFfpyYng==} engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-node@3.972.79': + resolution: {integrity: sha512-RIw5dof1EHkWubrZzPC941CDtnFG1iAXsxbFgLkhdYZXHc4icU13c/uxSMI0J5eUx9bxa7LjfpdjfClBB1QsDA==} + engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-process@3.972.67': resolution: {integrity: sha512-IlUEejorGTWKb4/Dm7K5Yw4QxUmXLThLhrvBmzVBqZFTbW72cv9LTcITmo1dsnYriALE4h68mOq4LB99x6sQ7Q==} engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-process@3.972.68': + resolution: {integrity: sha512-nLP3Pda2MQTFJ25hKBMmUuB9Uv+bTZQNlufbeCwklP549Vwnkd8bRLJoCKp5k6xjmdyptrPrOfGOhN0mKuca8A==} + engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-sso@3.973.11': resolution: {integrity: sha512-gAQBkBZxUB84d71+pPcI9L+jh2ujhuAVxc/4FgGiWFDjkPBlMKxzd5XDtkSXTFX8Ro7ansnT88+XadasxMeCRw==} engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-sso@3.973.12': + resolution: {integrity: sha512-EmgyyHn+f9WCcelp3L/vci+LGbX8GigWaVphRArjVo5Pktkr9YnLy/mQ6VDkDyBD72dtfRNTgHmD2ts4rTDXKQ==} + engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-web-identity@3.972.73': resolution: {integrity: sha512-SnlEmQa6SjOgs6iOPLUQl1Eyq4AKiAdPQlkOhFhqNfDtDCwibMGvL6QlkSmf3o6vAUSImzdPCxowT5dfQUZP1A==} engines: {node: '>=20.0.0'} + '@aws-sdk/credential-provider-web-identity@3.972.74': + resolution: {integrity: sha512-0YfczxGXF3RjGj8z7QG/Ho2HnLGKDHfPSHiTs47UU1U/+mmwISDN+rvGKt2zh+3FX8NdT4xd95LGBGyhQw2dgQ==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/middleware-sdk-s3@3.972.73': + resolution: {integrity: sha512-oy7sRA5HvHcAvkcKX6F8RI240jcOf3c8y/Gqjs9qemIibdKQqGBIi0uwa+47ZRYqGLpdEO28TQU4G73yUzo06Q==} + engines: {node: '>=20.0.0'} + '@aws-sdk/nested-clients@3.997.41': resolution: {integrity: sha512-RDHqPGQWlF6tatA/Tp3rg6oIwtgN9IVderxE+9av2Y93Dfyu+mO1hZ5Bu2jpfZg2rwdNbsssnwM+sLafIczMlQ==} engines: {node: '>=20.0.0'} + '@aws-sdk/nested-clients@3.997.42': + resolution: {integrity: sha512-XWRyon2MTHXD/zMoo0Mbge6Vwf+iE0qQaM/RyGO6NfZ9WukCFiQL27nQVZjYy2JwSIg+iXZxKOX95OBXqlSM4w==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/s3-request-presigner@3.1108.0': + resolution: {integrity: sha512-X0lX/rlyhlpQY97cwM2Rebuuj4HRMkp1wVYjJx1DHMTUI6Fehsh3/mZOd9U2HIbp1/nSciBmoD0dkpc3uXlUfA==} + engines: {node: '>=20.0.0'} + '@aws-sdk/signature-v4-multi-region@3.996.43': resolution: {integrity: sha512-lKekx8bLBXSv4O+cslk9Zfnw2XKSkWBs3uWL5QGhH2ZAQfNS7FE0vcSSN2vD/AhxX54ZTywWxR4STThoeOXlBA==} engines: {node: '>=20.0.0'} + '@aws-sdk/signature-v4-multi-region@3.996.44': + resolution: {integrity: sha512-ZSfQ35Qn4MhSY+A0Whyr+KBx+wJKZUyBsOrjB2pSHOafRzbFe47T8XcXM8hZqUAC69qnqIy0C9ArxTuud0CC2w==} + engines: {node: '>=20.0.0'} + '@aws-sdk/token-providers@3.1103.0': resolution: {integrity: sha512-N4wy26MNn31ItGVHYHPrEuCIFY4MBBjC+C5v1lJKqIUSA7OZBdhleCY53zCCrXn27hsk7YNOaTuhQu807S4AfQ==} engines: {node: '>=20.0.0'} + '@aws-sdk/token-providers@3.1108.0': + resolution: {integrity: sha512-rI80zxDxGJ6904eC/YbjkdjY6JdaZvQ01kOmrMvw7cFQGIHo27fhnIVbMSVDS4T6foQImjxYSRoOu/uSJscXDw==} + engines: {node: '>=20.0.0'} + '@aws-sdk/types@3.974.2': resolution: {integrity: sha512-3W6IUtSxFbH6X7Wb7DzGCV5QiFQsd0g8bOfntpmDxQlzBoKWUMBu/JPQR0DwkE+Hpnxd6db1tXbOwdeHddG6cA==} engines: {node: '>=20.0.0'} + '@aws-sdk/types@3.974.3': + resolution: {integrity: sha512-ECAqfpNsef+7MO8qtR0h9KcFIBAygaE7Cm6UOiQl+ft+uVap+1G7bNEjs4mdJE2OnA4m6k7i8peH8uGIAsOMGw==} + engines: {node: '>=20.0.0'} + '@aws-sdk/xml-builder@3.972.37': resolution: {integrity: sha512-zKq4HQum8JwDyEuyfuI4bbiAcU0KxP6qy+9PR/IsR92IyE/DaBAikzAS50tjxip4bqIIANpCcG+Yyj6CVhXupg==} engines: {node: '>=20.0.0'} + '@aws-sdk/xml-builder@3.972.38': + resolution: {integrity: sha512-grf7mzfVxBS5AlsuTvBN7uDpzqohFww9fRPCO+EBSUdvtsYMcPSKdz54h/7XiscqNcUM1Ae1MF7JLHmiYYuzbQ==} + engines: {node: '>=20.0.0'} + '@aws/lambda-invoke-store@0.3.0': resolution: {integrity: sha512-sl4Bm6yiMNYrZKkqqDFWN0UfnWhlS8ivKxrYl+6t0gCLrqr8y3B2IqZZbFRkfaVVp7C/baApyh71P+LeE1A2sQ==} engines: {node: '>=18.0.0'} @@ -2546,6 +2624,14 @@ snapshots: '@jridgewell/gen-mapping': 0.3.13 '@jridgewell/trace-mapping': 0.3.31 + '@aws-sdk/checksums@3.1000.27': + dependencies: + '@aws-sdk/core': 3.977.7 + '@aws-sdk/types': 3.974.3 + '@smithy/core': 3.31.1 + '@smithy/types': 4.16.1 + tslib: 2.8.1 + '@aws-sdk/client-ecr@3.1104.0': dependencies: '@aws-sdk/core': 3.977.6 @@ -2579,6 +2665,20 @@ snapshots: '@smithy/types': 4.16.1 tslib: 2.8.1 + '@aws-sdk/client-s3@3.1108.0': + dependencies: + '@aws-sdk/checksums': 3.1000.27 + '@aws-sdk/core': 3.977.7 + '@aws-sdk/credential-provider-node': 3.972.79 + '@aws-sdk/middleware-sdk-s3': 3.972.73 + '@aws-sdk/signature-v4-multi-region': 3.996.44 + '@aws-sdk/types': 3.974.3 + '@smithy/core': 3.31.1 + '@smithy/fetch-http-handler': 5.6.13 + '@smithy/node-http-handler': 4.9.13 + '@smithy/types': 4.16.1 + tslib: 2.8.1 + '@aws-sdk/core@3.977.6': dependencies: '@aws-sdk/types': 3.974.2 @@ -2590,6 +2690,17 @@ snapshots: bowser: 2.14.1 tslib: 2.8.1 + '@aws-sdk/core@3.977.7': + dependencies: + '@aws-sdk/types': 3.974.3 + '@aws-sdk/xml-builder': 3.972.38 + '@aws/lambda-invoke-store': 0.3.0 + '@smithy/core': 3.31.1 + '@smithy/signature-v4': 5.6.12 + '@smithy/types': 4.16.1 + bowser: 2.14.1 + tslib: 2.8.1 + '@aws-sdk/credential-provider-env@3.972.67': dependencies: '@aws-sdk/core': 3.977.6 @@ -2598,6 +2709,14 @@ snapshots: '@smithy/types': 4.16.1 tslib: 2.8.1 + '@aws-sdk/credential-provider-env@3.972.68': + dependencies: + '@aws-sdk/core': 3.977.7 + '@aws-sdk/types': 3.974.3 + '@smithy/core': 3.31.1 + '@smithy/types': 4.16.1 + tslib: 2.8.1 + '@aws-sdk/credential-provider-http@3.972.69': dependencies: '@aws-sdk/core': 3.977.6 @@ -2608,6 +2727,16 @@ snapshots: '@smithy/types': 4.16.1 tslib: 2.8.1 + '@aws-sdk/credential-provider-http@3.972.70': + dependencies: + '@aws-sdk/core': 3.977.7 + '@aws-sdk/types': 3.974.3 + '@smithy/core': 3.31.1 + '@smithy/fetch-http-handler': 5.6.13 + '@smithy/node-http-handler': 4.9.13 + '@smithy/types': 4.16.1 + tslib: 2.8.1 + '@aws-sdk/credential-provider-ini@3.973.12': dependencies: '@aws-sdk/core': 3.977.6 @@ -2624,6 +2753,22 @@ snapshots: '@smithy/types': 4.16.1 tslib: 2.8.1 + '@aws-sdk/credential-provider-ini@3.973.13': + dependencies: + '@aws-sdk/core': 3.977.7 + '@aws-sdk/credential-provider-env': 3.972.68 + '@aws-sdk/credential-provider-http': 3.972.70 + '@aws-sdk/credential-provider-login': 3.972.75 + '@aws-sdk/credential-provider-process': 3.972.68 + '@aws-sdk/credential-provider-sso': 3.973.12 + '@aws-sdk/credential-provider-web-identity': 3.972.74 + '@aws-sdk/nested-clients': 3.997.42 + '@aws-sdk/types': 3.974.3 + '@smithy/core': 3.31.1 + '@smithy/credential-provider-imds': 4.4.16 + '@smithy/types': 4.16.1 + tslib: 2.8.1 + '@aws-sdk/credential-provider-login@3.972.74': dependencies: '@aws-sdk/core': 3.977.6 @@ -2633,6 +2778,15 @@ snapshots: '@smithy/types': 4.16.1 tslib: 2.8.1 + '@aws-sdk/credential-provider-login@3.972.75': + dependencies: + '@aws-sdk/core': 3.977.7 + '@aws-sdk/nested-clients': 3.997.42 + '@aws-sdk/types': 3.974.3 + '@smithy/core': 3.31.1 + '@smithy/types': 4.16.1 + tslib: 2.8.1 + '@aws-sdk/credential-provider-node@3.972.78': dependencies: '@aws-sdk/credential-provider-env': 3.972.67 @@ -2647,6 +2801,20 @@ snapshots: '@smithy/types': 4.16.1 tslib: 2.8.1 + '@aws-sdk/credential-provider-node@3.972.79': + dependencies: + '@aws-sdk/credential-provider-env': 3.972.68 + '@aws-sdk/credential-provider-http': 3.972.70 + '@aws-sdk/credential-provider-ini': 3.973.13 + '@aws-sdk/credential-provider-process': 3.972.68 + '@aws-sdk/credential-provider-sso': 3.973.12 + '@aws-sdk/credential-provider-web-identity': 3.972.74 + '@aws-sdk/types': 3.974.3 + '@smithy/core': 3.31.1 + '@smithy/credential-provider-imds': 4.4.16 + '@smithy/types': 4.16.1 + tslib: 2.8.1 + '@aws-sdk/credential-provider-process@3.972.67': dependencies: '@aws-sdk/core': 3.977.6 @@ -2655,6 +2823,14 @@ snapshots: '@smithy/types': 4.16.1 tslib: 2.8.1 + '@aws-sdk/credential-provider-process@3.972.68': + dependencies: + '@aws-sdk/core': 3.977.7 + '@aws-sdk/types': 3.974.3 + '@smithy/core': 3.31.1 + '@smithy/types': 4.16.1 + tslib: 2.8.1 + '@aws-sdk/credential-provider-sso@3.973.11': dependencies: '@aws-sdk/core': 3.977.6 @@ -2665,6 +2841,16 @@ snapshots: '@smithy/types': 4.16.1 tslib: 2.8.1 + '@aws-sdk/credential-provider-sso@3.973.12': + dependencies: + '@aws-sdk/core': 3.977.7 + '@aws-sdk/nested-clients': 3.997.42 + '@aws-sdk/token-providers': 3.1108.0 + '@aws-sdk/types': 3.974.3 + '@smithy/core': 3.31.1 + '@smithy/types': 4.16.1 + tslib: 2.8.1 + '@aws-sdk/credential-provider-web-identity@3.972.73': dependencies: '@aws-sdk/core': 3.977.6 @@ -2674,6 +2860,24 @@ snapshots: '@smithy/types': 4.16.1 tslib: 2.8.1 + '@aws-sdk/credential-provider-web-identity@3.972.74': + dependencies: + '@aws-sdk/core': 3.977.7 + '@aws-sdk/nested-clients': 3.997.42 + '@aws-sdk/types': 3.974.3 + '@smithy/core': 3.31.1 + '@smithy/types': 4.16.1 + tslib: 2.8.1 + + '@aws-sdk/middleware-sdk-s3@3.972.73': + dependencies: + '@aws-sdk/core': 3.977.7 + '@aws-sdk/signature-v4-multi-region': 3.996.44 + '@aws-sdk/types': 3.974.3 + '@smithy/core': 3.31.1 + '@smithy/types': 4.16.1 + tslib: 2.8.1 + '@aws-sdk/nested-clients@3.997.41': dependencies: '@aws-sdk/core': 3.977.6 @@ -2685,6 +2889,26 @@ snapshots: '@smithy/types': 4.16.1 tslib: 2.8.1 + '@aws-sdk/nested-clients@3.997.42': + dependencies: + '@aws-sdk/core': 3.977.7 + '@aws-sdk/signature-v4-multi-region': 3.996.44 + '@aws-sdk/types': 3.974.3 + '@smithy/core': 3.31.1 + '@smithy/fetch-http-handler': 5.6.13 + '@smithy/node-http-handler': 4.9.13 + '@smithy/types': 4.16.1 + tslib: 2.8.1 + + '@aws-sdk/s3-request-presigner@3.1108.0': + dependencies: + '@aws-sdk/core': 3.977.7 + '@aws-sdk/signature-v4-multi-region': 3.996.44 + '@aws-sdk/types': 3.974.3 + '@smithy/core': 3.31.1 + '@smithy/types': 4.16.1 + tslib: 2.8.1 + '@aws-sdk/signature-v4-multi-region@3.996.43': dependencies: '@aws-sdk/types': 3.974.2 @@ -2692,6 +2916,13 @@ snapshots: '@smithy/types': 4.16.1 tslib: 2.8.1 + '@aws-sdk/signature-v4-multi-region@3.996.44': + dependencies: + '@aws-sdk/types': 3.974.3 + '@smithy/signature-v4': 5.6.12 + '@smithy/types': 4.16.1 + tslib: 2.8.1 + '@aws-sdk/token-providers@3.1103.0': dependencies: '@aws-sdk/core': 3.977.6 @@ -2701,16 +2932,35 @@ snapshots: '@smithy/types': 4.16.1 tslib: 2.8.1 + '@aws-sdk/token-providers@3.1108.0': + dependencies: + '@aws-sdk/core': 3.977.7 + '@aws-sdk/nested-clients': 3.997.42 + '@aws-sdk/types': 3.974.3 + '@smithy/core': 3.31.1 + '@smithy/types': 4.16.1 + tslib: 2.8.1 + '@aws-sdk/types@3.974.2': dependencies: '@smithy/types': 4.16.1 tslib: 2.8.1 + '@aws-sdk/types@3.974.3': + dependencies: + '@smithy/types': 4.16.1 + tslib: 2.8.1 + '@aws-sdk/xml-builder@3.972.37': dependencies: '@smithy/types': 4.16.1 tslib: 2.8.1 + '@aws-sdk/xml-builder@3.972.38': + dependencies: + '@smithy/types': 4.16.1 + tslib: 2.8.1 + '@aws/lambda-invoke-store@0.3.0': {} '@babel/helper-string-parser@7.29.7': {} From a86a93a846c1bac2770140aafb8de154c1082ae3 Mon Sep 17 00:00:00 2001 From: Omar Alsoulah Date: Thu, 13 Aug 2026 18:12:14 +0300 Subject: [PATCH 10/28] =?UTF-8?q?sanad:=20invoke=20route=20=E2=80=94=20det?= =?UTF-8?q?erministic=20live-deployment=20selection?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- control-plane/artifacts/sanad-web/lib/agents/registry.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/control-plane/artifacts/sanad-web/lib/agents/registry.ts b/control-plane/artifacts/sanad-web/lib/agents/registry.ts index 2962cd6ba..1d34af819 100644 --- a/control-plane/artifacts/sanad-web/lib/agents/registry.ts +++ b/control-plane/artifacts/sanad-web/lib/agents/registry.ts @@ -266,6 +266,7 @@ export async function getLiveDeployment(agentId: string, env: string) { inArray(deployments.status, ["active", "paused"]) ) ) + .orderBy(desc(deployments.createdAt)) .limit(1); return rows[0] ?? null; } From 32fc6f5a983fedc912d32a4c971b94d5d78fdffc Mon Sep 17 00:00:00 2001 From: Omar Alsoulah Date: Thu, 13 Aug 2026 18:27:14 +0300 Subject: [PATCH 11/28] sanad: run completion + pricing + read apis + lost-run reaper --- .../app/api/internal/cron/reap-runs/route.ts | 35 +++++ .../app/api/v1/runs/[id]/complete/route.ts | 82 +++++++++++ .../sanad-web/app/api/v1/runs/[id]/route.ts | 27 ++++ .../app/api/v1/runs/[id]/trace/route.ts | 34 +++++ .../sanad-web/app/api/v1/runs/route.ts | 45 ++++++ .../sanad-web/lib/agents/registry.ts | 18 +++ .../sanad-web/lib/compute/machines.ts | 19 ++- .../artifacts/sanad-web/lib/models/catalog.ts | 8 ++ .../artifacts/sanad-web/lib/runs/reaper.ts | 86 +++++++++++ .../artifacts/sanad-web/lib/runs/store.ts | 133 +++++++++++++++++- .../sanad-web/tests/unit/run-cost.test.ts | 16 +++ .../sanad-web/tests/unit/run-reaper.test.ts | 27 ++++ 12 files changed, 527 insertions(+), 3 deletions(-) create mode 100644 control-plane/artifacts/sanad-web/app/api/internal/cron/reap-runs/route.ts create mode 100644 control-plane/artifacts/sanad-web/app/api/v1/runs/[id]/complete/route.ts create mode 100644 control-plane/artifacts/sanad-web/app/api/v1/runs/[id]/route.ts create mode 100644 control-plane/artifacts/sanad-web/app/api/v1/runs/[id]/trace/route.ts create mode 100644 control-plane/artifacts/sanad-web/app/api/v1/runs/route.ts create mode 100644 control-plane/artifacts/sanad-web/lib/runs/reaper.ts create mode 100644 control-plane/artifacts/sanad-web/tests/unit/run-cost.test.ts create mode 100644 control-plane/artifacts/sanad-web/tests/unit/run-reaper.test.ts diff --git a/control-plane/artifacts/sanad-web/app/api/internal/cron/reap-runs/route.ts b/control-plane/artifacts/sanad-web/app/api/internal/cron/reap-runs/route.ts new file mode 100644 index 000000000..fa8f60b36 --- /dev/null +++ b/control-plane/artifacts/sanad-web/app/api/internal/cron/reap-runs/route.ts @@ -0,0 +1,35 @@ +import { timingSafeEqual } from "crypto"; +import { NextRequest } from "next/server"; +import { ok, err } from "@/lib/http/envelope"; +import { DEFAULT_STALE_MS, sweepLostRuns } from "@/lib/runs/reaper"; + +// Floor for staleAfterMs — a caller-supplied 0 or negative value must not +// reap every currently-running run instantly. +const MIN_STALE_MS = 60_000; + +function secretMatches(header: string | null): boolean { + const secret = process.env.CRON_SECRET; + if (!secret || !header) return false; // unset CRON_SECRET => always 401, fail closed + const a = Buffer.from(header); + const b = Buffer.from(secret); + return a.length === b.length && timingSafeEqual(a, b); +} + +/** + * Cron entrypoint for the lost-run reaper — not session- or machine-authed, + * just a shared secret the scheduler holds (same shape as + * ROUTER_SHARED_SECRET's x-router-secret check in + * app/api/v1/compute/route/route.ts). + */ +export async function POST(req: NextRequest) { + if (!secretMatches(req.headers.get("x-cron-secret"))) { + return err(401, "unauthorized", "Invalid cron credential"); + } + + const raw = (await req.json().catch(() => ({}))) as { staleAfterMs?: unknown }; + const requested = typeof raw.staleAfterMs === "number" ? raw.staleAfterMs : DEFAULT_STALE_MS; + const staleAfterMs = Math.max(requested, MIN_STALE_MS); + + const reaped = await sweepLostRuns(staleAfterMs); + return ok({ reaped }); +} diff --git a/control-plane/artifacts/sanad-web/app/api/v1/runs/[id]/complete/route.ts b/control-plane/artifacts/sanad-web/app/api/v1/runs/[id]/complete/route.ts new file mode 100644 index 000000000..3598a2d22 --- /dev/null +++ b/control-plane/artifacts/sanad-web/app/api/v1/runs/[id]/complete/route.ts @@ -0,0 +1,82 @@ +import { NextRequest } from "next/server"; +import { z } from "zod"; +import { ok, err } from "@/lib/http/envelope"; +import { completeRun, getRun } from "@/lib/runs/store"; +import { getAgentById, getDeploymentById, getWorkspaceById } from "@/lib/agents/registry"; +import { getMachineByWorkspaceEnv } from "@/lib/compute/machines"; +import { machineTokenMatches } from "@/lib/compute/tokens"; + +const Body = z.object({ + status: z.enum(["succeeded", "failed", "cancelled"]), + errorCode: z.string().min(1).max(128).optional(), + output: z.unknown().optional(), + tokensIn: z.number().int().min(0), + tokensOut: z.number().int().min(0), + modelAlias: z.string().min(1).max(128).optional(), + traceUploaded: z.boolean(), +}); + +/** + * Completion ingest — Bearer-authed by the run's own workspace machine, not + * a user session. The machine holds no signed token from us; instead we + * walk run -> deployment -> agent -> workspace -> workspaceMachines to find + * the machine's runNonce, recompute deriveMachineToken(workspaceId, + * runNonce) (machineTokenMatches — same HMAC-then-timingSafeEqual pattern + * as app/api/v1/compute/route/route.ts:8-14), and compare. + * + * Every resolution failure along that chain — run not found, deployment or + * agent missing (shouldn't happen given FK integrity, but not trusted), + * no workspaceMachines row for this (workspace, env), or a row with no + * runNonce yet — collapses to the same 401. There is no way to distinguish + * "which link is missing" in the response without also telling a caller + * holding a stale or foreign token something about run/agent existence, so + * this fails closed uniformly. + * + * Idempotent by construction: completeRun's UPDATE only matches rows still + * in "queued"/"running", so a retried completion (or one racing the reaper) + * is a no-op — this always answers 200 with whatever the run's status ends + * up being, not necessarily the status the caller just posted. + */ +export async function POST( + req: NextRequest, + { params }: { params: Promise<{ id: string }> } +) { + const header = req.headers.get("authorization"); + const presented = header?.startsWith("Bearer ") ? header.slice(7).trim() : ""; + if (!presented) { + return err(401, "unauthorized", "Missing machine bearer token"); + } + + const { id: runId } = await params; + const run = await getRun(runId); + const deployment = run ? await getDeploymentById(run.deploymentId) : null; + const agent = deployment ? await getAgentById(deployment.agentId) : null; + const workspace = agent ? await getWorkspaceById(agent.workspaceId) : null; + const machine = + workspace && deployment + ? await getMachineByWorkspaceEnv(workspace.id, deployment.env) + : null; + + if (!run || !deployment || !agent || !workspace || !machine || !machine.runNonce) { + return err(401, "unauthorized", "Invalid machine credential"); + } + if (!machineTokenMatches(presented, workspace.id, machine.runNonce)) { + return err(401, "unauthorized", "Invalid machine credential"); + } + + let raw: unknown; + try { + raw = await req.json(); + } catch { + return err(400, "invalid_request", "Request body must be JSON"); + } + const parsed = Body.safeParse(raw); + if (!parsed.success) { + return err(400, "invalid_request", parsed.error.issues[0]?.message ?? "Invalid completion payload"); + } + + await completeRun(runId, parsed.data); + + const finalRow = await getRun(runId); + return ok({ runId, status: finalRow?.status ?? run.status }); +} diff --git a/control-plane/artifacts/sanad-web/app/api/v1/runs/[id]/route.ts b/control-plane/artifacts/sanad-web/app/api/v1/runs/[id]/route.ts new file mode 100644 index 000000000..a5cd840b9 --- /dev/null +++ b/control-plane/artifacts/sanad-web/app/api/v1/runs/[id]/route.ts @@ -0,0 +1,27 @@ +import { NextRequest } from "next/server"; +import { ok, err } from "@/lib/http/envelope"; +import { verifyBearer } from "@/lib/auth/session"; +import { getRunForOrg, serializeRun } from "@/lib/runs/store"; + +/** + * A single run — session-authed, org-scoped. A run id from another org is + * indistinguishable from one that doesn't exist at all (both 404 + * `not_found`) — same information-hiding rule as agent name resolution. + */ +export async function GET( + req: NextRequest, + { params }: { params: Promise<{ id: string }> } +) { + const session = await verifyBearer(req); + if (!session) { + return err(401, "unauthorized", "Invalid or revoked session token"); + } + + const { id } = await params; + const row = await getRunForOrg(id, session.orgId); + if (!row) { + return err(404, "not_found", "No such run"); + } + + return ok({ run: serializeRun(row) }); +} diff --git a/control-plane/artifacts/sanad-web/app/api/v1/runs/[id]/trace/route.ts b/control-plane/artifacts/sanad-web/app/api/v1/runs/[id]/trace/route.ts new file mode 100644 index 000000000..a6badb704 --- /dev/null +++ b/control-plane/artifacts/sanad-web/app/api/v1/runs/[id]/trace/route.ts @@ -0,0 +1,34 @@ +import { NextRequest, NextResponse } from "next/server"; +import { err } from "@/lib/http/envelope"; +import { verifyBearer } from "@/lib/auth/session"; +import { getRunForOrg, presignTraceGet } from "@/lib/runs/store"; + +/** + * Trace download — session-authed, org-scoped (same rule as the run read + * routes: a foreign run id 404s exactly like an unknown one). Redirects to + * a short-lived (300s, presignTraceGet) S3 GET URL rather than proxying the + * object through this server. `trace_unavailable` covers both "run hasn't + * uploaded one yet" and "run never will" (failed before upload) — the + * client can't act on the distinction, so it isn't surfaced. + */ +export async function GET( + req: NextRequest, + { params }: { params: Promise<{ id: string }> } +) { + const session = await verifyBearer(req); + if (!session) { + return err(401, "unauthorized", "Invalid or revoked session token"); + } + + const { id } = await params; + const row = await getRunForOrg(id, session.orgId); + if (!row) { + return err(404, "not_found", "No such run"); + } + if (!row.traceUploaded) { + return err(404, "trace_unavailable", "This run has no uploaded trace"); + } + + const url = await presignTraceGet(id); + return NextResponse.redirect(url, 307); +} diff --git a/control-plane/artifacts/sanad-web/app/api/v1/runs/route.ts b/control-plane/artifacts/sanad-web/app/api/v1/runs/route.ts new file mode 100644 index 000000000..c169b8daa --- /dev/null +++ b/control-plane/artifacts/sanad-web/app/api/v1/runs/route.ts @@ -0,0 +1,45 @@ +import { NextRequest } from "next/server"; +import { ok, err } from "@/lib/http/envelope"; +import { verifyBearer } from "@/lib/auth/session"; +import { getAgentByName } from "@/lib/agents/registry"; +import { listRuns, serializeRun } from "@/lib/runs/store"; + +const DEFAULT_LIMIT = 20; +const MAX_LIMIT = 100; + +/** Clamp to [1, MAX_LIMIT]; anything unparsable falls back to the default. */ +function parseLimit(raw: string | null): number { + if (!raw) return DEFAULT_LIMIT; + const n = Number.parseInt(raw, 10); + if (!Number.isFinite(n) || n <= 0) return DEFAULT_LIMIT; + return Math.min(n, MAX_LIMIT); +} + +/** + * Runs list — session-authed, org-scoped (see lib/runs/store.ts's listRuns). + * `agent` filters by name, resolved to an id within the caller's own org + * first: an agent name that doesn't exist (here, or at all) yields an empty + * list rather than an error — same non-leaking shape either way. + */ +export async function GET(req: NextRequest) { + const session = await verifyBearer(req); + if (!session) { + return err(401, "unauthorized", "Invalid or revoked session token"); + } + + const sp = req.nextUrl.searchParams; + const agentName = sp.get("agent"); + const env = sp.get("env") ?? undefined; + const status = sp.get("status") ?? undefined; + const limit = parseLimit(sp.get("limit")); + + let agentId: string | undefined; + if (agentName) { + const agent = await getAgentByName(session.orgId, agentName); + if (!agent) return ok({ runs: [] }); + agentId = agent.id; + } + + const rows = await listRuns({ orgId: session.orgId, agentId, env, status, limit }); + return ok({ runs: rows.map(serializeRun) }); +} diff --git a/control-plane/artifacts/sanad-web/lib/agents/registry.ts b/control-plane/artifacts/sanad-web/lib/agents/registry.ts index 1d34af819..17825e989 100644 --- a/control-plane/artifacts/sanad-web/lib/agents/registry.ts +++ b/control-plane/artifacts/sanad-web/lib/agents/registry.ts @@ -290,6 +290,24 @@ export async function getWorkspaceById(id: string) { return rows[0] ?? null; } +/** + * Fetch an agent row by id. Unlike getAgentByName, not org-scoped — used by + * the run-completion route's machine-auth path, which resolves + * run.deploymentId -> deployment.agentId -> agent.workspaceId to recompute + * the expected agentd token; the caller there is a machine bearer token, + * not a session, so there is no orgId to scope by yet. + */ +export async function getAgentById(id: string) { + const rows = await db.select().from(agents).where(eq(agents.id, id)).limit(1); + return rows[0] ?? null; +} + +/** Fetch a deployment row by id — same machine-auth path as getAgentById above. */ +export async function getDeploymentById(id: string) { + const rows = await db.select().from(deployments).where(eq(deployments.id, id)).limit(1); + return rows[0] ?? null; +} + /** List every agent in the org, across all its workspaces. */ export async function listAgentsForOrg(orgId: string) { return db diff --git a/control-plane/artifacts/sanad-web/lib/compute/machines.ts b/control-plane/artifacts/sanad-web/lib/compute/machines.ts index 0f1fcc278..b657aa73b 100644 --- a/control-plane/artifacts/sanad-web/lib/compute/machines.ts +++ b/control-plane/artifacts/sanad-web/lib/compute/machines.ts @@ -12,7 +12,7 @@ * exported from there for this reuse. */ import { createHash } from "crypto"; -import { eq } from "drizzle-orm"; +import { and, eq } from "drizzle-orm"; import { db } from "../db"; import { workspaceMachines } from "../db/schema"; import { @@ -235,6 +235,23 @@ async function ensureInner( }; } +/** + * Fetch a workspace machine row by (workspaceId, env) — the run-completion + * route's machine-auth path needs this to recompute the expected agentd + * token (deriveMachineToken(workspaceId, machine.runNonce)) for the run's + * deployment env. A missing row (or a row with no runNonce yet — never + * finished its first RunTask) both mean "no credential to check against", + * handled by the caller as a 401. + */ +export async function getMachineByWorkspaceEnv(workspaceId: string, env: string): Promise { + const [row] = await db + .select() + .from(workspaceMachines) + .where(and(eq(workspaceMachines.workspaceId, workspaceId), eq(workspaceMachines.env, env))) + .limit(1); + return row ?? null; +} + /** Router route lookup: hash12 → task IP, for workspace machines. */ export async function machineIpByHash(hash12: string): Promise { const [row] = await db diff --git a/control-plane/artifacts/sanad-web/lib/models/catalog.ts b/control-plane/artifacts/sanad-web/lib/models/catalog.ts index 14407d31f..8557e731e 100644 --- a/control-plane/artifacts/sanad-web/lib/models/catalog.ts +++ b/control-plane/artifacts/sanad-web/lib/models/catalog.ts @@ -7,3 +7,11 @@ export const MODEL_CATALOG = [ ] as const; export const DEFAULT_MODEL_ALIAS = "kimi-k3"; + +// Placeholder pricing (USD per million tokens) — flagged for Omar's sign-off +// before GA. An alias missing from this map is NOT an error: costUsdMicros +// (lib/runs/store.ts) treats it as free (cost 0) rather than throwing, so an +// unpriced/experimental model never blocks a run from completing. +export const MODEL_PRICING: Record = { + "kimi-k3": { inUsdPerMTok: 0.6, outUsdPerMTok: 2.5 }, +}; diff --git a/control-plane/artifacts/sanad-web/lib/runs/reaper.ts b/control-plane/artifacts/sanad-web/lib/runs/reaper.ts new file mode 100644 index 000000000..1eb1c37b7 --- /dev/null +++ b/control-plane/artifacts/sanad-web/lib/runs/reaper.ts @@ -0,0 +1,86 @@ +/** + * Lost-run reaper: runs stuck in "running" whose backing workspace machine + * has gone silent (or never registered a machine row for that + * workspace+env at all) — almost always a Fargate task that died mid-turn + * without ever calling POST .../runs/{id}/complete. Left alone, such a run + * would sit in "running" forever, and Task 5's ?wait=1 poll + * (pollRunSettled) would spin its full 10s budget on every retry with no + * way to ever observe a terminal state. sweepLostRuns marks each one + * `lost` (errorCode "machine_lost") so callers stop waiting. + * + * The ownership chain is runs -> deployments -> agents -> workspaceMachines + * (four tables), resolved here as three narrow, independently-typed queries + * joined in memory rather than one wide SQL join: (1) running runs, (2) + * deployments+agents for those runs' deploymentIds (one leftJoin), (3) + * workspaceMachines for the workspaceIds in play. Keeps every query small + * and keeps the "no machine row at all" case (the other half of the + * contract, beyond staleness) a plain absent-from-the-map lookup instead of + * a NULL-across-an-outer-join special case. + */ +import { eq, inArray } from "drizzle-orm"; +import { db } from "../db"; +import { agents, deployments, runs, workspaceMachines } from "../db/schema"; + +export const DEFAULT_STALE_MS = 300_000; + +export async function sweepLostRuns(staleAfterMs: number): Promise { + const cutoffMs = Date.now() - staleAfterMs; + + const runningRows = await db + .select({ id: runs.id, deploymentId: runs.deploymentId }) + .from(runs) + .where(eq(runs.status, "running")); + if (runningRows.length === 0) return 0; + + const deploymentIds = [...new Set(runningRows.map((r) => r.deploymentId))]; + const deploymentRows = await db + .select({ + deploymentId: deployments.id, + env: deployments.env, + workspaceId: agents.workspaceId, + }) + .from(deployments) + .leftJoin(agents, eq(agents.id, deployments.agentId)) + .where(inArray(deployments.id, deploymentIds)); + const byDeploymentId = new Map(deploymentRows.map((d) => [d.deploymentId, d])); + + const workspaceIds = [ + ...new Set( + deploymentRows + .map((d) => d.workspaceId) + .filter((id): id is string => !!id) + ), + ]; + const machineRows = workspaceIds.length + ? await db + .select({ + workspaceId: workspaceMachines.workspaceId, + env: workspaceMachines.env, + lastSeenAt: workspaceMachines.lastSeenAt, + }) + .from(workspaceMachines) + .where(inArray(workspaceMachines.workspaceId, workspaceIds)) + : []; + const lastSeenByKey = new Map( + machineRows.map((m) => [`${m.workspaceId}:${m.env}`, m.lastSeenAt]) + ); + + let reaped = 0; + for (const row of runningRows) { + const dep = byDeploymentId.get(row.deploymentId); + const lastSeenAt = dep ? lastSeenByKey.get(`${dep.workspaceId}:${dep.env}`) : undefined; + // Stale (or no machine row at all — the `!lastSeenAt` branch covers both + // "no matching workspaceMachines row" and "row exists but lastSeenAt is + // still null", e.g. a machine that never finished provisioning). + const isLost = !lastSeenAt || lastSeenAt.getTime() < cutoffMs; + if (!isLost) continue; + + await db + .update(runs) + .set({ status: "lost", errorCode: "machine_lost", finishedAt: new Date() }) + .where(eq(runs.id, row.id)); + reaped++; + } + + return reaped; +} diff --git a/control-plane/artifacts/sanad-web/lib/runs/store.ts b/control-plane/artifacts/sanad-web/lib/runs/store.ts index a351fc99d..3ab8015f8 100644 --- a/control-plane/artifacts/sanad-web/lib/runs/store.ts +++ b/control-plane/artifacts/sanad-web/lib/runs/store.ts @@ -5,11 +5,12 @@ * machine uses to upload/read a run's wire trace. */ import { randomBytes } from "crypto"; -import { and, eq } from "drizzle-orm"; +import { and, desc, eq, getTableColumns, inArray } from "drizzle-orm"; import { S3Client, PutObjectCommand, GetObjectCommand } from "@aws-sdk/client-s3"; import { getSignedUrl } from "@aws-sdk/s3-request-presigner"; import { db } from "../db"; -import { runs } from "../db/schema"; +import { agents, deployments, runs, workspaces } from "../db/schema"; +import { MODEL_PRICING } from "../models/catalog"; export type RunRow = typeof runs.$inferSelect; @@ -107,6 +108,134 @@ export async function markRunFailed(id: string, errorCode: string): Promise { + const modelAlias = p.modelAlias ?? null; + await db + .update(runs) + .set({ + status: p.status, + errorCode: p.errorCode ?? null, + output: p.output ?? null, + tokensIn: p.tokensIn, + tokensOut: p.tokensOut, + modelAlias, + costUsdMicros: costUsdMicros(modelAlias, p.tokensIn, p.tokensOut), + traceUploaded: p.traceUploaded, + finishedAt: new Date(), + }) + .where(and(eq(runs.id, runId), inArray(runs.status, ["queued", "running"]))); +} + +// -- read APIs ------------------------------------------------------------ +// Every read below joins runs -> deployments -> agents -> workspaces and +// filters on workspaces.orgId — the established information-hiding rule +// (see lib/agents/registry.ts's getAgentByName): a run id belonging to +// another org must 404 exactly like one that doesn't exist at all, never +// leak via a 403. + +/** JSON-safe projection of a run row for the read APIs. */ +export function serializeRun(row: RunRow) { + return { + id: row.id, + deploymentId: row.deploymentId, + agentVersionId: row.agentVersionId, + status: row.status, + errorCode: row.errorCode, + triggerPrincipal: row.triggerPrincipal, + output: row.output, + tokensIn: row.tokensIn, + tokensOut: row.tokensOut, + costUsdMicros: row.costUsdMicros, + modelAlias: row.modelAlias, + traceUploaded: row.traceUploaded, + startedAt: row.startedAt, + finishedAt: row.finishedAt, + createdAt: row.createdAt, + }; +} + +/** A single run, scoped to the org — a foreign or unknown run id both return null. */ +export async function getRunForOrg(runId: string, orgId: string): Promise { + const rows = await db + .select(getTableColumns(runs)) + .from(runs) + .innerJoin(deployments, eq(runs.deploymentId, deployments.id)) + .innerJoin(agents, eq(deployments.agentId, agents.id)) + .innerJoin(workspaces, eq(agents.workspaceId, workspaces.id)) + .where(and(eq(runs.id, runId), eq(workspaces.orgId, orgId))) + .limit(1); + return rows[0] ?? null; +} + +/** Org-scoped run list for `GET /api/v1/runs`, newest-first. */ +export async function listRuns(p: { + orgId: string; + agentId?: string; + env?: string; + status?: string; + limit: number; +}): Promise { + const conditions = [eq(workspaces.orgId, p.orgId)]; + if (p.agentId) conditions.push(eq(agents.id, p.agentId)); + if (p.env) conditions.push(eq(deployments.env, p.env)); + if (p.status) conditions.push(eq(runs.status, p.status)); + + return db + .select(getTableColumns(runs)) + .from(runs) + .innerJoin(deployments, eq(runs.deploymentId, deployments.id)) + .innerJoin(agents, eq(deployments.agentId, agents.id)) + .innerJoin(workspaces, eq(agents.workspaceId, workspaces.id)) + .where(and(...conditions)) + .orderBy(desc(runs.createdAt)) + .limit(p.limit); +} + // -- trace presigner ---------------------------------------------------- // Lazy client, like lib/compute/aws.ts's clients — railway mode never // touches AWS, so nothing here should construct a client at import time. diff --git a/control-plane/artifacts/sanad-web/tests/unit/run-cost.test.ts b/control-plane/artifacts/sanad-web/tests/unit/run-cost.test.ts new file mode 100644 index 000000000..8d43ed9da --- /dev/null +++ b/control-plane/artifacts/sanad-web/tests/unit/run-cost.test.ts @@ -0,0 +1,16 @@ +import { describe, it, expect } from "vitest"; +import { costUsdMicros } from "@/lib/runs/store"; + +describe("costUsdMicros", () => { + it("prices kimi-k3 tokens", () => { + // 1M in @ $0.60 + 1M out @ $2.50 = $3.10 = 3_100_000 micros + expect(costUsdMicros("kimi-k3", 1_000_000, 1_000_000)).toBe(3_100_000); + }); + it("unknown alias costs zero, never throws", () => { + expect(costUsdMicros("nope", 5_000, 5_000)).toBe(0); + expect(costUsdMicros(null, 5_000, 5_000)).toBe(0); + }); + it("rounds to integer micros", () => { + expect(Number.isInteger(costUsdMicros("kimi-k3", 123, 457))).toBe(true); + }); +}); diff --git a/control-plane/artifacts/sanad-web/tests/unit/run-reaper.test.ts b/control-plane/artifacts/sanad-web/tests/unit/run-reaper.test.ts new file mode 100644 index 000000000..9039bc783 --- /dev/null +++ b/control-plane/artifacts/sanad-web/tests/unit/run-reaper.test.ts @@ -0,0 +1,27 @@ +import { describe, it, expect, vi } from "vitest"; + +const updates: any[] = []; +const staleRows = [{ id: "r_aaaaaaaaaaaa" }, { id: "r_bbbbbbbbbbbb" }]; +vi.mock("@/lib/db", () => ({ + db: { + select: vi.fn(() => ({ + from: vi.fn(() => ({ + leftJoin: vi.fn(() => ({ where: vi.fn(async () => staleRows) })), + where: vi.fn(async () => staleRows), + })), + })), + update: vi.fn(() => ({ + set: vi.fn((v: any) => { updates.push(v); return { where: vi.fn(async () => {}) }; }), + })), + }, +})); + +import { sweepLostRuns } from "@/lib/runs/reaper"; + +describe("sweepLostRuns", () => { + it("marks stale running runs lost and returns the count", async () => { + const n = await sweepLostRuns(300_000); + expect(n).toBe(2); + expect(updates[0]).toMatchObject({ status: "lost", errorCode: "machine_lost" }); + }); +}); From c400a4633b17f8b6fa9bc928f15150dee5876a96 Mon Sep 17 00:00:00 2001 From: Omar Alsoulah Date: Thu, 13 Aug 2026 18:37:53 +0300 Subject: [PATCH 12/28] =?UTF-8?q?sanad:=20run=20reaper=20=E2=80=94=20statu?= =?UTF-8?q?s-guarded=20batch=20update=20with=20returning=20count?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../artifacts/sanad-web/lib/runs/reaper.ts | 31 ++++++++++----- .../sanad-web/tests/unit/run-reaper.test.ts | 38 +++++++++++++++++-- 2 files changed, 56 insertions(+), 13 deletions(-) diff --git a/control-plane/artifacts/sanad-web/lib/runs/reaper.ts b/control-plane/artifacts/sanad-web/lib/runs/reaper.ts index 1eb1c37b7..7e8a13f73 100644 --- a/control-plane/artifacts/sanad-web/lib/runs/reaper.ts +++ b/control-plane/artifacts/sanad-web/lib/runs/reaper.ts @@ -17,7 +17,7 @@ * contract, beyond staleness) a plain absent-from-the-map lookup instead of * a NULL-across-an-outer-join special case. */ -import { eq, inArray } from "drizzle-orm"; +import { and, eq, inArray } from "drizzle-orm"; import { db } from "../db"; import { agents, deployments, runs, workspaceMachines } from "../db/schema"; @@ -65,7 +65,7 @@ export async function sweepLostRuns(staleAfterMs: number): Promise { machineRows.map((m) => [`${m.workspaceId}:${m.env}`, m.lastSeenAt]) ); - let reaped = 0; + const staleIds: string[] = []; for (const row of runningRows) { const dep = byDeploymentId.get(row.deploymentId); const lastSeenAt = dep ? lastSeenByKey.get(`${dep.workspaceId}:${dep.env}`) : undefined; @@ -73,14 +73,25 @@ export async function sweepLostRuns(staleAfterMs: number): Promise { // "no matching workspaceMachines row" and "row exists but lastSeenAt is // still null", e.g. a machine that never finished provisioning). const isLost = !lastSeenAt || lastSeenAt.getTime() < cutoffMs; - if (!isLost) continue; - - await db - .update(runs) - .set({ status: "lost", errorCode: "machine_lost", finishedAt: new Date() }) - .where(eq(runs.id, row.id)); - reaped++; + if (isLost) staleIds.push(row.id); } + if (staleIds.length === 0) return 0; + + // The runningRows snapshot above can go stale before this UPDATE runs — a + // run in the candidate set may have genuinely completed via POST + // .../runs/{id}/complete in the interim. Re-checking status = "running" + // in the WHERE (same guard completeRun uses) makes this a no-op for any + // row that already left "running", instead of clobbering its real + // status/output/tokens/cost with "lost". One batched statement rather + // than a per-row loop, and RETURNING reports exactly which rows this + // UPDATE actually flipped — the count is `returned.length`, not + // `staleIds.length`, so a caller never learns "reaped N" for a run that + // this call didn't actually touch. + const returned = await db + .update(runs) + .set({ status: "lost", errorCode: "machine_lost", finishedAt: new Date() }) + .where(and(inArray(runs.id, staleIds), eq(runs.status, "running"))) + .returning({ id: runs.id }); - return reaped; + return returned.length; } diff --git a/control-plane/artifacts/sanad-web/tests/unit/run-reaper.test.ts b/control-plane/artifacts/sanad-web/tests/unit/run-reaper.test.ts index 9039bc783..e0b811b1a 100644 --- a/control-plane/artifacts/sanad-web/tests/unit/run-reaper.test.ts +++ b/control-plane/artifacts/sanad-web/tests/unit/run-reaper.test.ts @@ -1,7 +1,15 @@ import { describe, it, expect, vi } from "vitest"; +import { PgDialect } from "drizzle-orm/pg-core"; const updates: any[] = []; +const whereArgs: any[] = []; const staleRows = [{ id: "r_aaaaaaaaaaaa" }, { id: "r_bbbbbbbbbbbb" }]; +// Simulates a race: r_bbbbbbbbbbbb genuinely completed via POST +// .../runs/{id}/complete between the stale-candidate select and the +// guarded UPDATE below, so the status="running" re-check excludes it from +// RETURNING even though it was in the stale-candidate set. +const returningRows = [{ id: "r_aaaaaaaaaaaa" }]; + vi.mock("@/lib/db", () => ({ db: { select: vi.fn(() => ({ @@ -11,7 +19,15 @@ vi.mock("@/lib/db", () => ({ })), })), update: vi.fn(() => ({ - set: vi.fn((v: any) => { updates.push(v); return { where: vi.fn(async () => {}) }; }), + set: vi.fn((v: any) => { + updates.push(v); + return { + where: vi.fn((w: any) => { + whereArgs.push(w); + return { returning: vi.fn(async () => returningRows) }; + }), + }; + }), })), }, })); @@ -19,9 +35,25 @@ vi.mock("@/lib/db", () => ({ import { sweepLostRuns } from "@/lib/runs/reaper"; describe("sweepLostRuns", () => { - it("marks stale running runs lost and returns the count", async () => { + it("marks stale running runs lost via a status-guarded batch update, counting only what RETURNING confirms", async () => { const n = await sweepLostRuns(300_000); - expect(n).toBe(2); + + // The guarded UPDATE only actually flipped 1 of the 2 stale candidates + // (simulated race above) — the returned count must come from + // RETURNING's length, not the 2-row candidate-select count. This is + // the regression the guard exists to prevent: silently reporting + // "reaped" for a run the UPDATE didn't touch. + expect(n).toBe(1); + expect(updates).toHaveLength(1); // one batched statement, not one per stale row expect(updates[0]).toMatchObject({ status: "lost", errorCode: "machine_lost" }); + + // Render the real WHERE condition sweepLostRuns built (drizzle's own + // PgDialect, not a stand-in) to confirm it re-checks status = "running" + // in addition to the id list — dropping this guard is exactly what let + // the reaper clobber a run that completed mid-sweep. + const { sql, params } = new PgDialect().sqlToQuery(whereArgs[0]); + expect(sql).toMatch(/"status"\s*=\s*\$\d/); + expect(params).toContain("running"); + expect(sql).toMatch(/"id"\s+in\s*\(/i); }); }); From c8a87207b68ddaf10f97aa92a6aa2f996437b03b Mon Sep 17 00:00:00 2001 From: Omar Alsoulah Date: Thu, 13 Aug 2026 18:48:31 +0300 Subject: [PATCH 13/28] =?UTF-8?q?sanad:=20worker=20assembly=20=E2=80=94=20?= =?UTF-8?q?sidecar=20spec,=20input=20rendering,=20ReturnOutput=20stop-turn?= =?UTF-8?q?=20tool?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/kimi_cli/soul/toolset.py | 4 ++ src/kimi_cli/worker/__init__.py | 17 +++++++++ src/kimi_cli/worker/assembly.py | 45 ++++++++++++++++++++++ src/kimi_cli/worker/return_output.py | 56 ++++++++++++++++++++++++++++ src/kimi_cli/worker/sidecar.py | 47 +++++++++++++++++++++++ tests/worker/test_assembly.py | 41 ++++++++++++++++++++ tests/worker/test_return_output.py | 43 +++++++++++++++++++++ tests/worker/test_sidecar.py | 34 +++++++++++++++++ 8 files changed, 287 insertions(+) create mode 100644 src/kimi_cli/worker/__init__.py create mode 100644 src/kimi_cli/worker/assembly.py create mode 100644 src/kimi_cli/worker/return_output.py create mode 100644 src/kimi_cli/worker/sidecar.py create mode 100644 tests/worker/test_assembly.py create mode 100644 tests/worker/test_return_output.py create mode 100644 tests/worker/test_sidecar.py diff --git a/src/kimi_cli/soul/toolset.py b/src/kimi_cli/soul/toolset.py index 5d66344aa..ee76b1576 100644 --- a/src/kimi_cli/soul/toolset.py +++ b/src/kimi_cli/soul/toolset.py @@ -340,6 +340,10 @@ def dedup_triggered(self) -> bool: def force_stop_turn(self) -> bool: return self._force_stop_turn + def request_stop_turn(self) -> None: + """Ask the soul to end the turn after the current step (used by worker tools).""" + self._force_stop_turn = True + def handle(self, tool_call: ToolCall) -> HandleResult: token = current_tool_call.set(tool_call) try: diff --git a/src/kimi_cli/worker/__init__.py b/src/kimi_cli/worker/__init__.py new file mode 100644 index 000000000..3e79e46c7 --- /dev/null +++ b/src/kimi_cli/worker/__init__.py @@ -0,0 +1,17 @@ +from kimi_cli.worker.assembly import ( + RETURN_OUTPUT_TOOL, + WorkerInputError, + derive_agent_spec, + render_input_prompt, +) +from kimi_cli.worker.sidecar import WorkerSpec, WorkerSpecError, load_worker_spec + +__all__ = [ + "RETURN_OUTPUT_TOOL", + "WorkerInputError", + "WorkerSpec", + "WorkerSpecError", + "derive_agent_spec", + "load_worker_spec", + "render_input_prompt", +] diff --git a/src/kimi_cli/worker/assembly.py b/src/kimi_cli/worker/assembly.py new file mode 100644 index 000000000..a7a7b4aff --- /dev/null +++ b/src/kimi_cli/worker/assembly.py @@ -0,0 +1,45 @@ +"""Run assembly shared by `sanad dev` (local) and the cloud RunRunner — parity by construction.""" + +import json +from pathlib import Path +from typing import Any + +import yaml + +from kimi_cli.worker.sidecar import WorkerSpec + +RETURN_OUTPUT_TOOL = "kimi_cli.worker.return_output:ReturnOutput" + + +class WorkerInputError(Exception): + pass + + +def render_input_prompt(spec: WorkerSpec, payload: dict[str, Any]) -> str: + declared = set(spec.interface.inputs) + given = set(payload) + if unknown := given - declared: + raise WorkerInputError(f"unknown inputs: {sorted(unknown)}") + if missing := declared - given: + raise WorkerInputError(f"missing inputs: {sorted(missing)}") + body = json.dumps(payload, sort_keys=True, ensure_ascii=False) + outputs = ", ".join(sorted(spec.interface.outputs)) or "output" + return ( + "Perform your task with these inputs:\n\n" + f"\n{body}\n\n\n" + "When the task is complete you MUST call the ReturnOutput tool " + f"exactly once with the declared outputs: {outputs}." + ) + + +def derive_agent_spec(agent_file: Path, out_dir: Path) -> Path: + out_dir.mkdir(parents=True, exist_ok=True) + derived = out_dir / "worker-agent.yaml" + derived.write_text( + yaml.safe_dump( + {"extend": str(agent_file.resolve()), "tools": [RETURN_OUTPUT_TOOL]}, + sort_keys=False, + ), + encoding="utf-8", + ) + return derived diff --git a/src/kimi_cli/worker/return_output.py b/src/kimi_cli/worker/return_output.py new file mode 100644 index 000000000..76e707a43 --- /dev/null +++ b/src/kimi_cli/worker/return_output.py @@ -0,0 +1,56 @@ +"""The worker interface contract: declared outputs come back through this tool.""" + +import json +import os +from pathlib import Path +from typing import Any, override + +from kosong.tooling import CallableTool2, ToolReturnValue +from pydantic import BaseModel, Field + +from kimi_cli.soul.toolset import KimiToolset +from kimi_cli.worker.sidecar import load_worker_spec + + +class Params(BaseModel): + output: dict[str, Any] = Field(description="The declared output document for this run.") + + +class ReturnOutput(CallableTool2[Params]): + name: str = "ReturnOutput" + description: str = ( + "Return the run's final output document. Call exactly once, with every declared " + "output key, when the task is complete. This ends the run." + ) + params: type[Params] = Params + + def __init__(self, toolset: KimiToolset) -> None: + super().__init__() + self._toolset = toolset + + @override + async def __call__(self, params: Params) -> ToolReturnValue: + spec = load_worker_spec(Path(os.environ["KIMI_WORKER_INTERFACE_FILE"])) + declared = set(spec.interface.outputs) + given = set(params.output) + if declared and (given != declared): + message = ( + f"Output keys {sorted(given)} do not match declared outputs " + f"{sorted(declared)}. Call ReturnOutput again with exactly the " + "declared keys." + ) + return ToolReturnValue( + is_error=True, + output=message, + message=message, + display=[], + ) + out_file = Path(os.environ["KIMI_WORKER_OUTPUT_FILE"]) + out_file.write_text(json.dumps(params.output, ensure_ascii=False), encoding="utf-8") + self._toolset.request_stop_turn() + return ToolReturnValue( + is_error=False, + output="Output recorded. The run is complete.", + message="Output recorded. The run is complete.", + display=[], + ) diff --git a/src/kimi_cli/worker/sidecar.py b/src/kimi_cli/worker/sidecar.py new file mode 100644 index 000000000..4f39c56a8 --- /dev/null +++ b/src/kimi_cli/worker/sidecar.py @@ -0,0 +1,47 @@ +"""worker.yaml — the P0 interface/budget sidecar (replaced by manifest-v1's stanzas).""" + +from pathlib import Path + +import yaml +from pydantic import BaseModel, Field, field_validator + + +class WorkerSpecError(Exception): + pass + + +class InterfaceSpec(BaseModel): + inputs: dict[str, str] = Field(default_factory=dict) + outputs: dict[str, str] = Field(default_factory=dict) + + @field_validator("inputs", "outputs") + @classmethod + def _no_empty_types(cls, v: dict[str, str]) -> dict[str, str]: + for key, typ in v.items(): + if not key or not typ.strip(): + raise ValueError(f"empty type for {key!r}") + return v + + +class BudgetSpec(BaseModel): + max_turn_seconds: int = 900 + max_steps_per_turn: int = 100 + max_tokens_per_run: int = 2_000_000 + + +class WorkerSpec(BaseModel): + interface: InterfaceSpec = Field(default_factory=InterfaceSpec) + budgets: BudgetSpec = Field(default_factory=BudgetSpec) + + +def load_worker_spec(path: Path) -> WorkerSpec: + try: + raw = yaml.safe_load(path.read_text(encoding="utf-8")) + except FileNotFoundError as e: + raise WorkerSpecError(f"worker spec not found: {path}") from e + except yaml.YAMLError as e: + raise WorkerSpecError(f"invalid YAML in {path}: {e}") from e + try: + return WorkerSpec.model_validate(raw or {}) + except ValueError as e: + raise WorkerSpecError(str(e)) from e diff --git a/tests/worker/test_assembly.py b/tests/worker/test_assembly.py new file mode 100644 index 000000000..55a760194 --- /dev/null +++ b/tests/worker/test_assembly.py @@ -0,0 +1,41 @@ +from pathlib import Path + +import pytest + +from kimi_cli.worker.assembly import WorkerInputError, derive_agent_spec, render_input_prompt +from kimi_cli.worker.sidecar import load_worker_spec + + +def _spec(tmp_path: Path): + p = tmp_path / "worker.yaml" + p.write_text("interface:\n inputs: {invoice_no: string}\n outputs: {decision: string}\n") + return load_worker_spec(p) + + +def test_render_is_deterministic(tmp_path: Path) -> None: + spec = _spec(tmp_path) + out = render_input_prompt(spec, {"invoice_no": "INV-1"}) + assert "" in out + assert '"invoice_no": "INV-1"' in out + assert "ReturnOutput tool exactly once" in out + assert out == render_input_prompt(spec, {"invoice_no": "INV-1"}) + + +def test_unknown_input_rejected(tmp_path: Path) -> None: + with pytest.raises(WorkerInputError): + render_input_prompt(_spec(tmp_path), {"bogus": 1}) + + +def test_missing_input_rejected(tmp_path: Path) -> None: + with pytest.raises(WorkerInputError): + render_input_prompt(_spec(tmp_path), {}) + + +def test_derived_spec_extends_and_adds_tool(tmp_path: Path) -> None: + agent = tmp_path / "agent.yaml" + agent.write_text("version: '1'\nname: test\nsystem_prompt_path: prompt.md\n") + (tmp_path / "prompt.md").write_text("hi") + derived = derive_agent_spec(agent, tmp_path / "out") + text = derived.read_text() + assert str(agent) in text + assert "kimi_cli.worker.return_output:ReturnOutput" in text diff --git a/tests/worker/test_return_output.py b/tests/worker/test_return_output.py new file mode 100644 index 000000000..472dffc85 --- /dev/null +++ b/tests/worker/test_return_output.py @@ -0,0 +1,43 @@ +import json +from pathlib import Path + +import pytest + +from kimi_cli.worker.return_output import Params, ReturnOutput + + +class FakeToolset: + def __init__(self) -> None: + self.stopped = False + + def request_stop_turn(self) -> None: + self.stopped = True + + +def _env(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Path: + iface = tmp_path / "worker.yaml" + iface.write_text("interface:\n inputs: {}\n outputs: {decision: string}\n") + out_file = tmp_path / "output.json" + monkeypatch.setenv("KIMI_WORKER_INTERFACE_FILE", str(iface)) + monkeypatch.setenv("KIMI_WORKER_OUTPUT_FILE", str(out_file)) + return out_file + + +async def test_writes_output_and_stops(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + out_file = _env(monkeypatch, tmp_path) + toolset = FakeToolset() + tool = ReturnOutput(toolset) # type: ignore[arg-type] + result = await tool(Params(output={"decision": "approve"})) + assert not result.is_error + assert json.loads(out_file.read_text()) == {"decision": "approve"} + assert toolset.stopped + + +async def test_undeclared_output_key_errors( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + out_file = _env(monkeypatch, tmp_path) + tool = ReturnOutput(FakeToolset()) # type: ignore[arg-type] + result = await tool(Params(output={"bogus": 1})) + assert result.is_error + assert not out_file.exists() diff --git a/tests/worker/test_sidecar.py b/tests/worker/test_sidecar.py new file mode 100644 index 000000000..7cb133dfa --- /dev/null +++ b/tests/worker/test_sidecar.py @@ -0,0 +1,34 @@ +from pathlib import Path + +import pytest + +from kimi_cli.worker.sidecar import WorkerSpec, WorkerSpecError, load_worker_spec + +VALID = """\ +interface: + inputs: {invoice_no: string, amount: number} + outputs: {decision: "enum[approve, hold]", summary: string} +budgets: + max_turn_seconds: 60 +""" + + +def test_load_valid(tmp_path: Path) -> None: + p = tmp_path / "worker.yaml" + p.write_text(VALID) + spec: WorkerSpec = load_worker_spec(p) + assert spec.interface.inputs == {"invoice_no": "string", "amount": "number"} + assert spec.budgets.max_turn_seconds == 60 + assert spec.budgets.max_steps_per_turn == 100 # default + + +def test_missing_file(tmp_path: Path) -> None: + with pytest.raises(WorkerSpecError): + load_worker_spec(tmp_path / "nope.yaml") + + +def test_empty_output_type_rejected(tmp_path: Path) -> None: + p = tmp_path / "worker.yaml" + p.write_text('interface:\n inputs: {}\n outputs: {decision: ""}\n') + with pytest.raises(WorkerSpecError): + load_worker_spec(p) From 1ad0f6a191363896b9a5cdd58530c5c904da9c92 Mon Sep 17 00:00:00 2001 From: Omar Alsoulah Date: Thu, 13 Aug 2026 18:52:02 +0300 Subject: [PATCH 14/28] =?UTF-8?q?sanad:=20worker=20assembly=20=E2=80=94=20?= =?UTF-8?q?derived=20spec=20preserves=20base=20tools?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/kimi_cli/worker/assembly.py | 28 +++++++++++++++++++++- tests/worker/test_assembly.py | 41 ++++++++++++++++++++++++++++++--- 2 files changed, 65 insertions(+), 4 deletions(-) diff --git a/src/kimi_cli/worker/assembly.py b/src/kimi_cli/worker/assembly.py index a7a7b4aff..b92df5103 100644 --- a/src/kimi_cli/worker/assembly.py +++ b/src/kimi_cli/worker/assembly.py @@ -6,6 +6,7 @@ import yaml +from kimi_cli.agentspec import load_agent_spec from kimi_cli.worker.sidecar import WorkerSpec RETURN_OUTPUT_TOOL = "kimi_cli.worker.return_output:ReturnOutput" @@ -33,11 +34,36 @@ def render_input_prompt(spec: WorkerSpec, payload: dict[str, Any]) -> str: def derive_agent_spec(agent_file: Path, out_dir: Path) -> Path: + """Write a derived agent spec that extends `agent_file` and adds ReturnOutput. + + Two things the naive `{extend: ..., tools: [ReturnOutput]}` shape gets wrong: + + 1. `extend` alone is not enough to keep the base tools: agentspec.py's + extend-resolution treats a `tools` list on the extending spec as a full + replacement of the base tools, not an addition (see `_load_agent_spec` in + agentspec.py — only `system_prompt_args` is merged; `tools` is overwritten + wholesale). So we resolve the base spec's tools here and explicitly restate + them plus ReturnOutput, while still using `extend` for everything else + (prompt, model, subagents, ...). + 2. The fields (`extend`, `tools`, ...) must be nested under an `agent:` key — + `_load_agent_spec` reads `AgentSpec(**data.get("agent", {}))`, so a flat + top-level `{extend: ..., tools: [...]}` document is silently ignored + (treated as an empty spec) rather than raising, which would otherwise fail + later with a confusing "Agent name is required". + """ + base = load_agent_spec(agent_file) + tools = list(base.tools) + if RETURN_OUTPUT_TOOL not in tools: + tools.append(RETURN_OUTPUT_TOOL) + out_dir.mkdir(parents=True, exist_ok=True) derived = out_dir / "worker-agent.yaml" derived.write_text( yaml.safe_dump( - {"extend": str(agent_file.resolve()), "tools": [RETURN_OUTPUT_TOOL]}, + { + "version": "1", + "agent": {"extend": str(agent_file.resolve()), "tools": tools}, + }, sort_keys=False, ), encoding="utf-8", diff --git a/tests/worker/test_assembly.py b/tests/worker/test_assembly.py index 55a760194..f538dd37a 100644 --- a/tests/worker/test_assembly.py +++ b/tests/worker/test_assembly.py @@ -2,7 +2,13 @@ import pytest -from kimi_cli.worker.assembly import WorkerInputError, derive_agent_spec, render_input_prompt +from kimi_cli.agentspec import load_agent_spec +from kimi_cli.worker.assembly import ( + RETURN_OUTPUT_TOOL, + WorkerInputError, + derive_agent_spec, + render_input_prompt, +) from kimi_cli.worker.sidecar import load_worker_spec @@ -31,11 +37,40 @@ def test_missing_input_rejected(tmp_path: Path) -> None: render_input_prompt(_spec(tmp_path), {}) -def test_derived_spec_extends_and_adds_tool(tmp_path: Path) -> None: +def _agent_with_tools(tmp_path: Path) -> Path: agent = tmp_path / "agent.yaml" - agent.write_text("version: '1'\nname: test\nsystem_prompt_path: prompt.md\n") + agent.write_text( + "version: '1'\n" + "agent:\n" + " name: test\n" + " system_prompt_path: prompt.md\n" + " tools:\n" + " - kimi_cli.tools.shell:Shell\n" + ) (tmp_path / "prompt.md").write_text("hi") + return agent + + +def test_derived_spec_extends_and_adds_tool(tmp_path: Path) -> None: + agent = _agent_with_tools(tmp_path) derived = derive_agent_spec(agent, tmp_path / "out") text = derived.read_text() assert str(agent) in text assert "kimi_cli.worker.return_output:ReturnOutput" in text + + # The real contract: base tools must survive through the derived spec once + # loaded via the actual agentspec machinery, not just appear as raw text. + # extend's `tools` field is a full replacement (see agentspec.py), so a naive + # `{extend: ..., tools: [ReturnOutput]}` would silently drop Shell et al. + base = load_agent_spec(agent) + resolved = load_agent_spec(derived) + assert resolved.tools == [*base.tools, RETURN_OUTPUT_TOOL] + + +def test_rederiving_does_not_duplicate_return_output(tmp_path: Path) -> None: + agent = _agent_with_tools(tmp_path) + out_dir = tmp_path / "out" + first = derive_agent_spec(agent, out_dir) + second = derive_agent_spec(first, out_dir / "again") + resolved = load_agent_spec(second) + assert resolved.tools.count(RETURN_OUTPUT_TOOL) == 1 From b9337dddc2fcf6b7086e5a4f1076f838a7708e63 Mon Sep 17 00:00:00 2001 From: Omar Alsoulah Date: Thu, 13 Aug 2026 19:02:20 +0300 Subject: [PATCH 15/28] =?UTF-8?q?sanad:=20agent=20dev=20=E2=80=94=20local?= =?UTF-8?q?=20ephemeral=20worker=20run=20with=20nudge-then-no=5Foutput=20c?= =?UTF-8?q?ontract?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/kimi_cli/cli/_lazy_group.py | 2 + src/kimi_cli/cli/worker.py | 124 ++++++++++++++++++++++++++++++++ tests_e2e/test_worker_dev.py | 72 +++++++++++++++++++ 3 files changed, 198 insertions(+) create mode 100644 src/kimi_cli/cli/worker.py create mode 100644 tests_e2e/test_worker_dev.py diff --git a/src/kimi_cli/cli/_lazy_group.py b/src/kimi_cli/cli/_lazy_group.py index 3799d6063..669fcd996 100644 --- a/src/kimi_cli/cli/_lazy_group.py +++ b/src/kimi_cli/cli/_lazy_group.py @@ -21,6 +21,7 @@ class LazySubcommandGroup(typer.core.TyperGroup): "blueprint": ("kimi_cli.cli.blueprint", "cli", "Validate and inspect the blueprint."), "vis": ("kimi_cli.cli.vis", "cli", "Run Kimi Agent Tracing Visualizer."), "web": ("kimi_cli.cli.web", "cli", "Run Kimi Code CLI web interface."), + "agent": ("kimi_cli.cli.worker", "cli", "Deploy and operate worker agents."), } lazy_command_order: tuple[str, ...] = ( "info", @@ -30,6 +31,7 @@ class LazySubcommandGroup(typer.core.TyperGroup): "blueprint", "vis", "web", + "agent", ) # Click options that support optional values. When the flag is present diff --git a/src/kimi_cli/cli/worker.py b/src/kimi_cli/cli/worker.py new file mode 100644 index 000000000..e2835124a --- /dev/null +++ b/src/kimi_cli/cli/worker.py @@ -0,0 +1,124 @@ +"""sanad agent — worker-agent verbs (dev now; deploy/runs/logs/pause/resume in Task 9).""" + +import asyncio +import json +import os +import tempfile +from pathlib import Path +from typing import TYPE_CHECKING, Annotated + +import typer + +if TYPE_CHECKING: + from kimi_cli.app import KimiCLI + +cli = typer.Typer(help="Deploy and operate worker agents.") + + +@cli.callback() +def agent_group() -> None: + """Deploy and operate worker agents.""" + # Typer collapses a group with exactly one registered command into that + # command directly (see typer.main.get_command), which would make `kimi + # agent dev` parse `dev` as a stray positional argument to `agent` itself. + # An explicit callback forces Group mode so `dev` stays a real subcommand + # name now, ahead of Task 9 adding deploy/runs/logs/pause/resume. + + +EXIT_OK = 0 +EXIT_FAILURE = 1 +EXIT_NO_OUTPUT = 3 +EXIT_BAD_INPUT = 4 + +NUDGE = ( + "You have not called the ReturnOutput tool. Call it now with the declared outputs. " + "This is your final step." +) + + +@cli.command() +def dev( + input_json: Annotated[str, typer.Option("--input", help="Run input as JSON.")], + agent_file: Annotated[Path, typer.Option("--agent-file")] = Path("agent.yaml"), + worker_file: Annotated[Path, typer.Option("--worker-file")] = Path("worker.yaml"), + work_dir: Annotated[Path, typer.Option("--work-dir")] = Path("."), + config_file: Annotated[Path | None, typer.Option("--config-file")] = None, +) -> None: + """Run the worker once locally with the same assembly the cloud runner uses.""" + raise typer.Exit(asyncio.run(_dev(input_json, agent_file, worker_file, work_dir, config_file))) + + +async def _dev( + input_json: str, + agent_file: Path, + worker_file: Path, + work_dir: Path, + config_file: Path | None, +) -> int: + from kimi_cli.worker import ( + WorkerInputError, + WorkerSpecError, + derive_agent_spec, + load_worker_spec, + render_input_prompt, + ) + + work_dir = work_dir.resolve() + try: + spec = load_worker_spec((work_dir / worker_file).resolve()) + prompt = render_input_prompt(spec, json.loads(input_json)) + except (WorkerInputError, WorkerSpecError, json.JSONDecodeError) as e: + typer.echo(f"error: {e}", err=True) + return EXIT_BAD_INPUT + + with tempfile.TemporaryDirectory(prefix="sanad-worker-") as tmp: + out_file = Path(tmp) / "output.json" + # Set env before KimiCLI.create (toolset loads tools at create time) so the + # ReturnOutput tool sees KIMI_WORKER_INTERFACE_FILE/KIMI_WORKER_OUTPUT_FILE + # at call time regardless of when it reads them. + os.environ["KIMI_WORKER_INTERFACE_FILE"] = str((work_dir / worker_file).resolve()) + os.environ["KIMI_WORKER_OUTPUT_FILE"] = str(out_file) + derived = derive_agent_spec((work_dir / agent_file).resolve(), Path(tmp)) + + from kaos.path import KaosPath + + from kimi_cli.app import KimiCLI + from kimi_cli.session import Session + + session = await Session.create(KaosPath(str(work_dir))) + cli_app = await KimiCLI.create( + session, + config=config_file, + runtime_afk=True, + ui_mode="print", + agent_file=derived, + max_steps_per_turn=spec.budgets.max_steps_per_turn, + ) + status = await _one_turn(cli_app, prompt, spec.budgets.max_turn_seconds) + if status != 0: + return status + if not out_file.exists(): + # One nudge, then give up (spec: nudge-retry then fail no_output). + status = await _one_turn(cli_app, NUDGE, spec.budgets.max_turn_seconds) + if status != 0: + return status + if not out_file.exists(): + typer.echo("error: run finished without calling ReturnOutput", err=True) + return EXIT_NO_OUTPUT + typer.echo(out_file.read_text(encoding="utf-8")) + return EXIT_OK + + +async def _one_turn(cli_app: "KimiCLI", prompt: str, max_seconds: int) -> int: + cancel = asyncio.Event() + try: + async with asyncio.timeout(max_seconds): + async for _msg in cli_app.run(prompt, cancel): + pass + except TimeoutError: + typer.echo("error: turn budget exceeded", err=True) + return EXIT_FAILURE + except Exception as e: # provider errors, RunCancelled, ... + typer.echo(f"error: {e}", err=True) + return EXIT_FAILURE + return 0 diff --git a/tests_e2e/test_worker_dev.py b/tests_e2e/test_worker_dev.py new file mode 100644 index 000000000..4094a17b1 --- /dev/null +++ b/tests_e2e/test_worker_dev.py @@ -0,0 +1,72 @@ +import json +import subprocess + +from tests_e2e.wire_helpers import ( + make_env, + make_home_dir, + make_work_dir, + repo_root, + write_scripted_config, +) + +# Loadable shape required by kimi_cli.agentspec.load_agent_spec: `version` + `agent:` +# nesting (see tests/worker/test_assembly.py::_agent_with_tools). A flat top-level +# `{name: ..., system_prompt_path: ...}` document is silently treated as an empty +# spec by `_load_agent_spec` (`AgentSpec(**data.get("agent", {}))`), which later +# fails with a confusing "Agent name is required" instead of loading. `tools` is +# also required on a non-extending spec (Inherit() with no base to inherit from +# raises "Tools are required"), so it must be listed explicitly, even if empty. +AGENT_YAML = "version: '1'\nagent:\n name: t\n system_prompt_path: prompt.md\n tools: []\n" +WORKER_YAML = "interface:\n inputs: {q: string}\n outputs: {answer: string}\n" + + +def _tool_call(payload: dict) -> str: + call = {"id": "tc-1", "name": "ReturnOutput", "arguments": json.dumps(payload)} + return f"tool_call: {json.dumps(call)}" + + +def _run_dev(tmp_path, scripts: list[str], input_json: str) -> subprocess.CompletedProcess: + config_path = write_scripted_config(tmp_path, scripts) + work_dir = make_work_dir(tmp_path) + home_dir = make_home_dir(tmp_path) + (work_dir / "agent.yaml").write_text(AGENT_YAML) + (work_dir / "prompt.md").write_text("You are a test agent.") + (work_dir / "worker.yaml").write_text(WORKER_YAML) + return subprocess.run( + [ + "uv", + "run", + "kimi", + "agent", + "dev", + "--input", + input_json, + "--config-file", + str(config_path), + "--work-dir", + str(work_dir), + ], + cwd=repo_root(), + env=make_env(home_dir), + capture_output=True, + text=True, + timeout=120, + ) + + +def test_dev_returns_output(tmp_path) -> None: + scripts = ["\n".join(["text: working", _tool_call({"output": {"answer": "42"}})])] + proc = _run_dev(tmp_path, scripts, '{"q": "meaning"}') + assert proc.returncode == 0, proc.stderr + assert json.loads(proc.stdout.strip()) == {"answer": "42"} + + +def test_dev_no_output_exit_code(tmp_path) -> None: + # Model never calls ReturnOutput: one text turn, then the nudge turn also returns text. + proc = _run_dev(tmp_path, ["text: done", "text: still no tool"], '{"q": "x"}') + assert proc.returncode == 3, (proc.stdout, proc.stderr) + + +def test_dev_bad_input_exit_code(tmp_path) -> None: + proc = _run_dev(tmp_path, ["text: unused"], '{"wrong_key": 1}') + assert proc.returncode == 4 From 3bea50b290e4626a195ca18a9b842ed8ca2d4404 Mon Sep 17 00:00:00 2001 From: Omar Alsoulah Date: Thu, 13 Aug 2026 19:23:30 +0300 Subject: [PATCH 16/28] =?UTF-8?q?sanad:=20agent=20verbs=20=E2=80=94=20depl?= =?UTF-8?q?oy=20bundle=20flow,=20runs/logs/pause/resume=20clients?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/kimi_cli/cli/worker.py | 266 +++++++++++++++++++++- src/kimi_cli/sanad/client.py | 152 ++++++++++++- src/kimi_cli/sanad/models.py | 27 +++ tests/worker/test_client_worker.py | 193 ++++++++++++++++ tests/worker/test_worker_cli.py | 343 +++++++++++++++++++++++++++++ 5 files changed, 973 insertions(+), 8 deletions(-) create mode 100644 tests/worker/test_client_worker.py create mode 100644 tests/worker/test_worker_cli.py diff --git a/src/kimi_cli/cli/worker.py b/src/kimi_cli/cli/worker.py index e2835124a..59572bfc6 100644 --- a/src/kimi_cli/cli/worker.py +++ b/src/kimi_cli/cli/worker.py @@ -1,9 +1,11 @@ -"""sanad agent — worker-agent verbs (dev now; deploy/runs/logs/pause/resume in Task 9).""" +"""sanad agent — worker-agent verbs: local dev run, plus the control-plane +deploy/runs/logs/pause/resume commands.""" import asyncio import json import os import tempfile +from datetime import UTC, datetime from pathlib import Path from typing import TYPE_CHECKING, Annotated @@ -11,6 +13,8 @@ if TYPE_CHECKING: from kimi_cli.app import KimiCLI + from kimi_cli.sanad.client import SanadClient + from kimi_cli.sanad.session import SanadSession cli = typer.Typer(help="Deploy and operate worker agents.") @@ -21,8 +25,8 @@ def agent_group() -> None: # Typer collapses a group with exactly one registered command into that # command directly (see typer.main.get_command), which would make `kimi # agent dev` parse `dev` as a stray positional argument to `agent` itself. - # An explicit callback forces Group mode so `dev` stays a real subcommand - # name now, ahead of Task 9 adding deploy/runs/logs/pause/resume. + # An explicit callback forces Group mode so each verb stays a real + # subcommand name even while the group only had one command (dev). EXIT_OK = 0 @@ -122,3 +126,259 @@ async def _one_turn(cli_app: "KimiCLI", prompt: str, max_seconds: int) -> int: typer.echo(f"error: {e}", err=True) return EXIT_FAILURE return 0 + + +# -- control-plane verbs (deploy/runs/logs/pause/resume) -------------------- +# +# These talk to the sanad control plane over SanadClient, unlike `dev` above +# which runs entirely offline. Session-token resolution reuses +# SanadSession.require_token() (env SANAD_SESSION_TOKEN, falling back to the +# OS keychain — see KeychainStore.get()) rather than re-deriving that +# precedence here. `_build_session`/`_build_client` are seams: tests +# monkeypatch them to inject a fake keychain / MockTransport, the same +# pattern `kimi_cli.sanad.cli._build_session` uses. + + +def _build_session() -> "SanadSession": + from kimi_cli.sanad.session import SanadSession + from kimi_cli.sanad.settings import SanadSettings + + return SanadSession(SanadSettings.load()) + + +def _build_client() -> "SanadClient": + from kimi_cli.sanad.client import SanadClient + from kimi_cli.sanad.settings import SanadSettings + + return SanadClient(SanadSettings.load()) + + +def _resolve_token() -> tuple[str | None, int]: + """Session token, or ``(None, EXIT_FAILURE)`` with the error already printed.""" + from kimi_cli.sanad.errors import SanadError + + session = _build_session() + try: + token = session.require_token() + except SanadError as e: + typer.echo(f"error: {e.message}", err=True) + return None, EXIT_FAILURE + finally: + session.close() + return token, EXIT_OK + + +def _collect_bundle_files( + work_dir: Path, agent_path: Path, worker_path: Path, system_prompt_path: Path +) -> tuple[dict[str, str], str | None]: + """Read agent.yaml + worker.yaml + the referenced system prompt as a files map. + + Keys are paths relative to ``work_dir`` (the shape the versions route + expects); a file outside ``work_dir`` or unreadable is reported as a + single error string rather than raised, so the caller can turn it into a + clean exit-4 message. + """ + files: dict[str, str] = {} + for abs_path in (agent_path, worker_path, system_prompt_path): + try: + key = str(abs_path.relative_to(work_dir)) + except ValueError: + return {}, f"{abs_path} is outside --work-dir {work_dir}" + try: + files[key] = abs_path.read_text(encoding="utf-8") + except OSError as e: + return {}, f"cannot read {abs_path}: {e}" + return files, None + + +@cli.command() +def deploy( + env: Annotated[str, typer.Option("--env")] = "dev", + workspace: Annotated[str, typer.Option("--workspace")] = "default", + agent_file: Annotated[Path, typer.Option("--agent-file")] = Path("agent.yaml"), + worker_file: Annotated[Path, typer.Option("--worker-file")] = Path("worker.yaml"), + work_dir: Annotated[Path, typer.Option("--work-dir")] = Path("."), +) -> None: + """Validate the local bundle, then upsert/version/deploy it to the control plane. + + The agent name comes from agent.yaml's own ``agent.name`` field, not a CLI + argument — deploy always ships the bundle it validates. + """ + raise typer.Exit(_deploy(env, workspace, agent_file, worker_file, work_dir)) + + +def _deploy(env: str, workspace: str, agent_file: Path, worker_file: Path, work_dir: Path) -> int: + from kimi_cli.agentspec import load_agent_spec + from kimi_cli.exception import AgentSpecError + from kimi_cli.sanad.errors import SanadError + from kimi_cli.worker import WorkerSpecError, load_worker_spec + + work_dir = work_dir.resolve() + agent_path = (work_dir / agent_file).resolve() + worker_path = (work_dir / worker_file).resolve() + + try: + resolved = load_agent_spec(agent_path) + load_worker_spec(worker_path) + except (AgentSpecError, WorkerSpecError, FileNotFoundError, OSError) as e: + typer.echo(f"error: {e}", err=True) + return EXIT_BAD_INPUT + + files, bundle_error = _collect_bundle_files( + work_dir, agent_path, worker_path, resolved.system_prompt_path + ) + if bundle_error is not None: + typer.echo(f"error: {bundle_error}", err=True) + return EXIT_BAD_INPUT + + # Validation (and every filesystem read) happens above, before any client + # is built or network call attempted — a broken bundle never touches the + # control plane. + token, code = _resolve_token() + if token is None: + return code + + client = _build_client() + try: + result = client.deploy_agent( + token, name=resolved.name, files=files, env=env, workspace=workspace + ) + except SanadError as e: + typer.echo(f"error: {e.message}", err=True) + return EXIT_FAILURE + finally: + client.close() + + typer.echo(result.model_dump_json(by_alias=True)) + return EXIT_OK + + +def _format_age(created_at: str, now: datetime) -> str: + try: + created = datetime.fromisoformat(created_at.replace("Z", "+00:00")) + except ValueError: + return "?" + if created.tzinfo is None: + created = created.replace(tzinfo=UTC) + seconds = max(0, int((now - created).total_seconds())) + if seconds < 60: + return f"{seconds}s" + minutes = seconds // 60 + if minutes < 60: + return f"{minutes}m" + hours = minutes // 60 + if hours < 24: + return f"{hours}h" + return f"{hours // 24}d" + + +@cli.command() +def runs( + agent: Annotated[str | None, typer.Option("--agent")] = None, + env: Annotated[str | None, typer.Option("--env")] = None, + limit: Annotated[int, typer.Option("--limit")] = 20, + as_json: Annotated[bool, typer.Option("--json")] = False, +) -> None: + """List recent runs as a compact table (id, status, cost, tokens, age).""" + raise typer.Exit(_runs(agent, env, limit, as_json)) + + +def _runs(agent: str | None, env: str | None, limit: int, as_json: bool) -> int: + from kimi_cli.sanad.errors import SanadError + + token, code = _resolve_token() + if token is None: + return code + + client = _build_client() + try: + rows = client.list_runs(token, agent=agent, env=env, limit=limit) + except SanadError as e: + typer.echo(f"error: {e.message}", err=True) + return EXIT_FAILURE + finally: + client.close() + + if as_json: + typer.echo(json.dumps([r.model_dump(by_alias=True) for r in rows])) + return EXIT_OK + + now = datetime.now(UTC) + typer.echo(f"{'ID':<16} {'STATUS':<10} {'COST':>10} {'IN':>8} {'OUT':>8} AGE") + for r in rows: + cost = f"${r.cost_usd_micros / 1_000_000:.4f}" + typer.echo( + f"{r.id:<16} {r.status:<10} {cost:>10} {r.tokens_in:>8} {r.tokens_out:>8} " + f"{_format_age(r.created_at, now)}" + ) + return EXIT_OK + + +@cli.command() +def logs( + run_id: Annotated[str, typer.Argument()], + follow: Annotated[ + bool, + typer.Option("--follow", help="Reserved for a future streaming mode; currently a no-op."), + ] = False, +) -> None: + """Print the trace URL for a finished run.""" + raise typer.Exit(_logs(run_id)) + + +def _logs(run_id: str) -> int: + from kimi_cli.sanad.errors import SanadError + + token, code = _resolve_token() + if token is None: + return code + + client = _build_client() + try: + url = client.get_run_trace_url(token, run_id) + except SanadError as e: + typer.echo(f"error: {e.message}", err=True) + return EXIT_FAILURE + finally: + client.close() + + typer.echo(url) + return EXIT_OK + + +def _set_status(name: str, env: str, status: str) -> int: + from kimi_cli.sanad.errors import SanadError + + token, code = _resolve_token() + if token is None: + return code + + client = _build_client() + try: + client.set_deployment_status(token, agent=name, env=env, status=status) + except SanadError as e: + typer.echo(f"error: {e.message}", err=True) + return EXIT_FAILURE + finally: + client.close() + + typer.echo(f"{name} ({env}): {status}") + return EXIT_OK + + +@cli.command() +def pause( + name: Annotated[str, typer.Argument()], + env: Annotated[str, typer.Option("--env")] = "dev", +) -> None: + """Pause the live deployment for NAME in --env.""" + raise typer.Exit(_set_status(name, env, "paused")) + + +@cli.command() +def resume( + name: Annotated[str, typer.Argument()], + env: Annotated[str, typer.Option("--env")] = "dev", +) -> None: + """Resume the paused deployment for NAME in --env.""" + raise typer.Exit(_set_status(name, env, "active")) diff --git a/src/kimi_cli/sanad/client.py b/src/kimi_cli/sanad/client.py index ee5e96c08..27e433f16 100644 --- a/src/kimi_cli/sanad/client.py +++ b/src/kimi_cli/sanad/client.py @@ -12,7 +12,15 @@ import httpx from kimi_cli.sanad.errors import SanadError -from kimi_cli.sanad.models import DevicePoll, DeviceStart, Me, MintResponse, UsageSummary +from kimi_cli.sanad.models import ( + DeployResult, + DevicePoll, + DeviceStart, + Me, + MintResponse, + RunRow, + UsageSummary, +) from kimi_cli.sanad.settings import SanadSettings @@ -31,23 +39,35 @@ def __init__( ) # -- low level -------------------------------------------------------- - def _request( + def _send( self, method: str, path: str, *, - json: dict | None = None, + json: dict[str, object] | None = None, + params: dict[str, str | int] | None = None, session_token: str | None = None, - ) -> object: + follow_redirects: bool = False, + ) -> httpx.Response: headers: dict[str, str] = {} if session_token is not None: headers["authorization"] = f"Bearer {session_token}" try: - resp = self._http.request(method, path, json=json, headers=headers) + return self._http.request( + method, + path, + json=json, + params=params, + headers=headers, + follow_redirects=follow_redirects, + ) except httpx.HTTPError as exc: raise SanadError( "network_error", f"Could not reach the sanad control plane: {exc}", retryable=True ) from exc + + def _unwrap(self, resp: httpx.Response) -> object: + """Unwrap a ``{data: ...}`` envelope, raising :class:`SanadError` on ``{error: ...}``.""" if resp.status_code == 204: return None payload: object = None @@ -75,6 +95,18 @@ def _request( return payload["data"] return payload + def _request( + self, + method: str, + path: str, + *, + json: dict[str, object] | None = None, + params: dict[str, str | int] | None = None, + session_token: str | None = None, + ) -> object: + resp = self._send(method, path, json=json, params=params, session_token=session_token) + return self._unwrap(resp) + # -- auth ------------------------------------------------------------- def device_start(self) -> DeviceStart: return DeviceStart.model_validate(self._request("POST", "/api/v1/auth/device/start")) @@ -120,6 +152,116 @@ def revoke_runtime_token_family(self, session_token: str, family_id: str) -> Non session_token=session_token, ) + # -- worker agents ------------------------------------------------------ + def deploy_agent( + self, + session_token: str, + *, + name: str, + files: dict[str, str], + env: str, + workspace: str = "default", + ) -> DeployResult: + """Upsert the agent, publish a new version, then deploy it to ``env``. + + Three sequential calls, in order — an agent must exist before it can + take a version, and a version must exist before it can be deployed. + Each response's envelope shape is the route's own (see + ``control-plane/artifacts/sanad-web/app/api/v1/agents/route.ts`` and + siblings): the agent-create response nests ``agentId`` (not ``id``), + which is easy to get wrong copying from the wire shape by eye. + """ + created = self._request( + "POST", + "/api/v1/agents", + json={"name": name, "workspace": workspace}, + session_token=session_token, + ) + agent_id = str(created["agentId"]) if isinstance(created, dict) else "" + + version = self._request( + "POST", + f"/api/v1/agents/{name}/versions", + json={"files": files}, + session_token=session_token, + ) + version_id = str(version["versionId"]) if isinstance(version, dict) else "" + content_hash = str(version["contentHash"]) if isinstance(version, dict) else "" + + deployment = self._request( + "POST", + f"/api/v1/agents/{name}/deployments", + json={"versionId": version_id, "env": env}, + session_token=session_token, + ) + deployment_id = str(deployment["deploymentId"]) if isinstance(deployment, dict) else "" + + return DeployResult( + agent_id=agent_id, + version_id=version_id, + deployment_id=deployment_id, + content_hash=content_hash, + ) + + def set_deployment_status( + self, session_token: str, *, agent: str, env: str, status: str + ) -> None: + """PATCH the deployment status (``active``/``paused``) for ``agent``/``env``.""" + self._request( + "PATCH", + f"/api/v1/agents/{agent}/deployments", + json={"env": env, "status": status}, + session_token=session_token, + ) + + def list_runs( + self, + session_token: str, + *, + agent: str | None = None, + env: str | None = None, + limit: int = 20, + ) -> list[RunRow]: + params: dict[str, str | int] = {"limit": limit} + if agent is not None: + params["agent"] = agent + if env is not None: + params["env"] = env + data = self._request("GET", "/api/v1/runs", params=params, session_token=session_token) + rows = data.get("runs") if isinstance(data, dict) else None + return [RunRow.model_validate(row) for row in (rows or [])] + + def get_run(self, session_token: str, run_id: str) -> RunRow: + data = self._request("GET", f"/api/v1/runs/{run_id}", session_token=session_token) + run = data.get("run") if isinstance(data, dict) else None + return RunRow.model_validate(run) + + def get_run_trace_url(self, session_token: str, run_id: str) -> str: + """Follow-less GET: the trace endpoint 307s to a presigned URL rather + than proxying the object, so we read ``location`` off the redirect + response itself instead of letting httpx chase it. + """ + resp = self._send( + "GET", + f"/api/v1/runs/{run_id}/trace", + session_token=session_token, + follow_redirects=False, + ) + if resp.is_redirect: + location = resp.headers.get("location") + if location: + return location + # Not a redirect (or a redirect without Location, which shouldn't + # happen): let _unwrap raise the server's error envelope (e.g. 404 + # trace_unavailable), or fall through to a generic failure below. + self._unwrap(resp) + raise SanadError( + "internal_error", + "Trace endpoint did not return a redirect.", + status=resp.status_code, + retryable=True, + ) + # -- high level ------------------------------------------------------- def poll_until_complete( self, diff --git a/src/kimi_cli/sanad/models.py b/src/kimi_cli/sanad/models.py index e96609b8e..a6e5b109a 100644 --- a/src/kimi_cli/sanad/models.py +++ b/src/kimi_cli/sanad/models.py @@ -90,3 +90,30 @@ class UsageSummary(_Camel): limit: int period_end: str | None = None by_model: list[UsageByModel] = [] + + +class DeployResult(_Camel): + """Result of the three-call ``deploy_agent`` flow (agent upsert → version → deployment). + + Not parsed from a single server envelope — the client assembles it from + the three POST responses — but it shares ``_Camel``'s alias generator so + the CLI can print it ``by_alias=True`` in the camelCase shape the server + itself uses (``agentId``/``versionId``/``deploymentId``/``contentHash``). + """ + + agent_id: str + version_id: str + deployment_id: str + content_hash: str + + +class RunRow(_Camel): + """One row from ``GET /api/v1/runs`` or ``GET /api/v1/runs/{id}``.""" + + id: str + status: str + error_code: str | None = None + created_at: str + cost_usd_micros: int + tokens_in: int + tokens_out: int diff --git a/tests/worker/test_client_worker.py b/tests/worker/test_client_worker.py new file mode 100644 index 000000000..b4fa00a77 --- /dev/null +++ b/tests/worker/test_client_worker.py @@ -0,0 +1,193 @@ +"""SanadClient worker-agent methods: deploy/runs/logs/pause/resume (Task 9). + +Envelope shapes here follow the actual routes (not the earlier plan sketch): +POST /api/v1/agents returns ``{data: {agentId, name, workspace}}`` (agentId, +not id); GET /api/v1/runs returns ``{data: {runs: [...]}}`` (nested, not a +bare list) — see control-plane/artifacts/sanad-web/app/api/v1/agents/route.ts +and .../runs/route.ts. +""" + +from __future__ import annotations + +import httpx + +from kimi_cli.sanad.client import SanadClient +from kimi_cli.sanad.errors import SanadError +from kimi_cli.sanad.settings import SanadSettings + + +def _client(handler) -> SanadClient: + settings = SanadSettings(api_base_url="https://cp.test") + return SanadClient(settings, transport=httpx.MockTransport(handler)) + + +def test_deploy_agent_three_calls_in_order() -> None: + calls: list[str] = [] + + def handler(request: httpx.Request) -> httpx.Response: + calls.append(f"{request.method} {request.url.path}") + if request.url.path == "/api/v1/agents": + return httpx.Response(200, json={"data": {"agentId": "ag_1", "name": "t"}}) + if request.url.path == "/api/v1/agents/t/versions": + return httpx.Response( + 200, json={"data": {"versionId": "av_1", "contentHash": "aa" * 32}} + ) + return httpx.Response(200, json={"data": {"deploymentId": "dp_1"}}) + + out = _client(handler).deploy_agent("sess", name="t", files={"agent.yaml": "x"}, env="dev") + assert calls == [ + "POST /api/v1/agents", + "POST /api/v1/agents/t/versions", + "POST /api/v1/agents/t/deployments", + ] + assert out.agent_id == "ag_1" + assert out.version_id == "av_1" + assert out.deployment_id == "dp_1" + assert out.content_hash == "aa" * 32 + + +def test_deploy_agent_sends_files_and_env_downstream() -> None: + seen: dict[str, object] = {} + + def handler(request: httpx.Request) -> httpx.Response: + import json as _json + + if request.url.path == "/api/v1/agents": + body = _json.loads(request.content) + seen["create"] = body + return httpx.Response(200, json={"data": {"agentId": "ag_1", "name": "t"}}) + if request.url.path == "/api/v1/agents/t/versions": + seen["files"] = _json.loads(request.content)["files"] + return httpx.Response( + 200, json={"data": {"versionId": "av_1", "contentHash": "bb" * 32}} + ) + seen["deploy"] = _json.loads(request.content) + return httpx.Response(200, json={"data": {"deploymentId": "dp_1"}}) + + _client(handler).deploy_agent( + "sess", name="t", files={"agent.yaml": "x", "worker.yaml": "y"}, env="dev", workspace="ws1" + ) + assert seen["create"] == {"name": "t", "workspace": "ws1"} + assert seen["files"] == {"agent.yaml": "x", "worker.yaml": "y"} + assert seen["deploy"] == {"versionId": "av_1", "env": "dev"} + + +def test_list_runs_unwraps_nested_envelope() -> None: + def handler(request: httpx.Request) -> httpx.Response: + assert request.url.params["agent"] == "t" + assert "env" not in request.url.params + return httpx.Response( + 200, + json={ + "data": { + "runs": [ + { + "id": "r_abcabcabcabc", + "status": "succeeded", + "errorCode": None, + "createdAt": "2026-08-13T00:00:00Z", + "costUsdMicros": 12, + "tokensIn": 5, + "tokensOut": 7, + } + ] + } + }, + ) + + rows = _client(handler).list_runs("sess", agent="t", env=None) + assert rows[0].id == "r_abcabcabcabc" + assert rows[0].status == "succeeded" + assert rows[0].error_code is None + assert rows[0].cost_usd_micros == 12 + assert rows[0].tokens_in == 5 + assert rows[0].tokens_out == 7 + + +def test_get_run_unwraps_nested_envelope() -> None: + def handler(request: httpx.Request) -> httpx.Response: + assert request.url.path == "/api/v1/runs/r_1" + return httpx.Response( + 200, + json={ + "data": { + "run": { + "id": "r_1", + "status": "failed", + "errorCode": "provider_error", + "createdAt": "2026-08-13T00:00:00Z", + "costUsdMicros": 0, + "tokensIn": 1, + "tokensOut": 0, + } + } + }, + ) + + row = _client(handler).get_run("sess", "r_1") + assert row.id == "r_1" + assert row.status == "failed" + assert row.error_code == "provider_error" + + +def test_get_run_trace_url_reads_redirect_location() -> None: + def handler(request: httpx.Request) -> httpx.Response: + assert request.url.path == "/api/v1/runs/r_1/trace" + return httpx.Response(307, headers={"location": "https://s3.example.test/trace.json"}) + + url = _client(handler).get_run_trace_url("sess", "r_1") + assert url == "https://s3.example.test/trace.json" + + +def test_get_run_trace_url_raises_on_trace_unavailable() -> None: + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response( + 404, + json={ + "error": { + "code": "trace_unavailable", + "message": "This run has no uploaded trace", + "requestId": "r", + "retryable": False, + } + }, + ) + + try: + _client(handler).get_run_trace_url("sess", "r_1") + raise AssertionError("expected SanadError") + except SanadError as exc: + assert exc.code == "trace_unavailable" + assert exc.status == 404 + + +def test_set_deployment_status_patches_env_and_status() -> None: + def handler(request: httpx.Request) -> httpx.Response: + assert request.method == "PATCH" + assert request.url.path == "/api/v1/agents/t/deployments" + return httpx.Response( + 200, json={"data": {"agentId": "ag_1", "env": "dev", "status": "paused"}} + ) + + _client(handler).set_deployment_status("sess", agent="t", env="dev", status="paused") + + +def test_set_deployment_status_raises_not_deployed() -> None: + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response( + 404, + json={ + "error": { + "code": "not_deployed", + "message": "no active deployment for env", + "requestId": "r", + "retryable": False, + } + }, + ) + + try: + _client(handler).set_deployment_status("sess", agent="t", env="dev", status="paused") + raise AssertionError("expected SanadError") + except SanadError as exc: + assert exc.code == "not_deployed" diff --git a/tests/worker/test_worker_cli.py b/tests/worker/test_worker_cli.py new file mode 100644 index 000000000..e2e8e1c46 --- /dev/null +++ b/tests/worker/test_worker_cli.py @@ -0,0 +1,343 @@ +"""CLI verb tests for `kimi agent deploy/runs/logs/pause/resume` (Task 9). + +Session-token resolution and the SanadClient are both injected via the +module-level seams (`_build_session`/`_build_client`) the same way +tests/sanad/test_cli.py fakes `_build_session` for the `sanad` app. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import httpx +from typer.testing import CliRunner + +from kimi_cli.cli import worker as worker_cli +from kimi_cli.sanad.client import SanadClient +from kimi_cli.sanad.session import SanadSession +from kimi_cli.sanad.settings import SanadSettings +from tests.sanad.test_session import FakeKeychain, ok + +runner = CliRunner() + +AGENT_YAML = "version: '1'\nagent:\n name: t\n system_prompt_path: prompt.md\n tools: []\n" +WORKER_YAML = "interface:\n inputs: {q: string}\n outputs: {answer: string}\n" + + +def _write_bundle(work_dir: Path) -> None: + (work_dir / "agent.yaml").write_text(AGENT_YAML) + (work_dir / "prompt.md").write_text("You are a test agent.") + (work_dir / "worker.yaml").write_text(WORKER_YAML) + + +def _install_client(monkeypatch, handler) -> SanadClient: + """Client used for the actual deploy/runs/logs/pause/resume network calls.""" + client = SanadClient( + SanadSettings(api_base_url="https://cp.test"), transport=httpx.MockTransport(handler) + ) + monkeypatch.setattr(worker_cli, "_build_client", lambda: client) + return client + + +def _install_signed_in(monkeypatch, token: str = "sess-1") -> None: + """Session used purely for token resolution — its own client is never called.""" + + def _boom(request: httpx.Request) -> httpx.Response: + raise AssertionError("session's own client should never be called by worker verbs") + + session = SanadSession( + client=SanadClient( + SanadSettings(api_base_url="https://cp.test"), transport=httpx.MockTransport(_boom) + ), + keychain=FakeKeychain(token), # type: ignore[arg-type] + ) + monkeypatch.setattr(worker_cli, "_build_session", lambda: session) + + +def _install_signed_out(monkeypatch) -> None: + session = SanadSession( + client=SanadClient(SanadSettings(api_base_url="https://cp.test")), + keychain=FakeKeychain(None), # type: ignore[arg-type] + ) + monkeypatch.setattr(worker_cli, "_build_session", lambda: session) + + +# -- deploy ------------------------------------------------------------- + + +def test_deploy_broken_agent_yaml_exits_4_without_any_http_call(tmp_path, monkeypatch) -> None: + called: list[str] = [] + + def handler(request: httpx.Request) -> httpx.Response: + called.append(request.url.path) + return ok({}) + + # Signed in (so a missing token can't be the reason we never call out) — + # but the bundle never validates, so this handler must stay untouched. + _install_signed_in(monkeypatch) + _install_client(monkeypatch, handler) + + (tmp_path / "agent.yaml").write_text("not: {valid") # invalid YAML + (tmp_path / "worker.yaml").write_text(WORKER_YAML) + + result = runner.invoke( + worker_cli.cli, ["deploy", "--work-dir", str(tmp_path)], catch_exceptions=False + ) + + assert result.exit_code == 4, result.output + assert called == [] + + +def test_deploy_missing_worker_yaml_exits_4_without_any_http_call(tmp_path, monkeypatch) -> None: + called: list[str] = [] + + def handler(request: httpx.Request) -> httpx.Response: + called.append(request.url.path) + return ok({}) + + _install_signed_in(monkeypatch) + _install_client(monkeypatch, handler) + + (tmp_path / "agent.yaml").write_text(AGENT_YAML) + (tmp_path / "prompt.md").write_text("hi") + # worker.yaml intentionally absent + + result = runner.invoke( + worker_cli.cli, ["deploy", "--work-dir", str(tmp_path)], catch_exceptions=False + ) + + assert result.exit_code == 4, result.output + assert called == [] + + +def test_deploy_success_prints_camel_case_json(tmp_path, monkeypatch) -> None: + _write_bundle(tmp_path) + seen_files: dict[str, str] = {} + + def handler(request: httpx.Request) -> httpx.Response: + if request.url.path == "/api/v1/agents": + assert json.loads(request.content) == {"name": "t", "workspace": "default"} + return httpx.Response(200, json={"data": {"agentId": "ag_1", "name": "t"}}) + if request.url.path == "/api/v1/agents/t/versions": + seen_files.update(json.loads(request.content)["files"]) + return httpx.Response( + 200, json={"data": {"versionId": "av_1", "contentHash": "cc" * 32}} + ) + assert request.url.path == "/api/v1/agents/t/deployments" + assert json.loads(request.content) == {"versionId": "av_1", "env": "dev"} + return httpx.Response(200, json={"data": {"deploymentId": "dp_1"}}) + + _install_signed_in(monkeypatch) + _install_client(monkeypatch, handler) + + result = runner.invoke( + worker_cli.cli, ["deploy", "--work-dir", str(tmp_path)], catch_exceptions=False + ) + + assert result.exit_code == 0, result.output + payload = json.loads(result.output) + assert payload == { + "agentId": "ag_1", + "versionId": "av_1", + "deploymentId": "dp_1", + "contentHash": "cc" * 32, + } + assert seen_files == { + "agent.yaml": AGENT_YAML, + "worker.yaml": WORKER_YAML, + "prompt.md": "You are a test agent.", + } + + +def test_deploy_not_signed_in_exits_nonzero_without_http_call(tmp_path, monkeypatch) -> None: + _write_bundle(tmp_path) + called: list[str] = [] + + def handler(request: httpx.Request) -> httpx.Response: + called.append(request.url.path) + return ok({}) + + _install_signed_out(monkeypatch) + _install_client(monkeypatch, handler) + + result = runner.invoke( + worker_cli.cli, ["deploy", "--work-dir", str(tmp_path)], catch_exceptions=False + ) + + assert result.exit_code == 1, result.output + assert called == [] + assert "not signed in" in result.output.lower() + + +# -- runs ----------------------------------------------------------------- + + +def test_runs_table_renders_cost_and_tokens(monkeypatch) -> None: + def handler(request: httpx.Request) -> httpx.Response: + assert request.url.path == "/api/v1/runs" + return httpx.Response( + 200, + json={ + "data": { + "runs": [ + { + "id": "r_1", + "status": "succeeded", + "errorCode": None, + "createdAt": "2026-08-13T00:00:00Z", + "costUsdMicros": 12345, + "tokensIn": 100, + "tokensOut": 50, + } + ] + } + }, + ) + + _install_signed_in(monkeypatch) + _install_client(monkeypatch, handler) + + result = runner.invoke(worker_cli.cli, ["runs"], catch_exceptions=False) + + assert result.exit_code == 0, result.output + assert "r_1" in result.output + assert "$0.0123" in result.output + assert "100" in result.output + assert "50" in result.output + + +def test_runs_json_flag_emits_raw_rows(monkeypatch) -> None: + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, + json={ + "data": { + "runs": [ + { + "id": "r_1", + "status": "succeeded", + "errorCode": None, + "createdAt": "2026-08-13T00:00:00Z", + "costUsdMicros": 0, + "tokensIn": 0, + "tokensOut": 0, + } + ] + } + }, + ) + + _install_signed_in(monkeypatch) + _install_client(monkeypatch, handler) + + result = runner.invoke(worker_cli.cli, ["runs", "--json"], catch_exceptions=False) + + assert result.exit_code == 0, result.output + rows = json.loads(result.output) + assert rows == [ + { + "id": "r_1", + "status": "succeeded", + "errorCode": None, + "createdAt": "2026-08-13T00:00:00Z", + "costUsdMicros": 0, + "tokensIn": 0, + "tokensOut": 0, + } + ] + + +# -- logs ------------------------------------------------------------------- + + +def test_logs_prints_trace_url(monkeypatch) -> None: + def handler(request: httpx.Request) -> httpx.Response: + assert request.url.path == "/api/v1/runs/r_1/trace" + return httpx.Response(307, headers={"location": "https://s3.example.test/t.json"}) + + _install_signed_in(monkeypatch) + _install_client(monkeypatch, handler) + + result = runner.invoke(worker_cli.cli, ["logs", "r_1"], catch_exceptions=False) + + assert result.exit_code == 0, result.output + assert result.output.strip() == "https://s3.example.test/t.json" + + +def test_logs_trace_unavailable_exits_1(monkeypatch) -> None: + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response( + 404, + json={ + "error": { + "code": "trace_unavailable", + "message": "This run has no uploaded trace", + "requestId": "r", + "retryable": False, + } + }, + ) + + _install_signed_in(monkeypatch) + _install_client(monkeypatch, handler) + + result = runner.invoke(worker_cli.cli, ["logs", "r_1"], catch_exceptions=False) + + assert result.exit_code == 1, result.output + assert "no uploaded trace" in result.output.lower() + + +# -- pause / resume ----------------------------------------------------- + + +def test_pause_maps_to_patch_paused(monkeypatch) -> None: + def handler(request: httpx.Request) -> httpx.Response: + assert request.method == "PATCH" + assert request.url.path == "/api/v1/agents/t/deployments" + assert json.loads(request.content) == {"env": "dev", "status": "paused"} + return httpx.Response( + 200, json={"data": {"agentId": "ag_1", "env": "dev", "status": "paused"}} + ) + + _install_signed_in(monkeypatch) + _install_client(monkeypatch, handler) + + result = runner.invoke(worker_cli.cli, ["pause", "t"], catch_exceptions=False) + assert result.exit_code == 0, result.output + + +def test_resume_maps_to_patch_active(monkeypatch) -> None: + def handler(request: httpx.Request) -> httpx.Response: + assert json.loads(request.content) == {"env": "prod", "status": "active"} + return httpx.Response( + 200, json={"data": {"agentId": "ag_1", "env": "prod", "status": "active"}} + ) + + _install_signed_in(monkeypatch) + _install_client(monkeypatch, handler) + + result = runner.invoke(worker_cli.cli, ["resume", "t", "--env", "prod"], catch_exceptions=False) + assert result.exit_code == 0, result.output + + +def test_pause_not_deployed_exits_1_with_server_message(monkeypatch) -> None: + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response( + 404, + json={ + "error": { + "code": "not_deployed", + "message": "no active deployment for env", + "requestId": "r", + "retryable": False, + } + }, + ) + + _install_signed_in(monkeypatch) + _install_client(monkeypatch, handler) + + result = runner.invoke(worker_cli.cli, ["pause", "t"], catch_exceptions=False) + assert result.exit_code == 1, result.output + assert "no active deployment" in result.output.lower() From ab624fa09a7b1700e08c6913e86dc9b3ad17b81c Mon Sep 17 00:00:00 2001 From: Omar Alsoulah Date: Thu, 13 Aug 2026 19:42:39 +0300 Subject: [PATCH 17/28] =?UTF-8?q?sanad:=20RunRunner=20=E2=80=94=20one-turn?= =?UTF-8?q?=20wire=20runner=20with=20token=20budget=20and=20finished=20hoo?= =?UTF-8?q?k?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/sanad_terminal/run_runner.py | 214 ++++++++++++++++++ .../src/sanad_terminal/settings.py | 15 ++ .../src/sanad_terminal/wire_runner.py | 18 ++ terminal-server/tests/_fake_worker_wire.py | 109 +++++++++ terminal-server/tests/test_run_runner.py | 105 +++++++++ 5 files changed, 461 insertions(+) create mode 100644 terminal-server/src/sanad_terminal/run_runner.py create mode 100644 terminal-server/tests/_fake_worker_wire.py create mode 100644 terminal-server/tests/test_run_runner.py diff --git a/terminal-server/src/sanad_terminal/run_runner.py b/terminal-server/src/sanad_terminal/run_runner.py new file mode 100644 index 000000000..21556ebf2 --- /dev/null +++ b/terminal-server/src/sanad_terminal/run_runner.py @@ -0,0 +1,214 @@ +"""One ephemeral worker run = one wire subprocess. Sibling of CoderRunner. + +A run is server-minted (`r_`), machine-global (the machine is +single-workspace by construction, so the registry keys by bare run id — no +root-scoping needed the way CoderRunner's conversations need), and consumes +exactly one turn: P0 worker runs are afk (no browser attached, no approvals +UI), so a second `start_turn` is a programming error, not a queued follow-up. + +Token budget mirrors the wall-clock/step budgets in `wire_runner.py`, but +neither of those knows about token usage — that only exists inside +`StatusUpdate` event payloads, which is why `WireRunner.observe_event` exists +as a seam: the base class journals every event and hands it to this hook +as a no-op, and only RunRunner overrides it to accumulate usage and trip +`_trip_budget` when the run's token ceiling is exceeded. +""" + +from __future__ import annotations + +import asyncio +import re +from collections.abc import Awaitable, Callable, Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from sanad_terminal.wire_runner import TurnState, WireRunner, WireRunnerError, register_registry + +# Server-minted only (P0); the shape keeps ids path- and shell-safe. +RUN_ID_RE = re.compile(r"^r_[a-f0-9]{12}$") + + +@dataclass(frozen=True, slots=True) +class RunDirs: + root: Path + workspace: Path + home: Path + share: Path + bundle: Path + output_file: Path + interface_file: Path + + +def prepare_run_dirs(deployment_root: Path, run_id: str) -> RunDirs: + """Create (idempotently) the per-run directory layout under + `/runs//`, each of workspace/home/kimi-share/ + bundle locked to 0o700 — a run's files are never group- or world-readable. + """ + root = deployment_root / "runs" / run_id + sub = {name: root / name for name in ("workspace", "home", "kimi-share", "bundle")} + for d in sub.values(): + d.mkdir(parents=True, exist_ok=True) + d.chmod(0o700) + return RunDirs( + root=root, + workspace=sub["workspace"], + home=sub["home"], + share=sub["kimi-share"], + bundle=sub["bundle"], + output_file=root / "output.json", + interface_file=sub["bundle"] / "worker.yaml", + ) + + +class RunRunner(WireRunner): + """P0 posture: capabilities are false/false, so the base rejects every + inbound request (any gated tool call resolves DENIED) — a worker run has + no browser attached to answer an approval or question. Budgets (wall + clock, steps, tokens) are all mandatory: an afk run must be bounded on + every axis, not just the ones WireRunner already knows about. + """ + + def __init__( + self, + *, + run_id: str, + argv: Sequence[str], + cwd: Path, + env: dict[str, str], + uid: int | None = None, + gid: int | None = None, + max_turn_seconds: float, + max_steps_per_turn: int, + max_tokens_per_run: int, + on_finished: Callable[[RunRunner], Awaitable[None]] | None = None, + ) -> None: + super().__init__( + argv=argv, + cwd=cwd, + env=env, + uid=uid, + gid=gid, + client_name="sanad-worker", + capabilities={"supports_question": False, "supports_plan_mode": False}, + max_turn_seconds=max_turn_seconds, + max_steps_per_turn=max_steps_per_turn, + ) + self.run_id = run_id + self._max_tokens = max_tokens_per_run + self._tokens_in = 0 + self._tokens_out = 0 + self._model_alias: str | None = None + self._consumed = False + self._on_finished = on_finished + self._finished_fired = False + self._finish_task: asyncio.Task[None] | None = None + self._token_trip_task: asyncio.Task[None] | None = None + + async def start_turn(self, user_input: str, send_id: str | None = None) -> TurnState: + """Exactly one turn per run: a second call is a hard error unless it + replays the same `send_id` as the turn already in flight/finished + (the same idempotency the base class gives every runner).""" + if self._consumed: + cur = self._current + if send_id and cur is not None and cur.send_id == send_id: + return cur + raise WireRunnerError("run_consumed", "this run already executed its turn") + state = await super().start_turn(user_input, send_id) + self._consumed = True + return state + + def observe_event(self, envelope: dict[str, Any]) -> None: + """Accumulate token usage from `StatusUpdate` events and trip the + token budget when the run's ceiling is exceeded. Called synchronously + from `WireRunner._consume` for every journaled event.""" + if envelope.get("type") != "StatusUpdate": + return + payload = envelope.get("payload") or {} + if not isinstance(payload, dict): + return + # `StatusUpdate` (kimi_cli.wire.types) carries no model identifier + # field today — only context/token usage and plan-mode state — so + # `_model_alias` stays None here. If a future wire revision adds one + # (e.g. a `model` or `model_alias` key), capture it the same way + # `token_usage` is read below. + usage = payload.get("token_usage") or {} + if not isinstance(usage, dict): + return + self._tokens_in += ( + int(usage.get("input_other", 0) or 0) + + int(usage.get("input_cache_read", 0) or 0) + + int(usage.get("input_cache_creation", 0) or 0) + ) + self._tokens_out += int(usage.get("output", 0) or 0) + if self._tokens_in + self._tokens_out > self._max_tokens and self._current is not None: + self._schedule_trip(self._current, "token budget exceeded") + + def _schedule_trip(self, state: TurnState, reason: str) -> None: + """Task wrapper matching how the wall-clock/step watchers trip the + budget — `_trip_budget` itself is idempotent, so a burst of events + past the threshold still yields exactly one journaled breach.""" + if state.budget_tripped: + return + if self._token_trip_task is not None and not self._token_trip_task.done(): + return + self._token_trip_task = asyncio.create_task(self._trip_budget(state, reason)) + + def usage_totals(self) -> dict[str, Any]: + return { + "tokensIn": self._tokens_in, + "tokensOut": self._tokens_out, + "modelAlias": self._model_alias, + } + + def terminal_item(self) -> dict[str, Any] | None: + """The consumed turn's final `end`/`error` journal item, if any — + Task 12 reads this to decide the run's final status for the report.""" + state = self._current + if state is None or not state.items: + return None + for item in reversed(state.items): + if item.get("kind") in ("end", "error"): + return item + return None + + async def wait_finished_hooks(self) -> None: + """Await the `on_finished` callback task, if one was scheduled — lets + tests (and callers that need the side effect to have landed) block on + it deterministically instead of racing the background task.""" + if self._finish_task is not None: + await self._finish_task + + async def stop(self) -> None: + """Mirrors the base's own budget-task cleanup for the token-trip + task, which the base doesn't know about.""" + if self._token_trip_task is not None and not self._token_trip_task.done(): + self._token_trip_task.cancel() + self._token_trip_task = None + await super().stop() + + +# Registry of live runs, keyed by bare run id — the machine is +# single-workspace by construction, unlike CoderRunner's conversations which +# are scoped by workspace root. Registered with wire_runner so an active run +# holds the machine open (IdleStopper probe). +_runs: dict[str, RunRunner] = {} +register_registry(_runs) + + +def get_run(run_id: str) -> RunRunner | None: + return _runs.get(run_id) + + +def put_run(runner: RunRunner) -> None: + _runs[runner.run_id] = runner + + +async def drop_run(run_id: str) -> None: + runner = _runs.pop(run_id, None) + if runner is not None: + await runner.stop() + + +def live_run_count() -> int: + return len(_runs) diff --git a/terminal-server/src/sanad_terminal/settings.py b/terminal-server/src/sanad_terminal/settings.py index 66ae36124..00169da72 100644 --- a/terminal-server/src/sanad_terminal/settings.py +++ b/terminal-server/src/sanad_terminal/settings.py @@ -55,6 +55,16 @@ class TerminalSettings: # 24h-token ceilings; a runaway browser-driven turn burns quota unattended. coder_max_turn_seconds: float = 3600.0 coder_max_steps_per_turn: int = 200 + # -- worker runs (P0) ------------------------------------------------------- + # Default-off master switch for ephemeral worker runs; "1" is the only truthy. + worker_enabled: bool = False + # Per-run budgets — a worker run is afk (no attached browser can cancel it + # early), so these are the only backstop against a runaway subprocess. + worker_max_turn_seconds: float = 900.0 + worker_max_steps_per_turn: int = 100 + worker_max_tokens_per_run: int = 2_000_000 + # Keep the underlying compute warm between runs instead of scaling to zero. + keep_warm: bool = False @classmethod def load(cls, env: Mapping[str, str] | None = None) -> TerminalSettings: @@ -119,4 +129,9 @@ def load(cls, env: Mapping[str, str] | None = None) -> TerminalSettings: coder_enabled=e.get("CODER_ENABLED", "") == "1", coder_max_turn_seconds=float(e.get("CODER_MAX_TURN_SECONDS", "3600")), coder_max_steps_per_turn=int(e.get("CODER_MAX_STEPS_PER_TURN", "200")), + worker_enabled=e.get("WORKER_ENABLED", "") == "1", + worker_max_turn_seconds=float(e.get("WORKER_MAX_TURN_SECONDS", "900")), + worker_max_steps_per_turn=int(e.get("WORKER_MAX_STEPS_PER_TURN", "100")), + worker_max_tokens_per_run=int(e.get("WORKER_MAX_TOKENS_PER_RUN", "2000000")), + keep_warm=e.get("KEEP_WARM", "") == "1", ) diff --git a/terminal-server/src/sanad_terminal/wire_runner.py b/terminal-server/src/sanad_terminal/wire_runner.py index 55ccf0104..492977fe3 100644 --- a/terminal-server/src/sanad_terminal/wire_runner.py +++ b/terminal-server/src/sanad_terminal/wire_runner.py @@ -287,6 +287,8 @@ async def _consume(self, state: TurnState, queue: asyncio.Queue[dict[str, Any]]) kind = item.get("kind") if kind == "event": event = item.get("event") or {} + if isinstance(event, dict): + self.observe_event(event) if isinstance(event, dict) and event.get("type") == "StepBegin": state.steps += 1 if ( @@ -330,6 +332,14 @@ async def _consume(self, state: TurnState, queue: asyncio.Queue[dict[str, Any]]) self._trip_task.cancel() self._trip_task = None self._touch() + # Terminal-status hook (RunRunner only — `_on_finished` doesn't + # exist on the base/coder runners, so this is a no-op for them): + # fire exactly once per turn, as a background task so a slow + # callback (upload + report) never blocks the journal. + on_finished = getattr(self, "_on_finished", None) + if on_finished is not None and not getattr(self, "_finished_fired", False): + self._finished_fired = True + self._finish_task = asyncio.create_task(on_finished(self)) async with self._journal_cond: self._journal_cond.notify_all() @@ -489,6 +499,14 @@ def _dispatch(self, msg: dict[str, Any]) -> None: if fut is not None and not fut.done(): fut.set_result(msg) + def observe_event(self, envelope: dict[str, Any]) -> None: + """Hook fired for every wire event, after it's journaled. Base: no-op. + + Subclasses (RunRunner) override this to accumulate token usage from + StatusUpdate events and trip a token budget — a seam rather than a + base-class field so architect/coder behavior is untouched. + """ + def on_request(self, rid: Any, params: dict[str, Any]) -> bool: """Handle an inbound JSON-RPC request. Base: unhandled → caller rejects. diff --git a/terminal-server/tests/_fake_worker_wire.py b/terminal-server/tests/_fake_worker_wire.py new file mode 100644 index 000000000..e9eedb492 --- /dev/null +++ b/terminal-server/tests/_fake_worker_wire.py @@ -0,0 +1,109 @@ +"""A minimal stand-in for `sanad --wire` used in RunRunner tests. + +Modes are keyed on the prompt text: +- default: TurnBegin + one event + writes `{"answer": "fake"}` to + `$KIMI_WORKER_OUTPUT_FILE` (if set), then finishes — so + runner tests exercise the output-file path without a real + model. +- "HANG": TurnBegin, then the turn stays open until a cancel arrives + (the wall-clock-budget and cancel paths). +- "STEPHANG:": n StepBegin events, then hang until cancel (step budget). +- "TOKENS:": emits one StatusUpdate event whose token_usage totals `n` + output tokens, then hangs until cancel (the token-budget + path: the runner is expected to trip and cancel it). +""" + +import json +import os +import sys + + +def _write(obj: dict) -> None: + sys.stdout.write(json.dumps(obj) + "\n") + sys.stdout.flush() + + +def _event(type_name: str, payload: dict) -> None: + _write({"jsonrpc": "2.0", "method": "event", "params": {"type": type_name, "payload": payload}}) + + +def _read() -> dict | None: + raw = sys.stdin.readline() + if not raw: + return None + raw = raw.strip() + if not raw: + return {} + try: + msg = json.loads(raw) + return msg if isinstance(msg, dict) else {} + except ValueError: + return {} + + +def _hang_until_cancel(prompt_id) -> None: + """Keep the turn open; resolve it as cancelled when the bridge says so.""" + while True: + msg = _read() + if msg is None: + return + if msg.get("method") == "cancel": + _write({"jsonrpc": "2.0", "id": msg.get("id"), "result": {}}) + _write({"jsonrpc": "2.0", "id": prompt_id, "result": {"status": "cancelled"}}) + return + + +def _write_output_file() -> None: + path = os.environ.get("KIMI_WORKER_OUTPUT_FILE") + if not path: + return + with open(path, "w") as f: + json.dump({"answer": "fake"}, f) + + +def main() -> None: + while True: + msg = _read() + if msg is None: + return + method = msg.get("method") + mid = msg.get("id") + if method == "initialize": + caps = msg.get("params", {}).get("capabilities", {}) + _write( + { + "jsonrpc": "2.0", + "id": mid, + "result": { + "protocol_version": "1.10", + "server": {"name": "fake-worker", "version": "0"}, + "capabilities": caps, + }, + } + ) + elif method == "prompt": + user_input = msg.get("params", {}).get("user_input", "") + _event("TurnBegin", {"user_input": user_input}) + if user_input.startswith("STEPHANG:"): + for i in range(int(user_input.split(":", 1)[1])): + _event("StepBegin", {"step": i}) + _hang_until_cancel(mid) + elif user_input.startswith("TOKENS:"): + n = int(user_input.split(":", 1)[1]) + _event( + "StatusUpdate", + {"token_usage": {"input_other": 0, "output": n}}, + ) + _hang_until_cancel(mid) + elif "HANG" in user_input: + _hang_until_cancel(mid) + else: + _event("TextPart", {"type": "text", "text": "hello from worker"}) + _write_output_file() + _write({"jsonrpc": "2.0", "id": mid, "result": {"status": "finished"}}) + elif method == "cancel": + _write({"jsonrpc": "2.0", "id": mid, "result": {}}) + + +if __name__ == "__main__": + main() diff --git a/terminal-server/tests/test_run_runner.py b/terminal-server/tests/test_run_runner.py new file mode 100644 index 000000000..6df4dda5e --- /dev/null +++ b/terminal-server/tests/test_run_runner.py @@ -0,0 +1,105 @@ +import sys +from pathlib import Path + +import pytest + +from sanad_terminal.run_runner import ( + RUN_ID_RE, RunRunner, get_run, prepare_run_dirs, put_run, +) +from sanad_terminal.wire_runner import WireRunnerError + +FAKE_WIRE = Path(__file__).parent / "_fake_worker_wire.py" + + +def _runner(tmp_path: Path, run_id: str = "r_aaaaaaaaaaaa") -> RunRunner: + dirs = prepare_run_dirs(tmp_path, run_id) + return RunRunner( + run_id=run_id, + argv=(sys.executable, str(FAKE_WIRE)), + cwd=dirs.workspace, + env={"KIMI_WORKER_OUTPUT_FILE": str(dirs.output_file)}, + uid=None, gid=None, + max_turn_seconds=30.0, max_steps_per_turn=50, max_tokens_per_run=1000, + ) + + +def test_run_id_re() -> None: + assert RUN_ID_RE.match("r_0123456789ab") + assert not RUN_ID_RE.match("c_0123456789ab") + assert not RUN_ID_RE.match("r_0123456789ABCD") + + +def test_prepare_run_dirs_layout(tmp_path: Path) -> None: + dirs = prepare_run_dirs(tmp_path, "r_aaaaaaaaaaaa") + assert dirs.root == tmp_path / "runs" / "r_aaaaaaaaaaaa" + for d in (dirs.workspace, dirs.home, dirs.share, dirs.bundle): + assert d.is_dir() + assert (d.stat().st_mode & 0o777) == 0o700 + + +async def test_exactly_one_turn(tmp_path: Path) -> None: + runner = _runner(tmp_path) + await runner.start() + state = await runner.start_turn("go", send_id="s1") + async for item in runner.follow(state.turn_id, 0): + if item["kind"] in ("end", "error"): + break + with pytest.raises(WireRunnerError) as exc: + await runner.start_turn("again", send_id="s2") + assert exc.value.code == "run_consumed" + await runner.stop() + + +async def test_same_send_id_replays(tmp_path: Path) -> None: + runner = _runner(tmp_path) + await runner.start() + state = await runner.start_turn("go", send_id="s1") + assert (await runner.start_turn("go", send_id="s1")).turn_id == state.turn_id + await runner.stop() + + +async def test_token_budget_trips_and_journals(tmp_path: Path) -> None: + """`observe_event` (the RunRunner override) sums StatusUpdate token_usage + and, once the run's ceiling is exceeded, schedules the same `_trip_budget` + the wall-clock/step watchers use — journaling a `turn_budget_exceeded` + error and cancelling the turn.""" + run_id = "r_dddddddddddd" + dirs = prepare_run_dirs(tmp_path, run_id) + runner = RunRunner( + run_id=run_id, argv=(sys.executable, str(FAKE_WIRE)), + cwd=dirs.workspace, env={"KIMI_WORKER_OUTPUT_FILE": str(dirs.output_file)}, + uid=None, gid=None, max_turn_seconds=30.0, max_steps_per_turn=50, + max_tokens_per_run=100, + ) + await runner.start() + state = await runner.start_turn("TOKENS:500") + items = [item async for item in runner.follow(state.turn_id, 0)] + codes = [i.get("code") for i in items if i.get("kind") == "error"] + assert "turn_budget_exceeded" in codes + assert state.status == "cancelled" + totals = runner.usage_totals() + assert totals["tokensOut"] == 500 + await runner.stop() + + +async def test_on_finished_fires_once(tmp_path: Path) -> None: + fired: list[str] = [] + + async def on_finished(r: RunRunner) -> None: + fired.append(r.run_id) + + dirs = prepare_run_dirs(tmp_path, "r_bbbbbbbbbbbb") + runner = RunRunner( + run_id="r_bbbbbbbbbbbb", argv=(sys.executable, str(FAKE_WIRE)), + cwd=dirs.workspace, env={"KIMI_WORKER_OUTPUT_FILE": str(dirs.output_file)}, + uid=None, gid=None, max_turn_seconds=30.0, max_steps_per_turn=50, + max_tokens_per_run=1000, on_finished=on_finished, + ) + await runner.start() + state = await runner.start_turn("go") + async for item in runner.follow(state.turn_id, 0): + if item["kind"] in ("end", "error"): + break + await runner.wait_finished_hooks() # helper that awaits the callback task + assert fired == ["r_bbbbbbbbbbbb"] + await runner.stop() From 20e107353b1e015780d016aaa109f26763771bd2 Mon Sep 17 00:00:00 2001 From: Omar Alsoulah Date: Thu, 13 Aug 2026 19:59:54 +0300 Subject: [PATCH 18/28] =?UTF-8?q?sanad:=20RunRunner=20=E2=80=94=20status-g?= =?UTF-8?q?uarded=20token=20trip,=20defensive=20telemetry=20parsing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/sanad_terminal/run_runner.py | 54 ++++++++++++++++--- terminal-server/tests/_fake_worker_wire.py | 23 ++++++++ terminal-server/tests/test_run_runner.py | 50 +++++++++++++++++ 3 files changed, 120 insertions(+), 7 deletions(-) diff --git a/terminal-server/src/sanad_terminal/run_runner.py b/terminal-server/src/sanad_terminal/run_runner.py index 21556ebf2..d00ed5b16 100644 --- a/terminal-server/src/sanad_terminal/run_runner.py +++ b/terminal-server/src/sanad_terminal/run_runner.py @@ -23,12 +23,23 @@ from pathlib import Path from typing import Any +from loguru import logger + from sanad_terminal.wire_runner import TurnState, WireRunner, WireRunnerError, register_registry # Server-minted only (P0); the shape keeps ids path- and shell-safe. RUN_ID_RE = re.compile(r"^r_[a-f0-9]{12}$") +def _as_int(value: Any) -> int: + """Coerce a `token_usage` field to `int`, never raising: a wire + subprocess is untrusted input, and malformed telemetry (a string, list, + dict, None, ...) must degrade to 0 rather than fail the run.""" + if isinstance(value, (int, float)): + return int(value) + return 0 + + @dataclass(frozen=True, slots=True) class RunDirs: root: Path @@ -121,7 +132,20 @@ async def start_turn(self, user_input: str, send_id: str | None = None) -> TurnS def observe_event(self, envelope: dict[str, Any]) -> None: """Accumulate token usage from `StatusUpdate` events and trip the token budget when the run's ceiling is exceeded. Called synchronously - from `WireRunner._consume` for every journaled event.""" + from `WireRunner._consume` for every journaled event. + + Belt-and-suspenders defensive: this is telemetry parsing on data from + a subprocess, and it must never be able to kill a turn by raising + into `_consume` (which has no guard around this call) — a malformed + `token_usage` field would otherwise fail the whole run with no + journal item explaining why. + """ + try: + self._observe_event(envelope) + except Exception: + logger.exception("observe_event failed; ignoring malformed telemetry") + + def _observe_event(self, envelope: dict[str, Any]) -> None: if envelope.get("type") != "StatusUpdate": return payload = envelope.get("payload") or {} @@ -136,23 +160,39 @@ def observe_event(self, envelope: dict[str, Any]) -> None: if not isinstance(usage, dict): return self._tokens_in += ( - int(usage.get("input_other", 0) or 0) - + int(usage.get("input_cache_read", 0) or 0) - + int(usage.get("input_cache_creation", 0) or 0) + _as_int(usage.get("input_other")) + + _as_int(usage.get("input_cache_read")) + + _as_int(usage.get("input_cache_creation")) ) - self._tokens_out += int(usage.get("output", 0) or 0) + self._tokens_out += _as_int(usage.get("output")) if self._tokens_in + self._tokens_out > self._max_tokens and self._current is not None: self._schedule_trip(self._current, "token budget exceeded") def _schedule_trip(self, state: TurnState, reason: str) -> None: """Task wrapper matching how the wall-clock/step watchers trip the budget — `_trip_budget` itself is idempotent, so a burst of events - past the threshold still yields exactly one journaled breach.""" + past the threshold still yields exactly one journaled breach. + + Re-checks `state.status == "running"` (and `not state.budget_tripped`) + as the coroutine's first statement, mirroring `_budget_watch`'s own + guard: `observe_event` runs synchronously inside `_consume`'s event + handling, so it can schedule this task on what turns out to be the + run's LAST event before a natural `finished`/`cancelled`/`failed` end + — without the re-check, this task could still be pending when + `_consume` journals the `end` item and settles the turn, then wake up + and append a stray `turn_budget_exceeded` error AFTER it, corrupting + `terminal_item()` for an otherwise-successful run. + """ if state.budget_tripped: return if self._token_trip_task is not None and not self._token_trip_task.done(): return - self._token_trip_task = asyncio.create_task(self._trip_budget(state, reason)) + self._token_trip_task = asyncio.create_task(self._trip_if_still_running(state, reason)) + + async def _trip_if_still_running(self, state: TurnState, reason: str) -> None: + if state.status != "running" or state.budget_tripped: + return + await self._trip_budget(state, reason) def usage_totals(self) -> dict[str, Any]: return { diff --git a/terminal-server/tests/_fake_worker_wire.py b/terminal-server/tests/_fake_worker_wire.py index e9eedb492..c6bb6db32 100644 --- a/terminal-server/tests/_fake_worker_wire.py +++ b/terminal-server/tests/_fake_worker_wire.py @@ -11,6 +11,12 @@ - "TOKENS:": emits one StatusUpdate event whose token_usage totals `n` output tokens, then hangs until cancel (the token-budget path: the runner is expected to trip and cancel it). +- "TOKENS_THEN_FINISH:": emits the same over-budget StatusUpdate but does + NOT hang — it finishes the turn immediately afterward, the + same scheduling slice a late `_trip_budget` task could + otherwise race against (the "over-budget event was the + run's last one" case that must NOT retroactively mark a + successful run as budget-exceeded). """ import json @@ -88,6 +94,23 @@ def main() -> None: for i in range(int(user_input.split(":", 1)[1])): _event("StepBegin", {"step": i}) _hang_until_cancel(mid) + elif user_input.startswith("TOKENS_THEN_FINISH:"): + # No output-file write here (unlike the default path): that's + # a real disk I/O syscall, and the gap it introduces between + # the two stdout writes is enough for the server's reader + # loop to yield in between them — which gives a scheduled + # trip task room to interleave BEFORE the `finished` response + # is even read, defeating the point of this mode (both + # messages must land in the same read so the runner processes + # them in the same scheduling slice, same as a real model + # emitting a final StatusUpdate immediately before its + # response completes). + n = int(user_input.split(":", 1)[1]) + _event( + "StatusUpdate", + {"token_usage": {"input_other": 0, "output": n}}, + ) + _write({"jsonrpc": "2.0", "id": mid, "result": {"status": "finished"}}) elif user_input.startswith("TOKENS:"): n = int(user_input.split(":", 1)[1]) _event( diff --git a/terminal-server/tests/test_run_runner.py b/terminal-server/tests/test_run_runner.py index 6df4dda5e..d0221fb18 100644 --- a/terminal-server/tests/test_run_runner.py +++ b/terminal-server/tests/test_run_runner.py @@ -1,3 +1,4 @@ +import asyncio import sys from pathlib import Path @@ -82,6 +83,55 @@ async def test_token_budget_trips_and_journals(tmp_path: Path) -> None: await runner.stop() +async def test_token_budget_does_not_corrupt_a_natural_finish(tmp_path: Path) -> None: + """If the over-budget StatusUpdate is the run's LAST event before a + natural `finished`, `_consume` can journal the `end` item and settle the + turn in the same scheduling slice — before the trip task scheduled from + that event ever gets a chance to run. `_schedule_trip`'s wrapper + re-checks `state.status == "running"` as its first statement, so that + late task must no-op instead of appending a stray `turn_budget_exceeded` + error AFTER the turn already finished (which would corrupt + `terminal_item()` for what was actually a successful run).""" + run_id = "r_eeeeeeeeeeee" + dirs = prepare_run_dirs(tmp_path, run_id) + runner = RunRunner( + run_id=run_id, argv=(sys.executable, str(FAKE_WIRE)), + cwd=dirs.workspace, env={"KIMI_WORKER_OUTPUT_FILE": str(dirs.output_file)}, + uid=None, gid=None, max_turn_seconds=30.0, max_steps_per_turn=50, + max_tokens_per_run=100, + ) + await runner.start() + state = await runner.start_turn("TOKENS_THEN_FINISH:500") + async for item in runner.follow(state.turn_id, 0): + if item["kind"] in ("end", "error"): + break + # Give any late-scheduled trip task its chance to run (and no-op) before + # asserting the journal is settled. + for _ in range(10): + await asyncio.sleep(0) + assert state.status == "finished" + item = runner.terminal_item() + assert item is not None + assert item["kind"] == "end" + codes = [i.get("code") for i in state.items if i.get("kind") == "error"] + assert "turn_budget_exceeded" not in codes + await runner.stop() + + +async def test_observe_event_ignores_malformed_token_usage(tmp_path: Path) -> None: + """`observe_event` is telemetry parsing on subprocess-controlled data — + it must degrade malformed fields to 0 (and never raise into `_consume`, + which has no guard around this call).""" + runner = _runner(tmp_path) + runner.observe_event( + { + "type": "StatusUpdate", + "payload": {"token_usage": {"input_other": "garbage", "output": [1]}}, + } + ) + assert runner.usage_totals() == {"tokensIn": 0, "tokensOut": 0, "modelAlias": None} + + async def test_on_finished_fires_once(tmp_path: Path) -> None: fired: list[str] = [] From 479926a2390c4e113a850ad00152f4a6a6439dbb Mon Sep 17 00:00:00 2001 From: Omar Alsoulah Date: Thu, 13 Aug 2026 20:14:24 +0300 Subject: [PATCH 19/28] =?UTF-8?q?sanad:=20worker=20routes=20=E2=80=94=20ga?= =?UTF-8?q?ted=20run=20start/follow/cancel=20with=20bundle=20containment?= =?UTF-8?q?=20+=20keep-warm=20probe?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- terminal-server/pyproject.toml | 8 +- terminal-server/src/sanad_terminal/app.py | 22 ++ .../src/sanad_terminal/routes_worker.py | 248 ++++++++++++++++++ terminal-server/tests/test_routes_worker.py | 144 ++++++++++ 4 files changed, 420 insertions(+), 2 deletions(-) create mode 100644 terminal-server/src/sanad_terminal/routes_worker.py create mode 100644 terminal-server/tests/test_routes_worker.py diff --git a/terminal-server/pyproject.toml b/terminal-server/pyproject.toml index 03aeba99b..e04e170f1 100644 --- a/terminal-server/pyproject.toml +++ b/terminal-server/pyproject.toml @@ -5,8 +5,12 @@ description = "WebSocket PTY bridge + workspace API that runs the governed sanad readme = "README.md" requires-python = ">=3.12" dependencies = [ - # kimi-cli is a dependency solely so the `sanad` console script is - # co-installed in this environment; sanad_terminal never imports kimi_cli. + # kimi-cli is a dependency so the `sanad` console script is co-installed + # in this environment; sanad_terminal never imports kimi_cli EXCEPT for + # routes_worker.py's use of kimi_cli.worker (derive_agent_spec / + # load_worker_spec / render_input_prompt) and kimi_cli.exception + # (AgentSpecError) — the one sanctioned exception, since deriving a + # worker's agent spec is the CLI's own logic and must not be duplicated. "kimi-cli", "sanad-blueprint", "fastapi>=0.115.0", diff --git a/terminal-server/src/sanad_terminal/app.py b/terminal-server/src/sanad_terminal/app.py index 4b4ed4a05..23e45eefe 100644 --- a/terminal-server/src/sanad_terminal/app.py +++ b/terminal-server/src/sanad_terminal/app.py @@ -83,6 +83,10 @@ def create_app( idle_stopper.add_probe( lambda: runners_hold_machine(resolved.idle_stop_seconds) ) + # Worker-serving machines can opt out of scale-to-zero entirely (the + # control plane sets this per-workspace so a hot path never eats a + # cold-start latency hit). + idle_stopper.add_probe(lambda: resolved.keep_warm) @asynccontextmanager async def lifespan(_: FastAPI) -> AsyncIterator[None]: @@ -122,6 +126,10 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]: from sanad_terminal.coder_runner import shutdown_conversations await shutdown_conversations() + from sanad_terminal.run_runner import _runs, drop_run + + for run_id in list(_runs): + await drop_run(run_id) await manager.shutdown() await cp.aclose() @@ -158,6 +166,20 @@ async def _coder_disabled(request, exc): # noqa: ANN001, ANN202 content={"error": {"code": "coder_disabled", "message": "coder panel is not enabled"}}, ) + from sanad_terminal.routes_worker import WorkerDisabled + from sanad_terminal.routes_worker import router as worker_router + + app.include_router(worker_router) + + @app.exception_handler(WorkerDisabled) + async def _worker_disabled(request, exc): # noqa: ANN001, ANN202 + return JSONResponse( + status_code=404, + content={ + "error": {"code": "worker_disabled", "message": "worker runs are not enabled"} + }, + ) + register_error_handlers(app) if idle_stopper is not None: diff --git a/terminal-server/src/sanad_terminal/routes_worker.py b/terminal-server/src/sanad_terminal/routes_worker.py new file mode 100644 index 000000000..293945164 --- /dev/null +++ b/terminal-server/src/sanad_terminal/routes_worker.py @@ -0,0 +1,248 @@ +"""Internal Worker REST (P0) — flag-gated, ephemeral, single-turn agent +runs invoked by the control plane (`POST /v1/agents/{name}/invoke`). + +Unlike the coder/architect bridges, a worker run is server-minted, machine- +global (this machine serves one workspace), and afk: there is no browser +attached to answer approvals or drive a second turn, so `RunRunner` rejects +every inbound request and consumes exactly one `start_turn`. The bundle +(agent.yaml/prompt/worker.yaml/...) arrives inline in the request body and is +written under the run's own sandboxed `bundle/` directory — every relative +path in it must resolve inside that directory, or the run never starts. +""" + +from __future__ import annotations + +import json +import pwd +from collections.abc import AsyncIterator, Awaitable, Callable +from pathlib import Path +from typing import Annotated, Any + +from fastapi import APIRouter, Depends, Request +from fastapi.responses import JSONResponse, StreamingResponse +from pydantic import BaseModel, Field + +from kimi_cli.exception import AgentSpecError +from kimi_cli.worker import ( + WorkerInputError, + WorkerSpecError, + derive_agent_spec, + load_worker_spec, + render_input_prompt, +) +from sanad_terminal.routes_workspace import _settings, workspace_root +from sanad_terminal.run_runner import RUN_ID_RE, RunRunner, get_run, prepare_run_dirs, put_run +from sanad_terminal.wire_runner import WireRunnerError +from sanad_terminal.workspace import build_child_env + +router = APIRouter(prefix="/internal/worker") + +# Depended on purely for its side effect (the same task-mode bearer check +# every other /internal/* route uses) — a worker run's own directories are +# rooted at /runs//, never at the returned workspace path. +Authed = Annotated[Path, Depends(workspace_root)] + + +class WorkerDisabled(Exception): + pass + + +def _gate(request: Request) -> None: + if not _settings(request).worker_enabled: + raise WorkerDisabled() + + +Gated = Annotated[None, Depends(_gate)] + + +def _err(status: int, code: str, message: str) -> JSONResponse: + return JSONResponse(status_code=status, content={"error": {"code": code, "message": message}}) + + +class BundleBody(BaseModel): + files: dict[str, str] = Field(default_factory=dict) + + +class BudgetsBody(BaseModel): + max_turn_seconds: float = Field(alias="maxTurnSeconds") + max_steps_per_turn: int = Field(alias="maxStepsPerTurn") + max_tokens_per_run: int = Field(alias="maxTokensPerRun") + + model_config = {"populate_by_name": True} + + +class RunStartBody(BaseModel): + run_id: str = Field(alias="runId") + send_id: str = Field(alias="sendId") + input: dict[str, Any] = Field(default_factory=dict) + bundle: BundleBody + budgets: BudgetsBody + session_token: str = Field(alias="sessionToken") + trace_upload_url: str = Field(default="", alias="traceUploadUrl") + + model_config = {"populate_by_name": True} + + +def make_on_finished(request: Request) -> Callable[[RunRunner], Awaitable[None]] | None: + """P0 placeholder: nothing observes a run's completion yet, so `RunRunner` + is handed no callback and its terminal-status hook stays dormant. Task 12 + swaps this factory's body for one that uploads the trace and reports the + run's outcome to the control plane — the call site (`on_finished= + make_on_finished(request)`) doesn't change. + """ + del request + return None + + +def _turn_id(runner: RunRunner) -> str | None: + summary = runner.turn_summary() + return summary["turnId"] if summary else None + + +async def _ndjson(items: AsyncIterator[dict[str, Any]]) -> AsyncIterator[bytes]: + try: + async for item in items: + yield json.dumps(item).encode("utf-8") + b"\n" + except WireRunnerError as exc: + yield ( + json.dumps({"kind": "error", "code": exc.code, "message": exc.message}).encode( + "utf-8" + ) + + b"\n" + ) + + +def _stream(runner: RunRunner, turn_id: str, from_seq: int = 0) -> StreamingResponse: + return StreamingResponse( + _ndjson(runner.follow(turn_id, from_seq)), media_type="application/x-ndjson" + ) + + +@router.post("/runs", response_model=None) +async def start_run( + _: Gated, __: Authed, request: Request, body: RunStartBody +) -> StreamingResponse | JSONResponse: + if not RUN_ID_RE.fullmatch(body.run_id): + return _err(400, "bad_run_id", "malformed run id") + + existing = get_run(body.run_id) + if existing is not None: + turn_id = _turn_id(existing) + state = existing.get_turn(turn_id) if turn_id else None + if turn_id is not None and state is not None and state.send_id == body.send_id: + return _stream(existing, turn_id) + return _err(409, "busy_run", "a different run is already using this id") + + settings = _settings(request) + dirs = prepare_run_dirs(settings.data_dir, body.run_id) + + files = body.bundle.files + if not files: + return _err(400, "bad_bundle", "bundle must contain at least one file") + + bundle_root = dirs.bundle.resolve() + for rel, content in files.items(): + rel_path = Path(rel) + if rel_path.is_absolute(): + return _err(400, "bad_bundle", f"absolute path not allowed: {rel}") + resolved = (dirs.bundle / rel_path).resolve() + if not resolved.is_relative_to(bundle_root): + return _err(400, "bad_bundle_path", rel) + resolved.parent.mkdir(parents=True, exist_ok=True) + resolved.write_text(content, encoding="utf-8") + + try: + spec = load_worker_spec(dirs.interface_file) + except WorkerSpecError as exc: + return _err(400, "bad_bundle", str(exc)) + + try: + prompt = render_input_prompt(spec, body.input) + except WorkerInputError as exc: + return _err(400, "bad_input", str(exc)) + + try: + derived = derive_agent_spec(dirs.bundle / "agent.yaml", dirs.bundle) + except (AgentSpecError, OSError) as exc: + return _err(400, "bad_bundle", str(exc)) + + argv = [ + *settings.spawn_argv, + "--wire", + "--session", + body.run_id, + "--agent-file", + str(derived), + "--work-dir", + str(dirs.workspace), + ] + env = build_child_env( + user_dir=dirs.root, + session_token=body.session_token, + api_base_url=settings.child_api_base_url, + cols=80, + rows=24, + ) + env = { + **env, + "KIMI_WORKER_INTERFACE_FILE": str(dirs.interface_file), + "KIMI_WORKER_OUTPUT_FILE": str(dirs.output_file), + "KIMI_SHARE_DIR": str(dirs.share), + } + + uid = gid = None + if settings.agent_user: + pw = pwd.getpwnam(settings.agent_user) + uid, gid = pw.pw_uid, pw.pw_gid + + runner = RunRunner( + run_id=body.run_id, + argv=argv, + cwd=dirs.workspace, + env=env, + uid=uid, + gid=gid, + max_turn_seconds=min(body.budgets.max_turn_seconds, settings.worker_max_turn_seconds), + max_steps_per_turn=min(body.budgets.max_steps_per_turn, settings.worker_max_steps_per_turn), + max_tokens_per_run=min(body.budgets.max_tokens_per_run, settings.worker_max_tokens_per_run), + on_finished=make_on_finished(request), + ) + try: + await runner.start() + except WireRunnerError as exc: + await runner.stop() + return _err(503, exc.code, exc.message) + + put_run(runner) + + try: + state = await runner.start_turn(prompt, body.send_id) + except WireRunnerError as exc: + return _err(409, exc.code, exc.message) + + return _stream(runner, state.turn_id) + + +@router.get("/runs/{rid}/follow", response_model=None) +async def follow_run( + _: Gated, __: Authed, rid: str, from_seq: int = 0 +) -> StreamingResponse | JSONResponse: + if not RUN_ID_RE.fullmatch(rid): + return _err(400, "bad_run_id", "malformed run id") + runner = get_run(rid) + turn_id = _turn_id(runner) if runner is not None else None + if runner is None or turn_id is None: + return _err(404, "unknown_run", "no such run") + return _stream(runner, turn_id, from_seq) + + +@router.post("/runs/{rid}/cancel") +async def cancel_run(_: Gated, __: Authed, rid: str) -> JSONResponse: + if not RUN_ID_RE.fullmatch(rid): + return _err(400, "bad_run_id", "malformed run id") + runner = get_run(rid) + if runner is None: + return _err(404, "unknown_run", "no such run") + if runner.alive: + await runner.cancel() + return JSONResponse({"ok": True}) diff --git a/terminal-server/tests/test_routes_worker.py b/terminal-server/tests/test_routes_worker.py new file mode 100644 index 000000000..73059b97f --- /dev/null +++ b/terminal-server/tests/test_routes_worker.py @@ -0,0 +1,144 @@ +"""Worker runs P0: flag-gated /internal/worker/* — bundle containment, budget +clamping, NDJSON turn streaming, and same-runId+sendId replay.""" + +import json +import sys +from pathlib import Path + +from fastapi.testclient import TestClient +from sanad_terminal.app import create_app +from sanad_terminal.settings import TerminalSettings + +FAKE_WIRE = Path(__file__).parent / "_fake_worker_wire.py" + +# agent.yaml fields must live under a top-level `agent:` key (kimi_cli's +# agentspec loader silently treats a flat document as an empty spec, then +# fails on the (defaulted-to-Inherit) required fields) and `tools: []` is +# explicit because `tools` has no usable default (Inherit with nothing to +# inherit from, since this spec doesn't `extend`). +BUNDLE = { + "agent.yaml": ( + "version: '1'\nagent:\n name: t\n system_prompt_path: prompt.md\n tools: []\n" + ), + "prompt.md": "You are a worker.", + "worker.yaml": "interface:\n inputs: {q: string}\n outputs: {answer: string}\n", +} + + +def _body(run_id: str = "r_aaaaaaaaaaaa") -> dict: + return { + "runId": run_id, "sendId": run_id, "input": {"q": "hi"}, + "bundle": {"files": BUNDLE}, + "budgets": {"maxTurnSeconds": 30, "maxStepsPerTurn": 50, "maxTokensPerRun": 100000}, + "sessionToken": "sess_x", "traceUploadUrl": "https://s3.test/put", + } + + +def _make_client(tmp_path: Path, *, enabled: bool) -> TestClient: + settings = TerminalSettings( + mode="task", + fixed_user="user_1", + agentd_token="tok", + data_dir=tmp_path, + spawn_argv=(sys.executable, str(FAKE_WIRE)), + worker_enabled=enabled, + ) + return TestClient(create_app(settings, control_plane=None)) + + +AUTH = {"authorization": "Bearer tok"} + + +def test_disabled_is_404(tmp_path: Path) -> None: + with _make_client(tmp_path, enabled=False) as c: + r = c.post("/internal/worker/runs", json=_body(), headers=AUTH) + assert r.status_code == 404 + assert r.json()["error"]["code"] == "worker_disabled" + + +def test_bad_run_id_rejected(tmp_path: Path) -> None: + with _make_client(tmp_path, enabled=True) as c: + r = c.post("/internal/worker/runs", json=_body("nope"), headers=AUTH) + assert r.status_code == 400 + assert r.json()["error"]["code"] == "bad_run_id" + + +def test_bundle_traversal_rejected(tmp_path: Path) -> None: + body = _body() + body["bundle"]["files"] = {"../evil.yaml": "x", **BUNDLE} + with _make_client(tmp_path, enabled=True) as c: + r = c.post("/internal/worker/runs", json=body, headers=AUTH) + assert r.status_code == 400 + assert r.json()["error"]["code"] == "bad_bundle_path" + + +def test_bundle_absolute_path_rejected(tmp_path: Path) -> None: + body = _body() + body["bundle"]["files"] = {"/etc/evil.yaml": "x", **BUNDLE} + with _make_client(tmp_path, enabled=True) as c: + r = c.post("/internal/worker/runs", json=body, headers=AUTH) + assert r.status_code == 400 + assert r.json()["error"]["code"] == "bad_bundle" + + +def test_empty_bundle_rejected(tmp_path: Path) -> None: + body = _body() + body["bundle"]["files"] = {} + with _make_client(tmp_path, enabled=True) as c: + r = c.post("/internal/worker/runs", json=body, headers=AUTH) + assert r.status_code == 400 + assert r.json()["error"]["code"] == "bad_bundle" + + +def test_missing_worker_yaml_is_bad_bundle(tmp_path: Path) -> None: + body = _body() + body["bundle"]["files"] = {k: v for k, v in BUNDLE.items() if k != "worker.yaml"} + with _make_client(tmp_path, enabled=True) as c: + r = c.post("/internal/worker/runs", json=body, headers=AUTH) + assert r.status_code == 400 + assert r.json()["error"]["code"] == "bad_bundle" + + +def test_run_streams_ndjson_and_replays_by_send_id(tmp_path: Path) -> None: + with _make_client(tmp_path, enabled=True) as c: + r = c.post("/internal/worker/runs", json=_body(), headers=AUTH) + assert r.status_code == 200 + items = [json.loads(line) for line in r.text.strip().splitlines()] + assert items[0]["kind"] == "turn" + assert items[-1]["kind"] in ("end", "error") + # replay: same runId+sendId re-follows instead of 409 + r2 = c.post("/internal/worker/runs", json=_body(), headers=AUTH) + assert r2.status_code == 200 + items2 = [json.loads(line) for line in r2.text.strip().splitlines()] + assert items2 == items + + +def test_different_send_id_for_existing_run_is_409(tmp_path: Path) -> None: + with _make_client(tmp_path, enabled=True) as c: + r = c.post("/internal/worker/runs", json=_body(), headers=AUTH) + assert r.status_code == 200 + body = _body() + body["sendId"] = "different" + r2 = c.post("/internal/worker/runs", json=body, headers=AUTH) + assert r2.status_code == 409 + assert r2.json()["error"]["code"] == "busy_run" + + +def test_follow_unknown_run_is_404(tmp_path: Path) -> None: + with _make_client(tmp_path, enabled=True) as c: + r = c.get("/internal/worker/runs/r_bbbbbbbbbbbb/follow", headers=AUTH) + assert r.status_code == 404 + assert r.json()["error"]["code"] == "unknown_run" + + +def test_cancel_unknown_run_is_404(tmp_path: Path) -> None: + with _make_client(tmp_path, enabled=True) as c: + r = c.post("/internal/worker/runs/r_bbbbbbbbbbbb/cancel", headers=AUTH) + assert r.status_code == 404 + assert r.json()["error"]["code"] == "unknown_run" + + +def test_follow_and_cancel_require_auth(tmp_path: Path) -> None: + with _make_client(tmp_path, enabled=True) as c: + assert c.get("/internal/worker/runs/r_bbbbbbbbbbbb/follow").status_code == 401 + assert c.post("/internal/worker/runs/r_bbbbbbbbbbbb/cancel").status_code == 401 From 8b0275db9bbce670198e394ae314f2344e510e5b Mon Sep 17 00:00:00 2001 From: Omar Alsoulah Date: Thu, 13 Aug 2026 20:27:38 +0300 Subject: [PATCH 20/28] =?UTF-8?q?sanad:=20worker=20routes=20=E2=80=94=20bu?= =?UTF-8?q?ndle=20write=20hardening,=20symmetric=20spawn=20cleanup,=20auth?= =?UTF-8?q?=20test?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/sanad_terminal/routes_worker.py | 34 ++++++++-- terminal-server/tests/test_routes_worker.py | 64 +++++++++++++++++++ 2 files changed, 94 insertions(+), 4 deletions(-) diff --git a/terminal-server/src/sanad_terminal/routes_worker.py b/terminal-server/src/sanad_terminal/routes_worker.py index 293945164..2719442d6 100644 --- a/terminal-server/src/sanad_terminal/routes_worker.py +++ b/terminal-server/src/sanad_terminal/routes_worker.py @@ -31,7 +31,14 @@ render_input_prompt, ) from sanad_terminal.routes_workspace import _settings, workspace_root -from sanad_terminal.run_runner import RUN_ID_RE, RunRunner, get_run, prepare_run_dirs, put_run +from sanad_terminal.run_runner import ( + RUN_ID_RE, + RunRunner, + drop_run, + get_run, + prepare_run_dirs, + put_run, +) from sanad_terminal.wire_runner import WireRunnerError from sanad_terminal.workspace import build_child_env @@ -146,10 +153,22 @@ async def start_run( if rel_path.is_absolute(): return _err(400, "bad_bundle", f"absolute path not allowed: {rel}") resolved = (dirs.bundle / rel_path).resolve() - if not resolved.is_relative_to(bundle_root): + # `resolved == bundle_root` catches "." / "" / "sub/.." — every key + # that normalizes back to the bundle directory itself, which + # `is_relative_to` alone would happily accept (a path is relative to + # itself) and then blow up `write_text` with IsADirectoryError. + if resolved == bundle_root or not resolved.is_relative_to(bundle_root): return _err(400, "bad_bundle_path", rel) - resolved.parent.mkdir(parents=True, exist_ok=True) - resolved.write_text(content, encoding="utf-8") + try: + resolved.parent.mkdir(parents=True, exist_ok=True) + resolved.write_text(content, encoding="utf-8") + except OSError as exc: + # A key that conflicts with another (e.g. "a" and "a/b.txt" both + # present — one wants "a" as a file, the other as a directory) + # raises FileExistsError/IsADirectoryError/NotADirectoryError + # here rather than at the containment check above; surface it + # the same way instead of a bare 500. + return _err(400, "bad_bundle_path", f"{rel}: {exc}") try: spec = load_worker_spec(dirs.interface_file) @@ -218,6 +237,13 @@ async def start_run( try: state = await runner.start_turn(prompt, body.send_id) except WireRunnerError as exc: + # Symmetric with the start() failure above: a runner that never got + # a turn must not linger in the registry — it would keep + # `runners_hold_machine` reporting the machine as busy, and every + # retry of this runId would 409 `busy_run` forever since `get_run` + # would keep finding a dead, turn-less entry. `drop_run` both stops + # the runner and removes it, freeing the id for a fresh attempt. + await drop_run(body.run_id) return _err(409, exc.code, exc.message) return _stream(runner, state.turn_id) diff --git a/terminal-server/tests/test_routes_worker.py b/terminal-server/tests/test_routes_worker.py index 73059b97f..a4bc36ea1 100644 --- a/terminal-server/tests/test_routes_worker.py +++ b/terminal-server/tests/test_routes_worker.py @@ -5,9 +5,12 @@ import sys from pathlib import Path +import pytest from fastapi.testclient import TestClient from sanad_terminal.app import create_app +from sanad_terminal.run_runner import RunRunner, get_run from sanad_terminal.settings import TerminalSettings +from sanad_terminal.wire_runner import WireRunnerError FAKE_WIRE = Path(__file__).parent / "_fake_worker_wire.py" @@ -99,6 +102,37 @@ def test_missing_worker_yaml_is_bad_bundle(tmp_path: Path) -> None: assert r.json()["error"]["code"] == "bad_bundle" +def test_start_run_requires_auth(tmp_path: Path) -> None: + with _make_client(tmp_path, enabled=True) as c: + r = c.post("/internal/worker/runs", json=_body()) + assert r.status_code == 401 + + +@pytest.mark.parametrize("key", [".", "", "sub/.."]) +def test_bundle_key_normalizing_to_bundle_root_rejected(tmp_path: Path, key: str) -> None: + """A key that resolves to the bundle directory itself (not merely outside + it) must 400 `bad_bundle_path`, not 500 from `write_text` hitting a + directory (`is_relative_to` alone accepts it — a path is relative to + itself).""" + body = _body() + body["bundle"]["files"] = {key: "x", **BUNDLE} + with _make_client(tmp_path, enabled=True) as c: + r = c.post("/internal/worker/runs", json=body, headers=AUTH) + assert r.status_code == 400, r.text + assert r.json()["error"]["code"] == "bad_bundle_path" + + +def test_bundle_conflicting_file_and_directory_keys_rejected(tmp_path: Path) -> None: + """"a" wants to be a file; "a/b.txt" wants "a" to be a directory — must + 400, not crash `mkdir`/`write_text` with an unhandled OSError.""" + body = _body() + body["bundle"]["files"] = {"a": "x", "a/b.txt": "y", **BUNDLE} + with _make_client(tmp_path, enabled=True) as c: + r = c.post("/internal/worker/runs", json=body, headers=AUTH) + assert r.status_code == 400, r.text + assert r.json()["error"]["code"] == "bad_bundle_path" + + def test_run_streams_ndjson_and_replays_by_send_id(tmp_path: Path) -> None: with _make_client(tmp_path, enabled=True) as c: r = c.post("/internal/worker/runs", json=_body(), headers=AUTH) @@ -142,3 +176,33 @@ def test_follow_and_cancel_require_auth(tmp_path: Path) -> None: with _make_client(tmp_path, enabled=True) as c: assert c.get("/internal/worker/runs/r_bbbbbbbbbbbb/follow").status_code == 401 assert c.post("/internal/worker/runs/r_bbbbbbbbbbbb/cancel").status_code == 401 + + +def test_start_turn_failure_deregisters_the_run(tmp_path: Path, monkeypatch) -> None: + """`RunRunner.start()` can succeed (subprocess spawned, handshake done) + while `start_turn()` then fails — e.g. the child exited right after + `initialize`. The route must not leave that dead runner registered: + `runners_hold_machine` would keep reporting the machine busy for it, and + every retry of the same runId would 409 `busy_run` forever since + `get_run` would keep finding a turn-less entry. Forcing this cheaply + through the fake wire isn't practical (the child doesn't know at + `initialize` time whether it should exit — the only per-request signal is + the prompt text, sent later, by `start_turn` itself); a class-level + monkeypatch of `start_turn` after the real handshake exercises the same + route branch directly.""" + + async def _boom(self, user_input, send_id=None): # noqa: ANN001, ARG001 + raise WireRunnerError("not_started", "agent is not running") + + monkeypatch.setattr(RunRunner, "start_turn", _boom) + with _make_client(tmp_path, enabled=True) as c: + r = c.post("/internal/worker/runs", json=_body(), headers=AUTH) + assert r.status_code == 409 + assert r.json()["error"]["code"] == "not_started" + assert get_run("r_aaaaaaaaaaaa") is None + + # Un-poison the id: a retry with the real start_turn must succeed, + # not 409 forever. + monkeypatch.undo() + r2 = c.post("/internal/worker/runs", json=_body(), headers=AUTH) + assert r2.status_code == 200 From 07d04f60f2db40254502b1250ec0b75163b426af Mon Sep 17 00:00:00 2001 From: Omar Alsoulah Date: Thu, 13 Aug 2026 20:56:20 +0300 Subject: [PATCH 21/28] =?UTF-8?q?sanad:=20run=20completion=20=E2=80=94=20t?= =?UTF-8?q?race=20gzip=20upload,=20usage=20report,=20no=5Foutput=20fail-fa?= =?UTF-8?q?st?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/sanad_terminal/control_plane.py | 44 ++++ .../src/sanad_terminal/routes_worker.py | 123 ++++++++++- .../src/sanad_terminal/run_runner.py | 27 +++ terminal-server/tests/test_run_completion.py | 205 ++++++++++++++++++ terminal-server/tests/test_run_runner.py | 49 ++++- 5 files changed, 437 insertions(+), 11 deletions(-) create mode 100644 terminal-server/tests/test_run_completion.py diff --git a/terminal-server/src/sanad_terminal/control_plane.py b/terminal-server/src/sanad_terminal/control_plane.py index 799507471..227bf9a49 100644 --- a/terminal-server/src/sanad_terminal/control_plane.py +++ b/terminal-server/src/sanad_terminal/control_plane.py @@ -5,6 +5,7 @@ from typing import Any import httpx +from loguru import logger from pydantic import BaseModel, ConfigDict, ValidationError from pydantic.alias_generators import to_camel @@ -101,5 +102,48 @@ async def redeem_ticket(self, ticket: str) -> RedeemedTicket: "redeem_failed", f"malformed redeem response: {exc}", resp.status_code ) from exc + async def report_run_completion( + self, run_id: str, agentd_token: str, payload: dict[str, Any] + ) -> None: + """POST a worker run's terminal outcome to `/api/v1/runs/{run_id}/complete`. + + Authenticated with the machine's OWN bearer (the task-mode `AGENTD_TOKEN`, + the same credential every other `/internal/*` request on this machine + carries in reverse) — deliberately NOT the redeem-flow headers + `_redeem_headers` builds (`x-machine-token`/`x-terminal-secret`), which + authenticate the machine to the control plane for a *different* + purpose (ticket redemption) and aren't accepted by this endpoint. + + Fire-and-forget semantics end-to-end: a missing token (railway mode, + or any machine that never got one) skips the call entirely rather than + sending a bearer-less request that could only ever 401; any other + failure (network error or a non-2xx response) is logged and + swallowed, never raised. The control plane's reaper is the backstop + for a machine that dies before a retry — see the P0 worker-panel + design note this method implements. + """ + if not agentd_token: + logger.warning( + "run {} finished but no agentd token is configured; " + "skipping completion report (reaper will reap it)", + run_id, + ) + return + try: + resp = await self._http.post( + f"/api/v1/runs/{run_id}/complete", + json=payload, + headers={"Authorization": f"Bearer {agentd_token}"}, + ) + if not resp.is_success: + logger.warning( + "run completion report rejected run_id={} status={} body={}", + run_id, + resp.status_code, + resp.text, + ) + except httpx.HTTPError as exc: + logger.warning("run completion report failed run_id={}: {}", run_id, exc) + async def aclose(self) -> None: await self._http.aclose() diff --git a/terminal-server/src/sanad_terminal/routes_worker.py b/terminal-server/src/sanad_terminal/routes_worker.py index 2719442d6..330391205 100644 --- a/terminal-server/src/sanad_terminal/routes_worker.py +++ b/terminal-server/src/sanad_terminal/routes_worker.py @@ -14,12 +14,15 @@ import json import pwd +import shutil from collections.abc import AsyncIterator, Awaitable, Callable from pathlib import Path from typing import Annotated, Any +import httpx from fastapi import APIRouter, Depends, Request from fastapi.responses import JSONResponse, StreamingResponse +from loguru import logger from pydantic import BaseModel, Field from kimi_cli.exception import AgentSpecError @@ -30,9 +33,11 @@ load_worker_spec, render_input_prompt, ) +from sanad_terminal.control_plane import ControlPlaneClient from sanad_terminal.routes_workspace import _settings, workspace_root from sanad_terminal.run_runner import ( RUN_ID_RE, + RunDirs, RunRunner, drop_run, get_run, @@ -90,15 +95,110 @@ class RunStartBody(BaseModel): model_config = {"populate_by_name": True} -def make_on_finished(request: Request) -> Callable[[RunRunner], Awaitable[None]] | None: - """P0 placeholder: nothing observes a run's completion yet, so `RunRunner` - is handed no callback and its terminal-status hook stays dormant. Task 12 - swaps this factory's body for one that uploads the trace and reports the - run's outcome to the control plane — the call site (`on_finished= - make_on_finished(request)`) doesn't change. +def _map_terminal_status( + item: dict[str, Any] | None, output_file: Path +) -> tuple[str, str | None, dict[str, Any] | None]: + """(status, errorCode, output) from a consumed turn's terminal journal + item + whether the run wrote its declared output file. + + - `error` item: `failed`, carrying whatever `code` the journal has (the + wall-clock/step budget trips and a dead subprocess both land here; + the latter has no `code`, so `errorCode` comes back `None`). + - `end` item whose raw wire status is `cancelled` (an explicit + `/cancel` call, or the *second* item a token-budget trip produces — + `_trip_budget` journals the `error` first, then the wire's own `end` + follows once the cancel completes and becomes the true last item): + `cancelled`, no error code. + - `end` otherwise: the output file is the only signal of success a + one-way, afk run has — present means `succeeded`; missing means + `failed`/`no_output` and, per binding #P0, that is NOT nudged the way + `sanad dev` nudges a human — a cloud run just fails fast. + - No terminal item at all (should not happen; `on_finished` only fires + once a turn has ended) is treated the same as a missing output file + rather than raising, since the caller wraps everything anyway. """ - del request - return None + if item is not None and item.get("kind") == "error": + return "failed", item.get("code"), None + if item is not None and item.get("kind") == "end" and item.get("status") == "cancelled": + return "cancelled", None, None + if output_file.exists(): + try: + output = json.loads(output_file.read_text(encoding="utf-8")) + except (OSError, ValueError): + logger.exception("output file unreadable at {}", output_file) + return "failed", "no_output", None + return "succeeded", None, output + return "failed", "no_output", None + + +def _make_on_finished( + dirs: RunDirs, + body: RunStartBody, + *, + agentd_token: str, + control_plane: ControlPlaneClient, + upload_transport: httpx.AsyncBaseTransport | None = None, +) -> Callable[[RunRunner], Awaitable[None]]: + """Build the callback `RunRunner`'s terminal-status hook fires exactly + once, as a background task, after the NDJSON stream to the caller has + already ended (the `end`/`error` item is the last thing `follow()` + yields) — a slow trace upload or control-plane round trip here never + blocks a client. + + Closes over the request-scoped `dirs`/`body` so the call site only needs + to build this once per `/runs` call; `upload_transport` is the test seam + (production always passes `None`, i.e. a real `httpx.AsyncClient`). + """ + + async def _on_finished(runner: RunRunner) -> None: + try: + status, error_code, output = _map_terminal_status( + runner.terminal_item(), dirs.output_file + ) + + trace_uploaded = False + trace_bytes = await runner.collect_trace() + if trace_bytes and body.trace_upload_url: + try: + async with httpx.AsyncClient(transport=upload_transport) as client: + resp = await client.put(body.trace_upload_url, content=trace_bytes) + trace_uploaded = resp.is_success + if not trace_uploaded: + logger.warning( + "trace upload rejected run_id={} status={}", + runner.run_id, + resp.status_code, + ) + except Exception: + # Any failure here (network error, or anything else) must + # not stop the completion report from going out — the + # trace is best-effort, the status/usage report is not. + logger.exception("trace upload failed run_id={}", runner.run_id) + + payload: dict[str, Any] = { + "status": status, + "traceUploaded": trace_uploaded, + **runner.usage_totals(), + } + if error_code: + payload["errorCode"] = error_code + if output is not None: + payload["output"] = output + if not payload.get("modelAlias"): + payload.pop("modelAlias", None) + + await control_plane.report_run_completion(runner.run_id, agentd_token, payload) + + if status == "succeeded": + shutil.rmtree(dirs.root, ignore_errors=True) + except Exception: + # A reporting crash must never take the app down or leave a dead + # runner registered — `finally` below still runs `drop_run`. + logger.exception("on_finished crashed for run_id={}", runner.run_id) + finally: + await drop_run(runner.run_id) + + return _on_finished def _turn_id(runner: RunRunner) -> str | None: @@ -224,7 +324,12 @@ async def start_run( max_turn_seconds=min(body.budgets.max_turn_seconds, settings.worker_max_turn_seconds), max_steps_per_turn=min(body.budgets.max_steps_per_turn, settings.worker_max_steps_per_turn), max_tokens_per_run=min(body.budgets.max_tokens_per_run, settings.worker_max_tokens_per_run), - on_finished=make_on_finished(request), + on_finished=_make_on_finished( + dirs, + body, + agentd_token=settings.agentd_token, + control_plane=request.app.state.control_plane, + ), ) try: await runner.start() diff --git a/terminal-server/src/sanad_terminal/run_runner.py b/terminal-server/src/sanad_terminal/run_runner.py index d00ed5b16..ffca4eb3c 100644 --- a/terminal-server/src/sanad_terminal/run_runner.py +++ b/terminal-server/src/sanad_terminal/run_runner.py @@ -17,6 +17,7 @@ from __future__ import annotations import asyncio +import gzip import re from collections.abc import Awaitable, Callable, Sequence from dataclasses import dataclass @@ -212,6 +213,32 @@ def terminal_item(self) -> dict[str, Any] | None: return item return None + async def collect_trace(self) -> bytes | None: + """Locate this run's wire trace and gzip it for upload. + + The CLI writes its session journal at + `/sessions///wire.jsonl`, and + the run was spawned with `--session ` (see routes_worker.py), + so ` == self.run_id` — but the workdir-basename segment + is the CLI's own choice, not ours, hence the glob. `KIMI_SHARE_DIR` + is set on `self._env` by the caller (`routes_worker.py`'s bundle env + overlay); its absence (e.g. a bare `RunRunner` built without it, as + plain unit tests do) is not an error — there is simply no trace to + collect, same as a share dir that exists but has no matching file. + """ + share_dir = self._env.get("KIMI_SHARE_DIR") + if not share_dir: + return None + matches = sorted(Path(share_dir).glob(f"sessions/*/{self.run_id}/wire.jsonl")) + if not matches: + return None + try: + data = matches[0].read_bytes() + except OSError: + logger.exception("failed to read wire trace for run {}", self.run_id) + return None + return gzip.compress(data) + async def wait_finished_hooks(self) -> None: """Await the `on_finished` callback task, if one was scheduled — lets tests (and callers that need the side effect to have landed) block on diff --git a/terminal-server/tests/test_run_completion.py b/terminal-server/tests/test_run_completion.py new file mode 100644 index 000000000..d398d1848 --- /dev/null +++ b/terminal-server/tests/test_run_completion.py @@ -0,0 +1,205 @@ +"""Task 12: `_make_on_finished` — trace upload + completion report. + +Drives the real closure directly against a `FakeRunner` + `httpx.MockTransport` +(no real subprocess, no real network) — the same harness shape the task brief +prescribes, adapted in two ways once the brief's literal snippet was checked +against the real code it's testing: + +- `RunStartBody`'s Python attributes are its own field names (`run_id`, + `trace_upload_url`), never the `Field(alias=...)` camelCase wire name, even + with `populate_by_name=True` (verified directly against the installed + pydantic before writing this — a `type("B", (), {"runId": ...})()` stub, as + a literal transcription of the brief's snippet would have it, does not + match what `start_run` actually hands `_make_on_finished` in production). +- The PUT body is the gzip-compressed trace, not JSON — a handler that + unconditionally does `json.loads(request.content)` (as the brief's snippet + does) raises `UnicodeDecodeError` on the gzip magic bytes before it ever + gets to record the call. +""" + +import gzip +import json +from pathlib import Path +from typing import Any + +import httpx +from sanad_terminal.control_plane import ControlPlaneClient +from sanad_terminal.routes_worker import _make_on_finished +from sanad_terminal.run_runner import get_run, prepare_run_dirs, put_run + + +class FakeRunner: + run_id = "r_cccccccccccc" + + def __init__( + self, terminal_item: dict[str, Any] | None, trace: bytes | None = b'{"type":"metadata"}\n' + ) -> None: + self._terminal = terminal_item + self._trace = trace + + def terminal_item(self) -> dict[str, Any] | None: + return self._terminal + + def usage_totals(self) -> dict[str, Any]: + return {"tokensIn": 10, "tokensOut": 5, "modelAlias": "kimi-k3"} + + async def collect_trace(self) -> bytes | None: + return gzip.compress(self._trace) if self._trace is not None else None + + async def stop(self) -> None: + """`drop_run` (called from `_on_finished`'s `finally`) always calls + `runner.stop()` after popping the registry entry — a real RunRunner + method this fake stands in for when it's been registered via + `put_run` for a de-registration assertion.""" + + +def _body(trace_upload_url: str = "https://s3.test/put") -> Any: + # Duck-typed stand-in for RunStartBody — only the two attributes + # `_make_on_finished` actually reads, named the way a real RunStartBody + # instance exposes them (its own field names, not the wire aliases). + return type( + "B", (), {"trace_upload_url": trace_upload_url, "run_id": FakeRunner.run_id} + )() + + +def _mock_transport( + calls: list[tuple[str, str, Any]], + *, + put_status: int = 200, + post_status: int = 200, +) -> httpx.MockTransport: + """Records every request as (method, url, parsed-json-or-raw-bytes) and + answers PUT (trace upload, gzip body) and POST (completion report, JSON + body) with the given status codes.""" + + def handler(request: httpx.Request) -> httpx.Response: + if request.method == "PUT": + calls.append((request.method, str(request.url), request.content)) + return httpx.Response(put_status, text="ok" if put_status < 300 else "upload failed") + payload = json.loads(request.content) if request.content else {} + calls.append((request.method, str(request.url), payload)) + if post_status >= 300: + return httpx.Response( + post_status, json={"error": {"code": "internal", "message": "boom"}} + ) + return httpx.Response(post_status, json={"data": {}}) + + return httpx.MockTransport(handler) + + +def _harness( + tmp_path: Path, + output: dict | None, + *, + put_status: int = 200, + post_status: int = 200, + agentd_token: str = "tok", +): + calls: list[tuple[str, str, Any]] = [] + transport = _mock_transport(calls, put_status=put_status, post_status=post_status) + cp = ControlPlaneClient("https://cp.test", "secret", transport=transport) + dirs = prepare_run_dirs(tmp_path, FakeRunner.run_id) + if output is not None: + dirs.output_file.write_text(json.dumps(output)) + body = _body() + on_finished = _make_on_finished( + dirs, body, agentd_token=agentd_token, control_plane=cp, upload_transport=transport + ) + return on_finished, calls, dirs + + +async def test_success_reports_output_and_uploads(tmp_path: Path) -> None: + on_finished, calls, dirs = _harness(tmp_path, {"answer": "42"}) + runner = FakeRunner({"kind": "end", "status": "finished"}) + put_run(runner) # type: ignore[arg-type] + await on_finished(runner) # type: ignore[arg-type] + puts = [c for c in calls if c[0] == "PUT"] + posts = [c for c in calls if c[0] == "POST" and "/runs/" in c[1]] + assert len(puts) == 1 + assert posts[0][2]["status"] == "succeeded" + assert posts[0][2]["output"] == {"answer": "42"} + assert posts[0][2]["traceUploaded"] is True + assert posts[0][2]["tokensIn"] == 10 + assert posts[0][2]["tokensOut"] == 5 + assert posts[0][2]["modelAlias"] == "kimi-k3" + assert "errorCode" not in posts[0][2] + assert not dirs.root.exists() # cleaned on success + assert get_run(FakeRunner.run_id) is None # de-registered + + +async def test_no_output_fails_fast(tmp_path: Path) -> None: + on_finished, calls, dirs = _harness(tmp_path, None) + runner = FakeRunner({"kind": "end", "status": "finished"}) + await on_finished(runner) # type: ignore[arg-type] + post = next(c for c in calls if c[0] == "POST" and "/runs/" in c[1]) + assert post[2]["status"] == "failed" + assert post[2]["errorCode"] == "no_output" + assert "output" not in post[2] + assert dirs.root.exists() # kept for debugging + + +async def test_budget_error_maps_through(tmp_path: Path) -> None: + item = {"kind": "error", "code": "turn_budget_exceeded", "message": "m"} + on_finished, calls, _dirs = _harness(tmp_path, None) + runner = FakeRunner(item) + await on_finished(runner) # type: ignore[arg-type] + post = next(c for c in calls if c[0] == "POST" and "/runs/" in c[1]) + assert post[2] == {**post[2], "status": "failed", "errorCode": "turn_budget_exceeded"} + + +async def test_cancelled_end_maps_to_cancelled(tmp_path: Path) -> None: + """The real shape a token-budget trip (or an explicit /cancel) produces: + `_trip_budget` journals the `error` item first, but the wire's own `end` + (status cancelled) follows once the cancel completes and becomes the + turn's TRUE last item — `terminal_item()` returns that, not the earlier + error. The overall run status must still come out `cancelled`, not + `failed`/`no_output` (there was never going to be an output file for a + cancelled run).""" + on_finished, calls, _dirs = _harness(tmp_path, None) + runner = FakeRunner({"kind": "end", "status": "cancelled"}) + await on_finished(runner) # type: ignore[arg-type] + post = next(c for c in calls if c[0] == "POST" and "/runs/" in c[1]) + assert post[2]["status"] == "cancelled" + assert "errorCode" not in post[2] + assert "output" not in post[2] + + +async def test_upload_failure_reports_trace_uploaded_false(tmp_path: Path) -> None: + on_finished, calls, _dirs = _harness(tmp_path, {"answer": "42"}, put_status=500) + runner = FakeRunner({"kind": "end", "status": "finished"}) + await on_finished(runner) # type: ignore[arg-type] + puts = [c for c in calls if c[0] == "PUT"] + post = next(c for c in calls if c[0] == "POST" and "/runs/" in c[1]) + assert len(puts) == 1 # the attempt was made + assert post[2]["status"] == "succeeded" # the run itself still succeeded + assert post[2]["traceUploaded"] is False + + +async def test_report_failure_is_logged_not_raised(tmp_path: Path) -> None: + on_finished, calls, dirs = _harness(tmp_path, {"answer": "42"}, post_status=500) + runner = FakeRunner({"kind": "end", "status": "finished"}) + put_run(runner) # type: ignore[arg-type] + await on_finished(runner) # does not raise despite the 500 # type: ignore[arg-type] + posts = [c for c in calls if c[0] == "POST" and "/runs/" in c[1]] + assert len(posts) == 1 # the attempt was made + assert get_run(FakeRunner.run_id) is None # still de-registered (finally) + # Cleanup is keyed on the run's own outcome (succeeded), not on whether + # the control plane accepted the report — a rejected/failed report still + # means the local dir is safe to reclaim. + assert not dirs.root.exists() + + +async def test_no_agentd_token_skips_report_without_raising(tmp_path: Path) -> None: + """Railway mode (or any machine that never got AGENTD_TOKEN): the + factory still runs end to end, but `report_run_completion` logs and + skips rather than sending a bearer-less request.""" + on_finished, calls, dirs = _harness(tmp_path, {"answer": "42"}, agentd_token="") + runner = FakeRunner({"kind": "end", "status": "finished"}) + put_run(runner) # type: ignore[arg-type] + await on_finished(runner) # type: ignore[arg-type] + posts = [c for c in calls if c[0] == "POST" and "/runs/" in c[1]] + assert posts == [] # no attempt made at all — no token to send + puts = [c for c in calls if c[0] == "PUT"] + assert len(puts) == 1 # trace upload is independent of the report skip + assert not dirs.root.exists() # still cleaned up (status was succeeded) + assert get_run(FakeRunner.run_id) is None # still de-registered diff --git a/terminal-server/tests/test_run_runner.py b/terminal-server/tests/test_run_runner.py index d0221fb18..cc22a950d 100644 --- a/terminal-server/tests/test_run_runner.py +++ b/terminal-server/tests/test_run_runner.py @@ -1,11 +1,13 @@ import asyncio +import gzip import sys from pathlib import Path import pytest - from sanad_terminal.run_runner import ( - RUN_ID_RE, RunRunner, get_run, prepare_run_dirs, put_run, + RUN_ID_RE, + RunRunner, + prepare_run_dirs, ) from sanad_terminal.wire_runner import WireRunnerError @@ -132,6 +134,49 @@ async def test_observe_event_ignores_malformed_token_usage(tmp_path: Path) -> No assert runner.usage_totals() == {"tokensIn": 0, "tokensOut": 0, "modelAlias": None} +async def test_collect_trace_none_without_share_dir(tmp_path: Path) -> None: + """No `KIMI_SHARE_DIR` in env (a bare RunRunner, as most of this file's + fixtures build it) is not an error — there's simply nowhere to look.""" + runner = _runner(tmp_path) + assert await runner.collect_trace() is None + + +async def test_collect_trace_none_when_wire_jsonl_missing(tmp_path: Path) -> None: + run_id = "r_ffffffffffff" + dirs = prepare_run_dirs(tmp_path, run_id) + runner = RunRunner( + run_id=run_id, argv=(sys.executable, str(FAKE_WIRE)), + cwd=dirs.workspace, + env={"KIMI_WORKER_OUTPUT_FILE": str(dirs.output_file), "KIMI_SHARE_DIR": str(dirs.share)}, + uid=None, gid=None, max_turn_seconds=30.0, max_steps_per_turn=50, + max_tokens_per_run=1000, + ) + assert await runner.collect_trace() is None + + +async def test_collect_trace_locates_and_gzips_the_session_journal(tmp_path: Path) -> None: + """The CLI's own layout: /sessions///wire.jsonl — glob-located since the workdir-basename segment + is the CLI's choice, not ours.""" + run_id = "r_00000000ffff" + dirs = prepare_run_dirs(tmp_path, run_id) + runner = RunRunner( + run_id=run_id, argv=(sys.executable, str(FAKE_WIRE)), + cwd=dirs.workspace, + env={"KIMI_WORKER_OUTPUT_FILE": str(dirs.output_file), "KIMI_SHARE_DIR": str(dirs.share)}, + uid=None, gid=None, max_turn_seconds=30.0, max_steps_per_turn=50, + max_tokens_per_run=1000, + ) + session_dir = dirs.share / "sessions" / "some-workdir-basename" / run_id + session_dir.mkdir(parents=True) + raw = b'{"type":"metadata"}\n{"type":"turn_begin"}\n' + (session_dir / "wire.jsonl").write_bytes(raw) + + trace = await runner.collect_trace() + assert trace is not None + assert gzip.decompress(trace) == raw + + async def test_on_finished_fires_once(tmp_path: Path) -> None: fired: list[str] = [] From 8cbc20e442124d34b96df6ca2a67b03b01d43d0b Mon Sep 17 00:00:00 2001 From: Omar Alsoulah Date: Thu, 13 Aug 2026 20:56:32 +0300 Subject: [PATCH 22/28] =?UTF-8?q?sanad:=20worker=20route=20tests=20?= =?UTF-8?q?=E2=80=94=20mock=20control=20plane,=20deterministic=20replay/bu?= =?UTF-8?q?sy-run=20coverage?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- terminal-server/tests/test_routes_worker.py | 123 ++++++++++++++++++-- 1 file changed, 114 insertions(+), 9 deletions(-) diff --git a/terminal-server/tests/test_routes_worker.py b/terminal-server/tests/test_routes_worker.py index a4bc36ea1..2a1e2a0e1 100644 --- a/terminal-server/tests/test_routes_worker.py +++ b/terminal-server/tests/test_routes_worker.py @@ -3,12 +3,17 @@ import json import sys +import time +from collections.abc import AsyncIterator from pathlib import Path +from typing import Any +import httpx import pytest from fastapi.testclient import TestClient from sanad_terminal.app import create_app -from sanad_terminal.run_runner import RunRunner, get_run +from sanad_terminal.control_plane import ControlPlaneClient +from sanad_terminal.run_runner import RunRunner, get_run, put_run from sanad_terminal.settings import TerminalSettings from sanad_terminal.wire_runner import WireRunnerError @@ -37,6 +42,20 @@ def _body(run_id: str = "r_aaaaaaaaaaaa") -> dict: } +def _control_plane() -> ControlPlaneClient: + """A full worker turn now runs Task 12's `on_finished` for real, which + fires a background completion POST — `control_plane=None` here would let + `create_app` build a real client against the real `control_plane_url` + (defaulted to production), so a completed test turn would fire an actual + network request at the live control plane. Inject a mock transport + instead, same pattern as `test_routes_coder.py`'s `_control_plane`.""" + + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, json={"data": {}}) + + return ControlPlaneClient("https://cp.test", "unused", transport=httpx.MockTransport(handler)) + + def _make_client(tmp_path: Path, *, enabled: bool) -> TestClient: settings = TerminalSettings( mode="task", @@ -46,12 +65,50 @@ def _make_client(tmp_path: Path, *, enabled: bool) -> TestClient: spawn_argv=(sys.executable, str(FAKE_WIRE)), worker_enabled=enabled, ) - return TestClient(create_app(settings, control_plane=None)) + return TestClient(create_app(settings, _control_plane())) AUTH = {"authorization": "Bearer tok"} +class _StubRunner: + """A duck-typed stand-in for a still-registered `RunRunner`, used to test + `start_run`'s replay/busy-run branch in isolation from real subprocess + timing. Task 12 wired `on_finished` for real, and its `finally` clause + calls `drop_run` the instant a run's completion is reported — so a run + that finished via the real fake-wire subprocess is de-registered again + within microseconds, well before a second synchronous `TestClient.post` + call on the same test thread reliably lands (verified empirically: the + two-call-in-a-row version of these tests started flaking/failing once + `on_finished` stopped being a no-op). Only implements what `start_run`'s + `existing is not None` branch and `_stream`/`follow` touch.""" + + def __init__( + self, run_id: str, turn_id: str, send_id: str, items: list[dict[str, Any]] + ) -> None: + self.run_id = run_id + self._turn_id = turn_id + self._send_id = send_id + self._items = items + + def turn_summary(self) -> dict[str, Any]: + return {"turnId": self._turn_id} + + def get_turn(self, turn_id: str) -> Any: + if turn_id != self._turn_id: + return None + return type("S", (), {"send_id": self._send_id})() + + async def follow(self, turn_id: str, from_seq: int = 0) -> AsyncIterator[dict[str, Any]]: + assert turn_id == self._turn_id + for item in self._items[from_seq:]: + yield item + + async def stop(self) -> None: + """App-shutdown teardown (`create_app`'s lifespan) calls `drop_run` + on every still-registered run, which calls this.""" + + def test_disabled_is_404(tmp_path: Path) -> None: with _make_client(tmp_path, enabled=False) as c: r = c.post("/internal/worker/runs", json=_body(), headers=AUTH) @@ -133,24 +190,41 @@ def test_bundle_conflicting_file_and_directory_keys_rejected(tmp_path: Path) -> assert r.json()["error"]["code"] == "bad_bundle_path" -def test_run_streams_ndjson_and_replays_by_send_id(tmp_path: Path) -> None: +def test_run_streams_ndjson(tmp_path: Path) -> None: with _make_client(tmp_path, enabled=True) as c: r = c.post("/internal/worker/runs", json=_body(), headers=AUTH) assert r.status_code == 200 items = [json.loads(line) for line in r.text.strip().splitlines()] assert items[0]["kind"] == "turn" assert items[-1]["kind"] in ("end", "error") - # replay: same runId+sendId re-follows instead of 409 - r2 = c.post("/internal/worker/runs", json=_body(), headers=AUTH) - assert r2.status_code == 200 - items2 = [json.loads(line) for line in r2.text.strip().splitlines()] - assert items2 == items -def test_different_send_id_for_existing_run_is_409(tmp_path: Path) -> None: +def test_replay_reuses_existing_run_when_send_id_matches(tmp_path: Path) -> None: + """Same runId+sendId against a run that's STILL REGISTERED re-follows its + existing journal (`_stream(existing, turn_id)`) instead of re-running — + exercised directly against a stub runner rather than a real completed + run, since a real run is de-registered (`drop_run`, in `on_finished`'s + `finally`) within microseconds of finishing, well before a second + request from the same test can reliably observe it as still-registered + (see `_StubRunner`'s docstring).""" with _make_client(tmp_path, enabled=True) as c: + items = [ + {"seq": 0, "kind": "turn", "turnId": "t_stub"}, + {"seq": 1, "kind": "event", "event": {"type": "TextPart"}}, + {"seq": 2, "kind": "end", "status": "finished"}, + ] + put_run(_StubRunner("r_aaaaaaaaaaaa", "t_stub", "r_aaaaaaaaaaaa", items)) # type: ignore[arg-type] r = c.post("/internal/worker/runs", json=_body(), headers=AUTH) assert r.status_code == 200 + assert [json.loads(line) for line in r.text.strip().splitlines()] == items + + +def test_different_send_id_for_existing_run_is_409(tmp_path: Path) -> None: + """A different sendId against a STILL REGISTERED run (its current turn's + sendId doesn't match) is a conflict — see `test_replay_...` above for why + this drives a stub runner rather than a real completed one.""" + with _make_client(tmp_path, enabled=True) as c: + put_run(_StubRunner("r_aaaaaaaaaaaa", "t_stub", "original-send-id", [])) # type: ignore[arg-type] body = _body() body["sendId"] = "different" r2 = c.post("/internal/worker/runs", json=body, headers=AUTH) @@ -158,6 +232,37 @@ def test_different_send_id_for_existing_run_is_409(tmp_path: Path) -> None: assert r2.json()["error"]["code"] == "busy_run" +def test_repeat_request_after_completion_starts_a_fresh_run(tmp_path: Path) -> None: + """Documents the real, intentional P0 consequence of Task 12's immediate + `drop_run`: once a run has finished, uploaded its trace, and reported its + outcome, it's gone from the registry — a caller that repeats the exact + same runId+sendId afterward does NOT get 409 `busy_run` or a replay, it + just starts a brand new run under the same id (a fresh turnId; the P0 + design accepts this because the original caller already received the + complete stream, including the terminal item, before `on_finished` (and + therefore `drop_run`) ever runs).""" + with _make_client(tmp_path, enabled=True) as c: + r1 = c.post("/internal/worker/runs", json=_body(), headers=AUTH) + assert r1.status_code == 200 + turn1 = json.loads(r1.text.strip().splitlines()[0])["turnId"] + # `on_finished` (and therefore `drop_run`) runs as a background task + # AFTER the StreamingResponse above already finished — its landing + # isn't ordered against this test thread's next line, so wait for it + # deterministically instead of racing it (both directions of that + # race are exactly what broke the two tests above before they were + # rewritten onto `_StubRunner`). + for _ in range(200): + if get_run("r_aaaaaaaaaaaa") is None: + break + time.sleep(0.01) + else: + pytest.fail("run was never de-registered after completion") + r2 = c.post("/internal/worker/runs", json=_body(), headers=AUTH) + assert r2.status_code == 200 + turn2 = json.loads(r2.text.strip().splitlines()[0])["turnId"] + assert turn1 != turn2 + + def test_follow_unknown_run_is_404(tmp_path: Path) -> None: with _make_client(tmp_path, enabled=True) as c: r = c.get("/internal/worker/runs/r_bbbbbbbbbbbb/follow", headers=AUTH) From ade6a836cc527b443c2e9b9f9914ebe0e8ddb499 Mon Sep 17 00:00:00 2001 From: Omar Alsoulah Date: Thu, 13 Aug 2026 21:20:24 +0300 Subject: [PATCH 23/28] =?UTF-8?q?sanad:=20worker=20trace=20upload=20?= =?UTF-8?q?=E2=80=94=20injectable=20transport=20seam,=20end-to-end=20trace?= =?UTF-8?q?=20test?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- terminal-server/src/sanad_terminal/app.py | 8 ++ .../src/sanad_terminal/routes_worker.py | 1 + terminal-server/tests/_fake_worker_wire.py | 43 +++++++++ terminal-server/tests/test_routes_worker.py | 90 ++++++++++++++++++- 4 files changed, 138 insertions(+), 4 deletions(-) diff --git a/terminal-server/src/sanad_terminal/app.py b/terminal-server/src/sanad_terminal/app.py index 23e45eefe..20c52ebd7 100644 --- a/terminal-server/src/sanad_terminal/app.py +++ b/terminal-server/src/sanad_terminal/app.py @@ -9,6 +9,7 @@ from collections.abc import AsyncIterator from contextlib import asynccontextmanager +import httpx from fastapi import FastAPI, WebSocket from fastapi.responses import JSONResponse from loguru import logger @@ -51,6 +52,7 @@ def create_app( settings: TerminalSettings | None = None, control_plane: ControlPlaneClient | None = None, + worker_upload_transport: httpx.AsyncBaseTransport | None = None, ) -> FastAPI: resolved = settings or TerminalSettings.load() cp = control_plane or ControlPlaneClient( @@ -138,6 +140,12 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]: app.state.control_plane = cp app.state.manager = manager app.state.idle_stopper = idle_stopper + # Test seam for worker runs' trace-upload client (routes_worker.py's + # `_make_on_finished` -> `httpx.AsyncClient(transport=...)`), threaded the + # same way `control_plane` is: production leaves this `None` (a real + # client); tests inject an `httpx.MockTransport` so a completed worker + # turn's trace PUT never reaches the network by accident. + app.state.worker_upload_transport = worker_upload_transport from sanad_terminal.routes_workspace import register_error_handlers, router diff --git a/terminal-server/src/sanad_terminal/routes_worker.py b/terminal-server/src/sanad_terminal/routes_worker.py index 330391205..2b8d2df07 100644 --- a/terminal-server/src/sanad_terminal/routes_worker.py +++ b/terminal-server/src/sanad_terminal/routes_worker.py @@ -329,6 +329,7 @@ async def start_run( body, agentd_token=settings.agentd_token, control_plane=request.app.state.control_plane, + upload_transport=request.app.state.worker_upload_transport, ), ) try: diff --git a/terminal-server/tests/_fake_worker_wire.py b/terminal-server/tests/_fake_worker_wire.py index c6bb6db32..96758d2da 100644 --- a/terminal-server/tests/_fake_worker_wire.py +++ b/terminal-server/tests/_fake_worker_wire.py @@ -17,6 +17,15 @@ otherwise race against (the "over-budget event was the run's last one" case that must NOT retroactively mark a successful run as budget-exceeded). +- "WRITE_TRACE": same as default (output file + finished), but ALSO writes + a plausible `$KIMI_SHARE_DIR/sessions// + /wire.jsonl` — the layout `RunRunner. + collect_trace()` globs for — so a route-level test can + exercise the real trace-upload PUT end to end instead of + relying on the fixture's usual silence there (every OTHER + mode deliberately writes nothing under KIMI_SHARE_DIR, so + `collect_trace()` returns None for them by design — this + is the one opt-in exception). """ import json @@ -67,6 +76,35 @@ def _write_output_file() -> None: json.dump({"answer": "fake"}, f) +def _arg_after(flag: str) -> str | None: + argv = sys.argv + if flag in argv: + idx = argv.index(flag) + if idx + 1 < len(argv): + return argv[idx + 1] + return None + + +def _write_trace_file() -> None: + """Mirror the real CLI's session-journal layout closely enough for + `RunRunner.collect_trace()`'s glob to find it: `$KIMI_SHARE_DIR/sessions/ + //wire.jsonl`. `--session ` and + `--work-dir ` are both real argv `routes_worker.py` always passes + (see the `argv` list it builds), so this reads them back rather than + needing a dedicated env var just for the fixture.""" + share_dir = os.environ.get("KIMI_SHARE_DIR") + session_id = _arg_after("--session") + if not share_dir or not session_id: + return + work_dir = _arg_after("--work-dir") or "workspace" + basename = os.path.basename(work_dir.rstrip("/")) or "workspace" + session_dir = os.path.join(share_dir, "sessions", basename, session_id) + os.makedirs(session_dir, exist_ok=True) + with open(os.path.join(session_dir, "wire.jsonl"), "w") as f: + f.write(json.dumps({"type": "metadata", "session_id": session_id}) + "\n") + f.write(json.dumps({"type": "turn_begin"}) + "\n") + + def main() -> None: while True: msg = _read() @@ -120,6 +158,11 @@ def main() -> None: _hang_until_cancel(mid) elif "HANG" in user_input: _hang_until_cancel(mid) + elif "WRITE_TRACE" in user_input: + _event("TextPart", {"type": "text", "text": "hello from worker"}) + _write_output_file() + _write_trace_file() + _write({"jsonrpc": "2.0", "id": mid, "result": {"status": "finished"}}) else: _event("TextPart", {"type": "text", "text": "hello from worker"}) _write_output_file() diff --git a/terminal-server/tests/test_routes_worker.py b/terminal-server/tests/test_routes_worker.py index 2a1e2a0e1..7103eb7af 100644 --- a/terminal-server/tests/test_routes_worker.py +++ b/terminal-server/tests/test_routes_worker.py @@ -1,6 +1,7 @@ """Worker runs P0: flag-gated /internal/worker/* — bundle containment, budget clamping, NDJSON turn streaming, and same-runId+sendId replay.""" +import gzip import json import sys import time @@ -42,21 +43,53 @@ def _body(run_id: str = "r_aaaaaaaaaaaa") -> dict: } -def _control_plane() -> ControlPlaneClient: +def _control_plane(calls: list[tuple[str, dict]] | None = None) -> ControlPlaneClient: """A full worker turn now runs Task 12's `on_finished` for real, which fires a background completion POST — `control_plane=None` here would let `create_app` build a real client against the real `control_plane_url` (defaulted to production), so a completed test turn would fire an actual network request at the live control plane. Inject a mock transport - instead, same pattern as `test_routes_coder.py`'s `_control_plane`.""" + instead, same pattern as `test_routes_coder.py`'s `_control_plane`. + Optionally records (url, json body) pairs into `calls` for tests that + want to inspect what got reported.""" + sink = calls if calls is not None else [] def handler(request: httpx.Request) -> httpx.Response: + body = json.loads(request.content) if request.content else {} + sink.append((str(request.url), body)) return httpx.Response(200, json={"data": {}}) return ControlPlaneClient("https://cp.test", "unused", transport=httpx.MockTransport(handler)) -def _make_client(tmp_path: Path, *, enabled: bool) -> TestClient: +def _upload_transport(calls: list[tuple[str, bytes]] | None = None) -> httpx.MockTransport: + """The trace-upload seam threaded through `create_app`'s + `worker_upload_transport` -> `app.state.worker_upload_transport` -> + `_make_on_finished`'s `upload_transport` param. Without this, a full + worker turn whose fake wire happened to write a trace file would PUT to + `body.trace_upload_url` (`https://s3.test/put` in `_body()`) over a REAL + `httpx.AsyncClient` — harmless only by fixture accident (the fake wire + never writes one, so `collect_trace()` returns `None`), not by any actual + seam. Every route test now gets a mock here regardless, so that accident + stops being load-bearing; `test_trace_upload_hits_the_injected_transport_ + end_to_end` is the one test that deliberately makes the fake wire write a + trace, to exercise this for real.""" + sink = calls if calls is not None else [] + + def handler(request: httpx.Request) -> httpx.Response: + sink.append((str(request.url), request.content)) + return httpx.Response(200, text="ok") + + return httpx.MockTransport(handler) + + +def _make_client( + tmp_path: Path, + *, + enabled: bool, + control_plane_calls: list[tuple[str, dict]] | None = None, + upload_calls: list[tuple[str, bytes]] | None = None, +) -> TestClient: settings = TerminalSettings( mode="task", fixed_user="user_1", @@ -65,7 +98,9 @@ def _make_client(tmp_path: Path, *, enabled: bool) -> TestClient: spawn_argv=(sys.executable, str(FAKE_WIRE)), worker_enabled=enabled, ) - return TestClient(create_app(settings, _control_plane())) + return TestClient( + create_app(settings, _control_plane(control_plane_calls), _upload_transport(upload_calls)) + ) AUTH = {"authorization": "Bearer tok"} @@ -199,6 +234,53 @@ def test_run_streams_ndjson(tmp_path: Path) -> None: assert items[-1]["kind"] in ("end", "error") +def test_trace_upload_hits_the_injected_transport_end_to_end(tmp_path: Path) -> None: + """Positively tests the trace-upload seam (see `_upload_transport`'s + docstring for why it was only accidentally safe before): a real full + turn whose fake wire actually writes a `wire.jsonl` (`WRITE_TRACE` mode) + must PUT exactly once to the injected mock, with gzip-magic-byte content, + and the completion report must say `traceUploaded: true` — exercising + `RunRunner.collect_trace()` end to end through the real route, not just + through `_make_on_finished` called directly (as `test_run_completion.py` + does with a `FakeRunner`).""" + upload_calls: list[tuple[str, bytes]] = [] + control_plane_calls: list[tuple[str, dict]] = [] + with _make_client( + tmp_path, + enabled=True, + control_plane_calls=control_plane_calls, + upload_calls=upload_calls, + ) as c: + body = _body() + body["input"] = {"q": "WRITE_TRACE"} + r = c.post("/internal/worker/runs", json=body, headers=AUTH) + assert r.status_code == 200 + items = [json.loads(line) for line in r.text.strip().splitlines()] + assert items[-1]["kind"] == "end" + assert items[-1]["status"] == "finished" + + # `on_finished` (upload + report + drop) is a background task fired + # after the stream above already ended — wait for it deterministically + # (see `test_repeat_request_after_completion_starts_a_fresh_run`). + for _ in range(200): + if get_run("r_aaaaaaaaaaaa") is None: + break + time.sleep(0.01) + else: + pytest.fail("run was never de-registered after completion") + + assert len(upload_calls) == 1 + url, content = upload_calls[0] + assert url == "https://s3.test/put" + assert content[:2] == b"\x1f\x8b" # gzip magic bytes — the object IS gzip + assert gzip.decompress(content).startswith(b'{"type": "metadata"') + + posts = [call for call in control_plane_calls if "/complete" in call[0]] + assert len(posts) == 1 + assert posts[0][1]["status"] == "succeeded" + assert posts[0][1]["traceUploaded"] is True + + def test_replay_reuses_existing_run_when_send_id_matches(tmp_path: Path) -> None: """Same runId+sendId against a run that's STILL REGISTERED re-follows its existing journal (`_stream(existing, turn_id)`) instead of re-running — From 12eb992cdd787b00c0cba211336916440140b273 Mon Sep 17 00:00:00 2001 From: Omar Alsoulah Date: Thu, 13 Aug 2026 21:30:06 +0300 Subject: [PATCH 24/28] =?UTF-8?q?sanad:=20worker=20parity=20e2e=20?= =?UTF-8?q?=E2=80=94=20dev=20and=20cloud=20runner=20agree=20on=20output=20?= =?UTF-8?q?and=20events?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests_e2e/test_worker_parity.py | 210 ++++++++++++++++++++++++++++++++ 1 file changed, 210 insertions(+) create mode 100644 tests_e2e/test_worker_parity.py diff --git a/tests_e2e/test_worker_parity.py b/tests_e2e/test_worker_parity.py new file mode 100644 index 000000000..3e5a56b96 --- /dev/null +++ b/tests_e2e/test_worker_parity.py @@ -0,0 +1,210 @@ +"""Task 13: DX-4 parity e2e — `sanad agent dev` (local) and the RunRunner +route (cloud) must run the SAME scripted bundle through the SAME real CLI +and produce the SAME output document. + +Fixture shape and spawn_argv follow the binding notes in the task brief: +- agent.yaml is the loadable nested `version: '1'` + `agent: {...}` form + (kimi_cli.agentspec.load_agent_spec silently treats a flat document as an + empty spec — see tests/worker/test_assembly.py::_agent_with_tools and + tests_e2e/test_worker_dev.py's AGENT_YAML). +- The cloud path spawns via RunRunner with `cwd=dirs.workspace` (a tmp dir) + and a from-scratch child env (`build_child_env`), so a bare `uv run kimi` + would fail to resolve this repo's project from that cwd — spawn_argv uses + `uv run --project kimi` (routes_worker.py appends + `--wire --session ... --agent-file ... --work-dir ...` itself). The + scripted-echo scripts path rides the CONFIG FILE (provider.env, applied + inside the child by `create_llm`), not the parent env, so `--config-file` + is baked into spawn_argv. +- The cloud control-plane and trace-upload transports are mocked (same + pattern as terminal-server/tests/test_routes_worker.py's `_make_client`) so + this test never makes a real network call. +- A successful cloud run has its run directory removed (`shutil.rmtree` in + routes_worker.py's `_on_finished`, once status == "succeeded"), so the + output is read from the mocked completion POST body, not from disk. +""" + +from __future__ import annotations + +import json +import subprocess +import time +from pathlib import Path + +import httpx +from fastapi.testclient import TestClient + +from tests_e2e.wire_helpers import ( + make_env, + make_home_dir, + make_work_dir, + repo_root, + write_scripted_config, +) + +# Loadable nested shape required by kimi_cli.agentspec.load_agent_spec (see +# tests/worker/test_assembly.py::_agent_with_tools and +# tests_e2e/test_worker_dev.py's AGENT_YAML) — a flat top-level document is +# silently treated as an empty spec and fails later with a confusing error. +# `tools: []` is required too (Inherit() with no base to inherit from raises +# "Tools are required"). +BUNDLE = { + "agent.yaml": "version: '1'\nagent:\n name: t\n system_prompt_path: prompt.md\n tools: []\n", + "prompt.md": "You are a worker.", + "worker.yaml": "interface:\n inputs: {q: string}\n outputs: {answer: string}\n", +} +SCRIPTS = [ + "\n".join( + [ + "text: thinking", + "tool_call: " + + json.dumps( + { + "id": "tc-1", + "name": "ReturnOutput", + "arguments": json.dumps({"output": {"answer": "42"}}), + } + ), + ] + ) +] + + +def _dev_output(tmp_path: Path) -> dict: + dev_dir = tmp_path / "dev" + dev_dir.mkdir() + config_path = write_scripted_config(dev_dir, SCRIPTS) + work_dir = make_work_dir(dev_dir) + home_dir = make_home_dir(dev_dir) + for name, text in BUNDLE.items(): + (work_dir / name).write_text(text) + proc = subprocess.run( + [ + "uv", + "run", + "kimi", + "agent", + "dev", + "--input", + '{"q": "meaning"}', + "--config-file", + str(config_path), + "--work-dir", + str(work_dir), + ], + cwd=repo_root(), + env=make_env(home_dir), + capture_output=True, + text=True, + timeout=180, + ) + assert proc.returncode == 0, proc.stderr + return json.loads(proc.stdout.strip()) + + +def _control_plane(calls: list[tuple[str, dict]]): + """Mock transport for the completion POST — mirrors + test_routes_worker.py's `_control_plane`. NEVER let the cloud path make + a real network call; the run's output is read back from here since a + successful run's directory is removed on disk (Task 12).""" + from sanad_terminal.control_plane import ControlPlaneClient + + def handler(request: httpx.Request) -> httpx.Response: + body = json.loads(request.content) if request.content else {} + calls.append((str(request.url), body)) + return httpx.Response(200, json={"data": {}}) + + return ControlPlaneClient("https://cp.test", "unused", transport=httpx.MockTransport(handler)) + + +def _upload_transport() -> httpx.MockTransport: + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, text="ok") + + return httpx.MockTransport(handler) + + +def _cloud_output(tmp_path: Path) -> tuple[dict, list[str], list[dict]]: + from sanad_terminal.app import create_app + from sanad_terminal.settings import TerminalSettings + + cloud = tmp_path / "cloud" + cloud.mkdir() + config_path = write_scripted_config(cloud, SCRIPTS) + settings = TerminalSettings( + mode="task", + fixed_user="u1", + agentd_token="tok", + data_dir=cloud, + # `uv run kimi` alone would resolve to whatever project owns the + # child's cwd (RunRunner spawns with cwd=dirs.workspace, a tmp dir + # with no pyproject.toml) — `--project` pins the real repo. Routes + # appends `--wire --session ... --agent-file ... --work-dir ...`, so + # this is deliberately the base command *without* `--wire`, plus the + # scripted-echo config file (the child env is built from scratch, so + # KIMI_SCRIPTED_ECHO_SCRIPTS must ride the config, not the parent env). + spawn_argv=( + "uv", + "run", + "--project", + str(repo_root()), + "kimi", + "--config-file", + str(config_path), + ), + worker_enabled=True, + ) + control_plane_calls: list[tuple[str, dict]] = [] + with TestClient( + create_app(settings, _control_plane(control_plane_calls), _upload_transport()) + ) as c: + body = { + "runId": "r_dddddddddddd", + "sendId": "r_dddddddddddd", + "input": {"q": "meaning"}, + "bundle": {"files": BUNDLE}, + "budgets": {"maxTurnSeconds": 120, "maxStepsPerTurn": 50, "maxTokensPerRun": 100000}, + "sessionToken": "sess_x", + "traceUploadUrl": "https://invalid.test/put", + } + r = c.post("/internal/worker/runs", json=body, headers={"authorization": "Bearer tok"}) + assert r.status_code == 200, r.text + items = [json.loads(line) for line in r.text.strip().splitlines()] + + # `on_finished` (report + drop) fires as a background task AFTER the + # StreamingResponse above already ended, same as + # test_routes_worker.py's completion tests — wait for the completion + # POST to actually land instead of racing it. + from sanad_terminal.run_runner import get_run + + for _ in range(200): + if get_run("r_dddddddddddd") is None: + break + time.sleep(0.05) + else: + raise AssertionError("cloud run was never de-registered after completion") + + events = [i["event"]["type"] for i in items if i["kind"] == "event"] + completions = [call for call in control_plane_calls if "/complete" in call[0]] + assert len(completions) == 1, control_plane_calls + payload = completions[0][1] + assert payload["status"] == "succeeded", payload + return payload["output"], events, items + + +def test_dev_and_cloud_agree(tmp_path: Path) -> None: + dev_out = _dev_output(tmp_path) + cloud_out, cloud_events, cloud_items = _cloud_output(tmp_path) + # INVARIANT (may not be weakened): same output document from both paths. + assert dev_out == cloud_out == {"answer": "42"} + + # Event-sequence parity is checked at the cloud-journal level only: dev + # (`kimi agent dev`) prints just the final output JSON on stdout, it + # doesn't expose per-event wire journal — see this file's module + # docstring / task report for why full sequence-level dev/cloud diffing + # was out of scope for the P0 bar. Sanity-check the cloud journal saw a + # full turn: begin, the tool call, and a terminated end item. + assert "TurnBegin" in cloud_events, cloud_events + assert any(t in ("ToolCall", "ToolCallPart") for t in cloud_events), cloud_events + end_items = [i for i in cloud_items if i["kind"] in ("end", "error")] + assert end_items, cloud_items + assert end_items[-1].get("status") == "finished", end_items[-1] From 2ea7b84190251f9b5d7756233b67738a82a09e66 Mon Sep 17 00:00:00 2001 From: Omar Alsoulah Date: Thu, 13 Aug 2026 21:50:30 +0300 Subject: [PATCH 25/28] =?UTF-8?q?sanad:=20agent=20pages=20+=20per-agent=20?= =?UTF-8?q?openapi=20=E2=80=94=20org=20list,=20run=20history,=20typed=20in?= =?UTF-8?q?voke=20schema?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sanad-web/app/agents/[name]/page.tsx | 180 ++++++++++++++++++ .../artifacts/sanad-web/app/agents/format.ts | 23 +++ .../artifacts/sanad-web/app/agents/page.tsx | 140 ++++++++++++++ .../v1/agents/[name]/openapi.json/route.ts | 79 ++++++++ .../artifacts/sanad-web/lib/agents/openapi.ts | 88 +++++++++ .../sanad-web/lib/agents/registry.ts | 53 +++++- .../artifacts/sanad-web/package.json | 1 + .../tests/unit/agent-openapi.test.ts | 21 ++ control-plane/pnpm-lock.yaml | 3 + 9 files changed, 587 insertions(+), 1 deletion(-) create mode 100644 control-plane/artifacts/sanad-web/app/agents/[name]/page.tsx create mode 100644 control-plane/artifacts/sanad-web/app/agents/format.ts create mode 100644 control-plane/artifacts/sanad-web/app/agents/page.tsx create mode 100644 control-plane/artifacts/sanad-web/app/api/v1/agents/[name]/openapi.json/route.ts create mode 100644 control-plane/artifacts/sanad-web/lib/agents/openapi.ts create mode 100644 control-plane/artifacts/sanad-web/tests/unit/agent-openapi.test.ts diff --git a/control-plane/artifacts/sanad-web/app/agents/[name]/page.tsx b/control-plane/artifacts/sanad-web/app/agents/[name]/page.tsx new file mode 100644 index 000000000..72b31e744 --- /dev/null +++ b/control-plane/artifacts/sanad-web/app/agents/[name]/page.tsx @@ -0,0 +1,180 @@ +import { auth } from "@clerk/nextjs/server"; +import { notFound, redirect } from "next/navigation"; +import Link from "next/link"; +import type { CSSProperties } from "react"; +import Nav from "../../ui/Nav"; +import { chip, surface, type } from "../../ui/theme"; +import { getAgentDetailByName, getLiveDeployment } from "@/lib/agents/registry"; +import { listRuns } from "@/lib/runs/store"; +import { formatAge, formatUsd } from "../format"; + +export const metadata = { title: "sanad — agent" }; + +/** + * Agent detail (P0 minimal page, Task 14): owner + status + the live + * deployment per env, and the last 20 runs — read-only, server rendered. + * No pause/resume affordance here; those stay CLI verbs (`sanad agent + * pause`/`resume`) in P0. + */ +export default async function AgentDetailPage({ + params, +}: { + params: Promise<{ name: string }>; +}) { + const { userId } = await auth(); + if (!userId) redirect("/"); + + const orgId = `personal_${userId}`; + const { name } = await params; + const agent = await getAgentDetailByName(orgId, name); + if (!agent) notFound(); + + const [dev, prod, runs] = await Promise.all([ + getLiveDeployment(agent.id, "dev"), + getLiveDeployment(agent.id, "prod"), + listRuns({ orgId, agentId: agent.id, limit: 20 }), + ]); + + const deployments = [ + { env: "dev", deployment: dev }, + { env: "prod", deployment: prod }, + ]; + + return ( +
+
+ ); +} + +const s: Record = { + main: { + maxWidth: "1000px", + margin: "0 auto", + padding: "2.5rem 2.5rem 5rem", + width: "100%", + }, + breadcrumb: { ...type.small, marginBottom: "1.5rem" }, + header: { marginBottom: "2.5rem" }, + sub: { + margin: "0.5rem 0 0", + display: "flex", + alignItems: "center", + color: "var(--ink-muted)", + fontSize: "0.875rem", + }, + dot: { margin: "0 0.55rem", color: "var(--rule-strong)" }, + section: { marginBottom: "3rem" }, + empty: { ...type.small }, + mono: { + fontFamily: "var(--font-mono)", + fontSize: "0.8rem", + color: "var(--ink)", + }, + table: { width: "100%", borderCollapse: "collapse" }, + th: { + padding: "0 0.25rem 0.5rem", + fontFamily: "var(--font-mono)", + fontSize: "0.68rem", + textTransform: "uppercase", + letterSpacing: "0.12em", + color: "var(--ink-muted)", + fontWeight: 500, + textAlign: "left", + borderBottom: "1px solid var(--rule)", + }, + td: { + padding: "0.7rem 0.25rem", + fontSize: "0.875rem", + color: "var(--ink-soft)", + borderBottom: "1px solid var(--rule)", + }, +}; diff --git a/control-plane/artifacts/sanad-web/app/agents/format.ts b/control-plane/artifacts/sanad-web/app/agents/format.ts new file mode 100644 index 000000000..721858704 --- /dev/null +++ b/control-plane/artifacts/sanad-web/app/agents/format.ts @@ -0,0 +1,23 @@ +/** + * Display formatting for the agents pages. Colocated rather than in lib/ — + * nothing else in the app needs a relative-age string or a run's dollar cost + * formatted this way yet, so this stays local until a second caller shows up. + */ + +/** "3m ago" / "2h ago" / "5d ago" — coarse relative age, newest unit only. */ +export function formatAge(date: Date): string { + const ms = Date.now() - date.getTime(); + const seconds = Math.floor(ms / 1000); + if (seconds < 60) return "just now"; + const minutes = Math.floor(seconds / 60); + if (minutes < 60) return `${minutes}m ago`; + const hours = Math.floor(minutes / 60); + if (hours < 24) return `${hours}h ago`; + const days = Math.floor(hours / 24); + return `${days}d ago`; +} + +/** A run's cost, stored in micros (1 USD = 1_000_000 micros), as "$0.0031". */ +export function formatUsd(micros: number): string { + return `$${(micros / 1e6).toFixed(4)}`; +} diff --git a/control-plane/artifacts/sanad-web/app/agents/page.tsx b/control-plane/artifacts/sanad-web/app/agents/page.tsx new file mode 100644 index 000000000..4b21528d2 --- /dev/null +++ b/control-plane/artifacts/sanad-web/app/agents/page.tsx @@ -0,0 +1,140 @@ +import { auth } from "@clerk/nextjs/server"; +import { redirect } from "next/navigation"; +import Link from "next/link"; +import type { CSSProperties } from "react"; +import Nav from "../ui/Nav"; +import { chip, surface, type } from "../ui/theme"; +import { getLiveDeployment, listAgentsForOrgWithOwnerEmail } from "@/lib/agents/registry"; +import { listRuns } from "@/lib/runs/store"; +import { formatAge } from "./format"; + +export const metadata = { title: "sanad — agents" }; + +/** + * Agents (P0 minimal page, Task 14): every agent in the org with its owner, + * status, live deployment per env, and last run — read-only, server + * rendered. Per-agent deployment/run lookups run in parallel (Promise.all) + * rather than one mega-join, matching the same N-small-queries convention + * ProjectsPage uses for its per-project session counts; P0's agent counts + * are small enough that this stays cheap. + */ +export default async function AgentsPage() { + const { userId } = await auth(); + if (!userId) redirect("/"); + + const orgId = `personal_${userId}`; + const agentRows = await listAgentsForOrgWithOwnerEmail(orgId); + + const rows = await Promise.all( + agentRows.map(async (agent) => { + const [dev, prod, lastRuns] = await Promise.all([ + getLiveDeployment(agent.id, "dev"), + getLiveDeployment(agent.id, "prod"), + listRuns({ orgId, agentId: agent.id, limit: 1 }), + ]); + return { + ...agent, + devStatus: dev?.status ?? null, + prodStatus: prod?.status ?? null, + lastRun: lastRuns[0] ?? null, + }; + }) + ); + + return ( +
+
+ ); +} + +const s: Record = { + main: { + maxWidth: "1000px", + margin: "0 auto", + padding: "3.5rem 2.5rem 5rem", + width: "100%", + }, + header: { marginBottom: "2.5rem" }, + empty: { ...type.small }, + inlineCode: { + fontFamily: "var(--font-mono)", + fontSize: "0.85em", + background: "var(--paper-sunken)", + border: "1px solid var(--rule)", + borderRadius: "5px", + padding: "0.05rem 0.35rem", + color: "var(--ink)", + }, + table: { width: "100%", borderCollapse: "collapse" }, + th: { + padding: "0 0.25rem 0.5rem", + fontFamily: "var(--font-mono)", + fontSize: "0.68rem", + textTransform: "uppercase", + letterSpacing: "0.12em", + color: "var(--ink-muted)", + fontWeight: 500, + textAlign: "left", + borderBottom: "1px solid var(--rule)", + }, + td: { + padding: "0.7rem 0.25rem", + fontSize: "0.875rem", + color: "var(--ink-soft)", + borderBottom: "1px solid var(--rule)", + }, +}; diff --git a/control-plane/artifacts/sanad-web/app/api/v1/agents/[name]/openapi.json/route.ts b/control-plane/artifacts/sanad-web/app/api/v1/agents/[name]/openapi.json/route.ts new file mode 100644 index 000000000..9393393de --- /dev/null +++ b/control-plane/artifacts/sanad-web/app/api/v1/agents/[name]/openapi.json/route.ts @@ -0,0 +1,79 @@ +import { NextRequest, NextResponse } from "next/server"; +import { parse as parseYaml } from "yaml"; +import { err } from "@/lib/http/envelope"; +import { verifyBearer } from "@/lib/auth/session"; +import { getActiveDeployment, getAgentByName, getVersionBundle } from "@/lib/agents/registry"; +import { buildAgentOpenApi } from "@/lib/agents/openapi"; + +type WorkerYaml = { + interface?: { + inputs?: Record; + outputs?: Record; + }; +}; + +/** + * Per-agent OpenAPI document (RT-3). Unlike every other /api/v1/agents/* + * route, the response is the raw OpenAPI object itself — no {data, meta} + * envelope — since this is meant to be fed straight to OpenAPI tooling. + * Errors still use the shared envelope (err()) though: a 401/404/500 here is + * a control-plane response, not part of the document being described. + */ +export async function GET( + req: NextRequest, + { params }: { params: Promise<{ name: string }> } +) { + const session = await verifyBearer(req); + if (!session) { + return err(401, "unauthorized", "Invalid or revoked session token"); + } + + const { name } = await params; + const agent = await getAgentByName(session.orgId, name); + if (!agent) { + return err(404, "not_found", "No such agent"); + } + + // Prefer prod's active deployment; fall back to dev's. A paused deployment + // does not count — the OpenAPI document describes what a caller can + // actually invoke right now, same as getActiveDeployment's "active" only + // filter (unlike the invoke route's getLiveDeployment, which also needs to + // see paused rows to return 409 instead of 404). + const deployment = + (await getActiveDeployment(agent.id, "prod")) ?? + (await getActiveDeployment(agent.id, "dev")); + if (!deployment) { + return err(404, "not_deployed", "agent has no active deployment"); + } + + const bundle = await getVersionBundle(deployment.agentVersionId); + const workerYamlText = bundle?.files["worker.yaml"]; + if (!workerYamlText) { + // The deployed version's bundle is missing its interface sidecar — + // that's a data-integrity problem, not a client mistake. + console.error( + `openapi: version ${deployment.agentVersionId} bundle is missing worker.yaml` + ); + return err(500, "internal_error", "agent version bundle is missing worker.yaml", true); + } + + let parsed: WorkerYaml; + try { + parsed = (parseYaml(workerYamlText) ?? {}) as WorkerYaml; + } catch (e) { + console.error( + `openapi: failed to parse worker.yaml for version ${deployment.agentVersionId}`, + e + ); + return err(500, "internal_error", "worker.yaml could not be parsed", true); + } + + const doc = buildAgentOpenApi({ + agentName: agent.name, + interfaceSpec: { + inputs: parsed.interface?.inputs ?? {}, + outputs: parsed.interface?.outputs ?? {}, + }, + }); + return NextResponse.json(doc); +} diff --git a/control-plane/artifacts/sanad-web/lib/agents/openapi.ts b/control-plane/artifacts/sanad-web/lib/agents/openapi.ts new file mode 100644 index 000000000..a861987c8 --- /dev/null +++ b/control-plane/artifacts/sanad-web/lib/agents/openapi.ts @@ -0,0 +1,88 @@ +/** + * Per-agent OpenAPI document (RT-3): a pure, DB-free projection of a + * worker.yaml `interface` stanza into an OpenAPI 3.1 description of the + * agent's one invoke endpoint. Kept separate from the route (Task 14) so the + * type-mapping and shape rules are unit-testable without touching the DB or + * Next's request/response plumbing. + */ + +/** P0 type map: everything that isn't "number" or "boolean" is a string. */ +function jsonSchemaType(t: string): { type: "number" | "boolean" | "string" } { + if (t === "number") return { type: "number" }; + if (t === "boolean") return { type: "boolean" }; + return { type: "string" }; +} + +/** + * A worker.yaml inputs/outputs map -> a JSON Schema object's properties + + * required. Keys are sorted so the document (and its `required` array) is + * stable regardless of the source map's insertion order — the same + * canonicalization concern as registry.ts's bundleContentHash. + */ +function schemaFrom(fields: Record): { + properties: Record; + required: string[]; +} { + const keys = Object.keys(fields).sort(); + const properties: Record = {}; + for (const key of keys) properties[key] = jsonSchemaType(fields[key]); + return { properties, required: keys }; +} + +export function buildAgentOpenApi(p: { + agentName: string; + interfaceSpec: { inputs: Record; outputs: Record }; +}): object { + const invokePath = `/api/v1/agents/${p.agentName}/invoke`; + const request = schemaFrom(p.interfaceSpec.inputs); + const response = schemaFrom(p.interfaceSpec.outputs); + + return { + openapi: "3.1.0", + info: { + title: `${p.agentName} — sanad agent`, + version: "1.0.0", + }, + paths: { + [invokePath]: { + post: { + operationId: "invokeAgent", + summary: `Invoke ${p.agentName}`, + security: [{ invokeToken: [] }], + requestBody: { + required: true, + content: { + "application/json": { + schema: { + type: "object", + properties: request.properties, + required: request.required, + }, + }, + }, + }, + responses: { + "200": { + description: "Run result", + content: { + "application/json": { + schema: { + type: "object", + properties: response.properties, + required: response.required, + }, + }, + }, + }, + }, + }, + }, + }, + components: { + securitySchemes: { + invokeToken: { type: "http", scheme: "bearer" }, + }, + }, + security: [{ invokeToken: [] }], + }; +} diff --git a/control-plane/artifacts/sanad-web/lib/agents/registry.ts b/control-plane/artifacts/sanad-web/lib/agents/registry.ts index 17825e989..deabdc5cf 100644 --- a/control-plane/artifacts/sanad-web/lib/agents/registry.ts +++ b/control-plane/artifacts/sanad-web/lib/agents/registry.ts @@ -1,7 +1,7 @@ import { createHash } from "crypto"; import { and, desc, eq, inArray } from "drizzle-orm"; import { db } from "../db"; -import { agents, agentVersions, deployments, workspaces } from "../db/schema"; +import { agents, agentVersions, deployments, users, workspaces } from "../db/schema"; export class OwnerRequiredError extends Error { readonly code = "owner_required"; @@ -325,3 +325,54 @@ export async function listAgentsForOrg(orgId: string) { .innerJoin(workspaces, eq(agents.workspaceId, workspaces.id)) .where(eq(workspaces.orgId, orgId)); } + +/** + * Org-scoped agent list with the owner's email — the agents dashboard page + * (Task 14) shows a human-readable owner, unlike /api/v1/agents's JSON + * response (listAgentsForOrg), whose callers already hold ownerUserId and + * can resolve it themselves. Joins users the same way getSessionMembership + * and the team page do. + */ +export async function listAgentsForOrgWithOwnerEmail(orgId: string) { + return db + .select({ + id: agents.id, + name: agents.name, + workspaceId: agents.workspaceId, + workspaceName: workspaces.name, + ownerUserId: agents.ownerUserId, + ownerEmail: users.email, + status: agents.status, + createdAt: agents.createdAt, + }) + .from(agents) + .innerJoin(workspaces, eq(agents.workspaceId, workspaces.id)) + .innerJoin(users, eq(users.id, agents.ownerUserId)) + .where(eq(workspaces.orgId, orgId)) + .orderBy(desc(agents.createdAt)); +} + +/** + * Single-agent detail (the agent page, Task 14): the same org-scoped lookup + * as getAgentByName, plus the owner's email in the same query rather than a + * second round trip keyed on ownerUserId. + */ +export async function getAgentDetailByName(orgId: string, name: string) { + const rows = await db + .select({ + id: agents.id, + workspaceId: agents.workspaceId, + name: agents.name, + ownerUserId: agents.ownerUserId, + ownerEmail: users.email, + status: agents.status, + description: agents.description, + createdAt: agents.createdAt, + }) + .from(agents) + .innerJoin(workspaces, eq(agents.workspaceId, workspaces.id)) + .innerJoin(users, eq(users.id, agents.ownerUserId)) + .where(and(eq(workspaces.orgId, orgId), eq(agents.name, name))) + .limit(1); + return rows[0] ?? null; +} diff --git a/control-plane/artifacts/sanad-web/package.json b/control-plane/artifacts/sanad-web/package.json index 819e96aea..b9f28266d 100644 --- a/control-plane/artifacts/sanad-web/package.json +++ b/control-plane/artifacts/sanad-web/package.json @@ -36,6 +36,7 @@ "react": "19.1.0", "react-dom": "19.1.0", "svix": "^1.63.0", + "yaml": "^2.9.0", "zod": "catalog:" }, "devDependencies": { diff --git a/control-plane/artifacts/sanad-web/tests/unit/agent-openapi.test.ts b/control-plane/artifacts/sanad-web/tests/unit/agent-openapi.test.ts new file mode 100644 index 000000000..8f3a7fd81 --- /dev/null +++ b/control-plane/artifacts/sanad-web/tests/unit/agent-openapi.test.ts @@ -0,0 +1,21 @@ +import { describe, it, expect } from "vitest"; +import { buildAgentOpenApi } from "@/lib/agents/openapi"; + +describe("buildAgentOpenApi", () => { + const doc: any = buildAgentOpenApi({ + agentName: "invoice-triage", + interfaceSpec: { inputs: { q: "string", n: "number" }, outputs: { answer: "string" } }, + }); + it("declares the invoke path with typed request properties", () => { + const body = doc.paths["/api/v1/agents/invoice-triage/invoke"].post + .requestBody.content["application/json"].schema; + expect(body.properties.q).toEqual({ type: "string" }); + expect(body.properties.n).toEqual({ type: "number" }); + expect(body.required).toEqual(["n", "q"]); + }); + it("declares bearer auth", () => { + expect(doc.components.securitySchemes.invokeToken).toEqual({ + type: "http", scheme: "bearer", + }); + }); +}); diff --git a/control-plane/pnpm-lock.yaml b/control-plane/pnpm-lock.yaml index d61ad4c7d..d19e8de87 100644 --- a/control-plane/pnpm-lock.yaml +++ b/control-plane/pnpm-lock.yaml @@ -269,6 +269,9 @@ importers: svix: specifier: ^1.63.0 version: 1.99.1 + yaml: + specifier: ^2.9.0 + version: 2.9.0 zod: specifier: 'catalog:' version: 3.25.76 From 9c8dcea318f3a57f70b0cfa9077b99e5335c7bb9 Mon Sep 17 00:00:00 2001 From: Omar Alsoulah Date: Thu, 13 Aug 2026 22:02:01 +0300 Subject: [PATCH 26/28] =?UTF-8?q?sanad:=20agent=20openapi=20route=20contra?= =?UTF-8?q?ct=20tests=20=E2=80=94=20auth,=20fallback=20order,=20scoping,?= =?UTF-8?q?=20parse=20failure?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../contract/agent-openapi-route.test.ts | 188 ++++++++++++++++++ 1 file changed, 188 insertions(+) create mode 100644 control-plane/artifacts/sanad-web/tests/contract/agent-openapi-route.test.ts diff --git a/control-plane/artifacts/sanad-web/tests/contract/agent-openapi-route.test.ts b/control-plane/artifacts/sanad-web/tests/contract/agent-openapi-route.test.ts new file mode 100644 index 000000000..04c941b4b --- /dev/null +++ b/control-plane/artifacts/sanad-web/tests/contract/agent-openapi-route.test.ts @@ -0,0 +1,188 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { NextRequest } from "next/server"; + +vi.mock("@/lib/auth/session", () => ({ + verifyBearer: vi.fn(), +})); +vi.mock("@/lib/agents/registry", () => ({ + getAgentByName: vi.fn(), + getActiveDeployment: vi.fn(), + getVersionBundle: vi.fn(), +})); + +import { verifyBearer } from "@/lib/auth/session"; +import { getActiveDeployment, getAgentByName, getVersionBundle } from "@/lib/agents/registry"; +import { GET } from "@/app/api/v1/agents/[name]/openapi.json/route"; + +const SESSION = { sessionId: "sess_1", userId: "user_1", orgId: "org_1" }; + +const AGENT = { + id: "ag_1", + workspaceId: "ws_1", + name: "invoice-triage", + ownerUserId: "user_1", + status: "active", + description: null, + createdAt: new Date("2026-01-01T00:00:00Z"), +}; + +const PROD_WORKER_YAML = "interface:\n inputs: {q: string}\n outputs: {answer: string}\n"; +const DEV_WORKER_YAML = "interface:\n inputs: {n: number}\n outputs: {ok: boolean}\n"; + +function deployment(env: "dev" | "prod", overrides: Record = {}) { + return { + id: `dp_${env}`, + agentId: AGENT.id, + agentVersionId: `av_${env}`, + env, + status: "active", + maxTurnSeconds: 900, + maxStepsPerTurn: 100, + maxTokensPerRun: 2_000_000, + createdAt: new Date("2026-01-01T00:00:00Z"), + updatedAt: new Date("2026-01-01T00:00:00Z"), + ...overrides, + }; +} + +function req(bearer?: string): NextRequest { + return new NextRequest( + "http://localhost/api/v1/agents/invoice-triage/openapi.json", + { headers: bearer !== undefined ? { authorization: `Bearer ${bearer}` } : {} } + ); +} + +function ctx(name = "invoice-triage") { + return { params: Promise.resolve({ name }) }; +} + +function invokeSchema(doc: any) { + return doc.paths["/api/v1/agents/invoice-triage/invoke"].post.requestBody + .content["application/json"].schema; +} + +beforeEach(() => { + vi.clearAllMocks(); +}); + +describe("GET /api/v1/agents/[name]/openapi.json", () => { + it("401s with no/invalid bearer", async () => { + vi.mocked(verifyBearer).mockResolvedValue(null); + + const res = await GET(req(), ctx()); + + expect(res.status).toBe(401); + const body = await res.json(); + expect(body.error.code).toBe("unauthorized"); + expect(getAgentByName).not.toHaveBeenCalled(); + }); + + it("404s not_found for an unknown or foreign-org agent name", async () => { + vi.mocked(verifyBearer).mockResolvedValue(SESSION); + // getAgentByName's inferred return type collapses to non-nullable + // (no `noUncheckedIndexedAccess` in this project's tsconfig — `rows[0] + // ?? null` types as just `rows[0]`'s element type), same reason + // logout.test.ts casts its null fixtures `as never`. + vi.mocked(getAgentByName).mockResolvedValue(null as never); + + const res = await GET(req("tok"), ctx()); + + expect(res.status).toBe(404); + const body = await res.json(); + expect(body.error.code).toBe("not_found"); + }); + + it("prefers prod's active deployment when both prod and dev are active", async () => { + vi.mocked(verifyBearer).mockResolvedValue(SESSION); + vi.mocked(getAgentByName).mockResolvedValue(AGENT); + vi.mocked(getActiveDeployment).mockImplementation(async (_agentId, env) => + env === "prod" ? deployment("prod") : deployment("dev") + ); + vi.mocked(getVersionBundle).mockImplementation(async (versionId) => + versionId === "av_prod" + ? { files: { "worker.yaml": PROD_WORKER_YAML } } + : { files: { "worker.yaml": DEV_WORKER_YAML } } + ); + + const res = await GET(req("tok"), ctx()); + + expect(res.status).toBe(200); + const doc = await res.json(); + const schema = invokeSchema(doc); + // prod's interface (q: string) was used, not dev's (n: number) + expect(schema.properties.q).toEqual({ type: "string" }); + expect(schema.properties.n).toBeUndefined(); + }); + + it("falls back to dev's active deployment when prod has none (paused/never deployed)", async () => { + vi.mocked(verifyBearer).mockResolvedValue(SESSION); + vi.mocked(getAgentByName).mockResolvedValue(AGENT); + // getActiveDeployment only ever returns "active" rows — a paused prod + // deployment surfaces here exactly like no prod deployment at all: null. + vi.mocked(getActiveDeployment).mockImplementation(async (_agentId, env) => + env === "prod" ? (null as never) : deployment("dev") + ); + vi.mocked(getVersionBundle).mockResolvedValue({ files: { "worker.yaml": DEV_WORKER_YAML } }); + + const res = await GET(req("tok"), ctx()); + + expect(res.status).toBe(200); + const doc = await res.json(); + const schema = invokeSchema(doc); + expect(schema.properties.n).toEqual({ type: "number" }); + }); + + it("404s not_deployed when neither env has an active deployment", async () => { + vi.mocked(verifyBearer).mockResolvedValue(SESSION); + vi.mocked(getAgentByName).mockResolvedValue(AGENT); + vi.mocked(getActiveDeployment).mockResolvedValue(null as never); + + const res = await GET(req("tok"), ctx()); + + expect(res.status).toBe(404); + const body = await res.json(); + expect(body.error.code).toBe("not_deployed"); + expect(getVersionBundle).not.toHaveBeenCalled(); + }); + + it("500s internal_error when the deployed worker.yaml is unparseable", async () => { + vi.mocked(verifyBearer).mockResolvedValue(SESSION); + vi.mocked(getAgentByName).mockResolvedValue(AGENT); + vi.mocked(getActiveDeployment).mockImplementation(async (_agentId, env) => + env === "prod" ? deployment("prod") : (null as never) + ); + vi.mocked(getVersionBundle).mockResolvedValue({ + files: { "worker.yaml": "interface:\n inputs: [1, 2\n" }, + }); + + const res = await GET(req("tok"), ctx()); + + expect(res.status).toBe(500); + const body = await res.json(); + expect(body.error.code).toBe("internal_error"); + }); + + it("returns the raw OpenAPI document on success — no {data,meta} envelope", async () => { + vi.mocked(verifyBearer).mockResolvedValue(SESSION); + vi.mocked(getAgentByName).mockResolvedValue(AGENT); + vi.mocked(getActiveDeployment).mockImplementation(async (_agentId, env) => + env === "prod" ? deployment("prod") : (null as never) + ); + vi.mocked(getVersionBundle).mockResolvedValue({ files: { "worker.yaml": PROD_WORKER_YAML } }); + + const res = await GET(req("tok"), ctx()); + + expect(res.status).toBe(200); + const doc = await res.json(); + expect(doc.openapi).toBe("3.1.0"); + expect(doc.data).toBeUndefined(); + expect(doc.error).toBeUndefined(); + expect(doc.components.securitySchemes.invokeToken).toEqual({ + type: "http", + scheme: "bearer", + }); + const schema = invokeSchema(doc); + expect(schema.properties.q).toEqual({ type: "string" }); + expect(schema.required).toEqual(["q"]); + }); +}); From 63819a0d605997e283531eb00f6029f66b8ff1da Mon Sep 17 00:00:00 2001 From: Omar Alsoulah Date: Thu, 13 Aug 2026 22:25:53 +0300 Subject: [PATCH 27/28] =?UTF-8?q?sanad:=20run=20lifecycle=20hardening=20?= =?UTF-8?q?=E2=80=94=20maintained=20staleness=20signal,=20guarded=20transi?= =?UTF-8?q?tions,=20honest=20invoke=20errors?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../app/api/v1/agents/[name]/invoke/route.ts | 67 +++++++- .../app/api/v1/runs/[id]/complete/route.ts | 8 +- .../sanad-web/lib/compute/machines.ts | 30 +++- .../artifacts/sanad-web/lib/runs/reaper.ts | 12 +- .../artifacts/sanad-web/lib/runs/store.ts | 31 +++- .../unit/invoke-route-infra-errors.test.ts | 153 ++++++++++++++++++ .../sanad-web/tests/unit/run-reaper.test.ts | 53 ++++-- .../tests/unit/run-store-transitions.test.ts | 79 +++++++++ 8 files changed, 406 insertions(+), 27 deletions(-) create mode 100644 control-plane/artifacts/sanad-web/tests/unit/invoke-route-infra-errors.test.ts create mode 100644 control-plane/artifacts/sanad-web/tests/unit/run-store-transitions.test.ts diff --git a/control-plane/artifacts/sanad-web/app/api/v1/agents/[name]/invoke/route.ts b/control-plane/artifacts/sanad-web/app/api/v1/agents/[name]/invoke/route.ts index 50a5694b6..06aa20fca 100644 --- a/control-plane/artifacts/sanad-web/app/api/v1/agents/[name]/invoke/route.ts +++ b/control-plane/artifacts/sanad-web/app/api/v1/agents/[name]/invoke/route.ts @@ -35,6 +35,28 @@ function sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } +/** + * Best-effort parse of a machine error response body as `{"error":{code, + * message}}` (the same envelope shape lib/http/envelope.ts's err() emits) — + * used to tell a genuine 4xx rejection (bad bundle, bad input: the caller's + * fault, not retryable) apart from a 5xx/network/garbled response (the + * machine's fault, retryable as machine_error). Returns null for anything + * that isn't that exact shape, including unparseable JSON. + */ +function parseMachineErrorEnvelope(text: string): { code: string; message: string } | null { + try { + const body = JSON.parse(text); + const code = body?.error?.code; + const message = body?.error?.message; + if (typeof code === "string" && typeof message === "string") { + return { code, message }; + } + } catch { + // Not JSON — falls through to the generic machine_error path. + } + return null; +} + function machineWakingResponse(): NextResponse { // err() has no header support and this is the one response that needs // Retry-After, so build the envelope directly here (same shape as err()). @@ -156,17 +178,33 @@ export async function POST( return err(500, "internal_error", "agent's workspace is missing", true); } + // Presign before waking the machine: it's pure config + signing (no + // machine involved), so failing fast here — instead of after paying for a + // cold start — avoids a bare, post-wake 500 leaving the run stuck + // "queued" with a machine already up and nothing to talk to it about. + let traceUploadUrl: string; + try { + traceUploadUrl = await presignTracePut(runId); + } catch (e) { + console.error(`invoke: failed to presign trace upload for run ${runId}`, e); + await markRunFailed(runId, "storage_unconfigured"); + return err(500, "storage_unconfigured", "SANAD_RUNS_BUCKET is not configured"); + } + let target: MachineTarget; try { const woken = await wakeMachine(workspace.id, info.env, workspace.keepWarm); if (woken === "timeout") { - await markRunFailed(runId, "wake_timeout"); + // Infra-side failure: clear the idempotency key so a caller retrying + // with the same Idempotency-Key gets a fresh attempt instead of an + // eternal replay of this failure (see markRunFailed's docstring). + await markRunFailed(runId, "wake_timeout", { clearIdempotencyKey: true }); return machineWakingResponse(); } target = woken; } catch (e) { console.error(`invoke: failed to wake workspace machine for run ${runId}`, e); - await markRunFailed(runId, "machine_error"); + await markRunFailed(runId, "machine_error", { clearIdempotencyKey: true }); return err(502, "machine_error", "failed to reach the workspace machine", true); } @@ -183,7 +221,6 @@ export async function POST( maxTokensPerRun: deployment.maxTokensPerRun, }; const sessionToken = await mintSession(agent.ownerUserId, info.orgId, undefined, "worker-run", workspace.id); - const traceUploadUrl = await presignTracePut(runId); let machineRes: Response; try { @@ -198,14 +235,34 @@ export async function POST( } as RequestInit & { duplex: "half" }); } catch (e) { console.error(`invoke: machine fetch failed for run ${runId}`, e); - await markRunFailed(runId, "machine_error"); + await markRunFailed(runId, "machine_error", { clearIdempotencyKey: true }); return err(502, "machine_error", "failed to reach the workspace machine", true); } if (!machineRes.ok) { const detail = await machineRes.text().catch(() => ""); + if (machineRes.status >= 400 && machineRes.status < 500) { + const parsed = parseMachineErrorEnvelope(detail); + if (parsed) { + // A 4xx with a parseable envelope is the caller's/bundle's fault + // (bad input, bad bundle, …), not an infra problem — pass it + // through verbatim, non-retryable, and keep the idempotency key: a + // caller fixing their bundle and retrying with the same key should + // NOT get a fresh run, they should get the same "you did this + // wrong" answer until they change something (matches genuine + // run-failure replay semantics, e.g. no_output/budget). + console.error( + `invoke: machine rejected run ${runId} with ${machineRes.status} ${parsed.code}`, + detail + ); + await markRunFailed(runId, parsed.code); + return err(machineRes.status, parsed.code, parsed.message); + } + } + // 5xx, network-shaped, or an unparseable body — machine's fault, not the + // caller's; infra failure, so clear the idempotency key too. console.error(`invoke: machine rejected run ${runId} with status ${machineRes.status}`, detail); - await markRunFailed(runId, "machine_error"); + await markRunFailed(runId, "machine_error", { clearIdempotencyKey: true }); return err(502, "machine_error", "workspace machine rejected the run", true); } diff --git a/control-plane/artifacts/sanad-web/app/api/v1/runs/[id]/complete/route.ts b/control-plane/artifacts/sanad-web/app/api/v1/runs/[id]/complete/route.ts index 3598a2d22..a86775f65 100644 --- a/control-plane/artifacts/sanad-web/app/api/v1/runs/[id]/complete/route.ts +++ b/control-plane/artifacts/sanad-web/app/api/v1/runs/[id]/complete/route.ts @@ -3,7 +3,7 @@ import { z } from "zod"; import { ok, err } from "@/lib/http/envelope"; import { completeRun, getRun } from "@/lib/runs/store"; import { getAgentById, getDeploymentById, getWorkspaceById } from "@/lib/agents/registry"; -import { getMachineByWorkspaceEnv } from "@/lib/compute/machines"; +import { getMachineByWorkspaceEnv, touchMachineLastSeen } from "@/lib/compute/machines"; import { machineTokenMatches } from "@/lib/compute/tokens"; const Body = z.object({ @@ -64,6 +64,12 @@ export async function POST( return err(401, "unauthorized", "Invalid machine credential"); } + // Proof of life: this POST only reaches here on a valid machine + // credential, so it's as good a staleness signal as a warm-attach probe — + // refresh it so lib/runs/reaper.ts's staleness check doesn't reap a run + // whose machine has been silently alive this whole time. + await touchMachineLastSeen(machine.id); + let raw: unknown; try { raw = await req.json(); diff --git a/control-plane/artifacts/sanad-web/lib/compute/machines.ts b/control-plane/artifacts/sanad-web/lib/compute/machines.ts index b657aa73b..76c56288c 100644 --- a/control-plane/artifacts/sanad-web/lib/compute/machines.ts +++ b/control-plane/artifacts/sanad-web/lib/compute/machines.ts @@ -162,12 +162,16 @@ async function ensureInner( if (warm) { const stale = row.imageRef !== config.workspaceImage; if (!stale) { - if (row.keepWarm !== opts.keepWarm) { - await db - .update(workspaceMachines) - .set({ keepWarm: opts.keepWarm, updatedAt: new Date() }) - .where(eq(workspaceMachines.id, row.id)); - } + // Touch lastSeenAt on every warm attach, not just when keepWarm + // changed — this is the reaper's staleness signal (lib/runs/reaper.ts). + // It was previously written once at cold start (below) and never + // refreshed, so a run on a machine that cold-started more than + // staleAfterMs ago would look "silent" to the reaper even while the + // machine is actively answering warm probes. + await db + .update(workspaceMachines) + .set({ keepWarm: opts.keepWarm, lastSeenAt: new Date(), updatedAt: new Date() }) + .where(eq(workspaceMachines.id, row.id)); return { machineId: row.id, hash12: row.hash12, @@ -252,6 +256,20 @@ export async function getMachineByWorkspaceEnv(workspaceId: string, env: string) return row ?? null; } +/** + * Touch a machine's lastSeenAt — the reaper's staleness signal + * (lib/runs/reaper.ts). Called from the run-completion route + * (POST .../runs/{id}/complete) right after the machine bearer token is + * verified: a completion POST is proof the machine is alive right now, same + * as a successful warm-attach probe in ensureWorkspaceMachine above. + */ +export async function touchMachineLastSeen(machineId: string): Promise { + await db + .update(workspaceMachines) + .set({ lastSeenAt: new Date(), updatedAt: new Date() }) + .where(eq(workspaceMachines.id, machineId)); +} + /** Router route lookup: hash12 → task IP, for workspace machines. */ export async function machineIpByHash(hash12: string): Promise { const [row] = await db diff --git a/control-plane/artifacts/sanad-web/lib/runs/reaper.ts b/control-plane/artifacts/sanad-web/lib/runs/reaper.ts index 7e8a13f73..b99c29187 100644 --- a/control-plane/artifacts/sanad-web/lib/runs/reaper.ts +++ b/control-plane/artifacts/sanad-web/lib/runs/reaper.ts @@ -27,7 +27,7 @@ export async function sweepLostRuns(staleAfterMs: number): Promise { const cutoffMs = Date.now() - staleAfterMs; const runningRows = await db - .select({ id: runs.id, deploymentId: runs.deploymentId }) + .select({ id: runs.id, deploymentId: runs.deploymentId, startedAt: runs.startedAt }) .from(runs) .where(eq(runs.status, "running")); if (runningRows.length === 0) return 0; @@ -72,7 +72,15 @@ export async function sweepLostRuns(staleAfterMs: number): Promise { // Stale (or no machine row at all — the `!lastSeenAt` branch covers both // "no matching workspaceMachines row" and "row exists but lastSeenAt is // still null", e.g. a machine that never finished provisioning). - const isLost = !lastSeenAt || lastSeenAt.getTime() < cutoffMs; + const machineStale = !lastSeenAt || lastSeenAt.getTime() < cutoffMs; + // Belt-and-suspenders against a machine-staleness false positive (e.g. + // lastSeenAt not yet refreshed on a machine that just picked up this + // run): a run that only just started can never be reaped, regardless of + // what the machine row says. A null startedAt (shouldn't happen for a + // "running" row — markRunRunning always sets it) is treated the same + // way: never reap on unproven age. + const startedStale = row.startedAt !== null && row.startedAt.getTime() < cutoffMs; + const isLost = machineStale && startedStale; if (isLost) staleIds.push(row.id); } if (staleIds.length === 0) return 0; diff --git a/control-plane/artifacts/sanad-web/lib/runs/store.ts b/control-plane/artifacts/sanad-web/lib/runs/store.ts index 3ab8015f8..650ecf1cf 100644 --- a/control-plane/artifacts/sanad-web/lib/runs/store.ts +++ b/control-plane/artifacts/sanad-web/lib/runs/store.ts @@ -94,17 +94,42 @@ export async function getRun(id: string): Promise { return rows[0] ?? null; } +/** + * Flip a queued run to "running". Guarded to only match status="queued" — + * without this, a fast completion POST (or the reaper) landing before this + * UPDATE runs could resurrect an already-terminal ("succeeded"/"failed"/ + * "lost") row back to "running" forever, since an unconditional UPDATE by id + * has no way to know the row moved on in the meantime. + */ export async function markRunRunning(id: string): Promise { await db .update(runs) .set({ status: "running", startedAt: new Date() }) - .where(eq(runs.id, id)); + .where(and(eq(runs.id, id), eq(runs.status, "queued"))); } -export async function markRunFailed(id: string, errorCode: string): Promise { +/** + * Flip a run to "failed". `clearIdempotencyKey` should be set true only for + * infra-side failures (wake_timeout, machine_error) — a caller retrying with + * the same Idempotency-Key after those must get a fresh attempt, not an + * eternal replay of `{status:"failed"}` (createRun's onConflictDoNothing + * replays by (deploymentId, idempotencyKey), so a poisoned key can never + * succeed again). Genuine run failures (no_output, budget) keep their key — + * replaying "failed" for those is the correct, intended behavior. + */ +export async function markRunFailed( + id: string, + errorCode: string, + opts?: { clearIdempotencyKey?: boolean } +): Promise { await db .update(runs) - .set({ status: "failed", errorCode, finishedAt: new Date() }) + .set({ + status: "failed", + errorCode, + finishedAt: new Date(), + ...(opts?.clearIdempotencyKey ? { idempotencyKey: null } : {}), + }) .where(eq(runs.id, id)); } diff --git a/control-plane/artifacts/sanad-web/tests/unit/invoke-route-infra-errors.test.ts b/control-plane/artifacts/sanad-web/tests/unit/invoke-route-infra-errors.test.ts new file mode 100644 index 000000000..798b5b92e --- /dev/null +++ b/control-plane/artifacts/sanad-web/tests/unit/invoke-route-infra-errors.test.ts @@ -0,0 +1,153 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { NextRequest } from "next/server"; + +/** + * Focused unit coverage for Finding 3 (machine 4xx must pass through, not + * get flattened into a retryable 502) and Finding 8 (a presign failure must + * short-circuit BEFORE the machine is ever woken). The full route contract + * — auth, gates, idempotent replay, etc — lives in + * tests/contract/invoke-route.test.ts; this file only exercises the two + * infra-error branches these findings touch. + */ + +vi.mock("@/lib/tokens/invoke", () => ({ verifyInvokeBearer: vi.fn() })); +vi.mock("@/lib/agents/registry", () => ({ + getAgentByName: vi.fn(), + getLiveDeployment: vi.fn(), + getVersionBundle: vi.fn(), + getWorkspaceById: vi.fn(), +})); +vi.mock("@/lib/billing/quota", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, assertWithinQuota: vi.fn() }; +}); +vi.mock("@/lib/compute/machines", () => ({ ensureWorkspaceMachine: vi.fn() })); +vi.mock("@/lib/auth/session", () => ({ mintSession: vi.fn() })); +vi.mock("@/lib/runs/store", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + createRun: vi.fn(), + getRun: vi.fn(), + markRunFailed: vi.fn(), + markRunRunning: vi.fn(), + presignTracePut: vi.fn(), + }; +}); + +import { verifyInvokeBearer } from "@/lib/tokens/invoke"; +import { getAgentByName, getLiveDeployment, getVersionBundle, getWorkspaceById } from "@/lib/agents/registry"; +import { assertWithinQuota } from "@/lib/billing/quota"; +import { ensureWorkspaceMachine } from "@/lib/compute/machines"; +import { mintSession } from "@/lib/auth/session"; +import { createRun, markRunFailed, presignTracePut } from "@/lib/runs/store"; +import { POST } from "@/app/api/v1/agents/[name]/invoke/route"; + +const TOKEN_INFO = { tokenId: "tok_1", agentId: "ag_1", env: "prod", orgId: "org_1" }; +const AGENT = { id: "ag_1", workspaceId: "ws_1", ownerUserId: "user_1", name: "invoice-triage" }; +const DEPLOYMENT = { + id: "dp_1", + agentId: "ag_1", + agentVersionId: "av_1", + env: "prod", + status: "active", + maxTurnSeconds: 900, + maxStepsPerTurn: 100, + maxTokensPerRun: 2_000_000, +}; +const WORKSPACE = { id: "ws_1", keepWarm: false }; +const MACHINE_TARGET = { + machineId: "wm_1", + hash12: "abc123def456", + baseUrl: "http://10.0.0.9:4100", + agentdToken: "agentd-tok", + coldStart: false, +}; + +function req(body: unknown = {}): NextRequest { + return new NextRequest("http://localhost/api/v1/agents/invoice-triage/invoke", { + method: "POST", + headers: { authorization: "Bearer itok_abc", "content-type": "application/json" }, + body: JSON.stringify(body), + }); +} + +function ctx(name = "invoice-triage") { + return { params: Promise.resolve({ name }) }; +} + +beforeEach(() => { + vi.clearAllMocks(); + vi.mocked(verifyInvokeBearer).mockResolvedValue(TOKEN_INFO); + vi.mocked(getAgentByName).mockResolvedValue(AGENT as never); + vi.mocked(getLiveDeployment).mockResolvedValue(DEPLOYMENT as never); + vi.mocked(assertWithinQuota).mockResolvedValue(undefined); + vi.mocked(getWorkspaceById).mockResolvedValue(WORKSPACE as never); + vi.mocked(createRun).mockResolvedValue({ id: "r_1", existing: false }); + vi.mocked(getVersionBundle).mockResolvedValue({ files: {} }); + vi.mocked(mintSession).mockResolvedValue("session-tok" as never); + vi.mocked(presignTracePut).mockResolvedValue("https://s3.example/put"); + vi.mocked(ensureWorkspaceMachine).mockResolvedValue(MACHINE_TARGET as never); + vi.stubGlobal("fetch", vi.fn()); +}); + +describe("invoke route — Finding 3: machine 4xx passthrough", () => { + it("passes through a machine 400 bad_bundle as non-retryable, without touching the idempotency key", async () => { + vi.mocked(global.fetch).mockResolvedValue( + new Response(JSON.stringify({ error: { code: "bad_bundle", message: "bundle failed to parse" } }), { + status: 400, + }) as never + ); + + const res = await POST(req(), ctx()); + + expect(res.status).toBe(400); + const body = await res.json(); + expect(body.error.code).toBe("bad_bundle"); + expect(body.error.message).toBe("bundle failed to parse"); + expect(body.error.retryable).toBe(false); + + // Genuine caller/bundle error, not an infra failure — markRunFailed is + // called WITHOUT clearIdempotencyKey, so a retry with the same + // Idempotency-Key replays this same failure rather than getting a fresh + // attempt (see lib/runs/store.ts markRunFailed's docstring). + expect(markRunFailed).toHaveBeenCalledWith("r_1", "bad_bundle"); + }); + + it("falls back to the generic retryable 502 machine_error for a 5xx", async () => { + vi.mocked(global.fetch).mockResolvedValue(new Response("boom", { status: 500 }) as never); + + const res = await POST(req(), ctx()); + + expect(res.status).toBe(502); + const body = await res.json(); + expect(body.error.code).toBe("machine_error"); + expect(body.error.retryable).toBe(true); + expect(markRunFailed).toHaveBeenCalledWith("r_1", "machine_error", { clearIdempotencyKey: true }); + }); + + it("falls back to the generic 502 for an unparseable 4xx body", async () => { + vi.mocked(global.fetch).mockResolvedValue(new Response("not json", { status: 422 }) as never); + + const res = await POST(req(), ctx()); + + expect(res.status).toBe(502); + const body = await res.json(); + expect(body.error.code).toBe("machine_error"); + }); +}); + +describe("invoke route — Finding 8: presign before wake", () => { + it("500s storage_unconfigured and never wakes the machine when presigning fails", async () => { + vi.mocked(presignTracePut).mockRejectedValue(new Error("SANAD_RUNS_BUCKET is not configured")); + + const res = await POST(req(), ctx()); + + expect(res.status).toBe(500); + const body = await res.json(); + expect(body.error.code).toBe("storage_unconfigured"); + expect(ensureWorkspaceMachine).not.toHaveBeenCalled(); + expect(markRunFailed).toHaveBeenCalledWith("r_1", "storage_unconfigured"); + expect(global.fetch).not.toHaveBeenCalled(); + }); +}); diff --git a/control-plane/artifacts/sanad-web/tests/unit/run-reaper.test.ts b/control-plane/artifacts/sanad-web/tests/unit/run-reaper.test.ts index e0b811b1a..20377e10b 100644 --- a/control-plane/artifacts/sanad-web/tests/unit/run-reaper.test.ts +++ b/control-plane/artifacts/sanad-web/tests/unit/run-reaper.test.ts @@ -3,21 +3,47 @@ import { PgDialect } from "drizzle-orm/pg-core"; const updates: any[] = []; const whereArgs: any[] = []; -const staleRows = [{ id: "r_aaaaaaaaaaaa" }, { id: "r_bbbbbbbbbbbb" }]; + +const NOW = Date.now(); +const OLD = new Date(NOW - 400_000); // older than the 300_000ms cutoff below +const FRESH = new Date(NOW); // well within the cutoff + +// Three "running" candidates sharing one deployment (so they all resolve to +// the same — stale — machine lastSeenAt below), differing only in +// startedAt: +// - r_aaaaaaaaaaaa: stale machine + stale startedAt -> reaped +// - r_bbbbbbbbbbbb: stale machine + stale startedAt -> would-be reaped, but +// simulates a race (see returningRows below) — the original regression +// coverage. +// - r_cccccccccccc: stale machine + FRESH startedAt -> must never even +// enter the stale-candidate set, regardless of machine staleness +// (Finding 1c: a recently-started run is never reaped). +const runningRows = [ + { id: "r_aaaaaaaaaaaa", deploymentId: "dp_1", startedAt: OLD }, + { id: "r_bbbbbbbbbbbb", deploymentId: "dp_1", startedAt: OLD }, + { id: "r_cccccccccccc", deploymentId: "dp_1", startedAt: FRESH }, +]; +const deploymentRows = [{ deploymentId: "dp_1", env: "prod", workspaceId: "ws_1" }]; +const machineRows = [{ workspaceId: "ws_1", env: "prod", lastSeenAt: OLD }]; // Simulates a race: r_bbbbbbbbbbbb genuinely completed via POST // .../runs/{id}/complete between the stale-candidate select and the // guarded UPDATE below, so the status="running" re-check excludes it from // RETURNING even though it was in the stale-candidate set. const returningRows = [{ id: "r_aaaaaaaaaaaa" }]; +let selectCall = 0; vi.mock("@/lib/db", () => ({ db: { - select: vi.fn(() => ({ - from: vi.fn(() => ({ - leftJoin: vi.fn(() => ({ where: vi.fn(async () => staleRows) })), - where: vi.fn(async () => staleRows), - })), - })), + select: vi.fn(() => { + selectCall += 1; + const call = selectCall; + return { + from: vi.fn(() => ({ + leftJoin: vi.fn(() => ({ where: vi.fn(async () => deploymentRows) })), + where: vi.fn(async () => (call === 1 ? runningRows : machineRows)), + })), + }; + }), update: vi.fn(() => ({ set: vi.fn((v: any) => { updates.push(v); @@ -40,9 +66,9 @@ describe("sweepLostRuns", () => { // The guarded UPDATE only actually flipped 1 of the 2 stale candidates // (simulated race above) — the returned count must come from - // RETURNING's length, not the 2-row candidate-select count. This is - // the regression the guard exists to prevent: silently reporting - // "reaped" for a run the UPDATE didn't touch. + // RETURNING's length, not the candidate-select count. This is the + // regression the guard exists to prevent: silently reporting "reaped" + // for a run the UPDATE didn't touch. expect(n).toBe(1); expect(updates).toHaveLength(1); // one batched statement, not one per stale row expect(updates[0]).toMatchObject({ status: "lost", errorCode: "machine_lost" }); @@ -55,5 +81,12 @@ describe("sweepLostRuns", () => { expect(sql).toMatch(/"status"\s*=\s*\$\d/); expect(params).toContain("running"); expect(sql).toMatch(/"id"\s+in\s*\(/i); + + // Finding 1c: r_cccccccccccc has a stale machine (same deployment as + // r_aaaa/r_bbbb) but a FRESH startedAt — it must never enter the + // candidate id list at all, i.e. never appear in the update's params. + expect(params).toContain("r_aaaaaaaaaaaa"); + expect(params).toContain("r_bbbbbbbbbbbb"); + expect(params).not.toContain("r_cccccccccccc"); }); }); diff --git a/control-plane/artifacts/sanad-web/tests/unit/run-store-transitions.test.ts b/control-plane/artifacts/sanad-web/tests/unit/run-store-transitions.test.ts new file mode 100644 index 000000000..64edc4c58 --- /dev/null +++ b/control-plane/artifacts/sanad-web/tests/unit/run-store-transitions.test.ts @@ -0,0 +1,79 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { PgDialect } from "drizzle-orm/pg-core"; + +const sets: any[] = []; +const wheres: any[] = []; + +vi.mock("@/lib/db", () => ({ + db: { + update: vi.fn(() => ({ + set: vi.fn((v: any) => { + sets.push(v); + return { + where: vi.fn(async (w: any) => { + wheres.push(w); + }), + }; + }), + })), + }, +})); + +import { markRunRunning, markRunFailed } from "@/lib/runs/store"; + +beforeEach(() => { + sets.length = 0; + wheres.length = 0; +}); + +describe("markRunRunning (Finding 4)", () => { + it("guards the transition to only match a queued row", async () => { + await markRunRunning("r_1"); + + expect(sets[0]).toMatchObject({ status: "running" }); + + // A fast completion POST (or the reaper) landing before this UPDATE + // must not resurrect an already-terminal row — the WHERE has to carry + // BOTH id and status="queued", not just id. + const { sql, params } = new PgDialect().sqlToQuery(wheres[0]); + expect(sql).toMatch(/"id"\s*=\s*\$\d/); + expect(sql).toMatch(/"status"\s*=\s*\$\d/); + expect(params).toContain("r_1"); + expect(params).toContain("queued"); + }); +}); + +describe("markRunFailed (Finding 2)", () => { + it("nulls the idempotency key when clearIdempotencyKey is set (infra failure)", async () => { + await markRunFailed("r_1", "wake_timeout", { clearIdempotencyKey: true }); + + expect(sets[0]).toMatchObject({ + status: "failed", + errorCode: "wake_timeout", + idempotencyKey: null, + }); + }); + + it("nulls the idempotency key for machine_error too", async () => { + await markRunFailed("r_1", "machine_error", { clearIdempotencyKey: true }); + + expect(sets[0]).toMatchObject({ + status: "failed", + errorCode: "machine_error", + idempotencyKey: null, + }); + }); + + it("preserves the idempotency key for a genuine run failure (no opts)", async () => { + await markRunFailed("r_1", "no_output"); + + expect(sets[0]).toMatchObject({ status: "failed", errorCode: "no_output" }); + expect(sets[0]).not.toHaveProperty("idempotencyKey"); + }); + + it("preserves the idempotency key when clearIdempotencyKey is explicitly false", async () => { + await markRunFailed("r_1", "turn_budget_exceeded", { clearIdempotencyKey: false }); + + expect(sets[0]).not.toHaveProperty("idempotencyKey"); + }); +}); From 084ba19f35d0dc1aff0d6135685f928be452347f Mon Sep 17 00:00:00 2001 From: Omar Alsoulah Date: Thu, 13 Aug 2026 22:28:34 +0300 Subject: [PATCH 28/28] sanad: invoke + completion contract tests --- .../tests/contract/invoke-route.test.ts | 204 ++++++++++++++++++ .../tests/contract/run-complete-route.test.ts | 175 +++++++++++++++ 2 files changed, 379 insertions(+) create mode 100644 control-plane/artifacts/sanad-web/tests/contract/invoke-route.test.ts create mode 100644 control-plane/artifacts/sanad-web/tests/contract/run-complete-route.test.ts diff --git a/control-plane/artifacts/sanad-web/tests/contract/invoke-route.test.ts b/control-plane/artifacts/sanad-web/tests/contract/invoke-route.test.ts new file mode 100644 index 000000000..9a0736718 --- /dev/null +++ b/control-plane/artifacts/sanad-web/tests/contract/invoke-route.test.ts @@ -0,0 +1,204 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { NextRequest } from "next/server"; + +vi.mock("@/lib/tokens/invoke", () => ({ verifyInvokeBearer: vi.fn() })); +vi.mock("@/lib/agents/registry", () => ({ + getAgentByName: vi.fn(), + getLiveDeployment: vi.fn(), + getVersionBundle: vi.fn(), + getWorkspaceById: vi.fn(), +})); +// assertWithinQuota is real business logic that reaches the db on the +// non-mocked path — no route test case here exercises quota rejection, but +// the route calls it unconditionally, so it still needs a resolved stub. +vi.mock("@/lib/billing/quota", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, assertWithinQuota: vi.fn() }; +}); +vi.mock("@/lib/compute/machines", () => ({ ensureWorkspaceMachine: vi.fn() })); +vi.mock("@/lib/auth/session", () => ({ mintSession: vi.fn() })); +// invokeGate/newRunId are pure (no db) — keep them real so the 403/404/409 +// gate-priority logic under test is the actual implementation, not a +// hand-rolled stand-in that could drift from it. +vi.mock("@/lib/runs/store", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + createRun: vi.fn(), + getRun: vi.fn(), + markRunFailed: vi.fn(), + markRunRunning: vi.fn(), + presignTracePut: vi.fn(), + }; +}); + +import { verifyInvokeBearer } from "@/lib/tokens/invoke"; +import { getAgentByName, getLiveDeployment, getVersionBundle, getWorkspaceById } from "@/lib/agents/registry"; +import { assertWithinQuota } from "@/lib/billing/quota"; +import { ensureWorkspaceMachine } from "@/lib/compute/machines"; +import { mintSession } from "@/lib/auth/session"; +import { createRun, getRun, markRunFailed, presignTracePut } from "@/lib/runs/store"; +import { POST } from "@/app/api/v1/agents/[name]/invoke/route"; + +const TOKEN_INFO = { tokenId: "tok_1", agentId: "ag_1", env: "prod", orgId: "org_1" }; +const AGENT = { id: "ag_1", workspaceId: "ws_1", ownerUserId: "user_1", name: "invoice-triage" }; +const DEPLOYMENT = { + id: "dp_1", + agentId: "ag_1", + agentVersionId: "av_1", + env: "prod", + status: "active", + maxTurnSeconds: 900, + maxStepsPerTurn: 100, + maxTokensPerRun: 2_000_000, +}; +const WORKSPACE = { id: "ws_1", keepWarm: false }; +const MACHINE_TARGET = { + machineId: "wm_1", + hash12: "abc123def456", + baseUrl: "http://10.0.0.9:4100", + agentdToken: "agentd-tok", + coldStart: false, +}; + +function req(opts: { bearer?: string | null; body?: unknown; wait?: boolean } = {}): NextRequest { + const bearer = "bearer" in opts ? opts.bearer : "itok_abc"; + const body = opts.body ?? {}; + const qs = opts.wait ? "?wait=1" : ""; + return new NextRequest(`http://localhost/api/v1/agents/invoice-triage/invoke${qs}`, { + method: "POST", + headers: { + ...(bearer ? { authorization: `Bearer ${bearer}` } : {}), + "content-type": "application/json", + }, + body: JSON.stringify(body), + }); +} + +function ctx(name = "invoice-triage") { + return { params: Promise.resolve({ name }) }; +} + +/** Wires the happy-path chain through workspace resolution — individual + * tests override whichever mock they need to diverge on. */ +function mockHappyPathThroughGate() { + vi.mocked(verifyInvokeBearer).mockResolvedValue(TOKEN_INFO); + vi.mocked(getAgentByName).mockResolvedValue(AGENT as never); + vi.mocked(getLiveDeployment).mockResolvedValue(DEPLOYMENT as never); + vi.mocked(assertWithinQuota).mockResolvedValue(undefined); + vi.mocked(getWorkspaceById).mockResolvedValue(WORKSPACE as never); +} + +beforeEach(() => { + vi.clearAllMocks(); + vi.stubGlobal("fetch", vi.fn()); +}); + +describe("POST /api/v1/agents/[name]/invoke", () => { + it("401s with no itok", async () => { + vi.mocked(verifyInvokeBearer).mockResolvedValue(null); + + const res = await POST(req({ bearer: null }), ctx()); + + expect(res.status).toBe(401); + const body = await res.json(); + expect(body.error.code).toBe("unauthorized"); + expect(getAgentByName).not.toHaveBeenCalled(); + }); + + it("403s token_scope for a cross-agent token", async () => { + vi.mocked(verifyInvokeBearer).mockResolvedValue(TOKEN_INFO); + // The path agent ("ag_2") doesn't match the token's agentId ("ag_1"). + vi.mocked(getAgentByName).mockResolvedValue({ ...AGENT, id: "ag_2" } as never); + vi.mocked(getLiveDeployment).mockResolvedValue(DEPLOYMENT as never); + + const res = await POST(req(), ctx()); + + expect(res.status).toBe(403); + const body = await res.json(); + expect(body.error.code).toBe("token_scope"); + // Token-scope outranks quota — the route must reject before even + // checking quota. + expect(assertWithinQuota).not.toHaveBeenCalled(); + expect(createRun).not.toHaveBeenCalled(); + }); + + it("404s not_deployed when there is no live deployment for the env", async () => { + mockHappyPathThroughGate(); + vi.mocked(getLiveDeployment).mockResolvedValue(null as never); + + const res = await POST(req(), ctx()); + + expect(res.status).toBe(404); + const body = await res.json(); + expect(body.error.code).toBe("not_deployed"); + expect(createRun).not.toHaveBeenCalled(); + }); + + it("409s paused for a paused deployment", async () => { + mockHappyPathThroughGate(); + vi.mocked(getLiveDeployment).mockResolvedValue({ ...DEPLOYMENT, status: "paused" } as never); + + const res = await POST(req(), ctx()); + + expect(res.status).toBe(409); + const body = await res.json(); + expect(body.error.code).toBe("paused"); + expect(createRun).not.toHaveBeenCalled(); + }); + + it("replays an idempotent invoke without ever calling the machine", async () => { + mockHappyPathThroughGate(); + vi.mocked(createRun).mockResolvedValue({ id: "r_1", existing: true }); + vi.mocked(getRun).mockResolvedValue({ + id: "r_1", + status: "succeeded", + output: { text: "cached result" }, + } as never); + + const res = await POST(req({ body: {} }), ctx()); + + expect(res.status).toBe(200); + const body = await res.json(); + expect(body.data).toEqual({ runId: "r_1", status: "succeeded", output: { text: "cached result" } }); + expect(ensureWorkspaceMachine).not.toHaveBeenCalled(); + expect(global.fetch).not.toHaveBeenCalled(); + expect(getWorkspaceById).not.toHaveBeenCalled(); + }); + + it("passes through a machine 400 as non-retryable (Finding 3)", async () => { + mockHappyPathThroughGate(); + vi.mocked(createRun).mockResolvedValue({ id: "r_1", existing: false }); + vi.mocked(presignTracePut).mockResolvedValue("https://s3.example/put"); + vi.mocked(ensureWorkspaceMachine).mockResolvedValue(MACHINE_TARGET as never); + vi.mocked(getVersionBundle).mockResolvedValue({ files: {} }); + vi.mocked(mintSession).mockResolvedValue("session-tok" as never); + vi.mocked(global.fetch).mockResolvedValue( + new Response(JSON.stringify({ error: { code: "bad_bundle", message: "bundle failed to parse" } }), { + status: 400, + }) as never + ); + + const res = await POST(req(), ctx()); + + expect(res.status).toBe(400); + const body = await res.json(); + expect(body.error).toMatchObject({ code: "bad_bundle", message: "bundle failed to parse", retryable: false }); + expect(markRunFailed).toHaveBeenCalledWith("r_1", "bad_bundle"); + }); + + it("500s storage_unconfigured before ever waking the machine (Finding 8)", async () => { + mockHappyPathThroughGate(); + vi.mocked(createRun).mockResolvedValue({ id: "r_1", existing: false }); + vi.mocked(presignTracePut).mockRejectedValue(new Error("SANAD_RUNS_BUCKET is not configured")); + + const res = await POST(req(), ctx()); + + expect(res.status).toBe(500); + const body = await res.json(); + expect(body.error.code).toBe("storage_unconfigured"); + expect(ensureWorkspaceMachine).not.toHaveBeenCalled(); + expect(global.fetch).not.toHaveBeenCalled(); + expect(markRunFailed).toHaveBeenCalledWith("r_1", "storage_unconfigured"); + }); +}); diff --git a/control-plane/artifacts/sanad-web/tests/contract/run-complete-route.test.ts b/control-plane/artifacts/sanad-web/tests/contract/run-complete-route.test.ts new file mode 100644 index 000000000..5d46dae97 --- /dev/null +++ b/control-plane/artifacts/sanad-web/tests/contract/run-complete-route.test.ts @@ -0,0 +1,175 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { NextRequest } from "next/server"; + +vi.mock("@/lib/runs/store", () => ({ + completeRun: vi.fn(), + getRun: vi.fn(), +})); +vi.mock("@/lib/agents/registry", () => ({ + getAgentById: vi.fn(), + getDeploymentById: vi.fn(), + getWorkspaceById: vi.fn(), +})); +vi.mock("@/lib/compute/machines", () => ({ + getMachineByWorkspaceEnv: vi.fn(), + touchMachineLastSeen: vi.fn(), +})); +vi.mock("@/lib/compute/tokens", () => ({ + machineTokenMatches: vi.fn(), +})); + +import { completeRun, getRun } from "@/lib/runs/store"; +import { getAgentById, getDeploymentById, getWorkspaceById } from "@/lib/agents/registry"; +import { getMachineByWorkspaceEnv, touchMachineLastSeen } from "@/lib/compute/machines"; +import { machineTokenMatches } from "@/lib/compute/tokens"; +import { POST } from "@/app/api/v1/runs/[id]/complete/route"; + +const RUN = { id: "r_1", deploymentId: "dp_1", status: "running" }; +const DEPLOYMENT = { id: "dp_1", agentId: "ag_1", env: "prod" }; +const AGENT = { id: "ag_1", workspaceId: "ws_1" }; +const WORKSPACE = { id: "ws_1" }; +const MACHINE = { id: "wm_1", workspaceId: "ws_1", env: "prod", runNonce: "nonce-1" }; + +const VALID_BODY = { + status: "succeeded" as const, + output: { text: "done" }, + tokensIn: 100, + tokensOut: 50, + modelAlias: "kimi-k3", + traceUploaded: true, +}; + +function req(opts: { bearer?: string | null; body?: unknown } = {}): NextRequest { + // "bearer" absent from opts -> default token; explicit `bearer: null` -> + // no Authorization header at all (distinct from JS's destructuring + // default, which can't tell "key omitted" from "key set to undefined"). + const bearer = "bearer" in opts ? opts.bearer : "correct-token"; + const body = opts.body ?? VALID_BODY; + return new NextRequest("http://localhost/api/v1/runs/r_1/complete", { + method: "POST", + headers: { + ...(bearer ? { authorization: `Bearer ${bearer}` } : {}), + "content-type": "application/json", + }, + body: JSON.stringify(body), + }); +} + +function ctx(id = "r_1") { + return { params: Promise.resolve({ id }) }; +} + +function mockFullChain() { + vi.mocked(getRun).mockResolvedValue(RUN as never); + vi.mocked(getDeploymentById).mockResolvedValue(DEPLOYMENT as never); + vi.mocked(getAgentById).mockResolvedValue(AGENT as never); + vi.mocked(getWorkspaceById).mockResolvedValue(WORKSPACE as never); + vi.mocked(getMachineByWorkspaceEnv).mockResolvedValue(MACHINE as never); +} + +beforeEach(() => { + vi.clearAllMocks(); +}); + +describe("POST /api/v1/runs/[id]/complete", () => { + it("401s with a wrong bearer (machine credential doesn't match)", async () => { + mockFullChain(); + vi.mocked(machineTokenMatches).mockReturnValue(false); + + const res = await POST(req({ bearer: "wrong-token" }), ctx()); + + expect(res.status).toBe(401); + const body = await res.json(); + expect(body.error.code).toBe("unauthorized"); + expect(completeRun).not.toHaveBeenCalled(); + // Auth never succeeded — the staleness signal must not be touched for a + // credential that didn't check out. + expect(touchMachineLastSeen).not.toHaveBeenCalled(); + }); + + it("401s with no Authorization header at all", async () => { + mockFullChain(); + + const res = await POST(req({ bearer: null }), ctx()); + + expect(res.status).toBe(401); + expect(getRun).not.toHaveBeenCalled(); + }); + + it("401s when there is no machine row for the run's (workspace, env)", async () => { + vi.mocked(getRun).mockResolvedValue(RUN as never); + vi.mocked(getDeploymentById).mockResolvedValue(DEPLOYMENT as never); + vi.mocked(getAgentById).mockResolvedValue(AGENT as never); + vi.mocked(getWorkspaceById).mockResolvedValue(WORKSPACE as never); + vi.mocked(getMachineByWorkspaceEnv).mockResolvedValue(null); + + const res = await POST(req(), ctx()); + + expect(res.status).toBe(401); + expect(machineTokenMatches).not.toHaveBeenCalled(); + }); + + it("401s when the machine row has no runNonce yet", async () => { + vi.mocked(getRun).mockResolvedValue(RUN as never); + vi.mocked(getDeploymentById).mockResolvedValue(DEPLOYMENT as never); + vi.mocked(getAgentById).mockResolvedValue(AGENT as never); + vi.mocked(getWorkspaceById).mockResolvedValue(WORKSPACE as never); + vi.mocked(getMachineByWorkspaceEnv).mockResolvedValue({ ...MACHINE, runNonce: null } as never); + + const res = await POST(req(), ctx()); + + expect(res.status).toBe(401); + const body = await res.json(); + expect(body.error.code).toBe("unauthorized"); + }); + + it("calls completeRun with the parsed body and returns 200 for a valid token", async () => { + mockFullChain(); + vi.mocked(machineTokenMatches).mockReturnValue(true); + vi.mocked(completeRun).mockResolvedValue(undefined); + vi.mocked(getRun).mockResolvedValueOnce(RUN as never).mockResolvedValueOnce({ + ...RUN, + status: "succeeded", + } as never); + + const res = await POST(req({ body: VALID_BODY }), ctx()); + + expect(res.status).toBe(200); + const body = await res.json(); + expect(body.data).toEqual({ runId: "r_1", status: "succeeded" }); + expect(completeRun).toHaveBeenCalledWith("r_1", VALID_BODY); + }); + + it("is a no-op (still 200) when completing an already-terminal run", async () => { + // completeRun's own contract (unit-tested in lib/runs/store.ts) is that + // its UPDATE only matches status IN (queued, running) — a retried + // completion for a run that's already terminal changes nothing. At the + // route level, that means: completeRun is still called (the route + // doesn't pre-check status), but the re-read after it reflects the + // run's real prior terminal status, not whatever the retried POST body + // claimed, and the route still answers 200. + mockFullChain(); + vi.mocked(machineTokenMatches).mockReturnValue(true); + vi.mocked(completeRun).mockResolvedValue(undefined); + const alreadyTerminal = { ...RUN, status: "succeeded" }; + vi.mocked(getRun).mockResolvedValueOnce(alreadyTerminal as never).mockResolvedValueOnce(alreadyTerminal as never); + + const res = await POST(req({ body: { ...VALID_BODY, status: "failed" } }), ctx()); + + expect(res.status).toBe(200); + const body = await res.json(); + expect(body.data.status).toBe("succeeded"); // unchanged by the retried "failed" POST + expect(completeRun).toHaveBeenCalled(); + }); + + it("touches the machine's lastSeenAt once auth succeeds (Finding 1b)", async () => { + mockFullChain(); + vi.mocked(machineTokenMatches).mockReturnValue(true); + vi.mocked(completeRun).mockResolvedValue(undefined); + + await POST(req(), ctx()); + + expect(touchMachineLastSeen).toHaveBeenCalledWith("wm_1"); + expect(touchMachineLastSeen).toHaveBeenCalledTimes(1); + }); +});