Add structured logging, MCP request logging, and an OpenTelemetry pipeline across server apps

AGENTS.md

@@ -70,6 +70,11 @@ a given Git URL into `.reference` and pull latest before using it.
   vocabulary the HTTP protocol plugins compose (core never imports it, keeping it
   carrier-agnostic).
 - `packages/core/storage-*`: storage adapters and test support.
+- `packages/core/observability`: the shared structured-logging + OTLP layer
+  (`observabilityLayer`): JSON logs to stderr with trace correlation, plus
+  OTLP traces/logs/metrics export via `effect/unstable/observability` (works
+  on Bun and workerd). Every server app boots it; see RUNNING.md for the env
+  vars.
 - `packages/plugins/*`: protocol and provider plugins; their runtime, React, API,
   and testing helpers live with the owning plugin.
 - `packages/react`: shared React UI and atom/client integration.

RUNNING.md

@@ -40,6 +40,42 @@ develop on its `main`, publish a bump, then bump the dependency here. The
 The e2e globalsetup files are the source of truth for "how do I boot a
 working instance of X" — read them before inventing a boot path.
 
+## Observability: structured logs + OTel
+
+Every server app (local, host-selfhost, host-cloudflare, cloud) boots the
+shared `@executor-js/observability` layer: structured JSON logging plus an
+optional OTLP traces/logs/metrics pipeline. Config is env-driven and inert
+when unset:
+
+- `OTEL_EXPORTER_OTLP_ENDPOINT` — OTLP/HTTP base URL (e.g.
+  `http://localhost:4318` or a Traceway/collector endpoint);
+  `/v1/{traces,logs,metrics}` are appended. Unset → no export, JSON logging
+  stays on.
+- `OTEL_EXPORTER_OTLP_HEADERS` — standard `key=value,key2=value2` format
+  (auth tokens etc.).
+- `LOG_LEVEL` — minimum level (`trace`/`debug`/`info`/`warn`/`error`),
+  default `info`. Applies to both the console lines and OTLP log records.
+
+Notes:
+
+- JSON log lines go to STDERR, never stdout — the MCP stdio transport owns
+  stdout as its JSON-RPC channel. Lines carry `trace_id`/`span_id` when a
+  span is active, so console logs correlate with exported traces.
+- On the Workers apps the vars are bindings (wrangler vars / `.dev.vars`),
+  not `process.env`. Cloud keeps its existing Axiom trace pipeline
+  (`AXIOM_TOKEN` etc.) and Sentry correlation; the OTLP endpoint there adds
+  logs + metrics only.
+- MCP protocol traffic is logged as structured events (`mcp.tool.start/end`
+  with outcome + duration, `mcp.session.*` lifecycle, `mcp.auth.outcome`,
+  elicitation at debug), and `mcp.tool.calls` / `mcp.tool.duration_ms`
+  metrics are emitted per tool call. `EXECUTOR_MCP_DEBUG=1` still enables
+  the verbose stderr debug hook.
+
+Quick local check: run an OTLP collector (or a stub printing posts to
+`/v1/*`), then
+`OTEL_EXPORTER_OTLP_ENDPOINT=http://127.0.0.1:4318 LOG_LEVEL=debug bun run start`
+in `apps/local` and drive an MCP `execute` call.
+
 ## E2E: running, viewing, sharing
 
 `e2e/AGENTS.md` covers writing scenarios. Operationally:

apps/cloud/package.json

@@ -41,6 +41,7 @@
     "@executor-js/execution": "workspace:*",
     "@executor-js/fumadb": "workspace:*",
     "@executor-js/host-mcp": "workspace:*",
+    "@executor-js/observability": "workspace:*",
     "@executor-js/plugin-google": "workspace:*",
     "@executor-js/plugin-graphql": "workspace:*",
     "@executor-js/plugin-mcp": "workspace:*",

apps/cloud/src/app.ts

@@ -23,7 +23,7 @@ import {
   CloudHostConfig,
   CloudPluginsProvider,
 } from "./engine/execution-stack";
-import { WorkerTelemetryLive } from "./observability/telemetry";
+import { CloudObservabilityLive, WorkerTelemetryLive } from "./observability/telemetry";
 
 // ===========================================================================
 // The Executor CLOUD app, as ONE `ExecutorApp.make` call.
@@ -121,7 +121,12 @@ const { appLayer, toWebHandler, mcpExport } = ExecutorApp.make({
   // platform. A boot-time WorkOS misconfig is unrecoverable -> `orDie`.
   boot: controlPlane.pipe(
     Layer.merge(
-      Layer.mergeAll(WorkerTelemetryLive, HttpServer.layerServices, AutumnService.Default),
+      Layer.mergeAll(
+        WorkerTelemetryLive,
+        CloudObservabilityLive,
+        HttpServer.layerServices,
+        AutumnService.Default,
+      ),
     ),
     // oxlint-disable-next-line executor/no-effect-escape-hatch -- boundary: a boot-time WorkOS misconfiguration is unrecoverable
     Layer.orDie,

apps/cloud/src/env-augment.d.ts

@@ -10,6 +10,13 @@ declare global {
       AXIOM_DATASET?: string;
       AXIOM_TRACES_URL?: string;
       AXIOM_TRACES_SAMPLE_RATIO?: string;
+      /** OTLP/HTTP base endpoint for structured logs + metrics export (traces
+       *  keep flowing through the Axiom pipeline above); unset disables it. */
+      OTEL_EXPORTER_OTLP_ENDPOINT?: string;
+      /** OTLP export headers in the standard `key=value,key2=value2` format. */
+      OTEL_EXPORTER_OTLP_HEADERS?: string;
+      /** Minimum structured-log level (trace/debug/info/warn/error/fatal). */
+      LOG_LEVEL?: string;
       SENTRY_DSN?: string;
       SENTRY_OTEL_LOG_PAYLOAD?: string;
       SENTRY_OTEL_VERIFY?: string;

apps/cloud/src/observability/telemetry.ts

@@ -45,6 +45,8 @@ import { ATTR_SERVICE_NAME, ATTR_SERVICE_VERSION } from "@opentelemetry/semantic
 import { env } from "cloudflare:workers";
 import { Effect, Layer } from "effect";
 
+import { observabilityLayer } from "@executor-js/observability";
+
 import {
   CountingSpanExporter,
   CountingSpanProcessor,
@@ -139,3 +141,21 @@ const makeTelemetryLive = (): Layer.Layer<never> =>
 export const WorkerTelemetryLive: Layer.Layer<never> = makeTelemetryLive();
 
 export const DoTelemetryLive: Layer.Layer<never> = makeTelemetryLive();
+
+// Structured JSON stdout logging + OTLP logs/metrics export. Traces are
+// deliberately NOT exported here (`traces: false`): they keep flowing through
+// the Axiom WebTracerProvider pipeline above, which the non-Effect fetch
+// paths (server.ts's raw `http.server` span) and Sentry correlation depend
+// on. `Layer.unwrap(Effect.sync(...))` defers the `env` read to layer build
+// time, matching `makeTelemetryLive`'s lazy gate.
+export const CloudObservabilityLive: Layer.Layer<never> = Layer.unwrap(
+  Effect.sync(() =>
+    observabilityLayer({
+      serviceName: SERVICE_NAME,
+      endpoint: env.OTEL_EXPORTER_OTLP_ENDPOINT,
+      headers: env.OTEL_EXPORTER_OTLP_HEADERS,
+      logLevel: env.LOG_LEVEL,
+      traces: false,
+    }),
+  ),
+);

apps/host-cloudflare/package.json

@@ -23,6 +23,7 @@
     "@executor-js/execution": "workspace:*",
     "@executor-js/fumadb": "workspace:*",
     "@executor-js/host-mcp": "workspace:*",
+    "@executor-js/observability": "workspace:*",
     "@executor-js/plugin-encrypted-secrets": "workspace:*",
     "@executor-js/plugin-google": "workspace:*",
     "@executor-js/plugin-graphql": "workspace:*",

apps/host-cloudflare/src/app.ts

@@ -1,4 +1,4 @@
-import { Effect } from "effect";
+import { Effect, Layer } from "effect";
 import { HttpEffect, HttpRouter } from "effect/unstable/http";
 
 import { dbProviderLayer, ExecutorApp, textFailureStrategy } from "@executor-js/api/server";
@@ -12,7 +12,7 @@ import {
   makeCloudflareHostConfig,
   makeCloudflarePluginsProvider,
 } from "./execution";
-import { ErrorCaptureLive } from "./observability";
+import { ErrorCaptureLive, makeCloudflareObservabilityLayer } from "./observability";
 import { cloudflareAccountMiddleware } from "./account/account-provider";
 import { makeCloudflareApprovalHandler } from "./mcp";
 import { makeCloudflareMcpAgentHandler } from "./mcp/agent-handler";
@@ -73,7 +73,7 @@ export const makeCloudflareApp = async (env: CloudflareEnv) => {
       ],
     },
     config: { mountPrefix: "/api", failure: textFailureStrategy },
-    boot: identityLayer,
+    boot: Layer.merge(identityLayer, makeCloudflareObservabilityLayer(env)),
   });
 
   return { appLayer, toWebHandler, mcpAgentHandler };

apps/host-cloudflare/src/config.ts

@@ -45,6 +45,12 @@ export interface CloudflareEnv {
   readonly EXECUTOR_SECRET_KEY?: string;
   readonly ALLOW_LOCAL_NETWORK?: string;
   readonly VITE_PUBLIC_SITE_URL?: string;
+  /** OTLP/HTTP base endpoint for traces/logs/metrics export; unset disables export. */
+  readonly OTEL_EXPORTER_OTLP_ENDPOINT?: string;
+  /** OTLP export headers in the standard `key=value,key2=value2` format. */
+  readonly OTEL_EXPORTER_OTLP_HEADERS?: string;
+  /** Minimum structured-log level (trace/debug/info/warn/error/fatal). */
+  readonly LOG_LEVEL?: string;
   /**
    * Dev/single-user escape hatch: when "true", skip Cloudflare Access entirely
    * and treat every request as a fixed admin. For local `wrangler dev` and

apps/host-cloudflare/src/observability.ts

@@ -1,7 +1,27 @@
-// Cloudflare host `ErrorCapture` — the shared console implementation with a
-// `cloudflare-` trace-id prefix. Worker stdout is routed to Logpush/the
-// dashboard, so the squashed cause is grep-able by the opaque 500 traceId.
+// Cloudflare host observability.
+//
+// `ErrorCaptureLive` — the shared console implementation with a `cloudflare-`
+// trace-id prefix. Worker stdout is routed to Logpush/the dashboard, so the
+// squashed cause is grep-able by the opaque 500 traceId.
+//
+// `makeCloudflareObservabilityLayer` — structured JSON stdout logging plus the
+// OTLP traces/logs/metrics pipeline (fetch-based, workerd-safe), enabled by
+// the `OTEL_EXPORTER_OTLP_ENDPOINT` binding. A Worker has no module-scope
+// bindings, so the layer closes over the per-fetch `env` instead of process.env.
+
+import type { Layer } from "effect";
 
 import { consoleErrorCapture } from "@executor-js/api/server";
+import { observabilityLayer } from "@executor-js/observability";
+
+import type { CloudflareEnv } from "./config";
 
 export const ErrorCaptureLive = consoleErrorCapture("cloudflare");
+
+export const makeCloudflareObservabilityLayer = (env: CloudflareEnv): Layer.Layer<never> =>
+  observabilityLayer({
+    serviceName: "executor-cloudflare",
+    endpoint: env.OTEL_EXPORTER_OTLP_ENDPOINT,
+    headers: env.OTEL_EXPORTER_OTLP_HEADERS,
+    logLevel: env.LOG_LEVEL,
+  });

apps/host-selfhost/package.json

@@ -26,6 +26,7 @@
     "@executor-js/execution": "workspace:*",
     "@executor-js/fumadb": "workspace:*",
     "@executor-js/host-mcp": "workspace:*",
+    "@executor-js/observability": "workspace:*",
     "@executor-js/plugin-encrypted-secrets": "workspace:*",
     "@executor-js/plugin-google": "workspace:*",
     "@executor-js/plugin-graphql": "workspace:*",

apps/host-selfhost/src/app.ts

@@ -20,7 +20,7 @@ import {
 } from "./execution";
 import { makeSelfHostMcpSeams } from "./mcp";
 import { selfHostPlugins } from "./plugins";
-import { ErrorCaptureLive } from "./observability";
+import { ErrorCaptureLive, SelfHostObservabilityLive } from "./observability";
 import { oauthCallbackSignInRedirectLocation } from "./auth/oauth-callback-login";
 
 // ===========================================================================
@@ -126,8 +126,13 @@ export const makeSelfHostApp = async (options: MakeSelfHostAppOptions = {}) => {
     config: { mountPrefix: "/api", failure: textFailureStrategy },
     // The boot-scoped context provideMerge'd under everything: the long-lived DB
     // handle (read by the DbProvider seam, Better Auth, and the MCP store) + the
-    // resolved identity (captured once by the execution middleware + MCP auth).
-    boot: Layer.merge(Layer.succeed(SelfHostDb)(dbHandle), identityLayer),
+    // resolved identity (captured once by the execution middleware + MCP auth)
+    // + structured logging / OTLP telemetry.
+    boot: Layer.mergeAll(
+      Layer.succeed(SelfHostDb)(dbHandle),
+      identityLayer,
+      SelfHostObservabilityLive,
+    ),
   });
 
   return {

apps/host-selfhost/src/observability.ts

@@ -1,11 +1,25 @@
 // ---------------------------------------------------------------------------
-// Self-host `ErrorCapture` — the shared console implementation with a
-// `selfhost-` trace id prefix. Prints the squashed + pretty cause to stderr
-// and returns a short correlation id that surfaces in the opaque 500 traceId,
-// so operators can grep their logs. Cloud swaps in a Sentry-backed impl behind
-// the same tag.
+// Self-host observability.
+//
+// `ErrorCaptureLive` — the shared console implementation with a `selfhost-`
+// trace id prefix. Prints the squashed + pretty cause to stderr and returns a
+// short correlation id that surfaces in the opaque 500 traceId, so operators
+// can grep their logs. Cloud swaps in a Sentry-backed impl behind the same tag.
+//
+// `SelfHostObservabilityLive` — structured JSON stdout logging plus the OTLP
+// traces/logs/metrics pipeline, enabled by `OTEL_EXPORTER_OTLP_ENDPOINT`.
 // ---------------------------------------------------------------------------
 
+import type { Layer } from "effect";
+
 import { consoleErrorCapture } from "@executor-js/api/server";
+import { observabilityLayer } from "@executor-js/observability";
 
 export const ErrorCaptureLive = consoleErrorCapture("selfhost");
+
+export const SelfHostObservabilityLive: Layer.Layer<never> = observabilityLayer({
+  serviceName: "executor-selfhost",
+  endpoint: process.env.OTEL_EXPORTER_OTLP_ENDPOINT,
+  headers: process.env.OTEL_EXPORTER_OTLP_HEADERS,
+  logLevel: process.env.LOG_LEVEL,
+});

apps/local/package.json

@@ -29,6 +29,7 @@
     "@executor-js/fumadb": "workspace:*",
     "@executor-js/host-mcp": "workspace:*",
     "@executor-js/integrations-registry": "workspace:*",
+    "@executor-js/observability": "workspace:*",
     "@executor-js/plugin-desktop-settings": "workspace:*",
     "@executor-js/plugin-example": "workspace:*",
     "@executor-js/plugin-file-secrets": "workspace:*",

apps/local/src/app.ts

@@ -12,7 +12,7 @@ import { makeQuickJsExecutor } from "@executor-js/runtime-quickjs";
 
 import { getExecutorBundle, type LocalExecutor } from "./executor";
 import { makeLocalIdentityLayer } from "./identity";
-import { ErrorCaptureLive } from "./observability";
+import { ErrorCaptureLive, LocalObservabilityLive } from "./observability";
 
 // ===========================================================================
 // The LOCAL Executor app, as ONE `ExecutorApp.make` call.
@@ -118,8 +118,9 @@ export const makeLocalApiHandler = async (token: string): Promise<LocalApiHandle
     config: { failure: textFailureStrategy },
     // The boot-scoped context provideMerge'd under everything: the identity
     // provider (captured once by the fixed-execution middleware) + the fixed
-    // execution seam (the one executor + engine + extension map).
-    boot: Layer.merge(identity, fixedExecution),
+    // execution seam (the one executor + engine + extension map) + structured
+    // logging / OTLP telemetry.
+    boot: Layer.mergeAll(identity, fixedExecution, LocalObservabilityLive),
   });
 
   const web = toWebHandler();

apps/local/src/main.ts

@@ -5,6 +5,7 @@ import { makeQuickJsExecutor } from "@executor-js/runtime-quickjs";
 import { makeLocalApiHandler } from "./app";
 import { createExecutorHandle, disposeExecutor, getExecutorBundle } from "./executor";
 import { createMcpRequestHandler, type McpRequestHandler } from "./mcp";
+import { disposeObservabilityRuntime } from "./observability";
 
 // ---------------------------------------------------------------------------
 // Local server handlers.
@@ -36,7 +37,12 @@ export type ServerHandlers = {
 };
 
 class ServerHandlersDisposeError extends Data.TaggedError("ServerHandlersDisposeError")<{
-  readonly operation: "api.dispose" | "mcp.close" | "disposeExecutor" | "runtime.dispose";
+  readonly operation:
+    | "api.dispose"
+    | "mcp.close"
+    | "disposeExecutor"
+    | "runtime.dispose"
+    | "observability.dispose";
   readonly cause: unknown;
 }> {}
 
@@ -63,6 +69,9 @@ const closeServerHandlers = async (handlers: ServerHandlers): Promise<void> => {
       // after the surfaces are closed so server shutdown (and failed startup
       // cleanup via disposeServerHandlers) releases the owned data-dir lock.
       yield* ignoreDisposeFailure("disposeExecutor", () => disposeExecutor());
+      // Flush buffered OTLP telemetry from the MCP surface's runtime last, so
+      // logs emitted during the disposals above still export.
+      yield* ignoreDisposeFailure("observability.dispose", () => disposeObservabilityRuntime());
     }),
   );
 };