fix(tracing): close four silent gaps in Phoenix span coverage

.env.local.example

@@ -130,14 +130,13 @@ ENABLE_GUEST_CHAT=true
 # PHOENIX_API_KEY=your_key_here
 # EVAL_REPLAY_TRACING_ENABLED=false
 #
-# Optional OpenInference masking for production traces.
+# Masks LLM prompt/output content on AI SDK spans (recordInputs/recordOutputs).
+# Only the exact string "true" masks — "TRUE"/"1"/"yes" are ignored and record.
+# NOT de-identification: the chat root span is built outside the AI SDK and still
+# carries session id, user id, and request metadata (model, search mode,
+# correlation id) regardless of these flags.
 # OPENINFERENCE_HIDE_INPUTS=true
 # OPENINFERENCE_HIDE_OUTPUTS=true
-# OPENINFERENCE_HIDE_INPUT_MESSAGES=true
-# OPENINFERENCE_HIDE_OUTPUT_MESSAGES=true
-# OPENINFERENCE_HIDE_INPUT_IMAGES=true
-# OPENINFERENCE_HIDE_INPUT_TEXT=true
-# OPENINFERENCE_BASE64_IMAGE_MAX_LENGTH=10000
 
 # Performance diagnostics logging.
 # ENABLE_PERF_LOGGING=true

app/api/health/route.test.ts

@@ -0,0 +1,74 @@
+import { NextRequest } from 'next/server'
+
+import { afterEach, describe, expect, it, vi } from 'vitest'
+
+const mockExecute = vi.fn()
+
+vi.mock('@/lib/db', () => ({
+  db: {
+    execute: (...args: unknown[]) => mockExecute(...args)
+  }
+}))
+
+import { GET } from './route'
+
+function makeRequest(query = ''): NextRequest {
+  return new NextRequest(`http://localhost/api/health${query}`)
+}
+
+afterEach(() => {
+  vi.unstubAllEnvs()
+  mockExecute.mockReset()
+  delete globalThis.__polymorphTracingState
+})
+
+describe('GET /api/health', () => {
+  it('includes tracing in the body for check=phoenix, reflecting the current global state', async () => {
+    mockExecute.mockResolvedValue(undefined)
+    globalThis.__polymorphTracingState = 'disabled-https'
+
+    const response = await GET(makeRequest('?check=phoenix'))
+    const body = await response.json()
+
+    expect(body.tracing).toBe('disabled-https')
+  })
+
+  it('includes tracing in the body for check=all, reflecting the current global state', async () => {
+    mockExecute.mockResolvedValue(undefined)
+    globalThis.__polymorphTracingState = 'enabled'
+
+    const response = await GET(makeRequest('?check=all'))
+    const body = await response.json()
+
+    expect(body.tracing).toBe('enabled')
+  })
+
+  it('omits tracing from the body when check is unset', async () => {
+    mockExecute.mockResolvedValue(undefined)
+    globalThis.__polymorphTracingState = 'enabled'
+
+    const response = await GET(makeRequest())
+    const body = await response.json()
+
+    expect(body).not.toHaveProperty('tracing')
+  })
+
+  it('omits tracing from the body for an unrecognized check value', async () => {
+    mockExecute.mockResolvedValue(undefined)
+    globalThis.__polymorphTracingState = 'enabled'
+
+    const response = await GET(makeRequest('?check=db'))
+    const body = await response.json()
+
+    expect(body).not.toHaveProperty('tracing')
+  })
+
+  it('falls back to unknown when the global tracing state was never set', async () => {
+    mockExecute.mockResolvedValue(undefined)
+
+    const response = await GET(makeRequest('?check=phoenix'))
+    const body = await response.json()
+
+    expect(body.tracing).toBe('unknown')
+  })
+})

app/api/health/route.ts

@@ -4,6 +4,16 @@ import { sql } from 'drizzle-orm'
 
 import { db } from '@/lib/db'
 
+type TracingState =
+  | 'enabled'
+  | 'disabled-off'
+  | 'disabled-https'
+  | 'init-failed'
+
+declare global {
+  var __polymorphTracingState: TracingState | undefined
+}
+
 export const dynamic = 'force-dynamic'
 
 export async function GET(req: NextRequest) {
@@ -63,6 +73,12 @@ export async function GET(req: NextRequest) {
   }
   if (dbError) body.dbError = dbError
   if (phoenixStatus !== undefined) body.phoenix = phoenixStatus
+  // `phoenix: 'ok'` only means the collector is reachable. `tracing` says
+  // whether THIS process registered an exporter — the blind-deploy signature
+  // is phoenix: 'ok' with tracing: 'disabled-https'.
+  if (checks === 'phoenix' || checks === 'all') {
+    body.tracing = globalThis.__polymorphTracingState ?? 'unknown'
+  }
 
   return NextResponse.json(body, { status: isHealthy ? 200 : 503 })
 }

docs/architecture/STREAMING-OPERATIONS.md

@@ -108,15 +108,22 @@ When the client navigates away or closes the tab, the browser terminates the SSE
 4. Tool executions (search, fetch)
 5. Title and related question generation
 
-The `onFinish` callback checks `isAborted` and skips persistence if the stream was aborted:
+The `onFinish` callback skips **persistence** if the stream was aborted or produced no response message, but trace flushing runs unconditionally in a `finally` block — aborted streams' spans still reach Phoenix:
 
 ```typescript
 onFinish: async ({ responseMessage, isAborted }) => {
-  if (isAborted || !responseMessage) return
-  // ... persist
+  try {
+    if (!isAborted && responseMessage) {
+      // ... persist
+    }
+  } finally {
+    await flushTraces() // always runs, aborted or not
+  }
 }
 ```
 
+Aborts are the traces most worth keeping, and flushing after the persistence attempt means the exported spans include DB write latency.
+
 ### Database Persistence Errors
 
 The persistence layer (`persistStreamResults`) is designed to never break the stream:

docs/getting-started/ENVIRONMENT-OPERATIONS.md

@@ -52,9 +52,9 @@ EVAL_REPLAY_TRACING_ENABLED=false
 
 Set these in the Vercel dashboard under **Settings → Environment Variables** for the Production environment. Since Vercel serverless functions run outside of any private network, the Phoenix endpoint must be publicly reachable (with auth via `PHOENIX_API_KEY`).
 
-For production masking, configure OpenInference environment variables according to the sensitivity of your trace data: `OPENINFERENCE_HIDE_INPUTS`, `OPENINFERENCE_HIDE_OUTPUTS`, `OPENINFERENCE_HIDE_INPUT_MESSAGES`, `OPENINFERENCE_HIDE_OUTPUT_MESSAGES`, `OPENINFERENCE_HIDE_INPUT_IMAGES`, `OPENINFERENCE_HIDE_INPUT_TEXT`, and `OPENINFERENCE_BASE64_IMAGE_MAX_LENGTH`.
+For production masking, configure `OPENINFERENCE_HIDE_INPUTS` and `OPENINFERENCE_HIDE_OUTPUTS`. These mask LLM prompt and output content on AI SDK spans (via `recordInputs`/`recordOutputs`). Only the exact string `true` masks — `TRUE`/`1`/`yes` are ignored and record normally. They are **not** de-identification: the `chat-response` root span is built outside the AI SDK and still carries session id, user id, and request metadata regardless. See [Phoenix operations](../operations/PHOENIX-OPERATIONS.md) for the full caveats.
 
-**`PHOENIX_API_KEY` vs `OTEL_EXPORTER_OTLP_HEADERS`:** `instrumentation.ts` (lines 29-31) reads `PHOENIX_API_KEY` and explicitly sets the `Authorization: Bearer` header on the `OTLPTraceExporter`. The standard OTel env var `OTEL_EXPORTER_OTLP_HEADERS=Authorization=Bearer <key>` accomplishes the same thing at the SDK level. Setting `PHOENIX_API_KEY` alone is sufficient. Adding `OTEL_EXPORTER_OTLP_HEADERS` is harmless as a belt-and-suspenders approach but not required.
+**`PHOENIX_API_KEY` vs `OTEL_EXPORTER_OTLP_HEADERS`:** `instrumentation.ts` (lines 63-64) reads `PHOENIX_API_KEY` and explicitly sets the `Authorization: Bearer` header on the `OTLPTraceExporter`. The standard OTel env var `OTEL_EXPORTER_OTLP_HEADERS=Authorization=Bearer <key>` accomplishes the same thing at the SDK level. Setting `PHOENIX_API_KEY` alone is sufficient. Adding `OTEL_EXPORTER_OTLP_HEADERS` is harmless as a belt-and-suspenders approach but not required.
 
 **Local development:** Set `ENABLE_TRACING=true` and leave `PHOENIX_COLLECTOR_ENDPOINT` at the default (`http://localhost:6006`) if running Phoenix locally via Docker.
 

docs/operations/PHOENIX-OPERATIONS.md

@@ -73,17 +73,17 @@ Set these env vars in the Vercel dashboard (Settings → Environment Variables,
 
 The app exports traces to `${PHOENIX_COLLECTOR_ENDPOINT}/v1/traces` with `Authorization: Bearer $PHOENIX_API_KEY` from `instrumentation.ts`. Use low-cardinality Phoenix projects such as `polymorph-prod`; keep per-request details in metadata (`correlationId`, `otelTraceId`, model, mode, and eval case fields).
 
-For production, set OpenInference masking according to the data you are comfortable storing in Phoenix:
-
-| Variable                                | Typical production value |
-| --------------------------------------- | ------------------------ |
-| `OPENINFERENCE_HIDE_INPUTS`             | `true`                   |
-| `OPENINFERENCE_HIDE_OUTPUTS`            | `true`                   |
-| `OPENINFERENCE_HIDE_INPUT_MESSAGES`     | `true`                   |
-| `OPENINFERENCE_HIDE_OUTPUT_MESSAGES`    | `true`                   |
-| `OPENINFERENCE_HIDE_INPUT_IMAGES`       | `true`                   |
-| `OPENINFERENCE_HIDE_INPUT_TEXT`         | `true`                   |
-| `OPENINFERENCE_BASE64_IMAGE_MAX_LENGTH` | `10000`                  |
+For production, set OpenInference masking according to the data you are comfortable storing in Phoenix. `OPENINFERENCE_HIDE_INPUTS` / `OPENINFERENCE_HIDE_OUTPUTS` mask LLM prompt and output content on AI SDK spans (via `recordInputs`/`recordOutputs`).
+
+Two limits worth knowing before you rely on them:
+
+- **Only the exact string `true` masks.** `TRUE`, `1`, and `yes` are ignored and the span records normally. The control fails toward recording, not toward hiding, so a typo silently leaves content exposed.
+- **This is not de-identification.** The `chat-response` root span is constructed outside the AI SDK (`lib/streaming/create-chat-stream-response.ts`) and so is unreachable by these flags. It still carries the session id (chat id), user id, and request metadata — correlation id, model id, search mode, user mode, intent — no matter how the flags are set. Masking removes message content, not who sent it.
+
+| Variable                     | Typical production value |
+| ---------------------------- | ------------------------ |
+| `OPENINFERENCE_HIDE_INPUTS`  | `true`                   |
+| `OPENINFERENCE_HIDE_OUTPUTS` | `true`                   |
 
 See [Environment Reference](../getting-started/ENVIRONMENT-OPERATIONS.md#tracing-arize-phoenix) for details.
 

docs/reference/API-AUXILIARY-ENDPOINTS.md

@@ -172,7 +172,8 @@ For Vercel monitoring, use the canonical production alias (`https://polymorph.fy
   "status": "ok",
   "timestamp": "2025-01-15T10:30:00.000Z",
   "db": "connected",
-  "phoenix": "ok"
+  "phoenix": "ok",
+  "tracing": "enabled"
 }
 ```
 
@@ -189,4 +190,6 @@ For Vercel monitoring, use the canonical production alias (`https://polymorph.fy
 
 > **Note:** Phoenix status is advisory-only and does not affect the HTTP status code. The endpoint returns 503 only when the database is unreachable.
 
+> **`phoenix: 'ok'` is not proof tracing is active.** With `check=phoenix` or `check=all`, the response also includes `tracing`, one of `enabled` | `disabled-off` | `disabled-https` | `init-failed` | `unknown`. `phoenix` reports whether the Phoenix collector is reachable over the network; `tracing` reports whether _this process_ actually registered an OTel exporter (set in `instrumentation.ts`). The blind-deploy signature to watch for is `phoenix: 'ok'` together with `tracing: 'disabled-https'` — the collector is up, but this deployment silently disabled export because `PHOENIX_COLLECTOR_ENDPOINT` wasn't `https://` in production (see [Environment Operations → Tracing](../getting-started/ENVIRONMENT-OPERATIONS.md#tracing-arize-phoenix)).
+
 ---

instrumentation.test.ts

@@ -0,0 +1,38 @@
+import { afterEach, describe, expect, it, vi } from 'vitest'
+
+import { register } from './instrumentation'
+
+afterEach(() => {
+  vi.unstubAllEnvs()
+  delete globalThis.__polymorphTracingState
+})
+
+describe('register', () => {
+  it('sets tracing state to disabled-off when ENABLE_TRACING is not true', async () => {
+    vi.stubEnv('ENABLE_TRACING', 'false')
+
+    await register()
+
+    expect(globalThis.__polymorphTracingState).toBe('disabled-off')
+  })
+
+  it('sets tracing state to disabled-https when the collector endpoint is plain HTTP in production', async () => {
+    vi.stubEnv('ENABLE_TRACING', 'true')
+    vi.stubEnv('VERCEL_ENV', 'production')
+    vi.stubEnv('NEXT_PUBLIC_APP_URL', 'https://example.com')
+    vi.stubEnv('PHOENIX_COLLECTOR_ENDPOINT', 'http://collector.example.com')
+    const consoleErrorSpy = vi
+      .spyOn(console, 'error')
+      .mockImplementation(() => {})
+
+    await register()
+
+    expect(globalThis.__polymorphTracingState).toBe('disabled-https')
+    expect(consoleErrorSpy).toHaveBeenCalledWith(
+      expect.stringContaining(
+        'PHOENIX_COLLECTOR_ENDPOINT must use HTTPS in production'
+      )
+    )
+    consoleErrorSpy.mockRestore()
+  })
+})

instrumentation.ts

@@ -1,8 +1,20 @@
 import { isProductionTarget, validateEnv } from '@/lib/config/env'
 
+type TracingState =
+  | 'enabled'
+  | 'disabled-off'
+  | 'disabled-https'
+  | 'init-failed'
+
+declare global {
+  var __polymorphTracingState: TracingState | undefined
+}
+
 export async function register() {
   validateEnv()
 
+  globalThis.__polymorphTracingState = 'disabled-off'
+
   if (process.env.ENABLE_TRACING === 'true') {
     try {
       const { SEMRESATTRS_PROJECT_NAME } =
@@ -23,6 +35,7 @@ export async function register() {
         console.error(
           '[otel] PHOENIX_COLLECTOR_ENDPOINT must use HTTPS in production. Tracing disabled.'
         )
+        globalThis.__polymorphTracingState = 'disabled-https'
         return
       }
 
@@ -59,11 +72,14 @@ export async function register() {
         ]
       })
 
+      globalThis.__polymorphTracingState = 'enabled'
+
       console.log(
         `[otel] Tracing enabled → ${collectorEndpoint} (project: ${process.env.PHOENIX_PROJECT_NAME ?? 'polymorph-local'})`
       )
     } catch (err) {
       console.error('[otel] Failed to initialize tracing:', err)
+      globalThis.__polymorphTracingState = 'init-failed'
       // Tracing is optional — app continues without it
     }
   }

lib/agents/__tests__/researcher.test.ts

@@ -101,7 +101,10 @@ vi.mock('@/lib/utils/registry', () => ({
   isProviderEnabled: vi.fn().mockReturnValue(true)
 }))
 vi.mock('@/lib/utils/telemetry', () => ({
-  isTracingEnabled: vi.fn().mockReturnValue(false)
+  isTracingEnabled: vi.fn().mockReturnValue(false),
+  telemetryRecordingOptions: vi
+    .fn()
+    .mockReturnValue({ recordInputs: true, recordOutputs: true })
 }))
 vi.mock('@/lib/agents/prompts/search-mode-prompts', () => ({
   ARTIFACT_INTAKE_PROTOCOL: 'Artifact intake protocol',

lib/agents/__tests__/title-generator.test.ts

@@ -10,7 +10,10 @@ vi.mock('@/lib/utils/registry', () => ({
 }))
 
 vi.mock('@/lib/utils/telemetry', () => ({
-  isTracingEnabled: vi.fn().mockReturnValue(false)
+  isTracingEnabled: vi.fn().mockReturnValue(false),
+  telemetryRecordingOptions: vi
+    .fn()
+    .mockReturnValue({ recordInputs: true, recordOutputs: true })
 }))
 
 import { generateText } from 'ai'

lib/agents/chat/__tests__/registry.test.ts

@@ -46,7 +46,11 @@ const toolWiringMocks = vi.hoisted(() => {
     stepCountIs: vi.fn(maxSteps => ({ maxSteps })),
     tool: vi.fn(config => config),
     getModel: vi.fn(model => ({ model })),
-    isTracingEnabled: vi.fn(() => false)
+    isTracingEnabled: vi.fn(() => false),
+    telemetryRecordingOptions: vi.fn(() => ({
+      recordInputs: true,
+      recordOutputs: true
+    }))
   }
 })
 
@@ -81,7 +85,8 @@ vi.mock('@/lib/utils/registry', () => ({
 }))
 
 vi.mock('@/lib/utils/telemetry', () => ({
-  isTracingEnabled: toolWiringMocks.isTracingEnabled
+  isTracingEnabled: toolWiringMocks.isTracingEnabled,
+  telemetryRecordingOptions: toolWiringMocks.telemetryRecordingOptions
 }))
 
 vi.mock('@/lib/tools/create-canvas-artifact/server', () => ({

lib/agents/chat/factory.ts

@@ -9,7 +9,10 @@ import type { SearchMode, UserMode } from '@/lib/types/search'
 import { createModelId } from '@/lib/utils'
 import { selectModelForModeAndType } from '@/lib/utils/model-selection'
 import { getModel } from '@/lib/utils/registry'
-import { isTracingEnabled } from '@/lib/utils/telemetry'
+import {
+  isTracingEnabled,
+  telemetryRecordingOptions
+} from '@/lib/utils/telemetry'
 
 import { type ChatAgentTools, createChatAgentTools } from './toolset'
 
@@ -141,6 +144,7 @@ export function createConfiguredChatAgent(
       experimental_telemetry: {
         isEnabled: telemetryEnabled ?? isTracingEnabled(),
         functionId: `${definition.agentId}-agent`,
+        ...telemetryRecordingOptions(),
         metadata: {
           modelId: model,
           agentType: definition.agentId,

lib/agents/generate-related-questions.ts

@@ -1,10 +1,14 @@
 import { type ModelMessage, Output, streamText } from 'ai'
 
+import {
+  isTracingEnabled,
+  telemetryRecordingOptions
+} from '@/lib/utils/telemetry'
+
 import { getRelatedQuestionsModel } from '../config/model-types'
 import { relatedQuestionSchema } from '../schema/related'
 import { createModelId } from '../utils'
 import { getModel } from '../utils/registry'
-import { isTracingEnabled } from '../utils/telemetry'
 
 import { RELATED_QUESTIONS_PROMPT } from './prompts/related-questions-prompt'
 
@@ -36,6 +40,7 @@ export function createRelatedQuestionsStream(
     experimental_telemetry: {
       isEnabled: isTracingEnabled(),
       functionId: 'related-questions',
+      ...telemetryRecordingOptions(),
       metadata: {
         modelId,
         agentType: 'related-questions-generator',