feat(server): standardize setup-failure reporting via reportSetupFailure helper

Scenarios that can't execute (connect failure, missing fixture, capability
not advertised) currently hand-roll a try/catch around connect and pin the
error onto whichever check ID happens to be first. That mislabels the
failure and silently drops any *other* checks the scenario would emit (e.g.
a connect failure in tools-list also makes tools-name-format vanish).

Add `reportSetupFailure(scenarioName, error, specReferences?)` to
sdk-client.ts, emitting a single dedicated `<scenario>-setup` check as
FAILURE with the error detail. Route the setup path of four representative
multi-check server scenarios through it (server-initialize, tools-list,
prompts-list, resources-list): connect runs in its own try/catch that returns
the setup check; genuine post-connect failures keep their real check IDs.

Rebased onto the version-aware connection abstraction (modelcontextprotocol#318): scenarios now
take a RunContext and connect via ctx.connect(); the setup/exec split and the
reportSetupFailure helper carry over unchanged.

This is the "-setup check as a first cut" from modelcontextprotocol#248. Refs modelcontextprotocol#248.

Author: rinaldofesta

src/connection/sdk-client.test.ts

@@ -0,0 +1,40 @@
+import { describe, it, expect } from 'vitest';
+import { reportSetupFailure } from './sdk-client';
+
+describe('reportSetupFailure', () => {
+  it('emits a single FAILURE check id-d "<scenario>-setup"', () => {
+    const checks = reportSetupFailure(
+      'tools-list',
+      new Error('connect ECONNREFUSED')
+    );
+
+    expect(checks).toHaveLength(1);
+    expect(checks[0]).toMatchObject({
+      id: 'tools-list-setup',
+      status: 'FAILURE',
+      errorMessage: 'Setup failed: connect ECONNREFUSED'
+    });
+  });
+
+  it('stringifies a non-Error thrown value', () => {
+    const checks = reportSetupFailure('prompts-list', 'boom');
+
+    expect(checks[0]?.errorMessage).toBe('Setup failed: boom');
+  });
+
+  it('attaches spec references when provided and omits the field otherwise', () => {
+    const withRefs = reportSetupFailure('resources-list', new Error('nope'), [
+      { id: 'MCP-Resources-List' }
+    ]);
+    expect(withRefs[0]?.specReferences).toEqual([{ id: 'MCP-Resources-List' }]);
+
+    const withoutRefs = reportSetupFailure('resources-list', new Error('nope'));
+    expect(withoutRefs[0]).not.toHaveProperty('specReferences');
+  });
+
+  it('sets a timestamp', () => {
+    const checks = reportSetupFailure('server-initialize', new Error('x'));
+    expect(typeof checks[0]?.timestamp).toBe('string');
+    expect(checks[0]?.timestamp.length).toBeGreaterThan(0);
+  });
+});

src/connection/sdk-client.ts

@@ -8,12 +8,55 @@ import {
   LoggingMessageNotificationSchema,
   ProgressNotificationSchema
 } from '@modelcontextprotocol/sdk/types.js';
+import { ConformanceCheck } from '../types';
 
 export interface MCPClientConnection {
   client: Client;
   close: () => Promise<void>;
 }
 
+/**
+ * Emit a single `<scenarioName>-setup` check as FAILURE for a scenario that
+ * could not get far enough to evaluate its real checks (connect failure,
+ * missing fixture, capability not advertised, etc.).
+ *
+ * See #248: previously each scenario hand-rolled a try/catch around connect
+ * and pinned the setup error onto whichever check ID happened to be first.
+ * That mislabels the failure — the error ends up under a check that has
+ * nothing to do with the actual problem, and any *other* checks the scenario
+ * would have emitted silently disappear. Routing setup failures through this
+ * helper gives them a dedicated, semantically honest ID and a consistent
+ * output shape across scenarios.
+ *
+ * The convention is that a scenario that cannot execute counts as a FAILURE;
+ * the escape hatches are scenario filtering (`--suite`/`--scenario`) and the
+ * expected-failures baseline, not in-scenario skipping or silent passes.
+ *
+ * @param scenarioName The scenario's `name`; the emitted check id is
+ *   `<scenarioName>-setup`.
+ * @param error The thrown setup error.
+ * @param specReferences Optional spec references to attach to the check.
+ * @returns A one-element array, so a scenario can `return reportSetupFailure(...)`.
+ */
+export function reportSetupFailure(
+  scenarioName: string,
+  error: unknown,
+  specReferences?: ConformanceCheck['specReferences']
+): ConformanceCheck[] {
+  const message = error instanceof Error ? error.message : String(error);
+  return [
+    {
+      id: `${scenarioName}-setup`,
+      name: `${scenarioName} setup`,
+      description: `Scenario "${scenarioName}" could not be set up (connect/fixture/capability)`,
+      status: 'FAILURE',
+      timestamp: new Date().toISOString(),
+      errorMessage: `Setup failed: ${message}`,
+      ...(specReferences ? { specReferences } : {})
+    }
+  ];
+}
+
 /**
  * Create and connect an MCP client to a server
  */

src/scenarios/server/lifecycle.test.ts

@@ -2,9 +2,15 @@ import { testContext } from '../../connection/testing';
 import { ServerInitializeScenario } from './lifecycle';
 import { connectToServer } from '../../connection/sdk-client';
 
-vi.mock('../../connection/sdk-client', () => ({
-  connectToServer: vi.fn()
-}));
+vi.mock(import('../../connection/sdk-client'), async (importOriginal) => {
+  const actual = await importOriginal();
+  return {
+    ...actual,
+    // Only the connection factory is mocked; reportSetupFailure stays real so
+    // the scenario's setup-failure path is exercised end-to-end.
+    connectToServer: vi.fn()
+  };
+});
 
 describe('ServerInitializeScenario', () => {
   const serverUrl = 'http://localhost:3000/mcp';
@@ -98,4 +104,25 @@ describe('ServerInitializeScenario', () => {
       }
     });
   });
+
+  it('reports a single setup FAILURE when the connection cannot be established', async () => {
+    vi.mocked(connectToServer).mockRejectedValueOnce(
+      new Error('connect ECONNREFUSED 127.0.0.1:3000')
+    );
+
+    const checks = await new ServerInitializeScenario().run(
+      testContext(serverUrl)
+    );
+
+    // A connect failure should not be mislabeled as the initialize or
+    // session-id check failing (#248): a single dedicated setup check instead.
+    expect(checks).toHaveLength(1);
+    expect(checks[0]).toMatchObject({
+      id: 'server-initialize-setup',
+      status: 'FAILURE',
+      errorMessage: 'Setup failed: connect ECONNREFUSED 127.0.0.1:3000'
+    });
+    // The session-id check is never reached, so no raw fetch is attempted.
+    expect(fetchMock).not.toHaveBeenCalled();
+  });
 });

src/scenarios/server/lifecycle.ts

@@ -8,7 +8,10 @@ import {
   DRAFT_PROTOCOL_VERSION
 } from '../../types';
 import type { RunContext } from '../../connection';
-import { connectToServer } from '../../connection/sdk-client';
+import {
+  connectToServer,
+  reportSetupFailure
+} from '../../connection/sdk-client';
 
 const VISIBLE_ASCII_REGEX = /^[\x21-\x7E]+$/;
 
@@ -70,22 +73,10 @@ and validates session ID format if one is assigned.`;
 
       await connection.close();
     } catch (error) {
-      checks.push({
-        id: 'server-initialize',
-        name: 'ServerInitialize',
-        description:
-          'Server responds to initialize request with valid structure',
-        status: 'FAILURE',
-        timestamp: new Date().toISOString(),
-        errorMessage: `Failed to initialize: ${error instanceof Error ? error.message : String(error)}`,
-        specReferences: [
-          {
-            id: 'MCP-Initialize',
-            url: 'https://modelcontextprotocol.io/specification/2025-06-18/basic/lifecycle#initialization'
-          }
-        ]
-      });
-      return checks;
+      // The handshake never completed, so neither the initialize check nor the
+      // session-id check below can be evaluated. Report a single setup failure
+      // rather than mislabeling it as one specific check failing (#248).
+      return reportSetupFailure(this.name, error);
     }
 
     // Check: Session ID visible ASCII validation

src/scenarios/server/prompts.ts

@@ -3,7 +3,8 @@
  */
 
 import { ClientScenario, ConformanceCheck } from '../../types';
-import type { RunContext } from '../../connection';
+import type { Connection, RunContext } from '../../connection';
+import { reportSetupFailure } from '../../connection/sdk-client';
 import type {
   ListPromptsResult,
   GetPromptResult
@@ -28,9 +29,16 @@ export class PromptsListScenario implements ClientScenario {
   async run(ctx: RunContext): Promise<ConformanceCheck[]> {
     const checks: ConformanceCheck[] = [];
 
+    let conn: Connection;
     try {
-      const conn = await ctx.connect();
+      conn = await ctx.connect();
+    } catch (error) {
+      // A connect failure isn't a `prompts-list` failure; report it as a setup
+      // failure rather than mislabeling the check (#248).
+      return reportSetupFailure(this.name, error);
+    }
 
+    try {
       const result = await conn.request<ListPromptsResult>('prompts/list');
 
       // Validate response structure

src/scenarios/server/resources.ts

@@ -7,7 +7,12 @@ import {
   ConformanceCheck,
   DRAFT_PROTOCOL_VERSION
 } from '../../types';
-import { JsonRpcError, type RunContext } from '../../connection';
+import {
+  JsonRpcError,
+  type Connection,
+  type RunContext
+} from '../../connection';
+import { reportSetupFailure } from '../../connection/sdk-client';
 import type {
   ListResourcesResult,
   ReadResourceResult,
@@ -36,9 +41,16 @@ export class ResourcesListScenario implements ClientScenario {
   async run(ctx: RunContext): Promise<ConformanceCheck[]> {
     const checks: ConformanceCheck[] = [];
 
+    let conn: Connection;
     try {
-      const conn = await ctx.connect();
+      conn = await ctx.connect();
+    } catch (error) {
+      // A connect failure isn't a `resources-list` failure; report it as a
+      // setup failure rather than mislabeling the check (#248).
+      return reportSetupFailure(this.name, error);
+    }
 
+    try {
       const result = await conn.request<ListResourcesResult>('resources/list');
 
       // Validate response structure

src/scenarios/server/tools.ts

@@ -7,14 +7,15 @@ import {
   ConformanceCheck,
   DRAFT_PROTOCOL_VERSION
 } from '../../types';
-import type { RunContext } from '../../connection';
+import type { Connection, RunContext } from '../../connection';
 import type {
   ListToolsResult,
   CallToolResult
 } from '../../spec-types/2025-06-18';
 import {
   connectToServer,
-  NotificationCollector
+  NotificationCollector,
+  reportSetupFailure
 } from '../../connection/sdk-client';
 import {
   CreateMessageRequestSchema,
@@ -117,9 +118,17 @@ export class ToolsListScenario implements ClientScenario {
   async run(ctx: RunContext): Promise<ConformanceCheck[]> {
     const checks: ConformanceCheck[] = [];
 
+    let conn: Connection;
     try {
-      const conn = await ctx.connect();
+      conn = await ctx.connect();
+    } catch (error) {
+      // A connect failure isn't a `tools-list` failure; pinning it there would
+      // also drop the `tools-name-format` check entirely. Report it as setup
+      // (#248).
+      return reportSetupFailure(this.name, error);
+    }
 
+    try {
       const result = await conn.request<ListToolsResult>('tools/list');
 
       // Validate response structure