feat: add integration tests with real MCP servers

- Add stdio and HTTP test fixture servers
- Integration tests for stdio transport probing
- Integration tests for streamable-http transport probing
- Tests for normalization end-to-end
- Tests for custom messages
- Tests for environment variable passing
- Use dynamic port allocation (port 0) to avoid conflicts
- Add proper process cleanup with SIGTERM/SIGKILL
- Add forceExit to Jest config for clean test exit
- Use client.close() for proper connection cleanup

Author: SamMorrowDrums

action.yml

@@ -89,6 +89,10 @@ inputs:
     description: 'Git ref to compare against (auto-detects merge-base or previous tag if not set)'
     required: false
     default: ''
+  fail_on_error:
+    description: 'Fail the action if probe errors occur (not just API differences)'
+    required: false
+    default: 'true'
   env_vars:
     description: 'Environment variables (newline-separated KEY=VALUE pairs)'
     required: false
@@ -156,6 +160,7 @@ runs:
         INPUT_START_COMMAND: ${{ inputs.start_command }}
         INPUT_SERVER_TIMEOUT: ${{ inputs.server_timeout }}
         INPUT_COMPARE_REF: ${{ inputs.compare_ref }}
+        INPUT_FAIL_ON_ERROR: ${{ inputs.fail_on_error }}
         INPUT_TRANSPORT: ${{ inputs.transport }}
         INPUT_SERVER_URL: ${{ inputs.server_url }}
         INPUT_CONFIGURATIONS: ${{ inputs.configurations }}

dist/__tests__/fixtures/http-server.d.ts

@@ -0,0 +1,7 @@
+#!/usr/bin/env npx tsx
+/**
+ * Minimal MCP Server for integration testing (streamable-http transport)
+ *
+ * Run with: npx tsx http-server.ts <port>
+ */
+export {};

dist/__tests__/fixtures/stdio-server.d.ts

@@ -0,0 +1,7 @@
+#!/usr/bin/env npx tsx
+/**
+ * Minimal MCP Server for integration testing (stdio transport)
+ *
+ * This server exposes tools, prompts, and resources for testing the probe functionality.
+ */
+export {};

dist/index.js

@@ -42,6 +42,7 @@ export interface ActionInputs {
     configurations: TestConfiguration[];
     customMessages: CustomMessage[];
     compareRef: string;
+    failOnError: boolean;
     envVars: string;
     serverTimeout: number;
 }

dist/index.js.map

@@ -3,6 +3,8 @@ export default {
   preset: "ts-jest/presets/default-esm",
   testEnvironment: "node",
   extensionsToTreatAsEsm: [".ts"],
+  // Force exit after tests complete - needed because npx tsx leaves handles open
+  forceExit: true,
   moduleNameMapper: {
     "^(\\.{1,2}/.*)\\.js$": "$1",
   },

dist/types.d.ts

@@ -0,0 +1,103 @@
+#!/usr/bin/env npx tsx
+/**
+ * Minimal MCP Server for integration testing (streamable-http transport)
+ *
+ * Run with: npx tsx http-server.ts <port>
+ */
+
+import { Server } from "@modelcontextprotocol/sdk/server/index.js";
+import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/streamableHttp.js";
+import { CallToolRequestSchema, ListToolsRequestSchema } from "@modelcontextprotocol/sdk/types.js";
+import * as http from "http";
+
+const server = new Server(
+  {
+    name: "test-http-server",
+    version: "1.0.0",
+  },
+  {
+    capabilities: {
+      tools: {},
+    },
+  }
+);
+
+// Define tools
+server.setRequestHandler(ListToolsRequestSchema, async () => {
+  return {
+    tools: [
+      {
+        name: "echo",
+        description: "Echoes back the input",
+        inputSchema: {
+          type: "object" as const,
+          properties: {
+            message: { type: "string", description: "Message to echo" },
+          },
+          required: ["message"],
+        },
+      },
+    ],
+  };
+});
+
+server.setRequestHandler(CallToolRequestSchema, async (request) => {
+  const { name, arguments: args } = request.params;
+
+  if (name === "echo") {
+    return {
+      content: [{ type: "text", text: (args as { message: string }).message }],
+    };
+  }
+
+  throw new Error(`Unknown tool: ${name}`);
+});
+
+// Create HTTP server with streamable transport
+async function main() {
+  const httpServer = http.createServer();
+  const transport = new StreamableHTTPServerTransport({ sessionIdGenerator: undefined });
+
+  httpServer.on("request", async (req, res) => {
+    // Simple CORS support
+    res.setHeader("Access-Control-Allow-Origin", "*");
+    res.setHeader("Access-Control-Allow-Methods", "GET, POST, OPTIONS");
+    res.setHeader("Access-Control-Allow-Headers", "Content-Type");
+
+    if (req.method === "OPTIONS") {
+      res.writeHead(200);
+      res.end();
+      return;
+    }
+
+    if (req.url === "/mcp" || req.url?.startsWith("/mcp?")) {
+      await transport.handleRequest(req, res);
+    } else {
+      res.writeHead(404);
+      res.end("Not found");
+    }
+  });
+
+  await server.connect(transport);
+
+  // Use port 0 to let the OS assign a free port, unless specific port given
+  const requestedPort = parseInt(process.argv[2] || "0", 10);
+  httpServer.listen(requestedPort, () => {
+    const addr = httpServer.address();
+    const actualPort = typeof addr === "object" && addr ? addr.port : requestedPort;
+    // Output format that tests parse: "listening on port XXXXX"
+    console.log(`Test HTTP MCP server listening on port ${actualPort}`);
+  });
+
+  // Handle shutdown
+  process.on("SIGTERM", () => {
+    httpServer.close();
+    process.exit(0);
+  });
+  process.on("SIGINT", () => {
+    httpServer.close();
+    process.exit(0);
+  });
+}
+
+main().catch(console.error);

jest.config.mjs

@@ -0,0 +1,158 @@
+#!/usr/bin/env npx tsx
+/**
+ * Minimal MCP Server for integration testing (stdio transport)
+ *
+ * This server exposes tools, prompts, and resources for testing the probe functionality.
+ */
+
+import { Server } from "@modelcontextprotocol/sdk/server/index.js";
+import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
+import {
+  CallToolRequestSchema,
+  GetPromptRequestSchema,
+  ListPromptsRequestSchema,
+  ListResourcesRequestSchema,
+  ListToolsRequestSchema,
+  ReadResourceRequestSchema,
+} from "@modelcontextprotocol/sdk/types.js";
+
+const server = new Server(
+  {
+    name: "test-stdio-server",
+    version: "1.0.0",
+  },
+  {
+    capabilities: {
+      tools: {},
+      prompts: {},
+      resources: {},
+    },
+  }
+);
+
+// Define tools
+server.setRequestHandler(ListToolsRequestSchema, async () => {
+  return {
+    tools: [
+      {
+        name: "greet",
+        description: "Greets a person by name",
+        inputSchema: {
+          type: "object" as const,
+          properties: {
+            name: { type: "string", description: "Name to greet" },
+          },
+          required: ["name"],
+        },
+      },
+      {
+        name: "add",
+        description: "Adds two numbers",
+        inputSchema: {
+          type: "object" as const,
+          properties: {
+            a: { type: "number", description: "First number" },
+            b: { type: "number", description: "Second number" },
+          },
+          required: ["a", "b"],
+        },
+      },
+    ],
+  };
+});
+
+server.setRequestHandler(CallToolRequestSchema, async (request) => {
+  const { name, arguments: args } = request.params;
+
+  if (name === "greet") {
+    return {
+      content: [{ type: "text", text: `Hello, ${(args as { name: string }).name}!` }],
+    };
+  }
+
+  if (name === "add") {
+    const { a, b } = args as { a: number; b: number };
+    // Return as embedded JSON to test normalization
+    return {
+      content: [
+        {
+          type: "text",
+          text: JSON.stringify({ result: a + b, operation: "add", inputs: { b, a } }),
+        },
+      ],
+    };
+  }
+
+  throw new Error(`Unknown tool: ${name}`);
+});
+
+// Define prompts
+server.setRequestHandler(ListPromptsRequestSchema, async () => {
+  return {
+    prompts: [
+      {
+        name: "code-review",
+        description: "Review code for issues",
+        arguments: [
+          { name: "code", description: "The code to review", required: true },
+          { name: "language", description: "Programming language", required: false },
+        ],
+      },
+    ],
+  };
+});
+
+server.setRequestHandler(GetPromptRequestSchema, async (request) => {
+  if (request.params.name === "code-review") {
+    const args = request.params.arguments || {};
+    return {
+      messages: [
+        {
+          role: "user" as const,
+          content: {
+            type: "text" as const,
+            text: `Please review this ${args.language || "code"}:\n\n${args.code}`,
+          },
+        },
+      ],
+    };
+  }
+  throw new Error(`Unknown prompt: ${request.params.name}`);
+});
+
+// Define resources
+server.setRequestHandler(ListResourcesRequestSchema, async () => {
+  return {
+    resources: [
+      {
+        uri: "test://readme",
+        name: "README",
+        description: "Project readme file",
+        mimeType: "text/plain",
+      },
+    ],
+  };
+});
+
+server.setRequestHandler(ReadResourceRequestSchema, async (request) => {
+  if (request.params.uri === "test://readme") {
+    return {
+      contents: [
+        {
+          uri: "test://readme",
+          mimeType: "text/plain",
+          text: "# Test Server\n\nThis is a test MCP server.",
+        },
+      ],
+    };
+  }
+  throw new Error(`Unknown resource: ${request.params.uri}`);
+});
+
+// Start the server
+async function main() {
+  const transport = new StdioServerTransport();
+  await server.connect(transport);
+}
+
+main().catch(console.error);