feat: add automatic HTTP server lifecycle management

- Add start_command option for HTTP transport to spawn server automatically
- Add startup_wait_ms option to control server startup delay (default: 2000ms)
- Server process is automatically killed after probing completes
- Update README with HTTP server lifecycle documentation
- Update action.yml with new configuration options

Author: SamMorrowDrums

README.md

@@ -209,9 +209,15 @@ When using `configurations`, each object supports:
 |-------|-------------|----------|
 | `name` | Identifier for this configuration (appears in report) | Yes |
 | `transport` | `stdio` or `streamable-http` | No (default: `stdio`) |
-| `start_command` | Server start command | Yes (unless using external server) |
-| `server_url` | URL for HTTP transport | Required if transport is `streamable-http` |
+| `start_command` | Server start command (stdio: spawns process, HTTP: starts server in background) | Yes for stdio, optional for HTTP |
+| `server_url` | URL for HTTP transport | Required for `streamable-http` |
+| `startup_wait_ms` | Milliseconds to wait for HTTP server to start (when using `start_command`) | No (default: 2000) |
+| `pre_test_command` | Command to run before probing (alternative to `start_command` for HTTP) | No |
+| `pre_test_wait_ms` | Milliseconds to wait after `pre_test_command` | No |
+| `post_test_command` | Command to run after probing (cleanup, used with `pre_test_command`) | No |
+| `headers` | HTTP headers for this configuration | No |
 | `env_vars` | Additional environment variables | No |
+| `custom_messages` | Config-specific custom messages | No |
 
 ## How It Works
 
@@ -265,26 +271,45 @@ The default transport communicates with your server via stdin/stdout using JSON-
 
 ### Streamable HTTP Transport
 
-For servers exposing an HTTP endpoint:
+For servers exposing an HTTP endpoint, the action can automatically manage the server lifecycle. Use `start_command` and the action will spawn your server, wait for it to start, probe it, then shut it down:
 
 ```yaml
 - uses: SamMorrowDrums/mcp-conformance-action@v1
   with:
     setup_node: true
     install_command: npm ci
     build_command: npm run build
-    start_command: node dist/http.js
-    transport: streamable-http
-    server_url: http://localhost:3000/mcp
+    configurations: |
+      [{
+        "name": "http-server",
+        "transport": "streamable-http",
+        "start_command": "node dist/http.js",
+        "server_url": "http://localhost:3000/mcp",
+        "startup_wait_ms": 2000
+      }]
 ```
 
 The action will:
 1. Start the server using `start_command`
-2. Poll the endpoint until it responds (up to `server_timeout` seconds)
+2. Wait `startup_wait_ms` (default: 2000ms) for the server to initialize
 3. Send MCP requests via HTTP POST
 4. Terminate the server after tests complete
 
-For pre-deployed servers, omit `start_command`:
+For more control, you can use `pre_test_command` and `post_test_command` to manage server lifecycle yourself:
+
+```yaml
+configurations: |
+  [{
+    "name": "http-server",
+    "transport": "streamable-http",
+    "server_url": "http://localhost:3000/mcp",
+    "pre_test_command": "node dist/http.js &",
+    "pre_test_wait_ms": 2000,
+    "post_test_command": "pkill -f 'node dist/http.js' || true"
+  }]
+```
+
+For pre-deployed servers, omit both `start_command` and `pre_test_command`:
 
 ```yaml
 - uses: SamMorrowDrums/mcp-conformance-action@v1

action.yml

@@ -72,7 +72,18 @@ inputs:
     required: false
     default: ''
   configurations:
-    description: 'JSON array of test configurations for multiple transports. Each object: name, transport, start_command, server_url, env_vars, custom_messages'
+    description: |
+      JSON array of test configurations for multiple transports/scenarios. 
+      Each object supports: name, transport, start_command, args, server_url, headers, env_vars, custom_messages.
+      
+      For HTTP transport, use start_command to have the action manage server lifecycle:
+        - start_command: Command to start the server (action spawns, probes, then kills it)
+        - startup_wait_ms: Milliseconds to wait for server startup (default: 2000)
+      
+      Or use pre/post commands for manual control:
+        - pre_test_command: Command to run before probing (e.g., start server in background)
+        - pre_test_wait_ms: Milliseconds to wait after pre_test_command
+        - post_test_command: Command to run after probing (e.g., kill server process)
     required: false
     default: ''
   custom_messages:

dist/index.js

@@ -35477,6 +35477,8 @@ async function checkoutPrevious() {
 var external_path_ = __nccwpck_require__(6928);
 // EXTERNAL MODULE: external "fs"
 var external_fs_ = __nccwpck_require__(9896);
+// EXTERNAL MODULE: external "child_process"
+var external_child_process_ = __nccwpck_require__(5317);
 ;// CONCATENATED MODULE: ./node_modules/zod/v4/core/core.js
 /** A special constant with type `never` */
 const NEVER = Object.freeze({
@@ -53107,6 +53109,7 @@ function probeResultToFiles(result) {
 
 
 
+
 /**
  * Parse configurations from input
  */
@@ -53245,6 +53248,68 @@ async function runBuild(dir, inputs) {
 function sleep(ms) {
     return new Promise((resolve) => setTimeout(resolve, ms));
 }
+/**
+ * Start an HTTP server process for the given configuration.
+ * Returns the spawned process which should be killed after probing.
+ */
+async function startHttpServer(config, workDir, envVars) {
+    if (!config.start_command) {
+        return null;
+    }
+    lib_core.info(`  Starting HTTP server: ${config.start_command}`);
+    // Merge environment variables
+    const env = {};
+    for (const [key, value] of Object.entries(process.env)) {
+        if (value !== undefined) {
+            env[key] = value;
+        }
+    }
+    for (const [key, value] of Object.entries(envVars)) {
+        env[key] = value;
+    }
+    const serverProcess = (0,external_child_process_.spawn)("sh", ["-c", config.start_command], {
+        cwd: workDir,
+        env,
+        detached: true,
+        stdio: ["ignore", "pipe", "pipe"],
+    });
+    // Log server output for debugging
+    serverProcess.stdout?.on("data", (data) => {
+        lib_core.debug(`  [server stdout]: ${data.toString().trim()}`);
+    });
+    serverProcess.stderr?.on("data", (data) => {
+        lib_core.debug(`  [server stderr]: ${data.toString().trim()}`);
+    });
+    // Wait for server to start up
+    const waitMs = config.startup_wait_ms ?? config.pre_test_wait_ms ?? 2000;
+    lib_core.info(`  Waiting ${waitMs}ms for server to start...`);
+    await sleep(waitMs);
+    // Check if process is still running
+    if (serverProcess.exitCode !== null) {
+        throw new Error(`HTTP server exited prematurely with code ${serverProcess.exitCode}`);
+    }
+    lib_core.info("  HTTP server started");
+    return serverProcess;
+}
+/**
+ * Stop an HTTP server process
+ */
+function stopHttpServer(serverProcess) {
+    if (!serverProcess) {
+        return;
+    }
+    lib_core.info("  Stopping HTTP server...");
+    try {
+        // Kill the process group (negative PID kills the group)
+        if (serverProcess.pid) {
+            process.kill(-serverProcess.pid, "SIGTERM");
+        }
+    }
+    catch (error) {
+        // Process might already be dead
+        lib_core.debug(`  Error stopping server: ${error}`);
+    }
+}
 /**
  * Run pre-test command if specified
  */
@@ -53302,13 +53367,24 @@ async function probeWithConfig(config, workDir, globalEnvVars, globalHeaders, gl
         });
     }
     else {
-        return await probeServer({
-            transport: "streamable-http",
-            url: config.server_url,
-            headers,
-            envVars,
-            customMessages,
-        });
+        // For HTTP transport, optionally start the server if start_command is provided
+        let serverProcess = null;
+        try {
+            if (config.start_command) {
+                serverProcess = await startHttpServer(config, workDir, envVars);
+            }
+            return await probeServer({
+                transport: "streamable-http",
+                url: config.server_url,
+                headers,
+                envVars,
+                customMessages,
+            });
+        }
+        finally {
+            // Always stop the server if we started it
+            stopHttpServer(serverProcess);
+        }
     }
 }
 /**

dist/types.d.ts

@@ -14,6 +14,8 @@ export interface TestConfiguration {
     pre_test_command?: string;
     /** Milliseconds to wait after pre_test_command before starting the server */
     pre_test_wait_ms?: number;
+    /** Milliseconds to wait for HTTP server to start (when using start_command with HTTP transport) */
+    startup_wait_ms?: number;
     /** Command to run after stopping the MCP server for this config (cleanup) */
     post_test_command?: string;
 }

src/runner.ts

@@ -6,6 +6,7 @@ import * as exec from "@actions/exec";
 import * as core from "@actions/core";
 import * as path from "path";
 import * as fs from "fs";
+import { spawn, ChildProcess } from "child_process";
 import { probeServer, probeResultToFiles } from "./probe.js";
 import { createWorktree, removeWorktree, checkout, checkoutPrevious } from "./git.js";
 import type {
@@ -178,6 +179,81 @@ function sleep(ms: number): Promise<void> {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }
 
+/**
+ * Start an HTTP server process for the given configuration.
+ * Returns the spawned process which should be killed after probing.
+ */
+async function startHttpServer(
+  config: TestConfiguration,
+  workDir: string,
+  envVars: Record<string, string>
+): Promise<ChildProcess | null> {
+  if (!config.start_command) {
+    return null;
+  }
+
+  core.info(`  Starting HTTP server: ${config.start_command}`);
+
+  // Merge environment variables
+  const env: Record<string, string> = {};
+  for (const [key, value] of Object.entries(process.env)) {
+    if (value !== undefined) {
+      env[key] = value;
+    }
+  }
+  for (const [key, value] of Object.entries(envVars)) {
+    env[key] = value;
+  }
+
+  const serverProcess = spawn("sh", ["-c", config.start_command], {
+    cwd: workDir,
+    env,
+    detached: true,
+    stdio: ["ignore", "pipe", "pipe"],
+  });
+
+  // Log server output for debugging
+  serverProcess.stdout?.on("data", (data) => {
+    core.debug(`  [server stdout]: ${data.toString().trim()}`);
+  });
+  serverProcess.stderr?.on("data", (data) => {
+    core.debug(`  [server stderr]: ${data.toString().trim()}`);
+  });
+
+  // Wait for server to start up
+  const waitMs = config.startup_wait_ms ?? config.pre_test_wait_ms ?? 2000;
+  core.info(`  Waiting ${waitMs}ms for server to start...`);
+  await sleep(waitMs);
+
+  // Check if process is still running
+  if (serverProcess.exitCode !== null) {
+    throw new Error(`HTTP server exited prematurely with code ${serverProcess.exitCode}`);
+  }
+
+  core.info("  HTTP server started");
+  return serverProcess;
+}
+
+/**
+ * Stop an HTTP server process
+ */
+function stopHttpServer(serverProcess: ChildProcess | null): void {
+  if (!serverProcess) {
+    return;
+  }
+
+  core.info("  Stopping HTTP server...");
+  try {
+    // Kill the process group (negative PID kills the group)
+    if (serverProcess.pid) {
+      process.kill(-serverProcess.pid, "SIGTERM");
+    }
+  } catch (error) {
+    // Process might already be dead
+    core.debug(`  Error stopping server: ${error}`);
+  }
+}
+
 /**
  * Run pre-test command if specified
  */
@@ -246,13 +322,24 @@ async function probeWithConfig(
       customMessages,
     });
   } else {
-    return await probeServer({
-      transport: "streamable-http",
-      url: config.server_url,
-      headers,
-      envVars,
-      customMessages,
-    });
+    // For HTTP transport, optionally start the server if start_command is provided
+    let serverProcess: ChildProcess | null = null;
+    try {
+      if (config.start_command) {
+        serverProcess = await startHttpServer(config, workDir, envVars);
+      }
+
+      return await probeServer({
+        transport: "streamable-http",
+        url: config.server_url,
+        headers,
+        envVars,
+        customMessages,
+      });
+    } finally {
+      // Always stop the server if we started it
+      stopHttpServer(serverProcess);
+    }
   }
 }
 

src/types.ts

@@ -15,6 +15,8 @@ export interface TestConfiguration {
   pre_test_command?: string;
   /** Milliseconds to wait after pre_test_command before starting the server */
   pre_test_wait_ms?: number;
+  /** Milliseconds to wait for HTTP server to start (when using start_command with HTTP transport) */
+  startup_wait_ms?: number;
   /** Command to run after stopping the MCP server for this config (cleanup) */
   post_test_command?: string;
 }