feat: add pre/post test command hooks for dynamic configuration

Adds three new optional fields to TestConfiguration for lifecycle hooks:
- pre_test_command: runs before starting the MCP server
- pre_test_wait_ms: wait time after pre_test_command before probing
- post_test_command: cleanup command after stopping the server

This enables testing servers with dynamic dependencies like mock services.

Also removes hardcoded version string from startup output.

Closes #11

Author: SamMorrowDrums

dist/index.js

@@ -10,6 +10,12 @@ export interface TestConfiguration {
     headers?: Record<string, string>;
     env_vars?: string;
     custom_messages?: CustomMessage[];
+    /** Command to run before starting the MCP server for this config */
+    pre_test_command?: string;
+    /** Milliseconds to wait after pre_test_command before starting the server */
+    pre_test_wait_ms?: number;
+    /** Command to run after stopping the MCP server for this config (cleanup) */
+    post_test_command?: string;
 }
 export interface CustomMessage {
     id: number;

dist/index.js.map

@@ -198,7 +198,7 @@ async function runInitialBuild(inputs: ActionInputs): Promise<void> {
  */
 async function run(): Promise<void> {
   try {
-    core.info("🚀 MCP Conformance Action v2.0.0");
+    core.info("🚀 MCP Conformance Action");
     core.info("");
 
     // Get inputs

dist/types.d.ts

@@ -171,6 +171,43 @@ async function runBuild(dir: string, inputs: ActionInputs): Promise<void> {
   }
 }
 
+/**
+ * Sleep for a specified number of milliseconds
+ */
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+/**
+ * Run pre-test command if specified
+ */
+async function runPreTestCommand(config: TestConfiguration, workDir: string): Promise<void> {
+  if (config.pre_test_command) {
+    core.info(`  Running pre-test command: ${config.pre_test_command}`);
+    await exec.exec("sh", ["-c", config.pre_test_command], { cwd: workDir });
+
+    if (config.pre_test_wait_ms && config.pre_test_wait_ms > 0) {
+      core.info(`  Waiting ${config.pre_test_wait_ms}ms for service to be ready...`);
+      await sleep(config.pre_test_wait_ms);
+    }
+  }
+}
+
+/**
+ * Run post-test command if specified (cleanup)
+ */
+async function runPostTestCommand(config: TestConfiguration, workDir: string): Promise<void> {
+  if (config.post_test_command) {
+    core.info(`  Running post-test command: ${config.post_test_command}`);
+    try {
+      await exec.exec("sh", ["-c", config.post_test_command], { cwd: workDir });
+    } catch (error) {
+      // Log but don't fail - cleanup errors shouldn't break the test
+      core.warning(`  Post-test command failed: ${error}`);
+    }
+  }
+}
+
 /**
  * Probe a server with a specific configuration
  */
@@ -186,6 +223,9 @@ async function probeWithConfig(
   const headers = { ...globalHeaders, ...config.headers };
   const customMessages = config.custom_messages || globalCustomMessages;
 
+  // Run pre-test command before probing
+  await runPreTestCommand(config, workDir);
+
   if (config.transport === "stdio") {
     const command = config.start_command || "";
     const parts = command.split(/\s+/);
@@ -304,13 +344,19 @@ export async function runSingleConfigTest(
   // Test current branch
   core.info("🔄 Testing current branch...");
   const branchStart = Date.now();
-  const branchResult = await probeWithConfig(
-    config,
-    ctx.workDir,
-    globalEnvVars,
-    globalHeaders,
-    globalCustomMessages
-  );
+  let branchResult: ProbeResult;
+  try {
+    branchResult = await probeWithConfig(
+      config,
+      ctx.workDir,
+      globalEnvVars,
+      globalHeaders,
+      globalCustomMessages
+    );
+  } finally {
+    // Always run post-test cleanup
+    await runPostTestCommand(config, ctx.workDir);
+  }
   result.branchTime = Date.now() - branchStart;
 
   if (branchResult.error) {
@@ -346,13 +392,19 @@ export async function runSingleConfigTest(
     // Probe on base
     core.info("🔄 Testing comparison ref...");
     const baseStart = Date.now();
-    const baseResult = await probeWithConfig(
-      config,
-      baseWorkDir,
-      globalEnvVars,
-      globalHeaders,
-      globalCustomMessages
-    );
+    let baseResult: ProbeResult;
+    try {
+      baseResult = await probeWithConfig(
+        config,
+        baseWorkDir,
+        globalEnvVars,
+        globalHeaders,
+        globalCustomMessages
+      );
+    } finally {
+      // Always run post-test cleanup
+      await runPostTestCommand(config, baseWorkDir);
+    }
     result.baseTime = Date.now() - baseStart;
 
     if (baseResult.error) {

src/index.ts

@@ -11,6 +11,12 @@ export interface TestConfiguration {
   headers?: Record<string, string>;
   env_vars?: string;
   custom_messages?: CustomMessage[];
+  /** Command to run before starting the MCP server for this config */
+  pre_test_command?: string;
+  /** Milliseconds to wait after pre_test_command before starting the server */
+  pre_test_wait_ms?: number;
+  /** Command to run after stopping the MCP server for this config (cleanup) */
+  post_test_command?: string;
 }
 
 export interface CustomMessage {