Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 45 additions & 0 deletions src/config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,36 @@ for (const key of BROWSERSTACK_LOCAL_OPTION_KEYS) {
}
}

/**
* Default o11y base URL for the `tfaRcaTurn` collaborative-RCA tool —
* PRODUCTION (verified: api-automation.browserstack.com routes the
* /ext/v1 testRca + rcaChat endpoints). Same value for every user served by
* the process; overridable at startup via `O11Y_TFA_RCA_BASE_URL` to target a
* staging tenant (e.g. api-observability-<tenant>.bsstag.com) where a build's
* representatives actually live. Per-process config, never a per-call arg.
*/
const DEFAULT_O11Y_TFA_RCA_BASE_URL = "https://api-automation.browserstack.com";

/**
* Base URL for the Automate test-runs API (`/ext/v1/builds/{id}/testRuns`) that
* `listTestIds` calls. Overridable at startup via `BROWSERSTACK_AUTOMATION_BASE_URL`
* so the tool can target a non-prod env (e.g. rengg-tfa staging) where a build
* actually lives, instead of the prod default. Per-process config, never a
* per-call arg.
*/
const DEFAULT_BROWSERSTACK_AUTOMATION_BASE_URL =
"https://api-automation.browserstack.com";

/**
* Base URL of the Test Observability (TRA) web UI used to build the
* human-facing "view the full report" links returned by the RCA tools.
* Same value for every user served by the process; overridable at startup via
* `BROWSERSTACK_O11Y_UI_BASE_URL` (e.g. to point at a staging UI). Per-process
* config, never a per-call arg.
*/
const DEFAULT_BROWSERSTACK_O11Y_UI_BASE_URL =
"https://automation.browserstack.com";

/**
* USE_OWN_LOCAL_BINARY_PROCESS:
* If true, the system will not start a new local binary process, but will use the user's own process.
Expand All @@ -41,6 +71,9 @@ export class Config {
public readonly USE_OWN_LOCAL_BINARY_PROCESS: boolean,
public readonly REMOTE_MCP: boolean,
public readonly UPLOAD_BASE_DIR: string | undefined,
public readonly O11Y_TFA_RCA_BASE_URL: string,
public readonly BROWSERSTACK_AUTOMATION_BASE_URL: string,
public readonly BROWSERSTACK_O11Y_UI_BASE_URL: string,
) {}
}

Expand All @@ -52,6 +85,18 @@ const config = new Config(
process.env.MCP_UPLOAD_BASE_DIR && process.env.MCP_UPLOAD_BASE_DIR.length > 0
? process.env.MCP_UPLOAD_BASE_DIR
: undefined,
process.env.O11Y_TFA_RCA_BASE_URL &&
process.env.O11Y_TFA_RCA_BASE_URL.length > 0
? process.env.O11Y_TFA_RCA_BASE_URL
: DEFAULT_O11Y_TFA_RCA_BASE_URL,
process.env.BROWSERSTACK_AUTOMATION_BASE_URL &&
process.env.BROWSERSTACK_AUTOMATION_BASE_URL.length > 0
? process.env.BROWSERSTACK_AUTOMATION_BASE_URL
: DEFAULT_BROWSERSTACK_AUTOMATION_BASE_URL,
process.env.BROWSERSTACK_O11Y_UI_BASE_URL &&
process.env.BROWSERSTACK_O11Y_UI_BASE_URL.length > 0
? process.env.BROWSERSTACK_O11Y_UI_BASE_URL
: DEFAULT_BROWSERSTACK_O11Y_UI_BASE_URL,
);

export default config;
2 changes: 2 additions & 0 deletions src/server-factory.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import addBuildInsightsTools from "./tools/build-insights.js";
import { setupOnInitialized } from "./oninitialized.js";
import { BrowserStackConfig } from "./lib/types.js";
import addRCATools from "./tools/rca-agent.js";
import addTfaRcaCollaborationTools from "./tools/tfa-rca-collaboration.js";

/**
* Wrapper class for BrowserStack MCP Server
Expand Down Expand Up @@ -61,6 +62,7 @@ export class BrowserStackMcpServer {
addSelfHealTools,
addBuildInsightsTools,
addRCATools,
addTfaRcaCollaborationTools,
];

toolAdders.forEach((adder) => {
Expand Down
16 changes: 16 additions & 0 deletions src/tools/rca-agent-utils/constants.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,18 @@
import { z } from "zod";
import appConfig from "../../config.js";
import { TestStatus } from "./types.js";

/**
* Base URL for the Automate test-runs API used by `listTestIds`. Process-startup
* config resolved in `src/config.ts` from `BROWSERSTACK_AUTOMATION_BASE_URL`
* (default prod). Set it to a rengg/staging host to list a build that lives
* there. Read per call so a per-server-instance config is honored; never read
* `process.env` here.
*/
export function getAutomationBaseUrl(): string {
return appConfig.BROWSERSTACK_AUTOMATION_BASE_URL;
}

export const FETCH_RCA_PARAMS = {
testId: z
.array(z.number().int())
Expand Down Expand Up @@ -34,4 +46,8 @@ export const LIST_TEST_IDS_PARAMS = {
.describe(
"Filter tests by status. If not provided, all tests are returned. Example for RCA usecase always use failed status",
),
includeFailureDetail: z
.boolean()
.optional()
.describe("Add per-test failure signature for clustering. Default false."),
};
91 changes: 85 additions & 6 deletions src/tools/rca-agent-utils/get-failed-test-id.ts
Original file line number Diff line number Diff line change
@@ -1,12 +1,24 @@
import logger from "../../logger.js";
import { TestStatus, FailedTestInfo, TestRun, TestDetails } from "./types.js";
import { getAutomationBaseUrl } from "./constants.js";
import {
TestStatus,
FailedTestInfo,
TestRun,
TestDetails,
TestFailureSignature,
} from "./types.js";

// Cap on the failure summary line — keep the response payload lean (we never
// return full stack traces into the MCP client's context window).
const ERROR_SUMMARY_MAX = 200;

export async function getTestIds(
buildId: string,
authString: string,
status?: TestStatus,
includeFailureDetail = false,
): Promise<FailedTestInfo[]> {
const baseUrl = `https://api-automation.browserstack.com/ext/v1/builds/${buildId}/testRuns`;
const baseUrl = `${getAutomationBaseUrl()}/ext/v1/builds/${buildId}/testRuns`;
let url = status ? `${baseUrl}?test_statuses=${status}` : baseUrl;
let allFailedTests: FailedTestInfo[] = [];
let requestNumber = 0;
Expand Down Expand Up @@ -36,7 +48,11 @@ export async function getTestIds(

// Extract failed IDs from current page
if (data.hierarchy && data.hierarchy.length > 0) {
const currentFailedTests = extractFailedTestIds(data.hierarchy, status);
const currentFailedTests = extractFailedTestIds(
data.hierarchy,
status,
includeFailureDetail,
);
allFailedTests = allFailedTests.concat(currentFailedTests);
}

Expand Down Expand Up @@ -68,28 +84,91 @@ export async function getTestIds(
export function extractFailedTestIds(
hierarchy: TestDetails[],
status?: TestStatus,
includeFailureDetail = false,
): FailedTestInfo[] {
let failedTests: FailedTestInfo[] = [];

for (const node of hierarchy) {
// Match on status alone — the observability_url `details=<id>` check below
// already filters to real test nodes (suite/hook nodes carry no status and
// no such URL). Do NOT also require run_count: JUnit-uploaded builds report
// run_count=0 even for genuinely failed tests, which would drop them all.
if (node.details?.status === status) {
if (node.details?.observability_url) {
const idMatch = node.details.observability_url.match(/details=(\d+)/);
if (idMatch) {
failedTests.push({
const entry: FailedTestInfo = {
test_id: idMatch[1],
test_name: node.display_name || `Test ${idMatch[1]}`,
});
};
if (includeFailureDetail) {
const signature = buildFailureSignature(node.details);
if (signature) entry.failure = signature;
}
failedTests.push(entry);
}
}
}

if (node.children && node.children.length > 0) {
failedTests = failedTests.concat(
extractFailedTestIds(node.children, status),
extractFailedTestIds(node.children, status, includeFailureDetail),
);
}
}

return failedTests;
}

// Build a trimmed failure signature from a test node's `details`. Returns
// undefined when no signal is available so the field is simply omitted.
function buildFailureSignature(details: any): TestFailureSignature | undefined {
if (!details) return undefined;

const signature: TestFailureSignature = {};

if (details.failure_categories != null) {
signature.category = Array.isArray(details.failure_categories)
? details.failure_categories.filter(Boolean).join(", ")
: String(details.failure_categories);
}

const errorSummary = extractFirstFailureLine(details);
if (errorSummary) signature.error_summary = errorSummary;

if (details.file_path) signature.file_path = String(details.file_path);
if (typeof details.is_flaky === "boolean")
signature.is_flaky = details.is_flaky;
if (typeof details.is_always_failing === "boolean")
signature.is_always_failing = details.is_always_failing;
if (typeof details.is_new_failure === "boolean")
signature.is_new_failure = details.is_new_failure;

return Object.keys(signature).length > 0 ? signature : undefined;
}

// First non-empty line of the first retry's TEST_FAILURE log, capped. Handles
// both string entries and object entries ({ message } / { text }).
function extractFirstFailureLine(details: any): string | undefined {
const retries = details?.retries;
if (!Array.isArray(retries)) return undefined;

for (const retry of retries) {
const failures = retry?.logs?.TEST_FAILURE;
if (!failures) continue;
const entries = Array.isArray(failures) ? failures : [failures];
for (const failure of entries) {
const text =
typeof failure === "string"
? failure
: (failure?.message ?? failure?.text ?? "");
const firstLine = String(text)
.split("\n")
.map((line) => line.trim())
.find((line) => line.length > 0);
if (firstLine) return firstLine.slice(0, ERROR_SUMMARY_MAX);
}
}

return undefined;
}
13 changes: 13 additions & 0 deletions src/tools/rca-agent-utils/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,22 @@ export interface TestRun {
};
}

// Trimmed per-test failure signature for downstream clustering.
// Never carries full stack traces — error_summary is a single capped line.
export interface TestFailureSignature {
category?: string;
error_summary?: string;
file_path?: string;
is_flaky?: boolean;
is_always_failing?: boolean;
is_new_failure?: boolean;
}

export interface FailedTestInfo {
test_id: number;
test_name: string;
// Present only when listTestIds is called with includeFailureDetail=true.
failure?: TestFailureSignature;
}

export enum RCAState {
Expand Down
10 changes: 8 additions & 2 deletions src/tools/rca-agent.ts
Original file line number Diff line number Diff line change
Expand Up @@ -145,15 +145,21 @@ export async function listTestIdsTool(
args: {
buildId: string;
status?: TestStatus;
includeFailureDetail?: boolean;
},
config: BrowserStackConfig,
): Promise<CallToolResult> {
try {
const { buildId, status } = args;
const { buildId, status, includeFailureDetail } = args;
const authString = getBrowserStackAuth(config);

// Get test IDs
const testIds = await getTestIds(buildId, authString, status);
const testIds = await getTestIds(
buildId,
authString,
status,
includeFailureDetail,
);

return {
content: [
Expand Down
Loading