Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions .github/actions/conformance/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,9 @@
Contract:
- MCP_CONFORMANCE_SCENARIO env var -> scenario name
- MCP_CONFORMANCE_CONTEXT env var -> optional JSON (for client-credentials scenarios)
- MCP_CONFORMANCE_PROTOCOL_VERSION env var -> spec version the harness mock
server is speaking (e.g. "2025-11-25", "2026-07-28"). Always set; defaults
to the harness's LATEST_SPEC_VERSION when --spec-version is omitted.
- Server URL as last CLI argument (sys.argv[1])
- Must exit 0 within 30 seconds

Expand Down Expand Up @@ -50,6 +53,13 @@
)
logger = logging.getLogger(__name__)

#: Spec version the harness is running this scenario at (e.g. "2025-11-25",
#: "2026-07-28"). The harness always sets this (it falls back to its own
#: LATEST_SPEC_VERSION when --spec-version is omitted), so None means we were
#: invoked outside the harness. Handlers that need to take the stateless 2026
#: path will branch on this once the SDK has one; today it is logged only.
PROTOCOL_VERSION: str | None = os.environ.get("MCP_CONFORMANCE_PROTOCOL_VERSION")

# Type for async scenario handler functions
ScenarioHandler = Callable[[str], Coroutine[Any, None, None]]

Expand Down Expand Up @@ -347,6 +357,7 @@ def main() -> None:

server_url = sys.argv[1]
scenario = os.environ.get("MCP_CONFORMANCE_SCENARIO")
logger.debug(f"Conformance protocol version: {PROTOCOL_VERSION!r}")

if scenario:
logger.debug(f"Running explicit scenario '{scenario}' against {server_url}")
Expand Down
149 changes: 149 additions & 0 deletions .github/actions/conformance/expected-failures.2026-07-28.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,149 @@
# Expected failures for the carried-forward 2026-07-28 legs
# (`--suite all --spec-version 2026-07-28` for both server and client).
#
# This baseline is separate from expected-failures.yml because entries are
# keyed by scenario name only: a scenario that passes at its default version
# in the 2025 legs but fails when forced to 2026-07-28 (or vice versa) cannot
# be expressed in a shared file (the passing leg would flag the entry as
# stale). Like expected-failures.yml, this single file covers both
# directions: the client 2026 leg reads the `client:` section and the server
# 2026 leg reads the `server:` section. Both burn down independently of the
# 2025 legs.
#
# Baseline established against @modelcontextprotocol/conformance pinned in
# .github/workflows/conformance.yml (CONFORMANCE_VERSION = 0.2.0-alpha.4).
# New conformance releases are adopted by deliberately bumping that pin and
# reconciling both this file and expected-failures.yml in the same change.
#
# Entries are grouped by what unblocks them. As each gap closes the
# corresponding scenarios start passing and MUST be removed from this list
# (the runner fails on stale entries), so the baseline burns down per
# milestone.

client:
# --- No stateless client path on main yet ---
# client.py drives the 2025 stateful lifecycle (initialize handshake +
# session). The 2026-mode mock server is stateless, so the call sequence
# never reaches the assertion. Unblocks when client.py's is_modern_protocol()
# branch takes the per-request _meta path.
- tools_call

# --- SEP-837 (application_type during DCR) ---
# The sep-837-application-type-present check only fires on 2026-version
# runs; the client omits application_type during Dynamic Client
# Registration, so every auth scenario that reaches DCR fails it on this
# leg (the same scenarios pass at their default version in the 2025 legs).
- auth/metadata-default
- auth/metadata-var1
- auth/metadata-var2
- auth/metadata-var3
- auth/scope-from-www-authenticate
- auth/scope-from-scopes-supported
- auth/scope-omitted-when-undefined
- auth/token-endpoint-auth-basic
- auth/token-endpoint-auth-post
- auth/token-endpoint-auth-none
- auth/offline-access-not-supported

# --- Auth scenarios cut short by the 2026 connection lifecycle ---
# The auth fixture flow drives the 2025 stateful lifecycle; the 2026-mode
# mock rejects the MCP POST before the scope-escalation behaviour these
# scenarios measure, so no authorization requests are observed. Unblocks
# when client.py's auth flow speaks the 2026 per-request lifecycle.
- auth/scope-step-up
- auth/scope-retry-limit

# --- Same gaps as the 2025 baseline (fail identically when forced to 2026-07-28) ---
# SEP-2575 (request metadata / _meta envelope): client does not populate the
# _meta envelope or the MCP-Protocol-Version header semantics yet.
- request-metadata
# SEP-2322 (multi-round-trip requests): client does not echo requestState /
# handle IncompleteResult yet.
- sep-2322-client-request-state
# SEP-2243 (HTTP standardization): no fixture handler / client header support yet.
- http-custom-headers
- http-invalid-tool-headers
# SEP-2106 (JSON Schema $ref handling): client still dereferences network $refs.
- json-schema-ref-no-deref
# SEP-2468 (authorization response iss parameter): not implemented in the client.
- auth/iss-supported
- auth/iss-not-advertised
- auth/iss-supported-missing
- auth/iss-wrong-issuer
- auth/iss-unexpected
- auth/iss-normalized
- auth/metadata-issuer-mismatch
# SEP-2352 (authorization server migration): client does not re-register when
# PRM authorization_servers changes.
- auth/authorization-server-migration
# auth/enterprise-managed-authorization (SEP-990) is in the 2025 baseline but
# NOT here: the harness skips it as inapplicable at --spec-version 2026-07-28
# (it is an extension scenario not carried into the 2026 wire), so it is
# neither run nor evaluated on this leg.

server:
# --- No stateless server path on main yet (carried-forward 2025-era scenarios) ---
# mcp-everything-server only runs in 2025 stateful mode. With
# --spec-version 2026-07-28 the harness sends stateless requests
# (MCP-Protocol-Version: 2026-07-28, _meta envelope, no initialize), which
# the server rejects before the handler runs. These scenarios all pass on
# the 2025 legs; they unblock once mcp-everything-server routes 2026
# requests through a stateless path.
- completion-complete
- tools-list
- tools-call-simple-text
- tools-call-image
- tools-call-audio
- tools-call-embedded-resource
- tools-call-mixed-content
- tools-call-error
- tools-call-with-progress
- server-sse-multiple-streams
- resources-list
- resources-read-text
- resources-read-binary
- resources-templates-read
- prompts-list
- prompts-get-simple
- prompts-get-with-args
- prompts-get-embedded-resource
- prompts-get-with-image
- dns-rebinding-protection
# SEP-2106 (JSON Schema 2020-12 in tool inputSchema): the fixture tool's
# schema has none of the 2020-12 keywords the scenario checks. The scenario
# is in `--suite all` but not `--suite active`, so this is the only leg that
# runs it; it fails identically at 2025-11-25 (not a 2026-path regression).
- json-schema-2020-12

# --- Draft scenarios (same failures and reasons as the `--suite draft` leg) ---
# SEP-2575 (stateless HTTP / _meta envelope): server has no stateless mode,
# _meta-derived capabilities, error-code mappings, or server/discover yet.
- server-stateless
# SEP-2322 (multi-round-trip requests / IncompleteResult): not implemented.
- input-required-result-basic-elicitation
- input-required-result-basic-sampling
- input-required-result-basic-list-roots
- input-required-result-request-state
- input-required-result-multiple-input-requests
- input-required-result-multi-round
- input-required-result-non-tool-request
- input-required-result-result-type
- input-required-result-tampered-state
- input-required-result-capability-check
# SEP-2549 (caching): no ttlMs/cacheScope support.
- caching
# SEP-2243 (HTTP header standardization): -32001 HeaderMismatch handling and
# case-insensitive/whitespace-trimmed header validation not implemented.
- http-header-validation
- http-custom-header-server-validation

# --- WARNING-only entries ---
# These scenarios emit no FAILURE checks, only SHOULD-level WARNINGs, but
# the expected-failures evaluator counts WARNINGs as failures. Same entries
# as the draft suite in expected-failures.yml.
# SEP-2164: server returns -32600 (not -32602) and omits error.data.uri.
- sep-2164-resource-not-found
# SEP-2322 SHOULD-level behaviours (re-request missing inputResponses,
# ignore unrecognized inputResponses keys).
- input-required-result-missing-input-response
- input-required-result-ignore-extra-params
5 changes: 3 additions & 2 deletions .github/actions/conformance/expected-failures.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,10 @@
# CI exits 0 if only these fail, exits 1 on unexpected failures or stale entries.
#
# Baseline established against @modelcontextprotocol/conformance pinned in
# .github/workflows/conformance.yml (CONFORMANCE_VERSION = 0.2.0-alpha.3).
# .github/workflows/conformance.yml (CONFORMANCE_VERSION = 0.2.0-alpha.4).
# New conformance releases are adopted by deliberately bumping that pin and
# reconciling this file in the same change.
# reconciling both this file and expected-failures.2026-07-28.yml in the same
# change.
#
# Entries are grouped by SEP. As each SEP lands in the SDK the corresponding
# scenarios start passing and MUST be removed from this list (the runner fails
Expand Down
18 changes: 16 additions & 2 deletions .github/workflows/conformance.yml
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,9 @@ permissions:

env:
# Pinned conformance harness version. Bump deliberately and reconcile
# .github/actions/conformance/expected-failures.yml in the same change.
CONFORMANCE_VERSION: "0.2.0-alpha.3"
# both .github/actions/conformance/expected-failures*.yml files in the
# same change.
CONFORMANCE_VERSION: "0.2.0-alpha.4"

jobs:
server-conformance:
Expand All @@ -43,6 +44,12 @@ jobs:
./.github/actions/conformance/run-server.sh
--suite draft
--expected-failures ./.github/actions/conformance/expected-failures.yml
- name: Run server conformance (2026-07-28 wire, all suite)
run: >-
./.github/actions/conformance/run-server.sh
--suite all
--spec-version 2026-07-28
--expected-failures ./.github/actions/conformance/expected-failures.2026-07-28.yml

client-conformance:
runs-on: ubuntu-latest
Expand All @@ -64,3 +71,10 @@ jobs:
--command 'uv run --frozen python .github/actions/conformance/client.py'
--suite all
--expected-failures ./.github/actions/conformance/expected-failures.yml
- name: Run client conformance (2026-07-28 wire, all suite)
run: >-
npx --yes @modelcontextprotocol/conformance@"$CONFORMANCE_VERSION" client
--command 'uv run --frozen python .github/actions/conformance/client.py'
--suite all
--spec-version 2026-07-28
--expected-failures ./.github/actions/conformance/expected-failures.2026-07-28.yml
Loading