From 38aa78a836ea2109be87d1eeff7f75eb1124a818 Mon Sep 17 00:00:00 2001 From: Jack Amadeo Date: Fri, 17 Jul 2026 22:26:27 -0400 Subject: [PATCH] chore: run full conformance CI suite the expected failures may need to be updated based on when this merges --- .github/workflows/conformance.yml | 53 +++++-------------- conformance/expected-failures-2026-07-28.yaml | 43 +++++++++++++++ conformance/src/bin/server.rs | 4 +- 3 files changed, 56 insertions(+), 44 deletions(-) create mode 100644 conformance/expected-failures-2026-07-28.yaml diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index 817c7d4ed..535f7ef59 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -81,7 +81,6 @@ jobs: echo "draft conformance server did not become ready" >&2 exit 1 - # Run discovery separately until #985 enables the full draft suite. - name: Run SEP-2575 discovery contract run: | endpoint=http://127.0.0.1:8002/mcp @@ -152,35 +151,14 @@ jobs: (.error.data.supported | index("2026-07-28") != null) ' /tmp/unsupported-version.json - # Keep this explicit list until the full draft suite is enabled by #985. - - name: Run supported draft server scenarios + - name: Run 2026-07-28 server suite run: | - for scenario in \ - sep-2164-resource-not-found \ - caching \ - http-header-validation \ - http-custom-header-server-validation \ - input-required-result-basic-elicitation \ - input-required-result-basic-sampling \ - input-required-result-basic-list-roots \ - input-required-result-request-state \ - input-required-result-multiple-input-requests \ - input-required-result-multi-round \ - input-required-result-missing-input-response \ - input-required-result-non-tool-request \ - input-required-result-result-type \ - input-required-result-unsupported-methods \ - input-required-result-tampered-state \ - input-required-result-capability-check \ - input-required-result-ignore-extra-params \ - input-required-result-validate-input \ - ; do - npx -y "@modelcontextprotocol/conformance@${DRAFT_CONFORMANCE_VERSION}" server \ - --url http://127.0.0.1:8002/mcp \ - --scenario "$scenario" \ - --spec-version draft \ - -o conformance-results - done + npx -y "@modelcontextprotocol/conformance@${DRAFT_CONFORMANCE_VERSION}" server \ + --url http://127.0.0.1:8002/mcp \ + --suite all \ + --spec-version 2026-07-28 \ + --expected-failures conformance/expected-failures-2026-07-28.yaml \ + -o conformance-results - name: Stop conformance servers if: always() @@ -218,21 +196,14 @@ jobs: --spec-version 2025-11-25 \ -o conformance-client-results/full - # SEP-2322 MRTR client scenario (spec 2026-07-28). - - name: Run draft SEP-2322 client scenario + - name: Run 2026-07-28 client suite run: | npx -y "@modelcontextprotocol/conformance@${DRAFT_CONFORMANCE_VERSION}" client \ --command "$(pwd)/target/debug/conformance-client" \ - --scenario sep-2322-client-request-state \ - -o conformance-client-results/mrtr - - - name: Run draft SEP-2575 client scenario - run: | - npx -y "@modelcontextprotocol/conformance@${DRAFT_CONFORMANCE_VERSION}" client \ - --command "$(pwd)/target/debug/conformance-client" \ - --scenario request-metadata \ - --spec-version draft \ - -o conformance-client-results/sep-2575 + --suite all \ + --spec-version 2026-07-28 \ + --expected-failures conformance/expected-failures-2026-07-28.yaml \ + -o conformance-client-results/draft - name: Upload results if: always() diff --git a/conformance/expected-failures-2026-07-28.yaml b/conformance/expected-failures-2026-07-28.yaml new file mode 100644 index 000000000..07257b8ef --- /dev/null +++ b/conformance/expected-failures-2026-07-28.yaml @@ -0,0 +1,43 @@ +# Known failures for the pinned 2026-07-28 draft conformance suite +# (@modelcontextprotocol/conformance DRAFT_CONFORMANCE_VERSION). +# +# The full suites run in CI with `--expected-failures` pointing at this file: +# - a scenario failing that is NOT listed here fails the build +# - a scenario listed here that starts passing also fails the build (stale entry), +# so remove it from this list when the underlying issue is fixed. +# +# When bumping DRAFT_CONFORMANCE_VERSION, diff +# `conformance list --spec-version 2026-07-28` and update #977. + +server: + # SEP-2575 stateless lifecycle gaps: discover capability declaration, + # missing-capability rejection, HTTP 404 for removed methods, and + # diagnostic tools (test_missing_capability, test_streaming_elicitation, + # test_logging_tool) not yet implemented. + - server-stateless + # SEP-2106: composition/conditional/$anchor keywords are stripped from + # published tool input schemas. + - json-schema-2020-12 + +client: + # Client does not yet send MCP-Protocol-Version header pre-initialize as + # required by the 2026-07-28 stateless lifecycle. + - tools_call + # Auth feature gaps in the 2026-07-28 auth scenarios. + - auth/scope-step-up + - auth/offline-access-scope + - auth/offline-access-not-supported + - auth/authorization-server-migration + - auth/iss-supported + - auth/iss-not-advertised + - auth/iss-supported-missing + - auth/iss-wrong-issuer + - auth/iss-unexpected + - auth/iss-normalized + - auth/metadata-issuer-mismatch + # SEP-2243 client-side header handling gaps. + - http-standard-headers + - http-custom-headers + - http-invalid-tool-headers + # SEP-2106 client-side: $ref must not be dereferenced. + - json-schema-ref-no-deref diff --git a/conformance/src/bin/server.rs b/conformance/src/bin/server.rs index 4df4812d6..1a53c1fd9 100644 --- a/conformance/src/bin/server.rs +++ b/conformance/src/bin/server.rs @@ -1228,9 +1228,7 @@ async fn main() -> anyhow::Result<()> { let server = ConformanceServer::new(); let stateless = std::env::var_os("STATELESS").is_some(); - let config = StreamableHttpServerConfig::default() - .with_stateful_mode(!stateless) - .with_json_response(stateless); + let config = StreamableHttpServerConfig::default().with_stateful_mode(!stateless); let service = StreamableHttpService::new( move || Ok(server.clone()), LocalSessionManager::default().into(),