From 411efb53638cf57ec5ac948f52810451e3d52249 Mon Sep 17 00:00:00 2001 From: Max Isbey <224885523+maxisbey@users.noreply.github.com> Date: Wed, 21 Jan 2026 16:27:12 +0000 Subject: [PATCH 1/5] ci: use conformance repo's composite GitHub Action Replace custom npx scripts with the composite action from modelcontextprotocol/conformance. This simplifies the workflow by: - Removing the manual actions/setup-node step (handled by the action) - Replacing run-server.sh's npx call with the action - Replacing the inline npx call for client tests with the action - Removing continue-on-error (Python SDK passes all tests) References conformance repo at commit ab789d6 (expected-failures branch) until a tagged release is available. --- .github/workflows/conformance.yml | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index 248e5bf6a..31f325178 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -16,30 +16,34 @@ permissions: jobs: server-conformance: runs-on: ubuntu-latest - continue-on-error: true steps: - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - uses: astral-sh/setup-uv@61cb8a9741eeb8a550a1b8544337180c0fc8476b # v7.2.0 with: enable-cache: true version: 0.9.5 - - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 - with: - node-version: 24 - run: uv sync --frozen --all-extras --package mcp-everything-server - - run: ./.github/actions/conformance/run-server.sh + - name: Start everything-server + run: | + uv run --frozen mcp-everything-server --port 3001 & + timeout 15 bash -c 'until curl -s http://localhost:3001/mcp > /dev/null 2>&1; do sleep 0.5; done' + echo "Server ready" + - uses: modelcontextprotocol/conformance@ab789d6f03b7b3561b80a6e835bd07aa8d87c10a + with: + mode: server + url: http://localhost:3001/mcp client-conformance: runs-on: ubuntu-latest - continue-on-error: true steps: - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - uses: astral-sh/setup-uv@61cb8a9741eeb8a550a1b8544337180c0fc8476b # v7.2.0 with: enable-cache: true version: 0.9.5 - - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 - with: - node-version: 24 - run: uv sync --frozen --all-extras --package mcp - - run: npx @modelcontextprotocol/conformance@0.1.10 client --command 'uv run --frozen python .github/actions/conformance/client.py' --suite all + - uses: modelcontextprotocol/conformance@ab789d6f03b7b3561b80a6e835bd07aa8d87c10a + with: + mode: client + command: "uv run --frozen python .github/actions/conformance/client.py" + suite: all From 51cd38134ddd13e7d1cd27ead9a6d4210e01ec21 Mon Sep 17 00:00:00 2001 From: Max Isbey <224885523+maxisbey@users.noreply.github.com> Date: Wed, 21 Jan 2026 16:28:29 +0000 Subject: [PATCH 2/5] ci: remove unused run-server.sh (replaced by conformance action) --- .github/actions/conformance/run-server.sh | 30 ----------------------- 1 file changed, 30 deletions(-) delete mode 100755 .github/actions/conformance/run-server.sh diff --git a/.github/actions/conformance/run-server.sh b/.github/actions/conformance/run-server.sh deleted file mode 100755 index 01af13612..000000000 --- a/.github/actions/conformance/run-server.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash -set -e - -PORT="${PORT:-3001}" -SERVER_URL="http://localhost:${PORT}/mcp" - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -cd "$SCRIPT_DIR/../../.." - -# Start everything-server -uv run --frozen mcp-everything-server --port "$PORT" & -SERVER_PID=$! -trap "kill $SERVER_PID 2>/dev/null || true; wait $SERVER_PID 2>/dev/null || true" EXIT - -# Wait for server to be ready -MAX_RETRIES=30 -RETRY_COUNT=0 -while ! curl -s "$SERVER_URL" > /dev/null 2>&1; do - RETRY_COUNT=$((RETRY_COUNT + 1)) - if [ $RETRY_COUNT -ge $MAX_RETRIES ]; then - echo "Server failed to start after ${MAX_RETRIES} retries" >&2 - exit 1 - fi - sleep 0.5 -done - -echo "Server ready at $SERVER_URL" - -# Run conformance tests -npx @modelcontextprotocol/conformance@0.1.10 server --url "$SERVER_URL" "$@" From 34cddfe34198c62bb529b30eaf7a67b704d9d22e Mon Sep 17 00:00:00 2001 From: Max Isbey <224885523+maxisbey@users.noreply.github.com> Date: Wed, 21 Jan 2026 16:30:40 +0000 Subject: [PATCH 3/5] ci: add conformance-baseline.yml (empty, no expected failures) --- .github/workflows/conformance.yml | 2 ++ conformance-baseline.yml | 9 +++++++++ 2 files changed, 11 insertions(+) create mode 100644 conformance-baseline.yml diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index 31f325178..2dbe441b7 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -32,6 +32,7 @@ jobs: with: mode: server url: http://localhost:3001/mcp + expected-failures: ./conformance-baseline.yml client-conformance: runs-on: ubuntu-latest @@ -47,3 +48,4 @@ jobs: mode: client command: "uv run --frozen python .github/actions/conformance/client.py" suite: all + expected-failures: ./conformance-baseline.yml diff --git a/conformance-baseline.yml b/conformance-baseline.yml new file mode 100644 index 000000000..7a4089fda --- /dev/null +++ b/conformance-baseline.yml @@ -0,0 +1,9 @@ +# MCP Conformance Test Baseline +# Lists scenarios expected to fail. CI will fail if: +# - A scenario not listed here fails (regression) +# - A scenario listed here passes (stale entry, remove it) +# +# server: +# - scenario-name +# client: +# - scenario-name From 136bafe16131dd5c520767f83d2f3bf5cd04eb37 Mon Sep 17 00:00:00 2001 From: Max Isbey <224885523+maxisbey@users.noreply.github.com> Date: Wed, 21 Jan 2026 16:33:46 +0000 Subject: [PATCH 4/5] ci: update conformance action ref (now builds from source) --- .github/workflows/conformance.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index 2dbe441b7..1a064bd66 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -28,7 +28,7 @@ jobs: uv run --frozen mcp-everything-server --port 3001 & timeout 15 bash -c 'until curl -s http://localhost:3001/mcp > /dev/null 2>&1; do sleep 0.5; done' echo "Server ready" - - uses: modelcontextprotocol/conformance@ab789d6f03b7b3561b80a6e835bd07aa8d87c10a + - uses: modelcontextprotocol/conformance@a7a585ff08fe34a922300e659c2c94619b18b9cf with: mode: server url: http://localhost:3001/mcp @@ -43,7 +43,7 @@ jobs: enable-cache: true version: 0.9.5 - run: uv sync --frozen --all-extras --package mcp - - uses: modelcontextprotocol/conformance@ab789d6f03b7b3561b80a6e835bd07aa8d87c10a + - uses: modelcontextprotocol/conformance@a7a585ff08fe34a922300e659c2c94619b18b9cf with: mode: client command: "uv run --frozen python .github/actions/conformance/client.py" From b606880db18d409910011fb9150124f697e0ca1a Mon Sep 17 00:00:00 2001 From: Max Isbey <224885523+maxisbey@users.noreply.github.com> Date: Wed, 21 Jan 2026 16:38:32 +0000 Subject: [PATCH 5/5] ci: update conformance action ref to clean commit --- .github/workflows/conformance.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index 1a064bd66..07b95d100 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -28,7 +28,7 @@ jobs: uv run --frozen mcp-everything-server --port 3001 & timeout 15 bash -c 'until curl -s http://localhost:3001/mcp > /dev/null 2>&1; do sleep 0.5; done' echo "Server ready" - - uses: modelcontextprotocol/conformance@a7a585ff08fe34a922300e659c2c94619b18b9cf + - uses: modelcontextprotocol/conformance@4a42c25058c4636aca1ae7f6547ee30805de36ee with: mode: server url: http://localhost:3001/mcp @@ -43,7 +43,7 @@ jobs: enable-cache: true version: 0.9.5 - run: uv sync --frozen --all-extras --package mcp - - uses: modelcontextprotocol/conformance@a7a585ff08fe34a922300e659c2c94619b18b9cf + - uses: modelcontextprotocol/conformance@4a42c25058c4636aca1ae7f6547ee30805de36ee with: mode: client command: "uv run --frozen python .github/actions/conformance/client.py"