diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml new file mode 100644 index 00000000..66acdf85 --- /dev/null +++ b/.github/workflows/conformance.yml @@ -0,0 +1,55 @@ +name: Conformance Tests + +on: + # TODO: enable automatic triggering when tested. + # push: + # branches: [main] + # pull_request: + workflow_dispatch: + +concurrency: + group: conformance-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + server-conformance: + runs-on: ubuntu-latest + steps: + - name: Check out code + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4 + - name: Set up Go + uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5 + with: + go-version: "^1.25" + - name: Start everything-server + run: | + go run ./conformance/everything-server/main.go -http=":3001" & + # Wait for the server to be ready. + timeout 15 bash -c 'until curl -s http://localhost:3001/mcp; do sleep 0.5; done' + - name: "Run conformance tests" + uses: modelcontextprotocol/conformance@c2f3fdaf781dcd5a862cb0d2f6454c1c210bf0f0 # v0.1.11 + with: + mode: server + url: http://localhost:3001/mcp + suite: active + expected-failures: ./conformance/baseline.yml + + client-conformance: + runs-on: ubuntu-latest + steps: + - name: Check out code + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4 + - name: Set up Go + uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5 + with: + go-version: "^1.25" + - name: "Run conformance tests" + uses: modelcontextprotocol/conformance@c2f3fdaf781dcd5a862cb0d2f6454c1c210bf0f0 # v0.1.11 + with: + mode: client + command: go run ./conformance/everything-client/main.go + suite: core + expected-failures: ./conformance/baseline.yml diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 0aa95dfe..e2b5a521 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -32,5 +32,5 @@ jobs: uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # v4.1.0 with: node-version: "20" - - name: Run conformance tests - run: ./scripts/conformance.sh + - name: Run server conformance tests + run: ./scripts/server-conformance.sh diff --git a/conformance/baseline.yml b/conformance/baseline.yml new file mode 100644 index 00000000..c5835490 --- /dev/null +++ b/conformance/baseline.yml @@ -0,0 +1,21 @@ +server: + # All tests should pass. +client: +- auth/basic-cimd +- auth/metadata-default +- auth/metadata-var1 +- auth/metadata-var2 +- auth/metadata-var3 +- auth/2025-03-26-oauth-metadata-backcompat +- auth/2025-03-26-oauth-endpoint-fallback +- auth/scope-from-www-authenticate +- auth/scope-from-scopes-supported +- auth/scope-omitted-when-undefined +- auth/scope-step-up +- auth/scope-retry-limit +- auth/token-endpoint-auth-basic +- auth/token-endpoint-auth-post +- auth/token-endpoint-auth-none +- auth/client-credentials-jwt +- auth/client-credentials-basic +- sse-retry diff --git a/conformance/everything-client/main.go b/conformance/everything-client/main.go new file mode 100644 index 00000000..9674dbbc --- /dev/null +++ b/conformance/everything-client/main.go @@ -0,0 +1,248 @@ +// Copyright 2025 The Go MCP SDK Authors. All rights reserved. +// Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file. + +// The conformance client implements features required for MCP conformance testing. +// It mirrors the functionality of the TypeScript conformance client at +// https://github.com/modelcontextprotocol/typescript-sdk/blob/main/src/conformance/everything-client.ts +package main + +import ( + "context" + "fmt" + "log" + "os" + "slices" + "sort" + "strings" + + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +// scenarioHandler is the function signature for all conformance test scenarios. +// It takes a context and the server URL to connect to. +type scenarioHandler func(ctx context.Context, serverURL string) error + +var ( + // registry stores all registered scenario handlers. + registry = make(map[string]scenarioHandler) +) + +// registerScenario registers a new scenario handler with the given name. +// This function should be called during init() by scenario implementations. +func registerScenario(name string, handler scenarioHandler) { + if _, exists := registry[name]; exists { + log.Fatalf("Scenario %q is already registered", name) + } + registry[name] = handler +} + +func init() { + registerScenario("initialize", runBasicClient) + registerScenario("tools_call", runToolsCallClient) + registerScenario("elicitation-sep1034-client-defaults", runElicitationDefaultsClient) + registerScenario("sse-retry", runSSERetryClient) +} + +// ============================================================================ +// Basic scenarios +// ============================================================================ + +func runBasicClient(ctx context.Context, serverURL string) error { + session, err := connectToServer(ctx, serverURL) + if err != nil { + return err + } + defer session.Close() + + _, err = session.ListTools(ctx, nil) + if err != nil { + return fmt.Errorf("session.ListTools(): %v", err) + } + + return nil +} + +func runToolsCallClient(ctx context.Context, serverURL string) error { + session, err := connectToServer(ctx, serverURL) + if err != nil { + return err + } + defer session.Close() + + tools, err := session.ListTools(ctx, nil) + if err != nil { + return fmt.Errorf("session.ListTools(): %v", err) + } + + idx := slices.IndexFunc(tools.Tools, func(t *mcp.Tool) bool { + return t.Name == "add_numbers" + }) + if idx == -1 { + return fmt.Errorf("tool %q not found", "add_numbers") + } + + _, err = session.CallTool(ctx, &mcp.CallToolParams{ + Name: "add_numbers", + Arguments: map[string]any{"a": 5, "b": 3}, + }) + if err != nil { + return fmt.Errorf("session.CallTool('add_numbers'): %v", err) + } + + return nil +} + +// ============================================================================ +// Elicitation scenarios +// ============================================================================ + +func runElicitationDefaultsClient(ctx context.Context, serverURL string) error { + elicitationHandler := func(ctx context.Context, req *mcp.ElicitRequest) (*mcp.ElicitResult, error) { + return &mcp.ElicitResult{ + Action: "accept", + Content: map[string]any{}, + }, nil + } + + session, err := connectToServer(ctx, serverURL, withClientOptions(&mcp.ClientOptions{ + ElicitationHandler: elicitationHandler, + })) + if err != nil { + return err + } + defer session.Close() + + tools, err := session.ListTools(ctx, nil) + if err != nil { + return fmt.Errorf("session.ListTools(): %v", err) + } + + var testToolName = "test_client_elicitation_defaults" + idx := slices.IndexFunc(tools.Tools, func(t *mcp.Tool) bool { + return t.Name == testToolName + }) + if idx == -1 { + return fmt.Errorf("tool %q not found", testToolName) + } + + _, err = session.CallTool(ctx, &mcp.CallToolParams{ + Name: testToolName, + Arguments: map[string]any{}, + }) + if err != nil { + return fmt.Errorf("session.CallTool(%q): %v", testToolName, err) + } + + return nil +} + +// ============================================================================ +// SSE retry scenario +// ============================================================================ + +func runSSERetryClient(ctx context.Context, serverURL string) error { + // TODO: this scenario is not passing yet. It requires a fix in the client SSE handling. + session, err := connectToServer(ctx, serverURL) + if err != nil { + return err + } + defer session.Close() + log.Printf("Connected to server %q", serverURL) + + tools, err := session.ListTools(ctx, nil) + if err != nil { + return fmt.Errorf("session.ListTools(): %v", err) + } + + var testToolName = "test_reconnection" + idx := slices.IndexFunc(tools.Tools, func(t *mcp.Tool) bool { + return t.Name == testToolName + }) + if idx == -1 { + return fmt.Errorf("tool %q not found", testToolName) + } + + _, err = session.CallTool(ctx, &mcp.CallToolParams{ + Name: testToolName, + Arguments: map[string]any{}, + }) + if err != nil { + return fmt.Errorf("session.CallTool(%q): %v", testToolName, err) + } + + return nil +} + +// ============================================================================ +// Main entry point +// ============================================================================ + +func main() { + if len(os.Args) != 2 { + printUsageAndExit("Usage: %s ", os.Args[0]) + } + + serverURL := os.Args[1] + scenarioName := os.Getenv("MCP_CONFORMANCE_SCENARIO") + + if scenarioName == "" { + printUsageAndExit("MCP_CONFORMANCE_SCENARIO not set") + } + + handler, ok := registry[scenarioName] + if !ok { + printUsageAndExit("Unknown scenario: %q", scenarioName) + } + + ctx := context.Background() + if err := handler(ctx, serverURL); err != nil { + log.Fatalf("Scenario %q failed: %v", scenarioName, err) + } +} + +func printUsageAndExit(format string, args ...any) { + var scenarios []string + for name := range registry { + scenarios = append(scenarios, name) + } + sort.Strings(scenarios) + + msg := fmt.Sprintf(format, args...) + log.Fatalf("%s\nAvailable scenarios:\n - %s", msg, strings.Join(scenarios, "\n - ")) +} + +type connectConfig struct { + clientOptions *mcp.ClientOptions +} + +type connectOption func(*connectConfig) + +func withClientOptions(opts *mcp.ClientOptions) connectOption { + return func(c *connectConfig) { + c.clientOptions = opts + } +} + +// connectToServer connects to the MCP server and returns a client session. +// The caller is responsible for closing the session. +func connectToServer(ctx context.Context, serverURL string, opts ...connectOption) (*mcp.ClientSession, error) { + config := &connectConfig{} + for _, opt := range opts { + opt(config) + } + + client := mcp.NewClient(&mcp.Implementation{ + Name: "test-client", + Version: "1.0.0", + }, config.clientOptions) + + transport := &mcp.StreamableClientTransport{Endpoint: serverURL} + + session, err := client.Connect(ctx, transport, nil) + if err != nil { + return nil, fmt.Errorf("client.Connect(): %v", err) + } + + return session, nil +} diff --git a/examples/server/conformance/main.go b/conformance/everything-server/main.go similarity index 100% rename from examples/server/conformance/main.go rename to conformance/everything-server/main.go diff --git a/scripts/client-conformance.sh b/scripts/client-conformance.sh new file mode 100755 index 00000000..c093c75f --- /dev/null +++ b/scripts/client-conformance.sh @@ -0,0 +1,85 @@ +#!/bin/bash +# Copyright 2025 The Go MCP SDK Authors. All rights reserved. +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file. + +# Run MCP conformance tests against the Go SDK conformance client. + +set -e + +RESULT_DIR="" +WORKDIR="" +CONFORMANCE_REPO="" +FINAL_EXIT_CODE=0 + +usage() { + echo "Usage: $0 [options]" + echo "" + echo "Run MCP conformance tests against the Go SDK conformance client." + echo "" + echo "Options:" + echo " --result_dir Save results to the specified directory" + echo " --conformance_repo Run conformance tests from a local checkout" + echo " instead of using the latest npm release" + echo " --help Show this help message" +} + +# Parse arguments. +while [[ $# -gt 0 ]]; do + case $1 in + --result_dir) + RESULT_DIR="$2" + shift 2 + ;; + --conformance_repo) + CONFORMANCE_REPO="$2" + shift 2 + ;; + --help) + usage + exit 0 + ;; + *) + echo "Unknown option: $1" + usage + exit 1 + ;; + esac +done + +# Set up the work directory. +if [ -n "$RESULT_DIR" ]; then + mkdir -p "$RESULT_DIR" + WORKDIR="$RESULT_DIR" +else + WORKDIR=$(mktemp -d) +fi + +# Build the conformance server. +go build -o "$WORKDIR/conformance-client" ./conformance/everything-client + +# Run conformance tests from the work directory to avoid writing results to the repo. +echo "Running conformance tests..." +if [ -n "$CONFORMANCE_REPO" ]; then + # Run from local checkout using npm run start. + (cd "$WORKDIR" && \ + npm --prefix "$CONFORMANCE_REPO" run start -- \ + client --command "$WORKDIR/conformance-client" \ + --suite core \ + ${RESULT_DIR:+--output-dir "$RESULT_DIR"}) || FINAL_EXIT_CODE=$? +else + (cd "$WORKDIR" && \ + npx @modelcontextprotocol/conformance@latest \ + client --command "$WORKDIR/conformance-client" \ + --suite core \ + ${RESULT_DIR:+--output-dir "$RESULT_DIR"}) || FINAL_EXIT_CODE=$? +fi + +echo "" +if [ -n "$RESULT_DIR" ]; then + echo "See $RESULT_DIR for details." +else + echo "Run with --result_dir to save results." +fi + +exit $FINAL_EXIT_CODE diff --git a/scripts/conformance.sh b/scripts/server-conformance.sh similarity index 76% rename from scripts/conformance.sh rename to scripts/server-conformance.sh index 9e7d3007..0826086a 100755 --- a/scripts/conformance.sh +++ b/scripts/server-conformance.sh @@ -12,6 +12,7 @@ SERVER_PID="" RESULT_DIR="" WORKDIR="" CONFORMANCE_REPO="" +FINAL_EXIT_CODE=0 usage() { echo "Usage: $0 [options]" @@ -50,11 +51,9 @@ done cleanup() { if [ -n "$SERVER_PID" ]; then + echo "Stopping server..." kill "$SERVER_PID" 2>/dev/null || true - fi - # Clean up the work directory unless --result_dir was specified. - if [ -z "$RESULT_DIR" ] && [ -n "$WORKDIR" ]; then - rm -rf "$WORKDIR" + wait "$SERVER_PID" 2>/dev/null || true fi } trap cleanup EXIT @@ -68,7 +67,7 @@ else fi # Build the conformance server. -go build -o "$WORKDIR/conformance-server" ./examples/server/conformance +go build -o "$WORKDIR/conformance-server" ./conformance/everything-server # Start the server in the background echo "Starting conformance server on port $PORT..." @@ -79,17 +78,10 @@ echo "Server pid is $SERVER_PID" # Wait for server to be ready echo "Waiting for server to be ready..." -for i in {1..30}; do - if curl -s "http://localhost:$PORT" > /dev/null 2>&1; then - echo "Server is ready." - break - fi - if [ "$i" -eq 30 ]; then - echo "Server failed to start within 15 seconds." - exit 1 - fi - sleep 0.5 -done +if ! timeout 15 bash -c "until curl -s http://localhost:$PORT > /dev/null 2>&1; do sleep 0.5; done"; then + echo "Server failed to start within 15 seconds." + exit 1 +fi # Run conformance tests from the work directory to avoid writing results to the repo. echo "Running conformance tests..." @@ -97,10 +89,13 @@ if [ -n "$CONFORMANCE_REPO" ]; then # Run from local checkout using npm run start. (cd "$WORKDIR" && \ npm --prefix "$CONFORMANCE_REPO" run start -- \ - server --url "http://localhost:$PORT") + server --url "http://localhost:$PORT" \ + ${RESULT_DIR:+--output-dir "$RESULT_DIR"}) || FINAL_EXIT_CODE=$? else (cd "$WORKDIR" && \ - npx @modelcontextprotocol/conformance@latest server --url "http://localhost:$PORT") + npx @modelcontextprotocol/conformance@latest \ + server --url "http://localhost:$PORT" \ + ${RESULT_DIR:+--output-dir "$RESULT_DIR"}) || FINAL_EXIT_CODE=$? fi echo "" @@ -109,3 +104,5 @@ if [ -n "$RESULT_DIR" ]; then else echo "Run with --result_dir to save results." fi + +exit $FINAL_EXIT_CODE