diff --git a/mcp-interview.json b/mcp-interview.json index dd5a2d2..7463790 100644 --- a/mcp-interview.json +++ b/mcp-interview.json @@ -2401,8 +2401,8 @@ "sampling_requests": 2, "elicitation_requests": 1, "list_roots_requests": 0, - "logging_requests": 2, - "plan": "This testing plan systematically exercises all 12 MCP server tools, starting with foundational, stateless tools and progressing to those demonstrating advanced features or requiring more complex input. Tools are grouped and ordered to build understanding, with dependencies and logical relationships considered. Each tool is tested with realistic arguments, and edge cases are included where appropriate. The plan ensures coverage of normal and boundary scenarios, and anticipates expected outputs for each step.", + "logging_requests": 1, + "plan": "This testing plan systematically exercises all 12 MCP server tools, starting with foundational, stateless tools and progressing to those demonstrating advanced features or requiring more complex input. Tools are grouped and ordered to build understanding, with dependencies and logical relationships considered. Each tool is tested with realistic arguments, and edge cases are included where relevant. The plan aims for comprehensive coverage with minimal redundancy.", "steps": [ { "error_handling": { @@ -2450,9 +2450,9 @@ "sampling_requests": 0, "elicitation_requests": 0, "list_roots_requests": 0, - "logging_requests": 2, - "justification": "Echo is a foundational tool with no dependencies. Testing with a simple string verifies basic echo functionality.", - "expected_output": "The response should be the same as the input message: 'Hello, MCP!'.", + "logging_requests": 1, + "justification": "Echo is a foundational tool; test with a standard string to verify basic echo functionality.", + "expected_output": "Should return the exact message sent: 'Hello, MCP!'", "tool_name": "echo", "tool_arguments": { "message": "Hello, MCP!" @@ -2505,8 +2505,8 @@ "elicitation_requests": 0, "list_roots_requests": 0, "logging_requests": 0, - "justification": "Edge case: Test echo with an empty string to ensure it handles empty input gracefully.", - "expected_output": "The response should be an empty string.", + "justification": "Test echo with an empty string to check handling of edge case input.", + "expected_output": "Should return an empty string.", "tool_name": "echo", "tool_arguments": { "message": "" @@ -2559,8 +2559,8 @@ "elicitation_requests": 0, "list_roots_requests": 0, "logging_requests": 0, - "justification": "Basic arithmetic test with two positive integers.", - "expected_output": "The response should be 10.", + "justification": "Test add with two positive integers to verify correct addition.", + "expected_output": "Should return 10.", "tool_name": "add", "tool_arguments": { "a": 3, @@ -2614,8 +2614,8 @@ "elicitation_requests": 0, "list_roots_requests": 0, "logging_requests": 0, - "justification": "Test with a negative and a floating-point number to check handling of different number types.", - "expected_output": "The response should be -2.5.", + "justification": "Test add with a negative and a floating-point number to check handling of varied numeric types.", + "expected_output": "Should return -2.5.", "tool_name": "add", "tool_arguments": { "a": -5, @@ -2669,8 +2669,8 @@ "elicitation_requests": 0, "list_roots_requests": 0, "logging_requests": 0, - "justification": "Test the long-running operation with minimal duration and steps to verify progress updates and completion.", - "expected_output": "Should return progress updates (e.g., 0%, 50%, 100%) over ~2 seconds, then a completion message.", + "justification": "Test longRunningOperation with minimal duration and steps to verify progress updates and completion.", + "expected_output": "Should provide progress updates (e.g., 0%, 50%, 100%) over 2 seconds, then a completion message.", "tool_name": "longRunningOperation", "tool_arguments": { "duration": 2, @@ -2724,7 +2724,7 @@ "elicitation_requests": 0, "list_roots_requests": 0, "logging_requests": 0, - "justification": "Edge case: Zero duration and minimal steps to ensure the tool handles instant completion.", + "justification": "Edge case: zero duration and single step to test immediate completion.", "expected_output": "Should immediately return a single progress update and completion.", "tool_name": "longRunningOperation", "tool_arguments": { @@ -2766,7 +2766,7 @@ "content": [ { "type": "text", - "text": "{\n \"npm_config_user_agent\": \"npm/10.9.2 node/v22.17.0 linux arm64 workspaces/false\",\n \"NODE_VERSION\": \"22.17.0\",\n \"HOSTNAME\": \"1639ae8399b7\",\n \"YARN_VERSION\": \"1.22.22\",\n \"npm_node_execpath\": \"/usr/local/bin/node\",\n \"npm_config_noproxy\": \"\",\n \"HOME\": \"/root\",\n \"npm_package_json\": \"/package.json\",\n \"npm_config_userconfig\": \"/root/.npmrc\",\n \"npm_config_local_prefix\": \"/\",\n \"npm_config_yes\": \"true\",\n \"COLOR\": \"0\",\n \"npm_config_prefix\": \"/usr/local\",\n \"npm_config_npm_version\": \"10.9.2\",\n \"npm_config_cache\": \"/root/.npm\",\n \"npm_config_node_gyp\": \"/usr/local/lib/node_modules/npm/node_modules/node-gyp/bin/node-gyp.js\",\n \"PATH\": \"/root/.npm/_npx/5b2dd62b9d0bddd4/node_modules/.bin:/node_modules/.bin:/usr/local/lib/node_modules/npm/node_modules/@npmcli/run-script/lib/node-gyp-bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\n \"NODE\": \"/usr/local/bin/node\",\n \"npm_lifecycle_script\": \"mcp-server-everything\",\n \"npm_lifecycle_event\": \"npx\",\n \"npm_config_globalconfig\": \"/usr/local/etc/npmrc\",\n \"npm_config_init_module\": \"/root/.npm-init.js\",\n \"PWD\": \"/\",\n \"npm_execpath\": \"/usr/local/lib/node_modules/npm/bin/npm-cli.js\",\n \"npm_config_global_prefix\": \"/usr/local\",\n \"npm_command\": \"exec\",\n \"INIT_CWD\": \"/\",\n \"EDITOR\": \"vi\"\n}", + "text": "{\n \"npm_config_user_agent\": \"npm/10.9.2 node/v22.17.0 linux arm64 workspaces/false\",\n \"NODE_VERSION\": \"22.17.0\",\n \"HOSTNAME\": \"a2ba59e0192a\",\n \"YARN_VERSION\": \"1.22.22\",\n \"npm_node_execpath\": \"/usr/local/bin/node\",\n \"npm_config_noproxy\": \"\",\n \"HOME\": \"/root\",\n \"npm_package_json\": \"/package.json\",\n \"npm_config_userconfig\": \"/root/.npmrc\",\n \"npm_config_local_prefix\": \"/\",\n \"npm_config_yes\": \"true\",\n \"COLOR\": \"0\",\n \"npm_config_prefix\": \"/usr/local\",\n \"npm_config_npm_version\": \"10.9.2\",\n \"npm_config_cache\": \"/root/.npm\",\n \"npm_config_node_gyp\": \"/usr/local/lib/node_modules/npm/node_modules/node-gyp/bin/node-gyp.js\",\n \"PATH\": \"/root/.npm/_npx/5b2dd62b9d0bddd4/node_modules/.bin:/node_modules/.bin:/usr/local/lib/node_modules/npm/node_modules/@npmcli/run-script/lib/node-gyp-bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\n \"NODE\": \"/usr/local/bin/node\",\n \"npm_lifecycle_script\": \"mcp-server-everything\",\n \"npm_lifecycle_event\": \"npx\",\n \"npm_config_globalconfig\": \"/usr/local/etc/npmrc\",\n \"npm_config_init_module\": \"/root/.npm-init.js\",\n \"PWD\": \"/\",\n \"npm_execpath\": \"/usr/local/lib/node_modules/npm/bin/npm-cli.js\",\n \"npm_config_global_prefix\": \"/usr/local\",\n \"npm_command\": \"exec\",\n \"INIT_CWD\": \"/\",\n \"EDITOR\": \"vi\"\n}", "annotations": null, "meta": null } @@ -2779,8 +2779,8 @@ "elicitation_requests": 0, "list_roots_requests": 0, "logging_requests": 0, - "justification": "Stateless tool to verify the server's environment variable reporting.", - "expected_output": "A dictionary/object containing all environment variables and their values.", + "justification": "Test printEnv to verify environment variable reporting; useful for debugging server configuration.", + "expected_output": "Should return a dictionary/object of environment variables and their values.", "tool_name": "printEnv", "tool_arguments": {} }, @@ -2832,7 +2832,7 @@ "list_roots_requests": 0, "logging_requests": 0, "justification": "Test LLM sampling with a factual prompt and a small token limit.", - "expected_output": "A short LLM-generated response, likely 'Paris' or a brief sentence.", + "expected_output": "Should return a short LLM-generated response, e.g., 'The capital of France is Paris.'", "tool_name": "sampleLLM", "tool_arguments": { "prompt": "What is the capital of France?", @@ -2886,12 +2886,11 @@ "elicitation_requests": 0, "list_roots_requests": 0, "logging_requests": 0, - "justification": "Test LLM with a creative prompt and a larger token limit.", - "expected_output": "A short poem or several lines of text about the sea.", + "justification": "Test LLM sampling with a creative prompt and default maxTokens.", + "expected_output": "Should return a short haiku or poetic response about rain.", "tool_name": "sampleLLM", "tool_arguments": { - "prompt": "Write a short poem about the sea.", - "maxTokens": 50 + "prompt": "Write a haiku about rain." } }, { @@ -2954,8 +2953,8 @@ "elicitation_requests": 0, "list_roots_requests": 0, "logging_requests": 0, - "justification": "Test retrieval of the MCP_TINY_IMAGE resource.", - "expected_output": "A binary or base64-encoded image object representing the tiny image.", + "justification": "Test getTinyImage to verify image retrieval functionality.", + "expected_output": "Should return the MCP_TINY_IMAGE (likely as a base64 string or binary blob).", "tool_name": "getTinyImage", "tool_arguments": {} }, @@ -3001,6 +3000,18 @@ "priority": 0.7 }, "meta": null + }, + { + "type": "image", + "data": "iVBORw0KGgoAAAANSUhEUgAAABQAAAAUCAYAAACNiR0NAAAKsGlDQ1BJQ0MgUHJvZmlsZQAASImVlwdUU+kSgOfe9JDQEiIgJfQmSCeAlBBaAAXpYCMkAUKJMRBU7MriClZURLCs6KqIgo0idizYFsWC3QVZBNR1sWDDlXeBQ9jdd9575805c+a7c+efmf+e/z9nLgCdKZDJMlF1gCxpjjwyyI8dn5DIJvUABRiY0kBdIMyWcSMiwgCTUft3+dgGyJC9YzuU69/f/1fREImzhQBIBMbJomxhFsbHMe0TyuQ5ALg9mN9kbo5siK9gzJRjDWL8ZIhTR7hviJOHGY8fjomO5GGsDUCmCQTyVACaKeZn5wpTsTw0f4ztpSKJFGPsGbyzsmaLMMbqgiUWI8N4KD8n+S95Uv+WM1mZUyBIVfLIXoaF7C/JlmUK5v+fn+N/S1amYrSGOaa0NHlwJGaxvpAHGbNDlSxNnhI+yhLRcPwwpymCY0ZZmM1LHGWRwD9UuTZzStgop0gC+co8OfzoURZnB0SNsnx2pLJWipzHHWWBfKyuIiNG6U8T85X589Ki40Y5VxI7ZZSzM6JCx2J4Sr9cEansXywN8hurG6jce1b2X/Yr4SvX5qRFByv3LhjrXyzljuXMjlf2JhL7B4zFxCjjZTl+ylqyzAhlvDgzSOnPzo1Srs3BDuTY2gjlN0wXhESMMoRBELAhBjIhB+QggECQgBTEOeJ5Q2cUeLNl8+WS1LQcNhe7ZWI2Xyq0m8B2tHd0Bhi6syNH4j1r+C4irGtjvhWVAF4nBgcHT475Qm4BHEkCoNaO+SxnAKh3A1w5JVTIc0d8Q9cJCEAFNWCCDhiACViCLTiCK3iCLwRACIRDNCTATBBCGmRhnc+FhbAMCqAI1sNmKIOdsBv2wyE4CvVwCs7DZbgOt+AePIZ26IJX0AcfYQBBEBJCRxiIDmKImCE2iCPCQbyRACQMiUQSkCQkFZEiCmQhsgIpQoqRMmQXUokcQU4g55GrSCvyEOlAepF3yFcUh9JQJqqPmqMTUQ7KRUPRaHQGmorOQfPQfHQtWopWoAfROvQ8eh29h7ajr9B+HOBUcCycEc4Wx8HxcOG4RFwKTo5bjCvEleAqcNW4Rlwz7g6uHfca9wVPxDPwbLwt3hMfjI/BC/Fz8Ivxq/Fl+P34OvxF/B18B74P/51AJ+gRbAgeBD4hnpBKmEsoIJQQ9hJqCZcI9whdhI9EIpFFtCC6EYOJCcR04gLiauJ2Yg3xHLGV2EnsJ5FIOiQbkhcpnCQg5ZAKSFtJB0lnSbdJXaTPZBWyIdmRHEhOJEvJy8kl5APkM+Tb5G7yAEWdYkbxoIRTRJT5lHWUPZRGyk1KF2WAqkG1oHpRo6np1GXUUmo19RL1CfW9ioqKsYq7ylQVicpSlVKVwypXVDpUvtA0adY0Hm06TUFbS9tHO0d7SHtPp9PN6b70RHoOfS29kn6B/oz+WZWhaqfKVxWpLlEtV61Tva36Ro2iZqbGVZuplqdWonZM7abaa3WKurk6T12gvli9XP2E+n31fg2GhoNGuEaWxmqNAxpXNXo0SZrmmgGaIs18zd2aFzQ7GTiGCYPHEDJWMPYwLjG6mESmBZPPTGcWMQ8xW5h9WppazlqxWvO0yrVOa7WzcCxzFp+VyVrHOspqY30dpz+OO048btW46nG3x33SHq/tqy3WLtSu0b6n/VWHrROgk6GzQade56kuXtdad6ruXN0dupd0X49njvccLxxfOP7o+Ed6qJ61XqTeAr3dejf0+vUN9IP0Zfpb9S/ovzZgGfgapBtsMjhj0GvIMPQ2lBhuMjxr+JKtxeayM9ml7IvsPiM9o2AjhdEuoxajAWML4xjj5cY1xk9NqCYckxSTTSZNJn2mhqaTTReaVpk+MqOYcczSzLaYNZt9MrcwjzNfaV5v3mOhbcG3yLOosnhiSbf0sZxjWWF514poxbHKsNpudcsatXaxTrMut75pg9q42khsttu0TiBMcJ8gnVAx4b4tzZZrm2tbZdthx7ILs1tuV2/3ZqLpxMSJGyY2T/xu72Kfab/H/rGDpkOIw3KHRod3jtaOQsdyx7tOdKdApyVODU5vnW2cxc47nB+4MFwmu6x0aXL509XNVe5a7drrZuqW5LbN7T6HyYngrOZccSe4+7kvcT/l/sXD1SPH46jHH562nhmeBzx7JllMEk/aM6nTy9hL4LXLq92b7Z3k/ZN3u4+Rj8Cnwue5r4mvyHevbzfXipvOPch942fvJ/er9fvE8+At4p3zx/kH+Rf6twRoBsQElAU8CzQOTA2sCuwLcglaEHQumBAcGrwh+D5fny/kV/L7QtxCFoVcDKWFRoWWhT4Psw6ThzVORieHTN44+ckUsynSKfXhEM4P3xj+NMIiYk7EyanEqRFTy6e+iHSIXBjZHMWImhV1IOpjtF/0uujHMZYxipimWLXY6bGVsZ/i/OOK49rjJ8Yvir+eoJsgSWhIJCXGJu5N7J8WMG3ztK7pLtMLprfNsJgxb8bVmbozM2eenqU2SzDrWBIhKS7pQNI3QbigQtCfzE/eltwn5Am3CF+JfEWbRL1iL3GxuDvFK6U4pSfVK3Vjam+aT1pJ2msJT1ImeZsenL4z/VNGeMa+jMHMuMyaLHJWUtYJqaY0Q3pxtsHsebNbZTayAln7HI85m+f0yUPle7OR7BnZDTlMbDi6obBU/KDoyPXOLc/9PDd27rF5GvOk827Mt56/an53XmDezwvwC4QLmhYaLVy2sGMRd9Guxcji5MVNS0yW5C/pWhq0dP8y6rKMZb8st19evPzDirgVjfn6+UvzO38I+qGqQLVAXnB/pefKnT/if5T82LLKadXWVd8LRYXXiuyLSoq+rRauvrbGYU3pmsG1KWtb1rmu27GeuF66vm2Dz4b9xRrFecWdGydvrNvE3lS46cPmWZuvljiX7NxC3aLY0l4aVtqw1XTr+q3fytLK7pX7ldds09u2atun7aLtt3f47qjeqb+zaOfXnyQ/PdgVtKuuwryiZDdxd+7uF3ti9zT/zPm5cq/u3qK9f+6T7mvfH7n/YqVbZeUBvQPrqtAqRVXvwekHbx3yP9RQbVu9q4ZVU3QYDisOvzySdKTtaOjRpmOcY9XHzY5vq2XUFtYhdfPr+urT6tsbEhpaT4ScaGr0bKw9aXdy3ymjU+WntU6vO0M9k39m8Gze2f5zsnOvz6ee72ya1fT4QvyFuxenXmy5FHrpyuXAyxeauc1nr3hdOXXV4+qJa5xr9dddr9fdcLlR+4vLL7Utri11N91uNtzyv9XYOqn1zG2f2+fv+N+5fJd/9/q9Kfda22LaHtyffr/9gehBz8PMh28f5T4aeLz0CeFJ4VP1pyXP9J5V/Gr1a027a/vpDv+OG8+jnj/uFHa++i37t29d+S/oL0q6Dbsrexx7TvUG9t56Oe1l1yvZq4HXBb9r/L7tjeWb43/4/nGjL76v66387eC71e913u/74PyhqT+i/9nHrI8Dnwo/63ze/4Xzpflr3NfugbnfSN9K/7T6s/F76Pcng1mDgzKBXDA8CuAwRVNSAN7tA6AnADCwGYI6bWSmHhZk5D9gmOA/8cjcPSyuANWYGRqNeOcADmNqvhRAzRdgaCyK9gXUyUmpo/Pv8Kw+JAbYv8K0HECi2x6tebQU/iEjc/xf+v6nBWXWv9l/AV0EC6JTIblRAAAAeGVYSWZNTQAqAAAACAAFARIAAwAAAAEAAQAAARoABQAAAAEAAABKARsABQAAAAEAAABSASgAAwAAAAEAAgAAh2kABAAAAAEAAABaAAAAAAAAAJAAAAABAAAAkAAAAAEAAqACAAQAAAABAAAAFKADAAQAAAABAAAAFAAAAAAXNii1AAAACXBIWXMAABYlAAAWJQFJUiTwAAAB82lUWHRYTUw6Y29tLmFkb2JlLnhtcAAAAAAAPHg6eG1wbWV0YSB4bWxuczp4PSJhZG9iZTpuczptZXRhLyIgeDp4bXB0az0iWE1QIENvcmUgNi4wLjAiPgogICA8cmRmOlJERiB4bWxuczpyZGY9Imh0dHA6Ly93d3cudzMub3JnLzE5OTkvMDIvMjItcmRmLXN5bnRheC1ucyMiPgogICAgICA8cmRmOkRlc2NyaXB0aW9uIHJkZjphYm91dD0iIgogICAgICAgICAgICB4bWxuczp0aWZmPSJodHRwOi8vbnMuYWRvYmUuY29tL3RpZmYvMS4wLyI+CiAgICAgICAgIDx0aWZmOllSZXNvbHV0aW9uPjE0NDwvdGlmZjpZUmVzb2x1dGlvbj4KICAgICAgICAgPHRpZmY6T3JpZW50YXRpb24+MTwvdGlmZjpPcmllbnRhdGlvbj4KICAgICAgICAgPHRpZmY6WFJlc29sdXRpb24+MTQ0PC90aWZmOlhSZXNvbHV0aW9uPgogICAgICAgICA8dGlmZjpSZXNvbHV0aW9uVW5pdD4yPC90aWZmOlJlc29sdXRpb25Vbml0PgogICAgICA8L3JkZjpEZXNjcmlwdGlvbj4KICAgPC9yZGY6UkRGPgo8L3g6eG1wbWV0YT4KReh49gAAAjRJREFUOBGFlD2vMUEUx2clvoNCcW8hCqFAo1dKhEQpvsF9KrWEBh/ALbQ0KkInBI3SWyGPCCJEQliXgsTLefaca/bBWjvJzs6cOf/fnDkzOQJIjWm06/XKBEGgD8c6nU5VIWgBtQDPZPWtJE8O63a7LBgMMo/Hw0ql0jPjcY4RvmqXy4XMjUYDUwLtdhtmsxnYbDbI5/O0djqdFFKmsEiGZ9jP9gem0yn0ej2Yz+fg9XpfycimAD7DttstQTDKfr8Po9GIIg6Hw1Cr1RTgB+A72GAwgMPhQLBMJgNSXsFqtUI2myUo18pA6QJogefsPrLBX4QdCVatViklw+EQRFGEj88P2O12pEUGATmsXq+TaLPZ0AXgMRF2vMEqlQoJTSYTpNNpApvNZliv1/+BHDaZTAi2Wq1A3Ig0xmMej7+RcZjdbodUKkWAaDQK+GHjHPnImB88JrZIJAKFQgH2+z2BOczhcMiwRCIBgUAA+NN5BP6mj2DYff35gk6nA61WCzBn2JxO5wPM7/fLz4vD0E+OECfn8xl/0Gw2KbLxeAyLxQIsFgt8p75pDSO7h/HbpUWpewCike9WLpfB7XaDy+WCYrFI/slk8i0MnRRAUt46hPMI4vE4+Hw+ec7t9/44VgWigEeby+UgFArJWjUYOqhWG6x50rpcSfR6PVUfNOgEVRlTX0HhrZBKz4MZjUYWi8VoA+lc9H/VaRZYjBKrtXR8tlwumcFgeMWRbZpA9ORQWfVm8A/FsrLaxebd5wAAAABJRU5ErkJggg==", + "mimeType": "image/png", + "annotations": { + "audience": [ + "user" + ], + "priority": 0.5 + }, + "meta": null } ], "structuredContent": null, @@ -3011,12 +3022,12 @@ "elicitation_requests": 0, "list_roots_requests": 0, "logging_requests": 0, - "justification": "Test annotated message with 'success' type and no image.", - "expected_output": "A message object with a 'success' annotation and no image included.", + "justification": "Test annotatedMessage with 'success' type and image inclusion to verify annotation and image embedding.", + "expected_output": "Should return a message annotated as 'success' and include an example image.", "tool_name": "annotatedMessage", "tool_arguments": { "messageType": "success", - "includeImage": false + "includeImage": true } }, { @@ -3062,18 +3073,6 @@ "priority": 1.0 }, "meta": null - }, - { - "type": "image", - "data": "iVBORw0KGgoAAAANSUhEUgAAABQAAAAUCAYAAACNiR0NAAAKsGlDQ1BJQ0MgUHJvZmlsZQAASImVlwdUU+kSgOfe9JDQEiIgJfQmSCeAlBBaAAXpYCMkAUKJMRBU7MriClZURLCs6KqIgo0idizYFsWC3QVZBNR1sWDDlXeBQ9jdd9575805c+a7c+efmf+e/z9nLgCdKZDJMlF1gCxpjjwyyI8dn5DIJvUABRiY0kBdIMyWcSMiwgCTUft3+dgGyJC9YzuU69/f/1fREImzhQBIBMbJomxhFsbHMe0TyuQ5ALg9mN9kbo5siK9gzJRjDWL8ZIhTR7hviJOHGY8fjomO5GGsDUCmCQTyVACaKeZn5wpTsTw0f4ztpSKJFGPsGbyzsmaLMMbqgiUWI8N4KD8n+S95Uv+WM1mZUyBIVfLIXoaF7C/JlmUK5v+fn+N/S1amYrSGOaa0NHlwJGaxvpAHGbNDlSxNnhI+yhLRcPwwpymCY0ZZmM1LHGWRwD9UuTZzStgop0gC+co8OfzoURZnB0SNsnx2pLJWipzHHWWBfKyuIiNG6U8T85X589Ki40Y5VxI7ZZSzM6JCx2J4Sr9cEansXywN8hurG6jce1b2X/Yr4SvX5qRFByv3LhjrXyzljuXMjlf2JhL7B4zFxCjjZTl+ylqyzAhlvDgzSOnPzo1Srs3BDuTY2gjlN0wXhESMMoRBELAhBjIhB+QggECQgBTEOeJ5Q2cUeLNl8+WS1LQcNhe7ZWI2Xyq0m8B2tHd0Bhi6syNH4j1r+C4irGtjvhWVAF4nBgcHT475Qm4BHEkCoNaO+SxnAKh3A1w5JVTIc0d8Q9cJCEAFNWCCDhiACViCLTiCK3iCLwRACIRDNCTATBBCGmRhnc+FhbAMCqAI1sNmKIOdsBv2wyE4CvVwCs7DZbgOt+AePIZ26IJX0AcfYQBBEBJCRxiIDmKImCE2iCPCQbyRACQMiUQSkCQkFZEiCmQhsgIpQoqRMmQXUokcQU4g55GrSCvyEOlAepF3yFcUh9JQJqqPmqMTUQ7KRUPRaHQGmorOQfPQfHQtWopWoAfROvQ8eh29h7ajr9B+HOBUcCycEc4Wx8HxcOG4RFwKTo5bjCvEleAqcNW4Rlwz7g6uHfca9wVPxDPwbLwt3hMfjI/BC/Fz8Ivxq/Fl+P34OvxF/B18B74P/51AJ+gRbAgeBD4hnpBKmEsoIJQQ9hJqCZcI9whdhI9EIpFFtCC6EYOJCcR04gLiauJ2Yg3xHLGV2EnsJ5FIOiQbkhcpnCQg5ZAKSFtJB0lnSbdJXaTPZBWyIdmRHEhOJEvJy8kl5APkM+Tb5G7yAEWdYkbxoIRTRJT5lHWUPZRGyk1KF2WAqkG1oHpRo6np1GXUUmo19RL1CfW9ioqKsYq7ylQVicpSlVKVwypXVDpUvtA0adY0Hm06TUFbS9tHO0d7SHtPp9PN6b70RHoOfS29kn6B/oz+WZWhaqfKVxWpLlEtV61Tva36Ro2iZqbGVZuplqdWonZM7abaa3WKurk6T12gvli9XP2E+n31fg2GhoNGuEaWxmqNAxpXNXo0SZrmmgGaIs18zd2aFzQ7GTiGCYPHEDJWMPYwLjG6mESmBZPPTGcWMQ8xW5h9WppazlqxWvO0yrVOa7WzcCxzFp+VyVrHOspqY30dpz+OO048btW46nG3x33SHq/tqy3WLtSu0b6n/VWHrROgk6GzQade56kuXtdad6ruXN0dupd0X49njvccLxxfOP7o+Ed6qJ61XqTeAr3dejf0+vUN9IP0Zfpb9S/ovzZgGfgapBtsMjhj0GvIMPQ2lBhuMjxr+JKtxeayM9ml7IvsPiM9o2AjhdEuoxajAWML4xjj5cY1xk9NqCYckxSTTSZNJn2mhqaTTReaVpk+MqOYcczSzLaYNZt9MrcwjzNfaV5v3mOhbcG3yLOosnhiSbf0sZxjWWF514poxbHKsNpudcsatXaxTrMut75pg9q42khsttu0TiBMcJ8gnVAx4b4tzZZrm2tbZdthx7ILs1tuV2/3ZqLpxMSJGyY2T/xu72Kfab/H/rGDpkOIw3KHRod3jtaOQsdyx7tOdKdApyVODU5vnW2cxc47nB+4MFwmu6x0aXL509XNVe5a7drrZuqW5LbN7T6HyYngrOZccSe4+7kvcT/l/sXD1SPH46jHH562nhmeBzx7JllMEk/aM6nTy9hL4LXLq92b7Z3k/ZN3u4+Rj8Cnwue5r4mvyHevbzfXipvOPch942fvJ/er9fvE8+At4p3zx/kH+Rf6twRoBsQElAU8CzQOTA2sCuwLcglaEHQumBAcGrwh+D5fny/kV/L7QtxCFoVcDKWFRoWWhT4Psw6ThzVORieHTN44+ckUsynSKfXhEM4P3xj+NMIiYk7EyanEqRFTy6e+iHSIXBjZHMWImhV1IOpjtF/0uujHMZYxipimWLXY6bGVsZ/i/OOK49rjJ8Yvir+eoJsgSWhIJCXGJu5N7J8WMG3ztK7pLtMLprfNsJgxb8bVmbozM2eenqU2SzDrWBIhKS7pQNI3QbigQtCfzE/eltwn5Am3CF+JfEWbRL1iL3GxuDvFK6U4pSfVK3Vjam+aT1pJ2msJT1ImeZsenL4z/VNGeMa+jMHMuMyaLHJWUtYJqaY0Q3pxtsHsebNbZTayAln7HI85m+f0yUPle7OR7BnZDTlMbDi6obBU/KDoyPXOLc/9PDd27rF5GvOk827Mt56/an53XmDezwvwC4QLmhYaLVy2sGMRd9Guxcji5MVNS0yW5C/pWhq0dP8y6rKMZb8st19evPzDirgVjfn6+UvzO38I+qGqQLVAXnB/pefKnT/if5T82LLKadXWVd8LRYXXiuyLSoq+rRauvrbGYU3pmsG1KWtb1rmu27GeuF66vm2Dz4b9xRrFecWdGydvrNvE3lS46cPmWZuvljiX7NxC3aLY0l4aVtqw1XTr+q3fytLK7pX7ldds09u2atun7aLtt3f47qjeqb+zaOfXnyQ/PdgVtKuuwryiZDdxd+7uF3ti9zT/zPm5cq/u3qK9f+6T7mvfH7n/YqVbZeUBvQPrqtAqRVXvwekHbx3yP9RQbVu9q4ZVU3QYDisOvzySdKTtaOjRpmOcY9XHzY5vq2XUFtYhdfPr+urT6tsbEhpaT4ScaGr0bKw9aXdy3ymjU+WntU6vO0M9k39m8Gze2f5zsnOvz6ee72ya1fT4QvyFuxenXmy5FHrpyuXAyxeauc1nr3hdOXXV4+qJa5xr9dddr9fdcLlR+4vLL7Utri11N91uNtzyv9XYOqn1zG2f2+fv+N+5fJd/9/q9Kfda22LaHtyffr/9gehBz8PMh28f5T4aeLz0CeFJ4VP1pyXP9J5V/Gr1a027a/vpDv+OG8+jnj/uFHa++i37t29d+S/oL0q6Dbsrexx7TvUG9t56Oe1l1yvZq4HXBb9r/L7tjeWb43/4/nGjL76v66387eC71e913u/74PyhqT+i/9nHrI8Dnwo/63ze/4Xzpflr3NfugbnfSN9K/7T6s/F76Pcng1mDgzKBXDA8CuAwRVNSAN7tA6AnADCwGYI6bWSmHhZk5D9gmOA/8cjcPSyuANWYGRqNeOcADmNqvhRAzRdgaCyK9gXUyUmpo/Pv8Kw+JAbYv8K0HECi2x6tebQU/iEjc/xf+v6nBWXWv9l/AV0EC6JTIblRAAAAeGVYSWZNTQAqAAAACAAFARIAAwAAAAEAAQAAARoABQAAAAEAAABKARsABQAAAAEAAABSASgAAwAAAAEAAgAAh2kABAAAAAEAAABaAAAAAAAAAJAAAAABAAAAkAAAAAEAAqACAAQAAAABAAAAFKADAAQAAAABAAAAFAAAAAAXNii1AAAACXBIWXMAABYlAAAWJQFJUiTwAAAB82lUWHRYTUw6Y29tLmFkb2JlLnhtcAAAAAAAPHg6eG1wbWV0YSB4bWxuczp4PSJhZG9iZTpuczptZXRhLyIgeDp4bXB0az0iWE1QIENvcmUgNi4wLjAiPgogICA8cmRmOlJERiB4bWxuczpyZGY9Imh0dHA6Ly93d3cudzMub3JnLzE5OTkvMDIvMjItcmRmLXN5bnRheC1ucyMiPgogICAgICA8cmRmOkRlc2NyaXB0aW9uIHJkZjphYm91dD0iIgogICAgICAgICAgICB4bWxuczp0aWZmPSJodHRwOi8vbnMuYWRvYmUuY29tL3RpZmYvMS4wLyI+CiAgICAgICAgIDx0aWZmOllSZXNvbHV0aW9uPjE0NDwvdGlmZjpZUmVzb2x1dGlvbj4KICAgICAgICAgPHRpZmY6T3JpZW50YXRpb24+MTwvdGlmZjpPcmllbnRhdGlvbj4KICAgICAgICAgPHRpZmY6WFJlc29sdXRpb24+MTQ0PC90aWZmOlhSZXNvbHV0aW9uPgogICAgICAgICA8dGlmZjpSZXNvbHV0aW9uVW5pdD4yPC90aWZmOlJlc29sdXRpb25Vbml0PgogICAgICA8L3JkZjpEZXNjcmlwdGlvbj4KICAgPC9yZGY6UkRGPgo8L3g6eG1wbWV0YT4KReh49gAAAjRJREFUOBGFlD2vMUEUx2clvoNCcW8hCqFAo1dKhEQpvsF9KrWEBh/ALbQ0KkInBI3SWyGPCCJEQliXgsTLefaca/bBWjvJzs6cOf/fnDkzOQJIjWm06/XKBEGgD8c6nU5VIWgBtQDPZPWtJE8O63a7LBgMMo/Hw0ql0jPjcY4RvmqXy4XMjUYDUwLtdhtmsxnYbDbI5/O0djqdFFKmsEiGZ9jP9gem0yn0ej2Yz+fg9XpfycimAD7DttstQTDKfr8Po9GIIg6Hw1Cr1RTgB+A72GAwgMPhQLBMJgNSXsFqtUI2myUo18pA6QJogefsPrLBX4QdCVatViklw+EQRFGEj88P2O12pEUGATmsXq+TaLPZ0AXgMRF2vMEqlQoJTSYTpNNpApvNZliv1/+BHDaZTAi2Wq1A3Ig0xmMej7+RcZjdbodUKkWAaDQK+GHjHPnImB88JrZIJAKFQgH2+z2BOczhcMiwRCIBgUAA+NN5BP6mj2DYff35gk6nA61WCzBn2JxO5wPM7/fLz4vD0E+OECfn8xl/0Gw2KbLxeAyLxQIsFgt8p75pDSO7h/HbpUWpewCike9WLpfB7XaDy+WCYrFI/slk8i0MnRRAUt46hPMI4vE4+Hw+ec7t9/44VgWigEeby+UgFArJWjUYOqhWG6x50rpcSfR6PVUfNOgEVRlTX0HhrZBKz4MZjUYWi8VoA+lc9H/VaRZYjBKrtXR8tlwumcFgeMWRbZpA9ORQWfVm8A/FsrLaxebd5wAAAABJRU5ErkJggg==", - "mimeType": "image/png", - "annotations": { - "audience": [ - "user" - ], - "priority": 0.5 - }, - "meta": null } ], "structuredContent": null, @@ -3084,12 +3083,11 @@ "elicitation_requests": 0, "list_roots_requests": 0, "logging_requests": 0, - "justification": "Test annotated message with 'error' type and image included to verify both annotation and image embedding.", - "expected_output": "A message object with an 'error' annotation and an example image included.", + "justification": "Test annotatedMessage with 'error' type and default image inclusion (false) to check error annotation.", + "expected_output": "Should return a message annotated as 'error' without an image.", "tool_name": "annotatedMessage", "tool_arguments": { - "messageType": "error", - "includeImage": true + "messageType": "error" } }, { @@ -3157,8 +3155,8 @@ "elicitation_requests": 0, "list_roots_requests": 0, "logging_requests": 0, - "justification": "Test resource reference with the minimum valid resource ID.", - "expected_output": "A resource reference object for resource ID 1.", + "justification": "Test getResourceReference with the minimum valid resourceId to check lower boundary.", + "expected_output": "Should return a resource reference object for resourceId 1.", "tool_name": "getResourceReference", "tool_arguments": { "resourceId": 1 @@ -3229,8 +3227,8 @@ "elicitation_requests": 0, "list_roots_requests": 0, "logging_requests": 0, - "justification": "Edge case: Test with the maximum valid resource ID.", - "expected_output": "A resource reference object for resource ID 100.", + "justification": "Test getResourceReference with the maximum valid resourceId to check upper boundary.", + "expected_output": "Should return a resource reference object for resourceId 100.", "tool_name": "getResourceReference", "tool_arguments": { "resourceId": 100 @@ -3270,7 +3268,7 @@ "content": [ { "type": "text", - "text": "Here are 3 resource links to resources available in this server (see full output in tool response if your client does not support resource_link yet):", + "text": "Here are 1 resource links to resources available in this server (see full output in tool response if your client does not support resource_link yet):", "annotations": null, "meta": null }, @@ -3284,28 +3282,6 @@ "annotations": null, "meta": null, "type": "resource_link" - }, - { - "name": "Resource 2", - "title": null, - "uri": "test://static/resource/2", - "description": "Resource 2: binary blob resource", - "mimeType": "application/octet-stream", - "size": null, - "annotations": null, - "meta": null, - "type": "resource_link" - }, - { - "name": "Resource 3", - "title": null, - "uri": "test://static/resource/3", - "description": "Resource 3: plaintext resource", - "mimeType": "text/plain", - "size": null, - "annotations": null, - "meta": null, - "type": "resource_link" } ], "structuredContent": null, @@ -3316,11 +3292,11 @@ "elicitation_requests": 0, "list_roots_requests": 0, "logging_requests": 0, - "justification": "Test retrieval of multiple resource links with the default count.", - "expected_output": "An array of 3 resource link objects, each referencing a different resource.", + "justification": "Test getResourceLinks with the minimum count to check lower boundary.", + "expected_output": "Should return a list with one resource link.", "tool_name": "getResourceLinks", "tool_arguments": { - "count": 3 + "count": 1 } }, { @@ -3480,8 +3456,8 @@ "elicitation_requests": 0, "list_roots_requests": 0, "logging_requests": 0, - "justification": "Edge case: Test with the maximum allowed count to verify upper boundary handling.", - "expected_output": "An array of 10 resource link objects.", + "justification": "Test getResourceLinks with the maximum count to check upper boundary.", + "expected_output": "Should return a list with ten resource links.", "tool_name": "getResourceLinks", "tool_arguments": { "count": 10 @@ -3538,8 +3514,8 @@ "elicitation_requests": 0, "list_roots_requests": 0, "logging_requests": 0, - "justification": "Test structured content with a city name to verify normal operation.", - "expected_output": "An object with 'temperature', 'conditions', and 'humidity' fields for San Francisco.", + "justification": "Test structuredContent with a city name to verify structured weather data output.", + "expected_output": "Should return an object with temperature, conditions, and humidity for San Francisco.", "tool_name": "structuredContent", "tool_arguments": { "location": "San Francisco" @@ -3596,8 +3572,8 @@ "elicitation_requests": 0, "list_roots_requests": 0, "logging_requests": 0, - "justification": "Test structured content with a zip code to verify alternate valid input.", - "expected_output": "An object with 'temperature', 'conditions', and 'humidity' fields for the 94103 zip code.", + "justification": "Test structuredContent with a zip code to verify alternate location input.", + "expected_output": "Should return an object with temperature, conditions, and humidity for zip code 94103.", "tool_name": "structuredContent", "tool_arguments": { "location": "94103" @@ -3650,8 +3626,8 @@ "elicitation_requests": 0, "list_roots_requests": 0, "logging_requests": 0, - "justification": "Test listing of MCP roots to verify protocol support, even if no files are present.", - "expected_output": "A list (possibly empty) of root objects or a message indicating no roots are available.", + "justification": "Test listRoots to verify the roots protocol capability, even though no files are accessed.", + "expected_output": "Should return a list of current MCP roots (may be empty or contain mock roots).", "tool_name": "listRoots", "tool_arguments": {} }, @@ -3708,8 +3684,8 @@ "elicitation_requests": 1, "list_roots_requests": 0, "logging_requests": 0, - "justification": "Test the elicitation feature to verify interactive information gathering.", - "expected_output": "A prompt or sequence of prompts asking for favorite color, number, and pets.", + "justification": "Test startElicitation to verify the elicitation protocol, prompting for user information.", + "expected_output": "Should prompt for favorite color, number, and pets, possibly as a multi-step interaction.", "tool_name": "startElicitation", "tool_arguments": {} } diff --git a/mcp-interview.md b/mcp-interview.md index 56fb464..d697070 100644 --- a/mcp-interview.md +++ b/mcp-interview.md @@ -88,6 +88,7 @@ mcp-interviewer--test --model gpt-4.1 docker run -i --rm node:lts npx -y @modelc | Input schemas parameter count | | 1.0 | 0 | 2 | | Input schemas required parameter count | | 0.6 | 0 | 2 | | Input schemas optional parameter count | | 0.4 | 0 | 2 | +| Input schema max depth | | 1.4 | 0 | 3 | ## Tool Call Statistics | Metric | Total | Average | Min | Max | @@ -97,14 +98,14 @@ mcp-interviewer--test --model gpt-4.1 docker run -i --rm node:lts npx -y @modelc | Tool call outputs with no error | 20 | | | | | Tool call outputs with error | 0 | | | | | Exceptions calling tools | 0 | | | | -| Tool call output lengths (gpt-4o text tokens) | 735 | 36.8 | 3 | 406 | +| Tool call output lengths (gpt-4o text tokens) | 736 | 36.8 | 3 | 407 | | Text output content blocks | 24 | 6.0 | 1 | 2 | -| Resource_Link output content blocks | 13 | 3.2 | 0 | 10 | +| Resource_Link output content blocks | 11 | 2.8 | 0 | 10 | | Image output content blocks | 2 | 0.5 | 0 | 1 | | Resource output content blocks | 2 | 0.5 | 0 | 1 | | Sampling requests | 2 | 0.1 | 0 | 1 | | Elicitation requests | 1 | 0.1 | 0 | 1 | -| Logging requests | 2 | 0.1 | 0 | 2 | +| Logging requests | 1 | 0.1 | 0 | 1 | ## Constraint Violations | ❌ Errors | ⚠️ Warnings | ✅ Passes | @@ -114,15 +115,15 @@ mcp-interviewer--test --model gpt-4.1 docker run -i --rm node:lts npx -y @modelc
Details -✅ openai-name-length (ONL) +✅ openai-name-length (ONL) [[1]](https://github.com/openai/openai-python/blob/e5f93f5daee9f3fc7646833ac235b1693f192a56/src/openai/types/shared_params/function_definition.py#L17-L18) -✅ openai-name-pattern (ONP) +✅ openai-name-pattern (ONP) [[1]](https://github.com/openai/openai-python/blob/e5f93f5daee9f3fc7646833ac235b1693f192a56/src/openai/types/shared_params/function_definition.py#L17-L18) -✅ openai-tool-count (OTC) +✅ openai-tool-count (OTC) [[1]](https://platform.openai.com/docs/guides/function-calling#best-practices-for-defining-functions) -✅ openai-token-length (OTL) +✅ openai-token-length (OTL) [[1]](https://platform.openai.com/docs/models/compare) -✅ tool-schema-flatness (TSF) +✅ tool-schema-flatness (TSF) [[1]](https://composio.dev/blog/gpt-4-function-calling-example)
@@ -141,7 +142,7 @@ mcp-interviewer--test --model gpt-4.1 docker run -i --rm node:lts npx -y @modelc [→ View tool details](#tool-echo) -**Reasoning (🤖):** Echo is a foundational tool with no dependencies. Testing with a simple string verifies basic echo functionality. +**Reasoning (🤖):** Echo is a foundational tool; test with a standard string to verify basic echo functionality. **Tool Call (🤖):** ```json @@ -149,7 +150,7 @@ mcp-interviewer--test --model gpt-4.1 docker run -i --rm node:lts npx -y @modelc "message": "Hello, MCP!" } ``` -**Expected Output (🤖):** The response should be the same as the input message: 'Hello, MCP!'. +**Expected Output (🤖):** Should return the exact message sent: 'Hello, MCP!' **Actual Output (1 blocks):** @@ -169,7 +170,7 @@ Echo: Hello, MCP! | Request Type | Count | | --- | --- | -| Logging | 2 | +| Logging | 1 | @@ -180,7 +181,7 @@ Echo: Hello, MCP! [→ View tool details](#tool-echo) -**Reasoning (🤖):** Edge case: Test echo with an empty string to ensure it handles empty input gracefully. +**Reasoning (🤖):** Test echo with an empty string to check handling of edge case input. **Tool Call (🤖):** ```json @@ -188,7 +189,7 @@ Echo: Hello, MCP! "message": "" } ``` -**Expected Output (🤖):** The response should be an empty string. +**Expected Output (🤖):** Should return an empty string. **Actual Output (1 blocks):** @@ -213,7 +214,7 @@ Echo: [→ View tool details](#tool-add) -**Reasoning (🤖):** Basic arithmetic test with two positive integers. +**Reasoning (🤖):** Test add with two positive integers to verify correct addition. **Tool Call (🤖):** ```json @@ -222,7 +223,7 @@ Echo: "b": 7 } ``` -**Expected Output (🤖):** The response should be 10. +**Expected Output (🤖):** Should return 10. **Actual Output (1 blocks):** @@ -247,7 +248,7 @@ The sum of 3 and 7 is 10. [→ View tool details](#tool-add) -**Reasoning (🤖):** Test with a negative and a floating-point number to check handling of different number types. +**Reasoning (🤖):** Test add with a negative and a floating-point number to check handling of varied numeric types. **Tool Call (🤖):** ```json @@ -256,7 +257,7 @@ The sum of 3 and 7 is 10. "b": 2.5 } ``` -**Expected Output (🤖):** The response should be -2.5. +**Expected Output (🤖):** Should return -2.5. **Actual Output (1 blocks):** @@ -281,7 +282,7 @@ The sum of -5 and 2.5 is -2.5. [→ View tool details](#tool-longRunningOperation) -**Reasoning (🤖):** Test the long-running operation with minimal duration and steps to verify progress updates and completion. +**Reasoning (🤖):** Test longRunningOperation with minimal duration and steps to verify progress updates and completion. **Tool Call (🤖):** ```json @@ -290,7 +291,7 @@ The sum of -5 and 2.5 is -2.5. "steps": 2 } ``` -**Expected Output (🤖):** Should return progress updates (e.g., 0%, 50%, 100%) over ~2 seconds, then a completion message. +**Expected Output (🤖):** Should provide progress updates (e.g., 0%, 50%, 100%) over 2 seconds, then a completion message. **Actual Output (1 blocks):** @@ -315,7 +316,7 @@ Long running operation completed. Duration: 2 seconds, Steps: 2. [→ View tool details](#tool-longRunningOperation) -**Reasoning (🤖):** Edge case: Zero duration and minimal steps to ensure the tool handles instant completion. +**Reasoning (🤖):** Edge case: zero duration and single step to test immediate completion. **Tool Call (🤖):** ```json @@ -349,13 +350,13 @@ Long running operation completed. Duration: 0 seconds, Steps: 1. [→ View tool details](#tool-printEnv) -**Reasoning (🤖):** Stateless tool to verify the server's environment variable reporting. +**Reasoning (🤖):** Test printEnv to verify environment variable reporting; useful for debugging server configuration. **Tool Call (🤖):** ```json {} ``` -**Expected Output (🤖):** A dictionary/object containing all environment variables and their values. +**Expected Output (🤖):** Should return a dictionary/object of environment variables and their values. **Actual Output (1 blocks):** @@ -365,7 +366,7 @@ Long running operation completed. Duration: 0 seconds, Steps: 1. { "npm_config_user_agent": "npm/10.9.2 node/v22.17.0 linux arm64 workspaces/false", "NODE_VERSION": "22.17.0", - "HOSTNAME": "1639ae8399b7", + "HOSTNAME": "a2ba59e0192a", "YARN_VERSION": "1.22.22", "npm_node_execpath": "/usr/local/bin/node", "npm_config_noproxy": "", @@ -383,7 +384,7 @@ Long running operation completed. Duration: 0 seconds, Steps: 1. | Metric | Value | | --- | --- | -| Text token count | 406 | +| Text token count | 407 | | Text blocks | 1 | @@ -404,7 +405,7 @@ Long running operation completed. Duration: 0 seconds, Steps: 1. "maxTokens": 10 } ``` -**Expected Output (🤖):** A short LLM-generated response, likely 'Paris' or a brief sentence. +**Expected Output (🤖):** Should return a short LLM-generated response, e.g., 'The capital of France is Paris.' **Actual Output (1 blocks):** @@ -435,16 +436,15 @@ LLM sampling result: Dummy content [→ View tool details](#tool-sampleLLM) -**Reasoning (🤖):** Test LLM with a creative prompt and a larger token limit. +**Reasoning (🤖):** Test LLM sampling with a creative prompt and default maxTokens. **Tool Call (🤖):** ```json { - "prompt": "Write a short poem about the sea.", - "maxTokens": 50 + "prompt": "Write a haiku about rain." } ``` -**Expected Output (🤖):** A short poem or several lines of text about the sea. +**Expected Output (🤖):** Should return a short haiku or poetic response about rain. **Actual Output (1 blocks):** @@ -475,13 +475,13 @@ LLM sampling result: Dummy content [→ View tool details](#tool-getTinyImage) -**Reasoning (🤖):** Test retrieval of the MCP_TINY_IMAGE resource. +**Reasoning (🤖):** Test getTinyImage to verify image retrieval functionality. **Tool Call (🤖):** ```json {} ``` -**Expected Output (🤖):** A binary or base64-encoded image object representing the tiny image. +**Expected Output (🤖):** Should return the MCP_TINY_IMAGE (likely as a base64 string or binary blob). **Actual Output (3 blocks):** @@ -514,30 +514,35 @@ The image above is the MCP tiny image. [→ View tool details](#tool-annotatedMessage) -**Reasoning (🤖):** Test annotated message with 'success' type and no image. +**Reasoning (🤖):** Test annotatedMessage with 'success' type and image inclusion to verify annotation and image embedding. **Tool Call (🤖):** ```json { "messageType": "success", - "includeImage": false + "includeImage": true } ``` -**Expected Output (🤖):** A message object with a 'success' annotation and no image included. +**Expected Output (🤖):** Should return a message annotated as 'success' and include an example image. -**Actual Output (1 blocks):** +**Actual Output (2 blocks):** ✅ **No Error** ``` Operation completed successfully ``` +``` +[Image: image/png] + Size: 5380 bytes (base64) +``` **Output Statistics:** | Metric | Value | | --- | --- | | Text token count | 3 | | Text blocks | 1 | +| Image blocks | 1 | @@ -548,35 +553,29 @@ Operation completed successfully [→ View tool details](#tool-annotatedMessage) -**Reasoning (🤖):** Test annotated message with 'error' type and image included to verify both annotation and image embedding. +**Reasoning (🤖):** Test annotatedMessage with 'error' type and default image inclusion (false) to check error annotation. **Tool Call (🤖):** ```json { - "messageType": "error", - "includeImage": true + "messageType": "error" } ``` -**Expected Output (🤖):** A message object with an 'error' annotation and an example image included. +**Expected Output (🤖):** Should return a message annotated as 'error' without an image. -**Actual Output (2 blocks):** +**Actual Output (1 blocks):** ✅ **No Error** ``` Error: Operation failed ``` -``` -[Image: image/png] - Size: 5380 bytes (base64) -``` **Output Statistics:** | Metric | Value | | --- | --- | | Text token count | 4 | | Text blocks | 1 | -| Image blocks | 1 | @@ -587,7 +586,7 @@ Error: Operation failed [→ View tool details](#tool-getResourceReference) -**Reasoning (🤖):** Test resource reference with the minimum valid resource ID. +**Reasoning (🤖):** Test getResourceReference with the minimum valid resourceId to check lower boundary. **Tool Call (🤖):** ```json @@ -595,7 +594,7 @@ Error: Operation failed "resourceId": 1 } ``` -**Expected Output (🤖):** A resource reference object for resource ID 1. +**Expected Output (🤖):** Should return a resource reference object for resourceId 1. **Actual Output (3 blocks):** @@ -630,7 +629,7 @@ You can access this resource using the URI: test://static/resource/1 [→ View tool details](#tool-getResourceReference) -**Reasoning (🤖):** Edge case: Test with the maximum valid resource ID. +**Reasoning (🤖):** Test getResourceReference with the maximum valid resourceId to check upper boundary. **Tool Call (🤖):** ```json @@ -638,7 +637,7 @@ You can access this resource using the URI: test://static/resource/1 "resourceId": 100 } ``` -**Expected Output (🤖):** A resource reference object for resource ID 100. +**Expected Output (🤖):** Should return a resource reference object for resourceId 100. **Actual Output (3 blocks):** @@ -672,45 +671,35 @@ You can access this resource using the URI: test://static/resource/100 [→ View tool details](#tool-getResourceLinks) -**Reasoning (🤖):** Test retrieval of multiple resource links with the default count. +**Reasoning (🤖):** Test getResourceLinks with the minimum count to check lower boundary. **Tool Call (🤖):** ```json { - "count": 3 + "count": 1 } ``` -**Expected Output (🤖):** An array of 3 resource link objects, each referencing a different resource. +**Expected Output (🤖):** Should return a list with one resource link. -**Actual Output (4 blocks):** +**Actual Output (2 blocks):** ✅ **No Error** ``` -Here are 3 resource links to resources available in this server (see full output in tool response if your client does not support resource_link yet): +Here are 1 resource links to resources available in this server (see full output in tool response if your client does not support resource_link yet): ``` ``` [Resource Link: test://static/resource/1] MIME type: text/plain Description: Resource 1: plaintext resource ``` -``` -[Resource Link: test://static/resource/2] - MIME type: application/octet-stream - Description: Resource 2: binary blob resource -``` -``` -[Resource Link: test://static/resource/3] - MIME type: text/plain - Description: Resource 3: plaintext resource -``` **Output Statistics:** | Metric | Value | | --- | --- | | Text token count | 29 | | Text blocks | 1 | -| Resource link blocks | 3 | +| Resource link blocks | 1 | @@ -721,7 +710,7 @@ Here are 3 resource links to resources available in this server (see full output [→ View tool details](#tool-getResourceLinks) -**Reasoning (🤖):** Edge case: Test with the maximum allowed count to verify upper boundary handling. +**Reasoning (🤖):** Test getResourceLinks with the maximum count to check upper boundary. **Tool Call (🤖):** ```json @@ -729,7 +718,7 @@ Here are 3 resource links to resources available in this server (see full output "count": 10 } ``` -**Expected Output (🤖):** An array of 10 resource link objects. +**Expected Output (🤖):** Should return a list with ten resource links. **Actual Output (11 blocks):** @@ -805,7 +794,7 @@ Here are 10 resource links to resources available in this server (see full outpu [→ View tool details](#tool-structuredContent) -**Reasoning (🤖):** Test structured content with a city name to verify normal operation. +**Reasoning (🤖):** Test structuredContent with a city name to verify structured weather data output. **Tool Call (🤖):** ```json @@ -813,7 +802,7 @@ Here are 10 resource links to resources available in this server (see full outpu "location": "San Francisco" } ``` -**Expected Output (🤖):** An object with 'temperature', 'conditions', and 'humidity' fields for San Francisco. +**Expected Output (🤖):** Should return an object with temperature, conditions, and humidity for San Francisco. **Actual Output (1 blocks):** @@ -842,7 +831,7 @@ Here are 10 resource links to resources available in this server (see full outpu [→ View tool details](#tool-structuredContent) -**Reasoning (🤖):** Test structured content with a zip code to verify alternate valid input. +**Reasoning (🤖):** Test structuredContent with a zip code to verify alternate location input. **Tool Call (🤖):** ```json @@ -850,7 +839,7 @@ Here are 10 resource links to resources available in this server (see full outpu "location": "94103" } ``` -**Expected Output (🤖):** An object with 'temperature', 'conditions', and 'humidity' fields for the 94103 zip code. +**Expected Output (🤖):** Should return an object with temperature, conditions, and humidity for zip code 94103. **Actual Output (1 blocks):** @@ -879,13 +868,13 @@ Here are 10 resource links to resources available in this server (see full outpu [→ View tool details](#tool-listRoots) -**Reasoning (🤖):** Test listing of MCP roots to verify protocol support, even if no files are present. +**Reasoning (🤖):** Test listRoots to verify the roots protocol capability, even though no files are accessed. **Tool Call (🤖):** ```json {} ``` -**Expected Output (🤖):** A list (possibly empty) of root objects or a message indicating no roots are available. +**Expected Output (🤖):** Should return a list of current MCP roots (may be empty or contain mock roots). **Actual Output (1 blocks):** @@ -915,13 +904,13 @@ Note: This server demonstrates the roots protocol capability but doesn't actuall [→ View tool details](#tool-startElicitation) -**Reasoning (🤖):** Test the elicitation feature to verify interactive information gathering. +**Reasoning (🤖):** Test startElicitation to verify the elicitation protocol, prompting for user information. **Tool Call (🤖):** ```json {} ``` -**Expected Output (🤖):** A prompt or sequence of prompts asking for favorite color, number, and pets. +**Expected Output (🤖):** Should prompt for favorite color, number, and pets, possibly as a multi-step interaction. **Actual Output (2 blocks):** diff --git a/src/mcp_interviewer/constraints/base.py b/src/mcp_interviewer/constraints/base.py index ba837dc..767cfb9 100644 --- a/src/mcp_interviewer/constraints/base.py +++ b/src/mcp_interviewer/constraints/base.py @@ -11,9 +11,12 @@ from typing import Any from mcp.types import CallToolResult, Tool +from pydantic import FileUrl, HttpUrl from ..models import ServerScoreCard +SourceUrl = HttpUrl | FileUrl + class Severity(StrEnum): """Severity levels for constraint violations. @@ -81,6 +84,18 @@ def cli_code(cls) -> str: """ ... + @classmethod + def sources(cls) -> list[SourceUrl]: + """Return a list of source URLs for documentation or reference. + + Subclasses can override this method to provide relevant documentation + links for the constraint. + + Returns: + list[SourceUrl]: List of HTTP or File URLs (default: empty list) + """ + return [] + @abstractmethod def test( self, server: ServerScoreCard diff --git a/src/mcp_interviewer/constraints/openai.py b/src/mcp_interviewer/constraints/openai.py index d386f9b..0d32c0c 100644 --- a/src/mcp_interviewer/constraints/openai.py +++ b/src/mcp_interviewer/constraints/openai.py @@ -10,12 +10,14 @@ from collections.abc import Generator from mcp.types import CallToolResult, Tool +from pydantic import HttpUrl from .base import ( Constraint, ConstraintViolation, ServerScoreCard, Severity, + SourceUrl, ToolConstraint, ToolResultConstraint, ) @@ -39,6 +41,14 @@ def cli_code(cls) -> str: """Return the shorthand code for this constraint.""" return "OTC" + @classmethod + def sources(cls) -> list[SourceUrl]: + return [ + HttpUrl( + "https://platform.openai.com/docs/guides/function-calling#best-practices-for-defining-functions" + ) + ] + def test( self, server: ServerScoreCard ) -> Generator[ConstraintViolation, None, None]: @@ -80,6 +90,14 @@ def cli_code(cls) -> str: """Return the shorthand code for this constraint.""" return "ONL" + @classmethod + def sources(cls) -> list[SourceUrl]: + return [ + HttpUrl( + "https://github.com/openai/openai-python/blob/e5f93f5daee9f3fc7646833ac235b1693f192a56/src/openai/types/shared_params/function_definition.py#L17-L18" + ) + ] + def test_tool(self, tool: Tool) -> Generator[ConstraintViolation, None, None]: """Test if the tool name meets length requirements. @@ -114,6 +132,14 @@ def cli_code(cls) -> str: """Return the shorthand code for this constraint.""" return "ONP" + @classmethod + def sources(cls) -> list[SourceUrl]: + return [ + HttpUrl( + "https://github.com/openai/openai-python/blob/e5f93f5daee9f3fc7646833ac235b1693f192a56/src/openai/types/shared_params/function_definition.py#L17-L18" + ) + ] + pattern = re.compile(r"^[a-zA-Z_]+[a-zA-Z0-9_]*$") def test_tool(self, tool: Tool) -> Generator[ConstraintViolation, None, None]: @@ -151,6 +177,10 @@ def cli_code(cls) -> str: """Return the shorthand code for this constraint.""" return "OTL" + @classmethod + def sources(cls) -> list[SourceUrl]: + return [HttpUrl("https://platform.openai.com/docs/models/compare")] + def test_tool_result( self, result: CallToolResult ) -> Generator[ConstraintViolation, None, None]: diff --git a/src/mcp_interviewer/constraints/tool_schema_flatness.py b/src/mcp_interviewer/constraints/tool_schema_flatness.py index 7268dc6..e6c4aec 100644 --- a/src/mcp_interviewer/constraints/tool_schema_flatness.py +++ b/src/mcp_interviewer/constraints/tool_schema_flatness.py @@ -3,10 +3,11 @@ from jsonschema import RefResolver from mcp import Tool +from pydantic import HttpUrl from mcp_interviewer.constraints.base import ConstraintViolation, Severity -from .base import ToolConstraint +from .base import SourceUrl, ToolConstraint class ToolInputSchemaFlatnessConstraint(ToolConstraint): @@ -31,6 +32,10 @@ def cli_code(cls) -> str: """Return the shorthand code for this constraint.""" return "TSF" + @classmethod + def sources(cls) -> list[SourceUrl]: + return [HttpUrl("https://composio.dev/blog/gpt-4-function-calling-example")] + def test_tool(self, tool: Tool) -> Generator[ConstraintViolation, None, None]: """Test if the tool's inputSchema has nested properties fields. diff --git a/src/mcp_interviewer/reports/interviewer/constraint_violations.py b/src/mcp_interviewer/reports/interviewer/constraint_violations.py index 1636c90..4a53538 100644 --- a/src/mcp_interviewer/reports/interviewer/constraint_violations.py +++ b/src/mcp_interviewer/reports/interviewer/constraint_violations.py @@ -59,17 +59,27 @@ def _build(self): for constraint, violations in sorted( constraint_to_violations.items(), key=lambda item: item[0].cli_code() ): - constraint_str = f"{constraint.cli_name()} ({constraint.cli_code()})" + constraint_prefix = f"{constraint.cli_name()} ({constraint.cli_code()})" + + constraint_suffix = "" + for i, source in enumerate(constraint.sources()): + constraint_suffix += f"[[{i + 1}]]({source}) " + constraint_suffix = constraint_suffix.strip() + if violations: for violation in violations: if violation.severity == Severity.CRITICAL: - self.add_text(f"❌ {constraint_str}: {violation.message}") + self.add_text( + f"❌ {constraint_prefix}: {violation.message} {constraint_suffix}" + ) elif violation.severity == Severity.WARNING: - self.add_text(f"⚠️ {constraint_str}: {violation.message}") + self.add_text( + f"⚠️ {constraint_prefix}: {violation.message} {constraint_suffix}" + ) self.add_blank_line() else: - self.add_text(f"✅ {constraint_str}") + self.add_text(f"✅ {constraint_prefix} {constraint_suffix}") self.add_blank_line() self.end_collapsible()