test: add very basic test suite for fetch functionality

Author: tgambet

package.json

@@ -14,7 +14,8 @@
     "lint:fix": "eslint src/** --fix",
     "format": "prettier --write \"**/*.{ts,json,md,yml,js}\"",
     "format:check": "prettier --check .",
-    "test": "node --import tsx --test tests/**",
+    "test": "node --import tsx --test tests/**/*.test.ts",
+    "test:watch": "node --import tsx --test --watch tests/**/*.test.ts",
     "inspect": "pnpx @modelcontextprotocol/inspector",
     "fix": "pnpm format && pnpm lint:fix"
   },

src/fetch.prompt.ts

@@ -1,9 +1,8 @@
-import { PromptCallback } from '@modelcontextprotocol/sdk/server/mcp.js';
-import { z } from 'zod';
+import { z, ZodTypeAny } from 'zod';
+import { config } from './config/config.js';
 import { DEFAULT_USER_AGENT_MANUAL } from './constants.js';
 import { cache } from './utils/lru-cache.js';
 import { processURL } from './utils/process-url.js';
-import { config } from './config/config.js';
 
 const name = 'fetch';
 
@@ -13,7 +12,10 @@ const parameters = {
   url: z.string().describe('URL to fetch.'),
 };
 
-const execute: PromptCallback<typeof parameters> = async ({ url }) => {
+type Args = z.objectOutputType<typeof parameters, ZodTypeAny>;
+
+// PromptCallback<typeof parameters>
+const execute = async ({ url }: Args) => {
   const userAgent = config['user-agent'] ?? DEFAULT_USER_AGENT_MANUAL;
 
   const cacheKey = `${url}||${userAgent}||false`;
@@ -37,7 +39,7 @@ const execute: PromptCallback<typeof parameters> = async ({ url }) => {
       {
         role: 'user',
         content: { type: 'text', text: result },
-      },
+      } as const,
     ],
   };
 };

src/fetch.tool.ts

@@ -1,5 +1,4 @@
-import { ToolCallback } from '@modelcontextprotocol/sdk/server/mcp.js';
-import { z } from 'zod';
+import { z, ZodTypeAny } from 'zod';
 import { config } from './config/config.js';
 import { DEFAULT_USER_AGENT_AUTONOMOUS } from './constants.js';
 import { checkRobotsTxt } from './utils/check-robots-txt.js';
@@ -37,9 +36,12 @@ const parameters = {
     ),
 };
 
-const execute: ToolCallback<typeof parameters> =
+type Args = z.objectOutputType<typeof parameters, ZodTypeAny>;
+
+// ToolCallback<typeof parameters>
+const execute =
   // TODO: use signal to handle cancellation
-  async ({ url, max_length, start_index, raw }) => {
+  async ({ url, max_length, start_index, raw }: Args) => {
     const userAgent = config['user-agent'] ?? DEFAULT_USER_AGENT_AUTONOMOUS;
 
     const cacheKey = `${url}||${userAgent}||${raw.toString()}`;
@@ -61,7 +63,7 @@ const execute: ToolCallback<typeof parameters> =
     const result = paginate(url, content, prefix, start_index, max_length);
 
     return {
-      content: [{ type: 'text', text: result }],
+      content: [{ type: 'text' as const, text: result }],
     };
   };
 

src/main.ts

@@ -10,19 +10,19 @@ import { config } from './config/config.js';
 import { fetchPrompt } from './fetch.prompt.js';
 import { fetchTool } from './fetch.tool.js';
 
-const server = new McpServer({
+const mcp = new McpServer({
   name: 'mcp-fetch-node',
   version: '1.x.x',
 });
 
-server.tool(
+mcp.tool(
   fetchTool.name,
   fetchTool.description,
   fetchTool.parameters,
   fetchTool.execute,
 );
 
-server.prompt(
+mcp.prompt(
   fetchPrompt.name,
   fetchPrompt.description,
   fetchPrompt.parameters,
@@ -38,7 +38,7 @@ app.get('/sse', async (_req, res) => {
 
   transports.set(transport.sessionId, transport);
 
-  await server.connect(transport);
+  await mcp.connect(transport);
 
   res.on('close', () => {
     transport.close().catch((err: unknown) => {
@@ -66,7 +66,7 @@ app.post('/messages', async (req, res) => {
   await transport.handlePostMessage(req, res);
 });
 
-const expressServer = app.listen(config.port);
+const server = app.listen(config.port);
 
 console.log(`Server is running on port ${config.port.toString()}`);
 
@@ -76,8 +76,8 @@ await readline.question('Press enter to exit...\n');
 
 readline.close();
 
-expressServer.closeAllConnections();
+server.closeAllConnections();
 
-await promisify(expressServer.close)();
+await promisify(server.close.bind(server))();
 
-await server.close();
+await mcp.close();

src/utils/extract.ts

@@ -48,7 +48,7 @@ export function extract(html: string) {
     const { document } = parseHTML(result);
 
     // Remove unwanted elements
-    document.body
+    document
       .querySelectorAll(
         [
           '[hidden]',
@@ -67,7 +67,7 @@ export function extract(html: string) {
       ?.forEach((a: any) => a.remove());
 
     // Remove nav-liked lists
-    document.querySelectorAll('ul, table, section').forEach((node: any) => {
+    document.querySelectorAll('ul, table').forEach((node: any) => {
       const list = node.cloneNode(true);
       list.querySelectorAll('a').forEach((a: any) => {
         a.innerHTML = '';
@@ -85,7 +85,7 @@ export function extract(html: string) {
     });
 
     // Sanitize again
-    result = sanitizeHtml(document.body.innerHTML as string, {
+    result = sanitizeHtml(document.documentElement.innerHTML as string, {
       allowedAttributes: { a: ['href'] },
     });
 

tests/fetch.prompt.test.ts

@@ -0,0 +1,28 @@
+import assert from 'node:assert';
+import { describe, it } from 'node:test';
+import { fetchPrompt } from '../src/fetch.prompt.js';
+
+describe('Fetch Prompt', () => {
+  it('should fetch content via prompt', async () => {
+    const result = await fetchPrompt.execute({
+      url: 'https://example.com',
+    });
+
+    assert.ok(result.messages);
+    assert.equal(result.messages[0].role, 'user');
+    assert.ok(
+      (result.messages[0].content.text as string).includes('Example Domain'),
+    );
+  });
+
+  it('should handle invalid URLs in prompt', async () => {
+    try {
+      await fetchPrompt.execute({
+        url: 'not-a-valid-url',
+      });
+      assert.fail('Should have thrown an error');
+    } catch (error) {
+      assert.ok(error instanceof Error);
+    }
+  });
+});

tests/fetch.tool.test.ts

@@ -0,0 +1,44 @@
+import assert from 'node:assert';
+import { describe, it } from 'node:test';
+import { fetchTool } from '../src/fetch.tool.js';
+
+describe('Fetch Tool', () => {
+  it('should fetch and return content from a valid URL', async () => {
+    const result = await fetchTool.execute({
+      url: 'https://example.com',
+      max_length: 1000,
+      start_index: 0,
+      raw: false,
+    });
+
+    assert.ok(result.content);
+    assert.equal(result.content[0].type, 'text');
+    assert.ok(result.content[0].text.includes('Example Domain'));
+  });
+
+  it('should respect max_length parameter', async () => {
+    const maxLength = 100;
+    const result = await fetchTool.execute({
+      url: 'https://example.com',
+      max_length: maxLength,
+      start_index: 0,
+      raw: false,
+    });
+
+    assert.ok(result.content[0].text.length <= maxLength + 200); // Adding buffer for prefix text
+  });
+
+  it('should handle invalid URLs', async () => {
+    try {
+      await fetchTool.execute({
+        url: 'not-a-valid-url',
+        max_length: 1000,
+        start_index: 0,
+        raw: false,
+      });
+      assert.fail('Should have thrown an error');
+    } catch (error) {
+      assert.ok(error instanceof Error);
+    }
+  });
+});

tests/index.test.ts

@@ -0,0 +1,54 @@
+import assert from 'node:assert';
+import { describe, it } from 'node:test';
+import { paginate } from '../src/utils/paginate.js';
+import { extract } from '../src/utils/extract.js';
+import { processURL } from '../src/utils/process-url.js';
+
+describe('Utility Functions', () => {
+  describe('paginate', () => {
+    it('should correctly paginate content', () => {
+      const content = 'Hello World!';
+      const result = paginate('test-url', content, '', 0, 5);
+      assert.ok(result.includes('Hello'));
+      assert.ok(result.includes('truncated'));
+    });
+
+    it('should handle start index beyond content length', () => {
+      const content = 'Hello World!';
+      const result = paginate('test-url', content, '', 20, 5);
+      assert.ok(result.includes('No more content available'));
+    });
+  });
+
+  describe('extract', () => {
+    it('should extract content from HTML', () => {
+      const html = '<div><h1>Title</h1><p>Content</p><nav>Menu</nav></div>';
+      const result = extract(html);
+      assert.ok(result.includes('Title'));
+      assert.ok(result.includes('Content'));
+      assert.ok(!result.includes('Menu')); // nav should be removed
+    });
+  });
+
+  describe('processURL', () => {
+    it('should process HTML content', async () => {
+      const [content, prefix] = await processURL(
+        'https://example.com',
+        'test-user-agent',
+        false,
+      );
+      assert.ok(content.length > 0);
+      assert.equal(prefix, '');
+    });
+
+    it('should handle raw content request', async () => {
+      const [content, prefix] = await processURL(
+        'https://example.com',
+        'test-user-agent',
+        true,
+      );
+      assert.ok(content.includes('<html'));
+      assert.ok(prefix.includes('raw'));
+    });
+  });
+});