Skip to content

Commit 50150ef

Browse files
Copilotdata-douser
andcommitted
Add extension integration test for MCP prompt error handling with invalid paths
Add mcp-prompt-e2e.integration.test.ts to the VS Code extension test suite that spawns the real MCP server inside the Extension Development Host and verifies that prompts return user-friendly warnings (not protocol errors) when given invalid file paths: - explain_codeql_query with nonexistent relative path returns "does not exist" - explain_codeql_query with valid absolute path returns no warning - document_codeql_query with nonexistent path returns "does not exist" - Server lists prompts including explain_codeql_query Register the new test in esbuild.config.js so it is bundled for CI. Co-authored-by: data-douser <70299490+data-douser@users.noreply.github.com>
1 parent 488c575 commit 50150ef

File tree

2 files changed

+184
-0
lines changed

2 files changed

+184
-0
lines changed

extensions/vscode/esbuild.config.js

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ const testSuiteConfig = {
3737
'test/suite/bridge.integration.test.ts',
3838
'test/suite/copydb-e2e.integration.test.ts',
3939
'test/suite/extension.integration.test.ts',
40+
'test/suite/mcp-prompt-e2e.integration.test.ts',
4041
'test/suite/mcp-resource-e2e.integration.test.ts',
4142
'test/suite/mcp-server.integration.test.ts',
4243
'test/suite/mcp-tool-e2e.integration.test.ts',
Lines changed: 183 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,183 @@
1+
/**
2+
* End-to-end integration tests for MCP server prompt error handling.
3+
*
4+
* These run inside the Extension Development Host with the REAL VS Code API.
5+
* They spawn the actual `ql-mcp` server process, connect via
6+
* StdioClientTransport, and invoke prompts with invalid file paths to verify
7+
* that the server returns user-friendly warnings instead of throwing raw MCP
8+
* protocol errors.
9+
*
10+
* This test suite exists to catch the class of bugs where:
11+
* - A relative or nonexistent `queryPath` triggers a cryptic -32001 error
12+
* - Invalid paths propagate silently into the LLM context without any warning
13+
* - Path traversal attempts are not detected
14+
*/
15+
16+
import * as assert from 'assert';
17+
import * as fs from 'fs';
18+
import * as path from 'path';
19+
import * as vscode from 'vscode';
20+
import { Client } from '@modelcontextprotocol/sdk/client/index.js';
21+
import { StdioClientTransport } from '@modelcontextprotocol/sdk/client/stdio.js';
22+
23+
const EXTENSION_ID = 'advanced-security.vscode-codeql-development-mcp-server';
24+
25+
/**
26+
* Resolve the MCP server entry point.
27+
*/
28+
function resolveServerPath(): string {
29+
const extPath = vscode.extensions.getExtension(EXTENSION_ID)?.extensionUri.fsPath;
30+
if (!extPath) throw new Error('Extension not found');
31+
32+
const monorepo = path.resolve(extPath, '..', '..', 'server', 'dist', 'codeql-development-mcp-server.js');
33+
try {
34+
fs.accessSync(monorepo);
35+
return monorepo;
36+
} catch {
37+
// Fall through
38+
}
39+
40+
const vsix = path.resolve(extPath, 'server', 'dist', 'codeql-development-mcp-server.js');
41+
try {
42+
fs.accessSync(vsix);
43+
return vsix;
44+
} catch {
45+
throw new Error(`MCP server not found at ${monorepo} or ${vsix}`);
46+
}
47+
}
48+
49+
suite('MCP Prompt Error Handling Integration Tests', () => {
50+
let client: Client;
51+
let transport: StdioClientTransport;
52+
53+
suiteSetup(async function () {
54+
this.timeout(30_000);
55+
56+
const ext = vscode.extensions.getExtension(EXTENSION_ID);
57+
assert.ok(ext, `Extension ${EXTENSION_ID} not found`);
58+
if (!ext.isActive) await ext.activate();
59+
60+
const serverPath = resolveServerPath();
61+
62+
const env: Record<string, string> = {
63+
...process.env as Record<string, string>,
64+
TRANSPORT_MODE: 'stdio',
65+
};
66+
67+
transport = new StdioClientTransport({
68+
command: 'node',
69+
args: [serverPath],
70+
env,
71+
stderr: 'pipe',
72+
});
73+
74+
client = new Client({ name: 'prompt-e2e-test', version: '1.0.0' });
75+
await client.connect(transport);
76+
console.log('[mcp-prompt-e2e] Connected to MCP server');
77+
});
78+
79+
suiteTeardown(async function () {
80+
this.timeout(10_000);
81+
try { if (client) await client.close(); } catch { /* best-effort */ }
82+
try { if (transport) await transport.close(); } catch { /* best-effort */ }
83+
});
84+
85+
test('Server should list prompts including explain_codeql_query', async function () {
86+
this.timeout(15_000);
87+
88+
const response = await client.listPrompts();
89+
assert.ok(response.prompts, 'Server should return prompts');
90+
assert.ok(response.prompts.length > 0, 'Server should have at least one prompt');
91+
92+
const names = response.prompts.map(p => p.name);
93+
assert.ok(
94+
names.includes('explain_codeql_query'),
95+
`Prompts should include explain_codeql_query. Found: ${names.join(', ')}`,
96+
);
97+
98+
console.log(`[mcp-prompt-e2e] Server provides ${response.prompts.length} prompts`);
99+
});
100+
101+
test('explain_codeql_query with nonexistent relative path should return warning, not throw', async function () {
102+
this.timeout(15_000);
103+
104+
// This simulates a user entering a relative path in the VS Code slash
105+
// command input that does not exist on disk.
106+
const result = await client.getPrompt({
107+
name: 'explain_codeql_query',
108+
arguments: {
109+
queryPath: 'nonexistent/path/to/query.ql',
110+
language: 'javascript',
111+
},
112+
});
113+
114+
// The prompt MUST return messages — not throw a protocol error.
115+
assert.ok(result.messages, 'Prompt should return messages');
116+
assert.ok(result.messages.length > 0, 'Prompt should return at least one message');
117+
118+
const text = result.messages[0]?.content as unknown as { type: string; text: string };
119+
assert.ok(text?.text, 'First message should have text content');
120+
121+
// The response should contain a user-friendly warning about the invalid path.
122+
assert.ok(
123+
text.text.includes('does not exist'),
124+
`Response should warn that the path does not exist. Got:\n${text.text.slice(0, 500)}`,
125+
);
126+
127+
console.log('[mcp-prompt-e2e] explain_codeql_query correctly returned warning for nonexistent path');
128+
});
129+
130+
test('explain_codeql_query with valid absolute path should not include a warning', async function () {
131+
this.timeout(15_000);
132+
133+
// Use this very test file as a known-existing absolute path.
134+
const extPath = vscode.extensions.getExtension(EXTENSION_ID)?.extensionUri.fsPath;
135+
assert.ok(extPath, 'Extension path should be available');
136+
const existingFile = path.resolve(extPath, 'package.json');
137+
138+
const result = await client.getPrompt({
139+
name: 'explain_codeql_query',
140+
arguments: {
141+
queryPath: existingFile,
142+
language: 'javascript',
143+
},
144+
});
145+
146+
assert.ok(result.messages, 'Prompt should return messages');
147+
assert.ok(result.messages.length > 0, 'Prompt should return at least one message');
148+
149+
const text = result.messages[0]?.content as unknown as { type: string; text: string };
150+
assert.ok(text?.text, 'First message should have text content');
151+
152+
// With a valid existing path, there should be no warning.
153+
assert.ok(
154+
!text.text.includes('does not exist'),
155+
`Response should NOT contain a "does not exist" warning for valid path. Got:\n${text.text.slice(0, 500)}`,
156+
);
157+
158+
console.log('[mcp-prompt-e2e] explain_codeql_query returned clean response for valid path');
159+
});
160+
161+
test('document_codeql_query with nonexistent path should return warning, not throw', async function () {
162+
this.timeout(15_000);
163+
164+
const result = await client.getPrompt({
165+
name: 'document_codeql_query',
166+
arguments: {
167+
queryPath: 'does-not-exist/MyQuery.ql',
168+
language: 'python',
169+
},
170+
});
171+
172+
assert.ok(result.messages, 'Prompt should return messages');
173+
assert.ok(result.messages.length > 0, 'Prompt should return at least one message');
174+
175+
const text = result.messages[0]?.content as unknown as { type: string; text: string };
176+
assert.ok(
177+
text?.text?.includes('does not exist'),
178+
`Response should warn that the path does not exist. Got:\n${(text?.text ?? '').slice(0, 500)}`,
179+
);
180+
181+
console.log('[mcp-prompt-e2e] document_codeql_query correctly returned warning for nonexistent path');
182+
});
183+
});

0 commit comments

Comments
 (0)