Skip to content

Commit 926ea89

Browse files
committed
Setup extended integration test playground with real CodeQL databases
Add config-driven extended test runner that validates MCP server tools against real CodeQL databases from GitHub. Infrastructure: - run-extended-tests.ts: 7-scenario test orchestrator with server log capture - download-databases.ts: DB discovery from vscode-codeql storage + GitHub API download - repos.json: 4 test repos (gin-gonic/gin, expressjs/express, checkstyle/checkstyle, PyCQA/flake8) - Three-tier auth: VS Code session -> GH_TOKEN -> gh auth token Test scenarios: - Database resolution and listing - Query execution (BQRS decode, SARIF interpretation) - CallGraphFromTo with external predicates - BQRS info and interpretation tools Build infrastructure: - esbuild config updated for extended test bundling - ESLint config updated for test files - .gitignore updated for downloaded databases Closes #167
1 parent 93d646d commit 926ea89

File tree

7 files changed

+859
-2
lines changed

7 files changed

+859
-2
lines changed

.gitignore

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,9 @@ codeql-development-mcp-server.code-workspace
4747
# The 'codeql test run` command generates `<QueryBaseName>.testproj` test database directories
4848
*.testproj
4949

50+
# CodeQL CLI diagnostic files generated during query runs
51+
**/diagnostic/cli-diagnostics-*.json
52+
5053
# Prevent accidentally committing integration test output files in root directory
5154
# These should only be in client/integration-tests/primitives/tools/*/after/ directories
5255
/evaluator-log.json

extensions/vscode/esbuild.config.js

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,22 @@ const testSuiteConfig = {
5252
},
5353
};
5454

55+
// Extended integration tests — standalone (no vscode API dependency).
56+
// Built separately so they can run via `node` without the Extension Host.
57+
const extendedTestConfig = {
58+
...shared,
59+
entryPoints: [
60+
'test/extended/run-extended-tests.ts',
61+
],
62+
outdir: 'dist/test/extended',
63+
outfile: undefined,
64+
outExtension: { '.js': '.cjs' },
65+
external: [], // No externals — fully self-contained
66+
logOverride: {
67+
'require-resolve-not-external': 'silent',
68+
},
69+
};
70+
5571
const isWatch = process.argv.includes('--watch');
5672

5773
if (isWatch) {
@@ -67,6 +83,10 @@ if (isWatch) {
6783
await build(testSuiteConfig);
6884
console.log('✅ Test suite build completed successfully');
6985
console.log(`📦 Generated: dist/test/suite/*.cjs`);
86+
87+
await build(extendedTestConfig);
88+
console.log('✅ Extended test build completed successfully');
89+
console.log(`📦 Generated: dist/test/extended/*.cjs`);
7090
} catch (error) {
7191
console.error('❌ Build failed:', error);
7292
process.exit(1);

extensions/vscode/eslint.config.mjs

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -46,9 +46,10 @@ export default [
4646
sourceType: 'module',
4747
parser: typescript.parser,
4848
globals: {
49-
process: 'readonly',
50-
console: 'readonly',
5149
Buffer: 'readonly',
50+
console: 'readonly',
51+
fetch: 'readonly',
52+
process: 'readonly',
5253
__dirname: 'readonly',
5354
__filename: 'readonly',
5455
},

extensions/vscode/package.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -155,6 +155,7 @@
155155
"test": "npm run test:coverage && npm run test:integration",
156156
"test:coverage": "vitest --run --coverage",
157157
"test:integration": "npm run download:vscode && vscode-test",
158+
"test:integration:extended": "npm run bundle && node dist/test/extended/run-extended-tests.cjs",
158159
"test:integration:label": "vscode-test --label",
159160
"test:watch": "vitest --watch",
160161
"vscode:prepublish": "npm run clean && npm run lint && npm run bundle && npm run bundle:server",
Lines changed: 258 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,258 @@
1+
/**
2+
* Download CodeQL databases using the GitHub REST API.
3+
*
4+
* When running inside the VS Code Extension Development Host, this uses
5+
* the VS Code GitHub authentication session (same auth as vscode-codeql).
6+
* When running standalone, it falls back to the GH_TOKEN env var.
7+
*
8+
* Downloads are cached on disk and reused if less than 24 hours old.
9+
*/
10+
11+
import { createWriteStream, existsSync, mkdirSync, readFileSync, readdirSync, statSync } from 'fs';
12+
import { join } from 'path';
13+
import { execFileSync } from 'child_process';
14+
import { homedir } from 'os';
15+
import { pipeline } from 'stream/promises';
16+
17+
const MAX_AGE_MS = 24 * 60 * 60 * 1000; // 24 hours
18+
19+
export interface RepoConfig {
20+
callGraphFromTo?: { sourceFunction: string; targetFunction: string };
21+
language: string;
22+
owner: string;
23+
repo: string;
24+
}
25+
26+
/**
27+
* Get a GitHub token. Tries VS Code auth session first, then GH_TOKEN env var,
28+
* then `gh auth token` CLI.
29+
*/
30+
async function getGitHubToken(): Promise<string | undefined> {
31+
// Try VS Code authentication (when running in Extension Host)
32+
try {
33+
const vscode = await import('vscode');
34+
const session = await vscode.authentication.getSession('github', ['repo'], { createIfNone: false });
35+
if (session?.accessToken) {
36+
console.log(' 🔑 Using VS Code GitHub authentication');
37+
return session.accessToken;
38+
}
39+
} catch {
40+
// Not in VS Code — fall through
41+
}
42+
43+
// Try GH_TOKEN env var
44+
if (process.env.GH_TOKEN) {
45+
console.log(' 🔑 Using GH_TOKEN environment variable');
46+
return process.env.GH_TOKEN;
47+
}
48+
49+
// Try `gh auth token` CLI
50+
try {
51+
const { execFileSync } = await import('child_process');
52+
const token = execFileSync('gh', ['auth', 'token'], { encoding: 'utf8', timeout: 5000 }).trim();
53+
if (token) {
54+
console.log(' 🔑 Using GitHub CLI (gh auth token)');
55+
return token;
56+
}
57+
} catch {
58+
// gh CLI not available or not authenticated
59+
}
60+
61+
return undefined;
62+
}
63+
64+
/**
65+
* Download a CodeQL database for a repository via GitHub REST API.
66+
* Returns the path to the extracted database, or null if download failed.
67+
*/
68+
async function downloadDatabase(
69+
repo: RepoConfig,
70+
databaseDir: string,
71+
token: string,
72+
): Promise<string | null> {
73+
const { language, owner, repo: repoName } = repo;
74+
const repoDir = join(databaseDir, owner, repoName);
75+
const dbDir = join(repoDir, language);
76+
const zipPath = join(repoDir, `${language}.zip`);
77+
const markerFile = join(dbDir, 'codeql-database.yml');
78+
79+
// Check cache
80+
if (existsSync(markerFile)) {
81+
try {
82+
const mtime = statSync(markerFile).mtimeMs;
83+
if (Date.now() - mtime < MAX_AGE_MS) {
84+
console.log(` ✓ Cached: ${owner}/${repoName} (${language})`);
85+
return dbDir;
86+
}
87+
} catch {
88+
// Fall through to download
89+
}
90+
}
91+
92+
mkdirSync(repoDir, { recursive: true });
93+
94+
const url = `https://api.github.com/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repoName)}/code-scanning/codeql/databases/${encodeURIComponent(language)}`;
95+
console.log(` ⬇ Downloading: ${owner}/${repoName} (${language})...`);
96+
97+
try {
98+
const response = await fetch(url, {
99+
headers: {
100+
Accept: 'application/zip',
101+
Authorization: `Bearer ${token}`,
102+
'User-Agent': 'codeql-development-mcp-server-extended-tests',
103+
},
104+
});
105+
106+
if (!response.ok) {
107+
console.error(` ✗ Download failed: ${response.status} ${response.statusText}`);
108+
return null;
109+
}
110+
111+
if (!response.body) {
112+
console.error(` ✗ Empty response body`);
113+
return null;
114+
}
115+
116+
// Stream to zip file
117+
const dest = createWriteStream(zipPath);
118+
// @ts-expect-error — ReadableStream → NodeJS.ReadableStream interop
119+
await pipeline(response.body, dest);
120+
121+
// Extract
122+
console.log(` 📦 Extracting: ${owner}/${repoName} (${language})...`);
123+
mkdirSync(dbDir, { recursive: true });
124+
execFileSync('unzip', ['-o', '-q', zipPath, '-d', dbDir]);
125+
126+
// Flatten if single nested directory (zip often has one top-level dir)
127+
const entries = readdirSync(dbDir);
128+
if (entries.length === 1 && !existsSync(join(dbDir, 'codeql-database.yml'))) {
129+
const nested = join(dbDir, entries[0]);
130+
if (existsSync(join(nested, 'codeql-database.yml'))) {
131+
// Copy all contents up, then remove the nested directory
132+
execFileSync('bash', ['-c', `cp -a "${nested}"/. "${dbDir}/" && rm -rf "${nested}"`]);
133+
}
134+
}
135+
136+
if (!existsSync(markerFile)) {
137+
console.error(` ✗ Extraction failed: ${markerFile} not found`);
138+
return null;
139+
}
140+
141+
// Clean up zip
142+
try { const { unlinkSync } = await import('fs'); unlinkSync(zipPath); } catch { /* best effort */ }
143+
144+
console.log(` ✓ Ready: ${owner}/${repoName} (${language})`);
145+
return dbDir;
146+
} catch (err) {
147+
console.error(` ✗ Error downloading ${owner}/${repoName}: ${err}`);
148+
return null;
149+
}
150+
}
151+
152+
/**
153+
* Get the default vscode-codeql global storage paths (platform-dependent).
154+
*/
155+
function getVscodeCodeqlStoragePaths(): string[] {
156+
const home = homedir();
157+
const candidates = [
158+
join(home, 'Library', 'Application Support', 'Code', 'User', 'globalStorage', 'GitHub.vscode-codeql'),
159+
join(home, 'Library', 'Application Support', 'Code', 'User', 'globalStorage', 'github.vscode-codeql'),
160+
join(home, '.config', 'Code', 'User', 'globalStorage', 'GitHub.vscode-codeql'),
161+
join(home, '.config', 'Code', 'User', 'globalStorage', 'github.vscode-codeql'),
162+
];
163+
return candidates.filter(p => existsSync(p));
164+
}
165+
166+
/**
167+
* Scan directories for CodeQL databases (by codeql-database.yml marker).
168+
*/
169+
function scanForDatabases(dir: string, found: Map<string, { language: string; path: string }>, depth: number): void {
170+
if (depth > 4) return;
171+
const markerPath = join(dir, 'codeql-database.yml');
172+
if (existsSync(markerPath)) {
173+
try {
174+
const yml = readFileSync(markerPath, 'utf8');
175+
const langMatch = yml.match(/primaryLanguage:\s*(\S+)/);
176+
found.set(dir, { language: langMatch?.[1] ?? 'unknown', path: dir });
177+
} catch { /* skip */ }
178+
return;
179+
}
180+
try {
181+
for (const entry of readdirSync(dir)) {
182+
if (entry.startsWith('.') || entry === 'node_modules') continue;
183+
const full = join(dir, entry);
184+
try { if (statSync(full).isDirectory()) scanForDatabases(full, found, depth + 1); } catch { /* skip */ }
185+
}
186+
} catch { /* skip */ }
187+
}
188+
189+
/**
190+
* Discover and/or download databases for the requested repos.
191+
* Returns a map of "owner/repo" → database path.
192+
*/
193+
export async function resolveAllDatabases(
194+
repos: RepoConfig[],
195+
additionalDirs: string[],
196+
): Promise<{ databases: Map<string, string>; missing: RepoConfig[] }> {
197+
const databases = new Map<string, string>();
198+
const missing: RepoConfig[] = [];
199+
200+
// First: discover existing databases on disk
201+
const searchDirs = [...additionalDirs, ...getVscodeCodeqlStoragePaths()];
202+
const envDirs = process.env.CODEQL_DATABASES_BASE_DIRS;
203+
if (envDirs) searchDirs.push(...envDirs.split(':').filter(Boolean));
204+
205+
console.log(` Searching ${searchDirs.length} directories for existing databases...`);
206+
const existing = new Map<string, { language: string; path: string }>();
207+
for (const dir of searchDirs) {
208+
if (existsSync(dir)) scanForDatabases(dir, existing, 0);
209+
}
210+
console.log(` Found ${existing.size} existing database(s) on disk`);
211+
212+
// Match existing databases to requested repos
213+
for (const repo of repos) {
214+
let found = false;
215+
for (const [dbPath, info] of existing) {
216+
if (info.language === repo.language) {
217+
const pathLower = dbPath.toLowerCase();
218+
if (pathLower.includes(repo.repo.toLowerCase()) || pathLower.includes(repo.owner.toLowerCase())) {
219+
databases.set(`${repo.owner}/${repo.repo}`, dbPath);
220+
found = true;
221+
console.log(` ✓ Found: ${repo.owner}/${repo.repo}${dbPath}`);
222+
break;
223+
}
224+
}
225+
}
226+
if (!found) missing.push(repo);
227+
}
228+
229+
// Second: try to download missing databases
230+
if (missing.length > 0) {
231+
const token = await getGitHubToken();
232+
if (token) {
233+
console.log(`\n ⬇ Attempting to download ${missing.length} missing database(s)...`);
234+
const downloadDir = additionalDirs[0] || join(homedir(), '.codeql-mcp-test-databases');
235+
mkdirSync(downloadDir, { recursive: true });
236+
237+
const stillMissing: RepoConfig[] = [];
238+
for (const repo of missing) {
239+
const dbPath = await downloadDatabase(repo, downloadDir, token);
240+
if (dbPath) {
241+
databases.set(`${repo.owner}/${repo.repo}`, dbPath);
242+
} else {
243+
stillMissing.push(repo);
244+
}
245+
}
246+
return { databases, missing: stillMissing };
247+
} else {
248+
console.log(`\n ⚠️ No GitHub token available for downloading missing databases.`);
249+
console.log(` 💡 Options to provide databases:`);
250+
console.log(` 1. Open VS Code, use "CodeQL: Download Database from GitHub"`);
251+
console.log(` 2. Set GH_TOKEN env var for automatic download`);
252+
console.log(` 3. Set CODEQL_DATABASES_BASE_DIRS to point to existing databases`);
253+
}
254+
}
255+
256+
return { databases, missing };
257+
}
258+
Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
{
2+
"description": "Repositories for extended MCP integration testing with CallGraphFromTo source/target function pairs.",
3+
"repositories": [
4+
{
5+
"owner": "gin-gonic",
6+
"repo": "gin",
7+
"language": "go",
8+
"callGraphFromTo": { "sourceFunction": "handleHTTPRequest", "targetFunction": "ServeHTTP" }
9+
},
10+
{
11+
"owner": "expressjs",
12+
"repo": "express",
13+
"language": "javascript",
14+
"callGraphFromTo": { "sourceFunction": "json", "targetFunction": "send" }
15+
},
16+
{
17+
"owner": "checkstyle",
18+
"repo": "checkstyle",
19+
"language": "java",
20+
"callGraphFromTo": { "sourceFunction": "process", "targetFunction": "log" }
21+
},
22+
{
23+
"owner": "PyCQA",
24+
"repo": "flake8",
25+
"language": "python",
26+
"callGraphFromTo": { "sourceFunction": "run", "targetFunction": "report" }
27+
}
28+
],
29+
"settings": {
30+
"databaseDir": ".tmp/extended-test-databases",
31+
"fixtureSearchDirs": [
32+
"test/fixtures/single-folder-workspace/codeql-storage/databases",
33+
"test/fixtures/multi-root-workspace/folder-a/codeql-storage/databases"
34+
],
35+
"timeoutMs": 600000
36+
}
37+
}

0 commit comments

Comments
 (0)