Fixes and integration tests for MCP-provided prompts and VSIX bundle

.github/workflows/build-extension.yml → .github/workflows/build-and-test-extension.yml

@@ -1,73 +1,85 @@
-name: Build Extension - CodeQL Development MCP Server
+name: Build and Test Extension - CodeQL Development MCP Server
 
 on:
   pull_request:
     branches: ['main']
     paths:
-      - '.github/workflows/build-extension.yml'
+      - '.github/workflows/build-and-test-extension.yml'
       - '.node-version'
       - 'extensions/vscode/**'
       - 'server/dist/**'
       - 'server/ql/*/tools/src/**'
+      - 'server/src/**'
   push:
     branches: ['main']
     paths:
-      - '.github/workflows/build-extension.yml'
+      - '.github/workflows/build-and-test-extension.yml'
       - '.node-version'
       - 'extensions/vscode/**'
       - 'server/dist/**'
       - 'server/ql/*/tools/src/**'
+      - 'server/src/**'
   workflow_dispatch:
 
 permissions:
   contents: read
 
 jobs:
-  build-extension:
-    name: Build Extension
+  build-and-test-extension:
+    name: Build and Test Extension
     runs-on: ubuntu-latest
 
     steps:
-      - name: Build Extension - Checkout repository
+      - name: Checkout repository
         uses: actions/checkout@v6
 
-      - name: Build Extension - Setup Node.js environment
+      - name: Setup Node.js environment
         uses: actions/setup-node@v6
         with:
           cache: 'npm'
           node-version-file: '.node-version'
 
-      - name: Build Extension - Install dependencies
+      - name: Install dependencies
         run: npm ci --include=optional
 
-      - name: Build Extension - Build server (dependency)
+      - name: Build server (dependency)
         run: npm run build -w server
 
-      - name: Build Extension - Run extension tests with coverage
+      - name: Run extension unit tests with coverage
         working-directory: extensions/vscode
         run: npm run test:coverage
 
-      - name: Build Extension - Bundle extension and server
+      - name: Bundle extension and server
         working-directory: extensions/vscode
         run: |
           npm run clean
           npm run lint
           npm run bundle
           npm run bundle:server
 
-      - name: Build Extension - Verify VSIX packaging
+      - name: Setup CodeQL environment for integration tests
+        uses: ./.github/actions/setup-codeql-environment
+        with:
+          add-to-path: 'true'
+          install-language-runtimes: 'false'
+
+      - name: Run Extension Host integration tests
+        working-directory: extensions/vscode
+        run: xvfb-run -a npm run test:integration
+
+      - name: Verify VSIX packaging
         working-directory: extensions/vscode
         run: npx @vscode/vsce package --no-dependencies --out codeql-development-mcp-server.vsix
 
-      - name: Build Extension - Verify VSIX contents
+      - name: Verify VSIX contents
         working-directory: extensions/vscode
         run: |
           echo "## VSIX Contents" >> $GITHUB_STEP_SUMMARY
           echo '```' >> $GITHUB_STEP_SUMMARY
           npx @vscode/vsce ls --no-dependencies --tree 2>&1 | head -50 >> $GITHUB_STEP_SUMMARY
           echo '```' >> $GITHUB_STEP_SUMMARY
 
-      - name: Build Extension - Check for uncommitted changes
+      - name: Check for uncommitted changes
         run: |
           if [ -n "$(git status --porcelain)" ]; then
             echo "❌ Uncommitted changes detected after build:"
@@ -78,12 +90,13 @@ jobs:
             echo "✅ No uncommitted changes after build"
           fi
 
-      - name: Build Extension - Summary
+      - name: Summary
         run: |
-          echo "## Build Extension Summary" >> $GITHUB_STEP_SUMMARY
+          echo "## Build and Test Extension Summary" >> $GITHUB_STEP_SUMMARY
           echo "✅ ESLint checks completed" >> $GITHUB_STEP_SUMMARY
-          echo "✅ All tests passed with coverage" >> $GITHUB_STEP_SUMMARY
+          echo "✅ Unit tests passed with coverage" >> $GITHUB_STEP_SUMMARY
           echo "✅ Extension bundled successfully" >> $GITHUB_STEP_SUMMARY
           echo "✅ Server bundled into extension" >> $GITHUB_STEP_SUMMARY
+          echo "✅ Extension Host integration tests passed" >> $GITHUB_STEP_SUMMARY
           echo "✅ VSIX packaging verified" >> $GITHUB_STEP_SUMMARY
           echo "✅ No uncommitted changes detected" >> $GITHUB_STEP_SUMMARY

.gitignore

@@ -19,6 +19,8 @@ codeql-development-mcp-server.code-workspace
 
 ## Globs to ignore
 *.log
+# Allow *.log files committed as test fixtures
+!extensions/vscode/test/fixtures/**/*.log
 *.pid
 *.swp
 *.swo

docs/getting-started.md

@@ -18,7 +18,7 @@ See the [VS Code Extension guide](./vscode/extension.md) for details.
 
 The `.vsix` can be downloaded from
 [GitHub Releases](https://github.com/advanced-security/codeql-development-mcp-server/releases)
-or built from source (`npm run package:vscode` at the repository root).
+or built from source (`npm run package:vsix` at the repository root).
 
 ### From npm
 

docs/testing.md

@@ -4,25 +4,40 @@
 
 ## Overview
 
-Every release of the CodeQL Development MCP Server is validated through three progressively broader layers of testing. Each layer builds on the previous one, moving from fast unit checks to real-world agentic validation.
+Every release of the CodeQL Development MCP Server is validated through four progressively broader layers of testing. Each layer builds on the previous one, moving from fast unit checks to real-world agentic validation.
 
-| Layer                  | Scope                                           | Framework                      | Location                      |
-| ---------------------- | ----------------------------------------------- | ------------------------------ | ----------------------------- |
-| 1 — Unit tests         | TypeScript source code                          | Vitest                         | `server/test/**/*.test.ts`    |
-| 2 — Integration tests  | Individual MCP tools against a running server   | Custom MCP client              | `client/integration-tests/**` |
-| 3 — Agentic validation | End-to-end CodeQL workflows driven by AI agents | GitHub Copilot agents + skills | `.github/{agents,skills}/**`  |
+| Layer                            | Scope                                           | Framework                      | Location                                    |
+| -------------------------------- | ----------------------------------------------- | ------------------------------ | ------------------------------------------- |
+| 1a — Server unit tests           | MCP server TypeScript source code               | Vitest                         | `server/test/**/*.test.ts`                  |
+| 1b — Extension unit tests        | VS Code extension TypeScript source code        | Vitest                         | `extensions/vscode/test/**/*.test.ts`       |
+| 2a — MCP tool integration tests  | Individual MCP tools against a running server   | Custom MCP client              | `client/integration-tests/**`               |
+| 2b — Extension integration tests | Extension behaviour inside a VS Code host       | @vscode/test-cli + Mocha       | `extensions/vscode/test/suite/**/*.test.ts` |
+| 3 — Agentic validation           | End-to-end CodeQL workflows driven by AI agents | GitHub Copilot agents + skills | `.github/{agents,skills}/**`                |
 
 ## Layer 1 — Unit Tests
 
+### 1a — Server unit tests
+
 Unit tests verify the TypeScript implementation of the MCP server.
 
 - **Framework**: [Vitest](https://vitest.dev/) with code coverage via `@vitest/coverage-v8`.
 - **Location**: `server/test/**/*.test.ts`, mirroring `server/src/**/*.ts`.
 - **Run command**: `npm run test -w server` (or `npm run test:server` from the repo root).
 - **Conventions**: Tests follow the Arrange-Act-Assert (AAA) pattern, use descriptive names, and mock external dependencies where necessary.
 
+### 1b — Extension unit tests
+
+Unit tests verify the VS Code extension's TypeScript code outside of the Extension Host (no VS Code API dependency).
+
+- **Framework**: [Vitest](https://vitest.dev/) with code coverage via `@vitest/coverage-v8`.
+- **Location**: `extensions/vscode/test/**/*.test.ts` (excluding `test/suite/`).
+- **Run command**: `npm run test:coverage -w extensions/vscode` (or `npm run test:vscode` from the repo root, which also runs integration tests).
+- **Conventions**: Same AAA pattern and naming conventions as the server unit tests.
+
 ## Layer 2 — Integration Tests
 
+### 2a — MCP tool integration tests
+
 Integration tests exercise individual MCP tools against a live server instance using the custom MCP client.
 
 - **Client**: `client/src/ql-mcp-client.js` — starts the MCP server, invokes tools, and validates results.
@@ -33,6 +48,22 @@ Integration tests exercise individual MCP tools against a live server instance u
   - No mocks — tests use real CodeQL databases and queries bundled under `server/ql/`.
   - The `before/monitoring-state.json` file supplies tool arguments. For file-based tests, the integration-test runner diffs filesystem state from `before/` to `after/`; for monitoring-based tests, `after/` artifacts are generally not diffed and are only interpreted for specific validations (for example, `codeql_query_run` interpreted output).
 
+### 2b — Extension integration tests
+
+Extension integration tests run inside a real VS Code Extension Development Host with the full VS Code API. They validate activation, MCP server registration, CodeQL extension bridging, and workspace-aware scenarios.
+
+- **Framework**: [@vscode/test-cli](https://github.com/nicolo-ribaudo/vscode-test-cli) + [Mocha](https://mochajs.org/) (TDD UI).
+- **Location**: `extensions/vscode/test/suite/**/*.integration.test.ts` (compiled to `dist/test/suite/*.test.cjs` by esbuild).
+- **Configuration**: `extensions/vscode/.vscode-test.mjs` — defines three test profiles:
+  - `noWorkspace` — no workspace folder open
+  - `singleFolder` — single-folder workspace fixture
+  - `multiRoot` — multi-root workspace fixture
+- **Run commands**:
+  - `npm run test:integration -w extensions/vscode` — run all profiles
+  - `npm run test:integration:label -w extensions/vscode -- noWorkspace` — run a single profile
+- **Prerequisites**: `npm run bundle -w extensions/vscode` must be run first to compile the extension and test suite.
+- **CI**: Runs under `xvfb-run` on Linux for headless display support (see `.github/workflows/build-and-test-extension.yml`).
+
 ## Layer 3 — Agentic Validation
 
 Agentic validation uses GitHub Copilot agents and skills to solve CodeQL query development challenges, exercising MCP tools in realistic end-to-end scenarios.
@@ -49,14 +80,20 @@ Agentic validation uses GitHub Copilot agents and skills to solve CodeQL query d
 From the repository root:
 
 ```bash
-# Build everything and run all layers (unit + integration)
+# Build everything and run all layers (1a + 1b + 2a + 2b)
 npm run build-and-test
 
-# Run only unit tests
+# Run only server unit tests (1a)
 npm run test:server
 
-# Run only integration tests
+# Run extension unit tests + integration tests (1b + 2b)
+npm run test:vscode
+
+# Run only MCP tool integration tests (2a)
 npm run test:client
+
+# Run only extension integration tests (2b)
+npm run test:integration -w extensions/vscode
 ```
 
-> **Note**: Layer 3 (agentic validation) runs as part of the CI/CD pipeline via GitHub Actions workflows and is not included in the local `build-and-test` command.
+> **Note**: `npm run test:vscode` runs `npm test -w extensions/vscode`, which executes both Vitest unit tests (`test:coverage`) and Extension Host integration tests (`test:integration`) in sequence. Layer 3 (agentic validation) runs as part of the CI/CD pipeline via GitHub Actions workflows and is not included in the local `build-and-test` command.

docs/vscode/extension.md

@@ -39,7 +39,7 @@ Or in VS Code: **Extensions** sidebar → `⋯` menu → **Install from VSIX…*
 From the repository root:
 
 ```bash
-npm run package:vscode
+npm run package:vsix
 code --install-extension extensions/vscode/codeql-development-mcp-server.vsix
 ```
 

extensions/vscode/.gitignore

@@ -3,3 +3,6 @@ dist/
 server/
 coverage/
 *.vsix
+
+# @vscode/test-cli downloads and runtime data
+.vscode-test/

extensions/vscode/.vscode-test.mjs

@@ -0,0 +1,51 @@
+/**
+ * Configuration for @vscode/test-cli.
+ *
+ * Defines integration test profiles that run inside a VS Code Extension
+ * Development Host with the REAL VS Code API. The compiled Mocha test
+ * suite (dist/test/suite/*.cjs) is discovered by the runner in index.ts.
+ *
+ * Run all profiles:  npx vscode-test
+ * Run one profile:   npx vscode-test --label noWorkspace
+ *
+ * Prerequisites:
+ *   npm run bundle          # builds extension + test suite
+ *   npm run bundle:server   # bundles MCP server for e2e tests
+ */
+
+import { defineConfig } from '@vscode/test-cli';
+
+export default defineConfig([
+  {
+    label: 'noWorkspace',
+    files: 'dist/test/suite/*.test.cjs',
+    version: 'stable',
+    mocha: {
+      ui: 'tdd',
+      color: true,
+      timeout: 60_000,
+    },
+  },
+  {
+    label: 'singleFolder',
+    files: 'dist/test/suite/*.test.cjs',
+    version: 'stable',
+    workspaceFolder: './test/fixtures/single-folder-workspace',
+    mocha: {
+      ui: 'tdd',
+      color: true,
+      timeout: 60_000,
+    },
+  },
+  {
+    label: 'multiRoot',
+    files: 'dist/test/suite/*.test.cjs',
+    version: 'stable',
+    workspaceFolder: './test/fixtures/multi-root-workspace/test.code-workspace',
+    mocha: {
+      ui: 'tdd',
+      color: true,
+      timeout: 60_000,
+    },
+  },
+]);

extensions/vscode/.vscodeignore

@@ -1,6 +1,7 @@
 # Source and config (not needed at runtime)
 .vscode/**
 .vscode-test/**
+.vscode-test.mjs
 src/**
 test/**
 scripts/**

extensions/vscode/package.json

@@ -1,7 +1,7 @@
 {
   "name": "vscode-codeql-development-mcp-server",
   "displayName": "CodeQL Development MCP Server",
-  "description": "Automatically installs, configures, and manages the CodeQL Development MCP Server in VS Code.",
+  "description": "LLM-assisted development of CodeQL queries, libraries, and tests via #ql-mcp prompts, resources, and tools.",
   "version": "2.24.1",
   "publisher": "advanced-security",
   "license": "SEE LICENSE IN LICENSE",
@@ -65,7 +65,9 @@
         },
         "codeql-mcp.serverArgs": {
           "type": "array",
-          "items": { "type": "string" },
+          "items": {
+            "type": "string"
+          },
           "default": [],
           "description": "Custom arguments for the MCP server command. When empty, the bundled server entry point is used automatically. Set to e.g. ['/path/to/server/dist/codeql-development-mcp-server.js'] for local development."
         },
@@ -84,19 +86,25 @@
         },
         "codeql-mcp.additionalDatabaseDirs": {
           "type": "array",
-          "items": { "type": "string" },
+          "items": {
+            "type": "string"
+          },
           "default": [],
           "description": "Additional directories to search for CodeQL databases. Appended to CODEQL_DATABASES_BASE_DIRS alongside the vscode-codeql storage paths."
         },
         "codeql-mcp.additionalMrvaRunResultsDirs": {
           "type": "array",
-          "items": { "type": "string" },
+          "items": {
+            "type": "string"
+          },
           "default": [],
           "description": "Additional directories containing MRVA run result subdirectories. Appended to CODEQL_MRVA_RUN_RESULTS_DIRS alongside the vscode-codeql variant analysis storage path."
         },
         "codeql-mcp.additionalQueryRunResultsDirs": {
           "type": "array",
-          "items": { "type": "string" },
+          "items": {
+            "type": "string"
+          },
           "default": [],
           "description": "Additional directories containing query run result subdirectories. Appended to CODEQL_QUERY_RUN_RESULTS_DIRS alongside the vscode-codeql query storage path."
         }
@@ -133,8 +141,10 @@
     "lint": "eslint src/ test/",
     "lint:fix": "eslint src/ test/ --fix",
     "package": "vsce package --no-dependencies --out codeql-development-mcp-server.vsix",
-    "test": "vitest --run",
+    "test": "npm run test:coverage && npm run test:integration",
     "test:coverage": "vitest --run --coverage",
+    "test:integration": "vscode-test",
+    "test:integration:label": "vscode-test --label",
     "test:watch": "vitest --watch",
     "vscode:prepublish": "npm run clean && npm run lint && npm run bundle && npm run bundle:server",
     "watch": "node esbuild.config.js --watch"
@@ -145,6 +155,8 @@
     "@types/node": "^25.3.0",
     "@types/vscode": "^1.109.0",
     "@vitest/coverage-v8": "^4.0.18",
+    "@vscode/test-cli": "^0.0.12",
+    "@vscode/test-electron": "^2.5.2",
     "@vscode/vsce": "^3.7.1",
     "esbuild": "^0.27.3",
     "eslint": "^10.0.0",