Skip to content
Merged
Show file tree
Hide file tree
Changes from 10 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 58 additions & 2 deletions .github/actions/setup-codeql-environment/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -75,8 +75,9 @@ runs:
echo " CodeQL Version: $CODEQL_VERSION"
echo " Cache Key: $CODEQL_CACHE_KEY"

- name: Cache `gh-codeql` extension and CodeQL packages
id: cache-codeql
- name: Cache `gh-codeql` extension and CodeQL packages (Unix)
id: cache-codeql-unix
if: runner.os != 'Windows'
uses: actions/cache@v4
with:
path: |
Expand All @@ -86,6 +87,18 @@ runs:
restore-keys: |
gh-codeql-${{ runner.os }}-${{ steps.codeql-version.outputs.codeql-version }}-

- name: Cache `gh-codeql` extension and CodeQL packages (Windows)
id: cache-codeql-windows
if: runner.os == 'Windows'
uses: actions/cache@v4
with:
path: |
~\AppData\Local\GitHub\gh-codeql
~\.codeql\packages
key: ${{ steps.codeql-version.outputs.codeql-cache-key }}
restore-keys: |
gh-codeql-${{ runner.os }}-${{ steps.codeql-version.outputs.codeql-version }}-

# Install GitHub CLI CodeQL extension and set `codeql` CLI version
- name: Install GitHub CLI CodeQL extension and set version
id: install-gh-codeql
Expand Down Expand Up @@ -136,6 +149,49 @@ runs:

echo "✅ GitHub CLI CodeQL extension installed successfully"

# On Windows, gh codeql install-stub creates a bash script which is not
# discoverable by Node.js child_process.spawn() or execFile(), since
# these functions only resolve real executables (.exe), not scripts.
# Find the actual codeql.exe binary from the gh-codeql distribution
# and add its directory to PATH so that spawn('codeql', ...) works.
# This workaround can be removed once github/gh-codeql#21 is merged,
# which adds native Windows support to install-stub.
- name: Add CodeQL binary directory to PATH (Windows)
if: runner.os == 'Windows'
shell: bash
run: |
echo "🔧 Locating actual codeql.exe binary for Windows compatibility..."

# The gh-codeql extension stores the CodeQL CLI binary under the
# GitHub CLI extensions directory. On Windows runners this is:
# $LOCALAPPDATA/GitHub CLI/extensions/gh-codeql/dist/release/<version>/
GH_EXTENSIONS_DIR="${LOCALAPPDATA:-$HOME/AppData/Local}/GitHub CLI/extensions/gh-codeql"

if [ ! -d "$GH_EXTENSIONS_DIR" ]; then
echo "⚠️ gh-codeql extensions directory not found at: $GH_EXTENSIONS_DIR"
echo "Searching more broadly under LOCALAPPDATA..."
GH_EXTENSIONS_DIR="${LOCALAPPDATA:-$HOME/AppData/Local}"
fi

# Find the codeql.exe binary in the gh-codeql distribution
CODEQL_EXE=$(find "$GH_EXTENSIONS_DIR" -name "codeql.exe" -type f 2>/dev/null | head -1)

if [ -z "$CODEQL_EXE" ]; then
echo "❌ Error: codeql.exe not found under $GH_EXTENSIONS_DIR"
echo "Directory listing (top 30 files):"
find "$GH_EXTENSIONS_DIR" -maxdepth 5 -type f 2>/dev/null | head -30
exit 1
fi

CODEQL_BIN_DIR=$(dirname "$CODEQL_EXE")
echo "Found codeql.exe at: $CODEQL_EXE"
echo "Adding $CODEQL_BIN_DIR to PATH"

# Prepend the directory containing codeql.exe to PATH
echo "$CODEQL_BIN_DIR" >> "$GITHUB_PATH"

echo "✅ Added CodeQL binary directory to PATH for Windows"

- name: Setup CodeQL environment variables
id: setup-codeql-env
shell: bash
Expand Down
1 change: 1 addition & 0 deletions .github/agents/ql-mcp-tool-tester.md
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ My `ql-mcp-tool-tester` agent:
- NEVER makes anything up about CodeQL CLI behavior or MCP protocol.
- NEVER modifies the MCP server or client code; focuses solely on testing and validating the tools/primitives.
- NEVER "pipes" or redirects `npm test` or `npm run test*` command outputs in any way. Just observe the raw output and use exit codes to determine success/failure.
- **NEVER uses `os.tmpdir()`, `/tmp`, or any OS-level temporary directory** in test code, fixtures, or tool invocations. The OS temp directory is world-readable and triggers CWE-377/CWE-378 vulnerabilities. All temporary files MUST use the project-local `<repoRoot>/.tmp/` directory. In integration test fixtures the `{{tmpdir}}` placeholder resolves to this project-local directory at runtime — it does NOT resolve to the OS temp directory.

## Related Skills

Expand Down
1 change: 1 addition & 0 deletions .github/instructions/server_test_ts.instructions.md
Original file line number Diff line number Diff line change
Expand Up @@ -35,3 +35,4 @@ This file contains instructions for working with TypeScript test files in the `s
- NEVER write tests that depend on external resources or network calls without proper mocking.
- NEVER write overly complex tests that test multiple concerns in a single test case.
- NEVER skip writing tests for new functionality or bug fixes.
- **NEVER use `os.tmpdir()`, `/tmp`, or any OS-level temporary directory** in test code or test fixtures. The OS temp directory is world-readable and triggers CWE-377/CWE-378 vulnerabilities. Instead, ALWAYS use the project-local `.tmp/` directory via `getProjectTmpDir()`, `createProjectTempDir()`, or `getProjectTmpBase()` from `server/src/utils/temp-dir.ts`. For integration test fixtures, use the `{{tmpdir}}` placeholder which resolves at runtime to `<repoRoot>/.tmp/`.
67 changes: 57 additions & 10 deletions .github/workflows/client-integration-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,17 +6,17 @@ on:
paths:
- '.github/actions/setup-codeql-environment/action.yml'
- '.github/workflows/client-integration-tests.yml'
- '.node-version'
- '.codeql-version'
- '.node-version'
- 'client/**'
- 'server/**'
pull_request:
branches: [main]
paths:
- '.github/actions/setup-codeql-environment/action.yml'
- '.github/workflows/client-integration-tests.yml'
- '.node-version'
- '.codeql-version'
- '.node-version'
- 'client/**'
- 'server/**'
workflow_dispatch:
Expand All @@ -26,7 +26,13 @@ permissions:

jobs:
integration-tests:
runs-on: ubuntu-latest
name: Integration Tests (${{ matrix.os }})
runs-on: ${{ matrix.os }}

strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, windows-latest]

env:
HTTP_HOST: 'localhost'
Expand All @@ -44,9 +50,14 @@ jobs:
cache: 'npm'
node-version-file: '.node-version'

- name: MCP Integration Tests - Install OS dependencies
- name: MCP Integration Tests - Install OS dependencies (Ubuntu)
if: runner.os == 'Linux'
run: sudo apt-get install -y jq

- name: MCP Integration Tests - Install OS dependencies (Windows)
if: runner.os == 'Windows'
run: choco install jq -y

- name: MCP Integration Tests - Install node dependencies for client and server workspaces
run: npm ci --workspace=client && npm ci --workspace=server

Expand All @@ -55,44 +66,80 @@ jobs:
with:
install-language-runtimes: false

## Verify that the CodeQL CLI is spawnable from Node.js, not just from
## bash. On Windows, Node.js spawn()/execFile() require a real .exe
## binary on PATH, not a bash stub or .cmd wrapper. Fail fast here
## instead of waiting for integration tests to time out.
- name: MCP Integration Tests - Verify CodeQL CLI is spawnable from Node.js
shell: bash
run: |
node -e "
const { execFile } = require('child_process');
execFile('codeql', ['version', '--format=terse'], (err, stdout) => {
if (err) {
console.error('❌ CodeQL CLI is not spawnable from Node.js:', err.message);
console.error('This typically means codeql.exe is not on PATH (Windows).');
process.exit(1);
}
console.log('✅ CodeQL CLI is spawnable from Node.js, version:', stdout.trim());
});
"

## Install packs used in the integration tests.
- name: MCP Integration Tests - Install CodeQL packs
shell: bash
run: ./server/scripts/install-packs.sh

## Extract test databases used in the integration tests.
- name: MCP Integration Tests - Extract test databases
shell: bash
run: ./server/scripts/extract-test-databases.sh

## Configure npm to use bash for running scripts on Windows, since the
## integration test scripts are bash scripts that cmd.exe cannot execute.
- name: MCP Integration Tests - Configure npm script shell (Windows)
if: runner.os == 'Windows'
shell: bash
run: npm config set script-shell "$(which bash)"

## Run integration tests. This script builds the server bundle and runs tests.
## We do NOT use 'npm run build-and-test' as it runs query unit tests which
## have a dedicated workflow (query-unit-tests.yml).
- name: MCP Integration Tests - Run integration tests
shell: bash
run: npm run test:integration --workspace=client

- name: MCP Integration Tests - Stop the background MCP server process
if: always()
shell: bash
run: |
if [ -f server.pid ]; then
PID=$(cat server.pid)
echo "Stopping server with PID $PID"
if kill -0 $PID 2>/dev/null; then
kill $PID || true
if kill -0 "$PID" 2>/dev/null; then
kill "$PID" || true
sleep 2
# Force kill if still running
if kill -0 $PID 2>/dev/null; then
if kill -0 "$PID" 2>/dev/null; then
echo "Force killing server process"
kill -9 $PID || true
kill -9 "$PID" || true
fi
else
echo "Server process was not running"
fi
rm server.pid
rm -f server.pid
else
echo "No server.pid file found"
fi

# Clean up log files
if [ -f server.log ]; then
echo "Removing server.log"
rm server.log
rm -f server.log
fi

- name: MCP Integration Tests - Summary
shell: bash
run: |
echo "## Integration Tests Summary (${{ matrix.os }})" >> $GITHUB_STEP_SUMMARY
echo "✅ MCP server integration tests passed on ${{ matrix.os }}" >> $GITHUB_STEP_SUMMARY
4 changes: 2 additions & 2 deletions client/integration-tests/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -38,15 +38,15 @@ Common mistakes to avoid:
- ❌ **DO NOT** commit files like `evaluator-log.json`, `query-results.bqrs`, `*.bqrs` files to the repository root
- ❌ **DO NOT** commit temporary files created during integration test development to the repository root
- ✅ **DO** place all test output files in the appropriate `client/integration-tests/primitives/tools/<tool_name>/<test_name>/after/` directory
- ✅ **DO** use `/tmp/` paths for temporary files during development and testing
- ✅ **DO** use `{{tmpdir}}` as a placeholder for the project-local temporary directory in test fixture paths (resolved at runtime to `<repoRoot>/.tmp/`)

**File generation best practices:**

1. **Generate test files correctly**: When creating integration tests that involve file generation (e.g., BQRS files, evaluator logs):
- Run the actual tool command to generate authentic files
- Copy the generated files from their temporary location to the correct `after/` directory
- Never fabricate or "make up" binary file contents
2. **Use proper paths**: Always use absolute paths or `/tmp/` paths when running commands that generate files during development
2. **Use proper paths**: Always use `{{tmpdir}}/` as a placeholder for the project-local temp directory in test fixture JSON files (e.g., `"output": "{{tmpdir}}/results.sarif"`). This resolves at runtime to `<repoRoot>/.tmp/`, **not** the OS temp directory, to avoid CWE-377/CWE-378 (world-readable temp files).
3. **Verify placement**: Before committing, verify that generated files are in the correct `after/` directory, not in the repository root

The `.gitignore` file has been updated to help prevent accidental commits of common integration test output files in the root directory.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,6 @@
"database": "server/ql/javascript/examples/test/ExampleQuery1/ExampleQuery1.testproj",
"queries": "server/ql/javascript/examples/src/ExampleQuery1/ExampleQuery1.ql",
"format": "sarif-latest",
"output": "/tmp/integration-test-analyze-results.sarif"
"output": "{{tmpdir}}/integration-test-analyze-results.sarif"
}
}
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"sessions": [],
"parameters": {
"database": "/tmp/codeql-integration-test-db",
"database": "{{tmpdir}}/codeql-integration-test-db",
"language": "javascript",
"source-root": "server/ql/javascript/examples/test/ExampleQuery1",
"overwrite": true
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,17 +10,17 @@
"arguments": {
"query": "client/integration-tests/static/javascript/src/ExampleQuery1/ExampleQuery1.ql",
"database": "client/integration-tests/static/javascript/test/ExampleQuery1/ExampleQuery1.testproj",
"output": "/tmp/test-query-results.bqrs",
"evaluator-log": "/tmp/test-evaluator-log.json",
"output": "{{tmpdir}}/test-query-results.bqrs",
"evaluator-log": "{{tmpdir}}/test-evaluator-log.json",
"evaluator-log-level": 5,
"tuple-counting": true
},
"response": {
"stdout": "Evaluation completed successfully",
"stderr": "WARNING: Ignoring local version because local version support is off.",
"filesGenerated": [
"/tmp/test-query-results.bqrs",
"/tmp/test-evaluator-log.json"
"{{tmpdir}}/test-query-results.bqrs",
"{{tmpdir}}/test-evaluator-log.json"
]
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@
"arguments": {
"query": "client/integration-tests/static/javascript/src/ExampleQuery1/ExampleQuery1.ql",
"database": "client/integration-tests/static/javascript/test/ExampleQuery1/ExampleQuery1.testproj",
"output": "/tmp/test-query-results.bqrs",
"evaluator-log": "/tmp/test-evaluator-log.json",
"output": "{{tmpdir}}/test-query-results.bqrs",
"evaluator-log": "{{tmpdir}}/test-evaluator-log.json",
"evaluator-log-level": 5,
"tuple-counting": true
}
Comment thread
data-douser marked this conversation as resolved.
Outdated
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"sessions": [],
"parameters": {
"basePath": "/tmp/codeql-test-query",
"basePath": "{{tmpdir}}/codeql-test-query",
"queryName": "TestQuery",
"language": "javascript",
"description": "Integration test query"
Expand Down
52 changes: 48 additions & 4 deletions client/src/lib/integration-test-runner.js
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
*/

import fs from "fs";
import os from "os";
import path from "path";
import { fileURLToPath } from "url";
import {
Expand All @@ -13,6 +12,44 @@ import {
removeDirectory
} from "./file-utils.js";

/**
* Repository root, calculated once at module load.
* Mirrors `server/src/utils/temp-dir.ts`.
*/
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
const repoRoot = path.resolve(__dirname, "..", "..", "..");

/**
* Project-local temporary directory (`<repoRoot>/.tmp`).
* All temporary files are kept here instead of the OS temp directory
* to avoid CWE-377/CWE-378 (world-readable temp files).
*/
const PROJECT_TMP_BASE = path.join(repoRoot, ".tmp");

/**
* Resolve `{{tmpdir}}` placeholders in string values of a parameters object.
* Test fixtures use `{{tmpdir}}` as a cross-platform placeholder for the
* project-local temporary directory (`<repoRoot>/.tmp`), which avoids
* writing to the world-readable OS temp directory (CWE-377 / CWE-378).
*
* @param {Record<string, unknown>} params - Tool parameters object (mutated in place)
* @param {object} [logger] - Optional logger for diagnostics
* @returns {Record<string, unknown>} The same object, with placeholders resolved
*/
export function resolvePathPlaceholders(params, logger) {
fs.mkdirSync(PROJECT_TMP_BASE, { recursive: true });
for (const [key, value] of Object.entries(params)) {
if (typeof value === "string" && value.includes("{{tmpdir}}")) {
params[key] = value.replace(/\{\{tmpdir\}\}/g, PROJECT_TMP_BASE);
if (logger) {
logger.log(` Resolved ${key}: {{tmpdir}} → ${params[key]}`);
}
}
}
return params;
}

/**
* Integration test runner class
*/
Expand Down Expand Up @@ -208,8 +245,11 @@ export class IntegrationTestRunner {
return;
}

// Create temp directory for test execution
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), `mcp-test-${toolName}-${testCase}-`));
// Create temp directory for test execution under project .tmp/
fs.mkdirSync(PROJECT_TMP_BASE, { recursive: true });
const tempDir = fs.mkdtempSync(
path.join(PROJECT_TMP_BASE, `mcp-test-${toolName}-${testCase}-`)
);

try {
// Copy before files to temp directory
Expand Down Expand Up @@ -414,7 +454,10 @@ export class IntegrationTestRunner {
* Run a file-based test with custom configuration
*/
async runFileBasedConfigurableTest(toolName, testCase, testConfig, beforeDir, afterDir) {
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), `mcp-test-${toolName}-${testCase}-`));
fs.mkdirSync(PROJECT_TMP_BASE, { recursive: true });
const tempDir = fs.mkdtempSync(
path.join(PROJECT_TMP_BASE, `mcp-test-${toolName}-${testCase}-`)
);
Comment thread
data-douser marked this conversation as resolved.

try {
// Copy before files to temp directory
Expand Down Expand Up @@ -630,6 +673,7 @@ export class IntegrationTestRunner {
if (monitoringState.parameters) {
params = monitoringState.parameters;
this.logger.log(`Using parameters from monitoring-state.json`);
resolvePathPlaceholders(params, this.logger);

// Helper function to ensure database is extracted
const ensureDatabaseExtracted = async (dbPath) => {
Expand Down
Loading
Loading