Fixes for extension .lock database contention and tool improvements to avoid LLM use of grep

client/integration-tests/primitives/tools/codeql_resolve_files/find_qll_files/README.md

@@ -0,0 +1,26 @@
+# Integration Test: codeql_resolve_files/find_qll_files
+
+## Purpose
+
+Tests the `codeql_resolve_files` tool to ensure it can find QL library files by extension within a CodeQL pack directory.
+
+## Test Scenario
+
+This test validates that the `codeql_resolve_files` tool can:
+
+1. Accept a directory path and file extension filter
+2. Use the `codeql resolve files` CLI command to find matching files
+3. Return a list of file paths matching the specified criteria
+
+## Test Parameters
+
+- `dir`: "server/ql/javascript/examples/src"
+- `include-extension`: [".ql"]
+
+## Expected Behavior
+
+The tool should:
+
+1. Invoke `codeql resolve files` with the specified directory and extension filter
+2. Return file paths for all `.ql` files found in the directory tree
+3. Return a successful result with the file listing

client/integration-tests/primitives/tools/codeql_resolve_files/find_qll_files/after/monitoring-state.json

@@ -0,0 +1,9 @@
+{
+  "toolName": "codeql_resolve_files",
+  "parameters": {
+    "dir": "server/ql/javascript/examples/src",
+    "include-extension": [".ql"]
+  },
+  "success": true,
+  "description": "Successfully resolved .ql files in JavaScript examples directory"
+}

client/integration-tests/primitives/tools/codeql_resolve_files/find_qll_files/before/monitoring-state.json

@@ -0,0 +1,9 @@
+{
+  "toolName": "codeql_resolve_files",
+  "parameters": {
+    "dir": "server/ql/javascript/examples/src",
+    "include-extension": [".ql"]
+  },
+  "expectedSuccess": true,
+  "description": "Test codeql_resolve_files for finding .ql files in JavaScript examples"
+}

client/integration-tests/primitives/tools/search_ql_code/search_predicate_name/README.md

@@ -0,0 +1,28 @@
+# Integration Test: search_ql_code/search_predicate_name
+
+## Purpose
+
+Tests the `search_ql_code` tool to ensure it can search QL source files for a predicate name pattern and return structured results with file paths and line numbers.
+
+## Test Scenario
+
+This test validates that the `search_ql_code` tool can:
+
+1. Accept a regex pattern for a predicate name
+2. Search `.ql` and `.qll` files in a given directory
+3. Return structured JSON results with file paths, line numbers, and matching lines
+4. Report the correct number of files searched and matches found
+
+## Test Parameters
+
+- `pattern`: "isSource"
+- `paths`: ["server/ql/javascript/examples/src"]
+- `contextLines`: 1
+
+## Expected Behavior
+
+The tool should:
+
+1. Recursively search `.ql` and `.qll` files in the specified directory
+2. Return matches with file paths, line numbers, and context lines
+3. Report `filesSearched`, `totalMatches`, `returnedMatches`, and `truncated` fields

client/integration-tests/primitives/tools/search_ql_code/search_predicate_name/after/monitoring-state.json

@@ -0,0 +1,23 @@
+{
+  "toolName": "search_ql_code",
+  "parameters": {
+    "pattern": "isSource",
+    "paths": ["server/ql/javascript/examples/src"],
+    "contextLines": 1
+  },
+  "success": true,
+  "description": "Successfully searched QL files for predicate name pattern",
+  "searchResult": {
+    "totalMatches": 1,
+    "returnedMatches": 1,
+    "truncated": false,
+    "filesSearched": 1,
+    "results": [
+      {
+        "filePath": "server/ql/javascript/examples/src/ExampleQuery1/ExampleQuery1.ql",
+        "lineNumber": 1,
+        "lineContent": "contains isSource match"
+      }
+    ]
+  }
+}

client/integration-tests/primitives/tools/search_ql_code/search_predicate_name/before/monitoring-state.json

@@ -0,0 +1,10 @@
+{
+  "toolName": "search_ql_code",
+  "parameters": {
+    "pattern": "isSource",
+    "paths": ["server/ql/javascript/examples/src"],
+    "contextLines": 1
+  },
+  "expectedSuccess": true,
+  "description": "Test search_ql_code for predicate name search in JavaScript examples"
+}

extensions/vscode/esbuild.config.js

@@ -35,6 +35,7 @@ const testSuiteConfig = {
   entryPoints: [
     'test/suite/index.ts',
     'test/suite/bridge.integration.test.ts',
+    'test/suite/copydb-e2e.integration.test.ts',
     'test/suite/extension.integration.test.ts',
     'test/suite/mcp-resource-e2e.integration.test.ts',
     'test/suite/mcp-server.integration.test.ts',

extensions/vscode/package.json

@@ -48,33 +48,13 @@
     "configuration": {
       "title": "CodeQL MCP Server",
       "properties": {
-        "codeql-mcp.autoInstall": {
-          "type": "boolean",
-          "default": true,
-          "description": "Automatically install and update the CodeQL Development MCP Server on activation."
-        },
-        "codeql-mcp.serverVersion": {
-          "type": "string",
-          "default": "latest",
-          "description": "The npm version of codeql-development-mcp-server to install. Use 'latest' for the most recent release."
-        },
-        "codeql-mcp.serverCommand": {
-          "type": "string",
-          "default": "node",
-          "description": "Command to launch the MCP server. The default 'node' runs the bundled server. Override to 'npx' to download from npm, or provide a custom path."
-        },
-        "codeql-mcp.serverArgs": {
+        "codeql-mcp.additionalDatabaseDirs": {
           "type": "array",
           "items": {
             "type": "string"
           },
           "default": [],
-          "description": "Custom arguments for the MCP server command. When empty, the bundled server entry point is used automatically. Set to e.g. ['/path/to/server/dist/codeql-development-mcp-server.js'] for local development."
-        },
-        "codeql-mcp.watchCodeqlExtension": {
-          "type": "boolean",
-          "default": true,
-          "description": "Watch for CodeQL databases and query results created by the CodeQL extension."
+          "description": "Additional directories to search for CodeQL databases. Appended to CODEQL_DATABASES_BASE_DIRS. When copyDatabases is enabled (default), they are appended alongside the managed databases directory; when copyDatabases is disabled, they are appended alongside the original vscode-codeql database storage directories."
         },
         "codeql-mcp.additionalEnv": {
           "type": "object",
@@ -84,29 +64,54 @@
             "type": "string"
           }
         },
-        "codeql-mcp.additionalDatabaseDirs": {
+        "codeql-mcp.additionalMrvaRunResultsDirs": {
           "type": "array",
           "items": {
             "type": "string"
           },
           "default": [],
-          "description": "Additional directories to search for CodeQL databases. Appended to CODEQL_DATABASES_BASE_DIRS alongside the vscode-codeql storage paths."
+          "description": "Additional directories containing MRVA run result subdirectories. Appended to CODEQL_MRVA_RUN_RESULTS_DIRS alongside the vscode-codeql variant analysis storage path."
         },
-        "codeql-mcp.additionalMrvaRunResultsDirs": {
+        "codeql-mcp.additionalQueryRunResultsDirs": {
           "type": "array",
           "items": {
             "type": "string"
           },
           "default": [],
-          "description": "Additional directories containing MRVA run result subdirectories. Appended to CODEQL_MRVA_RUN_RESULTS_DIRS alongside the vscode-codeql variant analysis storage path."
+          "description": "Additional directories containing query run result subdirectories. Appended to CODEQL_QUERY_RUN_RESULTS_DIRS alongside the vscode-codeql query storage path."
         },
-        "codeql-mcp.additionalQueryRunResultsDirs": {
+        "codeql-mcp.autoInstall": {
+          "type": "boolean",
+          "default": true,
+          "description": "Automatically install and update the CodeQL Development MCP Server on activation."
+        },
+        "codeql-mcp.copyDatabases": {
+          "type": "boolean",
+          "default": true,
+          "markdownDescription": "Copy CodeQL databases from the `GitHub.vscode-codeql` extension storage into a managed directory, removing query-server lock files so the MCP server CLI can operate without contention. Disable to use databases in-place (may fail when the CodeQL query server is running)."
+        },
+        "codeql-mcp.serverArgs": {
           "type": "array",
           "items": {
             "type": "string"
           },
           "default": [],
-          "description": "Additional directories containing query run result subdirectories. Appended to CODEQL_QUERY_RUN_RESULTS_DIRS alongside the vscode-codeql query storage path."
+          "description": "Custom arguments for the MCP server command. When empty, the bundled server entry point is used automatically. Set to e.g. ['/path/to/server/dist/codeql-development-mcp-server.js'] for local development."
+        },
+        "codeql-mcp.serverCommand": {
+          "type": "string",
+          "default": "node",
+          "description": "Command to launch the MCP server. The default 'node' runs the bundled server. Override to 'npx' to download from npm, or provide a custom path."
+        },
+        "codeql-mcp.serverVersion": {
+          "type": "string",
+          "default": "latest",
+          "description": "The npm version of codeql-development-mcp-server to install. Use 'latest' for the most recent release."
+        },
+        "codeql-mcp.watchCodeqlExtension": {
+          "type": "boolean",
+          "default": true,
+          "description": "Watch for CodeQL databases and query results created by the CodeQL extension."
         }
       }
     },

extensions/vscode/src/bridge/database-copier.ts

@@ -0,0 +1,164 @@
+import { existsSync } from 'fs';
+import { cp, mkdir, readdir, rm, stat, unlink } from 'fs/promises';
+import { join } from 'path';
+import type { Logger } from '../common/logger';
+
+/**
+ * Copies CodeQL databases from `GitHub.vscode-codeql` extension storage
+ * to a managed directory, removing `.lock` files that the CodeQL query
+ * server creates in `<dataset>/default/cache/`.
+ *
+ * This avoids lock contention when the `ql-mcp` server runs CLI commands
+ * against databases that are simultaneously registered by the
+ * `vscode-codeql` query server.
+ *
+ * Each database is identified by its top-level directory name (which
+ * contains `codeql-database.yml`). A database is only re-copied when its
+ * source has been modified more recently than the existing copy.
+ */
+export class DatabaseCopier {
+  constructor(
+    private readonly destinationBase: string,
+    private readonly logger: Logger,
+  ) {}
+
+  /**
+   * Synchronise databases from one or more source directories into the
+   * managed destination. Only databases that are newer than the existing
+   * copy (or missing entirely) are re-copied.
+   *
+   * @returns The list of database paths in the managed destination that
+   *          are ready for use (absolute paths).
+   */
+  async syncAll(sourceDirs: string[]): Promise<string[]> {
+    try {
+      await mkdir(this.destinationBase, { recursive: true });
+    } catch (err) {
+      this.logger.error(
+        `Failed to create managed database directory ${this.destinationBase}: ${err instanceof Error ? err.message : String(err)}`,
+      );
+      return [];
+    }
+
+    const copied: string[] = [];
+
+    for (const sourceDir of sourceDirs) {
+      if (!existsSync(sourceDir)) {
+        continue;
+      }
+
+      let entries: string[];
+      try {
+        entries = await readdir(sourceDir);
+      } catch {
+        continue;
+      }
+
+      for (const entry of entries) {
+        const srcDbPath = join(sourceDir, entry);
+        if (!(await isCodeQLDatabase(srcDbPath))) {
+          continue;
+        }
+
+        const destDbPath = join(this.destinationBase, entry);
+
+        if (await this.needsCopy(srcDbPath, destDbPath)) {
+          await this.copyDatabase(srcDbPath, destDbPath);
+        }
+
+        if (await isCodeQLDatabase(destDbPath)) {
+          copied.push(destDbPath);
+        }
+      }
+    }
+
+    return copied;
+  }
+
+  /**
+   * Copy a single database directory, then strip any `.lock` files that
+   * the CodeQL query server may have left behind.
+   */
+  private async copyDatabase(src: string, dest: string): Promise<void> {
+    this.logger.info(`Copying database ${src} → ${dest}`);
+    try {
+      // Remove stale destination if present
+      try {
+        if (existsSync(dest)) {
+          await rm(dest, { recursive: true, force: true });
+        }
+      } catch (rmErr) {
+        this.logger.error(
+          `Failed to remove stale destination ${dest}: ${rmErr instanceof Error ? rmErr.message : String(rmErr)}`,
+        );
+        return;
+      }
+
+      await cp(src, dest, { recursive: true });
+      await removeLockFiles(dest);
+      this.logger.info(`Database copied successfully: ${dest}`);
+    } catch (err) {
+      this.logger.error(
+        `Failed to copy database ${src}: ${err instanceof Error ? err.message : String(err)}`,
+      );
+    }
+  }
+
+  /**
+   * A copy is needed when the destination does not exist, or the source
+   * `codeql-database.yml` is newer than the destination's.
+   */
+  private async needsCopy(src: string, dest: string): Promise<boolean> {
+    const destYml = join(dest, 'codeql-database.yml');
+    if (!existsSync(destYml)) {
+      return true;
+    }
+
+    const srcYml = join(src, 'codeql-database.yml');
+    try {
+      const srcMtime = (await stat(srcYml)).mtimeMs;
+      const destMtime = (await stat(destYml)).mtimeMs;
+      return srcMtime > destMtime;
+    } catch {
+      // If stat fails, re-copy to be safe
+      return true;
+    }
+  }
+}
+
+/** Check whether a directory looks like a CodeQL database. */
+async function isCodeQLDatabase(dirPath: string): Promise<boolean> {
+  try {
+    return (await stat(dirPath)).isDirectory() && existsSync(join(dirPath, 'codeql-database.yml'));
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Recursively remove all `.lock` files under the given directory.
+ * These are empty sentinel files created by the CodeQL query server in
+ * `<dataset>/default/cache/.lock`.
+ */
+async function removeLockFiles(dir: string): Promise<void> {
+  let entries: string[];
+  try {
+    entries = await readdir(dir);
+  } catch {
+    return;
+  }
+
+  for (const entry of entries) {
+    const fullPath = join(dir, entry);
+    try {
+      const st = await stat(fullPath);
+      if (st.isDirectory()) {
+        await removeLockFiles(fullPath);
+      } else if (entry === '.lock') {
+        await unlink(fullPath);
+      }
+    } catch {
+      // Best-effort removal
+    }
+  }
+}