Fixes for extension .lock database contention and tool improvements to avoid LLM use of grep

.github/instructions/server_src_ts.instructions.md

@@ -24,12 +24,9 @@ This file contains instructions for working with TypeScript source code files in
 - PREFER the import of functionality from `@modelcontextprotocol/sdk` over direct implementation, unless absolutely necessary.
 - PREFER to implement each MCP server primitive in its own file named after the primitive, e.g., `server/src/<lib-example>/<primitive-example>.ts`.
 - PREFER many simple MCP server primitives that each do one thing well over fewer complex MCP server primitives that do many things.
-- PREFER copying and/or adapting existing `*.prompt.md` files matching one of the following patterns:
-  - `ql/.github/prompts/*.prompt.md`
-  - `ql/languages/*/tools/dev/*.prompt.md`
-  - `ql/resources/{codeql,qlt}/*.prompt.md`
 
 ## CONTRAINTS
 
 - NEVER leave any trailing whitespace on any line.
-- NEVER guess at what a `codeql` or `qlt` CLI subcommand does; ALWAYS verify against the official `codeql <subcommand> -h -vv` or `qlt <subcommand> -h` documentation, respectively.
+- NEVER guess at what a `codeql` CLI subcommand does; ALWAYS verify against the official `codeql <subcommand> -h -vv` documentation.
+- **NEVER use stat/lstat followed by a separate read/open on the same path** — this is a TOCTOU (Time-of-Check-Time-of-Use) race condition (CWE-367). Instead, attempt the operation directly (e.g., `readFileSync`) within a try/catch block. If you need to know the file size before reading, read first and then check the buffer size — do NOT stat then read. For directory traversal, `lstatSync` is acceptable since it is the operation itself (checking entry type), not a precursor to a separate operation.

client/integration-tests/primitives/tools/codeql_resolve_files/find_ql_files/README.md

@@ -0,0 +1,26 @@
+# Integration Test: codeql_resolve_files/find_ql_files
+
+## Purpose
+
+Tests the `codeql_resolve_files` tool to ensure it can find QL query files by extension within a CodeQL pack directory.
+
+## Test Scenario
+
+This test validates that the `codeql_resolve_files` tool can:
+
+1. Accept a directory path and file extension filter
+2. Use the `codeql resolve files` CLI command to find matching files
+3. Return a list of file paths matching the specified criteria
+
+## Test Parameters
+
+- `dir`: "server/ql/javascript/examples/src"
+- `include-extension`: [".ql"]
+
+## Expected Behavior
+
+The tool should:
+
+1. Invoke `codeql resolve files` with the specified directory and extension filter
+2. Return file paths for all `.ql` files found in the directory tree
+3. Return a successful result with the file listing

client/integration-tests/primitives/tools/codeql_resolve_files/find_ql_files/after/monitoring-state.json

@@ -0,0 +1,14 @@
+{
+  "sessions": [
+    {
+      "id": "integration_test_session",
+      "calls": [
+        {
+          "tool": "codeql_resolve_files",
+          "timestamp": "2025-09-25T16:06:00.000Z",
+          "status": "success"
+        }
+      ]
+    }
+  ]
+}

client/integration-tests/primitives/tools/codeql_resolve_files/find_ql_files/before/monitoring-state.json

@@ -0,0 +1,7 @@
+{
+  "sessions": [],
+  "parameters": {
+    "dir": "server/ql/javascript/examples/src",
+    "include-extension": [".ql"]
+  }
+}

client/integration-tests/primitives/tools/profile_codeql_query_from_logs/multi_query_raw_log/after/query-evaluation-detail.txt

@@ -0,0 +1,62 @@
+# CodeQL Evaluator Profile — Predicate Detail
+# Log format: raw
+# CodeQL version: 2.23.1
+# Use read_file with line ranges from the JSON response to access individual predicates.
+
+== Query: QueryA.ql ==
+   Total: 200.00ms | Predicates evaluated: 2 | Cache hits: 1
+
+--- QueryA::source/0#abc123 ---
+    Eval order: 1 of 2
+    Duration:   65.00 ms
+    Result:     5 tuples
+    Strategy:   SIMPLE_INTENSIONAL
+    Position:   /workspace/QueryA.ql:10,1-12,5
+    Pipeline stages (1):
+      [1] 50.00ms -> 5 tuples (counts=[5])
+
+--- QueryA::sink/0#def456 ---
+    Eval order: 2 of 2
+    Duration:   45.00 ms
+    Result:     3 tuples
+    Strategy:   SIMPLE_INTENSIONAL
+    Position:   /workspace/QueryA.ql:14,1-16,5
+    Dependencies (1):
+      - QueryA::source/0#abc123
+    Pipeline stages (1):
+      [1] 30.00ms -> 3 tuples (counts=[3])
+
+== Query: QueryB.ql ==
+   Total: 300.00ms | Predicates evaluated: 3 | Cache hits: 0
+
+--- QueryB::entryPoint/0#jkl012 ---
+    Eval order: 1 of 3
+    Duration:   95.00 ms
+    Result:     8 tuples
+    Strategy:   EXTENSIONAL
+    Position:   /workspace/QueryB.ql:8,1-10,3
+    Pipeline stages (1):
+      [1] 80.00ms -> 8 tuples (counts=[8])
+
+--- QueryB::flowStep/2#mno345 ---
+    Eval order: 2 of 3
+    Duration:   95.00 ms
+    Result:     12 tuples
+    Strategy:   SIMPLE_INTENSIONAL
+    Position:   /workspace/QueryB.ql:12,1-18,5
+    Dependencies (1):
+      - QueryB::entryPoint/0#jkl012
+    Pipeline stages (1):
+      [1] 80.00ms -> 12 tuples (counts=[12])
+
+--- QueryB::result/3#pqr678 ---
+    Eval order: 3 of 3
+    Duration:   45.00 ms
+    Result:     2 tuples
+    Strategy:   SIMPLE_INTENSIONAL
+    Position:   /workspace/QueryB.ql:20,1-22,5
+    Dependencies (2):
+      - QueryB::flowStep/2#mno345
+      - QueryB::entryPoint/0#jkl012
+    Pipeline stages (1):
+      [1] 30.00ms -> 2 tuples (counts=[2])

client/integration-tests/primitives/tools/profile_codeql_query_from_logs/multi_query_raw_log/after/query-evaluation-profile.json

@@ -14,7 +14,19 @@
           "resultSize": 5,
           "pipelineCount": 1,
           "evaluationStrategy": "SIMPLE_INTENSIONAL",
-          "dependencies": []
+          "dependencies": [],
+          "pipelineStages": [
+            {
+              "durationMs": 50,
+              "resultSize": 5,
+              "counts": [
+                5
+              ],
+              "duplicationPercentages": [
+                0
+              ]
+            }
+          ]
         },
         {
           "predicateName": "QueryA::sink/0#def456",
@@ -25,6 +37,18 @@
           "evaluationStrategy": "SIMPLE_INTENSIONAL",
           "dependencies": [
             "QueryA::source/0#abc123"
+          ],
+          "pipelineStages": [
+            {
+              "durationMs": 30,
+              "resultSize": 3,
+              "counts": [
+                3
+              ],
+              "duplicationPercentages": [
+                0
+              ]
+            }
           ]
         }
       ],
@@ -42,7 +66,19 @@
           "resultSize": 8,
           "pipelineCount": 1,
           "evaluationStrategy": "EXTENSIONAL",
-          "dependencies": []
+          "dependencies": [],
+          "pipelineStages": [
+            {
+              "durationMs": 80,
+              "resultSize": 8,
+              "counts": [
+                8
+              ],
+              "duplicationPercentages": [
+                0
+              ]
+            }
+          ]
         },
         {
           "predicateName": "QueryB::flowStep/2#mno345",
@@ -53,6 +89,18 @@
           "evaluationStrategy": "SIMPLE_INTENSIONAL",
           "dependencies": [
             "QueryB::entryPoint/0#jkl012"
+          ],
+          "pipelineStages": [
+            {
+              "durationMs": 80,
+              "resultSize": 12,
+              "counts": [
+                12
+              ],
+              "duplicationPercentages": [
+                0
+              ]
+            }
           ]
         },
         {
@@ -65,6 +113,18 @@
           "dependencies": [
             "QueryB::flowStep/2#mno345",
             "QueryB::entryPoint/0#jkl012"
+          ],
+          "pipelineStages": [
+            {
+              "durationMs": 30,
+              "resultSize": 2,
+              "counts": [
+                2
+              ],
+              "duplicationPercentages": [
+                0
+              ]
+            }
           ]
         }
       ],

client/integration-tests/primitives/tools/profile_codeql_query_from_logs/single_query_raw_log/after/query-evaluation-detail.txt

@@ -0,0 +1,154 @@
+# CodeQL Evaluator Profile — Predicate Detail
+# Log format: raw
+# CodeQL version: 2.23.1
+# Use read_file with line ranges from the JSON response to access individual predicates.
+
+== Query: ExampleQuery1.ql ==
+   Total: 203.81ms | Predicates evaluated: 9 | Cache hits: 1
+
+--- Files::Container.splitAbsolutePath/2#dispred#43b82b7b_120#join_rhs ---
+    Eval order: 2 of 9
+    Duration:   58.52 ms
+    Result:     27 tuples
+    Strategy:   SIMPLE_INTENSIONAL
+    Position:   /home/runner/.codeql/packages/codeql/util/2.0.18/codeql/util/FileSystem.qll:220,5-221,90
+    Dependencies (1):
+      - cached_Files::Container.splitAbsolutePath/2#dispred#43b82b7b
+    RA operations (2 steps):
+      {3} r1 = SCAN `cached_Files::Container.splitAbsolutePath/2#dispred#43b82b7b` OUTPUT In.1, In.2, In.0
+      return r1
+    Pipeline stages (1):
+      [1] 27.98ms -> 27 tuples (counts=[27])
+
+--- files ---
+    Eval order: 1 of 9
+    Duration:   20.94 ms
+    Result:     1 tuples
+    Strategy:   EXTENSIONAL
+    Position:   /home/runner/.codeql/packages/codeql/javascript-all/2.6.11/semmlecode.javascript.dbscheme:21,1-24,3
+
+--- Files::Impl::File.getURL/0#dispred#3f789d74#bf ---
+    Eval order: 6 of 9
+    Duration:   11.28 ms
+    Result:     1 tuples
+    Strategy:   SIMPLE_INTENSIONAL
+    Position:   /home/runner/.codeql/packages/codeql/util/2.0.18/codeql/util/FileSystem.qll:220,5-221,90
+    Dependencies (2):
+      - m#Files::Impl::File.getURL/0#dispred#3f789d74#bf
+      - Files::Container.getAbsolutePath/0#dispred#051b95e1#bf
+    RA operations (3 steps):
+      {4} r1 = JOIN `m#Files::Impl::File.getURL/0#dispred#3f789d74#bf` WITH `Files::Container.getAbsolutePath/0#dispred#051b95e1#bf` ON FIRST 1 OUTPUT Lhs.0, _, Rhs.1, _
+      {2}    | REWRITE WITH Tmp.1 := "file://", Tmp.3 := ":0:0:0:0", Out.1 := (Tmp.1 ++ In.2 ++ Tmp.3) KEEPING 2
+      return r1
+    Pipeline stages (1):
+      [1] 6.74ms -> 1 tuples (counts=[1, 1])
+
+--- m#Files::Impl::File.getURL/0#dispred#3f789d74#bf ---
+    Eval order: 3 of 9
+    Duration:   9.62 ms
+    Result:     1 tuples
+    Strategy:   SIMPLE_INTENSIONAL
+    Position:   /home/runner/.codeql/packages/codeql/util/2.0.18/codeql/util/FileSystem.qll:220,5-221,90
+    Dependencies (2):
+      - Files::Container.splitAbsolutePath/2#dispred#43b82b7b_120#join_rhs
+      - files
+    RA operations (4 steps):
+      {2} r1 = CONSTANT(unique string, unique int)["ExampleQuery1.js",1]
+      {1}    | JOIN WITH `Files::Container.splitAbsolutePath/2#dispred#43b82b7b_120#join_rhs` ON FIRST 2 OUTPUT Rhs.2
+      {1}    | JOIN WITH files ON FIRST 1 OUTPUT Lhs.0
+      return r1
+    Pipeline stages (1):
+      [1] 3.57ms -> 1 tuples (counts=[1, 1, 1])
+
+--- #select#query ---
+    Eval order: 9 of 9
+    Duration:   9.01 ms
+    Result:     1 tuples
+    Strategy:   SIMPLE_INTENSIONAL
+    Position:   /home/runner/work/codeql-development-mcp-server/codeql-development-mcp-server/server/ql/javascript/examples/src/ExampleQuery1/ExampleQuery1.ql:13,1-15,80
+    Dependencies (5):
+      - m#Files::Impl::File.getURL/0#dispred#3f789d74#bf
+      - Files::Container.getAbsolutePath/0#dispred#051b95e1#bf
+      - Files::Container.getAbsolutePath/0#dispred#051b95e1#bf_0#antijoin_rhs
+      - Files::Impl::File.getURL/0#dispred#3f789d74#bf
+      - Files::Impl::File.getURL/0#dispred#3f789d74#bf_0#antijoin_rhs
+    RA operations (21 steps):
+      {2} r1 = JOIN `m#Files::Impl::File.getURL/0#dispred#3f789d74#bf` WITH `Files::Container.getAbsolutePath/0#dispred#051b95e1#bf` ON FIRST 1 OUTPUT Lhs.0, Rhs.1
+
+      {1} r2 = `m#Files::Impl::File.getURL/0#dispred#3f789d74#bf` AND NOT `Files::Container.getAbsolutePath/0#dispred#051b95e1#bf_0#antijoin_rhs`(FIRST 1)
+      {2}    | SCAN OUTPUT In.0, _
+      {2}    | REWRITE WITH Out.1 := "(no string representation)"
+
+      {2} r3 = r1 UNION r2
+      {4}    | JOIN WITH `Files::Impl::File.getURL/0#dispred#3f789d74#bf` ON FIRST 1 OUTPUT Lhs.0, Lhs.1, Rhs.1, _
+      {4}    | REWRITE WITH Out.3 := "Example test code file found for codeql_test_extract example query."
+
+      {1} r4 = `m#Files::Impl::File.getURL/0#dispred#3f789d74#bf` AND NOT `Files::Container.getAbsolutePath/0#dispred#051b95e1#bf_0#antijoin_rhs`(FIRST 1)
+      {2}    | SCAN OUTPUT In.0, _
+      {2}    | REWRITE WITH Out.1 := "(no string representation)"
+
+      {2} r5 = r1 UNION r4
+      {2}    | AND NOT `Files::Impl::File.getURL/0#dispred#3f789d74#bf_0#antijoin_rhs`(FIRST 1)
+      {4}    | SCAN OUTPUT In.0, In.1, _, _
+      {4}    | REWRITE WITH Out.2 := "", Out.3 := "Example test code file found for codeql_test_extract example query."
+
+      {4} r6 = r3 UNION r5
+      return r6
+    Pipeline stages (1):
+      [1] 4.21ms -> 1 tuples (counts=[1, -1, -1, 0, 0, -1, 1, 1, 1, -1, -1, 0, 0, -1, 1, -1, 0, 0, -1, 1])
+
+--- Files::Container.getAbsolutePath/0#dispred#051b95e1#bf ---
+    Eval order: 5 of 9
+    Duration:   4.14 ms
+    Result:     1 tuples
+    Strategy:   SIMPLE_INTENSIONAL
+    Position:   /home/runner/.codeql/packages/codeql/util/2.0.18/codeql/util/FileSystem.qll:63,5-87,66
+    Dependencies (2):
+      - m#Files::Container.getAbsolutePath/0#dispred#051b95e1#bf
+      - files
+    RA operations (2 steps):
+      {2} r1 = JOIN `m#Files::Container.getAbsolutePath/0#dispred#051b95e1#bf` WITH files ON FIRST 1 OUTPUT Lhs.0, Rhs.1
+      return r1
+    Pipeline stages (1):
+      [1] 0.47ms -> 1 tuples (counts=[1])
+
+--- Files::Container.getAbsolutePath/0#dispred#051b95e1#bf_0#antijoin_rhs ---
+    Eval order: 8 of 9
+    Duration:   3.71 ms
+    Result:     1 tuples
+    Strategy:   SIMPLE_INTENSIONAL
+    Position:   /home/runner/work/codeql-development-mcp-server/codeql-development-mcp-server/server/ql/javascript/examples/src/ExampleQuery1/ExampleQuery1.ql:13,1-15,80
+    Dependencies (1):
+      - Files::Container.getAbsolutePath/0#dispred#051b95e1#bf
+    RA operations (2 steps):
+      {1} r1 = SCAN `Files::Container.getAbsolutePath/0#dispred#051b95e1#bf` OUTPUT In.0
+      return r1
+    Pipeline stages (1):
+      [1] 0.30ms -> 1 tuples (counts=[1])
+
+--- m#Files::Container.getAbsolutePath/0#dispred#051b95e1#bf ---
+    Eval order: 4 of 9
+    Duration:   2.81 ms
+    Result:     1 tuples
+    Strategy:   SIMPLE_INTENSIONAL
+    Position:   /home/runner/.codeql/packages/codeql/util/2.0.18/codeql/util/FileSystem.qll:63,5-87,66
+    Dependencies (1):
+      - m#Files::Impl::File.getURL/0#dispred#3f789d74#bf
+    RA operations (1 steps):
+      return `m#Files::Impl::File.getURL/0#dispred#3f789d74#bf`
+    Pipeline stages (1):
+      [1] 2.43ms -> 1 tuples (counts=[])
+
+--- Files::Impl::File.getURL/0#dispred#3f789d74#bf_0#antijoin_rhs ---
+    Eval order: 7 of 9
+    Duration:   0.72 ms
+    Result:     1 tuples
+    Strategy:   SIMPLE_INTENSIONAL
+    Position:   /home/runner/work/codeql-development-mcp-server/codeql-development-mcp-server/server/ql/javascript/examples/src/ExampleQuery1/ExampleQuery1.ql:13,1-15,80
+    Dependencies (1):
+      - Files::Impl::File.getURL/0#dispred#3f789d74#bf
+    RA operations (2 steps):
+      {1} r1 = SCAN `Files::Impl::File.getURL/0#dispred#3f789d74#bf` OUTPUT In.0
+      return r1
+    Pipeline stages (1):
+      [1] 0.35ms -> 1 tuples (counts=[1])