Fix minimal default scope in extract-test-databases.sh for efficient running of client integration tests (#228)

* Fix extract-test-databases.sh minimal default scope

* Fixes for test db extraction on windows

Author: data-douser

.github/workflows/client-integration-tests.yml

@@ -93,6 +93,8 @@ jobs:
         run: ./server/scripts/install-packs.sh
 
       ## Extract test databases used in the integration tests.
+      ## Defaults to integration scope (javascript/examples only).
+      ## Query unit tests auto-extract their own databases via `codeql test run`.
       - name: MCP Integration Tests - Extract test databases
         shell: bash
         run: ./server/scripts/extract-test-databases.sh

client/src/lib/integration-test-runner.js

@@ -739,10 +739,11 @@ export class IntegrationTestRunner {
             if (!fs.existsSync(absoluteDbPath) && dbPath.endsWith(".testproj")) {
               // For paths like "test/ExpressSqlInjection/ExpressSqlInjection.testproj",
               // the test source directory is "test/ExpressSqlInjection"
-              const parts = dbPath.split(path.sep);
+              // Always split on "/" because fixture paths use forward slashes regardless of OS.
+              const parts = dbPath.split("/");
               const lastPart = parts[parts.length - 1];
               const testName = lastPart.replace(".testproj", "");
-              const parentDir = parts.slice(0, -1).join(path.sep);
+              const parentDir = parts.slice(0, -1).join("/");
 
               // Check if the parent directory name matches the test name
               const parentDirName = parts[parts.length - 2];
@@ -785,6 +786,28 @@ export class IntegrationTestRunner {
       // Call the tool with appropriate parameters (timeout is handled by this.callTool)
       this.logger.log(`Calling tool ${toolName}`);
 
+      // Clean up stale interpretedOutput from prior test runs so that
+      // directory comparisons only see output from this invocation.
+      if (toolName === "codeql_query_run" && params.interpretedOutput) {
+        const outputPath = String(params.interpretedOutput);
+        const normalizedOutput = path.normalize(outputPath);
+        // Safety: reject absolute paths and directory traversals to prevent
+        // accidental deletion of files outside the working directory (CWE-22).
+        if (
+          path.isAbsolute(normalizedOutput) ||
+          normalizedOutput.startsWith("..") ||
+          normalizedOutput.includes(`${path.sep}..`)
+        ) {
+          this.logger.log(`  Skipping interpretedOutput cleanup: unsafe path "${outputPath}"`);
+        } else {
+          try {
+            fs.rmSync(outputPath, { recursive: true, force: true });
+          } catch {
+            // Ignore — path may not exist yet
+          }
+        }
+      }
+
       const result = await this.callTool(toolName, params);
 
       // For monitoring tests, we primarily check if the tool executed successfully

server/scripts/extract-test-databases.sh

@@ -3,19 +3,27 @@ set -euo pipefail
 
 ## Parse command line arguments
 LANGUAGE=""
+SCOPE=""
 
 usage() {
 	cat << EOF
 Usage: $0 [OPTIONS]
 
 Extract test databases for CodeQL queries associated with the MCP server.
 
+By default, only a minimal set of databases for client integration tests is
+pre-extracted (currently: javascript/examples only). This is not an
+exhaustive list of databases the integration test suite may use; additional
+databases may be extracted on demand, so full extraction is rarely needed.
+
 OPTIONS:
-    --language <lang>  Extract databases only for the specified language
+    --scope <scope>    Extract databases for a specific use case
+                       Valid values:
+                         integration  - Only databases needed by client integration tests (default)
+                         all          - All test databases for all languages
+    --language <lang>  Extract databases only for the specified language (implies --scope all)
                        Valid values: actions, cpp, csharp, go, java, javascript, python, ruby, rust, swift
     -h, --help         Show this help message
-
-By default, the script extracts databases for all supported languages.
 EOF
 }
 
@@ -25,6 +33,10 @@ while [[ $# -gt 0 ]]; do
 			LANGUAGE="$2"
 			shift 2
 			;;
+		--scope)
+			SCOPE="$2"
+			shift 2
+			;;
 		-h|--help)
 			usage
 			exit 0
@@ -37,6 +49,18 @@ while [[ $# -gt 0 ]]; do
 	esac
 done
 
+## Validate scope if provided
+if [ -n "${SCOPE}" ]; then
+	case "${SCOPE}" in
+		integration|all) ;;
+		*)
+			echo "Error: Invalid scope '${SCOPE}'" >&2
+			echo "Valid scopes: integration, all" >&2
+			exit 1
+			;;
+	esac
+fi
+
 ## Validate language if provided
 VALID_LANGUAGES=("actions" "cpp" "csharp" "go" "java" "javascript" "python" "ruby" "rust" "swift")
 if [ -n "${LANGUAGE}" ]; then
@@ -91,7 +115,14 @@ extract_test_databases() {
 	done < <(find "${_base_dir}/test" -mindepth 1 -maxdepth 1 -type d -print0)
 }
 
-## Extract test databases for integration tests.
+## Extract test databases based on scope and language filters.
+##
+## Default (no flags): only databases needed by client integration tests
+##   (currently just server/ql/javascript/examples).
+## --scope all: all languages × examples + tools.
+## --language: filter to a single language (implies --scope all).
+
+# --language implies --scope all for that language
 if [ -n "${LANGUAGE}" ]; then
 	echo "Extracting test databases for language: ${LANGUAGE}"
 	# Special handling for JavaScript which has both examples and tools
@@ -101,7 +132,7 @@ if [ -n "${LANGUAGE}" ]; then
 	if [ -d "server/ql/${LANGUAGE}/tools" ]; then
 		extract_test_databases "server/ql/${LANGUAGE}/tools"
 	fi
-else
+elif [ "${SCOPE}" = "all" ]; then
 	echo "Extracting test databases for all languages..."
 	for lang in "${VALID_LANGUAGES[@]}"; do
 		# Special handling for JavaScript which has both examples and tools
@@ -112,6 +143,9 @@ else
 			extract_test_databases "server/ql/${lang}/tools"
 		fi
 	done
+else
+	echo "Extracting test databases for integration tests only..."
+	extract_test_databases "server/ql/javascript/examples"
 fi
 
 echo "INFO: Test database extraction complete!"