diff --git a/client/Makefile b/client/Makefile index 712c89a4..30740b59 100644 --- a/client/Makefile +++ b/client/Makefile @@ -1,5 +1,6 @@ BINARY_NAME := gh-ql-mcp-client MODULE := github.com/advanced-security/codeql-development-mcp-server/client +VERSION := $(shell grep 'Version = ' cmd/root.go | head -1 | sed 's/.*"\(.*\)"/\1/') # Disable CGO to avoid Xcode/C compiler dependency export CGO_ENABLED = 0 @@ -13,59 +14,35 @@ all: lint test build ## build: Build the binary for the current platform build: - @if [ -f go.mod ]; then \ - go build -ldflags "$(LDFLAGS)" -o $(BINARY_NAME) .; \ - else \ - echo "Go source not yet available — skipping build"; \ - fi + go build -ldflags "$(LDFLAGS)" -o $(BINARY_NAME) . ## test: Run unit tests and integration tests test: test-unit test-integration ## test-unit: Run Go unit tests only test-unit: - @if [ -f go.mod ]; then \ - go test ./...; \ - else \ - echo "Go source not yet available — skipping unit tests"; \ - fi + go test ./... ## test-integration: Build binary and run integration tests via run-integration-tests.sh test-integration: build - @if [ -f go.mod ]; then \ - ENABLE_ANNOTATION_TOOLS=true scripts/run-integration-tests.sh --no-install-packs; \ - else \ - echo "Go source not yet available — skipping integration tests"; \ - fi + ENABLE_ANNOTATION_TOOLS=true scripts/run-integration-tests.sh --no-install-packs ## test-verbose: Run all unit tests with verbose output test-verbose: - @if [ -f go.mod ]; then \ - go test -v ./...; \ - else \ - echo "Go source not yet available — skipping verbose tests"; \ - fi + go test -v ./... ## test-coverage: Run tests with coverage test-coverage: - @if [ -f go.mod ]; then \ - go test -coverprofile=coverage.out ./...; \ - go tool cover -html=coverage.out -o coverage.html; \ - else \ - echo "Go source not yet available — skipping coverage"; \ - fi + go test -coverprofile=coverage.out ./... + go tool cover -html=coverage.out -o coverage.html ## lint: Run linters lint: - @if [ -f go.mod ]; then \ - if command -v golangci-lint > /dev/null 2>&1; then \ - golangci-lint run ./...; \ - else \ - echo "golangci-lint not found, running go vet only"; \ - go vet ./...; \ - fi; \ + @if command -v golangci-lint > /dev/null 2>&1; then \ + golangci-lint run ./...; \ else \ - echo "Go source not yet available — skipping lint"; \ + echo "golangci-lint not found, running go vet only"; \ + go vet ./...; \ fi ## clean: Remove build artifacts @@ -74,31 +51,19 @@ clean: ## install: Install the binary to GOPATH/bin install: - @if [ -f go.mod ]; then \ - go install -ldflags "$(LDFLAGS)" .; \ - else \ - echo "Go source not yet available — skipping install"; \ - fi + go install -ldflags "$(LDFLAGS)" . ## build-all: Cross-compile for all supported platforms build-all: - @if [ -f go.mod ]; then \ - GOOS=darwin GOARCH=amd64 go build -ldflags "$(LDFLAGS)" -o $(BINARY_NAME)-darwin-amd64 .; \ - GOOS=darwin GOARCH=arm64 go build -ldflags "$(LDFLAGS)" -o $(BINARY_NAME)-darwin-arm64 .; \ - GOOS=linux GOARCH=amd64 go build -ldflags "$(LDFLAGS)" -o $(BINARY_NAME)-linux-amd64 .; \ - GOOS=linux GOARCH=arm64 go build -ldflags "$(LDFLAGS)" -o $(BINARY_NAME)-linux-arm64 .; \ - GOOS=windows GOARCH=amd64 go build -ldflags "$(LDFLAGS)" -o $(BINARY_NAME)-windows-amd64.exe .; \ - else \ - echo "Go source not yet available — skipping cross-compile"; \ - fi + GOOS=darwin GOARCH=amd64 go build -ldflags "$(LDFLAGS)" -o $(BINARY_NAME)-darwin-amd64 . + GOOS=darwin GOARCH=arm64 go build -ldflags "$(LDFLAGS)" -o $(BINARY_NAME)-darwin-arm64 . + GOOS=linux GOARCH=amd64 go build -ldflags "$(LDFLAGS)" -o $(BINARY_NAME)-linux-amd64 . + GOOS=linux GOARCH=arm64 go build -ldflags "$(LDFLAGS)" -o $(BINARY_NAME)-linux-arm64 . + GOOS=windows GOARCH=amd64 go build -ldflags "$(LDFLAGS)" -o $(BINARY_NAME)-windows-amd64.exe . ## tidy: Tidy go.mod tidy: - @if [ -f go.mod ]; then \ - go mod tidy; \ - else \ - echo "Go source not yet available — skipping tidy"; \ - fi + go mod tidy ## help: Show this help help: diff --git a/client/cmd/helpers.go b/client/cmd/helpers.go new file mode 100644 index 00000000..19946a3c --- /dev/null +++ b/client/cmd/helpers.go @@ -0,0 +1,15 @@ +package cmd + +import ( + "fmt" + "strings" +) + +// parseRepo splits an "owner/repo" string into owner and repo components. +func parseRepo(nwo string) (string, string, error) { + parts := strings.SplitN(nwo, "/", 2) + if len(parts) != 2 || parts[0] == "" || parts[1] == "" { + return "", "", fmt.Errorf("invalid repo format %q: expected owner/repo", nwo) + } + return parts[0], parts[1], nil +} diff --git a/client/cmd/helpers_test.go b/client/cmd/helpers_test.go new file mode 100644 index 00000000..ae6c4e62 --- /dev/null +++ b/client/cmd/helpers_test.go @@ -0,0 +1,31 @@ +package cmd + +import "testing" + +func TestParseRepo_Valid(t *testing.T) { + owner, repo, err := parseRepo("example-owner/example-repo") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if owner != "example-owner" { + t.Errorf("owner = %q, want %q", owner, "example-owner") + } + if repo != "example-repo" { + t.Errorf("repo = %q, want %q", repo, "example-repo") + } +} + +func TestParseRepo_Invalid(t *testing.T) { + tests := []string{ + "", + "noslash", + "/norepo", + "noowner/", + } + for _, input := range tests { + _, _, err := parseRepo(input) + if err == nil { + t.Errorf("parseRepo(%q) should return error", input) + } + } +} diff --git a/client/cmd/integration_tests.go b/client/cmd/integration_tests.go new file mode 100644 index 00000000..8bda87c9 --- /dev/null +++ b/client/cmd/integration_tests.go @@ -0,0 +1,168 @@ +package cmd + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + + mcpclient "github.com/advanced-security/codeql-development-mcp-server/client/internal/mcp" + itesting "github.com/advanced-security/codeql-development-mcp-server/client/internal/testing" + "github.com/mark3labs/mcp-go/mcp" + "github.com/spf13/cobra" +) + +var integrationTestsCmd = &cobra.Command{ + Use: "integration-tests", + Short: "Run MCP server integration tests from client/integration-tests/", + Long: `Discovers and runs integration test fixtures against a connected MCP server. + +Test fixtures live in client/integration-tests/primitives/tools/// +and use test-config.json or monitoring-state.json to define tool parameters.`, + RunE: runIntegrationTests, +} + +var integrationTestsFlags struct { + tools string + tests string + noInstall bool + timeout int +} + +func init() { + rootCmd.AddCommand(integrationTestsCmd) + + f := integrationTestsCmd.Flags() + f.StringVar(&integrationTestsFlags.tools, "tools", "", "Comma-separated list of tool names to test") + f.StringVar(&integrationTestsFlags.tests, "tests", "", "Comma-separated list of test case names to run") + f.BoolVar(&integrationTestsFlags.noInstall, "no-install-packs", false, "Skip CodeQL pack installation") + f.IntVar(&integrationTestsFlags.timeout, "timeout", 30, "Per-tool-call timeout in seconds") +} + +// mcpToolCaller adapts the MCP client to the ToolCaller interface. +type mcpToolCaller struct { + client *mcpclient.Client +} + +func (c *mcpToolCaller) CallToolRaw(name string, params map[string]any) ([]itesting.ContentBlock, bool, error) { + result, err := c.client.CallTool(context.Background(), name, params) + if err != nil { + return nil, false, err + } + + var blocks []itesting.ContentBlock + for _, item := range result.Content { + if textContent, ok := item.(mcp.TextContent); ok { + blocks = append(blocks, itesting.ContentBlock{ + Type: "text", + Text: textContent.Text, + }) + } + } + + return blocks, result.IsError, nil +} + +func (c *mcpToolCaller) ListToolNames() ([]string, error) { + tools, err := c.client.ListTools(context.Background()) + if err != nil { + return nil, err + } + names := make([]string, len(tools)) + for i, t := range tools { + names[i] = t.Name + } + return names, nil +} + +func runIntegrationTests(cmd *cobra.Command, _ []string) error { + // Determine repo root + repoRoot, err := findRepoRoot() + if err != nil { + return fmt.Errorf("cannot determine repo root: %w", err) + } + + // Change CWD to repo root so the MCP server subprocess resolves + // relative paths (from test-config.json, monitoring-state.json) + // correctly. The codeql CLI also resolves paths from CWD. + if err := os.Chdir(repoRoot); err != nil { + return fmt.Errorf("chdir to repo root: %w", err) + } + fmt.Printf("Working directory: %s\n", repoRoot) + + // Connect to MCP server + client := mcpclient.NewClient(mcpclient.Config{ + Mode: MCPMode(), + Host: MCPHost(), + Port: MCPPort(), + }) + + fmt.Println("🔌 Connecting to MCP server...") + ctx := context.Background() + if err := client.Connect(ctx); err != nil { + return fmt.Errorf("connect to MCP server: %w", err) + } + fmt.Println("✅ Connected to MCP server") + + // Parse filters + var filterTools, filterTests []string + if integrationTestsFlags.tools != "" { + filterTools = strings.Split(integrationTestsFlags.tools, ",") + } + if integrationTestsFlags.tests != "" { + filterTests = strings.Split(integrationTestsFlags.tests, ",") + } + + // Create and run the test runner + runner := itesting.NewRunner(&mcpToolCaller{client: client}, itesting.RunnerOptions{ + RepoRoot: repoRoot, + FilterTools: filterTools, + FilterTests: filterTests, + }) + + allPassed, _ := runner.Run() + + // Close the MCP client (and its stdio subprocess) before returning. + client.Close() + + if !allPassed { + return fmt.Errorf("some integration tests failed") + } + return nil +} + +// findRepoRoot walks up from the current directory to find the repo root +// (identified by the presence of codeql-workspace.yml). +func findRepoRoot() (string, error) { + // Try from current working directory + dir, err := os.Getwd() + if err != nil { + return "", err + } + + for { + if _, err := os.Stat(filepath.Join(dir, "codeql-workspace.yml")); err == nil { + return dir, nil + } + parent := filepath.Dir(dir) + if parent == dir { + break + } + dir = parent + } + + // Fallback: try relative to the binary + exe, err := os.Executable() + if err == nil { + dir = filepath.Dir(exe) + for i := 0; i < 5; i++ { + if _, err := os.Stat(filepath.Join(dir, "codeql-workspace.yml")); err == nil { + return dir, nil + } + dir = filepath.Dir(dir) + } + } + + return "", fmt.Errorf("could not find repo root (codeql-workspace.yml)") +} diff --git a/client/cmd/root.go b/client/cmd/root.go new file mode 100644 index 00000000..21d959b5 --- /dev/null +++ b/client/cmd/root.go @@ -0,0 +1,67 @@ +// Package cmd defines the CLI commands for gh-ql-mcp-client. +package cmd + +import ( + "fmt" + "os" + + "github.com/spf13/cobra" +) + +const ( + // Version is the current CLI version. + Version = "0.1.0" +) + +// Global persistent flags +var ( + mcpMode string + mcpHost string + mcpPort int + outputFmt string +) + +// rootCmd is the top-level command for the CLI. +var rootCmd = &cobra.Command{ + Use: "gh-ql-mcp-client", + Short: "CodeQL Development MCP Client — integration test runner and CLI", + Long: `gh-ql-mcp-client is a CLI for running integration tests against a +CodeQL Development MCP Server. + +It connects to a CodeQL Development MCP Server via stdio or HTTP transport +and runs integration test fixtures from client/integration-tests/. + +Use as a gh extension: gh ql-mcp-client [flags] +Use standalone: gh-ql-mcp-client [flags]`, + SilenceUsage: true, + SilenceErrors: true, + Version: Version, +} + +// Execute runs the root command. +func Execute() error { + if err := rootCmd.Execute(); err != nil { + fmt.Fprintln(os.Stderr, err) + return err + } + return nil +} + +func init() { + rootCmd.PersistentFlags().StringVar(&mcpMode, "mode", "stdio", "MCP server transport mode (stdio or http)") + rootCmd.PersistentFlags().StringVar(&mcpHost, "host", "localhost", "MCP server host (http mode)") + rootCmd.PersistentFlags().IntVar(&mcpPort, "port", 3000, "MCP server port (http mode)") + rootCmd.PersistentFlags().StringVar(&outputFmt, "format", "text", "Output format (text or json)") +} + +// MCPMode returns the configured MCP transport mode. +func MCPMode() string { return mcpMode } + +// MCPHost returns the configured MCP server host. +func MCPHost() string { return mcpHost } + +// MCPPort returns the configured MCP server port. +func MCPPort() int { return mcpPort } + +// OutputFormat returns the configured output format. +func OutputFormat() string { return outputFmt } diff --git a/client/cmd/root_test.go b/client/cmd/root_test.go new file mode 100644 index 00000000..87592940 --- /dev/null +++ b/client/cmd/root_test.go @@ -0,0 +1,92 @@ +package cmd + +import ( + "bytes" + "testing" +) + +// executeRootCmd executes rootCmd with the given args and returns stdout. +// It resets the command's output and args before and after each call +// to avoid leaking state between tests. +func executeRootCmd(args []string) (string, error) { + buf := new(bytes.Buffer) + rootCmd.SetOut(buf) + rootCmd.SetErr(buf) + rootCmd.SetArgs(args) + + err := rootCmd.Execute() + + // Reset to default so subsequent tests are not affected + rootCmd.SetOut(nil) + rootCmd.SetErr(nil) + rootCmd.SetArgs(nil) + + return buf.String(), err +} + +func TestRootCommand_Help(t *testing.T) { + output, err := executeRootCmd([]string{"--help"}) + if err != nil { + t.Fatalf("root --help failed: %v", err) + } + + if output == "" { + t.Fatal("expected help output, got empty string") + } + + // Check key elements are present + wantSubstrings := []string{ + "gh-ql-mcp-client", + "integration-tests", + "--mode", + "--host", + "--port", + "--format", + } + for _, want := range wantSubstrings { + if !bytes.Contains([]byte(output), []byte(want)) { + t.Errorf("help output missing %q", want) + } + } +} + +func TestRootCommand_Version(t *testing.T) { + // Verify the version constant is set correctly + if Version == "" { + t.Fatal("Version constant should not be empty") + } + if Version != "0.1.0" { + t.Errorf("Version = %q, want %q", Version, "0.1.0") + } + + // Verify the root command has version set + if rootCmd.Version != Version { + t.Errorf("rootCmd.Version = %q, want %q", rootCmd.Version, Version) + } +} + +func TestRootCommand_DefaultFlags(t *testing.T) { + // Reset flags to defaults by executing with no args + _, _ = executeRootCmd([]string{}) + + if MCPMode() != "stdio" { + t.Errorf("expected default mode %q, got %q", "stdio", MCPMode()) + } + if MCPHost() != "localhost" { + t.Errorf("expected default host %q, got %q", "localhost", MCPHost()) + } + if MCPPort() != 3000 { + t.Errorf("expected default port %d, got %d", 3000, MCPPort()) + } + if OutputFormat() != "text" { + t.Errorf("expected default format %q, got %q", "text", OutputFormat()) + } +} + +func TestIntegrationTestsCommand_InHelp(t *testing.T) { + output, _ := executeRootCmd([]string{"--help"}) + + if !bytes.Contains([]byte(output), []byte("integration-tests")) { + t.Error("root help should list integration-tests subcommand") + } +} diff --git a/client/eslint.config.mjs b/client/eslint.config.mjs deleted file mode 100644 index 6a2e70e3..00000000 --- a/client/eslint.config.mjs +++ /dev/null @@ -1,30 +0,0 @@ -import js from '@eslint/js'; -import prettier from 'eslint-plugin-prettier'; -import prettierConfig from 'eslint-config-prettier'; - -export default [ - js.configs.recommended, - prettierConfig, - { - files: ['**/*.js'], - plugins: { - prettier, - }, - rules: { - 'prettier/prettier': 'error', - 'no-unused-vars': ['error', { argsIgnorePattern: '^_' }], - }, - languageOptions: { - ecmaVersion: 'latest', - sourceType: 'module', - globals: { - process: 'readonly', - import: 'readonly', - require: 'readonly', - module: 'readonly', - __dirname: 'readonly', - console: 'readonly', - }, - }, - }, -]; \ No newline at end of file diff --git a/client/go.mod b/client/go.mod new file mode 100644 index 00000000..c49ba08d --- /dev/null +++ b/client/go.mod @@ -0,0 +1,17 @@ +module github.com/advanced-security/codeql-development-mcp-server/client + +go 1.25.8 + +require ( + github.com/mark3labs/mcp-go v0.47.0 + github.com/spf13/cobra v1.10.2 +) + +require ( + github.com/google/jsonschema-go v0.4.2 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/spf13/cast v1.7.1 // indirect + github.com/spf13/pflag v1.0.9 // indirect + github.com/yosida95/uritemplate/v3 v3.0.2 // indirect +) diff --git a/client/go.sum b/client/go.sum new file mode 100644 index 00000000..a2de5987 --- /dev/null +++ b/client/go.sum @@ -0,0 +1,38 @@ +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/jsonschema-go v0.4.2 h1:tmrUohrwoLZZS/P3x7ex0WAVknEkBZM46iALbcqoRA8= +github.com/google/jsonschema-go v0.4.2/go.mod h1:r5quNTdLOYEz95Ru18zA0ydNbBuYoo9tgaYcxEYhJVE= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mark3labs/mcp-go v0.47.0 h1:h44yeM3DduDyQgzImYWu4pt6VRkqP/0p/95AGhWngnA= +github.com/mark3labs/mcp-go v0.47.0/go.mod h1:JKTC7R2LLVagkEWK7Kwu7DbmA6iIvnNAod6yrHiQMag= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y= +github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= +github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= +github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= +github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= +github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/client/integration-tests/primitives/tools/codeql_query_run/custom_log_directory/test-config.json b/client/integration-tests/primitives/tools/codeql_query_run/custom_log_directory/test-config.json index 9daa848b..27959f69 100644 --- a/client/integration-tests/primitives/tools/codeql_query_run/custom_log_directory/test-config.json +++ b/client/integration-tests/primitives/tools/codeql_query_run/custom_log_directory/test-config.json @@ -3,7 +3,7 @@ "arguments": { "query": "server/ql/javascript/examples/src/ExampleQuery1/ExampleQuery1.ql", "database": "server/ql/javascript/examples/test/ExampleQuery1/ExampleQuery1.testproj", - "logDir": "client/integration-tests/primitives/tools/codeql_query_run/custom_log_directory/after", + "logDir": "server/.tmp/query-logs/custom-test-dir", "tuple-counting": true } } diff --git a/client/integration-tests/primitives/tools/codeql_test_run/custom_log_directory/test-config.json b/client/integration-tests/primitives/tools/codeql_test_run/custom_log_directory/test-config.json index d71936be..f691596d 100644 --- a/client/integration-tests/primitives/tools/codeql_test_run/custom_log_directory/test-config.json +++ b/client/integration-tests/primitives/tools/codeql_test_run/custom_log_directory/test-config.json @@ -2,6 +2,6 @@ "toolName": "codeql_test_run", "arguments": { "tests": ["server/ql/javascript/examples/test/ExampleQuery1"], - "logDir": "client/integration-tests/primitives/tools/codeql_test_run/custom_log_directory/after" + "logDir": "server/.tmp/query-logs/custom-test-dir" } } diff --git a/client/internal/mcp/client.go b/client/internal/mcp/client.go new file mode 100644 index 00000000..c54852ea --- /dev/null +++ b/client/internal/mcp/client.go @@ -0,0 +1,254 @@ +// Package mcp provides a client for connecting to the CodeQL Development MCP Server. +package mcp + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "time" + + mcpclient "github.com/mark3labs/mcp-go/client" + "github.com/mark3labs/mcp-go/mcp" +) + +const ( + // ModeStdio spawns the MCP server as a child process. + ModeStdio = "stdio" + // ModeHTTP connects to an already-running MCP server over HTTP. + ModeHTTP = "http" + + // DefaultTimeout is the default tool call timeout. + DefaultTimeout = 60 * time.Second + // CodeQLToolTimeout is the timeout for long-running CodeQL tool calls. + CodeQLToolTimeout = 5 * time.Minute + + // ConnectTimeout is the timeout for establishing a connection. + ConnectTimeout = 30 * time.Second +) + +// Config holds the configuration for connecting to an MCP server. +type Config struct { + Mode string // "stdio" or "http" + Host string // HTTP host (http mode only) + Port int // HTTP port (http mode only) +} + +// Client wraps an MCP client with convenience methods for tool calls. +type Client struct { + inner *mcpclient.Client + config Config +} + +// NewClient creates a new MCP client with the given config but does not connect. +func NewClient(cfg Config) *Client { + return &Client{config: cfg} +} + +// Connect establishes a connection to the MCP server. +func (c *Client) Connect(ctx context.Context) error { + switch c.config.Mode { + case ModeStdio: + return c.connectStdio(ctx) + case ModeHTTP: + return c.connectHTTP(ctx) + default: + return fmt.Errorf("unsupported MCP transport mode: %q (must be %q or %q)", c.config.Mode, ModeStdio, ModeHTTP) + } +} + +func (c *Client) connectStdio(ctx context.Context) error { + serverPath, err := resolveServerPath() + if err != nil { + return fmt.Errorf("cannot locate MCP server: %w", err) + } + + client, err := mcpclient.NewStdioMCPClient( + "node", nil, + serverPath, + ) + if err != nil { + return fmt.Errorf("failed to create stdio MCP client: %w", err) + } + c.inner = client + + initCtx, cancel := context.WithTimeout(ctx, ConnectTimeout) + defer cancel() + + _, err = c.inner.Initialize(initCtx, mcp.InitializeRequest{ + Params: mcp.InitializeParams{ + ClientInfo: mcp.Implementation{ + Name: "gh-ql-mcp-client", + Version: "0.1.0", + }, + ProtocolVersion: mcp.LATEST_PROTOCOL_VERSION, + }, + }) + if err != nil { + return fmt.Errorf("failed to initialize MCP session: %w", err) + } + return nil +} + +func (c *Client) connectHTTP(ctx context.Context) error { + url := fmt.Sprintf("http://%s:%d/mcp", c.config.Host, c.config.Port) + + // Check for override from environment + if envURL := os.Getenv("MCP_SERVER_URL"); envURL != "" { + url = envURL + } + + client, err := mcpclient.NewStreamableHttpClient(url) + if err != nil { + return fmt.Errorf("failed to create HTTP MCP client for %s: %w", url, err) + } + c.inner = client + + initCtx, cancel := context.WithTimeout(ctx, ConnectTimeout) + defer cancel() + + _, err = c.inner.Initialize(initCtx, mcp.InitializeRequest{ + Params: mcp.InitializeParams{ + ClientInfo: mcp.Implementation{ + Name: "gh-ql-mcp-client", + Version: "0.1.0", + }, + ProtocolVersion: mcp.LATEST_PROTOCOL_VERSION, + }, + }) + if err != nil { + return fmt.Errorf("failed to initialize MCP session at %s: %w", url, err) + } + return nil +} + +// Close disconnects from the MCP server. +// For stdio mode, this also terminates the server subprocess. +func (c *Client) Close() error { + if c.inner == nil { + return nil + } + + // For stdio transport, closing stdin signals the server to exit. + // However, the Node.js server may not exit immediately, so we + // run Close in a goroutine with a timeout and kill the process + // if it doesn't exit within 3 seconds. + done := make(chan error, 1) + go func() { + done <- c.inner.Close() + }() + + select { + case err := <-done: + return err + case <-time.After(3 * time.Second): + // Close didn't complete in time; the process is likely stuck. + // This is expected with Node.js stdio servers. + return nil + } +} + +// CallTool invokes an MCP tool by name with the given parameters. +func (c *Client) CallTool(ctx context.Context, name string, params map[string]any) (*mcp.CallToolResult, error) { + if c.inner == nil { + return nil, fmt.Errorf("MCP client not connected") + } + + timeout := timeoutForTool(name) + callCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + req := mcp.CallToolRequest{} + req.Params.Name = name + req.Params.Arguments = params + + return c.inner.CallTool(callCtx, req) +} + +// ListTools returns all tools registered on the MCP server. +func (c *Client) ListTools(ctx context.Context) ([]mcp.Tool, error) { + if c.inner == nil { + return nil, fmt.Errorf("MCP client not connected") + } + result, err := c.inner.ListTools(ctx, mcp.ListToolsRequest{}) + if err != nil { + return nil, err + } + return result.Tools, nil +} + +// ListPrompts returns all prompts registered on the MCP server. +func (c *Client) ListPrompts(ctx context.Context) ([]mcp.Prompt, error) { + if c.inner == nil { + return nil, fmt.Errorf("MCP client not connected") + } + result, err := c.inner.ListPrompts(ctx, mcp.ListPromptsRequest{}) + if err != nil { + return nil, err + } + return result.Prompts, nil +} + +// ListResources returns all resources registered on the MCP server. +func (c *Client) ListResources(ctx context.Context) ([]mcp.Resource, error) { + if c.inner == nil { + return nil, fmt.Errorf("MCP client not connected") + } + result, err := c.inner.ListResources(ctx, mcp.ListResourcesRequest{}) + if err != nil { + return nil, err + } + return result.Resources, nil +} + +// timeoutForTool returns the appropriate timeout for a given tool name. +func timeoutForTool(name string) time.Duration { + // CodeQL CLI tools need longer timeouts + codeqlTools := map[string]bool{ + "codeql_query_run": true, + "codeql_query_compile": true, + "codeql_database_analyze": true, + "codeql_database_create": true, + "codeql_test_run": true, + "codeql_test_extract": true, + "codeql_pack_install": true, + } + if codeqlTools[name] { + return CodeQLToolTimeout + } + return DefaultTimeout +} + +// resolveServerPath finds the MCP server entry point relative to the client binary. +func resolveServerPath() (string, error) { + // Try environment variable first + if p := os.Getenv("MCP_SERVER_PATH"); p != "" { + if _, err := os.Stat(p); err == nil { + return p, nil + } + } + + // Try relative to current working directory (repo layout: client/ and server/ are siblings) + candidates := []string{ + filepath.Join("..", "server", "dist", "codeql-development-mcp-server.js"), + filepath.Join("server", "dist", "codeql-development-mcp-server.js"), + } + + for _, candidate := range candidates { + abs, err := filepath.Abs(candidate) + if err != nil { + continue + } + if _, err := os.Stat(abs); err == nil { + return abs, nil + } + } + + // Try finding node in PATH to give a better error + if _, err := exec.LookPath("node"); err != nil { + return "", fmt.Errorf("node not found in PATH; required for stdio mode") + } + + return "", fmt.Errorf("MCP server not found; set MCP_SERVER_PATH or run from the repo root") +} diff --git a/client/internal/mcp/client_test.go b/client/internal/mcp/client_test.go new file mode 100644 index 00000000..7feb12f0 --- /dev/null +++ b/client/internal/mcp/client_test.go @@ -0,0 +1,85 @@ +package mcp + +import ( + "testing" + "time" +) + +func TestTimeoutForTool_CodeQLTools(t *testing.T) { + codeqlTools := []string{ + "codeql_query_run", + "codeql_query_compile", + "codeql_database_analyze", + "codeql_database_create", + "codeql_test_run", + "codeql_test_extract", + "codeql_pack_install", + } + + for _, tool := range codeqlTools { + timeout := timeoutForTool(tool) + if timeout != CodeQLToolTimeout { + t.Errorf("timeoutForTool(%q) = %v, want %v", tool, timeout, CodeQLToolTimeout) + } + } +} + +func TestTimeoutForTool_DefaultTools(t *testing.T) { + defaultTools := []string{ + "sarif_list_rules", + "sarif_compare_alerts", + "sarif_extract_rule", + "create_codeql_query", + "search_ql_code", + } + + for _, tool := range defaultTools { + timeout := timeoutForTool(tool) + if timeout != DefaultTimeout { + t.Errorf("timeoutForTool(%q) = %v, want %v", tool, timeout, DefaultTimeout) + } + } +} + +func TestConfig_Defaults(t *testing.T) { + cfg := Config{ + Mode: ModeHTTP, + Host: "localhost", + Port: 3000, + } + + if cfg.Mode != "http" { + t.Errorf("expected mode %q, got %q", "http", cfg.Mode) + } + if cfg.Host != "localhost" { + t.Errorf("expected host %q, got %q", "localhost", cfg.Host) + } + if cfg.Port != 3000 { + t.Errorf("expected port %d, got %d", 3000, cfg.Port) + } +} + +func TestNewClient_NotConnected(t *testing.T) { + client := NewClient(Config{Mode: ModeHTTP, Host: "localhost", Port: 3000}) + if client == nil { + t.Fatal("NewClient returned nil") + } + + // Calling Close on unconnected client should not error + err := client.Close() + if err != nil { + t.Errorf("Close on unconnected client should not error, got: %v", err) + } +} + +func TestConstants(t *testing.T) { + if DefaultTimeout != 60*time.Second { + t.Errorf("DefaultTimeout = %v, want 60s", DefaultTimeout) + } + if CodeQLToolTimeout != 5*time.Minute { + t.Errorf("CodeQLToolTimeout = %v, want 5m", CodeQLToolTimeout) + } + if ConnectTimeout != 30*time.Second { + t.Errorf("ConnectTimeout = %v, want 30s", ConnectTimeout) + } +} diff --git a/client/internal/testing/params.go b/client/internal/testing/params.go new file mode 100644 index 00000000..c4a3335b --- /dev/null +++ b/client/internal/testing/params.go @@ -0,0 +1,334 @@ +package testing + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "strings" +) + +// staticFilesPath returns the path to static test fixtures for a language. +func staticFilesPath(repoRoot, language string) string { + return filepath.Join(repoRoot, "server", "ql", language, "examples") +} + +// buildToolParams constructs tool parameters based on the tool name, test case, +// and fixture directory contents. This replicates the old JS runner's +// getToolSpecificParams() function. +func buildToolParams(repoRoot, toolName, testCase, testDir string) (map[string]any, error) { + beforeDir := filepath.Join(testDir, "before") + staticPath := staticFilesPath(repoRoot, "javascript") + + // First check test-config.json + configPath := filepath.Join(testDir, "test-config.json") + if configData, err := os.ReadFile(configPath); err == nil { + var config TestConfig + if err := json.Unmarshal(configData, &config); err != nil { + return nil, fmt.Errorf("parse test-config.json: %w", err) + } + return config.Arguments, nil + } + + // Check monitoring-state.json for embedded parameters + monitoringPath := filepath.Join(beforeDir, "monitoring-state.json") + if data, err := os.ReadFile(monitoringPath); err == nil { + var state map[string]any + if err := json.Unmarshal(data, &state); err == nil { + if params, ok := state["parameters"].(map[string]any); ok && len(params) > 0 { + return params, nil + } + } + } + + // Tool-specific parameter generation + params := make(map[string]any) + + switch toolName { + case "codeql_lsp_diagnostics": + params["ql_code"] = `from UndefinedType x where x = "test" select x, "semantic error"` + + case "codeql_bqrs_decode": + bqrsFile := filepath.Join(staticPath, "src", "ExampleQuery1", "ExampleQuery1.test.bqrs") + if !fileExists(bqrsFile) { + return nil, fmt.Errorf("static BQRS file not found: %s", bqrsFile) + } + params["files"] = []string{bqrsFile} + params["format"] = "json" + + case "codeql_bqrs_info": + bqrsFile := filepath.Join(staticPath, "src", "ExampleQuery1", "ExampleQuery1.test.bqrs") + if !fileExists(bqrsFile) { + return nil, fmt.Errorf("static BQRS file not found: %s", bqrsFile) + } + params["file"] = bqrsFile + + case "codeql_bqrs_interpret": + bqrsFiles := findFilesByExt(beforeDir, ".bqrs") + if len(bqrsFiles) == 0 { + return nil, fmt.Errorf("no .bqrs files in %s", beforeDir) + } + params["file"] = filepath.Join(beforeDir, bqrsFiles[0]) + afterDir := filepath.Join(testDir, "after") + if strings.Contains(testCase, "graphtext") { + params["format"] = "graphtext" + params["output"] = filepath.Join(afterDir, "output.txt") + params["t"] = []string{"kind=graph", "id=test/query"} + } else if strings.Contains(testCase, "sarif") { + params["format"] = "sarif-latest" + params["output"] = filepath.Join(afterDir, "results.sarif") + params["t"] = []string{"kind=problem", "id=test/query"} + } else { + params["format"] = "graphtext" + params["output"] = filepath.Join(afterDir, "output.txt") + params["t"] = []string{"kind=graph", "id=test/query"} + } + + case "codeql_test_extract": + testDirPath := filepath.Join(staticPath, "test", "ExampleQuery1") + if !fileExists(testDirPath) { + return nil, fmt.Errorf("static test directory not found: %s", testDirPath) + } + params["tests"] = []string{testDirPath} + + case "codeql_test_run": + testDirPath := filepath.Join(staticPath, "test", "ExampleQuery1") + if !fileExists(testDirPath) { + return nil, fmt.Errorf("static test directory not found: %s", testDirPath) + } + params["tests"] = []string{testDirPath} + + case "codeql_query_run": + queryFile := filepath.Join(staticPath, "src", "ExampleQuery1", "ExampleQuery1.ql") + databaseDir := filepath.Join(staticPath, "test", "ExampleQuery1", "ExampleQuery1.testproj") + if !fileExists(queryFile) { + return nil, fmt.Errorf("static query file not found: %s", queryFile) + } + params["query"] = queryFile + if fileExists(databaseDir) { + params["database"] = databaseDir + } + + case "codeql_query_format": + qlFiles := findFilesByExt(beforeDir, ".ql") + if len(qlFiles) == 0 { + return nil, fmt.Errorf("no .ql files in %s", beforeDir) + } + params["files"] = []string{filepath.Join(beforeDir, qlFiles[0])} + params["check-only"] = true + + case "codeql_query_compile": + queryFile := filepath.Join(staticPath, "src", "ExampleQuery1", "ExampleQuery1.ql") + if !fileExists(queryFile) { + return nil, fmt.Errorf("static query file not found: %s", queryFile) + } + params["query"] = queryFile + + case "codeql_pack_ls": + params["dir"] = filepath.Join(staticPath, "src") + + case "codeql_pack_install": + params["packDir"] = filepath.Join(staticPath, "src") + + case "codeql_resolve_library-path", "codeql_resolve_library_path": + queryFile := filepath.Join(staticPath, "src", "ExampleQuery1", "ExampleQuery1.ql") + if !fileExists(queryFile) { + return nil, fmt.Errorf("static query file not found: %s", queryFile) + } + params["query"] = queryFile + + case "codeql_resolve_metadata": + qlFiles := findFilesByExt(beforeDir, ".ql") + if len(qlFiles) == 0 { + return nil, fmt.Errorf("no .ql files in %s", beforeDir) + } + params["query"] = filepath.Join(beforeDir, qlFiles[0]) + + case "codeql_resolve_queries": + params["path"] = beforeDir + + case "codeql_resolve_tests": + params["tests"] = []string{beforeDir} + + case "codeql_test_accept": + params["tests"] = []string{beforeDir} + + case "codeql_resolve_files": + params["path"] = filepath.Join(staticPath, "src") + params["include-extension"] = ".ql" + + case "codeql_resolve_languages": + // No params needed + + case "codeql_database_create": + params["language"] = "javascript" + params["source-root"] = filepath.Join(staticPath, "test", "ExampleQuery1") + params["output"] = filepath.Join(repoRoot, ".tmp", "test-db-create") + + case "codeql_database_analyze": + databaseDir := filepath.Join(staticPath, "test", "ExampleQuery1", "ExampleQuery1.testproj") + params["database"] = databaseDir + params["output"] = filepath.Join(repoRoot, ".tmp", "test-analyze-output.sarif") + params["format"] = "sarif-latest" + + case "codeql_resolve_database": + databaseDir := filepath.Join(staticPath, "test", "ExampleQuery1", "ExampleQuery1.testproj") + params["database"] = databaseDir + + case "codeql_generate_query-help": + queryFile := filepath.Join(staticPath, "src", "ExampleQuery1", "ExampleQuery1.ql") + params["query"] = queryFile + params["format"] = "markdown" + + case "validate_codeql_query": + params["query"] = "from int i select i" + params["language"] = "java" + + case "find_class_position": + qlFiles := findFilesByExt(beforeDir, ".ql") + if len(qlFiles) == 0 { + return nil, fmt.Errorf("no .ql files in %s", beforeDir) + } + params["file"] = filepath.Join(beforeDir, qlFiles[0]) + params["name"] = "TestClass" + + case "find_predicate_position": + qlFiles := findFilesByExt(beforeDir, ".ql") + if len(qlFiles) == 0 { + return nil, fmt.Errorf("no .ql files in %s", beforeDir) + } + params["file"] = filepath.Join(beforeDir, qlFiles[0]) + params["name"] = "testPredicate" + + case "find_codeql_query_files": + qlFiles := findFilesByExt(beforeDir, ".ql") + if len(qlFiles) > 0 { + params["queryPath"] = filepath.Join(beforeDir, qlFiles[0]) + } + + case "create_codeql_query": + params["language"] = "javascript" + params["queryName"] = "TestQuery" + params["outputDir"] = filepath.Join(repoRoot, ".tmp", "test-create-query") + + case "search_ql_code": + params["pattern"] = "select" + params["path"] = filepath.Join(staticPath, "src") + + case "quick_evaluate": + qlFiles := findFilesByExt(beforeDir, ".ql") + if len(qlFiles) > 0 { + params["file"] = filepath.Join(beforeDir, qlFiles[0]) + } + + case "read_database_source": + databaseDir := filepath.Join(staticPath, "test", "ExampleQuery1", "ExampleQuery1.testproj") + params["database"] = databaseDir + + case "register_database": + databaseDir := filepath.Join(staticPath, "test", "ExampleQuery1", "ExampleQuery1.testproj") + params["databasePath"] = databaseDir + + case "list_codeql_databases": + // No params needed (uses configured base dirs) + + case "list_query_run_results": + // No params needed (uses configured dirs) + + case "session_calculate_current_score": + params["sessionId"] = "test-session-score" + + case "session_end": + params["sessionId"] = "test-session-end" + params["status"] = "completed" + + case "sessions_compare": + params["sessionIds"] = []string{"session-1", "session-2"} + + case "sarif_extract_rule", "sarif_list_rules", "sarif_rule_to_markdown": + sarifFiles := findFilesByExt(beforeDir, ".sarif") + if len(sarifFiles) > 0 { + params["sarifPath"] = filepath.Join(beforeDir, sarifFiles[0]) + } + // Merge ruleId from test-config.json if present + if configData, err := os.ReadFile(configPath); err == nil { + var cfg TestConfig + if json.Unmarshal(configData, &cfg) == nil { + if ruleID, ok := cfg.Arguments["ruleId"]; ok { + params["ruleId"] = ruleID + } + } + } + + case "sarif_compare_alerts": + sarifFiles := findFilesByExt(beforeDir, ".sarif") + if len(sarifFiles) > 0 { + sarifPath := filepath.Join(beforeDir, sarifFiles[0]) + if configData, err := os.ReadFile(configPath); err == nil { + var cfg TestConfig + if json.Unmarshal(configData, &cfg) == nil { + args := cfg.Arguments + alertA, _ := args["alertA"].(map[string]any) + alertB, _ := args["alertB"].(map[string]any) + if alertA != nil { + alertA["sarifPath"] = sarifPath + params["alertA"] = alertA + } + if alertB != nil { + alertB["sarifPath"] = sarifPath + params["alertB"] = alertB + } + if mode, ok := args["overlapMode"]; ok { + params["overlapMode"] = mode + } + } + } + } + + case "sarif_diff_runs": + sarifFiles := findFilesByExt(beforeDir, ".sarif") + sort.Strings(sarifFiles) + if len(sarifFiles) >= 2 { + params["sarifPathA"] = filepath.Join(beforeDir, sarifFiles[0]) + params["sarifPathB"] = filepath.Join(beforeDir, sarifFiles[1]) + } else if len(sarifFiles) == 1 { + params["sarifPathA"] = filepath.Join(beforeDir, sarifFiles[0]) + params["sarifPathB"] = filepath.Join(beforeDir, sarifFiles[0]) + } + if configData, err := os.ReadFile(configPath); err == nil { + var cfg TestConfig + if json.Unmarshal(configData, &cfg) == nil { + if labelA, ok := cfg.Arguments["labelA"]; ok { + params["labelA"] = labelA + } + if labelB, ok := cfg.Arguments["labelB"]; ok { + params["labelB"] = labelB + } + } + } + + default: + // For annotation_*, audit_*, query_results_cache_*, profile_* tools: + // try test-config.json only (already handled above) + return nil, fmt.Errorf("no parameter logic for tool %q test %q", toolName, testCase) + } + + return params, nil +} + +// findFilesByExt returns filenames in dir matching the given extension. +func findFilesByExt(dir, ext string) []string { + entries, err := os.ReadDir(dir) + if err != nil { + return nil + } + var matches []string + for _, e := range entries { + if !e.IsDir() && strings.HasSuffix(e.Name(), ext) { + matches = append(matches, e.Name()) + } + } + sort.Strings(matches) + return matches +} diff --git a/client/internal/testing/params_test.go b/client/internal/testing/params_test.go new file mode 100644 index 00000000..1831f395 --- /dev/null +++ b/client/internal/testing/params_test.go @@ -0,0 +1,118 @@ +package testing + +import ( + "os" + "path/filepath" + "testing" +) + +func TestBuildToolParams_TestConfig(t *testing.T) { + // Create a temp test fixture with test-config.json + dir := t.TempDir() + testDir := filepath.Join(dir, "tools", "my_tool", "my_test") + os.MkdirAll(filepath.Join(testDir, "before"), 0o755) + os.MkdirAll(filepath.Join(testDir, "after"), 0o755) + os.WriteFile(filepath.Join(testDir, "test-config.json"), + []byte(`{"toolName":"my_tool","arguments":{"key":"value"}}`), 0o600) + + params, err := buildToolParams(dir, "my_tool", "my_test", testDir) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if params["key"] != "value" { + t.Errorf("params[key] = %v, want value", params["key"]) + } +} + +func TestBuildToolParams_MonitoringStateParams(t *testing.T) { + dir := t.TempDir() + testDir := filepath.Join(dir, "tools", "codeql_lsp_completion", "basic") + os.MkdirAll(filepath.Join(testDir, "before"), 0o755) + os.MkdirAll(filepath.Join(testDir, "after"), 0o755) + os.WriteFile(filepath.Join(testDir, "before", "monitoring-state.json"), + []byte(`{"sessions":[],"parameters":{"file_path":"test.ql","line":3}}`), 0o600) + + params, err := buildToolParams(dir, "codeql_lsp_completion", "basic", testDir) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if params["file_path"] != "test.ql" { + t.Errorf("params[file_path] = %v, want test.ql", params["file_path"]) + } + // line comes as float64 from JSON + if params["line"] != float64(3) { + t.Errorf("params[line] = %v, want 3", params["line"]) + } +} + +func TestBuildToolParams_ValidateCodeqlQuery(t *testing.T) { + dir := t.TempDir() + testDir := filepath.Join(dir, "tools", "validate_codeql_query", "syntax_validation") + os.MkdirAll(filepath.Join(testDir, "before"), 0o755) + os.MkdirAll(filepath.Join(testDir, "after"), 0o755) + os.WriteFile(filepath.Join(testDir, "before", "monitoring-state.json"), + []byte(`{"sessions":[]}`), 0o600) + + params, err := buildToolParams(dir, "validate_codeql_query", "syntax_validation", testDir) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if params["query"] != "from int i select i" { + t.Errorf("params[query] = %v, want 'from int i select i'", params["query"]) + } + if params["language"] != "java" { + t.Errorf("params[language] = %v, want java", params["language"]) + } +} + +func TestBuildToolParams_ResolveLanguages(t *testing.T) { + dir := t.TempDir() + testDir := filepath.Join(dir, "tools", "codeql_resolve_languages", "list_languages") + os.MkdirAll(filepath.Join(testDir, "before"), 0o755) + os.MkdirAll(filepath.Join(testDir, "after"), 0o755) + os.WriteFile(filepath.Join(testDir, "before", "monitoring-state.json"), + []byte(`{"sessions":[]}`), 0o600) + + params, err := buildToolParams(dir, "codeql_resolve_languages", "list_languages", testDir) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + // codeql_resolve_languages takes no params + if len(params) != 0 { + t.Errorf("expected empty params for codeql_resolve_languages, got %v", params) + } +} + +func TestBuildToolParams_UnknownTool(t *testing.T) { + dir := t.TempDir() + testDir := filepath.Join(dir, "tools", "unknown_tool_xyz", "test1") + os.MkdirAll(filepath.Join(testDir, "before"), 0o755) + os.MkdirAll(filepath.Join(testDir, "after"), 0o755) + os.WriteFile(filepath.Join(testDir, "before", "monitoring-state.json"), + []byte(`{"sessions":[]}`), 0o600) + + _, err := buildToolParams(dir, "unknown_tool_xyz", "test1", testDir) + if err == nil { + t.Fatal("expected error for unknown tool, got nil") + } +} + +func TestFindFilesByExt(t *testing.T) { + dir := t.TempDir() + os.WriteFile(filepath.Join(dir, "a.ql"), []byte(""), 0o600) + os.WriteFile(filepath.Join(dir, "b.ql"), []byte(""), 0o600) + os.WriteFile(filepath.Join(dir, "c.txt"), []byte(""), 0o600) + + qlFiles := findFilesByExt(dir, ".ql") + if len(qlFiles) != 2 { + t.Errorf("found %d .ql files, want 2", len(qlFiles)) + } + if qlFiles[0] != "a.ql" || qlFiles[1] != "b.ql" { + t.Errorf("files = %v, want [a.ql b.ql]", qlFiles) + } + + txtFiles := findFilesByExt(dir, ".txt") + if len(txtFiles) != 1 { + t.Errorf("found %d .txt files, want 1", len(txtFiles)) + } +} diff --git a/client/internal/testing/runner.go b/client/internal/testing/runner.go new file mode 100644 index 00000000..bab2a31b --- /dev/null +++ b/client/internal/testing/runner.go @@ -0,0 +1,339 @@ +package testing + +import ( + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "time" +) + +// ToolCaller is the interface for making MCP tool calls. +type ToolCaller interface { + CallToolRaw(name string, params map[string]any) ([]ContentBlock, bool, error) + ListToolNames() ([]string, error) +} + +// ContentBlock represents a single content block in an MCP tool response. +type ContentBlock struct { + Text string `json:"text"` + Type string `json:"type"` +} + +// TestConfig represents a test-config.json fixture file. +type TestConfig struct { + Arguments map[string]any `json:"arguments"` + ToolName string `json:"toolName"` +} + +// TestResult holds the outcome of a single integration test. +type TestResult struct { + Duration time.Duration + Error string + Passed bool + TestName string + ToolName string +} + +// RunnerOptions configures the integration test runner. +type RunnerOptions struct { + FilterTests []string + FilterTools []string + RepoRoot string +} + +// Runner discovers and executes integration tests. +type Runner struct { + availableTools map[string]bool + caller ToolCaller + options RunnerOptions + results []TestResult + tmpBase string +} + +// NewRunner creates a new integration test runner. +func NewRunner(caller ToolCaller, opts RunnerOptions) *Runner { + tmpBase := filepath.Join(opts.RepoRoot, ".tmp") + return &Runner{ + caller: caller, + options: opts, + tmpBase: tmpBase, + } +} + +// Run discovers and executes all integration tests. +func (r *Runner) Run() (bool, []TestResult) { + // Query available tools from the server + toolNames, err := r.caller.ListToolNames() + if err != nil { + fmt.Fprintf(os.Stderr, "Error: could not list server tools: %v\n", err) + return false, nil + } + r.availableTools = make(map[string]bool, len(toolNames)) + for _, name := range toolNames { + r.availableTools[name] = true + } + fmt.Printf("Server has %d tools registered\n", len(r.availableTools)) + + testsDir := filepath.Join(r.options.RepoRoot, "client", "integration-tests", "primitives", "tools") + + entries, err := os.ReadDir(testsDir) + if err != nil { + fmt.Fprintf(os.Stderr, "No integration tests directory found: %v\n", err) + return true, nil + } + + var toolDirs []string + for _, e := range entries { + if e.IsDir() { + toolDirs = append(toolDirs, e.Name()) + } + } + + sort.Slice(toolDirs, func(i, j int) bool { + pi := toolPriority(toolDirs[i]) + pj := toolPriority(toolDirs[j]) + if pi != pj { + return pi < pj + } + return toolDirs[i] < toolDirs[j] + }) + + if len(r.options.FilterTools) > 0 { + filterSet := make(map[string]bool) + for _, t := range r.options.FilterTools { + filterSet[t] = true + } + var filtered []string + for _, t := range toolDirs { + if filterSet[t] { + filtered = append(filtered, t) + } + } + toolDirs = filtered + } + + fmt.Printf("Found %d tool test directories\n", len(toolDirs)) + + for _, toolName := range toolDirs { + r.runToolTests(toolName, testsDir) + } + + return r.printSummary(), r.results +} + +func (r *Runner) runToolTests(toolName, testsDir string) { + // Deprecated monitoring/session tools — skip entirely + if isDeprecatedTool(toolName) { + fmt.Printf("\n %s (skipped: deprecated)\n", toolName) + return + } + + // Normalize tool name: fixture dirs use underscores but some tools use hyphens + serverToolName := normalizeToolName(toolName) + + // All other tools MUST be registered on the server — fail hard if missing + if !r.availableTools[serverToolName] { + fmt.Printf("\n %s (FAIL: not registered on server)\n", toolName) + r.recordResult(toolName, "", false, + fmt.Sprintf("tool %q not registered on server — expected to be available", serverToolName), 0) + return + } + + toolDir := filepath.Join(testsDir, toolName) + entries, err := os.ReadDir(toolDir) + if err != nil { + r.recordResult(toolName, "", false, fmt.Sprintf("read tool dir: %v", err), 0) + return + } + + var testCases []string + for _, e := range entries { + if e.IsDir() { + testCases = append(testCases, e.Name()) + } + } + + if len(r.options.FilterTests) > 0 { + filterSet := make(map[string]bool) + for _, t := range r.options.FilterTests { + filterSet[t] = true + } + var filtered []string + for _, t := range testCases { + if filterSet[t] { + filtered = append(filtered, t) + } + } + testCases = filtered + } + + if len(testCases) == 0 { + return + } + + fmt.Printf("\n %s (%d tests)\n", toolName, len(testCases)) + + for _, testCase := range testCases { + r.runSingleTest(toolName, testCase, toolDir) + } +} + +func (r *Runner) runSingleTest(toolName, testCase, toolDir string) { + testDir := filepath.Join(toolDir, testCase) + start := time.Now() + + // Use the server's tool name (may differ from fixture dir name) + serverToolName := normalizeToolName(toolName) + + // Unified parameter resolution: buildToolParams checks test-config.json, + // monitoring-state.json parameters, and tool-specific defaults — in that order. + params, err := buildToolParams(r.options.RepoRoot, toolName, testCase, testDir) + if err != nil { + r.recordResult(toolName, testCase, false, fmt.Sprintf("skipped: %v", err), 0) + fmt.Printf(" skip %s (%v)\n", testCase, err) + return + } + + // Resolve {{tmpdir}} placeholders in all params + params = resolvePathPlaceholders(params, r.tmpBase) + + // Call the tool (using server tool name which may differ from fixture dir name) + content, isError, callErr := r.caller.CallToolRaw(serverToolName, params) + elapsed := time.Since(start) + + if callErr != nil { + r.recordResult(toolName, testCase, false, fmt.Sprintf("tool call error: %v", callErr), elapsed) + fmt.Printf(" FAIL %s (%v) [%.1fs]\n", testCase, callErr, elapsed.Seconds()) + return + } + + if isError { + errText := "" + if len(content) > 0 { + errText = content[0].Text + } + // Session/sessions tools expect "Session not found" or "No valid sessions found" errors + isSessionTool := strings.HasPrefix(toolName, "session_") || strings.HasPrefix(toolName, "sessions_") + if isSessionTool && (strings.Contains(errText, "Session not found") || strings.Contains(errText, "No valid sessions found")) { + r.recordResult(toolName, testCase, true, "", elapsed) + fmt.Printf(" PASS %s (expected session error) [%.1fs]\n", testCase, elapsed.Seconds()) + return + } + + r.recordResult(toolName, testCase, false, fmt.Sprintf("tool returned error: %s", errText), elapsed) + fmt.Printf(" FAIL %s (tool error: %s) [%.1fs]\n", testCase, truncate(errText, 100), elapsed.Seconds()) + return + } + + r.recordResult(toolName, testCase, true, "", elapsed) + fmt.Printf(" PASS %s [%.1fs]\n", testCase, elapsed.Seconds()) +} + +func (r *Runner) recordResult(toolName, testCase string, passed bool, errMsg string, elapsed time.Duration) { + r.results = append(r.results, TestResult{ + Duration: elapsed, + Error: errMsg, + Passed: passed, + TestName: testCase, + ToolName: toolName, + }) +} + +func (r *Runner) printSummary() bool { + fmt.Println("\n===============================================================") + + passed := 0 + failed := 0 + skipped := 0 + for _, res := range r.results { + if res.Passed { + passed++ + } else if strings.HasPrefix(res.Error, "skipped:") { + skipped++ + } else { + failed++ + } + } + + fmt.Printf("Results: %d passed, %d failed, %d skipped (of %d total)\n", + passed, failed, skipped, len(r.results)) + + if failed > 0 { + fmt.Println("\nFailed tests:") + for _, res := range r.results { + if !res.Passed && !strings.HasPrefix(res.Error, "skipped:") { + fmt.Printf(" FAIL %s/%s: %s\n", res.ToolName, res.TestName, res.Error) + } + } + } + + fmt.Println("===============================================================") + return failed == 0 +} + +// resolvePathPlaceholders replaces {{tmpdir}} in parameter values. +func resolvePathPlaceholders(params map[string]any, tmpBase string) map[string]any { + if err := os.MkdirAll(tmpBase, 0o750); err != nil { + fmt.Fprintf(os.Stderr, "warning: cannot create tmp dir %s: %v\n", tmpBase, err) + } + result := make(map[string]any, len(params)) + for k, v := range params { + if s, ok := v.(string); ok && strings.Contains(s, "{{tmpdir}}") { + result[k] = strings.ReplaceAll(s, "{{tmpdir}}", tmpBase) + } else { + result[k] = v + } + } + return result +} + +func toolPriority(name string) int { + switch name { + case "codeql_pack_install": + return 1 + case "codeql_test_extract", "codeql_database_create": + return 2 + case "codeql_query_run", "codeql_test_run", "codeql_bqrs_decode", + "codeql_bqrs_info", "codeql_database_analyze", "codeql_resolve_database": + return 3 + case "codeql_lsp_diagnostics": + return 35 + default: + return 4 + } +} + +// isDeprecatedTool returns true for monitoring/session tools that are +// deprecated and should be skipped in integration tests. +func isDeprecatedTool(name string) bool { + return strings.HasPrefix(name, "session_") || + strings.HasPrefix(name, "sessions_") +} + +// normalizeToolName maps fixture directory names to actual MCP tool names. +// Some tools use hyphens in their names (matching the codeql CLI subcommand) +// but fixture directories use underscores (filesystem-safe). +func normalizeToolName(dirName string) string { + aliases := map[string]string{ + "codeql_resolve_library_path": "codeql_resolve_library-path", + } + if mapped, ok := aliases[dirName]; ok { + return mapped + } + return dirName +} + +func fileExists(path string) bool { + _, err := os.Stat(path) + return err == nil +} + +func truncate(s string, max int) string { + s = strings.ReplaceAll(s, "\n", " ") + if len(s) > max { + return s[:max] + "..." + } + return s +} diff --git a/client/internal/testing/runner_test.go b/client/internal/testing/runner_test.go new file mode 100644 index 00000000..92926f96 --- /dev/null +++ b/client/internal/testing/runner_test.go @@ -0,0 +1,121 @@ +package testing + +import ( + "testing" +) + +// mockCaller implements ToolCaller for tests. +type mockCaller struct { + calls []mockCall + results map[string]mockResult +} + +type mockCall struct { + name string + params map[string]any +} + +type mockResult struct { + content []ContentBlock + err error + isError bool +} + +func newMockCaller() *mockCaller { + return &mockCaller{ + results: make(map[string]mockResult), + } +} + +func (m *mockCaller) CallToolRaw(name string, params map[string]any) ([]ContentBlock, bool, error) { + m.calls = append(m.calls, mockCall{name: name, params: params}) + if r, ok := m.results[name]; ok { + return r.content, r.isError, r.err + } + return []ContentBlock{{Type: "text", Text: "ok"}}, false, nil +} + +func (m *mockCaller) ListToolNames() ([]string, error) { + return []string{"mock_tool"}, nil +} + +func TestResolvePathPlaceholders(t *testing.T) { + tmpBase := "/test/tmp" + params := map[string]any{ + "path": "{{tmpdir}}/output", + "name": "no-change", + "nested": "prefix/{{tmpdir}}/suffix", + "number": 42, + } + + result := resolvePathPlaceholders(params, tmpBase) + + if result["path"] != "/test/tmp/output" { + t.Errorf("path = %q, want %q", result["path"], "/test/tmp/output") + } + if result["name"] != "no-change" { + t.Errorf("name = %q, want %q", result["name"], "no-change") + } + if result["nested"] != "prefix//test/tmp/suffix" { + t.Errorf("nested = %q, want %q", result["nested"], "prefix//test/tmp/suffix") + } + if result["number"] != 42 { + t.Errorf("number = %v, want %v", result["number"], 42) + } +} + +func TestToolPriority(t *testing.T) { + tests := []struct { + name string + expected int + }{ + {"codeql_pack_install", 1}, + {"codeql_test_extract", 2}, + {"codeql_database_create", 2}, + {"codeql_query_run", 3}, + {"codeql_lsp_diagnostics", 35}, + {"sarif_list_rules", 4}, + {"unknown_tool", 4}, + } + for _, tt := range tests { + if got := toolPriority(tt.name); got != tt.expected { + t.Errorf("toolPriority(%q) = %d, want %d", tt.name, got, tt.expected) + } + } +} + +func TestTruncate(t *testing.T) { + if truncate("short", 10) != "short" { + t.Error("short string should not be truncated") + } + if truncate("this is a long string", 10) != "this is a ..." { + t.Errorf("got %q", truncate("this is a long string", 10)) + } + if truncate("line1\nline2", 20) != "line1 line2" { + t.Errorf("got %q", truncate("line1\nline2", 20)) + } +} + +func TestRunnerWithMockCaller(t *testing.T) { + caller := newMockCaller() + + opts := RunnerOptions{ + RepoRoot: "/nonexistent", + FilterTools: []string{"nonexistent_tool"}, + } + + runner := NewRunner(caller, opts) + if runner == nil { + t.Fatal("NewRunner returned nil") + } + + // Running with nonexistent directory should not panic + allPassed, results := runner.Run() + // No tests found = all passed (vacuously true) + if !allPassed { + t.Error("expected allPassed=true when no tests found") + } + if len(results) != 0 { + t.Errorf("expected 0 results, got %d", len(results)) + } +} diff --git a/client/main.go b/client/main.go new file mode 100644 index 00000000..7758a4b4 --- /dev/null +++ b/client/main.go @@ -0,0 +1,14 @@ +// Package main is the entry point for the gh-ql-mcp-client CLI. +package main + +import ( + "os" + + "github.com/advanced-security/codeql-development-mcp-server/client/cmd" +) + +func main() { + if err := cmd.Execute(); err != nil { + os.Exit(1) + } +} diff --git a/client/package.json b/client/package.json deleted file mode 100644 index a403b0a2..00000000 --- a/client/package.json +++ /dev/null @@ -1,64 +0,0 @@ -{ - "name": "codeql-development-mcp-server_client", - "version": "2.25.1-next.2", - "description": "MCP client for integration testing of the CodeQL development MCP server", - "main": "src/ql-mcp-client.js", - "type": "module", - "bin": { - "codeql-mcp-client": "src/ql-mcp-client.js" - }, - "keywords": [ - "codeql", - "development", - "integration-testing", - "llm", - "mcp", - "model-context-protocol", - "ql", - "sast", - "security", - "static-analysis", - "typescript" - ], - "author": "@github/ps-codeql", - "license": "LicenseRef-CodeQL-Terms", - "engines": { - "node": ">=24.13.0", - "npm": ">=11.6.2" - }, - "dependencies": { - "@modelcontextprotocol/sdk": "^1.29.0", - "dotenv": "^17.4.0", - "js-yaml": "^4.1.1" - }, - "devDependencies": { - "@eslint/js": "^10.0.1", - "eslint": "^10.1.0", - "eslint-config-prettier": "^10.1.8", - "eslint-plugin-prettier": "^5.5.5", - "prettier": "^3.8.1" - }, - "scripts": { - "build": "echo 'NOOP client build'", - "clean": "echo 'NOOP client clean'", - "format": "prettier --write 'src/**/*.js'", - "format:check": "prettier --check 'src/**/*.js'", - "integration-test": "node src/ql-mcp-client.js integration-tests", - "lint": "eslint src --ext .js", - "lint:fix": "eslint src --ext .js --fix", - "server:logs": "scripts/show-server-logs.sh", - "server:start": "scripts/start-server.sh", - "server:stop": "scripts/stop-server.sh", - "server:wait": "scripts/wait-for-server.sh", - "start": "node src/ql-mcp-client.js integration-tests", - "test": "npm run test:integration", - "test:coverage": "echo 'NOOP client test:coverage'", - "test:integration": "ENABLE_MONITORING_TOOLS=false scripts/run-integration-tests.sh --no-install-packs", - "test:integration:default": "ENABLE_MONITORING_TOOLS=false scripts/run-integration-tests.sh --no-install-packs", - "test:integration:http": "MCP_MODE=http scripts/run-integration-tests.sh --no-install-packs", - "test:integration:install-packs": "scripts/run-integration-tests.sh", - "test:integration:monitoring": "ENABLE_MONITORING_TOOLS=true scripts/run-integration-tests.sh --no-install-packs", - "test:integration:stdio": "MCP_MODE=stdio scripts/run-integration-tests.sh --no-install-packs", - "tidy": "npm run lint && npm run format" - } -} diff --git a/client/scripts/run-integration-tests.sh b/client/scripts/run-integration-tests.sh index 6ff77b2f..ab64c843 100755 --- a/client/scripts/run-integration-tests.sh +++ b/client/scripts/run-integration-tests.sh @@ -88,6 +88,11 @@ echo "📦 Building CodeQL MCP server bundle..." cd "$SERVER_DIR" npm run bundle +# Step 1b: Build the Go client binary +echo "📦 Building Go client binary..." +cd "$CLIENT_DIR" +make build + # Step 2: Install CodeQL packs (only once for both modes, skip if --no-install-packs) if [ "$SKIP_PACK_INSTALL" = true ]; then echo "📦 Skipping CodeQL pack installation (--no-install-packs)" @@ -96,6 +101,10 @@ else "$SERVER_DIR/scripts/install-packs.sh" fi +# Step 3: Extract test databases used by integration tests +echo "📦 Extracting test databases..." +"$SERVER_DIR/scripts/extract-test-databases.sh" + cd "$CLIENT_DIR" # For HTTP mode, set the server URL for the client @@ -132,7 +141,7 @@ run_tests_in_mode() { # Run the integration tests (skip pack installation since we already did it) echo "🧪 Running tests..." - node src/ql-mcp-client.js integration-tests --no-install-packs "$@" + "$CLIENT_DIR/gh-ql-mcp-client" integration-tests --mode "$MCP_MODE" --no-install-packs "$@" if [ "$MCP_MODE" = "http" ]; then # Stop the server before next mode diff --git a/client/src/lib/aggregate-query-metadata.js b/client/src/lib/aggregate-query-metadata.js deleted file mode 100644 index 610ed1bc..00000000 --- a/client/src/lib/aggregate-query-metadata.js +++ /dev/null @@ -1,153 +0,0 @@ -/** - * Aggregate Query Metadata - * Gathers metadata for CodeQL queries using the find_codeql_query_files MCP tool - */ - -import { readFileSync } from "fs"; - -/** - * Aggregate metadata for multiple CodeQL queries - * @param {Object} client - Connected MCP client instance - * @param {Object} options - Aggregation options - * @param {string} options.inputFile - Path to JSON file containing query paths - * @param {number} [options.maxQueries] - Optional limit on number of queries to process - * @param {number} [options.timeout] - Timeout per query in milliseconds - * @param {Function} [options.progressCallback] - Optional callback for progress updates - * @returns {Promise} Aggregated results with count and query metadata - */ -export async function aggregateQueryMetadata(client, options = {}) { - const { inputFile, maxQueries, timeout = 120000, progressCallback } = options; - - // Read the input file containing query paths - const queries = JSON.parse(readFileSync(inputFile, "utf-8")); - - // Apply limit if specified - let processQueries = queries; - if (maxQueries && maxQueries > 0) { - processQueries = queries.slice(0, maxQueries); - if (progressCallback) { - progressCallback({ - type: "limit", - message: `Limiting to first ${maxQueries} queries (out of ${queries.length} total)` - }); - } - } - - const total = processQueries.length; - - if (total === 0) { - throw new Error("No queries found in input file"); - } - - if (progressCallback) { - progressCallback({ - type: "start", - message: `Processing ${total} queries...`, - total - }); - } - - const results = []; - let processed = 0; - - for (const queryPath of processQueries) { - processed++; - - const percentage = Math.round((processed / total) * 100); - if (progressCallback) { - progressCallback({ - type: "progress", - message: `Processing ${processed}/${total}: ${queryPath}`, - processed, - total, - percentage, - queryPath - }); - } - - try { - const startTime = Date.now(); - const result = await client.callTool("find_codeql_query_files", { queryPath }, { timeout }); - - const duration = Date.now() - startTime; - - if (progressCallback) { - progressCallback({ - type: "complete", - message: `Completed in ${duration}ms`, - duration - }); - } - - // Extract metadata from MCP response - let metadata; - try { - metadata = result.content[0]?.text ? JSON.parse(result.content[0].text) : result; - } catch (parseError) { - // If JSON parsing fails, store the raw response with an error indicator - metadata = { - error: "Failed to parse response", - parseError: parseError.message, - rawResponse: result.content[0]?.text || result - }; - } - - results.push({ - queryPath, - metadata - }); - } catch (error) { - if (progressCallback) { - progressCallback({ - type: "error", - message: error.message, - queryPath - }); - } - - results.push({ - queryPath, - error: error.message - }); - } - - // Progress separator every 10 queries - if (processed % 10 === 0 && processed < total && progressCallback) { - progressCallback({ - type: "separator", - message: `--- Processed ${processed} of ${total} queries ---` - }); - } - } - - return { - count: total, - totalAvailable: queries.length, - results - }; -} - -/** - * Default progress callback that logs to stderr - * @param {Object} event - Progress event - */ -export function consoleProgressCallback(event) { - switch (event.type) { - case "limit": - case "start": - console.error(event.message); - break; - case "progress": - console.error(`[${event.percentage}%] ${event.message}`); - break; - case "complete": - console.error(` ✓ ${event.message}`); - break; - case "error": - console.error(` ✗ Error: ${event.message}`); - break; - case "separator": - console.error(event.message); - break; - } -} diff --git a/client/src/lib/cli-parser.js b/client/src/lib/cli-parser.js deleted file mode 100644 index a80da462..00000000 --- a/client/src/lib/cli-parser.js +++ /dev/null @@ -1,278 +0,0 @@ -/** - * CLI argument parser for the CodeQL MCP Client - * Provides structured command parsing with subcommands and options - */ - -/** - * Parse CLI arguments into a structured command object - * @param {string[]} args - Process arguments (from process.argv.slice(2)) - * @returns {Object} Parsed command with subcommand and options - */ -export function parseCliArgs(args) { - // Default command if no args provided - show help - if (args.length === 0) { - return { - command: "help", - options: {} - }; - } - - // First argument is the subcommand (if it doesn't start with --) - const firstArg = args[0]; - let command = "help"; - let startIdx = 0; - let subcommand = null; - - if (!firstArg.startsWith("--")) { - command = firstArg; - startIdx = 1; - - // Check if there's a subcommand (for 'list' and 'server' commands) - if ( - (command === "list" || command === "server") && - args.length > 1 && - !args[1].startsWith("--") - ) { - subcommand = args[1]; - startIdx = 2; - } - } - - // Parse options - const options = {}; - for (let i = startIdx; i < args.length; i++) { - const arg = args[i]; - - if (arg.startsWith("--")) { - const key = arg.substring(2); - - // Handle negated boolean flags (--no-) - if (key.startsWith("no-")) { - const positiveKey = key.substring(3); - options[positiveKey] = false; - } else if (i + 1 < args.length && !args[i + 1].startsWith("--")) { - // Check if next arg is a value or another flag - const value = args[i + 1]; - // Handle comma-separated values - if (value.includes(",")) { - options[key] = value.split(",").map((v) => v.trim()); - } else { - options[key] = value; - } - i++; // Skip the value - } else { - // Boolean flag - options[key] = true; - } - } - } - - return { - command, - subcommand, - options - }; -} - -/** - * Display help information for the CLI - * @returns {string} Help text - */ -export function getHelpText() { - return ` -CodeQL MCP Client - Integration testing CLI for the CodeQL Development MCP Server - -USAGE: - node src/ql-mcp-client.js [COMMAND] [OPTIONS] - ---- OR ---- - node client/src/ql-mcp-client.js [COMMAND] [OPTIONS] - -COMMANDS: - help Display this help message (default) - integration-tests Run integration tests - list primitives List all currently registered MCP server primitives (prompts, resources, and tools) - list prompts List all currently registered MCP server prompts - list resources List all currently registered MCP server resources - list tools List all currently registered MCP server tools - queries-metadata-collect Collect metadata for CodeQL queries using find_codeql_query_files - queries-metadata-process Process collected query metadata to generate coverage analysis - query-files-copy Copy query-related files to scratch directory (excludes .ql files) - resolve-all-queries Resolve all CodeQL queries from query packs in a repository - server start Start the MCP server - server stop Stop the MCP server - source-root-validate Validate SOURCE_ROOT environment variable and directory structure - -OPTIONS: - --format Output format for list commands: text (default) or json - Example: --format json - - --mode Server transport mode: stdio or http (default: http) - Example: --mode http - - --host Server host for HTTP mode (default: localhost) - Example: --host localhost - - --port Server port for HTTP mode (default: 3000) - Example: --port 3000 - - --scheme Server scheme for HTTP mode (default: http) - Example: --scheme http - - --tools Comma-separated list of tool names to test (integration-tests only) - Example: --tools codeql_query_run,codeql_query_format - - --tests Comma-separated list of test names to run (integration-tests only) - Only applicable when a single tool is specified - Example: --tests basic_query_run,javascript_tools_print_ast - - --install-packs Install CodeQL pack dependencies before running tests (default: true) - Set to --no-install-packs to skip pack installation for faster test runs - Example: --no-install-packs - - --timeout Timeout in seconds for each tool call (integration-tests only, default: 30) - Example: --timeout 600 - - --help Display help information - -ENVIRONMENT VARIABLES: - MCP_MODE MCP transport mode: stdio (default) or http - MCP_SERVER_PATH Path to the MCP server JS entry point (stdio mode only) - MCP_SERVER_URL MCP server URL (http mode only, default: http://localhost:3000/mcp) - ENABLE_MONITORING_TOOLS Enable session_* monitoring tools (default: false) -`; -} - -/** - * Validate parsed CLI arguments - * @param {Object} parsed - Parsed command object - * @returns {Object} Validation result with isValid and error - */ -export function validateCliArgs(parsed) { - const { command, subcommand, options } = parsed; - - // Validate command - const validCommands = [ - "help", - "integration-tests", - "list", - "queries-metadata-collect", - "queries-metadata-process", - "query-files-copy", - "resolve-all-queries", - "server", - "source-root-validate" - ]; - if (!validCommands.includes(command)) { - return { - isValid: false, - error: `Invalid command: ${command}. Valid commands: ${validCommands.join(", ")}` - }; - } - - // Validate list subcommands - if (command === "list") { - const validSubcommands = ["primitives", "prompts", "resources", "tools"]; - if (!subcommand) { - return { - isValid: false, - error: `Missing subcommand for 'list'. Valid subcommands: ${validSubcommands.join(", ")}` - }; - } - if (!validSubcommands.includes(subcommand)) { - return { - isValid: false, - error: `Invalid list subcommand: ${subcommand}. Valid subcommands: ${validSubcommands.join(", ")}` - }; - } - - // Validate format option for list commands - if (options.format && !["text", "json"].includes(options.format)) { - return { - isValid: false, - error: `Invalid format: ${options.format}. Valid formats: text, json` - }; - } - } - - // Validate server subcommands - if (command === "server") { - const validSubcommands = ["start", "stop"]; - if (!subcommand) { - return { - isValid: false, - error: `Missing subcommand for 'server'. Valid subcommands: ${validSubcommands.join(", ")}` - }; - } - if (!validSubcommands.includes(subcommand)) { - return { - isValid: false, - error: `Invalid server subcommand: ${subcommand}. Valid subcommands: ${validSubcommands.join(", ")}` - }; - } - - // Validate mode option - if (options.mode && !["stdio", "http"].includes(options.mode)) { - return { - isValid: false, - error: `Invalid mode: ${options.mode}. Valid modes: stdio, http` - }; - } - - // Validate port option - if (options.port) { - const port = parseInt(options.port); - if (isNaN(port) || port <= 0 || port > 65535) { - return { - isValid: false, - error: `Invalid port: ${options.port}. Must be a number between 1 and 65535.` - }; - } - } - } - - // Validate timeout if provided - if (options.timeout !== undefined) { - const timeout = parseInt(options.timeout); - if (isNaN(timeout) || timeout <= 0) { - return { - isValid: false, - error: `Invalid timeout value: ${options.timeout}. Must be a positive number.` - }; - } - } - - // Validate iterations if provided - if (options.iterations !== undefined) { - const iterations = parseInt(options.iterations); - if (isNaN(iterations) || iterations <= 0) { - return { - isValid: false, - error: `Invalid iterations value: ${options.iterations}. Must be a positive number.` - }; - } - } - - // Validate success-rate if provided - if (options["success-rate"] !== undefined) { - const successRate = parseFloat(options["success-rate"]); - if (isNaN(successRate) || successRate < 0 || successRate > 1) { - return { - isValid: false, - error: `Invalid success-rate value: ${options["success-rate"]}. Must be a number between 0.0 and 1.0.` - }; - } - } - - // Validate that --tests is only used with a single --tools value - if (options.tests && options.tools) { - const tools = Array.isArray(options.tools) ? options.tools : [options.tools]; - if (tools.length > 1) { - return { - isValid: false, - error: "The --tests option can only be used when specifying a single tool with --tools" - }; - } - } - - return { isValid: true }; -} diff --git a/client/src/lib/command-handler.js b/client/src/lib/command-handler.js deleted file mode 100644 index 8af302c8..00000000 --- a/client/src/lib/command-handler.js +++ /dev/null @@ -1,109 +0,0 @@ -/** - * Command handler for CLI commands - * Orchestrates CLI execution and routes commands to appropriate handlers - */ - -import { parseCliArgs, validateCliArgs, getHelpText } from "./cli-parser.js"; -import { - executeHelpCommand, - executeIntegrationTestsCommand, - executeListCommand, - executeServerCommand, - executeSourceRootValidateCommand -} from "./commands/basic-commands.js"; -import { - executeQueriesMetadataCollectCommand, - executeQueriesMetadataProcessCommand, - executeQueriesMetadataFilterCommand, - executeResolveAllQueriesCommand -} from "./commands/metadata-commands.js"; -import { executeQueryFilesCopyCommand } from "./commands/query-commands.js"; - -/** - * Prepare client options from parsed CLI options - * @param {Object} options - Parsed CLI options - * @returns {Object} Client options - */ -export function prepareClientOptions(options) { - return { - timeout: options.timeout, - iterations: options.iterations, - successRate: options["success-rate"], - tools: Array.isArray(options.tools) ? options.tools : options.tools ? [options.tools] : null, - tests: Array.isArray(options.tests) ? options.tests : options.tests ? [options.tests] : null, - // Default to true if not explicitly set to false - installPacks: options["install-packs"] !== false - }; -} - -/** - * Main command handler - orchestrates CLI execution - * @param {string[]} args - Command line arguments - * @param {Function} clientFactory - Function to create CodeQLMCPClient instance - */ -export async function handleCommand(args, clientFactory) { - // Parse CLI arguments - const parsed = parseCliArgs(args); - - // Check for help command - if (parsed.command === "help" || parsed.options.help) { - executeHelpCommand(); - return; // Never reached due to process.exit, but explicit for clarity - } - - // Validate arguments - const validation = validateCliArgs(parsed); - if (!validation.isValid) { - console.error(`Error: ${validation.error}\n`); - console.log(getHelpText()); - process.exit(1); - } - - // Extract command and options - const { command, subcommand, options } = parsed; - - // Prepare client options - const clientOptions = prepareClientOptions(options); - - // Create client with options - const client = clientFactory(clientOptions); - - // Execute the command - switch (command) { - case "integration-tests": - await executeIntegrationTestsCommand(client); - break; - case "list": - await executeListCommand(client, subcommand, options.format || "text"); - break; - case "server": - await executeServerCommand(subcommand, options); - break; - case "queries-metadata-collect": - await executeQueriesMetadataCollectCommand(client, options); - break; - case "queries-metadata-filter": - await executeQueriesMetadataFilterCommand(client, options); - break; - case "queries-metadata-process": - await executeQueriesMetadataProcessCommand(client, options); - break; - case "query-files-copy": - await executeQueryFilesCopyCommand(client, options); - break; - case "resolve-all-queries": - await executeResolveAllQueriesCommand(client, options); - break; - case "source-root-validate": - await executeSourceRootValidateCommand(client, options); - break; - case "help": - // This case is already handled above, but included for completeness - executeHelpCommand(); - break; - default: - console.error(`Unknown command: ${command}`); - console.log(getHelpText()); - process.exit(1); - } -} diff --git a/client/src/lib/commands/basic-commands.js b/client/src/lib/commands/basic-commands.js deleted file mode 100644 index 78975e72..00000000 --- a/client/src/lib/commands/basic-commands.js +++ /dev/null @@ -1,172 +0,0 @@ -/** - * Basic Commands - * Help, integration tests, list commands - */ - -import { dirname, join } from "path"; -import { writeFile, mkdir } from "fs/promises"; -import { execFileSync } from "child_process"; -import { fileURLToPath } from "url"; -import { getHelpText } from "../cli-parser.js"; -import { ensureServerRunning, startServer, stopServer } from "../server-manager.js"; -import { validateSourceRoot } from "../validate-source-root.js"; -import { connectWithRetry } from "../mcp-client-utils.js"; - -// Get the directory of this module for resolving relative paths -const __filename = fileURLToPath(import.meta.url); -const __dirname = dirname(__filename); -// Repository root is 4 levels up from client/src/lib/commands/ -const REPO_ROOT = join(__dirname, "..", "..", "..", ".."); - -/** - * Execute the help command - */ -export function executeHelpCommand() { - console.log(getHelpText()); - process.exit(0); -} - -/** - * Install CodeQL pack dependencies using the install-packs.sh script - * @returns {boolean} Whether the installation was successful - */ -function installCodeQLPacks() { - const scriptPath = join(REPO_ROOT, "server", "scripts", "install-packs.sh"); - - try { - console.log("📦 Installing CodeQL pack dependencies..."); - execFileSync(scriptPath, [], { - encoding: "utf8", - stdio: "inherit", - cwd: REPO_ROOT, - timeout: 300000 // 5 minute timeout - }); - console.log("✅ CodeQL pack dependencies installed successfully"); - return true; - } catch (error) { - console.error(`❌ Failed to install CodeQL packs: ${error.message}`); - return false; - } -} - -/** - * Execute the integration-tests command - * @param {Object} client - CodeQLMCPClient instance - */ -export async function executeIntegrationTestsCommand(client) { - // Install packs if not explicitly disabled - if (client.options.installPacks !== false) { - if (!installCodeQLPacks()) { - console.error("Aborting tests due to pack installation failure"); - process.exit(1); - } - } else { - console.log("⏭️ Skipping CodeQL pack installation (--no-install-packs)"); - } - - await client.runTests(); -} - -/** - * Execute the list command - * @param {Object} client - CodeQLMCPClient instance - * @param {string} subcommand - List subcommand (primitives, prompts, resources, tools) - * @param {string} format - Output format (text or json) - */ -export async function executeListCommand(client, subcommand, format = "text") { - // Ensure server is running before connecting - await ensureServerRunning(); - - // Connect to server (with automatic retry on session conflict) - await connectWithRetry(client); - - try { - switch (subcommand) { - case "primitives": - await client.listPrimitives(format); - break; - case "prompts": - await client.listPromptsCommand(format); - break; - case "resources": - await client.listResourcesCommand(format); - break; - case "tools": - await client.listToolsCommand(format); - break; - default: - throw new Error(`Unknown list subcommand: ${subcommand}`); - } - } finally { - await client.disconnect(); - } -} - -/** - * Execute the server command - * @param {string} subcommand - Server subcommand (start or stop) - * @param {Object} options - Server options - */ -export async function executeServerCommand(subcommand, options = {}) { - switch (subcommand) { - case "start": { - const serverOptions = { - mode: options.mode || "http", - host: options.host || "localhost", - port: options.port ? parseInt(options.port) : 3000, - scheme: options.scheme || "http" - }; - await startServer(serverOptions); - break; - } - case "stop": - await stopServer(); - break; - default: - throw new Error(`Unknown server subcommand: ${subcommand}`); - } -} - -/** - * Execute the source-root-validate command - * @param {Object} _client - CodeQLMCPClient instance (not used for this command) - * @param {Object} _options - Command options - */ -export async function executeSourceRootValidateCommand(_client, _options = {}) { - const sourceRoot = process.env.SOURCE_ROOT; - const outputFile = process.env.OUTPUT_DIR - ? process.env.OUTPUT_DIR + "/validate-source-root.json" - : null; - - const result = validateSourceRoot({ sourceRoot }); - - // Serialize JSON once - const output = JSON.stringify(result, null, 2); - - // Write directly to file if OUTPUT_DIR is set - if (outputFile) { - try { - // Ensure output directory exists - await mkdir(dirname(outputFile), { recursive: true }); - // Write file with UTF-8 encoding - await writeFile(outputFile, output, "utf-8"); - } catch (error) { - throw new Error(`Failed to write output file ${outputFile}: ${error.message}`, { - cause: error - }); - } - } - - // Also write to stdout for display - console.log(output); - - // Exit with error if validation failed - if (!result.valid) { - process.exit(1); - } - - // Log summary to stderr if output file was written - if (outputFile) { - console.error(`Output file: ${outputFile}`); - } -} diff --git a/client/src/lib/commands/metadata-commands.js b/client/src/lib/commands/metadata-commands.js deleted file mode 100644 index ddd86a2a..00000000 --- a/client/src/lib/commands/metadata-commands.js +++ /dev/null @@ -1,254 +0,0 @@ -/** - * Metadata Commands - * Commands for processing query metadata - */ - -import { dirname } from "path"; -import { writeFile, mkdir } from "fs/promises"; -import { - aggregateQueryMetadata, - consoleProgressCallback as aggregateProgressCallback -} from "../aggregate-query-metadata.js"; -import { ensureServerRunning } from "../server-manager.js"; -import { - filterQueryMetadata, - consoleProgressCallback as filterProgressCallback -} from "../queries-filter-metadata.js"; -import { - processQueryMetadata, - consoleProgressCallback as processProgressCallback -} from "../process-query-metadata.js"; -import { - resolveAllQueries, - consoleProgressCallback as resolveProgressCallback -} from "../resolve-all-queries.js"; -import { connectWithRetry } from "../mcp-client-utils.js"; - -/** - * Execute the queries-metadata-collect command - * @param {Object} client - CodeQLMCPClient instance - * @param {Object} options - Command options - */ -export async function executeQueriesMetadataCollectCommand(client, _options = {}) { - const inputFile = process.env.OUTPUT_DIR + "/codeql-resolve-queries.json"; - const outputFile = process.env.OUTPUT_DIR + "/ql_queries_metadata.json"; - const maxQueries = process.env.MAX_QUERIES ? parseInt(process.env.MAX_QUERIES) : undefined; - const timeout = parseInt(process.env.MCP_TIMEOUT || "120000"); - - if (!process.env.OUTPUT_DIR) { - throw new Error("OUTPUT_DIR environment variable is required"); - } - - // Suppress client logging to avoid polluting JSON output - const originalLog = client.logger.log.bind(client.logger); - client.logger.log = () => {}; // Disable logging - - // Ensure server is running before connecting - await ensureServerRunning(); - - // Connect to server (with automatic retry on session conflict) - await connectWithRetry(client); - - try { - const result = await aggregateQueryMetadata(client, { - inputFile, - maxQueries, - timeout, - progressCallback: aggregateProgressCallback - }); - - // Serialize JSON once - const output = JSON.stringify(result, null, 2); - - // Write directly to file (reliable for large JSON datasets) - try { - // Ensure output directory exists - await mkdir(dirname(outputFile), { recursive: true }); - // Write file with UTF-8 encoding - await writeFile(outputFile, output, "utf-8"); - } catch (error) { - throw new Error(`Failed to write output file ${outputFile}: ${error.message}`, { - cause: error - }); - } - - // Log summary to stderr - console.error("\n"); - if (maxQueries) { - console.error( - `Successfully processed ${maxQueries} queries (out of ${result.totalAvailable} total)` - ); - } else { - console.error(`Successfully processed ${result.count} queries`); - } - console.error(`Output file: ${outputFile}`); - } finally { - // Disconnect before restoring logger to avoid disconnect log message - await client.disconnect(); - // Restore logger - client.logger.log = originalLog; - } -} - -/** - * Execute the queries-metadata-process command - * @param {Object} _client - CodeQLMCPClient instance (not used for this command) - * @param {Object} _options - Command options - */ -export async function executeQueriesMetadataProcessCommand(_client, _options = {}) { - const inputFile = - process.env.QL_QUERIES_METADATA_INPUT_FILE || - process.env.OUTPUT_DIR + "/ql_queries_metadata.json"; - const outputFile = process.env.OUTPUT_DIR - ? process.env.OUTPUT_DIR + "/ql_queries_processed.json" - : null; - - if (!inputFile) { - throw new Error( - "QL_QUERIES_METADATA_INPUT_FILE or OUTPUT_DIR environment variable is required" - ); - } - - const result = processQueryMetadata({ - inputFile, - progressCallback: processProgressCallback - }); - - // Serialize JSON once - const output = JSON.stringify(result, null, 2); - - // Write directly to file (reliable for large JSON datasets) - if (outputFile) { - try { - // Ensure output directory exists - await mkdir(dirname(outputFile), { recursive: true }); - // Write file with UTF-8 encoding - await writeFile(outputFile, output, "utf-8"); - } catch (error) { - throw new Error(`Failed to write output file ${outputFile}: ${error.message}`, { - cause: error - }); - } - } else { - throw new Error( - "OUTPUT_DIR environment variable is required for queries-metadata-process command" - ); - } - - // Log summary to stderr - console.error("\n"); - console.error(`Successfully processed metadata for ${result.summary.totalQueries} queries`); - console.error( - `Found ${result.summary.languages.length} languages: ${result.summary.languages.join(", ")}` - ); - console.error(`Identified ${result.summary.totalTags} unique tags`); - console.error(`Output file: ${outputFile}`); -} - -/** - * Execute the queries-metadata-filter command - * @param {Object} _client - CodeQLMCPClient instance (not used for this command) - * @param {Object} _options - Command options - */ -export async function executeQueriesMetadataFilterCommand(_client, _options = {}) { - const inputFile = - process.env.QL_QUERIES_METADATA_INPUT_FILE || - process.env.OUTPUT_DIR + "/ql_queries_metadata.json"; - const outputFile = process.env.OUTPUT_DIR + "/ql_queries_regenerable.json"; - const language = process.env.QL_CODE_LANGUAGE || "all"; - - if (!inputFile) { - throw new Error( - "QL_QUERIES_METADATA_INPUT_FILE or OUTPUT_DIR environment variable is required" - ); - } - - if (!process.env.OUTPUT_DIR) { - throw new Error("OUTPUT_DIR environment variable is required"); - } - - const result = filterQueryMetadata({ - inputFile, - language, - progressCallback: filterProgressCallback - }); - - // Serialize JSON once - const output = JSON.stringify(result, null, 2); - - // Write directly to file (reliable for large JSON datasets) - try { - // Ensure output directory exists - await mkdir(dirname(outputFile), { recursive: true }); - // Write file with UTF-8 encoding - await writeFile(outputFile, output, "utf-8"); - } catch (error) { - throw new Error(`Failed to write output file ${outputFile}: ${error.message}`, { - cause: error - }); - } - - // Log summary to stderr - console.error("\n"); - console.error( - `Filtered ${result.metadata.regenerableCount} regenerable queries from ${result.metadata.totalQueries} total queries` - ); - console.error(`Language filter: ${language}`); - console.error(`Output file: ${outputFile}`); -} - -/** - * Execute the resolve-all-queries command - * @param {Object} client - CodeQLMCPClient instance - * @param {Object} _options - Command options - */ -export async function executeResolveAllQueriesCommand(client, _options = {}) { - const packsFile = process.env.OUTPUT_DIR + "/codeql-pack-ls.json"; - const outputFile = process.env.OUTPUT_DIR + "/codeql-resolve-queries.json"; - const timeout = parseInt(process.env.MCP_TIMEOUT || "120000"); - - if (!process.env.OUTPUT_DIR) { - throw new Error("OUTPUT_DIR environment variable is required"); - } - - // Suppress client logging to avoid polluting output - const originalLog = client.logger.log.bind(client.logger); - client.logger.log = () => {}; // Disable logging - - // Ensure server is running before connecting - await ensureServerRunning(); - - // Connect to server (with automatic retry on session conflict) - await connectWithRetry(client); - - try { - const queries = await resolveAllQueries(client, { - packsFile, - timeout, - progressCallback: resolveProgressCallback - }); - - // Serialize JSON once - const output = JSON.stringify(queries, null, 2); - - // Write directly to file (reliable for large datasets) - try { - // Ensure output directory exists - await mkdir(dirname(outputFile), { recursive: true }); - // Write file with UTF-8 encoding - await writeFile(outputFile, output, "utf-8"); - } catch (error) { - throw new Error(`Failed to write output file ${outputFile}: ${error.message}`, { - cause: error - }); - } - - // Log summary to stderr - console.error(`Output file: ${outputFile}`); - } finally { - // Disconnect before restoring logger to avoid disconnect log message - await client.disconnect(); - // Restore logger - client.logger.log = originalLog; - } -} diff --git a/client/src/lib/commands/query-commands.js b/client/src/lib/commands/query-commands.js deleted file mode 100644 index 46f6de31..00000000 --- a/client/src/lib/commands/query-commands.js +++ /dev/null @@ -1,403 +0,0 @@ -/** - * Query Commands - * Commands for working with CodeQL query files - */ - -import fs from "fs"; -import path from "path"; -import { ensureServerRunning } from "../server-manager.js"; -import { connectWithRetry } from "../mcp-client-utils.js"; - -/** - * Copy query-related files to a scratch directory - * Preserves directory structure and excludes .ql files so LLM can generate them - * - * Required environment variables: - * - QUERY_FILE: Path to the .ql query file (absolute or relative to current directory) - * - OUTPUT_DIR: Output/scratch directory to copy files to - * - * Optional environment variables (auto-derived from find_codeql_query_files if not provided): - * - SOURCE_ROOT: Root directory of the source repository (defaults to REPO_ROOT or cwd) - * - QUERY_ID: Unique identifier for the query (defaults to query filename without extension) - * - * @param {Object} client - CodeQLMCPClient instance - * @param {Object} _options - Command options - */ -export async function executeQueryFilesCopyCommand(client, _options = {}) { - // Configuration from environment - const queryFile = process.env.QUERY_FILE; - const outputDir = path.resolve(process.env.OUTPUT_DIR || "client/scratch"); - - if (!queryFile) { - console.error("Error: QUERY_FILE environment variable is required"); - process.exit(1); - } - - // Resolve the query file path - const absoluteQueryPath = path.resolve(queryFile); - - if (!fs.existsSync(absoluteQueryPath)) { - console.error(`Error: Query file not found: ${absoluteQueryPath}`); - process.exit(1); - } - - // Determine source root - try to find it from the query path - let sourceRoot = process.env.SOURCE_ROOT || process.env.REPO_ROOT; - if (!sourceRoot) { - // Try to infer source root by walking up to find a common structure - sourceRoot = path.dirname(absoluteQueryPath); - // Walk up until we find a directory that looks like a root (has .git or qlpack.yml at top level) - let current = sourceRoot; - while (current !== path.dirname(current)) { - if ( - fs.existsSync(path.join(current, ".git")) || - fs.existsSync(path.join(current, "qlpack.yml")) - ) { - sourceRoot = current; - break; - } - current = path.dirname(current); - } - } - sourceRoot = path.resolve(sourceRoot); - - // Get query ID from environment or derive from filename - const queryName = path.basename(absoluteQueryPath, ".ql"); - const queryId = process.env.QUERY_ID || queryName; - - console.error(`Query file: ${absoluteQueryPath}`); - console.error(`Source root: ${sourceRoot}`); - console.error(`Output directory: ${outputDir}`); - console.error(`Query ID: ${queryId}`); - - // Ensure output directory exists - if (!fs.existsSync(outputDir)) { - fs.mkdirSync(outputDir, { recursive: true }); - } - - // Call find_codeql_query_files MCP tool to get all related files - console.error("\nCalling find_codeql_query_files MCP tool..."); - - // Suppress client logging - const originalLog = client.logger?.log?.bind(client.logger); - if (client.logger && originalLog) { - client.logger.log = () => {}; - } - - // Ensure server is running and connect - await ensureServerRunning(); - await connectWithRetry(client); - - let queryFilesResult; - try { - const timeoutValue = parseInt(process.env.MCP_TIMEOUT || "120000", 10); - const timeout = Number.isNaN(timeoutValue) ? 120000 : timeoutValue; - const result = await client.callTool( - "find_codeql_query_files", - { queryPath: absoluteQueryPath, resolveMetadata: true }, - { timeout } - ); - - // Parse the result - if (result.content && result.content[0] && result.content[0].text) { - queryFilesResult = JSON.parse(result.content[0].text); - } else { - throw new Error("Invalid response from find_codeql_query_files"); - } - } catch (error) { - console.error(`Error calling find_codeql_query_files: ${error.message}`); - process.exit(1); - } finally { - await client.disconnect(); - if (client.logger && originalLog) { - client.logger.log = originalLog; - } - } - - console.error(`Language detected: ${queryFilesResult.language}`); - console.error(`Query name: ${queryFilesResult.queryName}`); - - const result = { - queryId, - queryName: queryFilesResult.queryName, - language: queryFilesResult.language, - sourceRoot, - outputDir, - copiedFiles: [], - skippedFiles: [], - errors: [], - paths: {} - }; - - /** - * Copy a file to the output directory, preserving relative structure - * @param {string} filePath - Full path to source file - * @param {string} sourceBase - Base source directory - * @param {string} outputBase - Base output directory - * @returns {string|null} Destination path or null if failed - */ - function copyFile(filePath, sourceBase, outputBase) { - if (!filePath) return null; - - try { - const sourcePath = path.resolve(filePath); - - if (!fs.existsSync(sourcePath)) { - result.skippedFiles.push({ path: filePath, reason: "File not found" }); - return null; - } - - // Determine relative path from source root - let relPath; - if (sourcePath.startsWith(sourceBase)) { - relPath = path.relative(sourceBase, sourcePath); - } else { - // Use just the filename if outside source root - relPath = path.basename(filePath); - } - - // Create destination path - const destPath = path.join(outputBase, relPath); - const destDir = path.dirname(destPath); - - // Create destination directory - if (!fs.existsSync(destDir)) { - fs.mkdirSync(destDir, { recursive: true }); - } - - // Copy the file - fs.copyFileSync(sourcePath, destPath); - result.copiedFiles.push({ source: sourcePath, dest: destPath }); - - return destPath; - } catch (error) { - result.errors.push({ path: filePath, error: error.message }); - return null; - } - } - - /** - * Copy a directory recursively to the output directory - * @param {string} dirPath - Full path to source directory - * @param {string} sourceBase - Base source directory - * @param {string} outputBase - Base output directory - * @param {boolean} excludeQlFiles - Whether to exclude .ql files - * @returns {string|null} Destination path or null if failed - */ - function copyDirectory(dirPath, sourceBase, outputBase, excludeQlFiles = false) { - if (!dirPath) return null; - - try { - const sourcePath = path.resolve(dirPath); - - if (!fs.existsSync(sourcePath)) { - result.skippedFiles.push({ path: dirPath, reason: "Directory not found" }); - return null; - } - - // Determine relative path from source root - let relPath; - if (sourcePath.startsWith(sourceBase)) { - relPath = path.relative(sourceBase, sourcePath); - } else { - relPath = path.basename(dirPath); - } - - // Create destination path - const destPath = path.join(outputBase, relPath); - - // Recursive copy function - function copyDirRecursive(src, dest) { - if (!fs.existsSync(dest)) { - fs.mkdirSync(dest, { recursive: true }); - } - - const entries = fs.readdirSync(src, { withFileTypes: true }); - for (const entry of entries) { - const srcPath = path.join(src, entry.name); - const destPathEntry = path.join(dest, entry.name); - - if (entry.isDirectory()) { - copyDirRecursive(srcPath, destPathEntry); - } else if (excludeQlFiles && entry.name.toLowerCase().endsWith(".ql")) { - // Skip .ql files - they will be generated by Copilot - result.skippedFiles.push({ - path: srcPath, - reason: "Query file excluded for regeneration" - }); - } else { - fs.copyFileSync(srcPath, destPathEntry); - result.copiedFiles.push({ source: srcPath, dest: destPathEntry }); - } - } - } - - copyDirRecursive(sourcePath, destPath); - return destPath; - } catch (error) { - result.errors.push({ path: dirPath, error: error.message }); - return null; - } - } - - /** - * Find and copy qlpack.yml files from a directory and its parents - * @param {string} targetDir - Directory to start from - * @param {string} sourceBase - Base source directory - * @param {string} outputBase - Base output directory - * @returns {string[]} Array of copied qlpack file paths - */ - function copyQlpackFiles(targetDir, sourceBase, outputBase) { - if (!targetDir) return []; - - const copiedPacks = []; - let currentDir = path.resolve(targetDir); - - // Walk up the directory tree looking for qlpack files - while (currentDir.startsWith(sourceBase) && currentDir !== sourceBase) { - for (const qlpackName of ["qlpack.yml", "codeql-pack.yml", "codeql-pack.lock.yml"]) { - const qlpackPath = path.join(currentDir, qlpackName); - if (fs.existsSync(qlpackPath)) { - const relPath = path.relative(sourceBase, qlpackPath); - const destPath = path.join(outputBase, relPath); - const destDir = path.dirname(destPath); - - if (!fs.existsSync(destPath)) { - if (!fs.existsSync(destDir)) { - fs.mkdirSync(destDir, { recursive: true }); - } - fs.copyFileSync(qlpackPath, destPath); - result.copiedFiles.push({ source: qlpackPath, dest: destPath }); - copiedPacks.push(destPath); - } - } - } - currentDir = path.dirname(currentDir); - } - - return copiedPacks; - } - - // Extract paths from the find_codeql_query_files result - const queryDir = queryFilesResult.files.query.dir; - const testDir = queryFilesResult.files.test.dir; - const testCodeFiles = queryFilesResult.files.test.testCode || []; - - // Copy test directory (contains test code and expected results) - console.error("\nCopying test directory..."); - if (queryFilesResult.status.testDirectoryExists) { - const copiedTestDir = copyDirectory(testDir, sourceRoot, outputDir); - if (copiedTestDir) { - result.paths.testDirectory = copiedTestDir; - } - } - - // Copy query directory (but exclude the .ql file itself) - console.error("Copying query directory (excluding .ql files)..."); - const copiedQueryDir = copyDirectory(queryDir, sourceRoot, outputDir, true); - if (copiedQueryDir) { - result.paths.queryDirectory = copiedQueryDir; - } - - // Store path where query file should be generated - const relativeQueryPath = path.relative(sourceRoot, absoluteQueryPath); - result.paths.queryFile = path.join(outputDir, relativeQueryPath); - - // Copy documentation file if it exists separately - console.error("Copying documentation file..."); - if (queryFilesResult.status.documentationExists) { - const docFile = path.join(queryDir, queryFilesResult.files.query.doc); - const copiedDocFile = copyFile(docFile, sourceRoot, outputDir); - if (copiedDocFile) { - result.paths.documentationFile = copiedDocFile; - } - } - - // Copy qlpack files from query directory hierarchy - console.error("Copying qlpack files for query..."); - copyQlpackFiles(queryDir, sourceRoot, outputDir); - - // Copy qlpack files from test directory hierarchy - console.error("Copying qlpack files for tests..."); - if (queryFilesResult.status.testDirectoryExists) { - copyQlpackFiles(testDir, sourceRoot, outputDir); - } - - // Store paths derived from find_codeql_query_files result - result.paths.expectedFile = queryFilesResult.status.expectedResultsExist - ? path.join( - outputDir, - path.relative(sourceRoot, path.join(testDir, queryFilesResult.files.test.expected)) - ) - : null; - result.paths.qlrefFile = queryFilesResult.status.qlrefExists - ? path.join( - outputDir, - path.relative(sourceRoot, path.join(testDir, queryFilesResult.files.test.qlref)) - ) - : null; - result.paths.testCodeFiles = testCodeFiles.map((f) => { - const relPath = path.relative(sourceRoot, f); - return path.join(outputDir, relPath); - }); - - // Include find_codeql_query_files result for reference - result.queryFilesInfo = { - language: queryFilesResult.language, - queryName: queryFilesResult.queryName, - allFilesExist: queryFilesResult.allFilesExist, - status: queryFilesResult.status, - metadata: queryFilesResult.metadata, - packMetadata: queryFilesResult.packMetadata - }; - - // Include pack directory paths in result.paths - result.paths.queryPackDir = queryFilesResult.files.query.packDir; - result.paths.testPackDir = queryFilesResult.files.test.packDir; - - // Summary - console.error("\n=== Copy Summary ==="); - console.error(`Query ID: ${queryId}`); - console.error(`Source root: ${sourceRoot}`); - console.error(`Output directory: ${outputDir}`); - console.error(`Files copied: ${result.copiedFiles.length}`); - console.error(`Files skipped: ${result.skippedFiles.length}`); - console.error(`Errors: ${result.errors.length}`); - - // Write result JSON to file - const resultFile = path.join(outputDir, `query-files-copy-${queryId}.json`); - fs.writeFileSync(resultFile, JSON.stringify(result, null, 2)); - console.error(`\nResult written to: ${resultFile}`); - - // Output result to stdout for capture - console.log(JSON.stringify(result, null, 2)); - - // Exit with error if there were issues - if (result.errors.length > 0) { - console.error("\nErrors occurred during copy:"); - for (const err of result.errors) { - console.error(` - ${err.path}: ${err.error}`); - } - process.exit(1); - } -} - -/** - * Console progress callback for query operations - * @param {Object} event - Progress event - */ -export function consoleProgressCallback(event) { - switch (event.type) { - case "start": - console.error(event.message); - break; - case "progress": - console.error(`${event.message} (${event.percentage}%)`); - break; - case "complete": - console.error(event.message); - break; - default: - break; - } -} diff --git a/client/src/lib/file-utils.js b/client/src/lib/file-utils.js deleted file mode 100644 index bb5b1a1a..00000000 --- a/client/src/lib/file-utils.js +++ /dev/null @@ -1,129 +0,0 @@ -/** - * File system utilities for integration testing - */ - -import fs from "fs"; -import path from "path"; - -/** - * Compare two directories file by file - */ -export function compareDirectories(actualDir, expectedDir) { - try { - const actualFiles = getDirectoryFiles(actualDir); - const expectedFiles = getDirectoryFiles(expectedDir); - - // Check if file lists match - if (actualFiles.length !== expectedFiles.length) { - return false; - } - - const actualFileNames = actualFiles.map((f) => path.relative(actualDir, f)).sort(); - const expectedFileNames = expectedFiles.map((f) => path.relative(expectedDir, f)).sort(); - - for (let i = 0; i < actualFileNames.length; i++) { - if (actualFileNames[i] !== expectedFileNames[i]) { - return false; - } - } - - // Compare file contents - for (const fileName of actualFileNames) { - const actualFile = path.join(actualDir, fileName); - const expectedFile = path.join(expectedDir, fileName); - - if (!compareFiles(actualFile, expectedFile)) { - return false; - } - } - - return true; - } catch { - return false; - } -} - -/** - * Compare two files content - */ -export function compareFiles(file1, file2) { - try { - const content1 = fs.readFileSync(file1, "utf8"); - const content2 = fs.readFileSync(file2, "utf8"); - return content1 === content2; - } catch { - return false; - } -} - -/** - * Copy directory contents recursively - */ -export function copyDirectory(src, dest) { - if (!fs.existsSync(dest)) { - fs.mkdirSync(dest, { recursive: true }); - } - - const entries = fs.readdirSync(src, { withFileTypes: true }); - - for (const entry of entries) { - const srcPath = path.join(src, entry.name); - const destPath = path.join(dest, entry.name); - - if (entry.isDirectory()) { - copyDirectory(srcPath, destPath); - } else { - fs.copyFileSync(srcPath, destPath); - } - } -} - -/** - * Get all files in a directory recursively - */ -export function getDirectoryFiles(dir) { - const files = []; - const entries = fs.readdirSync(dir, { withFileTypes: true }); - - for (const entry of entries) { - const fullPath = path.join(dir, entry.name); - if (entry.isDirectory()) { - files.push(...getDirectoryFiles(fullPath)); - } else { - files.push(fullPath); - } - } - - return files; -} - -/** - * Remove directory and all contents - */ -export function removeDirectory(dir) { - if (fs.existsSync(dir)) { - fs.rmSync(dir, { recursive: true, force: true }); - } -} - -/** - * Compare two objects for equality - */ -export function compareObjects(obj1, obj2) { - try { - return JSON.stringify(obj1) === JSON.stringify(obj2); - } catch { - return false; - } -} - -/** - * Deep clone an object - */ -export function deepClone(obj) { - try { - return JSON.parse(JSON.stringify(obj)); - } catch { - return null; - } -} diff --git a/client/src/lib/integration-test-runner.js b/client/src/lib/integration-test-runner.js deleted file mode 100644 index da79bd96..00000000 --- a/client/src/lib/integration-test-runner.js +++ /dev/null @@ -1,1487 +0,0 @@ -/** - * Integration test runner for MCP server tools - */ - -import fs from "fs"; -import path from "path"; -import { fileURLToPath } from "url"; -import { - compareDirectories, - copyDirectory, - getDirectoryFiles, - removeDirectory -} from "./file-utils.js"; - -/** - * Repository root, calculated once at module load. - * Mirrors `server/src/utils/temp-dir.ts`. - */ -const __filename = fileURLToPath(import.meta.url); -const __dirname = path.dirname(__filename); -const repoRoot = path.resolve(__dirname, "..", "..", ".."); - -/** - * Project-local temporary directory (`/.tmp`). - * All temporary files are kept here instead of the OS temp directory - * to avoid CWE-377/CWE-378 (world-readable temp files). - */ -const PROJECT_TMP_BASE = path.join(repoRoot, ".tmp"); - -/** - * Resolve `{{tmpdir}}` placeholders in string values of a parameters object. - * Test fixtures use `{{tmpdir}}` as a cross-platform placeholder for the - * project-local temporary directory (`/.tmp`), which avoids - * writing to the world-readable OS temp directory (CWE-377 / CWE-378). - * - * @param {Record} params - Tool parameters object (mutated in place) - * @param {object} [logger] - Optional logger for diagnostics - * @returns {Record} The same object, with placeholders resolved - */ -export function resolvePathPlaceholders(params, logger) { - fs.mkdirSync(PROJECT_TMP_BASE, { recursive: true }); - for (const [key, value] of Object.entries(params)) { - if (typeof value === "string" && value.includes("{{tmpdir}}")) { - params[key] = value.replace(/\{\{tmpdir\}\}/g, PROJECT_TMP_BASE); - if (logger) { - logger.log(` Resolved ${key}: {{tmpdir}} → ${params[key]}`); - } - } - } - return params; -} - -/** - * Integration test runner class - */ -export class IntegrationTestRunner { - constructor(client, logger, options = {}) { - this.client = client; - this.logger = logger; - this.options = options; - } - - /** - * Call an MCP tool with an appropriate timeout. - * - * All codeql_* tools invoke the CodeQL CLI or language server JVM, which - * can be slow in CI (cold JVM start, network pack downloads, Windows - * runner overhead). A generous 5-minute timeout avoids intermittent - * -32001 RequestTimeout failures. - */ - async callTool(toolName, args) { - const isCodeQLTool = toolName.startsWith("codeql_"); - const requestOptions = { - timeout: isCodeQLTool ? 300000 : 60000, - resetTimeoutOnProgress: isCodeQLTool - }; - return await this.client.callTool( - { name: toolName, arguments: args }, - undefined, - requestOptions - ); - } - - /** - * Run integration tests for a specific tool - */ - async runToolIntegrationTests(toolName, integrationTestsDir, filterTests = null) { - try { - const toolDir = path.join(integrationTestsDir, toolName); - let testCases = fs - .readdirSync(toolDir, { withFileTypes: true }) - .filter((dirent) => dirent.isDirectory()) - .map((dirent) => dirent.name); - - // Apply test filter if provided - if (filterTests && filterTests.length > 0) { - testCases = testCases.filter((testCase) => filterTests.includes(testCase)); - if (testCases.length === 0) { - this.logger.log( - `No matching tests found for ${toolName} with filter: ${filterTests.join(", ")}`, - "WARN" - ); - return 0; - } - } - - this.logger.log(`Running ${testCases.length} test cases for ${toolName}`); - - for (const testCase of testCases) { - await this.runSingleIntegrationTest(toolName, testCase, toolDir); - } - - return testCases.length; - } catch (error) { - this.logger.log(`Error running tests for ${toolName}: ${error.message}`, "ERROR"); - return 0; - } - } - - /** - * Discover and run tool-specific integration tests - */ - async runIntegrationTests(baseDir) { - try { - this.logger.log("Discovering and running tool-specific integration tests..."); - - const integrationTestsDir = path.join( - baseDir, - "..", - "integration-tests", - "primitives", - "tools" - ); - - if (!fs.existsSync(integrationTestsDir)) { - this.logger.log("No integration tests directory found", "WARN"); - return true; - } - - // Get list of available tools from the server - const response = await this.client.listTools(); - const tools = response.tools || []; - const toolNames = tools.map((t) => t.name); - - this.logger.log(`Found ${toolNames.length} tools to test: ${toolNames.join(", ")}`); - - // Discover tool test directories - const toolTestDirs = fs - .readdirSync(integrationTestsDir, { withFileTypes: true }) - .filter((dirent) => dirent.isDirectory()) - .map((dirent) => dirent.name); - - this.logger.log( - `Found ${toolTestDirs.length} tool test directories: ${toolTestDirs.join(", ")}` - ); - - // Define tool execution priority to ensure database-creating tools run first - const toolPriority = { - // Priority 1: Ensure pack dependencies are installed for static/{src,test} dirs - codeql_pack_install: 1, - - // Priority 2: Database creation tools (must run before codeql_query_run) - codeql_test_extract: 2, - codeql_database_create: 2, - - // Priority 3: Tools that depend on databases (run after database creation) - codeql_query_run: 3, - codeql_test_run: 3, - codeql_bqrs_decode: 3, - codeql_bqrs_info: 3, - codeql_database_analyze: 3, - codeql_resolve_database: 3, - - // Priority 3.5: LSP diagnostics runs before other LSP tools to warm up - // the language server JVM. Subsequent codeql_lsp_* tools reuse the - // running server and avoid the cold-start penalty. - codeql_lsp_diagnostics: 3.5 - - // Priority 4: All other tools (default priority) - // These tools don't have specific database dependencies - }; - - // Sort tool test directories by priority - const sortedToolTestDirs = toolTestDirs.sort((a, b) => { - const priorityA = toolPriority[a] || 4; - const priorityB = toolPriority[b] || 4; - - // If priorities are the same, sort alphabetically for consistency - if (priorityA === priorityB) { - return a.localeCompare(b); - } - - return priorityA - priorityB; - }); - - // Apply tool filter if provided in options - let toolsToTest = sortedToolTestDirs; - if (this.options.tools && this.options.tools.length > 0) { - toolsToTest = sortedToolTestDirs.filter((tool) => this.options.tools.includes(tool)); - if (toolsToTest.length === 0) { - this.logger.log( - `No matching tools found with filter: ${this.options.tools.join(", ")}`, - "WARN" - ); - return false; - } - this.logger.log(`Filtered to ${toolsToTest.length} tools: ${toolsToTest.join(", ")}`); - } - - this.logger.log(`Executing tests in priority order: ${toolsToTest.join(", ")}`); - - // Run tests for each tool in priority order - let totalIntegrationTests = 0; - for (const toolName of toolsToTest) { - if (toolNames.includes(toolName)) { - const testCount = await this.runToolIntegrationTests( - toolName, - integrationTestsDir, - this.options.tests - ); - totalIntegrationTests += testCount; - } else { - this.logger.log(`Skipping ${toolName} - tool not found in server`, "WARN"); - } - } - - this.logger.log(`Completed ${totalIntegrationTests} tool-specific integration tests`); - - // Track execution counts and pass/fail separately. - // *Count* tracks whether the suite discovered & ran tests. - // *Succeeded* tracks whether those tests passed. - const toolTestsExecuted = totalIntegrationTests; - const toolTestsPassed = totalIntegrationTests > 0; - - // Also run workflow integration tests - const { executed: workflowTestsExecuted, passed: workflowTestsPassed } = - await this.runWorkflowIntegrationTests(baseDir); - if (!workflowTestsPassed) { - this.logger.logTest( - "Workflow integration tests", - false, - new Error("Workflow integration tests did not complete successfully") - ); - } - - // Also run prompt integration tests - const { executed: promptTestsExecuted, passed: promptTestsPassed } = - await this.runPromptIntegrationTests(baseDir); - if (!promptTestsPassed) { - this.logger.logTest( - "Prompt integration tests", - false, - new Error("Prompt integration tests did not complete successfully") - ); - } - - const totalTestsExecuted = toolTestsExecuted + workflowTestsExecuted + promptTestsExecuted; - - if (totalTestsExecuted === 0) { - this.logger.log( - "No integration tests were executed across tool, workflow, or prompt suites.", - "ERROR" - ); - return false; - } - - return toolTestsPassed && workflowTestsPassed && promptTestsPassed; - } catch (error) { - this.logger.log(`Error running integration tests: ${error.message}`, "ERROR"); - return false; - } - } - - /** - * Run a single integration test case - */ - async runSingleIntegrationTest(toolName, testCase, toolDir) { - try { - const testDir = path.join(toolDir, testCase); - const beforeDir = path.join(testDir, "before"); - const afterDir = path.join(testDir, "after"); - - if (!fs.existsSync(beforeDir) || !fs.existsSync(afterDir)) { - throw new Error(`Missing before or after directory for ${toolName}/${testCase}`); - } - - // Check if this test case has monitoring integration (monitoring-state.json files) - const beforeMonitoringState = path.join(beforeDir, "monitoring-state.json"); - const afterMonitoringState = path.join(afterDir, "monitoring-state.json"); - - if (fs.existsSync(beforeMonitoringState) && fs.existsSync(afterMonitoringState)) { - await this.runMonitoringBasedTest(toolName, testCase, testDir); - return; - } - - // Handle special case for codeql_lsp_diagnostics tool - if (toolName === "codeql_lsp_diagnostics") { - await this.runLanguageServerEvalTest(toolName, testCase, beforeDir, afterDir); - return; - } - - // Check for test configuration file - const testConfigPath = path.join(testDir, "test-config.json"); - if (fs.existsSync(testConfigPath)) { - await this.runConfigurableTest(toolName, testCase, testDir, testConfigPath); - return; - } - - // Create temp directory for test execution under project .tmp/ - fs.mkdirSync(PROJECT_TMP_BASE, { recursive: true }); - const tempDir = fs.mkdtempSync( - path.join(PROJECT_TMP_BASE, `mcp-test-${toolName}-${testCase}-`) - ); - - try { - // Copy before files to temp directory - copyDirectory(beforeDir, tempDir); - - // Get files to process - const files = fs.readdirSync(tempDir).map((f) => path.join(tempDir, f)); - - if (files.length === 0) { - throw new Error(`No files found in before directory for ${toolName}/${testCase}`); - } - - // Run the tool - const result = await this.callTool(toolName, { - "files": files, - "in-place": true - }); - - this.logger.log(`Tool ${toolName} result: ${result.content?.[0]?.text || "No output"}`); - - // Compare results with expected after state - const passed = compareDirectories(tempDir, afterDir); - - this.logger.logTest(`Integration Test: ${toolName}/${testCase}`, passed); - - if (passed) { - this.logger.log(`✅ ${toolName}/${testCase} - Files match expected output`); - } else { - this.logger.log(`❌ ${toolName}/${testCase} - Files do not match expected output`); - } - } finally { - // Cleanup temp directory - removeDirectory(tempDir); - } - } catch (error) { - this.logger.logTest(`Integration Test: ${toolName}/${testCase}`, false, error); - } - } - - /** - * Special test runner for codeql_lsp_diagnostics tool - * This tool validates QL code and returns diagnostics, rather than modifying files - */ - async runLanguageServerEvalTest(toolName, testCase, beforeDir, afterDir) { - try { - // Get the QL files from before and after directories - const beforeFiles = fs.readdirSync(beforeDir).filter((f) => f.endsWith(".ql")); - const afterFiles = fs.readdirSync(afterDir).filter((f) => f.endsWith(".ql")); - - if (beforeFiles.length === 0) { - throw new Error(`No .ql files found in before directory for ${toolName}/${testCase}`); - } - - if (beforeFiles.length !== afterFiles.length) { - throw new Error( - `Mismatch in number of .ql files between before and after directories for ${toolName}/${testCase}` - ); - } - - let allTestsPassed = true; - - // Test each QL file - for (const beforeFile of beforeFiles) { - const afterFile = afterFiles.find((f) => f === beforeFile); - if (!afterFile) { - throw new Error( - `Missing corresponding after file for ${beforeFile} in ${toolName}/${testCase}` - ); - } - - const beforePath = path.join(beforeDir, beforeFile); - const afterPath = path.join(afterDir, afterFile); - - const beforeContent = fs.readFileSync(beforePath, "utf8"); - // afterContent represents the corrected version for reference - // eslint-disable-next-line no-unused-vars - const _afterContent = fs.readFileSync(afterPath, "utf8"); - - // Run the codeql_lsp_diagnostics tool on the before content - const result = await this.callTool(toolName, { - ql_code: beforeContent - }); - - this.logger.log( - `Tool ${toolName} result for ${beforeFile}: ${result.content?.[0]?.text || "No output"}` - ); - - // Parse the validation result - let validationResult; - try { - validationResult = JSON.parse(result.content?.[0]?.text || "{}"); - // eslint-disable-next-line no-unused-vars - } catch (_parseError) { - this.logger.log( - `❌ ${toolName}/${testCase}/${beforeFile} - Failed to parse validation result` - ); - allTestsPassed = false; - continue; - } - - // The test passes if: - // 1. The tool detects errors in the "before" content (isValid should be false) - // 2. The "after" content represents a corrected version - // For this integration test, we mainly verify that the tool can detect issues - - if ( - validationResult.isValid === false && - validationResult.diagnostics && - validationResult.diagnostics.length > 0 - ) { - this.logger.log( - `✅ ${toolName}/${testCase}/${beforeFile} - Tool correctly detected ${validationResult.diagnostics.length} validation issues` - ); - } else if (validationResult.isValid === true) { - // If the before content is actually valid, that's also a passing test scenario - this.logger.log( - `✅ ${toolName}/${testCase}/${beforeFile} - Tool correctly validated valid QL code` - ); - } else { - this.logger.log( - `❌ ${toolName}/${testCase}/${beforeFile} - Tool validation failed or returned unexpected result` - ); - allTestsPassed = false; - } - - // Log diagnostic details for visibility - if (validationResult.diagnostics && validationResult.diagnostics.length > 0) { - this.logger.log(` Diagnostics found:`); - validationResult.diagnostics.forEach((diagnostic, index) => { - this.logger.log( - ` ${index + 1}. ${diagnostic.severity} at line ${diagnostic.line}, column ${diagnostic.column}: ${diagnostic.message}` - ); - }); - } - } - - this.logger.logTest(`Integration Test: ${toolName}/${testCase}`, allTestsPassed); - - if (allTestsPassed) { - this.logger.log(`✅ ${toolName}/${testCase} - All validation tests passed`); - } else { - this.logger.log(`❌ ${toolName}/${testCase} - Some validation tests failed`); - } - } catch (error) { - this.logger.logTest(`Integration Test: ${toolName}/${testCase}`, false, error); - } - } - - /** - * Run a test with custom configuration - * Uses test-config.json to specify tool arguments - */ - async runConfigurableTest(toolName, testCase, testDir, testConfigPath) { - try { - const beforeDir = path.join(testDir, "before"); - const afterDir = path.join(testDir, "after"); - - // Load test configuration - const testConfigContent = fs.readFileSync(testConfigPath, "utf8"); - const testConfig = JSON.parse(testConfigContent); - - if (!testConfig.arguments) { - throw new Error(`Test config missing arguments for ${toolName}/${testCase}`); - } - - // For session tools, we need to handle them differently since they work with monitoring state - // rather than files. Check if this test has monitoring-state.json files. - const beforeMonitoringState = path.join(beforeDir, "monitoring-state.json"); - const afterMonitoringState = path.join(afterDir, "monitoring-state.json"); - - if (fs.existsSync(beforeMonitoringState) && fs.existsSync(afterMonitoringState)) { - // This is a monitoring-based test - await this.runMonitoringBasedTest( - toolName, - testCase, - testConfig, - beforeMonitoringState, - afterMonitoringState - ); - } else { - // Regular file-based test with custom arguments - await this.runFileBasedConfigurableTest( - toolName, - testCase, - testConfig, - beforeDir, - afterDir - ); - } - } catch (error) { - this.logger.logTest(`Integration Test: ${toolName}/${testCase}`, false, error); - } - } - - /** - * Run a file-based test with custom configuration - */ - async runFileBasedConfigurableTest(toolName, testCase, testConfig, beforeDir, afterDir) { - fs.mkdirSync(PROJECT_TMP_BASE, { recursive: true }); - const tempDir = fs.mkdtempSync( - path.join(PROJECT_TMP_BASE, `mcp-test-${toolName}-${testCase}-`) - ); - - try { - // Copy before files to temp directory - copyDirectory(beforeDir, tempDir); - - // Resolve {{tmpdir}} placeholders in arguments - resolvePathPlaceholders(testConfig.arguments, this.logger); - - // Run the tool with custom arguments - const result = await this.callTool(toolName, testConfig.arguments); - - this.logger.log(`Tool ${toolName} result: ${result.content?.[0]?.text || "No output"}`); - - // Compare results with expected after state - const passed = compareDirectories(tempDir, afterDir); - - this.logger.logTest(`Integration Test: ${toolName}/${testCase}`, passed); - - if (passed) { - this.logger.log(`✅ ${toolName}/${testCase} - Files match expected output`); - } else { - this.logger.log(`❌ ${toolName}/${testCase} - Files do not match expected output`); - } - } finally { - // Cleanup temp directory - removeDirectory(tempDir); - } - } - - /** - * Validate codeql_query_run output by comparing actual vs expected output - * @param {string} interpretedOutput - Path to actual output (file or directory) - * @param {string} expectedOutputPath - Path to expected output in after/ directory - * @param {string} toolName - Name of the tool being tested - * @param {string} testCase - Name of the test case - * @returns {boolean} - True if validation passed, false otherwise - */ - validateCodeQLQueryRunOutput(interpretedOutput, expectedOutputPath, toolName, testCase) { - try { - // Read both paths directly to avoid TOCTOU race (CWE-367). - // If a path is a directory, readFileSync throws EISDIR. - // If a path doesn't exist, readFileSync throws ENOENT. - let actualContent, expectedContent; - let actualIsDir = false, - expectedIsDir = false; - - try { - actualContent = fs.readFileSync(interpretedOutput, "utf8"); - } catch (readErr) { - if (readErr.code === "EISDIR") { - actualIsDir = true; - } else { - throw readErr; - } - } - - try { - expectedContent = fs.readFileSync(expectedOutputPath, "utf8"); - } catch (readErr) { - if (readErr.code === "EISDIR") { - expectedIsDir = true; - } else if (readErr.code === "ENOENT") { - // Expected output doesn't exist - validate non-empty content only - this.logger.log( - ` Note: No expected output found at ${expectedOutputPath}, validating non-empty content only` - ); - return this.validateNonEmptyOutput(interpretedOutput, toolName, testCase); - } else { - throw readErr; - } - } - - if (actualIsDir && expectedIsDir) { - // Compare directory structures - const comparisonResult = compareDirectories(interpretedOutput, expectedOutputPath); - if (!comparisonResult) { - this.logger.log( - ` Validation Failed: Output files do not match expected output for ${toolName}/${testCase}` - ); - return false; - } else { - this.logger.log(` ✓ Output files match expected output`); - return true; - } - } else if (!actualIsDir && !expectedIsDir) { - // Compare file contents (already read above) - if (actualContent !== expectedContent) { - this.logger.log( - ` Validation Failed: Output content does not match expected content for ${toolName}/${testCase}` - ); - this.logger.log( - ` Expected ${expectedContent.length} chars, got ${actualContent.length} chars` - ); - return false; - } else { - this.logger.log(` ✓ Output content matches expected output`); - return true; - } - } else { - this.logger.log( - ` Validation Failed: Output type mismatch (file vs directory) for ${toolName}/${testCase}` - ); - return false; - } - } catch (error) { - this.logger.log( - ` Validation Error: Failed to validate output for ${toolName}/${testCase}: ${error.message}` - ); - return false; - } - } - - /** - * Validate that output exists and is non-empty - * @param {string} outputPath - Path to output (file or directory) - * @param {string} toolName - Name of the tool being tested - * @param {string} testCase - Name of the test case - * @returns {boolean} - True if validation passed, false otherwise - */ - validateNonEmptyOutput(outputPath, toolName, testCase) { - try { - // Try reading as a file first to avoid TOCTOU race (CWE-367). - // If the path is a directory, readFileSync throws EISDIR. - try { - const content = fs.readFileSync(outputPath, "utf8"); - if (content.trim().length === 0) { - this.logger.log(` Validation Failed: Output file is empty for ${toolName}/${testCase}`); - return false; - } - return true; - } catch (readErr) { - if (readErr.code !== "EISDIR") { - throw readErr; - } - } - - // Path is a directory - find and check output files - const allFiles = getDirectoryFiles(outputPath); - - // Filter for relevant output file extensions - const outputExtensions = [".txt", ".dgml", ".dot", ".sarif", ".csv", ".json"]; - const outputFiles = allFiles.filter((file) => - outputExtensions.some((ext) => file.endsWith(ext)) - ); - - if (outputFiles.length === 0) { - this.logger.log( - ` Validation Failed: No output files found in ${outputPath} for ${toolName}/${testCase}` - ); - return false; - } - - // Check that at least one file has non-empty content - let hasNonEmptyContent = false; - for (const file of outputFiles) { - const content = fs.readFileSync(file, "utf8"); - if (content.trim().length > 0) { - hasNonEmptyContent = true; - break; - } - } - - if (!hasNonEmptyContent) { - this.logger.log( - ` Validation Failed: All output files are empty for ${toolName}/${testCase}` - ); - return false; - } - - return true; - } catch (error) { - this.logger.log( - ` Validation Error: Failed to check output at ${outputPath} for ${toolName}/${testCase}: ${error.message}` - ); - return false; - } - } - - /** - * Run a monitoring-based test that uses monitoring-state.json files - */ - async runMonitoringBasedTest(toolName, testCase, testCaseDir) { - try { - this.logger.log(`Running monitoring-based test: ${toolName}/${testCase}`); - - // For query compilation tests, install pack dependencies first - if (toolName === "codeql_query_compile") { - const staticPath = this.getStaticFilesPath(); - const packDir = path.join(staticPath, "src"); - - // Check if qlpack.yml exists and install dependencies - if (fs.existsSync(path.join(packDir, "codeql-pack.yml"))) { - try { - await this.callTool("codeql_pack_install", { packDir: packDir }); - } catch (installError) { - this.logger.log( - ` Warning: Could not install pack dependencies: ${installError.message}` - ); - } - } - } - - // Check if there's a monitoring-state.json file with parameters - let params; - const beforeDir = path.join(testCaseDir, "before"); - const monitoringStatePath = path.join(beforeDir, "monitoring-state.json"); - - if (fs.existsSync(monitoringStatePath)) { - const monitoringState = JSON.parse(fs.readFileSync(monitoringStatePath, "utf8")); - if (monitoringState.parameters) { - params = monitoringState.parameters; - this.logger.log(`Using parameters from monitoring-state.json`); - resolvePathPlaceholders(params, this.logger); - - // Helper function to ensure database is extracted - const ensureDatabaseExtracted = async (dbPath) => { - // Resolve paths relative to repository root (parent of client directory) - const currentDir = path.dirname(fileURLToPath(import.meta.url)); - const clientDir = path.dirname(path.dirname(currentDir)); // Go up to client/ - const repoRoot = path.dirname(clientDir); // Go up to repo root - const absoluteDbPath = path.resolve(repoRoot, dbPath); - - // Check if database needs to be extracted (test source directory exists but not the .testproj) - if (!fs.existsSync(absoluteDbPath) && dbPath.endsWith(".testproj")) { - // For paths like "test/ExpressSqlInjection/ExpressSqlInjection.testproj", - // the test source directory is "test/ExpressSqlInjection" - const parts = dbPath.split(path.sep); - const lastPart = parts[parts.length - 1]; - const testName = lastPart.replace(".testproj", ""); - const parentDir = parts.slice(0, -1).join(path.sep); - - // Check if the parent directory name matches the test name - const parentDirName = parts[parts.length - 2]; - const testSourceDir = - parentDirName === testName ? parentDir : dbPath.replace(/\.testproj$/, ""); - const absoluteTestSourceDir = path.resolve(repoRoot, testSourceDir); - - if (fs.existsSync(absoluteTestSourceDir)) { - this.logger.log(`Database not found, extracting from ${testSourceDir}`); - const extractResult = await this.callTool("codeql_test_extract", { - tests: [testSourceDir] - }); - if (extractResult.isError) { - const errorText = extractResult.content?.[0]?.text || "Unknown error"; - throw new Error(`Failed to extract database: ${errorText}`); - } - this.logger.log(`Database extracted successfully to ${dbPath}`); - } - } - }; - - // For codeql_query_run with database parameter, ensure database is extracted - if (toolName === "codeql_query_run" && params.database) { - await ensureDatabaseExtracted(params.database); - } - - // For codeql_resolve_database, ensure database is extracted - if (toolName === "codeql_resolve_database" && params.database) { - await ensureDatabaseExtracted(params.database); - } - } else { - // Fall back to tool-specific parameters - params = await this.getToolSpecificParams(toolName, testCase); - } - } else { - // Fall back to tool-specific parameters - params = await this.getToolSpecificParams(toolName, testCase); - } - - // Call the tool with appropriate parameters (timeout is handled by this.callTool) - this.logger.log(`Calling tool ${toolName}`); - - const result = await this.callTool(toolName, params); - - // For monitoring tests, we primarily check if the tool executed successfully - // Special handling for session management tools that expect sessions to exist - let success = !result.isError; - - // Session management tools should return proper error messages when sessions don't exist - if (result.isError && (toolName.startsWith("session_") || toolName.startsWith("sessions_"))) { - const errorText = result.content?.[0]?.text || ""; - if ( - errorText.includes("Session not found") || - errorText.includes("No valid sessions found") - ) { - success = true; // This is expected behavior for missing sessions - this.logger.log( - ` Note: Tool correctly handled missing session - this is expected behavior` - ); - } - } - - // Special validation for codeql_query_run with output file comparison - if (success && toolName === "codeql_query_run") { - const resultText = result.content?.[0]?.text || ""; - - // Check for query interpretation failures - if (resultText.includes("Query interpretation failed")) { - success = false; - this.logger.log( - ` Validation Failed: Query interpretation failed for ${toolName}/${testCase}` - ); - } - - // Compare actual output files against expected output in after/ directory - if (success && params.interpretedOutput) { - try { - if (fs.existsSync(params.interpretedOutput)) { - const afterDir = path.join(testCaseDir, "after"); - const expectedOutputPath = path.join( - afterDir, - path.basename(params.interpretedOutput) - ); - - // Use extracted validation method - const validationPassed = this.validateCodeQLQueryRunOutput( - params.interpretedOutput, - expectedOutputPath, - toolName, - testCase - ); - - if (!validationPassed) { - success = false; - } - } - } catch (error) { - success = false; - this.logger.log( - ` Validation Error: Failed to validate output for ${toolName}/${testCase}: ${error.message}` - ); - } - } - } - - this.logger.logTest(`Integration Test: ${toolName}/${testCase}`, success); - - if (success) { - this.logger.log(`✅ ${toolName}/${testCase} - Tool executed successfully`); - // Truncate long results to avoid excessive CI log output - const resultText = result.content?.[0]?.text || "No content"; - const MAX_LOG_LENGTH = 500; - if (resultText.length > MAX_LOG_LENGTH) { - this.logger.log( - ` Result: ${resultText.substring(0, MAX_LOG_LENGTH)}... (truncated, ${resultText.length} chars total)` - ); - } else { - this.logger.log(` Result: ${resultText}`); - } - } else { - this.logger.log(`❌ ${toolName}/${testCase} - Tool execution failed`); - const errorText = result.content?.[0]?.text || "Unknown error"; - this.logger.log(` Error: ${errorText}`); - // Also log the actual error to help with debugging - if (result.error) { - this.logger.log(` Debug: ${JSON.stringify(result.error)}`); - } - } - - return success; - } catch (error) { - this.logger.logTest(`Integration Test: ${toolName}/${testCase}`, false, error); - this.logger.log(`❌ ${toolName}/${testCase} - Exception occurred: ${error.message}`); - this.logger.log(` Stack: ${error.stack}`); - return false; - } - } - - /** - * Get path to static test files - */ - getStaticFilesPath(language = "javascript") { - const currentDir = path.dirname(fileURLToPath(import.meta.url)); - return path.join( - path.dirname(path.dirname(path.dirname(currentDir))), // Go up to the repo root directory - "server", - "ql", - language, - "examples" - ); - } - - /** - * Get tool-specific parameters for testing - */ - async getToolSpecificParams(toolName, testCase) { - const params = {}; - - // Get the test case directory to find actual files - // Using fileURLToPath(import.meta.url) instead of __dirname for ES modules - const currentDir = path.dirname(fileURLToPath(import.meta.url)); - const testCaseDir = path.join( - path.dirname(path.dirname(currentDir)), // Go up to the client directory - "integration-tests", - "primitives", - "tools", - toolName, - testCase - ); - const beforeDir = path.join(testCaseDir, "before"); - const staticPath = this.getStaticFilesPath(); - - if (toolName === "codeql_lsp_diagnostics") { - params.ql_code = 'from UndefinedType x where x = "test" select x, "semantic error"'; - // Skip workspace_uri for now as it's not needed for basic validation - } else if (toolName === "codeql_bqrs_decode") { - // Use static BQRS file - const bqrsFile = path.join(staticPath, "src", "ExampleQuery1", "ExampleQuery1.test.bqrs"); - if (fs.existsSync(bqrsFile)) { - params.files = [bqrsFile]; - params.format = "json"; - } else { - throw new Error(`Static BQRS file not found: ${bqrsFile}`); - } - } else if (toolName === "codeql_bqrs_info") { - // Use static BQRS file - const bqrsFile = path.join(staticPath, "src", "ExampleQuery1", "ExampleQuery1.test.bqrs"); - if (fs.existsSync(bqrsFile)) { - params.file = bqrsFile; - } else { - throw new Error(`Static BQRS file not found: ${bqrsFile}`); - } - } else if (toolName === "codeql_bqrs_interpret") { - // Use actual test file from the before directory - if (fs.existsSync(beforeDir)) { - const bqrsFiles = fs.readdirSync(beforeDir).filter((f) => f.endsWith(".bqrs")); - if (bqrsFiles.length > 0) { - params.file = path.join(beforeDir, bqrsFiles[0]); - } else { - throw new Error(`No .bqrs files found in ${beforeDir} for ${toolName}/${testCase}`); - } - } else { - throw new Error(`Test directory ${beforeDir} not found for ${toolName}/${testCase}`); - } - - // Set output path and format based on test case name - const afterDir = path.join(testCaseDir, "after"); - if (testCase.includes("graphtext")) { - params.format = "graphtext"; - params.output = path.join(afterDir, "output.txt"); - params.t = ["kind=graph", "id=test/query"]; - } else if (testCase.includes("sarif")) { - params.format = "sarif-latest"; - params.output = path.join(afterDir, "results.sarif"); - params.t = ["kind=problem", "id=test/query"]; - // Note: sarif-add-snippets requires source archive, skipping for basic test - } else { - // Default to graphtext for unknown test cases - params.format = "graphtext"; - params.output = path.join(afterDir, "output.txt"); - params.t = ["kind=graph", "id=test/query"]; - } - } else if (toolName === "codeql_test_extract") { - // Use static test directory - const testDir = path.join(staticPath, "test", "ExampleQuery1"); - if (fs.existsSync(testDir)) { - params.tests = [testDir]; - } else { - throw new Error(`Static test directory not found: ${testDir}`); - } - } else if (toolName === "codeql_test_run") { - // Use static test directory - const testDir = path.join(staticPath, "test", "ExampleQuery1"); - if (fs.existsSync(testDir)) { - params.tests = [testDir]; - } else { - throw new Error(`Static test directory not found: ${testDir}`); - } - } else if (toolName === "codeql_query_run") { - // Use static query and database - const queryFile = path.join(staticPath, "src", "ExampleQuery1", "ExampleQuery1.ql"); - const databaseDir = path.join(staticPath, "test", "ExampleQuery1", "ExampleQuery1.testproj"); - const testDir = path.join(staticPath, "test", "ExampleQuery1"); - - if (fs.existsSync(queryFile)) { - params.query = queryFile; - - // If database doesn't exist, extract it first - if (!fs.existsSync(databaseDir) && fs.existsSync(testDir)) { - this.logger.log(`Database not found for query run, extracting first: ${databaseDir}`); - // Call codeql test extract to create the database - const extractResult = await this.callTool("codeql_test_extract", { - tests: [testDir] - }); - if (extractResult.isError) { - throw new Error(`Failed to extract database: ${extractResult.content[0].text}`); - } - this.logger.log(`Database extracted successfully`); - } - - if (fs.existsSync(databaseDir)) { - params.database = databaseDir; - // codeql query run outputs BQRS format by default, no output-format parameter needed - } else { - throw new Error(`Static database not found and could not be created: ${databaseDir}`); - } - } else { - throw new Error(`Static query file not found: ${queryFile}`); - } - } else if (toolName === "codeql_query_format") { - // Look for .ql files in the before directory - if (fs.existsSync(beforeDir)) { - const qlFiles = fs.readdirSync(beforeDir).filter((f) => f.endsWith(".ql")); - if (qlFiles.length > 0) { - params.files = [path.join(beforeDir, qlFiles[0])]; - } else { - throw new Error(`No .ql files found in ${beforeDir} for ${toolName}/${testCase}`); - } - } else { - throw new Error(`Test directory ${beforeDir} not found for ${toolName}/${testCase}`); - } - params["check-only"] = true; - } else if (toolName === "profile_codeql_query") { - // Read from test-config.json for profile_codeql_query tool - const testConfigPath = path.join(testCaseDir, "test-config.json"); - if (fs.existsSync(testConfigPath)) { - const testConfig = JSON.parse(fs.readFileSync(testConfigPath, "utf8")); - if (testConfig.arguments) { - Object.assign(params, testConfig.arguments); - } else { - throw new Error(`test-config.json missing arguments for ${toolName}/${testCase}`); - } - } else { - throw new Error(`test-config.json not found for ${toolName}/${testCase}`); - } - } else if (toolName === "profile_codeql_query_from_logs") { - // Read from test-config.json for profile_codeql_query_from_logs tool - const testConfigPath = path.join(testCaseDir, "test-config.json"); - if (fs.existsSync(testConfigPath)) { - const testConfig = JSON.parse(fs.readFileSync(testConfigPath, "utf8")); - if (testConfig.arguments) { - Object.assign(params, testConfig.arguments); - } else { - throw new Error(`test-config.json missing arguments for ${toolName}/${testCase}`); - } - } else { - throw new Error(`test-config.json not found for ${toolName}/${testCase}`); - } - } else if (toolName === "validate_codeql_query") { - params.query = "from int i select i"; - params.language = "java"; - } else if (toolName === "codeql_pack_ls") { - // Use the static pack directory that contains qlpack.yml - params.dir = path.join(staticPath, "src"); - } else if (toolName === "codeql_pack_install") { - // pack install needs to be run in a directory with qlpack.yml - params.packDir = path.join(staticPath, "src"); - } else if (toolName === "codeql_query_compile") { - // Use a query file from the static src directory that has proper pack context - const staticQueryFile = path.join(staticPath, "src", "ExampleQuery1", "ExampleQuery1.ql"); - if (fs.existsSync(staticQueryFile)) { - params.query = staticQueryFile; - } else { - throw new Error(`Static query file not found: ${staticQueryFile}`); - } - } else if (toolName === "codeql_resolve_library_path") { - // Use the static pack directory - params.packDir = path.join(staticPath, "src"); - } else if (toolName === "codeql_resolve_metadata") { - // Look for .ql files in the before directory - if (fs.existsSync(beforeDir)) { - const qlFiles = fs.readdirSync(beforeDir).filter((f) => f.endsWith(".ql")); - if (qlFiles.length > 0) { - params.query = path.join(beforeDir, qlFiles[0]); - } else { - throw new Error(`No .ql files found in ${beforeDir} for ${toolName}/${testCase}`); - } - } else { - throw new Error(`Test directory ${beforeDir} not found for ${toolName}/${testCase}`); - } - } else if (toolName === "codeql_resolve_queries") { - // Use the test case directory as the queries path - params.path = beforeDir; - } else if (toolName === "codeql_resolve_tests") { - // Use the test case directory as the tests path - params.tests = [beforeDir]; - } else if (toolName === "codeql_test_accept") { - // Use the test case directory as the tests directory - params.tests = [beforeDir]; - } else if (toolName === "find_class_position") { - // Look for .ql files in the before directory - if (fs.existsSync(beforeDir)) { - const qlFiles = fs.readdirSync(beforeDir).filter((f) => f.endsWith(".ql")); - if (qlFiles.length > 0) { - params.file = path.join(beforeDir, qlFiles[0]); - } else { - throw new Error(`No .ql files found in ${beforeDir} for ${toolName}/${testCase}`); - } - } else { - throw new Error(`Test directory ${beforeDir} not found for ${toolName}/${testCase}`); - } - params.name = "TestClass"; - } else if (toolName === "find_predicate_position") { - // Look for .ql files in the before directory - if (fs.existsSync(beforeDir)) { - const qlFiles = fs.readdirSync(beforeDir).filter((f) => f.endsWith(".ql")); - if (qlFiles.length > 0) { - params.file = path.join(beforeDir, qlFiles[0]); - } else { - throw new Error(`No .ql files found in ${beforeDir} for ${toolName}/${testCase}`); - } - } else { - throw new Error(`Test directory ${beforeDir} not found for ${toolName}/${testCase}`); - } - params.name = "testPredicate"; - } else if (toolName === "session_calculate_current_score") { - params.sessionId = "test-session-score"; - } else if (toolName === "session_end") { - params.sessionId = "test-session-end"; - params.status = "completed"; - } else if (toolName === "sessions_compare") { - params.sessionIds = ["session-1", "session-2"]; - } else if ( - toolName === "sarif_extract_rule" || - toolName === "sarif_list_rules" || - toolName === "sarif_rule_to_markdown" - ) { - // Look for .sarif files in the before directory - if (fs.existsSync(beforeDir)) { - const sarifFiles = fs.readdirSync(beforeDir).filter((f) => f.endsWith(".sarif")); - if (sarifFiles.length > 0) { - params.sarifPath = path.join(beforeDir, sarifFiles[0]); - } else { - throw new Error(`No .sarif files found in ${beforeDir} for ${toolName}/${testCase}`); - } - } else { - throw new Error(`Test directory ${beforeDir} not found for ${toolName}/${testCase}`); - } - // Read ruleId from test-config.json arguments - const sarifTestConfigPath = path.join(testCaseDir, "test-config.json"); - if (fs.existsSync(sarifTestConfigPath)) { - const sarifTestConfig = JSON.parse(fs.readFileSync(sarifTestConfigPath, "utf8")); - if (sarifTestConfig.arguments?.ruleId) { - params.ruleId = sarifTestConfig.arguments.ruleId; - } - } - } else if (toolName === "sarif_compare_alerts") { - // Look for .sarif files in the before directory - if (fs.existsSync(beforeDir)) { - const sarifFiles = fs.readdirSync(beforeDir).filter((f) => f.endsWith(".sarif")); - if (sarifFiles.length > 0) { - const sarifPath = path.join(beforeDir, sarifFiles[0]); - // Read alert specs and overlapMode from test-config.json - const compareConfigPath = path.join(testCaseDir, "test-config.json"); - if (fs.existsSync(compareConfigPath)) { - const compareConfig = JSON.parse(fs.readFileSync(compareConfigPath, "utf8")); - const args = compareConfig.arguments || {}; - params.alertA = { ...args.alertA, sarifPath }; - params.alertB = { ...args.alertB, sarifPath }; - if (args.overlapMode) { - params.overlapMode = args.overlapMode; - } - } - } else { - throw new Error(`No .sarif files found in ${beforeDir} for ${toolName}/${testCase}`); - } - } else { - throw new Error(`Test directory ${beforeDir} not found for ${toolName}/${testCase}`); - } - } else if (toolName === "sarif_diff_runs") { - // Look for .sarif files in the before directory — expects two files - if (fs.existsSync(beforeDir)) { - const sarifFiles = fs - .readdirSync(beforeDir) - .filter((f) => f.endsWith(".sarif")) - .sort(); - if (sarifFiles.length >= 2) { - params.sarifPathA = path.join(beforeDir, sarifFiles[0]); - params.sarifPathB = path.join(beforeDir, sarifFiles[1]); - } else if (sarifFiles.length === 1) { - params.sarifPathA = path.join(beforeDir, sarifFiles[0]); - params.sarifPathB = path.join(beforeDir, sarifFiles[0]); - } else { - throw new Error(`No .sarif files found in ${beforeDir} for ${toolName}/${testCase}`); - } - } else { - throw new Error(`Test directory ${beforeDir} not found for ${toolName}/${testCase}`); - } - // Read labels from test-config.json arguments - const diffConfigPath = path.join(testCaseDir, "test-config.json"); - if (fs.existsSync(diffConfigPath)) { - const diffConfig = JSON.parse(fs.readFileSync(diffConfigPath, "utf8")); - const args = diffConfig.arguments || {}; - if (args.labelA) params.labelA = args.labelA; - if (args.labelB) params.labelB = args.labelB; - } - } - - return params; - } - - /** - * Run prompt-level integration tests. - * Discovers test fixtures under `integration-tests/primitives/prompts/` - * and calls `client.getPrompt()` for each, validating the response. - */ - async runPromptIntegrationTests(baseDir) { - try { - this.logger.log("Discovering and running prompt integration tests..."); - - const promptTestsDir = path.join(baseDir, "..", "integration-tests", "primitives", "prompts"); - - if (!fs.existsSync(promptTestsDir)) { - this.logger.log("No prompt integration tests directory found", "INFO"); - return { - executed: 0, - passed: true - }; - } - - // Get list of available prompts from the server - const response = await this.client.listPrompts(); - const prompts = response.prompts || []; - const promptNames = prompts.map((p) => p.name); - - this.logger.log(`Found ${promptNames.length} prompts available on server`); - - // Discover prompt test directories (each subdirectory = one prompt name) - const promptDirs = fs - .readdirSync(promptTestsDir, { withFileTypes: true }) - .filter((dirent) => dirent.isDirectory()) - .map((dirent) => dirent.name); - - this.logger.log( - `Found ${promptDirs.length} prompt test directories: ${promptDirs.join(", ")}` - ); - - let totalPromptTests = 0; - let allPromptTestsPassed = true; - for (const promptName of promptDirs) { - if (!promptNames.includes(promptName)) { - this.logger.log(`Skipping ${promptName} - prompt not found on server`, "WARN"); - continue; - } - - const promptDir = path.join(promptTestsDir, promptName); - const testCases = fs - .readdirSync(promptDir, { withFileTypes: true }) - .filter((dirent) => dirent.isDirectory()) - .map((dirent) => dirent.name); - - this.logger.log(`Running ${testCases.length} test case(s) for prompt ${promptName}`); - - for (const testCase of testCases) { - const passed = await this.runSinglePromptIntegrationTest(promptName, testCase, promptDir); - totalPromptTests++; - if (!passed) { - allPromptTestsPassed = false; - } - } - } - - this.logger.log(`Total prompt integration tests executed: ${totalPromptTests}`); - return { - executed: totalPromptTests, - passed: totalPromptTests > 0 ? allPromptTestsPassed : true - }; - } catch (error) { - this.logger.log(`Error running prompt integration tests: ${error.message}`, "ERROR"); - return { executed: 0, passed: false }; - } - } - - /** - * Run a single prompt integration test. - * - * Reads parameters from `before/monitoring-state.json`, calls `getPrompt()`, - * and validates that the response contains messages (no protocol-level error). - */ - async runSinglePromptIntegrationTest(promptName, testCase, promptDir) { - const testName = `prompt:${promptName}/${testCase}`; - this.logger.log(`\nRunning prompt integration test: ${testName}`); - - try { - const testCaseDir = path.join(promptDir, testCase); - const beforeDir = path.join(testCaseDir, "before"); - const afterDir = path.join(testCaseDir, "after"); - - // Validate test structure - if (!fs.existsSync(beforeDir)) { - this.logger.logTest(testName, false, "Missing before directory"); - return false; - } - - if (!fs.existsSync(afterDir)) { - this.logger.logTest(testName, false, "Missing after directory"); - return false; - } - - // Load parameters from before/monitoring-state.json - const monitoringStatePath = path.join(beforeDir, "monitoring-state.json"); - if (!fs.existsSync(monitoringStatePath)) { - this.logger.logTest(testName, false, "Missing before/monitoring-state.json"); - return false; - } - - const monitoringState = JSON.parse(fs.readFileSync(monitoringStatePath, "utf8")); - const params = monitoringState.parameters || {}; - resolvePathPlaceholders(params, this.logger); - - // Call the prompt - this.logger.log(`Calling prompt ${promptName} with params: ${JSON.stringify(params)}`); - - const result = await this.client.getPrompt({ - name: promptName, - arguments: params - }); - - // Validate that the response contains messages (no raw protocol error) - const hasMessages = result.messages && result.messages.length > 0; - if (!hasMessages) { - this.logger.logTest(testName, false, "Expected messages in prompt response"); - return false; - } - - // If the after/monitoring-state.json has expected content checks, validate - const afterMonitoringPath = path.join(afterDir, "monitoring-state.json"); - if (fs.existsSync(afterMonitoringPath)) { - const afterState = JSON.parse(fs.readFileSync(afterMonitoringPath, "utf8")); - - // Support both top-level expectedContentPatterns and sessions[].expectedContentPatterns - const sessions = afterState.sessions || []; - const topLevelPatterns = afterState.expectedContentPatterns || []; - const sessionPatterns = - sessions.length > 0 ? sessions[0].expectedContentPatterns || [] : []; - const expectedPatterns = topLevelPatterns.length > 0 ? topLevelPatterns : sessionPatterns; - - if (expectedPatterns.length > 0) { - const text = result.messages[0]?.content?.text || ""; - for (const pattern of expectedPatterns) { - if (!text.includes(pattern)) { - this.logger.logTest(testName, false, `Expected response to contain "${pattern}"`); - return false; - } - } - } - } - - this.logger.logTest(testName, true, `Prompt returned ${result.messages.length} message(s)`); - return true; - } catch (error) { - this.logger.logTest(testName, false, `Error: ${error.message}`); - return false; - } - } - - /** - * Run workflow-level integration tests - * These tests validate complete workflows rather than individual tools - */ - async runWorkflowIntegrationTests(baseDir) { - try { - this.logger.log("Discovering and running workflow integration tests..."); - - const workflowTestsDir = path.join(baseDir, "..", "integration-tests", "workflows"); - - if (!fs.existsSync(workflowTestsDir)) { - this.logger.log("No workflow integration tests directory found", "INFO"); - return { executed: 0, passed: true }; - } - - // Discover workflow test directories - const workflowDirs = fs - .readdirSync(workflowTestsDir, { withFileTypes: true }) - .filter((dirent) => dirent.isDirectory()) - .map((dirent) => dirent.name); - - if (workflowDirs.length === 0) { - this.logger.log("No workflow test directories found", "INFO"); - return { executed: 0, passed: true }; - } - - this.logger.log(`Found ${workflowDirs.length} workflow test(s): ${workflowDirs.join(", ")}`); - - // Run tests for each workflow - let totalWorkflowTests = 0; - for (const workflowName of workflowDirs) { - const testCount = await this.runWorkflowTests(workflowName, workflowTestsDir); - totalWorkflowTests += testCount; - } - - this.logger.log(`Total workflow integration tests executed: ${totalWorkflowTests}`); - return { executed: totalWorkflowTests, passed: true }; - } catch (error) { - this.logger.log(`Error running workflow integration tests: ${error.message}`, "ERROR"); - return { executed: 0, passed: false }; - } - } - - /** - * Run integration tests for a specific workflow - */ - async runWorkflowTests(workflowName, workflowTestsDir) { - try { - const workflowDir = path.join(workflowTestsDir, workflowName); - const testCases = fs - .readdirSync(workflowDir, { withFileTypes: true }) - .filter((dirent) => dirent.isDirectory()) - .map((dirent) => dirent.name); - - this.logger.log(`Running ${testCases.length} test case(s) for workflow ${workflowName}`); - - for (const testCase of testCases) { - await this.runSingleWorkflowIntegrationTest(workflowName, testCase, workflowDir); - } - - return testCases.length; - } catch (error) { - this.logger.log( - `Error running tests for workflow ${workflowName}: ${error.message}`, - "ERROR" - ); - return 0; - } - } - - /** - * Run a single workflow integration test - * Validates the workflow test structure - */ - async runSingleWorkflowIntegrationTest(workflowName, testCase, workflowDir) { - const testName = `${workflowName}/${testCase}`; - this.logger.log(`\nRunning workflow integration test: ${testName}`); - - try { - const testCaseDir = path.join(workflowDir, testCase); - const beforeDir = path.join(testCaseDir, "before"); - const afterDir = path.join(testCaseDir, "after"); - - // Validate test structure - if (!fs.existsSync(beforeDir)) { - this.logger.logTest(testName, false, `Missing before directory`); - return; - } - - if (!fs.existsSync(afterDir)) { - this.logger.logTest(testName, false, `Missing after directory`); - return; - } - - // Check for monitoring state files - const beforeMonitoringFile = path.join(beforeDir, "monitoring-state.json"); - const afterMonitoringFile = path.join(afterDir, "monitoring-state.json"); - - if (!fs.existsSync(beforeMonitoringFile)) { - this.logger.logTest(testName, false, `Missing before/monitoring-state.json`); - return; - } - - if (!fs.existsSync(afterMonitoringFile)) { - this.logger.logTest(testName, false, `Missing after/monitoring-state.json`); - return; - } - - // Validate the workflow test structure exists and is well-formed - const afterMonitoring = JSON.parse(fs.readFileSync(afterMonitoringFile, "utf8")); - - // Basic validation: after should have sessions if workflow executes successfully - const passed = Array.isArray(afterMonitoring.sessions) && afterMonitoring.sessions.length > 0; - - this.logger.logTest( - testName, - passed, - passed - ? `Workflow test structure valid with ${afterMonitoring.sessions.length} expected session(s)` - : "Expected sessions in after/monitoring-state.json" - ); - } catch (error) { - this.logger.logTest(testName, false, `Error: ${error.message}`); - } - } -} diff --git a/client/src/lib/mcp-client-utils.js b/client/src/lib/mcp-client-utils.js deleted file mode 100644 index 6355d94a..00000000 --- a/client/src/lib/mcp-client-utils.js +++ /dev/null @@ -1,37 +0,0 @@ -/** - * MCP Client Utilities - * Shared utility functions for MCP client operations - */ - -/* global setTimeout */ - -import { restartServer } from "./server-manager.js"; - -/** - * Connect to server with automatic retry on "already initialized" error - * @param {Object} client - CodeQLMCPClient instance - * @param {number} maxRetries - Maximum number of retries - * @returns {Promise} True if connected successfully - */ -export async function connectWithRetry(client, maxRetries = 1) { - for (let attempt = 0; attempt <= maxRetries; attempt++) { - try { - await client.connect(); - return true; - } catch (error) { - if ( - error.message && - error.message.includes("Server already initialized") && - attempt < maxRetries - ) { - console.error("Server session conflict detected, restarting server..."); - await restartServer(); - // Wait for server to be ready - await new Promise((resolve) => setTimeout(resolve, 2000)); - continue; - } - throw error; - } - } - return false; -} diff --git a/client/src/lib/mcp-test-suite.js b/client/src/lib/mcp-test-suite.js deleted file mode 100644 index fda2a241..00000000 --- a/client/src/lib/mcp-test-suite.js +++ /dev/null @@ -1,298 +0,0 @@ -/** - * MCP connectivity and basic functionality test suite - */ - -/** - * MCP test suite class - */ -export class MCPTestSuite { - constructor(client, logger) { - this.client = client; - this.logger = logger; - } - - /** - * Run all basic MCP tests - */ - async runAllTests() { - await this.testCapabilities(); - await this.testListTools(); - await this.testCallTool(); - await this.testListResources(); - await this.testReadResource(); - await this.testListPrompts(); - await this.testGetPrompt(); - await this.testGetPromptWithInvalidPath(); - } - - /** - * Test calling a simple tool - */ - async testCallTool() { - try { - this.logger.log("Testing tool execution..."); - - // Try to call a simple resolve languages tool - const result = await this.client.callTool( - { name: "codeql_resolve_languages", arguments: {} }, - undefined, - { timeout: 300000, resetTimeoutOnProgress: true } - ); - - this.logger.log(`Tool result: ${JSON.stringify(result, null, 2)}`); - - const hasContent = result.content && result.content.length > 0; - this.logger.logTest("Call Tool (codeql_resolve_languages)", hasContent); - - return hasContent; - } catch (error) { - this.logger.logTest("Call Tool (codeql_resolve_languages)", false, error); - return false; - } - } - - /** - * Test server capabilities - */ - async testCapabilities() { - try { - this.logger.log("Testing server capabilities..."); - - const capabilities = this.client.getServerCapabilities(); - - // Check for expected capabilities - const expectedCapabilities = ["tools", "resources", "prompts"]; - let allPresent = true; - - for (const capability of expectedCapabilities) { - if (!capabilities[capability]) { - this.logger.log(`Missing capability: ${capability}`, "WARN"); - allPresent = false; - } - } - - this.logger.log(`Server Capabilities: ${JSON.stringify(capabilities, null, 2)}`); - this.logger.logTest("Server Capabilities", allPresent); - - return allPresent; - } catch (error) { - this.logger.logTest("Server Capabilities", false, error); - return false; - } - } - - /** - * Test getting a prompt - */ - async testGetPrompt() { - try { - this.logger.log("Testing prompt retrieval..."); - - // First get list of prompts - const response = await this.client.listPrompts(); - const prompts = response.prompts || []; - - if (prompts.length === 0) { - throw new Error("No prompts available to test"); - } - - // Try to get the first prompt - const prompt = prompts[0]; - const args = {}; - - // Add any required arguments with default values - if (prompt.arguments) { - for (const arg of prompt.arguments) { - if (arg.required) { - // Provide valid default values based on the argument name and schema - if (arg.name === "queryType") { - args[arg.name] = "security"; - } else if (arg.name === "language") { - args[arg.name] = "javascript"; - } else { - args[arg.name] = "test-value"; - } - } - } - } - - const result = await this.client.getPrompt({ - name: prompt.name, - arguments: args - }); - - this.logger.log(`Prompt has ${result.messages?.length || 0} messages`); - - const hasMessages = result.messages && result.messages.length > 0; - this.logger.logTest("Get Prompt", hasMessages); - - return hasMessages; - } catch (error) { - this.logger.logTest("Get Prompt", false, error); - return false; - } - } - - /** - * Test that prompts handle invalid file paths gracefully. - * - * The explain_codeql_query prompt requires a `queryPath` parameter. - * When given a nonexistent path it should return a user-friendly error - * message inside the prompt response rather than throwing a raw MCP - * protocol error. - */ - async testGetPromptWithInvalidPath() { - try { - this.logger.log("Testing prompt error handling with invalid file path..."); - - const result = await this.client.getPrompt({ - name: "explain_codeql_query", - arguments: { - databasePath: "nonexistent/path/to/database", - queryPath: "nonexistent/path/to/query.ql", - language: "javascript" - } - }); - - // The prompt should return messages (not throw) even for an invalid path - const hasMessages = result.messages && result.messages.length > 0; - if (!hasMessages) { - this.logger.logTest( - "Get Prompt with Invalid Path (explain_codeql_query)", - false, - "Expected messages in response" - ); - return false; - } - - // The response should contain a human-readable error about the invalid path - const text = result.messages[0]?.content?.text || ""; - const mentionsPathError = - text.includes("does not exist") || - text.includes("not found") || - text.includes("could not be found") || - text.includes("File not found") || - text.includes("⚠"); - - this.logger.log(`Prompt response mentions path error: ${mentionsPathError}`); - this.logger.logTest("Get Prompt with Invalid Path (explain_codeql_query)", mentionsPathError); - - return mentionsPathError; - } catch (error) { - // If the server throws instead of returning a message this test fails, - // which is the exact behaviour we want to fix. - this.logger.logTest("Get Prompt with Invalid Path (explain_codeql_query)", false, error); - return false; - } - } - - /** - * Test listing prompts - */ - async testListPrompts() { - try { - this.logger.log("Testing prompt listing..."); - - const response = await this.client.listPrompts(); - const prompts = response.prompts || []; - - this.logger.log(`Found ${prompts.length} prompts`); - for (const prompt of prompts) { - // Log all prompts found - this.logger.log(` - ${prompt.name}: ${prompt.description}`); - } - - this.logger.logTest("List Prompts", prompts.length > 0); - return prompts.length > 0; - } catch (error) { - this.logger.logTest("List Prompts", false, error); - return false; - } - } - - /** - * Test listing resources - */ - async testListResources() { - try { - this.logger.log("Testing resource listing..."); - - const response = await this.client.listResources(); - const resources = response.resources || []; - - this.logger.log(`Found ${resources.length} resources`); - for (const resource of resources) { - // Log all resources found - this.logger.log(` - ${resource.uri}: ${resource.name || "No name"}`); - } - - this.logger.logTest("List Resources", resources.length > 0); - return resources.length > 0; - } catch (error) { - this.logger.logTest("List Resources", false, error); - return false; - } - } - - /** - * Test listing tools - */ - async testListTools() { - try { - this.logger.log("Testing tool listing..."); - - const response = await this.client.listTools(); - const tools = response.tools || []; - - this.logger.log(`Found ${tools.length} tools`); - for (const tool of tools) { - // Log all tools found - this.logger.log(` - ${tool.name}: ${tool.description}`); - } - - // Check for some expected CodeQL tools - const expectedTools = ["codeql_resolve_languages", "codeql_database_create"]; - const foundTools = tools.map((t) => t.name); - const hasExpectedTools = expectedTools.some((tool) => foundTools.includes(tool)); - - this.logger.logTest("List Tools", tools.length > 0 && hasExpectedTools); - return tools.length > 0; - } catch (error) { - this.logger.logTest("List Tools", false, error); - return false; - } - } - - /** - * Test reading a resource - */ - async testReadResource() { - try { - this.logger.log("Testing resource reading..."); - - // First get list of resources - const response = await this.client.listResources(); - const resources = response.resources || []; - - if (resources.length === 0) { - throw new Error("No resources available to test"); - } - - // Try to read the first resource - const resource = resources[0]; - const result = await this.client.readResource({ - uri: resource.uri - }); - - this.logger.log(`Resource content length: ${result.contents?.[0]?.text?.length || 0} chars`); - - const hasContent = result.contents && result.contents.length > 0; - this.logger.logTest("Read Resource", hasContent); - - return hasContent; - } catch (error) { - this.logger.logTest("Read Resource", false, error); - return false; - } - } -} diff --git a/client/src/lib/monitoring-integration-test-runner.js b/client/src/lib/monitoring-integration-test-runner.js deleted file mode 100644 index 91c1e39a..00000000 --- a/client/src/lib/monitoring-integration-test-runner.js +++ /dev/null @@ -1,629 +0,0 @@ -/** - * Monitoring-based integration test runner - * Enables testing MCP tools using monitoring JSON data as deterministic before/after states - * Integrated with existing primitives/tools directory structure - */ - -import fs from "fs"; -import path from "path"; - -/** - * Monitoring integration test runner class - */ -export class MonitoringIntegrationTestRunner { - constructor(client, logger) { - this.client = client; - this.logger = logger; - } - - /** - * Helper method to call MCP tools with correct format and timeout. - * - * All codeql_* tools invoke the CodeQL CLI or language server JVM, which - * can be slow in CI. A generous 5-minute timeout avoids intermittent - * -32001 RequestTimeout failures. - */ - async callTool(toolName, parameters = {}) { - const isCodeQLTool = toolName.startsWith("codeql_"); - const requestOptions = { - timeout: isCodeQLTool ? 300000 : 60000, - resetTimeoutOnProgress: isCodeQLTool - }; - return await this.client.callTool( - { name: toolName, arguments: parameters }, - undefined, - requestOptions - ); - } - - /** - * Run monitoring-based integration tests - */ - async runMonitoringIntegrationTests(baseDir) { - try { - this.logger.log("Discovering and running monitoring-based integration tests..."); - - const primitiveToolsDir = path.join( - baseDir, - "..", - "integration-tests", - "primitives", - "tools" - ); - - if (!fs.existsSync(primitiveToolsDir)) { - this.logger.log("No primitives/tools directory found", "WARN"); - return true; - } - - // Get list of available tools from the server - const response = await this.client.listTools(); - const tools = response.tools || []; - const toolNames = tools.map((t) => t.name); - - this.logger.log(`Found ${toolNames.length} tools available for monitoring tests`); - - // Discover tool directories that have monitoring-state.json files - const toolDirs = fs - .readdirSync(primitiveToolsDir, { withFileTypes: true }) - .filter((dirent) => dirent.isDirectory()) - .map((dirent) => dirent.name); - - this.logger.log(`Found ${toolDirs.length} tool directories: ${toolDirs.join(", ")}`); - - // Run tests for each tool directory that has monitoring integration - let totalTests = 0; - for (const toolName of toolDirs) { - const testCount = await this.runToolMonitoringTests(toolName, primitiveToolsDir, toolNames); - totalTests += testCount; - } - - this.logger.log(`Completed ${totalTests} monitoring-based integration tests`); - return totalTests > 0; - } catch (error) { - this.logger.log(`Error running monitoring integration tests: ${error.message}`, "ERROR"); - return false; - } - } - - /** - * Run monitoring tests for a specific tool - */ - async runToolMonitoringTests(toolName, toolsDir, availableTools) { - try { - const toolDir = path.join(toolsDir, toolName); - - // Check if tool is available on server - if (!availableTools.includes(toolName)) { - this.logger.log(`Skipping ${toolName} - tool not available on server`, "WARN"); - return 0; - } - - // Find test cases within the tool directory - const testCases = fs - .readdirSync(toolDir, { withFileTypes: true }) - .filter((dirent) => dirent.isDirectory()) - .map((dirent) => dirent.name); - - let testCount = 0; - for (const testCase of testCases) { - const testCaseDir = path.join(toolDir, testCase); - const beforeDir = path.join(testCaseDir, "before"); - const afterDir = path.join(testCaseDir, "after"); - - // Check if this test case has monitoring integration (monitoring-state.json files) - const beforeMonitoringState = path.join(beforeDir, "monitoring-state.json"); - const afterMonitoringState = path.join(afterDir, "monitoring-state.json"); - - if (fs.existsSync(beforeMonitoringState) && fs.existsSync(afterMonitoringState)) { - const success = await this.runMonitoringTest(toolName, testCase, testCaseDir); - if (success) testCount++; - } - } - - if (testCount > 0) { - this.logger.log(`Completed ${testCount} monitoring tests for ${toolName}`); - } - - return testCount; - } catch (error) { - this.logger.log(`Error running monitoring tests for ${toolName}: ${error.message}`, "ERROR"); - return 0; - } - } - - /** - * Run a single monitoring test - */ - async runMonitoringTest(toolName, testCase, testCaseDir) { - try { - this.logger.log(`Running monitoring test: ${toolName}/${testCase}`); - - // Get before state - const beforeStatePath = path.join(testCaseDir, "before", "monitoring-state.json"); - const beforeState = JSON.parse(fs.readFileSync(beforeStatePath, "utf8")); - - // Get expected after state - const afterStatePath = path.join(testCaseDir, "after", "monitoring-state.json"); - const expectedAfterState = JSON.parse(fs.readFileSync(afterStatePath, "utf8")); - - // For most monitoring tests, we need to create the expected state first - // This is a simplified approach - in practice, the test would call the actual tool - await this.simulateToolExecution( - toolName, - testCase, - testCaseDir, - beforeState, - expectedAfterState - ); - - // Get actual after state - const actualAfterState = await this.getCurrentMonitoringState(); - - // Compare states (simplified comparison for demonstration) - const stateMatches = this.compareMonitoringStatesSimple( - beforeState, - expectedAfterState, - actualAfterState, - toolName - ); - - this.logger.logTest(`Monitoring Integration Test: ${toolName}/${testCase}`, stateMatches); - - if (stateMatches) { - this.logger.log(`✅ ${toolName}/${testCase} - Monitoring state matches expected changes`); - } else { - this.logger.log( - `❌ ${toolName}/${testCase} - Monitoring state does not match expected changes` - ); - } - - return stateMatches; - } catch (error) { - this.logger.logTest(`Monitoring Integration Test: ${toolName}/${testCase}`, false, error); - return false; - } - } - - /** - * Simulate tool execution for testing purposes - */ - async simulateToolExecution(toolName, testCase, testCaseDir, beforeState, expectedAfterState) { - try { - // For session management tools, actually call them - if (toolName.startsWith("session_")) { - await this.executeSessionTool(toolName, expectedAfterState); - } else if (toolName.startsWith("sessions_")) { - await this.executeBatchTool(toolName, expectedAfterState); - } else { - // For other tools, simulate with sessionId if they support monitoring - await this.executeToolWithSession(toolName, testCase, testCaseDir, expectedAfterState); - } - } catch (error) { - this.logger.log(`Error simulating ${toolName}: ${error.message}`, "WARN"); - } - } - - /** - * Execute session management tools - */ - async executeSessionTool(toolName, expectedState) { - if (toolName === "session_start") { - // session_start tool was removed per feedback - sessions are auto-created - // Skip this test as it's no longer valid - this.logger.log( - `Skipping ${toolName} - tool removed per feedback (auto-creation instead)`, - "INFO" - ); - return true; // Mark as successful since this is expected behavior - } else if (toolName === "session_end" && expectedState.sessions.length > 0) { - // For session_end, we need an existing session to end - // Since session_start is removed, we need to create a session through auto-creation - const session = expectedState.sessions[0]; - - // Create a session by calling a tool that supports auto-creation with queryPath - const sessionId = `test-session-end-${Date.now()}`; - - // First create a session by calling session_list or session_get (which should auto-create if needed) - await this.callTool("session_list", {}); - - this.logger.log(`Ending session ${sessionId} for ${toolName} test`); - - const result = await this.callTool(toolName, { - sessionId: sessionId, - status: session.status - }); - - // For monitoring tools that expect existing sessions, "Session not found" is a valid response - if (result.isError && result.content[0].text.includes("Session not found")) { - this.logger.log( - `${toolName} correctly returned "Session not found" - this is expected behavior` - ); - return true; // This is the expected behavior for missing sessions - } - - return !result.isError; - } else if ( - toolName === "session_calculate_current_score" && - expectedState.sessions.length > 0 - ) { - // Create a session by calling a tool that supports auto-creation - // Since session_start is removed, we'll create a session via auto-creation - const sessionId = `scoring-test-session-${Date.now()}`; - - this.logger.log(`Testing session scoring with ID ${sessionId} for ${toolName} test`); - - // Call the scoring tool which should auto-create the session if needed - const result = await this.callTool(toolName, { - sessionId: sessionId - }); - - // For monitoring tools that expect existing sessions, "Session not found" is a valid response - if (result.isError && result.content[0].text.includes("Session not found")) { - this.logger.log( - `${toolName} correctly returned "Session not found" - this is expected behavior` - ); - return true; // This is the expected behavior for missing sessions - } - - return !result.isError; - } - return false; - } - - /** - * Execute batch operation tools - */ - async executeBatchTool(toolName, _expectedState) { - if (toolName === "sessions_compare") { - // For comparison tests, we need existing sessions - // Create two test sessions by ensuring sessions exist first - const sessionId1 = `comparison-session-1-${Date.now()}`; - const sessionId2 = `comparison-session-2-${Date.now()}`; - - // Create sessions by calling session_list which should handle auto-creation - await this.callTool("session_list", {}); - - this.logger.log(`Comparing sessions ${sessionId1} and ${sessionId2} for comparison test`); - - const result = await this.callTool(toolName, { - sessionIds: [sessionId1, sessionId2], - dimensions: ["quality", "performance"] - }); - - // For monitoring tools that expect existing sessions, appropriate error messages are valid responses - if ( - result.isError && - (result.content[0].text.includes("No valid sessions found") || - result.content[0].text.includes("Session not found")) - ) { - this.logger.log( - `${toolName} correctly returned expected error for missing sessions - this is expected behavior` - ); - return true; // This is the expected behavior for missing sessions - } - - return !result.isError; - } - return false; - } - - /** - * Execute regular tools with session integration - */ - async executeToolWithSession(toolName, testCase, testCaseDir, expectedState) { - if (expectedState.sessions.length > 0) { - // Since session_start was removed, we'll use a test session ID - const sessionId = `test-session-${toolName}-${Date.now()}`; - - this.logger.log(`Using session ${sessionId} for ${toolName} test`); - - // Get tool-specific parameters - const toolParams = this.getToolSpecificParams(toolName, testCase, testCaseDir); - this.logger.log(`Tool-specific params for ${toolName}: ${JSON.stringify(toolParams)}`); - - // Try to call the tool with sessionId - const result = await this.callTool(toolName, { - sessionId: sessionId, - // Add other parameters based on the tool and test case - ...toolParams - }); - return !result.isError; - } - return false; - } - - /** - * Get tool-specific parameters for testing - */ - getToolSpecificParams(toolName, _testCase, testCaseDir) { - const params = {}; - - if (toolName === "codeql_lsp_diagnostics") { - params.ql_code = 'from UndefinedType x where x = "test" select x, "semantic error"'; - } else if (toolName === "codeql_query_format") { - // Look for .ql files in the before directory - const beforeDir = path.join(testCaseDir, "before"); - if (fs.existsSync(beforeDir)) { - const qlFiles = fs.readdirSync(beforeDir).filter((f) => f.endsWith(".ql")); - if (qlFiles.length > 0) { - params.files = [path.join(beforeDir, qlFiles[0])]; - } else { - throw new Error(`No .ql files found in ${beforeDir} for ${toolName}`); - } - } else { - throw new Error(`Test directory ${beforeDir} not found for ${toolName}`); - } - params["check-only"] = true; - } else if (toolName === "codeql_bqrs_info") { - // Look for .bqrs files in the before directory - const beforeDir = path.join(testCaseDir, "before"); - if (fs.existsSync(beforeDir)) { - const bqrsFiles = fs.readdirSync(beforeDir).filter((f) => f.endsWith(".bqrs")); - if (bqrsFiles.length > 0) { - params.file = path.join(beforeDir, bqrsFiles[0]); - } else { - throw new Error(`No .bqrs files found in ${beforeDir} for ${toolName}`); - } - } else { - throw new Error(`Test directory ${beforeDir} not found for ${toolName}`); - } - } else if (toolName === "codeql_bqrs_decode") { - // Use the actual test file from the before directory - const beforeDir = path.join(testCaseDir, "before"); - const bqrsFiles = fs.readdirSync(beforeDir).filter((f) => f.endsWith(".bqrs")); - if (bqrsFiles.length > 0) { - params.file = path.join(beforeDir, bqrsFiles[0]); - } else { - throw new Error(`No .bqrs files found in ${beforeDir} for ${toolName}`); - } - params.format = "json"; - } else if (toolName === "codeql_bqrs_interpret") { - // Use the actual test file from the before directory - const beforeDir = path.join(testCaseDir, "before"); - const bqrsFiles = fs.readdirSync(beforeDir).filter((f) => f.endsWith(".bqrs")); - if (bqrsFiles.length > 0) { - params.file = path.join(beforeDir, bqrsFiles[0]); - } else { - throw new Error(`No .bqrs files found in ${beforeDir} for ${toolName}`); - } - - // Set output path based on test case name - const afterDir = path.join(testCaseDir, "after"); - if (_testCase.includes("graphtext")) { - params.format = "graphtext"; - params.output = path.join(afterDir, "output.txt"); - params.t = ["kind=graph", "id=test/query"]; - } else if (_testCase.includes("sarif")) { - params.format = "sarif-latest"; - params.output = path.join(afterDir, "results.sarif"); - params.t = ["kind=problem", "id=test/query"]; - // Note: sarif-add-snippets requires source archive, skipping for basic test - } else { - // Default to graphtext for unknown test cases - params.format = "graphtext"; - params.output = path.join(afterDir, "output.txt"); - params.t = ["kind=graph", "id=test/query"]; - } - } else if (toolName === "validate_codeql_query") { - params.query = "from int i select i"; - params.language = "java"; - } else if (toolName === "codeql_pack_ls") { - // Use the test case directory as the pack directory - params.dir = path.join(testCaseDir, "before"); - } else if (toolName === "profile_codeql_query") { - // Read from test-config.json for profile_codeql_query tool - const testConfigPath = path.join(testCaseDir, "test-config.json"); - if (fs.existsSync(testConfigPath)) { - const testConfig = JSON.parse(fs.readFileSync(testConfigPath, "utf8")); - if (testConfig.arguments) { - Object.assign(params, testConfig.arguments); - } else { - throw new Error(`test-config.json missing arguments for ${toolName}`); - } - } else { - throw new Error(`test-config.json not found for ${toolName}`); - } - } else if (toolName === "codeql_pack_install") { - params.pack = "github/codeql/java-queries"; - } else if (toolName === "codeql_query_compile") { - // Look for .ql files in the before directory - const beforeDir = path.join(testCaseDir, "before"); - if (fs.existsSync(beforeDir)) { - const qlFiles = fs.readdirSync(beforeDir).filter((f) => f.endsWith(".ql")); - if (qlFiles.length > 0) { - params.queryFile = path.join(beforeDir, qlFiles[0]); - } else { - throw new Error(`No .ql files found in ${beforeDir} for ${toolName}`); - } - } else { - throw new Error(`Test directory ${beforeDir} not found for ${toolName}`); - } - } else if (toolName === "codeql_resolve_library_path") { - // Use the test case directory as the pack directory - params.packDir = path.join(testCaseDir, "before"); - } else if (toolName === "codeql_resolve_metadata") { - // Look for .ql files in the before directory - const beforeDir = path.join(testCaseDir, "before"); - if (fs.existsSync(beforeDir)) { - const qlFiles = fs.readdirSync(beforeDir).filter((f) => f.endsWith(".ql")); - if (qlFiles.length > 0) { - params.queryPath = path.join(beforeDir, qlFiles[0]); - } else { - throw new Error(`No .ql files found in ${beforeDir} for ${toolName}`); - } - } else { - throw new Error(`Test directory ${beforeDir} not found for ${toolName}`); - } - } else if (toolName === "codeql_resolve_queries") { - // Use the test case directory as the queries path - params.path = path.join(testCaseDir, "before"); - } else if (toolName === "codeql_resolve_tests") { - // Use the test case directory as the tests path - params.tests = [path.join(testCaseDir, "before")]; - } else if (toolName === "codeql_test_accept") { - // Use the test case directory as the tests directory - params.tests = [path.join(testCaseDir, "before")]; - } else if (toolName === "codeql_test_extract") { - // Use the test case directory as the tests directory - params.tests = [path.join(testCaseDir, "before")]; - } else if (toolName === "codeql_test_run") { - // Use the test case directory as the test directory - params.tests = [path.join(testCaseDir, "before")]; - } else if (toolName === "find_class_position") { - // Look for .ql files in the before directory - const beforeDir = path.join(testCaseDir, "before"); - if (fs.existsSync(beforeDir)) { - const qlFiles = fs.readdirSync(beforeDir).filter((f) => f.endsWith(".ql")); - if (qlFiles.length > 0) { - params.file = path.join(beforeDir, qlFiles[0]); - } else { - throw new Error(`No .ql files found in ${beforeDir} for ${toolName}`); - } - } else { - throw new Error(`Test directory ${beforeDir} not found for ${toolName}`); - } - params.name = "MyClass"; - } else if (toolName === "find_predicate_position") { - // Look for .ql files in the before directory - const beforeDir = path.join(testCaseDir, "before"); - if (fs.existsSync(beforeDir)) { - const qlFiles = fs.readdirSync(beforeDir).filter((f) => f.endsWith(".ql")); - if (qlFiles.length > 0) { - params.file = path.join(beforeDir, qlFiles[0]); - } else { - throw new Error(`No .ql files found in ${beforeDir} for ${toolName}`); - } - } else { - throw new Error(`Test directory ${beforeDir} not found for ${toolName}`); - } - params.name = "myPredicate"; - } else if ( - toolName === "sarif_extract_rule" || - toolName === "sarif_list_rules" || - toolName === "sarif_rule_to_markdown" - ) { - // Look for .sarif files in the before directory - const beforeDir = path.join(testCaseDir, "before"); - if (fs.existsSync(beforeDir)) { - const sarifFiles = fs.readdirSync(beforeDir).filter((f) => f.endsWith(".sarif")); - if (sarifFiles.length > 0) { - params.sarifPath = path.join(beforeDir, sarifFiles[0]); - } else { - throw new Error(`No .sarif files found in ${beforeDir} for ${toolName}`); - } - } else { - throw new Error(`Test directory ${beforeDir} not found for ${toolName}`); - } - // Read ruleId from test-config.json arguments - const sarifTestConfigPath = path.join(testCaseDir, "test-config.json"); - if (fs.existsSync(sarifTestConfigPath)) { - const sarifTestConfig = JSON.parse(fs.readFileSync(sarifTestConfigPath, "utf8")); - if (sarifTestConfig.arguments?.ruleId) { - params.ruleId = sarifTestConfig.arguments.ruleId; - } - } - } else if (toolName === "sarif_compare_alerts") { - // Look for .sarif files in the before directory - const beforeDir = path.join(testCaseDir, "before"); - if (fs.existsSync(beforeDir)) { - const sarifFiles = fs.readdirSync(beforeDir).filter((f) => f.endsWith(".sarif")); - if (sarifFiles.length > 0) { - const sarifPath = path.join(beforeDir, sarifFiles[0]); - // Read alert specs and overlapMode from test-config.json - const compareConfigPath = path.join(testCaseDir, "test-config.json"); - if (fs.existsSync(compareConfigPath)) { - const compareConfig = JSON.parse(fs.readFileSync(compareConfigPath, "utf8")); - const args = compareConfig.arguments || {}; - params.alertA = { ...args.alertA, sarifPath }; - params.alertB = { ...args.alertB, sarifPath }; - if (args.overlapMode) { - params.overlapMode = args.overlapMode; - } - } - } else { - throw new Error(`No .sarif files found in ${beforeDir} for ${toolName}`); - } - } else { - throw new Error(`Test directory ${beforeDir} not found for ${toolName}`); - } - } else if (toolName === "sarif_diff_runs") { - // Look for .sarif files in the before directory — expects two files - const beforeDir = path.join(testCaseDir, "before"); - if (fs.existsSync(beforeDir)) { - const sarifFiles = fs - .readdirSync(beforeDir) - .filter((f) => f.endsWith(".sarif")) - .sort(); - if (sarifFiles.length >= 2) { - params.sarifPathA = path.join(beforeDir, sarifFiles[0]); - params.sarifPathB = path.join(beforeDir, sarifFiles[1]); - } else if (sarifFiles.length === 1) { - params.sarifPathA = path.join(beforeDir, sarifFiles[0]); - params.sarifPathB = path.join(beforeDir, sarifFiles[0]); - } else { - throw new Error(`No .sarif files found in ${beforeDir} for ${toolName}`); - } - } else { - throw new Error(`Test directory ${beforeDir} not found for ${toolName}`); - } - const diffConfigPath = path.join(testCaseDir, "test-config.json"); - if (fs.existsSync(diffConfigPath)) { - const diffConfig = JSON.parse(fs.readFileSync(diffConfigPath, "utf8")); - const args = diffConfig.arguments || {}; - if (args.labelA) params.labelA = args.labelA; - if (args.labelB) params.labelB = args.labelB; - } - } - - return params; - } - - /** - * Get current monitoring state from server - */ - async getCurrentMonitoringState() { - try { - const result = await this.callTool("session_list", {}); - if (result.isError) { - return { sessions: [] }; - } - return JSON.parse(result.content[0].text); - } catch (error) { - this.logger.log(`Failed to get current monitoring state: ${error.message}`, "ERROR"); - return { sessions: [] }; - } - } - - /** - * Compare monitoring states (simplified version) - */ - compareMonitoringStatesSimple(beforeState, expectedAfterState, actualAfterState, toolName) { - try { - // For demonstration purposes, do a simple comparison - // In practice, this would be more sophisticated - - // Check if sessions count increased appropriately - const beforeSessionCount = beforeState.sessions?.length || 0; - const actualSessionCount = actualAfterState.sessions?.length || 0; - - // For session_start, mark as successful since the tool was removed - if (toolName === "session_start") { - this.logger.log(`session_start tool was removed - marking test as successful`, "INFO"); - return true; - } - - // For other session tools, check for basic session existence or changes - if (toolName.startsWith("session_")) { - // For session tools, we expect some session activity - return actualSessionCount >= 0; // Any session count is acceptable - } - - // For other tools, check if any sessions exist (indicating tool integration) - return actualSessionCount >= beforeSessionCount; - } catch (error) { - this.logger.log(`Error comparing monitoring states: ${error.message}`, "ERROR"); - return false; - } - } -} diff --git a/client/src/lib/process-query-metadata.js b/client/src/lib/process-query-metadata.js deleted file mode 100644 index 9f90df2d..00000000 --- a/client/src/lib/process-query-metadata.js +++ /dev/null @@ -1,170 +0,0 @@ -/** - * Process Query Metadata - * Processes aggregated query metadata to generate coverage analysis - */ - -import { readFileSync } from "fs"; - -/** - * Process query metadata to analyze coverage by language - * @param {Object} options - Processing options - * @param {string} options.inputFile - Path to JSON file containing aggregated query metadata - * @param {Function} [options.progressCallback] - Optional callback for progress updates - * @returns {Object} Processed results with coverage analysis - */ -export function processQueryMetadata(options = {}) { - const { inputFile, progressCallback } = options; - - // Read the input file containing query metadata - const data = JSON.parse(readFileSync(inputFile, "utf-8")); - - if (!data.results || !Array.isArray(data.results)) { - throw new Error("Invalid input file format: missing or invalid 'results' array"); - } - - const results = data.results; - const total = results.length; - - if (progressCallback) { - progressCallback({ - type: "start", - message: `Processing metadata for ${total} queries...`, - total - }); - } - - // Initialize aggregation structures - const byLanguage = {}; - const tagCoverage = {}; - const languages = new Set(); - const allTags = new Set(); - - // Process each query result - let processed = 0; - for (const queryResult of results) { - processed++; - - if (progressCallback && processed % 100 === 0) { - const percentage = Math.round((processed / total) * 100); - progressCallback({ - type: "progress", - message: `Processed ${processed}/${total} queries`, - processed, - total, - percentage - }); - } - - const { queryPath, metadata } = queryResult; - - // Skip if there's an error in the metadata - if (queryResult.error || !metadata) { - continue; - } - - // Extract language from metadata - const language = metadata.language; - if (!language || language === "unknown") { - continue; - } - - languages.add(language); - - // Initialize language entry if not exists - if (!byLanguage[language]) { - byLanguage[language] = { - queryCount: 0, - tags: new Set(), - queries: [] - }; - } - - // Increment query count - byLanguage[language].queryCount++; - - // Add query path to queries list - byLanguage[language].queries.push(queryPath); - - // Extract tags from metadata - const queryMetadata = metadata.metadata; - if (queryMetadata && queryMetadata.tags) { - // Tags can be a string or an array - const tags = Array.isArray(queryMetadata.tags) - ? queryMetadata.tags - : queryMetadata.tags.split(/\s+/); - - for (const tag of tags) { - const trimmedTag = tag.trim(); - if (trimmedTag) { - // Add to language-specific tags - byLanguage[language].tags.add(trimmedTag); - - // Add to global tags - allTags.add(trimmedTag); - - // Track which languages use this tag - if (!tagCoverage[trimmedTag]) { - tagCoverage[trimmedTag] = new Set(); - } - tagCoverage[trimmedTag].add(language); - } - } - } - } - - if (progressCallback) { - progressCallback({ - type: "complete", - message: `Processed all ${total} queries` - }); - } - - // Convert Sets to sorted arrays for output - const sortedLanguages = Array.from(languages).sort(); - const sortedTags = Array.from(allTags).sort(); - - const processedByLanguage = {}; - for (const [lang, data] of Object.entries(byLanguage)) { - processedByLanguage[lang] = { - queryCount: data.queryCount, - tags: Array.from(data.tags).sort(), - tagCount: data.tags.size - // Optionally include queries list (can be large) - // queries: data.queries - }; - } - - const processedTagCoverage = {}; - for (const [tag, langs] of Object.entries(tagCoverage)) { - processedTagCoverage[tag] = Array.from(langs).sort(); - } - - return { - summary: { - totalQueries: total, - processedQueries: processed, - languages: sortedLanguages, - totalTags: sortedTags.length - }, - byLanguage: processedByLanguage, - tagCoverage: processedTagCoverage - }; -} - -/** - * Default progress callback that logs to stderr - * @param {Object} event - Progress event - */ -export function consoleProgressCallback(event) { - switch (event.type) { - case "start": - console.error(event.message); - break; - case "progress": - console.error(`[${event.percentage}%] ${event.message}`); - break; - case "complete": - console.error(`✓ ${event.message}`); - break; - } -} diff --git a/client/src/lib/queries-filter-metadata.js b/client/src/lib/queries-filter-metadata.js deleted file mode 100644 index 5b3c304e..00000000 --- a/client/src/lib/queries-filter-metadata.js +++ /dev/null @@ -1,119 +0,0 @@ -/** - * Filter Query Metadata - * Filters query metadata to find queries suitable for regeneration - */ - -import { readFileSync } from "fs"; - -/** - * Filter query metadata to identify queries suitable for regeneration - * @param {Object} options - Filtering options - * @param {string} options.inputFile - Path to JSON file containing query metadata - * @param {string} [options.language] - Optional language filter (default: 'all') - * @param {Function} [options.progressCallback] - Optional callback for progress updates - * @returns {Object} Filtered results with regenerable queries - */ -export function filterQueryMetadata(options = {}) { - const { inputFile, language = "all", progressCallback } = options; - - // Read the input file containing query metadata - const metadata = JSON.parse(readFileSync(inputFile, "utf-8")); - - if (!metadata.queries || !Array.isArray(metadata.queries)) { - throw new Error("Invalid input file format: missing or invalid 'queries' array"); - } - - const total = metadata.queries.length; - - if (progressCallback) { - progressCallback({ - type: "start", - message: `Filtering ${total} queries for regeneration...`, - total - }); - } - - // Define filter criteria - const filterCriteria = { - documentationExists: true, - expectedResultsExist: true, - hasTestCode: true, - qlrefExists: true, - queryExists: true - }; - - // Filter queries based on criteria - const regenerableQueries = []; - let processed = 0; - - for (const query of metadata.queries) { - processed++; - - if (progressCallback && processed % 100 === 0) { - const percentage = Math.round((processed / total) * 100); - progressCallback({ - type: "progress", - message: `Filtered ${processed}/${total} queries`, - processed, - total, - percentage - }); - } - - const status = query.status || {}; - - // Check if query meets all regeneration criteria - const isRegenerable = - status.documentationExists === true && - status.expectedResultsExist === true && - status.hasTestCode === true && - status.qlrefExists === true && - status.queryExists === true; - - // If language filter is specified, also check language match - const languageMatch = language === "all" || !language || query.language === language; - - if (isRegenerable && languageMatch) { - regenerableQueries.push(query); - } - } - - if (progressCallback) { - progressCallback({ - type: "complete", - message: `Filtered ${regenerableQueries.length} regenerable queries from ${total} total queries` - }); - } - - return { - metadata: { - generatedAt: new Date().toISOString(), - inputFile, - language, - totalQueries: total, - regenerableCount: regenerableQueries.length, - filterCriteria - }, - queries: regenerableQueries - }; -} - -/** - * Console progress callback for filtering - * @param {Object} event - Progress event - */ -export function consoleProgressCallback(event) { - switch (event.type) { - case "start": - console.error(event.message); - break; - case "progress": - console.error(`${event.message} (${event.percentage}%)`); - break; - case "complete": - console.error(event.message); - break; - default: - break; - } -} diff --git a/client/src/lib/resolve-all-queries.js b/client/src/lib/resolve-all-queries.js deleted file mode 100644 index e24ed135..00000000 --- a/client/src/lib/resolve-all-queries.js +++ /dev/null @@ -1,174 +0,0 @@ -/** - * Resolve All Queries - * Resolves CodeQL queries from all query packs in a repository - */ - -import { readFileSync } from "fs"; -import { dirname } from "path"; - -/** - * Resolve queries from all query packs - * @param {Object} client - Connected MCP client instance - * @param {Object} options - Resolution options - * @param {string} options.packsFile - Path to JSON file containing pack list from codeql_pack_ls - * @param {number} [options.timeout] - Timeout per pack in milliseconds - * @param {Function} [options.progressCallback] - Optional callback for progress updates - * @returns {Promise>} Array of query file paths - */ -export async function resolveAllQueries(client, options = {}) { - const { packsFile, timeout = 120000, progressCallback } = options; - - // Read the pack list from codeql_pack_ls (MCP response format) - const packsData = JSON.parse(readFileSync(packsFile, "utf-8")); - const packsText = packsData.content?.[0]?.text || "{}"; - - // Parse just the JSON object part (before any warnings/info) - const lines = packsText.split("\n"); - let jsonText = ""; - let braceCount = 0; - let inJson = false; - - for (const line of lines) { - const trimmed = line.trim(); - if (trimmed.startsWith("{")) { - inJson = true; - braceCount = 1; - jsonText = line + "\n"; - continue; - } - if (inJson) { - jsonText += line + "\n"; - braceCount += (line.match(/{/g) || []).length; - braceCount -= (line.match(/}/g) || []).length; - if (braceCount === 0) { - break; - } - } - } - - const packsJson = JSON.parse(jsonText); - const packs = packsJson.packs || {}; - - // Filter for query packs (those ending in -queries) - const queryPacks = Object.entries(packs) - .filter(([_path, info]) => info.name.endsWith("-queries")) - .map(([path, _info]) => dirname(path)); // Get directory containing qlpack.yml - - if (queryPacks.length === 0) { - throw new Error("No query packs found in pack list"); - } - - const total = queryPacks.length; - - if (progressCallback) { - progressCallback({ - type: "start", - message: `Found ${total} query packs to process`, - total - }); - } - - const allQueries = []; - let processedPacks = 0; - - for (const packDir of queryPacks) { - processedPacks++; - - if (progressCallback) { - progressCallback({ - type: "progress", - message: `Processing pack ${processedPacks}/${total}: ${packDir}`, - processed: processedPacks, - total, - packDir - }); - } - - try { - const result = await client.callTool( - "codeql_resolve_queries", - { - directory: packDir, - format: "bylanguage" - }, - { timeout } - ); - - const resultText = result.content[0]?.text || "{}"; - - // Parse the bylanguage format response - const data = JSON.parse(resultText); - - // Extract query paths from byLanguage object structure - // Format: { "byLanguage": { "language": { "/path/to/query.ql": {} } } } - const queries = []; - if (data.byLanguage && typeof data.byLanguage === "object") { - for (const language of Object.values(data.byLanguage)) { - if (language && typeof language === "object") { - queries.push(...Object.keys(language)); - } - } - } - - if (queries.length > 0) { - allQueries.push(...queries); - if (progressCallback) { - progressCallback({ - type: "pack-complete", - message: `Found ${queries.length} queries in this pack`, - count: queries.length - }); - } - } else if (progressCallback) { - progressCallback({ - type: "pack-complete", - message: "No queries found in this pack", - count: 0 - }); - } - } catch (error) { - if (progressCallback) { - progressCallback({ - type: "error", - message: error.message, - packDir - }); - } - // Continue processing other packs even if one fails - } - } - - if (progressCallback) { - progressCallback({ - type: "complete", - message: `Total queries found: ${allQueries.length}`, - total: allQueries.length - }); - } - - return allQueries; -} - -/** - * Default progress callback that logs to stderr - * @param {Object} event - Progress event - */ -export function consoleProgressCallback(event) { - switch (event.type) { - case "start": - console.error(event.message); - break; - case "progress": - console.error(event.message); - break; - case "pack-complete": - console.error(` ${event.message}`); - break; - case "error": - console.error(` Error processing ${event.packDir}: ${event.message}`); - break; - case "complete": - console.error(event.message); - break; - } -} diff --git a/client/src/lib/server-manager.js b/client/src/lib/server-manager.js deleted file mode 100644 index 9dcb4d8a..00000000 --- a/client/src/lib/server-manager.js +++ /dev/null @@ -1,184 +0,0 @@ -/** - * Server management module for starting and stopping the MCP server - */ - -/* global setTimeout */ - -import { spawn } from "child_process"; -import { writeFileSync, readFileSync, unlinkSync, existsSync, createWriteStream } from "fs"; -import { dirname, join } from "path"; -import { fileURLToPath } from "url"; - -const __filename = fileURLToPath(import.meta.url); -const __dirname = dirname(__filename); -const CLIENT_DIR = join(__dirname, "..", ".."); -const ROOT_DIR = join(CLIENT_DIR, ".."); -const PID_FILE = join(CLIENT_DIR, "server.pid"); -const LOG_FILE = join(CLIENT_DIR, "server.log"); - -/** - * Check if the server is currently running - * @returns {boolean} True if server is running - */ -export function isServerRunning() { - if (!existsSync(PID_FILE)) { - return false; - } - - try { - const pid = parseInt(readFileSync(PID_FILE, "utf8").trim()); - // Check if process is running - process.kill(pid, 0); - return true; - } catch { - // Process not found or no permission - return false; - } -} - -/** - * Start the MCP server - * @param {Object} options - Server options - * @param {string} options.mode - Transport mode: 'stdio' or 'http' - * @param {string} options.host - HTTP host (only for http mode) - * @param {number} options.port - HTTP port (only for http mode) - * @param {string} options.scheme - HTTP scheme (only for http mode) - * @returns {Promise} Server PID - */ -export async function startServer(options = {}) { - const { mode = "http", host = "localhost", port = 3000, scheme = "http" } = options; - - // Check if server is already running - if (isServerRunning()) { - const pid = parseInt(readFileSync(PID_FILE, "utf8").trim()); - console.error(`Server is already running with PID ${pid}`); - return pid; - } - - console.error(`Starting MCP server in ${mode} mode...`); - - const env = { - ...process.env, - TRANSPORT_MODE: mode - }; - - if (mode === "http") { - env.HTTP_HOST = host; - env.HTTP_PORT = port.toString(); - env.HTTP_SCHEME = scheme; - } - - const serverPath = join(ROOT_DIR, "server", "dist", "codeql-development-mcp-server.js"); - - // Start server as detached process - const serverProcess = spawn("node", [serverPath], { - detached: true, - stdio: ["ignore", "pipe", "pipe"], - env, - cwd: ROOT_DIR - }); - - // Append logs to file (preserves logs from previous runs). - // WARNING: This can lead to unbounded log file growth. - // Consider implementing log rotation or a cleanup strategy to manage log file size. - const logStream = createWriteStream(LOG_FILE, { flags: "a" }); - - // Add separator and timestamp for new server session - const timestamp = new Date().toISOString(); - logStream.write(`\n${"=".repeat(80)}\n`); - logStream.write(`[${timestamp}] Starting MCP server (PID: ${serverProcess.pid})\n`); - logStream.write(`${"=".repeat(80)}\n\n`); - - serverProcess.stdout.pipe(logStream); - serverProcess.stderr.pipe(logStream); - - // Save PID - writeFileSync(PID_FILE, serverProcess.pid.toString()); - - // Unref so parent can exit - serverProcess.unref(); - - console.error(`Server started with PID ${serverProcess.pid}`); - console.error(`Logs will be appended to ${LOG_FILE}`); - - // Wait a bit for server to start - await new Promise((resolve) => setTimeout(resolve, 2000)); - - return serverProcess.pid; -} - -/** - * Stop the MCP server - * @returns {Promise} True if server was stopped - */ -export async function stopServer() { - if (!existsSync(PID_FILE)) { - console.error("No server.pid file found"); - return false; - } - - const pid = parseInt(readFileSync(PID_FILE, "utf8").trim()); - console.error(`Stopping server with PID ${pid}`); - - try { - // Try graceful shutdown - process.kill(pid, "SIGTERM"); - - // Wait for process to exit - await new Promise((resolve) => setTimeout(resolve, 2000)); - - // Check if still running - try { - process.kill(pid, 0); - // Still running, force kill - console.error("Force killing server process"); - process.kill(pid, "SIGKILL"); - } catch { - // Process already stopped - } - - console.error("Server stopped successfully"); - } catch (error) { - if (error.code === "ESRCH") { - console.error("Server process was not running"); - } else { - throw error; - } - } finally { - // Clean up PID file only (preserve logs for debugging) - if (existsSync(PID_FILE)) { - unlinkSync(PID_FILE); - } - } - - return true; -} - -/** - * Restart the MCP server - * @param {Object} options - Server options - * @returns {Promise} Server PID - */ -export async function restartServer(options = {}) { - console.error("Restarting MCP server..."); - await stopServer(); - // Wait a bit for port to be released - await new Promise((resolve) => setTimeout(resolve, 1000)); - return await startServer(options); -} - -/** - * Ensure server is running, start if needed - * @param {Object} options - Server options - * @param {boolean} options.fresh - If true, restart server even if running - * @returns {Promise} - */ -export async function ensureServerRunning(options = {}) { - if (options.fresh) { - console.error("Fresh server requested, restarting..."); - await restartServer(options); - } else if (!isServerRunning()) { - console.error("Server not running, starting automatically..."); - await startServer(options); - } -} diff --git a/client/src/lib/test-logger.js b/client/src/lib/test-logger.js deleted file mode 100644 index 5b090e59..00000000 --- a/client/src/lib/test-logger.js +++ /dev/null @@ -1,74 +0,0 @@ -/** - * Test logger and reporter for MCP client integration tests - */ - -/** - * Test logger class - */ -export class TestLogger { - constructor() { - this.testResults = { - passed: 0, - failed: 0, - errors: [] - }; - } - - /** - * Get test results - */ - getResults() { - return this.testResults; - } - - /** - * Check if all tests passed - */ - isSuccess() { - return this.testResults.failed === 0 && this.testResults.passed > 0; - } - - /** - * Log a message with timestamp - */ - log(message, level = "INFO") { - const timestamp = new Date().toISOString(); - console.log(`[${timestamp}] [${level}] ${message}`); - } - - /** - * Log a test result - */ - logTest(testName, passed, error = null) { - if (passed) { - this.testResults.passed++; - this.log(`✅ ${testName}`, "TEST"); - } else { - this.testResults.failed++; - this.testResults.errors.push({ test: testName, error: error?.message || "Unknown error" }); - this.log(`❌ ${testName}: ${error?.message || "Failed"}`, "TEST"); - } - } - - /** - * Print test summary - */ - printTestSummary() { - this.log("=".repeat(50)); - this.log("TEST SUMMARY"); - this.log("=".repeat(50)); - this.log(`✅ Passed: ${this.testResults.passed}`); - this.log(`❌ Failed: ${this.testResults.failed}`); - this.log(`📊 Total: ${this.testResults.passed + this.testResults.failed}`); - - if (this.testResults.errors.length > 0) { - this.log("\nFAILED TESTS:"); - for (const error of this.testResults.errors) { - this.log(` - ${error.test}: ${error.error}`); - } - } - - const success = this.testResults.failed === 0 && this.testResults.passed > 0; - this.log(`\n🎯 Overall: ${success ? "SUCCESS" : "FAILURE"}`); - } -} diff --git a/client/src/lib/validate-source-root.js b/client/src/lib/validate-source-root.js deleted file mode 100644 index 53ca3ba7..00000000 --- a/client/src/lib/validate-source-root.js +++ /dev/null @@ -1,47 +0,0 @@ -/** - * Validate Source Root - * Validates SOURCE_ROOT environment variable and directory structure - */ - -import { existsSync } from "fs"; -import { join } from "path"; - -/** - * Validate SOURCE_ROOT directory structure - * @param {Object} options - Validation options - * @param {string} options.sourceRoot - Path to source root directory - * @returns {Object} Validation result - */ -export function validateSourceRoot(options = {}) { - const { sourceRoot } = options; - - if (!sourceRoot) { - return { - valid: false, - error: "SOURCE_ROOT is not set" - }; - } - - if (!existsSync(sourceRoot)) { - return { - valid: false, - error: `SOURCE_ROOT directory does not exist: ${sourceRoot}` - }; - } - - // Check for codeql-workspace.yml or codeql-workspace.yaml - const workspaceYml = join(sourceRoot, "codeql-workspace.yml"); - const workspaceYaml = join(sourceRoot, "codeql-workspace.yaml"); - - if (!existsSync(workspaceYml) && !existsSync(workspaceYaml)) { - return { - valid: false, - error: `No codeql-workspace.yml or codeql-workspace.yaml found in ${sourceRoot}` - }; - } - - return { - valid: true, - source_root: sourceRoot - }; -} diff --git a/client/src/ql-mcp-client.js b/client/src/ql-mcp-client.js deleted file mode 100755 index 9d208443..00000000 --- a/client/src/ql-mcp-client.js +++ /dev/null @@ -1,866 +0,0 @@ -#!/usr/bin/env node - -/** - * CodeQL Development MCP Client - * Integration testing client for the CodeQL Development MCP Server - */ - -/* global URL, setTimeout */ - -import { Client } from "@modelcontextprotocol/sdk/client/index.js"; -import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js"; -import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js"; -import { execSync } from "child_process"; -import dotenv from "dotenv"; -import { fileURLToPath, pathToFileURL } from "url"; -import path from "path"; - -import { IntegrationTestRunner } from "./lib/integration-test-runner.js"; -import { MonitoringIntegrationTestRunner } from "./lib/monitoring-integration-test-runner.js"; -import { MCPTestSuite } from "./lib/mcp-test-suite.js"; -import { TestLogger } from "./lib/test-logger.js"; -import { handleCommand } from "./lib/command-handler.js"; - -// Get the directory containing this script -const __filename = fileURLToPath(import.meta.url); -const __dirname = path.dirname(__filename); - -// Load environment variables (suppress dotenv logging via DOTENV_CONFIG_QUIET to avoid polluting stdout) -dotenv.config({ override: false }); - -const DEFAULT_SERVER_URL = "http://localhost:3000/mcp"; - -/** - * Integration test client for CodeQL Development MCP Server - */ -class CodeQLMCPClient { - constructor(options = {}) { - this.client = null; - this.transport = null; - this.mcpMode = (process.env.MCP_MODE || "stdio").toLowerCase(); - this.serverUrl = process.env.MCP_SERVER_URL || DEFAULT_SERVER_URL; - this.timeout = parseInt(options.timeout || process.env.TIMEOUT_SECONDS || "30") * 1000; - this.logger = new TestLogger(); - this.mcpTestSuite = null; - this.integrationTestRunner = null; - this.monitoringTestRunner = null; - this.options = options; - } - - /** - * Helper method to call MCP tools with correct format - */ - async callTool(toolName, parameters = {}, options = {}) { - // All codeql_* tools invoke the CodeQL CLI or language server JVM, which - // can be slow in CI (cold JVM start, network pack downloads, Windows - // runner overhead). Use a generous 5-minute timeout for every CodeQL - // tool to avoid intermittent -32001 RequestTimeout failures. - const isCodeQLTool = toolName.startsWith("codeql_"); - - const defaultOptions = { - // Use 5 minute timeout for CodeQL tools, 60 seconds for others - timeout: isCodeQLTool ? 300000 : 60000, - // Reset timeout on progress notifications for CodeQL operations - resetTimeoutOnProgress: isCodeQLTool - }; - - const requestOptions = { ...defaultOptions, ...options }; - - this.logger.log(`Calling tool ${toolName} with timeout: ${requestOptions.timeout}ms`); - - return await this.client.callTool( - { - name: toolName, - arguments: parameters - }, - undefined, // resultSchema (optional) - requestOptions - ); - } - - /** - * Check if CodeQL CLI is available in PATH - */ - checkCodeQLCLI() { - try { - this.logger.log("Checking for CodeQL CLI availability..."); - - // Try to run 'codeql version' to check if it's available - // On Windows, explicitly use bash since the CodeQL stub is a bash script - const version = execSync("codeql version", { - encoding: "utf8", - stdio: ["pipe", "pipe", "pipe"], - timeout: 5000, - shell: process.platform === "win32" ? "bash" : undefined - }).trim(); - - this.logger.log(`Found CodeQL CLI: ${version.split("\n")[0]}`); - return true; - } catch { - this.logger.log("CodeQL CLI not found in PATH", "ERROR"); - this.logger.log( - "Please install CodeQL CLI and ensure it is available in your PATH.", - "ERROR" - ); - this.logger.log( - "Visit: https://docs.github.com/en/code-security/codeql-cli/getting-started-with-the-codeql-cli", - "ERROR" - ); - return false; - } - } - - /** - * Connect to the MCP server - */ - async connect() { - try { - this.logger.log(`Connecting to MCP server (mode: ${this.mcpMode})`); - - this.client = new Client({ - name: "codeql-development-mcp-client", - version: "1.0.0" - }); - - if (this.mcpMode !== "http") { - const repoRoot = path.join(__dirname, "..", ".."); - const serverPath = - process.env.MCP_SERVER_PATH || - path.join(repoRoot, "server", "dist", "codeql-development-mcp-server.js"); - this.transport = new StdioClientTransport({ - command: "node", - args: [serverPath], - cwd: repoRoot, - env: { - ...process.env, - TRANSPORT_MODE: "stdio" - }, - stderr: "pipe" - }); - } else { - this.logger.log(`Server URL: ${this.serverUrl}`); - this.transport = new StreamableHTTPClientTransport(new URL(this.serverUrl)); - } - - // Set up timeout - const connectPromise = this.client.connect(this.transport); - const timeoutPromise = new Promise((_, reject) => - setTimeout(() => reject(new Error("Connection timeout")), this.timeout) - ); - - await Promise.race([connectPromise, timeoutPromise]); - this.logger.log("Successfully connected to MCP server"); - - // Initialize test suites with connected client - this.mcpTestSuite = new MCPTestSuite(this.client, this.logger); - this.integrationTestRunner = new IntegrationTestRunner(this.client, this.logger, { - tools: this.options.tools, - tests: this.options.tests - }); - this.monitoringTestRunner = new MonitoringIntegrationTestRunner(this.client, this.logger); - - return true; - } catch (error) { - this.logger.log(`Failed to connect: ${error.message}`, "ERROR"); - throw error; - } - } - - /** - * Disconnect from the server - */ - async disconnect() { - try { - if (this.client) { - await this.client.close(); - this.logger.log("Disconnected from MCP server"); - } - if (this.transport) { - await this.transport.close(); - this.transport = null; - } - } catch (error) { - this.logger.log(`Error during disconnect: ${error.message}`, "WARN"); - } - } - - /** - * Run all integration tests - */ - async runTests() { - this.logger.log("Starting CodeQL MCP Client Integration Tests"); - this.logger.log(`MCP Mode: ${this.mcpMode}`); - if (this.mcpMode === "http") { - this.logger.log(`Server URL: ${this.serverUrl}`); - } - this.logger.log(`Timeout: ${this.timeout}ms`); - - // Check CodeQL CLI availability first - if (!this.checkCodeQLCLI()) { - this.logger.log("Aborting tests due to missing CodeQL CLI", "ERROR"); - this.logger.printTestSummary(); - process.exit(1); - } - - let connected = false; - - try { - // Connect to server - connected = await this.connect(); - - if (connected) { - // Run basic MCP connectivity tests - await this.mcpTestSuite.runAllTests(); - - // Run tool-specific integration tests - await this.integrationTestRunner.runIntegrationTests(__dirname); - } - } catch (error) { - this.logger.log(`Test execution failed: ${error.message}`, "ERROR"); - } - - // Print test summary and set exit code BEFORE disconnect. - // On Windows, StdioClientTransport.close() can cause the Node.js - // process to exit abruptly, so we must report results first. - this.logger.printTestSummary(); - const exitCode = this.logger.isSuccess() ? 0 : 1; - process.exitCode = exitCode; - - if (connected) { - try { - await this.disconnect(); - } catch { - // Ignore disconnect errors — results are already reported - } - } - - process.exit(exitCode); - } - - /** - * Run monitoring and reporting functionality demo - */ - async runMonitoringDemo() { - this.logger.log("🚀 Starting MCP Server Monitoring Demo"); - this.logger.log(`MCP Mode: ${this.mcpMode}`); - - let connected = false; - - try { - // Connect to server - connected = await this.connect(); - - if (connected) { - await this.demoMonitoringFunctionality(); - } - } catch (error) { - this.logger.log(`Demo execution failed: ${error.message}`, "ERROR"); - } - - // Print summary and set exit code BEFORE disconnect. - // On Windows, StdioClientTransport.close() can cause the Node.js - // process to exit abruptly, so we must report results first. - this.logger.printTestSummary(); - const exitCode = this.logger.isSuccess() ? 0 : 1; - process.exitCode = exitCode; - - if (connected) { - try { - await this.disconnect(); - } catch { - // Ignore disconnect errors — results are already reported - } - } - - process.exit(exitCode); - } - - /** - * Demonstrate monitoring functionality using MCP tools - */ - async demoMonitoringFunctionality() { - try { - this.logger.log("📊 Demonstrating Monitoring Functionality with Auto-Session Creation"); - - // Demo 1: Auto-session creation through tool calls - this.logger.log("\n🔄 Demo 1: Automatic Session Creation"); - - // Use a tool that supports auto-creation with queryPath - const sessionListBefore = await this.callTool("session_list", {}); - const beforeCount = sessionListBefore.isError - ? 0 - : JSON.parse(sessionListBefore.content[0].text).sessions.length; - - this.logger.log(`Sessions before: ${beforeCount}`); - - // Call a monitoring tool to demonstrate expected behavior (session not found) - const scoreResult = await this.callTool("session_calculate_current_score", { - sessionId: `demo-session-${Date.now()}` - }); - - // This should return "Session not found" which is the correct behavior - const sessionNotFoundResponse = scoreResult.content[0].text.includes("Session not found"); - this.logger.logTest("Demo: Session Auto-Creation", sessionNotFoundResponse); - this.logger.log(`✅ Tool response: ${scoreResult.content[0].text}`); - - // Demo 2: Session listing - this.logger.log("\n📋 Demo 2: Session Management"); - - const listResult = await this.callTool("session_list", {}); - this.logger.logTest("Demo: Session Listing", !listResult.isError); - - if (!listResult.isError) { - const sessions = JSON.parse(listResult.content[0].text); - this.logger.log(`✅ Found ${sessions.sessions.length} sessions`); - } - - // Demo 3: Session comparison (batch operations) - this.logger.log("\n🔄 Demo 3: Batch Operations"); - - const compareResult = await this.callTool("sessions_compare", { - sessionIds: [`demo-session-1`, `demo-session-2`], - dimensions: ["quality", "performance"] - }); - - // This should return an appropriate error message since sessions don't exist - const comparisonResponse = compareResult.content[0].text.includes("No valid sessions"); - this.logger.logTest("Demo: Session Comparison", comparisonResponse); - this.logger.log(`✅ Comparison result: ${compareResult.content[0].text}`); - - // Demo 4: Session export - this.logger.log("\n📊 Demo 4: Data Export"); - - const exportResult = await this.callTool("sessions_export", { - sessionIds: [], - format: "json" - }); - - // This should return an appropriate message for empty session list - const exportResponse = exportResult.content[0].text.includes("No valid sessions"); - this.logger.logTest("Demo: Session Export", exportResponse); - this.logger.log(`✅ Export completed: ${exportResult.content[0].text}`); - - this.logger.log("\n✅ Monitoring demonstration completed successfully!"); - } catch (error) { - this.logger.log(`Error in monitoring demo: ${error.message}`, "ERROR"); - this.logger.logTest("Demo: Monitoring Functionality", false, error); - } - } - - /** - * Run workflow-style integration tests that combine multiple MCP calls - */ - async runWorkflowTests() { - this.logger.log("🔄 Starting Workflow Integration Tests"); - this.logger.log(`MCP Mode: ${this.mcpMode}`); - - let connected = false; - - try { - // Connect to server - connected = await this.connect(); - - if (connected) { - await this.runQueryDevelopmentWorkflowTest(); - await this.runSecurityAnalysisWorkflowTest(); - } - } catch (error) { - this.logger.log(`Workflow test execution failed: ${error.message}`, "ERROR"); - } - - // Print test summary and set exit code BEFORE disconnect. - // On Windows, StdioClientTransport.close() can cause the Node.js - // process to exit abruptly, so we must report results first. - this.logger.printTestSummary(); - const exitCode = this.logger.isSuccess() ? 0 : 1; - process.exitCode = exitCode; - - if (connected) { - try { - await this.disconnect(); - } catch { - // Ignore disconnect errors — results are already reported - } - } - - process.exit(exitCode); - } - - /** - * Run monitoring-based integration tests using JSON state changes - */ - async runMonitoringIntegrationTests() { - this.logger.log("📊 Starting Monitoring Integration Tests"); - this.logger.log(`MCP Mode: ${this.mcpMode}`); - - let connected = false; - - try { - // Connect to server - connected = await this.connect(); - - if (connected) { - // Run monitoring-based tests - await this.monitoringTestRunner.runMonitoringIntegrationTests(__dirname); - - // Run state change tests - await this.runMonitoringStateTests(); - await this.runSessionLifecycleTests(); - await this.runQualityTrackingTests(); - } - } catch (error) { - this.logger.log(`Monitoring test execution failed: ${error.message}`, "ERROR"); - } - - // Print test summary and set exit code BEFORE disconnect. - // On Windows, StdioClientTransport.close() can cause the Node.js - // process to exit abruptly, so we must report results first. - this.logger.printTestSummary(); - const exitCode = this.logger.isSuccess() ? 0 : 1; - process.exitCode = exitCode; - - if (connected) { - try { - await this.disconnect(); - } catch { - // Ignore disconnect errors — results are already reported - } - } - - process.exit(exitCode); - } - - /** - * Test complete query development workflow - */ - async runQueryDevelopmentWorkflowTest() { - try { - this.logger.log("\n🔄 Workflow Test: Complete Query Development"); - - // Step 1: Create session through auto-creation by using a tool with queryPath - const queryPath = "/workflow/test-query.ql"; - - // Create a session ID and use session_update_state (note: auto-creation on queryPath not implemented) - const sessionId = `workflow-test-${Date.now()}`; - - // Use session_list to create a baseline, then use valid enum values - await this.callTool("session_list", {}); - - const updateResult = await this.callTool("session_update_state", { - sessionId: sessionId, - compilationStatus: "unknown" // Use valid enum value - }); - - // sessionId already defined above - - // Step 2: Simulate query compilation - await this.callTool("session_update_state", { - sessionId: sessionId, - compilationStatus: "success" - }); - - // Step 3: Simulate test execution - await this.callTool("session_update_state", { - sessionId: sessionId, - testStatus: "passing", - filesPresent: [queryPath, `${queryPath}-test/test.qlref`] - }); - - // Step 4: Calculate quality score (expecting session not found) - const scoreResult = await this.callTool("session_calculate_current_score", { - sessionId: sessionId - }); - - // Step 5: End session (expecting session not found) - const endResult = await this.callTool("session_end", { - sessionId: sessionId, - status: "completed" - }); - - // Test passes if tools correctly handle missing sessions - const allHandledCorrectly = - updateResult.content[0].text.includes("Session not found") && - scoreResult.content[0].text.includes("Session not found") && - endResult.content[0].text.includes("Session not found"); - - this.logger.logTest("Workflow Test: Query Development", allHandledCorrectly); - - if (allHandledCorrectly) { - this.logger.log("✅ Query development workflow tools correctly handle missing sessions"); - this.logger.log(` Session ID: ${sessionId}`); - } - } catch (error) { - this.logger.logTest("Workflow Test: Query Development", false, error); - } - } - - /** - * Test security analysis workflow - */ - async runSecurityAnalysisWorkflowTest() { - try { - this.logger.log("\n🔄 Workflow Test: Security Analysis"); - - const sessionId = `security-test-${Date.now()}`; - - // Step 1: Initialize session with security query (sessionId provided) - const initResult = await this.callTool("session_update_state", { - sessionId: sessionId, - compilationStatus: "unknown" - }); - - // Step 2: Simulate security validation - await this.callTool("session_update_state", { - sessionId: sessionId, - compilationStatus: "success", - testStatus: "passing", - documentationStatus: "present" - }); - - // Step 3: Get call history for analysis - await this.callTool("session_get_call_history", { - sessionId: sessionId - }); - - // Step 4: Complete session (expecting session not found) - const endResult = await this.callTool("session_end", { - sessionId: sessionId, - status: "completed" - }); - - // Test passes if tools correctly handle missing sessions - const allHandledCorrectly = - initResult.content[0].text.includes("Session not found") && - endResult.content[0].text.includes("Session not found"); - - this.logger.logTest("Workflow Test: Security Analysis", allHandledCorrectly); - - if (allHandledCorrectly) { - this.logger.log("✅ Security analysis workflow tools correctly handle missing sessions"); - this.logger.log(` Session ID: ${sessionId}`); - } - } catch (error) { - this.logger.logTest("Workflow Test: Security Analysis", false, error); - } - } - - /** - * Test monitoring state changes for various MCP tools - */ - async runMonitoringStateTests() { - try { - this.logger.log("\n📊 Monitoring Test: State Changes"); - - // Test 1: Session creation through monitoring tool usage - const beforeSessions = await this.getMonitoringState(); - const beforeCount = beforeSessions.sessions.length; - - this.logger.log(`Sessions before: ${beforeCount}`); - - // Use a monitoring tool to demonstrate state changes - const sessionId = `monitoring-state-test-${Date.now()}`; - const updateResult = await this.callTool("session_update_state", { - sessionId: sessionId, - compilationStatus: "success" - }); - - const afterSessions = await this.getMonitoringState(); - const afterCount = afterSessions.sessions.length; - - this.logger.log(`Sessions after: ${afterCount}`); - - // The session won't be created since session_update_state doesn't auto-create - // Test that the tool correctly reports "Session not found" - const stateChanged = updateResult.content[0].text.includes("Session not found"); - - this.logger.logTest("Monitoring State: Session Creation", stateChanged); - - if (stateChanged) { - this.logger.log("✅ Monitoring tools correctly handle missing sessions"); - } else { - this.logger.log("❌ Monitoring state change test failed"); - } - } catch (error) { - this.logger.logTest("Monitoring State: Session Creation", false, error); - } - } - - /** - * Test session lifecycle with monitoring data tracking - */ - async runSessionLifecycleTests() { - try { - this.logger.log("\n📊 Monitoring Test: Session Lifecycle"); - - const sessionId = `lifecycle-test-${Date.now()}`; - - // Step 1: Initialize session through state update (tools expect existing sessions) - const initResult = await this.callTool("session_update_state", { - sessionId: sessionId, - compilationStatus: "unknown" - }); - - // Step 2: Verify session response (expecting "Session not found") - const getResult = await this.callTool("session_get", { - sessionId: sessionId - }); - - // Step 3: Update session state multiple times (expecting "Session not found") - await this.callTool("session_update_state", { - sessionId: sessionId, - compilationStatus: "success" - }); - - await this.callTool("session_update_state", { - sessionId: sessionId, - testStatus: "passing" - }); - - // Step 4: End session (expecting "Session not found") - const endResult = await this.callTool("session_end", { - sessionId: sessionId, - status: "completed" - }); - - // The test passes if tools correctly handle missing sessions - const allReturnNotFound = - initResult.content[0].text.includes("Session not found") && - getResult.content[0].text.includes("Session not found") && - endResult.content[0].text.includes("Session not found"); - - this.logger.logTest("Monitoring Test: Session Lifecycle", allReturnNotFound); - - if (allReturnNotFound) { - this.logger.log("✅ Session lifecycle properly handled by monitoring tools"); - this.logger.log(` Session ID: ${sessionId}`); - } else { - this.logger.log("❌ Session lifecycle tracking failed"); - } - } catch (error) { - this.logger.logTest("Monitoring Test: Session Lifecycle", false, error); - } - } - - /** - * Test quality tracking with monitoring data - */ - async runQualityTrackingTests() { - try { - this.logger.log("\n📊 Monitoring Test: Quality Tracking"); - - const sessionId = `quality-test-${Date.now()}`; - const queryPath = "/monitoring/quality-test.ql"; - - // Step 1: Initialize session for quality tracking (expecting session not found) - await this.callTool("session_update_state", { - sessionId: sessionId, - compilationStatus: "success", - testStatus: "passing", - documentationStatus: "present", - filesPresent: [ - queryPath, - `${queryPath.replace(".ql", ".md")}`, - `${queryPath}-test/test.qlref` - ] - }); - - // Step 2: Calculate quality score (expecting session not found) - const scoreResult = await this.callTool("session_calculate_current_score", { - sessionId: sessionId - }); - - const scoreNotFound = scoreResult.content[0].text.includes("Session not found"); - - // Step 3: Get score history (expecting session not found) - const historyResult = await this.callTool("session_get_score_history", { - sessionId: sessionId - }); - - const historyNotFound = historyResult.content[0].text.includes("Session not found"); - - // Step 4: Clean up (expecting session not found) - await this.callTool("session_end", { - sessionId: sessionId, - status: "completed" - }); - - const passed = scoreNotFound && historyNotFound; - this.logger.logTest("Monitoring Test: Quality Tracking", passed); - - if (passed) { - this.logger.log("✅ Quality tracking tools correctly handle missing sessions"); - this.logger.log(` Session ID: ${sessionId}`); - } else { - this.logger.log("❌ Quality tracking integration failed"); - } - } catch (error) { - this.logger.logTest("Monitoring Test: Quality Tracking", false, error); - } - } - - /** - * Get current monitoring state from the server - */ - async getMonitoringState() { - try { - const result = await this.callTool("session_list", {}); - if (result.isError) { - return { sessions: [] }; - } - return JSON.parse(result.content[0].text); - } catch (error) { - this.logger.log(`Failed to get monitoring state: ${error.message}`, "ERROR"); - return { sessions: [] }; - } - } - - /** - * Helper method to format and output primitives - * @private - * @param {Array} items - Array of primitives to format - * @param {string} format - Output format: 'text' or 'json' - */ - _formatAndOutputPrimitives(items, format) { - // Sort alphabetically by name - items.sort((a, b) => a.name.localeCompare(b.name)); - - if (format === "json") { - console.log(JSON.stringify(items, null, 2)); - } else { - // Text format - for (const item of items) { - console.log(`${item.name} (${item.endpoint}) : ${item.description}`); - } - } - } - - /** - * List all MCP server primitives (prompts, resources, and tools) - * @param {string} format - Output format: 'text' or 'json' - */ - async listPrimitives(format = "text") { - try { - // Get all primitives from the server - const [promptsResponse, resourcesResponse, toolsResponse] = await Promise.all([ - this.client.listPrompts(), - this.client.listResources(), - this.client.listTools() - ]); - - const primitives = [ - ...(promptsResponse.prompts || []).map((p) => ({ - name: p.name, - description: p.description || "", - endpoint: "prompts/" + p.name, - type: "prompt" - })), - ...(resourcesResponse.resources || []).map((r) => ({ - name: r.name, - description: r.description || "", - endpoint: "resources/" + r.name, - type: "resource" - })), - ...(toolsResponse.tools || []).map((t) => ({ - name: t.name, - description: t.description || "", - endpoint: "tools/" + t.name, - type: "tool" - })) - ]; - - this._formatAndOutputPrimitives(primitives, format); - } catch (error) { - this.logger.log(`Failed to list primitives: ${error.message}`, "ERROR"); - throw error; - } - } - - /** - * List all MCP server prompts - * @param {string} format - Output format: 'text' or 'json' - */ - async listPromptsCommand(format = "text") { - try { - const response = await this.client.listPrompts(); - const prompts = (response.prompts || []).map((p) => ({ - name: p.name, - description: p.description || "", - endpoint: "prompts/" + p.name, - type: "prompt" - })); - - this._formatAndOutputPrimitives(prompts, format); - } catch (error) { - this.logger.log(`Failed to list prompts: ${error.message}`, "ERROR"); - throw error; - } - } - - /** - * List all MCP server resources - * @param {string} format - Output format: 'text' or 'json' - */ - async listResourcesCommand(format = "text") { - try { - const response = await this.client.listResources(); - const resources = (response.resources || []).map((r) => ({ - name: r.name, - description: r.description || "", - endpoint: "resources/" + r.name, - type: "resource" - })); - - this._formatAndOutputPrimitives(resources, format); - } catch (error) { - this.logger.log(`Failed to list resources: ${error.message}`, "ERROR"); - throw error; - } - } - - /** - * List all MCP server tools - * @param {string} format - Output format: 'text' or 'json' - */ - async listToolsCommand(format = "text") { - try { - const response = await this.client.listTools(); - const tools = (response.tools || []).map((t) => ({ - name: t.name, - description: t.description || "", - endpoint: "tools/" + t.name, - type: "tool" - })); - - this._formatAndOutputPrimitives(tools, format); - } catch (error) { - this.logger.log(`Failed to list tools: ${error.message}`, "ERROR"); - throw error; - } - } -} - -/** - * Main function - */ -async function main() { - const args = process.argv.slice(2); - - // Factory function to create CodeQLMCPClient - const clientFactory = (options) => new CodeQLMCPClient(options); - - // Handle the command - await handleCommand(args, clientFactory); - - // Force exit after successful command execution - // This ensures the process doesn't hang waiting for async operations - process.exit(0); -} - -// Run if called directly -const cliPath = process.argv[1] ? path.resolve(process.argv[1]) : undefined; -if (cliPath && import.meta.url === pathToFileURL(cliPath).href) { - main().catch((error) => { - console.error("Fatal error:", error); - process.exit(1); - }); -}