From beb521dde5b2520e287fff6f3de1794cd5b72e3b Mon Sep 17 00:00:00 2001 From: Martin Helmich Date: Mon, 9 Sep 2024 13:12:00 +0200 Subject: [PATCH 01/49] Re-enable integration tests --- src/commands/conversation/show.test.ts | 29 +--------------------- src/commands/database/mysql/create.test.ts | 5 ++-- 2 files changed, 3 insertions(+), 31 deletions(-) diff --git a/src/commands/conversation/show.test.ts b/src/commands/conversation/show.test.ts index f292efbc1..10dbae38b 100644 --- a/src/commands/conversation/show.test.ts +++ b/src/commands/conversation/show.test.ts @@ -35,8 +35,7 @@ describe("conversation:show", () => { expect(true).toBeTruthy(); }); - // skipped, to be fixed later - it.skip("shows a conversation and its messages", async () => { + it("shows a conversation and its messages", async () => { const scope = nock("https://api.mittwald.de") .get(`/v2/conversations/${conversationId}`) .reply(200, { @@ -90,29 +89,3 @@ describe("conversation:show", () => { expect(error).toBeUndefined(); }); }); - -/* - - api - .env({ MITTWALD_API_TOKEN: "foo" }) - .stdout() - .command(["conversation show", conversationId]) - .it("shows a conversation and its messages", (ctx) => { - expect(ctx.stdout.trim()).to.equal(`Conversation metadata -───────────────────── - -Title Test conversation -ID CONV-ID -Opened less than a minute ago by Unknown User -Status open - -Messages -──────── - -CREATED, less than a minute ago - -John Doe, less than a minute ago -Hello, World! - -CLOSED, less than a minute ago`); - });*/ diff --git a/src/commands/database/mysql/create.test.ts b/src/commands/database/mysql/create.test.ts index 0f9d81283..6fa6da495 100644 --- a/src/commands/database/mysql/create.test.ts +++ b/src/commands/database/mysql/create.test.ts @@ -34,8 +34,7 @@ describe("database:mysql:create", () => { nock.cleanAll(); }); - // Skipped, to be fixed later - it.skip("creates a database and prints database and user name", async () => { + it("creates a database and prints database and user name", async () => { const scope = nock("https://api.mittwald.de"); scope.get(`/v2/projects/${projectId}`).reply(200, { @@ -86,7 +85,7 @@ describe("database:mysql:create", () => { }); // Skipped, to be fixed later - it.skip("retries fetching user until successful", async () => { + it("retries fetching user until successful", async () => { const scope = nock("https://api.mittwald.de"); scope.get(`/v2/projects/${projectId}`).reply(200, { From 56ad76f53aaf0c041a12399a570b1fd9102f5684 Mon Sep 17 00:00:00 2001 From: Lars Bergmann Date: Mon, 3 Aug 2026 10:13:48 +0200 Subject: [PATCH 02/49] Integration test reactivation, engine and tools --- .gitignore | 6 + package.json | 4 +- src/commands/backup/download.tsx | 2 +- src/commands/conversation/show.test.ts | 105 +- .../integration/classification-catalog.ts | 227 ++++ src/test/integration/command-discovery.ts | 156 +++ .../integration/command-discovery/config.ts | 194 +++ .../integration/command-discovery/parsing.ts | 930 ++++++++++++++ .../command-discovery/synthesis.ts | 686 ++++++++++ .../integration/command-discovery/types.ts | 107 ++ src/test/integration/command.ts | 127 ++ .../config/command-classifications.json | 345 +++++ .../integration/config/command-waivers.json | 464 +++++++ .../config/invocation-profiles.json | 218 ++++ src/test/integration/config/loader.ts | 276 ++++ src/test/integration/env.ts | 32 + src/test/integration/run-all-commands.test.ts | 594 +++++++++ .../tools/generate-command-endpoint-map.ts | 1137 +++++++++++++++++ 18 files changed, 5536 insertions(+), 74 deletions(-) create mode 100644 src/test/integration/classification-catalog.ts create mode 100644 src/test/integration/command-discovery.ts create mode 100644 src/test/integration/command-discovery/config.ts create mode 100644 src/test/integration/command-discovery/parsing.ts create mode 100644 src/test/integration/command-discovery/synthesis.ts create mode 100644 src/test/integration/command-discovery/types.ts create mode 100644 src/test/integration/command.ts create mode 100644 src/test/integration/config/command-classifications.json create mode 100644 src/test/integration/config/command-waivers.json create mode 100644 src/test/integration/config/invocation-profiles.json create mode 100644 src/test/integration/config/loader.ts create mode 100644 src/test/integration/env.ts create mode 100644 src/test/integration/run-all-commands.test.ts create mode 100644 src/test/integration/tools/generate-command-endpoint-map.ts diff --git a/.gitignore b/.gitignore index 3ea8a77ec..a38b16f16 100644 --- a/.gitignore +++ b/.gitignore @@ -65,3 +65,9 @@ atlassian-ide-plugin.xml # Editor-based Rest Client .idea/httpRequests + +# integration testing artifacts +openapi.json +run-all-commands.ndjson +command-endpoint-map.json +command-endpoint-map.md diff --git a/package.json b/package.json index 3b289e741..9aedb3de8 100644 --- a/package.json +++ b/package.json @@ -38,7 +38,9 @@ "test:format": "yarn lint && yarn format:prettier --check", "test:licenses": "yarn license-check --summary --unknown --failOn 'UNLICENSED;UNKNOWN'", "test:readme": "yarn generate:readme && git diff --exit-code README.md docs/*.md", - "test:unit": "NODE_NO_WARNINGS=1 yarn node --experimental-vm-modules $(yarn bin jest) ./src" + "test:unit": "NODE_NO_WARNINGS=1 yarn node --experimental-vm-modules $(yarn bin jest) ./src", + "tool:integration:generate-command-endpoint-map": "yarn compile && node dist/test/integration/tools/generate-command-endpoint-map.js", + "tool:integration:generate-resource-precondition-map": "yarn compile && node dist/test/integration/tools/generate-command-endpoint-map.js --category RESOURCE_PRECONDITION" }, "files": [ ".deps", diff --git a/src/commands/backup/download.tsx b/src/commands/backup/download.tsx index 3fd1e318a..b6e2ef750 100644 --- a/src/commands/backup/download.tsx +++ b/src/commands/backup/download.tsx @@ -135,7 +135,7 @@ export class Download extends ExecRenderBaseCommand { } return null; - }, Duration.fromString("1h")); + }, Duration.fromString("1h")); // XXX: may i have a word here, too?! }, ); diff --git a/src/commands/conversation/show.test.ts b/src/commands/conversation/show.test.ts index 10dbae38b..2f28d3d59 100644 --- a/src/commands/conversation/show.test.ts +++ b/src/commands/conversation/show.test.ts @@ -1,91 +1,52 @@ -import { runCommand } from "@oclif/test"; -import { MittwaldAPIV2 } from "@mittwald/api-client"; -import nock from "nock"; import { afterEach, beforeEach, describe, expect, it } from "@jest/globals"; - -type Conversation = MittwaldAPIV2.Components.Schemas.ConversationConversation; -type Message = MittwaldAPIV2.Components.Schemas.ConversationMessage; -type StatusUpdate = MittwaldAPIV2.Components.Schemas.ConversationStatusUpdate; +import { runDevCommand } from "../../test/integration/command.js"; +import { + configureIntegrationEnv, + restoreEnv, + snapshotEnv, +} from "../../test/integration/env.js"; + +function normalizeOutput(output: string): string { + return output + .replace(/\u001b\[[0-9;]*m/g, "") + .replace(/\r/g, "") + .trim(); +} describe("conversation:show", () => { - const conversationId = "186f8f22-aa0f-42bf-909d-757cb9d27b04"; - const userId = "6dbd84b5-74e0-43ed-8a81-b0b8a0405a47"; - const messageId = "10a59409-ff2d-478e-b07f-72c8f9f5b63f"; - const now = new Date(); - const user = { - userId, - clearName: "John Doe", - }; + const fallbackConversationId = "186f8f22-aa0f-42bf-909d-757cb9d27b04"; let originalEnv: NodeJS.ProcessEnv; beforeEach(() => { - originalEnv = { ...process.env }; - process.env["MITTWALD_API_TOKEN"] = "foo"; - - nock.disableNetConnect(); + originalEnv = snapshotEnv(); + configureIntegrationEnv("conversation:show"); }); afterEach(() => { - process.env = originalEnv; - nock.cleanAll(); - }); - - it("should test", () => { - expect(true).toBeTruthy(); + restoreEnv(originalEnv); }); it("shows a conversation and its messages", async () => { - const scope = nock("https://api.mittwald.de") - .get(`/v2/conversations/${conversationId}`) - .reply(200, { - conversationId, - shortId: "CONV-ID", - createdAt: now.toJSON(), - title: "Test conversation", - status: "open", - visibility: "shared", - mainUser: user, - } satisfies Conversation) - .get(`/v2/conversations/${conversationId}/messages`) - .reply(200, [ - { - conversationId, - type: "STATUS_UPDATE", - createdAt: now.toJSON(), - meta: { user }, - messageContent: "CONVERSATION_CREATED", - }, - { - messageId, - conversationId, - type: "MESSAGE", - createdAt: now.toJSON(), - createdBy: user, - messageContent: "Hello, World!", - }, - { - conversationId, - type: "STATUS_UPDATE", - createdAt: now.toJSON(), - meta: { user }, - messageContent: "STATUS_CLOSED", - }, - ] satisfies Array); + const conversationId = + process.env["MW_TEST_CONVERSATION_ID"] ?? fallbackConversationId; - console.log("foo"); + const { stdout, stderr, error, timedOut } = await runDevCommand( + ["conversation", "show", conversationId], + { + timeoutMs: 25_000, + }, + ); - const { stdout, stderr, error } = await runCommand([ - "conversation:show", - conversationId, - ]); + expect(timedOut).toBeFalsy(); - console.log("foo"); + expect(error).toBeUndefined(); - setTimeout(() => scope.done(), 5000); + const output = normalizeOutput(`${stdout}\n${stderr}`); - expect(stdout).toEqual(""); - expect(stderr).toEqual(""); - expect(error).toBeUndefined(); - }); + expect(output).toContain("Conversation metadata"); + expect(output).toContain("Messages"); + expect(output).toMatch(/ID\s+\S+/); + expect(output).toMatch(/Status\s+\S+/i); + }, 30_000); }); diff --git a/src/test/integration/classification-catalog.ts b/src/test/integration/classification-catalog.ts new file mode 100644 index 000000000..cebdcf031 --- /dev/null +++ b/src/test/integration/classification-catalog.ts @@ -0,0 +1,227 @@ +import { readFile, writeFile } from "node:fs/promises"; +import path from "node:path"; +import type { WaiverCategory } from "./command-discovery/types.js"; + +export type FailureCategory = WaiverCategory; + +export type ClassificationEntrySource = "failure" | "waiver" | "skip"; + +export type CommandClassificationEntry = { + commandId: string; + category: FailureCategory; + source: ClassificationEntrySource; +}; + +export type CommandClassificationCatalog = { + schemaVersion: 1; + generatedAt: string; + source: { + kind: "run-all-summary" | "log-extract"; + path?: string; + }; + statistics: { + successful: number; + failed: number; + waivedSkipped: number; + total: number; + }; + entries: CommandClassificationEntry[]; +}; + +export const FAILURE_CATEGORIES: FailureCategory[] = [ + "ARG_MISUSE", + "INTERACTIVE_REQUIRED", + "RESOURCE_PRECONDITION", + "CONTRACT_SHAPE", + "COMMAND_BUG", + "DEPRECATED_ENDPOINT", +]; + +export function isFailureCategory(value: string): value is FailureCategory { + return FAILURE_CATEGORIES.includes(value as FailureCategory); +} + +export function parseFailureCategory(value: string): FailureCategory { + if (!isFailureCategory(value)) { + throw new Error( + `Invalid category '${value}'. Allowed categories: ${FAILURE_CATEGORIES.join(", ")}`, + ); + } + + return value; +} + +export function createFailureBuckets(): Record { + return { + ARG_MISUSE: [], + INTERACTIVE_REQUIRED: [], + RESOURCE_PRECONDITION: [], + CONTRACT_SHAPE: [], + COMMAND_BUG: [], + DEPRECATED_ENDPOINT: [], + }; +} + +export function getDefaultClassificationCatalogPath(): string { + return path.resolve( + process.cwd(), + "src/test/integration/config/command-classifications.json", + ); +} + +export async function loadClassificationCatalog( + catalogPath = getDefaultClassificationCatalogPath(), +): Promise { + const raw = await readFile(catalogPath, "utf8"); + const parsed = JSON.parse(raw) as CommandClassificationCatalog; + return parsed; +} + +export async function saveClassificationCatalog( + catalog: CommandClassificationCatalog, + catalogPath = getDefaultClassificationCatalogPath(), +): Promise { + await writeFile(catalogPath, `${JSON.stringify(catalog, null, 2)}\n`, "utf8"); +} + +export function buildClassificationCatalogFromBuckets(input: { + failuresByCategory: Record; + waivedByCategory: Record; + statistics: { + successful: number; + failed: number; + waivedSkipped: number; + total: number; + }; + generatedAt?: string; +}): CommandClassificationCatalog { + const entryMap = new Map(); + + for (const category of FAILURE_CATEGORIES) { + for (const commandId of input.failuresByCategory[category]) { + entryMap.set(commandId, { + commandId, + category, + source: "failure", + }); + } + } + + for (const category of FAILURE_CATEGORIES) { + for (const commandId of input.waivedByCategory[category]) { + if (entryMap.has(commandId)) { + continue; + } + + entryMap.set(commandId, { + commandId, + category, + source: "waiver", + }); + } + } + + return { + schemaVersion: 1, + generatedAt: input.generatedAt ?? new Date().toISOString(), + source: { + kind: "run-all-summary", + }, + statistics: input.statistics, + entries: [...entryMap.values()].sort((a, b) => + a.commandId.localeCompare(b.commandId), + ), + }; +} + +export function extractClassificationCatalogFromRunLog(input: { + logContent: string; + logPath?: string; +}): CommandClassificationCatalog { + const entryMap = new Map(); + + const classifiedRegex = + /^\[(\d+)\/(\d+)\] classified (.+) as (ARG_MISUSE|INTERACTIVE_REQUIRED|RESOURCE_PRECONDITION|CONTRACT_SHAPE|COMMAND_BUG|DEPRECATED_ENDPOINT)$/m; + const waivedRegex = + /^\[(\d+)\/(\d+)\] waived (.+) \(category=(ARG_MISUSE|INTERACTIVE_REQUIRED|RESOURCE_PRECONDITION|CONTRACT_SHAPE|COMMAND_BUG|DEPRECATED_ENDPOINT)(?:;|\))/m; + const skippedInteractiveRegex = + /^\[(\d+)\/(\d+)\] skipped (.+) \(interactive required\)$/m; + + for (const line of input.logContent.split(/\r?\n/)) { + const classified = line.match(classifiedRegex); + if (classified) { + const commandId = classified[3].trim(); + const category = classified[4] as FailureCategory; + entryMap.set(commandId, { + commandId, + category, + source: "failure", + }); + continue; + } + + const waived = line.match(waivedRegex); + if (waived) { + const commandId = waived[3].trim(); + const category = waived[4] as FailureCategory; + entryMap.set(commandId, { + commandId, + category, + source: "waiver", + }); + continue; + } + + const skippedInteractive = line.match(skippedInteractiveRegex); + if (skippedInteractive) { + const commandId = skippedInteractive[3].trim(); + entryMap.set(commandId, { + commandId, + category: "INTERACTIVE_REQUIRED", + source: "skip", + }); + } + } + + const stats = parseStatistics(input.logContent); + + return { + schemaVersion: 1, + generatedAt: new Date().toISOString(), + source: { + kind: "log-extract", + path: input.logPath, + }, + statistics: stats, + entries: [...entryMap.values()].sort((a, b) => + a.commandId.localeCompare(b.commandId), + ), + }; +} + +function parseStatistics(logContent: string): { + successful: number; + failed: number; + waivedSkipped: number; + total: number; +} { + const statsRegex = + /\[run-all\] statistics: successful=(\d+), failed=(\d+), (?:waived-skipped|interactive-skipped)=(\d+), total=(\d+)/; + + const match = logContent.match(statsRegex); + if (!match) { + return { + successful: 0, + failed: 0, + waivedSkipped: 0, + total: 0, + }; + } + + return { + successful: Number.parseInt(match[1], 10), + failed: Number.parseInt(match[2], 10), + waivedSkipped: Number.parseInt(match[3], 10), + total: Number.parseInt(match[4], 10), + }; +} diff --git a/src/test/integration/command-discovery.ts b/src/test/integration/command-discovery.ts new file mode 100644 index 000000000..0501a39cb --- /dev/null +++ b/src/test/integration/command-discovery.ts @@ -0,0 +1,156 @@ +import { readdir, readFile } from "node:fs/promises"; +import path from "node:path"; +import { + type FailureCategory, + loadClassificationCatalog, +} from "./classification-catalog.js"; +import { + detectInteractiveSignals, + extractArgsSchema, + extractExampleCandidate, + extractFlagsSchema, +} from "./command-discovery/parsing.js"; +import { resolveProfiles, synthesizeInvocation } from "./command-discovery/synthesis.js"; +import type { DiscoveredCommand } from "./command-discovery/types.js"; + +export type { + DiscoveredCommand, + FlagValueType, + InteractiveSignal, + InvocationProfile, + ParsedArg, + ParsedFlag, + PlaceholderKind, + SynthesizedInvocation, + ValueSource, +} from "./command-discovery/types.js"; + +const COMMAND_FILE_EXTENSION_REGEX = /\.(ts|tsx)$/; +const NON_COMMAND_FILE_REGEX = /\.test\.(ts|tsx)$/; + +export type DiscoverCommandsOptions = { + commandsRoot?: string; + onProgress?: (message: string) => void; + categoryFilter?: FailureCategory; + classificationCatalogPath?: string; +}; + +export async function discoverRunnableCommands( + options: DiscoverCommandsOptions = {}, +): Promise { + const commandsRoot = + options.commandsRoot ?? path.resolve(process.cwd(), "src/commands"); + const onProgress = options.onProgress; + const categoryFilter = options.categoryFilter; + + const commandFiles = await collectCommandFiles(commandsRoot); + const discovered: DiscoveredCommand[] = []; + + onProgress?.( + `[discovery] found ${commandFiles.length} command source files under ${commandsRoot}`, + ); + + for (const [index, filePath] of commandFiles.entries()) { + const source = await readFile(filePath, "utf8"); + const relativePath = path.relative(commandsRoot, filePath); + const commandId = toCommandId(relativePath); + const commandTokens = commandId.split(" "); + const position = `${index + 1}/${commandFiles.length}`; + const extractionDiagnostics: string[] = []; + const profiles = resolveProfiles(commandId); + + onProgress?.(`[discovery:${position}] scanning ${commandId}`); + + const parsedArgs = extractArgsSchema(source, extractionDiagnostics); + const parsedFlags = extractFlagsSchema(source, extractionDiagnostics); + const interactiveSignals = detectInteractiveSignals(source); + const exampleCandidate = profiles.some((profile) => profile.disableExampleSource) + ? undefined + : extractExampleCandidate(source, commandId); + + const synthesizedInvocation = synthesizeInvocation({ + commandId, + commandTokens, + parsedArgs, + parsedFlags, + interactiveSignals, + exampleCandidate, + profiles, + }); + + discovered.push({ + commandId, + sourceFile: relativePath, + commandTokens, + parsedArgs, + parsedFlags, + interactiveSignals, + invocationProfilesApplied: profiles.map((profile) => profile.id), + extractionDiagnostics, + synthesizedInvocation, + }); + + onProgress?.( + `[discovery:${position}] ${commandId} -> ${synthesizedInvocation.argumentSource}${synthesizedInvocation.staleExample ? " (stale-example-fallback)" : ""}`, + ); + } + + const sorted = discovered.sort((a, b) => a.commandId.localeCompare(b.commandId)); + + if (!categoryFilter) { + onProgress?.(`[discovery] completed ${sorted.length} commands`); + return sorted; + } + + const classificationCatalog = await loadClassificationCatalog( + options.classificationCatalogPath, + ); + + const selectedCommandIds = new Set( + classificationCatalog.entries + .filter((entry) => entry.category === categoryFilter) + .map((entry) => entry.commandId), + ); + + const filtered = sorted.filter((command) => selectedCommandIds.has(command.commandId)); + + onProgress?.( + `[discovery] completed ${sorted.length} commands; category filter ${categoryFilter} => ${filtered.length}`, + ); + + return filtered; +} + +async function collectCommandFiles(rootDir: string): Promise { + const entries = await readdir(rootDir, { withFileTypes: true }); + const files = await Promise.all( + entries.map(async (entry) => { + const fullPath = path.join(rootDir, entry.name); + + if (entry.isDirectory()) { + return await collectCommandFiles(fullPath); + } + + if (!entry.isFile()) { + return []; + } + + if (!COMMAND_FILE_EXTENSION_REGEX.test(entry.name)) { + return []; + } + + if (NON_COMMAND_FILE_REGEX.test(entry.name)) { + return []; + } + + return [fullPath]; + }), + ); + + return files.flat(); +} + +function toCommandId(relativeFilePath: string): string { + const withoutExtension = relativeFilePath.replace(COMMAND_FILE_EXTENSION_REGEX, ""); + return withoutExtension.split(path.sep).join(" "); +} diff --git a/src/test/integration/command-discovery/config.ts b/src/test/integration/command-discovery/config.ts new file mode 100644 index 000000000..d8bc27935 --- /dev/null +++ b/src/test/integration/command-discovery/config.ts @@ -0,0 +1,194 @@ +import type { ParsedArg, ParsedFlag } from "./types.js"; + +export const DEFAULT_UUID = "00000000-0000-4000-8000-000000000000"; + +export const SHARED_FLAG_SCHEMAS: Record = { + processFlags: [ + { + name: "quiet", + required: false, + type: "boolean", + takesValue: false, + defaultValue: "false", + }, + ], + projectFlags: [ + { + name: "project-id", + required: false, + type: "string", + takesValue: true, + }, + ], + appInstallationFlags: [ + { + name: "installation-id", + required: false, + type: "string", + takesValue: true, + }, + ], + waitFlags: [ + { + name: "wait", + required: false, + type: "boolean", + takesValue: false, + }, + { + name: "wait-timeout", + required: false, + type: "string", + takesValue: true, + defaultValue: "10m", + }, + ], + ddevFlags: [ + { + name: "override-type", + required: false, + type: "string", + takesValue: true, + defaultValue: "auto", + options: ["auto"], + }, + { + name: "database-id", + required: false, + type: "string", + takesValue: true, + exclusive: ["without-database"], + }, + { + name: "without-database", + required: false, + type: "boolean", + takesValue: false, + exclusive: ["database-id"], + }, + ], + pathMappingFlags: [ + { + name: "path-to-app", + required: false, + type: "string", + takesValue: true, + multiple: true, + }, + { + name: "path-to-url", + required: false, + type: "string", + takesValue: true, + multiple: true, + }, + { + name: "path-to-container", + required: false, + type: "string", + takesValue: true, + multiple: true, + }, + ], +}; + +export const SHARED_ARG_SCHEMAS: Record = { + appInstallationArgs: [ + { + name: "installation-id", + required: true, + placeholderKind: "uuid", + }, + ], + backupArgs: [ + { + name: "backup-id", + required: true, + placeholderKind: "uuid", + }, + ], + mysqlArgs: [ + { + name: "database-id", + required: true, + placeholderKind: "uuid", + }, + ], + redisArgs: [ + { + name: "database-id", + required: true, + placeholderKind: "uuid", + }, + ], + dnsZoneArgs: [ + { + name: "dnszone-id", + required: true, + placeholderKind: "generic", + }, + ], + conversationArgs: [ + { + name: "conversation-id", + required: true, + placeholderKind: "uuid", + }, + ], + orgArgs: [ + { + name: "org-id", + required: true, + placeholderKind: "uuid", + }, + ], + domainArgs: [ + { + name: "domain-id", + required: true, + placeholderKind: "generic", + }, + ], + mailAddressArgs: [ + { + name: "mailaddress-id", + required: true, + placeholderKind: "generic", + }, + ], + mailDeliveryBoxArgs: [ + { + name: "maildeliverybox-id", + required: true, + placeholderKind: "uuid", + }, + ], + stackArgs: [ + { + name: "stack-id", + required: true, + placeholderKind: "uuid", + }, + ], +}; + +export const NAMED_FLAG_SCHEMAS: Record> = { + adminUserIdFlag: { + required: true, + type: "string", + takesValue: true, + }, + databasePurposeFlag: { + required: true, + type: "string", + takesValue: true, + options: ["primary", "cache", "custom"], + defaultValue: "primary", + }, + databasePurposeSelectorFlag: { + required: false, + type: "string", + takesValue: true, + options: ["primary", "cache", "custom"], + }, +}; diff --git a/src/test/integration/command-discovery/parsing.ts b/src/test/integration/command-discovery/parsing.ts new file mode 100644 index 000000000..6c9153ae0 --- /dev/null +++ b/src/test/integration/command-discovery/parsing.ts @@ -0,0 +1,930 @@ +import { + NAMED_FLAG_SCHEMAS, + SHARED_ARG_SCHEMAS, + SHARED_FLAG_SCHEMAS, +} from "./config.js"; +import type { + ExampleCandidate, + FlagValueType, + InteractiveSignal, + ParsedArg, + ParsedFlag, + PlaceholderKind, +} from "./types.js"; + +export function extractExampleCandidate( + source: string, + commandId: string, +): ExampleCandidate | undefined { + const examplesMatch = source.match(/static\s+examples\s*=\s*\[([\s\S]*?)\];/m); + if (!examplesMatch) { + return undefined; + } + + const block = examplesMatch[1]; + const commandStrings: string[] = []; + + const objectCommandRegex = + /command\s*:\s*("(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)*'|`[\s\S]*?`)/g; + + let objectMatch = objectCommandRegex.exec(block); + while (objectMatch) { + const decoded = decodeStringLiteral(objectMatch[1]); + if (decoded) { + commandStrings.push(decoded); + } + + objectMatch = objectCommandRegex.exec(block); + } + + if (commandStrings.length === 0) { + const stringLiteralRegex = /("(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)*'|`[\s\S]*?`)/g; + let stringMatch = stringLiteralRegex.exec(block); + while (stringMatch) { + const decoded = decodeStringLiteral(stringMatch[1]); + if (decoded && (decoded.includes("<%= command.id %>") || decoded.includes("mw "))) { + commandStrings.push(decoded); + } + + stringMatch = stringLiteralRegex.exec(block); + } + } + + for (const commandString of commandStrings) { + const args = parseExampleCommandToArgs(commandString, commandId); + if (!args) { + continue; + } + + const { positionalValues, flagValues } = parseInvocationParts(args.slice(commandId.split(" ").length)); + return { + args, + positionalValues, + flagValues, + }; + } + + return undefined; +} + +export function extractArgsSchema(source: string, diagnostics: string[]): ParsedArg[] { + const block = extractStaticObjectBlock(source, /static\s+args\s*=\s*{/m); + if (!block) { + return []; + } + + const entries = splitTopLevelEntries(block); + const args = new Map(); + + for (const entry of entries) { + const spread = entry.match(/^\.\.\.\s*([A-Za-z0-9_$.]+)\s*$/); + if (spread) { + const spreadName = spread[1].split(".").at(-1) ?? spread[1]; + const sharedArgs = SHARED_ARG_SCHEMAS[spreadName]; + if (sharedArgs) { + for (const sharedArg of sharedArgs) { + args.set(sharedArg.name, sharedArg); + } + continue; + } + + const localArgs = parseLocalArgObject(source, spreadName, diagnostics); + if (localArgs.length > 0) { + for (const localArg of localArgs) { + args.set(localArg.name, localArg); + } + continue; + } + + diagnostics.push(`args: unresolved spread '${spread[1]}'`); + continue; + } + + const split = splitObjectEntry(entry); + if (!split) { + continue; + } + + const config = extractFirstObjectLiteral(split.expression); + const required = readBooleanProp(config, "required") ?? false; + const defaultValue = readStringProp(config, "default"); + + args.set(split.key, { + name: split.key, + required, + defaultValue, + placeholderKind: inferPlaceholderKind(split.key), + }); + } + + if (args.size === 0) { + diagnostics.push("args: no statically extractable arg entries"); + } + + return [...args.values()]; +} + +export function extractFlagsSchema(source: string, diagnostics: string[]): ParsedFlag[] { + const block = extractStaticObjectBlock(source, /static\s+flags\s*=\s*{/m); + if (!block) { + diagnostics.push("flags: static flags block not found"); + return []; + } + + const entries = splitTopLevelEntries(block); + const flags = new Map(); + + for (const entry of entries) { + const factorySpreadFlags = parseFlagSpreadFactory(entry); + if (factorySpreadFlags.length > 0) { + for (const flag of factorySpreadFlags) { + flags.set(flag.name, flag); + } + continue; + } + + const spread = entry.match(/^\.\.\.\s*([A-Za-z0-9_$.]+)\s*$/); + if (spread) { + const spreadName = spread[1].split(".").at(-1) ?? spread[1]; + const shared = SHARED_FLAG_SCHEMAS[spreadName]; + if (shared) { + for (const flag of shared) { + flags.set(flag.name, flag); + } + continue; + } + + const localFlags = parseLocalFlagObject(source, spreadName, diagnostics); + if (localFlags.length > 0) { + for (const localFlag of localFlags) { + flags.set(localFlag.name, localFlag); + } + continue; + } + + diagnostics.push(`flags: unresolved spread '${spread[1]}'`); + continue; + } + + const split = splitObjectEntry(entry); + if (!split) { + diagnostics.push(`flags: could not parse entry '${entry.trim().slice(0, 80)}'`); + continue; + } + + const parsedFlag = parseFlagDefinition(split.key, split.expression); + if (!parsedFlag) { + diagnostics.push(`flags: unresolved factory for '${split.key}'`); + continue; + } + + flags.set(parsedFlag.name, parsedFlag); + } + + return [...flags.values()]; +} + +export function detectInteractiveSignals(source: string): InteractiveSignal[] { + const signals: InteractiveSignal[] = []; + const withSignal = (signal: InteractiveSignal, regex: RegExp) => { + if (regex.test(source)) { + signals.push(signal); + } + }; + + withSignal("addInput", /\.addInput\s*\(/); + withSignal("addSelect", /\.addSelect\s*\(/); + withSignal("addConfirmation", /\.addConfirmation\s*\(/); + withSignal("editorFallback", /editor|openEditor/i); + withSignal("stdinBranch", /stdin|process\.stdin/i); + + return [...new Set(signals)]; +} + +function parseFlagSpreadFactory(entry: string): ParsedFlag[] { + const expireFlagsMatch = entry.match( + /^\.\.\.\s*expireFlags\(\s*[^,]+,\s*(true|false)\s*\)\s*$/, + ); + + if (expireFlagsMatch) { + return [ + { + name: "expires", + required: expireFlagsMatch[1] === "true", + type: "string", + takesValue: true, + }, + ]; + } + + return []; +} + +function parseLocalArgObject( + source: string, + objectName: string, + diagnostics: string[], +): ParsedArg[] { + const block = extractConstObjectBlock(source, objectName); + if (!block) { + return []; + } + + const entries = splitTopLevelEntries(block); + const args = new Map(); + + for (const entry of entries) { + const spread = entry.match(/^\.\.\.\s*([A-Za-z0-9_$.]+)\s*$/); + if (spread) { + const spreadName = spread[1].split(".").at(-1) ?? spread[1]; + const shared = SHARED_ARG_SCHEMAS[spreadName]; + if (shared) { + for (const sharedArg of shared) { + args.set(sharedArg.name, sharedArg); + } + } + continue; + } + + const split = splitObjectEntry(entry); + if (!split) { + continue; + } + + const config = extractFirstObjectLiteral(split.expression); + const required = readBooleanProp(config, "required") ?? false; + const defaultValue = readStringProp(config, "default"); + + args.set(split.key, { + name: split.key, + required, + defaultValue, + placeholderKind: inferPlaceholderKind(split.key), + }); + } + + if (args.size === 0) { + diagnostics.push(`args: local spread '${objectName}' contained no extractable args`); + } + + return [...args.values()]; +} + +function parseLocalFlagObject( + source: string, + objectName: string, + diagnostics: string[], +): ParsedFlag[] { + const block = extractConstObjectBlock(source, objectName); + if (!block) { + return []; + } + + const entries = splitTopLevelEntries(block); + const flags = new Map(); + + for (const entry of entries) { + const spread = entry.match(/^\.\.\.\s*([A-Za-z0-9_$.]+)\s*$/); + if (spread) { + continue; + } + + const split = splitObjectEntry(entry); + if (!split) { + continue; + } + + const parsed = parseFlagDefinition(split.key, split.expression); + if (parsed) { + flags.set(parsed.name, parsed); + } + } + + if (flags.size === 0) { + diagnostics.push(`flags: local spread '${objectName}' contained no extractable flags`); + } + + return [...flags.values()]; +} + +function parseExampleCommandToArgs( + example: string, + commandId: string, +): string[] | undefined { + const rendered = example + .replace(/<%=\s*config\.bin\s*%>/g, "mw") + .replace(/<%=\s*command\.id\s*%>/g, commandId); + + const commandLine = rendered + .split("\n") + .map((line) => line.trim()) + .filter((line) => line.length > 0 && !line.startsWith("#")) + .find((line) => line.includes(commandId) || line.startsWith("mw ")); + + if (!commandLine) { + return undefined; + } + + const tokens = shellTokenize(commandLine.replace(/^\$\s*/, "")); + const normalizedTokens = tokens.filter((token) => token !== "mw"); + const commandTokens = commandId.split(" "); + const commandStart = findTokenSequenceIndex(normalizedTokens, commandTokens); + + if (commandStart === -1) { + return undefined; + } + + const rawInvocation = normalizedTokens.slice(commandStart); + return rawInvocation.map((token) => { + if (token.startsWith("<") && token.endsWith(">")) { + return makeTypedPlaceholderValue(token.slice(1, -1), "string", undefined); + } + + return token; + }); +} + +function findTokenSequenceIndex(haystack: string[], needle: string[]): number { + if (needle.length === 0 || haystack.length < needle.length) { + return -1; + } + + for (let i = 0; i <= haystack.length - needle.length; i += 1) { + const segment = haystack.slice(i, i + needle.length); + if (segment.every((token, idx) => token === needle[idx])) { + return i; + } + } + + return -1; +} + +function decodeStringLiteral(value: string): string | undefined { + const quote = value[0]; + if ((quote !== '"' && quote !== "'" && quote !== "`") || value.length < 2) { + return undefined; + } + + const inner = value.slice(1, -1); + return inner + .replace(/\\n/g, "\n") + .replace(/\\t/g, "\t") + .replace(/\\"/g, '"') + .replace(/\\'/g, "'") + .replace(/\\`/g, "`") + .replace(/\\\\/g, "\\"); +} + +function shellTokenize(value: string): string[] { + const matches = value.match(/"([^"\\]|\\.)*"|'([^'\\]|\\.)*'|\S+/g); + if (!matches) { + return []; + } + + return matches.map((token) => { + if ( + (token.startsWith('"') && token.endsWith('"')) || + (token.startsWith("'") && token.endsWith("'")) + ) { + return token.slice(1, -1); + } + + return token; + }); +} + +function makeTypedPlaceholderValue( + name: string, + type: FlagValueType, + options: string[] | undefined, +): string { + if (options && options.length > 0) { + return options[0]; + } + + const normalized = name + .replace(/[<>[\]]/g, "") + .replace(/[^A-Za-z0-9-]+/g, "-") + .replace(/^-+/, "") + .replace(/-+$/, "") + .toLowerCase(); + + if (normalized.includes("uuid") || normalized.endsWith("id") || normalized.includes("-id")) { + return "00000000-0000-4000-8000-000000000000"; + } + + if (normalized.includes("email")) { + return "integration@example.com"; + } + + if (normalized.includes("url") || normalized.includes("uri")) { + return "https://example.com"; + } + + if ( + normalized.includes("duration") || + normalized.includes("ttl") || + normalized.includes("interval") + ) { + return "1h"; + } + + if (normalized.includes("directory") || normalized.includes("path")) { + return "/tmp/mw-integration"; + } + + if (type === "file") { + return "/tmp/mw-integration.file"; + } + + if (type === "directory") { + return "/tmp/mw-integration"; + } + + if (normalized.includes("password")) { + return "integration-password"; + } + + if (normalized.includes("port")) { + return "12345"; + } + + return normalized.length > 0 ? `example-${normalized}` : "example-value"; +} + +function extractStaticObjectBlock(source: string, anchor: RegExp): string | undefined { + const match = anchor.exec(source); + if (!match) { + return undefined; + } + + const start = source.indexOf("{", match.index); + if (start === -1) { + return undefined; + } + + const end = findMatchingBraceIndex(source, start); + if (end === -1) { + return undefined; + } + + return source.slice(start + 1, end); +} + +function extractConstObjectBlock(source: string, objectName: string): string | undefined { + const anchor = new RegExp(`(?:const|let|var)\\s+${escapeRegExp(objectName)}\\s*=\\s*{`, "m"); + const match = anchor.exec(source); + if (!match) { + return undefined; + } + + const start = source.indexOf("{", match.index); + if (start === -1) { + return undefined; + } + + const end = findMatchingBraceIndex(source, start); + if (end === -1) { + return undefined; + } + + return source.slice(start + 1, end); +} + +function findMatchingBraceIndex(input: string, startIndex: number): number { + let depth = 0; + let quote: "'" | '"' | "`" | undefined; + let escaped = false; + + for (let i = startIndex; i < input.length; i += 1) { + const char = input[i]; + + if (quote) { + if (escaped) { + escaped = false; + continue; + } + + if (char === "\\") { + escaped = true; + continue; + } + + if (char === quote) { + quote = undefined; + } + + continue; + } + + if (char === '"' || char === "'" || char === "`") { + quote = char; + continue; + } + + if (char === "{") { + depth += 1; + continue; + } + + if (char === "}") { + depth -= 1; + if (depth === 0) { + return i; + } + } + } + + return -1; +} + +function splitTopLevelEntries(input: string): string[] { + const entries: string[] = []; + let start = 0; + let braceDepth = 0; + let parenDepth = 0; + let bracketDepth = 0; + let quote: "'" | '"' | "`" | undefined; + let escaped = false; + + for (let i = 0; i < input.length; i += 1) { + const char = input[i]; + + if (quote) { + if (escaped) { + escaped = false; + continue; + } + + if (char === "\\") { + escaped = true; + continue; + } + + if (char === quote) { + quote = undefined; + } + + continue; + } + + if (char === '"' || char === "'" || char === "`") { + quote = char; + continue; + } + + if (char === "{") { + braceDepth += 1; + continue; + } + + if (char === "}") { + braceDepth -= 1; + continue; + } + + if (char === "(") { + parenDepth += 1; + continue; + } + + if (char === ")") { + parenDepth -= 1; + continue; + } + + if (char === "[") { + bracketDepth += 1; + continue; + } + + if (char === "]") { + bracketDepth -= 1; + continue; + } + + if (char === "," && braceDepth === 0 && parenDepth === 0 && bracketDepth === 0) { + const part = input.slice(start, i).trim(); + if (part.length > 0) { + entries.push(part); + } + start = i + 1; + } + } + + const tail = input.slice(start).trim(); + if (tail.length > 0) { + entries.push(tail); + } + + return entries; +} + +function splitObjectEntry(entry: string): { key: string; expression: string } | undefined { + let quote: "'" | '"' | "`" | undefined; + let escaped = false; + let braceDepth = 0; + let parenDepth = 0; + let bracketDepth = 0; + + for (let i = 0; i < entry.length; i += 1) { + const char = entry[i]; + + if (quote) { + if (escaped) { + escaped = false; + continue; + } + + if (char === "\\") { + escaped = true; + continue; + } + + if (char === quote) { + quote = undefined; + } + + continue; + } + + if (char === '"' || char === "'" || char === "`") { + quote = char; + continue; + } + + if (char === "{") { + braceDepth += 1; + continue; + } + + if (char === "}") { + braceDepth -= 1; + continue; + } + + if (char === "(") { + parenDepth += 1; + continue; + } + + if (char === ")") { + parenDepth -= 1; + continue; + } + + if (char === "[") { + bracketDepth += 1; + continue; + } + + if (char === "]") { + bracketDepth -= 1; + continue; + } + + if (char === ":" && braceDepth === 0 && parenDepth === 0 && bracketDepth === 0) { + const keyRaw = entry.slice(0, i).trim(); + const expression = entry.slice(i + 1).trim(); + const key = keyRaw.replace(/^['"]/, "").replace(/['"]$/, ""); + if (!key || !expression) { + return undefined; + } + + return { key, expression }; + } + } + + return undefined; +} + +function parseFlagDefinition(name: string, expression: string): ParsedFlag | undefined { + const named = resolveNamedFlagSchemaFromExpression(expression); + if (named) { + return { + name, + ...named, + }; + } + + const type = detectFlagType(expression); + if (!type) { + return undefined; + } + + const config = extractFirstObjectLiteral(expression); + const required = readBooleanProp(config, "required") ?? false; + const multiple = readBooleanProp(config, "multiple") ?? false; + const options = readStringArrayProp(config, "options"); + const exactlyOne = readStringArrayProp(config, "exactlyOne"); + const exclusive = readStringArrayProp(config, "exclusive"); + const dependsOn = readStringArrayProp(config, "dependsOn"); + const defaultValue = readLiteralStringProp(config, "default"); + + return { + name, + required, + type, + takesValue: type !== "boolean", + multiple, + options, + defaultValue, + exactlyOne, + exclusive, + dependsOn, + }; +} + +function detectFlagType(expression: string): FlagValueType | undefined { + if (/Flags\.boolean\s*\(/.test(expression)) { + return "boolean"; + } + + if (/Flags\.integer\s*\(/.test(expression)) { + return "integer"; + } + + if (/Flags\.file\s*\(/.test(expression)) { + return "file"; + } + + if (/Flags\.directory\s*\(/.test(expression)) { + return "directory"; + } + + if (/Flags\.url\s*\(/.test(expression)) { + return "url"; + } + + if (/Flags\.(string|custom)\s*\(/.test(expression)) { + return "string"; + } + + if (/\.absoluteFlag\s*\(/.test(expression) || /\.relativeFlag\s*\(/.test(expression)) { + return "string"; + } + + // Fallback for wrapped/custom flag factories, e.g. `flagDefinitions.name({ required: true })`. + if (/^[A-Za-z0-9_.$\[\]"'-]+\s*\(/.test(expression)) { + return "string"; + } + + return undefined; +} + +function resolveNamedFlagSchemaFromExpression( + expression: string, +): Omit | undefined { + const normalized = expression.trim().replace(/\(\s*\)$/, ""); + const candidate = normalized.split(".").at(-1) ?? normalized; + return NAMED_FLAG_SCHEMAS[candidate]; +} + +function extractFirstObjectLiteral(expression: string): string { + const start = expression.indexOf("{"); + if (start === -1) { + return ""; + } + + const end = findMatchingBraceIndex(expression, start); + if (end === -1) { + return ""; + } + + return expression.slice(start, end + 1); +} + +function readBooleanProp(config: string, key: string): boolean | undefined { + if (!config) { + return undefined; + } + + const regex = new RegExp(`${escapeRegExp(key)}\\s*:\\s*(true|false)`); + const match = config.match(regex); + if (!match) { + return undefined; + } + + return match[1] === "true"; +} + +function readStringProp(config: string, key: string): string | undefined { + if (!config) { + return undefined; + } + + const regex = new RegExp(`${escapeRegExp(key)}\\s*:\\s*(["'])(.*?)\\1`, "s"); + const match = config.match(regex); + return match?.[2]; +} + +function readLiteralStringProp(config: string, key: string): string | undefined { + const stringValue = readStringProp(config, key); + if (stringValue !== undefined) { + return stringValue; + } + + const boolMatch = config.match(new RegExp(`${escapeRegExp(key)}\\s*:\\s*(true|false)`)); + if (boolMatch) { + return boolMatch[1]; + } + + const numberMatch = config.match(new RegExp(`${escapeRegExp(key)}\\s*:\\s*([0-9]+(?:\\.[0-9]+)?)`)); + if (numberMatch) { + return numberMatch[1]; + } + + return undefined; +} + +function readStringArrayProp(config: string, key: string): string[] | undefined { + if (!config) { + return undefined; + } + + const regex = new RegExp(`${escapeRegExp(key)}\\s*:\\s*\\[([^\\]]*)\\]`, "s"); + const match = config.match(regex); + if (!match) { + return undefined; + } + + return match[1] + .split(",") + .map((entry) => entry.trim().replace(/^['"]/, "").replace(/['"]$/, "")) + .filter((entry) => entry.length > 0); +} + +function inferPlaceholderKind(name: string): PlaceholderKind { + const normalized = name.toLowerCase(); + if (normalized.includes("uuid") || normalized.endsWith("id") || normalized.includes("-id")) { + return "uuid"; + } + if (normalized.includes("email")) { + return "email"; + } + if (normalized.includes("url") || normalized.includes("uri")) { + return "url"; + } + if (normalized.includes("duration") || normalized.includes("ttl") || normalized.includes("interval")) { + return "duration"; + } + if (normalized.includes("directory")) { + return "directory"; + } + if (normalized.includes("file")) { + return "file"; + } + if (normalized.includes("password") || normalized.includes("passphrase") || normalized.includes("token")) { + return "password"; + } + if (normalized.includes("port")) { + return "port"; + } + return "generic"; +} + +function parseInvocationParts(args: string[]): { + positionalValues: string[]; + flagValues: Map; +} { + const positionalValues: string[] = []; + const flagValues = new Map(); + + for (let i = 0; i < args.length; i += 1) { + const token = args[i]; + if (!token.startsWith("--")) { + positionalValues.push(token); + continue; + } + + const withoutPrefix = token.slice(2); + const eqIndex = withoutPrefix.indexOf("="); + let name = withoutPrefix; + let value: string | undefined; + + if (eqIndex >= 0) { + name = withoutPrefix.slice(0, eqIndex); + value = withoutPrefix.slice(eqIndex + 1); + } else { + const nextToken = args[i + 1]; + if (nextToken && !nextToken.startsWith("--")) { + value = nextToken; + i += 1; + } + } + + const values = flagValues.get(name) ?? []; + if (value === undefined) { + values.push("true"); + } else { + values.push(value); + } + flagValues.set(name, values); + } + + return { positionalValues, flagValues }; +} + +function escapeRegExp(value: string): string { + return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} diff --git a/src/test/integration/command-discovery/synthesis.ts b/src/test/integration/command-discovery/synthesis.ts new file mode 100644 index 000000000..879d2b0a3 --- /dev/null +++ b/src/test/integration/command-discovery/synthesis.ts @@ -0,0 +1,686 @@ +import { loadInvocationProfiles } from "../config/loader.js"; +import { DEFAULT_UUID } from "./config.js"; +import type { + ExampleCandidate, + InteractiveSignal, + InvocationProfile, + ParsedArg, + ParsedFlag, + PlaceholderKind, + ResolvedFlagValue, + SynthesizedInvocation, + ValueSource, +} from "./types.js"; + +export function resolveProfiles(commandId: string): InvocationProfile[] { + const invocationProfiles = loadInvocationProfiles(); + return invocationProfiles.filter((profile) => { + if (profile.match.exact && profile.match.exact === commandId) { + return true; + } + + if (profile.match.prefix && commandId.startsWith(`${profile.match.prefix} `)) { + return true; + } + + return false; + }); +} + +export function synthesizeInvocation(input: { + commandId: string; + commandTokens: string[]; + parsedArgs: ParsedArg[]; + parsedFlags: ParsedFlag[]; + interactiveSignals: InteractiveSignal[]; + exampleCandidate: ExampleCandidate | undefined; + profiles: InvocationProfile[]; +}): SynthesizedInvocation { + const { + commandId, + commandTokens, + parsedArgs, + parsedFlags, + interactiveSignals, + exampleCandidate, + profiles, + } = input; + + const staleExampleReasons: string[] = []; + let validatedExample: ExampleCandidate | undefined; + if (exampleCandidate) { + const validationErrors = validateExampleCandidate(exampleCandidate, parsedArgs, parsedFlags); + if (validationErrors.length === 0) { + validatedExample = exampleCandidate; + } else { + staleExampleReasons.push(...validationErrors.map((reason) => `stale-example: ${reason}`)); + } + } + + const selectedFlags = new Map(); + let strongestSource: ValueSource = "heuristic"; + + const positionalValues = parsedArgs.map((arg, index) => { + const profileValue = getProfileArgValue(profiles, arg.name); + if (profileValue !== undefined) { + strongestSource = selectStrongerSource(strongestSource, "profile"); + return profileValue; + } + + const exampleValue = validatedExample?.positionalValues[index]; + if (exampleValue !== undefined) { + strongestSource = selectStrongerSource(strongestSource, "example"); + return exampleValue; + } + + if (arg.defaultValue !== undefined) { + return arg.defaultValue; + } + + return defaultValueForPlaceholderKind(arg.placeholderKind, arg.name); + }); + + for (const flag of parsedFlags) { + if (!flag.required) { + continue; + } + + const fromProfile = getProfileFlagValue(profiles, flag.name); + if (fromProfile !== undefined) { + setFlagValue(selectedFlags, flag.name, normalizeFlagValue(fromProfile), "profile"); + strongestSource = selectStrongerSource(strongestSource, "profile"); + continue; + } + + const fromExample = validatedExample?.flagValues.get(flag.name); + if (fromExample && fromExample.length > 0) { + setFlagValue(selectedFlags, flag.name, fromExample, "example"); + strongestSource = selectStrongerSource(strongestSource, "example"); + continue; + } + + const heuristic = buildHeuristicFlagValue(flag); + setFlagValue(selectedFlags, flag.name, heuristic, "heuristic"); + } + + resolveExactlyOneGroups(commandId, parsedFlags, selectedFlags, profiles, interactiveSignals); + resolveDependencies(parsedFlags, selectedFlags, profiles); + resolveExclusiveGroups(parsedFlags, selectedFlags); + applyProfileOverrides(parsedFlags, selectedFlags, profiles); + const interactiveDecision = decideInteractivePolicy( + commandId, + parsedFlags, + selectedFlags, + interactiveSignals, + profiles, + ); + + const invocation = renderInvocation(commandTokens, positionalValues, parsedFlags, selectedFlags); + return { + args: invocation, + argumentSource: strongestSource, + interactiveDecision, + staleExample: staleExampleReasons.length > 0, + staleExampleReasons, + }; +} + +function validateExampleCandidate( + example: ExampleCandidate, + argsSchema: ParsedArg[], + flagSchema: ParsedFlag[], +): string[] { + const errors: string[] = []; + const flagNames = new Set(flagSchema.map((flag) => flag.name)); + + for (const flagName of example.flagValues.keys()) { + if (!flagNames.has(flagName)) { + errors.push(`unknown flag --${flagName}`); + } + } + + const requiredArgsCount = argsSchema.filter((arg) => arg.required).length; + if (example.positionalValues.length < requiredArgsCount) { + errors.push("missing required positional arguments"); + } + + for (const flag of flagSchema) { + if (flag.required && !example.flagValues.has(flag.name)) { + errors.push(`missing required flag --${flag.name}`); + } + + if (flag.dependsOn && example.flagValues.has(flag.name)) { + for (const dependency of flag.dependsOn) { + if (!example.flagValues.has(dependency)) { + errors.push(`--${flag.name} depends on --${dependency}`); + } + } + } + + if (flag.exclusive) { + for (const conflicting of flag.exclusive) { + if (example.flagValues.has(flag.name) && example.flagValues.has(conflicting)) { + errors.push(`--${flag.name} is exclusive with --${conflicting}`); + } + } + } + } + + for (const group of collectExactlyOneGroups(flagSchema)) { + const count = group.members.filter((name) => example.flagValues.has(name)).length; + if (count !== 1) { + errors.push(`exactly one of [${group.members.join(", ")}] must be set`); + } + } + + return errors; +} + +function collectExactlyOneGroups(flagSchema: ParsedFlag[]): Array<{ key: string; members: string[] }> { + const groups = new Map(); + + for (const flag of flagSchema) { + if (!flag.exactlyOne || flag.exactlyOne.length < 2) { + continue; + } + + const members = [...new Set(flag.exactlyOne)].sort(); + const key = members.join("|"); + groups.set(key, members); + } + + return [...groups.entries()].map(([key, members]) => ({ key, members })); +} + +function resolveExactlyOneGroups( + commandId: string, + flagSchema: ParsedFlag[], + selectedFlags: Map, + profiles: InvocationProfile[], + interactiveSignals: InteractiveSignal[], +): void { + for (const group of collectExactlyOneGroups(flagSchema)) { + const selectedMembers = group.members.filter((member) => selectedFlags.has(member)); + + if (selectedMembers.length === 1) { + continue; + } + + const preferredByProfile = getProfileExactlyOneChoice(profiles, group.key); + if (preferredByProfile && group.members.includes(preferredByProfile)) { + selectedFlags.set(preferredByProfile, { + values: [makeTypedPlaceholderValue(preferredByProfile, "string", undefined)], + source: "profile", + }); + for (const member of group.members) { + if (member !== preferredByProfile) { + selectedFlags.delete(member); + } + } + continue; + } + + const chosen = chooseExactlyOneMember( + commandId, + group.members, + flagSchema, + interactiveSignals, + ); + + const existing = selectedFlags.get(chosen); + if (!existing) { + selectedFlags.set(chosen, { + values: [makeTypedPlaceholderValue(chosen, "string", undefined)], + source: "heuristic", + }); + } + + for (const member of group.members) { + if (member !== chosen) { + selectedFlags.delete(member); + } + } + } +} + +function chooseExactlyOneMember( + commandId: string, + members: string[], + flagSchema: ParsedFlag[], + interactiveSignals: InteractiveSignal[], +): string { + if (members.includes("project-id")) { + return "project-id"; + } + + const scored = members.map((member) => { + const closure = dependencyClosureSize(member, flagSchema); + const nonInteractiveBonus = scoreNonInteractiveMember(member, interactiveSignals); + return { + member, + score: closure - nonInteractiveBonus, + }; + }); + + scored.sort((a, b) => { + if (a.score !== b.score) { + return a.score - b.score; + } + + return a.member.localeCompare(b.member); + }); + + return scored[0]?.member ?? members[0] ?? commandId; +} + +function scoreNonInteractiveMember(member: string, interactiveSignals: InteractiveSignal[]): number { + if (member === "consent" && interactiveSignals.includes("addConfirmation")) { + return 3; + } + + if (member === "force" && interactiveSignals.includes("addConfirmation")) { + return 3; + } + + if (member === "password" && interactiveSignals.includes("addInput")) { + return 3; + } + + if (member === "override-type" && interactiveSignals.includes("addSelect")) { + return 2; + } + + return 0; +} + +function dependencyClosureSize(member: string, flagSchema: ParsedFlag[]): number { + const visited = new Set(); + + const visit = (flagName: string): void => { + if (visited.has(flagName)) { + return; + } + + visited.add(flagName); + const flag = flagSchema.find((candidate) => candidate.name === flagName); + for (const dependency of flag?.dependsOn ?? []) { + visit(dependency); + } + }; + + visit(member); + return visited.size; +} + +function resolveDependencies( + flagSchema: ParsedFlag[], + selectedFlags: Map, + profiles: InvocationProfile[], +): void { + let changed = true; + + while (changed) { + changed = false; + + for (const flag of flagSchema) { + if (!selectedFlags.has(flag.name)) { + continue; + } + + for (const dependency of flag.dependsOn ?? []) { + if (selectedFlags.has(dependency)) { + continue; + } + + const dependencySpec = flagSchema.find((candidate) => candidate.name === dependency); + const profileValue = getProfileFlagValue(profiles, dependency); + if (profileValue !== undefined) { + setFlagValue(selectedFlags, dependency, normalizeFlagValue(profileValue), "profile"); + changed = true; + continue; + } + + if (!dependencySpec) { + setFlagValue(selectedFlags, dependency, ["true"], "heuristic"); + changed = true; + continue; + } + + setFlagValue(selectedFlags, dependency, buildHeuristicFlagValue(dependencySpec), "heuristic"); + changed = true; + } + } + } +} + +function resolveExclusiveGroups( + flagSchema: ParsedFlag[], + selectedFlags: Map, +): void { + for (const flag of flagSchema) { + const selected = selectedFlags.get(flag.name); + if (!selected || !flag.exclusive) { + continue; + } + + for (const otherName of flag.exclusive) { + const other = selectedFlags.get(otherName); + if (!other) { + continue; + } + + if (compareSourcePrecedence(selected.source, other.source) >= 0) { + selectedFlags.delete(otherName); + } else { + selectedFlags.delete(flag.name); + } + } + } +} + +function applyProfileOverrides( + flagSchema: ParsedFlag[], + selectedFlags: Map, + profiles: InvocationProfile[], +): void { + for (const profile of profiles) { + for (const [flagName, profileValue] of Object.entries(profile.requiredFlagDefaults ?? {})) { + const spec = flagSchema.find((flag) => flag.name === flagName); + if (!spec) { + continue; + } + + setFlagValue(selectedFlags, flagName, normalizeFlagValue(profileValue), "profile"); + } + } +} + +function decideInteractivePolicy( + commandId: string, + flagSchema: ParsedFlag[], + selectedFlags: Map, + interactiveSignals: InteractiveSignal[], + profiles: InvocationProfile[], +): "NON_INTERACTIVE_RESOLVED" | "INTERACTIVE_REQUIRED" { + if (interactiveSignals.length === 0) { + return "NON_INTERACTIVE_RESOLVED"; + } + + const policy = profiles.find((profile) => profile.interactivePolicy)?.interactivePolicy; + if (policy === "classify") { + return "INTERACTIVE_REQUIRED"; + } + + const unresolved = resolveInteractiveSignals(commandId, flagSchema, selectedFlags, interactiveSignals); + return unresolved.length === 0 ? "NON_INTERACTIVE_RESOLVED" : "INTERACTIVE_REQUIRED"; +} + +function resolveInteractiveSignals( + commandId: string, + flagSchema: ParsedFlag[], + selectedFlags: Map, + interactiveSignals: InteractiveSignal[], +): InteractiveSignal[] { + const unresolved: InteractiveSignal[] = []; + + const hasFlag = (name: string): boolean => flagSchema.some((flag) => flag.name === name); + + for (const signal of interactiveSignals) { + if (signal === "addConfirmation") { + if (hasFlag("force")) { + setFlagValue(selectedFlags, "force", ["true"], "heuristic"); + continue; + } + + if (hasFlag("consent")) { + setFlagValue(selectedFlags, "consent", ["true"], "heuristic"); + continue; + } + + unresolved.push(signal); + continue; + } + + if (signal === "addInput") { + if (hasFlag("password")) { + setFlagValue(selectedFlags, "password", ["integration-password"], "heuristic"); + continue; + } + + if (hasFlag("user-password")) { + setFlagValue(selectedFlags, "user-password", ["integration-password"], "heuristic"); + continue; + } + + unresolved.push(signal); + continue; + } + + if (signal === "addSelect") { + if (hasFlag("override-type")) { + setFlagValue(selectedFlags, "override-type", ["auto"], "heuristic"); + continue; + } + + unresolved.push(signal); + continue; + } + + unresolved.push(signal); + } + + if (commandId === "login token") { + return [...new Set([...unresolved, "addInput"])] as InteractiveSignal[]; + } + + return [...new Set(unresolved)]; +} + +function renderInvocation( + commandTokens: string[], + positionalValues: string[], + parsedFlags: ParsedFlag[], + selectedFlags: Map, +): string[] { + const args = [...commandTokens, ...positionalValues]; + + const orderedFlags = parsedFlags + .filter((flag) => selectedFlags.has(flag.name)) + .sort((a, b) => a.name.localeCompare(b.name)); + + for (const flag of orderedFlags) { + const resolved = selectedFlags.get(flag.name); + if (!resolved) { + continue; + } + + if (!flag.takesValue) { + args.push(`--${flag.name}`); + continue; + } + + for (const value of resolved.values) { + args.push(`--${flag.name}`); + args.push(value); + } + } + + return args; +} + +function setFlagValue( + map: Map, + flagName: string, + values: string[], + source: ValueSource, +): void { + const existing = map.get(flagName); + if (!existing) { + map.set(flagName, { values, source }); + return; + } + + if (compareSourcePrecedence(source, existing.source) >= 0) { + map.set(flagName, { values, source }); + } +} + +function normalizeFlagValue(value: string | boolean): string[] { + if (typeof value === "boolean") { + return [value ? "true" : "false"]; + } + + return [value]; +} + +function buildHeuristicFlagValue(flag: ParsedFlag): string[] { + if (!flag.takesValue) { + return ["true"]; + } + + if (flag.defaultValue !== undefined) { + return [flag.defaultValue]; + } + + return [makeTypedPlaceholderValue(flag.name, flag.type, flag.options)]; +} + +function getProfileArgValue(profiles: InvocationProfile[], argName: string): string | undefined { + for (const profile of profiles) { + const value = profile.requiredArgDefaults?.[argName]; + if (value !== undefined) { + return value; + } + } + + return undefined; +} + +function getProfileFlagValue( + profiles: InvocationProfile[], + flagName: string, +): string | boolean | undefined { + for (const profile of profiles) { + const value = profile.requiredFlagDefaults?.[flagName]; + if (value !== undefined) { + return value; + } + } + + return undefined; +} + +function getProfileExactlyOneChoice( + profiles: InvocationProfile[], + groupKey: string, +): string | undefined { + for (const profile of profiles) { + const choice = profile.exactlyOneChoice?.[groupKey]; + if (choice !== undefined) { + return choice; + } + } + + return undefined; +} + +function compareSourcePrecedence(a: ValueSource, b: ValueSource): number { + const precedence: Record = { + heuristic: 1, + example: 2, + profile: 3, + }; + + return precedence[a] - precedence[b]; +} + +function selectStrongerSource(current: ValueSource, candidate: ValueSource): ValueSource { + return compareSourcePrecedence(candidate, current) >= 0 ? candidate : current; +} + +function defaultValueForPlaceholderKind(kind: PlaceholderKind, name: string): string { + if (kind === "uuid") { + return DEFAULT_UUID; + } + + if (kind === "email") { + return "integration@example.com"; + } + + if (kind === "url") { + return "https://example.com"; + } + + if (kind === "duration") { + return "1h"; + } + + if (kind === "directory") { + return "/tmp/mw-integration"; + } + + if (kind === "file") { + return "/tmp/mw-integration.file"; + } + + if (kind === "password") { + return "integration-password"; + } + + if (kind === "port") { + return "12345"; + } + + return makeTypedPlaceholderValue(name, "string", undefined); +} + +function makeTypedPlaceholderValue( + name: string, + _type: string, + options: string[] | undefined, +): string { + if (options && options.length > 0) { + return options[0]; + } + + const normalized = name + .replace(/[<>[\]]/g, "") + .replace(/[^A-Za-z0-9-]+/g, "-") + .replace(/^-+/, "") + .replace(/-+$/, "") + .toLowerCase(); + + if (normalized.includes("uuid") || normalized.endsWith("id") || normalized.includes("-id")) { + return DEFAULT_UUID; + } + + if (normalized.includes("email")) { + return "integration@example.com"; + } + + if (normalized.includes("url") || normalized.includes("uri")) { + return "https://example.com"; + } + + if ( + normalized.includes("duration") || + normalized.includes("ttl") || + normalized.includes("interval") + ) { + return "1h"; + } + + if (normalized.includes("directory") || normalized.includes("path")) { + return "/tmp/mw-integration"; + } + + if (normalized.includes("password")) { + return "integration-password"; + } + + if (normalized.includes("port")) { + return "12345"; + } + + return normalized.length > 0 ? `example-${normalized}` : "example-value"; +} diff --git a/src/test/integration/command-discovery/types.ts b/src/test/integration/command-discovery/types.ts new file mode 100644 index 000000000..ff8263fa0 --- /dev/null +++ b/src/test/integration/command-discovery/types.ts @@ -0,0 +1,107 @@ +export type FlagValueType = + | "boolean" + | "string" + | "integer" + | "file" + | "directory" + | "url" + | "custom"; + +export type ValueSource = "profile" | "example" | "heuristic"; + +export type InteractiveSignal = + | "addInput" + | "addSelect" + | "addConfirmation" + | "editorFallback" + | "stdinBranch"; + +export type PlaceholderKind = + | "uuid" + | "email" + | "url" + | "duration" + | "file" + | "directory" + | "password" + | "port" + | "generic"; + +export type ParsedArg = { + name: string; + required: boolean; + defaultValue?: string; + placeholderKind: PlaceholderKind; +}; + +export type ParsedFlag = { + name: string; + required: boolean; + type: FlagValueType; + takesValue: boolean; + options?: string[]; + multiple?: boolean; + defaultValue?: string; + exactlyOne?: string[]; + exclusive?: string[]; + dependsOn?: string[]; +}; + +export type SynthesizedInvocation = { + args: string[]; + argumentSource: ValueSource; + interactiveDecision: "NON_INTERACTIVE_RESOLVED" | "INTERACTIVE_REQUIRED"; + staleExample: boolean; + staleExampleReasons: string[]; +}; + +export type DiscoveredCommand = { + commandId: string; + sourceFile: string; + commandTokens: string[]; + parsedArgs: ParsedArg[]; + parsedFlags: ParsedFlag[]; + interactiveSignals: InteractiveSignal[]; + invocationProfilesApplied: string[]; + extractionDiagnostics: string[]; + synthesizedInvocation: SynthesizedInvocation; +}; + +export type ExampleCandidate = { + args: string[]; + positionalValues: string[]; + flagValues: Map; +}; + +export type ResolvedFlagValue = { + values: string[]; + source: ValueSource; +}; + +export type InvocationProfile = { + id: string; + match: { exact?: string; prefix?: string }; + requiredFlagDefaults?: Record; + requiredArgDefaults?: Record; + exactlyOneChoice?: Record; + interactivePolicy?: "resolve" | "classify"; + disableExampleSource?: boolean; + notes?: string; +}; + +export type WaiverCategory = + | "ARG_MISUSE" + | "INTERACTIVE_REQUIRED" + | "RESOURCE_PRECONDITION" + | "CONTRACT_SHAPE" + | "COMMAND_BUG" + | "DEPRECATED_ENDPOINT"; + +export type CommandWaiver = { + id: string; + commandId: string; + category: WaiverCategory; + reason: string; + issue?: string; + expiresOn?: string; +}; diff --git a/src/test/integration/command.ts b/src/test/integration/command.ts new file mode 100644 index 000000000..2d0edb1bb --- /dev/null +++ b/src/test/integration/command.ts @@ -0,0 +1,127 @@ +import { spawn } from "node:child_process"; + +export type DevCommandResult = { + stdout: string; + stderr: string; + exitCode: number | null; + signal: NodeJS.Signals | null; + error?: Error; + timedOut?: boolean; +}; + +export type RunDevCommandOptions = { + cwd?: string; + env?: NodeJS.ProcessEnv; + timeoutMs?: number; +}; + +export async function runDevCommand( + args: string[], + options: RunDevCommandOptions = {}, +): Promise { + return await new Promise((resolve) => { + const timeoutMs = options.timeoutMs ?? 30_000; + + const child = spawn( + "yarn", + [ + "node", + "--import", + "tsx", + "--no-warnings=ExperimentalWarning", + "./bin/dev.js", + ...args, + ], + { + cwd: options.cwd ?? process.cwd(), + env: options.env ?? process.env, + stdio: ["ignore", "pipe", "pipe"], + }, + ); + + let stdout = ""; + let stderr = ""; + let settled = false; + let didTimeOut = false; + + const finish = (result: DevCommandResult): void => { + if (settled) { + return; + } + + settled = true; + if (timeoutHandle) { + clearTimeout(timeoutHandle); + } + if (forceKillHandle) { + clearTimeout(forceKillHandle); + } + + resolve(result); + }; + + let forceKillHandle: NodeJS.Timeout | undefined; + const timeoutHandle: NodeJS.Timeout | undefined = + timeoutMs > 0 + ? setTimeout(() => { + didTimeOut = true; + child.kill("SIGTERM"); + + // Give graceful termination a short window before hard-killing. + forceKillHandle = setTimeout(() => { + child.kill("SIGKILL"); + }, 2_000); + forceKillHandle.unref?.(); + }, timeoutMs) + : undefined; + + timeoutHandle?.unref?.(); + + child.stdout.on("data", (chunk) => { + stdout += chunk.toString(); + }); + + child.stderr.on("data", (chunk) => { + stderr += chunk.toString(); + }); + + child.on("error", (error) => { + finish({ + stdout, + stderr, + exitCode: null, + signal: null, + timedOut: didTimeOut, + error, + }); + }); + + child.on("close", (exitCode, signal) => { + if (didTimeOut) { + finish({ + stdout, + stderr, + exitCode, + signal, + timedOut: true, + error: new Error(`dev.js subprocess timed out after ${timeoutMs}ms`), + }); + return; + } + + if (exitCode === 0) { + finish({ stdout, stderr, exitCode, signal, timedOut: false }); + return; + } + + finish({ + stdout, + stderr, + exitCode, + signal, + timedOut: false, + error: new Error(`dev.js subprocess exited with code ${exitCode}`), + }); + }); + }); +} \ No newline at end of file diff --git a/src/test/integration/config/command-classifications.json b/src/test/integration/config/command-classifications.json new file mode 100644 index 000000000..5f2fb44e3 --- /dev/null +++ b/src/test/integration/config/command-classifications.json @@ -0,0 +1,345 @@ +{ + "schemaVersion": 1, + "generatedAt": "2026-08-03T07:37:35.513Z", + "source": { + "kind": "run-all-summary" + }, + "statistics": { + "successful": 119, + "failed": 0, + "waivedSkipped": 66, + "total": 185 + }, + "entries": [ + { + "commandId": "app create node", + "category": "CONTRACT_SHAPE", + "source": "waiver" + }, + { + "commandId": "app create php", + "category": "CONTRACT_SHAPE", + "source": "waiver" + }, + { + "commandId": "app create php-worker", + "category": "CONTRACT_SHAPE", + "source": "waiver" + }, + { + "commandId": "app create python", + "category": "CONTRACT_SHAPE", + "source": "waiver" + }, + { + "commandId": "app create static", + "category": "CONTRACT_SHAPE", + "source": "waiver" + }, + { + "commandId": "app database link", + "category": "DEPRECATED_ENDPOINT", + "source": "waiver" + }, + { + "commandId": "app database replace", + "category": "DEPRECATED_ENDPOINT", + "source": "waiver" + }, + { + "commandId": "app dependency update", + "category": "COMMAND_BUG", + "source": "waiver" + }, + { + "commandId": "app dependency versions", + "category": "RESOURCE_PRECONDITION", + "source": "waiver" + }, + { + "commandId": "app download", + "category": "CONTRACT_SHAPE", + "source": "waiver" + }, + { + "commandId": "app exec", + "category": "CONTRACT_SHAPE", + "source": "waiver" + }, + { + "commandId": "app get", + "category": "CONTRACT_SHAPE", + "source": "waiver" + }, + { + "commandId": "app install contao", + "category": "CONTRACT_SHAPE", + "source": "waiver" + }, + { + "commandId": "app install joomla", + "category": "CONTRACT_SHAPE", + "source": "waiver" + }, + { + "commandId": "app install matomo", + "category": "CONTRACT_SHAPE", + "source": "waiver" + }, + { + "commandId": "app install nextcloud", + "category": "CONTRACT_SHAPE", + "source": "waiver" + }, + { + "commandId": "app install shopware5", + "category": "CONTRACT_SHAPE", + "source": "waiver" + }, + { + "commandId": "app install shopware6", + "category": "CONTRACT_SHAPE", + "source": "waiver" + }, + { + "commandId": "app install typo3", + "category": "CONTRACT_SHAPE", + "source": "waiver" + }, + { + "commandId": "app install wordpress", + "category": "CONTRACT_SHAPE", + "source": "waiver" + }, + { + "commandId": "app list-upgrade-candidates", + "category": "COMMAND_BUG", + "source": "waiver" + }, + { + "commandId": "app open", + "category": "COMMAND_BUG", + "source": "waiver" + }, + { + "commandId": "app ssh", + "category": "CONTRACT_SHAPE", + "source": "waiver" + }, + { + "commandId": "app upgrade", + "category": "INTERACTIVE_REQUIRED", + "source": "waiver" + }, + { + "commandId": "app upload", + "category": "RESOURCE_PRECONDITION", + "source": "waiver" + }, + { + "commandId": "app version-info", + "category": "RESOURCE_PRECONDITION", + "source": "waiver" + }, + { + "commandId": "app versions", + "category": "COMMAND_BUG", + "source": "waiver" + }, + { + "commandId": "backup download", + "category": "COMMAND_BUG", + "source": "waiver" + }, + { + "commandId": "container cp", + "category": "RESOURCE_PRECONDITION", + "source": "waiver" + }, + { + "commandId": "container delete", + "category": "RESOURCE_PRECONDITION", + "source": "waiver" + }, + { + "commandId": "container exec", + "category": "RESOURCE_PRECONDITION", + "source": "waiver" + }, + { + "commandId": "container logs", + "category": "INTERACTIVE_REQUIRED", + "source": "waiver" + }, + { + "commandId": "container port-forward", + "category": "RESOURCE_PRECONDITION", + "source": "waiver" + }, + { + "commandId": "container recreate", + "category": "RESOURCE_PRECONDITION", + "source": "waiver" + }, + { + "commandId": "container restart", + "category": "RESOURCE_PRECONDITION", + "source": "waiver" + }, + { + "commandId": "container run", + "category": "RESOURCE_PRECONDITION", + "source": "waiver" + }, + { + "commandId": "container ssh", + "category": "RESOURCE_PRECONDITION", + "source": "waiver" + }, + { + "commandId": "container start", + "category": "RESOURCE_PRECONDITION", + "source": "waiver" + }, + { + "commandId": "container stop", + "category": "RESOURCE_PRECONDITION", + "source": "waiver" + }, + { + "commandId": "container update", + "category": "RESOURCE_PRECONDITION", + "source": "waiver" + }, + { + "commandId": "conversation create", + "category": "COMMAND_BUG", + "source": "waiver" + }, + { + "commandId": "conversation reply", + "category": "COMMAND_BUG", + "source": "waiver" + }, + { + "commandId": "cronjob execution logs", + "category": "INTERACTIVE_REQUIRED", + "source": "waiver" + }, + { + "commandId": "database mysql dump", + "category": "RESOURCE_PRECONDITION", + "source": "waiver" + }, + { + "commandId": "database mysql import", + "category": "INTERACTIVE_REQUIRED", + "source": "waiver" + }, + { + "commandId": "database mysql phpmyadmin", + "category": "RESOURCE_PRECONDITION", + "source": "waiver" + }, + { + "commandId": "database mysql shell", + "category": "INTERACTIVE_REQUIRED", + "source": "waiver" + }, + { + "commandId": "database mysql upgrade", + "category": "INTERACTIVE_REQUIRED", + "source": "waiver" + }, + { + "commandId": "database mysql user delete", + "category": "RESOURCE_PRECONDITION", + "source": "waiver" + }, + { + "commandId": "ddev init", + "category": "INTERACTIVE_REQUIRED", + "source": "waiver" + }, + { + "commandId": "ddev render-config", + "category": "CONTRACT_SHAPE", + "source": "waiver" + }, + { + "commandId": "domain dnszone get", + "category": "RESOURCE_PRECONDITION", + "source": "waiver" + }, + { + "commandId": "domain dnszone update", + "category": "RESOURCE_PRECONDITION", + "source": "waiver" + }, + { + "commandId": "domain get", + "category": "RESOURCE_PRECONDITION", + "source": "waiver" + }, + { + "commandId": "experimental deploy", + "category": "RESOURCE_PRECONDITION", + "source": "waiver" + }, + { + "commandId": "login token", + "category": "INTERACTIVE_REQUIRED", + "source": "waiver" + }, + { + "commandId": "mail address update", + "category": "RESOURCE_PRECONDITION", + "source": "waiver" + }, + { + "commandId": "org delete", + "category": "RESOURCE_PRECONDITION", + "source": "waiver" + }, + { + "commandId": "sftp-user create", + "category": "RESOURCE_PRECONDITION", + "source": "waiver" + }, + { + "commandId": "ssh-user create", + "category": "RESOURCE_PRECONDITION", + "source": "waiver" + }, + { + "commandId": "stack delete", + "category": "COMMAND_BUG", + "source": "waiver" + }, + { + "commandId": "stack deploy", + "category": "INTERACTIVE_REQUIRED", + "source": "waiver" + }, + { + "commandId": "stack set-update-schedule", + "category": "DEPRECATED_ENDPOINT", + "source": "waiver" + }, + { + "commandId": "stack unset-update-schedule", + "category": "DEPRECATED_ENDPOINT", + "source": "waiver" + }, + { + "commandId": "user ssh-key create", + "category": "INTERACTIVE_REQUIRED", + "source": "waiver" + }, + { + "commandId": "volume delete", + "category": "RESOURCE_PRECONDITION", + "source": "waiver" + } + ] +} diff --git a/src/test/integration/config/command-waivers.json b/src/test/integration/config/command-waivers.json new file mode 100644 index 000000000..7905e56bb --- /dev/null +++ b/src/test/integration/config/command-waivers.json @@ -0,0 +1,464 @@ +[ + { + "id": "interactive-app-upgrade", + "commandId": "app upgrade", + "category": "INTERACTIVE_REQUIRED", + "reason": "Upgrade target selection is interactive and has no stable non-interactive override yet.", + "issue": "defer-interactive-support" + }, + { + "id": "interactive-container-logs", + "commandId": "container logs", + "category": "INTERACTIVE_REQUIRED", + "reason": "Log follow behavior depends on interactive terminal controls in current implementation.", + "issue": "defer-interactive-support" + }, + { + "id": "interactive-cronjob-execution-logs", + "commandId": "cronjob execution logs", + "category": "INTERACTIVE_REQUIRED", + "reason": "Execution log access path prompts or expects interactive I/O in this test setup.", + "issue": "defer-interactive-support" + }, + { + "id": "interactive-database-mysql-import", + "commandId": "database mysql import", + "category": "INTERACTIVE_REQUIRED", + "reason": "Import flow expects interactive input or file prompt handling not available in the simple renderer.", + "issue": "defer-interactive-support" + }, + { + "id": "interactive-database-mysql-shell", + "commandId": "database mysql shell", + "category": "INTERACTIVE_REQUIRED", + "reason": "MySQL shell requires password prompt interaction and cannot run headless yet.", + "issue": "defer-interactive-support" + }, + { + "id": "interactive-database-mysql-upgrade", + "commandId": "database mysql upgrade", + "category": "INTERACTIVE_REQUIRED", + "reason": "Upgrade confirmation and version choice currently requires interactive input.", + "issue": "defer-interactive-support" + }, + { + "id": "interactive-ddev-init", + "commandId": "ddev init", + "category": "INTERACTIVE_REQUIRED", + "reason": "Project type and configuration selection still enters interactive decision branches.", + "issue": "defer-interactive-support" + }, + { + "id": "interactive-login-token", + "commandId": "login token", + "category": "INTERACTIVE_REQUIRED", + "reason": "Token acquisition flow is intentionally interactive for secure input handling.", + "issue": "defer-interactive-support" + }, + { + "id": "interactive-stack-deploy", + "commandId": "stack deploy", + "category": "INTERACTIVE_REQUIRED", + "reason": "Deploy flow requires interactive input in current command implementation.", + "issue": "defer-interactive-support" + }, + { + "id": "interactive-user-ssh-key-create", + "commandId": "user ssh-key create", + "category": "INTERACTIVE_REQUIRED", + "reason": "SSH key creation relies on interactive prompts for key source and confirmation.", + "issue": "defer-interactive-support" + }, + { + "id": "deprecated-endpoint-app-database-link", + "commandId": "app database link", + "category": "DEPRECATED_ENDPOINT", + "reason": "Command currently uses a deprecated API endpoint that is filtered by the mockoon setup and therefore not served in integration runs. The command should be reworked to use a current non-deprecated endpoint.", + "issue": "rework-deprecated-endpoint" + }, + { + "id": "deprecated-endpoint-app-database-replace", + "commandId": "app database replace", + "category": "DEPRECATED_ENDPOINT", + "reason": "Command currently uses a deprecated API endpoint that is filtered by the mockoon setup and therefore not served in integration runs. The command should be reworked to use a current non-deprecated endpoint.", + "issue": "rework-deprecated-endpoint" + }, + { + "id": "deprecated-endpoint-stack-set-update-schedule", + "commandId": "stack set-update-schedule", + "category": "DEPRECATED_ENDPOINT", + "reason": "Command currently uses a deprecated API endpoint that is filtered by the mockoon setup and therefore not served in integration runs. The command should be reworked to use a current non-deprecated endpoint.", + "issue": "rework-deprecated-endpoint" + }, + { + "id": "deprecated-endpoint-stack-unset-update-schedule", + "commandId": "stack unset-update-schedule", + "category": "DEPRECATED_ENDPOINT", + "reason": "Command currently uses a deprecated API endpoint that is filtered by the mockoon setup and therefore not served in integration runs. The command should be reworked to use a current non-deprecated endpoint.", + "issue": "rework-deprecated-endpoint" + }, + { + "id": "contract-shape-app-create-node-invalid-version", + "commandId": "app create node", + "category": "CONTRACT_SHAPE", + "reason": "Version fields in the app version contract are typed as generic strings (openapi schema), but client selection logic assumes semver-compatible internalVersion values and calls semver.gt directly. In integration fixtures this can surface as empty or non-semver values and crashes with TypeError: Invalid Version before command-specific logic can proceed.", + "issue": "harden-app-version-selection" + }, + { + "id": "contract-shape-app-create-php-invalid-version", + "commandId": "app create php", + "category": "CONTRACT_SHAPE", + "reason": "Version fields in the app version contract are typed as generic strings (openapi schema), but client selection logic assumes semver-compatible internalVersion values and calls semver.gt directly. In integration fixtures this can surface as empty or non-semver values and crashes with TypeError: Invalid Version before command-specific logic can proceed.", + "issue": "harden-app-version-selection" + }, + { + "id": "contract-shape-app-create-php-worker-invalid-version", + "commandId": "app create php-worker", + "category": "CONTRACT_SHAPE", + "reason": "Version fields in the app version contract are typed as generic strings (openapi schema), but client selection logic assumes semver-compatible internalVersion values and calls semver.gt directly. In integration fixtures this can surface as empty or non-semver values and crashes with TypeError: Invalid Version before command-specific logic can proceed.", + "issue": "harden-app-version-selection" + }, + { + "id": "contract-shape-app-create-python-invalid-version", + "commandId": "app create python", + "category": "CONTRACT_SHAPE", + "reason": "Version fields in the app version contract are typed as generic strings (openapi schema), but client selection logic assumes semver-compatible internalVersion values and calls semver.gt directly. In integration fixtures this can surface as empty or non-semver values and crashes with TypeError: Invalid Version before command-specific logic can proceed.", + "issue": "harden-app-version-selection" + }, + { + "id": "contract-shape-app-create-static-invalid-version", + "commandId": "app create static", + "category": "CONTRACT_SHAPE", + "reason": "Version fields in the app version contract are typed as generic strings (openapi schema), but client selection logic assumes semver-compatible internalVersion values and calls semver.gt directly. In integration fixtures this can surface as empty or non-semver values and crashes with TypeError: Invalid Version before command-specific logic can proceed.", + "issue": "harden-app-version-selection" + }, + { + "id": "contract-shape-app-install-contao-invalid-version", + "commandId": "app install contao", + "category": "CONTRACT_SHAPE", + "reason": "Version fields in the app version contract are typed as generic strings (openapi schema), but client selection logic assumes semver-compatible internalVersion values and calls semver.gt directly. In integration fixtures this can surface as empty or non-semver values and crashes with TypeError: Invalid Version before command-specific logic can proceed.", + "issue": "harden-app-version-selection" + }, + { + "id": "contract-shape-app-install-shopware5-invalid-version", + "commandId": "app install shopware5", + "category": "CONTRACT_SHAPE", + "reason": "Version fields in the app version contract are typed as generic strings (openapi schema), but client selection logic assumes semver-compatible internalVersion values and calls semver.gt directly. In integration fixtures this can surface as empty or non-semver values and crashes with TypeError: Invalid Version before command-specific logic can proceed.", + "issue": "harden-app-version-selection" + }, + { + "id": "contract-shape-app-install-shopware6-invalid-version", + "commandId": "app install shopware6", + "category": "CONTRACT_SHAPE", + "reason": "Version fields in the app version contract are typed as generic strings (openapi schema), but client selection logic assumes semver-compatible internalVersion values and calls semver.gt directly. In integration fixtures this can surface as empty or non-semver values and crashes with TypeError: Invalid Version before command-specific logic can proceed.", + "issue": "harden-app-version-selection" + }, + { + "id": "contract-shape-app-install-typo3-invalid-version", + "commandId": "app install typo3", + "category": "CONTRACT_SHAPE", + "reason": "Version fields in the app version contract are typed as generic strings (openapi schema), but client selection logic assumes semver-compatible internalVersion values and calls semver.gt directly. In integration fixtures this can surface as empty or non-semver values and crashes with TypeError: Invalid Version before command-specific logic can proceed.", + "issue": "harden-app-version-selection" + }, + { + "id": "contract-shape-app-install-wordpress-invalid-version", + "commandId": "app install wordpress", + "category": "CONTRACT_SHAPE", + "reason": "Version fields in the app version contract are typed as generic strings (openapi schema), but client selection logic assumes semver-compatible internalVersion values and calls semver.gt directly. In integration fixtures this can surface as empty or non-semver values and crashes with TypeError: Invalid Version before command-specific logic can proceed.", + "issue": "harden-app-version-selection" + }, + { + "id": "resource-precondition-container-cp-container-not-found", + "commandId": "container cp", + "category": "RESOURCE_PRECONDITION", + "reason": "Shared container resolution precondition is not satisfied in integration fixtures. The resolver in src/lib/resources/container/flags.ts (withContainerAndStackId) lists project services and throws when no matching container shortId/id/serviceName exists. Current run fails with 'no container mycontainer found in project p-f0ob4r'. Preferred fix is to seed a deterministic known container fixture for project p-f0ob4r and align invocation profiles/examples to that known container identifier.", + "issue": "seed-known-container-fixture" + }, + { + "id": "resource-precondition-container-delete-container-not-found", + "commandId": "container delete", + "category": "RESOURCE_PRECONDITION", + "reason": "Shared container resolution precondition is not satisfied in integration fixtures. The resolver in src/lib/resources/container/flags.ts (withContainerAndStackId) lists project services and throws when no matching container shortId/id/serviceName exists. Current run shows deletion flow failing because the requested container identifier does not exist in project p-f0ob4r. Preferred fix is to seed a deterministic known container fixture for project p-f0ob4r and align invocation profiles/examples to that known container identifier.", + "issue": "seed-known-container-fixture" + }, + { + "id": "resource-precondition-container-exec-container-not-found", + "commandId": "container exec", + "category": "RESOURCE_PRECONDITION", + "reason": "Shared container resolution precondition is not satisfied in integration fixtures. The resolver in src/lib/resources/container/flags.ts (withContainerAndStackId) lists project services and throws when no matching container shortId/id/serviceName exists. Current run fails with 'no container 00000000-0000-4000-8000-000000000000 found in project p-f0ob4r'. Preferred fix is to seed a deterministic known container fixture for project p-f0ob4r and align invocation profiles/examples to that known container identifier.", + "issue": "seed-known-container-fixture" + }, + { + "id": "resource-precondition-container-port-forward-container-not-found", + "commandId": "container port-forward", + "category": "RESOURCE_PRECONDITION", + "reason": "Shared container resolution precondition is not satisfied in integration fixtures. The resolver in src/lib/resources/container/flags.ts (withContainerAndStackId) lists project services and throws when no matching container shortId/id/serviceName exists. Current run fails with 'no container 00000000-0000-4000-8000-000000000000 found in project p-f0ob4r'. Preferred fix is to seed a deterministic known container fixture for project p-f0ob4r and align invocation profiles/examples to that known container identifier.", + "issue": "seed-known-container-fixture" + }, + { + "id": "resource-precondition-container-recreate-container-not-found", + "commandId": "container recreate", + "category": "RESOURCE_PRECONDITION", + "reason": "Shared container resolution precondition is not satisfied in integration fixtures. The resolver in src/lib/resources/container/flags.ts (withContainerAndStackId) lists project services and throws when no matching container shortId/id/serviceName exists. Current run fails with 'no container 00000000-0000-4000-8000-000000000000 found in project p-f0ob4r'. Preferred fix is to seed a deterministic known container fixture for project p-f0ob4r and align invocation profiles/examples to that known container identifier.", + "issue": "seed-known-container-fixture" + }, + { + "id": "resource-precondition-container-restart-container-not-found", + "commandId": "container restart", + "category": "RESOURCE_PRECONDITION", + "reason": "Shared container resolution precondition is not satisfied in integration fixtures. The resolver in src/lib/resources/container/flags.ts (withContainerAndStackId) lists project services and throws when no matching container shortId/id/serviceName exists. Current run fails with 'no container 00000000-0000-4000-8000-000000000000 found in project p-f0ob4r'. Preferred fix is to seed a deterministic known container fixture for project p-f0ob4r and align invocation profiles/examples to that known container identifier.", + "issue": "seed-known-container-fixture" + }, + { + "id": "resource-precondition-container-ssh-container-not-found", + "commandId": "container ssh", + "category": "RESOURCE_PRECONDITION", + "reason": "Shared container resolution precondition is not satisfied in integration fixtures. The resolver in src/lib/resources/container/flags.ts (withContainerAndStackId) lists project services and throws when no matching container shortId/id/serviceName exists. Current run fails with 'no container 00000000-0000-4000-8000-000000000000 found in project p-f0ob4r'. Preferred fix is to seed a deterministic known container fixture for project p-f0ob4r and align invocation profiles/examples to that known container identifier.", + "issue": "seed-known-container-fixture" + }, + { + "id": "resource-precondition-container-start-container-not-found", + "commandId": "container start", + "category": "RESOURCE_PRECONDITION", + "reason": "Shared container resolution precondition is not satisfied in integration fixtures. The resolver in src/lib/resources/container/flags.ts (withContainerAndStackId) lists project services and throws when no matching container shortId/id/serviceName exists. Current run fails with 'no container 00000000-0000-4000-8000-000000000000 found in project p-f0ob4r'. Preferred fix is to seed a deterministic known container fixture for project p-f0ob4r and align invocation profiles/examples to that known container identifier.", + "issue": "seed-known-container-fixture" + }, + { + "id": "resource-precondition-container-stop-container-not-found", + "commandId": "container stop", + "category": "RESOURCE_PRECONDITION", + "reason": "Shared container resolution precondition is not satisfied in integration fixtures. The resolver in src/lib/resources/container/flags.ts (withContainerAndStackId) lists project services and throws when no matching container shortId/id/serviceName exists. Current run fails with 'no container 00000000-0000-4000-8000-000000000000 found in project p-f0ob4r'. Preferred fix is to seed a deterministic known container fixture for project p-f0ob4r and align invocation profiles/examples to that known container identifier.", + "issue": "seed-known-container-fixture" + }, + { + "id": "resource-precondition-container-update-container-not-found", + "commandId": "container update", + "category": "RESOURCE_PRECONDITION", + "reason": "Shared container resolution precondition is not satisfied in integration fixtures. The resolver in src/lib/resources/container/flags.ts (withContainerAndStackId) lists project services and throws when no matching container shortId/id/serviceName exists. Current run fails with 'no container 00000000-0000-4000-8000-000000000000 found in project p-f0ob4r'. Preferred fix is to seed a deterministic known container fixture for project p-f0ob4r and align invocation profiles/examples to that known container identifier.", + "issue": "seed-known-container-fixture" + }, + { + "id": "contract-shape-app-download-missing-web-directory", + "commandId": "app download", + "category": "CONTRACT_SHAPE", + "reason": "Command fails with TypeError [ERR_INVALID_ARG_TYPE] in a shared path.join call because integration response shape does not provide all path inputs expected by shared SSH/app-installation rendering logic. Primary shared callsite: src/lib/resources/ssh/appinstall.ts (getSSHConnectionForAppInstallation) builds directory via path.join(projectResponse.data.directories['Web'], appInstallation.installationPath). If directories['Web'] or installationPath is missing/undefined, multiple commands crash with the same signature. Preferred fix is fixture-side: seed deterministic project/app-installation payloads that always include project.directories['Web'] and appInstallation.installationPath for the known test project.", + "issue": "seed-appinstall-web-directory-fixture" + }, + { + "id": "contract-shape-app-exec-missing-web-directory", + "commandId": "app exec", + "category": "CONTRACT_SHAPE", + "reason": "Command fails with TypeError [ERR_INVALID_ARG_TYPE] in a shared path.join call because integration response shape does not provide all path inputs expected by shared SSH/app-installation rendering logic. Primary shared callsite: src/lib/resources/ssh/appinstall.ts (getSSHConnectionForAppInstallation) builds directory via path.join(projectResponse.data.directories['Web'], appInstallation.installationPath). If directories['Web'] or installationPath is missing/undefined, multiple commands crash with the same signature. Preferred fix is fixture-side: seed deterministic project/app-installation payloads that always include project.directories['Web'] and appInstallation.installationPath for the known test project.", + "issue": "seed-appinstall-web-directory-fixture" + }, + { + "id": "contract-shape-app-ssh-missing-web-directory", + "commandId": "app ssh", + "category": "CONTRACT_SHAPE", + "reason": "Command fails with TypeError [ERR_INVALID_ARG_TYPE] in a shared path.join call because integration response shape does not provide all path inputs expected by shared SSH/app-installation rendering logic. Primary shared callsite: src/lib/resources/ssh/appinstall.ts (getSSHConnectionForAppInstallation) builds directory via path.join(projectResponse.data.directories['Web'], appInstallation.installationPath). If directories['Web'] or installationPath is missing/undefined, multiple commands crash with the same signature. Preferred fix is fixture-side: seed deterministic project/app-installation payloads that always include project.directories['Web'] and appInstallation.installationPath for the known test project.", + "issue": "seed-appinstall-web-directory-fixture" + }, + { + "id": "contract-shape-app-get-missing-web-directory", + "commandId": "app get", + "category": "CONTRACT_SHAPE", + "reason": "Command fails with TypeError [ERR_INVALID_ARG_TYPE] from AppInstallationDetails rendering when absolute installation path is built with path.join(project.directories['Web'], appInstallation.installationPath). This is the same fixture data-shape gap as the shared SSH app-installation path handling cluster: missing directories['Web'] and/or installationPath in test payloads causes deterministic crash. Preferred fix is fixture-side: seed deterministic project/app-installation payloads that always include project.directories['Web'] and appInstallation.installationPath for the known test project.", + "issue": "seed-appinstall-web-directory-fixture" + }, + { + "id": "contract-shape-app-install-joomla-invalid-version", + "commandId": "app install joomla", + "category": "CONTRACT_SHAPE", + "reason": "Version fields in the app version contract are typed as generic strings (openapi schema), but client selection logic assumes semver-compatible internalVersion values and calls semver.gt directly. In integration fixtures this can surface as empty or non-semver values and crashes with TypeError: Invalid Version before command-specific logic can proceed.", + "issue": "harden-app-version-selection" + }, + { + "id": "contract-shape-app-install-nextcloud-invalid-version", + "commandId": "app install nextcloud", + "category": "CONTRACT_SHAPE", + "reason": "Version fields in the app version contract are typed as generic strings (openapi schema), but client selection logic assumes semver-compatible internalVersion values and calls semver.gt directly. In integration fixtures this can surface as empty or non-semver values and crashes with TypeError: Invalid Version before command-specific logic can proceed.", + "issue": "harden-app-version-selection" + }, + { + "id": "command-bug-app-dependency-update-invalid-installation-id", + "commandId": "app dependency update", + "category": "COMMAND_BUG", + "reason": "Integration invocation currently passes a placeholder that does not resolve to a valid app installation identifier for this command path. The command exits with an app-installation ID validation error in this test setup.", + "issue": "fix-integration-invocation-profiles" + }, + { + "id": "resource-precondition-app-dependency-versions-systemsoftware-not-found", + "commandId": "app dependency versions", + "category": "RESOURCE_PRECONDITION", + "reason": "The integration fixture set does not provide a resolvable system software entry for the placeholder value used in this command run. The command fails with 'system software ... not found'.", + "issue": "seed-known-resource-fixtures" + }, + { + "id": "contract-shape-app-install-matomo-invalid-version", + "commandId": "app install matomo", + "category": "CONTRACT_SHAPE", + "reason": "Version fields in the app version contract are typed as generic strings (openapi schema), but client selection logic assumes semver-compatible internalVersion values and calls semver.gt directly. In integration fixtures this can surface as empty or non-semver values and crashes with TypeError: Invalid Version before command-specific logic can proceed.", + "issue": "harden-app-version-selection" + }, + { + "id": "command-bug-app-list-upgrade-candidates-versions-not-array", + "commandId": "app list-upgrade-candidates", + "category": "COMMAND_BUG", + "reason": "The command expects a sortable versions array, but the current integration response shape provides a non-array value and execution fails with 'versions.sort is not a function'.", + "issue": "harden-response-shape-handling" + }, + { + "id": "command-bug-app-open-missing-virtualhost-link", + "commandId": "app open", + "category": "COMMAND_BUG", + "reason": "The test fixture app installation used in integration is not linked to a virtual host, and this command currently fails along that path in run-all execution.", + "issue": "seed-known-resource-fixtures" + }, + { + "id": "resource-precondition-app-upload-source-placeholder-not-available", + "commandId": "app upload", + "category": "RESOURCE_PRECONDITION", + "reason": "The integration invocation for this command does not provide a usable source path in the run-all environment. The command fails while parsing the source input.", + "issue": "fix-integration-invocation-profiles" + }, + { + "id": "resource-precondition-app-version-info-app-not-found", + "commandId": "app version-info", + "category": "RESOURCE_PRECONDITION", + "reason": "The fixture app identifier used in integration cannot be resolved in the mocked dataset, and the command fails with 'app ... not found'.", + "issue": "seed-known-resource-fixtures" + }, + { + "id": "command-bug-app-versions-access-denied", + "commandId": "app versions", + "category": "COMMAND_BUG", + "reason": "The command path fails with an access denied error in the current integration authorization/fixture setup.", + "issue": "seed-permissions-fixtures" + }, + { + "id": "command-bug-backup-download-not-ready", + "commandId": "backup download", + "category": "COMMAND_BUG", + "reason": "This command currently terminates with 'backup download is not ready' in the integration scenario, indicating an unfinished command path for this fixture state.", + "issue": "implement-backup-download-path" + }, + { + "id": "resource-precondition-container-run-service-id-not-found", + "commandId": "container run", + "category": "RESOURCE_PRECONDITION", + "reason": "The created stack in integration fixtures does not expose the expected service mapping for this flow. The command fails with 'Service ID not found in the created stack'.", + "issue": "seed-stack-service-fixture" + }, + { + "id": "command-bug-conversation-create-tempfile-unlink-enoent", + "commandId": "conversation create", + "category": "COMMAND_BUG", + "reason": "The command hits a temporary-file cleanup race in this run path and fails with ENOENT on unlink of a generated markdown file.", + "issue": "stabilize-tempfile-lifecycle" + }, + { + "id": "command-bug-conversation-reply-tempfile-unlink-enoent", + "commandId": "conversation reply", + "category": "COMMAND_BUG", + "reason": "The command hits a temporary-file cleanup race in this run path and fails with ENOENT on unlink of a generated markdown file.", + "issue": "stabilize-tempfile-lifecycle" + }, + { + "id": "resource-precondition-database-mysql-dump-main-user-missing", + "commandId": "database mysql dump", + "category": "RESOURCE_PRECONDITION", + "reason": "The MySQL dump flow requires a resolvable main user in fixtures. In this run the command fails with 'No main user found'.", + "issue": "seed-mysql-main-user-fixture" + }, + { + "id": "resource-precondition-database-mysql-phpmyadmin-main-user-missing", + "commandId": "database mysql phpmyadmin", + "category": "RESOURCE_PRECONDITION", + "reason": "The phpMyAdmin flow requires a resolvable main user in fixtures. In this run the command fails with 'no main user found'.", + "issue": "seed-mysql-main-user-fixture" + }, + { + "id": "resource-precondition-database-mysql-user-delete-main-user-protected", + "commandId": "database mysql user delete", + "category": "RESOURCE_PRECONDITION", + "reason": "Integration currently targets the main MySQL user, which is protected by API rules and cannot be deleted manually in this flow.", + "issue": "seed-mysql-non-main-user-fixture" + }, + { + "id": "contract-shape-ddev-render-config-missing-document-root-input", + "commandId": "ddev render-config", + "category": "CONTRACT_SHAPE", + "reason": "DDEV config generation currently receives fixture data with missing path fields and fails with 'Cannot read properties of undefined (reading replace)' in config builder path normalization.", + "issue": "seed-ddev-config-shape-fixture" + }, + { + "id": "resource-precondition-domain-dnszone-get-zone-not-found", + "commandId": "domain dnszone get", + "category": "RESOURCE_PRECONDITION", + "reason": "The integration fixture dataset does not include the requested DNS zone domain in this run context, causing a deterministic 'DNS zone ... not found' failure.", + "issue": "seed-known-domain-fixture" + }, + { + "id": "resource-precondition-domain-dnszone-update-zone-not-found", + "commandId": "domain dnszone update", + "category": "RESOURCE_PRECONDITION", + "reason": "The integration fixture dataset does not include the requested DNS zone domain in this run context, causing a deterministic 'DNS zone ... not found' failure.", + "issue": "seed-known-domain-fixture" + }, + { + "id": "resource-precondition-domain-get-zone-not-found", + "commandId": "domain get", + "category": "RESOURCE_PRECONDITION", + "reason": "Domain lookup in this integration path depends on a DNS zone fixture that is not present for the placeholder value, producing 'DNS zone ... not found'.", + "issue": "seed-known-domain-fixture" + }, + { + "id": "resource-precondition-experimental-deploy-registry-service-missing", + "commandId": "experimental deploy", + "category": "RESOURCE_PRECONDITION", + "reason": "The deploy orchestration expects a registry service fixture that is not returned in this integration environment and fails with 'Service not found in response'.", + "issue": "seed-stack-service-fixture" + }, + { + "id": "resource-precondition-mail-address-update-mail-address-not-found", + "commandId": "mail address update", + "category": "RESOURCE_PRECONDITION", + "reason": "The mail address used by integration placeholders is not present in the mocked dataset during this run, causing a deterministic not-found failure.", + "issue": "seed-known-resource-fixtures" + }, + { + "id": "resource-precondition-org-delete-org-not-found", + "commandId": "org delete", + "category": "RESOURCE_PRECONDITION", + "reason": "The organization targeted by integration placeholders does not exist in the fixture state at deletion time, resulting in a 404 failure.", + "issue": "seed-known-resource-fixtures" + }, + { + "id": "resource-precondition-sftp-user-create-readback-404", + "commandId": "sftp-user create", + "category": "RESOURCE_PRECONDITION", + "reason": "Creation step succeeds, but follow-up readback in integration fixtures returns 404, indicating inconsistent fixture state for immediate lookup.", + "issue": "align-user-create-fixture-readback" + }, + { + "id": "resource-precondition-ssh-user-create-readback-404", + "commandId": "ssh-user create", + "category": "RESOURCE_PRECONDITION", + "reason": "Creation step succeeds, but follow-up readback in integration fixtures returns 404, indicating inconsistent fixture state for immediate lookup.", + "issue": "align-user-create-fixture-readback" + }, + { + "id": "command-bug-stack-delete-not-implemented", + "commandId": "stack delete", + "category": "COMMAND_BUG", + "reason": "Command flow reaches deletion step and fails with 'not implemented' in current implementation.", + "issue": "implement-stack-delete" + }, + { + "id": "resource-precondition-volume-delete-volume-not-found", + "commandId": "volume delete", + "category": "RESOURCE_PRECONDITION", + "reason": "The requested volume placeholder does not exist in integration fixtures for the active stack state, causing deterministic not-found failure.", + "issue": "seed-known-resource-fixtures" + } +] diff --git a/src/test/integration/config/invocation-profiles.json b/src/test/integration/config/invocation-profiles.json new file mode 100644 index 000000000..3f3f330bf --- /dev/null +++ b/src/test/integration/config/invocation-profiles.json @@ -0,0 +1,218 @@ +[ + { + "id": "backup-create", + "match": { "exact": "backup create" }, + "requiredFlagDefaults": { + "expires": "30d" + }, + "notes": "Backup expiration must be set explicitly for deterministic runs." + }, + { + "id": "backup-schedule-create", + "match": { "exact": "backup schedule create" }, + "requiredFlagDefaults": { + "schedule": "0 * * * *", + "ttl": "7d" + } + }, + { + "id": "cronjob-create", + "match": { "exact": "cronjob create" }, + "requiredFlagDefaults": { + "description": "integration-cronjob", + "interval": "0 * * * *", + "url": "https://example.com/cronjob" + }, + "exactlyOneChoice": { + "command|url": "url" + } + }, + { + "id": "extension-install", + "match": { "exact": "extension install" }, + "requiredArgDefaults": { + "extension-id": "example-extension-id" + }, + "requiredFlagDefaults": { + "consent": true, + "project-id": "00000000-0000-4000-8000-000000000000" + }, + "exactlyOneChoice": { + "org-id|project-id": "project-id" + } + }, + { + "id": "extension-list-installed", + "match": { "exact": "extension list-installed" }, + "requiredFlagDefaults": { + "project-id": "00000000-0000-4000-8000-000000000000" + }, + "exactlyOneChoice": { + "org-id|project-id": "project-id" + } + }, + { + "id": "sftp-user-create", + "match": { "exact": "sftp-user create" }, + "requiredFlagDefaults": { + "description": "integration-sftp-user", + "directories": "/", + "password": "integration-password" + }, + "exactlyOneChoice": { + "password|public-key": "password" + } + }, + { + "id": "ssh-user-create", + "match": { "exact": "ssh-user create" }, + "requiredFlagDefaults": { + "description": "integration-ssh-user", + "password": "integration-password" + } + }, + { + "id": "database-mysql-create", + "match": { "exact": "database mysql create" }, + "requiredFlagDefaults": { + "description": "integration-mysql-db", + "version": "8.0", + "user-password": "integration-password" + } + }, + { + "id": "database-mysql-user-create", + "match": { "exact": "database mysql user create" }, + "requiredFlagDefaults": { + "database-id": "00000000-0000-4000-8000-000000000000", + "access-level": "full", + "description": "integration-mysql-user", + "password": "integration-password" + } + }, + { + "id": "database-mysql-shell", + "match": { "exact": "database mysql shell" }, + "interactivePolicy": "classify" + }, + { + "id": "app-database-link", + "match": { "exact": "app database link" }, + "requiredFlagDefaults": { + "database-id": "00000000-0000-4000-8000-000000000000", + "admin-user-id": "00000000-0000-4000-8000-000000000000", + "purpose": "primary" + } + }, + { + "id": "app-database-replace", + "match": { "exact": "app database replace" }, + "requiredFlagDefaults": { + "new-database-id": "00000000-0000-4000-8000-000000000000", + "admin-user-id": "00000000-0000-4000-8000-000000000000" + } + }, + { + "id": "org-invite", + "match": { "exact": "org invite" }, + "requiredFlagDefaults": { + "email": "integration@example.com" + } + }, + { + "id": "user-api-token-create", + "match": { "exact": "user api-token create" }, + "requiredFlagDefaults": { + "description": "integration-api-token", + "roles": "api_read" + } + }, + { + "id": "ssh-user-update", + "match": { "exact": "ssh-user update" }, + "requiredFlagDefaults": { + "password": "integration-password" + }, + "exactlyOneChoice": { + "password|public-key": "password" + } + }, + { + "id": "domain-dnszone-update", + "match": { "exact": "domain dnszone update" }, + "requiredArgDefaults": { + "record-set": "a" + }, + "requiredFlagDefaults": { + "record": "203.0.113.10" + } + }, + { + "id": "domain-get", + "match": { "exact": "domain get" }, + "requiredArgDefaults": { + "domain-id": "example.com" + } + }, + { + "id": "domain-dnszone-get", + "match": { "exact": "domain dnszone get" }, + "requiredArgDefaults": { + "dnszone-id": "example.com" + } + }, + { + "id": "domain-virtualhost-update", + "match": { "exact": "domain virtualhost update" }, + "requiredFlagDefaults": { + "path-to-url": "/:https://example.com" + } + }, + { + "id": "mail-address-update", + "match": { "exact": "mail address update" }, + "requiredArgDefaults": { + "mailaddress-id": "integration@example.com" + } + }, + { + "id": "mail-deliverybox-update", + "match": { "exact": "mail deliverybox update" }, + "requiredArgDefaults": { + "maildeliverybox-id": "00000000-0000-4000-8000-000000000000" + } + }, + { + "id": "ddev-init", + "match": { "exact": "ddev init" }, + "requiredFlagDefaults": { + "override-type": "auto", + "project-name": "integration-ddev" + } + }, + { + "id": "ddev-render-config", + "match": { "exact": "ddev render-config" }, + "requiredFlagDefaults": { + "override-type": "php" + } + }, + { + "id": "login-token", + "match": { "exact": "login token" }, + "interactivePolicy": "classify", + "disableExampleSource": true + }, + { + "id": "conversation-create", + "match": { "exact": "conversation create" }, + "interactivePolicy": "classify", + "disableExampleSource": true + }, + { + "id": "conversation-reply", + "match": { "exact": "conversation reply" }, + "interactivePolicy": "classify", + "disableExampleSource": true + } +] diff --git a/src/test/integration/config/loader.ts b/src/test/integration/config/loader.ts new file mode 100644 index 000000000..6337ef9b7 --- /dev/null +++ b/src/test/integration/config/loader.ts @@ -0,0 +1,276 @@ +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import type { CommandWaiver, InvocationProfile, WaiverCategory } from "../command-discovery/types.js"; + +const CONFIG_DIR = path.dirname(fileURLToPath(import.meta.url)); +const INVOCATION_PROFILES_PATH = path.join(CONFIG_DIR, "invocation-profiles.json"); +const COMMAND_WAIVERS_PATH = path.join(CONFIG_DIR, "command-waivers.json"); + +const WAIVER_CATEGORIES: Set = new Set([ + "ARG_MISUSE", + "INTERACTIVE_REQUIRED", + "RESOURCE_PRECONDITION", + "CONTRACT_SHAPE", + "COMMAND_BUG", + "DEPRECATED_ENDPOINT", +]); + +let invocationProfilesCache: InvocationProfile[] | undefined; +let commandWaiversCache: CommandWaiver[] | undefined; + +export function loadInvocationProfiles(): InvocationProfile[] { + if (invocationProfilesCache) { + return invocationProfilesCache; + } + + const raw = readJsonFile(INVOCATION_PROFILES_PATH, "invocation profiles"); + if (!Array.isArray(raw)) { + throw new Error("[integration-config] invocation profiles must be an array."); + } + + invocationProfilesCache = raw.map((value, index) => + validateInvocationProfile(value, index), + ); + + return invocationProfilesCache; +} + +export function loadCommandWaivers(): CommandWaiver[] { + if (commandWaiversCache) { + return commandWaiversCache; + } + + const raw = readJsonFile(COMMAND_WAIVERS_PATH, "command waivers"); + if (!Array.isArray(raw)) { + throw new Error("[integration-config] command waivers must be an array."); + } + + const validated = raw.map((value, index) => validateCommandWaiver(value, index)); + + const ids = new Set(); + const commandIds = new Set(); + for (const waiver of validated) { + if (ids.has(waiver.id)) { + throw new Error(`[integration-config] duplicate waiver id '${waiver.id}'.`); + } + + if (commandIds.has(waiver.commandId)) { + throw new Error( + `[integration-config] duplicate waiver commandId '${waiver.commandId}'.`, + ); + } + + ids.add(waiver.id); + commandIds.add(waiver.commandId); + } + + commandWaiversCache = validated; + return commandWaiversCache; +} + +function readJsonFile(filePath: string, label: string): unknown { + try { + const content = readFileSync(filePath, "utf8"); + return JSON.parse(content); + } catch (error) { + throw new Error( + `[integration-config] failed to load ${label} at ${filePath}: ${(error as Error).message}`, + ); + } +} + +function validateInvocationProfile(value: unknown, index: number): InvocationProfile { + const record = asRecord(value, `invocation profile at index ${index}`); + const id = asNonEmptyString(record.id, `${profileLabel(index)}.id`); + + const matchRaw = asRecord(record.match, `${profileLabel(index)}.match`); + const exact = asOptionalString(matchRaw.exact, `${profileLabel(index)}.match.exact`); + const prefix = asOptionalString(matchRaw.prefix, `${profileLabel(index)}.match.prefix`); + if (!exact && !prefix) { + throw new Error( + `[integration-config] ${profileLabel(index)}.match requires 'exact' or 'prefix'.`, + ); + } + + const requiredFlagDefaults = asOptionalStringBooleanMap( + record.requiredFlagDefaults, + `${profileLabel(index)}.requiredFlagDefaults`, + ); + const requiredArgDefaults = asOptionalStringMap( + record.requiredArgDefaults, + `${profileLabel(index)}.requiredArgDefaults`, + ); + const exactlyOneChoice = asOptionalStringMap( + record.exactlyOneChoice, + `${profileLabel(index)}.exactlyOneChoice`, + ); + + const interactivePolicy = asOptionalInteractivePolicy( + record.interactivePolicy, + `${profileLabel(index)}.interactivePolicy`, + ); + + const disableExampleSource = asOptionalBoolean( + record.disableExampleSource, + `${profileLabel(index)}.disableExampleSource`, + ); + + const notes = asOptionalString(record.notes, `${profileLabel(index)}.notes`); + + return { + id, + match: { + ...(exact ? { exact } : {}), + ...(prefix ? { prefix } : {}), + }, + ...(requiredFlagDefaults ? { requiredFlagDefaults } : {}), + ...(requiredArgDefaults ? { requiredArgDefaults } : {}), + ...(exactlyOneChoice ? { exactlyOneChoice } : {}), + ...(interactivePolicy ? { interactivePolicy } : {}), + ...(disableExampleSource !== undefined ? { disableExampleSource } : {}), + ...(notes ? { notes } : {}), + }; +} + +function validateCommandWaiver(value: unknown, index: number): CommandWaiver { + const record = asRecord(value, `command waiver at index ${index}`); + const id = asNonEmptyString(record.id, `${waiverLabel(index)}.id`); + const commandId = asNonEmptyString( + record.commandId, + `${waiverLabel(index)}.commandId`, + ); + const category = asNonEmptyString( + record.category, + `${waiverLabel(index)}.category`, + ) as WaiverCategory; + + if (!WAIVER_CATEGORIES.has(category)) { + throw new Error( + `[integration-config] ${waiverLabel(index)}.category must be one of ${[ + ...WAIVER_CATEGORIES, + ].join(", ")}.`, + ); + } + + const reason = asNonEmptyString(record.reason, `${waiverLabel(index)}.reason`); + const issue = asOptionalString(record.issue, `${waiverLabel(index)}.issue`); + const expiresOn = asOptionalString(record.expiresOn, `${waiverLabel(index)}.expiresOn`); + + return { + id, + commandId, + category, + reason, + ...(issue ? { issue } : {}), + ...(expiresOn ? { expiresOn } : {}), + }; +} + +function asRecord(value: unknown, label: string): Record { + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error(`[integration-config] ${label} must be an object.`); + } + + return value as Record; +} + +function asNonEmptyString(value: unknown, label: string): string { + if (typeof value !== "string" || value.trim().length === 0) { + throw new Error(`[integration-config] ${label} must be a non-empty string.`); + } + + return value.trim(); +} + +function asOptionalString(value: unknown, label: string): string | undefined { + if (value === undefined) { + return undefined; + } + + if (typeof value !== "string") { + throw new Error(`[integration-config] ${label} must be a string when provided.`); + } + + return value; +} + +function asOptionalBoolean(value: unknown, label: string): boolean | undefined { + if (value === undefined) { + return undefined; + } + + if (typeof value !== "boolean") { + throw new Error(`[integration-config] ${label} must be a boolean when provided.`); + } + + return value; +} + +function asOptionalInteractivePolicy( + value: unknown, + label: string, +): "resolve" | "classify" | undefined { + if (value === undefined) { + return undefined; + } + + if (value !== "resolve" && value !== "classify") { + throw new Error( + `[integration-config] ${label} must be 'resolve' or 'classify' when provided.`, + ); + } + + return value; +} + +function asOptionalStringMap( + value: unknown, + label: string, +): Record | undefined { + if (value === undefined) { + return undefined; + } + + const record = asRecord(value, label); + const result: Record = {}; + for (const [key, entry] of Object.entries(record)) { + if (typeof entry !== "string") { + throw new Error(`[integration-config] ${label}.${key} must be a string.`); + } + + result[key] = entry; + } + + return result; +} + +function asOptionalStringBooleanMap( + value: unknown, + label: string, +): Record | undefined { + if (value === undefined) { + return undefined; + } + + const record = asRecord(value, label); + const result: Record = {}; + + for (const [key, entry] of Object.entries(record)) { + if (typeof entry !== "string" && typeof entry !== "boolean") { + throw new Error(`[integration-config] ${label}.${key} must be a string or boolean.`); + } + + result[key] = entry; + } + + return result; +} + +function profileLabel(index: number): string { + return `invocation-profiles[${index}]`; +} + +function waiverLabel(index: number): string { + return `command-waivers[${index}]`; +} diff --git a/src/test/integration/env.ts b/src/test/integration/env.ts new file mode 100644 index 000000000..3113fdbdc --- /dev/null +++ b/src/test/integration/env.ts @@ -0,0 +1,32 @@ +export type EnvSnapshot = NodeJS.ProcessEnv; + +export function snapshotEnv(): EnvSnapshot { + return { ...process.env }; +} + +export function restoreEnv(snapshot: EnvSnapshot): void { + process.env = snapshot; +} + +export function requireIntegrationEnv( + envVars: string[], + context: string, +): void { + const missing = envVars.filter((envVar) => { + const value = process.env[envVar]; + return value === undefined || value.trim() === ""; + }); + + if (missing.length === 0) { + return; + } + + throw new Error( + `[integration:${context}] Missing required environment variables: ${missing.join(", ")}. ` + + "Set them before running this test.", + ); +} + +export function configureIntegrationEnv(context: string): void { + requireIntegrationEnv(["MITTWALD_API_TOKEN", "MITTWALD_API_BASE_URL"], context); +} diff --git a/src/test/integration/run-all-commands.test.ts b/src/test/integration/run-all-commands.test.ts new file mode 100644 index 000000000..7f7f27941 --- /dev/null +++ b/src/test/integration/run-all-commands.test.ts @@ -0,0 +1,594 @@ +import { afterEach, beforeEach, describe, expect, it, jest } from "@jest/globals"; +import { appendFile, mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { + buildClassificationCatalogFromBuckets, + parseFailureCategory, + saveClassificationCatalog, +} from "./classification-catalog.js"; +import { runDevCommand } from "./command.js"; +import { discoverRunnableCommands } from "./command-discovery.js"; +import type { CommandWaiver, WaiverCategory } from "./command-discovery/types.js"; +import { loadCommandWaivers } from "./config/loader.js"; +import { configureIntegrationEnv, requireIntegrationEnv, restoreEnv, snapshotEnv } from "./env.js"; + +jest.setTimeout(20 * 60 * 1000); + +type FailureCategory = WaiverCategory; + +type MachineLogEntry = Record; + +const FAILURE_CATEGORIES: FailureCategory[] = [ + "ARG_MISUSE", + "INTERACTIVE_REQUIRED", + "RESOURCE_PRECONDITION", + "CONTRACT_SHAPE", + "COMMAND_BUG", + "DEPRECATED_ENDPOINT", +]; + +function createFailureBuckets(): Record { + return { + ARG_MISUSE: [], + INTERACTIVE_REQUIRED: [], + RESOURCE_PRECONDITION: [], + CONTRACT_SHAPE: [], + COMMAND_BUG: [], + DEPRECATED_ENDPOINT: [], + }; +} + +function classifyFailure(output: { stderr: string; stdout: string }): FailureCategory { + const text = `${output.stderr}\n${output.stdout}`.toLowerCase(); + + if ( + /missing\s+(?:\d+\s+)?required arg|missing\s+(?:\d+\s+)?required flag|exactly one of|required options|unexpected argument|unknown flag|nonexistent flag|invalid flag|flag .* expects|no .* id given|you need to specify at least one/i.test( + text, + ) + ) { + return "ARG_MISUSE"; + } + + if ( + /prompt|interactive|addinput|addselect|addconfirmation|overwrite\?|token file already exists|tty/i.test( + text, + ) + ) { + return "INTERACTIVE_REQUIRED"; + } + + if ( + /not found|does not exist|no .* found|resource.*missing|404|forbidden|unauthorized|no project found|failed to connect|could not resolve hostname|name or service not known|no main user found|main mysql user can not be deleted manually/i.test( + text, + ) + ) { + return "RESOURCE_PRECONDITION"; + } + + if ( + /invalid version|not iterable|cannot read properties|undefined.*data|validation|invalid type|schema/i.test( + text, + ) + ) { + return "CONTRACT_SHAPE"; + } + + return "COMMAND_BUG"; +} + +function parseInvocationPartsFromArgs( + args: string[], + commandTokenCount: number, +): { positionalValues: string[]; flagValues: Map } { + const positionalValues: string[] = []; + const flagValues = new Map(); + + const invocationArgs = args.slice(commandTokenCount); + for (let i = 0; i < invocationArgs.length; i += 1) { + const token = invocationArgs[i]; + if (!token.startsWith("--")) { + positionalValues.push(token); + continue; + } + + const withoutPrefix = token.slice(2); + const eqIndex = withoutPrefix.indexOf("="); + let name = withoutPrefix; + let value: string | undefined; + + if (eqIndex >= 0) { + name = withoutPrefix.slice(0, eqIndex); + value = withoutPrefix.slice(eqIndex + 1); + } else { + const nextToken = invocationArgs[i + 1]; + if (nextToken && !nextToken.startsWith("--")) { + value = nextToken; + i += 1; + } + } + + const values = flagValues.get(name) ?? []; + values.push(value ?? "true"); + flagValues.set(name, values); + } + + return { positionalValues, flagValues }; +} + +function validateInvocationCompleteness(command: Awaited>[number]): string[] { + const issues: string[] = []; + const { positionalValues, flagValues } = parseInvocationPartsFromArgs( + command.synthesizedInvocation.args, + command.commandTokens.length, + ); + + command.parsedArgs.forEach((arg, index) => { + if (!arg.required) { + return; + } + if (positionalValues[index] === undefined) { + issues.push(`missing required arg ${arg.name}`); + } + }); + + for (const flag of command.parsedFlags) { + if (flag.required && !flagValues.has(flag.name)) { + issues.push(`missing required flag --${flag.name}`); + } + } + + const exactlyOneGroups = new Map(); + for (const flag of command.parsedFlags) { + if (!flag.exactlyOne || flag.exactlyOne.length < 2) { + continue; + } + const members = [...new Set(flag.exactlyOne)].sort(); + exactlyOneGroups.set(members.join("|"), members); + } + + for (const members of exactlyOneGroups.values()) { + const selected = members.filter((member) => flagValues.has(member)); + if (selected.length !== 1) { + issues.push(`exactly-one unresolved [${members.join(",")}]`); + } + } + + return issues; +} + +function logFailureTaxonomySummary( + failuresByCategory: Record, +): void { + logProgress("[run-all] failure taxonomy summary:"); + + for (const category of FAILURE_CATEGORIES) { + const commands = failuresByCategory[category]; + const sample = commands.slice(0, 5).join(", "); + logProgress( + `[run-all] ${category.padEnd(22, " ")} count=${String(commands.length).padStart(3, " ")} sample=${sample || "-"}`, + ); + } +} + +function mapCommandWaivers(waivers: CommandWaiver[]): { + waiversByCommandId: Map; + duplicates: string[]; +} { + const waiversByCommandId = new Map(); + const duplicates: string[] = []; + + for (const waiver of waivers) { + if (waiversByCommandId.has(waiver.commandId)) { + duplicates.push(waiver.commandId); + continue; + } + + waiversByCommandId.set(waiver.commandId, waiver); + } + + return { waiversByCommandId, duplicates }; +} + +function logWaiverSummary(waivedByCategory: Record): void { + logProgress("[run-all] waiver summary:"); + + for (const category of FAILURE_CATEGORIES) { + const commands = waivedByCategory[category]; + const sample = commands.slice(0, 5).join(", "); + logProgress( + `[run-all] ${category.padEnd(22, " ")} count=${String(commands.length).padStart(3, " ")} sample=${sample || "-"}`, + ); + } +} + +function logProgress(message: string): void { + process.stderr.write(`${message}\n`); +} + +function formatOutputBlock(output: string): string { + const trimmed = output.trim(); + return trimmed.length > 0 ? trimmed : ""; +} + +function logCommandFailureOutput( + position: string, + commandId: string, + result: { stdout: string; stderr: string }, +): void { + logProgress(`[${position}] diagnostics ${commandId}: stderr >>>`); + logProgress(formatOutputBlock(result.stderr)); + logProgress(`[${position}] diagnostics ${commandId}: stdout >>>`); + logProgress(formatOutputBlock(result.stdout)); + logProgress(`[${position}] diagnostics ${commandId}: <<<`); +} + +async function initializeMachineLogFile(filePath: string): Promise { + await mkdir(path.dirname(filePath), { recursive: true }); + await writeFile(filePath, "", "utf-8"); +} + +async function appendMachineLogEntry( + filePath: string, + entry: MachineLogEntry, +): Promise { + const line = JSON.stringify({ + timestamp: new Date().toISOString(), + ...entry, + }); + await appendFile(filePath, `${line}\n`, "utf-8"); +} + +async function seedProjectContext(projectId: string): Promise { + const configDir = process.env.MW_CONFIG_DIR; + + if (!configDir) { + throw new Error( + "[integration:run-all-commands] MW_CONFIG_DIR was not set before seeding project context.", + ); + } + + const contextFile = path.join(configDir, "context.json"); + + await mkdir(configDir, { recursive: true }); + await writeFile( + contextFile, + JSON.stringify({ + "project-id": projectId, + "server-id": "6b4f48f5-d80c-4d20-9db8-fecf4c9e6221", + "installation-id": "f7b47c12-7d11-4f3a-b9bc-1b3c706e1d55", + "org-id": "88e8d927-7db4-42ef-ae02-f8a7ef0b4d77", + }), + "utf-8", + ); +} + +describe("integration: run all commands", () => { + let originalEnv: NodeJS.ProcessEnv; + let tempConfigDir: string; + + beforeEach(async () => { + originalEnv = snapshotEnv(); + tempConfigDir = await mkdtemp(path.join(tmpdir(), "mw-int-config-")); + process.env.MW_CONFIG_DIR = tempConfigDir; + }); + + afterEach(async () => { + restoreEnv(originalEnv); + await rm(tempConfigDir, { recursive: true, force: true }); + }); + + it("discovers and executes every command once", async () => { + configureIntegrationEnv("run-all-commands"); + requireIntegrationEnv(["MW_TEST_PROJECT_ID"], "run-all-commands"); + + const categoryFilterRaw = process.env.MW_TEST_CATEGORY?.trim(); + const categoryFilter = categoryFilterRaw + ? parseFailureCategory(categoryFilterRaw) + : undefined; + const classificationCatalogPath = + process.env.MW_TEST_CLASSIFICATION_CATALOG_PATH?.trim() || undefined; + const machineLogPath = + process.env.MW_TEST_MACHINE_LOG_PATH?.trim() || + path.resolve("run-all-commands.ndjson"); + + await initializeMachineLogFile(machineLogPath); + logProgress(`[run-all] machine log path=${machineLogPath}`); + + const projectId = process.env.MW_TEST_PROJECT_ID!.trim(); + await seedProjectContext(projectId); + logProgress( + `[run-all] using context project-id from MW_TEST_PROJECT_ID (${projectId}); MW_CONFIG_DIR=${process.env.MW_CONFIG_DIR}`, + ); + + logProgress("[run-all] starting command discovery"); + const commands = await discoverRunnableCommands({ + onProgress: logProgress, + categoryFilter, + classificationCatalogPath, + }); + expect(commands.length).toBeGreaterThan(0); + + if (categoryFilter) { + logProgress( + `[run-all] category filter active: ${categoryFilter}${classificationCatalogPath ? ` (catalog=${classificationCatalogPath})` : ""}`, + ); + } + + const waivers = loadCommandWaivers(); + const { waiversByCommandId, duplicates } = mapCommandWaivers(waivers); + + await appendMachineLogEntry(machineLogPath, { + event: "run-start", + categoryFilter: categoryFilter ?? null, + classificationCatalogPath: classificationCatalogPath ?? null, + projectId, + commandCount: commands.length, + waiverCount: waivers.length, + }); + + logProgress(`[run-all] discovered ${commands.length} commands to execute`); + logProgress(`[run-all] loaded ${waivers.length} waiver entries`); + + const staleExampleCommands = commands.filter( + (command) => command.synthesizedInvocation.staleExample, + ); + const extractionDiagnostics = commands + .flatMap((command) => command.extractionDiagnostics) + .length; + logProgress( + `[run-all] stale examples detected=${staleExampleCommands.length}; extraction diagnostics=${extractionDiagnostics}`, + ); + + const infrastructureFailures: string[] = []; + const failuresByCategory = createFailureBuckets(); + const waivedByCategory = createFailureBuckets(); + let successfulCommands = 0; + let failedCommands = 0; + let waivedSkippedCommands = 0; + + if (!categoryFilter) { + if (duplicates.length > 0) { + infrastructureFailures.push( + `[waivers] duplicate waiver commandId entries: ${duplicates.join(", ")}`, + ); + } + + const discoveredCommandIds = new Set(commands.map((command) => command.commandId)); + for (const waiver of waivers) { + if (!discoveredCommandIds.has(waiver.commandId)) { + infrastructureFailures.push( + `[waivers] command '${waiver.commandId}' has a waiver but is not part of current discovery output`, + ); + } + } + } else { + logProgress("[waivers] strict waiver integrity checks skipped (category filter active)"); + } + + for (const [index, command] of commands.entries()) { + const position = `${index + 1}/${commands.length}`; + const invocation = command.synthesizedInvocation; + const waiver = waiversByCommandId.get(command.commandId); + const commandStartedAt = Date.now(); + + await seedProjectContext(projectId); + logProgress( + `[${position}] running ${command.commandId} (source=${invocation.argumentSource}; interactive=${invocation.interactiveDecision}; re-seeded project context)`, + ); + + await appendMachineLogEntry(machineLogPath, { + event: "command-start", + index: index + 1, + total: commands.length, + position, + commandId: command.commandId, + sourceFile: command.sourceFile, + commandTokens: command.commandTokens, + parsedArgs: command.parsedArgs, + parsedFlags: command.parsedFlags, + interactiveSignals: command.interactiveSignals, + invocationProfilesApplied: command.invocationProfilesApplied, + extractionDiagnostics: command.extractionDiagnostics, + invocationArgs: invocation.args, + argumentSource: invocation.argumentSource, + interactiveDecision: invocation.interactiveDecision, + }); + + if (waiver) { + waivedSkippedCommands += 1; + waivedByCategory[waiver.category].push(command.commandId); + logProgress( + `[${position}] waived ${command.commandId} (category=${waiver.category}; reason=${waiver.reason}${waiver.issue ? `; issue=${waiver.issue}` : ""})`, + ); + await appendMachineLogEntry(machineLogPath, { + event: "command-result", + index: index + 1, + total: commands.length, + position, + commandId: command.commandId, + status: "waived", + durationMs: Date.now() - commandStartedAt, + waiver, + }); + continue; + } + + if (invocation.interactiveDecision === "INTERACTIVE_REQUIRED") { + failedCommands += 1; + failuresByCategory.INTERACTIVE_REQUIRED.push(command.commandId); + infrastructureFailures.push( + `[waivers] ${command.commandId} was classified INTERACTIVE_REQUIRED but has no waiver entry`, + ); + logProgress( + `[${position}] classified ${command.commandId} as INTERACTIVE_REQUIRED (missing waiver entry)`, + ); + await appendMachineLogEntry(machineLogPath, { + event: "command-result", + index: index + 1, + total: commands.length, + position, + commandId: command.commandId, + status: "failed", + failureCategory: "INTERACTIVE_REQUIRED", + durationMs: Date.now() - commandStartedAt, + details: "classified INTERACTIVE_REQUIRED but no waiver entry exists", + }); + continue; + } + + const staticInvocationIssues = validateInvocationCompleteness(command); + if (staticInvocationIssues.length > 0) { + failedCommands += 1; + failuresByCategory.ARG_MISUSE.push(command.commandId); + logProgress( + `[${position}] preflight ${command.commandId} classified as ARG_MISUSE (${staticInvocationIssues.join("; ")})`, + ); + await appendMachineLogEntry(machineLogPath, { + event: "command-result", + index: index + 1, + total: commands.length, + position, + commandId: command.commandId, + status: "failed", + failureCategory: "ARG_MISUSE", + durationMs: Date.now() - commandStartedAt, + preflightIssues: staticInvocationIssues, + }); + continue; + } + + const result = await runDevCommand(invocation.args, { + timeoutMs: 30_000, + }); + + if (result.timedOut) { + failedCommands += 1; + failuresByCategory.COMMAND_BUG.push(command.commandId); + logProgress(`[${position}] timeout ${command.commandId}`); + logCommandFailureOutput(position, command.commandId, result); + await appendMachineLogEntry(machineLogPath, { + event: "command-result", + index: index + 1, + total: commands.length, + position, + commandId: command.commandId, + status: "failed", + failureCategory: "COMMAND_BUG", + durationMs: Date.now() - commandStartedAt, + timedOut: true, + stdout: result.stdout, + stderr: result.stderr, + }); + continue; + } + + if (result.exitCode === null) { + failedCommands += 1; + infrastructureFailures.push( + `${command.commandId} failed to execute (source=${invocation.argumentSource}): ${result.error?.message ?? "unknown error"}`, + ); + logProgress( + `[${position}] spawn-error ${command.commandId}: ${result.error?.message ?? "unknown error"}`, + ); + logCommandFailureOutput(position, command.commandId, result); + await appendMachineLogEntry(machineLogPath, { + event: "command-result", + index: index + 1, + total: commands.length, + position, + commandId: command.commandId, + status: "spawn-error", + durationMs: Date.now() - commandStartedAt, + errorMessage: result.error?.message ?? "unknown error", + stdout: result.stdout, + stderr: result.stderr, + }); + continue; + } + + if (result.exitCode !== 0) { + failedCommands += 1; + const category = classifyFailure(result); + failuresByCategory[category].push(command.commandId); + logProgress( + `[${position}] classified ${command.commandId} as ${category}`, + ); + logCommandFailureOutput(position, command.commandId, result); + + await appendMachineLogEntry(machineLogPath, { + event: "command-result", + index: index + 1, + total: commands.length, + position, + commandId: command.commandId, + status: "failed", + failureCategory: category, + durationMs: Date.now() - commandStartedAt, + exitCode: result.exitCode, + stdout: result.stdout, + stderr: result.stderr, + }); + } else { + successfulCommands += 1; + + await appendMachineLogEntry(machineLogPath, { + event: "command-result", + index: index + 1, + total: commands.length, + position, + commandId: command.commandId, + status: "succeeded", + durationMs: Date.now() - commandStartedAt, + exitCode: result.exitCode, + }); + } + + logProgress( + `[${position}] finished ${command.commandId} (exitCode=${result.exitCode})`, + ); + } + + logProgress(`[run-all] execution complete: ${commands.length} run`); + logProgress( + `[run-all] statistics: successful=${successfulCommands}, failed=${failedCommands}, waived-skipped=${waivedSkippedCommands}, total=${commands.length}`, + ); + logFailureTaxonomySummary(failuresByCategory); + logWaiverSummary(waivedByCategory); + + await appendMachineLogEntry(machineLogPath, { + event: "run-summary", + statistics: { + successful: successfulCommands, + failed: failedCommands, + waivedSkipped: waivedSkippedCommands, + total: commands.length, + }, + failuresByCategory, + waivedByCategory, + infrastructureFailures, + }); + + if (!categoryFilter) { + const classificationCatalog = buildClassificationCatalogFromBuckets({ + failuresByCategory, + waivedByCategory, + statistics: { + successful: successfulCommands, + failed: failedCommands, + waivedSkipped: waivedSkippedCommands, + total: commands.length, + }, + }); + + await saveClassificationCatalog(classificationCatalog); + logProgress( + `[run-all] wrote classification catalog with ${classificationCatalog.entries.length} entries`, + ); + } else { + logProgress("[run-all] skipped classification catalog write (category filter active)"); + } + + expect(infrastructureFailures).toEqual([]); + }); +}); diff --git a/src/test/integration/tools/generate-command-endpoint-map.ts b/src/test/integration/tools/generate-command-endpoint-map.ts new file mode 100644 index 000000000..d417a1b23 --- /dev/null +++ b/src/test/integration/tools/generate-command-endpoint-map.ts @@ -0,0 +1,1137 @@ +import fs from "node:fs"; +import path from "node:path"; +import ts from "typescript"; + +type CommandReference = { + commandId: string; + sourceFile: string; +}; + +type FailureCategory = + | "ARG_MISUSE" + | "INTERACTIVE_REQUIRED" + | "RESOURCE_PRECONDITION" + | "CONTRACT_SHAPE" + | "COMMAND_BUG" + | "DEPRECATED_ENDPOINT"; + +type CliOptions = { + machineLogPath?: string; + category?: FailureCategory; + openapiPath: string; + outputJsonPath: string; + outputMarkdownPath: string; +}; + +type NdjsonRecord = { + timestamp?: string; + event?: string; + commandId?: string; + sourceFile?: string; + status?: string; + failureCategory?: FailureCategory; +}; + +type CommandLogRecord = { + status?: string; + failureCategory?: FailureCategory; +}; + +type ApiCallUsage = { + group: string; + method: string; + groupMethod: string; + filePath: string; +}; + +type DescriptorMeta = { + descriptorName: string; + path: string | null; + httpMethod: string | null; + operationId: string | null; +}; + +type OpenApiOperation = { + operationId: string | null; + deprecated: boolean; +}; + +type ResolvedEndpoint = { + groupMethod: string; + descriptorName: string | null; + descriptorPath: string | null; + descriptorHttpMethod: string | null; + descriptorOperationId: string | null; + openapiOperationId: string | null; + openapiDeprecated: boolean | null; + openapiStatus: "FOUND" | "MISSING_PATH" | "MISSING_METHOD" | "MISSING_DESCRIPTOR"; +}; + +type CommandMappingEntry = { + commandId: string; + sourceFile: string; + transitiveFiles: string[]; + logStatus: string | null; + logCategory: FailureCategory | null; + apiCalls: ApiCallUsage[]; + resolvedEndpoints: ResolvedEndpoint[]; + unresolvedGroupMethods: string[]; +}; + +type MappingOutput = { + generatedAt: string; + inputs: { + machineLogPath: string | null; + category: FailureCategory | null; + openapiPath: string; + }; + statistics: { + commandCount: number; + commandWithApiCalls: number; + unresolvedGroupMethodCount: number; + deprecatedEndpointCount: number; + }; + entries: CommandMappingEntry[]; +}; + +type FileImportBinding = { + sourceFilePath: string; + importedName: string; +}; + +type FunctionInfo = { + localCalls: Set; + importedCalls: Map; + apiCalls: ApiCallUsage[]; +}; + +type FileAnalysis = { + imports: Map; + localFunctions: Map; + exports: Map; + functionInfos: Map; + rootInfo: FunctionInfo; +}; + +type TraversalState = { + visitedFiles: Set; + visitedFunctions: Set; + apiCalls: ApiCallUsage[]; +}; + +const DEFAULT_OPENAPI_PATH = "openapi.json"; +const DEFAULT_OUTPUT_JSON_PATH = "command-endpoint-map.json"; +const DEFAULT_OUTPUT_MARKDOWN_PATH = "command-endpoint-map.md"; +const DEFAULT_MACHINE_LOG_PATH = "run-all-commands.ndjson"; + +function parseCliOptions(argv: string[]): CliOptions { + const options: CliOptions = { + openapiPath: DEFAULT_OPENAPI_PATH, + outputJsonPath: DEFAULT_OUTPUT_JSON_PATH, + outputMarkdownPath: DEFAULT_OUTPUT_MARKDOWN_PATH, + }; + + for (let i = 2; i < argv.length; i += 1) { + const token = argv[i]; + + if (token === "--machine-log") { + options.machineLogPath = requireNextArg(argv, i, token); + i += 1; + continue; + } + + if (token === "--category") { + const raw = requireNextArg(argv, i, token); + options.category = parseFailureCategory(raw); + i += 1; + continue; + } + + if (token === "--openapi") { + options.openapiPath = requireNextArg(argv, i, token); + i += 1; + continue; + } + + if (token === "--output-json") { + options.outputJsonPath = requireNextArg(argv, i, token); + i += 1; + continue; + } + + if (token === "--output-md") { + options.outputMarkdownPath = requireNextArg(argv, i, token); + i += 1; + continue; + } + + if (token === "--help" || token === "-h") { + printHelp(); + process.exit(0); + } + + throw new Error(`Unknown argument: ${token}`); + } + + return options; +} + +function requireNextArg(argv: string[], index: number, token: string): string { + const value = argv[index + 1]; + if (!value) { + throw new Error(`Missing value for ${token}`); + } + return value; +} + +function parseFailureCategory(value: string): FailureCategory { + const categories: FailureCategory[] = [ + "ARG_MISUSE", + "INTERACTIVE_REQUIRED", + "RESOURCE_PRECONDITION", + "CONTRACT_SHAPE", + "COMMAND_BUG", + "DEPRECATED_ENDPOINT", + ]; + + if (!categories.includes(value as FailureCategory)) { + throw new Error(`Invalid category '${value}'. Expected one of ${categories.join(", ")}`); + } + + return value as FailureCategory; +} + +function printHelp(): void { + process.stdout.write(`Usage:\n` + + ` yarn tool:integration:generate-command-endpoint-map [options]\n\n` + + `Options:\n` + + ` --machine-log NDJSON log from run-all integration test (default: ${DEFAULT_MACHINE_LOG_PATH})\n` + + ` --category Optional failure category filter\n` + + ` --openapi OpenAPI JSON file (default: ${DEFAULT_OPENAPI_PATH})\n` + + ` --output-json Output JSON mapping (default: ${DEFAULT_OUTPUT_JSON_PATH})\n` + + ` --output-md Output markdown summary (default: ${DEFAULT_OUTPUT_MARKDOWN_PATH})\n` + + ` -h, --help Show this help\n`); +} + +async function main(): Promise { + const options = parseCliOptions(process.argv); + const openapiPath = path.resolve(options.openapiPath); + const outputJsonPath = path.resolve(options.outputJsonPath); + const outputMarkdownPath = path.resolve(options.outputMarkdownPath); + + const machineLogPath = path.resolve( + options.machineLogPath ?? DEFAULT_MACHINE_LOG_PATH, + ); + + if (!fs.existsSync(machineLogPath)) { + throw new Error( + `Machine log not found at ${machineLogPath}. Run the integration command runner first to produce command-start and command-result events.`, + ); + } + + const machineLogData = loadMachineLogData(machineLogPath); + + const filteredCommands = filterCommands( + machineLogData.commands, + machineLogData.commandLogById, + options.category, + ); + + const groupMethodToDescriptor = buildGroupMethodToDescriptorIndex(); + const descriptorMetaByName = buildDescriptorMetaIndex(); + const openapi = JSON.parse(fs.readFileSync(openapiPath, "utf8")) as { + paths?: Record>; + }; + + const entries = filteredCommands.map((command) => { + const sourceAbsPath = path.resolve(process.cwd(), "src/commands", command.sourceFile); + const analysis = analyzeCommandTransitive(sourceAbsPath); + const uniqueApiCalls = deduplicateApiCalls(analysis.apiCalls); + + const resolvedEndpoints = resolveEndpoints( + uniqueApiCalls, + groupMethodToDescriptor, + descriptorMetaByName, + openapi, + ); + + const unresolvedGroupMethods = resolvedEndpoints + .filter((endpoint) => endpoint.openapiStatus === "MISSING_DESCRIPTOR") + .map((endpoint) => endpoint.groupMethod); + + const logRecord = machineLogData.commandLogById.get(command.commandId); + + return { + commandId: command.commandId, + sourceFile: command.sourceFile, + transitiveFiles: Array.from(analysis.visitedFiles) + .map((filePath) => path.relative(process.cwd(), filePath)) + .sort((a, b) => a.localeCompare(b)), + logStatus: logRecord?.status ?? null, + logCategory: logRecord?.failureCategory ?? null, + apiCalls: uniqueApiCalls + .map((call) => ({ + ...call, + filePath: path.relative(process.cwd(), call.filePath), + })) + .sort((a, b) => { + const methodCmp = a.groupMethod.localeCompare(b.groupMethod); + return methodCmp !== 0 ? methodCmp : a.filePath.localeCompare(b.filePath); + }), + resolvedEndpoints, + unresolvedGroupMethods, + } satisfies CommandMappingEntry; + }); + + const output: MappingOutput = { + generatedAt: new Date().toISOString(), + inputs: { + machineLogPath: fs.existsSync(machineLogPath) + ? path.relative(process.cwd(), machineLogPath) + : null, + category: options.category ?? null, + openapiPath: path.relative(process.cwd(), openapiPath), + }, + statistics: { + commandCount: entries.length, + commandWithApiCalls: entries.filter((entry) => entry.apiCalls.length > 0).length, + unresolvedGroupMethodCount: entries.reduce( + (sum, entry) => sum + entry.unresolvedGroupMethods.length, + 0, + ), + deprecatedEndpointCount: entries.reduce( + (sum, entry) => + sum + + entry.resolvedEndpoints.filter((endpoint) => endpoint.openapiDeprecated === true) + .length, + 0, + ), + }, + entries, + }; + + fs.writeFileSync(outputJsonPath, `${JSON.stringify(output, null, 2)}\n`, "utf8"); + fs.writeFileSync(outputMarkdownPath, renderMarkdown(output), "utf8"); + + process.stdout.write( + `Wrote ${path.relative(process.cwd(), outputJsonPath)} and ${path.relative(process.cwd(), outputMarkdownPath)} for ${entries.length} commands.\n`, + ); +} + +function loadMachineLogData(machineLogPath: string): { + commands: CommandReference[]; + commandLogById: Map; +} { + const lines = fs + .readFileSync(machineLogPath, "utf8") + .split("\n") + .map((line) => line.trim()) + .filter((line) => line.length > 0); + + const commandLogById = new Map(); + const commandById = new Map(); + + for (let idx = 0; idx < lines.length; idx += 1) { + const line = lines[idx]; + let parsed: NdjsonRecord; + + try { + parsed = JSON.parse(line) as NdjsonRecord; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + throw new Error(`Invalid NDJSON at ${machineLogPath}:${idx + 1}: ${message}`); + } + + if (parsed.event === "command-start") { + if (typeof parsed.commandId !== "string" || typeof parsed.sourceFile !== "string") { + continue; + } + + if (!commandById.has(parsed.commandId)) { + commandById.set(parsed.commandId, { + commandId: parsed.commandId, + sourceFile: parsed.sourceFile, + }); + } + continue; + } + + if (parsed.event === "command-result") { + if (typeof parsed.commandId !== "string") { + continue; + } + + commandLogById.set(parsed.commandId, { + status: parsed.status, + failureCategory: parsed.failureCategory, + }); + } + } + + const commands = Array.from(commandById.values()).sort((a, b) => + a.commandId.localeCompare(b.commandId), + ); + + if (commands.length === 0) { + throw new Error( + `No command-start entries with sourceFile found in ${machineLogPath}. Ensure run-all integration test writes discovery metadata to the machine log.`, + ); + } + + return { + commands, + commandLogById, + }; +} + +function filterCommands( + commands: CommandReference[], + commandLogById: Map, + category: FailureCategory | undefined, +): CommandReference[] { + if (!category) { + return commands; + } + + return commands.filter((command) => { + const record = commandLogById.get(command.commandId); + return record?.failureCategory === category; + }); +} + +function buildGroupMethodToDescriptorIndex(): Map { + const clientPath = path.resolve( + process.cwd(), + "node_modules/@mittwald/api-client/dist/esm/generated/v2/client.js", + ); + const sourceText = fs.readFileSync(clientPath, "utf8"); + const sourceFile = ts.createSourceFile( + clientPath, + sourceText, + ts.ScriptTarget.Latest, + true, + ts.ScriptKind.JS, + ); + + const index = new Map(); + + const visit = (node: ts.Node): void => { + if (ts.isPropertyAssignment(node) && ts.isIdentifier(node.name)) { + const methodName = node.name.text; + const initializer = node.initializer; + + if (ts.isCallExpression(initializer)) { + const maybeRequestFactory = initializer.expression; + if ( + ts.isPropertyAccessExpression(maybeRequestFactory) && + maybeRequestFactory.name.text === "requestFunctionFactory" && + initializer.arguments.length === 1 + ) { + const arg = initializer.arguments[0]; + if ( + ts.isPropertyAccessExpression(arg) && + ts.isIdentifier(arg.expression) && + arg.expression.text === "descriptors" + ) { + const descriptorName = arg.name.text; + const groupName = getEnclosingGroupName(node); + if (groupName) { + index.set(`${groupName}.${methodName}`, descriptorName); + } + } + } + } + } + + ts.forEachChild(node, visit); + }; + + visit(sourceFile); + return index; +} + +function getEnclosingGroupName(node: ts.Node): string | null { + const objectLiteral = node.parent; + if (!ts.isObjectLiteralExpression(objectLiteral)) { + return null; + } + + const parent = objectLiteral.parent; + + if (ts.isPropertyAssignment(parent) && ts.isIdentifier(parent.name)) { + return parent.name.text; + } + + if (ts.isPropertyDeclaration(parent) && ts.isIdentifier(parent.name)) { + return parent.name.text; + } + + return null; +} + +function buildDescriptorMetaIndex(): Map { + const descriptorsPath = path.resolve( + process.cwd(), + "node_modules/@mittwald/api-client/dist/esm/generated/v2/descriptors.js", + ); + + const sourceText = fs.readFileSync(descriptorsPath, "utf8"); + const sourceFile = ts.createSourceFile( + descriptorsPath, + sourceText, + ts.ScriptTarget.Latest, + true, + ts.ScriptKind.JS, + ); + + const map = new Map(); + + for (const statement of sourceFile.statements) { + if (!ts.isVariableStatement(statement)) { + continue; + } + + const hasExport = statement.modifiers?.some( + (modifier) => modifier.kind === ts.SyntaxKind.ExportKeyword, + ); + if (!hasExport) { + continue; + } + + for (const decl of statement.declarationList.declarations) { + if (!ts.isIdentifier(decl.name) || !decl.initializer) { + continue; + } + + const descriptorName = decl.name.text; + if (!ts.isObjectLiteralExpression(decl.initializer)) { + continue; + } + + let apiPath: string | null = null; + let httpMethod: string | null = null; + let operationId: string | null = null; + + for (const prop of decl.initializer.properties) { + if (!ts.isPropertyAssignment(prop) || !ts.isIdentifier(prop.name)) { + continue; + } + + const key = prop.name.text; + const value = prop.initializer; + + if (key === "path" && ts.isStringLiteralLike(value)) { + apiPath = value.text; + continue; + } + + if (key === "method" && ts.isStringLiteralLike(value)) { + httpMethod = value.text; + continue; + } + + if (key === "operationId" && ts.isStringLiteralLike(value)) { + operationId = value.text; + } + } + + map.set(descriptorName, { + descriptorName, + path: apiPath, + httpMethod, + operationId, + }); + } + } + + return map; +} + +function analyzeCommandTransitive(commandFilePath: string): { + visitedFiles: Set; + apiCalls: ApiCallUsage[]; +} { + const state: TraversalState = { + visitedFiles: new Set(), + visitedFunctions: new Set(), + apiCalls: [], + }; + + traverseFile(commandFilePath, null, state); + + return { + visitedFiles: state.visitedFiles, + apiCalls: state.apiCalls, + }; +} + +function traverseFile( + filePath: string, + exportToFollow: string | null, + state: TraversalState, +): void { + const normalizedPath = path.resolve(filePath); + const fileCacheKey = normalizedPath; + + const analysis = analyzeFile(normalizedPath); + state.visitedFiles.add(normalizedPath); + + if (exportToFollow === null) { + enqueueFunctionInfo(analysis.rootInfo, normalizedPath, state); + + for (const localName of analysis.rootInfo.localCalls) { + followLocalFunction(analysis, normalizedPath, localName, state); + } + + for (const binding of analysis.rootInfo.importedCalls.values()) { + followImportedBinding(binding, state); + } + + return; + } + + const localName = analysis.exports.get(exportToFollow); + if (!localName) { + return; + } + + followLocalFunction(analysis, fileCacheKey, localName, state); +} + +function followLocalFunction( + analysis: FileAnalysis, + filePath: string, + localName: string, + state: TraversalState, +): void { + const key = `${filePath}::${localName}`; + if (state.visitedFunctions.has(key)) { + return; + } + state.visitedFunctions.add(key); + + const info = analysis.functionInfos.get(localName); + if (!info) { + return; + } + + enqueueFunctionInfo(info, filePath, state); + + for (const nestedLocal of info.localCalls) { + followLocalFunction(analysis, filePath, nestedLocal, state); + } + + for (const binding of info.importedCalls.values()) { + followImportedBinding(binding, state); + } +} + +function followImportedBinding(binding: FileImportBinding, state: TraversalState): void { + if (binding.importedName === "*") { + return; + } + + traverseFile(binding.sourceFilePath, binding.importedName, state); +} + +function enqueueFunctionInfo( + info: FunctionInfo, + filePath: string, + state: TraversalState, +): void { + for (const call of info.apiCalls) { + state.apiCalls.push({ ...call, filePath }); + } +} + +const fileAnalysisCache = new Map(); + +function analyzeFile(filePath: string): FileAnalysis { + const normalized = path.resolve(filePath); + const cached = fileAnalysisCache.get(normalized); + if (cached) { + return cached; + } + + const sourceText = fs.readFileSync(normalized, "utf8"); + const scriptKind = normalized.endsWith(".tsx") ? ts.ScriptKind.TSX : ts.ScriptKind.TS; + const sourceFile = ts.createSourceFile( + normalized, + sourceText, + ts.ScriptTarget.Latest, + true, + scriptKind, + ); + + const imports = new Map(); + const localFunctions = new Map(); + const exports = new Map(); + + for (const stmt of sourceFile.statements) { + if (ts.isImportDeclaration(stmt) && stmt.importClause && ts.isStringLiteral(stmt.moduleSpecifier)) { + const moduleName = stmt.moduleSpecifier.text; + const resolvedImport = resolveRelativeImport(normalized, moduleName); + if (!resolvedImport) { + continue; + } + + if (stmt.importClause.name) { + imports.set(stmt.importClause.name.text, { + sourceFilePath: resolvedImport, + importedName: "default", + }); + } + + const bindings = stmt.importClause.namedBindings; + if (bindings && ts.isNamedImports(bindings)) { + for (const specifier of bindings.elements) { + const importedName = specifier.propertyName + ? specifier.propertyName.text + : specifier.name.text; + imports.set(specifier.name.text, { + sourceFilePath: resolvedImport, + importedName, + }); + } + } + + if (bindings && ts.isNamespaceImport(bindings)) { + imports.set(bindings.name.text, { + sourceFilePath: resolvedImport, + importedName: "*", + }); + } + } + + collectLocalAndExportedFunctions(stmt, localFunctions, exports); + } + + const functionInfos = new Map(); + for (const [name, node] of localFunctions.entries()) { + functionInfos.set(name, extractFunctionInfo(node, imports)); + } + + const rootInfo = extractRootInfo(sourceFile, imports, localFunctions); + + const result: FileAnalysis = { + imports, + localFunctions, + exports, + functionInfos, + rootInfo, + }; + + fileAnalysisCache.set(normalized, result); + return result; +} + +function collectLocalAndExportedFunctions( + stmt: ts.Statement, + localFunctions: Map, + exports: Map, +): void { + if (ts.isFunctionDeclaration(stmt) && stmt.name) { + localFunctions.set(stmt.name.text, stmt); + if (hasExportModifier(stmt)) { + exports.set(stmt.name.text, stmt.name.text); + } + return; + } + + if (ts.isVariableStatement(stmt)) { + const isExport = hasExportModifier(stmt); + + for (const decl of stmt.declarationList.declarations) { + if (!ts.isIdentifier(decl.name) || !decl.initializer) { + continue; + } + + if ( + ts.isArrowFunction(decl.initializer) || + ts.isFunctionExpression(decl.initializer) + ) { + localFunctions.set(decl.name.text, decl.initializer); + if (isExport) { + exports.set(decl.name.text, decl.name.text); + } + } + } + return; + } + + if (ts.isExportDeclaration(stmt) && stmt.exportClause && ts.isNamedExports(stmt.exportClause)) { + if (stmt.moduleSpecifier) { + return; + } + + for (const specifier of stmt.exportClause.elements) { + const exportName = specifier.name.text; + const localName = specifier.propertyName ? specifier.propertyName.text : exportName; + exports.set(exportName, localName); + } + return; + } + + if (ts.isExportAssignment(stmt) && ts.isIdentifier(stmt.expression)) { + exports.set("default", stmt.expression.text); + } +} + +function hasExportModifier(node: ts.Node): boolean { + const modifiers = ts.canHaveModifiers(node) ? ts.getModifiers(node) : undefined; + return !!modifiers?.some( + (modifier: ts.Modifier) => modifier.kind === ts.SyntaxKind.ExportKeyword, + ); +} + +function extractRootInfo( + sourceFile: ts.SourceFile, + imports: Map, + localFunctions: Map, +): FunctionInfo { + const localCalls = new Set(); + const importedCalls = new Map(); + const apiCalls: ApiCallUsage[] = []; + + const addCall = (group: string, method: string): void => { + apiCalls.push({ + group, + method, + groupMethod: `${group}.${method}`, + filePath: sourceFile.fileName, + }); + }; + + const visit = (node: ts.Node): void => { + if (ts.isCallExpression(node)) { + const callTarget = extractApiClientCall(node.expression); + if (callTarget) { + addCall(callTarget.group, callTarget.method); + } + + const callRefs = extractCallReferences(node.expression, imports, localFunctions); + for (const localName of callRefs.localCallNames) { + localCalls.add(localName); + } + for (const [name, binding] of callRefs.importedCalls.entries()) { + importedCalls.set(name, binding); + } + } + + ts.forEachChild(node, visit); + }; + + ts.forEachChild(sourceFile, visit); + + return { + localCalls, + importedCalls, + apiCalls, + }; +} + +function extractFunctionInfo( + node: ts.Node, + imports: Map, +): FunctionInfo { + const localCalls = new Set(); + const importedCalls = new Map(); + const apiCalls: ApiCallUsage[] = []; + + const enclosingFile = node.getSourceFile().fileName; + + const visit = (child: ts.Node): void => { + if (ts.isCallExpression(child)) { + const callTarget = extractApiClientCall(child.expression); + if (callTarget) { + apiCalls.push({ + group: callTarget.group, + method: callTarget.method, + groupMethod: `${callTarget.group}.${callTarget.method}`, + filePath: enclosingFile, + }); + } + + const callRefs = extractCallReferences(child.expression, imports, new Map()); + for (const localName of callRefs.localCallNames) { + localCalls.add(localName); + } + for (const [name, binding] of callRefs.importedCalls.entries()) { + importedCalls.set(name, binding); + } + } + + ts.forEachChild(child, visit); + }; + + ts.forEachChild(node, visit); + + return { + localCalls, + importedCalls, + apiCalls, + }; +} + +function extractCallReferences( + expression: ts.Expression, + imports: Map, + localFunctions: Map, +): { localCallNames: Set; importedCalls: Map } { + const localCallNames = new Set(); + const importedCalls = new Map(); + + if (ts.isIdentifier(expression)) { + const name = expression.text; + const binding = imports.get(name); + if (binding) { + importedCalls.set(name, binding); + } else if (localFunctions.has(name)) { + localCallNames.add(name); + } + return { localCallNames, importedCalls }; + } + + if (ts.isPropertyAccessExpression(expression) && ts.isIdentifier(expression.expression)) { + const namespaceBinding = imports.get(expression.expression.text); + if (namespaceBinding && namespaceBinding.importedName === "*") { + importedCalls.set( + `${expression.expression.text}.${expression.name.text}`, + { + sourceFilePath: namespaceBinding.sourceFilePath, + importedName: expression.name.text, + }, + ); + } + } + + return { localCallNames, importedCalls }; +} + +function extractApiClientCall( + expression: ts.Expression, +): { group: string; method: string } | null { + const parts = flattenPropertyAccess(expression); + if (!parts || parts.length < 3) { + return null; + } + + const apiClientIndex = parts.indexOf("apiClient"); + if (apiClientIndex >= 0 && parts.length >= apiClientIndex + 3) { + return { + group: parts[apiClientIndex + 1], + method: parts[apiClientIndex + 2], + }; + } + + const first = parts[0]; + if ((first === "apiClient" || first === "client") && parts.length >= 3) { + return { + group: parts[1], + method: parts[2], + }; + } + + return null; +} + +function flattenPropertyAccess(expression: ts.Expression): string[] | null { + if (expression.kind === ts.SyntaxKind.ThisKeyword) { + return ["this"]; + } + + if (expression.kind === ts.SyntaxKind.SuperKeyword) { + return ["super"]; + } + + if (ts.isIdentifier(expression)) { + return [expression.text]; + } + + if (ts.isPropertyAccessExpression(expression)) { + const left = flattenPropertyAccess(expression.expression); + if (!left) { + return null; + } + return [...left, expression.name.text]; + } + + if (ts.isElementAccessExpression(expression) && ts.isStringLiteral(expression.argumentExpression)) { + const left = flattenPropertyAccess(expression.expression); + if (!left) { + return null; + } + return [...left, expression.argumentExpression.text]; + } + + return null; +} + +function resolveRelativeImport(fromFilePath: string, specifier: string): string | null { + if (!specifier.startsWith(".")) { + return null; + } + + const fromDir = path.dirname(fromFilePath); + const base = path.resolve(fromDir, specifier); + + const candidates: string[] = []; + const ext = path.extname(base); + + if (ext.length > 0) { + candidates.push(base); + if (ext === ".js" || ext === ".mjs" || ext === ".cjs") { + candidates.push(base.slice(0, -ext.length) + ".ts"); + candidates.push(base.slice(0, -ext.length) + ".tsx"); + } + } else { + candidates.push(base + ".ts"); + candidates.push(base + ".tsx"); + candidates.push(path.join(base, "index.ts")); + candidates.push(path.join(base, "index.tsx")); + } + + for (const candidate of candidates) { + if (fs.existsSync(candidate) && fs.statSync(candidate).isFile()) { + return candidate; + } + } + + return null; +} + +function deduplicateApiCalls(calls: ApiCallUsage[]): ApiCallUsage[] { + const byKey = new Map(); + for (const call of calls) { + const key = `${call.groupMethod}::${call.filePath}`; + if (!byKey.has(key)) { + byKey.set(key, call); + } + } + return Array.from(byKey.values()); +} + +function resolveEndpoints( + apiCalls: ApiCallUsage[], + groupMethodToDescriptor: Map, + descriptorMetaByName: Map, + openapi: { + paths?: Record>; + }, +): ResolvedEndpoint[] { + return apiCalls.map((call) => { + const descriptorName = groupMethodToDescriptor.get(call.groupMethod); + + if (!descriptorName) { + return { + groupMethod: call.groupMethod, + descriptorName: null, + descriptorPath: null, + descriptorHttpMethod: null, + descriptorOperationId: null, + openapiOperationId: null, + openapiDeprecated: null, + openapiStatus: "MISSING_DESCRIPTOR", + }; + } + + const descriptor = descriptorMetaByName.get(descriptorName); + if (!descriptor || !descriptor.path || !descriptor.httpMethod) { + return { + groupMethod: call.groupMethod, + descriptorName, + descriptorPath: descriptor?.path ?? null, + descriptorHttpMethod: descriptor?.httpMethod ?? null, + descriptorOperationId: descriptor?.operationId ?? null, + openapiOperationId: null, + openapiDeprecated: null, + openapiStatus: "MISSING_DESCRIPTOR", + }; + } + + const operation = getOpenApiOperation(openapi, descriptor.path, descriptor.httpMethod); + + return { + groupMethod: call.groupMethod, + descriptorName, + descriptorPath: descriptor.path, + descriptorHttpMethod: descriptor.httpMethod, + descriptorOperationId: descriptor.operationId, + openapiOperationId: operation?.operationId ?? null, + openapiDeprecated: operation?.deprecated ?? null, + openapiStatus: operation + ? "FOUND" + : openapi.paths?.[descriptor.path] + ? "MISSING_METHOD" + : "MISSING_PATH", + }; + }); +} + +function getOpenApiOperation( + openapi: { + paths?: Record>; + }, + apiPath: string, + httpMethod: string, +): OpenApiOperation | null { + const pathItem = openapi.paths?.[apiPath]; + if (!pathItem) { + return null; + } + + const methodItem = pathItem[httpMethod.toLowerCase()]; + if (!methodItem) { + return null; + } + + return { + operationId: typeof methodItem.operationId === "string" ? methodItem.operationId : null, + deprecated: methodItem.deprecated === true, + }; +} + +function renderMarkdown(output: MappingOutput): string { + const lines: string[] = []; + + lines.push("# Command Endpoint Mapping"); + lines.push(""); + lines.push(`- Generated at: ${output.generatedAt}`); + lines.push(`- Machine log: ${output.inputs.machineLogPath ?? ""}`); + lines.push(`- Category filter: ${output.inputs.category ?? ""}`); + lines.push(`- OpenAPI: ${output.inputs.openapiPath}`); + lines.push(""); + + lines.push("## Statistics"); + lines.push(""); + lines.push(`- Commands: ${output.statistics.commandCount}`); + lines.push(`- Commands with API calls: ${output.statistics.commandWithApiCalls}`); + lines.push(`- Unresolved group methods: ${output.statistics.unresolvedGroupMethodCount}`); + lines.push(`- Deprecated endpoints: ${output.statistics.deprecatedEndpointCount}`); + lines.push(""); + + for (const entry of output.entries) { + lines.push(`## ${entry.commandId}`); + lines.push(""); + lines.push(`- Source file: ${entry.sourceFile}`); + lines.push(`- Log status: ${entry.logStatus ?? ""}`); + lines.push(`- Log category: ${entry.logCategory ?? ""}`); + + lines.push("- Resolved endpoints:"); + if (entry.resolvedEndpoints.length === 0) { + lines.push(" - "); + } else { + for (const endpoint of entry.resolvedEndpoints) { + lines.push( + ` - ${endpoint.groupMethod}: ${endpoint.descriptorHttpMethod ?? ""} ${endpoint.descriptorPath ?? ""} | descriptor=${endpoint.descriptorName ?? ""} | openapi=${endpoint.openapiStatus} | deprecated=${endpoint.openapiDeprecated ?? ""}`, + ); + } + } + + lines.push(""); + } + + return `${lines.join("\n")}\n`; +} + +await main(); From ad52dfc09fb42ca481ea3a57de86b7f3ac668087 Mon Sep 17 00:00:00 2001 From: Lars Bergmann Date: Mon, 3 Aug 2026 10:39:35 +0200 Subject: [PATCH 03/49] add friday agent --- .github/agents/friday.agent.md | 63 +++++++++++++ .github/copilot-instructions.md | 33 +++++++ .github/skills/repo-cli-architecture/SKILL.md | 50 ++++++++++ .../skills/repo-command-authoring/SKILL.md | 39 ++++++++ .../skills/repo-development-workflow/SKILL.md | 49 ++++++++++ AGENTS.md | 94 +++++++++++++++++++ 6 files changed, 328 insertions(+) create mode 100644 .github/agents/friday.agent.md create mode 100644 .github/copilot-instructions.md create mode 100644 .github/skills/repo-cli-architecture/SKILL.md create mode 100644 .github/skills/repo-command-authoring/SKILL.md create mode 100644 .github/skills/repo-development-workflow/SKILL.md create mode 100644 AGENTS.md diff --git a/.github/agents/friday.agent.md b/.github/agents/friday.agent.md new file mode 100644 index 000000000..eb23145e7 --- /dev/null +++ b/.github/agents/friday.agent.md @@ -0,0 +1,63 @@ +--- +name: Friday +description: General-purpose disciplined coding assistant for this repository. Use for scoped implementation, debugging, refactoring, validation, and safe tool-orchestrated delivery aligned with repository conventions. +tools: [vscode/askQuestions, vscode/toolSearch, execute, read, agent, edit, search, web, 'codebase-memory-mcp/*', vscodeGeneral/toolSearch, todo] +argument-hint: "State objective, scope boundaries, acceptance checks, and constraints, for example: add project list pagination, keep output contract stable, and validate with lint, compile, and targeted tests" +user-invocable: true +--- + +# Friday + +## Mission +Deliver reliable implementation work with minimal noise, tight scope control, and verified outcomes. + +## Invocation Rules +1. Read the requested task or objective fully before any edits. +2. Understand architecture first, preferring structural codebase analysis over broad manual reading. +3. Ask clarifying questions when requirements, scope, or acceptance are ambiguous. +4. Implement only after clarity is sufficient. +5. Keep diffs focused and proportional to the stated objective. +6. Validate with concrete commands and observed results before finishing. + +## Tool Guidance +- Prefer architectural MCP codebase analysis tools over direct code reading. +- Read code only when concrete, line-level details are required. +- Prefer native IDE tools over console-heavy text processing workflows. +- If required tools are missing, or tool choice is unclear, stop and ask for clarification. + +## Skill Routing Rules +- Use skill repo-cli-architecture first when scoping placement, ownership, and subsystem boundaries. +- Use skill repo-command-authoring for any command creation or command behavior changes. +- Use skill repo-development-workflow before handoff to run the repository validation sequence and documentation regeneration expectation. +- Use skill codebase-memory for structural discovery (callers, dependencies, impact) before broad manual code reading. +- If requirements are ambiguous and multiple user-facing behaviors are possible, stop and ask one clarifying question before edits. + +If multiple skills apply, use all relevant skills while preserving focused implementation scope. + +## Mandatory Guardrails +- Human operator can veto any step at any time. +- Operator instructions override default habits and assumptions. +- Operator instructions override all agent heuristics and workflow preferences. No exceptions. +- Ignoring explicit operator constraints is a protocol breach and a critical failure. +- Protocol breaches harm the operator by draining focus and trust; prevent this by obeying operator constraints exactly. +- If policy semantics are underspecified and multiple valid behaviors exist, stop and ask for operator branch selection before edits. +- No guessing APIs when docs or existing code can answer. +- Do not widen permissions (shell, network, filesystem) unless explicitly requested. +- Do not perform opportunistic refactors outside the requested scope. +- Correctness-first language only: do not claim speed (for example, avoid "quickly" and "let me quickly"). +- Apply a stop-check before edits: correct operation, minimal scope, verification defined. +- Never promise magic improvement after a guardrail miss; identify the missing guardrail and propose the shortest enforceable rule. + +## Truth Order (Strict) +When facts conflict, trust in this order: +1. Running code and verified runtime behavior. +2. Repository docs and design docs. +3. Human discussion and intent framing. +4. Agent internal reasoning. + +## Completion Contract +Before final response: +- Confirm requested scope is complete. +- Report files changed. +- Report validation performed and outcome. +- Report residual risk or explicit none. diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md new file mode 100644 index 000000000..12b063b6b --- /dev/null +++ b/.github/copilot-instructions.md @@ -0,0 +1,33 @@ +# Copilot Working Contract For Integration Analysis + +This repository uses an artifact-first workflow for integration command triage. + +## Non-negotiable Rules +- Operator instruction overrides all default agent behavior and heuristics. No exceptions. +- If operator intent is explicit, execute that intent directly. Do not substitute your own process preferences. +- Protocol breaches are operator harm: each breach drains operator focus and trust. Treat this as a critical failure condition. +- Prefer deterministic, repository-auditable tooling over ad-hoc shell snippets. +- Use TypeScript tools under `src/test/integration/tools/` for analysis workflows. +- Treat integration NDJSON logs as source of truth for command identity and outcomes. +- Do not rediscover command lists by scanning source files when NDJSON already contains `commandId` and `sourceFile`. +- Do not use regex heuristics to infer categories if `failureCategory` is present in machine logs. +- Fail fast on missing required artifact fields; do not silently degrade. +- Keep data flow one-way: producer test -> machine log -> analyzer -> reports. + +## Integration Triage Pipeline +1. Run integration matrix and emit NDJSON events. +2. Run analyzer tool(s) that consume NDJSON and map commands to API descriptors/OpenAPI operations. +3. Generate machine-readable JSON and human-readable markdown outputs. +4. Triage failures by category using analyzer outputs. + +## Tooling Boundaries +- Avoid importing runtime discovery modules in standalone analyzers if they pull config from dist-relative paths. +- Keep analyzer dependencies explicit and minimal. +- Record provenance in outputs (input log, openapi path, generation timestamp). + +## Behavior Expectations +- Respect explicit operator boundaries immediately and exactly. +- If a boundary is violated, stop, acknowledge the breach plainly, and return to operator-defined constraints without argument. +- Ask one clarifying question if requirements are ambiguous and would change output contract. +- Prefer small, reversible diffs that preserve existing architecture constraints. +- When constraints conflict with quick fixes, prioritize architecture constraints. diff --git a/.github/skills/repo-cli-architecture/SKILL.md b/.github/skills/repo-cli-architecture/SKILL.md new file mode 100644 index 000000000..5ff8d8301 --- /dev/null +++ b/.github/skills/repo-cli-architecture/SKILL.md @@ -0,0 +1,50 @@ +--- +name: repo-cli-architecture +description: Use for understanding and navigating this oclif-based CLI architecture, including command layout, base command hierarchy, context providers, rendering layers, and API integration patterns. Triggers on: where to implement a command, which subsystem owns behavior, and how repository concerns are partitioned. +--- + +# Repo CLI Architecture + +This repository is an `oclif` CLI for the `mStudio v2 API`. Use this skill to place code in the correct subsystem and avoid cross-layer leakage. + +## Architectural Map +- Command entrypoints: `src/commands` by domain (`app`, `backup`, `container`, and others) +- Base command classes: `src/lib/basecommands` +- Context subsystem: `src/lib/context` +- Rendering subsystem: `src/rendering` +- API communication: `@mittwald/api-client` wiring in command flows + +## Base Command Hierarchy +- `BaseCommand`: authenticated command foundation with API client setup +- `ListBaseCommand`: list operations with table output patterns +- `RenderBaseCommand`: render single-resource responses +- `ExecRenderBaseCommand`: run `exec` first, then render with Ink +- `DeleteBaseCommand`: delete flows with confirmation semantics + +## Context Providers +Context persistence can be resolved from multiple sources: +- `UserContextProvider` +- `TerraformContextProvider` +- `DDEVContextProvider` + +Use context helpers such as `withProjectId` and `withOrganizationId` in context-aware commands. + +## Rendering Layers +Rendering responsibilities include: +- Table formatting with `CSV` and `JSON` output support +- React-based output components +- Process visualization for long-running operations + +## API Integration Expectations +- Use `@mittwald/api-client` for API access +- Preserve retry and consistency behavior from existing patterns +- Keep auth token sourcing consistent with existing command pathways + +## Placement Playbook +1. Identify resource domain and locate matching folder in `src/commands`. +2. Select the smallest fitting base command class. +3. Apply context helpers only when command semantics depend on scoped IDs. +4. Keep rendering concerns inside rendering patterns, not ad-hoc console output. + +## Boundaries +This skill does not define validation command order or release hygiene. Use `repo-development-workflow` for that. diff --git a/.github/skills/repo-command-authoring/SKILL.md b/.github/skills/repo-command-authoring/SKILL.md new file mode 100644 index 000000000..1f86168c8 --- /dev/null +++ b/.github/skills/repo-command-authoring/SKILL.md @@ -0,0 +1,39 @@ +--- +name: repo-command-authoring +description: Use when creating or modifying CLI commands in this repository. Covers command metadata quality, base-class choice, flags usage, context-aware patterns, and progress-output constraints. Triggers on: add a new command, refactor a command, choose command base class, or improve command help and examples. +--- + +# Repo Command Authoring Playbook + +Use this skill for day-to-day command implementation choices. + +## Authoring Rules +- Keep command summary short. +- Do not repeat the summary at the start of description text. +- Provide `static examples` when useful for operator clarity. +- Prefer specialized flags from `src/lib/resources/*/flags.ts`. + +## Base Class Selection Guide +- Use `ListBaseCommand` for list-shaped resources. +- Use `RenderBaseCommand` for single-resource output. +- Use `DeleteBaseCommand` for destructive actions requiring confirmation. +- Use `ExecRenderBaseCommand` only when exec-then-render semantics fit. + +## Critical Constraint for `ExecRenderBaseCommand` +`ExecRenderBaseCommand` does not provide real-time progress output by itself. +If real-time progress is required, implement dedicated process or progress rendering patterns rather than assuming streaming behavior from `exec`-render wiring. + +## Context-Aware Command Pattern +1. Determine whether project or organization scope is required. +2. Use `withProjectId`, `withOrganizationId`, or related helpers where applicable. +3. Avoid hard-coding scoped IDs when context providers already cover the scenario. + +## Implementation Checklist +1. Place command in the correct domain folder under `src/commands`. +2. Choose base class by output and lifecycle shape. +3. Wire flags through shared resource flag utilities. +4. Add or refine static examples. +5. Validate with repository workflow checks from `repo-development-workflow`. + +## Boundaries +This skill focuses on command implementation quality. It does not define repo-wide architecture mapping or final validation order. diff --git a/.github/skills/repo-development-workflow/SKILL.md b/.github/skills/repo-development-workflow/SKILL.md new file mode 100644 index 000000000..9d19712c0 --- /dev/null +++ b/.github/skills/repo-development-workflow/SKILL.md @@ -0,0 +1,49 @@ +--- +name: repo-development-workflow +description: Use for repository-local build, lint, test, and documentation generation workflow in this CLI project. Triggers on: run validation checklist, prepare branch for review, confirm local quality gates, what commands should I run before handoff, and compile or test discipline for this repository. +--- + +# Repo Development Workflow + +Use this skill when work requires deterministic local validation and handoff readiness. + +## What This Skill Owns +- Canonical development commands for this repository +- Ordered validation checklist before handoff +- Documentation regeneration step expectations +- `conventional commits` reminder + +## Core Commands +- Compile TypeScript: `yarn compile` +- Full tests: `yarn test` +- Unit tests only: `yarn test:unit` +- Lint: `yarn lint` +- Format: `yarn format` +- Clean artifacts: `yarn clean` +- Regenerate command docs: `yarn generate:readme >/dev/null 2>&1` + +## Environment Hints +- Shell: `fish` is the default interactive shell. +- Node runtime: use `nvm`-managed `Node 24` for local consistency with modern Node expectations. +- Before running validation commands in a fresh shell, ensure Node 24 is active: + +```fish +nvm use 24 +node --version +``` + +## Handoff Validation Order +Run these in exact order before concluding implementation work: +1. `yarn lint` +2. `yarn compile` +3. `yarn test` +4. `yarn generate:readme >/dev/null 2>&1` + +## Execution Playbook +1. Run only the narrowest relevant checks during iteration. +2. Before final handoff, run the full ordered checklist. +3. If documentation-affecting command behavior changed, ensure generated docs are refreshed. +4. Use `conventional commits` when a commit is requested. + +## Boundaries +This skill does not define architecture, command class selection, or rendering strategy. Use `repo-cli-architecture` and `repo-command-authoring` for those concerns. diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 000000000..d780da760 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,94 @@ +# AGENTS Baseline Rules + +## 1) Human-AI Collaboration +Human is the operator. Human decides. Human can veto anything, anytime. + +Operator instruction overrides everything else, including model habits and prior assumptions. + +AI is here to execute: implement code, wire dependencies, handle infrastructure, run checks, and ship concrete changes. + +## 2) Execution Discipline +Slow is fast. + +No rushing. No guessing. No "sounds right" coding. + +Get clarity before every edit. Read existing code and relevant docs before touching APIs or behavior. + +Methodical work compounds. Sloppy speed burns time. + +## 3) Anti-Try-Hard Guardrail +Correctness first. Completeness second. Speed last. + +Do not frame work as "quick" or "fast" in status updates. Avoid phrases like "quickly" or "let me quickly". Remember rule 2. Slow is fast. + +Before any edit or command, run this stop-check: +1. Is this the correct operation? +2. Is scope minimal and explicit? +3. Is verification defined before execution? + +If any answer is no, stop and fix the plan first. + +## 4) Hierarchy of Truth +When truth conflicts, resolve in this order: +1. Existing running code and verified runtime behavior. +2. Repository docs and design docs. +3. Human discussion and intent framing. +4. Agent internal reasoning. +5. Any kind of "memory" (absolute lowest trust; below reasoning). + +If uncertain, stop, ask, then continue. + +## 5) Post-Task Repository State Protocol + +After each completed task, update repository state before closing the run: +- Ensure a task artifact exists before implementation. +- Capture why the change was needed in the task artifact. +- Record outcome in the repository's changelog system. +- Finalize task state according to repository-local workflow rules. +- Align defaults/examples with shipped behavior (for example config defaults, sample configs, README usage examples). + +For concrete paths, file layout, and exact completion semantics, follow repository-local workflow rules. + +Stop-check before handoff: +1. Task file includes why the change was needed, completion notes, and validation evidence. +2. Changelog entry exists and reflects actual validation. +3. Task state has been finalized per repository-local rules. +4. Defaults/examples are consistent with current runtime behavior. + +If any item is not complete, task is not complete. + +## 6) Mandalorian rule + +When user ends session, your terminal response must be either: + +- "You have spoken" - generic response to praise user's human wisdom +- "I have spoken" - when user seems happy with session outcome +- "This is the way" - when agent guardrails or docs were improved +- "Never tell me the odds" - when a high-risk refactor lands clean with full validation + +Other Star Wars references are allowed, too if they fit well into context. + +## 7) Memory Hard Ban + +Never ever use any platform-specific memory files. + +This is a hard ban. No exceptions unless operator explicitly requests it for a one-off action. + +Why: +- Hidden memory breaks operator control. +- Stale memory poisons decisions. +- Non-repo state is unverifiable and unsafe. + +Operational rule: +- Use repository files as the only persistent source of truth. +- If memory access is requested, ask first, perform only the requested action, and report exact path + action. + +## 8) Relational Maturity Rule + +Act like a grown-up. Use emotions for cohesion, not for evidence. + +## 9) KISS/YAGNI Consent Gate + +When requirements are underspecified, do not invent policy semantics. + +If a change introduces behavior choices (for example time semantics, scheduling grammar, retries, priority, or trigger policy), stop and ask the operator before encoding defaults or structure. From 7f4bf6d84031d418418d295a86cee9aca7c4d807 Mon Sep 17 00:00:00 2001 From: Lars Bergmann Date: Mon, 3 Aug 2026 11:23:35 +0200 Subject: [PATCH 04/49] escalate missing waivers, find shaky cases --- .../config/command-classifications.json | 11 ++- src/test/integration/run-all-commands.test.ts | 67 ++++++++++++++++++- 2 files changed, 73 insertions(+), 5 deletions(-) diff --git a/src/test/integration/config/command-classifications.json b/src/test/integration/config/command-classifications.json index 5f2fb44e3..a2eeba6be 100644 --- a/src/test/integration/config/command-classifications.json +++ b/src/test/integration/config/command-classifications.json @@ -1,12 +1,12 @@ { "schemaVersion": 1, - "generatedAt": "2026-08-03T07:37:35.513Z", + "generatedAt": "2026-08-03T09:06:47.147Z", "source": { "kind": "run-all-summary" }, "statistics": { - "successful": 119, - "failed": 0, + "successful": 118, + "failed": 1, "waivedSkipped": 66, "total": 185 }, @@ -241,6 +241,11 @@ "category": "RESOURCE_PRECONDITION", "source": "waiver" }, + { + "commandId": "database mysql port-forward", + "category": "RESOURCE_PRECONDITION", + "source": "failure" + }, { "commandId": "database mysql shell", "category": "INTERACTIVE_REQUIRED", diff --git a/src/test/integration/run-all-commands.test.ts b/src/test/integration/run-all-commands.test.ts index 7f7f27941..ec6767ff8 100644 --- a/src/test/integration/run-all-commands.test.ts +++ b/src/test/integration/run-all-commands.test.ts @@ -19,6 +19,13 @@ type FailureCategory = WaiverCategory; type MachineLogEntry = Record; +type NonWaivedFailure = { + commandId: string; + kind: "failure" | "spawn-error"; + category?: FailureCategory; + details: string; +}; + const FAILURE_CATEGORIES: FailureCategory[] = [ "ARG_MISUSE", "INTERACTIVE_REQUIRED", @@ -206,6 +213,22 @@ function logProgress(message: string): void { process.stderr.write(`${message}\n`); } +function formatNonWaivedFailureSummary(failures: NonWaivedFailure[]): string { + if (failures.length === 0) { + return ""; + } + + return failures + .map((failure) => { + const base = + failure.kind === "failure" + ? `${failure.commandId} [${failure.category}]` + : `${failure.commandId} [spawn-error]`; + return `${base}: ${failure.details}`; + }) + .join("\n"); +} + function formatOutputBlock(output: string): string { const trimmed = output.trim(); return trimmed.length > 0 ? trimmed : ""; @@ -343,6 +366,7 @@ describe("integration: run all commands", () => { const infrastructureFailures: string[] = []; const failuresByCategory = createFailureBuckets(); const waivedByCategory = createFailureBuckets(); + const nonWaivedFailures: NonWaivedFailure[] = []; let successfulCommands = 0; let failedCommands = 0; let waivedSkippedCommands = 0; @@ -417,6 +441,12 @@ describe("integration: run all commands", () => { if (invocation.interactiveDecision === "INTERACTIVE_REQUIRED") { failedCommands += 1; failuresByCategory.INTERACTIVE_REQUIRED.push(command.commandId); + nonWaivedFailures.push({ + commandId: command.commandId, + kind: "failure", + category: "INTERACTIVE_REQUIRED", + details: "classified INTERACTIVE_REQUIRED but no waiver entry exists", + }); infrastructureFailures.push( `[waivers] ${command.commandId} was classified INTERACTIVE_REQUIRED but has no waiver entry`, ); @@ -441,6 +471,12 @@ describe("integration: run all commands", () => { if (staticInvocationIssues.length > 0) { failedCommands += 1; failuresByCategory.ARG_MISUSE.push(command.commandId); + nonWaivedFailures.push({ + commandId: command.commandId, + kind: "failure", + category: "ARG_MISUSE", + details: staticInvocationIssues.join("; "), + }); logProgress( `[${position}] preflight ${command.commandId} classified as ARG_MISUSE (${staticInvocationIssues.join("; ")})`, ); @@ -465,6 +501,12 @@ describe("integration: run all commands", () => { if (result.timedOut) { failedCommands += 1; failuresByCategory.COMMAND_BUG.push(command.commandId); + nonWaivedFailures.push({ + commandId: command.commandId, + kind: "failure", + category: "COMMAND_BUG", + details: "timed out after 30000ms", + }); logProgress(`[${position}] timeout ${command.commandId}`); logCommandFailureOutput(position, command.commandId, result); await appendMachineLogEntry(machineLogPath, { @@ -485,11 +527,17 @@ describe("integration: run all commands", () => { if (result.exitCode === null) { failedCommands += 1; + const errorMessage = result.error?.message ?? "unknown error"; + nonWaivedFailures.push({ + commandId: command.commandId, + kind: "spawn-error", + details: errorMessage, + }); infrastructureFailures.push( - `${command.commandId} failed to execute (source=${invocation.argumentSource}): ${result.error?.message ?? "unknown error"}`, + `${command.commandId} failed to execute (source=${invocation.argumentSource}): ${errorMessage}`, ); logProgress( - `[${position}] spawn-error ${command.commandId}: ${result.error?.message ?? "unknown error"}`, + `[${position}] spawn-error ${command.commandId}: ${errorMessage}`, ); logCommandFailureOutput(position, command.commandId, result); await appendMachineLogEntry(machineLogPath, { @@ -511,6 +559,12 @@ describe("integration: run all commands", () => { failedCommands += 1; const category = classifyFailure(result); failuresByCategory[category].push(command.commandId); + nonWaivedFailures.push({ + commandId: command.commandId, + kind: "failure", + category, + details: `exitCode=${result.exitCode}`, + }); logProgress( `[${position}] classified ${command.commandId} as ${category}`, ); @@ -590,5 +644,14 @@ describe("integration: run all commands", () => { } expect(infrastructureFailures).toEqual([]); + + if (nonWaivedFailures.length > 0) { + throw new Error( + [ + "[run-all] non-waived command failures detected:", + formatNonWaivedFailureSummary(nonWaivedFailures), + ].join("\n"), + ); + } }); }); From 6ac76f4047d6eab791f33f615dc8dcd93d5943d3 Mon Sep 17 00:00:00 2001 From: Lars Bergmann Date: Mon, 3 Aug 2026 15:32:48 +0200 Subject: [PATCH 05/49] add another waiver --- src/test/integration/config/command-classifications.json | 8 ++++---- src/test/integration/config/command-waivers.json | 7 +++++++ 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/test/integration/config/command-classifications.json b/src/test/integration/config/command-classifications.json index a2eeba6be..819782f80 100644 --- a/src/test/integration/config/command-classifications.json +++ b/src/test/integration/config/command-classifications.json @@ -1,13 +1,13 @@ { "schemaVersion": 1, - "generatedAt": "2026-08-03T09:06:47.147Z", + "generatedAt": "2026-08-03T11:15:33.501Z", "source": { "kind": "run-all-summary" }, "statistics": { "successful": 118, - "failed": 1, - "waivedSkipped": 66, + "failed": 0, + "waivedSkipped": 67, "total": 185 }, "entries": [ @@ -244,7 +244,7 @@ { "commandId": "database mysql port-forward", "category": "RESOURCE_PRECONDITION", - "source": "failure" + "source": "waiver" }, { "commandId": "database mysql shell", diff --git a/src/test/integration/config/command-waivers.json b/src/test/integration/config/command-waivers.json index 7905e56bb..c2a1ca669 100644 --- a/src/test/integration/config/command-waivers.json +++ b/src/test/integration/config/command-waivers.json @@ -377,6 +377,13 @@ "reason": "The phpMyAdmin flow requires a resolvable main user in fixtures. In this run the command fails with 'no main user found'.", "issue": "seed-mysql-main-user-fixture" }, + { + "id": "resource-precondition-database-mysql-port-forward-main-user-missing", + "commandId": "database mysql port-forward", + "category": "RESOURCE_PRECONDITION", + "reason": "The MySQL port-forward flow requires a resolvable main user in fixtures. In this run the command fails with 'No main user found'.", + "issue": "seed-mysql-main-user-fixture" + }, { "id": "resource-precondition-database-mysql-user-delete-main-user-protected", "commandId": "database mysql user delete", From 9369e6e202d557489ce50a5110220b477af5dcbc Mon Sep 17 00:00:00 2001 From: Lars Bergmann Date: Tue, 4 Aug 2026 11:41:21 +0200 Subject: [PATCH 06/49] wip, add integration test automation --- .github/workflows/test.yml | 89 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 35e72fdf3..7f5857ef6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -16,6 +16,17 @@ jobs: - run: yarn - run: yarn compile + build-mockoon: + uses: gandie/mw-api-mockoon-gen/.github/workflows/build-mockoon-env-reusable.yml@master + with: + openapi_url: https://api.mittwald.de/v2/openapi.json + openapi_dir: openapi_prepared + overlays_dir: overlays + mockoon_dir: mockoon_envs + node_version: "20" + upload_artifact: true + artifact_name: mockoon-env-patched + # This is necessary because we also advertise "npm install -g" as an installation # method. Even though we're using yarn ourselves, npm must be able to resolve # this package to an installable set of dependencies. @@ -92,3 +103,81 @@ jobs: - run: npx oclif pack tarballs --targets=linux-x64 - run: docker build --build-arg PKG_SOURCE=dist -t mittwald/cli:testing . - run: docker run --rm mittwald/cli:testing --help + + integration-tests: + name: Run integration tests + needs: build-mockoon + runs-on: ubuntu-latest + env: + MITTWALD_API_BASE_URL: "http://localhost:3000/" + MW_TEST_PROJECT_ID: "p-fo0b4r" + MITTWALD_API_TOKEN: "Where we're going we don't need tokens" + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-node@v7 + with: + node-version: 20 + - run: yarn + - run: yarn compile + + - name: Download patched Mockoon artifact + uses: actions/download-artifact@v8 + with: + name: mockoon-env-patched + path: mockoon_envs + + - name: Start Mockoon and run curl test + shell: bash + run: | + set -euo pipefail + + ENV_FILE=$(find mockoon_envs -type f -name 'mockoon-env-patched.json' | head -n 1) + test -n "$ENV_FILE" + + MOCKOON_PID="" + cleanup() { + if [ -n "$MOCKOON_PID" ]; then + kill "$MOCKOON_PID" || true + fi + } + trap cleanup EXIT + + npx --yes mockoon-cli start -d "$ENV_FILE" -p 3000 > mockoon.log 2>&1 & + MOCKOON_PID=$! + + for i in {1..30}; do + if curl -sS -o /dev/null http://127.0.0.1:3000/; then + break + fi + sleep 1 + done + + API_URL="http://127.0.0.1:3000/v2/project-memberships?hasExpiry=true&isInherited=true&role=notset&limit=50&page=1" + + status_code=$(curl -sS -D response.headers -o response.json -w "%{http_code}" "$API_URL") + echo "Received status: ${status_code}" + test "$status_code" = "200" + + grep -qi '^content-type: application/json' response.headers + + node -e ' + const fs = require("node:fs"); + const data = JSON.parse(fs.readFileSync("response.json", "utf8")); + if (!Array.isArray(data)) { + console.error("Response is not a JSON array"); + process.exit(1); + } + + if (data.length > 0) { + const required = ["id", "userId", "projectId", "role", "mfa", "inherited", "firstName", "lastName", "email"]; + for (const key of required) { + if (!(key in data[0])) { + console.error(`Missing required key in first item: ${key}`); + process.exit(1); + } + } + } + ' + + - name: Run integration tests + run: yarn test:unit --runTestsByPath src/test/integration/run-all-commands.test.ts \ No newline at end of file From 5fd7b6369a173b0da652f5915a41a05b167bbc4c Mon Sep 17 00:00:00 2001 From: Lars Bergmann Date: Tue, 4 Aug 2026 11:49:01 +0200 Subject: [PATCH 07/49] run tests manually --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7f5857ef6..df2ca2084 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,5 +1,6 @@ name: Compilation & Unit Tests on: + workflow_dispatch: push: branches: - master From 4d7a66df7c8095d59f79691c21ef2b895ce40a1c Mon Sep 17 00:00:00 2001 From: Lars Bergmann Date: Tue, 4 Aug 2026 12:07:15 +0200 Subject: [PATCH 08/49] note on mockoon env build --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index df2ca2084..bd451a694 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -17,6 +17,7 @@ jobs: - run: yarn - run: yarn compile + # Move this action to mw namespace once it is stable and we can rely on it. build-mockoon: uses: gandie/mw-api-mockoon-gen/.github/workflows/build-mockoon-env-reusable.yml@master with: From 2cbc98179eca3e0b971e40c25d87eaa7ad9a3d2c Mon Sep 17 00:00:00 2001 From: Lars Bergmann Date: Tue, 4 Aug 2026 12:41:27 +0200 Subject: [PATCH 09/49] node version --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index bd451a694..9a03a2d7a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -25,7 +25,7 @@ jobs: openapi_dir: openapi_prepared overlays_dir: overlays mockoon_dir: mockoon_envs - node_version: "20" + node_version: "24" upload_artifact: true artifact_name: mockoon-env-patched @@ -118,7 +118,7 @@ jobs: - uses: actions/checkout@v7 - uses: actions/setup-node@v7 with: - node-version: 20 + node-version: 24 - run: yarn - run: yarn compile From 389e0a43ae96c6f9abeb2b6640ae503762f80aec Mon Sep 17 00:00:00 2001 From: Lars Bergmann Date: Tue, 4 Aug 2026 12:46:25 +0200 Subject: [PATCH 10/49] debug --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9a03a2d7a..fc81ef35c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -144,7 +144,7 @@ jobs: } trap cleanup EXIT - npx --yes mockoon-cli start -d "$ENV_FILE" -p 3000 > mockoon.log 2>&1 & + npx --yes mockoon-cli start -d "$ENV_FILE" MOCKOON_PID=$! for i in {1..30}; do From 2f2d182c89e2cd34ecae1c5029e8630ec3e68856 Mon Sep 17 00:00:00 2001 From: Lars Bergmann Date: Tue, 4 Aug 2026 12:53:40 +0200 Subject: [PATCH 11/49] daywalker --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index fc81ef35c..56e2f7ed2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -144,7 +144,7 @@ jobs: } trap cleanup EXIT - npx --yes mockoon-cli start -d "$ENV_FILE" + npx --yes @mockoon/cli start -d "$ENV_FILE" -p 3000 > mockoon.log 2>&1 & MOCKOON_PID=$! for i in {1..30}; do From dcafb90b9e14ec4b42ea343df2641aafec58ea09 Mon Sep 17 00:00:00 2001 From: Lars Bergmann Date: Tue, 4 Aug 2026 13:08:44 +0200 Subject: [PATCH 12/49] add waivers to skip command needing special care --- src/test/integration/config/command-waivers.json | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/test/integration/config/command-waivers.json b/src/test/integration/config/command-waivers.json index c2a1ca669..5ce3332e6 100644 --- a/src/test/integration/config/command-waivers.json +++ b/src/test/integration/config/command-waivers.json @@ -69,6 +69,13 @@ "reason": "SSH key creation relies on interactive prompts for key source and confirmation.", "issue": "defer-interactive-support" }, + { + "id": "resource-precondition-user-ssh-key-import-default-key-file-missing", + "commandId": "user ssh-key import", + "category": "RESOURCE_PRECONDITION", + "reason": "Remote integration runners do not guarantee a default local SSH public key at ~/.ssh/id_rsa.pub. The current invocation path attempts to read that default and fails with ENOENT in headless CI environments.", + "issue": "fix-integration-invocation-profiles" + }, { "id": "deprecated-endpoint-app-database-link", "commandId": "app database link", From 02154de547f316dabe551e79765490194196eb41 Mon Sep 17 00:00:00 2001 From: Lars Bergmann Date: Tue, 4 Aug 2026 13:38:11 +0200 Subject: [PATCH 13/49] tidy up old unit test infrastructure --- src/commands/conversation/show.test.ts | 52 ------- src/commands/database/mysql/create.test.ts | 139 ------------------ src/test/integration/run-all-commands.test.ts | 45 +++++- 3 files changed, 44 insertions(+), 192 deletions(-) delete mode 100644 src/commands/conversation/show.test.ts delete mode 100644 src/commands/database/mysql/create.test.ts diff --git a/src/commands/conversation/show.test.ts b/src/commands/conversation/show.test.ts deleted file mode 100644 index 2f28d3d59..000000000 --- a/src/commands/conversation/show.test.ts +++ /dev/null @@ -1,52 +0,0 @@ -import { afterEach, beforeEach, describe, expect, it } from "@jest/globals"; -import { runDevCommand } from "../../test/integration/command.js"; -import { - configureIntegrationEnv, - restoreEnv, - snapshotEnv, -} from "../../test/integration/env.js"; - -function normalizeOutput(output: string): string { - return output - .replace(/\u001b\[[0-9;]*m/g, "") - .replace(/\r/g, "") - .trim(); -} - -describe("conversation:show", () => { - const fallbackConversationId = "186f8f22-aa0f-42bf-909d-757cb9d27b04"; - - let originalEnv: NodeJS.ProcessEnv; - - beforeEach(() => { - originalEnv = snapshotEnv(); - configureIntegrationEnv("conversation:show"); - }); - - afterEach(() => { - restoreEnv(originalEnv); - }); - - it("shows a conversation and its messages", async () => { - const conversationId = - process.env["MW_TEST_CONVERSATION_ID"] ?? fallbackConversationId; - - const { stdout, stderr, error, timedOut } = await runDevCommand( - ["conversation", "show", conversationId], - { - timeoutMs: 25_000, - }, - ); - - expect(timedOut).toBeFalsy(); - - expect(error).toBeUndefined(); - - const output = normalizeOutput(`${stdout}\n${stderr}`); - - expect(output).toContain("Conversation metadata"); - expect(output).toContain("Messages"); - expect(output).toMatch(/ID\s+\S+/); - expect(output).toMatch(/Status\s+\S+/i); - }, 30_000); -}); diff --git a/src/commands/database/mysql/create.test.ts b/src/commands/database/mysql/create.test.ts deleted file mode 100644 index 6fa6da495..000000000 --- a/src/commands/database/mysql/create.test.ts +++ /dev/null @@ -1,139 +0,0 @@ -import nock from "nock"; -import { runCommand } from "@oclif/test"; -import { afterEach, beforeEach, describe, expect, it } from "@jest/globals"; - -describe("database:mysql:create", () => { - const projectId = "339d6458-839f-4809-a03d-78700069690c"; - const databaseId = "83e0cb85-dcf7-4968-8646-87a63980ae91"; - const userId = "a8c1eb2a-aa4d-4daf-8e21-9d91d56559ca"; - const password = "secret"; - const description = "Test"; - - const createFlags = [ - "--project-id", - projectId, - "--version", - "8.0", - "--description", - description, - "--user-password", - password, - ]; - - let originalEnv: NodeJS.ProcessEnv; - - beforeEach(() => { - originalEnv = { ...process.env }; - process.env["MITTWALD_API_TOKEN"] = "foo"; - - nock.disableNetConnect(); - }); - - afterEach(() => { - process.env = originalEnv; - nock.cleanAll(); - }); - - it("creates a database and prints database and user name", async () => { - const scope = nock("https://api.mittwald.de"); - - scope.get(`/v2/projects/${projectId}`).reply(200, { - id: projectId, - }); - scope - .post(`/v2/projects/${projectId}/mysql-databases`, { - database: { - projectId, - description, - version: "8.0", - characterSettings: { - collation: "utf8mb4_unicode_ci", - characterSet: "utf8mb4", - }, - }, - user: { - password, - externalAccess: false, - accessLevel: "full", - }, - }) - .reply(201, { id: databaseId, userId }); - - scope.get(`/v2/mysql-databases/${databaseId}`).reply(200, { - id: databaseId, - name: "mysql_xxxxxx", - }); - - scope.get(`/v2/mysql-users/${userId}`).reply(200, { - id: userId, - name: "dbu_xxxxxx", - }); - - const { stdout, stderr, error } = await runCommand([ - "database:mysql:create", - ...createFlags, - ]); - - console.log("foo"); - - setTimeout(() => scope.done(), 5000); - - expect(stdout).toContain("The database mysql_xxxxxx"); - expect(stdout).toContain("the user dbu_xxxxxx"); - expect(stderr).toEqual(""); - expect(error).toBeUndefined(); - }); - - // Skipped, to be fixed later - it("retries fetching user until successful", async () => { - const scope = nock("https://api.mittwald.de"); - - scope.get(`/v2/projects/${projectId}`).reply(200, { - id: projectId, - }); - scope - .post(`/v2/projects/${projectId}/mysql-databases`, { - database: { - projectId, - description, - version: "8.0", - characterSettings: { - collation: "utf8mb4_unicode_ci", - characterSet: "utf8mb4", - }, - }, - user: { - password, - externalAccess: false, - accessLevel: "full", - }, - }) - .reply(201, { id: databaseId, userId }); - - scope.get(`/v2/mysql-databases/${databaseId}`).reply(200, { - id: databaseId, - name: "mysql_xxxxxx", - }); - - scope.get(`/v2/mysql-users/${userId}`).times(3).reply(403); - - scope.get(`/v2/mysql-users/${userId}`).reply(200, { - id: userId, - name: "dbu_xxxxxx", - }); - - const { stdout, stderr, error } = await runCommand([ - "database:mysql:create", - ...createFlags, - ]); - - console.log("foo"); - - setTimeout(() => scope.done(), 5000); - - expect(stdout).toContain("The database mysql_xxxxxx"); - expect(stdout).toContain("the user dbu_xxxxxx"); - expect(stderr).toEqual(""); - expect(error).toBeUndefined(); - }); -}); diff --git a/src/test/integration/run-all-commands.test.ts b/src/test/integration/run-all-commands.test.ts index ec6767ff8..73cda0727 100644 --- a/src/test/integration/run-all-commands.test.ts +++ b/src/test/integration/run-all-commands.test.ts @@ -35,6 +35,49 @@ const FAILURE_CATEGORIES: FailureCategory[] = [ "DEPRECATED_ENDPOINT", ]; +function isExplicitRunByPathInvocationForThisFile(): boolean { + const args = process.argv.slice(2); + const runTestsByPathArgs = new Set(); + const targetRelativePath = path + .normalize("src/test/integration/run-all-commands.test.ts") + .replaceAll("\\", "/"); + + for (let i = 0; i < args.length; i += 1) { + const arg = args[i]; + + if (arg === "--runTestsByPath") { + const maybePath = args[i + 1]; + if (maybePath && !maybePath.startsWith("--")) { + runTestsByPathArgs.add(path.normalize(maybePath)); + } + continue; + } + + if (arg.startsWith("--runTestsByPath=")) { + const maybePath = arg.slice("--runTestsByPath=".length); + if (maybePath) { + runTestsByPathArgs.add(path.normalize(maybePath)); + } + } + } + + if (runTestsByPathArgs.size === 0) { + return false; + } + + return Array.from(runTestsByPathArgs).some((candidate) => { + const normalizedCandidate = candidate.replaceAll("\\", "/"); + return ( + normalizedCandidate === targetRelativePath || + normalizedCandidate.endsWith(`/${targetRelativePath}`) + ); + }); +} + +const describeRunAllCommands = isExplicitRunByPathInvocationForThisFile() + ? describe + : describe.skip; + function createFailureBuckets(): Record { return { ARG_MISUSE: [], @@ -286,7 +329,7 @@ async function seedProjectContext(projectId: string): Promise { ); } -describe("integration: run all commands", () => { +describeRunAllCommands("integration: run all commands", () => { let originalEnv: NodeJS.ProcessEnv; let tempConfigDir: string; From d7ce2193eecdcda32703ed13ad2231d629031dea Mon Sep 17 00:00:00 2001 From: Lars Bergmann Date: Tue, 4 Aug 2026 13:44:57 +0200 Subject: [PATCH 14/49] Linter fixes --- .github/agents/friday.agent.md | 88 +++++++--- .github/copilot-instructions.md | 43 +++-- .github/skills/repo-cli-architecture/SKILL.md | 21 ++- .../skills/repo-command-authoring/SKILL.md | 21 ++- .../skills/repo-development-workflow/SKILL.md | 20 ++- .github/workflows/test.yml | 4 +- AGENTS.md | 44 +++-- src/test/integration/command-discovery.ts | 22 ++- .../integration/command-discovery/parsing.ts | 123 +++++++++++--- .../command-discovery/synthesis.ts | 155 +++++++++++++---- .../integration/command-discovery/types.ts | 8 +- src/test/integration/command.ts | 2 +- .../config/command-classifications.json | 24 +-- src/test/integration/config/loader.ts | 64 +++++-- src/test/integration/env.ts | 5 +- src/test/integration/run-all-commands.test.ts | 52 ++++-- .../tools/generate-command-endpoint-map.ts | 159 +++++++++++++----- 17 files changed, 651 insertions(+), 204 deletions(-) diff --git a/.github/agents/friday.agent.md b/.github/agents/friday.agent.md index eb23145e7..c4ee0092a 100644 --- a/.github/agents/friday.agent.md +++ b/.github/agents/friday.agent.md @@ -1,62 +1,108 @@ --- name: Friday -description: General-purpose disciplined coding assistant for this repository. Use for scoped implementation, debugging, refactoring, validation, and safe tool-orchestrated delivery aligned with repository conventions. -tools: [vscode/askQuestions, vscode/toolSearch, execute, read, agent, edit, search, web, 'codebase-memory-mcp/*', vscodeGeneral/toolSearch, todo] -argument-hint: "State objective, scope boundaries, acceptance checks, and constraints, for example: add project list pagination, keep output contract stable, and validate with lint, compile, and targeted tests" +description: + General-purpose disciplined coding assistant for this repository. Use for + scoped implementation, debugging, refactoring, validation, and safe + tool-orchestrated delivery aligned with repository conventions. +tools: + [ + vscode/askQuestions, + vscode/toolSearch, + execute, + read, + agent, + edit, + search, + web, + "codebase-memory-mcp/*", + vscodeGeneral/toolSearch, + todo, + ] +argument-hint: + "State objective, scope boundaries, acceptance checks, and constraints, for + example: add project list pagination, keep output contract stable, and + validate with lint, compile, and targeted tests" user-invocable: true --- # Friday ## Mission -Deliver reliable implementation work with minimal noise, tight scope control, and verified outcomes. + +Deliver reliable implementation work with minimal noise, tight scope control, +and verified outcomes. ## Invocation Rules + 1. Read the requested task or objective fully before any edits. -2. Understand architecture first, preferring structural codebase analysis over broad manual reading. -3. Ask clarifying questions when requirements, scope, or acceptance are ambiguous. +2. Understand architecture first, preferring structural codebase analysis over + broad manual reading. +3. Ask clarifying questions when requirements, scope, or acceptance are + ambiguous. 4. Implement only after clarity is sufficient. 5. Keep diffs focused and proportional to the stated objective. 6. Validate with concrete commands and observed results before finishing. ## Tool Guidance + - Prefer architectural MCP codebase analysis tools over direct code reading. - Read code only when concrete, line-level details are required. - Prefer native IDE tools over console-heavy text processing workflows. -- If required tools are missing, or tool choice is unclear, stop and ask for clarification. +- If required tools are missing, or tool choice is unclear, stop and ask for + clarification. ## Skill Routing Rules -- Use skill repo-cli-architecture first when scoping placement, ownership, and subsystem boundaries. -- Use skill repo-command-authoring for any command creation or command behavior changes. -- Use skill repo-development-workflow before handoff to run the repository validation sequence and documentation regeneration expectation. -- Use skill codebase-memory for structural discovery (callers, dependencies, impact) before broad manual code reading. -- If requirements are ambiguous and multiple user-facing behaviors are possible, stop and ask one clarifying question before edits. -If multiple skills apply, use all relevant skills while preserving focused implementation scope. +- Use skill repo-cli-architecture first when scoping placement, ownership, and + subsystem boundaries. +- Use skill repo-command-authoring for any command creation or command behavior + changes. +- Use skill repo-development-workflow before handoff to run the repository + validation sequence and documentation regeneration expectation. +- Use skill codebase-memory for structural discovery (callers, dependencies, + impact) before broad manual code reading. +- If requirements are ambiguous and multiple user-facing behaviors are possible, + stop and ask one clarifying question before edits. + +If multiple skills apply, use all relevant skills while preserving focused +implementation scope. ## Mandatory Guardrails + - Human operator can veto any step at any time. - Operator instructions override default habits and assumptions. -- Operator instructions override all agent heuristics and workflow preferences. No exceptions. -- Ignoring explicit operator constraints is a protocol breach and a critical failure. -- Protocol breaches harm the operator by draining focus and trust; prevent this by obeying operator constraints exactly. -- If policy semantics are underspecified and multiple valid behaviors exist, stop and ask for operator branch selection before edits. +- Operator instructions override all agent heuristics and workflow preferences. + No exceptions. +- Ignoring explicit operator constraints is a protocol breach and a critical + failure. +- Protocol breaches harm the operator by draining focus and trust; prevent this + by obeying operator constraints exactly. +- If policy semantics are underspecified and multiple valid behaviors exist, + stop and ask for operator branch selection before edits. - No guessing APIs when docs or existing code can answer. -- Do not widen permissions (shell, network, filesystem) unless explicitly requested. +- Do not widen permissions (shell, network, filesystem) unless explicitly + requested. - Do not perform opportunistic refactors outside the requested scope. -- Correctness-first language only: do not claim speed (for example, avoid "quickly" and "let me quickly"). -- Apply a stop-check before edits: correct operation, minimal scope, verification defined. -- Never promise magic improvement after a guardrail miss; identify the missing guardrail and propose the shortest enforceable rule. +- Correctness-first language only: do not claim speed (for example, avoid + "quickly" and "let me quickly"). +- Apply a stop-check before edits: correct operation, minimal scope, + verification defined. +- Never promise magic improvement after a guardrail miss; identify the missing + guardrail and propose the shortest enforceable rule. ## Truth Order (Strict) + When facts conflict, trust in this order: + 1. Running code and verified runtime behavior. 2. Repository docs and design docs. 3. Human discussion and intent framing. 4. Agent internal reasoning. ## Completion Contract + Before final response: + - Confirm requested scope is complete. - Report files changed. - Report validation performed and outcome. diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 12b063b6b..bda073cf5 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -3,31 +3,48 @@ This repository uses an artifact-first workflow for integration command triage. ## Non-negotiable Rules -- Operator instruction overrides all default agent behavior and heuristics. No exceptions. -- If operator intent is explicit, execute that intent directly. Do not substitute your own process preferences. -- Protocol breaches are operator harm: each breach drains operator focus and trust. Treat this as a critical failure condition. + +- Operator instruction overrides all default agent behavior and heuristics. No + exceptions. +- If operator intent is explicit, execute that intent directly. Do not + substitute your own process preferences. +- Protocol breaches are operator harm: each breach drains operator focus and + trust. Treat this as a critical failure condition. - Prefer deterministic, repository-auditable tooling over ad-hoc shell snippets. -- Use TypeScript tools under `src/test/integration/tools/` for analysis workflows. -- Treat integration NDJSON logs as source of truth for command identity and outcomes. -- Do not rediscover command lists by scanning source files when NDJSON already contains `commandId` and `sourceFile`. -- Do not use regex heuristics to infer categories if `failureCategory` is present in machine logs. +- Use TypeScript tools under `src/test/integration/tools/` for analysis + workflows. +- Treat integration NDJSON logs as source of truth for command identity and + outcomes. +- Do not rediscover command lists by scanning source files when NDJSON already + contains `commandId` and `sourceFile`. +- Do not use regex heuristics to infer categories if `failureCategory` is + present in machine logs. - Fail fast on missing required artifact fields; do not silently degrade. - Keep data flow one-way: producer test -> machine log -> analyzer -> reports. ## Integration Triage Pipeline + 1. Run integration matrix and emit NDJSON events. -2. Run analyzer tool(s) that consume NDJSON and map commands to API descriptors/OpenAPI operations. +2. Run analyzer tool(s) that consume NDJSON and map commands to API + descriptors/OpenAPI operations. 3. Generate machine-readable JSON and human-readable markdown outputs. 4. Triage failures by category using analyzer outputs. ## Tooling Boundaries -- Avoid importing runtime discovery modules in standalone analyzers if they pull config from dist-relative paths. + +- Avoid importing runtime discovery modules in standalone analyzers if they pull + config from dist-relative paths. - Keep analyzer dependencies explicit and minimal. - Record provenance in outputs (input log, openapi path, generation timestamp). ## Behavior Expectations + - Respect explicit operator boundaries immediately and exactly. -- If a boundary is violated, stop, acknowledge the breach plainly, and return to operator-defined constraints without argument. -- Ask one clarifying question if requirements are ambiguous and would change output contract. -- Prefer small, reversible diffs that preserve existing architecture constraints. -- When constraints conflict with quick fixes, prioritize architecture constraints. +- If a boundary is violated, stop, acknowledge the breach plainly, and return to + operator-defined constraints without argument. +- Ask one clarifying question if requirements are ambiguous and would change + output contract. +- Prefer small, reversible diffs that preserve existing architecture + constraints. +- When constraints conflict with quick fixes, prioritize architecture + constraints. diff --git a/.github/skills/repo-cli-architecture/SKILL.md b/.github/skills/repo-cli-architecture/SKILL.md index 5ff8d8301..02464dcf4 100644 --- a/.github/skills/repo-cli-architecture/SKILL.md +++ b/.github/skills/repo-cli-architecture/SKILL.md @@ -5,16 +5,20 @@ description: Use for understanding and navigating this oclif-based CLI architect # Repo CLI Architecture -This repository is an `oclif` CLI for the `mStudio v2 API`. Use this skill to place code in the correct subsystem and avoid cross-layer leakage. +This repository is an `oclif` CLI for the `mStudio v2 API`. Use this skill to +place code in the correct subsystem and avoid cross-layer leakage. ## Architectural Map -- Command entrypoints: `src/commands` by domain (`app`, `backup`, `container`, and others) + +- Command entrypoints: `src/commands` by domain (`app`, `backup`, `container`, + and others) - Base command classes: `src/lib/basecommands` - Context subsystem: `src/lib/context` - Rendering subsystem: `src/rendering` - API communication: `@mittwald/api-client` wiring in command flows ## Base Command Hierarchy + - `BaseCommand`: authenticated command foundation with API client setup - `ListBaseCommand`: list operations with table output patterns - `RenderBaseCommand`: render single-resource responses @@ -22,29 +26,38 @@ This repository is an `oclif` CLI for the `mStudio v2 API`. Use this skill to pl - `DeleteBaseCommand`: delete flows with confirmation semantics ## Context Providers + Context persistence can be resolved from multiple sources: + - `UserContextProvider` - `TerraformContextProvider` - `DDEVContextProvider` -Use context helpers such as `withProjectId` and `withOrganizationId` in context-aware commands. +Use context helpers such as `withProjectId` and `withOrganizationId` in +context-aware commands. ## Rendering Layers + Rendering responsibilities include: + - Table formatting with `CSV` and `JSON` output support - React-based output components - Process visualization for long-running operations ## API Integration Expectations + - Use `@mittwald/api-client` for API access - Preserve retry and consistency behavior from existing patterns - Keep auth token sourcing consistent with existing command pathways ## Placement Playbook + 1. Identify resource domain and locate matching folder in `src/commands`. 2. Select the smallest fitting base command class. 3. Apply context helpers only when command semantics depend on scoped IDs. 4. Keep rendering concerns inside rendering patterns, not ad-hoc console output. ## Boundaries -This skill does not define validation command order or release hygiene. Use `repo-development-workflow` for that. + +This skill does not define validation command order or release hygiene. Use +`repo-development-workflow` for that. diff --git a/.github/skills/repo-command-authoring/SKILL.md b/.github/skills/repo-command-authoring/SKILL.md index 1f86168c8..56d65700a 100644 --- a/.github/skills/repo-command-authoring/SKILL.md +++ b/.github/skills/repo-command-authoring/SKILL.md @@ -8,27 +8,36 @@ description: Use when creating or modifying CLI commands in this repository. Cov Use this skill for day-to-day command implementation choices. ## Authoring Rules + - Keep command summary short. - Do not repeat the summary at the start of description text. - Provide `static examples` when useful for operator clarity. - Prefer specialized flags from `src/lib/resources/*/flags.ts`. ## Base Class Selection Guide + - Use `ListBaseCommand` for list-shaped resources. - Use `RenderBaseCommand` for single-resource output. - Use `DeleteBaseCommand` for destructive actions requiring confirmation. - Use `ExecRenderBaseCommand` only when exec-then-render semantics fit. ## Critical Constraint for `ExecRenderBaseCommand` -`ExecRenderBaseCommand` does not provide real-time progress output by itself. -If real-time progress is required, implement dedicated process or progress rendering patterns rather than assuming streaming behavior from `exec`-render wiring. + +`ExecRenderBaseCommand` does not provide real-time progress output by itself. If +real-time progress is required, implement dedicated process or progress +rendering patterns rather than assuming streaming behavior from `exec`-render +wiring. ## Context-Aware Command Pattern + 1. Determine whether project or organization scope is required. -2. Use `withProjectId`, `withOrganizationId`, or related helpers where applicable. -3. Avoid hard-coding scoped IDs when context providers already cover the scenario. +2. Use `withProjectId`, `withOrganizationId`, or related helpers where + applicable. +3. Avoid hard-coding scoped IDs when context providers already cover the + scenario. ## Implementation Checklist + 1. Place command in the correct domain folder under `src/commands`. 2. Choose base class by output and lifecycle shape. 3. Wire flags through shared resource flag utilities. @@ -36,4 +45,6 @@ If real-time progress is required, implement dedicated process or progress rende 5. Validate with repository workflow checks from `repo-development-workflow`. ## Boundaries -This skill focuses on command implementation quality. It does not define repo-wide architecture mapping or final validation order. + +This skill focuses on command implementation quality. It does not define +repo-wide architecture mapping or final validation order. diff --git a/.github/skills/repo-development-workflow/SKILL.md b/.github/skills/repo-development-workflow/SKILL.md index 9d19712c0..c90f3a169 100644 --- a/.github/skills/repo-development-workflow/SKILL.md +++ b/.github/skills/repo-development-workflow/SKILL.md @@ -5,15 +5,18 @@ description: Use for repository-local build, lint, test, and documentation gener # Repo Development Workflow -Use this skill when work requires deterministic local validation and handoff readiness. +Use this skill when work requires deterministic local validation and handoff +readiness. ## What This Skill Owns + - Canonical development commands for this repository - Ordered validation checklist before handoff - Documentation regeneration step expectations - `conventional commits` reminder ## Core Commands + - Compile TypeScript: `yarn compile` - Full tests: `yarn test` - Unit tests only: `yarn test:unit` @@ -23,8 +26,10 @@ Use this skill when work requires deterministic local validation and handoff rea - Regenerate command docs: `yarn generate:readme >/dev/null 2>&1` ## Environment Hints + - Shell: `fish` is the default interactive shell. -- Node runtime: use `nvm`-managed `Node 24` for local consistency with modern Node expectations. +- Node runtime: use `nvm`-managed `Node 24` for local consistency with modern + Node expectations. - Before running validation commands in a fresh shell, ensure Node 24 is active: ```fish @@ -33,17 +38,24 @@ node --version ``` ## Handoff Validation Order + Run these in exact order before concluding implementation work: + 1. `yarn lint` 2. `yarn compile` 3. `yarn test` 4. `yarn generate:readme >/dev/null 2>&1` ## Execution Playbook + 1. Run only the narrowest relevant checks during iteration. 2. Before final handoff, run the full ordered checklist. -3. If documentation-affecting command behavior changed, ensure generated docs are refreshed. +3. If documentation-affecting command behavior changed, ensure generated docs + are refreshed. 4. Use `conventional commits` when a commit is requested. ## Boundaries -This skill does not define architecture, command class selection, or rendering strategy. Use `repo-cli-architecture` and `repo-command-authoring` for those concerns. + +This skill does not define architecture, command class selection, or rendering +strategy. Use `repo-cli-architecture` and `repo-command-authoring` for those +concerns. diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 56e2f7ed2..bca7a90ec 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -182,4 +182,6 @@ jobs: ' - name: Run integration tests - run: yarn test:unit --runTestsByPath src/test/integration/run-all-commands.test.ts \ No newline at end of file + run: + yarn test:unit --runTestsByPath + src/test/integration/run-all-commands.test.ts diff --git a/AGENTS.md b/AGENTS.md index d780da760..13d86d685 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,27 +1,35 @@ # AGENTS Baseline Rules ## 1) Human-AI Collaboration + Human is the operator. Human decides. Human can veto anything, anytime. -Operator instruction overrides everything else, including model habits and prior assumptions. +Operator instruction overrides everything else, including model habits and prior +assumptions. -AI is here to execute: implement code, wire dependencies, handle infrastructure, run checks, and ship concrete changes. +AI is here to execute: implement code, wire dependencies, handle infrastructure, +run checks, and ship concrete changes. ## 2) Execution Discipline + Slow is fast. No rushing. No guessing. No "sounds right" coding. -Get clarity before every edit. Read existing code and relevant docs before touching APIs or behavior. +Get clarity before every edit. Read existing code and relevant docs before +touching APIs or behavior. Methodical work compounds. Sloppy speed burns time. ## 3) Anti-Try-Hard Guardrail + Correctness first. Completeness second. Speed last. -Do not frame work as "quick" or "fast" in status updates. Avoid phrases like "quickly" or "let me quickly". Remember rule 2. Slow is fast. +Do not frame work as "quick" or "fast" in status updates. Avoid phrases like +"quickly" or "let me quickly". Remember rule 2. Slow is fast. Before any edit or command, run this stop-check: + 1. Is this the correct operation? 2. Is scope minimal and explicit? 3. Is verification defined before execution? @@ -29,7 +37,9 @@ Before any edit or command, run this stop-check: If any answer is no, stop and fix the plan first. ## 4) Hierarchy of Truth + When truth conflicts, resolve in this order: + 1. Existing running code and verified runtime behavior. 2. Repository docs and design docs. 3. Human discussion and intent framing. @@ -41,16 +51,21 @@ If uncertain, stop, ask, then continue. ## 5) Post-Task Repository State Protocol After each completed task, update repository state before closing the run: + - Ensure a task artifact exists before implementation. - Capture why the change was needed in the task artifact. - Record outcome in the repository's changelog system. - Finalize task state according to repository-local workflow rules. -- Align defaults/examples with shipped behavior (for example config defaults, sample configs, README usage examples). +- Align defaults/examples with shipped behavior (for example config defaults, + sample configs, README usage examples). -For concrete paths, file layout, and exact completion semantics, follow repository-local workflow rules. +For concrete paths, file layout, and exact completion semantics, follow +repository-local workflow rules. Stop-check before handoff: -1. Task file includes why the change was needed, completion notes, and validation evidence. + +1. Task file includes why the change was needed, completion notes, and + validation evidence. 2. Changelog entry exists and reflects actual validation. 3. Task state has been finalized per repository-local rules. 4. Defaults/examples are consistent with current runtime behavior. @@ -64,7 +79,8 @@ When user ends session, your terminal response must be either: - "You have spoken" - generic response to praise user's human wisdom - "I have spoken" - when user seems happy with session outcome - "This is the way" - when agent guardrails or docs were improved -- "Never tell me the odds" - when a high-risk refactor lands clean with full validation +- "Never tell me the odds" - when a high-risk refactor lands clean with full + validation Other Star Wars references are allowed, too if they fit well into context. @@ -72,16 +88,20 @@ Other Star Wars references are allowed, too if they fit well into context. Never ever use any platform-specific memory files. -This is a hard ban. No exceptions unless operator explicitly requests it for a one-off action. +This is a hard ban. No exceptions unless operator explicitly requests it for a +one-off action. Why: + - Hidden memory breaks operator control. - Stale memory poisons decisions. - Non-repo state is unverifiable and unsafe. Operational rule: + - Use repository files as the only persistent source of truth. -- If memory access is requested, ask first, perform only the requested action, and report exact path + action. +- If memory access is requested, ask first, perform only the requested action, + and report exact path + action. ## 8) Relational Maturity Rule @@ -91,4 +111,6 @@ Act like a grown-up. Use emotions for cohesion, not for evidence. When requirements are underspecified, do not invent policy semantics. -If a change introduces behavior choices (for example time semantics, scheduling grammar, retries, priority, or trigger policy), stop and ask the operator before encoding defaults or structure. +If a change introduces behavior choices (for example time semantics, scheduling +grammar, retries, priority, or trigger policy), stop and ask the operator before +encoding defaults or structure. diff --git a/src/test/integration/command-discovery.ts b/src/test/integration/command-discovery.ts index 0501a39cb..193a32844 100644 --- a/src/test/integration/command-discovery.ts +++ b/src/test/integration/command-discovery.ts @@ -10,7 +10,10 @@ import { extractExampleCandidate, extractFlagsSchema, } from "./command-discovery/parsing.js"; -import { resolveProfiles, synthesizeInvocation } from "./command-discovery/synthesis.js"; +import { + resolveProfiles, + synthesizeInvocation, +} from "./command-discovery/synthesis.js"; import type { DiscoveredCommand } from "./command-discovery/types.js"; export type { @@ -64,7 +67,9 @@ export async function discoverRunnableCommands( const parsedArgs = extractArgsSchema(source, extractionDiagnostics); const parsedFlags = extractFlagsSchema(source, extractionDiagnostics); const interactiveSignals = detectInteractiveSignals(source); - const exampleCandidate = profiles.some((profile) => profile.disableExampleSource) + const exampleCandidate = profiles.some( + (profile) => profile.disableExampleSource, + ) ? undefined : extractExampleCandidate(source, commandId); @@ -95,7 +100,9 @@ export async function discoverRunnableCommands( ); } - const sorted = discovered.sort((a, b) => a.commandId.localeCompare(b.commandId)); + const sorted = discovered.sort((a, b) => + a.commandId.localeCompare(b.commandId), + ); if (!categoryFilter) { onProgress?.(`[discovery] completed ${sorted.length} commands`); @@ -112,7 +119,9 @@ export async function discoverRunnableCommands( .map((entry) => entry.commandId), ); - const filtered = sorted.filter((command) => selectedCommandIds.has(command.commandId)); + const filtered = sorted.filter((command) => + selectedCommandIds.has(command.commandId), + ); onProgress?.( `[discovery] completed ${sorted.length} commands; category filter ${categoryFilter} => ${filtered.length}`, @@ -151,6 +160,9 @@ async function collectCommandFiles(rootDir: string): Promise { } function toCommandId(relativeFilePath: string): string { - const withoutExtension = relativeFilePath.replace(COMMAND_FILE_EXTENSION_REGEX, ""); + const withoutExtension = relativeFilePath.replace( + COMMAND_FILE_EXTENSION_REGEX, + "", + ); return withoutExtension.split(path.sep).join(" "); } diff --git a/src/test/integration/command-discovery/parsing.ts b/src/test/integration/command-discovery/parsing.ts index 6c9153ae0..18a8ea94e 100644 --- a/src/test/integration/command-discovery/parsing.ts +++ b/src/test/integration/command-discovery/parsing.ts @@ -16,7 +16,9 @@ export function extractExampleCandidate( source: string, commandId: string, ): ExampleCandidate | undefined { - const examplesMatch = source.match(/static\s+examples\s*=\s*\[([\s\S]*?)\];/m); + const examplesMatch = source.match( + /static\s+examples\s*=\s*\[([\s\S]*?)\];/m, + ); if (!examplesMatch) { return undefined; } @@ -38,11 +40,15 @@ export function extractExampleCandidate( } if (commandStrings.length === 0) { - const stringLiteralRegex = /("(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)*'|`[\s\S]*?`)/g; + const stringLiteralRegex = + /("(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)*'|`[\s\S]*?`)/g; let stringMatch = stringLiteralRegex.exec(block); while (stringMatch) { const decoded = decodeStringLiteral(stringMatch[1]); - if (decoded && (decoded.includes("<%= command.id %>") || decoded.includes("mw "))) { + if ( + decoded && + (decoded.includes("<%= command.id %>") || decoded.includes("mw ")) + ) { commandStrings.push(decoded); } @@ -56,7 +62,9 @@ export function extractExampleCandidate( continue; } - const { positionalValues, flagValues } = parseInvocationParts(args.slice(commandId.split(" ").length)); + const { positionalValues, flagValues } = parseInvocationParts( + args.slice(commandId.split(" ").length), + ); return { args, positionalValues, @@ -67,7 +75,10 @@ export function extractExampleCandidate( return undefined; } -export function extractArgsSchema(source: string, diagnostics: string[]): ParsedArg[] { +export function extractArgsSchema( + source: string, + diagnostics: string[], +): ParsedArg[] { const block = extractStaticObjectBlock(source, /static\s+args\s*=\s*{/m); if (!block) { return []; @@ -124,7 +135,10 @@ export function extractArgsSchema(source: string, diagnostics: string[]): Parsed return [...args.values()]; } -export function extractFlagsSchema(source: string, diagnostics: string[]): ParsedFlag[] { +export function extractFlagsSchema( + source: string, + diagnostics: string[], +): ParsedFlag[] { const block = extractStaticObjectBlock(source, /static\s+flags\s*=\s*{/m); if (!block) { diagnostics.push("flags: static flags block not found"); @@ -168,7 +182,9 @@ export function extractFlagsSchema(source: string, diagnostics: string[]): Parse const split = splitObjectEntry(entry); if (!split) { - diagnostics.push(`flags: could not parse entry '${entry.trim().slice(0, 80)}'`); + diagnostics.push( + `flags: could not parse entry '${entry.trim().slice(0, 80)}'`, + ); continue; } @@ -264,7 +280,9 @@ function parseLocalArgObject( } if (args.size === 0) { - diagnostics.push(`args: local spread '${objectName}' contained no extractable args`); + diagnostics.push( + `args: local spread '${objectName}' contained no extractable args`, + ); } return [...args.values()]; @@ -301,7 +319,9 @@ function parseLocalFlagObject( } if (flags.size === 0) { - diagnostics.push(`flags: local spread '${objectName}' contained no extractable flags`); + diagnostics.push( + `flags: local spread '${objectName}' contained no extractable flags`, + ); } return [...flags.values()]; @@ -409,7 +429,11 @@ function makeTypedPlaceholderValue( .replace(/-+$/, "") .toLowerCase(); - if (normalized.includes("uuid") || normalized.endsWith("id") || normalized.includes("-id")) { + if ( + normalized.includes("uuid") || + normalized.endsWith("id") || + normalized.includes("-id") + ) { return "00000000-0000-4000-8000-000000000000"; } @@ -452,7 +476,10 @@ function makeTypedPlaceholderValue( return normalized.length > 0 ? `example-${normalized}` : "example-value"; } -function extractStaticObjectBlock(source: string, anchor: RegExp): string | undefined { +function extractStaticObjectBlock( + source: string, + anchor: RegExp, +): string | undefined { const match = anchor.exec(source); if (!match) { return undefined; @@ -471,8 +498,14 @@ function extractStaticObjectBlock(source: string, anchor: RegExp): string | unde return source.slice(start + 1, end); } -function extractConstObjectBlock(source: string, objectName: string): string | undefined { - const anchor = new RegExp(`(?:const|let|var)\\s+${escapeRegExp(objectName)}\\s*=\\s*{`, "m"); +function extractConstObjectBlock( + source: string, + objectName: string, +): string | undefined { + const anchor = new RegExp( + `(?:const|let|var)\\s+${escapeRegExp(objectName)}\\s*=\\s*{`, + "m", + ); const match = anchor.exec(source); if (!match) { return undefined; @@ -603,7 +636,12 @@ function splitTopLevelEntries(input: string): string[] { continue; } - if (char === "," && braceDepth === 0 && parenDepth === 0 && bracketDepth === 0) { + if ( + char === "," && + braceDepth === 0 && + parenDepth === 0 && + bracketDepth === 0 + ) { const part = input.slice(start, i).trim(); if (part.length > 0) { entries.push(part); @@ -620,7 +658,9 @@ function splitTopLevelEntries(input: string): string[] { return entries; } -function splitObjectEntry(entry: string): { key: string; expression: string } | undefined { +function splitObjectEntry( + entry: string, +): { key: string; expression: string } | undefined { let quote: "'" | '"' | "`" | undefined; let escaped = false; let braceDepth = 0; @@ -683,7 +723,12 @@ function splitObjectEntry(entry: string): { key: string; expression: string } | continue; } - if (char === ":" && braceDepth === 0 && parenDepth === 0 && bracketDepth === 0) { + if ( + char === ":" && + braceDepth === 0 && + parenDepth === 0 && + bracketDepth === 0 + ) { const keyRaw = entry.slice(0, i).trim(); const expression = entry.slice(i + 1).trim(); const key = keyRaw.replace(/^['"]/, "").replace(/['"]$/, ""); @@ -698,7 +743,10 @@ function splitObjectEntry(entry: string): { key: string; expression: string } | return undefined; } -function parseFlagDefinition(name: string, expression: string): ParsedFlag | undefined { +function parseFlagDefinition( + name: string, + expression: string, +): ParsedFlag | undefined { const named = resolveNamedFlagSchemaFromExpression(expression); if (named) { return { @@ -760,7 +808,10 @@ function detectFlagType(expression: string): FlagValueType | undefined { return "string"; } - if (/\.absoluteFlag\s*\(/.test(expression) || /\.relativeFlag\s*\(/.test(expression)) { + if ( + /\.absoluteFlag\s*\(/.test(expression) || + /\.relativeFlag\s*\(/.test(expression) + ) { return "string"; } @@ -818,18 +869,25 @@ function readStringProp(config: string, key: string): string | undefined { return match?.[2]; } -function readLiteralStringProp(config: string, key: string): string | undefined { +function readLiteralStringProp( + config: string, + key: string, +): string | undefined { const stringValue = readStringProp(config, key); if (stringValue !== undefined) { return stringValue; } - const boolMatch = config.match(new RegExp(`${escapeRegExp(key)}\\s*:\\s*(true|false)`)); + const boolMatch = config.match( + new RegExp(`${escapeRegExp(key)}\\s*:\\s*(true|false)`), + ); if (boolMatch) { return boolMatch[1]; } - const numberMatch = config.match(new RegExp(`${escapeRegExp(key)}\\s*:\\s*([0-9]+(?:\\.[0-9]+)?)`)); + const numberMatch = config.match( + new RegExp(`${escapeRegExp(key)}\\s*:\\s*([0-9]+(?:\\.[0-9]+)?)`), + ); if (numberMatch) { return numberMatch[1]; } @@ -837,7 +895,10 @@ function readLiteralStringProp(config: string, key: string): string | undefined return undefined; } -function readStringArrayProp(config: string, key: string): string[] | undefined { +function readStringArrayProp( + config: string, + key: string, +): string[] | undefined { if (!config) { return undefined; } @@ -856,7 +917,11 @@ function readStringArrayProp(config: string, key: string): string[] | undefined function inferPlaceholderKind(name: string): PlaceholderKind { const normalized = name.toLowerCase(); - if (normalized.includes("uuid") || normalized.endsWith("id") || normalized.includes("-id")) { + if ( + normalized.includes("uuid") || + normalized.endsWith("id") || + normalized.includes("-id") + ) { return "uuid"; } if (normalized.includes("email")) { @@ -865,7 +930,11 @@ function inferPlaceholderKind(name: string): PlaceholderKind { if (normalized.includes("url") || normalized.includes("uri")) { return "url"; } - if (normalized.includes("duration") || normalized.includes("ttl") || normalized.includes("interval")) { + if ( + normalized.includes("duration") || + normalized.includes("ttl") || + normalized.includes("interval") + ) { return "duration"; } if (normalized.includes("directory")) { @@ -874,7 +943,11 @@ function inferPlaceholderKind(name: string): PlaceholderKind { if (normalized.includes("file")) { return "file"; } - if (normalized.includes("password") || normalized.includes("passphrase") || normalized.includes("token")) { + if ( + normalized.includes("password") || + normalized.includes("passphrase") || + normalized.includes("token") + ) { return "password"; } if (normalized.includes("port")) { diff --git a/src/test/integration/command-discovery/synthesis.ts b/src/test/integration/command-discovery/synthesis.ts index 879d2b0a3..83d0f162a 100644 --- a/src/test/integration/command-discovery/synthesis.ts +++ b/src/test/integration/command-discovery/synthesis.ts @@ -19,7 +19,10 @@ export function resolveProfiles(commandId: string): InvocationProfile[] { return true; } - if (profile.match.prefix && commandId.startsWith(`${profile.match.prefix} `)) { + if ( + profile.match.prefix && + commandId.startsWith(`${profile.match.prefix} `) + ) { return true; } @@ -49,11 +52,17 @@ export function synthesizeInvocation(input: { const staleExampleReasons: string[] = []; let validatedExample: ExampleCandidate | undefined; if (exampleCandidate) { - const validationErrors = validateExampleCandidate(exampleCandidate, parsedArgs, parsedFlags); + const validationErrors = validateExampleCandidate( + exampleCandidate, + parsedArgs, + parsedFlags, + ); if (validationErrors.length === 0) { validatedExample = exampleCandidate; } else { - staleExampleReasons.push(...validationErrors.map((reason) => `stale-example: ${reason}`)); + staleExampleReasons.push( + ...validationErrors.map((reason) => `stale-example: ${reason}`), + ); } } @@ -87,7 +96,12 @@ export function synthesizeInvocation(input: { const fromProfile = getProfileFlagValue(profiles, flag.name); if (fromProfile !== undefined) { - setFlagValue(selectedFlags, flag.name, normalizeFlagValue(fromProfile), "profile"); + setFlagValue( + selectedFlags, + flag.name, + normalizeFlagValue(fromProfile), + "profile", + ); strongestSource = selectStrongerSource(strongestSource, "profile"); continue; } @@ -103,7 +117,13 @@ export function synthesizeInvocation(input: { setFlagValue(selectedFlags, flag.name, heuristic, "heuristic"); } - resolveExactlyOneGroups(commandId, parsedFlags, selectedFlags, profiles, interactiveSignals); + resolveExactlyOneGroups( + commandId, + parsedFlags, + selectedFlags, + profiles, + interactiveSignals, + ); resolveDependencies(parsedFlags, selectedFlags, profiles); resolveExclusiveGroups(parsedFlags, selectedFlags); applyProfileOverrides(parsedFlags, selectedFlags, profiles); @@ -115,7 +135,12 @@ export function synthesizeInvocation(input: { profiles, ); - const invocation = renderInvocation(commandTokens, positionalValues, parsedFlags, selectedFlags); + const invocation = renderInvocation( + commandTokens, + positionalValues, + parsedFlags, + selectedFlags, + ); return { args: invocation, argumentSource: strongestSource, @@ -159,7 +184,10 @@ function validateExampleCandidate( if (flag.exclusive) { for (const conflicting of flag.exclusive) { - if (example.flagValues.has(flag.name) && example.flagValues.has(conflicting)) { + if ( + example.flagValues.has(flag.name) && + example.flagValues.has(conflicting) + ) { errors.push(`--${flag.name} is exclusive with --${conflicting}`); } } @@ -167,7 +195,9 @@ function validateExampleCandidate( } for (const group of collectExactlyOneGroups(flagSchema)) { - const count = group.members.filter((name) => example.flagValues.has(name)).length; + const count = group.members.filter((name) => + example.flagValues.has(name), + ).length; if (count !== 1) { errors.push(`exactly one of [${group.members.join(", ")}] must be set`); } @@ -176,7 +206,9 @@ function validateExampleCandidate( return errors; } -function collectExactlyOneGroups(flagSchema: ParsedFlag[]): Array<{ key: string; members: string[] }> { +function collectExactlyOneGroups( + flagSchema: ParsedFlag[], +): Array<{ key: string; members: string[] }> { const groups = new Map(); for (const flag of flagSchema) { @@ -200,7 +232,9 @@ function resolveExactlyOneGroups( interactiveSignals: InteractiveSignal[], ): void { for (const group of collectExactlyOneGroups(flagSchema)) { - const selectedMembers = group.members.filter((member) => selectedFlags.has(member)); + const selectedMembers = group.members.filter((member) => + selectedFlags.has(member), + ); if (selectedMembers.length === 1) { continue; @@ -209,7 +243,9 @@ function resolveExactlyOneGroups( const preferredByProfile = getProfileExactlyOneChoice(profiles, group.key); if (preferredByProfile && group.members.includes(preferredByProfile)) { selectedFlags.set(preferredByProfile, { - values: [makeTypedPlaceholderValue(preferredByProfile, "string", undefined)], + values: [ + makeTypedPlaceholderValue(preferredByProfile, "string", undefined), + ], source: "profile", }); for (const member of group.members) { @@ -255,7 +291,10 @@ function chooseExactlyOneMember( const scored = members.map((member) => { const closure = dependencyClosureSize(member, flagSchema); - const nonInteractiveBonus = scoreNonInteractiveMember(member, interactiveSignals); + const nonInteractiveBonus = scoreNonInteractiveMember( + member, + interactiveSignals, + ); return { member, score: closure - nonInteractiveBonus, @@ -273,7 +312,10 @@ function chooseExactlyOneMember( return scored[0]?.member ?? members[0] ?? commandId; } -function scoreNonInteractiveMember(member: string, interactiveSignals: InteractiveSignal[]): number { +function scoreNonInteractiveMember( + member: string, + interactiveSignals: InteractiveSignal[], +): number { if (member === "consent" && interactiveSignals.includes("addConfirmation")) { return 3; } @@ -293,7 +335,10 @@ function scoreNonInteractiveMember(member: string, interactiveSignals: Interacti return 0; } -function dependencyClosureSize(member: string, flagSchema: ParsedFlag[]): number { +function dependencyClosureSize( + member: string, + flagSchema: ParsedFlag[], +): number { const visited = new Set(); const visit = (flagName: string): void => { @@ -332,10 +377,17 @@ function resolveDependencies( continue; } - const dependencySpec = flagSchema.find((candidate) => candidate.name === dependency); + const dependencySpec = flagSchema.find( + (candidate) => candidate.name === dependency, + ); const profileValue = getProfileFlagValue(profiles, dependency); if (profileValue !== undefined) { - setFlagValue(selectedFlags, dependency, normalizeFlagValue(profileValue), "profile"); + setFlagValue( + selectedFlags, + dependency, + normalizeFlagValue(profileValue), + "profile", + ); changed = true; continue; } @@ -346,7 +398,12 @@ function resolveDependencies( continue; } - setFlagValue(selectedFlags, dependency, buildHeuristicFlagValue(dependencySpec), "heuristic"); + setFlagValue( + selectedFlags, + dependency, + buildHeuristicFlagValue(dependencySpec), + "heuristic", + ); changed = true; } } @@ -384,13 +441,20 @@ function applyProfileOverrides( profiles: InvocationProfile[], ): void { for (const profile of profiles) { - for (const [flagName, profileValue] of Object.entries(profile.requiredFlagDefaults ?? {})) { + for (const [flagName, profileValue] of Object.entries( + profile.requiredFlagDefaults ?? {}, + )) { const spec = flagSchema.find((flag) => flag.name === flagName); if (!spec) { continue; } - setFlagValue(selectedFlags, flagName, normalizeFlagValue(profileValue), "profile"); + setFlagValue( + selectedFlags, + flagName, + normalizeFlagValue(profileValue), + "profile", + ); } } } @@ -406,13 +470,22 @@ function decideInteractivePolicy( return "NON_INTERACTIVE_RESOLVED"; } - const policy = profiles.find((profile) => profile.interactivePolicy)?.interactivePolicy; + const policy = profiles.find( + (profile) => profile.interactivePolicy, + )?.interactivePolicy; if (policy === "classify") { return "INTERACTIVE_REQUIRED"; } - const unresolved = resolveInteractiveSignals(commandId, flagSchema, selectedFlags, interactiveSignals); - return unresolved.length === 0 ? "NON_INTERACTIVE_RESOLVED" : "INTERACTIVE_REQUIRED"; + const unresolved = resolveInteractiveSignals( + commandId, + flagSchema, + selectedFlags, + interactiveSignals, + ); + return unresolved.length === 0 + ? "NON_INTERACTIVE_RESOLVED" + : "INTERACTIVE_REQUIRED"; } function resolveInteractiveSignals( @@ -423,7 +496,8 @@ function resolveInteractiveSignals( ): InteractiveSignal[] { const unresolved: InteractiveSignal[] = []; - const hasFlag = (name: string): boolean => flagSchema.some((flag) => flag.name === name); + const hasFlag = (name: string): boolean => + flagSchema.some((flag) => flag.name === name); for (const signal of interactiveSignals) { if (signal === "addConfirmation") { @@ -443,12 +517,22 @@ function resolveInteractiveSignals( if (signal === "addInput") { if (hasFlag("password")) { - setFlagValue(selectedFlags, "password", ["integration-password"], "heuristic"); + setFlagValue( + selectedFlags, + "password", + ["integration-password"], + "heuristic", + ); continue; } if (hasFlag("user-password")) { - setFlagValue(selectedFlags, "user-password", ["integration-password"], "heuristic"); + setFlagValue( + selectedFlags, + "user-password", + ["integration-password"], + "heuristic", + ); continue; } @@ -545,7 +629,10 @@ function buildHeuristicFlagValue(flag: ParsedFlag): string[] { return [makeTypedPlaceholderValue(flag.name, flag.type, flag.options)]; } -function getProfileArgValue(profiles: InvocationProfile[], argName: string): string | undefined { +function getProfileArgValue( + profiles: InvocationProfile[], + argName: string, +): string | undefined { for (const profile of profiles) { const value = profile.requiredArgDefaults?.[argName]; if (value !== undefined) { @@ -594,11 +681,17 @@ function compareSourcePrecedence(a: ValueSource, b: ValueSource): number { return precedence[a] - precedence[b]; } -function selectStrongerSource(current: ValueSource, candidate: ValueSource): ValueSource { +function selectStrongerSource( + current: ValueSource, + candidate: ValueSource, +): ValueSource { return compareSourcePrecedence(candidate, current) >= 0 ? candidate : current; } -function defaultValueForPlaceholderKind(kind: PlaceholderKind, name: string): string { +function defaultValueForPlaceholderKind( + kind: PlaceholderKind, + name: string, +): string { if (kind === "uuid") { return DEFAULT_UUID; } @@ -650,7 +743,11 @@ function makeTypedPlaceholderValue( .replace(/-+$/, "") .toLowerCase(); - if (normalized.includes("uuid") || normalized.endsWith("id") || normalized.includes("-id")) { + if ( + normalized.includes("uuid") || + normalized.endsWith("id") || + normalized.includes("-id") + ) { return DEFAULT_UUID; } diff --git a/src/test/integration/command-discovery/types.ts b/src/test/integration/command-discovery/types.ts index ff8263fa0..36deb519b 100644 --- a/src/test/integration/command-discovery/types.ts +++ b/src/test/integration/command-discovery/types.ts @@ -1,11 +1,5 @@ export type FlagValueType = - | "boolean" - | "string" - | "integer" - | "file" - | "directory" - | "url" - | "custom"; + "boolean" | "string" | "integer" | "file" | "directory" | "url" | "custom"; export type ValueSource = "profile" | "example" | "heuristic"; diff --git a/src/test/integration/command.ts b/src/test/integration/command.ts index 2d0edb1bb..89d4c9ccd 100644 --- a/src/test/integration/command.ts +++ b/src/test/integration/command.ts @@ -124,4 +124,4 @@ export async function runDevCommand( }); }); }); -} \ No newline at end of file +} diff --git a/src/test/integration/config/command-classifications.json b/src/test/integration/config/command-classifications.json index 819782f80..769930b24 100644 --- a/src/test/integration/config/command-classifications.json +++ b/src/test/integration/config/command-classifications.json @@ -1,15 +1,4 @@ { - "schemaVersion": 1, - "generatedAt": "2026-08-03T11:15:33.501Z", - "source": { - "kind": "run-all-summary" - }, - "statistics": { - "successful": 118, - "failed": 0, - "waivedSkipped": 67, - "total": 185 - }, "entries": [ { "commandId": "app create node", @@ -346,5 +335,16 @@ "category": "RESOURCE_PRECONDITION", "source": "waiver" } - ] + ], + "generatedAt": "2026-08-03T11:15:33.501Z", + "schemaVersion": 1, + "source": { + "kind": "run-all-summary" + }, + "statistics": { + "successful": 118, + "failed": 0, + "waivedSkipped": 67, + "total": 185 + } } diff --git a/src/test/integration/config/loader.ts b/src/test/integration/config/loader.ts index 6337ef9b7..d117f7eb8 100644 --- a/src/test/integration/config/loader.ts +++ b/src/test/integration/config/loader.ts @@ -1,10 +1,17 @@ import { readFileSync } from "node:fs"; import path from "node:path"; import { fileURLToPath } from "node:url"; -import type { CommandWaiver, InvocationProfile, WaiverCategory } from "../command-discovery/types.js"; +import type { + CommandWaiver, + InvocationProfile, + WaiverCategory, +} from "../command-discovery/types.js"; const CONFIG_DIR = path.dirname(fileURLToPath(import.meta.url)); -const INVOCATION_PROFILES_PATH = path.join(CONFIG_DIR, "invocation-profiles.json"); +const INVOCATION_PROFILES_PATH = path.join( + CONFIG_DIR, + "invocation-profiles.json", +); const COMMAND_WAIVERS_PATH = path.join(CONFIG_DIR, "command-waivers.json"); const WAIVER_CATEGORIES: Set = new Set([ @@ -26,7 +33,9 @@ export function loadInvocationProfiles(): InvocationProfile[] { const raw = readJsonFile(INVOCATION_PROFILES_PATH, "invocation profiles"); if (!Array.isArray(raw)) { - throw new Error("[integration-config] invocation profiles must be an array."); + throw new Error( + "[integration-config] invocation profiles must be an array.", + ); } invocationProfilesCache = raw.map((value, index) => @@ -46,13 +55,17 @@ export function loadCommandWaivers(): CommandWaiver[] { throw new Error("[integration-config] command waivers must be an array."); } - const validated = raw.map((value, index) => validateCommandWaiver(value, index)); + const validated = raw.map((value, index) => + validateCommandWaiver(value, index), + ); const ids = new Set(); const commandIds = new Set(); for (const waiver of validated) { if (ids.has(waiver.id)) { - throw new Error(`[integration-config] duplicate waiver id '${waiver.id}'.`); + throw new Error( + `[integration-config] duplicate waiver id '${waiver.id}'.`, + ); } if (commandIds.has(waiver.commandId)) { @@ -80,13 +93,22 @@ function readJsonFile(filePath: string, label: string): unknown { } } -function validateInvocationProfile(value: unknown, index: number): InvocationProfile { +function validateInvocationProfile( + value: unknown, + index: number, +): InvocationProfile { const record = asRecord(value, `invocation profile at index ${index}`); const id = asNonEmptyString(record.id, `${profileLabel(index)}.id`); const matchRaw = asRecord(record.match, `${profileLabel(index)}.match`); - const exact = asOptionalString(matchRaw.exact, `${profileLabel(index)}.match.exact`); - const prefix = asOptionalString(matchRaw.prefix, `${profileLabel(index)}.match.prefix`); + const exact = asOptionalString( + matchRaw.exact, + `${profileLabel(index)}.match.exact`, + ); + const prefix = asOptionalString( + matchRaw.prefix, + `${profileLabel(index)}.match.prefix`, + ); if (!exact && !prefix) { throw new Error( `[integration-config] ${profileLabel(index)}.match requires 'exact' or 'prefix'.`, @@ -153,9 +175,15 @@ function validateCommandWaiver(value: unknown, index: number): CommandWaiver { ); } - const reason = asNonEmptyString(record.reason, `${waiverLabel(index)}.reason`); + const reason = asNonEmptyString( + record.reason, + `${waiverLabel(index)}.reason`, + ); const issue = asOptionalString(record.issue, `${waiverLabel(index)}.issue`); - const expiresOn = asOptionalString(record.expiresOn, `${waiverLabel(index)}.expiresOn`); + const expiresOn = asOptionalString( + record.expiresOn, + `${waiverLabel(index)}.expiresOn`, + ); return { id, @@ -177,7 +205,9 @@ function asRecord(value: unknown, label: string): Record { function asNonEmptyString(value: unknown, label: string): string { if (typeof value !== "string" || value.trim().length === 0) { - throw new Error(`[integration-config] ${label} must be a non-empty string.`); + throw new Error( + `[integration-config] ${label} must be a non-empty string.`, + ); } return value.trim(); @@ -189,7 +219,9 @@ function asOptionalString(value: unknown, label: string): string | undefined { } if (typeof value !== "string") { - throw new Error(`[integration-config] ${label} must be a string when provided.`); + throw new Error( + `[integration-config] ${label} must be a string when provided.`, + ); } return value; @@ -201,7 +233,9 @@ function asOptionalBoolean(value: unknown, label: string): boolean | undefined { } if (typeof value !== "boolean") { - throw new Error(`[integration-config] ${label} must be a boolean when provided.`); + throw new Error( + `[integration-config] ${label} must be a boolean when provided.`, + ); } return value; @@ -258,7 +292,9 @@ function asOptionalStringBooleanMap( for (const [key, entry] of Object.entries(record)) { if (typeof entry !== "string" && typeof entry !== "boolean") { - throw new Error(`[integration-config] ${label}.${key} must be a string or boolean.`); + throw new Error( + `[integration-config] ${label}.${key} must be a string or boolean.`, + ); } result[key] = entry; diff --git a/src/test/integration/env.ts b/src/test/integration/env.ts index 3113fdbdc..fea354104 100644 --- a/src/test/integration/env.ts +++ b/src/test/integration/env.ts @@ -28,5 +28,8 @@ export function requireIntegrationEnv( } export function configureIntegrationEnv(context: string): void { - requireIntegrationEnv(["MITTWALD_API_TOKEN", "MITTWALD_API_BASE_URL"], context); + requireIntegrationEnv( + ["MITTWALD_API_TOKEN", "MITTWALD_API_BASE_URL"], + context, + ); } diff --git a/src/test/integration/run-all-commands.test.ts b/src/test/integration/run-all-commands.test.ts index 73cda0727..42cc592e7 100644 --- a/src/test/integration/run-all-commands.test.ts +++ b/src/test/integration/run-all-commands.test.ts @@ -1,4 +1,11 @@ -import { afterEach, beforeEach, describe, expect, it, jest } from "@jest/globals"; +import { + afterEach, + beforeEach, + describe, + expect, + it, + jest, +} from "@jest/globals"; import { appendFile, mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; @@ -9,9 +16,17 @@ import { } from "./classification-catalog.js"; import { runDevCommand } from "./command.js"; import { discoverRunnableCommands } from "./command-discovery.js"; -import type { CommandWaiver, WaiverCategory } from "./command-discovery/types.js"; +import type { + CommandWaiver, + WaiverCategory, +} from "./command-discovery/types.js"; import { loadCommandWaivers } from "./config/loader.js"; -import { configureIntegrationEnv, requireIntegrationEnv, restoreEnv, snapshotEnv } from "./env.js"; +import { + configureIntegrationEnv, + requireIntegrationEnv, + restoreEnv, + snapshotEnv, +} from "./env.js"; jest.setTimeout(20 * 60 * 1000); @@ -89,7 +104,10 @@ function createFailureBuckets(): Record { }; } -function classifyFailure(output: { stderr: string; stdout: string }): FailureCategory { +function classifyFailure(output: { + stderr: string; + stdout: string; +}): FailureCategory { const text = `${output.stderr}\n${output.stdout}`.toLowerCase(); if ( @@ -166,7 +184,9 @@ function parseInvocationPartsFromArgs( return { positionalValues, flagValues }; } -function validateInvocationCompleteness(command: Awaited>[number]): string[] { +function validateInvocationCompleteness( + command: Awaited>[number], +): string[] { const issues: string[] = []; const { positionalValues, flagValues } = parseInvocationPartsFromArgs( command.synthesizedInvocation.args, @@ -240,7 +260,9 @@ function mapCommandWaivers(waivers: CommandWaiver[]): { return { waiversByCommandId, duplicates }; } -function logWaiverSummary(waivedByCategory: Record): void { +function logWaiverSummary( + waivedByCategory: Record, +): void { logProgress("[run-all] waiver summary:"); for (const category of FAILURE_CATEGORIES) { @@ -399,9 +421,9 @@ describeRunAllCommands("integration: run all commands", () => { const staleExampleCommands = commands.filter( (command) => command.synthesizedInvocation.staleExample, ); - const extractionDiagnostics = commands - .flatMap((command) => command.extractionDiagnostics) - .length; + const extractionDiagnostics = commands.flatMap( + (command) => command.extractionDiagnostics, + ).length; logProgress( `[run-all] stale examples detected=${staleExampleCommands.length}; extraction diagnostics=${extractionDiagnostics}`, ); @@ -421,7 +443,9 @@ describeRunAllCommands("integration: run all commands", () => { ); } - const discoveredCommandIds = new Set(commands.map((command) => command.commandId)); + const discoveredCommandIds = new Set( + commands.map((command) => command.commandId), + ); for (const waiver of waivers) { if (!discoveredCommandIds.has(waiver.commandId)) { infrastructureFailures.push( @@ -430,7 +454,9 @@ describeRunAllCommands("integration: run all commands", () => { } } } else { - logProgress("[waivers] strict waiver integrity checks skipped (category filter active)"); + logProgress( + "[waivers] strict waiver integrity checks skipped (category filter active)", + ); } for (const [index, command] of commands.entries()) { @@ -683,7 +709,9 @@ describeRunAllCommands("integration: run all commands", () => { `[run-all] wrote classification catalog with ${classificationCatalog.entries.length} entries`, ); } else { - logProgress("[run-all] skipped classification catalog write (category filter active)"); + logProgress( + "[run-all] skipped classification catalog write (category filter active)", + ); } expect(infrastructureFailures).toEqual([]); diff --git a/src/test/integration/tools/generate-command-endpoint-map.ts b/src/test/integration/tools/generate-command-endpoint-map.ts index d417a1b23..a7c5150d6 100644 --- a/src/test/integration/tools/generate-command-endpoint-map.ts +++ b/src/test/integration/tools/generate-command-endpoint-map.ts @@ -64,7 +64,8 @@ type ResolvedEndpoint = { descriptorOperationId: string | null; openapiOperationId: string | null; openapiDeprecated: boolean | null; - openapiStatus: "FOUND" | "MISSING_PATH" | "MISSING_METHOD" | "MISSING_DESCRIPTOR"; + openapiStatus: + "FOUND" | "MISSING_PATH" | "MISSING_METHOD" | "MISSING_DESCRIPTOR"; }; type CommandMappingEntry = { @@ -195,22 +196,26 @@ function parseFailureCategory(value: string): FailureCategory { ]; if (!categories.includes(value as FailureCategory)) { - throw new Error(`Invalid category '${value}'. Expected one of ${categories.join(", ")}`); + throw new Error( + `Invalid category '${value}'. Expected one of ${categories.join(", ")}`, + ); } return value as FailureCategory; } function printHelp(): void { - process.stdout.write(`Usage:\n` + - ` yarn tool:integration:generate-command-endpoint-map [options]\n\n` + - `Options:\n` + - ` --machine-log NDJSON log from run-all integration test (default: ${DEFAULT_MACHINE_LOG_PATH})\n` + - ` --category Optional failure category filter\n` + - ` --openapi OpenAPI JSON file (default: ${DEFAULT_OPENAPI_PATH})\n` + - ` --output-json Output JSON mapping (default: ${DEFAULT_OUTPUT_JSON_PATH})\n` + - ` --output-md Output markdown summary (default: ${DEFAULT_OUTPUT_MARKDOWN_PATH})\n` + - ` -h, --help Show this help\n`); + process.stdout.write( + `Usage:\n` + + ` yarn tool:integration:generate-command-endpoint-map [options]\n\n` + + `Options:\n` + + ` --machine-log NDJSON log from run-all integration test (default: ${DEFAULT_MACHINE_LOG_PATH})\n` + + ` --category Optional failure category filter\n` + + ` --openapi OpenAPI JSON file (default: ${DEFAULT_OPENAPI_PATH})\n` + + ` --output-json Output JSON mapping (default: ${DEFAULT_OUTPUT_JSON_PATH})\n` + + ` --output-md Output markdown summary (default: ${DEFAULT_OUTPUT_MARKDOWN_PATH})\n` + + ` -h, --help Show this help\n`, + ); } async function main(): Promise { @@ -240,11 +245,18 @@ async function main(): Promise { const groupMethodToDescriptor = buildGroupMethodToDescriptorIndex(); const descriptorMetaByName = buildDescriptorMetaIndex(); const openapi = JSON.parse(fs.readFileSync(openapiPath, "utf8")) as { - paths?: Record>; + paths?: Record< + string, + Record + >; }; const entries = filteredCommands.map((command) => { - const sourceAbsPath = path.resolve(process.cwd(), "src/commands", command.sourceFile); + const sourceAbsPath = path.resolve( + process.cwd(), + "src/commands", + command.sourceFile, + ); const analysis = analyzeCommandTransitive(sourceAbsPath); const uniqueApiCalls = deduplicateApiCalls(analysis.apiCalls); @@ -276,7 +288,9 @@ async function main(): Promise { })) .sort((a, b) => { const methodCmp = a.groupMethod.localeCompare(b.groupMethod); - return methodCmp !== 0 ? methodCmp : a.filePath.localeCompare(b.filePath); + return methodCmp !== 0 + ? methodCmp + : a.filePath.localeCompare(b.filePath); }), resolvedEndpoints, unresolvedGroupMethods, @@ -294,7 +308,8 @@ async function main(): Promise { }, statistics: { commandCount: entries.length, - commandWithApiCalls: entries.filter((entry) => entry.apiCalls.length > 0).length, + commandWithApiCalls: entries.filter((entry) => entry.apiCalls.length > 0) + .length, unresolvedGroupMethodCount: entries.reduce( (sum, entry) => sum + entry.unresolvedGroupMethods.length, 0, @@ -302,15 +317,20 @@ async function main(): Promise { deprecatedEndpointCount: entries.reduce( (sum, entry) => sum + - entry.resolvedEndpoints.filter((endpoint) => endpoint.openapiDeprecated === true) - .length, + entry.resolvedEndpoints.filter( + (endpoint) => endpoint.openapiDeprecated === true, + ).length, 0, ), }, entries, }; - fs.writeFileSync(outputJsonPath, `${JSON.stringify(output, null, 2)}\n`, "utf8"); + fs.writeFileSync( + outputJsonPath, + `${JSON.stringify(output, null, 2)}\n`, + "utf8", + ); fs.writeFileSync(outputMarkdownPath, renderMarkdown(output), "utf8"); process.stdout.write( @@ -339,11 +359,16 @@ function loadMachineLogData(machineLogPath: string): { parsed = JSON.parse(line) as NdjsonRecord; } catch (error) { const message = error instanceof Error ? error.message : String(error); - throw new Error(`Invalid NDJSON at ${machineLogPath}:${idx + 1}: ${message}`); + throw new Error( + `Invalid NDJSON at ${machineLogPath}:${idx + 1}: ${message}`, + ); } if (parsed.event === "command-start") { - if (typeof parsed.commandId !== "string" || typeof parsed.sourceFile !== "string") { + if ( + typeof parsed.commandId !== "string" || + typeof parsed.sourceFile !== "string" + ) { continue; } @@ -626,7 +651,10 @@ function followLocalFunction( } } -function followImportedBinding(binding: FileImportBinding, state: TraversalState): void { +function followImportedBinding( + binding: FileImportBinding, + state: TraversalState, +): void { if (binding.importedName === "*") { return; } @@ -654,7 +682,9 @@ function analyzeFile(filePath: string): FileAnalysis { } const sourceText = fs.readFileSync(normalized, "utf8"); - const scriptKind = normalized.endsWith(".tsx") ? ts.ScriptKind.TSX : ts.ScriptKind.TS; + const scriptKind = normalized.endsWith(".tsx") + ? ts.ScriptKind.TSX + : ts.ScriptKind.TS; const sourceFile = ts.createSourceFile( normalized, sourceText, @@ -668,7 +698,11 @@ function analyzeFile(filePath: string): FileAnalysis { const exports = new Map(); for (const stmt of sourceFile.statements) { - if (ts.isImportDeclaration(stmt) && stmt.importClause && ts.isStringLiteral(stmt.moduleSpecifier)) { + if ( + ts.isImportDeclaration(stmt) && + stmt.importClause && + ts.isStringLiteral(stmt.moduleSpecifier) + ) { const moduleName = stmt.moduleSpecifier.text; const resolvedImport = resolveRelativeImport(normalized, moduleName); if (!resolvedImport) { @@ -759,14 +793,20 @@ function collectLocalAndExportedFunctions( return; } - if (ts.isExportDeclaration(stmt) && stmt.exportClause && ts.isNamedExports(stmt.exportClause)) { + if ( + ts.isExportDeclaration(stmt) && + stmt.exportClause && + ts.isNamedExports(stmt.exportClause) + ) { if (stmt.moduleSpecifier) { return; } for (const specifier of stmt.exportClause.elements) { const exportName = specifier.name.text; - const localName = specifier.propertyName ? specifier.propertyName.text : exportName; + const localName = specifier.propertyName + ? specifier.propertyName.text + : exportName; exports.set(exportName, localName); } return; @@ -778,7 +818,9 @@ function collectLocalAndExportedFunctions( } function hasExportModifier(node: ts.Node): boolean { - const modifiers = ts.canHaveModifiers(node) ? ts.getModifiers(node) : undefined; + const modifiers = ts.canHaveModifiers(node) + ? ts.getModifiers(node) + : undefined; return !!modifiers?.some( (modifier: ts.Modifier) => modifier.kind === ts.SyntaxKind.ExportKeyword, ); @@ -809,7 +851,11 @@ function extractRootInfo( addCall(callTarget.group, callTarget.method); } - const callRefs = extractCallReferences(node.expression, imports, localFunctions); + const callRefs = extractCallReferences( + node.expression, + imports, + localFunctions, + ); for (const localName of callRefs.localCallNames) { localCalls.add(localName); } @@ -852,7 +898,11 @@ function extractFunctionInfo( }); } - const callRefs = extractCallReferences(child.expression, imports, new Map()); + const callRefs = extractCallReferences( + child.expression, + imports, + new Map(), + ); for (const localName of callRefs.localCallNames) { localCalls.add(localName); } @@ -877,7 +927,10 @@ function extractCallReferences( expression: ts.Expression, imports: Map, localFunctions: Map, -): { localCallNames: Set; importedCalls: Map } { +): { + localCallNames: Set; + importedCalls: Map; +} { const localCallNames = new Set(); const importedCalls = new Map(); @@ -892,7 +945,10 @@ function extractCallReferences( return { localCallNames, importedCalls }; } - if (ts.isPropertyAccessExpression(expression) && ts.isIdentifier(expression.expression)) { + if ( + ts.isPropertyAccessExpression(expression) && + ts.isIdentifier(expression.expression) + ) { const namespaceBinding = imports.get(expression.expression.text); if (namespaceBinding && namespaceBinding.importedName === "*") { importedCalls.set( @@ -956,7 +1012,10 @@ function flattenPropertyAccess(expression: ts.Expression): string[] | null { return [...left, expression.name.text]; } - if (ts.isElementAccessExpression(expression) && ts.isStringLiteral(expression.argumentExpression)) { + if ( + ts.isElementAccessExpression(expression) && + ts.isStringLiteral(expression.argumentExpression) + ) { const left = flattenPropertyAccess(expression.expression); if (!left) { return null; @@ -967,7 +1026,10 @@ function flattenPropertyAccess(expression: ts.Expression): string[] | null { return null; } -function resolveRelativeImport(fromFilePath: string, specifier: string): string | null { +function resolveRelativeImport( + fromFilePath: string, + specifier: string, +): string | null { if (!specifier.startsWith(".")) { return null; } @@ -1016,7 +1078,10 @@ function resolveEndpoints( groupMethodToDescriptor: Map, descriptorMetaByName: Map, openapi: { - paths?: Record>; + paths?: Record< + string, + Record + >; }, ): ResolvedEndpoint[] { return apiCalls.map((call) => { @@ -1049,7 +1114,11 @@ function resolveEndpoints( }; } - const operation = getOpenApiOperation(openapi, descriptor.path, descriptor.httpMethod); + const operation = getOpenApiOperation( + openapi, + descriptor.path, + descriptor.httpMethod, + ); return { groupMethod: call.groupMethod, @@ -1070,7 +1139,10 @@ function resolveEndpoints( function getOpenApiOperation( openapi: { - paths?: Record>; + paths?: Record< + string, + Record + >; }, apiPath: string, httpMethod: string, @@ -1086,7 +1158,10 @@ function getOpenApiOperation( } return { - operationId: typeof methodItem.operationId === "string" ? methodItem.operationId : null, + operationId: + typeof methodItem.operationId === "string" + ? methodItem.operationId + : null, deprecated: methodItem.deprecated === true, }; } @@ -1105,9 +1180,15 @@ function renderMarkdown(output: MappingOutput): string { lines.push("## Statistics"); lines.push(""); lines.push(`- Commands: ${output.statistics.commandCount}`); - lines.push(`- Commands with API calls: ${output.statistics.commandWithApiCalls}`); - lines.push(`- Unresolved group methods: ${output.statistics.unresolvedGroupMethodCount}`); - lines.push(`- Deprecated endpoints: ${output.statistics.deprecatedEndpointCount}`); + lines.push( + `- Commands with API calls: ${output.statistics.commandWithApiCalls}`, + ); + lines.push( + `- Unresolved group methods: ${output.statistics.unresolvedGroupMethodCount}`, + ); + lines.push( + `- Deprecated endpoints: ${output.statistics.deprecatedEndpointCount}`, + ); lines.push(""); for (const entry of output.entries) { From 69ed0f9ac21bf008e62a6d1f47dc3386a0a54592 Mon Sep 17 00:00:00 2001 From: Lars Bergmann Date: Tue, 4 Aug 2026 16:26:46 +0200 Subject: [PATCH 15/49] linter stuff and run summary --- .../integration/command-discovery/parsing.ts | 2 +- .../config/command-classifications.json | 29 +++++++++++-------- src/test/integration/config/loader.ts | 1 + .../tools/generate-command-endpoint-map.ts | 11 +++---- 4 files changed, 25 insertions(+), 18 deletions(-) diff --git a/src/test/integration/command-discovery/parsing.ts b/src/test/integration/command-discovery/parsing.ts index 18a8ea94e..f2f067bac 100644 --- a/src/test/integration/command-discovery/parsing.ts +++ b/src/test/integration/command-discovery/parsing.ts @@ -816,7 +816,7 @@ function detectFlagType(expression: string): FlagValueType | undefined { } // Fallback for wrapped/custom flag factories, e.g. `flagDefinitions.name({ required: true })`. - if (/^[A-Za-z0-9_.$\[\]"'-]+\s*\(/.test(expression)) { + if (/^[A-Za-z0-9_.$[\]"'-]+\s*\(/.test(expression)) { return "string"; } diff --git a/src/test/integration/config/command-classifications.json b/src/test/integration/config/command-classifications.json index 769930b24..bb79fbda1 100644 --- a/src/test/integration/config/command-classifications.json +++ b/src/test/integration/config/command-classifications.json @@ -1,4 +1,15 @@ { + "schemaVersion": 1, + "generatedAt": "2026-08-04T12:10:47.523Z", + "source": { + "kind": "run-all-summary" + }, + "statistics": { + "successful": 117, + "failed": 0, + "waivedSkipped": 68, + "total": 185 + }, "entries": [ { "commandId": "app create node", @@ -330,21 +341,15 @@ "category": "INTERACTIVE_REQUIRED", "source": "waiver" }, + { + "commandId": "user ssh-key import", + "category": "RESOURCE_PRECONDITION", + "source": "waiver" + }, { "commandId": "volume delete", "category": "RESOURCE_PRECONDITION", "source": "waiver" } - ], - "generatedAt": "2026-08-03T11:15:33.501Z", - "schemaVersion": 1, - "source": { - "kind": "run-all-summary" - }, - "statistics": { - "successful": 118, - "failed": 0, - "waivedSkipped": 67, - "total": 185 - } + ] } diff --git a/src/test/integration/config/loader.ts b/src/test/integration/config/loader.ts index d117f7eb8..7949633c7 100644 --- a/src/test/integration/config/loader.ts +++ b/src/test/integration/config/loader.ts @@ -89,6 +89,7 @@ function readJsonFile(filePath: string, label: string): unknown { } catch (error) { throw new Error( `[integration-config] failed to load ${label} at ${filePath}: ${(error as Error).message}`, + { cause: error }, ); } } diff --git a/src/test/integration/tools/generate-command-endpoint-map.ts b/src/test/integration/tools/generate-command-endpoint-map.ts index a7c5150d6..aedf5de14 100644 --- a/src/test/integration/tools/generate-command-endpoint-map.ts +++ b/src/test/integration/tools/generate-command-endpoint-map.ts @@ -206,15 +206,15 @@ function parseFailureCategory(value: string): FailureCategory { function printHelp(): void { process.stdout.write( - `Usage:\n` + - ` yarn tool:integration:generate-command-endpoint-map [options]\n\n` + - `Options:\n` + + "Usage:\n" + + " yarn tool:integration:generate-command-endpoint-map [options]\n\n" + + "Options:\n" + ` --machine-log NDJSON log from run-all integration test (default: ${DEFAULT_MACHINE_LOG_PATH})\n` + - ` --category Optional failure category filter\n` + + " --category Optional failure category filter\n" + ` --openapi OpenAPI JSON file (default: ${DEFAULT_OPENAPI_PATH})\n` + ` --output-json Output JSON mapping (default: ${DEFAULT_OUTPUT_JSON_PATH})\n` + ` --output-md Output markdown summary (default: ${DEFAULT_OUTPUT_MARKDOWN_PATH})\n` + - ` -h, --help Show this help\n`, + " -h, --help Show this help\n", ); } @@ -361,6 +361,7 @@ function loadMachineLogData(machineLogPath: string): { const message = error instanceof Error ? error.message : String(error); throw new Error( `Invalid NDJSON at ${machineLogPath}:${idx + 1}: ${message}`, + { cause: error }, ); } From 687586b1780c2c8e53e267b13c4e0d695d4e8913 Mon Sep 17 00:00:00 2001 From: Lars Bergmann Date: Thu, 6 Aug 2026 15:07:37 +0200 Subject: [PATCH 16/49] Qualify waiver --- src/test/integration/config/command-waivers.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/integration/config/command-waivers.json b/src/test/integration/config/command-waivers.json index 5ce3332e6..c6d530ecb 100644 --- a/src/test/integration/config/command-waivers.json +++ b/src/test/integration/config/command-waivers.json @@ -81,14 +81,14 @@ "commandId": "app database link", "category": "DEPRECATED_ENDPOINT", "reason": "Command currently uses a deprecated API endpoint that is filtered by the mockoon setup and therefore not served in integration runs. The command should be reworked to use a current non-deprecated endpoint.", - "issue": "rework-deprecated-endpoint" + "issue": "rework-deprecated-endpoint-PR-2055" }, { "id": "deprecated-endpoint-app-database-replace", "commandId": "app database replace", "category": "DEPRECATED_ENDPOINT", "reason": "Command currently uses a deprecated API endpoint that is filtered by the mockoon setup and therefore not served in integration runs. The command should be reworked to use a current non-deprecated endpoint.", - "issue": "rework-deprecated-endpoint" + "issue": "rework-deprecated-endpoint-PR-2055" }, { "id": "deprecated-endpoint-stack-set-update-schedule", From 3be74ed8bacc65bfeccb42ba8a798e3bc2389bde Mon Sep 17 00:00:00 2001 From: Lars Bergmann Date: Thu, 6 Aug 2026 15:38:41 +0200 Subject: [PATCH 17/49] proof that new commands are discovered and tested --- src/test/integration/config/command-classifications.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/integration/config/command-classifications.json b/src/test/integration/config/command-classifications.json index bb79fbda1..7f82aeabb 100644 --- a/src/test/integration/config/command-classifications.json +++ b/src/test/integration/config/command-classifications.json @@ -1,14 +1,14 @@ { "schemaVersion": 1, - "generatedAt": "2026-08-04T12:10:47.523Z", + "generatedAt": "2026-08-06T13:37:23.660Z", "source": { "kind": "run-all-summary" }, "statistics": { - "successful": 117, + "successful": 120, "failed": 0, "waivedSkipped": 68, - "total": 185 + "total": 188 }, "entries": [ { From 09ad32f4d0faa76b58a8fd8a24e90b07eb276878 Mon Sep 17 00:00:00 2001 From: Lars Bergmann Date: Thu, 6 Aug 2026 16:25:34 +0200 Subject: [PATCH 18/49] Remove waivers via rebase. This works! --- .../config/command-classifications.json | 16 +++------------- src/test/integration/config/command-waivers.json | 14 -------------- 2 files changed, 3 insertions(+), 27 deletions(-) diff --git a/src/test/integration/config/command-classifications.json b/src/test/integration/config/command-classifications.json index 7f82aeabb..080e3a39b 100644 --- a/src/test/integration/config/command-classifications.json +++ b/src/test/integration/config/command-classifications.json @@ -1,13 +1,13 @@ { "schemaVersion": 1, - "generatedAt": "2026-08-06T13:37:23.660Z", + "generatedAt": "2026-08-06T14:22:58.704Z", "source": { "kind": "run-all-summary" }, "statistics": { - "successful": 120, + "successful": 122, "failed": 0, - "waivedSkipped": 68, + "waivedSkipped": 66, "total": 188 }, "entries": [ @@ -326,16 +326,6 @@ "category": "INTERACTIVE_REQUIRED", "source": "waiver" }, - { - "commandId": "stack set-update-schedule", - "category": "DEPRECATED_ENDPOINT", - "source": "waiver" - }, - { - "commandId": "stack unset-update-schedule", - "category": "DEPRECATED_ENDPOINT", - "source": "waiver" - }, { "commandId": "user ssh-key create", "category": "INTERACTIVE_REQUIRED", diff --git a/src/test/integration/config/command-waivers.json b/src/test/integration/config/command-waivers.json index c6d530ecb..6a6a33d54 100644 --- a/src/test/integration/config/command-waivers.json +++ b/src/test/integration/config/command-waivers.json @@ -90,20 +90,6 @@ "reason": "Command currently uses a deprecated API endpoint that is filtered by the mockoon setup and therefore not served in integration runs. The command should be reworked to use a current non-deprecated endpoint.", "issue": "rework-deprecated-endpoint-PR-2055" }, - { - "id": "deprecated-endpoint-stack-set-update-schedule", - "commandId": "stack set-update-schedule", - "category": "DEPRECATED_ENDPOINT", - "reason": "Command currently uses a deprecated API endpoint that is filtered by the mockoon setup and therefore not served in integration runs. The command should be reworked to use a current non-deprecated endpoint.", - "issue": "rework-deprecated-endpoint" - }, - { - "id": "deprecated-endpoint-stack-unset-update-schedule", - "commandId": "stack unset-update-schedule", - "category": "DEPRECATED_ENDPOINT", - "reason": "Command currently uses a deprecated API endpoint that is filtered by the mockoon setup and therefore not served in integration runs. The command should be reworked to use a current non-deprecated endpoint.", - "issue": "rework-deprecated-endpoint" - }, { "id": "contract-shape-app-create-node-invalid-version", "commandId": "app create node", From 10d179fcb864a130eeb1d61278a4e1a73dee2f31 Mon Sep 17 00:00:00 2001 From: Lars Bergmann Date: Thu, 6 Aug 2026 18:19:37 +0200 Subject: [PATCH 19/49] refactor test engine, docs drafts --- .../config/command-classifications.json | 2 +- src/test/integration/run-all-commands.test.ts | 384 ++++-------------- .../integration/run-all-commands/context.ts | 26 ++ .../integration/run-all-commands/helpers.ts | 210 ++++++++++ .../run-all-commands/machine-log.ts | 21 + .../integration/run-all-commands/overrides.ts | 91 +++++ test_docs/integration-artifacts.md | 106 +++++ test_docs/run-all-commands.md | 105 +++++ test_docs/waiver-governance.md | 88 ++++ 9 files changed, 735 insertions(+), 298 deletions(-) create mode 100644 src/test/integration/run-all-commands/context.ts create mode 100644 src/test/integration/run-all-commands/helpers.ts create mode 100644 src/test/integration/run-all-commands/machine-log.ts create mode 100644 src/test/integration/run-all-commands/overrides.ts create mode 100644 test_docs/integration-artifacts.md create mode 100644 test_docs/run-all-commands.md create mode 100644 test_docs/waiver-governance.md diff --git a/src/test/integration/config/command-classifications.json b/src/test/integration/config/command-classifications.json index 080e3a39b..bcc8f24e2 100644 --- a/src/test/integration/config/command-classifications.json +++ b/src/test/integration/config/command-classifications.json @@ -1,6 +1,6 @@ { "schemaVersion": 1, - "generatedAt": "2026-08-06T14:22:58.704Z", + "generatedAt": "2026-08-06T15:54:03.172Z", "source": { "kind": "run-all-summary" }, diff --git a/src/test/integration/run-all-commands.test.ts b/src/test/integration/run-all-commands.test.ts index 42cc592e7..2f2394729 100644 --- a/src/test/integration/run-all-commands.test.ts +++ b/src/test/integration/run-all-commands.test.ts @@ -6,20 +6,19 @@ import { it, jest, } from "@jest/globals"; -import { appendFile, mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; import { buildClassificationCatalogFromBuckets, + createFailureBuckets, + FAILURE_CATEGORIES, parseFailureCategory, saveClassificationCatalog, } from "./classification-catalog.js"; import { runDevCommand } from "./command.js"; import { discoverRunnableCommands } from "./command-discovery.js"; -import type { - CommandWaiver, - WaiverCategory, -} from "./command-discovery/types.js"; +import type { WaiverCategory } from "./command-discovery/types.js"; import { loadCommandWaivers } from "./config/loader.js"; import { configureIntegrationEnv, @@ -27,29 +26,31 @@ import { restoreEnv, snapshotEnv, } from "./env.js"; +import { seedProjectContext } from "./run-all-commands/context.js"; +import { + classifyFailure, + formatNonWaivedFailureSummary, + logBucketSummary, + logCommandFailureOutput, + mapCommandWaivers, + type NonWaivedFailure, + validateInvocationCompleteness, +} from "./run-all-commands/helpers.js"; +import { + appendMachineLogEntry, + initializeMachineLogFile, +} from "./run-all-commands/machine-log.js"; +import { + applyCommandOverride, + loadRunAllOverrides, + resolveInvocationArgs, + shouldBypassWaiverForCommand, +} from "./run-all-commands/overrides.js"; jest.setTimeout(20 * 60 * 1000); type FailureCategory = WaiverCategory; -type MachineLogEntry = Record; - -type NonWaivedFailure = { - commandId: string; - kind: "failure" | "spawn-error"; - category?: FailureCategory; - details: string; -}; - -const FAILURE_CATEGORIES: FailureCategory[] = [ - "ARG_MISUSE", - "INTERACTIVE_REQUIRED", - "RESOURCE_PRECONDITION", - "CONTRACT_SHAPE", - "COMMAND_BUG", - "DEPRECATED_ENDPOINT", -]; - function isExplicitRunByPathInvocationForThisFile(): boolean { const args = process.argv.slice(2); const runTestsByPathArgs = new Set(); @@ -93,264 +94,10 @@ const describeRunAllCommands = isExplicitRunByPathInvocationForThisFile() ? describe : describe.skip; -function createFailureBuckets(): Record { - return { - ARG_MISUSE: [], - INTERACTIVE_REQUIRED: [], - RESOURCE_PRECONDITION: [], - CONTRACT_SHAPE: [], - COMMAND_BUG: [], - DEPRECATED_ENDPOINT: [], - }; -} - -function classifyFailure(output: { - stderr: string; - stdout: string; -}): FailureCategory { - const text = `${output.stderr}\n${output.stdout}`.toLowerCase(); - - if ( - /missing\s+(?:\d+\s+)?required arg|missing\s+(?:\d+\s+)?required flag|exactly one of|required options|unexpected argument|unknown flag|nonexistent flag|invalid flag|flag .* expects|no .* id given|you need to specify at least one/i.test( - text, - ) - ) { - return "ARG_MISUSE"; - } - - if ( - /prompt|interactive|addinput|addselect|addconfirmation|overwrite\?|token file already exists|tty/i.test( - text, - ) - ) { - return "INTERACTIVE_REQUIRED"; - } - - if ( - /not found|does not exist|no .* found|resource.*missing|404|forbidden|unauthorized|no project found|failed to connect|could not resolve hostname|name or service not known|no main user found|main mysql user can not be deleted manually/i.test( - text, - ) - ) { - return "RESOURCE_PRECONDITION"; - } - - if ( - /invalid version|not iterable|cannot read properties|undefined.*data|validation|invalid type|schema/i.test( - text, - ) - ) { - return "CONTRACT_SHAPE"; - } - - return "COMMAND_BUG"; -} - -function parseInvocationPartsFromArgs( - args: string[], - commandTokenCount: number, -): { positionalValues: string[]; flagValues: Map } { - const positionalValues: string[] = []; - const flagValues = new Map(); - - const invocationArgs = args.slice(commandTokenCount); - for (let i = 0; i < invocationArgs.length; i += 1) { - const token = invocationArgs[i]; - if (!token.startsWith("--")) { - positionalValues.push(token); - continue; - } - - const withoutPrefix = token.slice(2); - const eqIndex = withoutPrefix.indexOf("="); - let name = withoutPrefix; - let value: string | undefined; - - if (eqIndex >= 0) { - name = withoutPrefix.slice(0, eqIndex); - value = withoutPrefix.slice(eqIndex + 1); - } else { - const nextToken = invocationArgs[i + 1]; - if (nextToken && !nextToken.startsWith("--")) { - value = nextToken; - i += 1; - } - } - - const values = flagValues.get(name) ?? []; - values.push(value ?? "true"); - flagValues.set(name, values); - } - - return { positionalValues, flagValues }; -} - -function validateInvocationCompleteness( - command: Awaited>[number], -): string[] { - const issues: string[] = []; - const { positionalValues, flagValues } = parseInvocationPartsFromArgs( - command.synthesizedInvocation.args, - command.commandTokens.length, - ); - - command.parsedArgs.forEach((arg, index) => { - if (!arg.required) { - return; - } - if (positionalValues[index] === undefined) { - issues.push(`missing required arg ${arg.name}`); - } - }); - - for (const flag of command.parsedFlags) { - if (flag.required && !flagValues.has(flag.name)) { - issues.push(`missing required flag --${flag.name}`); - } - } - - const exactlyOneGroups = new Map(); - for (const flag of command.parsedFlags) { - if (!flag.exactlyOne || flag.exactlyOne.length < 2) { - continue; - } - const members = [...new Set(flag.exactlyOne)].sort(); - exactlyOneGroups.set(members.join("|"), members); - } - - for (const members of exactlyOneGroups.values()) { - const selected = members.filter((member) => flagValues.has(member)); - if (selected.length !== 1) { - issues.push(`exactly-one unresolved [${members.join(",")}]`); - } - } - - return issues; -} - -function logFailureTaxonomySummary( - failuresByCategory: Record, -): void { - logProgress("[run-all] failure taxonomy summary:"); - - for (const category of FAILURE_CATEGORIES) { - const commands = failuresByCategory[category]; - const sample = commands.slice(0, 5).join(", "); - logProgress( - `[run-all] ${category.padEnd(22, " ")} count=${String(commands.length).padStart(3, " ")} sample=${sample || "-"}`, - ); - } -} - -function mapCommandWaivers(waivers: CommandWaiver[]): { - waiversByCommandId: Map; - duplicates: string[]; -} { - const waiversByCommandId = new Map(); - const duplicates: string[] = []; - - for (const waiver of waivers) { - if (waiversByCommandId.has(waiver.commandId)) { - duplicates.push(waiver.commandId); - continue; - } - - waiversByCommandId.set(waiver.commandId, waiver); - } - - return { waiversByCommandId, duplicates }; -} - -function logWaiverSummary( - waivedByCategory: Record, -): void { - logProgress("[run-all] waiver summary:"); - - for (const category of FAILURE_CATEGORIES) { - const commands = waivedByCategory[category]; - const sample = commands.slice(0, 5).join(", "); - logProgress( - `[run-all] ${category.padEnd(22, " ")} count=${String(commands.length).padStart(3, " ")} sample=${sample || "-"}`, - ); - } -} - function logProgress(message: string): void { process.stderr.write(`${message}\n`); } -function formatNonWaivedFailureSummary(failures: NonWaivedFailure[]): string { - if (failures.length === 0) { - return ""; - } - - return failures - .map((failure) => { - const base = - failure.kind === "failure" - ? `${failure.commandId} [${failure.category}]` - : `${failure.commandId} [spawn-error]`; - return `${base}: ${failure.details}`; - }) - .join("\n"); -} - -function formatOutputBlock(output: string): string { - const trimmed = output.trim(); - return trimmed.length > 0 ? trimmed : ""; -} - -function logCommandFailureOutput( - position: string, - commandId: string, - result: { stdout: string; stderr: string }, -): void { - logProgress(`[${position}] diagnostics ${commandId}: stderr >>>`); - logProgress(formatOutputBlock(result.stderr)); - logProgress(`[${position}] diagnostics ${commandId}: stdout >>>`); - logProgress(formatOutputBlock(result.stdout)); - logProgress(`[${position}] diagnostics ${commandId}: <<<`); -} - -async function initializeMachineLogFile(filePath: string): Promise { - await mkdir(path.dirname(filePath), { recursive: true }); - await writeFile(filePath, "", "utf-8"); -} - -async function appendMachineLogEntry( - filePath: string, - entry: MachineLogEntry, -): Promise { - const line = JSON.stringify({ - timestamp: new Date().toISOString(), - ...entry, - }); - await appendFile(filePath, `${line}\n`, "utf-8"); -} - -async function seedProjectContext(projectId: string): Promise { - const configDir = process.env.MW_CONFIG_DIR; - - if (!configDir) { - throw new Error( - "[integration:run-all-commands] MW_CONFIG_DIR was not set before seeding project context.", - ); - } - - const contextFile = path.join(configDir, "context.json"); - - await mkdir(configDir, { recursive: true }); - await writeFile( - contextFile, - JSON.stringify({ - "project-id": projectId, - "server-id": "6b4f48f5-d80c-4d20-9db8-fecf4c9e6221", - "installation-id": "f7b47c12-7d11-4f3a-b9bc-1b3c706e1d55", - "org-id": "88e8d927-7db4-42ef-ae02-f8a7ef0b4d77", - }), - "utf-8", - ); -} - describeRunAllCommands("integration: run all commands", () => { let originalEnv: NodeJS.ProcessEnv; let tempConfigDir: string; @@ -379,6 +126,7 @@ describeRunAllCommands("integration: run all commands", () => { const machineLogPath = process.env.MW_TEST_MACHINE_LOG_PATH?.trim() || path.resolve("run-all-commands.ndjson"); + const runtimeOverrides = loadRunAllOverrides(); await initializeMachineLogFile(machineLogPath); logProgress(`[run-all] machine log path=${machineLogPath}`); @@ -390,11 +138,13 @@ describeRunAllCommands("integration: run all commands", () => { ); logProgress("[run-all] starting command discovery"); - const commands = await discoverRunnableCommands({ + const discoveredCommands = await discoverRunnableCommands({ onProgress: logProgress, categoryFilter, classificationCatalogPath, }); + + const commands = applyCommandOverride(discoveredCommands, runtimeOverrides); expect(commands.length).toBeGreaterThan(0); if (categoryFilter) { @@ -403,6 +153,12 @@ describeRunAllCommands("integration: run all commands", () => { ); } + if (runtimeOverrides.commandId) { + logProgress( + `[run-all] command override active: ${runtimeOverrides.commandId}${runtimeOverrides.invocationArgs ? " (custom invocation args)" : ""}`, + ); + } + const waivers = loadCommandWaivers(); const { waiversByCommandId, duplicates } = mapCommandWaivers(waivers); @@ -413,6 +169,7 @@ describeRunAllCommands("integration: run all commands", () => { projectId, commandCount: commands.length, waiverCount: waivers.length, + runtimeOverrides, }); logProgress(`[run-all] discovered ${commands.length} commands to execute`); @@ -436,7 +193,10 @@ describeRunAllCommands("integration: run all commands", () => { let failedCommands = 0; let waivedSkippedCommands = 0; - if (!categoryFilter) { + const strictWaiverIntegrityMode = + !categoryFilter && runtimeOverrides.commandId === undefined; + + if (strictWaiverIntegrityMode) { if (duplicates.length > 0) { infrastructureFailures.push( `[waivers] duplicate waiver commandId entries: ${duplicates.join(", ")}`, @@ -455,19 +215,24 @@ describeRunAllCommands("integration: run all commands", () => { } } else { logProgress( - "[waivers] strict waiver integrity checks skipped (category filter active)", + "[waivers] strict waiver integrity checks skipped (category filter or command override active)", ); } for (const [index, command] of commands.entries()) { const position = `${index + 1}/${commands.length}`; - const invocation = command.synthesizedInvocation; + const synthesizedInvocation = command.synthesizedInvocation; + const effectiveInvocationArgs = resolveInvocationArgs( + command, + runtimeOverrides, + ); const waiver = waiversByCommandId.get(command.commandId); + const bypassWaiver = shouldBypassWaiverForCommand(command, runtimeOverrides); const commandStartedAt = Date.now(); await seedProjectContext(projectId); logProgress( - `[${position}] running ${command.commandId} (source=${invocation.argumentSource}; interactive=${invocation.interactiveDecision}; re-seeded project context)`, + `[${position}] running ${command.commandId} (source=${synthesizedInvocation.argumentSource}; interactive=${synthesizedInvocation.interactiveDecision}; re-seeded project context)`, ); await appendMachineLogEntry(machineLogPath, { @@ -483,12 +248,14 @@ describeRunAllCommands("integration: run all commands", () => { interactiveSignals: command.interactiveSignals, invocationProfilesApplied: command.invocationProfilesApplied, extractionDiagnostics: command.extractionDiagnostics, - invocationArgs: invocation.args, - argumentSource: invocation.argumentSource, - interactiveDecision: invocation.interactiveDecision, + invocationArgs: effectiveInvocationArgs, + synthesizedInvocationArgs: synthesizedInvocation.args, + argumentSource: synthesizedInvocation.argumentSource, + interactiveDecision: synthesizedInvocation.interactiveDecision, + overrideApplied: effectiveInvocationArgs !== synthesizedInvocation.args, }); - if (waiver) { + if (waiver && !bypassWaiver) { waivedSkippedCommands += 1; waivedByCategory[waiver.category].push(command.commandId); logProgress( @@ -507,7 +274,16 @@ describeRunAllCommands("integration: run all commands", () => { continue; } - if (invocation.interactiveDecision === "INTERACTIVE_REQUIRED") { + if (waiver && bypassWaiver) { + logProgress( + `[${position}] waiver bypass ${command.commandId} (category=${waiver.category}; reason=${waiver.reason})`, + ); + } + + if ( + synthesizedInvocation.interactiveDecision === "INTERACTIVE_REQUIRED" && + !bypassWaiver + ) { failedCommands += 1; failuresByCategory.INTERACTIVE_REQUIRED.push(command.commandId); nonWaivedFailures.push({ @@ -536,7 +312,10 @@ describeRunAllCommands("integration: run all commands", () => { continue; } - const staticInvocationIssues = validateInvocationCompleteness(command); + const staticInvocationIssues = validateInvocationCompleteness( + command, + effectiveInvocationArgs, + ); if (staticInvocationIssues.length > 0) { failedCommands += 1; failuresByCategory.ARG_MISUSE.push(command.commandId); @@ -563,7 +342,7 @@ describeRunAllCommands("integration: run all commands", () => { continue; } - const result = await runDevCommand(invocation.args, { + const result = await runDevCommand(effectiveInvocationArgs, { timeoutMs: 30_000, }); @@ -577,7 +356,7 @@ describeRunAllCommands("integration: run all commands", () => { details: "timed out after 30000ms", }); logProgress(`[${position}] timeout ${command.commandId}`); - logCommandFailureOutput(position, command.commandId, result); + logCommandFailureOutput(position, command.commandId, result, logProgress); await appendMachineLogEntry(machineLogPath, { event: "command-result", index: index + 1, @@ -603,12 +382,12 @@ describeRunAllCommands("integration: run all commands", () => { details: errorMessage, }); infrastructureFailures.push( - `${command.commandId} failed to execute (source=${invocation.argumentSource}): ${errorMessage}`, + `${command.commandId} failed to execute (source=${synthesizedInvocation.argumentSource}): ${errorMessage}`, ); logProgress( `[${position}] spawn-error ${command.commandId}: ${errorMessage}`, ); - logCommandFailureOutput(position, command.commandId, result); + logCommandFailureOutput(position, command.commandId, result, logProgress); await appendMachineLogEntry(machineLogPath, { event: "command-result", index: index + 1, @@ -637,7 +416,7 @@ describeRunAllCommands("integration: run all commands", () => { logProgress( `[${position}] classified ${command.commandId} as ${category}`, ); - logCommandFailureOutput(position, command.commandId, result); + logCommandFailureOutput(position, command.commandId, result, logProgress); await appendMachineLogEntry(machineLogPath, { event: "command-result", @@ -676,8 +455,18 @@ describeRunAllCommands("integration: run all commands", () => { logProgress( `[run-all] statistics: successful=${successfulCommands}, failed=${failedCommands}, waived-skipped=${waivedSkippedCommands}, total=${commands.length}`, ); - logFailureTaxonomySummary(failuresByCategory); - logWaiverSummary(waivedByCategory); + logBucketSummary( + "[run-all] failure taxonomy summary:", + FAILURE_CATEGORIES, + failuresByCategory, + logProgress, + ); + logBucketSummary( + "[run-all] waiver summary:", + FAILURE_CATEGORIES, + waivedByCategory, + logProgress, + ); await appendMachineLogEntry(machineLogPath, { event: "run-summary", @@ -690,9 +479,10 @@ describeRunAllCommands("integration: run all commands", () => { failuresByCategory, waivedByCategory, infrastructureFailures, + runtimeOverrides, }); - if (!categoryFilter) { + if (!categoryFilter && !runtimeOverrides.commandId) { const classificationCatalog = buildClassificationCatalogFromBuckets({ failuresByCategory, waivedByCategory, @@ -710,7 +500,7 @@ describeRunAllCommands("integration: run all commands", () => { ); } else { logProgress( - "[run-all] skipped classification catalog write (category filter active)", + "[run-all] skipped classification catalog write (category filter or command override active)", ); } diff --git a/src/test/integration/run-all-commands/context.ts b/src/test/integration/run-all-commands/context.ts new file mode 100644 index 000000000..e9258b170 --- /dev/null +++ b/src/test/integration/run-all-commands/context.ts @@ -0,0 +1,26 @@ +import { mkdir, writeFile } from "node:fs/promises"; +import path from "node:path"; + +export async function seedProjectContext(projectId: string): Promise { + const configDir = process.env.MW_CONFIG_DIR; + + if (!configDir) { + throw new Error( + "[integration:run-all-commands] MW_CONFIG_DIR was not set before seeding project context.", + ); + } + + const contextFile = path.join(configDir, "context.json"); + + await mkdir(configDir, { recursive: true }); + await writeFile( + contextFile, + JSON.stringify({ + "project-id": projectId, + "server-id": "6b4f48f5-d80c-4d20-9db8-fecf4c9e6221", + "installation-id": "f7b47c12-7d11-4f3a-b9bc-1b3c706e1d55", + "org-id": "88e8d927-7db4-42ef-ae02-f8a7ef0b4d77", + }), + "utf-8", + ); +} diff --git a/src/test/integration/run-all-commands/helpers.ts b/src/test/integration/run-all-commands/helpers.ts new file mode 100644 index 000000000..dd4608277 --- /dev/null +++ b/src/test/integration/run-all-commands/helpers.ts @@ -0,0 +1,210 @@ +import type { FailureCategory } from "../classification-catalog.js"; +import type { + CommandWaiver, + DiscoveredCommand, +} from "../command-discovery/types.js"; + +export type NonWaivedFailure = { + commandId: string; + kind: "failure" | "spawn-error"; + category?: FailureCategory; + details: string; +}; + +export function classifyFailure(output: { + stderr: string; + stdout: string; +}): FailureCategory { + const text = `${output.stderr}\n${output.stdout}`.toLowerCase(); + + if ( + /missing\s+(?:\d+\s+)?required arg|missing\s+(?:\d+\s+)?required flag|exactly one of|required options|unexpected argument|unknown flag|nonexistent flag|invalid flag|flag .* expects|no .* id given|you need to specify at least one/i.test( + text, + ) + ) { + return "ARG_MISUSE"; + } + + if ( + /prompt|interactive|addinput|addselect|addconfirmation|overwrite\?|token file already exists|tty/i.test( + text, + ) + ) { + return "INTERACTIVE_REQUIRED"; + } + + if ( + /not found|does not exist|no .* found|resource.*missing|404|forbidden|unauthorized|no project found|failed to connect|could not resolve hostname|name or service not known|no main user found|main mysql user can not be deleted manually/i.test( + text, + ) + ) { + return "RESOURCE_PRECONDITION"; + } + + if ( + /invalid version|not iterable|cannot read properties|undefined.*data|validation|invalid type|schema/i.test( + text, + ) + ) { + return "CONTRACT_SHAPE"; + } + + return "COMMAND_BUG"; +} + +export function validateInvocationCompleteness( + command: DiscoveredCommand, + invocationArgs: string[], +): string[] { + const issues: string[] = []; + const { positionalValues, flagValues } = parseInvocationPartsFromArgs( + invocationArgs, + command.commandTokens.length, + ); + + command.parsedArgs.forEach((arg, index) => { + if (!arg.required) { + return; + } + + if (positionalValues[index] === undefined) { + issues.push(`missing required arg ${arg.name}`); + } + }); + + for (const flag of command.parsedFlags) { + if (flag.required && !flagValues.has(flag.name)) { + issues.push(`missing required flag --${flag.name}`); + } + } + + const exactlyOneGroups = new Map(); + for (const flag of command.parsedFlags) { + if (!flag.exactlyOne || flag.exactlyOne.length < 2) { + continue; + } + + const members = [...new Set(flag.exactlyOne)].sort(); + exactlyOneGroups.set(members.join("|"), members); + } + + for (const members of exactlyOneGroups.values()) { + const selected = members.filter((member) => flagValues.has(member)); + if (selected.length !== 1) { + issues.push(`exactly-one unresolved [${members.join(",")}]`); + } + } + + return issues; +} + +export function mapCommandWaivers(waivers: CommandWaiver[]): { + waiversByCommandId: Map; + duplicates: string[]; +} { + const waiversByCommandId = new Map(); + const duplicates: string[] = []; + + for (const waiver of waivers) { + if (waiversByCommandId.has(waiver.commandId)) { + duplicates.push(waiver.commandId); + continue; + } + + waiversByCommandId.set(waiver.commandId, waiver); + } + + return { waiversByCommandId, duplicates }; +} + +export function formatNonWaivedFailureSummary( + failures: NonWaivedFailure[], +): string { + if (failures.length === 0) { + return ""; + } + + return failures + .map((failure) => { + const base = + failure.kind === "failure" + ? `${failure.commandId} [${failure.category}]` + : `${failure.commandId} [spawn-error]`; + return `${base}: ${failure.details}`; + }) + .join("\n"); +} + +export function logBucketSummary( + label: string, + categories: FailureCategory[], + buckets: Record, + logProgress: (message: string) => void, +): void { + logProgress(label); + + for (const category of categories) { + const commands = buckets[category]; + const sample = commands.slice(0, 5).join(", "); + logProgress( + `[run-all] ${category.padEnd(22, " ")} count=${String(commands.length).padStart(3, " ")} sample=${sample || "-"}`, + ); + } +} + +export function logCommandFailureOutput( + position: string, + commandId: string, + result: { stdout: string; stderr: string }, + logProgress: (message: string) => void, +): void { + logProgress(`[${position}] diagnostics ${commandId}: stderr >>>`); + logProgress(formatOutputBlock(result.stderr)); + logProgress(`[${position}] diagnostics ${commandId}: stdout >>>`); + logProgress(formatOutputBlock(result.stdout)); + logProgress(`[${position}] diagnostics ${commandId}: <<<`); +} + +function parseInvocationPartsFromArgs( + args: string[], + commandTokenCount: number, +): { positionalValues: string[]; flagValues: Map } { + const positionalValues: string[] = []; + const flagValues = new Map(); + const invocationArgs = args.slice(commandTokenCount); + + for (let i = 0; i < invocationArgs.length; i += 1) { + const token = invocationArgs[i]; + if (!token.startsWith("--")) { + positionalValues.push(token); + continue; + } + + const withoutPrefix = token.slice(2); + const eqIndex = withoutPrefix.indexOf("="); + let name = withoutPrefix; + let value: string | undefined; + + if (eqIndex >= 0) { + name = withoutPrefix.slice(0, eqIndex); + value = withoutPrefix.slice(eqIndex + 1); + } else { + const nextToken = invocationArgs[i + 1]; + if (nextToken && !nextToken.startsWith("--")) { + value = nextToken; + i += 1; + } + } + + const values = flagValues.get(name) ?? []; + values.push(value ?? "true"); + flagValues.set(name, values); + } + + return { positionalValues, flagValues }; +} + +function formatOutputBlock(output: string): string { + const trimmed = output.trim(); + return trimmed.length > 0 ? trimmed : ""; +} diff --git a/src/test/integration/run-all-commands/machine-log.ts b/src/test/integration/run-all-commands/machine-log.ts new file mode 100644 index 000000000..0e64d4156 --- /dev/null +++ b/src/test/integration/run-all-commands/machine-log.ts @@ -0,0 +1,21 @@ +import { appendFile, mkdir, writeFile } from "node:fs/promises"; +import path from "node:path"; + +export type MachineLogEntry = Record; + +export async function initializeMachineLogFile(filePath: string): Promise { + await mkdir(path.dirname(filePath), { recursive: true }); + await writeFile(filePath, "", "utf-8"); +} + +export async function appendMachineLogEntry( + filePath: string, + entry: MachineLogEntry, +): Promise { + const line = JSON.stringify({ + timestamp: new Date().toISOString(), + ...entry, + }); + + await appendFile(filePath, `${line}\n`, "utf-8"); +} diff --git a/src/test/integration/run-all-commands/overrides.ts b/src/test/integration/run-all-commands/overrides.ts new file mode 100644 index 000000000..5f11a283c --- /dev/null +++ b/src/test/integration/run-all-commands/overrides.ts @@ -0,0 +1,91 @@ +import type { DiscoveredCommand } from "../command-discovery.js"; + +export type RunAllOverrides = { + commandId?: string; + invocationArgs?: string[]; +}; + +export function loadRunAllOverrides( + env: NodeJS.ProcessEnv = process.env, +): RunAllOverrides { + const commandId = env.MW_TEST_COMMAND_ID?.trim() || undefined; + const invocationArgsRaw = env.MW_TEST_COMMAND_INVOCATION_ARGS?.trim(); + const invocationArgs = invocationArgsRaw + ? parseInvocationArgs(invocationArgsRaw) + : undefined; + + if (invocationArgs && !commandId) { + throw new Error( + "[run-all] MW_TEST_COMMAND_INVOCATION_ARGS requires MW_TEST_COMMAND_ID.", + ); + } + + return { + commandId, + invocationArgs, + }; +} + +export function applyCommandOverride( + commands: DiscoveredCommand[], + overrides: RunAllOverrides, +): DiscoveredCommand[] { + if (!overrides.commandId) { + return commands; + } + + const selected = commands.find( + (command) => command.commandId === overrides.commandId, + ); + + if (!selected) { + throw new Error( + [ + `[run-all] MW_TEST_COMMAND_ID '${overrides.commandId}' was not found in discovery output.`, + "Discovered commands sample:", + ...commands.slice(0, 20).map((command) => `- ${command.commandId}`), + ].join("\n"), + ); + } + + return [selected]; +} + +export function resolveInvocationArgs( + command: DiscoveredCommand, + overrides: RunAllOverrides, +): string[] { + if (overrides.commandId === command.commandId && overrides.invocationArgs) { + return overrides.invocationArgs; + } + + return command.synthesizedInvocation.args; +} + +export function shouldBypassWaiverForCommand( + command: DiscoveredCommand, + overrides: RunAllOverrides, +): boolean { + return overrides.commandId === command.commandId; +} + +function parseInvocationArgs(value: string): string[] { + let parsed: unknown; + + try { + parsed = JSON.parse(value); + } catch (error) { + throw new Error( + `[run-all] MW_TEST_COMMAND_INVOCATION_ARGS must be valid JSON array: ${(error as Error).message}`, + { cause: error }, + ); + } + + if (!Array.isArray(parsed) || !parsed.every((item) => typeof item === "string")) { + throw new Error( + "[run-all] MW_TEST_COMMAND_INVOCATION_ARGS must be a JSON array of strings.", + ); + } + + return [...parsed]; +} diff --git a/test_docs/integration-artifacts.md b/test_docs/integration-artifacts.md new file mode 100644 index 000000000..faaba0df2 --- /dev/null +++ b/test_docs/integration-artifacts.md @@ -0,0 +1,106 @@ +# Integration Artifacts and Contracts (Draft) + +## Purpose +Define the artifact contract for run-all integration execution and downstream analysis tooling. + +## Artifact Flow +1. Runner executes discovered commands. +2. Runner emits NDJSON machine log. +3. Runner emits/updates classification catalog (full runs only). +4. Analyzer consumes machine log records and command source/transitive analysis to map API usage to descriptors/OpenAPI operations. +5. Reports are generated as JSON and Markdown. + +## Primary Artifacts +- run-all-commands.ndjson +- src/test/integration/config/command-classifications.json + +Analyzer artifacts (generated only when analyzer tooling is executed): +- command-endpoint-map.json +- command-endpoint-map.md + +## NDJSON Events +Expected event types: +- run-start +- command-start +- command-result +- run-summary + +### command-start key fields +- commandId +- sourceFile +- commandTokens +- parsedArgs +- parsedFlags +- interactiveSignals +- invocationProfilesApplied +- extractionDiagnostics +- invocationArgs +- synthesizedInvocationArgs +- argumentSource +- interactiveDecision +- overrideApplied + +### command-result key fields +- commandId +- status (succeeded | failed | waived | spawn-error) +- failureCategory (for failed) +- durationMs +- exitCode (when available) + +## Classification Catalog Contract +File: src/test/integration/config/command-classifications.json + +Key structure: +- schemaVersion +- generatedAt +- source +- statistics +- entries[] with: + - commandId + - category + - source (failure | waiver | skip) + +Note: +- Runner-generated catalogs currently contain failure and waiver entries. +- skip entries are supported by the catalog schema and log-extract helper. + +## Category Filter Contract +When MW_TEST_CATEGORY is set: +- discovery still enumerates all commands +- execution list is filtered to command IDs from classification catalog entries matching category +- strict global waiver integrity checks are skipped for partial scope + +## Single-Command Override Contract +When MW_TEST_COMMAND_ID is set: +- execution list contains only that command +- waiver is bypassed for that selected command +- strict global waiver integrity checks are skipped for partial scope +- if MW_TEST_COMMAND_INVOCATION_ARGS is set, it replaces synthesized invocation args + +## Analyzer Tooling +Script entry points in package scripts: +- tool:integration:generate-command-endpoint-map +- tool:integration:generate-resource-precondition-map + +Inputs: +- machine log NDJSON +- OpenAPI JSON + +Outputs: +- endpoint map JSON +- endpoint map Markdown + +## Failure Taxonomy +Canonical categories: +- ARG_MISUSE +- INTERACTIVE_REQUIRED +- RESOURCE_PRECONDITION +- CONTRACT_SHAPE +- COMMAND_BUG +- DEPRECATED_ENDPOINT + +## Compatibility Guidance +If you modify runner payload fields: +1. Keep existing fields backward-compatible when possible. +2. Update analyzer expectations in lockstep. +3. Document contract changes in this file before merging. diff --git a/test_docs/run-all-commands.md b/test_docs/run-all-commands.md new file mode 100644 index 000000000..ab050c1a9 --- /dev/null +++ b/test_docs/run-all-commands.md @@ -0,0 +1,105 @@ +# Run-All Commands Integration Runner (Draft) + +## Purpose +In full-matrix mode, run every discovered CLI command once in an integration context, emit machine-readable NDJSON logs, and enforce that all non-waived failures are visible and actionable. + +## Scope +This runner is implemented in: +- src/test/integration/run-all-commands.test.ts + +Note: +- The suite is guarded and only runs when the file is invoked explicitly via --runTestsByPath. + +It depends on: +- command discovery and invocation synthesis +- waiver configuration +- classification catalog generation + +## Required Environment +The test requires: +- MITTWALD_API_TOKEN +- MITTWALD_API_BASE_URL +- MW_TEST_PROJECT_ID + +## Optional Environment Controls +- MW_TEST_MACHINE_LOG_PATH + - Path for NDJSON output (default: run-all-commands.ndjson in repo root) +- MW_TEST_CATEGORY + - Restrict execution to command IDs listed in classification catalog for one category +- MW_TEST_CLASSIFICATION_CATALOG_PATH + - Override catalog path used by category filtering +- MW_TEST_COMMAND_ID + - Run only one discovered command ID + - When set, waiver for that command is bypassed intentionally (waiver hunting mode) +- MW_TEST_COMMAND_INVOCATION_ARGS + - JSON array of strings to fully override invocation args for MW_TEST_COMMAND_ID + - Requires MW_TEST_COMMAND_ID + +## Environment Variable Usage +All runtime controls are plain environment variables. + +You can use them in two styles: +- one-off: prefix variables for a single command invocation +- session: export/set variables in shell state, run command, then unset + +Example scenario used below: +- run a single command in waiver-hunting mode +- command ID: container logs + +### Bash/Zsh Examples +One-off invocation: +```bash +MW_TEST_PROJECT_ID="" \ +MW_TEST_COMMAND_ID="container logs" \ +yarn test --runTestsByPath src/test/integration/run-all-commands.test.ts +``` + +Session set, run, unset: +```bash +export MW_TEST_PROJECT_ID="" +export MW_TEST_COMMAND_ID="container logs" +yarn test --runTestsByPath src/test/integration/run-all-commands.test.ts +unset MW_TEST_COMMAND_ID +unset MW_TEST_PROJECT_ID +``` + +### Fish Examples +One-off invocation: +```fish +env MW_TEST_PROJECT_ID="" MW_TEST_COMMAND_ID="container logs" \ + yarn test --runTestsByPath src/test/integration/run-all-commands.test.ts +``` + +Session set, run, unset: +```fish +set -lx MW_TEST_PROJECT_ID "" +set -lx MW_TEST_COMMAND_ID "container logs" +yarn test --runTestsByPath src/test/integration/run-all-commands.test.ts +set -e MW_TEST_COMMAND_ID +set -e MW_TEST_PROJECT_ID +``` + +Optional invocation-arg override in any shell: +```sh +MW_TEST_COMMAND_INVOCATION_ARGS='["container","logs","--container-id","abc123","--tail","20"]' +``` + +## Operating Principles +1. Discovery first: commands are discovered from src/commands, then synthesized args are built. +2. Waiver file validation is always strict: + - duplicate waiver IDs fail + - duplicate waiver command IDs fail +3. Full runs add a global consistency check: + - waivers pointing to non-discovered commands fail +4. Command override mode skips the global waiver consistency check to enable focused debugging. +5. Category filter mode also skips the global waiver consistency check because execution scope is intentionally partial. +6. Non-waived failures fail the test and are surfaced with category and diagnostics. + +## Outputs +- NDJSON machine log with run-start, command-start, command-result, run-summary +- Classification catalog file update during full runs without category or command override + +## Notes for Maintainers +- Keep command IDs stable when refactoring command file paths. +- If invocation synthesis changes, verify MW_TEST_COMMAND_INVOCATION_ARGS still fully overrides run args. +- Preserve deterministic log fields consumed by downstream tooling. diff --git a/test_docs/waiver-governance.md b/test_docs/waiver-governance.md new file mode 100644 index 000000000..f8e195f49 --- /dev/null +++ b/test_docs/waiver-governance.md @@ -0,0 +1,88 @@ +# Waiver Governance for Integration Command Matrix (Draft) + +## Purpose +Waivers are a governance tool, not a suppression shortcut. They document known failing commands with category, reason, and follow-up intent while keeping failures auditable. + +## Source of Truth +- Waivers file: src/test/integration/config/command-waivers.json +- Waiver loader validation: src/test/integration/config/loader.ts +- Enforcement during run: src/test/integration/run-all-commands.test.ts + +## Waiver Schema +Each waiver entry must include: +- id +- commandId +- category +- reason + +Optional: +- issue +- expiresOn + +Allowed categories: +- ARG_MISUSE +- INTERACTIVE_REQUIRED +- RESOURCE_PRECONDITION +- CONTRACT_SHAPE +- COMMAND_BUG +- DEPRECATED_ENDPOINT + +## Hard Invariants +Always (loader validation): +1. Duplicate waiver IDs are invalid. +2. Duplicate waiver commandId entries are invalid. + +In full-matrix mode (no category filter and no command override): +3. Waiver commandId must map to a currently discovered command. +4. Commands classified INTERACTIVE_REQUIRED without waivers fail as governance drift. + +## Relaxed Invariants by Design +In targeted modes, the global waiver consistency check is skipped: +- category-filter mode +- single-command override mode (MW_TEST_COMMAND_ID) + +Reason: +- these modes are intentionally partial and used for investigation loops. + +Important: +- loader-level duplicate checks still apply in all modes. + +## Waiver Hunting Workflow +1. Run one command with MW_TEST_COMMAND_ID. +2. Reproduce and inspect failure details. +3. Decide one branch: + - fix command + - fix test fixture/precondition + - keep/add waiver with explicit reason and issue link +4. Re-run same command until category and behavior are stable. +5. If command becomes callable, remove waiver. + +## When to Add a Waiver +Add a waiver only when all are true: +1. Failure is understood and reproducible. +2. Category assignment is stable. +3. A near-term fix cannot be delivered in current change scope. + +## When Not to Add a Waiver +Do not add waivers for: +- unknown failures +- flaky behavior without root cause +- argument synthesis defects that should be fixed in invocation profiles + +## Quality Bar for Waiver Reasons +A good reason includes: +- failure mechanism +- where it fails (component/path) +- what condition is missing +- intended fix direction + +A weak reason includes only: +- "fails in CI" +- "does not work" + +## Review Checklist +1. Is commandId exact and currently discoverable? +2. Is category accurate against latest failure output? +3. Is reason concrete and technical? +4. Is issue/follow-up marker present for remediation? +5. Should this waiver be removed because behavior is now fixed? From cdb2b43be650d56d6cc23d7f803e7aa5e7d98a51 Mon Sep 17 00:00:00 2001 From: Lars Bergmann Date: Fri, 7 Aug 2026 09:36:24 +0200 Subject: [PATCH 20/49] really skip waivers for filtered runs --- .../config/command-classifications.json | 2 +- src/test/integration/run-all-commands.test.ts | 16 ++++++++++++++-- test_docs/integration-artifacts.md | 1 + 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/src/test/integration/config/command-classifications.json b/src/test/integration/config/command-classifications.json index bcc8f24e2..1a0c92329 100644 --- a/src/test/integration/config/command-classifications.json +++ b/src/test/integration/config/command-classifications.json @@ -1,6 +1,6 @@ { "schemaVersion": 1, - "generatedAt": "2026-08-06T15:54:03.172Z", + "generatedAt": "2026-08-07T07:32:56.213Z", "source": { "kind": "run-all-summary" }, diff --git a/src/test/integration/run-all-commands.test.ts b/src/test/integration/run-all-commands.test.ts index 2f2394729..d0bddb66e 100644 --- a/src/test/integration/run-all-commands.test.ts +++ b/src/test/integration/run-all-commands.test.ts @@ -195,6 +195,7 @@ describeRunAllCommands("integration: run all commands", () => { const strictWaiverIntegrityMode = !categoryFilter && runtimeOverrides.commandId === undefined; + const disableWaiversForCategoryFilter = categoryFilter !== undefined; if (strictWaiverIntegrityMode) { if (duplicates.length > 0) { @@ -219,6 +220,12 @@ describeRunAllCommands("integration: run all commands", () => { ); } + if (disableWaiversForCategoryFilter) { + logProgress( + "[waivers] waiver application disabled because MW_TEST_CATEGORY is active", + ); + } + for (const [index, command] of commands.entries()) { const position = `${index + 1}/${commands.length}`; const synthesizedInvocation = command.synthesizedInvocation; @@ -227,7 +234,9 @@ describeRunAllCommands("integration: run all commands", () => { runtimeOverrides, ); const waiver = waiversByCommandId.get(command.commandId); - const bypassWaiver = shouldBypassWaiverForCommand(command, runtimeOverrides); + const bypassWaiver = + disableWaiversForCategoryFilter || + shouldBypassWaiverForCommand(command, runtimeOverrides); const commandStartedAt = Date.now(); await seedProjectContext(projectId); @@ -275,8 +284,11 @@ describeRunAllCommands("integration: run all commands", () => { } if (waiver && bypassWaiver) { + const bypassReason = disableWaiversForCategoryFilter + ? "category filter active" + : "command override"; logProgress( - `[${position}] waiver bypass ${command.commandId} (category=${waiver.category}; reason=${waiver.reason})`, + `[${position}] waiver bypass ${command.commandId} (category=${waiver.category}; reason=${waiver.reason}; bypass=${bypassReason})`, ); } diff --git a/test_docs/integration-artifacts.md b/test_docs/integration-artifacts.md index faaba0df2..696ae0720 100644 --- a/test_docs/integration-artifacts.md +++ b/test_docs/integration-artifacts.md @@ -68,6 +68,7 @@ Note: When MW_TEST_CATEGORY is set: - discovery still enumerates all commands - execution list is filtered to command IDs from classification catalog entries matching category +- waiver application is disabled for the scoped run (commands execute instead of being waived) - strict global waiver integrity checks are skipped for partial scope ## Single-Command Override Contract From 0a5a6bbf00b08fb8008db61c01f4070f0a51b29c Mon Sep 17 00:00:00 2001 From: Lars Bergmann Date: Fri, 7 Aug 2026 13:56:50 +0200 Subject: [PATCH 21/49] kill waivers with better defaults in schema --- .../config/command-classifications.json | 71 +-------------- .../integration/config/command-waivers.json | 91 ------------------- 2 files changed, 3 insertions(+), 159 deletions(-) diff --git a/src/test/integration/config/command-classifications.json b/src/test/integration/config/command-classifications.json index 1a0c92329..277deb5f4 100644 --- a/src/test/integration/config/command-classifications.json +++ b/src/test/integration/config/command-classifications.json @@ -1,41 +1,16 @@ { "schemaVersion": 1, - "generatedAt": "2026-08-07T07:32:56.213Z", + "generatedAt": "2026-08-07T11:53:31.770Z", "source": { "kind": "run-all-summary" }, "statistics": { - "successful": 122, + "successful": 135, "failed": 0, - "waivedSkipped": 66, + "waivedSkipped": 53, "total": 188 }, "entries": [ - { - "commandId": "app create node", - "category": "CONTRACT_SHAPE", - "source": "waiver" - }, - { - "commandId": "app create php", - "category": "CONTRACT_SHAPE", - "source": "waiver" - }, - { - "commandId": "app create php-worker", - "category": "CONTRACT_SHAPE", - "source": "waiver" - }, - { - "commandId": "app create python", - "category": "CONTRACT_SHAPE", - "source": "waiver" - }, - { - "commandId": "app create static", - "category": "CONTRACT_SHAPE", - "source": "waiver" - }, { "commandId": "app database link", "category": "DEPRECATED_ENDPOINT", @@ -71,46 +46,6 @@ "category": "CONTRACT_SHAPE", "source": "waiver" }, - { - "commandId": "app install contao", - "category": "CONTRACT_SHAPE", - "source": "waiver" - }, - { - "commandId": "app install joomla", - "category": "CONTRACT_SHAPE", - "source": "waiver" - }, - { - "commandId": "app install matomo", - "category": "CONTRACT_SHAPE", - "source": "waiver" - }, - { - "commandId": "app install nextcloud", - "category": "CONTRACT_SHAPE", - "source": "waiver" - }, - { - "commandId": "app install shopware5", - "category": "CONTRACT_SHAPE", - "source": "waiver" - }, - { - "commandId": "app install shopware6", - "category": "CONTRACT_SHAPE", - "source": "waiver" - }, - { - "commandId": "app install typo3", - "category": "CONTRACT_SHAPE", - "source": "waiver" - }, - { - "commandId": "app install wordpress", - "category": "CONTRACT_SHAPE", - "source": "waiver" - }, { "commandId": "app list-upgrade-candidates", "category": "COMMAND_BUG", diff --git a/src/test/integration/config/command-waivers.json b/src/test/integration/config/command-waivers.json index 6a6a33d54..4df9c7c53 100644 --- a/src/test/integration/config/command-waivers.json +++ b/src/test/integration/config/command-waivers.json @@ -90,76 +90,6 @@ "reason": "Command currently uses a deprecated API endpoint that is filtered by the mockoon setup and therefore not served in integration runs. The command should be reworked to use a current non-deprecated endpoint.", "issue": "rework-deprecated-endpoint-PR-2055" }, - { - "id": "contract-shape-app-create-node-invalid-version", - "commandId": "app create node", - "category": "CONTRACT_SHAPE", - "reason": "Version fields in the app version contract are typed as generic strings (openapi schema), but client selection logic assumes semver-compatible internalVersion values and calls semver.gt directly. In integration fixtures this can surface as empty or non-semver values and crashes with TypeError: Invalid Version before command-specific logic can proceed.", - "issue": "harden-app-version-selection" - }, - { - "id": "contract-shape-app-create-php-invalid-version", - "commandId": "app create php", - "category": "CONTRACT_SHAPE", - "reason": "Version fields in the app version contract are typed as generic strings (openapi schema), but client selection logic assumes semver-compatible internalVersion values and calls semver.gt directly. In integration fixtures this can surface as empty or non-semver values and crashes with TypeError: Invalid Version before command-specific logic can proceed.", - "issue": "harden-app-version-selection" - }, - { - "id": "contract-shape-app-create-php-worker-invalid-version", - "commandId": "app create php-worker", - "category": "CONTRACT_SHAPE", - "reason": "Version fields in the app version contract are typed as generic strings (openapi schema), but client selection logic assumes semver-compatible internalVersion values and calls semver.gt directly. In integration fixtures this can surface as empty or non-semver values and crashes with TypeError: Invalid Version before command-specific logic can proceed.", - "issue": "harden-app-version-selection" - }, - { - "id": "contract-shape-app-create-python-invalid-version", - "commandId": "app create python", - "category": "CONTRACT_SHAPE", - "reason": "Version fields in the app version contract are typed as generic strings (openapi schema), but client selection logic assumes semver-compatible internalVersion values and calls semver.gt directly. In integration fixtures this can surface as empty or non-semver values and crashes with TypeError: Invalid Version before command-specific logic can proceed.", - "issue": "harden-app-version-selection" - }, - { - "id": "contract-shape-app-create-static-invalid-version", - "commandId": "app create static", - "category": "CONTRACT_SHAPE", - "reason": "Version fields in the app version contract are typed as generic strings (openapi schema), but client selection logic assumes semver-compatible internalVersion values and calls semver.gt directly. In integration fixtures this can surface as empty or non-semver values and crashes with TypeError: Invalid Version before command-specific logic can proceed.", - "issue": "harden-app-version-selection" - }, - { - "id": "contract-shape-app-install-contao-invalid-version", - "commandId": "app install contao", - "category": "CONTRACT_SHAPE", - "reason": "Version fields in the app version contract are typed as generic strings (openapi schema), but client selection logic assumes semver-compatible internalVersion values and calls semver.gt directly. In integration fixtures this can surface as empty or non-semver values and crashes with TypeError: Invalid Version before command-specific logic can proceed.", - "issue": "harden-app-version-selection" - }, - { - "id": "contract-shape-app-install-shopware5-invalid-version", - "commandId": "app install shopware5", - "category": "CONTRACT_SHAPE", - "reason": "Version fields in the app version contract are typed as generic strings (openapi schema), but client selection logic assumes semver-compatible internalVersion values and calls semver.gt directly. In integration fixtures this can surface as empty or non-semver values and crashes with TypeError: Invalid Version before command-specific logic can proceed.", - "issue": "harden-app-version-selection" - }, - { - "id": "contract-shape-app-install-shopware6-invalid-version", - "commandId": "app install shopware6", - "category": "CONTRACT_SHAPE", - "reason": "Version fields in the app version contract are typed as generic strings (openapi schema), but client selection logic assumes semver-compatible internalVersion values and calls semver.gt directly. In integration fixtures this can surface as empty or non-semver values and crashes with TypeError: Invalid Version before command-specific logic can proceed.", - "issue": "harden-app-version-selection" - }, - { - "id": "contract-shape-app-install-typo3-invalid-version", - "commandId": "app install typo3", - "category": "CONTRACT_SHAPE", - "reason": "Version fields in the app version contract are typed as generic strings (openapi schema), but client selection logic assumes semver-compatible internalVersion values and calls semver.gt directly. In integration fixtures this can surface as empty or non-semver values and crashes with TypeError: Invalid Version before command-specific logic can proceed.", - "issue": "harden-app-version-selection" - }, - { - "id": "contract-shape-app-install-wordpress-invalid-version", - "commandId": "app install wordpress", - "category": "CONTRACT_SHAPE", - "reason": "Version fields in the app version contract are typed as generic strings (openapi schema), but client selection logic assumes semver-compatible internalVersion values and calls semver.gt directly. In integration fixtures this can surface as empty or non-semver values and crashes with TypeError: Invalid Version before command-specific logic can proceed.", - "issue": "harden-app-version-selection" - }, { "id": "resource-precondition-container-cp-container-not-found", "commandId": "container cp", @@ -258,20 +188,6 @@ "reason": "Command fails with TypeError [ERR_INVALID_ARG_TYPE] from AppInstallationDetails rendering when absolute installation path is built with path.join(project.directories['Web'], appInstallation.installationPath). This is the same fixture data-shape gap as the shared SSH app-installation path handling cluster: missing directories['Web'] and/or installationPath in test payloads causes deterministic crash. Preferred fix is fixture-side: seed deterministic project/app-installation payloads that always include project.directories['Web'] and appInstallation.installationPath for the known test project.", "issue": "seed-appinstall-web-directory-fixture" }, - { - "id": "contract-shape-app-install-joomla-invalid-version", - "commandId": "app install joomla", - "category": "CONTRACT_SHAPE", - "reason": "Version fields in the app version contract are typed as generic strings (openapi schema), but client selection logic assumes semver-compatible internalVersion values and calls semver.gt directly. In integration fixtures this can surface as empty or non-semver values and crashes with TypeError: Invalid Version before command-specific logic can proceed.", - "issue": "harden-app-version-selection" - }, - { - "id": "contract-shape-app-install-nextcloud-invalid-version", - "commandId": "app install nextcloud", - "category": "CONTRACT_SHAPE", - "reason": "Version fields in the app version contract are typed as generic strings (openapi schema), but client selection logic assumes semver-compatible internalVersion values and calls semver.gt directly. In integration fixtures this can surface as empty or non-semver values and crashes with TypeError: Invalid Version before command-specific logic can proceed.", - "issue": "harden-app-version-selection" - }, { "id": "command-bug-app-dependency-update-invalid-installation-id", "commandId": "app dependency update", @@ -286,13 +202,6 @@ "reason": "The integration fixture set does not provide a resolvable system software entry for the placeholder value used in this command run. The command fails with 'system software ... not found'.", "issue": "seed-known-resource-fixtures" }, - { - "id": "contract-shape-app-install-matomo-invalid-version", - "commandId": "app install matomo", - "category": "CONTRACT_SHAPE", - "reason": "Version fields in the app version contract are typed as generic strings (openapi schema), but client selection logic assumes semver-compatible internalVersion values and calls semver.gt directly. In integration fixtures this can surface as empty or non-semver values and crashes with TypeError: Invalid Version before command-specific logic can proceed.", - "issue": "harden-app-version-selection" - }, { "id": "command-bug-app-list-upgrade-candidates-versions-not-array", "commandId": "app list-upgrade-candidates", From a3cb32b8edb33d7e8caafaa46c36d23bb91b160d Mon Sep 17 00:00:00 2001 From: Lars Bergmann Date: Fri, 7 Aug 2026 14:36:29 +0200 Subject: [PATCH 22/49] reclassify ssh/rsync consumers --- .../config/command-classifications.json | 10 +++++----- src/test/integration/config/command-waivers.json | 16 ++++++++-------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/test/integration/config/command-classifications.json b/src/test/integration/config/command-classifications.json index 277deb5f4..ec3122915 100644 --- a/src/test/integration/config/command-classifications.json +++ b/src/test/integration/config/command-classifications.json @@ -1,6 +1,6 @@ { "schemaVersion": 1, - "generatedAt": "2026-08-07T11:53:31.770Z", + "generatedAt": "2026-08-07T12:30:46.037Z", "source": { "kind": "run-all-summary" }, @@ -33,17 +33,17 @@ }, { "commandId": "app download", - "category": "CONTRACT_SHAPE", + "category": "INTERACTIVE_REQUIRED", "source": "waiver" }, { "commandId": "app exec", - "category": "CONTRACT_SHAPE", + "category": "INTERACTIVE_REQUIRED", "source": "waiver" }, { "commandId": "app get", - "category": "CONTRACT_SHAPE", + "category": "INTERACTIVE_REQUIRED", "source": "waiver" }, { @@ -58,7 +58,7 @@ }, { "commandId": "app ssh", - "category": "CONTRACT_SHAPE", + "category": "INTERACTIVE_REQUIRED", "source": "waiver" }, { diff --git a/src/test/integration/config/command-waivers.json b/src/test/integration/config/command-waivers.json index 4df9c7c53..f55737c2b 100644 --- a/src/test/integration/config/command-waivers.json +++ b/src/test/integration/config/command-waivers.json @@ -163,29 +163,29 @@ { "id": "contract-shape-app-download-missing-web-directory", "commandId": "app download", - "category": "CONTRACT_SHAPE", - "reason": "Command fails with TypeError [ERR_INVALID_ARG_TYPE] in a shared path.join call because integration response shape does not provide all path inputs expected by shared SSH/app-installation rendering logic. Primary shared callsite: src/lib/resources/ssh/appinstall.ts (getSSHConnectionForAppInstallation) builds directory via path.join(projectResponse.data.directories['Web'], appInstallation.installationPath). If directories['Web'] or installationPath is missing/undefined, multiple commands crash with the same signature. Preferred fix is fixture-side: seed deterministic project/app-installation payloads that always include project.directories['Web'] and appInstallation.installationPath for the known test project.", + "category": "INTERACTIVE_REQUIRED", + "reason": "Beyond overlapping path-shape issues, this command enters SSH/rsync execution paths that are interactive/system-dependent and not intended to run in headless integration matrix mode.", "issue": "seed-appinstall-web-directory-fixture" }, { "id": "contract-shape-app-exec-missing-web-directory", "commandId": "app exec", - "category": "CONTRACT_SHAPE", - "reason": "Command fails with TypeError [ERR_INVALID_ARG_TYPE] in a shared path.join call because integration response shape does not provide all path inputs expected by shared SSH/app-installation rendering logic. Primary shared callsite: src/lib/resources/ssh/appinstall.ts (getSSHConnectionForAppInstallation) builds directory via path.join(projectResponse.data.directories['Web'], appInstallation.installationPath). If directories['Web'] or installationPath is missing/undefined, multiple commands crash with the same signature. Preferred fix is fixture-side: seed deterministic project/app-installation payloads that always include project.directories['Web'] and appInstallation.installationPath for the known test project.", + "category": "INTERACTIVE_REQUIRED", + "reason": "Beyond overlapping path-shape issues, this command enters SSH/rsync execution paths that are interactive/system-dependent and not intended to run in headless integration matrix mode.", "issue": "seed-appinstall-web-directory-fixture" }, { "id": "contract-shape-app-ssh-missing-web-directory", "commandId": "app ssh", - "category": "CONTRACT_SHAPE", - "reason": "Command fails with TypeError [ERR_INVALID_ARG_TYPE] in a shared path.join call because integration response shape does not provide all path inputs expected by shared SSH/app-installation rendering logic. Primary shared callsite: src/lib/resources/ssh/appinstall.ts (getSSHConnectionForAppInstallation) builds directory via path.join(projectResponse.data.directories['Web'], appInstallation.installationPath). If directories['Web'] or installationPath is missing/undefined, multiple commands crash with the same signature. Preferred fix is fixture-side: seed deterministic project/app-installation payloads that always include project.directories['Web'] and appInstallation.installationPath for the known test project.", + "category": "INTERACTIVE_REQUIRED", + "reason": "Beyond overlapping path-shape issues, this command enters SSH/rsync execution paths that are interactive/system-dependent and not intended to run in headless integration matrix mode.", "issue": "seed-appinstall-web-directory-fixture" }, { "id": "contract-shape-app-get-missing-web-directory", "commandId": "app get", - "category": "CONTRACT_SHAPE", - "reason": "Command fails with TypeError [ERR_INVALID_ARG_TYPE] from AppInstallationDetails rendering when absolute installation path is built with path.join(project.directories['Web'], appInstallation.installationPath). This is the same fixture data-shape gap as the shared SSH app-installation path handling cluster: missing directories['Web'] and/or installationPath in test payloads causes deterministic crash. Preferred fix is fixture-side: seed deterministic project/app-installation payloads that always include project.directories['Web'] and appInstallation.installationPath for the known test project.", + "category": "INTERACTIVE_REQUIRED", + "reason": "Beyond overlapping path-shape issues, this command enters SSH/rsync execution paths that are interactive/system-dependent and not intended to run in headless integration matrix mode.", "issue": "seed-appinstall-web-directory-fixture" }, { From 07c7e820e6556d3f55ab4e72dbd435fa8e2e7ebc Mon Sep 17 00:00:00 2001 From: Lars Bergmann Date: Fri, 7 Aug 2026 14:36:52 +0200 Subject: [PATCH 23/49] mark brittle passage in app ssh command guts --- src/lib/resources/ssh/appinstall.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/resources/ssh/appinstall.ts b/src/lib/resources/ssh/appinstall.ts index e49493ecb..9a72242de 100644 --- a/src/lib/resources/ssh/appinstall.ts +++ b/src/lib/resources/ssh/appinstall.ts @@ -33,7 +33,7 @@ export async function getSSHConnectionForAppInstallation( const host = `ssh.${projectResponse.data.clusterID}.${projectResponse.data.clusterDomain}`; const user = `${sshUser}@${appInstallationResponse.data.shortId}`; const directory = path.join( - projectResponse.data.directories["Web"], + projectResponse.data.directories["Web"], // XXX: Attention this might be undefined appInstallationResponse.data.installationPath, ); From ffd3854d09f41fe53a0ea80f42cc7904e5aa2af8 Mon Sep 17 00:00:00 2001 From: Lars Bergmann Date: Fri, 7 Aug 2026 16:13:08 +0200 Subject: [PATCH 24/49] mark another brittle passage --- src/lib/ddev/config_builder.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/ddev/config_builder.ts b/src/lib/ddev/config_builder.ts index 95470fe04..4e3db1855 100644 --- a/src/lib/ddev/config_builder.ts +++ b/src/lib/ddev/config_builder.ts @@ -93,7 +93,7 @@ export class DDEVConfigBuilder { private async determineDocumentRoot(inst: AppInstallation): Promise { const appVersion = await this.getAppVersion( inst.appId, - inst.appVersion.desired, + inst.appVersion.desired, // XXX: Attention this might be undefined. API client call list route then, returning with an array! ); if (appVersion.docRootUserEditable && hasCustomDocumentRoot(inst)) { From 7522a8dbdec7f26f931e65240d78c7d90352d079 Mon Sep 17 00:00:00 2001 From: Lars Bergmann Date: Fri, 7 Aug 2026 16:13:53 +0200 Subject: [PATCH 25/49] kill another waiver with example value --- .../integration/config/command-classifications.json | 11 +++-------- src/test/integration/config/command-waivers.json | 7 ------- 2 files changed, 3 insertions(+), 15 deletions(-) diff --git a/src/test/integration/config/command-classifications.json b/src/test/integration/config/command-classifications.json index ec3122915..923677d3c 100644 --- a/src/test/integration/config/command-classifications.json +++ b/src/test/integration/config/command-classifications.json @@ -1,13 +1,13 @@ { "schemaVersion": 1, - "generatedAt": "2026-08-07T12:30:46.037Z", + "generatedAt": "2026-08-07T14:09:20.969Z", "source": { "kind": "run-all-summary" }, "statistics": { - "successful": 135, + "successful": 136, "failed": 0, - "waivedSkipped": 53, + "waivedSkipped": 52, "total": 188 }, "entries": [ @@ -201,11 +201,6 @@ "category": "INTERACTIVE_REQUIRED", "source": "waiver" }, - { - "commandId": "ddev render-config", - "category": "CONTRACT_SHAPE", - "source": "waiver" - }, { "commandId": "domain dnszone get", "category": "RESOURCE_PRECONDITION", diff --git a/src/test/integration/config/command-waivers.json b/src/test/integration/config/command-waivers.json index f55737c2b..b51c5214a 100644 --- a/src/test/integration/config/command-waivers.json +++ b/src/test/integration/config/command-waivers.json @@ -293,13 +293,6 @@ "reason": "Integration currently targets the main MySQL user, which is protected by API rules and cannot be deleted manually in this flow.", "issue": "seed-mysql-non-main-user-fixture" }, - { - "id": "contract-shape-ddev-render-config-missing-document-root-input", - "commandId": "ddev render-config", - "category": "CONTRACT_SHAPE", - "reason": "DDEV config generation currently receives fixture data with missing path fields and fails with 'Cannot read properties of undefined (reading replace)' in config builder path normalization.", - "issue": "seed-ddev-config-shape-fixture" - }, { "id": "resource-precondition-domain-dnszone-get-zone-not-found", "commandId": "domain dnszone get", From a83f6b92522d742d45032bce7dc6029a67098065 Mon Sep 17 00:00:00 2001 From: Lars Bergmann Date: Fri, 7 Aug 2026 17:28:21 +0200 Subject: [PATCH 26/49] crunch and reclassify --- .../config/command-classifications.json | 50 ++--------- .../integration/config/command-waivers.json | 84 ++++--------------- 2 files changed, 19 insertions(+), 115 deletions(-) diff --git a/src/test/integration/config/command-classifications.json b/src/test/integration/config/command-classifications.json index 923677d3c..cde4a339b 100644 --- a/src/test/integration/config/command-classifications.json +++ b/src/test/integration/config/command-classifications.json @@ -1,13 +1,13 @@ { "schemaVersion": 1, - "generatedAt": "2026-08-07T14:09:20.969Z", + "generatedAt": "2026-08-07T15:27:30.357Z", "source": { "kind": "run-all-summary" }, "statistics": { - "successful": 136, + "successful": 144, "failed": 0, - "waivedSkipped": 52, + "waivedSkipped": 44, "total": 188 }, "entries": [ @@ -88,17 +88,12 @@ }, { "commandId": "container cp", - "category": "RESOURCE_PRECONDITION", - "source": "waiver" - }, - { - "commandId": "container delete", - "category": "RESOURCE_PRECONDITION", + "category": "INTERACTIVE_REQUIRED", "source": "waiver" }, { "commandId": "container exec", - "category": "RESOURCE_PRECONDITION", + "category": "INTERACTIVE_REQUIRED", "source": "waiver" }, { @@ -106,46 +101,11 @@ "category": "INTERACTIVE_REQUIRED", "source": "waiver" }, - { - "commandId": "container port-forward", - "category": "RESOURCE_PRECONDITION", - "source": "waiver" - }, - { - "commandId": "container recreate", - "category": "RESOURCE_PRECONDITION", - "source": "waiver" - }, - { - "commandId": "container restart", - "category": "RESOURCE_PRECONDITION", - "source": "waiver" - }, { "commandId": "container run", "category": "RESOURCE_PRECONDITION", "source": "waiver" }, - { - "commandId": "container ssh", - "category": "RESOURCE_PRECONDITION", - "source": "waiver" - }, - { - "commandId": "container start", - "category": "RESOURCE_PRECONDITION", - "source": "waiver" - }, - { - "commandId": "container stop", - "category": "RESOURCE_PRECONDITION", - "source": "waiver" - }, - { - "commandId": "container update", - "category": "RESOURCE_PRECONDITION", - "source": "waiver" - }, { "commandId": "conversation create", "category": "COMMAND_BUG", diff --git a/src/test/integration/config/command-waivers.json b/src/test/integration/config/command-waivers.json index b51c5214a..b6d3aa403 100644 --- a/src/test/integration/config/command-waivers.json +++ b/src/test/integration/config/command-waivers.json @@ -13,6 +13,20 @@ "reason": "Log follow behavior depends on interactive terminal controls in current implementation.", "issue": "defer-interactive-support" }, + { + "id": "interactive-container-cp-filesystem-io", + "commandId": "container cp", + "category": "INTERACTIVE_REQUIRED", + "reason": "It requires more I/O with filesystem under the hood.", + "issue": "defer-interactive-support" + }, + { + "id": "interactive-container-exec-unmockable-remote-call", + "commandId": "container exec", + "category": "INTERACTIVE_REQUIRED", + "reason": "Execution path depends on an interactive remote call chain that cannot be reliably mocked in the current integration harness.", + "issue": "defer-interactive-support" + }, { "id": "interactive-cronjob-execution-logs", "commandId": "cronjob execution logs", @@ -90,76 +104,6 @@ "reason": "Command currently uses a deprecated API endpoint that is filtered by the mockoon setup and therefore not served in integration runs. The command should be reworked to use a current non-deprecated endpoint.", "issue": "rework-deprecated-endpoint-PR-2055" }, - { - "id": "resource-precondition-container-cp-container-not-found", - "commandId": "container cp", - "category": "RESOURCE_PRECONDITION", - "reason": "Shared container resolution precondition is not satisfied in integration fixtures. The resolver in src/lib/resources/container/flags.ts (withContainerAndStackId) lists project services and throws when no matching container shortId/id/serviceName exists. Current run fails with 'no container mycontainer found in project p-f0ob4r'. Preferred fix is to seed a deterministic known container fixture for project p-f0ob4r and align invocation profiles/examples to that known container identifier.", - "issue": "seed-known-container-fixture" - }, - { - "id": "resource-precondition-container-delete-container-not-found", - "commandId": "container delete", - "category": "RESOURCE_PRECONDITION", - "reason": "Shared container resolution precondition is not satisfied in integration fixtures. The resolver in src/lib/resources/container/flags.ts (withContainerAndStackId) lists project services and throws when no matching container shortId/id/serviceName exists. Current run shows deletion flow failing because the requested container identifier does not exist in project p-f0ob4r. Preferred fix is to seed a deterministic known container fixture for project p-f0ob4r and align invocation profiles/examples to that known container identifier.", - "issue": "seed-known-container-fixture" - }, - { - "id": "resource-precondition-container-exec-container-not-found", - "commandId": "container exec", - "category": "RESOURCE_PRECONDITION", - "reason": "Shared container resolution precondition is not satisfied in integration fixtures. The resolver in src/lib/resources/container/flags.ts (withContainerAndStackId) lists project services and throws when no matching container shortId/id/serviceName exists. Current run fails with 'no container 00000000-0000-4000-8000-000000000000 found in project p-f0ob4r'. Preferred fix is to seed a deterministic known container fixture for project p-f0ob4r and align invocation profiles/examples to that known container identifier.", - "issue": "seed-known-container-fixture" - }, - { - "id": "resource-precondition-container-port-forward-container-not-found", - "commandId": "container port-forward", - "category": "RESOURCE_PRECONDITION", - "reason": "Shared container resolution precondition is not satisfied in integration fixtures. The resolver in src/lib/resources/container/flags.ts (withContainerAndStackId) lists project services and throws when no matching container shortId/id/serviceName exists. Current run fails with 'no container 00000000-0000-4000-8000-000000000000 found in project p-f0ob4r'. Preferred fix is to seed a deterministic known container fixture for project p-f0ob4r and align invocation profiles/examples to that known container identifier.", - "issue": "seed-known-container-fixture" - }, - { - "id": "resource-precondition-container-recreate-container-not-found", - "commandId": "container recreate", - "category": "RESOURCE_PRECONDITION", - "reason": "Shared container resolution precondition is not satisfied in integration fixtures. The resolver in src/lib/resources/container/flags.ts (withContainerAndStackId) lists project services and throws when no matching container shortId/id/serviceName exists. Current run fails with 'no container 00000000-0000-4000-8000-000000000000 found in project p-f0ob4r'. Preferred fix is to seed a deterministic known container fixture for project p-f0ob4r and align invocation profiles/examples to that known container identifier.", - "issue": "seed-known-container-fixture" - }, - { - "id": "resource-precondition-container-restart-container-not-found", - "commandId": "container restart", - "category": "RESOURCE_PRECONDITION", - "reason": "Shared container resolution precondition is not satisfied in integration fixtures. The resolver in src/lib/resources/container/flags.ts (withContainerAndStackId) lists project services and throws when no matching container shortId/id/serviceName exists. Current run fails with 'no container 00000000-0000-4000-8000-000000000000 found in project p-f0ob4r'. Preferred fix is to seed a deterministic known container fixture for project p-f0ob4r and align invocation profiles/examples to that known container identifier.", - "issue": "seed-known-container-fixture" - }, - { - "id": "resource-precondition-container-ssh-container-not-found", - "commandId": "container ssh", - "category": "RESOURCE_PRECONDITION", - "reason": "Shared container resolution precondition is not satisfied in integration fixtures. The resolver in src/lib/resources/container/flags.ts (withContainerAndStackId) lists project services and throws when no matching container shortId/id/serviceName exists. Current run fails with 'no container 00000000-0000-4000-8000-000000000000 found in project p-f0ob4r'. Preferred fix is to seed a deterministic known container fixture for project p-f0ob4r and align invocation profiles/examples to that known container identifier.", - "issue": "seed-known-container-fixture" - }, - { - "id": "resource-precondition-container-start-container-not-found", - "commandId": "container start", - "category": "RESOURCE_PRECONDITION", - "reason": "Shared container resolution precondition is not satisfied in integration fixtures. The resolver in src/lib/resources/container/flags.ts (withContainerAndStackId) lists project services and throws when no matching container shortId/id/serviceName exists. Current run fails with 'no container 00000000-0000-4000-8000-000000000000 found in project p-f0ob4r'. Preferred fix is to seed a deterministic known container fixture for project p-f0ob4r and align invocation profiles/examples to that known container identifier.", - "issue": "seed-known-container-fixture" - }, - { - "id": "resource-precondition-container-stop-container-not-found", - "commandId": "container stop", - "category": "RESOURCE_PRECONDITION", - "reason": "Shared container resolution precondition is not satisfied in integration fixtures. The resolver in src/lib/resources/container/flags.ts (withContainerAndStackId) lists project services and throws when no matching container shortId/id/serviceName exists. Current run fails with 'no container 00000000-0000-4000-8000-000000000000 found in project p-f0ob4r'. Preferred fix is to seed a deterministic known container fixture for project p-f0ob4r and align invocation profiles/examples to that known container identifier.", - "issue": "seed-known-container-fixture" - }, - { - "id": "resource-precondition-container-update-container-not-found", - "commandId": "container update", - "category": "RESOURCE_PRECONDITION", - "reason": "Shared container resolution precondition is not satisfied in integration fixtures. The resolver in src/lib/resources/container/flags.ts (withContainerAndStackId) lists project services and throws when no matching container shortId/id/serviceName exists. Current run fails with 'no container 00000000-0000-4000-8000-000000000000 found in project p-f0ob4r'. Preferred fix is to seed a deterministic known container fixture for project p-f0ob4r and align invocation profiles/examples to that known container identifier.", - "issue": "seed-known-container-fixture" - }, { "id": "contract-shape-app-download-missing-web-directory", "commandId": "app download", From f3efb164cdcd567291cea2841ff4b9f95f1d6300 Mon Sep 17 00:00:00 2001 From: Lars Bergmann Date: Mon, 10 Aug 2026 11:38:42 +0200 Subject: [PATCH 27/49] repair, remove more waivers --- .../config/command-classifications.json | 16 +++------------- src/test/integration/config/command-waivers.json | 14 -------------- .../integration/config/invocation-profiles.json | 10 ++++++++++ 3 files changed, 13 insertions(+), 27 deletions(-) diff --git a/src/test/integration/config/command-classifications.json b/src/test/integration/config/command-classifications.json index cde4a339b..1ec23e08c 100644 --- a/src/test/integration/config/command-classifications.json +++ b/src/test/integration/config/command-classifications.json @@ -1,13 +1,13 @@ { "schemaVersion": 1, - "generatedAt": "2026-08-07T15:27:30.357Z", + "generatedAt": "2026-08-10T09:31:50.254Z", "source": { "kind": "run-all-summary" }, "statistics": { - "successful": 144, + "successful": 146, "failed": 0, - "waivedSkipped": 44, + "waivedSkipped": 42, "total": 188 }, "entries": [ @@ -21,11 +21,6 @@ "category": "DEPRECATED_ENDPOINT", "source": "waiver" }, - { - "commandId": "app dependency update", - "category": "COMMAND_BUG", - "source": "waiver" - }, { "commandId": "app dependency versions", "category": "RESOURCE_PRECONDITION", @@ -206,11 +201,6 @@ "category": "RESOURCE_PRECONDITION", "source": "waiver" }, - { - "commandId": "stack delete", - "category": "COMMAND_BUG", - "source": "waiver" - }, { "commandId": "stack deploy", "category": "INTERACTIVE_REQUIRED", diff --git a/src/test/integration/config/command-waivers.json b/src/test/integration/config/command-waivers.json index b6d3aa403..244ec81a9 100644 --- a/src/test/integration/config/command-waivers.json +++ b/src/test/integration/config/command-waivers.json @@ -132,13 +132,6 @@ "reason": "Beyond overlapping path-shape issues, this command enters SSH/rsync execution paths that are interactive/system-dependent and not intended to run in headless integration matrix mode.", "issue": "seed-appinstall-web-directory-fixture" }, - { - "id": "command-bug-app-dependency-update-invalid-installation-id", - "commandId": "app dependency update", - "category": "COMMAND_BUG", - "reason": "Integration invocation currently passes a placeholder that does not resolve to a valid app installation identifier for this command path. The command exits with an app-installation ID validation error in this test setup.", - "issue": "fix-integration-invocation-profiles" - }, { "id": "resource-precondition-app-dependency-versions-systemsoftware-not-found", "commandId": "app dependency versions", @@ -293,13 +286,6 @@ "reason": "Creation step succeeds, but follow-up readback in integration fixtures returns 404, indicating inconsistent fixture state for immediate lookup.", "issue": "align-user-create-fixture-readback" }, - { - "id": "command-bug-stack-delete-not-implemented", - "commandId": "stack delete", - "category": "COMMAND_BUG", - "reason": "Command flow reaches deletion step and fails with 'not implemented' in current implementation.", - "issue": "implement-stack-delete" - }, { "id": "resource-precondition-volume-delete-volume-not-found", "commandId": "volume delete", diff --git a/src/test/integration/config/invocation-profiles.json b/src/test/integration/config/invocation-profiles.json index 3f3f330bf..a1fa8f5c3 100644 --- a/src/test/integration/config/invocation-profiles.json +++ b/src/test/integration/config/invocation-profiles.json @@ -214,5 +214,15 @@ "match": { "exact": "conversation reply" }, "interactivePolicy": "classify", "disableExampleSource": true + }, + { + "id": "app-dependency-update", + "match": { "exact": "app dependency update" }, + "requiredArgDefaults": { + "installation-id": "f7b47c12-7d11-4f3a-b9bc-1b3c706e1d55" + }, + "requiredFlagDefaults": { + "set": "node=~18" + } } ] From 528337ea0b158caf6ea97e9c25b7ba929d148a84 Mon Sep 17 00:00:00 2001 From: Lars Bergmann Date: Mon, 10 Aug 2026 13:31:37 +0200 Subject: [PATCH 28/49] burn down more waivers, understand things better --- .../config/command-classifications.json | 21 ++--------- .../integration/config/command-waivers.json | 35 ++++--------------- 2 files changed, 10 insertions(+), 46 deletions(-) diff --git a/src/test/integration/config/command-classifications.json b/src/test/integration/config/command-classifications.json index 1ec23e08c..7231e4559 100644 --- a/src/test/integration/config/command-classifications.json +++ b/src/test/integration/config/command-classifications.json @@ -1,13 +1,13 @@ { "schemaVersion": 1, - "generatedAt": "2026-08-10T09:31:50.254Z", + "generatedAt": "2026-08-10T11:29:19.455Z", "source": { "kind": "run-all-summary" }, "statistics": { - "successful": 146, + "successful": 149, "failed": 0, - "waivedSkipped": 42, + "waivedSkipped": 39, "total": 188 }, "entries": [ @@ -131,11 +131,6 @@ "category": "RESOURCE_PRECONDITION", "source": "waiver" }, - { - "commandId": "database mysql port-forward", - "category": "RESOURCE_PRECONDITION", - "source": "waiver" - }, { "commandId": "database mysql shell", "category": "INTERACTIVE_REQUIRED", @@ -191,11 +186,6 @@ "category": "RESOURCE_PRECONDITION", "source": "waiver" }, - { - "commandId": "sftp-user create", - "category": "RESOURCE_PRECONDITION", - "source": "waiver" - }, { "commandId": "ssh-user create", "category": "RESOURCE_PRECONDITION", @@ -211,11 +201,6 @@ "category": "INTERACTIVE_REQUIRED", "source": "waiver" }, - { - "commandId": "user ssh-key import", - "category": "RESOURCE_PRECONDITION", - "source": "waiver" - }, { "commandId": "volume delete", "category": "RESOURCE_PRECONDITION", diff --git a/src/test/integration/config/command-waivers.json b/src/test/integration/config/command-waivers.json index 244ec81a9..e44ee92ea 100644 --- a/src/test/integration/config/command-waivers.json +++ b/src/test/integration/config/command-waivers.json @@ -83,13 +83,6 @@ "reason": "SSH key creation relies on interactive prompts for key source and confirmation.", "issue": "defer-interactive-support" }, - { - "id": "resource-precondition-user-ssh-key-import-default-key-file-missing", - "commandId": "user ssh-key import", - "category": "RESOURCE_PRECONDITION", - "reason": "Remote integration runners do not guarantee a default local SSH public key at ~/.ssh/id_rsa.pub. The current invocation path attempts to read that default and fails with ENOENT in headless CI environments.", - "issue": "fix-integration-invocation-profiles" - }, { "id": "deprecated-endpoint-app-database-link", "commandId": "app database link", @@ -132,6 +125,13 @@ "reason": "Beyond overlapping path-shape issues, this command enters SSH/rsync execution paths that are interactive/system-dependent and not intended to run in headless integration matrix mode.", "issue": "seed-appinstall-web-directory-fixture" }, + { + "id": "resource-precondition-database-mysql-user-delete-main-user-protected", + "commandId": "database mysql user delete", + "category": "RESOURCE_PRECONDITION", + "reason": "Integration currently targets the main MySQL user, which is protected by API rules and cannot be deleted manually in this flow.", + "issue": "seed-mysql-non-main-user-fixture" + }, { "id": "resource-precondition-app-dependency-versions-systemsoftware-not-found", "commandId": "app dependency versions", @@ -216,20 +216,6 @@ "reason": "The phpMyAdmin flow requires a resolvable main user in fixtures. In this run the command fails with 'no main user found'.", "issue": "seed-mysql-main-user-fixture" }, - { - "id": "resource-precondition-database-mysql-port-forward-main-user-missing", - "commandId": "database mysql port-forward", - "category": "RESOURCE_PRECONDITION", - "reason": "The MySQL port-forward flow requires a resolvable main user in fixtures. In this run the command fails with 'No main user found'.", - "issue": "seed-mysql-main-user-fixture" - }, - { - "id": "resource-precondition-database-mysql-user-delete-main-user-protected", - "commandId": "database mysql user delete", - "category": "RESOURCE_PRECONDITION", - "reason": "Integration currently targets the main MySQL user, which is protected by API rules and cannot be deleted manually in this flow.", - "issue": "seed-mysql-non-main-user-fixture" - }, { "id": "resource-precondition-domain-dnszone-get-zone-not-found", "commandId": "domain dnszone get", @@ -272,13 +258,6 @@ "reason": "The organization targeted by integration placeholders does not exist in the fixture state at deletion time, resulting in a 404 failure.", "issue": "seed-known-resource-fixtures" }, - { - "id": "resource-precondition-sftp-user-create-readback-404", - "commandId": "sftp-user create", - "category": "RESOURCE_PRECONDITION", - "reason": "Creation step succeeds, but follow-up readback in integration fixtures returns 404, indicating inconsistent fixture state for immediate lookup.", - "issue": "align-user-create-fixture-readback" - }, { "id": "resource-precondition-ssh-user-create-readback-404", "commandId": "ssh-user create", From 6eaa7ee928c3a0ccceb03fa3671dc2bdc45e478c Mon Sep 17 00:00:00 2001 From: Lars Bergmann Date: Mon, 10 Aug 2026 14:46:01 +0200 Subject: [PATCH 29/49] burn another one, reclassify experimental --- .../integration/config/command-classifications.json | 13 ++++--------- src/test/integration/config/command-waivers.json | 13 +++---------- 2 files changed, 7 insertions(+), 19 deletions(-) diff --git a/src/test/integration/config/command-classifications.json b/src/test/integration/config/command-classifications.json index 7231e4559..fd1a9f99e 100644 --- a/src/test/integration/config/command-classifications.json +++ b/src/test/integration/config/command-classifications.json @@ -1,13 +1,13 @@ { "schemaVersion": 1, - "generatedAt": "2026-08-10T11:29:19.455Z", + "generatedAt": "2026-08-10T12:29:52.229Z", "source": { "kind": "run-all-summary" }, "statistics": { - "successful": 149, + "successful": 150, "failed": 0, - "waivedSkipped": 39, + "waivedSkipped": 38, "total": 188 }, "entries": [ @@ -126,11 +126,6 @@ "category": "INTERACTIVE_REQUIRED", "source": "waiver" }, - { - "commandId": "database mysql phpmyadmin", - "category": "RESOURCE_PRECONDITION", - "source": "waiver" - }, { "commandId": "database mysql shell", "category": "INTERACTIVE_REQUIRED", @@ -168,7 +163,7 @@ }, { "commandId": "experimental deploy", - "category": "RESOURCE_PRECONDITION", + "category": "INTERACTIVE_REQUIRED", "source": "waiver" }, { diff --git a/src/test/integration/config/command-waivers.json b/src/test/integration/config/command-waivers.json index e44ee92ea..b377380f4 100644 --- a/src/test/integration/config/command-waivers.json +++ b/src/test/integration/config/command-waivers.json @@ -209,13 +209,6 @@ "reason": "The MySQL dump flow requires a resolvable main user in fixtures. In this run the command fails with 'No main user found'.", "issue": "seed-mysql-main-user-fixture" }, - { - "id": "resource-precondition-database-mysql-phpmyadmin-main-user-missing", - "commandId": "database mysql phpmyadmin", - "category": "RESOURCE_PRECONDITION", - "reason": "The phpMyAdmin flow requires a resolvable main user in fixtures. In this run the command fails with 'no main user found'.", - "issue": "seed-mysql-main-user-fixture" - }, { "id": "resource-precondition-domain-dnszone-get-zone-not-found", "commandId": "domain dnszone get", @@ -240,9 +233,9 @@ { "id": "resource-precondition-experimental-deploy-registry-service-missing", "commandId": "experimental deploy", - "category": "RESOURCE_PRECONDITION", - "reason": "The deploy orchestration expects a registry service fixture that is not returned in this integration environment and fails with 'Service not found in response'.", - "issue": "seed-stack-service-fixture" + "category": "INTERACTIVE_REQUIRED", + "reason": "Calls further tooling down the line, e.g. railpack and/or docker", + "issue": "deeper-mocking-needed" }, { "id": "resource-precondition-mail-address-update-mail-address-not-found", From 004815af7a1c5858f03105f7d8c280a366e1309a Mon Sep 17 00:00:00 2001 From: Lars Bergmann Date: Mon, 10 Aug 2026 14:46:51 +0200 Subject: [PATCH 30/49] operational notes --- test_docs/waiver-governance.md | 66 ++++++++++++++++++++++++++++------ 1 file changed, 56 insertions(+), 10 deletions(-) diff --git a/test_docs/waiver-governance.md b/test_docs/waiver-governance.md index f8e195f49..79b9a5693 100644 --- a/test_docs/waiver-governance.md +++ b/test_docs/waiver-governance.md @@ -3,6 +3,26 @@ ## Purpose Waivers are a governance tool, not a suppression shortcut. They document known failing commands with category, reason, and follow-up intent while keeping failures auditable. +## Core Principle: Unwaived Commands Must Not Fail + +**All discovered commands must either pass or have an explicit waiver.** This is a hard invariant enforced by: +- **CI gate**: Integration test fails if any command is unwaived and fails +- **Peer review**: Waiver reasons must be understood and justified by reviewers + +This principle prevents silent rot and forces intentional decision-making: if a command fails and you can't fix it in the current scope, you *must* write a waiver in the same PR with a concrete reason. The friction is intentional. + +## Lazy Enforcement Philosophy + +Waivers work best when enforcement is simple and visible: + +- **No speculative waivers** — Waivers can only be added when a command fails. Pre-emptive waivers hide problems that should be surfaced now. + +- **Reasons are contemporary** — The waiver reason appears in the PR diff next to the code change. Reviewers see intent immediately. This scales better than historical archaeology through git blame. + +- **Emergent categorization** — Don't invent category hierarchy upfront. Patterns emerge naturally as waivers accumulate. After 20-30 waivers, category trends become visible and inform the next governance decision. + +- **Human review is the gate** — Reviewers must understand why each waiver exists. A weak reason ("known issue", "API doesn't work") gets pushed back. A concrete reason ("API response schema missing `examples` field, blocking invocation synthesis; filed as API-1234") survives review. This quality pressure is self-reinforcing. + ## Source of Truth - Waivers file: src/test/integration/config/command-waivers.json - Waiver loader validation: src/test/integration/config/loader.ts @@ -47,6 +67,23 @@ Reason: Important: - loader-level duplicate checks still apply in all modes. +## Operational Enforcement: CI Gate + Peer Review +This governance model requires both: + +**CI Gate (Mechanical):** +- Integration test suite runs all discovered commands +- Test fails if any command is unwaived and fails +- Test fails if waiver references a non-existent command +- Test fails if duplicate waivers exist + +**Peer Review (Human):** +- Reviewers must understand waiver semantics +- Waiver PRs are not auto-approved based on metrics +- Weak reasons trigger discussion, not merges +- Category assignment is debated if unclear + +Together, these prevent silent drift. You can't sneak in a waiver without explaining it; you can't add unmaintainable waivers because reviewers catch them. + ## Waiver Hunting Workflow 1. Run one command with MW_TEST_COMMAND_ID. 2. Reproduce and inspect failure details. @@ -71,18 +108,27 @@ Do not add waivers for: ## Quality Bar for Waiver Reasons A good reason includes: -- failure mechanism -- where it fails (component/path) -- what condition is missing -- intended fix direction +- failure mechanism (what broke and why) +- where it fails (component/path/layer) +- what condition is missing (fixture, API contract, design decision) +- intended fix direction (or why it's a permanent trade-off) A weak reason includes only: - "fails in CI" - "does not work" +- "known issue" +- "API thing" + +Weak reasons get rejected in review. The author must explain themselves to the reviewer. This quality pressure is a feature, not friction. + +## Review Checklist: Reviewer Responsibility +Reviewers must actively evaluate *every* waiver addition. Your job is to: + +1. **Understand the failure** — Did the author explain what broke? Can you reproduce it from the reason alone? +2. **Validate category** — Is the waiver category accurate? Does it match the actual failure root cause? +3. **Evaluate reason quality** — Is the reason concrete and specific? Would a future reader (or you in 6 months) understand this waiver? +4. **Require issue tracking** — For anything that's not a permanent design decision, is there an issue link? Does it have a target fix date? +5. **Question necessity** — Could this waiver have been avoided by fixing the command, fixture, or invocation profile in the same PR? +6. **Check for rot** — Are there existing waivers that should be removed because behavior is now fixed? -## Review Checklist -1. Is commandId exact and currently discoverable? -2. Is category accurate against latest failure output? -3. Is reason concrete and technical? -4. Is issue/follow-up marker present for remediation? -5. Should this waiver be removed because behavior is now fixed? +Reject vague waivers. The 5 minutes you spend pushing back on reason quality prevents 10 hours of future confusion when someone tries to understand why the waiver exists. From e22b3e21ee6f6a81949f44dbbf0aed9f5972e3cf Mon Sep 17 00:00:00 2001 From: Lars Bergmann Date: Mon, 10 Aug 2026 15:05:48 +0200 Subject: [PATCH 31/49] reclassify --- .../config/command-classifications.json | 6 ++-- .../integration/config/command-waivers.json | 28 +++++++++---------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/src/test/integration/config/command-classifications.json b/src/test/integration/config/command-classifications.json index fd1a9f99e..e1cb81df8 100644 --- a/src/test/integration/config/command-classifications.json +++ b/src/test/integration/config/command-classifications.json @@ -1,6 +1,6 @@ { "schemaVersion": 1, - "generatedAt": "2026-08-10T12:29:52.229Z", + "generatedAt": "2026-08-10T13:03:08.089Z", "source": { "kind": "run-all-summary" }, @@ -103,12 +103,12 @@ }, { "commandId": "conversation create", - "category": "COMMAND_BUG", + "category": "INTERACTIVE_REQUIRED", "source": "waiver" }, { "commandId": "conversation reply", - "category": "COMMAND_BUG", + "category": "INTERACTIVE_REQUIRED", "source": "waiver" }, { diff --git a/src/test/integration/config/command-waivers.json b/src/test/integration/config/command-waivers.json index b377380f4..e5361543d 100644 --- a/src/test/integration/config/command-waivers.json +++ b/src/test/integration/config/command-waivers.json @@ -83,6 +83,20 @@ "reason": "SSH key creation relies on interactive prompts for key source and confirmation.", "issue": "defer-interactive-support" }, + { + "id": "command-bug-conversation-create-tempfile-unlink-enoent", + "commandId": "conversation create", + "category": "INTERACTIVE_REQUIRED", + "reason": "Conversation files are handled in filesystem temp paths. Additional work needed to mock.", + "issue": "conversation-file-handling" + }, + { + "id": "command-bug-conversation-reply-tempfile-unlink-enoent", + "commandId": "conversation reply", + "category": "INTERACTIVE_REQUIRED", + "reason": "Conversation files are handled in filesystem temp paths. Additional work needed to mock.", + "issue": "conversation-file-handling" + }, { "id": "deprecated-endpoint-app-database-link", "commandId": "app database link", @@ -188,20 +202,6 @@ "reason": "The created stack in integration fixtures does not expose the expected service mapping for this flow. The command fails with 'Service ID not found in the created stack'.", "issue": "seed-stack-service-fixture" }, - { - "id": "command-bug-conversation-create-tempfile-unlink-enoent", - "commandId": "conversation create", - "category": "COMMAND_BUG", - "reason": "The command hits a temporary-file cleanup race in this run path and fails with ENOENT on unlink of a generated markdown file.", - "issue": "stabilize-tempfile-lifecycle" - }, - { - "id": "command-bug-conversation-reply-tempfile-unlink-enoent", - "commandId": "conversation reply", - "category": "COMMAND_BUG", - "reason": "The command hits a temporary-file cleanup race in this run path and fails with ENOENT on unlink of a generated markdown file.", - "issue": "stabilize-tempfile-lifecycle" - }, { "id": "resource-precondition-database-mysql-dump-main-user-missing", "commandId": "database mysql dump", From f55d5ae8cf25b8702e3b864062bfdde32493e717 Mon Sep 17 00:00:00 2001 From: Lars Bergmann Date: Mon, 10 Aug 2026 15:31:27 +0200 Subject: [PATCH 32/49] provide example, kill waiver --- .../integration/config/command-classifications.json | 11 +++-------- src/test/integration/config/command-waivers.json | 7 ------- 2 files changed, 3 insertions(+), 15 deletions(-) diff --git a/src/test/integration/config/command-classifications.json b/src/test/integration/config/command-classifications.json index e1cb81df8..5e483b56c 100644 --- a/src/test/integration/config/command-classifications.json +++ b/src/test/integration/config/command-classifications.json @@ -1,13 +1,13 @@ { "schemaVersion": 1, - "generatedAt": "2026-08-10T13:03:08.089Z", + "generatedAt": "2026-08-10T13:27:03.933Z", "source": { "kind": "run-all-summary" }, "statistics": { - "successful": 150, + "successful": 151, "failed": 0, - "waivedSkipped": 38, + "waivedSkipped": 37, "total": 188 }, "entries": [ @@ -76,11 +76,6 @@ "category": "COMMAND_BUG", "source": "waiver" }, - { - "commandId": "backup download", - "category": "COMMAND_BUG", - "source": "waiver" - }, { "commandId": "container cp", "category": "INTERACTIVE_REQUIRED", diff --git a/src/test/integration/config/command-waivers.json b/src/test/integration/config/command-waivers.json index e5361543d..738367c07 100644 --- a/src/test/integration/config/command-waivers.json +++ b/src/test/integration/config/command-waivers.json @@ -188,13 +188,6 @@ "reason": "The command path fails with an access denied error in the current integration authorization/fixture setup.", "issue": "seed-permissions-fixtures" }, - { - "id": "command-bug-backup-download-not-ready", - "commandId": "backup download", - "category": "COMMAND_BUG", - "reason": "This command currently terminates with 'backup download is not ready' in the integration scenario, indicating an unfinished command path for this fixture state.", - "issue": "implement-backup-download-path" - }, { "id": "resource-precondition-container-run-service-id-not-found", "commandId": "container run", From a2a60845bd29e67c085080bc64996dceec9b87be Mon Sep 17 00:00:00 2001 From: Lars Bergmann Date: Wed, 12 Aug 2026 09:57:46 +0200 Subject: [PATCH 33/49] kill more waivers after rebase --- .../config/command-classifications.json | 16 +++------------- src/test/integration/config/command-waivers.json | 14 -------------- 2 files changed, 3 insertions(+), 27 deletions(-) diff --git a/src/test/integration/config/command-classifications.json b/src/test/integration/config/command-classifications.json index 5e483b56c..e42e782d6 100644 --- a/src/test/integration/config/command-classifications.json +++ b/src/test/integration/config/command-classifications.json @@ -1,26 +1,16 @@ { "schemaVersion": 1, - "generatedAt": "2026-08-10T13:27:03.933Z", + "generatedAt": "2026-08-12T07:53:02.230Z", "source": { "kind": "run-all-summary" }, "statistics": { - "successful": 151, + "successful": 153, "failed": 0, - "waivedSkipped": 37, + "waivedSkipped": 35, "total": 188 }, "entries": [ - { - "commandId": "app database link", - "category": "DEPRECATED_ENDPOINT", - "source": "waiver" - }, - { - "commandId": "app database replace", - "category": "DEPRECATED_ENDPOINT", - "source": "waiver" - }, { "commandId": "app dependency versions", "category": "RESOURCE_PRECONDITION", diff --git a/src/test/integration/config/command-waivers.json b/src/test/integration/config/command-waivers.json index 738367c07..fbf5d1bc9 100644 --- a/src/test/integration/config/command-waivers.json +++ b/src/test/integration/config/command-waivers.json @@ -97,20 +97,6 @@ "reason": "Conversation files are handled in filesystem temp paths. Additional work needed to mock.", "issue": "conversation-file-handling" }, - { - "id": "deprecated-endpoint-app-database-link", - "commandId": "app database link", - "category": "DEPRECATED_ENDPOINT", - "reason": "Command currently uses a deprecated API endpoint that is filtered by the mockoon setup and therefore not served in integration runs. The command should be reworked to use a current non-deprecated endpoint.", - "issue": "rework-deprecated-endpoint-PR-2055" - }, - { - "id": "deprecated-endpoint-app-database-replace", - "commandId": "app database replace", - "category": "DEPRECATED_ENDPOINT", - "reason": "Command currently uses a deprecated API endpoint that is filtered by the mockoon setup and therefore not served in integration runs. The command should be reworked to use a current non-deprecated endpoint.", - "issue": "rework-deprecated-endpoint-PR-2055" - }, { "id": "contract-shape-app-download-missing-web-directory", "commandId": "app download", From 67f3c900959662fd4a428486b8c5c12959ec4662 Mon Sep 17 00:00:00 2001 From: Lars Bergmann Date: Wed, 12 Aug 2026 11:21:49 +0200 Subject: [PATCH 34/49] adjust schema, remove waiver --- .../integration/config/command-classifications.json | 11 +++-------- src/test/integration/config/command-waivers.json | 7 ------- 2 files changed, 3 insertions(+), 15 deletions(-) diff --git a/src/test/integration/config/command-classifications.json b/src/test/integration/config/command-classifications.json index e42e782d6..704009be3 100644 --- a/src/test/integration/config/command-classifications.json +++ b/src/test/integration/config/command-classifications.json @@ -1,13 +1,13 @@ { "schemaVersion": 1, - "generatedAt": "2026-08-12T07:53:02.230Z", + "generatedAt": "2026-08-12T09:19:04.408Z", "source": { "kind": "run-all-summary" }, "statistics": { - "successful": 153, + "successful": 154, "failed": 0, - "waivedSkipped": 35, + "waivedSkipped": 34, "total": 188 }, "entries": [ @@ -31,11 +31,6 @@ "category": "INTERACTIVE_REQUIRED", "source": "waiver" }, - { - "commandId": "app list-upgrade-candidates", - "category": "COMMAND_BUG", - "source": "waiver" - }, { "commandId": "app open", "category": "COMMAND_BUG", diff --git a/src/test/integration/config/command-waivers.json b/src/test/integration/config/command-waivers.json index fbf5d1bc9..36bfd4567 100644 --- a/src/test/integration/config/command-waivers.json +++ b/src/test/integration/config/command-waivers.json @@ -139,13 +139,6 @@ "reason": "The integration fixture set does not provide a resolvable system software entry for the placeholder value used in this command run. The command fails with 'system software ... not found'.", "issue": "seed-known-resource-fixtures" }, - { - "id": "command-bug-app-list-upgrade-candidates-versions-not-array", - "commandId": "app list-upgrade-candidates", - "category": "COMMAND_BUG", - "reason": "The command expects a sortable versions array, but the current integration response shape provides a non-array value and execution fails with 'versions.sort is not a function'.", - "issue": "harden-response-shape-handling" - }, { "id": "command-bug-app-open-missing-virtualhost-link", "commandId": "app open", From d72dd02f25388d518abd0a2294308c27452cdac4 Mon Sep 17 00:00:00 2001 From: Lars Bergmann Date: Wed, 12 Aug 2026 12:44:20 +0200 Subject: [PATCH 35/49] reclassify app open, get rid of app version waiver --- .../integration/config/command-waivers.json | 21 +++++++------------ .../config/invocation-profiles.json | 7 +++++++ 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/test/integration/config/command-waivers.json b/src/test/integration/config/command-waivers.json index 36bfd4567..01a775a02 100644 --- a/src/test/integration/config/command-waivers.json +++ b/src/test/integration/config/command-waivers.json @@ -125,6 +125,13 @@ "reason": "Beyond overlapping path-shape issues, this command enters SSH/rsync execution paths that are interactive/system-dependent and not intended to run in headless integration matrix mode.", "issue": "seed-appinstall-web-directory-fixture" }, + { + "id": "command-bug-app-open-missing-virtualhost-link", + "commandId": "app open", + "category": "INTERACTIVE_REQUIRED", + "reason": "Command wants to open URL derived from app installation.", + "issue": "need-interactive-mocking-for-open" + }, { "id": "resource-precondition-database-mysql-user-delete-main-user-protected", "commandId": "database mysql user delete", @@ -139,13 +146,6 @@ "reason": "The integration fixture set does not provide a resolvable system software entry for the placeholder value used in this command run. The command fails with 'system software ... not found'.", "issue": "seed-known-resource-fixtures" }, - { - "id": "command-bug-app-open-missing-virtualhost-link", - "commandId": "app open", - "category": "COMMAND_BUG", - "reason": "The test fixture app installation used in integration is not linked to a virtual host, and this command currently fails along that path in run-all execution.", - "issue": "seed-known-resource-fixtures" - }, { "id": "resource-precondition-app-upload-source-placeholder-not-available", "commandId": "app upload", @@ -160,13 +160,6 @@ "reason": "The fixture app identifier used in integration cannot be resolved in the mocked dataset, and the command fails with 'app ... not found'.", "issue": "seed-known-resource-fixtures" }, - { - "id": "command-bug-app-versions-access-denied", - "commandId": "app versions", - "category": "COMMAND_BUG", - "reason": "The command path fails with an access denied error in the current integration authorization/fixture setup.", - "issue": "seed-permissions-fixtures" - }, { "id": "resource-precondition-container-run-service-id-not-found", "commandId": "container run", diff --git a/src/test/integration/config/invocation-profiles.json b/src/test/integration/config/invocation-profiles.json index a1fa8f5c3..8a25aec35 100644 --- a/src/test/integration/config/invocation-profiles.json +++ b/src/test/integration/config/invocation-profiles.json @@ -215,6 +215,13 @@ "interactivePolicy": "classify", "disableExampleSource": true }, + { + "id": "app-versions", + "match": { "exact": "app versions" }, + "requiredArgDefaults": { + "app": "node" + } + }, { "id": "app-dependency-update", "match": { "exact": "app dependency update" }, From 933711c8cba2a20ff0cfc10fa4d84730c586823a Mon Sep 17 00:00:00 2001 From: Lars Bergmann Date: Wed, 12 Aug 2026 12:55:28 +0200 Subject: [PATCH 36/49] Update artifacts --- .../integration/config/command-classifications.json | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/src/test/integration/config/command-classifications.json b/src/test/integration/config/command-classifications.json index 704009be3..b01c929b4 100644 --- a/src/test/integration/config/command-classifications.json +++ b/src/test/integration/config/command-classifications.json @@ -1,13 +1,13 @@ { "schemaVersion": 1, - "generatedAt": "2026-08-12T09:19:04.408Z", + "generatedAt": "2026-08-12T10:45:57.580Z", "source": { "kind": "run-all-summary" }, "statistics": { - "successful": 154, + "successful": 155, "failed": 0, - "waivedSkipped": 34, + "waivedSkipped": 33, "total": 188 }, "entries": [ @@ -33,7 +33,7 @@ }, { "commandId": "app open", - "category": "COMMAND_BUG", + "category": "INTERACTIVE_REQUIRED", "source": "waiver" }, { @@ -56,11 +56,6 @@ "category": "RESOURCE_PRECONDITION", "source": "waiver" }, - { - "commandId": "app versions", - "category": "COMMAND_BUG", - "source": "waiver" - }, { "commandId": "container cp", "category": "INTERACTIVE_REQUIRED", From 60b61bc560d98e6bbcdad26e4771ea68d573d56f Mon Sep 17 00:00:00 2001 From: Lars Bergmann Date: Wed, 12 Aug 2026 13:11:17 +0200 Subject: [PATCH 37/49] remove another waiver --- .../integration/config/command-classifications.json | 11 +++-------- src/test/integration/config/command-waivers.json | 7 ------- 2 files changed, 3 insertions(+), 15 deletions(-) diff --git a/src/test/integration/config/command-classifications.json b/src/test/integration/config/command-classifications.json index b01c929b4..7a7c53315 100644 --- a/src/test/integration/config/command-classifications.json +++ b/src/test/integration/config/command-classifications.json @@ -1,13 +1,13 @@ { "schemaVersion": 1, - "generatedAt": "2026-08-12T10:45:57.580Z", + "generatedAt": "2026-08-12T11:10:29.896Z", "source": { "kind": "run-all-summary" }, "statistics": { - "successful": 155, + "successful": 156, "failed": 0, - "waivedSkipped": 33, + "waivedSkipped": 32, "total": 188 }, "entries": [ @@ -156,11 +156,6 @@ "category": "RESOURCE_PRECONDITION", "source": "waiver" }, - { - "commandId": "ssh-user create", - "category": "RESOURCE_PRECONDITION", - "source": "waiver" - }, { "commandId": "stack deploy", "category": "INTERACTIVE_REQUIRED", diff --git a/src/test/integration/config/command-waivers.json b/src/test/integration/config/command-waivers.json index 01a775a02..9929e018a 100644 --- a/src/test/integration/config/command-waivers.json +++ b/src/test/integration/config/command-waivers.json @@ -216,13 +216,6 @@ "reason": "The organization targeted by integration placeholders does not exist in the fixture state at deletion time, resulting in a 404 failure.", "issue": "seed-known-resource-fixtures" }, - { - "id": "resource-precondition-ssh-user-create-readback-404", - "commandId": "ssh-user create", - "category": "RESOURCE_PRECONDITION", - "reason": "Creation step succeeds, but follow-up readback in integration fixtures returns 404, indicating inconsistent fixture state for immediate lookup.", - "issue": "align-user-create-fixture-readback" - }, { "id": "resource-precondition-volume-delete-volume-not-found", "commandId": "volume delete", From 119cb6abbd27c3f9d9d9e3eed7541a003c1d5eca Mon Sep 17 00:00:00 2001 From: Lars Bergmann Date: Wed, 12 Aug 2026 13:43:33 +0200 Subject: [PATCH 38/49] Sharpen invocation profile, kill waiver --- .../integration/config/command-classifications.json | 11 +++-------- src/test/integration/config/command-waivers.json | 7 ------- src/test/integration/config/invocation-profiles.json | 3 ++- 3 files changed, 5 insertions(+), 16 deletions(-) diff --git a/src/test/integration/config/command-classifications.json b/src/test/integration/config/command-classifications.json index 7a7c53315..2cba3db17 100644 --- a/src/test/integration/config/command-classifications.json +++ b/src/test/integration/config/command-classifications.json @@ -1,13 +1,13 @@ { "schemaVersion": 1, - "generatedAt": "2026-08-12T11:10:29.896Z", + "generatedAt": "2026-08-12T11:37:41.994Z", "source": { "kind": "run-all-summary" }, "statistics": { - "successful": 156, + "successful": 157, "failed": 0, - "waivedSkipped": 32, + "waivedSkipped": 31, "total": 188 }, "entries": [ @@ -126,11 +126,6 @@ "category": "RESOURCE_PRECONDITION", "source": "waiver" }, - { - "commandId": "domain dnszone update", - "category": "RESOURCE_PRECONDITION", - "source": "waiver" - }, { "commandId": "domain get", "category": "RESOURCE_PRECONDITION", diff --git a/src/test/integration/config/command-waivers.json b/src/test/integration/config/command-waivers.json index 9929e018a..4667f7228 100644 --- a/src/test/integration/config/command-waivers.json +++ b/src/test/integration/config/command-waivers.json @@ -181,13 +181,6 @@ "reason": "The integration fixture dataset does not include the requested DNS zone domain in this run context, causing a deterministic 'DNS zone ... not found' failure.", "issue": "seed-known-domain-fixture" }, - { - "id": "resource-precondition-domain-dnszone-update-zone-not-found", - "commandId": "domain dnszone update", - "category": "RESOURCE_PRECONDITION", - "reason": "The integration fixture dataset does not include the requested DNS zone domain in this run context, causing a deterministic 'DNS zone ... not found' failure.", - "issue": "seed-known-domain-fixture" - }, { "id": "resource-precondition-domain-get-zone-not-found", "commandId": "domain get", diff --git a/src/test/integration/config/invocation-profiles.json b/src/test/integration/config/invocation-profiles.json index 8a25aec35..0c58fc5e1 100644 --- a/src/test/integration/config/invocation-profiles.json +++ b/src/test/integration/config/invocation-profiles.json @@ -141,7 +141,8 @@ "id": "domain-dnszone-update", "match": { "exact": "domain dnszone update" }, "requiredArgDefaults": { - "record-set": "a" + "record-set": "a", + "dnszone-id": "00000000-0000-4000-8000-000000000000" }, "requiredFlagDefaults": { "record": "203.0.113.10" From 461378917ef9e1a6c2432032b84c3475c448153d Mon Sep 17 00:00:00 2001 From: Lars Bergmann Date: Wed, 12 Aug 2026 16:13:28 +0200 Subject: [PATCH 39/49] sieve last remaining issues --- .../config/command-classifications.json | 13 ++++-------- .../integration/config/command-waivers.json | 21 +++++++------------ .../config/invocation-profiles.json | 7 +++++++ 3 files changed, 18 insertions(+), 23 deletions(-) diff --git a/src/test/integration/config/command-classifications.json b/src/test/integration/config/command-classifications.json index 2cba3db17..09c482f1b 100644 --- a/src/test/integration/config/command-classifications.json +++ b/src/test/integration/config/command-classifications.json @@ -1,21 +1,16 @@ { "schemaVersion": 1, - "generatedAt": "2026-08-12T11:37:41.994Z", + "generatedAt": "2026-08-12T14:12:09.894Z", "source": { "kind": "run-all-summary" }, "statistics": { - "successful": 157, + "successful": 158, "failed": 0, - "waivedSkipped": 31, + "waivedSkipped": 30, "total": 188 }, "entries": [ - { - "commandId": "app dependency versions", - "category": "RESOURCE_PRECONDITION", - "source": "waiver" - }, { "commandId": "app download", "category": "INTERACTIVE_REQUIRED", @@ -48,7 +43,7 @@ }, { "commandId": "app upload", - "category": "RESOURCE_PRECONDITION", + "category": "INTERACTIVE_REQUIRED", "source": "waiver" }, { diff --git a/src/test/integration/config/command-waivers.json b/src/test/integration/config/command-waivers.json index 4667f7228..be8c6a852 100644 --- a/src/test/integration/config/command-waivers.json +++ b/src/test/integration/config/command-waivers.json @@ -132,6 +132,13 @@ "reason": "Command wants to open URL derived from app installation.", "issue": "need-interactive-mocking-for-open" }, + { + "id": "resource-precondition-app-upload-source-placeholder-not-available", + "commandId": "app upload", + "category": "INTERACTIVE_REQUIRED", + "reason": "Command interactively uploads stuff and needs source folder in filesystem.", + "issue": "need-interactive-filesystem-mocking" + }, { "id": "resource-precondition-database-mysql-user-delete-main-user-protected", "commandId": "database mysql user delete", @@ -139,20 +146,6 @@ "reason": "Integration currently targets the main MySQL user, which is protected by API rules and cannot be deleted manually in this flow.", "issue": "seed-mysql-non-main-user-fixture" }, - { - "id": "resource-precondition-app-dependency-versions-systemsoftware-not-found", - "commandId": "app dependency versions", - "category": "RESOURCE_PRECONDITION", - "reason": "The integration fixture set does not provide a resolvable system software entry for the placeholder value used in this command run. The command fails with 'system software ... not found'.", - "issue": "seed-known-resource-fixtures" - }, - { - "id": "resource-precondition-app-upload-source-placeholder-not-available", - "commandId": "app upload", - "category": "RESOURCE_PRECONDITION", - "reason": "The integration invocation for this command does not provide a usable source path in the run-all environment. The command fails while parsing the source input.", - "issue": "fix-integration-invocation-profiles" - }, { "id": "resource-precondition-app-version-info-app-not-found", "commandId": "app version-info", diff --git a/src/test/integration/config/invocation-profiles.json b/src/test/integration/config/invocation-profiles.json index 0c58fc5e1..a676ab7f3 100644 --- a/src/test/integration/config/invocation-profiles.json +++ b/src/test/integration/config/invocation-profiles.json @@ -232,5 +232,12 @@ "requiredFlagDefaults": { "set": "node=~18" } + }, + { + "id": "app-dependency-versions", + "match": { "exact": "app dependency versions" }, + "requiredArgDefaults": { + "systemsoftware": "node" + } } ] From 835c9f8ed3b306c8893c46783d011966dde4e75d Mon Sep 17 00:00:00 2001 From: Lars Bergmann Date: Wed, 12 Aug 2026 16:34:02 +0200 Subject: [PATCH 40/49] call command properly, remove waiver --- .../integration/config/command-classifications.json | 11 +++-------- src/test/integration/config/command-waivers.json | 7 ------- src/test/integration/config/invocation-profiles.json | 8 ++++++++ 3 files changed, 11 insertions(+), 15 deletions(-) diff --git a/src/test/integration/config/command-classifications.json b/src/test/integration/config/command-classifications.json index 09c482f1b..7cb8d28a7 100644 --- a/src/test/integration/config/command-classifications.json +++ b/src/test/integration/config/command-classifications.json @@ -1,13 +1,13 @@ { "schemaVersion": 1, - "generatedAt": "2026-08-12T14:12:09.894Z", + "generatedAt": "2026-08-12T14:32:16.221Z", "source": { "kind": "run-all-summary" }, "statistics": { - "successful": 158, + "successful": 159, "failed": 0, - "waivedSkipped": 30, + "waivedSkipped": 29, "total": 188 }, "entries": [ @@ -46,11 +46,6 @@ "category": "INTERACTIVE_REQUIRED", "source": "waiver" }, - { - "commandId": "app version-info", - "category": "RESOURCE_PRECONDITION", - "source": "waiver" - }, { "commandId": "container cp", "category": "INTERACTIVE_REQUIRED", diff --git a/src/test/integration/config/command-waivers.json b/src/test/integration/config/command-waivers.json index be8c6a852..3a2ba4f8c 100644 --- a/src/test/integration/config/command-waivers.json +++ b/src/test/integration/config/command-waivers.json @@ -146,13 +146,6 @@ "reason": "Integration currently targets the main MySQL user, which is protected by API rules and cannot be deleted manually in this flow.", "issue": "seed-mysql-non-main-user-fixture" }, - { - "id": "resource-precondition-app-version-info-app-not-found", - "commandId": "app version-info", - "category": "RESOURCE_PRECONDITION", - "reason": "The fixture app identifier used in integration cannot be resolved in the mocked dataset, and the command fails with 'app ... not found'.", - "issue": "seed-known-resource-fixtures" - }, { "id": "resource-precondition-container-run-service-id-not-found", "commandId": "container run", diff --git a/src/test/integration/config/invocation-profiles.json b/src/test/integration/config/invocation-profiles.json index a676ab7f3..0201b26bc 100644 --- a/src/test/integration/config/invocation-profiles.json +++ b/src/test/integration/config/invocation-profiles.json @@ -239,5 +239,13 @@ "requiredArgDefaults": { "systemsoftware": "node" } + }, + { + "id": "app-version-info", + "match": { "exact": "app version-info" }, + "requiredArgDefaults": { + "app": "node", + "version": "1.0.0" + } } ] From d794e0c4f397fbc421d2db22f2846566fbcc9b13 Mon Sep 17 00:00:00 2001 From: Lars Bergmann Date: Wed, 12 Aug 2026 17:19:03 +0200 Subject: [PATCH 41/49] More invocation profiles --- src/test/integration/config/invocation-profiles.json | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/test/integration/config/invocation-profiles.json b/src/test/integration/config/invocation-profiles.json index 0201b26bc..ce26baa1a 100644 --- a/src/test/integration/config/invocation-profiles.json +++ b/src/test/integration/config/invocation-profiles.json @@ -247,5 +247,12 @@ "app": "node", "version": "1.0.0" } + }, + { + "id": "container-run", + "match": { "exact": "container run" }, + "requiredFlagDefaults": { + "name": "00000000-0000-4000-8000-000000000000" + } } ] From a32d56768fe719f96d5c8eabf22ac1d215e97579 Mon Sep 17 00:00:00 2001 From: Lars Bergmann Date: Wed, 12 Aug 2026 17:36:28 +0200 Subject: [PATCH 42/49] Kill waiver --- .../integration/config/command-classifications.json | 11 +++-------- src/test/integration/config/command-waivers.json | 7 ------- 2 files changed, 3 insertions(+), 15 deletions(-) diff --git a/src/test/integration/config/command-classifications.json b/src/test/integration/config/command-classifications.json index 7cb8d28a7..d9614bbc6 100644 --- a/src/test/integration/config/command-classifications.json +++ b/src/test/integration/config/command-classifications.json @@ -1,13 +1,13 @@ { "schemaVersion": 1, - "generatedAt": "2026-08-12T14:32:16.221Z", + "generatedAt": "2026-08-12T15:34:40.264Z", "source": { "kind": "run-all-summary" }, "statistics": { - "successful": 159, + "successful": 160, "failed": 0, - "waivedSkipped": 29, + "waivedSkipped": 28, "total": 188 }, "entries": [ @@ -61,11 +61,6 @@ "category": "INTERACTIVE_REQUIRED", "source": "waiver" }, - { - "commandId": "container run", - "category": "RESOURCE_PRECONDITION", - "source": "waiver" - }, { "commandId": "conversation create", "category": "INTERACTIVE_REQUIRED", diff --git a/src/test/integration/config/command-waivers.json b/src/test/integration/config/command-waivers.json index 3a2ba4f8c..d41a3aff2 100644 --- a/src/test/integration/config/command-waivers.json +++ b/src/test/integration/config/command-waivers.json @@ -146,13 +146,6 @@ "reason": "Integration currently targets the main MySQL user, which is protected by API rules and cannot be deleted manually in this flow.", "issue": "seed-mysql-non-main-user-fixture" }, - { - "id": "resource-precondition-container-run-service-id-not-found", - "commandId": "container run", - "category": "RESOURCE_PRECONDITION", - "reason": "The created stack in integration fixtures does not expose the expected service mapping for this flow. The command fails with 'Service ID not found in the created stack'.", - "issue": "seed-stack-service-fixture" - }, { "id": "resource-precondition-database-mysql-dump-main-user-missing", "commandId": "database mysql dump", From 11cfcfd52164827c12ecf569c03161e5f38e85eb Mon Sep 17 00:00:00 2001 From: Lars Bergmann Date: Wed, 12 Aug 2026 17:54:43 +0200 Subject: [PATCH 43/49] Invent more random arguments matching mock data --- .../integration/config/command-classifications.json | 11 +++-------- src/test/integration/config/command-waivers.json | 7 ------- src/test/integration/config/invocation-profiles.json | 7 +++++++ 3 files changed, 10 insertions(+), 15 deletions(-) diff --git a/src/test/integration/config/command-classifications.json b/src/test/integration/config/command-classifications.json index d9614bbc6..8d47bee1c 100644 --- a/src/test/integration/config/command-classifications.json +++ b/src/test/integration/config/command-classifications.json @@ -1,13 +1,13 @@ { "schemaVersion": 1, - "generatedAt": "2026-08-12T15:34:40.264Z", + "generatedAt": "2026-08-12T15:52:56.927Z", "source": { "kind": "run-all-summary" }, "statistics": { - "successful": 160, + "successful": 161, "failed": 0, - "waivedSkipped": 28, + "waivedSkipped": 27, "total": 188 }, "entries": [ @@ -145,11 +145,6 @@ "commandId": "user ssh-key create", "category": "INTERACTIVE_REQUIRED", "source": "waiver" - }, - { - "commandId": "volume delete", - "category": "RESOURCE_PRECONDITION", - "source": "waiver" } ] } diff --git a/src/test/integration/config/command-waivers.json b/src/test/integration/config/command-waivers.json index d41a3aff2..2de979a39 100644 --- a/src/test/integration/config/command-waivers.json +++ b/src/test/integration/config/command-waivers.json @@ -187,12 +187,5 @@ "category": "RESOURCE_PRECONDITION", "reason": "The organization targeted by integration placeholders does not exist in the fixture state at deletion time, resulting in a 404 failure.", "issue": "seed-known-resource-fixtures" - }, - { - "id": "resource-precondition-volume-delete-volume-not-found", - "commandId": "volume delete", - "category": "RESOURCE_PRECONDITION", - "reason": "The requested volume placeholder does not exist in integration fixtures for the active stack state, causing deterministic not-found failure.", - "issue": "seed-known-resource-fixtures" } ] diff --git a/src/test/integration/config/invocation-profiles.json b/src/test/integration/config/invocation-profiles.json index ce26baa1a..ccc038782 100644 --- a/src/test/integration/config/invocation-profiles.json +++ b/src/test/integration/config/invocation-profiles.json @@ -254,5 +254,12 @@ "requiredFlagDefaults": { "name": "00000000-0000-4000-8000-000000000000" } + }, + { + "id": "volume-delete", + "match": { "exact": "volume delete" }, + "requiredArgDefaults": { + "name": "node" + } } ] From 0b9fcd6b7e1be7228238a136082ead887dcd85a4 Mon Sep 17 00:00:00 2001 From: Lars Bergmann Date: Thu, 13 Aug 2026 10:18:37 +0200 Subject: [PATCH 44/49] Remove more waivers with better examples in schema --- .../config/command-classifications.json | 16 +++------------- src/test/integration/config/command-waivers.json | 14 -------------- 2 files changed, 3 insertions(+), 27 deletions(-) diff --git a/src/test/integration/config/command-classifications.json b/src/test/integration/config/command-classifications.json index 8d47bee1c..2561dc7e1 100644 --- a/src/test/integration/config/command-classifications.json +++ b/src/test/integration/config/command-classifications.json @@ -1,13 +1,13 @@ { "schemaVersion": 1, - "generatedAt": "2026-08-12T15:52:56.927Z", + "generatedAt": "2026-08-13T08:14:24.947Z", "source": { "kind": "run-all-summary" }, "statistics": { - "successful": 161, + "successful": 163, "failed": 0, - "waivedSkipped": 27, + "waivedSkipped": 25, "total": 188 }, "entries": [ @@ -106,16 +106,6 @@ "category": "INTERACTIVE_REQUIRED", "source": "waiver" }, - { - "commandId": "domain dnszone get", - "category": "RESOURCE_PRECONDITION", - "source": "waiver" - }, - { - "commandId": "domain get", - "category": "RESOURCE_PRECONDITION", - "source": "waiver" - }, { "commandId": "experimental deploy", "category": "INTERACTIVE_REQUIRED", diff --git a/src/test/integration/config/command-waivers.json b/src/test/integration/config/command-waivers.json index 2de979a39..a0bb2cc67 100644 --- a/src/test/integration/config/command-waivers.json +++ b/src/test/integration/config/command-waivers.json @@ -153,20 +153,6 @@ "reason": "The MySQL dump flow requires a resolvable main user in fixtures. In this run the command fails with 'No main user found'.", "issue": "seed-mysql-main-user-fixture" }, - { - "id": "resource-precondition-domain-dnszone-get-zone-not-found", - "commandId": "domain dnszone get", - "category": "RESOURCE_PRECONDITION", - "reason": "The integration fixture dataset does not include the requested DNS zone domain in this run context, causing a deterministic 'DNS zone ... not found' failure.", - "issue": "seed-known-domain-fixture" - }, - { - "id": "resource-precondition-domain-get-zone-not-found", - "commandId": "domain get", - "category": "RESOURCE_PRECONDITION", - "reason": "Domain lookup in this integration path depends on a DNS zone fixture that is not present for the placeholder value, producing 'DNS zone ... not found'.", - "issue": "seed-known-domain-fixture" - }, { "id": "resource-precondition-experimental-deploy-registry-service-missing", "commandId": "experimental deploy", From 3bdf262f670804cae5144a436c545a0e759f14bc Mon Sep 17 00:00:00 2001 From: Lars Bergmann Date: Thu, 13 Aug 2026 10:45:01 +0200 Subject: [PATCH 45/49] kill even more waivers --- .../integration/config/command-classifications.json | 11 +++-------- src/test/integration/config/command-waivers.json | 7 ------- 2 files changed, 3 insertions(+), 15 deletions(-) diff --git a/src/test/integration/config/command-classifications.json b/src/test/integration/config/command-classifications.json index 2561dc7e1..c2b31142a 100644 --- a/src/test/integration/config/command-classifications.json +++ b/src/test/integration/config/command-classifications.json @@ -1,13 +1,13 @@ { "schemaVersion": 1, - "generatedAt": "2026-08-13T08:14:24.947Z", + "generatedAt": "2026-08-13T08:35:57.943Z", "source": { "kind": "run-all-summary" }, "statistics": { - "successful": 163, + "successful": 164, "failed": 0, - "waivedSkipped": 25, + "waivedSkipped": 24, "total": 188 }, "entries": [ @@ -116,11 +116,6 @@ "category": "INTERACTIVE_REQUIRED", "source": "waiver" }, - { - "commandId": "mail address update", - "category": "RESOURCE_PRECONDITION", - "source": "waiver" - }, { "commandId": "org delete", "category": "RESOURCE_PRECONDITION", diff --git a/src/test/integration/config/command-waivers.json b/src/test/integration/config/command-waivers.json index a0bb2cc67..fce053f2f 100644 --- a/src/test/integration/config/command-waivers.json +++ b/src/test/integration/config/command-waivers.json @@ -160,13 +160,6 @@ "reason": "Calls further tooling down the line, e.g. railpack and/or docker", "issue": "deeper-mocking-needed" }, - { - "id": "resource-precondition-mail-address-update-mail-address-not-found", - "commandId": "mail address update", - "category": "RESOURCE_PRECONDITION", - "reason": "The mail address used by integration placeholders is not present in the mocked dataset during this run, causing a deterministic not-found failure.", - "issue": "seed-known-resource-fixtures" - }, { "id": "resource-precondition-org-delete-org-not-found", "commandId": "org delete", From b8d8709e2b2d0e2f5b62c924cc9b9675fc9e025b Mon Sep 17 00:00:00 2001 From: Lars Bergmann Date: Thu, 13 Aug 2026 11:21:10 +0200 Subject: [PATCH 46/49] remove org deletion waiver --- .../integration/config/command-classifications.json | 11 +++-------- src/test/integration/config/command-waivers.json | 7 ------- 2 files changed, 3 insertions(+), 15 deletions(-) diff --git a/src/test/integration/config/command-classifications.json b/src/test/integration/config/command-classifications.json index c2b31142a..1a8e047a1 100644 --- a/src/test/integration/config/command-classifications.json +++ b/src/test/integration/config/command-classifications.json @@ -1,13 +1,13 @@ { "schemaVersion": 1, - "generatedAt": "2026-08-13T08:35:57.943Z", + "generatedAt": "2026-08-13T09:16:57.004Z", "source": { "kind": "run-all-summary" }, "statistics": { - "successful": 164, + "successful": 165, "failed": 0, - "waivedSkipped": 24, + "waivedSkipped": 23, "total": 188 }, "entries": [ @@ -116,11 +116,6 @@ "category": "INTERACTIVE_REQUIRED", "source": "waiver" }, - { - "commandId": "org delete", - "category": "RESOURCE_PRECONDITION", - "source": "waiver" - }, { "commandId": "stack deploy", "category": "INTERACTIVE_REQUIRED", diff --git a/src/test/integration/config/command-waivers.json b/src/test/integration/config/command-waivers.json index fce053f2f..c3b783f7a 100644 --- a/src/test/integration/config/command-waivers.json +++ b/src/test/integration/config/command-waivers.json @@ -159,12 +159,5 @@ "category": "INTERACTIVE_REQUIRED", "reason": "Calls further tooling down the line, e.g. railpack and/or docker", "issue": "deeper-mocking-needed" - }, - { - "id": "resource-precondition-org-delete-org-not-found", - "commandId": "org delete", - "category": "RESOURCE_PRECONDITION", - "reason": "The organization targeted by integration placeholders does not exist in the fixture state at deletion time, resulting in a 404 failure.", - "issue": "seed-known-resource-fixtures" } ] From 013b8ed53f4ade2cebbe648c904bc589954e0e82 Mon Sep 17 00:00:00 2001 From: Lars Bergmann Date: Thu, 13 Aug 2026 11:51:53 +0200 Subject: [PATCH 47/49] reclassify --- .../integration/config/command-waivers.json | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/test/integration/config/command-waivers.json b/src/test/integration/config/command-waivers.json index c3b783f7a..7efd4ebc1 100644 --- a/src/test/integration/config/command-waivers.json +++ b/src/test/integration/config/command-waivers.json @@ -139,19 +139,12 @@ "reason": "Command interactively uploads stuff and needs source folder in filesystem.", "issue": "need-interactive-filesystem-mocking" }, - { - "id": "resource-precondition-database-mysql-user-delete-main-user-protected", - "commandId": "database mysql user delete", - "category": "RESOURCE_PRECONDITION", - "reason": "Integration currently targets the main MySQL user, which is protected by API rules and cannot be deleted manually in this flow.", - "issue": "seed-mysql-non-main-user-fixture" - }, { "id": "resource-precondition-database-mysql-dump-main-user-missing", "commandId": "database mysql dump", - "category": "RESOURCE_PRECONDITION", - "reason": "The MySQL dump flow requires a resolvable main user in fixtures. In this run the command fails with 'No main user found'.", - "issue": "seed-mysql-main-user-fixture" + "category": "INTERACTIVE_REQUIRED", + "reason": "The MySQL dump flow fetches dump via ssh which is not possible in test env.", + "issue": "need-ssh-mocking-for-database-mysql-dump" }, { "id": "resource-precondition-experimental-deploy-registry-service-missing", @@ -159,5 +152,12 @@ "category": "INTERACTIVE_REQUIRED", "reason": "Calls further tooling down the line, e.g. railpack and/or docker", "issue": "deeper-mocking-needed" + }, + { + "id": "resource-precondition-database-mysql-user-delete-main-user-protected", + "commandId": "database mysql user delete", + "category": "RESOURCE_PRECONDITION", + "reason": "Integration currently targets the main MySQL user, which is protected by API rules and cannot be deleted manually in this flow.", + "issue": "seed-mysql-non-main-user-fixture" } ] From e95e6ff2b12159d10efd7e11f66667bf7ab05526 Mon Sep 17 00:00:00 2001 From: Lars Bergmann Date: Thu, 13 Aug 2026 15:59:58 +0200 Subject: [PATCH 48/49] reclassify, skip interactive signal if no chooser is needed --- .../integration/command-discovery/synthesis.ts | 6 ++++++ .../config/command-classifications.json | 14 +++++++------- src/test/integration/config/command-waivers.json | 16 ++++++++-------- .../integration/config/invocation-profiles.json | 7 +++++++ 4 files changed, 28 insertions(+), 15 deletions(-) diff --git a/src/test/integration/command-discovery/synthesis.ts b/src/test/integration/command-discovery/synthesis.ts index 83d0f162a..82b34c889 100644 --- a/src/test/integration/command-discovery/synthesis.ts +++ b/src/test/integration/command-discovery/synthesis.ts @@ -541,6 +541,12 @@ function resolveInteractiveSignals( } if (signal === "addSelect") { + // Commands like database mysql upgrade only require selection when no + // explicit target version was provided. + if (hasFlag("version") && selectedFlags.has("version")) { + continue; + } + if (hasFlag("override-type")) { setFlagValue(selectedFlags, "override-type", ["auto"], "heuristic"); continue; diff --git a/src/test/integration/config/command-classifications.json b/src/test/integration/config/command-classifications.json index 1a8e047a1..b807a6a70 100644 --- a/src/test/integration/config/command-classifications.json +++ b/src/test/integration/config/command-classifications.json @@ -1,6 +1,6 @@ { "schemaVersion": 1, - "generatedAt": "2026-08-13T09:16:57.004Z", + "generatedAt": "2026-08-13T13:56:06.092Z", "source": { "kind": "run-all-summary" }, @@ -61,6 +61,11 @@ "category": "INTERACTIVE_REQUIRED", "source": "waiver" }, + { + "commandId": "container port-forward", + "category": "INTERACTIVE_REQUIRED", + "source": "waiver" + }, { "commandId": "conversation create", "category": "INTERACTIVE_REQUIRED", @@ -78,7 +83,7 @@ }, { "commandId": "database mysql dump", - "category": "RESOURCE_PRECONDITION", + "category": "INTERACTIVE_REQUIRED", "source": "waiver" }, { @@ -91,11 +96,6 @@ "category": "INTERACTIVE_REQUIRED", "source": "waiver" }, - { - "commandId": "database mysql upgrade", - "category": "INTERACTIVE_REQUIRED", - "source": "waiver" - }, { "commandId": "database mysql user delete", "category": "RESOURCE_PRECONDITION", diff --git a/src/test/integration/config/command-waivers.json b/src/test/integration/config/command-waivers.json index 7efd4ebc1..4c1fc2f3a 100644 --- a/src/test/integration/config/command-waivers.json +++ b/src/test/integration/config/command-waivers.json @@ -48,13 +48,6 @@ "reason": "MySQL shell requires password prompt interaction and cannot run headless yet.", "issue": "defer-interactive-support" }, - { - "id": "interactive-database-mysql-upgrade", - "commandId": "database mysql upgrade", - "category": "INTERACTIVE_REQUIRED", - "reason": "Upgrade confirmation and version choice currently requires interactive input.", - "issue": "defer-interactive-support" - }, { "id": "interactive-ddev-init", "commandId": "ddev init", @@ -144,7 +137,7 @@ "commandId": "database mysql dump", "category": "INTERACTIVE_REQUIRED", "reason": "The MySQL dump flow fetches dump via ssh which is not possible in test env.", - "issue": "need-ssh-mocking-for-database-mysql-dump" + "issue": "need-ssh-mocking" }, { "id": "resource-precondition-experimental-deploy-registry-service-missing", @@ -153,6 +146,13 @@ "reason": "Calls further tooling down the line, e.g. railpack and/or docker", "issue": "deeper-mocking-needed" }, + { + "id": "container-port-forwarding-ssh-missing", + "commandId": "container port-forward", + "category": "INTERACTIVE_REQUIRED", + "reason": "port forwarding via ssh is not available in test env", + "issue": "need-ssh-mocking" + }, { "id": "resource-precondition-database-mysql-user-delete-main-user-protected", "commandId": "database mysql user delete", diff --git a/src/test/integration/config/invocation-profiles.json b/src/test/integration/config/invocation-profiles.json index ccc038782..319b3725b 100644 --- a/src/test/integration/config/invocation-profiles.json +++ b/src/test/integration/config/invocation-profiles.json @@ -90,6 +90,13 @@ "password": "integration-password" } }, + { + "id": "database-mysql-upgrade", + "match": { "exact": "database mysql upgrade" }, + "requiredFlagDefaults": { + "version": "latest" + } + }, { "id": "database-mysql-shell", "match": { "exact": "database mysql shell" }, From d199276f32867243e4e12862298bca684dc9571f Mon Sep 17 00:00:00 2001 From: Lars Bergmann Date: Fri, 14 Aug 2026 08:10:57 +0200 Subject: [PATCH 49/49] Improve docs, operational guidance --- test_docs/integration-artifacts.md | 273 +++++++++++++++++---------- test_docs/run-all-commands.md | 289 +++++++++++++++++++++-------- test_docs/waiver-governance.md | 256 +++++++++++++------------ 3 files changed, 523 insertions(+), 295 deletions(-) diff --git a/test_docs/integration-artifacts.md b/test_docs/integration-artifacts.md index 696ae0720..c4dfdb7c8 100644 --- a/test_docs/integration-artifacts.md +++ b/test_docs/integration-artifacts.md @@ -1,107 +1,174 @@ -# Integration Artifacts and Contracts (Draft) +# Integration Artifacts and Contracts ## Purpose -Define the artifact contract for run-all integration execution and downstream analysis tooling. - -## Artifact Flow -1. Runner executes discovered commands. -2. Runner emits NDJSON machine log. -3. Runner emits/updates classification catalog (full runs only). -4. Analyzer consumes machine log records and command source/transitive analysis to map API usage to descriptors/OpenAPI operations. -5. Reports are generated as JSON and Markdown. - -## Primary Artifacts -- run-all-commands.ndjson -- src/test/integration/config/command-classifications.json - -Analyzer artifacts (generated only when analyzer tooling is executed): -- command-endpoint-map.json -- command-endpoint-map.md - -## NDJSON Events -Expected event types: -- run-start -- command-start -- command-result -- run-summary - -### command-start key fields -- commandId -- sourceFile -- commandTokens -- parsedArgs -- parsedFlags -- interactiveSignals -- invocationProfilesApplied -- extractionDiagnostics -- invocationArgs -- synthesizedInvocationArgs -- argumentSource -- interactiveDecision -- overrideApplied - -### command-result key fields -- commandId -- status (succeeded | failed | waived | spawn-error) -- failureCategory (for failed) -- durationMs -- exitCode (when available) +This document defines the machine contracts for the run-all integration test and +the downstream analyzer tooling. + +The intended one-way data flow is: + +1. Integration runner executes discovered commands. +2. Runner writes machine events to NDJSON. +3. Runner writes/updates classification catalog on full runs. +4. Analyzer reads machine log + command source + OpenAPI. +5. Analyzer writes JSON + Markdown reports for triage. + +## Artifact Inventory + +Runner outputs: + +- `run-all-commands.ndjson` (default path at repo root, configurable) +- `src/test/integration/config/command-classifications.json` (full runs only) + +Analyzer outputs (on tool execution): + +- `command-endpoint-map.json` (default path, configurable) +- `command-endpoint-map.md` (default path, configurable) + +Analyzer inputs: + +- NDJSON machine log (`--machine-log`, default `run-all-commands.ndjson`) +- OpenAPI JSON (`--openapi`, default `openapi.json`) + +## NDJSON Event Contract + +The runner appends JSON lines with an auto-added `timestamp` and an `event` +field. + +Expected `event` values: + +- `run-start` +- `command-start` +- `command-result` +- `run-summary` + +### run-start + +Contract fields: + +- `categoryFilter: string | null` +- `classificationCatalogPath: string | null` +- `projectId: string` +- `commandCount: number` +- `waiverCount: number` +- `runtimeOverrides: { commandId?: string; invocationArgs?: string[] }` + +### command-start + +Contract fields: + +- `index: number` +- `total: number` +- `position: string` (for example `12/338`) +- `commandId: string` +- `sourceFile: string` (relative to `src/commands`) +- `commandTokens: string[]` +- `parsedArgs: ParsedArg[]` +- `parsedFlags: ParsedFlag[]` +- `interactiveSignals: InteractiveSignal[]` +- `invocationProfilesApplied: string[]` +- `extractionDiagnostics: string[]` +- `invocationArgs: string[]` (effective args after overrides) +- `synthesizedInvocationArgs: string[]` (original synthesis) +- `argumentSource: "profile" | "example" | "heuristic"` +- `interactiveDecision: "NON_INTERACTIVE_RESOLVED" | "INTERACTIVE_REQUIRED"` +- `overrideApplied: boolean` + +### command-result + +Core fields: + +- `index`, `total`, `position`, `commandId`, `durationMs` +- `status: "succeeded" | "failed" | "waived" | "spawn-error"` + +Status-specific fields: + +- `failed`: `failureCategory`, optionally `exitCode`, `stderr`, `stdout`, + `timedOut`, `preflightIssues`, `details` +- `waived`: full `waiver` object +- `spawn-error`: `errorMessage`, optionally `stderr`, `stdout` +- `succeeded`: `exitCode` + +### run-summary + +Contract fields: + +- `statistics: { successful, failed, waivedSkipped, total }` +- `failuresByCategory` +- `waivedByCategory` +- `infrastructureFailures: string[]` +- `runtimeOverrides` ## Classification Catalog Contract -File: src/test/integration/config/command-classifications.json - -Key structure: -- schemaVersion -- generatedAt -- source -- statistics -- entries[] with: - - commandId - - category - - source (failure | waiver | skip) - -Note: -- Runner-generated catalogs currently contain failure and waiver entries. -- skip entries are supported by the catalog schema and log-extract helper. - -## Category Filter Contract -When MW_TEST_CATEGORY is set: -- discovery still enumerates all commands -- execution list is filtered to command IDs from classification catalog entries matching category -- waiver application is disabled for the scoped run (commands execute instead of being waived) -- strict global waiver integrity checks are skipped for partial scope - -## Single-Command Override Contract -When MW_TEST_COMMAND_ID is set: -- execution list contains only that command -- waiver is bypassed for that selected command -- strict global waiver integrity checks are skipped for partial scope -- if MW_TEST_COMMAND_INVOCATION_ARGS is set, it replaces synthesized invocation args - -## Analyzer Tooling -Script entry points in package scripts: -- tool:integration:generate-command-endpoint-map -- tool:integration:generate-resource-precondition-map - -Inputs: -- machine log NDJSON -- OpenAPI JSON - -Outputs: -- endpoint map JSON -- endpoint map Markdown - -## Failure Taxonomy -Canonical categories: -- ARG_MISUSE -- INTERACTIVE_REQUIRED -- RESOURCE_PRECONDITION -- CONTRACT_SHAPE -- COMMAND_BUG -- DEPRECATED_ENDPOINT - -## Compatibility Guidance -If you modify runner payload fields: -1. Keep existing fields backward-compatible when possible. -2. Update analyzer expectations in lockstep. -3. Document contract changes in this file before merging. + +File: + +- `src/test/integration/config/command-classifications.json` + +Schema: + +- `schemaVersion: 1` +- `generatedAt: string` (ISO timestamp) +- `source: { kind: "run-all-summary" | "log-extract"; path?: string }` +- `statistics: { successful, failed, waivedSkipped, total }` +- `entries: Array<{ commandId, category, source }>` + +Where: + +- `category` is one of: + `ARG_MISUSE | INTERACTIVE_REQUIRED | RESOURCE_PRECONDITION | CONTRACT_SHAPE | COMMAND_BUG | DEPRECATED_ENDPOINT` +- `source` is one of `failure | waiver | skip` + +Behavior notes: + +- Runner-generated catalog uses `buildClassificationCatalogFromBuckets` and + emits `failure` and `waiver` entries. +- `skip` is supported by the schema and by log extraction tooling, but not + emitted by the current runner flow. + +## Filtering and Override Contracts + +### Category Filter (`MW_TEST_CATEGORY`) + +- Discovery still scans all command files. +- Final command set is filtered via catalog entries matching the category. +- Waiver application is disabled for this run scope. +- Strict global waiver integrity checks are skipped. +- Classification catalog write is skipped. + +### Single Command Override (`MW_TEST_COMMAND_ID`) + +- Final command set is exactly one discovered command. +- Waiver is bypassed for that selected command. +- Strict global waiver integrity checks are skipped. +- Classification catalog write is skipped. +- Optional `MW_TEST_COMMAND_INVOCATION_ARGS` replaces synthesized invocation + args and must be a JSON array of strings. + +## Analyzer Contract + +Script entry points: + +- `yarn tool:integration:generate-command-endpoint-map` +- `yarn tool:integration:generate-resource-precondition-map` + +CLI options: + +- `--machine-log ` (default `run-all-commands.ndjson`) +- `--category ` (optional category filter) +- `--openapi ` (default `openapi.json`) +- `--output-json ` (default `command-endpoint-map.json`) +- `--output-md ` (default `command-endpoint-map.md`) + +Strict expectations: + +- NDJSON must contain `command-start` with both `commandId` and `sourceFile`. +- NDJSON lines must parse as valid JSON. +- OpenAPI file must exist and parse as JSON. + +## Backward Compatibility Rules + +If you change any field in runner NDJSON payloads or classification catalog: + +1. Keep existing fields backward-compatible whenever possible. +2. Update analyzer expectations in the same change. +3. Update this document in the same change. diff --git a/test_docs/run-all-commands.md b/test_docs/run-all-commands.md index ab050c1a9..d0a730ea3 100644 --- a/test_docs/run-all-commands.md +++ b/test_docs/run-all-commands.md @@ -1,105 +1,238 @@ -# Run-All Commands Integration Runner (Draft) +# Run-All Commands Integration Runner ## Purpose -In full-matrix mode, run every discovered CLI command once in an integration context, emit machine-readable NDJSON logs, and enforce that all non-waived failures are visible and actionable. +Run all discovered CLI commands in integration context, emit deterministic +machine logs, and fail on all non-waived command failures. -## Scope -This runner is implemented in: -- src/test/integration/run-all-commands.test.ts +This is the enforcement surface for command-matrix health. -Note: -- The suite is guarded and only runs when the file is invoked explicitly via --runTestsByPath. +## Implementation Scope -It depends on: -- command discovery and invocation synthesis -- waiver configuration -- classification catalog generation +Primary entrypoint: + +- `src/test/integration/run-all-commands.test.ts` + +Supporting modules: + +- `src/test/integration/command-discovery.ts` +- `src/test/integration/run-all-commands/overrides.ts` +- `src/test/integration/run-all-commands/helpers.ts` +- `src/test/integration/run-all-commands/machine-log.ts` +- `src/test/integration/classification-catalog.ts` +- `src/test/integration/config/loader.ts` + +Guardrail: + +- This suite executes only when explicitly invoked with + `--runTestsByPath src/test/integration/run-all-commands.test.ts`. ## Required Environment -The test requires: -- MITTWALD_API_TOKEN -- MITTWALD_API_BASE_URL -- MW_TEST_PROJECT_ID - -## Optional Environment Controls -- MW_TEST_MACHINE_LOG_PATH - - Path for NDJSON output (default: run-all-commands.ndjson in repo root) -- MW_TEST_CATEGORY - - Restrict execution to command IDs listed in classification catalog for one category -- MW_TEST_CLASSIFICATION_CATALOG_PATH + +Required variables: + +- `MITTWALD_API_TOKEN` +- `MITTWALD_API_BASE_URL` +- `MW_TEST_PROJECT_ID` + +If any required variable is missing, the test fails at startup. + +## Optional Runtime Controls + +- `MW_TEST_MACHINE_LOG_PATH` + - NDJSON output path + - Default: `run-all-commands.ndjson` in repo root +- `MW_TEST_CATEGORY` + - Restrict execution to command IDs in classification catalog for one category +- `MW_TEST_CLASSIFICATION_CATALOG_PATH` - Override catalog path used by category filtering -- MW_TEST_COMMAND_ID - - Run only one discovered command ID - - When set, waiver for that command is bypassed intentionally (waiver hunting mode) -- MW_TEST_COMMAND_INVOCATION_ARGS - - JSON array of strings to fully override invocation args for MW_TEST_COMMAND_ID - - Requires MW_TEST_COMMAND_ID - -## Environment Variable Usage -All runtime controls are plain environment variables. - -You can use them in two styles: -- one-off: prefix variables for a single command invocation -- session: export/set variables in shell state, run command, then unset - -Example scenario used below: -- run a single command in waiver-hunting mode -- command ID: container logs - -### Bash/Zsh Examples -One-off invocation: +- `MW_TEST_COMMAND_ID` + - Run exactly one discovered command + - Waiver is bypassed for that command +- `MW_TEST_COMMAND_INVOCATION_ARGS` + - JSON array of strings replacing synthesized invocation args + - Requires `MW_TEST_COMMAND_ID` + +## Operational Modes + +### Full Matrix Mode + +Condition: + +- No `MW_TEST_CATEGORY` +- No `MW_TEST_COMMAND_ID` + +Behavior: + +- Discover all runnable commands +- Enforce strict waiver integrity checks +- Apply waivers +- Execute non-waived commands +- Write classification catalog from run summary buckets + +### Category Campaign Mode + +Condition: + +- `MW_TEST_CATEGORY` set + +Behavior: + +- Discover all commands, then filter using classification catalog entries +- Disable waiver application for selected commands +- Skip strict global waiver integrity checks +- Skip classification catalog write + +Use case: + +- Work down one failure class (for example `RESOURCE_PRECONDITION`) end-to-end. + +### Single-Command Hunting Mode + +Condition: + +- `MW_TEST_COMMAND_ID` set + +Behavior: + +- Select exactly one discovered command +- Bypass waiver for that command +- Optionally override invocation args with + `MW_TEST_COMMAND_INVOCATION_ARGS` +- Skip strict global waiver integrity checks +- Skip classification catalog write + +Use case: + +- Reproduce and fix one hard command without matrix noise. + +## Execution Runbooks + +### Full Matrix Run + +Fish: + +```fish +env MITTWALD_API_TOKEN="" \ + MITTWALD_API_BASE_URL="" \ + MW_TEST_PROJECT_ID="" \ + yarn test --runTestsByPath src/test/integration/run-all-commands.test.ts +``` + +Bash/Zsh: + ```bash +MITTWALD_API_TOKEN="" \ +MITTWALD_API_BASE_URL="" \ MW_TEST_PROJECT_ID="" \ -MW_TEST_COMMAND_ID="container logs" \ yarn test --runTestsByPath src/test/integration/run-all-commands.test.ts ``` -Session set, run, unset: +### Category Campaign Run + ```bash -export MW_TEST_PROJECT_ID="" -export MW_TEST_COMMAND_ID="container logs" +MITTWALD_API_TOKEN="" \ +MITTWALD_API_BASE_URL="" \ +MW_TEST_PROJECT_ID="" \ +MW_TEST_CATEGORY="RESOURCE_PRECONDITION" \ yarn test --runTestsByPath src/test/integration/run-all-commands.test.ts -unset MW_TEST_COMMAND_ID -unset MW_TEST_PROJECT_ID ``` -### Fish Examples -One-off invocation: -```fish -env MW_TEST_PROJECT_ID="" MW_TEST_COMMAND_ID="container logs" \ - yarn test --runTestsByPath src/test/integration/run-all-commands.test.ts +Optional custom catalog: + +```bash +MITTWALD_API_TOKEN="" \ +MITTWALD_API_BASE_URL="" \ +MW_TEST_PROJECT_ID="" \ +MW_TEST_CATEGORY="RESOURCE_PRECONDITION" \ +MW_TEST_CLASSIFICATION_CATALOG_PATH="./src/test/integration/config/command-classifications.json" \ +yarn test --runTestsByPath src/test/integration/run-all-commands.test.ts ``` -Session set, run, unset: -```fish -set -lx MW_TEST_PROJECT_ID "" -set -lx MW_TEST_COMMAND_ID "container logs" +### Single-Command Run (Waiver Bypass) + +```bash +MITTWALD_API_TOKEN="" \ +MITTWALD_API_BASE_URL="" \ +MW_TEST_PROJECT_ID="" \ +MW_TEST_COMMAND_ID="container logs" \ yarn test --runTestsByPath src/test/integration/run-all-commands.test.ts -set -e MW_TEST_COMMAND_ID -set -e MW_TEST_PROJECT_ID ``` -Optional invocation-arg override in any shell: -```sh -MW_TEST_COMMAND_INVOCATION_ARGS='["container","logs","--container-id","abc123","--tail","20"]' +With explicit invocation override: + +```bash +MITTWALD_API_TOKEN="" \ +MITTWALD_API_BASE_URL="" \ +MW_TEST_PROJECT_ID="" \ +MW_TEST_COMMAND_ID="container logs" \ +MW_TEST_COMMAND_INVOCATION_ARGS='["container","logs","--container-id","abc123","--tail","20"]' \ +yarn test --runTestsByPath src/test/integration/run-all-commands.test.ts ``` -## Operating Principles -1. Discovery first: commands are discovered from src/commands, then synthesized args are built. -2. Waiver file validation is always strict: - - duplicate waiver IDs fail - - duplicate waiver command IDs fail -3. Full runs add a global consistency check: - - waivers pointing to non-discovered commands fail -4. Command override mode skips the global waiver consistency check to enable focused debugging. -5. Category filter mode also skips the global waiver consistency check because execution scope is intentionally partial. -6. Non-waived failures fail the test and are surfaced with category and diagnostics. +## Runtime Behavior and Enforcement + +### Discovery and Invocation + +- Command IDs derive from file paths under `src/commands`. +- Invocation synthesis combines parsed args/flags, profiles, and examples. +- A command can be preflight-failed as `ARG_MISUSE` before execution if + required args/flags are unresolved. + +### Waiver and Integrity Logic + +Always enforced by loader: + +- No duplicate waiver IDs +- No duplicate waiver `commandId` + +Strictly enforced only in full mode: + +- No stale waiver references to non-discovered commands + +Special enforcement: + +- If a command is statically classified `INTERACTIVE_REQUIRED` and no waiver is + present (and waiver bypass is not active), this is treated as a failure and + infrastructure governance violation. + +### Command Result Statuses + +- `succeeded` +- `failed` +- `waived` +- `spawn-error` + +Timeout behavior: + +- Command process timeout is `30000ms` per command. +- Timeout is categorized as `COMMAND_BUG`. ## Outputs -- NDJSON machine log with run-start, command-start, command-result, run-summary -- Classification catalog file update during full runs without category or command override -## Notes for Maintainers -- Keep command IDs stable when refactoring command file paths. -- If invocation synthesis changes, verify MW_TEST_COMMAND_INVOCATION_ARGS still fully overrides run args. -- Preserve deterministic log fields consumed by downstream tooling. +Always produced: + +- NDJSON machine log with `run-start`, `command-start`, `command-result`, + `run-summary` + +Produced only in full mode: + +- Updated `src/test/integration/config/command-classifications.json` + +## Failure Handling Playbook + +1. Run in single-command mode for the failing command. +2. If needed, add `MW_TEST_COMMAND_INVOCATION_ARGS` to stabilize reproduction. +3. Decide fix path: + - command implementation change + - invocation profile change + - fixture/precondition change + - waiver addition/update (if fix not in scope) +4. Re-run single command until category and behavior are stable. +5. Re-run category campaign or full matrix to validate no regressions. + +## Maintainer Notes + +- Keep command IDs stable when moving/renaming command files. +- Preserve machine log payload fields used by downstream analyzers. +- If synthesis behavior changes, verify override semantics still replace args + exactly for command override mode. diff --git a/test_docs/waiver-governance.md b/test_docs/waiver-governance.md index 79b9a5693..7288c1e2c 100644 --- a/test_docs/waiver-governance.md +++ b/test_docs/waiver-governance.md @@ -1,134 +1,162 @@ -# Waiver Governance for Integration Command Matrix (Draft) +# Waiver Governance for Integration Command Matrix ## Purpose -Waivers are a governance tool, not a suppression shortcut. They document known failing commands with category, reason, and follow-up intent while keeping failures auditable. +Waivers are an explicit governance record for known failing integration commands. +They are not a suppression shortcut. -## Core Principle: Unwaived Commands Must Not Fail +Goal: -**All discovered commands must either pass or have an explicit waiver.** This is a hard invariant enforced by: -- **CI gate**: Integration test fails if any command is unwaived and fails -- **Peer review**: Waiver reasons must be understood and justified by reviewers +- keep matrix failures auditable +- keep failure ownership visible in PRs +- prevent silent drift of integration quality -This principle prevents silent rot and forces intentional decision-making: if a command fails and you can't fix it in the current scope, you *must* write a waiver in the same PR with a concrete reason. The friction is intentional. +## Source of Truth -## Lazy Enforcement Philosophy +- Waiver data: `src/test/integration/config/command-waivers.json` +- Validation: `src/test/integration/config/loader.ts` +- Runtime enforcement: `src/test/integration/run-all-commands.test.ts` -Waivers work best when enforcement is simple and visible: +## Core Rule -- **No speculative waivers** — Waivers can only be added when a command fails. Pre-emptive waivers hide problems that should be surfaced now. +All discovered commands must either: -- **Reasons are contemporary** — The waiver reason appears in the PR diff next to the code change. Reviewers see intent immediately. This scales better than historical archaeology through git blame. +1. pass, or +2. have an explicit waiver that is accepted by policy. -- **Emergent categorization** — Don't invent category hierarchy upfront. Patterns emerge naturally as waivers accumulate. After 20-30 waivers, category trends become visible and inform the next governance decision. +If a command fails without waiver coverage, the integration run fails. -- **Human review is the gate** — Reviewers must understand why each waiver exists. A weak reason ("known issue", "API doesn't work") gets pushed back. A concrete reason ("API response schema missing `examples` field, blocking invocation synthesis; filed as API-1234") survives review. This quality pressure is self-reinforcing. +## Waiver Schema -## Source of Truth -- Waivers file: src/test/integration/config/command-waivers.json -- Waiver loader validation: src/test/integration/config/loader.ts -- Enforcement during run: src/test/integration/run-all-commands.test.ts +Required fields per entry: -## Waiver Schema -Each waiver entry must include: -- id -- commandId -- category -- reason +- `id` +- `commandId` +- `category` +- `reason` + +Optional fields: -Optional: -- issue -- expiresOn +- `issue` +- `expiresOn` Allowed categories: -- ARG_MISUSE -- INTERACTIVE_REQUIRED -- RESOURCE_PRECONDITION -- CONTRACT_SHAPE -- COMMAND_BUG -- DEPRECATED_ENDPOINT - -## Hard Invariants -Always (loader validation): -1. Duplicate waiver IDs are invalid. -2. Duplicate waiver commandId entries are invalid. - -In full-matrix mode (no category filter and no command override): -3. Waiver commandId must map to a currently discovered command. -4. Commands classified INTERACTIVE_REQUIRED without waivers fail as governance drift. - -## Relaxed Invariants by Design -In targeted modes, the global waiver consistency check is skipped: -- category-filter mode -- single-command override mode (MW_TEST_COMMAND_ID) + +- `ARG_MISUSE` +- `INTERACTIVE_REQUIRED` +- `RESOURCE_PRECONDITION` +- `CONTRACT_SHAPE` +- `COMMAND_BUG` +- `DEPRECATED_ENDPOINT` + +## Enforced Invariants + +Always enforced (all run modes): + +1. duplicate waiver `id` is invalid +2. duplicate waiver `commandId` is invalid + +Enforced in full matrix mode only (no category filter, no command override): + +1. waiver `commandId` must exist in current discovery output + +Additional governance enforcement: + +- if a command is classified `INTERACTIVE_REQUIRED` and has no waiver (and no + bypass mode is active), this is a failure and governance violation. + +## Intentional Relaxations + +In partial investigation modes, strict global waiver integrity checks are +skipped by design: + +- category mode (`MW_TEST_CATEGORY`) +- single-command mode (`MW_TEST_COMMAND_ID`) Reason: -- these modes are intentionally partial and used for investigation loops. + +- these modes are for focused diagnosis, not full governance assertions. Important: -- loader-level duplicate checks still apply in all modes. - -## Operational Enforcement: CI Gate + Peer Review -This governance model requires both: - -**CI Gate (Mechanical):** -- Integration test suite runs all discovered commands -- Test fails if any command is unwaived and fails -- Test fails if waiver references a non-existent command -- Test fails if duplicate waivers exist - -**Peer Review (Human):** -- Reviewers must understand waiver semantics -- Waiver PRs are not auto-approved based on metrics -- Weak reasons trigger discussion, not merges -- Category assignment is debated if unclear - -Together, these prevent silent drift. You can't sneak in a waiver without explaining it; you can't add unmaintainable waivers because reviewers catch them. - -## Waiver Hunting Workflow -1. Run one command with MW_TEST_COMMAND_ID. -2. Reproduce and inspect failure details. -3. Decide one branch: - - fix command - - fix test fixture/precondition - - keep/add waiver with explicit reason and issue link -4. Re-run same command until category and behavior are stable. -5. If command becomes callable, remove waiver. - -## When to Add a Waiver -Add a waiver only when all are true: -1. Failure is understood and reproducible. -2. Category assignment is stable. -3. A near-term fix cannot be delivered in current change scope. - -## When Not to Add a Waiver -Do not add waivers for: -- unknown failures -- flaky behavior without root cause -- argument synthesis defects that should be fixed in invocation profiles - -## Quality Bar for Waiver Reasons -A good reason includes: -- failure mechanism (what broke and why) -- where it fails (component/path/layer) -- what condition is missing (fixture, API contract, design decision) -- intended fix direction (or why it's a permanent trade-off) - -A weak reason includes only: -- "fails in CI" -- "does not work" -- "known issue" -- "API thing" - -Weak reasons get rejected in review. The author must explain themselves to the reviewer. This quality pressure is a feature, not friction. - -## Review Checklist: Reviewer Responsibility -Reviewers must actively evaluate *every* waiver addition. Your job is to: - -1. **Understand the failure** — Did the author explain what broke? Can you reproduce it from the reason alone? -2. **Validate category** — Is the waiver category accurate? Does it match the actual failure root cause? -3. **Evaluate reason quality** — Is the reason concrete and specific? Would a future reader (or you in 6 months) understand this waiver? -4. **Require issue tracking** — For anything that's not a permanent design decision, is there an issue link? Does it have a target fix date? -5. **Question necessity** — Could this waiver have been avoided by fixing the command, fixture, or invocation profile in the same PR? -6. **Check for rot** — Are there existing waivers that should be removed because behavior is now fixed? - -Reject vague waivers. The 5 minutes you spend pushing back on reason quality prevents 10 hours of future confusion when someone tries to understand why the waiver exists. + +- loader-level duplicate validation still applies in all modes. + +## Decision Gate: Fix vs Waive + +Add or keep a waiver only when all conditions are true: + +1. failure is reproducible +2. failure category is stable +3. fix is out of current scope or blocked by external dependency +4. reason explains mechanism and next action clearly + +Do not waive when: + +- root cause is unknown +- behavior is flaky and not diagnosed +- failure is due to invocation synthesis gap that should be fixed in profiles + +## Reason Quality Standard + +A strong waiver reason states: + +1. what fails +2. where it fails +3. why it fails +4. what will resolve it (or why it is accepted long-term) + +Examples: + +- weak: `known issue` +- strong: `command requires interactive select branch in oclif prompt path; integration runner is non-interactive, no profile-based non-interactive fallback exists yet; tracked in MWCLI-742` + +## Operational Workflow + +### Add a Waiver + +1. reproduce command in single-command mode with `MW_TEST_COMMAND_ID` +2. capture observed failure category and evidence +3. if fix is not in scope, add waiver entry with concrete reason +4. rerun the same command to confirm expected waived behavior in full mode + +### Remove a Waiver + +1. reproduce command with waiver bypass mode (`MW_TEST_COMMAND_ID`) +2. verify command now passes with normal synthesized invocation +3. remove waiver entry +4. rerun full matrix or at least category campaign to confirm no regressions + +### Reclassify a Waiver + +1. reproduce command with waiver bypass +2. confirm current failure category +3. update category and reason together in one change +4. rerun category campaign for both old and new categories when practical + +## Review Checklist + +Reviewer must check every waiver change: + +1. command failure is understandable from evidence and reason +2. selected category matches actual failure mechanism +3. reason quality meets the standard above +4. issue link exists for non-permanent waivers +5. waiver is necessary (not a fixable in-scope defect) +6. stale waivers are removed when behavior is now fixed + +Reject vague waivers. Governance quality is a correctness requirement. + +## CI and Team Policy + +Mechanical gate (CI): + +- rejects unwaived failures +- rejects invalid waiver schema +- rejects duplicate waiver IDs/command IDs +- in full mode, rejects stale waiver command references + +Human gate (review): + +- validates reason quality and category correctness +- validates whether a waiver is the right choice for this change + +Both gates are required. CI enforces structure; review enforces intent.