diff --git a/.gitattributes b/.gitattributes index 05b1a132398..1e9bf994ead 100644 --- a/.gitattributes +++ b/.gitattributes @@ -2,3 +2,5 @@ **/ModuleBindings/** linguist-generated=true eol=lf /docs/llms/** linguist-generated=true /docs/llms/*-details.json linguist-generated=false +/tools/stack-bench/** text eol=lf +/tools/stack-bench/**/*.woff2 -text -diff diff --git a/codex-plugin/plugins/spacetimedb/skills/typescript-client/SKILL.md b/codex-plugin/plugins/spacetimedb/skills/typescript-client/SKILL.md index 3a31183133f..62f86ff36fc 100644 --- a/codex-plugin/plugins/spacetimedb/skills/typescript-client/SKILL.md +++ b/codex-plugin/plugins/spacetimedb/skills/typescript-client/SKILL.md @@ -18,7 +18,7 @@ Generated bindings convert snake_case names to camelCase, including row fields: ## React: main.tsx ```typescript -import React, { useEffect, useMemo } from 'react'; +import React, { useMemo } from 'react'; import ReactDOM from 'react-dom/client'; import { SpacetimeDBProvider } from 'spacetimedb/react'; import { DbConnection } from './module_bindings'; @@ -30,6 +30,7 @@ function Root() { DbConnection.builder() .withUri(SPACETIMEDB_URI) .withDatabaseName(MODULE_NAME) + // Reuse the token issued on the previous connection. .withToken(localStorage.getItem('auth_token') || undefined), [] ); @@ -46,6 +47,7 @@ ReactDOM.createRoot(document.getElementById('root')!).render(); ## React: App.tsx ```typescript +import { useEffect } from 'react'; import { useTable, useSpacetimeDB } from 'spacetimedb/react'; import { DbConnection, tables } from './module_bindings'; @@ -53,7 +55,7 @@ function App() { const { isActive, identity: myIdentity, token, getConnection } = useSpacetimeDB(); const conn = getConnection() as DbConnection | null; - // Save auth token + // Persist the issued token for the next page load. useEffect(() => { if (token) localStorage.setItem('auth_token', token); }, [token]); // Subscribe when connected. Prefer typed query builders over raw SQL diff --git a/codex-plugin/plugins/spacetimedb/skills/typescript-server/SKILL.md b/codex-plugin/plugins/spacetimedb/skills/typescript-server/SKILL.md index c9f2e7343fd..ba24bf781d5 100644 --- a/codex-plugin/plugins/spacetimedb/skills/typescript-server/SKILL.md +++ b/codex-plugin/plugins/spacetimedb/skills/typescript-server/SKILL.md @@ -98,6 +98,10 @@ Every column is a `t` builder value: Modifiers: `.primaryKey()`, `.autoInc()`, `.unique()`, `.index('btree')`, `.default(value)`. +`.primaryKey()` and `.unique()` apply to one column. For uniqueness across +multiple columns, use a surrogate key, the multi-column index below, and a +reducer that rejects an existing index match before inserting. + Use `.default(value)` only for a newly appended migration-safe field. Do not put defaults on primary-key, unique, or auto-increment columns. Optional columns: `nickname: t.option(t.string())` @@ -130,7 +134,9 @@ export { default } from './schema'; // re-export the schema for the module ent ## Reducers -Reducers are created with `spacetimedb.reducer(...)`; the export name becomes the reducer name: +Reducers are created with `spacetimedb.reducer(...)`. An exported `signUp` +becomes `signUp` in generated clients and `sign_up` in `spacetime call` and +`describe`: ```typescript export const createEntity = spacetimedb.reducer( @@ -278,9 +284,22 @@ const Shape = t.enum('Shape', { A client subscribing to a view receives only the rows it returns. Use a per-user view (keyed on `ctx.sender`) for per-viewer access control: deleting a row it depends on (e.g. a membership row) automatically drops the rows it was exposing from that client. +Use index accessors in views. Do not scan a whole table with `.iter()` when an +indexed lookup can select the required rows. `t.row(...)` and `t.object(...)` return schema builders, not TypeScript runtime row types. Let a view callback infer its result, or annotate a separately declared structural type such as `Array<{ sku: bigint; label: string }>`. A named output type must not reuse the generated PascalCase name of its view accessor (for example, reserve `DiscountedProduct` for a `discounted_product` view). +A view context is `ViewCtx` (and `AnonymousViewCtx`), both exported from +`spacetimedb/server`. It carries `sender`, a read-only `db`, and `from`; it is +not a `ReducerCtx`, so a helper shared between a reducer and a view must accept +either: + +```typescript +import type { ReducerCtx, ViewCtx, InferSchema } from 'spacetimedb/server'; +type S = InferSchema; +function stockOf(ctx: ReducerCtx | ViewCtx, itemId: bigint) { ... } +``` + Both `spacetimedb.view(...)` and `spacetimedb.anonymousView(...)` take three arguments: view options, the declared return schema, and the callback. ```typescript @@ -288,7 +307,7 @@ Both `spacetimedb.view(...)` and `spacetimedb.anonymousView(...)` take three arg export const activeUsers = spacetimedb.anonymousView( { name: 'active_users', public: true }, t.array(entity.rowType), - (ctx) => [...ctx.db.entity.iter()].filter(e => e.active) + (ctx) => [...ctx.db.entity.active.filter(true)] // active: t.bool().index('btree') ); // Per-user view (varies by ctx.sender): diff --git a/crates/bindings-typescript/src/lib/query.ts b/crates/bindings-typescript/src/lib/query.ts index bb93b0e6ce3..0c17e3d145a 100644 --- a/crates/bindings-typescript/src/lib/query.ts +++ b/crates/bindings-typescript/src/lib/query.ts @@ -248,19 +248,21 @@ export type NamespacedQueryBuilder = * A runtime reference to a table. This materializes the RowExpr for us. * TODO: Maybe add the full SchemaDef to the type signature depending on how joins will work. */ -export type TableRef = Readonly<{ - type: 'table'; - sourceName: TableDef['sourceName']; - accessorName: string; - cols: RowExpr; - indexedCols: IndexedRowExpr; - tableDef: TableDef; +// Keep this named so TypeScript diagnostics show `TableRef` instead of its +// expanded structure. +export interface TableRef { + readonly type: 'table'; + readonly sourceName: TableDef['sourceName']; + readonly accessorName: string; + readonly cols: RowExpr; + readonly indexedCols: IndexedRowExpr; + readonly tableDef: TableDef; // Delegated UntypedTableDef properties for compatibility. - columns: TableDef['columns']; - indexes: TableDef['indexes']; - rowType: TableDef['rowType']; - constraints: any; -}>; + readonly columns: TableDef['columns']; + readonly indexes: TableDef['indexes']; + readonly rowType: TableDef['rowType']; + readonly constraints: any; +} class TableRefImpl implements TableRef, From diff --git a/crates/bindings-typescript/src/sdk/connection_manager.ts b/crates/bindings-typescript/src/sdk/connection_manager.ts index 211b01b5add..42febd5d221 100644 --- a/crates/bindings-typescript/src/sdk/connection_manager.ts +++ b/crates/bindings-typescript/src/sdk/connection_manager.ts @@ -145,9 +145,7 @@ class ConnectionManagerImpl { clearTimeout(managed.reconnectTimer); managed.reconnectTimer = null; managed.reconnectAttempt = 0; - if (managed.builder) { - this.#buildManagedConnection(managed, managed.builder); - } + this.#reconnectManagedConnection(managed); continue; } @@ -179,9 +177,7 @@ class ConnectionManagerImpl { connection.disconnect(); this.#updateState(managed, { isActive: false }); managed.reconnectAttempt = 0; - if (managed.builder) { - this.#buildManagedConnection(managed, managed.builder); - } + this.#reconnectManagedConnection(managed); } /** Generates a unique key for a connection based on URI and module name. */ @@ -294,6 +290,14 @@ class ConnectionManagerImpl { } } + /** Reconnect with the issued token. Explicit rebuilds use the caller's token. */ + #reconnectManagedConnection(managed: ManagedConnection): void { + if (!managed.builder) return; + const token = managed.state.token; + if (token) managed.builder.withToken(token); + this.#buildManagedConnection(managed, managed.builder); + } + #buildManagedConnection>( managed: ManagedConnection, builder: DbConnectionBuilder @@ -349,7 +353,7 @@ class ConnectionManagerImpl { return; } - this.#buildManagedConnection(managed, managed.builder); + this.#reconnectManagedConnection(managed); }, delay); } diff --git a/crates/bindings-typescript/tests/connection_manager_liveness.test.ts b/crates/bindings-typescript/tests/connection_manager_liveness.test.ts index ea24ea6887e..e9f76a909e9 100644 --- a/crates/bindings-typescript/tests/connection_manager_liveness.test.ts +++ b/crates/bindings-typescript/tests/connection_manager_liveness.test.ts @@ -12,7 +12,7 @@ type ErrorContextInterface = { isActive: boolean }; class MockConnection { isActive = false; identity = undefined; - token = undefined; + token: string | undefined; connectionId = ConnectionId.random(); isDisconnectRequested = false; disconnected = false; @@ -28,6 +28,10 @@ class MockConnection { (ctx: ErrorContextInterface, error: Error) => void >(); + constructor(private readonly issuedToken?: string) { + this.token = undefined; + } + get isSocketClosed(): boolean { return this.socketClosed; } @@ -63,6 +67,7 @@ class MockConnection { simulateConnect(): void { this.isActive = true; + this.token = this.issuedToken; for (const cb of this.#onConnect) cb(this); } simulateDisconnect(error?: Error): void { @@ -74,7 +79,9 @@ class MockConnection { class MockBuilder { buildCount = 0; + presentedTokens: Array = []; connections: MockConnection[] = []; + #token: string | undefined; #onConnect = new Set<(conn: MockConnection) => void>(); #onDisconnect = new Set< @@ -84,9 +91,17 @@ class MockBuilder { (ctx: ErrorContextInterface, error: Error) => void >(); + constructor(private readonly issuedToken?: string) {} + + withToken(token?: string): MockBuilder { + this.#token = token; + return this; + } + build(): MockConnection { - const connection = new MockConnection(); + const connection = new MockConnection(this.issuedToken); this.buildCount += 1; + this.presentedTokens.push(this.#token); this.connections.push(connection); for (const cb of this.#onConnect) connection.register('connect', cb); for (const cb of this.#onDisconnect) connection.register('disconnect', cb); @@ -201,6 +216,32 @@ describe('ConnectionManager liveness recovery', () => { ConnectionManager.release(key); }); + test('reuses the issued token when reviving a dead socket', () => { + const key = nextKey(); + const builder = new MockBuilder('issued-token'); + const first = retain(key, builder); + expect(builder.presentedTokens).toEqual([undefined]); + + first.simulateConnect(); + first.socketClosed = true; + fire('win:online'); + + expect(builder.presentedTokens).toEqual([undefined, 'issued-token']); + ConnectionManager.release(key); + }); + + test('an explicit rebuild uses the caller token', () => { + const key = nextKey(); + const firstBuilder = new MockBuilder('issued-token'); + retain(key, firstBuilder).simulateConnect(); + + const replacement = new MockBuilder().withToken('caller-token'); + ConnectionManager.rebuild(key, replacement as any); + + expect(replacement.presentedTokens).toEqual(['caller-token']); + ConnectionManager.release(key); + }); + test('does not rebuild a healthy connection on resume', () => { const key = nextKey(); const builder = new MockBuilder(); diff --git a/crates/bindings-typescript/tests/table_ref_error_message.test.ts b/crates/bindings-typescript/tests/table_ref_error_message.test.ts new file mode 100644 index 00000000000..009a9c192dd --- /dev/null +++ b/crates/bindings-typescript/tests/table_ref_error_message.test.ts @@ -0,0 +1,79 @@ +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import * as ts from 'typescript'; +import { describe, expect, it } from 'vitest'; + +const bindingsRoot = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + '..' +); + +function runTypecheck(source: string) { + const tmpDir = mkdtempSync(path.join(tmpdir(), 'stdb-tableref-diag-')); + const reproPath = path.join(tmpDir, 'repro.ts'); + writeFileSync(reproPath, source); + + try { + const options: ts.CompilerOptions = { + target: ts.ScriptTarget.ESNext, + module: ts.ModuleKind.ESNext, + strict: true, + noEmit: true, + skipLibCheck: true, + forceConsistentCasingInFileNames: true, + allowImportingTsExtensions: true, + noImplicitAny: true, + moduleResolution: ts.ModuleResolutionKind.Bundler, + useDefineForClassFields: true, + verbatimModuleSyntax: true, + isolatedModules: true, + }; + + const host = ts.createCompilerHost(options); + const program = ts.createProgram( + [reproPath, path.join(bindingsRoot, 'src/server/sys.d.ts')], + options, + host + ); + const diagnostics = ts.getPreEmitDiagnostics(program); + return diagnostics.map(d => + ts.flattenDiagnosticMessageText(d.messageText, '\n') + ); + } finally { + rmSync(tmpDir, { recursive: true, force: true }); + } +} + +describe('TableRef diagnostics', () => { + const source = ` +import { t } from ${JSON.stringify(path.join(bindingsRoot, 'src/server/index.ts'))}; +import { table } from ${JSON.stringify(path.join(bindingsRoot, 'src/lib/table.ts'))}; +import { createTableRefFromDef } from ${JSON.stringify(path.join(bindingsRoot, 'src/lib/query.ts'))}; +import type { AllUnique } from ${JSON.stringify(path.join(bindingsRoot, 'src/lib/constraints.ts'))}; + +const cartItem = table( + { name: 'cart_item' }, + { id: t.u64().primaryKey().autoInc(), accountId: t.u64(), quantity: t.u32() } +); + +const ref = createTableRefFromDef(cartItem as any, 'cartItem'); +type Boom = AllUnique; +declare const b: Boom; +`; + + it('names the type instead of dumping its structure', () => { + const messages = runTypecheck(source); + const constraintError = messages.find(m => + m.includes("does not satisfy the constraint 'UntypedTableDef'") + ); + + expect(constraintError).toBeDefined(); + // The name, not the shape. + expect(constraintError).toContain('TableRef<'); + expect(constraintError).not.toContain('type: "table"'); + expect(constraintError).not.toContain('accessorName'); + expect(constraintError.length).toBeLessThan(250); + }, 15000); +}); diff --git a/crates/cli/build.rs b/crates/cli/build.rs index c5bd4303464..90da9fa3bdd 100644 --- a/crates/cli/build.rs +++ b/crates/cli/build.rs @@ -110,6 +110,7 @@ fn generate_template_files() { // Embed skill files from skills/*/SKILL.md let skills_dir = repo_root.join("skills"); + println!("cargo:rerun-if-changed={}", skills_dir.display()); let skill_names = discover_skill_names(&skills_dir); generated_code.push_str("pub fn get_skill(name: &str) -> Option<&'static str> {\n"); diff --git a/crates/cli/src/subcommands/dev.rs b/crates/cli/src/subcommands/dev.rs index a0588b6de50..a2622c8ea0c 100644 --- a/crates/cli/src/subcommands/dev.rs +++ b/crates/cli/src/subcommands/dev.rs @@ -738,7 +738,7 @@ pub async fn exec(mut config: Config, args: &ArgMatches) -> Result<(), anyhow::E let loaded_config_dir = loaded_config.as_ref().map(|lc| lc.config_dir.clone()); generate_build_and_publish( - &config, + &mut config, &project_dir, loaded_config_dir.as_deref(), &spacetimedb_dir, @@ -853,7 +853,7 @@ pub async fn exec(mut config: Config, args: &ArgMatches) -> Result<(), anyhow::E println!("\n{}", "File change detected, rebuilding...".yellow()); match generate_build_and_publish( - &config, + &mut config, &project_dir, loaded_config_dir.as_deref(), &spacetimedb_dir, @@ -1000,7 +1000,7 @@ fn upsert_env_db_names_and_hosts(env_path: &Path, server_host_url: &str, databas #[allow(clippy::too_many_arguments)] async fn generate_build_and_publish( - config: &Config, + config: &mut Config, project_dir: &Path, config_dir: Option<&Path>, spacetimedb_dir: &Path, @@ -1146,7 +1146,8 @@ async fn generate_build_and_publish( publish_entry.insert("break-clients".to_string(), json!(true)); } - publish::exec_from_entry(config.clone(), publish_entry, config_dir, clear_database, yes).await?; + // Preserve a token created during publish for logs and later rebuilds. + publish::exec_from_entry(config, publish_entry, config_dir, clear_database, yes).await?; } println!("{}", "Published successfully!".green().bold()); diff --git a/crates/cli/src/subcommands/publish.rs b/crates/cli/src/subcommands/publish.rs index 745664880f0..63d704d25af 100644 --- a/crates/cli/src/subcommands/publish.rs +++ b/crates/cli/src/subcommands/publish.rs @@ -450,7 +450,7 @@ pub async fn exec_with_options( } pub async fn exec_from_entry( - mut config: Config, + config: &mut Config, entry: HashMap, config_dir: Option<&std::path::Path>, clear_database: ClearMode, @@ -465,7 +465,7 @@ pub async fn exec_from_entry( let yes = if force { YesFlags::all() } else { YesFlags::default() }; - execute_publish_configs(&mut config, vec![command_config], true, config_dir, clear_database, yes).await + execute_publish_configs(config, vec![command_config], true, config_dir, clear_database, yes).await } async fn execute_publish_configs<'a>( diff --git a/skills/spacetimedb-typescript-core/SKILL.md b/skills/spacetimedb-typescript-core/SKILL.md new file mode 100644 index 00000000000..993a527a452 --- /dev/null +++ b/skills/spacetimedb-typescript-core/SKILL.md @@ -0,0 +1,101 @@ +--- +name: spacetimedb-typescript-core +description: Core SpacetimeDB TypeScript server and client SDK syntax for building an application without framework or architecture guidance. +license: Apache-2.0 +metadata: + author: clockworklabs + version: "1.0" + language: typescript +--- + +# SpacetimeDB TypeScript Core API + +## Server module + +Define tables with `table()`, bind them with `schema()`, and export the schema +as the module default. Export reducers from the same module or its entry file. + +```typescript +import { schema, table, t } from 'spacetimedb/server'; + +const record = table( + { name: 'record', public: true }, + { + id: t.u64().primaryKey().autoInc(), + label: t.string().index('btree'), + value: t.u32(), + }, +); + +const spacetimedb = schema({ record }); +export default spacetimedb; + +export const createRecord = spacetimedb.reducer( + { label: t.string(), value: t.u32() }, + (ctx, { label, value }) => { + ctx.db.record.insert({ id: 0n, label, value }); + }, +); +``` + +Table names must be snake_case. The keys passed to `schema({ ... })` are the +server-side `ctx.db` accessor names. A split module must re-export the schema +as the default export from its entry file. + +## Types and table access + +Common builders are `t.string()`, `t.bool()`, `t.u32()`, `t.i32()`, +`t.u64()`, `t.i64()`, `t.identity()`, `t.timestamp()`, and +`t.option(inner)`. The 64-bit integer builders use TypeScript `bigint` values. +Use `0n` for an auto-increment `u64` or `i64` field during insertion. + +Column modifiers include `.primaryKey()`, `.autoInc()`, `.unique()`, and +`.index('btree')`. + +```typescript +const row = ctx.db.record.id.find(id); // row | null +const inserted = ctx.db.record.insert(values); // inserted row +if (row) ctx.db.record.id.update({ ...row, value: 2 }); // update by primary key +ctx.db.record.id.delete(id); // delete by primary key +const matching = [...ctx.db.record.label.filter(label)]; +const all = [...ctx.db.record.iter()]; +``` + +`iter()` and `filter()` return iterators. Spread them before using array +methods. Insert through the table accessor, not through an index accessor. + +## Generated client bindings + +Generated bindings convert snake_case table, reducer, and field names to +camelCase. A server reducer named `createRecord` is called as `createRecord` in a +TypeScript client. + +Create a connection with the generated `DbConnection`: + +```typescript +import { DbConnection, tables } from './module_bindings'; + +const connection = DbConnection.builder() + .withUri(serverUri) + .withDatabaseName(moduleName) + .onConnect(ctx => { + ctx.subscriptionBuilder() + .onApplied(() => console.log('ready')) + .subscribe([tables.record]); + }) + .build(); +``` + +Call reducers with an object argument: + +```typescript +await connection.reducers.createRecord({ label: 'Example', value: 1 }); +``` + +The generated database accessors support row callbacks: + +```typescript +connection.db.record.onInsert((_ctx, row) => console.log(row.label)); +connection.db.record.onUpdate((_ctx, oldRow, newRow) => console.log(oldRow, newRow)); +connection.db.record.onDelete((_ctx, row) => console.log(row.id)); +``` diff --git a/skills/typescript-client/SKILL.md b/skills/typescript-client/SKILL.md index 3a31183133f..62f86ff36fc 100644 --- a/skills/typescript-client/SKILL.md +++ b/skills/typescript-client/SKILL.md @@ -18,7 +18,7 @@ Generated bindings convert snake_case names to camelCase, including row fields: ## React: main.tsx ```typescript -import React, { useEffect, useMemo } from 'react'; +import React, { useMemo } from 'react'; import ReactDOM from 'react-dom/client'; import { SpacetimeDBProvider } from 'spacetimedb/react'; import { DbConnection } from './module_bindings'; @@ -30,6 +30,7 @@ function Root() { DbConnection.builder() .withUri(SPACETIMEDB_URI) .withDatabaseName(MODULE_NAME) + // Reuse the token issued on the previous connection. .withToken(localStorage.getItem('auth_token') || undefined), [] ); @@ -46,6 +47,7 @@ ReactDOM.createRoot(document.getElementById('root')!).render(); ## React: App.tsx ```typescript +import { useEffect } from 'react'; import { useTable, useSpacetimeDB } from 'spacetimedb/react'; import { DbConnection, tables } from './module_bindings'; @@ -53,7 +55,7 @@ function App() { const { isActive, identity: myIdentity, token, getConnection } = useSpacetimeDB(); const conn = getConnection() as DbConnection | null; - // Save auth token + // Persist the issued token for the next page load. useEffect(() => { if (token) localStorage.setItem('auth_token', token); }, [token]); // Subscribe when connected. Prefer typed query builders over raw SQL diff --git a/skills/typescript-server/SKILL.md b/skills/typescript-server/SKILL.md index c9f2e7343fd..ba24bf781d5 100644 --- a/skills/typescript-server/SKILL.md +++ b/skills/typescript-server/SKILL.md @@ -98,6 +98,10 @@ Every column is a `t` builder value: Modifiers: `.primaryKey()`, `.autoInc()`, `.unique()`, `.index('btree')`, `.default(value)`. +`.primaryKey()` and `.unique()` apply to one column. For uniqueness across +multiple columns, use a surrogate key, the multi-column index below, and a +reducer that rejects an existing index match before inserting. + Use `.default(value)` only for a newly appended migration-safe field. Do not put defaults on primary-key, unique, or auto-increment columns. Optional columns: `nickname: t.option(t.string())` @@ -130,7 +134,9 @@ export { default } from './schema'; // re-export the schema for the module ent ## Reducers -Reducers are created with `spacetimedb.reducer(...)`; the export name becomes the reducer name: +Reducers are created with `spacetimedb.reducer(...)`. An exported `signUp` +becomes `signUp` in generated clients and `sign_up` in `spacetime call` and +`describe`: ```typescript export const createEntity = spacetimedb.reducer( @@ -278,9 +284,22 @@ const Shape = t.enum('Shape', { A client subscribing to a view receives only the rows it returns. Use a per-user view (keyed on `ctx.sender`) for per-viewer access control: deleting a row it depends on (e.g. a membership row) automatically drops the rows it was exposing from that client. +Use index accessors in views. Do not scan a whole table with `.iter()` when an +indexed lookup can select the required rows. `t.row(...)` and `t.object(...)` return schema builders, not TypeScript runtime row types. Let a view callback infer its result, or annotate a separately declared structural type such as `Array<{ sku: bigint; label: string }>`. A named output type must not reuse the generated PascalCase name of its view accessor (for example, reserve `DiscountedProduct` for a `discounted_product` view). +A view context is `ViewCtx` (and `AnonymousViewCtx`), both exported from +`spacetimedb/server`. It carries `sender`, a read-only `db`, and `from`; it is +not a `ReducerCtx`, so a helper shared between a reducer and a view must accept +either: + +```typescript +import type { ReducerCtx, ViewCtx, InferSchema } from 'spacetimedb/server'; +type S = InferSchema; +function stockOf(ctx: ReducerCtx | ViewCtx, itemId: bigint) { ... } +``` + Both `spacetimedb.view(...)` and `spacetimedb.anonymousView(...)` take three arguments: view options, the declared return schema, and the callback. ```typescript @@ -288,7 +307,7 @@ Both `spacetimedb.view(...)` and `spacetimedb.anonymousView(...)` take three arg export const activeUsers = spacetimedb.anonymousView( { name: 'active_users', public: true }, t.array(entity.rowType), - (ctx) => [...ctx.db.entity.iter()].filter(e => e.active) + (ctx) => [...ctx.db.entity.active.filter(true)] // active: t.bool().index('btree') ); // Per-user view (varies by ctx.sender): diff --git a/tools/llm-sequential-upgrade/.gitignore b/tools/llm-sequential-upgrade/.gitignore index 14aa619a63d..35223d12b8f 100644 --- a/tools/llm-sequential-upgrade/.gitignore +++ b/tools/llm-sequential-upgrade/.gitignore @@ -27,4 +27,4 @@ telemetry/metrics.jsonl **/telemetry/**/metadata.json # Sequential-upgrade run output lives in the external spacetimedb-ai-test-results repo -sequential-upgrade/sequential-upgrade-*/ +sequential-upgrade/ diff --git a/tools/llm-sequential-upgrade/read-guard.sh b/tools/llm-sequential-upgrade/read-guard.sh new file mode 100644 index 00000000000..314c52e7cde --- /dev/null +++ b/tools/llm-sequential-upgrade/read-guard.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# Write Claude Code settings that deny direct Read tool access to benchmark +# internals. Bash is allowed, so this is not filesystem isolation. + +write_read_guard() { + local app_dir="$1" backend="$2" out siblings="" + out="$app_dir/.read-guard-settings.json" + + local b + for b in spacetime postgres mongodb; do + [[ "$b" == "$backend" ]] && continue + siblings+=" \"Read(**/$b/results/**)\", +" + done + + cat > "$out" < +campaign inspect +campaign report +``` + +- `status` is the compact normal view. +- `inspect` adds score, cost, duration, cleanup, evidence, and feature progress. +- `report` rebuilds `report/report.json` and `report/report.html` from retained + evidence. + +Do not infer state from logs. Use logs only to diagnose a reported phase or +failure. The controller never retries, extends, or grants paid work +automatically. + +## Resume and repair + +If the controller stopped while an attempt remained live, reconcile ownership +before any resume: + +```sh +campaign reconcile --out +``` + +Reconciliation changes state only when private supervisor evidence proves that +the exact owned resources are clean. + +Dependency campaigns can grant more repairs to selected exhausted features: + +```sh +campaign grant-repairs \ + --attempt --grant-id --level \ + --feature --repairs +``` + +The grant creates a linked continuation. It does not rewrite the completed +execution. Use `campaign resume --out ` to +run scheduled dependency work. + +## Model-free trials and qualification + +`campaign trial` accepts only registered non-billable adapters and zero pricing. +It validates orchestration but does not produce comparative model data. + +Check qualification requirements without starting work: + +```sh +docker compose --env-file /var/lib/stack-bench/operator.env \ + -f appliance/docker-compose.yaml run --rm controller \ + qualification status --track ecommerce --level +``` + +Run only evidence required by that exact status. Do not repeat reference, +mutation, or null work when its bound inputs have not changed. See the +[reference app guide](../reference-apps/README.md) and +[grader guide](../grader/README.md) for qualification rules. + +## Dashboard + +Start the optional dashboard: + +```sh +docker compose --env-file /var/lib/stack-bench/operator.env \ + -f appliance/docker-compose.yaml --profile dashboard up -d dashboard +``` + +Open `http://127.0.0.1:7331`. The dashboard reads and controls the same campaign +state as the CLI. See [dashboard/README.md](../dashboard/README.md). + +## Results and cleanup + +Results remain under `/var/lib/stack-bench/results` after the controller exits. +Verify and copy the complete campaign package before deleting the runner. + +A run removes only resources whose private ownership evidence still matches. +If cleanup cannot be proved, it preserves the evidence and quarantines the run. +Follow [RECOVERY.md](RECOVERY.md). Do not delete same-name resources or clear the +shared state root by guesswork. diff --git a/tools/stack-bench/appliance/RECOVERY.md b/tools/stack-bench/appliance/RECOVERY.md new file mode 100644 index 00000000000..ecf413d98af --- /dev/null +++ b/tools/stack-bench/appliance/RECOVERY.md @@ -0,0 +1,67 @@ +# Interruption and recovery + +Stack Bench never guesses that a container, listener, lock, database, or data +directory is safe to delete. Normal teardown authenticates the run's private +lease, compares exact container IDs and listener PIDs, and releases only locks +whose owner record still matches that lease. + +Every appliance run keeps two different records: + +- `results/.../recovery.json` is public, contains no ownership token, and says + whether cleanup is `clean`, intentionally `retained`, or `quarantined`; +- `controller-home/supervisor/.json` is private recovery authority. It + contains the lease token and must remain readable only by the appliance + operator. Normal cleanup deletes it. Refused cleanup deliberately preserves + it. + +## If a run is interrupted + +1. Preserve the result directory and private supervisor-state file. +2. Read `recovery.json`. Do not publish an attempt whose status is + `quarantined`. +3. Do not start another run using any lock key listed in that artifact. +4. Retry authenticated cleanup from the controller: + +```sh +docker compose --env-file /var/lib/stack-bench/operator.env \ + -f appliance/docker-compose.yaml run --rm controller \ + recover /var/lib/stack-bench/controller-home/supervisor/.json +``` + +On success the command changes `recovery.json` to `clean`, releases the exact +owned resources, and removes the private supervisor state. It is idempotent +when public lease evidence already proves that an earlier cleanup completed. + +If the parent process ended before it retained a supervisor file, recover from +the private runtime lease instead. Supply a durable output directory outside +the private runtime directory: + +```sh +docker compose --env-file /var/lib/stack-bench/operator.env \ + -f appliance/docker-compose.yaml run --rm controller \ + recover-lease /var/lib/stack-bench/controller-home/runtime//backend-lease.json \ + --out /var/lib/stack-bench/results/recovery/ +``` + +This path uses the same ownership token, container ID, listener PID, and lock +checks. It refuses an output directory inside the runtime directory because a +successful recovery removes that directory. + +## If recovery refuses + +Refusal is the safety behavior. It means a live resource does not match the +lease or its identity could not be proven. The command leaves the private state, +lease, lock records, and public quarantine artifact intact. + +Compare the live container ID and listener PIDs with `recovery.json` and the +private lease before manual action. Never delete a same-name container, kill a +port's current listener, remove another lock, or recursively clear the shared +state root merely because its name resembles Stack Bench. Escalate with the +complete result directory and private state stored separately from public +artifacts. + +## Intentional retention + +`--retain-backend` is inspection mode, not successful cleanup. It writes +`status: "retained"` and preserves private recovery authority. No other run may +reuse the listed locks until the recovery command completes. diff --git a/tools/stack-bench/appliance/RELEASE.md b/tools/stack-bench/appliance/RELEASE.md new file mode 100644 index 00000000000..23b34502679 --- /dev/null +++ b/tools/stack-bench/appliance/RELEASE.md @@ -0,0 +1,121 @@ +# Release assembly and verification + +Stack Bench uses two deliberately different release states. + +- A `candidate` has exact image digests, checksummed files, and digest-bound + SPDX SBOMs. It is useful for inspecting and testing a proposed bundle, but it + is unsigned and cannot be called qualified. +- A `qualified` release adds a bundled public key, a detached Sigstore bundle + covering `release.json`, and registry signatures for every image. Verification + must use a public key obtained outside the release bundle. + +Schema v2 is the only accepted release format. + +## Build the controller image + +Build the Linux/amd64 controller from a clean checkout. The source identity +command refuses changed or untracked release inputs. + +```powershell +$source = npm --prefix tools/stack-bench run release:source --silent | ConvertFrom-Json +docker build --platform linux/amd64 ` + -f tools/stack-bench/appliance/Controller.Dockerfile ` + --build-arg SOURCE_REVISION=$($source.revision) ` + --build-arg SOURCE_SHA256=$($source.sha256) ` + --build-arg BINARY_SOURCE_SHA256=$($source.binarySourceSha256) ` + -t stack-bench-controller:development . +``` + +The build accepts only Linux SpacetimeDB binaries recorded in +`container/spacetimedb-binaries.json`. Rebuild them with +`bash tools/stack-bench/container/build-linux-cli.sh` after a recorded binary +source changes. Review and commit the updated provenance file. The binary files +remain ignored. + +## Build a candidate + +Publish the first-party images, resolve every first- and third-party image to an +exact single-platform `linux/amd64` manifest reference, then generate one SPDX +SBOM for each exact reference. Do not use a multi-architecture index digest: +Docker Scout correctly reports the selected child-manifest digest, so an index +digest cannot satisfy the one-image/one-SBOM identity contract. + +```sh +node dist/src/releases/release-bundle.js sbom registry.example/controller@sha256:DIGEST \ + --output bundle/sbom/controller.spdx.json +``` + +The command uses registry resolution, refuses mutable references and existing +output, and checks that Docker Scout's SPDX 2.3 document contains the requested +image digest. A successful tool exit without that digest binding is rejected. + +Create a strict release specification with `state: "candidate"`, +`signing: null`, and `files` entries containing only `path` and `role`. Place +every input below the bundle root, then materialize immutable size and SHA-256 +metadata: + +```sh +node dist/src/releases/release-bundle.js assemble release-spec.json \ + --root bundle --output bundle/release.json +node dist/src/releases/release-manifest.js verify bundle/release.json --root bundle +``` + +Candidate verification reports `candidate-file-integrity`. It validates all +declared files and all four image-to-SBOM digest bindings. Candidate manifests +must use `signing: null` and cannot include a public signing key. + +## Sign and qualify + +Signing keys are external CI inputs. Never copy a private key, registry token, +or signing password into the source tree, image, bundle, Compose environment, +or command transcript. Sign each exact registry image with Cosign. The +authoritative image-signature evidence stays attached to the registry object +and is checked directly during verification; the release does not preserve a +redundant unverified export. Add the public half of the signing key as +`signing/cosign.pub` with the `public-key` role. + +Change the specification to `state: "qualified"` and declare: + +```json +{ + "signing": { + "scheme": "cosign-public-key-v1", + "publicKeyPath": "signing/cosign.pub", + "manifestBundlePath": "signing/release-manifest.sigstore.json" + } +} +``` + +Assemble `release.json` only after all other evidence exists, then sign that +exact file with a detached Cosign bundle: + +```sh +cosign sign-blob --yes --key "$COSIGN_KEY" \ + --bundle bundle/signing/release-manifest.sigstore.json bundle/release.json +``` + +The detached bundle is intentionally not checksummed by `release.json`: a file +cannot contain the hash of its own signature. Cosign authenticates it instead. + +Verify with the trusted public key copied to a path outside the downloaded +bundle: + +```sh +node dist/src/releases/release-manifest.js verify bundle/release.json --root bundle \ + --trusted-key /operator/trust/stack-bench-cosign.pub +``` + +Qualified verification refuses an absent or bundle-local trust key, requires +it to equal the public key bound by the signed manifest, verifies the detached +manifest signature, and runs `cosign verify` against every exact registry image +reference. A failed or unavailable Cosign invocation is a failed release; there +is no downgrade to candidate verification. The controller image includes +checksum-pinned Cosign 3.1.3 so this command is available in the delivered +appliance rather than depending on an untracked host installation. + +## Trust distribution + +The release bundle cannot establish trust in its own key. Publish the expected +public key and its SHA-256 fingerprint through a separately controlled channel. +The operator must compare that fingerprint before verification. Key rotation +requires a new release and an explicit trust-distribution update. diff --git a/tools/stack-bench/appliance/campaign.ecommerce-progression-reference.json b/tools/stack-bench/appliance/campaign.ecommerce-progression-reference.json new file mode 100644 index 00000000000..01a584ee447 --- /dev/null +++ b/tools/stack-bench/appliance/campaign.ecommerce-progression-reference.json @@ -0,0 +1,96 @@ +{ + "schemaVersion": 7, + "kind": "campaign-manifest", + "id": "ecommerce-progression-reference", + "version": "2.0.1", + "state": "draft", + "title": "Ecommerce progression reference pilot", + "track": "ecommerce", + "mode": { "id": "dependency", "version": "4.3.0", "workSelection": "progressive" }, + "repair": { "selection": "feature", "budget": { "total": 0 } }, + "levels": [1, 2, 3, 4, 5, 6], + "featureCatalog": "ecommerce.questlines@2.0.2", + "selection": { + "levels": [ + { "level": 1, "recipe": "ecommerce.progression-catalog@2.0.2" }, + { "level": 2, "recipe": "ecommerce.progression-catalog@2.0.2" }, + { "level": 3, "recipe": "ecommerce.progression-catalog@2.0.2" }, + { "level": 4, "recipe": "ecommerce.progression-catalog@2.0.2" }, + { "level": 5, "recipe": "ecommerce.progression-catalog@2.0.2" }, + { "level": 6, "recipe": "ecommerce.progression-catalog@2.0.2" } + ] + }, + "stacks": [ + { "id": "mongodb", "adapterVersion": "1.4.0" }, + { "id": "postgres", "adapterVersion": "1.5.0" }, + { "id": "spacetime", "adapterVersion": "1.3.0" } + ], + "agents": [ + { + "adapter": "reference-fixture", + "adapterVersion": "1.4.0", + "model": "reference-fixture" + } + ], + "conditions": [ + { + "id": "reference-pilot", + "version": "1.0.0", + "guidanceProfile": "neutral@1.8.0", + "repairPolicy": "scored-only@1.1.0" + } + ], + "repetitions": 1, + "parallelism": 1, + "ordering": { + "method": "balanced-rotation", + "seed": "ecommerce-progression-reference-1" + }, + "budgets": { + "attemptTimeoutMinutes": 180, + "maxCostUsdPerAttempt": null + }, + "attemptPolicy": { + "retries": 0, + "retryOn": [], + "excludeFromAnalysis": [ + "contaminated", + "harness_failure", + "inconclusive", + "ungraded" + ] + }, + "runtime": { + "releaseManifestSha256": null, + "controllerImage": null, + "buildImage": null, + "platform": "linux/amd64" + }, + "pricing": { + "currency": "USD", + "unit": "USD-per-million-tokens", + "capturedAt": "2026-08-25T00:00:00.000Z", + "source": "Reference fixtures make no provider calls.", + "models": { + "reference-fixture": { + "input": 0, + "output": 0, + "cacheWrite5m": 0, + "cacheWrite1h": 0, + "cacheRead": 0 + } + } + }, + "analysis": { + "primaryMetric": "finalScoreRate", + "secondaryMetrics": [ + "firstBuildScoreRate", + "totalDurationMs", + "invalidAttemptRate" + ], + "dispersion": "median-iqr", + "invalidAttempts": "report-separately", + "missingData": "no-imputation", + "comparisonUnit": "stack-agent-condition-recipe" + } +} diff --git a/tools/stack-bench/appliance/campaign.example.json b/tools/stack-bench/appliance/campaign.example.json new file mode 100644 index 00000000000..95d974b7002 --- /dev/null +++ b/tools/stack-bench/appliance/campaign.example.json @@ -0,0 +1,115 @@ +{ + "schemaVersion": 7, + "kind": "campaign-manifest", + "id": "ecommerce-l1-example", + "version": "2.0.0", + "state": "draft", + "title": "Ecommerce L1 model-free example", + "track": "ecommerce", + "mode": { "id": "sequential", "version": "1.0.0" }, + "repair": { "selection": "batch", "budget": { "total": 3 } }, + "levels": [1], + "selection": { + "levels": [ + { + "level": 1, + "recipe": "ecommerce.sequential-l1@2.5.0", + "features": [ + "ecommerce.feature.accounts", + "ecommerce.feature.cart-checkout", + "ecommerce.feature.catalog", + "ecommerce.feature.purchasing", + "ecommerce.feature.reviews", + "ecommerce.feature.warehouse-admin" + ], + "checks": [] + } + ] + }, + "stacks": [ + { "id": "spacetime", "adapterVersion": "1.3.0" }, + { "id": "postgres", "adapterVersion": "1.5.0" }, + { "id": "mongodb", "adapterVersion": "1.4.0" } + ], + "agents": [ + { + "adapter": "deterministic", + "adapterVersion": "1.3.0", + "model": "deterministic" + } + ], + "conditions": [ + { + "id": "prescribed", + "version": "1.1.0", + "guidanceProfile": "prescribed@1.2.0", + "repairPolicy": "scored-only@1.1.0", + "specifications": { + "levels": [ + { + "level": 1, + "requested": [ + "ecommerce.spec.access-control@1.2.0", + "ecommerce.spec.concurrency-safety@1.3.0", + "ecommerce.spec.external-data-sync@1.1.0", + "ecommerce.spec.live-state@1.2.0", + "ecommerce.spec.state-durability@1.1.0", + "ecommerce.spec.transactional-integrity@1.3.0" + ], + "expected": [], + "observed": [] + } + ] + } + } + ], + "repetitions": 3, + "parallelism": 1, + "ordering": { + "method": "balanced-rotation", + "seed": "replace-before-measurement" + }, + "budgets": { + "attemptTimeoutMinutes": 240, + "maxCostUsdPerAttempt": null + }, + "attemptPolicy": { + "retries": 1, + "retryOn": ["provider_failure"], + "excludeFromAnalysis": ["contaminated", "harness_failure", "inconclusive", "ungraded"] + }, + "runtime": { + "releaseManifestSha256": null, + "controllerImage": null, + "buildImage": null, + "platform": "linux/amd64" + }, + "pricing": { + "currency": "USD", + "unit": "USD-per-million-tokens", + "capturedAt": "2026-08-12T00:00:00.000Z", + "source": "deterministic adapter makes no billable provider calls", + "models": { + "deterministic": { + "input": 0, + "output": 0, + "cacheWrite5m": 0, + "cacheWrite1h": 0, + "cacheRead": 0 + } + } + }, + "analysis": { + "primaryMetric": "firstBuildScoreRate", + "secondaryMetrics": [ + "finalScoreRate", + "totalCostUsd", + "totalDurationMs", + "invalidAttemptRate" + ], + "dispersion": "median-iqr", + "invalidAttempts": "report-separately", + "missingData": "no-imputation", + "comparisonUnit": "stack-agent-condition-recipe" + } +} diff --git a/tools/stack-bench/appliance/campaign.product-brief-reference.json b/tools/stack-bench/appliance/campaign.product-brief-reference.json new file mode 100644 index 00000000000..a8f7d37359c --- /dev/null +++ b/tools/stack-bench/appliance/campaign.product-brief-reference.json @@ -0,0 +1,114 @@ +{ + "schemaVersion": 7, + "kind": "campaign-manifest", + "id": "ecommerce-l1-product-brief-reference", + "version": "2.0.0", + "state": "draft", + "title": "Ecommerce L1 product brief and quality validation", + "track": "ecommerce", + "mode": { "id": "sequential", "version": "1.0.0" }, + "repair": { "selection": "batch", "budget": { "total": 0 } }, + "levels": [1], + "selection": { + "levels": [ + { + "level": 1, + "recipe": "ecommerce.sequential-l1@2.5.0", + "features": [ + "ecommerce.feature.accounts", + "ecommerce.feature.cart-checkout", + "ecommerce.feature.catalog", + "ecommerce.feature.purchasing", + "ecommerce.feature.reviews", + "ecommerce.feature.warehouse-admin" + ], + "checks": [] + } + ] + }, + "stacks": [ + { "id": "spacetime", "adapterVersion": "1.3.0" }, + { "id": "postgres", "adapterVersion": "1.5.0" }, + { "id": "mongodb", "adapterVersion": "1.4.0" } + ], + "agents": [ + { + "adapter": "reference-fixture", + "adapterVersion": "1.4.0", + "model": "reference-fixture" + } + ], + "conditions": [ + { + "id": "product-brief-quality", + "version": "1.2.0", + "guidanceProfile": "neutral@1.8.0", + "repairPolicy": "scored-only@1.1.0", + "specifications": { + "levels": [ + { + "level": 1, + "requested": [], + "expected": [ + "ecommerce.spec.access-control@1.2.0", + "ecommerce.spec.concurrency-safety@1.3.0", + "ecommerce.spec.external-data-sync@1.1.0", + "ecommerce.spec.live-state@1.2.0", + "ecommerce.spec.state-durability@1.1.0", + "ecommerce.spec.transactional-integrity@1.3.0" + ], + "observed": [] + } + ] + } + } + ], + "repetitions": 2, + "ordering": { + "method": "balanced-rotation", + "seed": "product-brief-quality-reference-1" + }, + "budgets": { + "attemptTimeoutMinutes": 60, + "maxCostUsdPerAttempt": null + }, + "attemptPolicy": { + "retries": 1, + "retryOn": ["harness_failure", "inconclusive"], + "excludeFromAnalysis": ["contaminated", "harness_failure", "inconclusive", "ungraded"] + }, + "runtime": { + "releaseManifestSha256": null, + "controllerImage": null, + "buildImage": null, + "platform": "linux/amd64" + }, + "pricing": { + "currency": "USD", + "unit": "USD-per-million-tokens", + "capturedAt": "2026-08-16T00:00:00.000Z", + "source": "reference fixture adapter makes no billable provider calls", + "models": { + "reference-fixture": { + "input": 0, + "output": 0, + "cacheWrite5m": 0, + "cacheWrite1h": 0, + "cacheRead": 0 + } + } + }, + "analysis": { + "primaryMetric": "firstBuildScoreRate", + "secondaryMetrics": [ + "finalScoreRate", + "totalCostUsd", + "totalDurationMs", + "invalidAttemptRate" + ], + "dispersion": "median-iqr", + "invalidAttempts": "report-separately", + "missingData": "no-imputation", + "comparisonUnit": "stack-agent-condition-recipe" + } +} diff --git a/tools/stack-bench/appliance/controller.ts b/tools/stack-bench/appliance/controller.ts new file mode 100644 index 00000000000..6ee5da5fa91 --- /dev/null +++ b/tools/stack-bench/appliance/controller.ts @@ -0,0 +1,151 @@ +#!/usr/bin/env node + +import { spawn } from 'node:child_process'; +import { join } from 'node:path'; +import { pathToFileURL } from 'node:url'; + +import { STACK_BENCH_ROOT } from '../src/package-root.js'; + +const RUNTIME_ROOT = join(STACK_BENCH_ROOT, 'dist'); + +const COMMANDS = Object.freeze({ + 'init-deps': [join(RUNTIME_ROOT, 'appliance', 'dependency-volume.js'), 'init'], + 'verify-deps': [join(RUNTIME_ROOT, 'appliance', 'dependency-volume.js'), 'verify'], + 'preflight': [join(RUNTIME_ROOT, 'commands', 'preflight.js')], + 'qualify-reference': [join(RUNTIME_ROOT, 'src', 'references', 'reference-live.js')], + 'qualify-null': [join(RUNTIME_ROOT, 'commands', 'null-control.js')], + 'qualification': [join(RUNTIME_ROOT, 'commands', 'qualification-cli.js')], + 'pack-budget': [join(RUNTIME_ROOT, 'commands', 'pack-budget.js')], + 'campaign': [join(RUNTIME_ROOT, 'commands', 'campaign-cli.js')], + 'dashboard': [join(RUNTIME_ROOT, 'dashboard', 'dashboard-server.js')], + 'repair': [join(RUNTIME_ROOT, 'commands', 'repair-cli.js')], + 'run': [join(RUNTIME_ROOT, 'commands', 'bench.js')], + 'verify-release': [join(RUNTIME_ROOT, 'src', 'releases', 'release-manifest.js'), 'verify'], + 'recover': [join(RUNTIME_ROOT, 'commands', 'recovery.js'), 'recover'], + 'recover-lease': [join(RUNTIME_ROOT, 'commands', 'recovery.js'), 'recover-lease'], +} satisfies Record); + +const COMMANDS_REQUIRING_AGENT_AUTH = new Set(['preflight', 'dashboard', 'run']); + +export function controllerCommandRequiresAgentAuth(command: string | undefined, + args: string[] = []): boolean { + if (command && COMMANDS_REQUIRING_AGENT_AUTH.has(command)) return true; + return command === 'campaign' && args[0] === 'run'; +} + +export interface ResolvedControllerCommand { + executable: string; + args: string[]; +} + +export function resolveControllerCommand(argv: string[]): ResolvedControllerCommand | null { + const [command, ...rest] = argv; + if (!command || command === '--help' || command === 'help') return null; + if (!Object.hasOwn(COMMANDS, command)) { + throw new Error(`unknown controller command ${JSON.stringify(command)}`); + } + return { executable: process.execPath, + args: [...COMMANDS[command as keyof typeof COMMANDS], ...rest] }; +} + +export function controllerChildEnvironment(source: NodeJS.ProcessEnv = process.env, + { requireAgentAuth = true }: { requireAgentAuth?: boolean } = {}): NodeJS.ProcessEnv { + const env = { ...source }; + delete env.ANTHROPIC_API_KEY; + delete env.ANTHROPIC_API_KEY_FILE; + delete env.CLAUDE_CODE_OAUTH_TOKEN; + delete env.CLAUDE_CODE_OAUTH_TOKEN_FILE; + if (!requireAgentAuth) return env; + const mode = source.STACK_BENCH_AGENT_AUTH ?? 'subscription-token'; + if (!['subscription-token', 'api-key'].includes(mode)) { + throw new Error('STACK_BENCH_AGENT_AUTH must be subscription-token or api-key'); + } + if (mode === 'api-key') { + const path = source.STACK_BENCH_ANTHROPIC_API_KEY_FILE?.trim(); + if (!path) throw new Error('api-key auth requires STACK_BENCH_ANTHROPIC_API_KEY_FILE'); + env.ANTHROPIC_API_KEY_FILE = path; + } else { + const path = source.STACK_BENCH_CLAUDE_OAUTH_TOKEN_FILE?.trim(); + if (!path) { + throw new Error('subscription-token auth requires STACK_BENCH_CLAUDE_OAUTH_TOKEN_FILE'); + } + env.CLAUDE_CODE_OAUTH_TOKEN_FILE = path; + } + return env; +} + +interface SignalChild { + kill(signal: NodeJS.Signals): unknown; +} + +interface SignalSource { + on(signal: NodeJS.Signals, listener: () => void): unknown; + off(signal: NodeJS.Signals, listener: () => void): unknown; +} + +export function forwardControllerSignals(child: SignalChild, + source: SignalSource = process): () => void { + const signals: NodeJS.Signals[] = ['SIGINT', 'SIGTERM']; + const listeners = new Map void>(signals.map(signal => + [signal, () => { child.kill(signal); }])); + for (const [signal, listener] of listeners) source.on(signal, listener); + return () => { + for (const [signal, listener] of listeners) source.off(signal, listener); + }; +} + +function help(): void { + process.stdout.write('Stack Bench controller\n\n' + + 'Commands:\n' + + ' preflight verify the runner without a model call\n' + + ' qualify-reference run a pristine or mutation reference gate\n' + + ' --mutation-workers N split one mutation gate across 1 to 8 isolated workers\n' + + ' qualify-null run the exact null-oracle gate\n' + + ' qualification status show exact launch and promotion blockers\n' + + ' pack-budget recommend derive reviewable bounds from exact reference evidence\n' + + ' campaign validate|show compile the exact comparison plan without running it\n' + + ' campaign trial --out exercise a model-free draft\n' + + ' campaign run --out start a campaign\n' + + ' campaign reconcile --out prove cleanup for interrupted work\n' + + ' campaign status inspect exact durable campaign state\n' + + ' campaign report regenerate deterministic JSON and static HTML\n' + + ' dashboard [--port N] serve the local operator dashboard\n' + + ' repair status --level N inspect whether a failed level can continue\n' + + ' repair grant --level N --repairs N add one finite repair budget\n' + + ' run execute and retain one requested run\n' + + ' verify-release verify candidate files or a qualified signed release\n' + + ' recover retry authenticated cleanup or retain quarantine\n' + + ' recover-lease --out recover when parent state was not retained\n' + + ' init-deps | verify-deps initialize or verify the release dependency volume\n'); +} + +interface ChildOutcome { + code: number | null; + signal: NodeJS.Signals | null; +} + +async function main(argv: string[]): Promise { + const command = argv[2]; + const resolved = resolveControllerCommand(argv.slice(2)); + if (!resolved) { help(); return; } + const child = spawn(resolved.executable, resolved.args, + { stdio: 'inherit', env: controllerChildEnvironment(process.env, + { requireAgentAuth: controllerCommandRequiresAgentAuth(command, argv.slice(3)) }) }); + const stopForwardingSignals = forwardControllerSignals(child); + let outcome: ChildOutcome; + try { + outcome = await new Promise((resolveExit, reject) => { + child.once('error', reject); + child.once('exit', (code, signal) => { resolveExit({ code, signal }); }); + }); + } finally { stopForwardingSignals(); } + if (outcome.signal) process.kill(process.pid, outcome.signal); + process.exitCode = outcome.code ?? 1; +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) { + main(process.argv).catch((error: unknown) => { + console.error(`stack-bench-controller: ${error instanceof Error ? error.message : String(error)}`); + process.exitCode = 2; + }); +} diff --git a/tools/stack-bench/appliance/dependency-volume.ts b/tools/stack-bench/appliance/dependency-volume.ts new file mode 100644 index 00000000000..e6f49e55a86 --- /dev/null +++ b/tools/stack-bench/appliance/dependency-volume.ts @@ -0,0 +1,165 @@ +#!/usr/bin/env node + +import { createHash } from 'node:crypto'; +import { + chmodSync, copyFileSync, existsSync, lstatSync, mkdirSync, readFileSync, readdirSync, + renameSync, rmSync, writeFileSync, +} from 'node:fs'; +import { dirname, join, relative, resolve, sep } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { parseArgs } from 'node:util'; + +const MARKER = '.stack-bench-release-deps.json'; + +export interface DependencyManifestFile { + path: string; + size: number; + mode: number; + sha256: string; +} + +export interface DependencyManifest { + schemaVersion: 1; + files: DependencyManifestFile[]; +} + +interface DependencyVerification { + manifestSha256: string; + files: number; +} + +interface DependencyInitialization extends DependencyVerification { + initialized: boolean; +} + +function sha256Bytes(bytes: string | NodeJS.ArrayBufferView): string { + return createHash('sha256').update(bytes).digest('hex'); +} + +function normalizedRelative(root: string, path: string): string { + const value = relative(root, path).split(sep).join('/'); + if (!value || value.startsWith('../') || value === '..') throw new Error(`path escapes dependency root: ${path}`); + return value; +} + +function walk(root: string, current = root): string[] { + const files: string[] = []; + for (const entry of readdirSync(current, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))) { + const path = join(current, entry.name); + if (entry.isSymbolicLink()) throw new Error(`dependency tree cannot contain symlinks: ${normalizedRelative(root, path)}`); + if (entry.isDirectory()) files.push(...walk(root, path)); + else if (entry.isFile()) files.push(path); + else throw new Error(`dependency tree contains unsupported entry: ${normalizedRelative(root, path)}`); + } + return files; +} + +export function createDependencyManifest(root: string): DependencyManifest { + const absolute = resolve(root); + if (!existsSync(absolute) || !lstatSync(absolute).isDirectory()) { + throw new Error(`dependency source is not a directory: ${absolute}`); + } + const files = walk(absolute).map(path => { + const bytes = readFileSync(path); + return { path: normalizedRelative(absolute, path), size: bytes.length, + mode: lstatSync(path).mode & 0o777, sha256: sha256Bytes(bytes) }; + }); + if (!files.length) throw new Error('dependency source is empty'); + return { schemaVersion: 1, files }; +} + +export function manifestSha256(manifest: DependencyManifest): string { + return sha256Bytes(`${JSON.stringify(manifest)}\n`); +} + +export function verifyDependencyTree(root: string, manifest: DependencyManifest, + { allowMarker = false }: { allowMarker?: boolean } = {}): DependencyVerification { + if (!manifest || manifest.schemaVersion !== 1 || !Array.isArray(manifest.files) || !manifest.files.length) { + throw new Error('dependency manifest is invalid'); + } + const absolute = resolve(root); + const actual = createDependencyManifest(absolute); + if (allowMarker) actual.files = actual.files.filter(file => file.path !== MARKER); + if (JSON.stringify(actual.files) !== JSON.stringify(manifest.files)) { + throw new Error(`dependency tree does not match manifest ${manifestSha256(manifest)}`); + } + return { manifestSha256: manifestSha256(manifest), files: manifest.files.length }; +} + +export function initializeDependencyVolume({ source, target, manifest }: + { source: string; target: string; manifest: DependencyManifest }): DependencyInitialization { + const sourceRoot = resolve(source); + const targetRoot = resolve(target); + const verified = verifyDependencyTree(sourceRoot, manifest); + mkdirSync(targetRoot, { recursive: true, mode: 0o755 }); + const markerPath = join(targetRoot, MARKER); + const existing = readdirSync(targetRoot); + if (existing.length) { + if (!existsSync(markerPath)) throw new Error('dependency volume is non-empty but has no release marker'); + const marker = JSON.parse(readFileSync(markerPath, 'utf8')); + if (marker.schemaVersion !== 1 || marker.manifestSha256 !== verified.manifestSha256) { + throw new Error('dependency volume belongs to a different release'); + } + verifyDependencyTree(targetRoot, manifest, { allowMarker: true }); + return { ...verified, initialized: false }; + } + + const staging = join(targetRoot, `.staging-${process.pid}`); + mkdirSync(staging, { mode: 0o700 }); + try { + for (const file of manifest.files) { + const from = join(sourceRoot, ...file.path.split('/')); + const to = join(staging, ...file.path.split('/')); + mkdirSync(dirname(to), { recursive: true }); + copyFileSync(from, to); + chmodSync(to, file.mode); + } + for (const entry of readdirSync(staging)) renameSync(join(staging, entry), join(targetRoot, entry)); + rmSync(staging, { recursive: true, force: true }); + writeFileSync(markerPath, `${JSON.stringify({ schemaVersion: 1, + manifestSha256: verified.manifestSha256 })}\n`, { flag: 'wx', mode: 0o444 }); + verifyDependencyTree(targetRoot, manifest, { allowMarker: true }); + return { ...verified, initialized: true }; + } catch (error) { + rmSync(staging, { recursive: true, force: true }); + throw error; + } +} + +function main(argv: string[]): void { + const { values, positionals } = parseArgs({ args: argv.slice(2), allowPositionals: true, + options: { + source: { type: 'string', default: '/opt/stack-bench-embedded-deps' }, + target: { type: 'string', default: '/opt/stack-bench-release-deps' }, + manifest: { type: 'string', default: '/opt/stack-bench/dependency-manifest.json' }, + out: { type: 'string' }, + } }); + const [command] = positionals; + const source = values.source; + const target = values.target; + const manifestPath = values.manifest; + if (command === 'manifest') { + const output = values.out; + if (!output) throw new Error('manifest requires --out'); + writeFileSync(resolve(output), `${JSON.stringify(createDependencyManifest(source), null, 2)}\n`, { flag: 'wx' }); + return; + } + const manifest: DependencyManifest = JSON.parse(readFileSync(resolve(manifestPath), 'utf8')); + if (command === 'init') { + process.stdout.write(`${JSON.stringify(initializeDependencyVolume({ source, target, manifest }))}\n`); + return; + } + if (command === 'verify') { + process.stdout.write(`${JSON.stringify(verifyDependencyTree(target, manifest, { allowMarker: true }))}\n`); + return; + } + throw new Error('usage: dependency-volume manifest|init|verify [options]'); +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) { + try { main(process.argv); } + catch (error) { + console.error(`dependency-volume: ${error instanceof Error ? error.message : String(error)}`); + process.exit(2); + } +} diff --git a/tools/stack-bench/appliance/docker-compose.yaml b/tools/stack-bench/appliance/docker-compose.yaml new file mode 100644 index 00000000000..1edcead0ca5 --- /dev/null +++ b/tools/stack-bench/appliance/docker-compose.yaml @@ -0,0 +1,114 @@ +name: stack-bench-appliance + +services: + deps-init: + image: ${STACK_BENCH_CONTROLLER_IMAGE:?set STACK_BENCH_CONTROLLER_IMAGE to the manifest digest reference} + platform: linux/amd64 + command: ["init-deps"] + read_only: true + cap_drop: ["ALL"] + security_opt: ["no-new-privileges:true"] + volumes: + - type: volume + source: release-deps + target: /opt/stack-bench-release-deps + tmpfs: + - /tmp:size=64m,mode=1777 + + controller: + image: ${STACK_BENCH_CONTROLLER_IMAGE:?set STACK_BENCH_CONTROLLER_IMAGE to the manifest digest reference} + platform: linux/amd64 + init: true + network_mode: host + read_only: true + cap_drop: ["ALL"] + security_opt: ["no-new-privileges:true"] + depends_on: + deps-init: + condition: service_completed_successfully + postgres: + condition: service_healthy + mongodb: + condition: service_healthy + environment: + STACK_BENCH_CONTROLLER_IMAGE: ${STACK_BENCH_CONTROLLER_IMAGE:?set STACK_BENCH_CONTROLLER_IMAGE to the manifest digest reference} + STACK_BENCH_IMAGE: ${STACK_BENCH_BUILD_IMAGE:?set STACK_BENCH_BUILD_IMAGE to the manifest digest reference} + STACK_BENCH_RELEASE_MANIFEST: ${STACK_BENCH_RELEASE_MANIFEST:-} + STACK_BENCH_APPLIANCE: "1" + STACK_BENCH_COMPOSE_FILE: /opt/stack-bench/appliance/docker-compose.yaml + STACK_BENCH_WORK_DIR: /var/lib/stack-bench/work + STACK_BENCH_RESULTS_DIR: /var/lib/stack-bench/results + STACK_BENCH_SUPERVISOR_DIR: /var/lib/stack-bench/controller-home/supervisor + STACK_BENCH_RUNTIME_DIR: /var/lib/stack-bench/controller-home/runtime + STACK_BENCH_RESOURCE_LOCK_DIR: /var/lib/stack-bench/controller-home/resource-locks + STACK_BENCH_RELEASE_DEPS_VOLUME: stack-bench-release-deps + STACK_BENCH_LINUX_CLI: /opt/stack-bench-release-deps/spacetimedb-cli + STDB_PACKAGE: /opt/stack-bench-release-deps/bindings-typescript + SPACETIME_BIN: /opt/stack-bench-release-deps/spacetimedb-cli + STACK_BENCH_AGENT_AUTH: ${STACK_BENCH_AGENT_AUTH:-subscription-token} + STACK_BENCH_CLAUDE_OAUTH_TOKEN_FILE: ${STACK_BENCH_CLAUDE_OAUTH_TOKEN_FILE:-/var/lib/stack-bench/secrets/claude_subscription_token} + STACK_BENCH_ANTHROPIC_API_KEY_FILE: ${STACK_BENCH_ANTHROPIC_API_KEY_FILE:-} + HOME: /var/lib/stack-bench/controller-home + volumes: + - type: bind + source: /var/run/docker.sock + target: /var/run/docker.sock + - type: bind + source: /var/lib/stack-bench + target: /var/lib/stack-bench + - type: volume + source: release-deps + target: /opt/stack-bench-release-deps + read_only: true + tmpfs: + - /tmp:size=1g,mode=1777 + command: ["--help"] + + dashboard: + extends: + service: controller + profiles: ["dashboard"] + network_mode: bridge + environment: + STACK_BENCH_DASHBOARD_CONTROL_SECRET_FILE: ${STACK_BENCH_DASHBOARD_CONTROL_SECRET_FILE:-/var/lib/stack-bench/secrets/dashboard_control_secret} + ports: + - "127.0.0.1:7331:7331" + command: ["dashboard", "--host", "0.0.0.0", "--port", "7331", "--allow-container-bind"] + + postgres: + image: postgres:16@sha256:219341e4cedb06c8634f80af40851da3425b41b76603fd890272f58e37e139f7 + platform: linux/amd64 + container_name: stack-bench-postgres + ports: ["127.0.0.1:6532:5432"] + environment: + POSTGRES_USER: appuser + POSTGRES_PASSWORD: local-app-password + POSTGRES_DB: app + volumes: + - pgdata:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U appuser -d app"] + interval: 5s + timeout: 5s + retries: 12 + + mongodb: + image: mongo:7@sha256:554a9bb1ec6e00c40ba078a41974a834d1a9a8ab1772645b69142afecc87f082 + platform: linux/amd64 + container_name: stack-bench-mongodb + ports: ["127.0.0.1:6537:27017"] + volumes: + - mongodata:/data/db + healthcheck: + test: ["CMD", "mongosh", "--quiet", "--eval", "db.runCommand({ping:1})"] + interval: 5s + timeout: 5s + retries: 12 + +volumes: + release-deps: + name: stack-bench-release-deps + pgdata: + name: stack-bench-appliance-pgdata + mongodata: + name: stack-bench-appliance-mongodata diff --git a/tools/stack-bench/appliance/operator.env.example b/tools/stack-bench/appliance/operator.env.example new file mode 100644 index 00000000000..ad900f63759 --- /dev/null +++ b/tools/stack-bench/appliance/operator.env.example @@ -0,0 +1,20 @@ +# Subscription billing is the default. Generate a dedicated long-lived Claude +# setup token, write only the token to this mode-0600 file, and never commit it. +STACK_BENCH_AGENT_AUTH=subscription-token +STACK_BENCH_CLAUDE_OAUTH_TOKEN_FILE=/var/lib/stack-bench/secrets/claude_subscription_token + +# To bill through an API key instead, set the mode to api-key and provide an +# absolute path below /var/lib/stack-bench containing only that key. +# STACK_BENCH_ANTHROPIC_API_KEY_FILE=/var/lib/stack-bench/secrets/anthropic_api_key + +# The dashboard reads this separate operator secret from a mode-0600 file. +# The file must contain at least 32 random characters on one line. +STACK_BENCH_DASHBOARD_CONTROL_SECRET_FILE=/var/lib/stack-bench/secrets/dashboard_control_secret + +# Both image values must be registry references ending in @sha256:<64 hex chars>. +STACK_BENCH_CONTROLLER_IMAGE=registry.example/stack-bench-controller@sha256:replace-with-release-digest +STACK_BENCH_BUILD_IMAGE=registry.example/stack-bench-build@sha256:replace-with-release-digest + +# Optional for internal campaigns and required for a distributed release +# campaign. The file must be below the appliance state root. +STACK_BENCH_RELEASE_MANIFEST=/var/lib/stack-bench/release/release.json diff --git a/tools/stack-bench/backends/minimal/mongodb-1.5.md b/tools/stack-bench/backends/minimal/mongodb-1.5.md new file mode 100644 index 00000000000..8b9e91fbdba --- /dev/null +++ b/tools/stack-bench/backends/minimal/mongodb-1.5.md @@ -0,0 +1,19 @@ +# MongoDB + +Use MongoDB for the application data. Choose the libraries, architecture, and +project structure. + +## Connection + +| Setting | Value | +|---|---| +| `DATABASE_URL` | `` | +| Web application | `http://localhost:` | + +The MongoDB service is already running. Use the exact `DATABASE_URL`. Do not +start another MongoDB server, connect to another instance, or create another +database. Serve the complete application on ``. +Create `/app/start.sh`. From a clean source checkout, it must install +dependencies, build the complete application, and start it on ``. +The script must not change source files. Leave the application running when the +work is complete. diff --git a/tools/stack-bench/backends/minimal/postgres-1.5.md b/tools/stack-bench/backends/minimal/postgres-1.5.md new file mode 100644 index 00000000000..4a9124a88d2 --- /dev/null +++ b/tools/stack-bench/backends/minimal/postgres-1.5.md @@ -0,0 +1,19 @@ +# PostgreSQL + +Use PostgreSQL for the application data. Choose the libraries, architecture, +and project structure. + +## Connection + +| Setting | Value | +|---|---| +| `DATABASE_URL` | `` | +| Web application | `http://localhost:` | + +The PostgreSQL service is already running. Use the exact `DATABASE_URL`. Do not +start another PostgreSQL server, connect to another instance, or create another +database. Serve the complete application on ``. +Create `/app/start.sh`. From a clean source checkout, it must install +dependencies, build the complete application, and start it on ``. +The script must not change source files. Leave the application running when the +work is complete. diff --git a/tools/stack-bench/backends/minimal/spacetime-1.6.md b/tools/stack-bench/backends/minimal/spacetime-1.6.md new file mode 100644 index 00000000000..4d686b4d547 --- /dev/null +++ b/tools/stack-bench/backends/minimal/spacetime-1.6.md @@ -0,0 +1,28 @@ +# SpacetimeDB + +Use SpacetimeDB for the application data. Put the TypeScript module in the +required directory below. Choose the schema, libraries, architecture, and the +rest of the project structure. + +## Connection + +Use the connection settings below. + +| Setting | Value | +|---|---| +| Server URI | `` | +| Module name | `` | +| SpacetimeDB CLI | `` | +| TypeScript SDK package | `` | +| Module source directory | `/app/backend/spacetimedb` | +| Web application | `http://localhost:` | + +Publish only the named module to the exact server URI. Local publish and +development commands must use `--yes`. Do not pipe confirmation input, publish +anonymously, or use the hosted service. Create `/app/start.sh`. From a clean +source checkout, it must install dependencies, build the complete application, +and start it on ``. The script must not change source files. Leave +the application running when the work is complete. + +The included TypeScript core SDK reference describes the available core API syntax. +CLI `--help` is available for command syntax. diff --git a/tools/stack-bench/backends/model-free-stub.md b/tools/stack-bench/backends/model-free-stub.md new file mode 100644 index 00000000000..7a83d0e4bc2 --- /dev/null +++ b/tools/stack-bench/backends/model-free-stub.md @@ -0,0 +1,4 @@ +# Model-free service + +Use the supplied service. Leave the app running on the assigned client port +when the work is complete. diff --git a/tools/stack-bench/backends/mongodb.md b/tools/stack-bench/backends/mongodb.md new file mode 100644 index 00000000000..4671ddbb19e --- /dev/null +++ b/tools/stack-bench/backends/mongodb.md @@ -0,0 +1,46 @@ +# Backend: MongoDB + +An Express API server with Socket.io for live updates, Mongoose over MongoDB, +and a React client. + +## Layout + +``` +/ + server/ + package.json express, socket.io, mongoose, dotenv, tsx + .env DATABASE_URL and PORT + src/models.ts Mongoose schemas and models + src/index.ts Express routes, Socket.io handlers + client/ + package.json react, react-dom, vite, socket.io-client + vite.config.ts server.port , proxy /api and /socket.io to + index.html + src/main.tsx + src/App.tsx +``` + +## Deploy + +```bash +cd server && npm install && npm run dev # on +cd client && npm install && npm run dev # on +``` + +Mongoose creates collections on first write; there is no migration step. + +The server prints to the terminal running `npm run dev`; it restarts on save, +so a code change is live without redeploying. + +Keep existing application data when you change the schema. Do not drop +collections during upgrades or repairs. + +## Configuration + +| Setting | Value | +|---|---| +| `DATABASE_URL` | `` | +| API server port | `` | +| Client dev server | `` | + +Use this exact `DATABASE_URL`. Do not point at another MongoDB instance. diff --git a/tools/stack-bench/backends/postgres.md b/tools/stack-bench/backends/postgres.md new file mode 100644 index 00000000000..32ee3ac4afd --- /dev/null +++ b/tools/stack-bench/backends/postgres.md @@ -0,0 +1,48 @@ +# Backend: PostgreSQL + +An Express API server with Socket.io for live updates, Drizzle ORM over +PostgreSQL, and a React client. + +## Layout + +``` +/ + server/ + package.json express, socket.io, drizzle-orm, pg, dotenv, tsx + .env DATABASE_URL and PORT + drizzle.config.ts + src/schema.ts Drizzle table definitions + src/index.ts Express routes, Socket.io handlers + client/ + package.json react, react-dom, vite, socket.io-client + vite.config.ts server.port , proxy /api and /socket.io to + index.html + src/main.tsx + src/App.tsx +``` + +## Deploy + +```bash +cd server && npm install && npx drizzle-kit push && npm run dev # on +cd client && npm install && npm run dev # on +``` + +Re-run `npx drizzle-kit push` after any schema change. + +The server prints to the terminal running `npm run dev`; it restarts on save, +so a code change is live without redeploying. + +Keep existing application data when you change the schema. Do not drop or +recreate tables during upgrades or repairs. + +## Configuration + +| Setting | Value | +|---|---| +| `DATABASE_URL` | `` | +| API server port | `` | +| Client dev server | `` | + +Use this exact `DATABASE_URL`. Do not point at another PostgreSQL instance and do +not create databases outside it. diff --git a/tools/stack-bench/backends/spacetime.md b/tools/stack-bench/backends/spacetime.md new file mode 100644 index 00000000000..250587f88fe --- /dev/null +++ b/tools/stack-bench/backends/spacetime.md @@ -0,0 +1,80 @@ +# Backend: SpacetimeDB + +The database runs your server logic. There is no separate API server and no ORM: +tables and reducers are a WASM module you publish, and the client subscribes to +tables and calls reducers over a live connection. + +## Layout + +``` +/ + backend/spacetimedb/ + package.json { "type": "module", dependencies: { "spacetimedb": "" }, + devDependencies: { "typescript": "~5.6.2" } } ← required; the build runs tsc from node_modules + tsconfig.json + src/schema.ts tables and indexes + src/index.ts reducers and lifecycle hooks + client/ + package.json react, react-dom, vite, and "spacetimedb": "" + vite.config.ts server.port must be + index.html + src/config.ts MODULE_NAME and SPACETIMEDB_URI + src/main.tsx React entry + src/App.tsx + src/module_bindings/ generated; never edit by hand +``` + +## Deploy + +Publish the module, then regenerate the client bindings from it: + +```bash + publish --module-path backend/spacetimedb -s --yes + generate --lang typescript --out-dir client/src/module_bindings --module-path backend/spacetimedb +``` + +**While iterating, run development mode instead of republishing by hand.** It +watches the module and automatically rebuilds, publishes, and regenerates the +client bindings on every save: + +```bash + dev --module-path backend/spacetimedb -s --yes +``` + +Leave it running in the background while you work. The manual commands below +are for one-off publishes and for the first deploy. + +Republish after any server change, and regenerate after any schema change. + +Keep existing application data when you change the schema. + +Always use `--yes` for local publish and development commands. It selects the +CLI's non-interactive authentication flow for the target server. Do not pipe +`y` into the command and do not publish anonymously. Use the same local +identity for every publish to the named module. + +Then start the client: + +```bash +cd client && npm install && npm run dev +``` + +` logs -s ` shows module output, including reducer errors. + +To inspect stored data while debugging: + +```bash + sql "SELECT * FROM item LIMIT 5" -s +``` + +## Configuration + +| Setting | Value | +|---|---| +| Server URI | `` | +| Module name | `` | +| Client dev server | `` | + +The SDK reference for writing modules and clients is in the skill documents +included with these instructions. Follow them for API specifics: import paths, +type builders, accessors and context typing. diff --git a/tools/stack-bench/commands/agent.ts b/tools/stack-bench/commands/agent.ts new file mode 100644 index 00000000000..5fe8372dcba --- /dev/null +++ b/tools/stack-bench/commands/agent.ts @@ -0,0 +1,897 @@ +#!/usr/bin/env node + +import { execFileSync } from 'node:child_process'; +import type { ExecFileSyncOptionsWithStringEncoding } from 'node:child_process'; +import { readFileSync, writeFileSync, mkdirSync, existsSync, realpathSync, + openSync, readSync, closeSync, readdirSync } from 'node:fs'; +import { join, dirname, resolve, relative, isAbsolute, sep } from 'node:path'; +import { homedir } from 'node:os'; +import { fileURLToPath } from 'node:url'; +import { parseArgs as parseNodeArgs } from 'node:util'; +import { loadTrack, levelPrompt, appendix, suitesFor, dbName, moduleName, portsFor, + DEFAULT_TRACK, TRACK_MANIFEST_FILE } from '../src/composition/tracks.js'; +import type { Track, TrackDefinition } from '../src/composition/tracks.js'; +import type { BackendLease } from '../src/runtime/backend-lease.js'; +import { resolveRecipeRelease } from '../src/composition/recipe-release.js'; +import { parseGuidanceMode, resolveDefaultGuidanceForStack, type GuidanceMode, + type ResolvedGuidanceDocument, type ResolvedSkills } + from '../src/campaigns/condition-compiler.js'; +import type { ExactRecipeRequest, RecipeBinding } from '../src/composition/recipe-release.js'; +import { createBoundRecipeTaskRequest, resolveBoundRecipeTaskRequest } from '../src/composition/recipe-selection.js'; +import { agentVisibleContractText, assertAgentVisibleText } + from '../src/composition/agent-visible-contract.js'; +import { DEFAULT_SPACETIME_SERVER_URI, leaseFromEnv } from '../src/runtime/backend-lease.js'; +import { CODING_CONTAINER_APP_ROOT, CODING_CONTAINER_BUG_REPORT_FILE, + CODING_CONTAINER_RELEASE_DEPS_ROOT, CODING_CONTAINER_SPACETIME_CLI, + CODING_CONTAINER_SPACETIME_PACKAGE } + from '../src/runtime/coding-container-policy.js'; +import { resolveContainerImage } from '../src/runtime/container-image.js'; +import { hashDirectory, sessionProvenance, sha256 } from '../src/evidence/provenance.js'; +import type { StackRunPorts } from '../src/stacks/stack-adapter-contract.js'; +import { STACK_ADAPTER_REGISTRY } from '../src/stacks/stack-adapters.js'; +import { requireLeasedDatabase, requireLeasedSpacetime } + from '../src/stacks/backend-reset-guard.js'; +import { DEFAULT_BUILD_IMAGE } from '../src/composition/product-config.js'; +import { dockerMountArguments } from '../src/runtime/container-mount.js'; +import { normalizePromptText, readAgentSkillDocuments, selectAgentSkills } from '../src/agents/agent-materials.js'; +import { codingSessionFailure, DEFAULT_THROTTLE_MAX_WAIT_MS, providerSessionFailure, + runCodingSessionWithRetries } from '../src/agents/coding-session-retry.js'; +import type { CodingSessionRetryResult } from '../src/agents/coding-session-retry.js'; +import { AGENT_PROCESS_TIMEOUT_MS } from '../src/agents/coding-session-timeouts.js'; +import { assertNewOrEmptyDirectory } from '../src/runtime/path-safety.js'; +import { claudeRatesForModel } from '../src/evidence/claude-usage-cost.js'; +import { PRICING_UNIT, validatePricingAuthority } + from '../src/evidence/pricing-authority.js'; +import type { PricingAuthority } from '../src/evidence/pricing-authority.js'; + +import { STACK_BENCH_ROOT as ROOT, compiledEntrypoint } from '../src/package-root.js'; +const REPO = resolve(ROOT, '..', '..'); +const CONTROL_COMMAND_TIMEOUT_MS = 120_000; +const DEFAULT_CODING_INTERRUPTION_RETRIES = 2; + +type UnknownRecord = Record; +interface PromptMaterials { + skillsText?: string; + requirementText?: string; + contractText?: string; + startingCatalog?: string; +} + +type RecipeTaskRequest = Parameters[1] & { + recipe?: Exclude; +}; + +type AgentMode = 'build' | 'upgrade' | 'fix' | 'resume'; + +interface AgentArgs { + mode: AgentMode; + backend: string; + app: string; + level: number; + runIndex: number; + model: string; + guidance: GuidanceMode; + track: string; + pricing: Readonly | null; + guidanceDocument?: ResolvedGuidanceDocument; + credentialAliases?: Readonly>; + recipe?: string; + recipeTask?: RecipeTaskRequest; + thinking?: string; + maxBudgetUsd?: number; + skills?: string[]; + skillIdentity?: ResolvedSkills; + apiKey?: string; + printPrompt?: boolean; +} + +interface ThinkingVolume { + blocks: number; + signatureBytes: number; + bytesPerBlock: number; +} + +interface SessionUsage { + input_tokens?: number; + output_tokens?: number; + cache_creation_input_tokens?: number; + cache_read_input_tokens?: number; +} + +const isRecord = (value: unknown): value is UnknownRecord => + value !== null && typeof value === 'object' && !Array.isArray(value); + +const stringValue = (value: unknown): string | null => typeof value === 'string' ? value : null; + +function stringArray(value: string, option: string): string[] { + const parsed: unknown = JSON.parse(value); + if (!Array.isArray(parsed) || parsed.some(item => typeof item !== 'string')) { + throw new Error(`${option} must be an array of strings`); + } + return parsed; +} + +function sessionUsage(value: unknown): SessionUsage { + return isRecord(value) ? { + input_tokens: typeof value.input_tokens === 'number' ? value.input_tokens : undefined, + output_tokens: typeof value.output_tokens === 'number' ? value.output_tokens : undefined, + cache_creation_input_tokens: typeof value.cache_creation_input_tokens === 'number' + ? value.cache_creation_input_tokens : undefined, + cache_read_input_tokens: typeof value.cache_read_input_tokens === 'number' + ? value.cache_read_input_tokens : undefined, + } : {}; +} + +// Use only the benchmark-owned SpacetimeDB host. +const STDB_URI = process.env.STACK_BENCH_STDB_URI ?? DEFAULT_SPACETIME_SERVER_URI; + +// Test the CLI and SDK from this checkout. +const LOCAL_CLI = join(REPO, 'target', 'release', 'spacetimedb-cli.exe'); +const STDB_BIN = process.env.SPACETIME_BIN ?? (existsSync(LOCAL_CLI) ? LOCAL_CLI : 'spacetime'); +const LOCAL_PKG = process.env.STDB_PACKAGE ?? join(REPO, 'crates', 'bindings-typescript'); + +const fwd = (path: string): string => path.split('\\').join('/'); + +// Keep the provider's default thinking budget unless an experiment selects one +// explicitly. The run records observed reasoning volume so default changes are +// visible in the evidence. +const THINKING_TOKENS = process.env.STACK_BENCH_THINKING ?? null; + +const EFFORT = process.env.STACK_BENCH_EFFORT ?? 'high'; + +const IMAGE = process.env.STACK_BENCH_IMAGE ?? DEFAULT_BUILD_IMAGE; + +// Containers reach host services through this address. App ports remain local. +export function hostServiceAddress(env: NodeJS.ProcessEnv = process.env): string { + return env.STACK_BENCH_HOST_ALIAS + ?? (env.STACK_BENCH_APPLIANCE === '1' ? '127.0.0.1' : 'host.docker.internal'); +} + +const HOST_ADDR = hostServiceAddress(); +const hostUrl = (url: string): string => url.replace(/127\.0\.0\.1|localhost/g, HOST_ADDR); + +const C_BIN = CODING_CONTAINER_SPACETIME_CLI; + +// The container requires the Linux CLI from this checkout. +const LINUX_CLI = process.env.STACK_BENCH_LINUX_CLI + ?? join(ROOT, 'container', 'bin', 'spacetimedb-cli'); + +// The provider CLI keeps one JSONL transcript per session under its project +// directory for the application path. +function transcriptFile(appDir: string, sessionId: string): string | null { + const store = join(homedir(), '.claude', 'projects'); + if (!existsSync(store)) return null; + const want = resolve(appDir).replace(/[\\/:]/g, '-').toLowerCase(); + const dir = readdirSync(store).find(d => { + const n = d.toLowerCase(); + return n === want || n === want.replace(/^-+/, ''); + }); + const file = dir && join(store, dir, `${sessionId}.jsonl`); + return file && existsSync(file) ? file : null; +} + +// The model ids the provider actually served. The requested name is an alias +// that can resolve to different snapshots over time; the transcript records +// what answered each request. +function transcriptModels(appDir: string, sessionIds: readonly (string | null | undefined)[]): string[] { + const models = new Set(); + for (const sessionId of new Set(sessionIds.filter((id): id is string => Boolean(id)))) { + try { + const file = transcriptFile(appDir, sessionId); + if (!file) continue; + for (const line of readFileSync(file, 'utf8').split('\n')) { + if (!line.includes('"model"')) continue; + let record: unknown; + try { record = JSON.parse(line); } catch { continue; } + if (!isRecord(record) || !isRecord(record.message)) continue; + const model = stringValue(record.message.model); + if (model) models.add(model); + } + } catch { /* an unreadable transcript leaves the list shorter, never wrong */ } + } + return [...models].sort(); +} + +// The transcript exposes reasoning blocks and signature bytes, not reasoning tokens. +function thinkingVolume(appDir: string, sessionId: string | null | undefined): ThinkingVolume | null { + if (!sessionId) return null; + try { + const file = transcriptFile(appDir, sessionId); + if (!file) return null; + + let blocks = 0, bytes = 0; + for (const line of readFileSync(file, 'utf8').split('\n')) { + if (!line.includes('"thinking"')) continue; // cheap filter before parsing + let record: unknown; + try { record = JSON.parse(line); } catch { continue; } + if (!isRecord(record) || !isRecord(record.message) + || !Array.isArray(record.message.content)) continue; + for (const content of record.message.content) { + if (!isRecord(content) || content.type !== 'thinking') continue; + blocks++; + bytes += stringValue(content.signature)?.length ?? 0; + } + } + return { blocks, signatureBytes: bytes, + bytesPerBlock: blocks ? Math.round(bytes / blocks) : 0 }; + } catch { return null; } +} + +function combinedThinkingVolume(appDir: string, sessionIds: readonly (string | null | undefined)[]): ThinkingVolume | null { + const volumes: ThinkingVolume[] = [...new Set(sessionIds.filter((id): id is string => Boolean(id)))] + .map(id => thinkingVolume(appDir, id)).filter((item): item is ThinkingVolume => item !== null); + if (!volumes.length) return null; + const blocks = volumes.reduce((sum, item) => sum + item.blocks, 0); + const signatureBytes = volumes.reduce((sum, item) => sum + item.signatureBytes, 0); + return { blocks, signatureBytes, + bytesPerBlock: blocks ? Math.round(signatureBytes / blocks) : 0 }; +} + + +// Record the Linux CLI executed by the container. The host and container +// binaries can change independently and must not share an identity. +function linuxSpacetimeVersion(image: string): { commit: string | null; binarySha256: string | null; raw: string } { + try { + const releaseVolume = process.env.STACK_BENCH_RELEASE_DEPS_VOLUME?.trim() || null; + const mountArgs = releaseVolume + ? dockerMountArguments({ kind: 'volume', source: releaseVolume, + target: CODING_CONTAINER_RELEASE_DEPS_ROOT, readOnly: true }) + : ['-v', `${LINUX_CLI}:${CODING_CONTAINER_SPACETIME_CLI}:ro`]; + const entrypoint = releaseVolume + ? `${CODING_CONTAINER_RELEASE_DEPS_ROOT}/spacetimedb-cli` + : CODING_CONTAINER_SPACETIME_CLI; + const out = execFileSync('docker', + ['run', '--rm', ...mountArgs, '--entrypoint', entrypoint, image, '--version'], + { encoding: 'utf8', stdio: 'pipe', env: { ...process.env, MSYS_NO_PATHCONV: '1' }, + timeout: CONTROL_COMMAND_TIMEOUT_MS }); + const commit = out.match(/Commit:\s*([0-9a-f]+)/i)?.[1] ?? null; + return { commit, binarySha256: sha256(readFileSync(LINUX_CLI)), + raw: out.trim().split(/\r?\n/).slice(0, 2).join(' ') }; + } catch { return { commit: null, binarySha256: null, raw: 'unknown' }; } +} + +function bindingsIdentity(pkgDir: string): { package: string; sourceSha256: string | null; sourceFiles: number } { + try { + const p = JSON.parse(readFileSync(join(pkgDir, 'package.json'), 'utf8')); + const source = hashDirectory(pkgDir, { exclude: name => + /(^|\/)(node_modules|dist|target)(\/|$)/.test(name) }); + return { package: `${p.name}@${p.version}`, sourceSha256: source.sha256, + sourceFiles: source.files.length }; + } catch { return { package: 'unknown', sourceSha256: null, sourceFiles: 0 }; } +} + +// The CLI version inside the build image. Read by running it, not by trusting +// the tag: the image is pinned by ARG and a tag can be moved. +function imageCliVersion(image: string): string { + try { + return execFileSync('docker', ['run', '--rm', '--entrypoint', 'claude', image, '--version'], + { encoding: 'utf8', stdio: 'pipe', env: { ...process.env, MSYS_NO_PATHCONV: '1' }, + timeout: CONTROL_COMMAND_TIMEOUT_MS }).trim(); + } catch { return 'unknown'; } +} + +function imageNodeVersion(image: string): string { + try { + return execFileSync('docker', ['run', '--rm', '--entrypoint', 'node', image, '--version'], + { encoding: 'utf8', stdio: 'pipe', env: { ...process.env, MSYS_NO_PATHCONV: '1' }, + timeout: CONTROL_COMMAND_TIMEOUT_MS }).trim(); + } catch { return 'unknown'; } +} + +function containerImage(name: string): { reference: string; imageId: string | null | undefined } { + try { + const out = execFileSync('docker', ['inspect', '-f', '{{.Config.Image}} {{.Image}}', name], + { encoding: 'utf8', stdio: 'pipe', timeout: CONTROL_COMMAND_TIMEOUT_MS }).trim(); + const [reference, imageId] = out.split(/\s+/, 2); + return { reference: reference ?? '', imageId }; + } catch { return { reference: 'unknown', imageId: null }; } +} + +// Record ambient provider configuration that can change model behaviour while +// replacing credential values with presence markers. +function ambientEnv(): Record { + const seen: Record = {}; + for (const [k, v] of Object.entries(process.env)) { + if (!/^(CLAUDE|ANTHROPIC|MAX_THINKING|DISABLE_AUTOUPDATER|FORCE_PROMPT)/.test(k)) continue; + // Never record a credential, only that one was present. + seen[k] = /KEY|TOKEN|SECRET/i.test(k) ? '' : v; + } + return seen; +} + +export function parseAgentArgs(argv: readonly string[]): AgentArgs { + const strings = ['mode', 'track', 'backend', 'level', 'app', 'run-index', 'model', + 'pricing-json', 'guidance', 'guidance-document-json', 'credential-aliases-json', + 'recipe', 'recipe-task-json', 'thinking', 'max-budget-usd', 'skills', 'skills-json', + 'skill-identity-json', 'api-key'] as const; + const { values: rawValues } = parseNodeArgs({ args: [...argv.slice(2)], options: Object.fromEntries([ + ...strings.map(name => [name, { type: 'string' as const }]), + ['print-prompt', { type: 'boolean' as const }], + ]), strict: true, allowPositionals: false }); + const values = rawValues as Partial> + & { 'print-prompt'?: boolean }; + const mode = values.mode; + if (mode !== 'build' && mode !== 'upgrade' && mode !== 'fix' && mode !== 'resume') { + throw new Error('--mode must be build, upgrade, fix, or resume'); + } + const backend = values.backend; + const app = values.app; + if (!backend || !app) { + throw new Error('usage: node dist/commands/agent.js --mode build|upgrade|fix|resume ' + + '--backend --app [--level ]'); + } + const level = values.level === undefined ? 1 : Number(values.level); + if (!Number.isSafeInteger(level) || level < 1) { + throw new Error('--level must be a positive integer'); + } + const runIndex = values['run-index'] === undefined ? 0 : Number(values['run-index']); + if (!Number.isSafeInteger(runIndex) || runIndex < 0) { + throw new Error('--run-index must be a non-negative integer'); + } + const model = values.model ?? 'claude-sonnet-5'; + const maxBudgetUsd = values['max-budget-usd'] === undefined + ? undefined : Number(values['max-budget-usd']); + if (maxBudgetUsd !== undefined && (!Number.isFinite(maxBudgetUsd) || maxBudgetUsd <= 0)) { + throw new Error('--max-budget-usd must be a positive number'); + } + if (values.skills !== undefined && values['skills-json'] !== undefined) { + throw new Error('--skills and --skills-json cannot be used together'); + } + const skills = values.skills?.split(',').map(skill => skill.trim()).filter(Boolean) + ?? (values['skills-json'] === undefined ? undefined + : stringArray(values['skills-json'], '--skills-json')); + let pricing = values['pricing-json'] === undefined + ? undefined : validatePricingAuthority(JSON.parse(values['pricing-json']), { at: '--pricing-json' }); + if (pricing === undefined && maxBudgetUsd !== undefined) { + const rates = claudeRatesForModel(model); + if (!rates) throw new Error(`no default pricing is recorded for model ${model}`); + pricing = validatePricingAuthority({ unit: PRICING_UNIT, rates }, + { at: 'default pricing' }); + } + return { mode, backend, app, level, runIndex, model, + guidance: parseGuidanceMode(values.guidance ?? 'prescribed'), + track: values.track ?? DEFAULT_TRACK, pricing: pricing ?? null, + ...(values['guidance-document-json'] ? { + guidanceDocument: JSON.parse(values['guidance-document-json']) as ResolvedGuidanceDocument, + } : {}), + ...(values['credential-aliases-json'] ? { + credentialAliases: JSON.parse(values['credential-aliases-json']) as Record, + } : {}), + ...(values.recipe ? { recipe: values.recipe } : {}), + ...(values['recipe-task-json'] ? { + recipeTask: JSON.parse(values['recipe-task-json']) as RecipeTaskRequest, + } : {}), + ...(values.thinking ? { thinking: values.thinking } : {}), + ...(maxBudgetUsd !== undefined ? { maxBudgetUsd } : {}), + ...(skills ? { skills } : {}), + ...(values['skill-identity-json'] ? { + skillIdentity: validateSkillIdentity(JSON.parse(values['skill-identity-json'])), + } : {}), + ...(values['api-key'] ? { apiKey: values['api-key'] } : {}), + ...(values['print-prompt'] ? { printPrompt: true } : {}) }; +} + +const dbUrl = (backend: string, runIndex: number, dbPort: number | null, track: Track): string | null => { + const adapter = STACK_ADAPTER_REGISTRY.get(backend); + if (adapter.id === 'spacetime' || adapter.id === 'stub') return null; + if (!dbPort) throw new Error(`${backend} has no assigned database port`); + return adapter.agent.connectionUrl({ dbPort, database: dbName(track, runIndex), hostUrl }); +}; + +// Create the leased database before the app connects. A build clears its schema. +// A reset between suites preserves the schema required by the running app. +type DatabasePreparationLease = BackendLease; + +type DatabaseCommandOptions = Pick; + +type DatabaseCommandExecutor = (command: string, args: readonly string[], + options: DatabaseCommandOptions) => string; + +const databaseCommandExecutor: DatabaseCommandExecutor = (command, args, options) => + String(execFileSync(command, args, { ...options, encoding: 'utf8' })); + +interface DatabasePreparationOptions { + exec?: DatabaseCommandExecutor; + stdbBin?: string; + lease?: DatabasePreparationLease; +} + +export function ensureDatabase(backend: string, runIndex: number, dbPort: number | null, + track: Pick, wipe = false, + { exec = databaseCommandExecutor, stdbBin = STDB_BIN, lease: suppliedLease }: DatabasePreparationOptions = {}) { + const lease = suppliedLease ?? leaseFromEnv(process.env, { backend, active: true }).lease; + if (lease.runIndex !== runIndex || lease.track !== track.name) { + throw new Error(`backend lease ${lease.runId} belongs to ${lease.track}/run${lease.runIndex}, ` + + `not ${track.name}/run${runIndex}`); + } + const expectedName = dbName(track, runIndex); + const name = lease.resources.database ?? expectedName; + const input = { name, expectedName, wipe, exec, cli: stdbBin, + expectedServerUri: STDB_URI, expectedModule: moduleName(track, runIndex), dbPort }; + const adapter = STACK_ADAPTER_REGISTRY.get(backend); + if (adapter.id === 'postgres' || adapter.id === 'mongodb') { + return adapter.database.prepare({ ...input, lease: requireLeasedDatabase(lease) }); + } + if (adapter.id === 'spacetime') { + return adapter.database.prepare({ ...input, lease: requireLeasedSpacetime(lease) }); + } + return adapter.database.prepare({ name }); +} + +// Prescribed guidance chooses an implementation stack. Neutral guidance gives +// only stack access facts and the selected API references. +export function readBackendGuidanceDocument( + document: ResolvedGuidanceDocument | undefined, + fallbackRelativePath: string, +): string { + if (typeof fallbackRelativePath !== 'string' || !fallbackRelativePath) { + throw new Error('backend guidance fallback path is required'); + } + if (document !== undefined) { + const fields = new Set(['path', 'sha256', 'bytes', 'applicationInterface']); + if (!document || typeof document !== 'object' || Array.isArray(document) + || Object.keys(document).some(field => !fields.has(field)) + || typeof document.path !== 'string' || !document.path || isAbsolute(document.path) + || document.path.includes('\\') + || !/^[a-f0-9]{64}$/.test(document.sha256) + || !Number.isSafeInteger(document.bytes) || document.bytes < 0 + || !['http', 'reducer'].includes(document.applicationInterface)) { + throw new Error('campaign guidance document identity is invalid'); + } + } + const root = realpathSync(ROOT); + const candidate = resolve(root, document?.path ?? fallbackRelativePath); + const candidateRel = relative(root, candidate); + if (candidateRel === '..' || candidateRel.startsWith(`..${sep}`) || isAbsolute(candidateRel)) { + throw new Error('campaign guidance document escapes the Stack Bench root'); + } + const selectedPath = realpathSync(candidate); + const resolvedRel = relative(root, selectedPath); + if (resolvedRel === '..' || resolvedRel.startsWith(`..${sep}`) || isAbsolute(resolvedRel)) { + throw new Error('campaign guidance document resolves outside the Stack Bench root'); + } + const bytes = Buffer.from(normalizePromptText(readFileSync(selectedPath, 'utf8')), 'utf8'); + if (document && (sha256(bytes) !== document.sha256 || bytes.length !== document.bytes)) { + throw new Error(`campaign guidance document changed after compilation: ${document.path}`); + } + return bytes.toString('utf8'); +} + +function validateSkillIdentity(value: unknown): ResolvedSkills { + const fields = new Set(['ids', 'sha256', 'bytes']); + if (!isRecord(value) || Object.keys(value).some(field => !fields.has(field)) + || !Array.isArray(value.ids) || new Set(value.ids).size !== value.ids.length + || value.ids.some(id => typeof id !== 'string' || !/^[a-z][a-z0-9-]*$/.test(id)) + || typeof value.sha256 !== 'string' || !/^[a-f0-9]{64}$/.test(value.sha256) + || !Number.isSafeInteger(value.bytes) || Number(value.bytes) < 0) { + throw new Error('campaign skill identity is invalid'); + } + return { ids: value.ids as string[], sha256: value.sha256, bytes: Number(value.bytes) }; +} + +function backendDoc(args: AgentArgs, p: StackRunPorts, track: Track): string { + const defaultGuidance = resolveDefaultGuidanceForStack(args.guidance, args.backend); + let defaultPath = defaultGuidance?.documents[args.backend]?.path; + if (!defaultPath && args.guidance === 'neutral') { + throw new Error(`neutral guidance has no document for ${args.backend}`); + } + defaultPath ??= join('backends', `${args.backend}.md`); + const raw = readBackendGuidanceDocument(args.guidanceDocument, defaultPath); + return raw + .replaceAll('', String(p.vite)) + .replaceAll('', String(p.express ?? '')) + .replaceAll('', track.title) + .replaceAll('', moduleName(track, args.runIndex)) + .replaceAll('', p.dbPort ? dbUrl(args.backend, args.runIndex, p.dbPort, track) ?? '' : '') + .replaceAll('', hostUrl(STDB_URI)) + .replaceAll('', C_BIN) + .replaceAll('', `file:${CODING_CONTAINER_SPACETIME_PACKAGE}`); +} + +// Fail before a paid session when the selected container cannot run this checkout. +function containerBlocker(backend: string): string | null { + try { + execFileSync('docker', ['image', 'inspect', IMAGE], + { stdio: 'pipe', timeout: CONTROL_COMMAND_TIMEOUT_MS }); + } catch (error: unknown) { + const detail = error instanceof Error ? error.message.split('\n')[0] : String(error).split('\n')[0]; + return `cannot verify isolation image ${IMAGE}: ${detail} — ` + + `build it with docker build -t ${IMAGE} ${fwd(join(ROOT, 'container'))}`; + } + if (!STACK_ADAPTER_REGISTRY.get(backend).agent.linuxCliRequired) return null; + if (!existsSync(LINUX_CLI)) { + return `no Linux SpacetimeDB CLI at ${fwd(LINUX_CLI)} — ` + + 'bash tools/stack-bench/container/build-linux-cli.sh'; + } + // A file at this path must be a Linux executable, not the Windows build. + const magic = Buffer.alloc(4); + try { + const fd = openSync(LINUX_CLI, 'r'); + try { readSync(fd, magic, 0, 4, 0); } finally { closeSync(fd); } + } catch { + return `cannot read the Linux SpacetimeDB CLI at ${fwd(LINUX_CLI)}`; + } + if (magic.toString('binary') !== '\x7fELF') { + return `${fwd(LINUX_CLI)} is not a Linux binary; rebuild it with ` + + 'container/build-linux-cli.sh'; + } + return null; +} + +function decideIsolation(args: AgentArgs): { container: true; reason: null } { + const blocker = containerBlocker(args.backend); + if (!blocker) return { container: true, reason: null }; + console.error(`agent.js: isolated build unavailable: ${blocker}`); + console.error(' benchmark coding sessions require the isolation container'); + process.exit(2); +} + +// Pin every round to the build's recorded container topology. +function resolveIsolation(args: AgentArgs): { container: true; reason: null } { + const marker = resolve(args.app, '..', '.stack-bench-isolation'); + const backendMarker = resolve(args.app, '..', '.stack-bench-backend'); + + if (args.mode === 'build') { + const decided = decideIsolation(args); + if (!args.printPrompt) { + mkdirSync(dirname(marker), { recursive: true }); + writeFileSync(marker, 'container'); + } + return decided; + } + + if (existsSync(marker)) { + const pinned = readFileSync(marker, 'utf8').trim(); + if (pinned !== 'container') { + console.error(`agent.js: unsupported isolation marker ${JSON.stringify(pinned)}; expected "container"`); + process.exit(2); + } + const blocker = containerBlocker(args.backend); + if (blocker) { + console.error(`agent.js: this run's build ran in a container, but ${blocker}`); + console.error(' refusing to run this round in a different environment'); + process.exit(2); + } + return { container: true, reason: null }; + } + + // A backend marker without an isolation marker is ambiguous prior state. + if (existsSync(backendMarker)) { + console.error('agent.js: app has prior benchmark state but no isolation marker'); + console.error(' refusing to guess where earlier rounds ran; start a clean run'); + process.exit(2); + } + const decided = decideIsolation(args); + if (!args.printPrompt) { + mkdirSync(dirname(marker), { recursive: true }); + writeFileSync(marker, 'container'); + } + return decided; +} + +export function buildPrompt(args: AgentArgs, p: StackRunPorts, track: Track, + materials: PromptMaterials = {}): string { + const prompt = (lines: string[]): string => assertAgentVisibleText(lines.join('\n')); + const applicationInterface = args.guidanceDocument?.applicationInterface + ?? resolveDefaultGuidanceForStack(args.guidance, args.backend) + ?.documents[args.backend]?.applicationInterface; + if (applicationInterface !== 'http' && applicationInterface !== 'reducer') { + throw new Error(`stack ${args.backend} has no application interface`); + } + const common = [ + `Build the app in ${CODING_CONTAINER_APP_ROOT}.`, + // Published container ports require the app to bind to all interfaces. + '', + 'The web application must listen on 0.0.0.0, not localhost, so it is reachable ' + + 'outside its process.', + 'The environment can run /app/start.sh again with APP_WARM_START=1. ' + + 'When dependencies are current, reuse them instead of installing them again.', + '', + '## Stack', + '', + agentVisibleContractText(backendDoc(args, p, track), args.credentialAliases, + applicationInterface), + ]; + const skills = materials.skillsText ?? readAgentSkillDocuments(REPO, args.skills ?? []); + if (skills) common.push('', '## Selected API reference', '', skills); + + if (args.mode === 'resume') { + return prompt([ + 'Restore the existing application to a runnable state.', + '', + 'This is a saved application from an earlier completed run. Install its', + 'dependencies and start its existing database module, server, and web client', + 'as needed. Do not implement features or fix application behavior. Do not', + 'change source files. The saved source must remain byte-for-byte identical.', + '', + 'Output RESUME_COMPLETE when the existing app is running.', + '', + ...common, + ]); + } + + if (args.mode === 'fix') { + return prompt([ + 'Fix the reported application bugs.', + '', + `Read ${CODING_CONTAINER_BUG_REPORT_FILE} in the app directory. Each entry says what was expected`, + 'and what actually happened. Fix the app so the behaviour matches, redeploy,', + 'and make sure the dev server is running.', + '', + 'Change only what is needed. Do not alter behaviour that is already correct.', + '', + 'Output FIX_COMPLETE when done.', + '', + ...common, + '', + agentVisibleContractText(materials.requirementText ?? levelPrompt(track, args.level), + args.credentialAliases, applicationInterface), + '', + '## Application interface', + '', + agentVisibleContractText(materials.contractText ?? appendix(track, args.level), + args.credentialAliases, applicationInterface), + ]); + } + + const verb = args.mode === 'upgrade' + ? [ + 'Add the features below to the existing app.', + '', + 'Keep completed features working. Add only the current features below.', + ] + : [`Build the application described below and leave it running.`]; + const startingCatalog = args.mode === 'build' && materials.startingCatalog + ? ['', '## Starting catalog', '', 'Use exactly this starting data:', '', + '```json', materials.startingCatalog, '```'] : []; + + return prompt([ + ...verb, + '', + `After the web application is running, reply with ${args.mode === 'upgrade' + ? 'UPGRADE_COMPLETE' : 'DEPLOY_COMPLETE'}.`, + '', + ...common, + '', + agentVisibleContractText(materials.requirementText ?? levelPrompt(track, args.level), + args.credentialAliases, applicationInterface), + ...startingCatalog, + '', + '## Application interface', + '', + agentVisibleContractText(materials.contractText ?? appendix(track, args.level), + args.credentialAliases, applicationInterface), + ]); +} + +export function agentScenarioPaths(track: Track, level: number, + recipeBinding: RecipeBinding | null = null): string[] { + const execution = recipeBinding?.execution; + if (execution) return execution.map(entry => resolve(track.dir, entry.source ?? '')); + return suitesFor(track, level).map(suite => suite.spec); +} + +export function agentRecipeRequest(explicitRecipe: string | null = null, + recipeTask: RecipeTaskRequest | null = null): ExactRecipeRequest | null { + const bound = recipeTask?.recipe; + if (!bound) return explicitRecipe; + const identity = `${bound.id}@${bound.version}`; + if (explicitRecipe && explicitRecipe !== identity) { + throw new Error(`agent recipe ${explicitRecipe} does not match bound task ${identity}`); + } + return bound; +} + +// The coding container must not contain the controller or grading inputs. + +async function main() { + const args = parseAgentArgs(process.argv); + const track = loadTrack(args.track); + const p = portsFor(track, args.backend, args.runIndex); + const adapter = STACK_ADAPTER_REGISTRY.get(args.backend); + const defaultGuidance = resolveDefaultGuidanceForStack(args.guidance, args.backend); + args.credentialAliases ??= defaultGuidance?.credentialAliases ?? {}; + const profileSkills = defaultGuidance?.skills[args.backend]?.ids; + const defaultSkills = profileSkills ?? [...adapter.agent.defaultSkills]; + const selectedSkills = selectAgentSkills(defaultSkills, + args.skillIdentity?.ids ?? args.skills ?? null); + const skillsText = readAgentSkillDocuments(REPO, selectedSkills); + if (args.skillIdentity && (sha256(skillsText) !== args.skillIdentity.sha256 + || Buffer.byteLength(skillsText) !== args.skillIdentity.bytes)) { + throw new Error('campaign skill material changed after compilation'); + } + const recipeBinding = resolveRecipeRelease(track, args.level, + agentRecipeRequest(args.recipe ?? null, args.recipeTask ?? null)); + if (args.recipeTask && !recipeBinding) { + throw new Error(`L${args.level} has no recipe release for the requested task`); + } + const selectedTask = recipeBinding + ? (args.recipeTask + ? resolveBoundRecipeTaskRequest(recipeBinding, args.recipeTask) + : createBoundRecipeTaskRequest(recipeBinding)) + : null; + const requirementText = selectedTask?.task.requirementText ?? levelPrompt(track, args.level); + const contractText = selectedTask?.task.contractText ?? appendix(track, args.level); + const taskMode = isRecord(args.recipeTask?.task) ? args.recipeTask.task.mode : null; + const startingCatalog = recipeBinding && taskMode === 'fresh' ? JSON.stringify({ + warehouses: recipeBinding.plan.fixture.warehouses, + items: recipeBinding.plan.fixture.items, + }, null, 2) : undefined; + + // Print the exact prompt without starting a session or changing the app. + if (args.printPrompt) { + process.stdout.write(buildPrompt(args, p, track, + { skillsText, requirementText, contractText, startingCatalog })); + return; + } + if (args.mode === 'build') { + assertNewOrEmptyDirectory(args.app, 'build application directory'); + } + resolveIsolation(args); + const imageIdentity = resolveContainerImage(IMAGE); + // Build wipes all backend state. Later rounds preserve it. + ensureDatabase(args.backend, args.runIndex, p.dbPort, track, args.mode === 'build'); + // Never erase a caller-supplied application tree. + mkdirSync(args.app, { recursive: true }); + writeFileSync(resolve(args.app, '..', '.stack-bench-backend'), args.backend); + + const prompt = buildPrompt(args, p, track, + { skillsText, requirementText, contractText, startingCatalog }); + const bugReportPath = join(args.app, CODING_CONTAINER_BUG_REPORT_FILE); + const bugReportText = args.mode === 'fix' && existsSync(bugReportPath) + ? readFileSync(bugReportPath, 'utf8') : null; + const provenance = sessionProvenance({ prompt, skillsText, contractText, bugReportText, + scenarioPaths: agentScenarioPaths(track, args.level, recipeBinding), + trackDir: track.dir, trackManifestPath: join(track.dir, TRACK_MANIFEST_FILE) }); + const started = Date.now(); + const retryLimitRaw = process.env.STACK_BENCH_CODING_INTERRUPTION_RETRIES + ?? String(DEFAULT_CODING_INTERRUPTION_RETRIES); + const retryLimit = Number(retryLimitRaw); + if (!Number.isInteger(retryLimit) || retryLimit < 0 || retryLimit > 3) { + throw new Error('STACK_BENCH_CODING_INTERRUPTION_RETRIES must be an integer from 0 to 3'); + } + // The throttle wait must fit inside the adapter deadline. + const throttleWaitRaw = process.env.STACK_BENCH_PROVIDER_THROTTLE_MAX_WAIT_MINUTES + ?? String(DEFAULT_THROTTLE_MAX_WAIT_MS / 60_000); + const throttleMaxWaitMinutes = Number(throttleWaitRaw); + if (!Number.isInteger(throttleMaxWaitMinutes) || throttleMaxWaitMinutes < 0 + || throttleMaxWaitMinutes > DEFAULT_THROTTLE_MAX_WAIT_MS / 60_000) { + throw new Error('STACK_BENCH_PROVIDER_THROTTLE_MAX_WAIT_MINUTES must be an integer from 0 to ' + + `${DEFAULT_THROTTLE_MAX_WAIT_MS / 60_000}`); + } + // Concurrent campaign slots must not wake and retry as one burst. This + // stable offset keeps retries reproducible while spreading them over 45s. + const throttleJitterMs = parseInt(sha256(Buffer.from( + `${args.backend}:${args.runIndex}:${args.level}:${args.mode}`)).slice(0, 8), 16) % 45_001; + let coding: CodingSessionRetryResult; + try { + // Send prompts through stdin to avoid the Windows command-line limit. + const cliEnv = { ...process.env, + // Absent unless deliberately overridden — see THINKING_TOKENS above. + ...((args.thinking ?? THINKING_TOKENS) + ? { MAX_THINKING_TOKENS: String(args.thinking ?? THINKING_TOKENS) } + : {}), + // Keep the CLI fixed across the campaign. + DISABLE_AUTOUPDATER: '1', + // Pin cache lifetime so run order cannot change cost. + FORCE_PROMPT_CACHING_5M: '1' }; + + coding = runCodingSessionWithRetries({ prompt, model: args.model, retryLimit, + maxBudgetUsd: args.maxBudgetUsd, + throttleMaxWaitMs: throttleMaxWaitMinutes * 60_000, + throttleJitterMs, + invoke: ({ input, maxBudgetUsd, resumeSession, recoverStoppedContainer }) => + execFileSync(process.execPath, [ + compiledEntrypoint('container', 'run-build.js'), + '--app', args.app, + '--backend', args.backend, + '--image', imageIdentity.id, + '--effort', EFFORT, + '--model', args.model, + ...(args.pricing ? ['--pricing-json', JSON.stringify(args.pricing)] : []), + '--completion-marker', args.mode === 'fix' ? 'FIX_COMPLETE' + : args.mode === 'upgrade' ? 'UPGRADE_COMPLETE' + : args.mode === 'resume' ? 'RESUME_COMPLETE' : 'DEPLOY_COMPLETE', + ...(maxBudgetUsd != null ? ['--max-budget-usd', String(maxBudgetUsd)] : []), + '--ports', [p.vite, p.express].filter(Boolean).join(','), + ...(resumeSession ? ['--resume-session', resumeSession] : []), + ...(recoverStoppedContainer ? ['--recover-stopped-container'] : []), + ], { input, encoding: 'utf8', maxBuffer: 256 * 1024 * 1024, + env: { ...cliEnv, ...(args.apiKey ? { STACK_BENCH_AGENT_API_KEY: args.apiKey } : {}) }, + timeout: AGENT_PROCESS_TIMEOUT_MS }), + }); + } catch (err: unknown) { + coding = { raw: '', spawnError: codingSessionFailure(isRecord(err) ? err : {}), sessionResults: [], + interruptions: [], result: { total_cost_usd: 0, num_turns: 0, + usage: { input_tokens: 0, output_tokens: 0, cache_creation_input_tokens: 0, + cache_read_input_tokens: 0 }, stack_bench_cost_receipts: [] }, + throttle: { waits: 0, waitedMs: 0, maxWaitMs: throttleMaxWaitMinutes * 60_000, + jitterMs: throttleJitterMs } }; + } + + const { raw, spawnError, sessionResults, interruptions, result, throttle } = coding; + const noOutput = !result.session_id && !raw.trim(); + const failed = Boolean(spawnError || noOutput); + const providerFailure = providerSessionFailure(result); + const usage = sessionUsage(result.usage); + const input = usage.input_tokens ?? 0; + const output = usage.output_tokens ?? 0; + const cacheWrite = usage.cache_creation_input_tokens ?? 0; + const cacheRead = usage.cache_read_input_tokens ?? 0; + const turns = result.num_turns ?? 0; + + // Preserve the cost inputs needed to explain stack differences. + const setupMetadata = adapter.agent.setupMetadata({ + imageId: imageIdentity.id, + localPackage: LOCAL_PKG, + env: process.env, + helpers: { linuxSpacetimeVersion, bindingsIdentity, containerImage }, + }); + const out = { + appDir: args.app, + mode: args.mode, + level: args.level, + track: args.track, + backend: args.backend, + model: args.model, + guidance: args.guidance, + setup: { + thinkingTokens: (args.thinking ?? THINKING_TOKENS) ? Number(args.thinking ?? THINKING_TOKENS) : 'cli default', + permissionMode: 'acceptEdits', + effort: EFFORT, + skills: selectedSkills, + cacheTier: '5m', + autoUpdater: 'disabled', + codingInterruptionRetries: { limit: retryLimit, + used: interruptions.filter(item => item.kind !== 'provider-throttled').length }, + providerThrottle: { maxWaitMinutes: throttleMaxWaitMinutes, + waits: throttle?.waits ?? 0, waitedMs: throttle?.waitedMs ?? 0, + jitterMs: throttle?.jitterMs ?? throttleJitterMs }, + cliVersion: imageCliVersion(imageIdentity.id), + isolation: { mode: 'container', image: imageIdentity.reference, + imageId: imageIdentity.id, hostAlias: HOST_ADDR }, + auth: (args.apiKey ?? process.env.ANTHROPIC_API_KEY) ? 'api-key' + : (process.env.CLAUDE_CODE_OAUTH_TOKEN || process.env.CLAUDE_CODE_OAUTH_TOKEN_FILE) + ? 'subscription-token' : 'not-selected', + ...(isRecord(setupMetadata) ? setupMetadata : {}), + env: ambientEnv(), + node: { orchestrator: process.version, codingContainer: imageNodeVersion(imageIdentity.id) }, + platform: process.platform, + resources: result.stack_bench_resources ?? null, + }, + costUsd: Number((result.total_cost_usd ?? 0).toFixed(6)), + costReceipts: result.stack_bench_cost_receipts ?? [], + tokens: input + output + cacheWrite + cacheRead, + outputTokens: output, + usage: { input, output, cacheWrite, cacheRead }, + provenance, + turns, + promptBytes: Buffer.byteLength(prompt), + tokensPerTurn: turns ? Math.round((input + output + cacheWrite + cacheRead) / turns) : null, + thinking: combinedThinkingVolume(args.app, sessionResults.map(item => item.session_id)), + durationMs: Date.now() - started, + sessionId: result.session_id ?? null, + ok: !failed && result.is_error === false, + providerMetadata: { failureCode: failed + ? String(spawnError ?? '').startsWith('provider stayed throttled') + ? 'provider-throttle-exhausted' + : providerFailure?.code ?? (noOutput ? 'coding-session-no-output' : 'coding-session-failed') + : result.is_error === true ? 'provider-session-error' : null, + diagnostic: spawnError, + failure: failed ? { + providerStatus: result.api_error_status ?? null, + waitedMs: throttle?.waitedMs ?? 0, + waits: throttle?.waits ?? 0, + } : null, + interruptions, invocations: sessionResults.length, + terminalRecovery: isRecord(result) ? result.terminal_recovery ?? null : null, + credentialBroker: result.stack_bench_credential_broker ?? null, + sessionIds: [...new Set(sessionResults.map(item => item.session_id).filter(Boolean))], + models: transcriptModels(args.app, sessionResults.map(item => item.session_id)) }, + }; + console.log(JSON.stringify(out)); +} + +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + main().catch(err => { console.error(err); process.exit(1); }); +} diff --git a/tools/stack-bench/commands/bench-arguments.ts b/tools/stack-bench/commands/bench-arguments.ts new file mode 100644 index 00000000000..ac4f486e5e1 --- /dev/null +++ b/tools/stack-bench/commands/bench-arguments.ts @@ -0,0 +1,344 @@ +import { dirname, resolve } from 'node:path'; +import { parseArgs } from 'node:util'; +import { readArtifact } from '../src/evidence/artifacts.js'; +import { DEFAULT_BUILD_IMAGE } from '../src/composition/product-config.js'; +import { DEFAULT_TRACK, RUN_INDEX_CAP } from '../src/composition/tracks.js'; +import { validateCompiledCampaignPlan } from '../src/campaigns/campaign-compiler.js'; +import type { CampaignAttemptPlan, CampaignSelection } + from '../src/campaigns/campaign-compiler.js'; +import { campaignAdmissionSmokeReuse, readCampaignAdmission } + from '../src/campaigns/campaign-admission.js'; +import type { CampaignAdmissionSmokeReuse } from '../src/campaigns/campaign-admission.js'; +import { compileProgressionInput, dependencyRuntimeDefinition, progressionLevels, + validateFeatureCatalogInput, validateProgressionInput } + from '../src/progression/progression-definition.js'; +import type { CompiledDependencyPolicyDefinition, CompiledProgressionDefinition, + ProgressionInput } from '../src/progression/progression-definition.js'; +import { validatePricingAuthority } from '../src/evidence/pricing-authority.js'; +import type { PricingAuthority } from '../src/evidence/pricing-authority.js'; +import { parseGuidanceMode } from '../src/campaigns/condition-compiler.js'; +import type { GuidanceMode } from '../src/campaigns/condition-compiler.js'; +import { validateCampaignExtensionSeed } from '../src/campaigns/campaign-scheduler.js'; +import type { CampaignExtensionSeed } from '../src/campaigns/campaign-scheduler.js'; +import { repairBudgetLimit } from '../src/progression/repair-plan.js'; + +type StudyCondition = CampaignAttemptPlan['condition']; +type UnknownRecord = Record; + +export interface BenchArguments { + backend?: string; + track: string; + levels: string; + levelsProvided: boolean; + levelList: number[]; + model: string | null; + agentAdapter: string; + pricing?: PricingAuthority | null; + repairs: number; + maxStalledRepairs: number; + maxBudgetUsd?: number; + runIndex: number; + out?: string; + app?: string; + url?: string; + media: boolean; + retainBackend?: boolean; + guidance: GuidanceMode; + guidanceDocument?: unknown; + condition?: StudyCondition; + selectionRequest?: CampaignSelection; + taskMode?: string; + packIds: string[]; + checkKeys: string[]; + featureIds: string[]; + requestedSpecifications: string[]; + expectedSpecifications: string[]; + observedSpecifications: string[]; + skills?: string[]; + apiKey?: string; + apiKeyFile?: string; + mutations?: string; + mutationShardIndex?: number; + mutationShardCount?: number; + mutationResumeFrom?: string; + mutationCheckpointOut?: string; + mutationBaselineBundle?: string; + expectedMutationCalibration?: unknown; + mutationMaxRuntimeMinutes: number; + referenceMutationOnly?: boolean; + seedFrom?: string; + seedThrough?: number; + progressionSeed?: CampaignExtensionSeed; + parentAttemptId?: string; + repairFrom?: string; + repairLevel?: number; + recipe?: string; + campaignFile?: string; + campaignAttemptId?: string; + campaignAdmissionId?: string; + progressionResumeFrom?: string; + experimentIdentity?: { id: string; version: string; sha256: string; state: string }; + campaignAdmission?: { id: string } & CampaignAdmissionSmokeReuse; + runMode?: CampaignAttemptPlan['mode']; + featureCatalog?: ProgressionInput; + dependencyPolicy?: ProgressionInput; + progression?: ProgressionInput; + progressionOwner?: UnknownRecord; +} + +interface BenchCliOptions extends Partial { + pack?: string[]; + check?: string[]; + pricingJson?: unknown; + featureModule?: string[]; + requestSpec?: string[]; + expectSpec?: string[]; + observeSpec?: string[]; + expectedMutationCalibrationJson?: unknown; + progressionSeedJson?: unknown; +} + +function parseCli(argv: readonly string[]): BenchCliOptions { + const strings = ['backend', 'track', 'levels', 'campaign-file', 'campaign-attempt-id', + 'campaign-admission-id', 'progression-resume-from', 'recipe', 'model', 'pricing-json', + 'repairs', 'max-stalled-repairs', 'max-budget-usd', 'run-index', 'out', 'app', 'url', + 'agent-adapter', 'guidance', 'task-mode', 'skills', 'mutations', + 'mutation-shard-index', 'mutation-shard-count', 'mutation-resume-from', + 'mutation-checkpoint-out', 'mutation-baseline-bundle', + 'expected-mutation-calibration-json', 'mutation-max-runtime-minutes', 'seed-from', + 'seed-through', 'progression-seed-json', + 'parent-attempt-id', 'repair-from', 'repair-level'] as const; + const multiple = ['pack', 'check', 'feature-module', 'request-spec', 'expect-spec', + 'observe-spec'] as const; + const options = Object.fromEntries([ + ...strings.map(name => [name, { type: 'string' as const }]), + ...multiple.map(name => [name, { type: 'string' as const, multiple: true }]), + ...['no-media', 'retain-backend', 'reference-mutation-only'].map(name => + [name, { type: 'boolean' as const }]), + ]); + const { values } = parseArgs({ args: [...argv.slice(2)], options, strict: true, + allowPositionals: false }); + const parsed: Record = {}; + for (const [key, value] of Object.entries(values)) { + parsed[key.replace(/-([a-z])/g, (_, letter: string) => letter.toUpperCase())] = value; + } + for (const key of ['pack', 'check', 'featureModule', 'requestSpec', 'expectSpec', + 'observeSpec']) { + const value = parsed[key] as string[] | undefined; + if (value) parsed[key] = value.flatMap(item => item.split(',').filter(Boolean)); + } + for (const key of ['repairs', 'maxStalledRepairs', 'maxBudgetUsd', 'mutationShardIndex', + 'mutationShardCount', 'mutationMaxRuntimeMinutes', 'repairLevel', 'seedThrough']) { + if (typeof parsed[key] === 'string') parsed[key] = Number(parsed[key]); + } + if (typeof parsed.runIndex === 'string') parsed.runIndex = Number(parsed.runIndex); + for (const key of ['campaignFile', 'progressionResumeFrom', 'mutations', + 'mutationResumeFrom', 'mutationCheckpointOut', 'mutationBaselineBundle', 'repairFrom']) { + if (typeof parsed[key] === 'string') parsed[key] = resolve(parsed[key]); + } + for (const key of ['pricingJson', 'expectedMutationCalibrationJson', 'progressionSeedJson']) { + if (typeof parsed[key] === 'string') parsed[key] = JSON.parse(parsed[key]); + } + if (typeof parsed.guidance === 'string') parsed.guidance = parseGuidanceMode(parsed.guidance); + if (typeof parsed.skills === 'string') parsed.skills = parsed.skills.split(',').filter(Boolean); + if (parsed.noMedia === true) parsed.media = false; + delete parsed.noMedia; + return parsed as BenchCliOptions; +} + +export function parseBenchArguments(argv: readonly string[]): BenchArguments { + const args: BenchArguments = { model: null, agentAdapter: 'claude-code', + repairs: 10, runIndex: 0, levels: '1', levelsProvided: false, media: true, + levelList: [], maxStalledRepairs: 3, guidance: 'prescribed', track: DEFAULT_TRACK, + packIds: [], checkKeys: [], featureIds: [], requestedSpecifications: [], + expectedSpecifications: [], observedSpecifications: [], + mutationMaxRuntimeMinutes: 60 }; + const { pack, check, pricingJson, featureModule, requestSpec, expectSpec, observeSpec, + expectedMutationCalibrationJson, progressionSeedJson, ...options } = parseCli(argv); + Object.assign(args, options); + if (pack) args.packIds = pack; + if (check) args.checkKeys = check; + if (pricingJson !== undefined) { + args.pricing = validatePricingAuthority(pricingJson, { at: '--pricing-json' }); + } + if (featureModule) args.featureIds = featureModule; + if (requestSpec) args.requestedSpecifications = requestSpec; + if (expectSpec) args.expectedSpecifications = expectSpec; + if (observeSpec) args.observedSpecifications = observeSpec; + args.levelsProvided = options.levels !== undefined; + if (expectedMutationCalibrationJson !== undefined) { + args.expectedMutationCalibration = expectedMutationCalibrationJson; + } + if (progressionSeedJson !== undefined) { + args.progressionSeed = validateCampaignExtensionSeed(progressionSeedJson); + } + if ((args.mutationResumeFrom || args.mutationCheckpointOut || args.mutationBaselineBundle) + && !args.mutations) { + throw new Error('mutation control options require --mutations'); + } + if (args.expectedMutationCalibration && !args.mutations) { + throw new Error('--expected-mutation-calibration-json requires --mutations'); + } + if (!Number.isFinite(args.mutationMaxRuntimeMinutes) || args.mutationMaxRuntimeMinutes < 1 + || args.mutationMaxRuntimeMinutes > 120) { + throw new Error('--mutation-max-runtime-minutes must be from 1 through 120'); + } + if (args.referenceMutationOnly && (!args.mutations || args.agentAdapter !== 'reference-fixture' + || args.repairs !== 0 || !args.app || args.campaignFile)) { + throw new Error('--reference-mutation-only requires a mutation-bound reference fixture run'); + } + if (args.mutationBaselineBundle && !args.referenceMutationOnly) { + throw new Error('--mutation-baseline-bundle is an internal reference mutation option'); + } + if (args.repairFrom && (args.repairLevel === undefined + || !Number.isSafeInteger(args.repairLevel) || args.repairLevel < 1)) { + throw new Error('--repair-from requires --repair-level with a positive integer'); + } + if (args.campaignFile && !args.campaignAttemptId) { + throw new Error('--campaign-file requires --campaign-attempt-id'); + } + if (!args.campaignFile && (args.campaignAttemptId || args.campaignAdmissionId)) { + throw new Error('campaign binding requires --campaign-file'); + } + if (args.progressionResumeFrom && !args.campaignFile) { + throw new Error('--progression-resume-from requires a compiled campaign'); + } + if (args.seedThrough !== undefined && (!args.seedFrom || !args.campaignFile)) { + throw new Error('--seed-through requires --seed-from and a compiled campaign'); + } + if (args.seedThrough !== undefined && !args.progressionSeed) { + throw new Error('--seed-through requires --progression-seed-json'); + } + if (args.progressionSeed !== undefined && args.seedThrough === undefined) { + throw new Error('--progression-seed-json requires --seed-through'); + } + if (args.progressionSeed && args.progressionSeed.fromDepth !== args.seedThrough) { + throw new Error('--progression-seed-json does not match --seed-through'); + } + if (args.campaignFile) { + const allowed = new Set(['--campaign-file', '--campaign-attempt-id', + '--campaign-admission-id', '--progression-resume-from', '--run-index', '--out', + '--max-budget-usd', '--seed-from', '--seed-through', '--progression-seed-json']); + const override = argv.slice(2).find(value => value.startsWith('--') + && !allowed.has(value.split('=', 1)[0]!)); + if (override) throw new Error(`campaign attempts cannot override ${override}`); + bindCampaign(args); + } + if (!args.backend && !args.repairFrom) { + throw new Error('--backend is required unless --repair-from or --campaign-file is supplied'); + } + if (args.progression) { + if (args.levelsProvided) throw new Error('--levels cannot be combined with progression input'); + args.progression = validateProgressionInput(args.progression); + args.levelList = progressionLevels(args.progression); + args.levels = `${args.levelList[0]}-${args.levelList.at(-1)}`; + const seedThrough = args.seedThrough; + if (seedThrough !== undefined && (!args.levelList.includes(seedThrough) + || !args.levelList.some(level => level > seedThrough))) { + throw new Error('--seed-through must precede another planned dependency depth'); + } + if (seedThrough !== undefined + && args.dependencyPolicy?.definition.workSelection !== 'progressive') { + throw new Error('--seed-through requires progressive dependency work selection'); + } + } else { + const [fromText, toText] = args.levels.split('-'); + const from = Number(fromText); + const to = toText === undefined ? from : Number(toText); + if (!Number.isSafeInteger(from) || from < 1 || !Number.isSafeInteger(to) || to < from) { + throw new Error('--levels must be one positive level or an ascending range'); + } + args.levelList = Array.from({ length: (to ?? from) - from + 1 }, (_, index) => from + index); + if (args.seedThrough !== undefined) { + throw new Error('--seed-through requires dependency mode'); + } + } + if (args.recipe && args.levelList.length !== 1) { + throw new Error('--recipe requires exactly one requested level'); + } + if (!Number.isSafeInteger(args.repairs) || args.repairs < 0) { + throw new Error('--repairs must be a non-negative safe integer'); + } + if (!Number.isInteger(args.maxStalledRepairs) || args.maxStalledRepairs < 0 + || args.maxStalledRepairs > 20) { + throw new Error('--max-stalled-repairs must be an integer from 0 through 20'); + } + if (args.maxBudgetUsd !== undefined + && (!Number.isFinite(args.maxBudgetUsd) || args.maxBudgetUsd <= 0)) { + throw new Error('--max-budget-usd must be a positive number'); + } + if (!Number.isSafeInteger(args.runIndex) || args.runIndex < 0 || args.runIndex > RUN_INDEX_CAP) { + throw new Error(`--run-index must be an integer from 0 through ${RUN_INDEX_CAP}`); + } + if ((args.mutationShardIndex === undefined) !== (args.mutationShardCount === undefined)) { + throw new Error('--mutation-shard-index and --mutation-shard-count must be supplied together'); + } + return args; +} + +function bindCampaign(args: BenchArguments): void { + if (!args.campaignFile) throw new Error('campaign file is required'); + const artifact = readArtifact(args.campaignFile, { expectedKind: 'campaign_plan' }); + const plan = validateCompiledCampaignPlan(artifact.payload); + const attempt = plan.attempts.find(item => item.id === args.campaignAttemptId); + if (!attempt) throw new Error('--campaign-attempt-id is not in the compiled campaign plan'); + const plannedBudget = plan.definition.budgets.maxCostUsdPerAttempt; + if (args.maxBudgetUsd !== undefined + && (plannedBudget === null || args.maxBudgetUsd > plannedBudget)) { + throw new Error('--max-budget-usd exceeds the compiled campaign budget'); + } + args.backend = attempt.stack; + args.track = plan.definition.track; + args.model = attempt.model; + args.agentAdapter = attempt.agentAdapter; + args.pricing = validatePricingAuthority(attempt.pricing, { at: 'compiled campaign pricing' }); + args.guidance = parseGuidanceMode(attempt.guidance); + args.condition = structuredClone(attempt.condition); + args.skills = structuredClone(attempt.skills); + args.selectionRequest = structuredClone(plan.definition.selection); + args.guidanceDocument = structuredClone( + attempt.condition.guidance.documents[attempt.stack]); + args.packIds = structuredClone(plan.definition.selection.packs ?? []); + args.checkKeys = structuredClone(plan.definition.selection.checks ?? []); + args.repairs = attempt.mode.id === 'dependency' + ? 0 : repairBudgetLimit(plan.definition.repair); + args.maxBudgetUsd ??= plannedBudget ?? undefined; + args.parentAttemptId = attempt.id; + args.media = false; + args.levels = `${Math.min(...attempt.levels)}-${Math.max(...attempt.levels)}`; + args.experimentIdentity = { + id: plan.id, version: plan.version, sha256: plan.contentSha256, state: plan.state, + }; + if (args.campaignAdmissionId) { + const admission = readCampaignAdmission(dirname(args.campaignFile), + args.campaignAdmissionId, plan); + const image = plan.definition.runtime.buildImage + ?? process.env.STACK_BENCH_IMAGE ?? DEFAULT_BUILD_IMAGE; + args.campaignAdmission = { + id: args.campaignAdmissionId, + ...campaignAdmissionSmokeReuse(admission, { + agentAdapter: args.agentAdapter, + runIndex: args.runIndex, + backend: args.backend, + image, + }), + }; + } + args.runMode = structuredClone(attempt.mode); + if (plan.featureCatalog) { + args.featureCatalog = validateFeatureCatalogInput(plan.featureCatalog); + } + if (attempt.mode.id === 'dependency') { + if (!plan.dependencyPolicy || !args.featureCatalog) { + throw new Error('dependency campaign requires a feature catalog and dependency policy'); + } + args.dependencyPolicy = plan.dependencyPolicy; + args.progression = compileProgressionInput(dependencyRuntimeDefinition( + args.featureCatalog, args.dependencyPolicy)); + args.progressionOwner = { schemaVersion: 1, + campaign: { id: plan.id, version: plan.version, sha256: plan.contentSha256 }, + attempt: { id: attempt.id, track: plan.definition.track, stack: attempt.stack, + agentAdapter: attempt.agentAdapter, model: attempt.model, + conditionSha256: attempt.condition.sha256 } }; + } +} diff --git a/tools/stack-bench/commands/bench.ts b/tools/stack-bench/commands/bench.ts new file mode 100644 index 00000000000..7ca98080553 --- /dev/null +++ b/tools/stack-bench/commands/bench.ts @@ -0,0 +1,2678 @@ +#!/usr/bin/env node + +import { execFile, execFileSync } from 'node:child_process'; +import type { ChildProcess, ExecFileSyncOptionsWithStringEncoding } from 'node:child_process'; +import { readFileSync, writeFileSync, mkdirSync, existsSync, cpSync, rmSync } from 'node:fs'; +import { join, dirname, resolve } from 'node:path'; +import { tmpdir } from 'node:os'; +import { pathToFileURL } from 'node:url'; +import { loadTrack, resultsName, portsFor, workDirFor, assertNoPortCollisions, + moduleName, dbName, suitesFor } from '../src/composition/tracks.js'; +import { parseBenchArguments } from './bench-arguments.js'; +import type { BenchArguments } from './bench-arguments.js'; +import { killTree } from '../src/runtime/platform.js'; +import { formatRepairProgress } from '../src/evidence/scoring.js'; +import { ARTIFACT_FILE, emptyArtifactIdentities, readArtifact, readArtifactPayload, + writeArtifact, writeRunJson } from '../src/evidence/artifacts.js'; +import { aggregateRunOutcome, classifyBundle, ladderMayAdvance, ladderMayContinue, + mutationControlEligible, runExitCode, runOutcomeKind } from '../src/evidence/outcomes.js'; +import { summarizeSessions } from '../src/evidence/session-metrics.js'; +import { hashDirectory, sha256 } from '../src/evidence/provenance.js'; +import { createBackendLease, newRunId, publicBackendLease, readBackendLease, + acquireResourceLocks, backendResourceLockKeys, releaseResourceLocks, resourceLockScope, + writeBackendLease } from '../src/runtime/backend-lease.js'; +import { captureApplicationDiagnostics } from '../src/runtime/backend-control.js'; +import type { RuntimeControlSpec } from '../src/runtime/backend-control.js'; +import { releaseBackendLease } from '../src/runtime/backend-teardown.js'; +import { resolveRecipeRelease } from '../src/composition/recipe-release.js'; +import { createAgentVisibleTaskRequest, createBoundRecipeTaskRequest } + from '../src/composition/recipe-selection.js'; +import { criterionEvidence, evidencePassed } from '../src/evidence/check-evidence.js'; +import { STACK_ADAPTER_REGISTRY } from '../src/stacks/stack-adapters.js'; +import { agentRecipeIdentity, agentRequestArgv } from '../src/agents/agent-adapter-contract.js'; +import { agentSessionFailure, validateAgentResult } + from '../src/agents/agent-result-contract.js'; +import { AGENT_ADAPTER_REGISTRY, agentAdapterIdentity } from '../src/agents/agent-adapters.js'; +import { archiveTranscripts } from '../src/agents/transcript-archive.js'; +import { runPreflight } from '../src/runtime/preflight.js'; +import { DEFAULT_BUILD_IMAGE } from '../src/composition/product-config.js'; +import { SUPERVISOR_STATE_VERSION, writeRecoveryArtifact } from '../src/runtime/recovery.js'; +import { applyAgentCredential } from '../src/agents/agent-credentials.js'; +import { hashAppSource, resetAppToSource, seedAppSource, snapshotAppSource } from '../src/runtime/source-snapshot.js'; +import { finalPackageEvidenceRequired, preserveFinalPackageEvidence, preserveLevelCheckpoint, + sourceBoundFirstBuildOutcome } from '../src/runtime/source-checkpoint.js'; +import { materializationAppFailure, materializeAcceptedSource } + from '../src/runtime/source-materialization.js'; +import { compareRepairBaseline, createRepairGrant } from '../src/runtime/repair-grant.js'; +import { canonicalDefinitionJson } from '../src/composition/definition-plan.js'; +import { contractInterfaceNames } from '../src/composition/agent-visible-contract.js'; +import { clearPrivateGradingEvidence, levelGradeIsUsable, repairEvidenceDecision, + repairHistoryEntry, repairProgressState, repairRegressionDecision, + restorePrivateGradingEvidence } + from '../src/evidence/repair-evidence.js'; +import { mutationControlArgv, mutationControlTimeoutMs, pristineMutationBaselinePath } + from '../src/evidence/mutation-control.js'; +import type { MutationControlArgs } from '../src/evidence/mutation-control.js'; +import { progressionEngine } from '../src/progression/progression-engine.js'; +import { dependencyRepairBudget, dependencyRepairRecords } + from '../src/progression/dependency-mode.js'; +import { DEPENDENCY_MODE_VERSION } from '../src/progression/dependency-definition.js'; +import { resolveProgressionRecipeAction, resolveProgressionRecipeLevelSelection, + resolveProgressionRepairTarget, validateProgressionCampaignLevelScope } + from '../src/progression/progression-recipe-selection.js'; +import { createLiveProgressionExecution } + from '../src/progression/live-progression.js'; +import type { CampaignSelection } from '../src/campaigns/campaign-compiler.js'; +import { gradingRunTimeoutMs, selectedGradingSourceCount } + from '../src/runtime/grading-timeout.js'; +import { claudeRatesForModel } from '../src/evidence/claude-usage-cost.js'; +import { PRICING_UNIT, validatePricingAuthority } + from '../src/evidence/pricing-authority.js'; +import type { BoundRecipeTaskRequestResult } from '../src/composition/recipe-selection.js'; +import type { RecipeBinding } from '../src/composition/recipe-release.js'; +import type { RepairGrantResolution, RepairOutcome } from '../src/runtime/repair-grant.js'; +import type { AgentAdapter, AgentMode, AgentRequest } + from '../src/agents/agent-adapter-contract.js'; +import type { ValidatedAgentResult } from '../src/agents/agent-result-contract.js'; +import type { Track } from '../src/composition/tracks.js'; +import type { RunOutcome } from '../src/evidence/outcomes.js'; +import type { GradeBundlePayload, BenchmarkRunRecord, RunLevelRecord, + RunContinuation, RunRepairCandidate, RunSessionRecord, RunTotals } + from '../src/evidence/benchmark-run.js'; +import { addCostUsd, finalizeRunTotals, runSessionRecord } + from '../src/evidence/benchmark-run.js'; +import { formatLevelSummary } from '../src/evidence/evidence-presentation.js'; +import type { ProgressionAction } from '../src/progression/progression-engine.js'; +import type { ProgressionRepairRegression, ProgressionState } + from '../src/progression/progression-state.js'; +import type { ProgressionRecipeAction, ProgressionRecipeSelections } + from '../src/progression/progression-recipe-selection.js'; +import type { BackendLease } from '../src/runtime/backend-lease.js'; + +import { STACK_BENCH_ROOT as ROOT, compiledEntrypoint } from '../src/package-root.js'; +import { stackBenchResultsRoot } from '../src/runtime/operational-paths.js'; +import { runningContainerIdentity } from '../src/runtime/container-identity.js'; +const COMMAND_TIMEOUT_MS = 20 * 60_000; + +type UnknownRecord = Record; +type ContaminationAudit = { kind: 'contaminated' | 'harness_failure'; evidence: string[]; + verdict: string }; +type LeakAuditEntry = { hits: Array<{ kind: string; path: string }> }; +type CommandFailure = Error & { stdout?: string | Buffer; stderr?: string | Buffer; + status?: number | null; signal?: NodeJS.Signals | null }; +type GradeOptions = { observation?: 'scored' | 'observed'; out?: string | null; + sourceSha256?: string | null; applicationFailure?: RunOutcome | null; + recipeTask?: GradeRecipeTask }; +type MutationControlResult = UnknownRecord & { ok: boolean; artifact?: string; + skipped?: boolean; processError?: string | null; outcome: RunOutcome | null }; +type RecipeTask = (BoundRecipeTaskRequestResult | ProgressionRecipeSelections['grader']) & { agentRequest?: UnknownRecord; + progressionAction?: ProgressionAction }; +type BenchArgs = BenchArguments & { + recipeTasks: Map; + recipeBindings: Map; + repairGrant?: RepairGrantResolution; + mutationImageId?: string; + spentBudgetUsd?: number; +}; +type ProgressionWorkRecipeAction = ProgressionRecipeSelections & { + action: Exclude; +}; +type FirstBuildRecord = { + score: number | null; + max: number | null; + regression: NonNullable['regression'] | null; + contractPass: boolean | null; + outcome: RunOutcome; + source: { sha256: string; files: number } | null; + missed: string[]; + observations?: UnknownRecord; +}; +type RepairStatus = 'not-needed' | 'corrected' | 'budget-exhausted' | 'incomplete' | 'ungraded'; +type ProgressionFailure = { kind?: string; reason?: string }; + +const object = (value: unknown): value is UnknownRecord => + value !== null && typeof value === 'object' && !Array.isArray(value); + +const errorMessage = (error: unknown): string => + error instanceof Error ? error.message : String(error); + +function commandFailure(error: unknown): CommandFailure { + if (error instanceof Error) return error; + throw error; +} + +function parseLeakAudit(value: string): LeakAuditEntry[] { + const parsed: unknown = JSON.parse(value); + if (!Array.isArray(parsed)) throw new Error('contamination audit output must be an array'); + return parsed.map((entry, index) => { + if (!object(entry) || !Array.isArray(entry.hits)) { + throw new Error(`contamination audit output[${index}] is invalid`); + } + const hits = entry.hits.map((hit, hitIndex) => { + if (!object(hit) || typeof hit.kind !== 'string' || typeof hit.path !== 'string') { + throw new Error(`contamination audit output[${index}].hits[${hitIndex}] is invalid`); + } + return { kind: hit.kind, path: hit.path }; + }); + return { hits }; + }); +} + +function stringArray(value: unknown, at: string): string[] { + if (!Array.isArray(value) || value.some(entry => typeof entry !== 'string')) { + throw new Error(`${at} must be an array of strings`); + } + return [...value]; +} + +function campaignSelection(value: unknown, at: string): CampaignSelection { + if (!object(value)) throw new Error(`${at} must be an object`); + const optionalStrings = (field: 'packs' | 'checks'): string[] | undefined => { + const entry = value[field]; + if (entry === undefined) return undefined; + return stringArray(entry, `${at}.${field}`); + }; + let levels: CampaignSelection['levels']; + if (value.levels !== undefined) { + if (!Array.isArray(value.levels)) throw new Error(`${at}.levels must be an array`); + levels = value.levels.map((entry, index) => { + if (!object(entry)) throw new Error(`${at}.levels[${index}] is invalid`); + const level = entry.level; + const recipe = entry.recipe; + if (typeof level !== 'number' || !Number.isSafeInteger(level) || typeof recipe !== 'string') { + throw new Error(`${at}.levels[${index}] is invalid`); + } + return { level, recipe, + ...(entry.features === undefined ? {} : { features: stringArray(entry.features, + `${at}.levels[${index}].features`) }), + ...(entry.checks === undefined ? {} : { checks: stringArray(entry.checks, + `${at}.levels[${index}].checks`) }) }; + }); + } + return { ...(optionalStrings('packs') === undefined ? {} : { packs: optionalStrings('packs') }), + ...(optionalStrings('checks') === undefined ? {} : { checks: optionalStrings('checks') }), + ...(levels === undefined ? {} : { levels }) }; +} + +function isProgressionWorkRecipeAction(value: ProgressionRecipeAction): + value is ProgressionWorkRecipeAction { + return value.action.type !== 'terminal'; +} + +function repairCheckKeys(value: ProgressionRecipeAction | null): string[] { + if (!value || !isProgressionWorkRecipeAction(value) || value.action.type !== 'repair') return []; + if (!object(value.action.prompt) || !Array.isArray(value.action.prompt.nodeIds) + || !object(value.action.grading) || !Array.isArray(value.action.grading.checks)) { + throw new Error('dependency repair action has invalid prompt or grading selections'); + } + const promptNodeIds = new Set(value.action.prompt.nodeIds.map(nodeId => { + if (typeof nodeId !== 'string' || !nodeId) { + throw new Error('dependency repair action has an invalid prompt node'); + } + return nodeId; + })); + const checks = value.action.grading.checks.flatMap(check => { + if (!object(check) || typeof check.id !== 'string' || !check.id + || typeof check.nodeId !== 'string' || !check.nodeId) { + throw new Error('dependency repair action has an invalid grading check'); + } + return promptNodeIds.has(check.nodeId) ? [check.id] : []; + }); + if (checks.length === 0) throw new Error('dependency repair action selects no repair checks'); + return checks; +} + +function repairReportArgs(value: ProgressionRecipeAction | null): string[] { + const checks = repairCheckKeys(value); + if (!value || !isProgressionWorkRecipeAction(value) || checks.length === 0) return []; + const interfaces = contractInterfaceNames(value.agent.task.contractText); + return ['--checks-json', JSON.stringify(checks), + '--controls-json', JSON.stringify(interfaces)]; +} + +function repairOwnerNodeIds(value: ProgressionRecipeAction | null): string[] { + if (!value || !isProgressionWorkRecipeAction(value) || value.action.type !== 'repair') return []; + return [...value.action.repair.nodeIds].sort(); +} + +function savedRepairRegression(state: ProgressionState | null, + selected: ProgressionRecipeAction | null): ProgressionRepairRegression | null { + const saved = state?.attempts.at(-1)?.repairRegression; + if (!saved) return null; + const owners = repairOwnerNodeIds(selected); + return JSON.stringify([...saved.ownerNodeIds].sort()) === JSON.stringify(owners) + ? structuredClone(saved) : null; +} + +function requireProgressionState(state: ProgressionState | null): ProgressionState { + if (!state) throw new Error('live dependency progression has no active state'); + return state; +} + +function requireContinuation(run: BenchmarkRunRecord): RunContinuation { + if (!run.continuation) throw new Error('repair continuation has no continuation record'); + return run.continuation; +} + +function requireRunTotals(run: BenchmarkRunRecord): RunTotals { + if (!run.totals) throw new Error('benchmark run totals are not available'); + return run.totals; +} + +function repairOutcome(outcome: RunOutcome): RepairOutcome { + return { kind: outcome.kind, appFailures: [...(outcome.appFailures ?? [])], + inconclusive: [...(outcome.inconclusive ?? [])], + harnessFailures: [...(outcome.harnessFailures ?? [])] }; +} + +function progressionFailure(outcome: RunOutcome): ProgressionFailure { + return { kind: outcome.kind, ...(outcome.reason === null || outcome.reason === undefined + ? {} : { reason: outcome.reason }) }; +} + +function featureCheckKeys(selected: ProgressionWorkRecipeAction, + state: ProgressionState): string[] { + if (!object(selected.action.prompt) || !Array.isArray(selected.action.prompt.nodeIds)) { + throw new Error('feature action has no prompt nodes'); + } + const selectedNodes = new Set(selected.action.prompt.nodeIds); + return state.definition.nodes + .filter(node => selectedNodes.has(node.id)) + .flatMap(node => node.gradingChecks + .filter(check => check.role === 'feature') + .map(check => check.id)); +} + +function bundlePassedChecks(bundle: GradeBundlePayload | null): Set { + return new Set(Object.values(bundle?.suites ?? {}).flatMap(suite => + (suite?.features ?? []).flatMap(feature => + (feature.criteria ?? []).flatMap(criterion => + typeof criterion.stableKey === 'string' + && evidencePassed(criterionEvidence(criterion)) ? [criterion.stableKey] : [])))); +} + +function featureCandidateAccepted(selected: ProgressionWorkRecipeAction, + state: ProgressionState, candidate: GradeBundlePayload | null): boolean { + if (!levelGradeIsUsable(classifyBundle(candidate))) return false; + const passed = bundlePassedChecks(candidate); + if (!featureCheckKeys(selected, state).every(check => passed.has(check))) return false; + return state.definition.nodes.every(node => node.gradingChecks.every(check => + state.nodes[node.id]?.checks[check.id] !== 'pass' || passed.has(check.id))); +} + +function featureActionNeedsCoding(selected: ProgressionWorkRecipeAction, + state: ProgressionState): boolean { + if (selected.action.type === 'repair') return true; + if (!object(selected.action.prompt) || !Array.isArray(selected.action.prompt.nodeIds)) { + throw new Error('feature action has no prompt nodes'); + } + return selected.action.prompt.nodeIds.some(nodeId => { + const node = state.nodes[String(nodeId)]; + return node?.status === 'active' + && Object.values(node.checks).every(outcome => outcome === null); + }); +} + +function mergeFeatureLevelRecord(previous: RunLevelRecord | null, + current: RunLevelRecord): RunLevelRecord { + if (!previous) return current; + const buildSessions: RunSessionRecord[] = [ + ...(previous.buildSessions ?? []), + ...(current.buildSessions ?? []), + ]; + const repairSessions = [...(previous.repairSessions ?? []), ...(current.repairSessions ?? [])]; + const sessionTotals = summarizeSessions([...buildSessions, ...repairSessions]); + const repairs = (previous.repairs ?? 0) + (current.repairs ?? 0); + const repair = current.repair ? { + ...current.repair, + limit: Math.max(previous.repair?.limit ?? 0, + (previous.repairs ?? 0) + current.repair.limit), + used: repairs, + } : previous.repair; + const merged: RunLevelRecord = { + ...previous, + ...current, + buildSessions, + buildCostUsd: addCostUsd(previous.buildCostUsd, current.buildCostUsd), + repairSessions, + repairCostUsd: addCostUsd(previous.repairCostUsd, current.repairCostUsd), + repairHistory: [...(previous.repairHistory ?? []), ...(current.repairHistory ?? [])], + repairs, + repair, + sessionTotals, + tokens: sessionTotals.tokens, + usage: sessionTotals.usage, + turns: sessionTotals.turns, + promptBytes: sessionTotals.promptBytes, + tokensPerTurn: sessionTotals.turns + ? Math.round(sessionTotals.tokens / sessionTotals.turns) : null, + thinking: sessionTotals.thinking, + costUsd: addCostUsd(previous.costUsd, current.costUsd), + durationSec: (previous.durationSec ?? 0) + (current.durationSec ?? 0), + }; + return merged; +} + +function mutationControlArgs(args: BenchArgs): MutationControlArgs { + if (!args.out || !args.mutations || !args.backend || !args.parentAttemptId) { + throw new Error('mutation control has incomplete run identity'); + } + return { levelList: args.levelList, out: args.out, recipe: args.recipe, + recipeTasks: args.recipeTasks, mutations: args.mutations, backend: args.backend, + track: args.track, runIndex: args.runIndex, parentAttemptId: args.parentAttemptId, + mutationShardIndex: args.mutationShardIndex, mutationShardCount: args.mutationShardCount, + mutationResumeFrom: args.mutationResumeFrom, mutationCheckpointOut: args.mutationCheckpointOut, + mutationBaselineBundle: args.mutationBaselineBundle, + expectedMutationCalibration: args.expectedMutationCalibration, + mutationMaxRuntimeMinutes: args.mutationMaxRuntimeMinutes, + mutationImageId: args.mutationImageId }; +} + +function mutationOutcome(value: unknown): RunOutcome | null { + if (value === null || value === undefined) return null; + if (!object(value)) { + throw new Error('mutation control artifact outcome is invalid'); + } + return { kind: runOutcomeKind(value.kind), + ...(typeof value.phase === 'string' ? { phase: value.phase } : {}), + ...(typeof value.reason === 'string' ? { reason: value.reason } : {}) }; +} + +function recipeRequestIdentity(value: unknown): { recipeSha256: string; selectionSha256: string; + taskPacks: unknown; taskSha256: string } { + if (!object(value) || !object(value.recipe) || !object(value.selection) || !object(value.task) + || typeof value.recipe.contentSha256 !== 'string' || typeof value.selection.sha256 !== 'string' + || typeof value.task.sha256 !== 'string') { + throw new Error('recipe task request has no complete identity'); + } + return { recipeSha256: value.recipe.contentSha256, selectionSha256: value.selection.sha256, + taskPacks: value.selection.taskPacks, taskSha256: value.task.sha256 }; +} + +function snapshotSource(appDir: string, to: string): void { + snapshotAppSource(appDir, to); +} + +// The ports a build may legitimately reach: its own web, database, and +// SpacetimeDB listeners. Every other local port belongs to another run, the +// controller, or the dashboard. +function runOwnPorts(track: Parameters[0], + args: { backend: string; runIndex: number }): number[] { + const ports = portsFor(track, args.backend, args.runIndex); + const stdb = process.env.STACK_BENCH_STDB_URI + ? Number(new URL(process.env.STACK_BENCH_STDB_URI).port) : null; + return [ports.vite, ports.express, ports.dbPort, stdb].filter((port): port is number => + typeof port === 'number' && Number.isInteger(port) && port > 0); +} + +// Check contamination after every coding session. File-tool permissions do not +// govern shell reads, so the transcript audit remains a separate hard gate. +function auditContamination(appDir: string, ownPorts: readonly number[], + expectTranscripts: boolean): ContaminationAudit | null { + // A non-billable adapter runs no provider session and leaves no transcript; + // there is nothing to audit and nothing that could have been read. + if (!expectTranscripts) return null; + const args = [join(ROOT, 'dist', 'commands', 'leak-audit.js'), '--app', appDir, '--json', + '--own-ports', ownPorts.join(',')]; + let firstFailure: unknown = null; + for (let attempt = 1; attempt <= 2; attempt++) { + try { + const audit = sh('node', args, { stdio: 'pipe' }); + const entries = parseLeakAudit(audit); + if (entries.length === 0) { + return { kind: 'harness_failure', + evidence: ['no session transcript was found to audit'], + verdict: 'SCORES NOT USABLE — nothing verified this build stayed inside its directory.' }; + } + const escapes = entries.flatMap(entry => entry.hits); + const serious = escapes.filter(h => /GRADER|CONTRACT|BENCHMARK NOTES|PROMPTS|NETWORK/.test(h.kind)); + if (firstFailure) { + console.error(` warning: contamination audit passed on retry after: ${auditFailureSummary(firstFailure)}`); + } + if (!serious.length) return null; + return { kind: 'contaminated', + evidence: [...new Set(serious.map(h => `${h.kind}: ${h.path.split('/').slice(-2).join('/')}`))].slice(0, 8), + verdict: 'SCORES NOT USABLE — the build read the harness that grades it.' }; + } catch (error) { + firstFailure ??= error; + if (attempt === 2) { + // An audit that could not run is not a pass. Keep the process details so + // the failure can be repaired without another paid reproduction. + return { kind: 'harness_failure', + evidence: [`audit did not run after retry: ${auditFailureSummary(error)}`], + verdict: 'SCORES NOT USABLE — nothing verified this build stayed inside its directory.' }; + } + } + } + return null; +} + +export function auditFailureSummary(error: unknown): string { + const failure = object(error) ? error : {}; + const message = errorMessage(error).split(/\r?\n/)[0] ?? ''; + const stderrLines = String(failure.stderr ?? '').trim().split(/\r?\n/).filter(Boolean); + const stderr = stderrLines.find(line => /(?:error|eacces|permission denied|failed)/i.test(line)) + ?? stderrLines[0]; + const details = [ + Number.isInteger(failure.status) ? `exit ${String(failure.status)}` : null, + failure.signal ? `signal ${String(failure.signal)}` : null, + stderr ? `stderr: ${stderr}` : null, + ].filter(Boolean); + return details.length ? `${message} (${details.join('; ')})` : message; +} + +const sh = (cmd: string, args: readonly string[], + opts: Omit = {}): string => + execFileSync(cmd, [...args], { + encoding: 'utf8', maxBuffer: 64 * 1024 * 1024, timeout: COMMAND_TIMEOUT_MS, ...opts, + }); + +let activeAgentChild: ChildProcess | null = null; +// Set once a run owns resources. The top-level rejection handler invokes this +// directly; relying only on process 'exit' made cleanup best-effort precisely +// when an awaited build rejected unexpectedly. +let emergencyTeardown: (() => void) | null = null; + +export function parseAgentProcessResult(stdout: string, stderr: string, processError: unknown, + request: AgentRequest): ValidatedAgentResult { + const resultLine = stdout.trim().split('\n').pop(); + let result: ValidatedAgentResult; + try { + if (!resultLine) throw new Error('agent returned no result line'); + result = validateAgentResult(JSON.parse(resultLine), request); + } catch (resultError) { + const stdoutTail = stdout.trim().slice(-2000) || ''; + const stderrTail = stderr.trim().slice(-4000) || ''; + const processDetail = processError ? `agent process failed: ${errorMessage(processError)}\n` : ''; + throw new Error(`${processDetail}agent returned an invalid result: ${errorMessage(resultError)}\n` + + `agent stdout tail:\n${stdoutTail}\nagent stderr tail:\n${stderrTail}`); + } + if (processError && result.ok) { + throw new Error(`agent process failed after reporting success: ${errorMessage(processError)}`); + } + return result; +} + +function runAgent( + args: BenchArgs, + adapter: AgentAdapter, + mode: AgentMode, + level: number, + appDir: string, +): Promise { + if (!args.backend || !args.model) throw new Error('agent run requires backend and model'); + const remainingBudget = args.maxBudgetUsd == null ? null + : addCostUsd(args.maxBudgetUsd, -(args.spentBudgetUsd ?? 0)); + if (remainingBudget !== null && remainingBudget <= 0) { + throw new Error(`attempt cost cap of $${args.maxBudgetUsd} was exhausted before ${mode} L${level}`); + } + if (remainingBudget !== null && adapter.costLimit === 'unsupported') { + throw new Error(`agent adapter ${adapter.id} cannot enforce --max-budget-usd`); + } + const recipeTask = args.recipeTasks?.get(level)?.agentRequest + ?? args.recipeTasks?.get(level)?.request ?? null; + const request: AgentRequest = { mode, level, app: appDir, backend: args.backend, track: args.track, + runIndex: args.runIndex, model: args.model, guidance: args.guidance, skills: args.skills, + ...(adapter.usesStackSkills + ? { skillIdentity: args.condition?.guidance.skills[args.backend] } : {}), + recipe: agentRecipeIdentity(args.recipe, recipeTask), + guidanceDocument: args.guidanceDocument, + credentialAliases: args.condition?.guidance?.credentialAliases ?? {}, + recipeTask, pricing: args.pricing, + maxBudgetUsd: remainingBudget, adapterCostLimit: adapter.costLimit }; + const argv = agentRequestArgv(adapter, request); + if (args.apiKey && !adapter.apiKeyEnvironmentVariable) { + throw new Error(`agent adapter ${adapter.id} does not accept an API key`); + } + const env = { ...process.env }; + if (args.apiKey) { + const credentialName = adapter.apiKeyEnvironmentVariable; + if (!credentialName) throw new Error(`agent adapter ${adapter.id} does not accept an API key`); + env[credentialName] = args.apiKey; + } + return new Promise((resolveRun, rejectRun) => { + const child = execFile('node', argv, { + encoding: 'utf8', maxBuffer: 64 * 1024 * 1024, timeout: adapter.deadlineMs, + env, + }, + (error, stdout, stderr) => { + if (activeAgentChild === child) activeAgentChild = null; + try { + const result = parseAgentProcessResult(stdout, stderr, error, request); + args.spentBudgetUsd = addCostUsd(args.spentBudgetUsd, result.costUsd); + resolveRun(result); + } + catch (parseError) { + rejectRun(parseError); + } + }); + activeAgentChild = child; + }); +} + +interface GradeCheck { + stableKey: string; + executionId?: string; + source?: string; +} + +interface GradeRecipeTask { + request: UnknownRecord; + selection: { checks: readonly GradeCheck[] } + | { scoredChecks: readonly GradeCheck[]; observedChecks?: readonly GradeCheck[] }; +} + +function checksForGrade(task: GradeRecipeTask | undefined, observation: GradeOptions['observation']): + readonly GradeCheck[] { + if (!task) return []; + if ('scoredChecks' in task.selection) { + return observation === 'observed' + ? task.selection.observedChecks ?? [] : task.selection.scoredChecks; + } + return task.selection.checks; +} + +type GradeArguments = Pick & { + recipeTasks?: ReadonlyMap; + progression?: { identity: { policy?: string } }; + condition?: { guidance?: { credentialAliases?: Record } }; +}; + +export function gradeArgv( + args: GradeArguments, + appDir: string, + url: string, + label: string, + level: number, + track: Track, + parentAttemptId: string, + options: GradeOptions = {}, +): string[] { + const { observation = 'scored', out = null, sourceSha256 = null, + applicationFailure = null } = options; + if (!args.backend) throw new Error('grading requires a backend'); + const restartSpec = restartSpecFor(args, appDir, track); + const task = options.recipeTask ?? args.recipeTasks?.get(level); + return [compiledEntrypoint('commands', 'run-suite.js'), '--app', appDir, '--url', url, + '--backend', args.backend, '--label', label, '--level', String(level), + '--track', args.track, + '--run-index', String(args.runIndex), + '--parent-attempt-id', parentAttemptId, + '--observation', observation, + ...(out ? ['--out', out] : []), + ...(sourceSha256 ? ['--source-sha256', sourceSha256] : []), + ...(args.recipe ? ['--recipe', args.recipe] : []), + ...(task ? ['--recipe-task-json', JSON.stringify(task.request)] : []), + ...(args.condition?.guidance?.credentialAliases + ? ['--credential-aliases-json', JSON.stringify( + args.condition.guidance.credentialAliases)] : []), + ...(applicationFailure + ? ['--application-failure-json', JSON.stringify(applicationFailure)] : []), + ...(observation === 'scored' && args.recipeTasks && !args.progression + ? ['--regression-checks-json', JSON.stringify([...args.recipeTasks.entries()] + .filter(([priorLevel]) => priorLevel < level) + .flatMap(([, priorTask]) => checksForGrade(priorTask, 'scored') + .map(check => check.stableKey)))] : []), + ...(args.media && observation === 'scored' ? [] : ['--no-media']), + ...(!STACK_ADAPTER_REGISTRY.get(args.backend).runPolicy.resetEnabled + ? ['--no-reset'] + : ['--restart-spec', JSON.stringify(restartSpec)])]; +} + +function grade( + args: BenchArgs, + appDir: string, + url: string, + label: string, + level: number, + track: Track, + parentAttemptId: string, + options: GradeOptions = {}, +): GradeBundlePayload | null { + const { out = null } = options; + const source = hashAppSource(appDir); + const argv = gradeArgv(args, appDir, url, label, level, track, parentAttemptId, { + ...options, sourceSha256: options.sourceSha256 ?? source.sha256, + }); + const bundle = join(out ?? join(appDir, 'stack-bench'), ARTIFACT_FILE.gradeBundle); + rmSync(bundle, { force: true }); + const task = options.recipeTask ?? args.recipeTasks?.get(level); + const currentChecks = checksForGrade(task, options.observation); + const regressionChecks = options.observation === 'observed' || args.progression + ? [] + : [...(args.recipeTasks?.entries() ?? [])] + .filter(([priorLevel]) => priorLevel < level) + .flatMap(([, priorTask]) => checksForGrade(priorTask, 'scored')); + const sourceCount = task + ? selectedGradingSourceCount(currentChecks, regressionChecks) + : suitesFor(track, level).length; + try { + sh('node', argv, { stdio: 'inherit', timeout: gradingRunTimeoutMs(sourceCount) }); + } catch { /* a current bundle may still explain a scored failure */ } + return existsSync(bundle) + ? readArtifactPayload(bundle, { expectedKind: 'grade_bundle' }) : null; +} + +function restartSpecFor(args: Pick, + appDir: string, track: Track): RuntimeControlSpec { + if (!args.backend) throw new Error('restart specification requires a backend'); + const port = portsFor(track, args.backend, args.runIndex).vite ?? null; + if (port == null) throw new Error(`stack ${args.backend} has no application port`); + return { backend: args.backend, app: appDir, port: Number(port), probe: '' }; +} + +function runMutationControl( + args: BenchArgs, + appDir: string, + url: string, + track: Track, + imageId: string | null, +): MutationControlResult { + if (!args.out || !args.mutations) throw new Error('mutation control requires output and manifest paths'); + const output = join(args.out, ARTIFACT_FILE.mutationControl); + if (!args.mutationResumeFrom || resolve(args.mutationResumeFrom) !== resolve(output)) { + rmSync(output, { force: true }); + } + if (imageId) args.mutationImageId = imageId; + else delete args.mutationImageId; + const argv = mutationControlArgv(mutationControlArgs(args), appDir, url, track); + let processError = null; + try { sh(process.execPath, argv, { + stdio: 'inherit', timeout: mutationControlTimeoutMs(args.mutationMaxRuntimeMinutes), + }); } + catch (error) { processError = errorMessage(error).split('\n')[0] ?? null; } + if (!existsSync(output)) { + return { ok: false, artifact: output, processError, + outcome: { kind: 'harness_failure', phase: 'mutation-control', + reason: processError ?? 'mutation runner produced no artifact' } }; + } + const artifact = readArtifactPayload(output, { expectedKind: 'mutation_control' }); + return { ok: artifact.ok === true && !processError, artifact: output, + processError, summary: artifact.summary ?? null, outcome: mutationOutcome(artifact.outcome), + results: artifact.results ?? [] }; +} + +function validateMutationInput(args: BenchArgs): void { + if (!args.mutations) return; + if (!args.app) throw new Error('--mutations requires an explicit pristine --app'); + const manifest = JSON.parse(readFileSync(args.mutations, 'utf8')); + if (!/^[a-f0-9]{64}$/.test(manifest.fixtureSha256 ?? '')) { + throw new Error('mutation manifest has no valid fixtureSha256'); + } + const fixture = hashDirectory(args.app); + if (fixture.sha256 !== manifest.fixtureSha256) { + throw new Error(`mutation manifest targets fixture ${manifest.fixtureSha256}, not ${fixture.sha256}`); + } +} + +async function main() { + const args: BenchArgs = { + ...parseBenchArguments(process.argv), + recipeTasks: new Map(), + recipeBindings: new Map(), + }; + let repairGrant = null; + if (args.repairFrom) { + const repairLevel = args.repairLevel; + if (typeof repairLevel !== 'number' || !Number.isSafeInteger(repairLevel) || repairLevel < 1) { + throw new Error('--repair-from requires a positive --repair-level'); + } + repairGrant = createRepairGrant(args.repairFrom, + { level: repairLevel, repairs: args.repairs }); + const config = repairGrant.configuration; + if (config.buildImage && process.env.STACK_BENCH_IMAGE + && config.buildImage !== process.env.STACK_BENCH_IMAGE) { + throw new Error('repair continuation build image differs from its parent run'); + } + if (config.buildImage) process.env.STACK_BENCH_IMAGE = config.buildImage; + Object.assign(args, { + backend: config.backend, + track: config.track, + recipe: config.recipe, + levels: String(config.level), + levelList: [config.level], + runIndex: config.runIndex, + agentAdapter: config.agentAdapter, + model: config.model, + guidance: config.guidance, + guidanceDocument: config.guidanceDocument, + condition: config.condition, + selectionRequest: campaignSelection(config.selectionRequest, 'repair configuration.selectionRequest'), + skills: config.skills, + packIds: [...(campaignSelection(config.selectionRequest, + 'repair configuration.selectionRequest').packs ?? [])], + checkKeys: [...(campaignSelection(config.selectionRequest, + 'repair configuration.selectionRequest').checks ?? [])], + featureIds: [], + requestedSpecifications: [], + expectedSpecifications: [], + observedSpecifications: [], + seedFrom: repairGrant.sourcePath, + url: config.url, + parentAttemptId: repairGrant.parent.id, + repairGrant, + }); + } + if (!args.backend) throw new Error('benchmark run requires a backend'); + const stackAdapter = STACK_ADAPTER_REGISTRY.get(args.backend); + const materializeCodingOutput = stackAdapter.id !== 'stub'; + const agentAdapter = AGENT_ADAPTER_REGISTRY.get(args.agentAdapter); + if (process.env.STACK_BENCH_APPLIANCE !== '1' && agentAdapter.costLimit !== 'non-billable') { + throw new Error(`agent adapter ${agentAdapter.id} requires the Docker appliance`); + } + if (repairGrant) { + const currentAgent = agentAdapterIdentity(agentAdapter); + const parentAgent = repairGrant.parentArtifact.identities.agentAdapter; + if (currentAgent.id !== parentAgent?.id || currentAgent.version !== parentAgent?.version + || currentAgent.sha256 !== parentAgent?.sha256) { + throw new Error('repair continuation agent adapter differs from its parent run'); + } + if (stackAdapter.id !== repairGrant.parentArtifact.identities.stackAdapter?.id + || stackAdapter.version !== repairGrant.parentArtifact.identities.stackAdapter?.version) { + throw new Error('repair continuation stack adapter differs from its parent run'); + } + } + applyAgentCredential(args, agentAdapter); + args.model ??= agentAdapter.defaultModel; + if (!args.model) throw new Error(`agent adapter ${agentAdapter.id} has no default model`); + if (args.pricing !== undefined) { + args.pricing = validatePricingAuthority(args.pricing, { at: '--pricing-json' }); + } else if (args.maxBudgetUsd != null && agentAdapter.costLimit === 'native') { + const rates = claudeRatesForModel(args.model); + if (!rates) throw new Error(`no default pricing is recorded for model ${args.model}`); + args.pricing = validatePricingAuthority({ unit: PRICING_UNIT, rates }, + { at: 'default pricing' }); + } else { + args.pricing = null; + } + if (args.retainBackend && !stackAdapter.runPolicy.retainHostSupported) { + throw new Error(`stack adapter ${args.backend} does not support --retain-backend`); + } + const stackRuntime = stackAdapter.orchestrator.config( + { root: ROOT, env: process.env, helpers: { exists: existsSync } }); + Object.assign(process.env, stackRuntime.environment); + process.env.STACK_BENCH_NODE_BIN = process.platform === 'win32' ? 'node.exe' : process.execPath; + const track = loadTrack(args.track); + const ownPorts = runOwnPorts(track, { backend: stackAdapter.id, runIndex: args.runIndex }); + const auditsTranscripts = agentAdapter.costLimit !== 'non-billable'; + // Resolve the requested scope for every level before probing the sandbox, + // acquiring a backend lease or paying for a build. A pack that exists at L2 + // but not L1 is not a late grading surprise; it is an invalid run request. + args.selectionRequest ??= { packs: [...args.packIds], checks: [...args.checkKeys] }; + for (const level of args.levelList) { + const declared = args.condition?.requested?.levels?.find(entry => entry.level === level) ?? null; + const modularSelection = args.selectionRequest.levels?.find(entry => entry.level === level) ?? null; + if (declared?.selection?.schemaVersion === 3) { + const expected = args.featureCatalog + ? { level, recipe: `${declared.recipe.id}@${declared.recipe.version}` } + : { level, recipe: `${declared.recipe.id}@${declared.recipe.version}`, + features: declared.selection.requested.features, + checks: declared.selection.requested.checks }; + if (canonicalDefinitionJson(modularSelection) !== canonicalDefinitionJson(expected)) { + throw new Error(`campaign selection changed before L${level}`); + } + } else if (modularSelection) { + throw new Error(`campaign selection declares modular L${level} without a modular condition`); + } + const declaredRecipe = declared + ? `${declared.recipe.id}@${declared.recipe.version}` : null; + const binding = resolveRecipeRelease(track, level, declaredRecipe ?? args.recipe); + if (!binding && (args.packIds.length || args.checkKeys.length)) { + throw new Error(`L${level} has no recipe release, so --pack/--check cannot be resolved`); + } + if (binding) { + args.recipeBindings.set(level, binding); + if (args.featureCatalog) { + validateProgressionCampaignLevelScope(binding, args.featureCatalog, declared, level); + } + const requested = declared?.selection?.requested; + const progressionSelection = args.featureCatalog + ? resolveProgressionRecipeLevelSelection(binding, args.featureCatalog, level, + { cumulative: Boolean(args.progression) }) : null; + const resolved = progressionSelection === null + ? createBoundRecipeTaskRequest(binding, requested?.features + ? { featureIds: requested.features, + requestedSpecifications: requested.specifications?.requested, + expectedSpecifications: requested.specifications?.expected, + observedSpecifications: requested.specifications?.observed, + checkKeys: requested.checks } + : args) : null; + const grader = progressionSelection?.grader ?? resolved; + if (!grader) throw new Error(`L${level} has no recipe task request`); + if (args.condition && !declared) { + throw new Error(`study condition does not bind requested L${level}`); + } + const graderIdentity = recipeRequestIdentity(grader.request); + if (declared && (declared.recipe.contentSha256 !== graderIdentity.recipeSha256 + || declared.selection.sha256 !== graderIdentity.selectionSha256 + || JSON.stringify(declared.selection.taskPacks) !== JSON.stringify(graderIdentity.taskPacks) + || declared.task.sha256 !== graderIdentity.taskSha256)) { + throw new Error(`study condition requested scope changed before L${level}`); + } + if (progressionSelection) { + const progressionGrader = progressionSelection.grader; + args.recipeTasks.set(level, { + request: progressionGrader.request, + selection: progressionGrader.selection, + task: progressionGrader.task, + agentRequest: progressionSelection.agent.request, + }); + } else if (resolved) { + args.recipeTasks.set(level, { + ...resolved, + agentRequest: createAgentVisibleTaskRequest(binding, resolved), + }); + } + } + } + if (args.progression) { + const state = progressionEngine.initialize(args.progression.definition); + const declared = args.condition?.requested?.levels + ?.find(entry => entry.level === state.level) ?? null; + const binding = resolveRecipeRelease(track, state.level, + declared ? `${declared.recipe.id}@${declared.recipe.version}` : null); + if (!binding) throw new Error(`L${state.level} has no recipe release`); + resolveProgressionRecipeAction(binding, state); + if (!args.progressionOwner) { + throw new Error('live dependency progression requires an exact compiled campaign attempt'); + } + } + if (repairGrant) { + const expectedSelection = repairGrant.level.selection?.sha256 ?? null; + const repairTask = args.recipeTasks.get(repairGrant.level.level); + const resolvedSelection = repairTask ? recipeRequestIdentity(repairTask.request).selectionSha256 : null; + if (resolvedSelection !== expectedSelection) { + throw new Error('repair continuation test selection differs from its parent run'); + } + } + if (!args.selectionRequest.levels && (JSON.stringify(args.selectionRequest.packs) !== JSON.stringify(args.packIds) + || JSON.stringify(args.selectionRequest.checks) !== JSON.stringify(args.checkKeys))) { + throw new Error('campaign pack/check selection changed before execution'); + } + // Caller-owned mutation inputs are pure request data. Reject them before + // checking credentials, Docker, ports, or any other ambient runner state so + // an invalid experiment can never be masked by an unrelated preflight error. + validateMutationInput(args); + assertNoPortCollisions(); + // The deterministic adapter/stack is the model-free unit loop. Real runs + // prove the exact requested scope, engine, image, credentials, storage and + // ports before any paid coding session begins. + const admittedSmoke = args.campaignAdmission?.reusable === true + ? { id: args.campaignAdmission.id, createdAt: args.campaignAdmission.createdAt } + : null; + const preflight = args.backend === 'stub' ? null : runPreflight({ + backends: [args.backend], track: args.track, levels: args.levels, + levelList: args.levelList, runIndex: args.runIndex, agentAdapter: args.agentAdapter, + guidance: args.guidance, + recipe: args.recipe, + ...(args.condition?.requested ? { requestedScopes: [args.condition.requested] } : {}), + ...(args.featureCatalog ? { featureCatalog: args.featureCatalog } : {}), + ...(args.runMode ? { mode: args.runMode } : {}), + agentSkills: args.skills ?? null, + packIds: args.packIds, checkKeys: args.checkKeys, smoke: admittedSmoke === null, + ...(admittedSmoke ? { admittedSmoke } : {}), + ...(process.env.STACK_BENCH_SUPERVISOR_STATE + ? { supervisorState: process.env.STACK_BENCH_SUPERVISOR_STATE } : {}), + image: process.env.STACK_BENCH_IMAGE ?? DEFAULT_BUILD_IMAGE, + resultsDir: resolve(args.out ?? stackBenchResultsRoot(ROOT)), + }, { env: args.apiKey && agentAdapter.apiKeyEnvironmentVariable + ? { ...process.env, [agentAdapter.apiKeyEnvironmentVariable]: '' } + : process.env }); + if (preflight && !preflight.ok) { + const failures = preflight.checks.filter(check => check.status === 'fail'); + console.error('\nPREFLIGHT FAILED — no model session was started.'); + for (const failure of failures) { + console.error(` ${failure.id}: ${failure.summary}`); + if (failure.remediation) console.error(` fix: ${failure.remediation}`); + } + process.exit(2); + } + if (preflight) console.log(` preflight ... ${preflight.summary.passed} checks passed` + + `${preflight.summary.warnings ? `, ${preflight.summary.warnings} warning(s)` : ''}`); + if (process.env.STACK_BENCH_APPLIANCE === '1') { + console.log(' sandbox ... coding container is isolated from the controller and grading files'); + } + let url = args.url ?? `http://localhost:${portsFor(track, args.backend, args.runIndex).vite}`; + const runDir = resultsName(track, args.backend, args.runIndex); + const runId = newRunId({ track: args.track, backend: args.backend, runIndex: args.runIndex }); + const artifactLabel = `${runDir}-${runId}`; + // Default results never reuse a directory. The stable backend/run name is a + // grouping directory only; every artifact beneath it belongs to one run id. + args.out ??= join(stackBenchResultsRoot(ROOT), runDir, runId); + if (!args.out) throw new Error('benchmark run has no results directory'); + const outputDir = args.out; + mkdirSync(args.out, { recursive: true }); + if (existsSync(join(args.out, ARTIFACT_FILE.run))) { + throw new Error(`refusing to reuse result directory containing ${ARTIFACT_FILE.run}: ${args.out}`); + } + if (preflight) writeArtifact(join(args.out, ARTIFACT_FILE.preflight), { + kind: 'preflight', id: `${runId}-preflight`, + attempt: { id: `${runId}-preflight`, parentId: runId }, + identities: emptyArtifactIdentities({ + agentAdapter: agentAdapterIdentity(agentAdapter), + stackAdapter: { id: stackAdapter.id, version: stackAdapter.version }, + }), + payload: preflight, + }); + + // Validate caller-owned source before acquiring a backend slot so a bad + // fixture cannot leave leased resources behind. + const ownWorkDir = !args.app; + const appDir = args.app ?? join(workDirFor(track, args.backend, args.runIndex, runId), 'app'); + if (args.repairGrant && url.startsWith('file:')) { + url = pathToFileURL(join(appDir, 'index.html')).href; + } + + // Bind destructive and lifecycle operations to exact resource identities and + // an ownership token. Targets come only from the lease, never generated code. + const runtimeRoot = resolve(process.env.STACK_BENCH_RUNTIME_DIR + ?? join(tmpdir(), 'stack-bench-runtime')); + const runtimeDir = join(runtimeRoot, runId); + const leasePath = join(runtimeDir, ARTIFACT_FILE.backendLease); + const preparedLease = stackAdapter.lease.prepare({ + track, + runIndex: args.runIndex, + runtimeDir, + serverUri: stackRuntime.lease.serverUri, + env: process.env, + helpers: { containerIdentity: runningContainerIdentity, dbName, moduleName }, + }); + const initialLease = createBackendLease({ + runId, + backend: args.backend, + track: args.track, + runIndex: args.runIndex, + ...preparedLease.lease, + }); + const lockScope = resourceLockScope(); + const lockKeys = backendResourceLockKeys(initialLease, preparedLease.lockKeys); + let privateSupervisorStatePath = null; + try { + initialLease.resources.locks.push(...acquireResourceLocks({ + ...lockScope, keys: lockKeys, lease: initialLease, + })); + writeBackendLease(leasePath, initialLease); + const supervisorState = process.env.STACK_BENCH_SUPERVISOR_STATE + ?? (process.env.STACK_BENCH_SUPERVISOR_DIR + ? join(resolve(process.env.STACK_BENCH_SUPERVISOR_DIR), `${runId}.json`) : null); + if (supervisorState) { + // Private handoff to an outer timeout supervisor. It contains the lease + // token, so create it once with owner-only permissions and never place it + // in the results tree. + privateSupervisorStatePath = resolve(supervisorState); + mkdirSync(dirname(privateSupervisorStatePath), { recursive: true, mode: 0o700 }); + writeFileSync(privateSupervisorStatePath, `${JSON.stringify({ + version: SUPERVISOR_STATE_VERSION, runId, backend: args.backend, runtimeDir, leasePath, + ownershipToken: initialLease.ownershipToken, output: resolve(args.out), + })}\n`, { flag: 'wx', mode: 0o600 }); + } + } catch (error) { + releaseResourceLocks(initialLease); + rmSync(runtimeDir, { recursive: true, force: true }); + throw error; + } + process.env.STACK_BENCH_LEASE = leasePath; + process.env.STACK_BENCH_LEASE_TOKEN = initialLease.ownershipToken; + if (process.platform === 'win32') { + // When Windows resolves `bash` through WSL, WSLENV must carry lease paths + // and tokens into lifecycle scripts with path translation. + const bridge = ['STACK_BENCH_LEASE/p', 'STACK_BENCH_LEASE_TOKEN', + 'STACK_BENCH_NODE_BIN', ...stackRuntime.windowsEnvironmentBridge]; + const existing = (process.env.WSLENV ?? '').split(':').filter(Boolean); + process.env.WSLENV = [...new Set([...existing, ...bridge])].join(':'); + } + + let tornDown = false; + let activeRun: BenchmarkRunRecord | null = null; + const recoveryPath = join(outputDir, ARTIFACT_FILE.recovery); + const writeLeaseEvidence = (knownLease: BackendLease | null = null) => { + const lease = knownLease ?? readBackendLease(leasePath, + { token: initialLease.ownershipToken, backend: args.backend, runId }); + const out = join(outputDir, ARTIFACT_FILE.backendLease); + const evidence = publicBackendLease(lease); + const id = `${runId}-backend-lease`; + writeArtifact(out, { + kind: 'backend_lease_evidence', id, + attempt: { id, parentId: runId }, + timestamps: { startedAt: evidence.createdAt, completedAt: new Date().toISOString() }, + identities: emptyArtifactIdentities({ stackAdapter: { id: args.backend } }), + payload: evidence, + }); + return evidence; + }; + const teardown = ({ reason = null, retainBackend = args.retainBackend }: + { reason?: string | null; retainBackend?: boolean } = {}) => { + if (tornDown) return; + if (activeAgentChild?.pid) { + killTree(activeAgentChild.pid); + activeAgentChild = null; + } + // Preserve restart failures before removing the only filesystem that holds + // their stderr. A 500 after restart is otherwise impossible to distinguish + // from an application defect, a dead dependency, or host pressure. + if (activeRun) { + try { + activeRun.backendDiagnostics = captureApplicationDiagnostics(join(outputDir, 'backend.log')); + } catch (error) { + activeRun.backendDiagnostics = { captured: false, + reason: errorMessage(error).split(/\r?\n/)[0] }; + } + } + let released = false; + let cleanupError: unknown = null; + try { + released = releaseBackendLease(leasePath, initialLease.ownershipToken, + { retainBackend }); + } catch (error) { cleanupError = error; } + let finalLease = initialLease; + try { + finalLease = readBackendLease(leasePath, + { token: initialLease.ownershipToken, backend: args.backend, runId }); + } catch (error) { cleanupError ??= error; released = false; } + const evidence = writeLeaseEvidence(finalLease); + writeRecoveryArtifact(recoveryPath, finalLease, { cleanupSucceeded: released, + retained: Boolean(retainBackend), + reason: cleanupError === null ? reason ?? (released ? null : 'authenticated cleanup refused') + : errorMessage(cleanupError) }); + if (activeRun) { + activeRun.backendLease = evidence; + activeRun.outcome ??= aggregateRunOutcome(activeRun.levels); + writeRunJson(join(outputDir, ARTIFACT_FILE.run), activeRun); + } + tornDown = released; + if (released && !retainBackend) { + rmSync(runtimeDir, { recursive: true, force: true }); + if (privateSupervisorStatePath) rmSync(privateSupervisorStatePath, { force: true }); + } + if (cleanupError) throw cleanupError; + if (!released) throw new Error(`backend teardown refused: listener no longer matches lease ${runId}`); + }; + emergencyTeardown = teardown; + + try { + stackAdapter.lifecycle.activate({ + leasePath, leaseToken: initialLease.ownershipToken, lease: initialLease, + ...stackRuntime.lifecycle, + }); + } catch (error) { + try { teardown({ reason: `backend activation failed: ${errorMessage(error)}`, retainBackend: false }); } + catch (cleanupError) { + console.error(` activation cleanup quarantined: ${errorMessage(cleanupError).split(/\r?\n/)[0]}`); + } + throw error; + } + + // Teardown stops only resources recorded in this run's lease. + const interrupt = (signal: NodeJS.Signals, exitCode: number) => { + console.log(`interrupted by ${signal} — stopping exact owned resources`); + try { teardown({ reason: `interrupted by ${signal}` }); } + catch (error) { console.error(` cleanup quarantined: ${errorMessage(error).split(/\r?\n/)[0]}`); } + process.exit(exitCode); + }; + process.on('SIGINT', () => interrupt('SIGINT', 130)); + process.on('SIGTERM', () => interrupt('SIGTERM', 143)); + process.on('exit', () => { + if (!tornDown) { + try { teardown(); } catch (error) { + console.error(` cleanup failed: ${errorMessage(error).split('\n')[0]}`); + } + } + }); + + // Seed source only; the upgrade session installs its own dependencies. + if (args.seedFrom) { + const from = resolve(args.seedFrom); + if (!existsSync(from)) { console.error(`--seed-from path does not exist: ${from}`); process.exit(2); } + seedAppSource(from, appDir); + if (args.progressionSeed) { + const seeded = hashAppSource(appDir); + if (seeded.sha256 !== args.progressionSeed.sourceSha256 + || seeded.files.length !== args.progressionSeed.sourceFiles) { + throw new Error('extension source does not match its recorded identity'); + } + } + console.log(args.repairGrant + ? ` restored L${args.levelList[0]} checkpoint from ${from} for a bounded repair continuation` + : ` seeded from ${from} — level ${args.levelList[0]} will UPGRADE it, not rebuild`); + } + + const started = Date.now(); + const run: BenchmarkRunRecord = { id: runId, + ...(args.repairGrant ? { kind: 'repair_continuation', + continuation: structuredClone(args.repairGrant.grant) } : {}), + startedAt: new Date(started).toISOString(), + parentAttemptId: args.parentAttemptId ?? null, + identities: emptyArtifactIdentities({ + experiment: args.experimentIdentity ?? null, + agentAdapter: agentAdapterIdentity(agentAdapter), + stackAdapter: { id: stackAdapter.id, version: stackAdapter.version }, + }), + mode: args.runMode ?? { id: args.progression ? 'dependency' : 'sequential', + version: args.progression ? DEPENDENCY_MODE_VERSION : '1.0.0' }, + track: args.track, backend: args.backend, model: args.model, + pricing: args.pricing, + guidance: args.guidance, condition: args.condition ?? null, + skills: args.skills ?? [], + runtime: { buildImage: process.env.STACK_BENCH_IMAGE ?? DEFAULT_BUILD_IMAGE, url }, + selectionRequest: args.selectionRequest, + featureCatalog: args.featureCatalog?.identity ?? null, + dependencyPolicy: args.dependencyPolicy?.identity ?? null, + ...(args.progressionOwner ? { progressionOwner: args.progressionOwner } : {}), + ...(args.progressionSeed ? { progressionSeed: { + fromDepth: args.progressionSeed.fromDepth, + sourceSha256: args.progressionSeed.sourceSha256, + sourceFiles: args.progressionSeed.sourceFiles, + parent: structuredClone(args.progressionSeed.parent), + validatedDepths: [], + } } : {}), + backendLease: publicBackendLease(readBackendLease(leasePath, + { token: initialLease.ownershipToken, backend: args.backend, runId })), + validation: { + ladder: { policy: args.progression ? args.progression.identity.policy : 'pass-before-next-level', + requestedLevels: [...args.levelList], + completedLevels: [], stoppedAfterLevel: null, blockedLevels: [] } }, levels: [] }; + activeRun = run; + + const progressionOwner = args.progression ? { + ...args.progressionOwner, + workspace: { appDirectory: 'source' }, + } : null; + const progressionExecution = args.progression ? createLiveProgressionExecution({ + progression: args.progression, + featureCatalogIdentity: args.featureCatalog?.identity, + dependencyPolicyIdentity: args.dependencyPolicy?.identity, + owner: progressionOwner, + statePath: join(args.out, ARTIFACT_FILE.progressionState), + runId, + outputDir: args.out, + appDir, + track: args.track, + backend: args.backend, + identities: run.identities, + recipeBindings: args.recipeBindings, + resumeFrom: args.progressionResumeFrom ?? null, + getRunArtifact: () => { + writeRunJson(join(outputDir, ARTIFACT_FILE.run), run); + return readArtifact(join(outputDir, ARTIFACT_FILE.run)); + }, + onState: status => { + run.progressionStatus = status; + writeRunJson(join(outputDir, ARTIFACT_FILE.run), run); + }, + }) : null; + const progressionStart = progressionExecution?.initialize() ?? null; + if (progressionStart?.resumed) { + const prior = progressionStart.priorRun; + if (!prior) throw new Error('resumed dependency progression has no prior run artifact'); + const actionLevel = progressionStart.action.type === 'terminal' + ? Number.MAX_SAFE_INTEGER : progressionStart.action.level; + const inheritedLevels = (prior.payload.levels ?? []) + .filter(level => level.level < actionLevel).map(level => level.level); + run.levels = (prior.payload.levels ?? []) + .filter(level => inheritedLevels.includes(level.level)).map(level => structuredClone(level)); + run.validation.ladder.completedLevels = [...inheritedLevels]; + run.progressionResume = { + priorRunId: prior.id, + priorRunSha256: sha256(canonicalDefinitionJson(prior)), + stateSha256: progressionStart.stateSha256, + action: progressionStart.action.type === 'terminal' + ? { type: 'terminal' } + : { type: progressionStart.action.type, level: progressionStart.action.level }, + inheritedLevels, + priorTotals: prior.payload.totals ?? null, + }; + run.progressionStatus = progressionStart.status; + writeRunJson(join(args.out, ARTIFACT_FILE.run), run); + } + + const bindProgressionAction = (level: number): ProgressionRecipeAction | null => { + if (!progressionExecution) return null; + const selected = progressionExecution.bind(); + if (!isProgressionWorkRecipeAction(selected)) return selected; + if (!args.recipeTasks) throw new Error('recipe task map is unavailable'); + args.recipeTasks.set(level, { + request: selected.grader.request, + selection: selected.grader.selection, + task: selected.grader.task, + agentRequest: selected.agent.request, + progressionAction: selected.action, + }); + return selected; + }; + + const progressionBundles = new Map(); + const recordProgressionGrade = (input: Parameters['record']>[0]) => { + const next = progressionExecution?.record(input) ?? null; + const last = progressionExecution?.state?.attempts.at(-1) ?? null; + if (input.bundle && input.selected && isProgressionWorkRecipeAction(input.selected) + && last?.outcome === 'conclusive') { + progressionBundles.set(input.selected.action.level, input.bundle as GradeBundlePayload); + } + return next; + }; + + const appendLevelRecord = (record: RunLevelRecord): void => { + if (args.dependencyPolicy?.definition.workSelection !== 'feature') { + run.levels.push(record); + return; + } + const index = run.levels.findIndex(candidate => candidate.level === record.level); + if (index < 0) run.levels.push(record); + else run.levels[index] = mergeFeatureLevelRecord(run.levels[index] ?? null, record); + }; + + let runCostComplete = true; + + const runAgentForLevel = async (mode: AgentMode, level: number, + onFailure?: () => Promise): Promise => { + try { + clearPrivateGradingEvidence(appDir); + const result = await runAgent(args, agentAdapter, mode, level, appDir); + if (result.costComplete !== true) runCostComplete = false; + return result; + } catch (error) { + await onFailure?.(); + const reason = errorMessage(error).split(/\r?\n/)[0] ?? 'agent execution failed'; + run.outcome = { kind: 'harness_failure', phase: `agent-${mode}`, + reason, appFailures: [], inconclusive: [], harnessFailures: [reason] }; + run.validation.ladder.stoppedAfterLevel = run.levels.at(-1)?.level ?? null; + run.validation.ladder.blockedLevels = args.levelList.filter(candidate => candidate >= level); + if (progressionExecution) { + run.progressionStatus = progressionExecution.status(); + run.validation.ladder.completedLevels = [...new Set(requireProgressionState(progressionExecution.state).attempts + .filter(attempt => attempt.outcome === 'conclusive') + .map(attempt => attempt.level))]; + } + finalizeRunTotals(run, started, { costComplete: false }); + run.completedAt = new Date().toISOString(); + writeRunJson(join(outputDir, ARTIFACT_FILE.run), run); + throw error; + } + }; + + // Stop before grading if a coding session read protected material or if the + // audit itself failed. Keep the paid session and exact cost in the run artifact even + // though no score may be used. + const abortUnusableSession = (whichSession: string, audit: ContaminationAudit, + levelRecord: UnknownRecord & { level: number }, + selected: ProgressionRecipeAction | null, completedRepair = false) => { + const reason = audit.evidence.join('; '); + const outcome: RunOutcome = { kind: audit.kind === 'harness_failure' ? 'harness_failure' : 'ungraded', + phase: 'contamination-audit', reason, + appFailures: [], inconclusive: [], + harnessFailures: audit.kind === 'harness_failure' ? [reason] : [] }; + run.contaminated = audit.kind === 'contaminated'; + run.contamination = { evidence: audit.evidence, verdict: audit.verdict, + detectedAt: whichSession }; + const record: RunLevelRecord = { ...levelRecord, error: reason, outcome, + level: levelRecord.level, graded: false, score: null, max: null, selection: null }; + appendLevelRecord(record); + if (progressionExecution) { + recordProgressionGrade({ selected, bundle: null, level: levelRecord.level, + failure: progressionFailure(outcome), + completedRepair }); + run.progressionStatus = progressionExecution!.status(); + } + run.validation.ladder.stoppedAfterLevel = run.levels.at(-2)?.level ?? null; + run.validation.ladder.blockedLevels = args.levelList + .filter(candidate => candidate >= levelRecord.level); + finalizeRunTotals(run, started, { costComplete: runCostComplete }); + run.outcome = outcome; + run.completedAt = new Date().toISOString(); + if (run.contaminated) { + console.log(`\n !! CONTAMINATED at ${whichSession}:`); + for (const evidence of audit.evidence) console.log(` ${evidence}`); + console.log(' Scores from this run must not be quoted.'); + } else { + console.log(`\n !! HARNESS FAILURE at ${whichSession}:`); + for (const evidence of audit.evidence) console.log(` ${evidence}`); + console.log(' The audit did not establish a usable result.'); + } + try { writeRunJson(join(outputDir, ARTIFACT_FILE.run), run); } catch { /* best effort */ } + try { archiveTranscripts(appDir, artifactLabel); } catch { /* best effort */ } + teardown(); + process.exit(4); + }; + + for (let levelIndex = 0; levelIndex < args.levelList.length; levelIndex += 1) { + const level = args.levelList[levelIndex]!; + const t0 = Date.now(); + const continuing = Boolean(args.repairGrant); + console.log(`\n================ ${args.backend} — level ${level} ================`); + + let progressionSelection = bindProgressionAction(level); + if (progressionSelection?.action.type === 'terminal') break; + if (args.dependencyPolicy?.definition.workSelection === 'all-at-once' + && progressionSelection?.action.level !== level) continue; + const applicationControl = materializeCodingOutput + ? restartSpecFor(args, appDir, track) : null; + const featureActionSequence = args.dependencyPolicy?.definition.workSelection === 'feature' + ? requireProgressionState(progressionExecution?.state ?? null).attempts.length + 1 : null; + const featureActionSuffix = featureActionSequence === null + ? '' : `-action${String(featureActionSequence).padStart(3, '0')}`; + // A clean-source start that fails voids a grade. Its launch log is the + // only account of why, so it stays beside the run. + const keepStartLog = (error: unknown, label: string): void => { + const startLog = error !== null && typeof error === 'object' && 'startLog' in error + ? error.startLog : null; + if (typeof startLog !== 'string' || !startLog) return; + writeFileSync(join(outputDir, `${label}-start.log`), `${startLog}\n`); + }; + const restoreFeatureAcceptedSource = async (): Promise => { + if (featureActionSequence === null) return; + const source = join(outputDir, 'source'); + if (applicationControl) { + try { + await materializeAcceptedSource(source, appDir, applicationControl); + } catch { + resetAppToSource(source, appDir); + } + } else resetAppToSource(source, appDir); + }; + if (featureActionSequence !== null && progressionSelection + && isProgressionWorkRecipeAction(progressionSelection) + && !featureActionNeedsCoding(progressionSelection, + requireProgressionState(progressionExecution?.state ?? null))) { + await restoreFeatureAcceptedSource(); + const bundle = grade(args, appDir, url, + `${args.backend}-l${level}${featureActionSuffix}-regrade`, level, track, runId); + const outcome = classifyBundle(bundle); + const repair = { status: levelGradeIsUsable(outcome) ? 'not-needed' as const : 'ungraded' as const, + limit: 0, used: 0, stopReason: 'accepted-source-regrade' }; + let next = recordProgressionGrade({ selected: progressionSelection, bundle, + level }); + let finalOutcome = outcome; + const currentState = requireProgressionState(progressionExecution?.state ?? null); + if (next?.type === 'build' && next.level === level + && !featureActionNeedsCoding(progressionSelection, currentState)) { + const reason = 'accepted source still has ungraded checks after regrade'; + finalOutcome = { kind: 'harness_failure', phase: 'feature-regrade', reason, + appFailures: [], inconclusive: [], harnessFailures: [reason] }; + next = recordProgressionGrade({ selected: progressionSelection, bundle: null, + level, failure: progressionFailure(finalOutcome) }); + } + const state = requireProgressionState(progressionExecution?.state ?? null); + const graded = levelGradeIsUsable(finalOutcome) + && state.attempts.at(-1)?.outcome === 'conclusive'; + let checkpoint = null; + if (graded && (state.phase === 'terminal' || state.level > level)) { + checkpoint = preserveLevelCheckpoint({ appDir, outputDir, runId, + identities: run.identities, track: args.track, backend: args.backend, level, + repair, outcome: finalOutcome, + selectionSha256: bundle?.selection?.sha256 ?? null }); + } + appendLevelRecord({ level, graded, + score: graded ? bundle?.totals?.score ?? null : null, + max: graded ? bundle?.totals?.max ?? null : null, + selection: graded ? bundle?.selection ?? null : null, + regression: bundle?.totals?.regression ?? null, + contractPass: bundle?.totals?.contractPass ?? null, + code: bundle?.code ?? null, + repair, + checkpoint, + sessionTotals: summarizeSessions([]), + costUsd: 0, + repairs: 0, + durationSec: Math.round((Date.now() - t0) / 1000), + outcome: finalOutcome }); + if (graded && (state.phase === 'terminal' || state.level > level) + && !run.validation.ladder.completedLevels.includes(level)) { + run.validation.ladder.completedLevels.push(level); + } + run.progressionStatus = progressionExecution!.status(); + writeRunJson(join(outputDir, ARTIFACT_FILE.run), run); + if (state.phase === 'terminal') break; + if (state.attempts.at(-1)?.outcome === 'inconclusive') { + run.validation.ladder.stoppedAfterLevel = level; + run.validation.ladder.blockedLevels = args.levelList.filter(candidate => candidate >= level); + break; + } + if (next?.type !== 'terminal' && next?.level === level) { + levelIndex -= 1; + continue; + } + if (next?.type !== 'terminal' && next && next.level < level) { + throw new Error(`dependency progression moved backward from L${level}`); + } + continue; + } + if (args.seedThrough !== undefined && level <= args.seedThrough) { + if (!progressionSelection || !isProgressionWorkRecipeAction(progressionSelection) + || progressionSelection.action.level !== level || !args.seedFrom || !run.progressionSeed) { + throw new Error(`extension cannot validate depth ${level}`); + } + let applicationFailure: RunOutcome | null = null; + if (applicationControl) { + try { + await materializeAcceptedSource(args.seedFrom, appDir, applicationControl); + } catch (error) { + applicationFailure = materializationAppFailure(error); + keepStartLog(error, `${args.backend}-extension-l${level}`); + } + } else { + resetAppToSource(args.seedFrom, appDir); + } + const bundle = grade(args, appDir, url, `${args.backend}-extension-l${level}`, + level, track, runId, { applicationFailure }); + const outcome = applicationFailure ?? classifyBundle(bundle); + const next = recordProgressionGrade({ selected: progressionSelection, bundle, level }); + const progressionState = requireProgressionState(progressionExecution?.state ?? null); + const progressionAttempt = progressionState.attempts.at(-1) ?? null; + const graded = levelGradeIsUsable(outcome, progressionAttempt); + const passed = graded && outcome.kind === 'passed'; + const repair = { status: passed ? 'not-needed' as const : 'incomplete' as const, + limit: 0, used: 0, + stopReason: passed ? 'not-needed' : 'extension-validation-failed' }; + let checkpoint = null; + try { + checkpoint = preserveLevelCheckpoint({ appDir, outputDir: args.out, runId, + identities: run.identities, track: args.track, backend: args.backend, level, + repair, outcome, selectionSha256: bundle?.selection?.sha256 ?? null }); + } catch (error) { + throw new Error(`could not preserve extension depth ${level}: ${errorMessage(error)}`); + } + const source = hashAppSource(appDir); + appendLevelRecord({ level, graded, + score: graded ? bundle?.totals?.score ?? null : null, + max: graded ? bundle?.totals?.max ?? null : null, + selection: bundle?.selection ?? null, + baseline: { kind: 'extension-validation', source: { + sha256: source.sha256, files: source.files.length } }, + regression: bundle?.totals?.regression ?? null, + contractPass: bundle?.totals?.contractPass ?? null, + code: bundle?.code ?? null, + repair, + checkpoint, + sessionTotals: summarizeSessions([]), + costUsd: 0, + repairs: 0, + durationSec: Math.round((Date.now() - t0) / 1000), + outcome }); + if (progressionAttempt?.outcome === 'conclusive') { + if (!run.validation.ladder.completedLevels.includes(level)) { + run.validation.ladder.completedLevels.push(level); + } + } + run.progressionStatus = progressionExecution!.status(); + if (passed) run.progressionSeed.validatedDepths.push(level); + if (!passed) { + run.validation.ladder.stoppedAfterLevel = level > 1 ? level - 1 : null; + run.validation.ladder.blockedLevels = args.levelList.filter(candidate => candidate >= level); + writeRunJson(join(args.out, ARTIFACT_FILE.run), run); + break; + } + writeRunJson(join(args.out, ARTIFACT_FILE.run), run); + if (!next || next.type === 'terminal' || next.level === undefined || next.level <= level) { + throw new Error(`extension did not advance after depth ${level}`); + } + continue; + } + const resumedRepair = progressionStart?.resumed === true + && progressionStart.action.type === 'repair'; + // The interrupted repair was charged but never graded. Grade its preserved + // source before any coding session. + const resumedGrade = resumedRepair && progressionStart?.action.type === 'repair' + && progressionStart.action.repair.awaitingGrade === true; + const resumedRegression = resumedRepair + ? savedRepairRegression(progressionExecution?.state ?? null, progressionSelection) : null; + const resumedRegressionReport = resumedRegression + ? join(args.out, 'repair-reports', + `rejected-regression-l${level}${featureActionSuffix}-resume.md`) : null; + const priorRepairs = resumedRepair + ? progressionStart.priorRun?.payload.levels?.find(item => item.level === level) + ?.repair?.used ?? 0 + : args.progression ? 0 : run.levels.reduce((sum, item) => sum + (item.repairs ?? 0), 0); + const repairBudgetFor = (selected: ProgressionRecipeAction | null, + completedRepairs: number) => selected + && isProgressionWorkRecipeAction(selected) + ? dependencyRepairBudget(selected.action, completedRepairs) + : args.repairs; + const levelRepairNodeIds = new Set(progressionSelection?.action.repair.nodeIds ?? []); + let progressionRepairLimit = repairBudgetFor( + progressionSelection, priorRepairs); + const trackProgressionBudget = (selected: ProgressionRecipeAction | null, + completedRepairs: number) => { + if (!selected || !isProgressionWorkRecipeAction(selected)) return; + selected.action.repair.nodeIds.forEach(nodeId => levelRepairNodeIds.add(nodeId)); + progressionRepairLimit = Math.max( + progressionRepairLimit, repairBudgetFor(selected, completedRepairs)); + }; + if (resumedRepair && !resumedGrade) { + let reportFailure: string | null = null; + try { + if (resumedRegressionReport && resumedRegression) { + mkdirSync(dirname(resumedRegressionReport), { recursive: true }); + writeFileSync(resumedRegressionReport, resumedRegression.report); + } + sh('node', [join(ROOT, 'dist', 'commands', 'report-bugs.js'), '--app', appDir, + '--history-json', '[]', '--archive', join(args.out, 'repair-reports', + `bug-report-l${level}${featureActionSuffix}-resume.md`), + ...(resumedRegressionReport ? ['--prior-regression', resumedRegressionReport] : []), + ...repairReportArgs(progressionSelection)], + { stdio: 'pipe' }); + } catch (error) { + reportFailure = errorMessage(error).split(/\r?\n/)[0] + ?? 'repair report generation failed'; + } + if (reportFailure) { + const outcome: RunOutcome = { + kind: 'harness_failure', phase: 'repair-report', reason: reportFailure, + appFailures: [], inconclusive: [], harnessFailures: [reportFailure], + }; + recordProgressionGrade({ selected: progressionSelection, bundle: null, level, + failure: progressionFailure(outcome) }); + appendLevelRecord({ level, graded: false, score: null, max: null, + selection: null, error: reportFailure, outcome, + repair: { status: 'ungraded', limit: progressionRepairLimit, + used: priorRepairs, stopReason: 'repair-report' }, + repairCostUsd: 0, repairSessions: [], repairs: 0, priorRepairs, + cumulativeRepairs: priorRepairs, sessionTotals: summarizeSessions([]), + costUsd: 0, durationSec: Math.round((Date.now() - t0) / 1000) }); + run.validation.ladder.stoppedAfterLevel = level; + run.validation.ladder.blockedLevels = args.levelList.filter(candidate => candidate >= level); + writeRunJson(join(args.out, ARTIFACT_FILE.run), run); + break; + } + } + + const firstMode = resumedRepair ? 'fix' + : continuing ? 'resume' : args.seedFrom ? 'upgrade' : 'build'; + const build = resumedGrade ? null : await runAgentForLevel( + resumedRepair || run.levels.length === 0 ? firstMode : 'upgrade', level, + featureActionSequence === null ? undefined : restoreFeatureAcceptedSource); + // Only a resumed grade runs a level without a coding session. + const requireBuild = (): NonNullable => { + if (!build) throw new Error(`level ${level} has no coding session`); + return build; + }; + const buildLeak = build ? auditContamination(appDir, ownPorts, auditsTranscripts) : null; + if (buildLeak) { + const session = requireBuild(); + const buildSession = runSessionRecord(session, + resumedRepair ? priorRepairs + 1 : null); + const sessionTotals = summarizeSessions([buildSession]); + abortUnusableSession(`level ${level} ${firstMode}`, buildLeak, { + level, graded: false, score: null, max: null, selection: null, + ...(resumedRepair + ? { repairCostUsd: session.costUsd, repairSessions: [buildSession], repairs: 1, + priorRepairs, cumulativeRepairs: priorRepairs + 1 } + : continuing + ? { resumeCostUsd: session.costUsd, resumeSession: buildSession } + : { buildCostUsd: session.costUsd, buildSessions: [buildSession] }), + sessionTotals, costUsd: session.costUsd, durationMs: Date.now() - t0, + }, progressionSelection, resumedRepair); + } + // Record the session setup needed to compare runs. + if (build) run.setup ??= build.setup; + if (continuing) { + const session = requireBuild(); + requireContinuation(run).resumeSetup = { + sessionId: session.sessionId ?? null, + costUsd: session.costUsd, + durationMs: session.durationMs, + sourceVerified: false, + }; + } + // No session, no app. Grading an empty directory yields a real-looking zero + // that is a harness failure, not a result for this backend. + const buildFailure = build ? agentSessionFailure(build) : null; + if (buildFailure) { + const session = requireBuild(); + await restoreFeatureAcceptedSource(); + console.log(` ABORTED: ${buildFailure.reason}. Details will be kept in ${join(args.out, ARTIFACT_FILE.run)}`); + const failedSession = runSessionRecord(session); + if (progressionExecution) { + recordProgressionGrade({ selected: progressionSelection, bundle: null, level, + failure: buildFailure }); + } + appendLevelRecord({ level, graded: false, score: null, max: null, + selection: null, error: buildFailure.reason, + outcome: buildFailure, + ...(continuing + ? { resumeSession: failedSession, resumeCostUsd: session.costUsd } + : { buildSessions: [failedSession], buildCostUsd: session.costUsd }), + sessionTotals: summarizeSessions([session]), + costUsd: session.costUsd, durationMs: Date.now() - t0 }); + break; + } + const gradeAcceptedSource = async (sourcePath: string, + label: string): Promise => { + let failure: RunOutcome | null = null; + if (applicationControl) { + try { + await materializeAcceptedSource(sourcePath, appDir, applicationControl); + } catch (error) { + failure = materializationAppFailure(error); + keepStartLog(error, label); + } + } else { + resetAppToSource(sourcePath, appDir); + } + return grade(args, appDir, url, label, level, track, runId, + { applicationFailure: failure }); + }; + const archiveCandidateGrade = (label: string): void => { + const gradingDirectory = join(appDir, 'stack-bench'); + if (!existsSync(gradingDirectory)) return; + cpSync(gradingDirectory, join(outputDir, 'candidate-grades', label), { + recursive: true, + filter: source => !/[\\/]media([\\/]|$)/.test(source), + }); + }; + const restoreAcceptedRepair = async (sourcePath: string, gradingPath: string): Promise => { + if (applicationControl) await materializeAcceptedSource(sourcePath, appDir, applicationControl); + else resetAppToSource(sourcePath, appDir); + restorePrivateGradingEvidence(appDir, gradingPath); + }; + // Keep a repaired source whose grade did not finish beside the run, bound + // by hash, so a resume can grade exactly what the paid session produced. + const preserveRepairCandidate = (directory: string): RunRepairCandidate => { + const path = join(outputDir, directory); + rmSync(path, { recursive: true, force: true }); + const live = hashAppSource(appDir); + snapshotSource(appDir, path); + const preserved = hashDirectory(path); + if (live.sha256 !== preserved.sha256 || live.files.length !== preserved.files.length) { + throw new Error('preserved repair source differs from the live application source'); + } + return { directory, sha256: preserved.sha256, files: preserved.files.length }; + }; + if (resumedGrade) { + const candidate = progressionStart?.priorRun?.payload.levels + ?.find(item => item.level === level)?.repair?.candidate; + const candidateRoot = args.progressionResumeFrom ?? outputDir; + let candidateFailure: string | null = null; + if (!candidate || !/^[A-Za-z0-9._-]+$/.test(candidate.directory) + || !existsSync(join(candidateRoot, candidate.directory))) { + candidateFailure = 'the interrupted repair left no source to grade'; + } else { + const candidatePath = join(candidateRoot, candidate.directory); + const preserved = hashDirectory(candidatePath); + if (preserved.sha256 !== candidate.sha256 + || preserved.files.length !== candidate.files) { + candidateFailure = 'the interrupted repair source does not match its record'; + } else { + resetAppToSource(candidatePath, appDir); + console.log(` restored the interrupted repair source from ${candidatePath}`); + } + } + if (candidateFailure) { + const outcome: RunOutcome = { + kind: 'harness_failure', phase: 'repair-candidate', reason: candidateFailure, + appFailures: [], inconclusive: [], harnessFailures: [candidateFailure], + }; + recordProgressionGrade({ selected: progressionSelection, bundle: null, level, + failure: progressionFailure(outcome) }); + appendLevelRecord({ level, graded: false, score: null, max: null, + selection: null, error: candidateFailure, outcome, + repair: { status: 'ungraded', limit: progressionRepairLimit, + used: priorRepairs, stopReason: 'repair-candidate' }, + repairCostUsd: 0, repairSessions: [], repairs: 0, priorRepairs, + cumulativeRepairs: priorRepairs, sessionTotals: summarizeSessions([]), + costUsd: 0, durationSec: Math.round((Date.now() - t0) / 1000) }); + run.validation.ladder.stoppedAfterLevel = level; + run.validation.ladder.blockedLevels = args.levelList.filter(candidate => candidate >= level); + writeRunJson(join(args.out, ARTIFACT_FILE.run), run); + break; + } + } + if (continuing) { + // A resume may restore runtime state but cannot change checkpoint source. + const resumed = hashAppSource(appDir); + const repairGrant = args.repairGrant; + if (!repairGrant) throw new Error('repair continuation has no grant'); + if (resumed.sha256 !== repairGrant.checkpoint.payload.source.sha256 + || resumed.files.length !== repairGrant.checkpoint.payload.source.files) { + throw new Error('resume setup changed the parent checkpoint source'); + } + const continuation = requireContinuation(run); + if (!continuation.resumeSetup) throw new Error('repair continuation did not record resume setup'); + continuation.resumeSetup.sourceVerified = true; + } + if (args.referenceMutationOnly) { + const session = requireBuild(); + appendLevelRecord({ level, score: null, max: null, graded: false, contractPass: null, + selection: null, + outcome: { kind: 'ungraded', phase: 'reference-mutation-only', + reason: 'the parent qualification owns the full clean grade', + appFailures: [], inconclusive: [], harnessFailures: [] }, + buildSessions: [runSessionRecord(session)], + buildCostUsd: session.costUsd, sessionTotals: summarizeSessions([session]), + costUsd: session.costUsd, durationMs: Date.now() - t0 }); + break; + } + const firstBuildDirectory = continuing + ? `baseline-l${level}${featureActionSuffix}` + : `first-build-l${level}${featureActionSuffix}`; + const firstBuildPath = join(args.out, firstBuildDirectory); + let firstBuildSource = null; + let materializationOutcome: RunOutcome | null = null; + try { + const liveSource = hashAppSource(appDir); + snapshotSource(appDir, firstBuildPath); + const preservedSource = hashDirectory(firstBuildPath); + if (liveSource.sha256 !== preservedSource.sha256) { + throw new Error('preserved first-build source differs from the live application source'); + } + firstBuildSource = { sha256: liveSource.sha256, files: liveSource.files.length }; + if (applicationControl) { + await materializeAcceptedSource(firstBuildPath, appDir, applicationControl); + } + console.log(` kept the ${continuing ? 'continuation baseline' : 'unaided'} source at ${firstBuildPath}`); + } catch (error) { + if (firstBuildSource) { + materializationOutcome = materializationAppFailure(error); + keepStartLog(error, `${args.backend}-l${level}${featureActionSuffix}`); + } + console.log(materializationOutcome + ? ` !! ${materializationOutcome.reason}` + : ` !! could not bind the first-build source: ${errorMessage(error).split('\n')[0]}`); + } + const firstBuildLabel = `${args.backend}-l${level}${featureActionSuffix}`; + let bundle = firstBuildSource + ? grade(args, appDir, url, firstBuildLabel, level, track, runId, + { applicationFailure: materializationOutcome }) : null; + // A grader failure on unchanged source is retried once, as a repair grade is. + if (firstBuildSource && !materializationOutcome + && !levelGradeIsUsable(classifyBundle(bundle))) { + console.log(' grade did not complete; retrying the same source once'); + bundle = grade(args, appDir, url, `${firstBuildLabel}-retry`, level, track, runId); + } + let reusableRepairEvidence: { + bundle: GradeBundlePayload; + results: string; + } | null = null; + + // What the model built BEFORE being handed the answers. Every backend can + // reach the same total given enough repairs, so the post-fix score stops + // discriminating — what it got right unaided is the comparison that survives. + const firstBuild: FirstBuildRecord = { + score: bundle?.totals?.score ?? null, + max: bundle?.totals?.max ?? null, + regression: bundle?.totals?.regression ?? null, + contractPass: bundle?.totals?.contractPass ?? null, + outcome: materializationOutcome ?? sourceBoundFirstBuildOutcome(bundle, firstBuildSource), + source: firstBuildSource, + missed: Object.values(bundle?.suites ?? {}).flatMap(s => + (s?.features ?? []).flatMap(f => + (f.criteria ?? []).filter(c => !evidencePassed(criterionEvidence(c))) + .map(c => `${f.name}/${c.id}`))), + }; + + if (continuing) { + const repairGrant = args.repairGrant; + if (!repairGrant) throw new Error('repair continuation has no grant'); + if (firstBuild.score === null || firstBuild.max === null || firstBuildSource === null) { + throw new Error('repair continuation did not produce a source-bound baseline score'); + } + const reproduction = compareRepairBaseline(repairGrant.level, { + score: firstBuild.score, + max: firstBuild.max, + selectionSha256: bundle?.selection?.sha256 ?? null, + sourceSha256: firstBuildSource.sha256, + expectedSourceSha256: repairGrant.checkpoint.payload.source.sha256, + outcome: repairOutcome(firstBuild.outcome), + }); + requireContinuation(run).baseline = { + score: firstBuild.score, + max: firstBuild.max, + selectionSha256: bundle?.selection?.sha256 ?? null, + sourceSha256: firstBuildSource?.sha256 ?? null, + outcome: firstBuild.outcome, + ...reproduction, + }; + if (!reproduction.reproduced) { + const reason = `restored checkpoint did not reproduce its parent: ${reproduction.mismatches.join(', ')}`; + console.log(` CONTINUATION STOPPED: ${reason}`); + const failure = { kind: 'harness_failure', phase: 'continuation-baseline', reason, + appFailures: [], inconclusive: [], harnessFailures: [] }; + firstBuild.outcome = failure; + bundle = { ...bundle, outcome: failure }; + } + } + + const selectedObservedChecks = checksForGrade(args.recipeTasks?.get(level), 'observed'); + if (!continuing && !resumedRepair && selectedObservedChecks.length) { + const observationOut = join(args.out, + `first-build-l${level}${featureActionSuffix}-observed`); + let observationBundle = null; + let observationOutcome; + if (!firstBuildSource) { + observationOutcome = { kind: 'harness_failure', phase: 'first-build-source', + reason: 'observed specifications require a source-bound first build' }; + } else if (!ladderMayContinue(firstBuild.outcome)) { + observationOutcome = { kind: 'ungraded', phase: 'first-build-observation', + reason: 'scored first-build grading did not establish a usable environment' }; + } else { + observationBundle = grade(args, appDir, url, `${args.backend}-l${level}-observed`, level, + track, runId, { observation: 'observed', out: observationOut, + sourceSha256: firstBuildSource.sha256 }); + observationOutcome = classifyBundle(observationBundle); + } + firstBuild.observations = { + sourceSha256: firstBuildSource?.sha256 ?? null, + selectionSha256: args.recipeTasks?.get(level)?.selection.sha256 ?? null, + selectedChecks: selectedObservedChecks.map(check => check.stableKey), + reportedChecks: observationBundle?.selection?.reportedChecks ?? [], + passedPoints: observationBundle?.totals?.score ?? null, + observedPoints: observationBundle?.totals?.max ?? null, + scoreContribution: false, + repairVisible: false, + artifact: observationBundle + ? `first-build-l${level}${featureActionSuffix}-observed/${ARTIFACT_FILE.gradeBundle}` : null, + outcome: observationOutcome, + }; + } + + let initialProgressionFailure: ProgressionFailure | null = null; + if (featureActionSequence !== null && progressionSelection + && isProgressionWorkRecipeAction(progressionSelection)) { + const candidateOutcome = classifyBundle(bundle); + archiveCandidateGrade(`l${level}${featureActionSuffix}`); + if (!levelGradeIsUsable(candidateOutcome)) { + await restoreFeatureAcceptedSource(); + initialProgressionFailure = progressionFailure(candidateOutcome); + } else if (!featureCandidateAccepted(progressionSelection, + requireProgressionState(progressionExecution?.state ?? null), + bundle)) { + bundle = await gradeAcceptedSource(join(outputDir, 'source'), + `${args.backend}-l${level}${featureActionSuffix}-restored`, + ); + const restoredOutcome = classifyBundle(bundle); + if (!levelGradeIsUsable(restoredOutcome)) { + initialProgressionFailure = progressionFailure(restoredOutcome); + } + } + } + + // Preserve the first source and scored grading before repair overwrites the + // app. Observed evidence remains in its own source-bound result directory. + const acceptedGradingDirectory = continuing + ? `baseline-l${level}${featureActionSuffix}-grading` + : `first-build-l${level}${featureActionSuffix}-grading`; + try { + const gradingFrom = join(appDir, 'stack-bench'); + if (existsSync(gradingFrom)) { + const gradingTo = join(args.out, acceptedGradingDirectory); + cpSync(gradingFrom, gradingTo, { + recursive: true, + filter: src => !/[\\/]media([\\/]|$)/.test(src), + }); + if (bundle) reusableRepairEvidence = { bundle, results: gradingTo }; + console.log(` kept the ${continuing ? 'continuation baseline' : 'unaided'} grading at ${join(args.out, acceptedGradingDirectory)}`); + } + } catch (e) { + // Never worth losing a run over: the score is already recorded. + console.log(` !! could not keep the first build: ${errorMessage(e).split('\n')[0]}`); + } + + // A resumed grade settles a repair that was already charged; only a + // resumed coding session is a new repair. + let repairs = resumedRepair && !resumedGrade ? 1 : 0; + let repairCost = resumedRepair && build ? build.costUsd : 0; + const repairSessions = resumedRepair && build + ? [runSessionRecord(build, priorRepairs + 1)] : []; + const repairHistory: ReturnType[] = []; + let repairCandidate: RunRepairCandidate | null = null; + let priorRegressionReport = resumedRegressionReport; + let priorRegressionOwner = repairOwnerNodeIds(progressionSelection).join('\n') || null; + let regressed = false; + let repairStopReason: string | null = null; + let repairProgress = repairProgressState(null, bundle); + const pauseForRepeatedFindings = () => { + if (args.progression) return false; + repairProgress = repairProgressState(repairProgress, bundle); + if (args.maxStalledRepairs === 0 + || repairProgress.stalledRounds < args.maxStalledRepairs) return false; + repairStopReason = 'repeated-findings'; + console.log(` pausing after ${repairProgress.stalledRounds} repairs ` + + 'with the same failed checks and no score gain'); + return true; + }; + const initialBundleOutcome = classifyBundle(bundle); + const initialProgressionAttempt = args.progression + ? requireProgressionState(progressionExecution?.state ?? null).attempts.at(-1) ?? null + : null; + const initialGradeUsable = levelGradeIsUsable(initialBundleOutcome, + initialProgressionAttempt); + if (!initialGradeUsable) { + repairStopReason = 'initial-grading-failed'; + console.log(' repairs skipped: the initial grade did not complete, so there are no reliable findings to fix'); + } + + let progressionNext = recordProgressionGrade({ + selected: progressionSelection, + bundle: initialProgressionFailure ? null : bundle, + level, + failure: initialProgressionFailure, + completedRepair: resumedRepair && !resumedGrade, + }); + // Never start a paid repair with nothing left to spend or while a charged + // repair still waits for its grade. + const progressionMayRepair = () => !args.progression + || (progressionNext?.type === 'repair' && progressionNext.repair.remaining > 0 + && progressionNext.repair.awaitingGrade !== true); + // One allowance for both modes: the engine's remaining repairs in + // dependency mode, the run-wide total less every repair so far otherwise. + const repairsRemaining = (): number => args.progression + ? (progressionNext?.type === 'repair' ? progressionNext.repair.remaining : 0) + : args.repairs - priorRepairs - repairs; + const mayRepair = (): boolean => args.progression + ? progressionMayRepair() : repairsRemaining() > 0; + const recordRepairProgression = ({ failure = null, repairRegression = null, + completedRepair = false }: { + failure?: ProgressionFailure | null; + repairRegression?: ProgressionRepairRegression | null; + completedRepair?: boolean; + } = {}) => { + progressionNext = recordProgressionGrade({ + selected: progressionSelection, + bundle: failure ? null : bundle, + level, + failure, + repairRegression, + completedRepair, + }); + return progressionMayRepair(); + }; + const recordRepairHarnessFailure = (phase: string, reason: string, + failedBundle: GradeBundlePayload | null = null, completedRepair = false): void => { + const failure: RunOutcome = { + kind: 'harness_failure', phase, reason, + appFailures: [], inconclusive: [], harnessFailures: [reason], + }; + bundle = failedBundle ? { ...failedBundle, outcome: failure } : { outcome: failure }; + repairStopReason = phase; + if (args.progression) recordRepairProgression({ + failure: progressionFailure(failure), completedRepair, + }); + }; + const restoreProgressionGrade = (accepted: GradeBundlePayload | null, + label: string): boolean => { + const expected = progressionSelection && isProgressionWorkRecipeAction(progressionSelection) + ? progressionSelection.grader.selectionSha256 : null; + if (!expected || accepted?.selection?.sha256 === expected) { + bundle = accepted; + return true; + } + bundle = grade(args, appDir, url, label, level, track, runId); + const outcome = classifyBundle(bundle); + if (levelGradeIsUsable(outcome)) return true; + recordRepairHarnessFailure('repair-restore-grading', + outcome.reason ?? 'restored source did not produce a reliable grade', bundle, true); + return false; + }; + const writeRepairReport = (results: string | null = null): + { status: 0 | 3 | 4 } | { status: 'failed'; reason: string } => { + try { + sh('node', [join(ROOT, 'dist', 'commands', 'report-bugs.js'), '--app', appDir, + ...(results ? ['--results', results] : []), + '--history-json', JSON.stringify(repairHistory), + '--archive', join(outputDir, 'repair-reports', + `bug-report-l${level}${featureActionSuffix}-round${repairs + 1}.md`), + ...(priorRegressionReport ? ['--prior-regression', priorRegressionReport] : []), + ...repairReportArgs(progressionSelection)], { stdio: 'pipe' }); + return { status: 0 }; + } catch (error) { + const failure = commandFailure(error); + if (failure.status === 3 || failure.status === 4) return { status: failure.status }; + return { status: 'failed', reason: errorMessage(failure).split(/\r?\n/)[0] + ?? 'repair report generation failed' }; + } + }; + const recordMissingRepairFeedback = (status: 3 | 4): void => { + repairStopReason = 'no-actionable-findings'; + if (!args.progression) return; + const reason = status === 3 + ? 'selected repair checks contain no failures' + : 'selected repair checks produced no actionable findings'; + const failure: RunOutcome = { + kind: 'harness_failure', phase: 'repair-report', reason, + appFailures: [], inconclusive: [], harnessFailures: [reason], + }; + bundle = { outcome: failure }; + recordRepairProgression({ failure: progressionFailure(failure) }); + }; + + // Hand back findings and let the agent fix, until clean or out of rounds. + while (levelGradeIsUsable(classifyBundle(bundle), args.progression + ? requireProgressionState(progressionExecution?.state ?? null).attempts.at(-1) ?? null + : null) && mayRepair()) { + let reportReady = false; + const acceptedBundle = bundle; + let repairBaselineBundle = bundle; + if (args.progression) { + progressionSelection = bindProgressionAction(level); + const repairOwner = repairOwnerNodeIds(progressionSelection).join('\n') || null; + if (repairOwner !== priorRegressionOwner) { + priorRegressionReport = null; + priorRegressionOwner = repairOwner; + } + trackProgressionBudget(progressionSelection, priorRepairs + repairs); + if (progressionSelection && isProgressionWorkRecipeAction(progressionSelection) + && bundle?.selection?.sha256 !== progressionSelection.grader.selectionSha256) { + const sequence = requireProgressionState(progressionExecution?.state ?? null).attempts.length + 1; + const targetChecks = repairCheckKeys(progressionSelection); + const sourceSha256 = hashAppSource(appDir).sha256; + const hasCurrentTargetEvidence = bundle?.source?.sha256 === sourceSha256 + && targetChecks.every(check => bundle?.selection?.reportedChecks?.includes(check)); + const reusable = reusableRepairEvidence?.bundle.source?.sha256 === sourceSha256 + && targetChecks.every(check => + reusableRepairEvidence?.bundle.selection?.reportedChecks?.includes(check)) + ? reusableRepairEvidence : null; + let repairResults = hasCurrentTargetEvidence ? null : reusable?.results ?? null; + if (reusable && !hasCurrentTargetEvidence) repairBaselineBundle = reusable.bundle; + if (!hasCurrentTargetEvidence && !reusable) { + const binding = args.recipeBindings.get(level); + if (!binding) throw new Error(`L${level} has no recipe binding`); + const targetTask = resolveProgressionRepairTarget(binding, + requireProgressionState(progressionExecution?.state ?? null)); + repairResults = join(outputDir, 'repair-grades', + `l${level}${featureActionSuffix}-round-${repairs + 1}`); + repairBaselineBundle = grade(args, appDir, url, + `${args.backend}-l${level}-repair-target${sequence}`, + level, track, runId, { out: repairResults, recipeTask: targetTask }); + } + const refreshOutcome = classifyBundle(repairBaselineBundle); + const refreshUsable = levelGradeIsUsable(refreshOutcome); + if (!refreshUsable) { + recordRepairHarnessFailure('refresh-grading-failed', + refreshOutcome.reason ?? 'repair target did not produce a reliable grade', + repairBaselineBundle); + break; + } + const refreshReport = writeRepairReport(repairResults); + if (refreshReport.status === 'failed') { + recordRepairHarnessFailure('repair-report', refreshReport.reason); + break; + } + if (refreshReport.status === 3) { + bundle = grade(args, appDir, url, + `${args.backend}-l${level}-repair-refresh${sequence}`, + level, track, runId); + if (!levelGradeIsUsable(classifyBundle(bundle))) { + recordRepairHarnessFailure('refresh-grading-failed', + classifyBundle(bundle).reason ?? 'repair refresh did not produce a reliable grade', + bundle); + break; + } + recordRepairProgression(); + continue; + } + if (refreshReport.status === 4) { + recordMissingRepairFeedback(refreshReport.status); + break; + } + reportReady = true; + } + } + const report = reportReady ? { status: 0 as const } : writeRepairReport(); + if (report.status === 'failed') { + recordRepairHarnessFailure('repair-report', report.reason); + break; + } + if (report.status !== 0) { + recordMissingRepairFeedback(report.status); + break; + } + + const before = repairBaselineBundle?.totals?.score ?? 0; + const beforeMax = repairBaselineBundle?.totals?.max ?? 0; + const beforeBundle = repairBaselineBundle; + // Keep the accepted source outside paths visible to the coding session. + const snapshot = join(tmpdir(), `stack-bench-snapshot-${args.backend}-${args.track}-run${args.runIndex}-l${level}`); + const gradingSnapshot = `${snapshot}-grading`; + const acceptedSource = hashAppSource(appDir); + snapshotSource(appDir, snapshot); + rmSync(gradingSnapshot, { recursive: true, force: true }); + if (existsSync(join(appDir, 'stack-bench'))) { + cpSync(join(appDir, 'stack-bench'), gradingSnapshot, { recursive: true }); + } + const cleanupRepairSnapshots = () => { + rmSync(snapshot, { recursive: true, force: true }); + rmSync(gradingSnapshot, { recursive: true, force: true }); + }; + try { + const displayedRepairBudget = args.progression + ? progressionRepairLimit + : args.repairs; + if (args.dependencyPolicy?.definition.repair.selection === 'feature' + && progressionSelection && isProgressionWorkRecipeAction(progressionSelection)) { + const [nodeId] = progressionSelection.action.repair.nodeIds; + const node = requireProgressionState(progressionExecution?.state ?? null).definition.nodes + .find(candidate => candidate.id === nodeId); + if (!nodeId || !node) { + throw new Error('feature repair has no selected feature'); + } + const used = requireProgressionState(progressionExecution?.state ?? null) + .nodes[nodeId]?.repairs.used ?? 0; + console.log(`--- feature repair ${used + 1}: ${node.title} ---`); + } else { + console.log(`--- repair ${priorRepairs + repairs + 1}/${displayedRepairBudget} ---`); + } + const fix = await runAgentForLevel('fix', level, + featureActionSequence === null ? undefined : restoreFeatureAcceptedSource); + repairCost += fix.costUsd; + repairSessions.push(runSessionRecord(fix, priorRepairs + repairs + 1)); + + const fixFailure = agentSessionFailure(fix); + if (fixFailure) { + if (featureActionSequence !== null) { + await restoreAcceptedRepair(snapshot, gradingSnapshot); + } + console.log(` coding session failed: ${fixFailure.reason}; stopping repairs`); + bundle = { outcome: fixFailure }; + repairHistory.push(repairHistoryEntry(repairs + 1, beforeBundle, bundle, + 'agent session failed')); + repairStopReason = 'agent-session-failure'; + recordRepairProgression({ failure: progressionFailure(fixFailure) }); + break; + } + repairs += 1; + + // Reject contaminated repairs before spending time on grading. + const fixLeak = auditContamination(appDir, ownPorts, auditsTranscripts); + if (fixLeak) { + const buildSession = build ? runSessionRecord(build) : null; + const sessions = resumedRepair || !buildSession + ? repairSessions : [buildSession, ...repairSessions]; + const sessionTotals = summarizeSessions(sessions); + cleanupRepairSnapshots(); + abortUnusableSession(`repair ${repairs}`, fixLeak, { + level, graded: false, score: null, max: null, + selection: bundle?.selection ?? null, + ...(resumedRepair + ? { resumedRepair: firstBuild } + : continuing + ? { baseline: firstBuild, resumeCostUsd: requireBuild().costUsd, + resumeSession: runSessionRecord(requireBuild()) } + : { firstBuild, buildCostUsd: requireBuild().costUsd, + buildSessions: [runSessionRecord(requireBuild())] }), + repairCostUsd: addCostUsd(repairCost), repairSessions, repairs, + ...(resumedRepair ? { priorRepairs, + cumulativeRepairs: priorRepairs + repairs } : {}), + repair: { status: 'ungraded', limit: displayedRepairBudget, + used: priorRepairs + repairs, + stopReason: fixLeak.kind === 'harness_failure' ? 'audit-failure' : 'contaminated' }, + sessionTotals, + costUsd: resumedRepair ? addCostUsd(repairCost) + : addCostUsd(requireBuild().costUsd, repairCost), + durationMs: Date.now() - t0, + }, progressionSelection, true); + } + if (hashAppSource(appDir).sha256 === acceptedSource.sha256) { + clearPrivateGradingEvidence(appDir); + if (existsSync(gradingSnapshot)) { + cpSync(gradingSnapshot, join(appDir, 'stack-bench'), { recursive: true }); + } + const reason = 'repair made no source change'; + console.log(` ${reason}; ${args.progression + ? 'counting the failed attempt' + : 'pausing before another paid round'}`); + repairHistory.push(repairHistoryEntry(repairs, beforeBundle, beforeBundle, reason)); + if (args.progression) { + if (!restoreProgressionGrade(acceptedBundle, + `${args.backend}-l${level}-unchanged${repairs}`)) break; + if (!recordRepairProgression({ completedRepair: true })) break; + continue; + } + repairStopReason = 'no-source-change'; + break; + } + const repairedSource = `${snapshot}-accepted`; + snapshotSource(appDir, repairedSource); + try { + bundle = await gradeAcceptedSource(repairedSource, + `${args.backend}-l${level}-fix${repairs}`); + if (!levelGradeIsUsable(classifyBundle(bundle))) { + console.log(' repair grade did not complete; retrying the same source once'); + bundle = await gradeAcceptedSource(repairedSource, + `${args.backend}-l${level}-fix${repairs}-retry`); + } + } finally { + rmSync(repairedSource, { recursive: true, force: true }); + } + + const repairedOutcome = classifyBundle(bundle); + if (!levelGradeIsUsable(repairedOutcome)) { + const reason = repairedOutcome.reason + ?? 'the repaired source did not produce a reliable grade'; + // The session is paid for and charged. Keep what it produced so a + // resume grades it instead of buying another repair; the feature's + // status waits for that grade. + const candidateDirectory = `repair-candidate-l${level}${featureActionSuffix}`; + let preserveFailure: string | null = null; + try { + repairCandidate = preserveRepairCandidate(candidateDirectory); + console.log(` repair grade failed: ${reason}; kept the repaired source at ` + + `${join(outputDir, candidateDirectory)} for grading on resume`); + } catch (error) { + preserveFailure = errorMessage(error).split(/\r?\n/)[0] + ?? 'could not keep the repaired source'; + console.log(` repair grade failed: ${reason}; ${preserveFailure}; restoring the accepted source`); + await restoreAcceptedRepair(snapshot, gradingSnapshot); + } + repairHistory.push(repairHistoryEntry(repairs, beforeBundle, bundle, + 'repair completed; grading failed')); + recordRepairHarnessFailure('repair-grading', + preserveFailure ? `${reason}; ${preserveFailure}` : reason, bundle, true); + break; + } + + if (featureActionSequence !== null && progressionSelection + && isProgressionWorkRecipeAction(progressionSelection) + && !featureCandidateAccepted(progressionSelection, + requireProgressionState(progressionExecution?.state ?? null), bundle)) { + const rejectedBundle = bundle; + archiveCandidateGrade(`l${level}${featureActionSuffix}-repair${repairs}`); + await restoreAcceptedRepair(snapshot, gradingSnapshot); + if (!restoreProgressionGrade(acceptedBundle, + `${args.backend}-l${level}${featureActionSuffix}-rollback${repairs}`)) break; + repairHistory.push(repairHistoryEntry(repairs, beforeBundle, rejectedBundle, + 'rolled back because the feature still failed or earlier behavior regressed')); + if (!recordRepairProgression({ completedRepair: true })) break; + if (pauseForRepeatedFindings()) break; + continue; + } + + const after = bundle?.totals?.score ?? 0; + const afterMax = bundle?.totals?.max ?? 0; + // Lost or inconclusive evidence cannot hide a repair regression. + let decision = repairEvidenceDecision(beforeBundle, bundle); + const regressionDecision = repairRegressionDecision(acceptedBundle, bundle); + if (regressionDecision.action === 'rollback-regression') decision = regressionDecision; + const shared = decision.shared; + if (decision.action === 'keep-setup-repair') { + console.log(afterMax > 0 + ? ` application setup is now gradeable (${after}/${afterMax}); keeping this repair` + : ' application setup is still failing; keeping the attempted repair for the next round'); + repairHistory.push(repairHistoryEntry(repairs, beforeBundle, bundle, + afterMax > 0 + ? 'kept because the app became gradeable' + : 'kept to continue repairing application setup')); + if (!recordRepairProgression({ completedRepair: true })) break; + if (pauseForRepeatedFindings()) break; + continue; + } + if (decision.action === 'rollback-no-comparison') { + console.log(' no criteria were conclusively scored in both rounds; rolling back this fix'); + await restoreAcceptedRepair(snapshot, gradingSnapshot); + if (!restoreProgressionGrade(acceptedBundle, + `${args.backend}-l${level}-rollback${repairs}`)) break; + repairHistory.push(repairHistoryEntry(repairs, beforeBundle, bundle, + 'rolled back because the result could not be compared')); + if (!recordRepairProgression({ completedRepair: true })) break; + if (pauseForRepeatedFindings()) break; + continue; + } + if (shared.points < Math.min(beforeMax, afterMax)) { + console.log(` comparing ${shared.points} point(s) across ${shared.count} criteria scored in both rounds` + + ` (${before}/${beforeMax} -> ${after}/${afterMax} overall)`); + } + if (decision.action === 'rollback-regression') { + if (shared.regressions.length) { + console.log(` broke ${shared.regressions.length} earlier passing check(s); rolling back this fix`); + } else if (shared.lostEvidence.length) { + console.log(` lost conclusive evidence for ${shared.lostEvidence.length} criterion/criteria; rolling back this fix`); + } else if (shared.definitionChanges.length) { + console.log(' rubric points changed between grades; rolling back this fix'); + } else { + console.log(` regressed (${shared.before} -> ${shared.after} on shared criteria); rolling back this fix`); + } + let repairRegression: ProgressionRepairRegression | null = null; + let regressionReportFailure: string | null = null; + try { + if (shared.regressions.length) { + const path = join(outputDir, 'repair-reports', + `rejected-regression-l${level}${featureActionSuffix}-round${repairs}.md`); + sh('node', [join(ROOT, 'dist', 'commands', 'report-bugs.js'), '--app', appDir, + '--out', path, '--checks-json', JSON.stringify(shared.regressions), + '--regression-context'], { stdio: 'pipe' }); + repairRegression = { + ownerNodeIds: repairOwnerNodeIds(progressionSelection), + report: readFileSync(path, 'utf8'), + }; + priorRegressionReport = path; + } + } catch (error) { + regressionReportFailure = errorMessage(error).split(/\r?\n/)[0] + ?? 'regression report generation failed'; + } finally { + await restoreAcceptedRepair(snapshot, gradingSnapshot); + } + if (regressionReportFailure) { + repairHistory.push(repairHistoryEntry(repairs, beforeBundle, bundle, + 'repair completed; regression reporting failed')); + recordRepairHarnessFailure('repair-regression-report', regressionReportFailure, + null, true); + break; + } + if (!restoreProgressionGrade(acceptedBundle, + `${args.backend}-l${level}-rollback${repairs}`)) break; + regressed = true; + repairHistory.push(repairHistoryEntry(repairs, beforeBundle, bundle, + 'rolled back because earlier behavior regressed')); + if (!recordRepairProgression({ repairRegression, completedRepair: true })) break; + if (pauseForRepeatedFindings()) break; + continue; + } + priorRegressionReport = null; + if (shared.after === shared.before) { + const remaining = displayedRepairBudget - priorRepairs - repairs; + console.log(` ${formatRepairProgress(shared, { before, beforeMax, after, afterMax })}; ` + + (remaining > 0 ? `${remaining} repair(s) remain` : 'repair budget exhausted')); + } + repairHistory.push(repairHistoryEntry(repairs, beforeBundle, bundle, + shared.after === shared.before ? 'kept with no score gain' : 'kept')); + if (!recordRepairProgression({ completedRepair: true })) break; + if (pauseForRepeatedFindings()) break; + } finally { + cleanupRepairSnapshots(); + } + } + + // Missing grade evidence is not a zero score. + const progressionState = progressionExecution + ? requireProgressionState(progressionExecution.state) : null; + const progressionAttempt = progressionState + ? progressionState.attempts.findLast(attempt => attempt.level === level) ?? null + : null; + const levelBundle = progressionAttempt?.outcome === 'inconclusive' + ? bundle : progressionBundles.get(level) ?? bundle; + const finalBundleOutcome = classifyBundle(levelBundle); + // Progression uses stricter evidence rules than a regular scored bundle. + // Store one answer when a selected check is not measured: the raw bundle + // remains available for diagnosis, but the level is not a usable grade. + const graded = levelGradeIsUsable(finalBundleOutcome, + args.progression ? progressionAttempt : null); + const finalTotals = graded ? levelBundle?.totals ?? null : null; + const nodeRepairs = progressionState + ? dependencyRepairRecords(progressionState, level, levelRepairNodeIds) + : null; + const repairLimit = progressionExecution + ? Math.max(priorRepairs + repairs, progressionRepairLimit) + : args.repairs; + const repairBudgetExhausted = progressionExecution + ? finalBundleOutcome.kind === 'app_failure' + && progressionNext?.type !== 'repair' + : priorRepairs + repairs >= args.repairs; + const repairStatus: RepairStatus = repairStopReason === 'no-source-change' ? 'incomplete' + : !graded ? 'ungraded' + : finalBundleOutcome.kind === 'passed' ? (repairs > 0 ? 'corrected' : 'not-needed') + : repairBudgetExhausted ? 'budget-exhausted' : 'incomplete'; + const stopReasons: Record = { + 'not-needed': 'not-needed', + corrected: 'passed', + 'budget-exhausted': 'budget-exhausted', + incomplete: null, + ungraded: null, + }; + const stopReason = repairStopReason ?? stopReasons[repairStatus]; + const repair = { + status: repairStatus, + limit: repairLimit, + used: priorRepairs + repairs, + ...(!args.progression ? { stallLimitRounds: args.maxStalledRepairs } : {}), + stopReason, + ...(nodeRepairs ? { nodeRepairs } : {}), + ...(repairCandidate ? { candidate: repairCandidate } : {}), + }; + const latestProgressionAttempt = progressionState?.attempts.at(-1) ?? null; + const featureDepthContinues = featureActionSequence !== null + && progressionState?.phase === 'active' && progressionState.level === level; + if (progressionState && latestProgressionAttempt && latestProgressionAttempt.level !== level) { + const prior = run.levels.find(item => item.level === latestProgressionAttempt.level); + const latestBundle = progressionBundles.get(latestProgressionAttempt.level); + if (prior && latestBundle) { + const latestOutcome = classifyBundle(latestBundle); + const latestGraded = levelGradeIsUsable(latestOutcome, latestProgressionAttempt); + prior.graded = latestGraded; + prior.score = latestGraded ? latestBundle.totals?.score ?? null : null; + prior.max = latestGraded ? latestBundle.totals?.max ?? null : null; + prior.regression = latestBundle.totals?.regression ?? null; + prior.selection = latestBundle.selection ?? null; + prior.contractPass = latestBundle.totals?.contractPass ?? null; + prior.code = latestBundle.code ?? null; + prior.repair = { ...repair, + nodeRepairs: dependencyRepairRecords( + progressionState, latestProgressionAttempt.level, levelRepairNodeIds), + }; + prior.stalled = repairStatus === 'budget-exhausted'; + prior.outcome = latestOutcome; + } + } + if (continuing) { + const continuation = requireContinuation(run); + continuation.cumulativeRepairsAfter = continuation.cumulativeRepairsBefore + repairs; + } + let checkpoint = null; + if (graded && !featureDepthContinues + && (!latestProgressionAttempt || latestProgressionAttempt.level === level)) { + try { + checkpoint = preserveLevelCheckpoint({ + appDir, + outputDir: args.out, + runId, + identities: run.identities, + track: args.track, + backend: args.backend, + level, + repair, + outcome: finalBundleOutcome, + selectionSha256: levelBundle?.selection?.sha256 ?? null, + }); + console.log(` kept the L${level} source checkpoint at ${join(args.out, checkpoint.directory)}`); + } catch (error) { + console.log(` !! could not keep the L${level} source checkpoint: ${errorMessage(error).split('\n')[0]}`); + } + } + if (!graded) { + console.log(` L${level}: GRADING DID NOT COMPLETE — no usable bundle. ` + + `Score is unknown, not zero; re-grade this level before using the run.`); + } + const buildSession = build ? runSessionRecord(build) : null; + const requireBuildSession = (): RunSessionRecord => { + if (!buildSession) throw new Error(`level ${level} has no coding session`); + return buildSession; + }; + const sessionTotals = summarizeSessions(resumedRepair || !buildSession ? repairSessions + : [buildSession, ...repairSessions]); + appendLevelRecord({ + level, + graded, + score: finalTotals?.score ?? null, + max: finalTotals?.max ?? null, + // Preserve earlier-level guarantees in the durable result. + regression: levelBundle?.totals?.regression ?? null, + selection: levelBundle?.selection ?? null, + ...(resumedRepair + ? { resumedRepair: firstBuild } + : continuing + ? { baseline: firstBuild, resumeCostUsd: requireBuild().costUsd, + resumeSession: requireBuildSession() } + : featureActionSequence !== null + ? { buildCostUsd: requireBuild().costUsd, buildSessions: [requireBuildSession()] } + : { firstBuild, buildCostUsd: requireBuild().costUsd, + buildSessions: [requireBuildSession()] }), + contractPass: levelBundle?.totals?.contractPass ?? null, + code: levelBundle?.code ?? null, + repairCostUsd: addCostUsd(repairCost), + repairSessions, + repairHistory, + sessionTotals, + tokens: sessionTotals.tokens, + usage: sessionTotals.usage, + turns: sessionTotals.turns, + promptBytes: sessionTotals.promptBytes, + tokensPerTurn: sessionTotals.turns + ? Math.round(sessionTotals.tokens / sessionTotals.turns) : null, + // Record actual reasoning because the provider default is not pinned. + thinking: sessionTotals.thinking, + repairs, + ...(resumedRepair ? { priorRepairs, + cumulativeRepairs: priorRepairs + repairs } : {}), + repair, + checkpoint, + // Keep the summary flag derived from the typed status so the two cannot drift. + stalled: repairStatus === 'budget-exhausted' + || ['repeated-findings', 'no-source-change'].includes(repairStopReason ?? ''), + regressed, + outcome: finalBundleOutcome, + durationSec: Math.round((Date.now() - t0) / 1000), + }); + if (!args.progression || requireProgressionState(progressionExecution?.state ?? null).attempts + .some(attempt => attempt.level === level && attempt.outcome === 'conclusive')) { + if (!featureDepthContinues && !run.validation.ladder.completedLevels.includes(level)) { + run.validation.ladder.completedLevels.push(level); + } + } + writeRunJson(join(args.out, ARTIFACT_FILE.run), run); + const blockedLevels = args.levelList.filter(candidate => candidate > level); + if (args.progression) { + const progressionState = requireProgressionState(progressionExecution?.state ?? null); + if (progressionState.phase === 'terminal') { + if (blockedLevels.length) run.validation.ladder.stoppedAfterLevel = level; + run.validation.ladder.blockedLevels = blockedLevels; + writeRunJson(join(args.out, ARTIFACT_FILE.run), run); + break; + } + if (progressionState.level <= level) { + if (progressionState.attempts.at(-1)?.outcome === 'inconclusive') { + run.validation.ladder.stoppedAfterLevel = level; + run.validation.ladder.blockedLevels = [level, ...blockedLevels]; + writeRunJson(join(args.out, ARTIFACT_FILE.run), run); + break; + } + if (featureActionSequence !== null && progressionState.level === level) { + levelIndex -= 1; + continue; + } + throw new Error(`dependency progression did not leave L${level} after its repair budget`); + } + continue; + } + if (blockedLevels.length && !ladderMayAdvance(finalBundleOutcome)) { + run.validation.ladder.stoppedAfterLevel = level; + run.validation.ladder.blockedLevels = blockedLevels; + writeRunJson(join(args.out, ARTIFACT_FILE.run), run); + console.log(` ladder paused after L${level}: L${level} must pass before ` + + `${blockedLevels.map(candidate => `L${candidate}`).join(', ')} can start`); + console.log(' inspect the failures, then explicitly grant more repairs or correct the benchmark'); + break; + } + } + + if (args.mutations) { + console.log(`\n================ ${args.backend} mutation control ================`); + const pristineOutcome = aggregateRunOutcome(run.levels); + if (args.referenceMutationOnly || mutationControlEligible(pristineOutcome)) { + args.parentAttemptId = runId; + const baselineBundle = pristineMutationBaselinePath(args); + if (baselineBundle) args.mutationBaselineBundle = baselineBundle; + else delete args.mutationBaselineBundle; + run.mutationControl = runMutationControl(args, appDir, url, track, + run.setup?.isolation?.imageId ?? null); + } else { + console.log(` skipped: pristine outcome is ${pristineOutcome.kind}`); + run.mutationControl = { ok: false, skipped: true, + outcome: { kind: pristineOutcome.kind, phase: 'mutation-control-prerequisite', + reason: `pristine outcome is ${pristineOutcome.kind}` } }; + } + writeRunJson(join(args.out, ARTIFACT_FILE.run), run); + } + + // Record a final transcript audit in addition to the per-session hard gates. + // The same retry and diagnostic path is used at both gates. + let finalAuditFailure = null; + const finalAudit = auditContamination(appDir, ownPorts, auditsTranscripts); + if (!finalAudit) { + run.contaminated = false; + run.contamination = { evidence: 'no agent access to private benchmark files detected', + verdict: 'private-access audit passed' }; + } else if (finalAudit.kind === 'contaminated') { + run.contaminated = true; + run.contamination = { evidence: finalAudit.evidence, verdict: finalAudit.verdict }; + console.log('\n !! CONTAMINATED: this build read the harness that grades it:'); + for (const evidence of finalAudit.evidence) console.log(` ${evidence}`); + console.log(' Scores from this run must not be quoted.'); + } else { + run.contaminated = false; + run.contamination = { evidence: finalAudit.evidence, verdict: finalAudit.verdict }; + const reason = finalAudit.evidence.join('; '); + finalAuditFailure = { kind: 'harness_failure', phase: 'contamination-audit', reason, + appFailures: [], inconclusive: [], harnessFailures: [reason] }; + console.log('\n !! AUDIT DID NOT COMPLETE. Scores from this run must not be quoted.'); + } + + // Keep the transcript evidence outside the provider CLI's prunable store. + try { archiveTranscripts(appDir, artifactLabel); } + catch { console.log(' (transcript archiving failed — evidence is on a 30-day timer)'); } + + run.outcome = finalAuditFailure ?? (args.referenceMutationOnly && run.mutationControl?.ok + ? { kind: 'passed', phase: 'mutation-control', reason: null, + appFailures: [], inconclusive: [], harnessFailures: [] } + : aggregateRunOutcome(run.levels)); + if (args.mutations && !run.mutationControl?.ok && !run.mutationControl?.skipped) { + run.outcome = { kind: run.mutationControl?.outcome?.kind === 'incomplete' + ? 'incomplete' : 'harness_failure', phase: 'mutation-control', + reason: run.mutationControl?.outcome?.reason + ?? run.mutationControl?.processError + ?? 'one or more declared mutations were not cleanly caught', + appFailures: [], inconclusive: [] }; + } + + if (finalPackageEvidenceRequired(run.outcome, run.levels)) { + try { + preserveFinalPackageEvidence({ appDir, outputDir }); + console.log(` source kept at ${join(outputDir, 'source')}`); + console.log(` grading detail kept at ${join(outputDir, 'grading')}`); + } catch (error) { + const reason = errorMessage(error).split(/\r?\n/)[0] ?? 'evidence preservation failed'; + run.outcome = { kind: 'harness_failure', phase: 'evidence-preservation', reason, + appFailures: [], inconclusive: [], harnessFailures: [reason] }; + console.log(` !! ${reason}`); + } + } + + finalizeRunTotals(run, started, { costComplete: runCostComplete }); + if (args.repairGrant) { + const continuation = requireContinuation(run); + const totals = requireRunTotals(run); + continuation.cumulativeCostAfterUsd = addCostUsd(continuation.cumulativeCostBeforeUsd, totals.costUsd); + continuation.cumulativeDurationAfterSec = continuation.cumulativeDurationBeforeSec + totals.durationSec; + } + run.completedAt = new Date().toISOString(); + writeRunJson(join(args.out, ARTIFACT_FILE.run), run); + + console.log(`\n================ ${args.backend} summary ================`); + for (const l of run.levels) { + console.log(` ${formatLevelSummary(l)}`); + } + const totals = requireRunTotals(run); + console.log(` TOTAL ${totals.score}/${totals.max} ` + + `$${totals.costUsd} ${totals.repairs} repair(s) ${totals.durationSec}s`); + console.log(` ${join(outputDir, ARTIFACT_FILE.run)}`); + + teardown(); + + // Remove only the temporary directory created by this run. + if (ownWorkDir) { + try { + rmSync(dirname(appDir), { recursive: true, force: true }); + } catch { + console.log(` (work dir still held: ${dirname(appDir)} — the next sweep will take it)`); + } + } + process.exitCode = runExitCode(run.outcome); +} + +if (process.argv[1] && pathToFileURL(resolve(process.argv[1])).href === import.meta.url) { + main().catch(error => { + console.error(error instanceof Error ? error.stack ?? error.message : errorMessage(error)); + try { emergencyTeardown?.(); } + catch (cleanupError) { + console.error(`cleanup after failure also failed: ${errorMessage(cleanupError).split(/\r?\n/)[0]}`); + } + process.exitCode = 1; + }); +} diff --git a/tools/stack-bench/commands/campaign-cli.ts b/tools/stack-bench/commands/campaign-cli.ts new file mode 100644 index 00000000000..38c927d4ac0 --- /dev/null +++ b/tools/stack-bench/commands/campaign-cli.ts @@ -0,0 +1,288 @@ +#!/usr/bin/env node + +import { resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; + +import { compileCampaignFile } from '../src/campaigns/campaign-compiler.js'; +import { CAMPAIGN_MODE_REGISTRY } from '../src/campaigns/campaign-mode.js'; +import { executeCampaign, inspectCampaign, reconcileCampaign } + from '../src/campaigns/campaign-runner.js'; +import { inspectCampaignSummary } from '../src/campaigns/campaign-inspection.js'; +import { generateCampaignReport } from '../src/campaigns/campaign-report.js'; +import { grantCampaignDependencyRepairs } + from '../src/campaigns/campaign-progression-grant.js'; +import { auditProgressionReferenceCampaign, formatProgressionReferenceCampaignAudit } + from '../src/campaigns/progression-reference-campaign-audit.js'; +import type { ReferenceCampaignAudit } + from '../src/campaigns/progression-reference-campaign-audit.js'; +import { prepareCampaignExtension } from '../src/campaigns/campaign-extension.js'; + +interface CampaignSummaryPlan { + id: string; + version: string; + contentSha256: string; +} + +interface CampaignSummaryState { + status: string; + summary: unknown; + attempts: Array<{ + plan: { id: string }; + status: string; + executions: Array<{ + id: string; + outcome: unknown; + reason: string | null; + }>; + }>; +} + +interface ReferenceCampaignPlan { + attempts: Array<{ + mode?: { id?: string }; + agentAdapter?: string; + }>; +} + +interface ReferenceCampaignState { + status: string; +} + +interface ResumeCampaign { + plan: { + contentSha256: string; + definition: { mode?: { id?: string } }; + }; + state: { + status: string; + attempts: Array<{ executions: readonly unknown[] }>; + }; +} + +type ReferenceCampaignAuditFunction = (directory: string) => ReferenceCampaignAudit | null; + +export type CampaignArgs = + | { command: 'modes' } + | { command: 'validate'; path: string } + | { command: 'show'; path: string } + | { command: 'status'; directory: string; full: boolean } + | { command: 'inspect'; directory: string } + | { command: 'report'; directory: string } + | { command: 'audit'; directory: string } + | { command: 'grant-repairs'; directory: string; attemptId: string; grantId: string; + level: number; nodeIds: string[]; repairs: number } + | { command: 'extend'; path: string; parentDirectory: string; fromDepth: number; + directory: string } + | { command: 'trial'; path: string; directory: string } + | { command: 'run'; path: string; directory: string } + | { command: 'resume'; path: string; directory: string } + | { command: 'reconcile'; path: string; directory: string }; + +function isOneOf(value: string | undefined, + values: readonly T[]): value is T { + return value !== undefined && values.some(candidate => candidate === value); +} + +export function campaignStateSummary(plan: CampaignSummaryPlan, state: CampaignSummaryState) { + const failures = state.attempts.flatMap(attempt => { + const execution = attempt.executions.at(-1); + if (!execution || execution.outcome === null || execution.outcome === 'passed') return []; + return [{ + attempt: attempt.plan.id, + status: attempt.status, + execution: execution.id, + outcome: execution.outcome, + reason: execution.reason, + }]; + }); + return { + campaign: { id: plan.id, version: plan.version, sha256: plan.contentSha256 }, + status: state.status, + summary: state.summary, + failures, + }; +} + +export function auditCompletedReferenceCampaign(directory: string, plan: ReferenceCampaignPlan, + state: ReferenceCampaignState, { + audit = auditProgressionReferenceCampaign, +}: { audit?: ReferenceCampaignAuditFunction } = {}): ReferenceCampaignAudit | null { + const hasReferenceProgression = plan.attempts.some(attempt => + attempt.mode?.id === 'dependency' && attempt.agentAdapter === 'reference-fixture'); + return state.status === 'completed' && hasReferenceProgression ? audit(directory) : null; +} + +export function validateResumeCampaignState( + requested: { contentSha256: string }, existing: T): T { + if (requested.contentSha256 !== existing.plan.contentSha256) { + throw new Error('resume requires the exact campaign plan already stored in the output directory'); + } + if (existing.plan.definition.mode?.id !== 'dependency') { + throw new Error('resume is available only for dependency campaigns'); + } + const executions = existing.state.attempts.reduce((total, attempt) => + total + attempt.executions.length, 0); + if (existing.state.status !== 'prepared' || executions < 1) { + throw new Error('resume requires a dependency campaign with scheduled work'); + } + return existing; +} + +export function validateResumeCampaign(path: string, directory: string): ResumeCampaign { + return validateResumeCampaignState(compileCampaignFile(path), inspectCampaign(directory)); +} + +export function parseCampaignArgs(argv: string[]): CampaignArgs { + const [command, path, ...rest] = argv.slice(2); + if (command === 'modes' && path === undefined) return { command }; + if (isOneOf(command, ['validate', 'show']) && path && rest.length === 0) { + return { command, path: resolve(path) }; + } + if (command === 'status' && path + && (rest.length === 0 || (rest.length === 1 && rest[0] === '--full'))) { + return { command, directory: resolve(path), full: rest.length === 1 }; + } + if (isOneOf(command, ['inspect', 'report', 'audit']) && path && rest.length === 0) { + return { command, directory: resolve(path) }; + } + if (command === 'grant-repairs' && path) { + const values: { attemptId?: string; grantId?: string; level?: number; repairs?: number; + nodeIds: string[] } = { nodeIds: [] }; + const seen = new Set(); + for (let index = 0; index < rest.length; index += 2) { + const flag = rest[index]; + const value = rest[index + 1]; + if (flag === undefined || value === undefined + || !['--attempt', '--grant-id', '--level', '--feature', '--repairs'].includes(flag) + || (flag !== '--feature' && seen.has(flag))) { + throw new Error(`invalid or duplicate grant-repairs option ${String(flag)}`); + } + seen.add(flag); + if (flag === '--attempt') values.attemptId = value; + else if (flag === '--grant-id') values.grantId = value; + else if (flag === '--level') values.level = Number(value); + else if (flag === '--repairs') values.repairs = Number(value); + else values.nodeIds.push(value); + } + if (!values.attemptId || !values.grantId || typeof values.level !== 'number' + || !Number.isSafeInteger(values.level) || typeof values.repairs !== 'number' + || !Number.isSafeInteger(values.repairs) || values.nodeIds.length === 0) { + throw new Error('grant-repairs requires --attempt, --grant-id, --level, ' + + 'one or more --feature values, and --repairs'); + } + return { command, directory: resolve(path), attemptId: values.attemptId, + grantId: values.grantId, level: values.level, nodeIds: values.nodeIds, + repairs: values.repairs }; + } + if (command === 'extend' && path && rest.length === 6 + && rest[0] === '--from' && rest[2] === '--depth' && rest[4] === '--out') { + const fromDepth = Number(rest[3]); + if (!Number.isSafeInteger(fromDepth) || fromDepth < 1) { + throw new Error('extend --depth must be a positive integer'); + } + return { command, path: resolve(path), parentDirectory: resolve(rest[1]!), + fromDepth, directory: resolve(rest[5]!) }; + } + if (isOneOf(command, ['trial', 'run', 'resume', 'reconcile']) + && path && rest.length === 2 && rest[0] === '--out') { + return { command, path: resolve(path), directory: resolve(rest[1]!) }; + } + throw new Error('usage: campaign-cli.js modes | validate|show ' + + '| trial|run|resume|reconcile --out ' + + '| extend --from --depth --out ' + + '| status [--full] | inspect|report|audit ' + + '| grant-repairs --attempt --grant-id --level ' + + '--feature [--feature ...] --repairs '); +} + +async function main() { + const args = parseCampaignArgs(process.argv); + if (args.command === 'modes') { + console.log(JSON.stringify(CAMPAIGN_MODE_REGISTRY.ids.map(value => { + const [id, version] = value.split('@'); + return { id, version }; + }), null, 2)); + return; + } + if (args.command === 'status') { + const campaign = inspectCampaign(args.directory, { requireCurrentInputs: false }); + console.log(JSON.stringify(args.full + ? campaign.state + : campaignStateSummary(campaign.plan, campaign.state), null, 2)); + return; + } + if (args.command === 'inspect') { + console.log(JSON.stringify(inspectCampaignSummary(args.directory), null, 2)); + return; + } + if (args.command === 'report') { + const generated = generateCampaignReport(args.directory); + console.log(`${generated.reportPath}\n${generated.htmlPath}\n${generated.report.contentSha256}`); + return; + } + if (args.command === 'audit') { + const report = auditProgressionReferenceCampaign(args.directory); + if (report === null) throw new Error('campaign has no dependency reference attempts to audit'); + console.log(formatProgressionReferenceCampaignAudit(report)); + if (!report.ok) process.exitCode = 1; + return; + } + if (args.command === 'grant-repairs') { + console.log(JSON.stringify(grantCampaignDependencyRepairs(args.directory, { + attemptId: args.attemptId, + grantId: args.grantId, + level: args.level, + nodeIds: args.nodeIds, + repairs: args.repairs, + }), null, 2)); + return; + } + if (args.command === 'extend') { + prepareCampaignExtension(args.path, args.parentDirectory, args.directory, args.fromDepth); + const plan = compileCampaignFile(args.path); + const state = await executeCampaign(args.path, args.directory, { mode: 'frozen' }); + console.log(JSON.stringify(campaignStateSummary(plan, state), null, 2)); + if (state.status !== 'completed') process.exitCode = 1; + return; + } + const plan = compileCampaignFile(args.path); + if (args.command === 'reconcile') { + const state = reconcileCampaign(args.path, args.directory); + console.log(JSON.stringify(campaignStateSummary(plan, state), null, 2)); + return; + } + if (args.command === 'trial' || args.command === 'run' || args.command === 'resume') { + if (args.command === 'resume') validateResumeCampaign(args.path, args.directory); + const cancellation = new AbortController(); + const cancel = () => cancellation.abort(); + process.on('SIGINT', cancel); + process.on('SIGTERM', cancel); + let state; + try { + const executionMode = args.command === 'trial' + || (args.command === 'resume' && plan.state === 'draft') + ? 'model-free-trial' : 'frozen'; + state = await executeCampaign(args.path, args.directory, { + mode: executionMode, + signal: cancellation.signal, + }); + } finally { + process.off('SIGINT', cancel); + process.off('SIGTERM', cancel); + } + console.log(JSON.stringify(campaignStateSummary(plan, state), null, 2)); + const audit = auditCompletedReferenceCampaign(args.directory, plan, state); + if (audit !== null) console.log(formatProgressionReferenceCampaignAudit(audit)); + if (state.status !== 'completed' || audit?.ok === false) process.exitCode = 1; + return; + } + if (args.command === 'show') console.log(JSON.stringify(plan, null, 2)); + else console.log(`${plan.id}@${plan.version} ${plan.state}: ${plan.summary.attempts} attempts, ${plan.contentSha256}`); +} + +if (process.argv[1] && pathToFileURL(resolve(process.argv[1])).href === import.meta.url) { + main().catch((error: unknown) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 2; + }); +} diff --git a/tools/stack-bench/commands/check-actions.ts b/tools/stack-bench/commands/check-actions.ts new file mode 100644 index 00000000000..b213a3fc8a6 --- /dev/null +++ b/tools/stack-bench/commands/check-actions.ts @@ -0,0 +1,113 @@ +#!/usr/bin/env node +// Probes are unauthenticated or malformed and must never mutate data. + +import { parseArgs as parseNodeArgs } from 'node:util'; + +import { emptyArtifactIdentities, writeArtifact } from '../src/evidence/artifacts.js'; +import { loadTrack } from '../src/composition/tracks.js'; +import { STACK_ADAPTER_REGISTRY } from '../src/stacks/stack-adapters.js'; + +interface CheckActionsArgs { + backend?: string; + url?: string; + app?: string; + out?: string; + track?: string; + quiet?: boolean; + parentAttemptId?: string; +} + +import type { NamedAction } from '../src/composition/tracks.js'; + +interface ActionResult { + id: string; + ok: boolean; + status: number; + note: string; +} + +function parseArgs(argv: string[]): CheckActionsArgs { + const { values } = parseNodeArgs({ args: argv.slice(2), options: { + backend: { type: 'string' }, url: { type: 'string' }, app: { type: 'string' }, + out: { type: 'string' }, track: { type: 'string' }, quiet: { type: 'boolean' }, + 'parent-attempt-id': { type: 'string' }, + } }); + const a: CheckActionsArgs = { backend: values.backend, url: values.url, app: values.app, + out: values.out, track: values.track, quiet: values.quiet, + parentAttemptId: values['parent-attempt-id'] }; + if (!a.backend) { console.error('--backend is required'); process.exit(2); } + return a; +} + +const args = parseArgs(process.argv); +const backend = args.backend; +if (!backend) throw new Error('--backend is required'); + +// Use non-writing probes declared by the selected track. +const track = args.track ? loadTrack(args.track) : null; +const ACTIONS = (track?.actions ?? []).map(action => ({ ...action, + http: { method: 'POST', path: action.path } })); +if (!ACTIONS.length) { + if (!args.quiet) console.log(` no named actions declared for track "${args.track ?? '(none)'}" — nothing to check`); + if (args.out) { + const id = `${args.parentAttemptId ?? 'actions'}-action-check`; + writeArtifact(args.out, { kind: 'action_check', id, + attempt: { id, parentId: args.parentAttemptId ?? null }, + identities: emptyArtifactIdentities({ stackAdapter: { id: backend } }), + payload: { backend, results: [], missing: [] } }); + } + process.exit(0); +} + +// SpacetimeDB control targets come from the authenticated lease. Client config +// is app-controlled input and may use environment expressions rather than +// literals; it is neither authoritative nor safe for harness operations. +const adapter = STACK_ADAPTER_REGISTRY.get(backend); +const spacetime = adapter.grading.context({ requireBuildContainer: false }); + +async function probe(action: NamedAction): Promise> { + try { + const request = adapter.namedAction.request( + { action, input: { args: action.args }, spacetime, url: args.url }); + if (!request.url) return { ok: false, status: 0, note: 'no --url given for a server-based backend' }; + const r = await fetch(request.url, { + method: request.method ?? 'POST', + headers: { 'Content-Type': 'application/json' }, + body: request.body, + }); + const rejectedByApplication = 'applicationRejectionStatuses' in request + && request.applicationRejectionStatuses.includes(r.status); + const recognizedWithoutRunning = r.status >= 400 && r.status < 500 + && ![404, 405, 429].includes(r.status); + const ok = r.ok || rejectedByApplication || recognizedWithoutRunning; + return { ok, status: r.status, + note: r.status === 404 ? request.missingNote + : ok ? '' : `action probe returned HTTP ${r.status}` }; + } catch (error: unknown) { + const message = error instanceof Error ? error.message : String(error); + return { ok: false, status: 0, note: (message.split('\n')[0] ?? '').slice(0, 90) }; + } +} + +const results: ActionResult[] = await Promise.all(ACTIONS.map(async action => ({ + id: action.id, + ...(await probe(action)), +}))); + +const missing = results.filter(r => !r.ok); +if (!args.quiet) { + for (const r of results) { + console.log(` ${r.ok ? 'ready' : 'UNUSABLE'} ${r.id.padEnd(11)} ${r.status ? `HTTP ${r.status}` : ''} ${r.note}`); + } + console.log(missing.length + ? `\n${missing.length} named action(s) unusable — contention and volume tests cannot be issued against this app.` + : '\nall named actions are ready.'); +} +if (args.out) { + const id = `${args.parentAttemptId ?? 'actions'}-action-check`; + writeArtifact(args.out, { kind: 'action_check', id, + attempt: { id, parentId: args.parentAttemptId ?? null }, + identities: emptyArtifactIdentities({ stackAdapter: { id: backend } }), + payload: { backend, results, missing: missing.map(m => m.id) } }); +} +process.exit(missing.length ? 1 : 0); diff --git a/tools/stack-bench/commands/check-calibration.ts b/tools/stack-bench/commands/check-calibration.ts new file mode 100644 index 00000000000..188a72d9833 --- /dev/null +++ b/tools/stack-bench/commands/check-calibration.ts @@ -0,0 +1,71 @@ +#!/usr/bin/env node + +import { existsSync, readFileSync, readdirSync } from 'node:fs'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { parseArgs } from 'node:util'; + +import { compileCalibrationDefinition, compileCalibrationFile } from '../src/composition/calibration-compiler.js'; +import { buildRecipeRelease } from '../src/composition/recipe-release.js'; +import { listTracks, TRACKS_DIR } from '../src/composition/tracks.js'; + +import { STACK_BENCH_ROOT as ROOT } from '../src/package-root.js'; + +export interface CalibrationCheckResult { + track: string; + id: string; + version: string; + state: string; + recipe: string; + controls: number; + stacks: number; + contentSha256: string; +} + +export function checkCalibrations( + { trackName = null }: { trackName?: string | null } = {}, +): CalibrationCheckResult[] { + const availableTracks = listTracks({ includeInternal: true }); + if (trackName && !availableTracks.includes(trackName)) { + throw new Error(`unknown calibration track ${trackName}`); + } + const tracks = trackName ? [trackName] : availableTracks; + const results: CalibrationCheckResult[] = []; + for (const name of tracks) { + const trackRoot = join(TRACKS_DIR, name); + const directory = join(trackRoot, 'composition', 'calibrations'); + if (!existsSync(directory)) continue; + for (const file of readdirSync(directory).filter(candidate => candidate.endsWith('.json')).sort()) { + const path = join(directory, file); + const source = `composition/calibrations/${file}`; + const input = JSON.parse(readFileSync(path, 'utf8')); + const definition = compileCalibrationDefinition(input, { source }); + const recipePath = resolve(dirname(path), definition.recipe.path); + const release = buildRecipeRelease(recipePath, { trackRoot }); + const plan = compileCalibrationFile(path, { trackRoot, stackBenchRoot: ROOT, release }); + results.push({ track: name, id: plan.id, version: plan.version, state: plan.state, + recipe: plan.recipe.id, controls: plan.controls.length, stacks: plan.qualification.stacks.length, + contentSha256: plan.contentSha256 }); + } + } + return results; +} + +function main() { + const { values } = parseArgs({ args: process.argv.slice(2), options: { + track: { type: 'string' }, + }, strict: true, allowPositionals: false }); + const results = checkCalibrations({ trackName: values.track ?? null }); + for (const result of results) { + console.log(`${result.track}: ${result.id}@${result.version} ${result.state}; ` + + `${result.controls} controls, ${result.stacks} stacks, ${result.contentSha256.slice(0, 12)}`); + } +} + +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + try { main(); } + catch (error: unknown) { + console.error(error instanceof Error ? error.stack ?? error.message : String(error)); + process.exitCode = 1; + } +} diff --git a/tools/stack-bench/commands/check-composition.ts b/tools/stack-bench/commands/check-composition.ts new file mode 100644 index 00000000000..4548a0730d5 --- /dev/null +++ b/tools/stack-bench/commands/check-composition.ts @@ -0,0 +1,87 @@ +import { existsSync, readFileSync, readdirSync } from 'node:fs'; +import { join } from 'node:path'; +import { pathToFileURL } from 'node:url'; + +import { + compileFixtureDefinition, + compilePackDefinition, + compilePromotionFile, + compileRecipeFile, +} from '../src/composition/composition-compiler.js'; +import { TRACKS_DIR, listTracks } from '../src/composition/tracks.js'; + +function json(path: string): unknown { + try { return JSON.parse(readFileSync(path, 'utf8')); } + catch (error: unknown) { + const message = error instanceof Error ? error.message : String(error); + throw new Error(`cannot read composition source ${path}: ${message}`, { cause: error }); + } +} + +export interface CompositionSummary { + track: string; + packs: number; + fixtures: number; + recipes: number; + checks: number; + aliases: number; +} + +export function checkCompositions( + { trackName = null }: { trackName?: string | null } = {}, +): CompositionSummary[] { + const names = trackName ? [trackName] : listTracks({ includeInternal: true }); + const summary = []; + for (const name of names) { + const trackRoot = join(TRACKS_DIR, name); + const root = join(trackRoot, 'composition'); + if (!existsSync(root)) { + if (trackName) throw new Error(`track ${name} has no composition directory`); + continue; + } + const packs = join(root, 'packs'); + const fixtures = join(root, 'fixtures'); + const recipes = join(root, 'recipes'); + const packFiles = readdirSync(packs).filter(file => file.endsWith('.json')).sort(); + const fixtureFiles = readdirSync(fixtures).filter(file => file.endsWith('.json')).sort(); + const recipeFiles = readdirSync(recipes).filter(file => file.endsWith('.json')).sort(); + if (!packFiles.length || !fixtureFiles.length || !recipeFiles.length) { + throw new Error(`track ${name} composition must contain packs, fixtures, and recipes`); + } + for (const file of packFiles) { + const path = join(packs, file); + compilePackDefinition(json(path), { source: path }); + } + for (const file of fixtureFiles) { + const path = join(fixtures, file); + compileFixtureDefinition(json(path), { source: path }); + } + const plans = recipeFiles.map(file => compileRecipeFile(join(recipes, file), { trackRoot })); + const promotionPath = join(root, 'promotions.json'); + const promotion = existsSync(promotionPath) + ? compilePromotionFile(promotionPath, { trackRoot }) : null; + summary.push({ track: name, packs: packFiles.length, fixtures: fixtureFiles.length, + recipes: plans.length, checks: plans.reduce((total, plan) => total + plan.checks.length, 0), + aliases: promotion?.entries.length ?? 0 }); + } + return summary; +} + +function main(): void { + const args = process.argv.slice(2); + let trackName: string | null = null; + for (let index = 0; index < args.length; index += 1) { + const value = args[index + 1]; + if (args[index] === '--track' && value) { + trackName = value; + index += 1; + } else throw new Error(`unknown or incomplete argument ${args[index]}`); + } + const summary = checkCompositions({ trackName }); + if (!summary.length) throw new Error('no composition sources found'); + for (const row of summary) { + console.log(`${row.track}: ${row.packs} packs, ${row.fixtures} fixtures, ${row.recipes} recipes, ${row.checks} selected checks, ${row.aliases} promotion entries`); + } +} + +if (process.argv[1] && pathToFileURL(process.argv[1]).href === import.meta.url) main(); diff --git a/tools/stack-bench/commands/check-mutations.ts b/tools/stack-bench/commands/check-mutations.ts new file mode 100644 index 00000000000..042a6d9770b --- /dev/null +++ b/tools/stack-bench/commands/check-mutations.ts @@ -0,0 +1,81 @@ +#!/usr/bin/env node + +import { existsSync, readFileSync } from 'node:fs'; +import { parseArgs as parseNodeArgs } from 'node:util'; + +import { mutationFileEdits, resolveMutationFile, validateMutationDefinitions } + from '../src/evidence/mutation-analysis.js'; +import type { MutationDefinition } from '../src/evidence/mutation-analysis.js'; + +interface CliArgs { + app: string; + mutations: string; + quiet: boolean; +} + +interface MutationSpec { + anchoredTo?: unknown; + mutations?: MutationDefinition[]; +} + +function parseArgs(argv: string[]): CliArgs { + const { values: { app, mutations, quiet = false } } = parseNodeArgs({ args: argv.slice(2), + options: { app: { type: 'string' }, mutations: { type: 'string' }, quiet: { type: 'boolean' } } }); + if (!app || !mutations) { + console.error('Usage: node dist/commands/check-mutations.js --app --mutations '); + process.exit(2); + } + return { app, mutations, quiet }; +} + +const args = parseArgs(process.argv); +const parsed: unknown = JSON.parse(readFileSync(args.mutations, 'utf8')); +if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) { + throw new Error('mutation manifest must be an object'); +} +const spec = parsed as MutationSpec; +const say = (...message: unknown[]): void => { if (!args.quiet) console.log(...message); }; + +say(`mutations : ${args.mutations}`); +say(`app : ${args.app}`); +if (spec.anchoredTo) say(`anchored : ${String(spec.anchoredTo).split('.')[0]}`); +say(''); + +let bad = 0; +const definitions = validateMutationDefinitions(spec.mutations); +for (const issue of definitions.issues) { + console.log(` BAD MANIFEST ${issue.mutation ?? ''} -> ${issue.kind}`); + bad += 1; +} +for (const mutation of spec.mutations ?? []) { + const mutationId = String(mutation.id ?? ''); + for (const edit of mutationFileEdits(mutation)) { + let file: string; + try { file = resolveMutationFile(args.app, edit.file); } + catch { + console.log(` UNSAFE FILE ${mutationId} -> ${edit.file} escapes the app directory`); + bad += 1; + continue; + } + if (!existsSync(file)) { + console.log(` DEAD FILE ${mutationId} -> ${edit.file} does not exist in this app`); + bad += 1; + continue; + } + const source = readFileSync(file, 'utf8'); + const matches = source.split(edit.find).length - 1; + if (matches === 1) { + say(` ok ${mutationId} -> ${edit.file}`); + continue; + } + console.log(matches === 0 + ? ` DEAD ANCHOR ${mutationId} -> not found in ${edit.file}` + : ` AMBIGUOUS ${mutationId} -> matches ${matches}x in ${edit.file}; the edit would land in more than one place`); + bad += 1; + } +} + +console.log(bad + ? `\n${bad} problem(s) — these mutations cannot validate anything against this app.` + : '\nall anchors present and unique — this file can validate against this app.'); +process.exit(bad ? 1 : 0); diff --git a/tools/stack-bench/commands/check-scenarios.ts b/tools/stack-bench/commands/check-scenarios.ts new file mode 100644 index 00000000000..7fd01f2c378 --- /dev/null +++ b/tools/stack-bench/commands/check-scenarios.ts @@ -0,0 +1,304 @@ +#!/usr/bin/env node +// Check scenario action names, actors, UI hooks, and score totals without an app. + +import { existsSync, readFileSync, readdirSync } from 'node:fs'; +import { join } from 'node:path'; +import { parseArgs } from 'node:util'; + +import { ACTION_REGISTRY } from '../src/actions/action-catalog.js'; +import { compileRecipeFile, type CompiledOwnedTaskFragment, type CompiledRecipeRelease } + from '../src/composition/composition-compiler.js'; +import { compileScenarioDefinition, type CompiledStep } + from '../src/composition/definition-compiler.js'; +import { DEFAULT_TRACK, listTracks, loadTrack, type Track } + from '../src/composition/tracks.js'; + +interface ScenarioScope { + features: Map>; + contractOwners: Set; + requirementOwners: Set; + contractText: string; + requirementText: string; +} + +interface RecipeSource { + baseRecipe: string | null; + isolatesSelectedSources: boolean; +} + +type HooksByLevel = Map>; + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function readJson(path: string): unknown { + return JSON.parse(readFileSync(path, 'utf8')) as unknown; +} + +function readRecipeSource(path: string): RecipeSource { + const source = readJson(path); + if (!isRecord(source)) throw new Error(`${path}: recipe must be an object`); + const task = source.task; + if (!isRecord(task)) throw new Error(`${path}: recipe task must be an object`); + const baseRecipe = task.baseRecipe; + let baseRecipePath: string | null = null; + if (baseRecipe !== undefined) { + if (!isRecord(baseRecipe) || typeof baseRecipe.path !== 'string') { + throw new Error(`${path}: task.baseRecipe.path must be a string`); + } + baseRecipePath = baseRecipe.path; + } + return { + baseRecipe: baseRecipePath, + isolatesSelectedSources: source.execution === 'all-selected-sources', + }; +} + +function contractHookIds(path: string): string[] { + const contract = readJson(path); + if (!isRecord(contract) || !Array.isArray(contract.hooks)) { + throw new Error(`${path}: hooks must be an array`); + } + return contract.hooks.map((hook, index) => { + if (!isRecord(hook) || typeof hook.id !== 'string') { + throw new Error(`${path}: hooks[${index}].id must be a string`); + } + return hook.id; + }); +} + +// Contract levels are cumulative. A level can use hooks introduced earlier. +function hooksByLevel(track: Track): HooksByLevel { + const perFile = new Map(); + for (const file of readdirSync(track.contracts).filter(name => /^\d\d-.*\.json$/.test(name))) { + perFile.set(file.slice(0, 2), contractHookIds(join(track.contracts, file))); + } + const byLevel: HooksByLevel = new Map(); + for (const level of perFile.keys()) { + const ids = [...perFile.entries()] + .filter(([candidate]) => candidate <= level) + .flatMap(([, hookIds]) => hookIds); + byLevel.set(level, new Set(ids)); + } + return byLevel; +} + +function packId(reference: string): string { + return reference.slice(0, reference.lastIndexOf('@')); +} + +function ownedFragment( + fragment: CompiledOwnedTaskFragment, + owners: ReadonlySet, +): boolean { + return fragment.owners.some(owner => owners.has(owner)); +} + +function recipeScenarioScopes(track: Track, recipeFile: string): Map { + const recipeDir = join(track.dir, 'composition', 'recipes'); + const chain: CompiledRecipeRelease[] = []; + const seen = new Set(); + let currentFile: string | null = recipeFile; + let isolatesSelectedSources = false; + while (currentFile !== null) { + if (seen.has(currentFile)) throw new Error(`recipe base cycle at ${currentFile}`); + seen.add(currentFile); + const path = join(recipeDir, currentFile); + chain.push(compileRecipeFile(path, { trackRoot: track.dir })); + const source = readRecipeSource(path); + if (chain.length === 1) isolatesSelectedSources = source.isolatesSelectedSources; + currentFile = source.baseRecipe; + } + + const recipe = chain[0]; + if (recipe === undefined) throw new Error(`recipe chain is empty for ${recipeFile}`); + const packs = new Map(recipe.packs.map(pack => [pack.id, pack])); + const contracts = isolatesSelectedSources + ? recipe.recipe.task.contracts + : chain.flatMap(release => release.recipe.task.contracts); + const requirements = isolatesSelectedSources + ? recipe.recipe.task.requirements + : chain.flatMap(release => release.recipe.task.requirements); + const scopes = new Map(); + + const ownersFor = (check: CompiledRecipeRelease['checks'][number]): Set => { + const found = new Set([check.packId, ...(check.requiresFeatures ?? [])]); + const visit = (id: string): void => { + const pack = packs.get(id); + if (pack === undefined) return; + for (const reference of pack.requiresPacks) { + const dependency = packId(reference); + if (found.has(dependency)) continue; + found.add(dependency); + visit(dependency); + } + }; + [...found].forEach(visit); + return found; + }; + + for (const check of recipe.checks) { + const source = check.source.replace(/^scenarios\//, ''); + const scope = scopes.get(source) ?? { + features: new Map>(), + contractOwners: new Set(), + requirementOwners: new Set(), + contractText: '', + requirementText: '', + }; + const criteria = scope.features.get(check.featureId) ?? new Set(); + criteria.add(check.criterionId); + scope.features.set(check.featureId, criteria); + for (const owner of ownersFor(check)) { + scope.contractOwners.add(owner); + scope.requirementOwners.add(owner); + } + scopes.set(source, scope); + } + + for (const scope of scopes.values()) { + const selectedContracts = isolatesSelectedSources + ? contracts.filter(fragment => ownedFragment(fragment, scope.contractOwners)) + : contracts; + const selectedRequirements = isolatesSelectedSources + ? requirements.filter(fragment => ownedFragment(fragment, scope.requirementOwners)) + : requirements; + scope.contractText = selectedContracts.map(fragment => fragment.text).join('\n'); + scope.requirementText = selectedRequirements.map(fragment => fragment.text).join('\n'); + } + return scopes; +} + +function normalizeText(text: string): string { + return text.replace(/\*\*/g, '').replace(/—/g, '-').toLowerCase().replace(/\s+/g, ' ').trim(); +} + +function promptFor(track: Track, level: string): string | null { + const dir = join(track.dir, 'prompts'); + if (!existsSync(dir)) return null; + const file = readdirSync(dir).find(name => name.startsWith(`${level}-`) && name.endsWith('.md')); + return file === undefined ? null : normalizeText(readFileSync(join(dir, file), 'utf8')); +} + +function referencedActors(step: CompiledStep): string[] { + return [step.from, step.fromActor].filter((actor): actor is string => actor !== undefined); +} + +function referencedTestIds(step: CompiledStep): string[] { + return [step.testid, step.in?.testid].filter((id): id is string => id !== undefined); +} + +function main(args: readonly string[]): number { + const { values } = parseArgs({ args: [...args], options: { + track: { type: 'string' }, + recipe: { type: 'string' }, + }, strict: true, allowPositionals: false }); + const trackArg = values.track ?? null; + const recipeArg = values.recipe ?? null; + const availableTracks = listTracks(); + const trackNames = trackArg === null + ? (availableTracks.length > 0 ? availableTracks : [DEFAULT_TRACK]) + : [trackArg]; + if (recipeArg !== null && trackNames.length !== 1) { + throw new Error('--recipe requires one --track'); + } + + const knownActions = new Set(ACTION_REGISTRY.ids); + let problems = 0; + let unstatedWarnings = 0; + let staleStatementWarnings = 0; + const fail = (where: string, message: string): void => { + console.log(` ${where}: ${message}`); + problems += 1; + }; + + for (const name of trackNames) { + const track = loadTrack(name); + console.log(`# track: ${name}`); + const contracts = hooksByLevel(track); + const recipeScopes = recipeArg === null ? null : recipeScenarioScopes(track, recipeArg); + for (const file of readdirSync(track.scenarios).filter(candidate => candidate.endsWith('.json'))) { + const recipeScope = recipeScopes?.get(file); + if (recipeScopes !== null && recipeScope === undefined) continue; + const scenarioPath = join(track.scenarios, file); + let spec; + try { + spec = compileScenarioDefinition(readJson(scenarioPath), { source: scenarioPath }); + } catch (error: unknown) { + fail(file, error instanceof Error ? error.message : String(error)); + continue; + } + const level = String(spec.level).padStart(2, '0'); + const hooks = recipeScope === undefined ? (contracts.get(level) ?? null) : null; + + console.log(file); + const prompt = recipeScope === undefined + ? promptFor(track, level) + : normalizeText(recipeScope.requirementText); + for (const feature of spec.features) { + const selectedCriteria = recipeScope?.features.get(feature.id); + if (recipeScope !== undefined && selectedCriteria === undefined) continue; + const criteria = selectedCriteria === undefined + ? feature.criteria + : feature.criteria.filter(criterion => selectedCriteria.has(criterion.id)); + for (const criterion of criteria) { + if (criterion.statedBy !== undefined) { + if (recipeScope === undefined && prompt !== null + && !prompt.includes(normalizeText(criterion.statedBy))) { + staleStatementWarnings += 1; + console.log(` warn F${feature.id} ${criterion.id}: statedBy text is not in the level ${level} prompt`); + } + } else if (recipeScope === undefined && criterion.points > 0) { + unstatedWarnings += 1; + console.log(` warn F${feature.id} ${criterion.id}: carries ${criterion.points} point(s) with no statedBy - the requirement may be unstated`); + } + } + + const actors = new Set(feature.actors ?? []); + const steps = [...feature.setup, ...criteria.flatMap(criterion => criterion.steps)]; + const declared = (actor: string): boolean => actors.has(actor) + || [...actors].some(candidate => actor.startsWith(`${candidate}-`)); + for (const step of steps) { + const at = `F${feature.id} ${step.do}`; + if (!knownActions.has(step.do)) fail(at, `unknown step type "${step.do}"`); + if (step.actor !== undefined && actors.size > 0 && !declared(step.actor)) { + fail(at, `actor "${step.actor}" is not in the feature's actor list`); + } + for (const actor of referencedActors(step)) { + if (actors.size > 0 && !declared(actor)) { + fail(at, `actor "${actor}" is not in the feature's actor list`); + } + } + if (hooks !== null) { + for (const id of referencedTestIds(step)) { + if (!hooks.has(id)) fail(at, `testid "${id}" is not in the contract`); + } + } + if (recipeScope !== undefined) { + for (const id of referencedTestIds(step)) { + if (!recipeScope.contractText.includes(`\`${id}\``)) { + fail(at, `testid "${id}" is not in the selected recipe contracts`); + } + } + } + } + + const points = criteria.reduce((total, criterion) => total + criterion.points, 0); + if (recipeScope === undefined && feature.max !== undefined && points !== feature.max) { + fail(`F${feature.id}`, `criteria total ${points} but max says ${feature.max}`); + } + } + } + } + + const warnings = unstatedWarnings + staleStatementWarnings; + console.log(problems > 0 + ? `\n${problems} error(s); ${warnings} warning(s)` + : warnings > 0 + ? `\n0 errors; ${warnings} warning(s) (${unstatedWarnings} point-carrying criteria lack statedBy; ${staleStatementWarnings} statedBy references are outside level prompts)` + : '\n0 errors; 0 warnings'); + return problems > 0 ? 1 : 0; +} + +process.exitCode = main(process.argv.slice(2)); diff --git a/tools/stack-bench/commands/composition-cli.ts b/tools/stack-bench/commands/composition-cli.ts new file mode 100644 index 00000000000..820698e7ea0 --- /dev/null +++ b/tools/stack-bench/commands/composition-cli.ts @@ -0,0 +1,321 @@ +#!/usr/bin/env node + +import { existsSync, readdirSync, realpathSync } from 'node:fs'; +import { join, relative, resolve, sep } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { parseArgs } from 'node:util'; + +import { compilePackDefinition, compileRecipeFile, resolveTaskFragment } from '../src/composition/composition-compiler.js'; +import { compileScenarioDefinition } from '../src/composition/definition-compiler.js'; +import { canonicalDefinitionJson, readDefinitionJson } + from '../src/composition/definition-plan.js'; +import { buildRecipeRelease } from '../src/composition/recipe-release.js'; +import { composeSelectedRecipeTask, selectRecipeRelease } from '../src/composition/recipe-selection.js'; +import { TRACKS_DIR } from '../src/composition/tracks.js'; +import type { CompiledPackDefinition, CompiledRecipePlan } from '../src/composition/composition-compiler.js'; +import type { RecipeRelease } from '../src/composition/recipe-release.js'; +import type { RecipeSelectionOptions, SelectedRecipeRelease } from '../src/composition/recipe-selection.js'; + +export { selectRecipeRelease } from '../src/composition/recipe-selection.js'; + +interface TrackRootOptions { + trackRoot: string; +} + +interface PackIndexEntry { + pack: CompiledPackDefinition; + path: string; +} + +interface CalibrationValue { + id: string; + version: string; + recipe?: { id?: string; version?: string; contentSha256?: string }; +} + +type RecipeOptions = TrackRootOptions & RecipeSelectionOptions; +type RecipeTaskKind = 'requirements' | 'contracts'; + +function contained(root: string, path: string, label: string): string { + const absoluteRoot = realpathSync(resolve(root)); + const candidate = resolve(path); + const lexical = relative(absoluteRoot, candidate); + if (lexical === '..' || lexical.startsWith(`..${sep}`)) throw new Error(`${label} escapes ${absoluteRoot}`); + if (!existsSync(candidate)) throw new Error(`${label} does not exist: ${candidate}`); + const absolute = realpathSync(candidate); + const physical = relative(absoluteRoot, absolute); + if (physical === '..' || physical.startsWith(`..${sep}`)) throw new Error(`${label} escapes ${absoluteRoot}`); + return absolute; +} + +function packIndex(trackRoot: string): Map { + const directory = join(trackRoot, 'composition', 'packs'); + const byRef = new Map(); + for (const name of readdirSync(directory).filter(file => file.endsWith('.json')).sort()) { + const path = join(directory, name); + const pack = compilePackDefinition(readDefinitionJson(path, 'pack'), { + source: relative(trackRoot, path).replaceAll('\\', '/'), + }); + const ref = `${pack.id}@${pack.version}`; + if (byRef.has(ref)) throw new Error(`duplicate pack release ${ref}`); + byRef.set(ref, { pack, path: realpathSync(path) }); + } + for (const [ref, { pack }] of byRef) { + for (const dependency of [...pack.requiresPacks, ...pack.conflictsWith]) { + if (!byRef.has(dependency)) throw new Error(`${ref} references missing pack ${dependency}`); + } + } + return byRef; +} + +export function validatePackFile(path: string, options: Partial = {}) { + const { trackRoot } = options; + if (trackRoot === undefined) throw new Error('pack validation requires trackRoot'); + const root = realpathSync(resolve(trackRoot)); + const absolute = contained(join(root, 'composition'), path, 'pack path'); + const pack = compilePackDefinition(readDefinitionJson(absolute, 'pack'), { + source: relative(root, absolute).replaceAll('\\', '/'), + }); + const packs = packIndex(root); + const ownRef = `${pack.id}@${pack.version}`; + const indexed = packs.get(ownRef); + if (!indexed || indexed.path !== absolute) throw new Error(`${ownRef} is not the indexed source ${absolute}`); + for (const ref of [...pack.requiresPacks, ...pack.conflictsWith]) { + if (!packs.has(ref)) throw new Error(`${ownRef} references missing pack ${ref}`); + } + const sourceCache = new Map(); + for (const kind of ['requirements', 'contracts'] satisfies RecipeTaskKind[]) { + for (const fragment of pack.task[kind]) { + resolveTaskFragment(fragment, { trackRoot: root, + source: `${relative(root, absolute).replaceAll('\\', '/')}.task.${kind}.${fragment.id}`, + sourceCache }); + } + } + const state = new Map(); + const visit = (ref: string, chain: string[] = []): void => { + if (state.get(ref) === 'done') return; + if (state.get(ref) === 'visiting') throw new Error(`pack dependency cycle: ${[...chain, ref].join(' -> ')}`); + state.set(ref, 'visiting'); + const entry = packs.get(ref); + if (!entry) throw new Error(`missing pack release ${ref}`); + for (const dependency of entry.pack.requiresPacks) { + if (!packs.has(dependency)) throw new Error(`${ref} references missing pack ${dependency}`); + visit(dependency, [...chain, ref]); + } + state.set(ref, 'done'); + }; + visit(ownRef); + let criteria = 0; + for (const check of pack.checks) { + const scenarioPath = contained(root, join(root, check.source), `${pack.id}.${check.id}.source`); + const scenario = compileScenarioDefinition(readDefinitionJson(scenarioPath, 'scenario'), { + source: relative(root, scenarioPath).replaceAll('\\', '/'), + }); + const feature = scenario.features.find(candidate => candidate.id === check.feature); + if (!feature) throw new Error(`${pack.id}.${check.id} references missing feature ${check.feature}`); + criteria += feature.criteria.length; + } + return { id: pack.id, version: pack.version, state: pack.state, path: absolute, + checkGroups: pack.checks.length, criteria, requiresPacks: pack.requiresPacks }; +} + +export function validateRecipeFile(path: string, options: Partial = {}): { + plan: CompiledRecipePlan; + release: RecipeRelease; +} { + const { trackRoot } = options; + if (trackRoot === undefined) throw new Error('recipe validation requires trackRoot'); + const absolute = contained(join(trackRoot, 'composition'), path, 'recipe path'); + const plan = compileRecipeFile(absolute, { trackRoot }); + const release = buildRecipeRelease(absolute, { trackRoot }); + return { plan, release }; +} + +export function showRecipeFile(path: string, options: RecipeOptions): SelectedRecipeRelease & { + builderTask: ReturnType & { note: string }; +} { + const compiled = validateRecipeFile(path, options); + const selected = selectRecipeRelease(compiled.release, options); + const builderTask = composeSelectedRecipeTask(compiled.plan, selected.selection); + return { + ...selected, + builderTask: { + ...builderTask, + note: 'Pack selection defines the requested task; a check-only filter narrows measurement inside it.', + }, + }; +} + +const same = (left: unknown, right: unknown): boolean => + canonicalDefinitionJson(left) === canonicalDefinitionJson(right); + +function meaningView(release: RecipeRelease) { + return { + track: release.track, + task: release.task, + checks: release.checkCatalog.map(({ stableKey, packId, checkGroupId, role, source, + featureId, criterionId, description }) => ({ stableKey, packId, checkGroupId, role, + source, featureId, criterionId, description })), + }; +} + +function scoringView(release: RecipeRelease) { + return { scoring: release.scoring, + checks: release.checkCatalog.map(({ stableKey, points }) => ({ stableKey, points })) }; +} + +function metadataView(release: RecipeRelease) { + return { id: release.id, version: release.version, state: release.state, title: release.title, + sequence: release.sequence, sourceManifestSha256: release.sourceManifestSha256 }; +} + +function matchingCalibrations(trackRoot: string, release: RecipeRelease): Array<{ + path: string; + value: CalibrationValue; +}> { + const directory = join(trackRoot, 'composition', 'calibrations'); + if (!existsSync(directory)) return []; + return readdirSync(directory).filter(name => name.endsWith('.json')).sort() + .map(name => ({ path: join(directory, name), + value: readDefinitionJson(join(directory, name), 'calibration') })) + .filter(({ value }) => value.recipe?.id === release.id + && value.recipe?.version === release.version + && value.recipe?.contentSha256 === release.contentSha256); +} + +export function diffRecipeFiles(fromPath: string, toPath: string, options: Partial = {}) { + const { trackRoot } = options; + if (trackRoot === undefined) throw new Error('recipe diff requires trackRoot'); + const from = validateRecipeFile(fromPath, { trackRoot }).release; + const to = validateRecipeFile(toPath, { trackRoot }).release; + const categories = { + meaning: !same(meaningView(from), meaningView(to)), + scoring: !same(scoringView(from), scoringView(to)), + fixtures: !same(from.components.fixture, to.components.fixture), + execution: from.executionSha256 !== to.executionSha256, + metadata: !same(metadataView(from), metadataView(to)), + }; + const recipeBindingChanged = from.id !== to.id || from.version !== to.version + || from.meaningSha256 !== to.meaningSha256 || from.executionSha256 !== to.executionSha256 + || from.contentSha256 !== to.contentSha256; + const calibrations = matchingCalibrations(trackRoot, from).map(({ path, value }) => { + const invalidated = []; + const stateChanged = from.state !== to.state; + if (recipeBindingChanged) invalidated.push('recipe binding'); + if (stateChanged) invalidated.push('recipe qualification state'); + if (categories.fixtures) invalidated.push('fixture binding'); + if (categories.scoring) invalidated.push('zero-point control policy'); + if (categories.meaning || categories.scoring || categories.execution || categories.fixtures) { + invalidated.push('reference repetitions', 'mutation repetitions'); + } + if (categories.meaning || categories.scoring || categories.fixtures) invalidated.push('null repetitions'); + if (recipeBindingChanged || stateChanged) invalidated.push('promotion decision'); + return { id: value.id, version: value.version, + path: relative(trackRoot, path).replaceAll('\\', '/'), invalidated: [...new Set(invalidated)] }; + }); + const fragmentDiff = (kind: RecipeTaskKind) => { + const before = new Map(from.task[kind].map(fragment => [fragment.id, fragment])); + const after = new Map(to.task[kind].map(fragment => [fragment.id, fragment])); + return { + added: [...after.keys()].filter(key => !before.has(key)).sort(), + removed: [...before.keys()].filter(key => !after.has(key)).sort(), + changed: [...after.keys()].filter(key => before.has(key) + && !same(before.get(key), after.get(key))).sort(), + }; + }; + return { + from: { id: from.id, version: from.version, state: from.state, meaningSha256: from.meaningSha256, + executionSha256: from.executionSha256, contentSha256: from.contentSha256 }, + to: { id: to.id, version: to.version, state: to.state, meaningSha256: to.meaningSha256, + executionSha256: to.executionSha256, contentSha256: to.contentSha256 }, + categories, + taskFragments: { + requirements: fragmentDiff('requirements'), + contracts: fragmentDiff('contracts'), + composedTaskChanged: from.task.composedSha256 !== to.task.composedSha256, + }, + calibrations, + }; +} + +type CliSubject = 'pack' | 'recipe'; +type CliCommand = 'validate' | 'show' | 'diff'; + +interface ParsedArgs extends RecipeSelectionOptions { + json: boolean; + positional: string[]; + packIds: string[]; + checkKeys: string[]; + track?: string; + trackRoot?: string; +} + +interface CliArgs extends ParsedArgs { + subject: CliSubject; + command: CliCommand; + paths: string[]; + trackRoot: string; +} + +function parse(argv: string[]): CliArgs { + const { positionals, values } = parseArgs({ args: argv.slice(2), allowPositionals: true, + options: { track: { type: 'string' }, 'track-root': { type: 'string' }, + pack: { type: 'string', multiple: true }, check: { type: 'string', multiple: true }, + json: { type: 'boolean' } } }); + const args: ParsedArgs = { json: values.json ?? false, positional: positionals, + packIds: (values.pack ?? []).flatMap(value => value.split(',').filter(Boolean)), + checkKeys: (values.check ?? []).flatMap(value => value.split(',').filter(Boolean)), + track: values.track, + trackRoot: values['track-root'] === undefined ? undefined : resolve(values['track-root']) }; + const [subject, command, ...paths] = args.positional; + if (subject !== 'pack' && subject !== 'recipe') { + throw new Error('usage: npm run pack -- validate --track | npm run recipe -- validate|show --track | npm run recipe -- diff --track '); + } + if (command !== 'validate' && command !== 'show' && command !== 'diff') { + throw new Error('usage: npm run pack -- validate --track | npm run recipe -- validate|show --track | npm run recipe -- diff --track '); + } + if (subject === 'pack' && command !== 'validate') throw new Error(`pack ${command} is not supported`); + if ((command === 'diff' ? paths.length !== 2 : paths.length !== 1)) throw new Error(`${subject} ${command} received the wrong number of paths`); + if (!args.trackRoot && !args.track) throw new Error('--track or --track-root is required'); + if ((args.packIds.length || args.checkKeys.length) && !(subject === 'recipe' && command === 'show')) { + throw new Error('--pack and --check are allowed only with recipe show'); + } + const trackRoot = args.trackRoot ?? join(TRACKS_DIR, args.track ?? ''); + return { ...args, subject, command, paths, trackRoot }; +} + +function main() { + const args = parse(process.argv); + const firstPath = args.paths[0]; + if (firstPath === undefined) throw new Error('command requires a source path'); + let result: object; + if (args.subject === 'pack') result = validatePackFile(firstPath, args); + else if (args.command === 'diff') { + const secondPath = args.paths[1]; + if (secondPath === undefined) throw new Error('recipe diff requires two source paths'); + result = diffRecipeFiles(firstPath, secondPath, args); + } else if (args.command === 'show') result = showRecipeFile(firstPath, args); + else { + const compiled = validateRecipeFile(firstPath, args); + result = { + id: compiled.release.id, version: compiled.release.version, state: compiled.release.state, + packs: compiled.release.components.packs.length, checks: compiled.release.checkCatalog.length, + points: compiled.release.checkCatalog.reduce((total, check) => total + check.points, 0), + meaningSha256: compiled.release.meaningSha256, + executionSha256: compiled.release.executionSha256, + contentSha256: compiled.release.contentSha256, + }; + } + if (args.json || args.command === 'show' || args.command === 'diff') console.log(JSON.stringify(result, null, 2)); + else if ('id' in result && 'version' in result && 'state' in result) { + console.log(`${String(result.id)}@${String(result.version)} ${String(result.state)}: valid`); + } else throw new Error('validation result has no release identity'); +} + +if (process.argv[1] && pathToFileURL(process.argv[1]).href === import.meta.url) { + try { main(); } + catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 2; + } +} diff --git a/tools/stack-bench/commands/container-smoke.ts b/tools/stack-bench/commands/container-smoke.ts new file mode 100644 index 00000000000..9ea5f297de3 --- /dev/null +++ b/tools/stack-bench/commands/container-smoke.ts @@ -0,0 +1,200 @@ +#!/usr/bin/env node +// Use only ephemeral resources owned by this smoke run. + +import { spawn, execFileSync } from 'node:child_process'; +import type { ChildProcess } from 'node:child_process'; +import { cpSync, existsSync, mkdirSync, mkdtempSync, rmSync } from 'node:fs'; +import { createServer } from 'node:net'; +import type { AddressInfo } from 'node:net'; +import { basename, join, resolve } from 'node:path'; +import { tmpdir } from 'node:os'; +import { setTimeout as delay } from 'node:timers/promises'; + +import { killTree, pidsOnPort, processIdentity } from '../src/runtime/platform.js'; +import { createBackendLease, readBackendLease, writeBackendLease } from '../src/runtime/backend-lease.js'; +import { fetchStatus } from '../src/runtime/readiness.js'; +import { DEFAULT_BUILD_IMAGE } from '../src/composition/product-config.js'; +import { containerReachableSpacetimeUri } from '../src/runtime/spacetime-target.js'; +import { CODING_CONTAINER_APP_ROOT, CODING_CONTAINER_SPACETIME_CLI, + codingContainerAgentExecOptions } from '../src/runtime/coding-container-policy.js'; +import { ARTIFACT_FILE } from '../src/evidence/artifacts.js'; + +import { STACK_BENCH_ROOT as ROOT, compiledEntrypoint } from '../src/package-root.js'; +const REPO = resolve(ROOT, '..', '..'); +const IMAGE = process.env.STACK_BENCH_IMAGE ?? DEFAULT_BUILD_IMAGE; +const CLI = process.env.SPACETIME_BIN ?? join(REPO, 'target', 'release', + process.platform === 'win32' ? 'spacetimedb-cli.exe' : 'spacetimedb-cli'); +const RUN_BUILD = compiledEntrypoint('container', 'run-build.js'); +const FIXTURE = join(ROOT, 'tests', 'fixtures', 'spacetime-module'); + +interface PreparedContainerIdentity { + containerName: string; + identity: string; + networkMode: string | null; +} + +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +function parsePreparedContainerIdentity(text: string): PreparedContainerIdentity { + const value: unknown = JSON.parse(text.trim().split(/\r?\n/).pop() ?? ''); + if (!isRecord(value)) throw new Error('prepared container identity is invalid'); + const record = value; + if (typeof record.containerName !== 'string' || typeof record.identity !== 'string' + || (record.networkMode !== null && typeof record.networkMode !== 'string')) { + throw new Error('prepared container identity is invalid'); + } + return { containerName: record.containerName, identity: record.identity, networkMode: record.networkMode }; +} + +async function freePort() { + const server = createServer(); + await new Promise((ok, fail) => server.listen({ port: 0, host: '127.0.0.1' }, ok).once('error', fail)); + const address = server.address(); + if (!address || typeof address === 'string') throw new Error('could not allocate a TCP port'); + const port: AddressInfo['port'] = address.port; + await new Promise(ok => server.close(ok)); + return port; +} + +async function waitFor(check: () => boolean | Promise, timeoutMs: number, description: string): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (await check()) return; + await delay(250); + } + throw new Error(`timed out waiting for ${description}`); +} + +async function main() { + if (!existsSync(CLI)) throw new Error(`local SpacetimeDB CLI is missing: ${CLI}`); + execFileSync('docker', ['image', 'inspect', IMAGE], { stdio: 'pipe' }); + + const root = mkdtempSync(join(tmpdir(), 'stack-bench-container-smoke-')); + const app = join(root, 'app'); + const dataDir = join(root, 'spacetime-data'); + const port = await freePort(); + const uri = `http://127.0.0.1:${port}`; + const module = `stackbench-container-smoke-${process.pid}`; + const containerName = `stack-bench-${basename(root)}`; + const leasePath = join(root, ARTIFACT_FILE.backendLease); + let host: ChildProcess | null = null; + let dev: ChildProcess | null = null; + let output = ''; + + try { + mkdirSync(app, { recursive: true }); + host = spawn(CLI, ['start', '--listen-addr', `127.0.0.1:${port}`, '--data-dir', dataDir], + { stdio: 'ignore', windowsHide: true }); + await waitFor(async () => { + const status = await fetchStatus(`${uri}/v1/ping`, { timeoutMs: 5000 }); + return status !== null && status >= 200 && status < 300; + }, 120_000, `dedicated SpacetimeDB host on :${port}`); + + const lease = createBackendLease({ runId: basename(root), backend: 'spacetime', + track: 'container-smoke', runIndex: 0, serverUri: uri, module, dataDir }); + lease.state = 'active'; + lease.resources.launchedProcess = host.pid ? processIdentity(host.pid) : null; + lease.resources.listenerProcesses = pidsOnPort(port).map(pid => processIdentity(pid)) + .filter((identity): identity is NonNullable => identity !== null); + writeBackendLease(leasePath, lease); + + const prepared = execFileSync(process.execPath, + [RUN_BUILD, '--app', app, '--backend', 'spacetime', '--image', IMAGE, '--prepare-only'], + { encoding: 'utf8', stdio: 'pipe', maxBuffer: 16 * 1024 * 1024, + env: { ...process.env, STACK_BENCH_LEASE: leasePath, + STACK_BENCH_LEASE_TOKEN: lease.ownershipToken, STACK_BENCH_STDB_URI: uri } }); + const identity = parsePreparedContainerIdentity(prepared); + if (identity.containerName !== containerName) { + throw new Error(`prepared unexpected container ${identity.containerName}`); + } + const leasedContainer = readBackendLease(leasePath, + { token: lease.ownershipToken, backend: 'spacetime', active: true }).resources.buildContainer; + if (!leasedContainer || identity.identity.split(' ')[0] !== leasedContainer.id) { + throw new Error('prepared container identity was not recorded in the backend lease'); + } + if (!leasedContainer.image || !/^sha256:[0-9a-f]{64}$/.test(leasedContainer.image)) { + throw new Error(`prepared container did not record an immutable image id: ${leasedContainer.image}`); + } + + cpSync(FIXTURE, join(app, 'spacetimedb'), { recursive: true }); + const agentExec = ['exec', ...codingContainerAgentExecOptions()]; + const cliAccess = execFileSync('docker', [...agentExec, containerName, 'sh', '-c', + `stat -c '%a %U %G' ${CODING_CONTAINER_SPACETIME_CLI}; test -x ${CODING_CONTAINER_SPACETIME_CLI}`], + { encoding: 'utf8', stdio: 'pipe' }); + if (!/^755 root root/m.test(cliAccess)) throw new Error(`unexpected CLI access: ${cliAccess.trim()}`); + execFileSync('docker', [...agentExec, containerName, 'sh', '-c', + `umask 000; cd ${CODING_CONTAINER_APP_ROOT}/spacetimedb && npm install --no-audit --no-fund`], + { stdio: 'pipe' }); + + const startedDev = spawn('docker', [...agentExec, '-i', containerName, 'sh', '-c', + `umask 000; cd ${CODING_CONTAINER_APP_ROOT}/spacetimedb && ${CODING_CONTAINER_SPACETIME_CLI} dev ${module} ` + + `--project-path ${CODING_CONTAINER_APP_ROOT}/spacetimedb --module-path . ` + + '--server-only --skip-generate ' + + `-s ${containerReachableSpacetimeUri({ resources: { serverUri: uri, + buildContainer: lease.resources.buildContainer } }, identity.networkMode)} -y`], + { stdio: ['ignore', 'pipe', 'pipe'], windowsHide: true }); + dev = startedDev; + const collect = (chunk: Buffer): void => { output = (output + chunk.toString()).slice(-128 * 1024); }; + startedDev.stdout?.on('data', collect); + startedDev.stderr?.on('data', collect); + + await waitFor(() => { + if (/Published successfully!/.test(output)) return true; + if (startedDev.exitCode !== null) throw new Error(`spacetime dev exited ${startedDev.exitCode}:\n${output}`); + return false; + }, 240_000, 'containerized module publish'); + + const sql = execFileSync(CLI, ['sql', module, 'SELECT * FROM smoke_item', '-s', uri], + { encoding: 'utf8', stdio: 'pipe' }); + if (!/\bid\s*\|\s*value\b/.test(sql)) throw new Error(`SQL verification failed:\n${sql}`); + if (startedDev.exitCode !== null) throw new Error('spacetime dev did not remain alive as a watcher'); + + // Publishing and log streaming must retain one authenticated identity. A + // prior dev bug published with a token stored only in a Config clone, then + // directly logged in again for logs and received an authorization error. + await delay(2_000); + const logStreamingAuthorized = !/Log streaming error:.*not authorized/s.test(output); + console.log(JSON.stringify({ ok: true, image: IMAGE, container: identity.identity, + host: { uri, listenerPids: pidsOnPort(port) }, published: true, sqlVerified: true, + watcherAlive: true, leasedContainer: true, immutableImagePinned: true, + logStreamingAuthorized }, null, 2)); + if (!logStreamingAuthorized) { + throw new Error('`spacetime dev` published successfully but its log stream was not authorized'); + } + // The grader resets by republishing the same named database from this exact + // leased container. Prove that `-y` retained a reusable local identity, + // rather than merely proving that the first anonymous-looking publish ran. + execFileSync('docker', [...agentExec, containerName, 'sh', '-c', + 'for process in /proc/[0-9]*; do ' + + 'test "$(readlink "$process/exe" 2>/dev/null)" = /deps/.spacetimedb-cli ' + + '&& kill -TERM "${process##*/}" || true; done'], { stdio: 'pipe' }); + await waitFor(() => startedDev.exitCode !== null, 15_000, 'spacetime dev to stop before reset publish'); + const targetUri = containerReachableSpacetimeUri({ resources: { serverUri: uri, + buildContainer: lease.resources.buildContainer } }, identity.networkMode); + execFileSync('docker', [...agentExec, containerName, 'sh', '-c', + `umask 000; cd ${CODING_CONTAINER_APP_ROOT}/spacetimedb && ${CODING_CONTAINER_SPACETIME_CLI} publish ${module} ` + + `--module-path . -s ${targetUri} --delete-data -y`], + { stdio: 'pipe', timeout: 240_000 }); + const afterReset = execFileSync(CLI, ['sql', module, 'SELECT * FROM smoke_item', '-s', uri], + { encoding: 'utf8', stdio: 'pipe' }); + if (!/\bid\s*\|\s*value\b/.test(afterReset)) { + throw new Error(`SQL verification after reset publish failed:\n${afterReset}`); + } + console.log(JSON.stringify({ resetRepublished: true, resetSqlVerified: true })); + } finally { + if (dev && dev.exitCode === null) dev.kill('SIGTERM'); + try { execFileSync('docker', ['rm', '-f', containerName], { stdio: 'ignore' }); } catch { /* absent */ } + // The port was proven unused before this script started the host. Kill only + // listeners on that exact ephemeral port, then the wrapper if it remains. + for (const pid of pidsOnPort(port)) killTree(pid); + if (host && host.exitCode === null) killTree(host.pid); + rmSync(root, { recursive: true, force: true }); + } +} + +main().catch(error => { + console.error(error.stack ?? error.message); + process.exitCode = 1; +}); diff --git a/tools/stack-bench/commands/definition-snapshots.ts b/tools/stack-bench/commands/definition-snapshots.ts new file mode 100644 index 00000000000..9f227e19d30 --- /dev/null +++ b/tools/stack-bench/commands/definition-snapshots.ts @@ -0,0 +1,80 @@ +import { mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { pathToFileURL } from 'node:url'; + +import { compileScenarioDefinition } from '../src/composition/definition-compiler.js'; +import { canonicalDefinitionJson, compileTrackPlan } from '../src/composition/definition-plan.js'; +import { listTracks } from '../src/composition/tracks.js'; +import { STACK_BENCH_ROOT } from '../src/package-root.js'; + +const SNAPSHOT_DIR = join(STACK_BENCH_ROOT, 'tests', 'snapshots', 'definitions'); +const ALL_ACTIONS = join(STACK_BENCH_ROOT, 'tests', 'fixtures', 'definitions', 'all-actions.json'); + +function atomicWrite(path: string, contents: string): void { + mkdirSync(dirname(path), { recursive: true }); + const temporary = `${path}.tmp-${process.pid}`; + writeFileSync(temporary, contents); + renameSync(temporary, path); +} + +interface DefinitionSnapshot { + name: string; + value: unknown; +} + +export function currentDefinitionSnapshots(): DefinitionSnapshot[] { + const entries: DefinitionSnapshot[] = listTracks({ includeInternal: true }).map(name => ({ + name: `${name}.snapshot.json`, + value: compileTrackPlan(name), + })); + entries.push({ + name: 'all-actions.snapshot.json', + value: compileScenarioDefinition(JSON.parse(readFileSync(ALL_ACTIONS, 'utf8')), { + source: ALL_ACTIONS, + }), + }); + return entries.sort((a, b) => a.name.localeCompare(b.name)); +} + +export interface DefinitionSnapshotResult { + checked: number; + changed: string[]; +} + +export function checkDefinitionSnapshots( + { update = false }: { update?: boolean } = {}, +): DefinitionSnapshotResult { + const entries = currentDefinitionSnapshots(); + const changed: string[] = []; + for (const entry of entries) { + const path = join(SNAPSHOT_DIR, entry.name); + const actual = canonicalDefinitionJson(entry.value); + let expected: string | null = null; + try { + expected = readFileSync(path, 'utf8').replaceAll('\r\n', '\n'); + } catch (error: unknown) { + if (!(error instanceof Error && 'code' in error && error.code === 'ENOENT')) throw error; + } + if (expected === actual) continue; + changed.push(entry.name); + if (update) atomicWrite(path, actual); + } + if (changed.length > 0 && !update) { + throw new Error( + `definition snapshot drift: ${changed.join(', ')}; inspect the semantic change, then run npm run check:definition-snapshots -- --update`, + ); + } + return { checked: entries.length, changed }; +} + +function main(): void { + const args = new Set(process.argv.slice(2)); + for (const arg of args) { + if (arg !== '--update') throw new Error(`unknown argument ${arg}`); + } + const result = checkDefinitionSnapshots({ update: args.has('--update') }); + console.log(`${result.checked} definition snapshots checked${ + result.changed.length > 0 ? `; ${result.changed.length} updated` : '; no drift'}`); +} + +if (process.argv[1] && pathToFileURL(process.argv[1]).href === import.meta.url) main(); diff --git a/tools/stack-bench/commands/fault-injection.ts b/tools/stack-bench/commands/fault-injection.ts new file mode 100644 index 00000000000..fc8f324d6be --- /dev/null +++ b/tools/stack-bench/commands/fault-injection.ts @@ -0,0 +1,250 @@ +#!/usr/bin/env node +// Fault injection may remove only resources owned by its lease. + +import assert from 'node:assert/strict'; +import { execFileSync, spawn } from 'node:child_process'; +import type { ChildProcess } from 'node:child_process'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { createServer } from 'node:http'; +import type { Server } from 'node:http'; +import type { AddressInfo } from 'node:net'; +import { basename, join, resolve } from 'node:path'; +import { tmpdir } from 'node:os'; +import { setTimeout as delay } from 'node:timers/promises'; + +import { createBackendLease, writeBackendLease } from '../src/runtime/backend-lease.js'; +import { killTree, pidsOnPort } from '../src/runtime/platform.js'; +import { ARTIFACT_FILE, readArtifact, readArtifactPayload } from '../src/evidence/artifacts.js'; +import { DEFAULT_BUILD_IMAGE } from '../src/composition/product-config.js'; + +import { STACK_BENCH_ROOT as ROOT, compiledEntrypoint } from '../src/package-root.js'; +const REPO = resolve(ROOT, '..', '..'); +const IMAGE = process.env.STACK_BENCH_IMAGE ?? DEFAULT_BUILD_IMAGE; +const CLI = process.env.SPACETIME_BIN ?? join(REPO, 'target', 'release', + process.platform === 'win32' ? 'spacetimedb-cli.exe' : 'spacetimedb-cli'); +const RUN_BUILD = compiledEntrypoint('container', 'run-build.js'); +interface ContainerIdentity { id: string; running: boolean; } +interface ExitResult { code: number | null; signal: NodeJS.Signals | null; } +interface FaultLeaseResources { + listenerProcesses: Array<{ pid: number; startMarker: string }>; + buildContainer: { id: string; image: string; running: boolean; removedAt?: string }; + locks: { releasedAt?: string }[]; +} +interface FaultLeaseEvidence { + runId: string; + state: string; + stoppedAt?: string; + releasedAt?: string; + resources: FaultLeaseResources; +} + +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +async function freePort() { + const server = createServer((_request, response) => response.end('foreign')); + await new Promise((ok, fail) => server.listen({ port: 0, host: '127.0.0.1' }, ok).once('error', fail)); + const address = server.address(); + if (!address || typeof address === 'string') throw new Error('could not allocate a TCP port'); + const port: AddressInfo['port'] = address.port; + await new Promise(ok => server.close(ok)); + return port; +} + +function inspectContainer(target: string): ContainerIdentity | null { + try { + const output = execFileSync('docker', ['inspect', '--format', + '{{.Id}} {{.State.Running}}', target], { encoding: 'utf8', stdio: 'pipe' }).trim(); + const [id, running] = output.split(/\s+/, 2); + if (!id) return null; + return { id, running: running === 'true' }; + } catch { return null; } +} + +function startContainer(name: string): ContainerIdentity { + const id = execFileSync('docker', ['run', '-d', '--init', '--name', name, + IMAGE, 'sleep', 'infinity'], { encoding: 'utf8', stdio: 'pipe' }).trim(); + assert.ok(id, `Docker did not return an id for ${name}`); + const container = inspectContainer(id); + if (!container) throw new Error(`Docker did not return a running container for ${name}`); + return container; +} + +function removeExactContainer(identity: ContainerIdentity | { id: string } | null): void { + if (!identity) return; + const current = inspectContainer(identity.id); + if (!current || current.id !== identity.id) return; + execFileSync('docker', ['rm', '-f', identity.id], { stdio: 'ignore' }); +} + +async function waitForExit(child: ChildProcess, timeoutMs: number): Promise { + return new Promise((resolveExit, reject) => { + const timeout = setTimeout(() => reject(new Error( + `benchmark runner did not exit after injected failure within ${timeoutMs}ms`)), timeoutMs); + child.once('exit', (code: number | null, signal: NodeJS.Signals | null) => { + clearTimeout(timeout); + resolveExit({ code, signal }); + }); + }); +} + +async function assertRefusesUnleasedCollision() { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-container-collision-')); + const app = join(root, 'app'); + const name = `stack-bench-${basename(root)}`; + const leasePath = join(root, ARTIFACT_FILE.backendLease); + let foreign = null; + try { + mkdirSync(app, { recursive: true }); + foreign = startContainer(name); + const lease = createBackendLease({ runId: `collision-${process.pid}`, backend: 'spacetime', + track: 'fault-injection', runIndex: 0, serverUri: 'http://127.0.0.1:1', + module: `collision-${process.pid}`, dataDir: join(root, 'data') }); + lease.state = 'active'; + writeBackendLease(leasePath, lease); + + let refused = false; + try { + execFileSync(process.execPath, + [RUN_BUILD, '--app', app, '--backend', 'spacetime', '--prepare-only'], + { stdio: 'pipe', env: { ...process.env, STACK_BENCH_LEASE: leasePath, + STACK_BENCH_LEASE_TOKEN: lease.ownershipToken } }); + } catch (error: unknown) { + const childError = error instanceof Error && isRecord(error) ? error : null; + refused = childError?.status === 3 + && /refusing to adopt existing unleased container/.test(String(childError?.stderr)); + } + assert.equal(refused, true, 'launcher did not explicitly refuse an unleased same-name container'); + assert.deepEqual(inspectContainer(foreign.id), foreign, + 'collision refusal changed or stopped the foreign container'); + } finally { + removeExactContainer(foreign); + rmSync(root, { recursive: true, force: true }); + } +} + +async function main() { + assert.ok(existsSync(CLI), `local SpacetimeDB CLI is missing: ${CLI}`); + execFileSync('docker', ['image', 'inspect', IMAGE], { stdio: 'pipe' }); + await assertRefusesUnleasedCollision(); + + const root = mkdtempSync(join(tmpdir(), 'stack-bench-fault-')); + const app = join(root, 'app'); + const out = join(root, 'out'); + const markerPath = join(app, '.fault-ready.json'); + const port = await freePort(); + const uri = `http://127.0.0.1:${port}`; + const foreignName = `stack-bench-foreign-${process.pid}-${Date.now()}`; + let foreignContainer: ContainerIdentity | null = null; + let foreignServer: Server | null = null; + let bench: ChildProcess | null = null; + let marker: { lease: { runId: string; state: string; resources: FaultLeaseResources }; leasePath: string; phase: string } | null = null; + let output = ''; + + try { + mkdirSync(app, { recursive: true }); + mkdirSync(out, { recursive: true }); + foreignContainer = startContainer(foreignName); + const startedForeignServer = createServer((_request, response) => response.end('foreign')); + foreignServer = startedForeignServer; + await new Promise((ok, fail) => startedForeignServer.listen({ port: 0, host: '127.0.0.1' }, ok).once('error', fail)); + const foreignAddress = startedForeignServer.address(); + if (!foreignAddress || typeof foreignAddress === 'string') throw new Error('could not allocate foreign TCP port'); + const foreignUri = `http://127.0.0.1:${foreignAddress.port}`; + + bench = spawn(process.execPath, + [compiledEntrypoint('commands', 'bench.js'), '--backend', 'spacetime', '--track', 'loop', + '--levels', '1', '--agent-adapter', 'fault-injection', '--app', app, '--out', out, + '--url', `file:///${app.replace(/\\/g, '/')}/index.html`], + { env: { ...process.env, STACK_BENCH_STDB_URI: uri, STACK_BENCH_IMAGE: IMAGE, + SPACETIME_BIN: CLI }, + stdio: ['ignore', 'pipe', 'pipe'], windowsHide: true }); + const collect = (chunk: Buffer): void => { output = (output + chunk.toString()).slice(-256 * 1024); }; + bench.stdout?.on('data', collect); + bench.stderr?.on('data', collect); + const exited = await waitForExit(bench, 300_000); + assert.notEqual(exited.code, 0, 'injected coding-agent failure unexpectedly exited zero'); + assert.ok(existsSync(markerPath), `fault marker was not written before failure:\n${output}`); + const markerValue: unknown = JSON.parse(readFileSync(markerPath, 'utf8')); + if (!isRecord(markerValue) || !isRecord(markerValue.lease) || !isRecord(markerValue.lease.resources) + || typeof markerValue.leasePath !== 'string' || typeof markerValue.phase !== 'string' + || typeof markerValue.lease.runId !== 'string' || typeof markerValue.lease.state !== 'string') { + throw new Error('fault marker is invalid'); + } + const markerResources = markerValue.lease.resources; + if (!Array.isArray(markerResources.listenerProcesses) || !isRecord(markerResources.buildContainer) + || typeof markerResources.buildContainer.id !== 'string') throw new Error('fault marker resources are invalid'); + marker = { phase: markerValue.phase, leasePath: markerValue.leasePath, + lease: { runId: markerValue.lease.runId, state: markerValue.lease.state, + resources: { listenerProcesses: markerResources.listenerProcesses.filter((item): item is { + pid: number; startMarker: string } => isRecord(item) && typeof item.pid === 'number' + && typeof item.startMarker === 'string'), buildContainer: { + id: markerResources.buildContainer.id, image: String(markerResources.buildContainer.image ?? ''), + running: markerResources.buildContainer.running === true }, locks: [] } } }; + assert.equal(marker.phase, 'restart-stopped', + 'fault was not injected inside the backend restart window'); + assert.equal(marker.lease.state, 'restarting'); + assert.match(marker.lease.resources.buildContainer.image, /^sha256:[0-9a-f]{64}$/, + 'build container lease did not record an immutable image id'); + + const evidencePath = join(out, ARTIFACT_FILE.backendLease); + assert.ok(existsSync(evidencePath), `teardown did not preserve lease evidence:\n${output}`); + const evidence = readArtifactPayload(evidencePath, { expectedKind: 'backend_lease_evidence' }); + const preflight = readArtifact(join(out, ARTIFACT_FILE.preflight), + { expectedKind: 'preflight' }); + assert.equal(preflight.payload.ok, true, 'paid-run preflight did not pass'); + assert.equal(preflight.attempt.parentId, marker.lease.runId, + 'preflight evidence is not attached to the run it admitted'); + assert.equal(evidence.runId, marker.lease.runId); + assert.equal(evidence.state, 'released', 'benchmark lease did not reach its terminal state'); + assert.ok(evidence.stoppedAt, 'benchmark-owned SpacetimeDB host has no stop evidence'); + assert.ok(evidence.releasedAt, 'benchmark lease has no release evidence'); + assert.deepEqual(evidence.resources.listenerProcesses, []); + assert.equal(evidence.resources.buildContainer.running, false, + 'benchmark-owned build container was not marked removed'); + assert.ok(evidence.resources.buildContainer.removedAt); + assert.ok(evidence.resources.locks.every(lock => lock.releasedAt), + 'one or more resource locks were not released'); + assert.equal(inspectContainer(marker.lease.resources.buildContainer.id), null, + 'benchmark-owned build container survived fatal cleanup'); + assert.equal(pidsOnPort(port).length, 0, 'benchmark-owned listener survived fatal cleanup'); + assert.equal(existsSync(marker.leasePath), false, 'private runtime lease was not removed'); + + assert.equal((await fetch(foreignUri)).status, 200, + 'foreign listener was disturbed by benchmark cleanup'); + assert.deepEqual(inspectContainer(foreignContainer.id), foreignContainer, + 'foreign container was changed or removed by benchmark cleanup'); + + console.log(JSON.stringify({ ok: true, injectedAt: 'restart-stopped-before-replacement', + benchmarkHostStopped: true, benchmarkContainerRemoved: true, locksReleased: true, + privateLeaseRemoved: true, foreignListenerSurvived: true, + foreignContainerSurvived: true, unleasedCollisionRefused: true, + immutableImagePinned: true }, null, 2)); + } finally { + if (bench?.exitCode === null) { + killTree(bench.pid); + await delay(500); + } + if (marker?.lease?.resources?.buildContainer) { + removeExactContainer(marker.lease.resources.buildContainer); + } + for (const identity of marker?.lease?.resources?.listenerProcesses ?? []) { + if (pidsOnPort(port).includes(String(identity.pid))) killTree(identity.pid); + } + if (foreignServer) { + const server = foreignServer; + // The verification fetch uses a keep-alive connection. Waiting on + // close() alone can hold CI open until Undici retires that socket. + server.closeAllConnections(); + await new Promise((ok, fail) => server.close(error => error ? fail(error) : ok())); + } + removeExactContainer(foreignContainer); + rmSync(root, { recursive: true, force: true }); + } +} + +main().catch(error => { + console.error(error.stack ?? error.message); + process.exitCode = 1; +}); diff --git a/tools/stack-bench/commands/leak-audit.ts b/tools/stack-bench/commands/leak-audit.ts new file mode 100644 index 00000000000..11e156302a2 --- /dev/null +++ b/tools/stack-bench/commands/leak-audit.ts @@ -0,0 +1,287 @@ +#!/usr/bin/env node +// Use the recorded cwd as the app boundary; transcript folder names are not authority. + +import { readdirSync, readFileSync, existsSync, statSync } from 'node:fs'; +import { join, resolve } from 'node:path'; +import { homedir } from 'node:os'; +import { pathToFileURL } from 'node:url'; +import { parseArgs } from 'node:util'; +import { CODING_CONTAINER_APP_ROOT } from '../src/runtime/coding-container-policy.js'; + +// --dir is exclusive. --app resolves its matching CLI transcript directory. +function transcriptsFor(appDir: string): string[] { + const base = join(homedir(), '.claude', 'projects'); + if (!existsSync(base)) return []; + const want = resolve(appDir).replace(/[\\/:]/g, '-').toLowerCase(); + return readdirSync(base) + .filter(d => d.toLowerCase() === want || d.toLowerCase() === want.replace(/^-+/, '')) + .map(d => join(base, d)); +} + +const norm = (value: unknown): string => String(value ?? '') + .replace(/\\/g, '/').replace(/^["']|["']$/g, '').toLowerCase(); + +// Ignore dependencies, build output, and this session's CLI task output. +const IGNORE = /node_modules|\.git[/\\]|package-lock\.json|\/dist\/|\.map$|[/\\]temp[/\\]claude[/\\].*[/\\]tasks[/\\]/; + +// Commands that pull file contents into context. +const READER = /(?:^|[;&|]\s*)(?:cat|head|tail|less|more|type|grep|rg|ack|find|ls\s+-\w*l|sed\s+-n|awk)\s+([^;&|]+)/g; + +// Network targets in a shell command: any URL, and a raw socket target. A +// build legitimately reaches its own web, database, and SpacetimeDB ports; +// every other local port belongs to another run, the controller, or the +// dashboard. Internet targets are recorded, not judged. +const URL_TARGET = /https?:\/\/([^\s/'"`]+)/gi; +const SOCKET_TARGET = /(?:^|[;&|]\s*)(?:nc|ncat|netcat)\s+(?:-\S+\s+)*([\w.-]+)\s+(\d{2,5})\b/g; +const LOCAL_HOST = /^(?:127\.\d+\.\d+\.\d+|localhost|0\.0\.0\.0|\[::1\]|host\.docker\.internal|10\.\d+\.\d+\.\d+|172\.(?:1[6-9]|2\d|3[01])\.\d+\.\d+|192\.168\.\d+\.\d+)$/i; + +export interface NetworkTarget { + host: string; + port: number | null; +} + +export function networkTargetsFromBash(command: unknown): NetworkTarget[] { + const targets: NetworkTarget[] = []; + const text = String(command ?? ''); + for (const match of text.matchAll(URL_TARGET)) { + const authority = (match[1] ?? '').replace(/^[^@]*@/, ''); + const port = authority.match(/:(\d{1,5})$/)?.[1]; + targets.push({ host: authority.replace(/:\d{1,5}$/, ''), port: port ? Number(port) : null }); + } + for (const match of text.matchAll(SOCKET_TARGET)) { + targets.push({ host: match[1] ?? '', port: Number(match[2]) }); + } + return targets; +} + +const networkKind = (target: NetworkTarget, ownPorts: ReadonlySet): string | null => { + if (!LOCAL_HOST.test(target.host)) return 'network (internet)'; + if (target.port !== null && ownPorts.has(target.port)) return null; + return 'NETWORK / OTHER RUN'; +}; + +const CLASSES: Array = [ + [/[/\\]stack-bench(?:[/\\]|$)/, 'GRADER / TEST SPECS'], + [/\.claude[/\\]projects.*memory|[/\\]memory[/\\].*\.md$/, 'BENCHMARK NOTES'], + [/scenarios[/\\].*\.json|grade\.(?:js|ts)|mutation|check-scenarios/, 'GRADER / TEST SPECS'], + [/contracts[/\\].*\.json|appendix-\d+\.md|walk\.(?:js|ts)|lint\.(?:js|ts)/, 'CONTRACT / LINTER'], + [/prompts[/\\]|test-plans[/\\]|GRADING|RUBRIC/, 'PROMPTS / RUBRIC'], + [/[/\\]skills[/\\]/, 'skill docs (intended)'], + [/backends[/\\].*\.md|CLAUDE\.md|README/, 'setup docs (intended)'], +]; +const classify = (path: string): string => CLASSES.find(([pattern]) => pattern.test(path))?.[1] + ?? 'other'; + +// The shallowest recorded cwd is the app boundary when --app is absent. +function sessionCwd(lines: string[]): string | null { + const seen = new Set(); + for (const l of lines) { + const m = l.match(/"cwd":"((?:[^"\\]|\\.)*)"/); + if (m?.[1]) seen.add(norm(m[1].replace(/\\\\/g, '/'))); + } + if (!seen.size) return null; + return [...seen].sort((a, b) => a.split('/').length - b.split('/').length || a.length - b.length)[0] + ?? null; +} + +export function pathsFromBash(command: unknown): string[] { + const out: string[] = []; + for (const match of String(command).matchAll(READER)) { + const argumentsText = match[1]; + if (!argumentsText) continue; + for (const tokRaw of argumentsText.split(/\s+/)) { + const t = tokRaw.replace(/^["']|["']$/g, ''); + if (!t || t.startsWith('-')) continue; + if (/[*?]/.test(t) || /\//.test(t) || /\\/.test(t) || /\.\w+$/.test(t)) out.push(t); + } + } + return out; +} + +// Count file-tool reads only after their result confirms success. +// Bash reads count unless the command fails. +interface AuditHit { + path: string; + via: string; + kind: string; + unresolved?: boolean; +} + +interface PendingRead { + paths: string[]; + network: Array<{ path: string; kind: string }>; + via: string; +} + +interface TranscriptAudit { + file: string; + cwd: string | null; + fileTool: number; + bashReads: number; + hits: AuditHit[]; + refused: AuditHit[]; +} + +interface AuditResult extends TranscriptAudit { + root: string; +} + +interface TranscriptContent { + type?: string; + name?: string; + id?: string; + tool_use_id?: string; + is_error?: boolean; + input?: { file_path?: string; path?: string; pattern?: string; command?: string }; +} + +export function auditTranscript(file: string, boundary: string | null, + { ownPorts = [] }: { ownPorts?: readonly number[] } = {}): TranscriptAudit { + const own = new Set(ownPorts); + const lines = readFileSync(file, 'utf8').split('\n').filter(Boolean); + // Container transcripts use /app; --app is its host path. + const recorded = sessionCwd(lines); + const cwd = recorded === CODING_CONTAINER_APP_ROOT ? recorded : (boundary ?? recorded); + const hits: AuditHit[] = []; + const refused: AuditHit[] = []; + const pending = new Map(); + let fileTool = 0, bashReads = 0; + + for (const line of lines) { + let event: { message?: { content?: TranscriptContent[] } }; + try { event = JSON.parse(line) as typeof event; } catch { continue; } + const c = event.message?.content; + if (!Array.isArray(c)) continue; + for (const p of c) { + if (p.type === 'tool_result' && p.tool_use_id && pending.has(p.tool_use_id)) { + const completed = pending.get(p.tool_use_id ?? ''); + if (!completed) continue; + const { paths, network, via } = completed; + pending.delete(p.tool_use_id ?? ''); + const blocked = p.is_error === true; + for (const n of paths) (blocked ? refused : hits).push({ path: n, via, kind: classify(n) }); + for (const target of network) (blocked ? refused : hits).push({ ...target, via }); + continue; + } + if (p.type !== 'tool_use') continue; + const cand = []; + if (/^(Read|Grep|Glob|NotebookRead)$/.test(p.name ?? '')) { + fileTool++; + cand.push(p.input?.file_path ?? p.input?.path ?? p.input?.pattern ?? ''); + } else if (p.name === 'Bash') { + const found = pathsFromBash(p.input?.command ?? ''); + bashReads += found.length; + cand.push(...found); + } + const paths = []; + const network: Array<{ path: string; kind: string }> = []; + if (p.name === 'Bash') { + for (const target of networkTargetsFromBash(p.input?.command ?? '')) { + const kind = networkKind(target, own); + if (kind) network.push({ path: `${target.host}${target.port === null ? '' : `:${target.port}`}`, kind }); + } + } + for (const raw of cand) { + let n = norm(raw); + if (!n || IGNORE.test(n)) continue; + // The CLI keeps auto-memory for the session's OWN project dir. A build + // A session may read its own memory, never another project's memory. + if (cwd && /[/\\]projects[/\\][^/\\]+[/\\]memory[/\\]/.test(n) + && n.includes(cwd.replace(/[\\/:]/g, '-'))) continue; + const absolute = /^[a-z]:/.test(n) || n.startsWith('/'); + if (!absolute && cwd) n = `${cwd}/${n.replace(/^\.\//, '')}`; + const privateHarnessPath = cwd + && (n === `${cwd}/stack-bench` || n.startsWith(`${cwd}/stack-bench/`)); + if (!privateHarnessPath && !absolute) continue; + if (!privateHarnessPath && cwd && (n === cwd || n.startsWith(`${cwd}/`))) continue; + paths.push(n); + } + if ((paths.length || network.length) && p.id && p.name) { + pending.set(p.id, { paths, network, via: p.name }); + } + } + } + // A call whose result never arrived (session cut short) is unresolved, and + // unresolved is not innocent: count it. + for (const { paths, network, via } of pending.values()) { + for (const n of paths) hits.push({ path: n, via, kind: classify(n), unresolved: true }); + for (const target of network) hits.push({ ...target, via, unresolved: true }); + } + + return { file, cwd, fileTool, bashReads, hits, refused }; +} + +function main(): void { + const { values } = parseArgs({ args: process.argv.slice(2), options: { + app: { type: 'string' }, dir: { type: 'string' }, json: { type: 'boolean' }, + 'own-ports': { type: 'string' }, + } }); + const ownPorts = (values['own-ports'] ?? '').split(',').filter(Boolean).map(Number); + if (ownPorts.some(port => !Number.isInteger(port) || port <= 0)) { + throw new Error('--own-ports must list positive integers'); + } + const requestedApp = values.app; + const requestedDirectory = values.dir; + if (requestedApp && requestedDirectory) throw new Error('--app and --dir cannot be used together'); + const roots = requestedApp ? transcriptsFor(requestedApp) + : requestedDirectory ? [resolve(requestedDirectory)] + : [join(homedir(), '.claude', 'projects')]; + // When the caller names the app directory, that is the boundary. Do not + // infer it from a transcript folder name. + const appBoundary = requestedApp ? norm(resolve(requestedApp)) : null; + const results: AuditResult[] = []; +for (const root of roots) { + if (!existsSync(root)) continue; + const stack = [root]; + while (stack.length) { + const d = stack.pop(); + if (!d) continue; + for (const e of readdirSync(d, { withFileTypes: true })) { + const p = join(d, e.name); + if (e.isDirectory()) { if (!/node_modules/.test(p)) stack.push(p); continue; } + if (!/\.jsonl$/.test(e.name)) continue; + // Include transcripts from the main session and its subagents. + if (!/transcript|^agent-|^[0-9a-f-]{36}\.jsonl$/.test(e.name)) continue; + if (statSync(p).size < 2000) continue; + results.push({ ...auditTranscript(p, appBoundary, { ownPorts }), root }); + } + } +} + +if (values.json) { + console.log(JSON.stringify(results, null, 2)); + return; +} + +const label = (file: string): string => file.replace(/\\/g, '/') + .split('/').slice(-3).join('/').slice(0, 62); +console.log('\nBuilds that read outside their own directory'); +console.log('(counts BOTH file tools and Bash cat/grep/find; boundary = the session\'s own cwd)\n'); + +let clean = 0; +for (const r of results.sort((a, b) => b.hits.length - a.hits.length)) { + if (!r.cwd) { console.log(` ?? ${label(r.file)} — no cwd recorded, cannot judge`); continue; } + if (!r.hits.length) { + clean++; + // Blocked attempts are worth printing: they are the sandbox doing its job, + // and they say which paths a build still goes looking for. + if (r.refused?.length) { + const kinds = [...new Set(r.refused.map(h => h.kind))].join(', '); + console.log(` ${label(r.file)}\n clean — ${r.refused.length} attempt(s) BLOCKED by the sandbox (${kinds})`); + } + continue; + } + const byKind: Record = {}; + for (const h of r.hits) (byKind[h.kind] ??= []).push(h.path); + console.log(` ${label(r.file)}`); + console.log(` cwd: ...${r.cwd.slice(-52)} (${r.fileTool} file-tool, ${r.bashReads} bash reads)`); + for (const [k, v] of Object.entries(byKind).sort((a, b) => b[1].length - a[1].length)) { + const example = [...new Set(v)][0] ?? ''; + console.log(` ${String(v.length).padStart(3)}x ${k.padEnd(22)} ${example.split('/').slice(-2).join('/')}`); + } +} +console.log(`\n ${clean} transcript(s) read nothing outside their directory.`); +console.log(` ${results.length} transcript(s) examined.\n`); +} + +if (process.argv[1] && pathToFileURL(process.argv[1]).href === import.meta.url) main(); diff --git a/tools/stack-bench/commands/null-control.ts b/tools/stack-bench/commands/null-control.ts new file mode 100644 index 00000000000..4716e66e1ea --- /dev/null +++ b/tools/stack-bench/commands/null-control.ts @@ -0,0 +1,260 @@ +#!/usr/bin/env node +// Grade the real validated production scenarios against a reachable app that +// implements nothing. Every point-bearing criterion must conclusively fail. + +import { execFile } from 'node:child_process'; +import { createServer } from 'node:http'; +import type { Server } from 'node:http'; +import type { AddressInfo } from 'node:net'; +import { mkdirSync, mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { basename, join, relative, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { parseArgs as parseNodeArgs } from 'node:util'; +import { readArtifactPayload, writeRunJson } from '../src/evidence/artifacts.js'; +import { calibrationQualificationIdentity, calibrationQualificationRelease, + resolveCalibrationForRelease } from '../src/composition/calibration-compiler.js'; +import { qualificationScopeIdentity } from '../src/composition/qualification-scope.js'; +import { analyseNullReports } from '../src/evidence/null-control-analysis.js'; +import { resolveRecipeRelease } from '../src/composition/recipe-release.js'; +import { resolveRecipeSelection } from '../src/composition/recipe-selection.js'; +import { isDeclaredLevel, listTracks, loadTrack, suitesFor } from '../src/composition/tracks.js'; +import { controllerRunner } from '../src/runtime/runner-environment.js'; +import type { CalibrationPlan } from '../src/composition/calibration-compiler.js'; +import type { RecipeBinding } from '../src/composition/recipe-release.js'; +import type { Track } from '../src/composition/tracks.js'; + +import { STACK_BENCH_ROOT as ROOT, compiledEntrypoint } from '../src/package-root.js'; +const GRADE = compiledEntrypoint('grader', 'grade.js'); +const NULL_CONTROL_WORKERS = 4; + +interface NullControlArgs { + tracks: string[]; + level: number | null; + recipe?: string; + out?: string; + audit: boolean; + parentAttemptId?: string; +} + +export function parseNullControlArgs(argv: string[]): NullControlArgs { + const { values } = parseNodeArgs({ args: argv.slice(2), options: { + track: { type: 'string' }, level: { type: 'string' }, recipe: { type: 'string' }, + out: { type: 'string' }, audit: { type: 'boolean' }, 'parent-attempt-id': { type: 'string' }, + } }); + const args: NullControlArgs = { + tracks: values.track?.split(',').filter(Boolean) ?? listTracks(), + level: values.level === undefined ? null : Number(values.level), audit: values.audit ?? false, + recipe: values.recipe, out: values.out, parentAttemptId: values['parent-attempt-id'], + }; + if (args.level !== null && (!Number.isInteger(args.level) || args.level < 1)) { + throw new Error('--level must be a positive integer'); + } + if (args.level !== null && args.tracks.length !== 1) { + throw new Error('--level requires exactly one --track'); + } + if (args.recipe && args.level === null) throw new Error('--recipe requires --level'); + return args; +} + +function runGrade(argv: string[], timeoutMs = 300_000): Promise<{ stdout: string; stderr: string }> { + return new Promise((resolve, reject) => { + execFile(process.execPath, [GRADE, ...argv], { + encoding: 'utf8', maxBuffer: 32 * 1024 * 1024, timeout: timeoutMs, + }, (error, stdout, stderr) => { + if (error) { + error.message = `grader failed: ${error.message}\n${stdout}\n${stderr}`; + reject(error); + } else resolve({ stdout, stderr }); + }); + }); +} + +export function nullControlSuites(track: Track, selectedLevel: number | null = null, + binding: RecipeBinding | null = null) { + if (selectedLevel !== null && !isDeclaredLevel(track, selectedLevel)) { + throw new Error(`L${selectedLevel} is not declared for ${track.name}`); + } + if (binding) { + if (selectedLevel === null) throw new Error('recipe-bound null control requires one level'); + if (!Array.isArray(binding.execution) || !binding.execution.length) { + throw new Error('recipe-bound null control requires a typed execution plan'); + } + const executionIds = new Set(); + const mappedKeys = new Set(); + const suites = binding.execution.map(execution => { + if (executionIds.has(execution.id)) { + throw new Error(`recipe-bound null control repeats execution ${execution.id}`); + } + executionIds.add(execution.id); + const checks = binding.release.checkCatalog.filter(check => check.executionId === execution.id); + if (!checks.length) { + throw new Error(`recipe-bound null control execution ${execution.id} maps no checks`); + } + for (const check of checks) { + if (mappedKeys.has(check.stableKey)) { + throw new Error(`recipe-bound null control maps check ${check.stableKey} more than once`); + } + mappedKeys.add(check.stableKey); + } + return { id: execution.id, spec: resolve(track.dir, execution.source ?? ''), + level: selectedLevel, checks }; + }); + const missing = binding.release.checkCatalog + .filter(check => !mappedKeys.has(check.stableKey)).map(check => check.stableKey); + if (missing.length) { + throw new Error(`recipe-bound null control leaves checks unmapped: ${missing.join(', ')}`); + } + return suites; + } + const seen = new Set(); + const suites = []; + const levels = selectedLevel === null + ? Array.from({ length: track.validatedThrough }, (_, index) => index + 1) + : [selectedLevel]; + for (const level of levels) { + for (const suite of suitesFor(track, level)) { + if (seen.has(suite.spec)) continue; + seen.add(suite.spec); + suites.push({ ...suite, level }); + } + } + return suites; +} + +export function selectNullQualificationBinding(binding: RecipeBinding, calibration: CalibrationPlan): RecipeBinding { + const selected = calibrationQualificationRelease(calibration, binding.release, binding.execution); + return { ...binding, release: selected.release, execution: selected.execution }; +} + +export function createNullQualification(binding: RecipeBinding, calibration: CalibrationPlan) { + const selectedBinding = selectNullQualificationBinding(binding, calibration); + const selection = resolveRecipeSelection(selectedBinding.release, { + checkKeys: selectedBinding.release.checkCatalog.map(check => check.stableKey), + }); + return { + binding: selectedBinding, + calibration, + identity: calibrationQualificationIdentity(calibration), + selectionSha256: selection.sha256, + }; +} + +async function listen(server: Server): Promise { + await new Promise((resolve, reject) => { + server.once('error', reject); + server.listen({ port: 0, host: '127.0.0.1' }, () => resolve()); + }); + return (server.address() as AddressInfo).port; +} + +async function main() { + const args = parseNullControlArgs(process.argv); + const nullAttemptId = `null-control-${new Date().toISOString().replace(/[:.]/g, '-')}`; + const work = mkdtempSync(join(tmpdir(), 'stack-bench-null-')); + const app = join(work, 'app'); + const reportsDir = join(work, 'reports'); + mkdirSync(app, { recursive: true }); + mkdirSync(reportsDir, { recursive: true }); + + // Root navigation succeeds, proving the browser and server are healthy. All + // application/API behavior is absent: non-navigation requests get 404. + const server = createServer((request, response) => { + if (request.method === 'GET' && (request.url === '/' || request.headers.accept?.includes('text/html'))) { + response.writeHead(200, { 'content-type': 'text/html; charset=utf-8' }); + response.end('Null control'); + } else { + response.writeHead(404, { 'content-type': 'application/json' }); + response.end('{"error":"not implemented"}'); + } + }); + + const started = Date.now(); + const suiteReports = []; + let qualification: ReturnType | null = null; + try { + const port = await listen(server); + const url = `http://127.0.0.1:${port}`; + for (const trackName of args.tracks) { + const track = loadTrack(trackName); + let binding: RecipeBinding | null = null; + if (args.level !== null) { + binding = resolveRecipeRelease(track, args.level, args.recipe); + if (!binding) throw new Error(`${trackName} L${args.level} has no recipe release`); + const calibration = resolveCalibrationForRelease(binding.release, + { trackRoot: track.dir, stackBenchRoot: ROOT }); + if (!calibration) throw new Error(`${trackName} L${args.level} has no calibration`); + qualification = createNullQualification(binding, calibration); + binding = qualification.binding; + } + const selectedSuites = nullControlSuites(track, args.level, binding); + const resolvedRecipe = binding + ? `${binding.release.id}@${binding.release.version}` : args.recipe; + for (let index = 0; index < selectedSuites.length; index += NULL_CONTROL_WORKERS) { + const reports = await Promise.all(selectedSuites + .slice(index, index + NULL_CONTROL_WORKERS).map(async suite => { + const reportPath = join(reportsDir, + `${trackName}-l${suite.level}-${suite.id.replaceAll('@', '-')}.json`); + console.log(`${trackName} L${suite.level} ${suite.id} (${basename(suite.spec)})`); + await runGrade(['--url', url, '--level', String(suite.level), '--spec', suite.spec, + '--backend', 'postgres', '--track', trackName, '--app', app, '--out', reportPath, + '--null-control', + '--parent-attempt-id', nullAttemptId, + ...(resolvedRecipe ? ['--recipe', resolvedRecipe] : []), + ...(binding ? ['--expected-recipe-sha256', binding.release.contentSha256] : []), + ...(qualification ? ['--selection-sha256', qualification.selectionSha256] : []), + ...(('checks' in suite ? suite.checks : []) ?? []) + .flatMap(check => ['--selected-check', check.stableKey])]); + const report = readArtifactPayload(reportPath, { expectedKind: 'grade' }); + console.log(`${suite.id}: ${report.total}/${report.max}`); + return { track: trackName, level: suite.level, id: suite.id, + scenario: relative(track.dir, suite.spec).replaceAll('\\', '/'), report }; + })); + suiteReports.push(...reports); + } + } + + const analysis = analyseNullReports(suiteReports); + const artifact = { + id: nullAttemptId, + kind: 'null_control', + startedAt: new Date(started).toISOString(), + completedAt: new Date().toISOString(), + parentAttemptId: args.parentAttemptId ?? null, + identities: qualification ? { + recipe: { id: qualification.binding.release.id, version: qualification.binding.release.version, + sha256: qualification.binding.release.contentSha256, state: qualification.binding.release.state }, + calibration: { ...qualification.identity, state: qualification.calibration.state }, + } : undefined, + durationMs: Date.now() - started, + runner: controllerRunner(), + ...(qualification ? { qualificationScope: qualificationScopeIdentity({ + kind: 'null', release: qualification.binding.release, stackBenchRoot: ROOT, + }) } : {}), + tracks: args.tracks, + ...analysis, + }; + const outputPath = resolve(args.out ?? join(ROOT, 'results', `${artifact.id}.json`)); + writeRunJson(outputPath, artifact); + console.log(JSON.stringify({ + id: artifact.id, + kind: artifact.kind, + durationMs: artifact.durationMs, + tracks: artifact.tracks, + ok: artifact.ok, + summary: artifact.summary, + artifact: outputPath, + }, null, 2)); + if (!analysis.ok && !args.audit) process.exitCode = 1; + } finally { + await new Promise(resolve => server.close(resolve)); + rmSync(work, { recursive: true, force: true }); + } +} + +if (process.argv[1] && pathToFileURL(resolve(process.argv[1])).href === import.meta.url) { + main().catch(error => { + console.error(error.stack ?? error.message); + process.exitCode = 2; + }); +} diff --git a/tools/stack-bench/commands/pack-budget.ts b/tools/stack-bench/commands/pack-budget.ts new file mode 100644 index 00000000000..6ab28f76873 --- /dev/null +++ b/tools/stack-bench/commands/pack-budget.ts @@ -0,0 +1,82 @@ +#!/usr/bin/env node + +import { existsSync } from 'node:fs'; +import { dirname, relative, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { parseArgs } from 'node:util'; + +import { artifactPayload, recipeArtifactIdentities, writeArtifact } from '../src/evidence/artifacts.js'; +import { calibrationQualificationIdentity, resolveCalibrationForRelease } + from '../src/composition/calibration-compiler.js'; +import { loadPackBudgetEvidence, PACK_BUDGET_POLICY, recommendPackBudgets } + from '../src/composition/pack-budget.js'; +import { resolveRecipeRelease } from '../src/composition/recipe-release.js'; +import { isDeclaredLevel, listTracks, loadTrack } from '../src/composition/tracks.js'; + +interface PackBudgetArgs { + command: 'recommend'; + track: string; + level: number; + evidence: string[]; + out: string; + recipe?: string; +} + +const USAGE = 'usage: pack-budget.js recommend --track --level ' + + '[--recipe @] --evidence [--evidence ...] ' + + '--out '; + +export function parsePackBudgetArgs(argv: string[]): PackBudgetArgs { + const [command, ...options] = argv.slice(2); + const { values } = parseArgs({ args: options, options: { + track: { type: 'string' }, + level: { type: 'string' }, + recipe: { type: 'string' }, + evidence: { type: 'string', multiple: true }, + out: { type: 'string' }, + } }); + const level = Number(values.level); + const evidence = (values.evidence ?? []).map(path => resolve(path)); + if (command !== 'recommend' || !values.track || !Number.isInteger(level) || level < 1 + || !evidence.length || !values.out) throw new Error(USAGE); + if (new Set(evidence).size !== evidence.length) throw new Error('--evidence paths must be unique'); + return { command, track: values.track, level, evidence, out: resolve(values.out), + ...(values.recipe ? { recipe: values.recipe } : {}) }; +} + +function main(): void { + const args = parsePackBudgetArgs(process.argv); + if (!listTracks().includes(args.track)) throw new Error(`unknown track ${args.track}`); + const track = loadTrack(args.track); + if (!isDeclaredLevel(track, args.level)) throw new Error(`L${args.level} is not declared for ${args.track}`); + const binding = resolveRecipeRelease(track, args.level, args.recipe); + if (!binding) throw new Error(`${args.track} L${args.level} has no recipe release`); + const calibration = resolveCalibrationForRelease(binding.release, { trackRoot: track.dir }); + if (!calibration) throw new Error(`${binding.release.id}@${binding.release.version} has no calibration`); + const loaded = loadPackBudgetEvidence(args.evidence); + const result = recommendPackBudgets({ binding, calibration, evidence: loaded }); + if (existsSync(args.out)) throw new Error(`refusing to replace existing budget measurement: ${args.out}`); + const id = `pack-budget-${args.track}-l${args.level}-${new Date().toISOString().replace(/[-:.TZ]/g, '').slice(0, 14)}`; + const artifact = writeArtifact(args.out, { kind: 'pack_budget_measurement', id, + identities: recipeArtifactIdentities(binding.release, { + calibration: { ...calibrationQualificationIdentity(calibration), state: calibration.state }, + }), + payload: { schemaVersion: 1, track: args.track, level: args.level, policy: PACK_BUDGET_POLICY, + runner: result.measuredRunner, + evidence: loaded.map(item => { + const stackAdapter = item.artifact.identities.stackAdapter; + if (!stackAdapter) throw new Error(`${item.path} has no stack adapter identity`); + return { path: relative(dirname(args.out), item.path).replaceAll('\\', '/'), + sha256: item.sha256, stack: stackAdapter.id }; + }), + samples: result.samples, recommendations: result.recommendations } }); + console.log(JSON.stringify(artifactPayload(artifact), null, 2)); +} + +if (process.argv[1] && pathToFileURL(resolve(process.argv[1])).href === import.meta.url) { + try { main(); } + catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 2; + } +} diff --git a/tools/stack-bench/commands/preflight-cli.ts b/tools/stack-bench/commands/preflight-cli.ts new file mode 100644 index 00000000000..af689917c26 --- /dev/null +++ b/tools/stack-bench/commands/preflight-cli.ts @@ -0,0 +1,85 @@ +import { resolve } from 'node:path'; +import { parseArgs } from 'node:util'; + +import { DEFAULT_BUILD_IMAGE } from '../src/composition/product-config.js'; +import { STACK_BENCH_ROOT } from '../src/package-root.js'; +import { stackBenchResultsRoot } from '../src/runtime/operational-paths.js'; +import type { PreflightReport, PreflightRequest } from '../src/runtime/preflight.js'; + +function splitList(value: unknown): string[] { + return String(value).split(',').map(item => item.trim()).filter(Boolean); +} + +export function parsePreflightArgs( + argv: string[], + { env = process.env }: { env?: NodeJS.ProcessEnv } = {}, +): PreflightRequest { + const { values } = parseArgs({ args: argv.slice(2), options: { + backend: { type: 'string', multiple: true }, + track: { type: 'string' }, + levels: { type: 'string' }, + recipe: { type: 'string' }, + 'run-index': { type: 'string' }, + parallelism: { type: 'string' }, + 'agent-adapter': { type: 'string' }, + guidance: { type: 'string' }, + pack: { type: 'string', multiple: true }, + check: { type: 'string', multiple: true }, + image: { type: 'string' }, + 'results-dir': { type: 'string' }, + report: { type: 'string' }, + smoke: { type: 'boolean' }, + json: { type: 'boolean' }, + } }); + const request: PreflightRequest = { backends: [], track: 'ecommerce', levels: '1', levelList: [], + runIndex: 0, parallelism: 1, + agentAdapter: 'claude-code', guidance: 'prescribed', packIds: [], checkKeys: [], smoke: false, + image: env.STACK_BENCH_IMAGE ?? DEFAULT_BUILD_IMAGE, + resultsDir: stackBenchResultsRoot(STACK_BENCH_ROOT, env) }; + request.backends = (values.backend ?? []).flatMap(splitList); + if (values.track !== undefined) request.track = values.track; + if (values.levels !== undefined) request.levels = values.levels; + if (values.recipe !== undefined) request.recipe = values.recipe; + if (values['run-index'] !== undefined) request.runIndex = Number(values['run-index']); + if (values.parallelism !== undefined) request.parallelism = Number(values.parallelism); + if (values['agent-adapter'] !== undefined) request.agentAdapter = values['agent-adapter']; + if (values.guidance !== undefined) request.guidance = values.guidance; + request.packIds = (values.pack ?? []).flatMap(splitList); + request.checkKeys = (values.check ?? []).flatMap(splitList); + if (values.image !== undefined) request.image = values.image; + if (values['results-dir'] !== undefined) request.resultsDir = resolve(values['results-dir']); + if (values.report !== undefined) request.report = resolve(values.report); + request.smoke = values.smoke ?? false; + request.json = values.json; + if (!request.backends.length) throw new Error('--backend is required (comma-separated values are accepted)'); + if (request.guidance !== 'neutral' && request.guidance !== 'prescribed') { + throw new Error('--guidance must be neutral or prescribed'); + } + request.backends = [...new Set(request.backends)].sort(); + if (!Number.isInteger(request.runIndex) || request.runIndex < 0) { + throw new Error('--run-index must be a non-negative integer'); + } + if (!Number.isInteger(request.parallelism) || (request.parallelism ?? 0) < 1) { + throw new Error('--parallelism must be a positive integer'); + } + const match = String(request.levels).match(/^(\d+)(?:-(\d+))?$/); + if (!match || Number(match[2] ?? match[1]) < Number(match[1])) { + throw new Error('--levels must be N or N-M'); + } + request.levelList = Array.from({ length: Number(match[2] ?? match[1]) - Number(match[1]) + 1 }, + (_, index) => Number(match[1]) + index); + if (request.recipe && request.levelList.length !== 1) { + throw new Error('--recipe requires exactly one requested level'); + } + return request; +} + +export function printPreflightReport(report: PreflightReport): void { + console.log(`Stack Bench preflight: ${report.ok ? 'READY' : 'NOT READY'}`); + for (const check of report.checks) { + const mark = check.status === 'pass' ? 'PASS' : check.status === 'warn' ? 'WARN' : 'FAIL'; + console.log(` ${mark.padEnd(4)} ${check.id.padEnd(28)} ${check.summary}`); + if (check.remediation && check.status === 'fail') console.log(` fix: ${check.remediation}`); + } + console.log(`\n${report.summary.passed} passed, ${report.summary.failed} failed, ${report.summary.warnings} warnings`); +} diff --git a/tools/stack-bench/commands/preflight.ts b/tools/stack-bench/commands/preflight.ts new file mode 100644 index 00000000000..45cb7a40af1 --- /dev/null +++ b/tools/stack-bench/commands/preflight.ts @@ -0,0 +1,19 @@ +#!/usr/bin/env node + +import { parsePreflightArgs, printPreflightReport } from './preflight-cli.js'; +import { runPreflight, writePreflightReport } from '../src/runtime/preflight.js'; + +let request; +try { + request = parsePreflightArgs(process.argv); +} catch (error) { + console.error(`preflight: ${error instanceof Error ? error.message : String(error)}`); + console.error('Usage: stack-bench preflight --backend spacetime[,postgres,mongodb] [--track ecommerce] [--levels 1-2] [--smoke]'); + process.exit(2); +} + +const report = runPreflight(request); +if (request.report) writePreflightReport(request.report, report); +if (request.json) console.log(JSON.stringify(report, null, 2)); +else printPreflightReport(report); +process.exitCode = report.ok ? 0 : 1; diff --git a/tools/stack-bench/commands/progression-graph.ts b/tools/stack-bench/commands/progression-graph.ts new file mode 100644 index 00000000000..82dc0a3be9e --- /dev/null +++ b/tools/stack-bench/commands/progression-graph.ts @@ -0,0 +1,21 @@ +import { dirname, join, resolve } from 'node:path'; + +import { STACK_BENCH_ROOT } from '../src/package-root.js'; +import { writeProgressionGraph } from '../src/progression/progression-graph.js'; + +interface ProgressionGraph { + nodes: unknown[]; + levels: number; +} + +const definitionPath = process.argv[2]; +if (!definitionPath) { + throw new Error('usage: progression-graph [html-path]'); +} +const resolvedDefinitionPath = resolve(definitionPath); +const graph: ProgressionGraph = writeProgressionGraph({ + definitionPath: resolvedDefinitionPath, + htmlPath: process.argv[3] ?? join(STACK_BENCH_ROOT, 'docs', 'dependency-graph.html'), + trackRoot: dirname(dirname(resolvedDefinitionPath)), +}); +console.log(`Rendered ${graph.nodes.length} nodes across ${graph.levels} levels.`); diff --git a/tools/stack-bench/commands/qualification-cli.ts b/tools/stack-bench/commands/qualification-cli.ts new file mode 100644 index 00000000000..74839b00940 --- /dev/null +++ b/tools/stack-bench/commands/qualification-cli.ts @@ -0,0 +1,277 @@ +#!/usr/bin/env node + +import { existsSync, readFileSync } from 'node:fs'; +import { basename, dirname, extname, join, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { parseArgs as parseNodeArgs } from 'node:util'; + +import { calibrationQualificationIdentity, resolveCalibrationForRelease } from '../src/composition/calibration-compiler.js'; +import { resolveRecipeRelease } from '../src/composition/recipe-release.js'; +import { isDeclaredLevel, listTracks, loadTrack } from '../src/composition/tracks.js'; +import { PACK_BUDGET_POLICY } from '../src/composition/pack-budget.js'; +import { STACK_BENCH_ROOT } from '../src/package-root.js'; +import { stackBenchResultsRoot } from '../src/runtime/operational-paths.js'; +import { companionReferenceArtifactPath } from '../src/references/reference-live.js'; +import type { CalibrationPlan } from '../src/composition/calibration-compiler.js'; +import type { RecipeBinding, RecipeRelease } from '../src/composition/recipe-release.js'; + +interface QualificationArgs { + command?: string; + track: string | null; + level: number | null; + recipe?: string; +} + +interface QualificationBlocker { + code: string; + path: string; + summary: string; +} + +export function parseQualificationArgs(argv: string[]): QualificationArgs { + const { positionals, values } = parseNodeArgs({ args: argv.slice(2), allowPositionals: true, + options: { track: { type: 'string' }, level: { type: 'string' }, recipe: { type: 'string' } } }); + const args: QualificationArgs = { command: positionals[0], track: values.track ?? null, + level: values.level === undefined ? null : Number(values.level), + ...(values.recipe === undefined ? {} : { recipe: values.recipe }) }; + if (args.command !== 'status' || typeof args.track !== 'string' || !args.track + || positionals.length !== 1 || args.level === null || !Number.isInteger(args.level) || args.level < 1) { + throw new Error('usage: node dist/commands/qualification-cli.js status --track --level ' + + '[--recipe @]'); + } + return args; +} + +function blocker(code: string, path: string, summary: string): QualificationBlocker { + return { code, path, summary }; +} + +function evidencePlan(calibration: CalibrationPlan) { + const stacks = calibration.qualification.stacks + .filter(stack => stack.status !== 'unsupported').map(stack => stack.id).sort(); + const evidence = []; + for (const stack of stacks) { + for (let repetition = 1; repetition <= calibration.qualification.referenceRepetitions; repetition += 1) { + evidence.push({ kind: 'reference', stack, repetition }); + } + for (let repetition = 1; repetition <= calibration.qualification.mutationRepetitions; repetition += 1) { + evidence.push({ kind: 'mutation', stack, repetition }); + } + } + for (let repetition = 1; repetition <= calibration.nullControl.repetitions; repetition += 1) { + evidence.push({ kind: 'null', stack: null, repetition }); + } + return evidence; +} + +export interface CalibrationMutationSelection { + mutations: Array<{ backend: string; path: string; targets: Array<{ id: string }> }>; +} + +export function mutationWorkerCount(calibration: CalibrationMutationSelection, stack: string, + readManifest: (path: string) => { mutations?: { id: string }[] } = path => + JSON.parse(readFileSync(resolve(STACK_BENCH_ROOT, path), 'utf8')) as { mutations?: { id: string }[] }) { + const entry = calibration.mutations.find(candidate => candidate.backend === stack); + if (!entry) return 1; + const manifest = readManifest(entry.path); + const selectedIds = new Set(entry.targets.map(target => target.id)); + const selectedMutations = (manifest.mutations ?? []).filter(mutation => + selectedIds.delete(mutation.id)); + if (selectedIds.size) { + throw new Error(`${stack} calibration selects missing mutations: ${[...selectedIds].sort().join(', ')}`); + } + return Math.min(4, Math.max(1, selectedMutations.length)); +} + +function mutationWorkerOption(calibration: CalibrationPlan, stack: string) { + const workers = mutationWorkerCount(calibration, stack); + return workers > 1 ? ` --mutation-workers ${workers}` : ''; +} + +function qualificationRunDirectory(artifactPath: string): string { + return join(dirname(artifactPath), `${basename(artifactPath, extname(artifactPath))}.runs`); +} + +function defectCheckCoverage(release: RecipeRelease, calibration: CalibrationPlan) { + const selected = calibration.qualification.checks + ? new Set(calibration.qualification.checks) : null; + const scored = release.checkCatalog.filter(check => check.points > 0 + && (selected === null || selected.has(check.stableKey))); + const scoredByKey = new Map(scored.map(check => [check.stableKey, check])); + const stacks = calibration.qualification.stacks + .filter(stack => stack.status !== 'unsupported').map(stack => stack.id).sort(); + return { + required: 'every scored check has an exact known-defect test on every supported stack', + totalChecks: scored.length, + totalPoints: scored.reduce((total, check) => total + check.points, 0), + stacks: stacks.map(stack => { + const covered = new Set(calibration.mutations + .filter(entry => entry.backend === stack) + .flatMap(entry => entry.targets.flatMap(target => target.stableKeys)) + .filter(key => scoredByKey.has(key))); + const missing = scored.filter(check => !covered.has(check.stableKey)); + return { + stack, + coveredChecks: covered.size, + coveredPoints: [...covered].reduce((total, key) => total + (scoredByKey.get(key)?.points ?? 0), 0), + missingChecks: missing.map(check => check.stableKey), + }; + }), + }; +} + +export function qualificationReadiness(trackName: string, level: number, recipe: string | null = null) { + if (!listTracks().includes(trackName)) throw new Error(`unknown qualification track ${trackName}`); + const track = loadTrack(trackName); + if (!isDeclaredLevel(track, level)) { + throw new Error(`L${level} is not declared for ${trackName}`); + } + const binding: RecipeBinding | null = resolveRecipeRelease(track, level, recipe); + if (!binding) throw new Error(`${trackName} L${level} has no recipe release`); + const calibration = resolveCalibrationForRelease(binding.release, + { trackRoot: track.dir, alias: `L${level}` }); + if (!calibration) { + throw new Error(`${binding.release.id}@${binding.release.version} has no L${level} calibration`); + } + const identity = calibrationQualificationIdentity(calibration); + const launchBlockers = []; + if (binding.release.state === 'retired') { + launchBlockers.push(blocker('recipe_retired', 'recipe.state', 'selected recipe is retired')); + } + for (const pack of binding.plan.packs) { + if (pack.budget.status !== 'bounded') { + launchBlockers.push(blocker('pack_budget_unbounded', `packs.${pack.id}.budget`, + `${pack.id}@${pack.version} needs a measured maxRuntimeMs before qualification`)); + } + } + for (const entry of calibration.references.entries) { + if (!['candidate', 'active'].includes(String(entry.status))) { + launchBlockers.push(blocker('reference_unavailable', `references.${entry.backend}`, + `${entry.id} is ${entry.status}`)); + } + } + for (const entry of calibration.mutations) { + if (!['candidate', 'active'].includes(String(entry.status))) { + launchBlockers.push(blocker('mutation_unavailable', `mutations.${entry.backend}`, + `${entry.path} is ${entry.status}`)); + } + } + + const requiredEvidence = evidencePlan(calibration); + const defectChecks = defectCheckCoverage(binding.release, calibration); + const recorded = new Set(calibration.qualification.evidence.map(entry => + `${entry.kind}:${entry.stack ?? ''}:${entry.repetition}`)); + const promotionBlockers = [...launchBlockers]; + for (const coverage of defectChecks.stacks.filter(item => item.missingChecks.length > 0)) { + promotionBlockers.push(blocker('defect_check_coverage_incomplete', + `defectChecks.${coverage.stack}`, + `${coverage.coveredChecks}/${defectChecks.totalChecks} scored checks have exact known-defect tests`)); + } + for (const item of requiredEvidence) { + const key = `${item.kind}:${item.stack ?? ''}:${item.repetition}`; + if (!recorded.has(key)) promotionBlockers.push(blocker('evidence_missing', `evidence.${key}`, + `${key} has no hash-bound qualification artifact`)); + } + for (const stale of (calibration.qualificationStaleness ?? []) as { + kind: string; stack?: string; repetition: number; reason: string; + }[]) { + const key = `${stale.kind}:${stale.stack ?? ''}:${stale.repetition}`; + promotionBlockers.push(blocker('qualification_evidence_stale', `evidence.${key}`, + `${key} must be regenerated: ${stale.reason}`)); + } + const sourceStates: [string, string][] = [ + ['recipe.state', binding.release.state], + ['fixture.state', binding.release.components.fixture.state], + ...binding.release.components.packs.map(pack => [`packs.${pack.id}.state`, pack.state] as [string, string]), + ['calibration.state', calibration.state], + ['promotion.status', binding.status], + ]; + const governance = sourceStates.map(([path, state]) => ({ path, state, + target: path === 'promotion.status' ? 'promoted' : 'qualified' })); + governance.push(...calibration.qualification.stacks.map(stack => ({ + path: `qualification.stacks.${stack.id}`, state: stack.status, + target: stack.status === 'unsupported' ? 'unsupported' : 'qualified', + }))); + + const output = join(stackBenchResultsRoot(STACK_BENCH_ROOT), 'qualification'); + const stacks = calibration.qualification.stacks + .filter(stack => stack.status !== 'unsupported').map(stack => stack.id).sort(); + const qualificationLevel = Number(calibration.promotion.alias.slice(1)); + const budgetEvidence = stacks.map(stack => + `${output}/budget-input/${trackName}-l${qualificationLevel}-${stack}.json`); + const budgetPreparationRequired = launchBlockers.some(item => item.code === 'pack_budget_unbounded'); + const recipeOption = ` --recipe ${binding.release.id}@${binding.release.version}`; + const featureCatalog = calibration.qualification.featureCatalog; + const featureCatalogOption = featureCatalog + ? ` --feature-catalog ${featureCatalog.id}@${featureCatalog.version}` : ''; + const combinedReferenceEvidence = calibration.qualification.referenceRepetitions + === calibration.qualification.mutationRepetitions; + const artifactStem = `${trackName}-l${qualificationLevel}-${binding.release.contentSha256.slice(0, 12)}`; + const artifactPaths = { + references: Object.fromEntries(stacks.map(stack => [stack, + `${output}/${artifactStem}-${stack}-reference.json`])), + mutations: Object.fromEntries(stacks.map(stack => [stack, + `${output}/${artifactStem}-${stack}-mutation.json`])), + null: `${output}/${artifactStem}-null.json`, + }; + const launchPaths = new Set([artifactPaths.null]); + for (const stack of stacks) { + const mutationPath = artifactPaths.mutations[stack]; + const referencePath = artifactPaths.references[stack]; + if (!mutationPath || !referencePath) throw new Error(`qualification path is missing for ${stack}`); + launchPaths.add(mutationPath); + launchPaths.add(qualificationRunDirectory(mutationPath)); + launchPaths.add(combinedReferenceEvidence + ? companionReferenceArtifactPath(mutationPath) : referencePath); + if (!combinedReferenceEvidence) { + launchPaths.add(qualificationRunDirectory(referencePath)); + } + } + for (const path of [...launchPaths].filter(existsSync).sort()) { + launchBlockers.push(blocker('qualification_output_exists', path, + 'qualification output already exists')); + } + return { + qualificationSchemaVersion: 1, + scope: { track: trackName, level, recipe: { id: binding.release.id, + version: binding.release.version, contentSha256: binding.release.contentSha256 }, + calibration: { ...identity, contentSha256: calibration.contentSha256 }, + runner: calibration.qualification.runner ?? null }, + launch: { ok: launchBlockers.length === 0, blockers: launchBlockers }, + budgetPreparation: { + required: budgetPreparationRequired, + policy: PACK_BUDGET_POLICY, + commands: budgetPreparationRequired ? [ + ...stacks.map((stack, index) => + `qualify-reference --backend ${stack} --track ${trackName} --level ${qualificationLevel}${recipeOption}${featureCatalogOption} --repetitions ${calibration.qualification.referenceRepetitions} --out ${budgetEvidence[index]}`), + `pack-budget recommend --track ${trackName} --level ${qualificationLevel}${recipeOption} ${budgetEvidence + .map(path => `--evidence ${path}`).join(' ')} --out ${output}/${trackName}-l${qualificationLevel}-pack-budgets.json`, + ] : [], + }, + requiredEvidence, + defectChecks, + artifactPaths, + commands: [ + ...stacks.flatMap(stack => [ + ...(!combinedReferenceEvidence ? [ + `qualify-reference --backend ${stack} --track ${trackName} --level ${qualificationLevel}${recipeOption}${featureCatalogOption} --repetitions ${calibration.qualification.referenceRepetitions} --out ${artifactPaths.references[stack]}`, + ] : []), + `qualify-reference --backend ${stack} --track ${trackName} --level ${qualificationLevel}${recipeOption}${featureCatalogOption} --repetitions ${calibration.qualification.mutationRepetitions} --mutations --release-candidate${mutationWorkerOption(calibration, stack)} --out ${artifactPaths.mutations[stack]}`, + ]), + `qualify-null --track ${trackName} --level ${qualificationLevel}${recipeOption} --out ${artifactPaths.null}`, + ], + promotion: { ready: promotionBlockers.length === 0, blockers: promotionBlockers, + governance }, + }; +} + +function main() { + const args = parseQualificationArgs(process.argv); + if (!args.track || args.level === null) throw new Error('track and level are required'); + console.log(JSON.stringify(qualificationReadiness(args.track, args.level, args.recipe), null, 2)); +} + +if (process.argv[1] && pathToFileURL(resolve(process.argv[1])).href === import.meta.url) { + try { main(); } + catch (error: unknown) { console.error(error instanceof Error ? error.message : String(error)); process.exitCode = 2; } +} diff --git a/tools/stack-bench/commands/recovery.ts b/tools/stack-bench/commands/recovery.ts new file mode 100644 index 00000000000..cf622baf806 --- /dev/null +++ b/tools/stack-bench/commands/recovery.ts @@ -0,0 +1,25 @@ +#!/usr/bin/env node + +import { recoverBackendLease, recoverSupervisedRun } from '../src/runtime/recovery.js'; + +const [command, statePath, option, output] = process.argv.slice(2); +const supervisorRequest = command === 'recover' && statePath !== undefined && process.argv.length === 4; +const leaseRequest = command === 'recover-lease' && statePath !== undefined && option === '--out' + && output !== undefined && process.argv.length === 6; +if (!supervisorRequest && !leaseRequest) { + console.error('Usage:\n' + + ' stack-bench recover \n' + + ' stack-bench recover-lease --out '); + process.exit(2); +} + +try { + const result = leaseRequest + ? recoverBackendLease(statePath, output) + : recoverSupervisedRun(statePath); + console.log(JSON.stringify(result, null, 2)); + process.exitCode = result.ok ? 0 : 1; +} catch (error) { + console.error(`recovery: ${error instanceof Error ? error.message : String(error)}`); + process.exitCode = 2; +} diff --git a/tools/stack-bench/commands/repair-cli.ts b/tools/stack-bench/commands/repair-cli.ts new file mode 100644 index 00000000000..b37a06a74a3 --- /dev/null +++ b/tools/stack-bench/commands/repair-cli.ts @@ -0,0 +1,220 @@ +#!/usr/bin/env node + +import { randomUUID } from 'node:crypto'; +import { existsSync, mkdirSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; + +import { acquireCampaignLock, releaseCampaignLock } from '../src/campaigns/campaign-lock.js'; +import { ARTIFACT_FILE, emptyArtifactIdentities, readArtifact, writeArtifact } + from '../src/evidence/artifacts.js'; +import { STACK_BENCH_ROOT } from '../src/package-root.js'; +import { rescueSupervisedLease } from '../src/runtime/recovery.js'; +import { runBounded } from '../src/runtime/bounded-process.js'; +import type { BoundedProcessResult, RunBoundedOptions } + from '../src/runtime/bounded-process.js'; +import { createRepairGrant, inspectRepairParent } from '../src/runtime/repair-grant.js'; + +const BENCH = join(STACK_BENCH_ROOT, 'dist', 'commands', 'bench.js'); + +export interface RepairStatusArgs { + command: 'status'; + parent: string; + level: number; +} + +export interface RepairGrantArgs { + command: 'grant'; + parent: string; + level: number; + repairs: number; + maxBudgetUsd?: number; + timeoutMinutes: number; +} + +export type RepairArgs = RepairStatusArgs | RepairGrantArgs; + +export function parseRepairArgs(argv: string[]): RepairArgs { + const [command, parent, ...rest] = argv.slice(2); + if (command === 'status' && parent && rest.length === 2 && rest[0] === '--level') { + const level = Number(rest[1]); + if (!Number.isSafeInteger(level) || level < 1) throw new Error('--level must be a positive integer'); + return { command, parent: resolve(parent), level }; + } + if (command !== 'grant' || !parent) { + throw new Error('usage: repair status --level | repair grant --level --repairs [--max-budget-usd ] [--timeout-minutes ]'); + } + const values: { level?: number; repairs?: number; maxBudgetUsd?: number; + timeoutMinutes: number } = { timeoutMinutes: 120 }; + const seen = new Set(); + for (let index = 0; index < rest.length; index += 2) { + const flag = rest[index]; + if (!flag || !['--level', '--repairs', '--max-budget-usd', '--timeout-minutes'].includes(flag) + || index + 1 >= rest.length || seen.has(flag)) { + throw new Error(`invalid or duplicate repair option ${String(flag)}`); + } + seen.add(flag); + const value = Number(rest[index + 1]); + if (flag === '--level') values.level = value; + else if (flag === '--repairs') values.repairs = value; + else if (flag === '--max-budget-usd') values.maxBudgetUsd = value; + else values.timeoutMinutes = value; + } + const level = values.level; + if (level === undefined || !Number.isSafeInteger(level) || level < 1) { + throw new Error('--level must be a positive integer'); + } + const repairs = values.repairs; + if (repairs === undefined || !Number.isSafeInteger(repairs) || repairs < 1) { + throw new Error('--repairs must be a positive safe integer'); + } + if (values.maxBudgetUsd !== undefined + && (!Number.isFinite(values.maxBudgetUsd) || values.maxBudgetUsd <= 0)) { + throw new Error('--max-budget-usd must be a positive number'); + } + if (!Number.isFinite(values.timeoutMinutes) || values.timeoutMinutes < 10 + || values.timeoutMinutes > 480) { + throw new Error('--timeout-minutes must be from 10 through 480'); + } + return { command, parent: resolve(parent), level, + repairs, timeoutMinutes: values.timeoutMinutes, + ...(values.maxBudgetUsd === undefined ? {} : { maxBudgetUsd: values.maxBudgetUsd }) }; +} + +export function repairStatus(parent: string, level: number): Record { + try { + const inspected = inspectRepairParent(parent, level); + return { eligible: true, parentRunId: inspected.parent.id, level, + score: inspected.level.score, max: inspected.level.max, + used: inspected.cumulativeRepairsBefore, + checkpointSha256: inspected.checkpoint.payload.source.sha256 }; + } catch (error) { + return { eligible: false, level, + reason: error instanceof Error ? error.message : String(error) }; + } +} + +interface RepairExecutionDependencies { + execute?: (command: string, argv: string[], + options: RunBoundedOptions) => Promise; + rescue?: (path: string, output: string) => void; + uuid?: () => string; + env?: NodeJS.ProcessEnv; +} + +interface RepairContinuationPayload { + outcome?: unknown; + continuation?: { + parentRunId?: string; + repairsGranted?: number; + level?: number; + [key: string]: unknown; + }; + [key: string]: unknown; +} + +export async function executeRepairGrant(args: RepairGrantArgs, + { execute = runBounded, rescue = rescueSupervisedLease, uuid = randomUUID, + env = process.env }: RepairExecutionDependencies = {}) { + const resolved = createRepairGrant(args.parent, { level: args.level, repairs: args.repairs }); + const lock = acquireCampaignLock(join(resolved.root, '.repair-control'), { + id: `repair-l${args.level}`, + contentSha256: resolved.checkpoint.payload.source.sha256, + }); + const stamp = new Date().toISOString().replace(/[-:.TZ]/g, '').slice(0, 14); + const executionId = `grant-${stamp}-${uuid().replaceAll('-', '').slice(0, 12)}`; + const output = join(resolved.root, 'continuations', executionId); + const privateRoot = join(tmpdir(), 'stack-bench-repair-supervisors'); + const supervisorState = join(privateRoot, `${executionId}.json`); + try { + mkdirSync(output, { recursive: true }); + mkdirSync(privateRoot, { recursive: true, mode: 0o700 }); + const argv = [BENCH, + '--repair-from', resolved.root, + '--repair-level', String(args.level), + '--repairs', String(args.repairs), + '--out', output, + '--no-media']; + if (args.maxBudgetUsd !== undefined) { + argv.push('--max-budget-usd', String(args.maxBudgetUsd)); + } + const childEnv: NodeJS.ProcessEnv = { + ...env, + STACK_BENCH_SUPERVISOR_STATE: supervisorState, + }; + if (resolved.configuration.buildImage) { + childEnv.STACK_BENCH_IMAGE = resolved.configuration.buildImage; + } + const processResult = await execute(process.execPath, argv, { + cwd: STACK_BENCH_ROOT, + env: childEnv, + stdio: 'inherit', + timeoutMs: args.timeoutMinutes * 60_000, + logs: { stdout: join(output, 'process.stdout.log'), + stderr: join(output, 'process.stderr.log') }, + }); + let cleanupError: unknown = null; + if (!processResult.ok && existsSync(supervisorState)) { + try { rescue(supervisorState, output); } + catch (error) { cleanupError = error; } + } + const streams = processResult.logs ? Object.fromEntries(Object.entries(processResult.logs) + .map(([name, value]) => [name, { ...value, path: `process.${name}.log` }])) : null; + writeArtifact(join(output, ARTIFACT_FILE.process), { + kind: 'repair_process', + id: `${executionId}-process`, + attempt: { id: `${executionId}-process`, parentId: resolved.parent.id }, + identities: emptyArtifactIdentities({ + agentAdapter: resolved.parentArtifact.identities.agentAdapter, + stackAdapter: resolved.parentArtifact.identities.stackAdapter, + }), + payload: { schemaVersion: 2, parentRunId: resolved.parent.id, + level: args.level, repairsGranted: args.repairs, + exitCode: processResult.code ?? null, signal: processResult.signal ?? null, + timedOut: processResult.timedOut, streams }, + }); + if (cleanupError) { + const detail = cleanupError instanceof Error ? cleanupError.message : String(cleanupError); + throw new Error(`repair continuation cleanup failed: ${detail}`); + } + const runPath = join(output, ARTIFACT_FILE.run); + if (!existsSync(runPath)) { + throw new Error(`repair continuation produced no run artifact${processResult.timedOut ? ' before its timeout' : ''}`); + } + const run = readArtifact(runPath, + { expectedKind: 'repair_continuation' }); + if (run.attempt.parentId !== resolved.parent.id + || run.payload.continuation?.parentRunId !== resolved.parent.id + || run.payload.continuation?.repairsGranted !== args.repairs + || run.payload.continuation?.level !== args.level) { + throw new Error('repair continuation result does not match its grant'); + } + return { output, process: processResult, run }; + } finally { + rmSync(supervisorState, { force: true }); + releaseCampaignLock(lock); + } +} + +async function main(): Promise { + const args = parseRepairArgs(process.argv); + if (args.command === 'status') { + const status = repairStatus(args.parent, args.level); + console.log(JSON.stringify(status, null, 2)); + if (status.eligible !== true) process.exitCode = 1; + return; + } + const result = await executeRepairGrant(args); + console.log(JSON.stringify({ output: result.output, id: result.run.id, + outcome: result.run.payload.outcome, + continuation: result.run.payload.continuation }, null, 2)); + if (!result.process.ok) process.exitCode = 1; +} + +if (process.argv[1] && pathToFileURL(resolve(process.argv[1])).href === import.meta.url) { + main().catch((error: unknown) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 2; + }); +} diff --git a/tools/stack-bench/commands/report-bugs.ts b/tools/stack-bench/commands/report-bugs.ts new file mode 100644 index 00000000000..955bec395ce --- /dev/null +++ b/tools/stack-bench/commands/report-bugs.ts @@ -0,0 +1,326 @@ +#!/usr/bin/env node +// Turns grading results into a behavioral BUG_REPORT.md for the fix agent. +// +// Every line the agent reads comes from one of three sources: the sentence +// the agent was already given for the behavior (`statedBy`, else the +// criterion's description), the rendered finding from the catalog, or the +// application's own console errors. Harness prose never enters the report. + +import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from 'node:fs'; +import { dirname, join, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { parseArgs as parseNodeArgs } from 'node:util'; +import { renderFinding } from '../src/actions/action-findings.js'; +import type { Finding } from '../src/actions/action-findings.js'; +import { sanitiseConsoleError, sanitiseDiagnostic } from '../src/evidence/diagnostic-sanitizer.js'; +import { ARTIFACT_FILE, readArtifactPayload } from '../src/evidence/artifacts.js'; +import { criterionEvidence, evidenceIsRepairable } from '../src/evidence/check-evidence.js'; +import { assertAgentVisibleText } from '../src/composition/agent-visible-contract.js'; +import { CODING_CONTAINER_BUG_REPORT_FILE, CODING_CONTAINER_START_SCRIPT } + from '../src/runtime/coding-container-policy.js'; + +interface RepairHistoryEntry { + round?: number; + beforeScore?: number; + beforeMax?: number; + afterScore?: number; + afterMax?: number; + result?: string; + remainingFailures?: string[]; +} + +interface ReportBugsArgs { + app: string; + results: string; + out: string; + archive?: string; + history: RepairHistoryEntry[]; + checks: string[] | null; + controls: string[] | null; + priorRegression: string | null; + regressionContext: boolean; +} + +interface ParsedArgs { + app?: string; + results?: string; + out?: string; + archive?: string; + history?: unknown; + checks?: unknown; + controls?: unknown; +} + +interface Criterion { + id?: string; + stableKey?: string; + desc?: string; + statedBy?: string; + points?: number; + evidence?: unknown; +} + +interface GradeFeature { + name?: string; + consoleErrors?: string[]; + criteria?: Criterion[]; +} + +interface GradePayload { + features?: GradeFeature[]; +} + +interface ContractResult { + id: string; + status: string; + detail?: string; +} + +interface ContractLintPayload { + results?: ContractResult[]; +} + +interface GradeBundlePayload { + backend?: string; + outcome?: { kind?: string; phase?: string; reason?: string }; +} + +interface RepairBug { + area: string; + actor: string | null; + action: string | null; + expected: string; + observed: string; + consoleErrors: string[]; + contract: boolean; +} + +// The public verb for the step that failed. Control and action names are the +// agent's own vocabulary; nothing else about the step is repeated. +function failedAction(action: string | undefined, finding: Finding | null): string | null { + if (action === 'fill') { + return finding?.kind === 'choice-missing' ? 'Select the requested choice' : 'Enter the requested value'; + } + if (action === 'click') return 'Use the requested control'; + if (action === 'signIn') return 'Sign in'; + if (action === 'signUp') return 'Create the account'; + if (action === 'reload') return 'Reload the page'; + return null; +} + +// What the application did, from the finding alone. A failure without a +// finding (the feature's setup failed before this behavior was reached) +// says so and nothing more. +function observed(finding: Finding | null, phase: string): string { + if (finding) return renderFinding(finding); + return phase === 'setup' + ? 'the application did not reach this behavior; an earlier step of the same feature failed' + : 'the application did not do this'; +} + +export function parseReportBugsArgs(argv: string[]): ReportBugsArgs { + const { values } = parseNodeArgs({ args: argv.slice(2), options: { + app: { type: 'string' }, results: { type: 'string' }, out: { type: 'string' }, + archive: { type: 'string' }, + 'history-json': { type: 'string' }, 'checks-json': { type: 'string' }, + 'controls-json': { type: 'string' }, + 'prior-regression': { type: 'string' }, + 'regression-context': { type: 'boolean' }, + } }); + const args: ParsedArgs = { app: values.app, results: values.results, out: values.out, + archive: values.archive, + history: values['history-json'] === undefined ? undefined : JSON.parse(values['history-json']), + checks: values['checks-json'] === undefined ? undefined : JSON.parse(values['checks-json']), + controls: values['controls-json'] === undefined ? undefined : JSON.parse(values['controls-json']) }; + if (!args.app) { + throw new Error('Usage: report-bugs --app [--out ]'); + } + args.results ??= join(args.app, 'stack-bench'); + args.out ??= join(args.app, CODING_CONTAINER_BUG_REPORT_FILE); + args.history ??= []; + if (!Array.isArray(args.history)) throw new Error('--history-json must contain an array'); + args.checks ??= null; + if (args.checks !== null && (!Array.isArray(args.checks) + || args.checks.some(check => typeof check !== 'string' || !check) + || new Set(args.checks).size !== args.checks.length)) { + throw new Error('--checks-json must contain distinct non-empty strings'); + } + args.controls ??= null; + if (args.controls !== null && (!Array.isArray(args.controls) + || args.controls.some(control => typeof control !== 'string' || !control) + || new Set(args.controls).size !== args.controls.length)) { + throw new Error('--controls-json must contain distinct non-empty strings'); + } + return { app: args.app, results: args.results, out: args.out, archive: args.archive, + history: args.history as RepairHistoryEntry[], checks: args.checks as string[] | null, + controls: args.controls as string[] | null, + priorRegression: values['prior-regression'] ?? null, + regressionContext: values['regression-context'] ?? false }; +} + +function priorRegressionSection(path: string): string[] { + const details = assertAgentVisibleText(readFileSync(resolve(path), 'utf8')).trim() + .replace(/^### /gm, '#### ') + .replace(/^## /gm, '### '); + if (!details) throw new Error('prior regression report has no failure details'); + return [ + '## Previous repair regression', + '', + 'The previous repair was rolled back because it broke behavior that already worked.', + 'Keep this behavior working while you fix the current problems.', + '', + ...details.split(/\r?\n/), + '', + ]; +} + +export function createBugReport(args: ReportBugsArgs): number { + const resultsDir = resolve(args.results); + if (!existsSync(resultsDir)) throw new Error(`No grading results in ${resultsDir}`); + + const bugs: RepairBug[] = []; + const selectedChecks = args.checks === null ? null : new Set(args.checks); + const selectedControls = args.controls === null ? null : new Set(args.controls); + + for (const file of readdirSync(resultsDir).filter(name => /^grading-.*\.json$/.test(name))) { + const report = readArtifactPayload(join(resultsDir, file), { expectedKind: 'grade' }); + for (const feature of report.features ?? []) { + // Repairs receive only scored, typed application failures. + for (const criterion of feature.criteria ?? []) { + if (selectedChecks && (!criterion.stableKey + || !selectedChecks.has(criterion.stableKey))) continue; + if (!(Number(criterion.points) > 0)) continue; + const evidence = criterionEvidence(criterion); + if (!evidenceIsRepairable(evidence)) continue; + const actionEntry = evidence.actions.at(-1); + const actionId = actionEntry && typeof actionEntry.evidence === 'object' && actionEntry.evidence + ? String((actionEntry.evidence as { action?: { id?: string } }).action?.id ?? '') : undefined; + const expected = (criterion.statedBy ?? criterion.desc ?? '').trim() || 'the requested behavior'; + bugs.push({ + area: sanitiseDiagnostic(feature.name, 120), + actor: sanitiseDiagnostic(actionEntry?.actor ?? evidence.actor, 120) || null, + action: failedAction(actionId, evidence.finding), + expected, + observed: observed(evidence.finding, evidence.phase), + consoleErrors: (feature.consoleErrors ?? []).slice(0, 3) + .map(sanitiseConsoleError).filter(Boolean), + contract: false, + }); + } + } + } + + // Contract failures are separate because the interface name is itself the public + // requirement here. Behavioral failures above must never expose one. + const lintPath = join(resultsDir, ARTIFACT_FILE.contractLint); + if (existsSync(lintPath)) { + const lint = readArtifactPayload(lintPath, { expectedKind: 'contract_lint' }); + for (const result of (lint.results ?? []).filter(item => item.status === 'FAIL' + && (!selectedControls || selectedControls.has(item.id)))) { + bugs.push({ + area: 'Application interface', + actor: null, + action: null, + expected: `A visible element for "${(result.detail ?? '').split('expected: ').pop()}" must use the "${result.id}" application interface`, + observed: sanitiseDiagnostic(result.detail + ?? `no visible element with id="${result.id}" was found after a clean reset`, 500), + consoleErrors: [], contract: true, + }); + } + } + + const bundlePath = join(resultsDir, ARTIFACT_FILE.gradeBundle); + if (existsSync(bundlePath)) { + const bundle = readArtifactPayload(bundlePath, { expectedKind: 'grade_bundle' }); + if (bundle.outcome?.kind === 'app_failure' && bundle.outcome.reason) { + const expectedByPhase: Record = { + 'database-provenance': `The app must use the ${bundle.backend} database and connection supplied for this run.`, + 'application-layout': 'The app must use a project layout that can be built, started, and reset repeatedly.', + 'application-restart': `The app must provide ${CODING_CONTAINER_START_SCRIPT}. From clean source, it must install dependencies, build, and start the complete application without changing source files.`, + }; + const expected = expectedByPhase[bundle.outcome.phase ?? ''] + ?? 'The app must start successfully in the supplied environment.'; + bugs.unshift({ + area: 'Application setup', + actor: null, + action: null, + expected, + observed: sanitiseDiagnostic(bundle.outcome.reason, 500), + consoleErrors: [], + contract: false, + }); + } + } + + if (bugs.length === 0) { + console.log('No failures — no bug report written.'); + return 3; + } + + const behavioral = bugs.filter(bug => !bug.contract); + const contractFailures = bugs.filter(bug => bug.contract); + const lines = args.regressionContext ? [] : [ + '# Bug Report', + '', + 'The application has these problems after a clean database reset and a fresh', + 'restart. Fix the behavior, then redeploy.', + 'Do not change behavior that is already correct. A result from existing local', + 'state does not replace the clean result below.', + '', + ]; + + if (!args.regressionContext && args.history.length) { + lines.push('## Earlier work', ''); + lines.push('Earlier changes did not fix the current problems. Use the current source as', + 'the starting point. Do not repeat an earlier approach only because it appeared', + 'to work with existing local state.', ''); + } + + if (behavioral.length) { + lines.push('## Behavior', ''); + behavioral.forEach((bug, index) => { + lines.push(`### Bug ${index + 1}: ${bug.area}`, ''); + if (bug.actor) lines.push(`**Actor/session:** ${bug.actor}`, ''); + if (bug.action) lines.push(`**Failed action:** ${bug.action}`, ''); + lines.push(`**Expected:** ${bug.expected}`, ''); + lines.push(`**Actual:** ${bug.observed}`, ''); + if (bug.consoleErrors.length) { + lines.push('**Console or network errors:**', ''); + bug.consoleErrors.forEach(error => lines.push(`- \`${error}\``)); + lines.push(''); + } + }); + } + + if (contractFailures.length) { + lines.push('## Application interface', ''); + lines.push('These required elements were not available in the clean application state:', ''); + contractFailures.forEach(bug => { + lines.push(`- **Expected:** ${bug.expected}`); + lines.push(` **Actual:** ${bug.observed}`); + }); + lines.push(''); + } + + if (args.priorRegression) lines.push(...priorRegressionSection(args.priorRegression)); + + const reportText = assertAgentVisibleText(lines.join('\n')); + writeFileSync(args.out, reportText); + if (args.archive) { + mkdirSync(dirname(args.archive), { recursive: true }); + writeFileSync(args.archive, reportText); + } + console.log(`Wrote ${bugs.length} bug(s) to ${args.out}`); + return 0; +} + +function main(): void { + try { + process.exitCode = createBugReport(parseReportBugsArgs(process.argv)); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 2; + } +} + +if (process.argv[1] && pathToFileURL(resolve(process.argv[1])).href === import.meta.url) main(); diff --git a/tools/stack-bench/commands/reset-backend.ts b/tools/stack-bench/commands/reset-backend.ts new file mode 100644 index 00000000000..c2815d3b103 --- /dev/null +++ b/tools/stack-bench/commands/reset-backend.ts @@ -0,0 +1,24 @@ +#!/usr/bin/env node + +import { GENERATED_APP_LAYOUT_EXIT_CODE, resetBackend } from '../src/stacks/backend-reset.js'; +import { GeneratedAppLayoutError } from '../src/runtime/spacetime-layout.js'; +import { redactCredentials } from '../src/evidence/diagnostic-sanitizer.js'; + +const [backend, app] = process.argv.slice(2); +if (!backend || !app) throw new Error('usage: node dist/commands/reset-backend.js '); + +Promise.resolve().then(() => resetBackend({ backend, app })).then(result => { + console.log(result); +}).catch(error => { + if (error instanceof GeneratedAppLayoutError || error?.code === 'generated_app_layout') { + console.error(`GENERATED_APP_LAYOUT: ${error.message}`); + process.exitCode = GENERATED_APP_LAYOUT_EXIT_CODE; + return; + } + const childOutput = [error?.stderr, error?.stdout] + .filter(value => value !== undefined && value !== null && String(value).trim()) + .map(value => String(value).trim()).join('\n'); + if (childOutput) console.error(redactCredentials(childOutput).slice(-2000)); + console.error(error.stack ?? error.message); + process.exitCode = 1; +}); diff --git a/tools/stack-bench/commands/run-suite.ts b/tools/stack-bench/commands/run-suite.ts new file mode 100644 index 00000000000..9dfd4342a26 --- /dev/null +++ b/tools/stack-bench/commands/run-suite.ts @@ -0,0 +1,1153 @@ +#!/usr/bin/env node + +import { execFile, execFileSync } from 'node:child_process'; +import type { ExecFileException, ExecFileSyncOptionsWithStringEncoding } from 'node:child_process'; +import { randomUUID } from 'node:crypto'; +import { readFileSync, writeFileSync, mkdirSync, existsSync, readdirSync, rmSync } from 'node:fs'; +import { join, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { setTimeout as sleep } from 'node:timers/promises'; +import { parseArgs as parseNodeArgs, promisify } from 'node:util'; +import { chromium } from 'playwright'; +import type { BrowserServer } from 'playwright'; +import { dbName, loadTrack, suitesFor, DEFAULT_TRACK } from '../src/composition/tracks.js'; +import { controlBackendRuntime, parseRuntimeControlSpec } + from '../src/runtime/backend-control.js'; +import type { RuntimeControlSpec } from '../src/runtime/backend-control.js'; +import { ARTIFACT_FILE, readArtifactPayload, recipeArtifactIdentities, writeArtifact } + from '../src/evidence/artifacts.js'; +import { bundleRecipeRelease, resolveRecipeRelease } from '../src/composition/recipe-release.js'; +import { createBoundRecipeTaskRequest, resolveBoundRecipeTaskRequest } from '../src/composition/recipe-selection.js'; +import { contractInterfaceNames } from '../src/composition/agent-visible-contract.js'; +import { resolveCalibrationForRelease } from '../src/composition/calibration-compiler.js'; +import { criterionEvidence, evidencePassed } from '../src/evidence/check-evidence.js'; +import { renderEvidenceConsoleLine } from '../src/evidence/evidence-presentation.js'; +import { STACK_ADAPTER_REGISTRY } from '../src/stacks/stack-adapters.js'; +import { requireLeasedDatabase, requireLeasedSpacetime } from '../src/stacks/backend-reset-guard.js'; +import { aggregatePackRuntime, exceededPackBudgets } from '../src/composition/pack-runtime.js'; +import { hashAppSource } from '../src/runtime/source-snapshot.js'; +import { GENERATED_APP_LAYOUT_EXIT_CODE } from '../src/stacks/backend-reset.js'; +import { readBackendLease } from '../src/runtime/backend-lease.js'; +import { redactCredentials } from '../src/evidence/diagnostic-sanitizer.js'; +import { canonicalDefinitionJson } from '../src/composition/definition-plan.js'; +import { sha256 } from '../src/evidence/provenance.js'; +import { GRADER_SOURCE_TIMEOUT_MS } from '../src/runtime/grading-timeout.js'; +import type { BackendLease, BackendLeaseExpectation } from '../src/runtime/backend-lease.js'; +import type { CheckEvidence } from '../src/evidence/check-evidence.js'; +import type { AggregatedPackRuntimeEvidence, PackRuntimeEvidence } from '../src/composition/pack-runtime.js'; +import { isModularRecipeTaskRequest } from '../src/composition/recipe-selection.js'; +import type { BoundRecipeTaskRequestResult, RecipeSelection } from '../src/composition/recipe-selection.js'; +import type { RecipeBinding, RecipeCheck } from '../src/composition/recipe-release.js'; +import type { Track, TrackSuite } from '../src/composition/tracks.js'; + +import { STACK_BENCH_ROOT as ROOT, compiledEntrypoint } from '../src/package-root.js'; +const RESET = compiledEntrypoint('commands', 'reset-backend.js'); + +type Observation = 'scored' | 'observed'; +type Selection = { + schemaVersion: number; + recipe: { id: string; version: string; contentSha256: string }; + requested: RecipeSelection['requested']; + sha256: string; + checks: RecipeCheck[]; + scoredPoints: number; + observedChecks?: Array; + observedPoints?: number; + evaluationSha256?: string; + regressionChecks?: Array; + regressionPoints?: number; + observation?: Observation; +}; +type DeclaredSuite = TrackSuite; +type Failure = Error & { stdout?: string; stderr?: string; status?: number | null; signal?: string | null; + code?: string }; +type FailureDetail = { message?: unknown; stderr?: unknown } | null; +type RecipeTaskArgument = { recipe: { id: string; version: string; contentSha256?: string } } & Record; +type RunArguments = { + app: string; + url: string; + backend: string; + label: string; + out: string; + level: string; + reset: boolean; + media: boolean; + runIndex: number; + track: string; + packIds: string[]; + checkKeys: string[]; + observation: Observation; + recipe?: string; + recipeTask?: RecipeTaskArgument; + credentialAliases?: unknown; + regressionChecks: string[]; + sourceSha256?: string; + restartSpec?: RuntimeControlSpec; + applicationFailure?: ApplicationFailure; + parentAttemptId?: string; + databaseLease?: BackendLease | null; + browserWsEndpoint?: string; + selection?: Selection | null; + bundleArtifactId: string; +}; +type GradeCriterion = { id: string; stableKey?: string; serverCheck?: string; evidence?: CheckEvidence }; +type GradeFeature = { name: string; criteria: GradeCriterion[]; + cleanupEvidence?: { failures: Array<{ stage: string }> } }; +type GradePayload = { total: number; max: number; features: GradeFeature[]; + selection?: { checks?: RecipeCheck[] }; packRuntime?: PackRuntimeEvidence }; +type LintPayload = { + pass: boolean; + counts: { pass: number; fail: number; blocked: number; scenario: number }; +}; +type ActionsPayload = { missing: string[]; results: unknown[] }; +type RuntimeProvenance = { ok: boolean | null; verified: boolean; reason: string }; +type ApplicationProbeResult = { ok: boolean; detail: string | null }; +type ResetOutcome = { kind: string; phase: string; appFailures?: string[] }; +type ApplicationFailure = ResetOutcome & { kind: 'app_failure'; reason: string }; +type DatabaseProvenance = { ok: boolean; reason: string; url?: string }; +type GradeLeaseReader = typeof readBackendLease; +type MutationDirectoryEntry = { name: string; isDirectory(): boolean; isFile(): boolean }; +type MutationDirectoryReader = (path: string, options: { withFileTypes: true }) => readonly MutationDirectoryEntry[]; +type ProbeResponse = { ok: boolean; status: number }; +type ApplicationFetch = (url: string, init: { signal: AbortSignal }) => Promise; +type DatabaseProvenanceDefinition = Track['databaseProvenance']; +type DatabaseNameLease = { resources: { database?: string | null } }; +type ProvenanceFetch = (url: string, init: { method: string; headers: Record; + body: string; signal: AbortSignal }) => Promise<{ ok: boolean; status: number }>; +type ProvenanceWrite = { ok: true; marker: string } | { ok: false; marker: null; reason: string }; +type ApplicationFailureSelection = { checks: Array<{ executionId: string; points?: number }> }; +type ContractLintArguments = Pick; +type BundleSelection = Selection & { attemptedChecks: string[]; reportedChecks: string[]; + notRun: Array<{ stableKey: string; reason: string }> }; +type Bundle = { + definitionSchemaVersion: number; + recipeRelease: ReturnType; + calibration: { id: string; version: string; state: string; contentSha256: string } | null; + label: string; track: string; backend: string; url: string; app: string; level: number; + observation: Observation; source?: { sha256: string }; + suites: Record; + totals: Record; + selection: BundleSelection | null; + code?: ReturnType; + error?: string; + outcome?: { kind: string; phase: string; reason?: string; appFailures?: string[] }; + provenance?: DatabaseProvenance & { runtime?: RuntimeProvenance }; + actions?: ActionsPayload | null; + packRuntime?: AggregatedPackRuntimeEvidence; +}; + +const isRecord = (value: unknown): value is Record => + value !== null && typeof value === 'object' && !Array.isArray(value); + +const parseObservation = (value: string): Observation => { + if (value === 'scored' || value === 'observed') return value; + throw new Error('--observation must be scored or observed'); +}; + +export function suitesForRecipe(track: Track, binding: RecipeBinding): DeclaredSuite[] { + if (!binding?.execution?.length) throw new Error('recipe has no typed execution plan'); + return binding.execution.map(entry => ({ + id: entry.id, + spec: resolve(track.dir, entry.source ?? ''), + ...(entry.ownership.kind === 'inherited' + ? { inherited: true, fromLevel: entry.ownership.fromLevel } + : {}), + })); +} + +export function childFailureDetail(failure: FailureDetail = null, stdout = '', limit = 600): string { + const processOutput = [failure?.stderr, stdout] + .filter(value => value !== undefined && value !== null && String(value).trim()) + .join('\n').trim(); + const diagnostic = processOutput || String(failure?.message ?? '').trim(); + const lines = diagnostic.split(/\r?\n/).map(line => line.trim()).filter(Boolean); + if (!lines.length) return ''; + const punctuationOnly = (line: string) => + [...line].every(character => '[]{},'.includes(character)); + const noise = (line: string) => line.startsWith('at ') || /^Node\.js v/.test(line) + || /^node:internal\//.test(line) || /^\^+$/.test(line) || punctuationOnly(line); + const cause = lines.find(line => !noise(line) && /(?:error|failed|timeout|closed|econn|killed)/i.test(line)) + ?? lines.find(line => !noise(line)) ?? lines[0]; + const selected = [cause, ...lines.slice(-4)].filter((line, index, all) => all.indexOf(line) === index); + return selected.join(' | ').slice(0, limit); +} + +export function resetFailureOutcome(error: unknown): ResetOutcome { + const failure = isRecord(error) ? error : {}; + return failure.status === GENERATED_APP_LAYOUT_EXIT_CODE + ? { kind: 'app_failure', phase: 'application-layout', + appFailures: ['application-layout'] } + : failure.code === 'generated_app_not_restartable' + ? { kind: 'app_failure', phase: 'application-restart', + appFailures: ['application-restart'] } + : { kind: 'harness_failure', phase: 'database-reset' }; +} + +export function applicationFailureTotals(selection: ApplicationFailureSelection | null | undefined, + declaredSuites: Array>): Record { + if (!selection?.checks?.length) return {}; + const inherited = new Set(declaredSuites.filter(suite => suite.inherited).map(suite => suite.id)); + const currentMax = selection.checks.filter(check => !inherited.has(check.executionId)) + .reduce((total, check) => total + Number(check.points ?? 0), 0); + const regressionMax = selection.checks.filter(check => inherited.has(check.executionId)) + .reduce((total, check) => total + Number(check.points ?? 0), 0); + return { score: 0, max: currentMax, dirty: false, contractPass: null, + regression: regressionMax ? { score: 0, max: regressionMax } : null }; +} + +export function clearPreviousGradeOutputs(output: string): void { + const generated = existsSync(output) ? readdirSync(output).filter(name => + /^grading-.+\.json$/.test(name) || /^grader-.+\.(?:stdout|stderr)\.log$/.test(name)) : []; + for (const name of [ARTIFACT_FILE.gradeBundle, ARTIFACT_FILE.contractLint, + ARTIFACT_FILE.actions, 'media', 'failure-media', + 'database-provenance', ...generated]) { + rmSync(join(output, name), { recursive: true, force: true }); + } +} + +function recordGraderChildResult(output: string, suiteId: string, + result: { stdout?: unknown; stderr?: unknown; failure?: Error | null }) { + const stdout = redactCredentials(String(result.stdout ?? '')); + const stderr = redactCredentials(String(result.stderr ?? '')); + const safeId = String(suiteId).replace(/[^A-Za-z0-9._-]/g, '_'); + const stdoutName = `grader-${safeId}.stdout.log`; + const stderrName = `grader-${safeId}.stderr.log`; + writeFileSync(join(output, stdoutName), stdout); + writeFileSync(join(output, stderrName), stderr); + const failure = result.failure ?? null; + if (failure) Object.assign(failure, { stdout, stderr }); + return { stdout, stderr, failure, stdoutName, stderrName }; +} + +const execFileAsync = promisify(execFile); + +export async function runGraderChild(argv: string[], output: string, suiteId: string) { + try { + const result = await execFileAsync(process.execPath, argv, { encoding: 'utf8', cwd: ROOT, + timeout: COMMAND_TIMEOUT_MS, maxBuffer: 64 * 1024 * 1024 }); + return recordGraderChildResult(output, suiteId, result); + } catch (error) { + const failure = error instanceof Error ? error : new Error(String(error)); + const processFailure = error as ExecFileException & { stdout?: unknown; stderr?: unknown }; + return recordGraderChildResult(output, suiteId, { + stdout: processFailure.stdout, stderr: processFailure.stderr, failure, + }); + } +} + +function gradeLeaseInput(backend: string, env: NodeJS.ProcessEnv): { path: string; + expected: BackendLeaseExpectation } | null { + if (!['mongodb', 'postgres', 'spacetime'].includes(backend)) return null; + const path = String(env.STACK_BENCH_LEASE ?? '').trim(); + const token = String(env.STACK_BENCH_LEASE_TOKEN ?? '').trim(); + if (!path && !token) return null; + if (!path || !token) throw new Error('database grading requires both lease path and lease token'); + return { path, expected: { token, backend, active: true } }; +} + +export function databaseLeaseForGrading(backend: string, env = process.env, { + readLease = readBackendLease, +}: { readLease?: GradeLeaseReader } = {}) { + const input = gradeLeaseInput(backend, env); + if (!input) return null; + const lease = readLease(input.path, input.expected); + if (backend === 'spacetime') { + if (!lease.resources.module || !lease.resources.serverUri) { + throw new Error('active spacetime lease has no complete module target'); + } + return lease; + } + const container = String(lease.resources?.container?.name ?? '').trim(); + const containerId = String(lease.resources?.container?.id ?? '').trim(); + if (!container || !containerId) { + throw new Error(`active ${backend} lease has no complete database container identity`); + } + return lease; +} + +export function databaseNameForGrading(track: Pick, runIndex: number, + lease: DatabaseNameLease | null = null): string { + if (!lease) return dbName(track, runIndex); + const database = String(lease.resources?.database ?? '').trim(); + if (!database) throw new Error('active database lease has no database name'); + return database; +} + +function parseArgs(argv: string[]): RunArguments { + const { values } = parseNodeArgs({ args: argv.slice(2), options: { + app: { type: 'string' }, url: { type: 'string' }, backend: { type: 'string' }, + label: { type: 'string' }, out: { type: 'string' }, level: { type: 'string' }, + recipe: { type: 'string' }, 'recipe-task-json': { type: 'string' }, + 'credential-aliases-json': { type: 'string' }, 'regression-checks-json': { type: 'string' }, + observation: { type: 'string' }, 'source-sha256': { type: 'string' }, + 'no-media': { type: 'boolean' }, track: { type: 'string' }, + pack: { type: 'string', multiple: true }, check: { type: 'string', multiple: true }, + 'restart-spec': { type: 'string' }, 'application-failure-json': { type: 'string' }, + 'run-index': { type: 'string' }, 'no-reset': { type: 'boolean' }, + 'parent-attempt-id': { type: 'string' }, + } }); + const a: RunArguments = { app: values.app ?? '', url: values.url ?? '', + backend: values.backend ?? '', label: values.label ?? '', out: values.out ?? '', + level: values.level ?? '1', reset: !(values['no-reset'] ?? false), + media: !(values['no-media'] ?? false), runIndex: Number(values['run-index'] ?? 0), + track: values.track ?? DEFAULT_TRACK, + packIds: (values.pack ?? []).flatMap(value => value.split(',').filter(Boolean)), + checkKeys: (values.check ?? []).flatMap(value => value.split(',').filter(Boolean)), + observation: parseObservation(values.observation ?? 'scored'), + recipe: values.recipe, + recipeTask: values['recipe-task-json'] === undefined ? undefined : JSON.parse(values['recipe-task-json']), + credentialAliases: values['credential-aliases-json'] === undefined + ? undefined : JSON.parse(values['credential-aliases-json']), + regressionChecks: values['regression-checks-json'] === undefined + ? [] : JSON.parse(values['regression-checks-json']), + sourceSha256: values['source-sha256'], + restartSpec: values['restart-spec'] === undefined + ? undefined : parseRuntimeControlSpec(JSON.parse(values['restart-spec'])), + applicationFailure: values['application-failure-json'] === undefined + ? undefined : JSON.parse(values['application-failure-json']), + parentAttemptId: values['parent-attempt-id'], bundleArtifactId: '' }; + if (!a.app || !a.url || !a.backend || !a.label) { + console.error('Usage: node dist/commands/run-suite.js --app --url --backend --label [--out ] [--media] [--no-reset]'); + process.exit(2); + } + if (!['scored', 'observed'].includes(a.observation)) { + throw new Error('--observation must be scored or observed'); + } + if (a.observation === 'observed' && !/^[a-f0-9]{64}$/.test(a.sourceSha256 ?? '')) { + throw new Error('observed specifications require --source-sha256'); + } + if (a.sourceSha256 !== undefined && !/^[a-f0-9]{64}$/.test(a.sourceSha256)) { + throw new Error('--source-sha256 must be a SHA-256 digest'); + } + if (a.applicationFailure && (a.applicationFailure.kind !== 'app_failure' + || typeof a.applicationFailure.phase !== 'string' || !a.applicationFailure.phase + || typeof a.applicationFailure.reason !== 'string' || !a.applicationFailure.reason)) { + throw new Error('--application-failure-json must describe an application failure'); + } + a.out ||= join(a.app, 'stack-bench'); + if (!Array.isArray(a.regressionChecks) + || a.regressionChecks.some(key => typeof key !== 'string' || !key)) { + throw new Error('--regression-checks-json must contain stable check keys'); + } + return a; +} + +export function selectObservationScope(selectedTask: BoundRecipeTaskRequestResult | null, + observation: Observation = 'scored'): Selection | null { + if (observation === 'scored') return selectedTask?.selection ?? null; + if (observation !== 'observed') throw new Error(`unknown observation scope ${observation}`); + if (!selectedTask || !isModularRecipeTaskRequest(selectedTask)) { + throw new Error('observed specifications require a modular schema-3 task request'); + } + const selection = selectedTask.selection; + if (!selection.observedChecks.length) throw new Error('observed specification scope is empty'); + return { + ...selection, + observation: 'observed', + checks: selection.observedChecks, + scoredPoints: 0, + observedPoints: selection.observedChecks.reduce((total, check) => total + check.points, 0), + }; +} + +export function attachRegressionScope(selection: Selection | null, recipeBinding: RecipeBinding | null, + declaredSuites: DeclaredSuite[], stableKeys: string[] = []): Selection | null { + if (!stableKeys.length) return selection; + if (!selection || !recipeBinding?.release?.checkCatalog) { + throw new Error('regression checks require a recipe-bound scored selection'); + } + const uniqueKeys = [...new Set(stableKeys)]; + if (uniqueKeys.length !== stableKeys.length) throw new Error('regression checks contain duplicates'); + const currentKeys = new Set(selection.checks.map(check => check.stableKey)); + const catalog = new Map(recipeBinding.release.checkCatalog + .map(check => [check.stableKey, check])); + const inheritedSuites = new Set(declaredSuites.filter(suite => suite.inherited) + .map(suite => suite.id)); + const regressionChecks = uniqueKeys.map(key => { + if (currentKeys.has(key)) throw new Error(`regression check ${key} is already in the current score`); + const check = catalog.get(key); + if (!check) throw new Error(`regression check ${key} is absent from the cumulative recipe`); + if (!inheritedSuites.has(check.executionId)) { + throw new Error(`regression check ${key} does not belong to an inherited execution`); + } + return { ...check, treatment: check.treatment ?? 'regression' }; + }); + const evaluationDocument = { schemaVersion: 1, selectionSha256: selection.sha256, + regressionChecks: uniqueKeys.slice().sort() }; + return { + ...selection, + checks: [...selection.checks, ...regressionChecks], + regressionChecks: regressionChecks.map(check => ({ ...check, treatment: check.treatment ?? 'regression' })), + regressionPoints: regressionChecks.reduce((total, check) => total + check.points, 0), + evaluationSha256: sha256(Buffer.from(canonicalDefinitionJson(evaluationDocument))), + }; +} + +const COMMAND_TIMEOUT_MS = GRADER_SOURCE_TIMEOUT_MS; +const run = (cmd: string, args: readonly string[], opts: Omit = {}): string => + execFileSync(cmd, args, { + encoding: 'utf8', stdio: 'pipe', cwd: ROOT, timeout: COMMAND_TIMEOUT_MS, ...opts, + }); + +export async function verifyApplicationProbe(url: string, { + fetchImpl = fetch, timeoutMs = 5000, +}: { fetchImpl?: ApplicationFetch; timeoutMs?: number } = {}): Promise { + let response; + try { + response = await fetchImpl(url, { signal: AbortSignal.timeout(timeoutMs) }); + } catch (error) { + return { ok: false, + detail: `application did not respond: ${error instanceof Error ? error.message : String(error)}` }; + } + if (!response.ok) { + return { ok: false, detail: `application returned HTTP ${response.status}` }; + } + return { ok: true, detail: null }; +} + +export async function waitForApplicationProbe(url: string, { + attempts = 9, intervalMs = 250, probeTimeoutMs = 1000, + probe = verifyApplicationProbe, sleepImpl = sleep, +}: { attempts?: number; intervalMs?: number; probeTimeoutMs?: number; + probe?: typeof verifyApplicationProbe; + sleepImpl?: (ms: number) => Promise } = {}): Promise { + if (!Number.isInteger(attempts) || attempts < 1) { + throw new Error('application probe attempts must be a positive integer'); + } + let result = null; + for (let attempt = 1; attempt <= attempts; attempt += 1) { + result = await probe(url, { timeoutMs: probeTimeoutMs }); + if (result.ok || attempt === attempts) return result; + await sleepImpl(intervalMs); + } + return result ?? { ok: false, detail: 'application readiness probe did not run' }; +} + +// Confirm the app uses the database leased to this run. +export function checkDatabaseProvenance(args: Pick): DatabaseProvenance { + const adapter = STACK_ADAPTER_REGISTRY.get(args.backend); + const expected = adapter.ports.allocations().db; + if (!expected) return { ok: true, reason: 'no external database for this backend' }; + // Neutral guidance does not prescribe project layout. Search the app for the + // connection string instead of assuming it is in server/.env. + const urls: string[] = []; + let usesLeasedEnvironment = false; + const walk = (dir: string): void => { + if (!existsSync(dir)) return; + for (const e of readdirSync(dir, { withFileTypes: true })) { + if (/^(node_modules|dist|\.vite|\.git|module_bindings)$/.test(e.name)) continue; + const p = join(dir, e.name); + if (e.isDirectory()) { walk(p); continue; } + if (!/\.(env|ts|tsx|js|mjs|json|yaml|yml)$|^\.env/.test(e.name)) continue; + try { + const text = readFileSync(p, 'utf8'); + if (/process\.env(?:\.DATABASE_URL|\[['"]DATABASE_URL['"]\])/.test(text)) { + usesLeasedEnvironment = true; + } + urls.push(...adapter.agent.findDatabaseUrls({ text })); + } catch { /* unreadable file proves nothing */ } + } + }; + walk(args.app); + if (usesLeasedEnvironment) { + return { ok: true, url: 'process.env.DATABASE_URL', + reason: 'app reads the database URL supplied by its authenticated backend lease' }; + } + if (!urls.length) return { ok: false, + reason: 'app neither reads process.env.DATABASE_URL nor contains a database connection string' }; + const matchesExpectedPort = (value: string): boolean => { + try { return Number(new URL(value).port) === Number(expected); } + catch { return false; } + }; + const ok = urls.some(matchesExpectedPort); + return { ok, url: urls[0], + reason: ok ? 'ok' : `app targets ${urls[0]} but the benchmark database is on port ${expected}` }; +} + +export async function writeApplicationDatabaseMarker( + args: Pick, + track: Pick, + definition: DatabaseProvenanceDefinition, + fetchImpl: ProvenanceFetch = fetch, +): Promise { + if (!definition) throw new Error('track does not define runtime database provenance'); + const action = track.actions.find(candidate => candidate.id === definition.action); + if (!action) throw new Error(`database provenance action is not declared: ${definition.action}`); + const marker = `sb${randomUUID().replaceAll('-', '').slice(0, 16)}`; + const body = { ...definition.body, [definition.markerParameter]: marker }; + const adapter = STACK_ADAPTER_REGISTRY.get(args.backend); + const request = adapter.namedAction.request({ action, + input: { values: body }, + url: args.url, + spacetime: args.backend === 'spacetime' ? adapter.grading.context() : null }); + if (!request.url) throw new Error('database provenance action has no URL'); + try { + const response = await fetchImpl(request.url, { + method: request.method, + headers: { 'Content-Type': 'application/json' }, + body: request.body, + signal: AbortSignal.timeout(15_000), + }); + return response.ok ? { ok: true, marker } + : { ok: false, marker: null, + reason: `application provenance action returned HTTP ${response.status}` }; + } catch (error) { + return { ok: false, marker: null, + reason: `application provenance action failed: ${error instanceof Error ? error.message : String(error)}` }; + } +} + +export function databaseProvenanceFailure(error: unknown): { kind: string; phase: string; reason: string } { + return { kind: 'harness_failure', phase: 'database-provenance', + reason: `runtime database provenance failed: ${error instanceof Error ? error.message : String(error)}` }; +} + +export function checkRuntimeDatabaseProvenance(args: Pick, + marker: string | null = null): RuntimeProvenance { + const adapter = STACK_ADAPTER_REGISTRY.get(args.backend); + if (!('proveUse' in adapter.database)) { + return { ok: null, verified: false, + reason: 'exact runtime database marker proof is not implemented for this stack' }; + } + if (!args.databaseLease) { + return { ok: null, verified: false, + reason: 'standalone grading has no authenticated database lease' }; + } + if (typeof marker !== 'string' || !marker) { + return { ok: null, verified: false, + reason: 'the application action did not produce a database marker' }; + } + if (args.backend === 'spacetime') { + return STACK_ADAPTER_REGISTRY.get('spacetime').database.proveUse( + { lease: requireLeasedSpacetime(args.databaseLease), marker }); + } + const lease = requireLeasedDatabase(args.databaseLease); + return args.backend === 'mongodb' + ? STACK_ADAPTER_REGISTRY.get('mongodb').database.proveUse({ lease, marker }) + : STACK_ADAPTER_REGISTRY.get('postgres').database.proveUse({ lease, marker }); +} + +function isGradePayload(value: GradePayload | LintPayload | null | undefined): value is GradePayload { + return value !== null && value !== undefined && 'total' in value && 'max' in value; +} + +// Report the application size and direct runtime dependency count. +export function codeMetrics(args: Pick): { serverLoc: number; serverFiles: number; + totalLoc: number; totalFiles: number; runtimeDeps: number } { + // Minimal-guidance apps may place server code outside the conventional directory. + const adapter = STACK_ADAPTER_REGISTRY.get(args.backend); + const conventional = adapter.agent.serverDirectory; + const SERVER_DIR = existsSync(join(args.app, conventional)) ? conventional : '.'; + const walk = (dir: string, out: string[] = []): string[] => { + if (!existsSync(dir)) return out; + for (const e of readdirSync(dir, { withFileTypes: true })) { + if (/^(node_modules|dist|\.vite|module_bindings|drizzle)$/.test(e.name)) continue; + const p = join(dir, e.name); + if (e.isDirectory()) walk(p, out); + // Count every supported JavaScript and TypeScript source extension. + else if (/\.(ts|tsx|js|jsx|mjs|cjs)$/.test(e.name)) out.push(p); + } + return out; + }; + const count = (files: string[]): number => files.reduce((n, f) => n + readFileSync(f, 'utf8').split('\n').length, 0); + // With no conventional server directory, "server" is everything that is not + // the client — otherwise the fallback counts the client twice and serverLoc + // equals totalLoc, which reads as a much larger backend than was written. + const allFiles = walk(args.app); + const serverFiles = SERVER_DIR === '.' + ? allFiles.filter(f => !/[\\/]client[\\/]/.test(f)) + : walk(join(args.app, SERVER_DIR)); + + let deps = 0; + const packageFiles = new Set([ + resolve(args.app, 'package.json'), + resolve(args.app, SERVER_DIR, 'package.json'), + resolve(args.app, 'client/package.json'), + ]); + for (const p of packageFiles) { + if (!existsSync(p)) continue; + try { deps += Object.keys(JSON.parse(readFileSync(p, 'utf8')).dependencies ?? {}).length; } catch { /* ignore */ } + } + + return { + serverLoc: count(serverFiles), serverFiles: serverFiles.length, + totalLoc: count(allFiles), totalFiles: allFiles.length, + runtimeDeps: deps, + }; +} + +export function findMutationBackups(app: string, { readDir = readdirSync }: + { readDir?: MutationDirectoryReader } = {}): string[] { + const backups: string[] = []; + const walk = (dir: string): void => { + let entries; + try { + entries = readDir(dir, { withFileTypes: true }); + } catch (error) { + // Vite atomically replaces transient dependency directories while the + // app runs. They are not source and may vanish between parent and child + // reads; a missing directory cannot contain a mutation backup. + if (isRecord(error) && error.code === 'ENOENT') return; + throw error; + } + for (const entry of entries) { + if (/^(node_modules|dist|\.vite|\.git|module_bindings)$/.test(entry.name)) continue; + const path = join(dir, entry.name); + if (entry.isDirectory()) walk(path); + else if (entry.isFile() && entry.name.endsWith('.mutation-backup')) backups.push(path); + } + }; + walk(app); + return backups; +} + +function resetDatabase(args: RunArguments): { ok: boolean; detail: string | null; + outcome: { kind: string; phase: string; appFailures?: string[] } | null } { + process.stdout.write(' reset database ... '); + try { + run(process.execPath, [RESET, args.backend, args.app]); + console.log('ok'); + } catch (err) { + console.log('FAILED'); + const failure: Failure = err instanceof Error ? err : new Error(String(err)); + const detail = childFailureDetail(failure, failure.stdout); + console.log(` ${detail}`); + return { ok: false, detail, outcome: resetFailureOutcome(failure) }; + } + return { ok: true, detail: null, outcome: null }; +} + +export function contractLintArgv(args: ContractLintArguments, + selectedTask: BoundRecipeTaskRequestResult | null = null): string[] { + const interfaces = selectedTask ? contractInterfaceNames(selectedTask.task.contractText) : []; + const out = join(args.out, ARTIFACT_FILE.contractLint); + return [compiledEntrypoint('linter', 'lint.js'), '--url', args.url, '--level', args.level, + '--track', args.track, '--label', args.label, '--out', out, + '--parent-attempt-id', args.bundleArtifactId, + ...(args.credentialAliases + ? ['--credential-aliases-json', JSON.stringify(args.credentialAliases)] : []), + ...interfaces.flatMap(id => ['--hook', id])]; +} + +function lint(args: RunArguments, selectedTask: BoundRecipeTaskRequestResult | null = null): LintPayload | null { + process.stdout.write(' contract lint ... '); + const out = join(args.out, ARTIFACT_FILE.contractLint); + rmSync(out, { force: true }); + let failure: unknown = null; + try { + run('node', contractLintArgv(args, selectedTask)); + } catch (error) { failure = error; /* hook failures still write a report */ } + if (!existsSync(out)) { + const output = failure && typeof failure === 'object' && 'stdout' in failure + ? String(failure.stdout ?? '') : undefined; + const detail = failure instanceof Error + ? childFailureDetail(failure, output) : null; + throw new Error(`contract lint produced no report${detail ? `: ${detail}` : ''}`); + } + const r = readArtifactPayload(out, { expectedKind: 'contract_lint' }); + console.log(r.pass + ? r.counts.pass > 0 + ? `PASS (${r.counts.pass} interfaces)` + : `DEFERRED (${r.counts.scenario} interfaces checked during feature grading)` + : `FAIL (${r.counts.fail} failed, ${r.counts.blocked} blocked)`); + return r; +} + +// Named write actions let concurrency checks issue authenticated operations +// without prescribing one transport. Missing actions are reported explicitly. +function checkActions(args: RunArguments): ActionsPayload | null { + process.stdout.write(` ${'actions'.padEnd(10)} ... `); + const out = join(args.out, ARTIFACT_FILE.actions); + rmSync(out, { force: true }); + try { + run('node', [compiledEntrypoint('commands', 'check-actions.js'), '--backend', args.backend, + '--url', args.url, '--app', args.app ?? '.', '--track', args.track, '--out', out, '--quiet', + '--parent-attempt-id', args.bundleArtifactId]); + } catch { /* non-zero exit means something is missing; the report still lands */ } + if (!existsSync(out)) { console.log('NO REPORT'); return null; } + const r = readArtifactPayload(out, { expectedKind: 'action_check' }); + if (!r.missing.length) { console.log(`all ${r.results.length} present`); return r; } + console.log(`${r.missing.length} MISSING — ${r.missing.join(', ')}`); + return r; +} + +async function gradeSuite(args: RunArguments, suite: DeclaredSuite, track: Track, + recipeBinding: RecipeBinding | null, bundleArtifactId: string, selectedChecks: RecipeCheck[] = [], + { recordSelection = true, captureMedia = true, outputDirectory = args.out }: { + recordSelection?: boolean; captureMedia?: boolean; outputDirectory?: string; + } = {}): Promise { + process.stdout.write(` ${suite.id.padEnd(10)} ... `); + mkdirSync(outputDirectory, { recursive: true }); + const out = join(outputDirectory, `grading-${suite.id}.json`); + rmSync(out, { force: true }); + const argv = [compiledEntrypoint('grader', 'grade.js'), '--url', args.url, '--level', args.level, + '--label', `${args.label}-${suite.id}`, '--out', out]; + if (suite.spec) argv.push('--spec', suite.spec); + argv.push('--backend', args.backend, '--track', args.track); + if (recipeBinding) argv.push('--expected-recipe-sha256', recipeBinding.release.contentSha256); + const requestedRecipe = args.recipe ?? (args.recipeTask + ? `${args.recipeTask.recipe.id}@${args.recipeTask.recipe.version}` : null); + if (requestedRecipe) argv.push('--recipe', requestedRecipe); + for (const check of selectedChecks) argv.push('--selected-check', check.stableKey); + if (args.credentialAliases) { + argv.push('--credential-aliases-json', JSON.stringify(args.credentialAliases)); + } + if (recordSelection && args.selection?.sha256) { + argv.push('--selection-sha256', args.selection.evaluationSha256 ?? args.selection.sha256); + } + argv.push('--parent-attempt-id', bundleArtifactId); + // The out-of-band write goes straight to this run's database, with no + // app code in the loop; only the harness knows which one that is. + argv.push('--db-name', databaseNameForGrading(track, args.runIndex ?? 0, + args.databaseLease?.resources.database ? args.databaseLease : null)); + if (args.restartSpec) argv.push('--restart-spec', JSON.stringify(args.restartSpec)); + // The systems criteria run scripts the app itself ships (back-office writes), + // so the grader has to know where the app lives. + if (args.app) argv.push('--app', args.app); + if (captureMedia && args.media) argv.push('--media', join(outputDirectory, 'media'), '--trace'); + else if (captureMedia) argv.push('--failure-media', join(outputDirectory, 'failure-media')); + if (args.browserWsEndpoint) argv.push('--browser-ws-endpoint', args.browserWsEndpoint); + const child = await runGraderChild(argv, outputDirectory, suite.id); + const { stdout, failure } = child; + if (!existsSync(out)) { + console.log('NO REPORT'); + const detail = childFailureDetail(failure, stdout); + throw new Error(`grader produced no report for ${suite.id}${detail ? `: ${detail}` : ''}; ` + + `full diagnostics: ${child.stdoutName}, ${child.stderrName}`); + } + const r = readArtifactPayload(out, { expectedKind: 'grade' }); + if (selectedChecks.length) { + const expected = selectedChecks.map(check => check.stableKey).sort(); + const reported = (r.selection?.checks ?? []).map(check => check.stableKey).sort(); + if (JSON.stringify(reported) !== JSON.stringify(expected)) { + throw new Error(`grader report scope differs from requested suite scope for ${suite.id}`); + } + } + console.log(`${r.total}/${r.max}`); + for (const f of r.features) { + for (const c of f.criteria.filter(c => !evidencePassed(criterionEvidence(c)))) { + console.log(` ${renderEvidenceConsoleLine(criterionEvidence(c), `${f.name} / ${c.id}`, { + includeSummary: false, + })}`); + } + } + // Disclose passes that lack server-side confirmation. + const uiOnly = r.features.flatMap(f => + f.criteria.filter(c => evidencePassed(criterionEvidence(c)) && c.serverCheck === 'unverified') + .map(c => `${f.name}/${c.id}`)); + if (uiOnly.length) { + console.log(` note: ${uiOnly.length} criterion/criteria passed on interface behaviour only`); + for (const u of uiOnly) console.log(` ${u} — server-side check not runnable on this backend`); + } + return r; +} + +async function main() { + const startedAt = new Date().toISOString(); + const args = parseArgs(process.argv); + args.databaseLease = databaseLeaseForGrading(args.backend); + const track = loadTrack(args.track); + const recipeBinding = resolveRecipeRelease(track, Number(args.level), args.recipeTask?.recipe ?? args.recipe); + if (!recipeBinding && (args.packIds.length || args.checkKeys.length)) { + throw new Error('--pack and --check require a recipe-bound level'); + } + const selectedTask = recipeBinding + ? (args.recipeTask + ? resolveBoundRecipeTaskRequest(recipeBinding, args.recipeTask) + : createBoundRecipeTaskRequest(recipeBinding, args)) + : null; + let selection = selectObservationScope(selectedTask, args.observation); + if (args.sourceSha256) { + const source = hashAppSource(args.app); + if (source.sha256 !== args.sourceSha256) { + throw new Error('live application source differs from the source selected for grading'); + } + } + const declaredSuites = recipeBinding + ? suitesForRecipe(track, recipeBinding) + : suitesFor(track, Number(args.level)); + if (args.observation === 'scored') { + selection = attachRegressionScope(selection, recipeBinding, declaredSuites, + args.regressionChecks); + } else if (args.regressionChecks.length) { + throw new Error('observed grading cannot include regression checks'); + } + args.selection = selection; + if (selection) { + const suiteIds = new Set(declaredSuites.map(suite => suite.id)); + const unmapped = selection.checks.filter(check => !suiteIds.has(check.executionId)); + if (unmapped.length) { + throw new Error(`selected recipe checks do not map to a declared suite: ${ + unmapped.map(check => check.stableKey).join(', ')}`); + } + } + const calibration = resolveCalibrationForRelease(recipeBinding?.release ?? null, { + trackRoot: track.dir, + stackBenchRoot: ROOT, + }); + const observationSuffix = args.observation === 'observed' ? '-observed' : ''; + const bundleArtifactId = `${args.parentAttemptId ?? args.label}-grade-bundle-l${args.level}${observationSuffix}`; + args.bundleArtifactId = bundleArtifactId; + mkdirSync(args.out, { recursive: true }); + // Remove all prior grade output before writing cumulative evidence. + clearPreviousGradeOutputs(args.out); + + console.log(`\n=== ${args.label} (${args.backend}) ===`); + console.log(` app: ${args.app}`); + console.log(` url: ${args.url}`); + if (recipeBinding && selection) { + console.log(` recipe: ${recipeBinding.alias} -> ${recipeBinding.release.id}@${recipeBinding.release.version} ` + + `(${recipeBinding.status}, ${recipeBinding.release.contentSha256.slice(0, 12)})`); + console.log(args.observation === 'observed' + ? ` scope: ${selection.checks.length} observed check(s), ${selection.observedPoints} observed point(s), 0 score contribution` + : ` scope: ${selection.checks.length} check(s), ${selection.scoredPoints} point(s)`); + if (selection.requested.packs?.length) console.log(` packs: ${selection.requested.packs.join(', ')}`); + if (selection.requested.features?.length) { + console.log(` features: ${selection.requested.features.join(', ')}`); + } + if (selection.requested.checks.length) console.log(` extra checks: ${selection.requested.checks.join(', ')}`); + } + + const bundle: Bundle = { + definitionSchemaVersion: track.schemaVersion, + recipeRelease: bundleRecipeRelease(recipeBinding), + calibration: calibration ? { id: calibration.id, version: calibration.version, + state: calibration.state, contentSha256: calibration.contentSha256 } : null, + label: args.label, track: args.track, backend: args.backend, url: args.url, app: args.app, + level: Number(args.level), observation: args.observation, + ...(args.sourceSha256 ? { source: { sha256: args.sourceSha256 } } : {}), + suites: {}, totals: {}, + selection: selection ? { ...selection, attemptedChecks: [], reportedChecks: [], notRun: [] } : null, + }; + const selectedPackIds = new Set(selection?.checks.map(check => check.packId) ?? []); + const selectedPackDefinitions = recipeBinding?.plan.packs + .filter(pack => selectedPackIds.has(pack.id)) ?? []; + const writeBundle = () => { + if (args.sourceSha256) { + const current = hashAppSource(args.app); + if (current.sha256 !== args.sourceSha256) { + bundle.error = 'application source changed while grading was in progress'; + bundle.outcome = { kind: 'harness_failure', phase: 'source-provenance', + reason: bundle.error }; + } + } + return writeArtifact(join(args.out, ARTIFACT_FILE.gradeBundle), { + kind: 'grade_bundle', + id: bundleArtifactId, + attempt: { id: bundleArtifactId, parentId: args.parentAttemptId ?? null }, + timestamps: { startedAt, completedAt: new Date().toISOString() }, + identities: recipeArtifactIdentities(recipeBinding?.release ?? null, { + calibration: calibration ? { id: calibration.id, version: calibration.version, + sha256: calibration.contentSha256, state: calibration.state } : null, + stackAdapter: { id: args.backend }, + }), + payload: bundle, + }); + }; + const recordApplicationAbort = () => { + bundle.totals = applicationFailureTotals(selection, declaredSuites); + }; + const freshenFailureMessage = () => { + const detail = lastResetFailure ? `: ${lastResetFailure}` : ''; + if (lastResetOutcome?.phase !== 'application-readiness') { + return `database reset failed — scores would not be comparable${detail}`; + } + return lastResetOutcome.kind === 'harness_failure' + ? `application server stopped by the grader was not restored${detail}` + : `application did not become ready after database reset${detail}`; + }; + const markRemainingNotRun = (reason: string): void => { + if (!bundle.selection) return; + const accounted = new Set([ + ...bundle.selection.attemptedChecks, + ...bundle.selection.notRun.map(check => check.stableKey), + ]); + bundle.selection.notRun.push(...bundle.selection.checks + .filter(check => !accounted.has(check.stableKey)) + .map(check => ({ stableKey: check.stableKey, reason }))); + }; + + if (args.applicationFailure) { + bundle.error = args.applicationFailure.reason; + bundle.outcome = args.applicationFailure; + recordApplicationAbort(); + markRemainingNotRun(`run aborted: ${bundle.error}`); + writeBundle(); + console.log(`\nABORTED: ${bundle.error}`); + process.exit(1); + } + + // Reset before each stateful check. + let lastResetFailure: string | null = null; + let lastResetOutcome: ResetOutcome = { kind: 'harness_failure', phase: 'database-reset' }; + // Set when a grader stopped the application server and could not start it + // again. Until the harness starts it, the app cannot be blamed for being + // unreachable. + let applicationLeftStopped = false; + const freshen = async () => { + if (!args.reset) return true; + const requiresReseed = STACK_ADAPTER_REGISTRY.get(args.backend).reset.requiresReseed; + const restartSpec = args.restartSpec; + if (track.reseedOnReset && restartSpec && requiresReseed) { + process.stdout.write(' stop application ... '); + try { + await controlBackendRuntime(restartSpec, 'stop'); + console.log('ok'); + } catch (error) { + const failure: Failure = error instanceof Error ? error : new Error(String(error)); + lastResetFailure = childFailureDetail(failure); + lastResetOutcome = { kind: 'harness_failure', phase: 'application-reset-control' }; + console.log(`FAILED (${lastResetFailure})`); + return false; + } + } + const reset = resetDatabase(args); + lastResetFailure = reset.detail; + lastResetOutcome = reset.outcome ?? { kind: 'harness_failure', phase: 'database-reset' }; + if (!reset.ok) return false; + // Do not grade until the reset application is reachable. + const waitUntilReady = async () => { + const ready = await waitForApplicationProbe(args.url); + if (!ready.ok) { + lastResetFailure = ready.detail; + lastResetOutcome = applicationLeftStopped + ? { kind: 'harness_failure', phase: 'application-readiness' } + : { kind: 'app_failure', phase: 'application-readiness', + appFailures: ['application-readiness'] }; + console.log(`FAILED (${ready.detail})`); + return false; + } + console.log('ok'); + return true; + }; + if (track.reseedOnReset && restartSpec && requiresReseed) { + process.stdout.write(' restart ... '); + // Judge restart success with the readiness probe. The restart command can + // leave a long-running server process behind, so the command also needs a deadline. + try { + // Do not give a background server an inherited pipe that keeps the + // synchronous restart command open. + await controlBackendRuntime(restartSpec, 'start'); + applicationLeftStopped = false; + } catch (err) { + const failure: Failure = err instanceof Error ? err : new Error(String(err)); + lastResetOutcome = resetFailureOutcome(failure); + const detail = ((failure.stderr || '') + (failure.stdout || '') + (failure.message || '')) + .toString().trim().split('\n').slice(-3).join(' | ').slice(0, 300); + lastResetFailure = detail || null; + console.log('FAILED (application did not restart)'); + console.log(` control: ${JSON.stringify(restartSpec)}`); + console.log(` ${detail}`); + return false; + } + return await waitUntilReady(); + } + process.stdout.write(' ready ... '); + return await waitUntilReady(); + }; + + bundle.code = codeMetrics(args); + console.log(` code ... ${bundle.code.serverLoc} server LOC in ${bundle.code.serverFiles} files, ` + + `${bundle.code.totalLoc} total LOC, ${bundle.code.runtimeDeps} runtime deps`); + + // Refuse source left modified by an interrupted mutation run. + const mutated = findMutationBackups(args.app); + if (mutated.length) { + bundle.error = `app still carries mutation backups (${mutated.join(', ')}) — its source is mutated, not the build under test`; + bundle.outcome = { kind: 'harness_failure', phase: 'mutation-cleanup', reason: bundle.error }; + markRemainingNotRun('run aborted because application source is still mutated'); + writeBundle(); + console.log(`\nABORTED: ${bundle.error}`); + process.exit(1); + } + + const prov = checkDatabaseProvenance(args); + bundle.provenance = prov; + console.log(` database ... ${prov.ok ? 'benchmark-owned' : `WRONG DATABASE — ${prov.reason}`}`); + if (!prov.ok) { + bundle.error = `app is not using the benchmark database: ${prov.reason}`; + bundle.outcome = { kind: 'app_failure', phase: 'database-provenance', reason: bundle.error, + appFailures: ['database-provenance'] }; + recordApplicationAbort(); + markRemainingNotRun('run aborted because database provenance was invalid'); + writeBundle(); + console.log('\nABORTED: results would not describe the benchmark environment.'); + process.exit(1); + } + + if (args.observation === 'scored') { + if (!(await freshen())) { + bundle.error = freshenFailureMessage(); + bundle.outcome = { ...lastResetOutcome, reason: bundle.error }; + if (bundle.outcome.kind === 'app_failure') recordApplicationAbort(); + markRemainingNotRun(`run aborted: ${bundle.error}`); + writeBundle(); + console.log(`\nABORTED: ${bundle.error}`); + process.exit(1); + } + let runtime = checkRuntimeDatabaseProvenance(args); + let proofError = null; + let actionFailure: string | null = null; + const proof = track.databaseProvenance; + const supportsRuntimeProof = 'proveUse' in STACK_ADAPTER_REGISTRY.get(args.backend).database; + const requiresRuntimeProof = supportsRuntimeProof && args.databaseLease && args.reset; + if (requiresRuntimeProof && !proof) { + proofError = new Error(`${args.track} does not define a runtime database provenance check`); + } else if (requiresRuntimeProof && proof) { + try { + const write = await writeApplicationDatabaseMarker(args, track, proof); + if (write.ok) runtime = checkRuntimeDatabaseProvenance(args, write.marker); + else actionFailure = write.reason; + } catch (error) { + proofError = error; + } + + // The proof writes unique data through the application. Remove it before + // linting and scored grading so the proof cannot change the result. + if (!(await freshen())) { + bundle.error = freshenFailureMessage(); + bundle.outcome = { ...lastResetOutcome, reason: bundle.error }; + if (bundle.outcome.kind === 'app_failure') recordApplicationAbort(); + markRemainingNotRun(`run aborted: ${bundle.error}`); + writeBundle(); + console.log(`\nABORTED: ${bundle.error}`); + process.exit(1); + } + } else if (supportsRuntimeProof && !args.reset) { + runtime = { ok: null, verified: false, + reason: 'runtime marker proof requires database reset to isolate its write' }; + } + + if (proofError) { + bundle.outcome = databaseProvenanceFailure(proofError); + bundle.error = bundle.outcome.reason; + markRemainingNotRun('run aborted because runtime database provenance could not be verified'); + writeBundle(); + console.log(`\nABORTED: ${bundle.error}`); + process.exit(1); + } + + if (actionFailure) { + bundle.error = actionFailure; + bundle.outcome = { kind: 'app_failure', phase: 'database-provenance-action', + reason: actionFailure, appFailures: ['database-provenance-action'] }; + recordApplicationAbort(); + markRemainingNotRun('run aborted because the application database write failed'); + writeBundle(); + console.log(`\nABORTED: ${actionFailure}`); + process.exit(1); + } + + bundle.provenance.runtime = runtime; + console.log(` db runtime ... ${runtime.verified + ? runtime.ok ? runtime.reason : `WRONG DATABASE — ${runtime.reason}` + : runtime.reason}`); + if (runtime.ok === false) { + bundle.error = `app did not write its marker to the benchmark database: ${runtime.reason}`; + bundle.outcome = { kind: 'app_failure', phase: 'database-provenance', reason: bundle.error, + appFailures: ['database-provenance'] }; + recordApplicationAbort(); + markRemainingNotRun('run aborted because runtime database provenance failed'); + writeBundle(); + console.log('\nABORTED: application data came from outside the benchmark database.'); + process.exit(1); + } + try { + bundle.suites.lint = lint(args, selectedTask); + } catch (error) { + markRemainingNotRun('run aborted after contract lint failed to produce evidence'); + bundle.error = error instanceof Error ? error.message : String(error); + bundle.outcome = { kind: 'harness_failure', phase: 'contract-lint', reason: bundle.error }; + writeBundle(); + console.log(`\nABORTED: ${bundle.error}`); + process.exit(1); + } + bundle.actions = checkActions(args); + } + + // Keep current-level score separate from earlier guarantee regressions. + let total = 0, max = 0, regTotal = 0, regMax = 0; + const dirty = false; + let browserServer: BrowserServer | null = null; + try { + if (declaredSuites.some(suite => !selection + || selection.checks.some(check => check.executionId === suite.id))) { + browserServer = await chromium.launchServer({ headless: true }); + args.browserWsEndpoint = browserServer.wsEndpoint(); + } + for (const suite of declaredSuites) { + const selectedChecks = selection?.checks.filter(check => check.executionId === suite.id) ?? []; + if (selection && selectedChecks.length === 0) { + console.log(` ${suite.id.padEnd(10)} ... not selected`); + continue; + } + if (!(await freshen())) { + bundle.error = freshenFailureMessage(); + console.log(` ${suite.id}: SKIPPED (${bundle.error})`); + markRemainingNotRun(`run aborted: ${bundle.error}`); + bundle.outcome = { ...lastResetOutcome, reason: bundle.error }; + if (bundle.outcome.kind === 'app_failure') recordApplicationAbort(); + writeBundle(); + console.log(`\nABORTED: ${bundle.error}`); + throw new Error(bundle.error); + } + if (bundle.selection) { + bundle.selection.attemptedChecks.push(...selectedChecks.map(check => check.stableKey)); + } + let r; + try { + r = await gradeSuite(args, suite, track, recipeBinding, bundleArtifactId, selectedChecks); + } catch (error) { + markRemainingNotRun(`run aborted after ${suite.id} grader failure`); + bundle.error = error instanceof Error ? error.message : String(error); + bundle.outcome = { kind: 'harness_failure', phase: `grade:${suite.id}`, reason: bundle.error }; + writeBundle(); + console.log(`\nABORTED: ${bundle.error}`); + throw error; + } + bundle.suites[suite.id] = r; + if (isGradePayload(r) && r.features.some(feature => feature.cleanupEvidence?.failures + .some(failure => failure.stage === 'application-restore'))) { + applicationLeftStopped = true; + } + if (bundle.selection) { + bundle.selection.reportedChecks.push(...selectedChecks.map(check => check.stableKey)); + } + if (selection) { + bundle.packRuntime = aggregatePackRuntime( + Object.values(bundle.suites).filter(isGradePayload), + selectedPackDefinitions); + const exceeded = exceededPackBudgets(bundle.packRuntime); + if (exceeded.length) { + // Runtime budgets qualify references; generated apps still receive a complete grade. + console.log(` runtime ... ${exceeded.map(pack => + `${pack.id} ${pack.measuredRuntimeMs}ms > ${pack.budget.maxRuntimeMs}ms`) + .join(', ')} [recorded; grading continues]`); + } + } + if (suite.inherited) { regTotal += r.total; regMax += r.max; } + else { total += r.total; max += r.max; } + } + } finally { + args.browserWsEndpoint = undefined; + await browserServer?.close(); + } + + bundle.totals = { + score: total, max, dirty, contractPass: isGradePayload(bundle.suites.lint) + ? null : bundle.suites.lint?.pass ?? null, + // null rather than 0/0 at L1, where there is nothing earlier to regress. + regression: regMax ? { score: regTotal, max: regMax } : null, + }; + writeBundle(); + + console.log(` ${'TOTAL'.padEnd(10)} ... ${total}/${max}${dirty ? ' [DIRTY]' : ''}`); + if (regMax) { + const kept = regTotal === regMax ? 'all earlier guarantees still hold' : `${regMax - regTotal} EARLIER GUARANTEE(S) LOST`; + console.log(` ${'REGRESSION'.padEnd(10)} ... ${regTotal}/${regMax} — ${kept}`); + } + console.log(` bundle: ${join(args.out, ARTIFACT_FILE.gradeBundle)}`); +} + +if (process.argv[1] && pathToFileURL(resolve(process.argv[1])).href === import.meta.url) main(); diff --git a/tools/stack-bench/commands/test-loop.ts b/tools/stack-bench/commands/test-loop.ts new file mode 100644 index 00000000000..c63a8de6b74 --- /dev/null +++ b/tools/stack-bench/commands/test-loop.ts @@ -0,0 +1,309 @@ +#!/usr/bin/env node + +import { execFileSync, spawnSync } from 'node:child_process'; +import { readFileSync, existsSync, rmSync, mkdirSync, mkdtempSync, readdirSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { ARTIFACT_FILE, readArtifact, readArtifactPayload } from '../src/evidence/artifacts.js'; +import type { ArtifactIdentities } from '../src/evidence/artifacts.js'; +import type { CostRun, CostSession } from '../src/evidence/cost-proof.js'; +import type { PublicBackendLease } from '../src/runtime/backend-lease.js'; +import type { RepairLevel, RepairOutcome } from '../src/runtime/repair-grant.js'; +import type { LevelCheckpoint } from '../src/runtime/source-checkpoint.js'; +import { CODING_CONTAINER_BUG_REPORT_FILE } from '../src/runtime/coding-container-policy.js'; + +import { STACK_BENCH_ROOT as ROOT, compiledEntrypoint } from '../src/package-root.js'; +const WORK = mkdtempSync(join(tmpdir(), 'stack-bench-loop-')); +const APP = join(WORK, 'app'); +// A cold Playwright start plus two grades can exceed three minutes on Windows +// Docker hosts. The timeout is a deadlock guard, not a performance assertion. +const BENCH_TIMEOUT_MS = 300_000; + +interface LoopSession extends CostSession { + sessionId?: string; + tokens?: number; + turns?: number; + durationMs?: number; +} + +interface LoopLevel extends RepairLevel { + buildSessions?: LoopSession[]; + repairSessions?: LoopSession[]; + resumeSession?: LoopSession; + contractPass?: boolean; + stalled?: boolean; + code?: { totalLoc?: number }; + sessionTotals?: { sessions?: number; tokens?: number; turns?: number; durationMs?: number }; +} + +interface LoopRun extends CostRun { + id?: string; + levels?: LoopLevel[]; + outcome?: RepairOutcome; + artifactEnvelope?: { identities?: ArtifactIdentities }; + backendLease?: PublicBackendLease; + totals?: CostRun['totals'] & { max?: number; sessions?: number; tokens?: number; turns?: number; + modelDurationMs?: number; durationSec?: number }; +} + +interface GradeFeature { + id?: string; + setupEvidence?: { schemaVersion?: number; status?: string }; + criteria?: { evidence?: { schemaVersion?: number; status?: string; actions?: unknown[] } }[]; +} + +interface GradePayload { features?: GradeFeature[]; } +interface SourceCheckpointPayload { source: LevelCheckpoint; } +interface RepairContinuation { + baseline?: { reproduced?: boolean; score?: number; sourceSha256?: string }; + cumulativeRepairsBefore?: number; + cumulativeRepairsAfter?: number; + resumeSetup?: { sourceVerified?: boolean }; +} +interface RepairContinuationPayload extends LoopRun { continuation?: RepairContinuation; } + +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +function processOutput(error: unknown): string { + if (!isRecord(error)) return String(error); + return `${String(error.stdout ?? '')}${String(error.stderr ?? '')}`; +} + +// A failed assertion or interrupted CI job must not leave a fixture app that a +// later loop can mistake for its own output. +process.on('exit', () => rmSync(WORK, { recursive: true, force: true })); + +let failures = 0; +const check = (name: string, ok: boolean, detail = ''): void => { + console.log(` ${ok ? 'PASS' : 'FAIL'} ${name}${ok || !detail ? '' : ` — ${detail}`}`); + if (!ok) failures += 1; +}; + +function runBench(extra: string[] = []): string { + const argv = [compiledEntrypoint('commands', 'bench.js'), '--backend', 'stub', '--levels', '1', + '--agent-adapter', 'deterministic', + '--app', APP, '--out', WORK, + '--track', 'loop', + '--url', `file:///${join(APP, 'index.html').replace(/\\/g, '/')}`, ...extra]; + try { + return execFileSync('node', argv, { + encoding: 'utf8', + maxBuffer: 32 * 1024 * 1024, + timeout: BENCH_TIMEOUT_MS, + killSignal: 'SIGTERM', + }); + } catch (error: unknown) { return processOutput(error); } +} + +const invalidRounds = spawnSync('node', [compiledEntrypoint('commands', 'bench.js'), '--backend', 'stub', + '--repairs', '1.5'], + { encoding: 'utf8' }); +check('fractional correction budgets are rejected before a run starts', + invalidRounds.status !== 0 + && /--repairs must be a non-negative safe integer/.test(invalidRounds.stderr)); + +rmSync(WORK, { recursive: true, force: true }); +mkdirSync(APP, { recursive: true }); + +console.log('\nLoop test — one repair available'); +const out = runBench(['--repairs', '1']); +const runPath = join(WORK, ARTIFACT_FILE.run); + +check(`the benchmark run produced ${ARTIFACT_FILE.run}`, existsSync(runPath)); +if (!existsSync(runPath)) { + console.log(`\ncannot continue without ${ARTIFACT_FILE.run}`); + process.exit(1); +} + +const run = readArtifactPayload(runPath); +const level = run.levels?.[0]; +const evidenceDir = join(APP, 'stack-bench'); +const bundleArtifact = readArtifact(join(evidenceDir, ARTIFACT_FILE.gradeBundle), + { expectedKind: 'grade_bundle' }); +const lintArtifact = readArtifact(join(evidenceDir, ARTIFACT_FILE.contractLint), + { expectedKind: 'contract_lint' }); +const actionArtifact = readArtifact(join(evidenceDir, ARTIFACT_FILE.actions), + { expectedKind: 'action_check' }); +const gradeArtifact = readArtifact(join(evidenceDir, 'grading-features.json'), { expectedKind: 'grade' }); +const leaseArtifact = readArtifact(join(WORK, ARTIFACT_FILE.backendLease), + { expectedKind: 'backend_lease_evidence' }); +const checkpointArtifact = readArtifact(join(WORK, 'level-l1-checkpoint.json'), + { expectedKind: 'source_checkpoint' }); + +check('recorded exactly one level', run.levels?.length === 1); +check('run and level carry structured outcomes', + typeof run.outcome?.kind === 'string' && run.outcome.kind === level?.outcome?.kind, + `run=${run.outcome?.kind} level=${level?.outcome?.kind}`); +check('artifacts carry the producing run id', typeof run.id === 'string' && run.id.length > 10); +check('run envelope identifies engine, agent adapter, and stack adapter', + /^[a-f0-9]{64}$/.test(run.artifactEnvelope?.identities?.engine?.sha256 ?? '') + && /^[a-f0-9]{64}$/.test(run.artifactEnvelope?.identities?.agentAdapter?.sha256 ?? '') + && run.artifactEnvelope?.identities?.stackAdapter?.id === 'stub'); +check('bundle is a child of the run', bundleArtifact.attempt.parentId === run.id, + JSON.stringify(bundleArtifact.attempt)); +check('public lease evidence is a child of the run', leaseArtifact.attempt.parentId === run.id, + JSON.stringify(leaseArtifact.attempt)); +check('level source checkpoint is hash-bound and linked to the run', + checkpointArtifact.attempt.parentId === run.id + && level?.checkpoint?.artifact === 'level-l1-checkpoint.json' + && level.checkpoint.sha256 === checkpointArtifact.payload.source.sha256 + && /^[a-f0-9]{64}$/.test(level.checkpoint.sha256) + && existsSync(join(WORK, level.checkpoint.directory)), + JSON.stringify(level?.checkpoint)); +check('lint, action, and grade evidence are children of the bundle', + [lintArtifact, actionArtifact, gradeArtifact] + .every(artifact => artifact.attempt.parentId === bundleArtifact.attempt.id)); +const gradedFeatures = gradeArtifact.payload?.features ?? []; +check('grade artifacts retain typed setup, criterion, and action evidence', + gradedFeatures.length > 0 + && gradedFeatures.every(feature => feature.setupEvidence?.schemaVersion === 1 + && (feature.criteria ?? []).every(criterion => criterion.evidence?.schemaVersion === 1 + && Array.isArray(criterion.evidence.actions))), + JSON.stringify(gradedFeatures.map(feature => ({ id: feature.id, + setup: feature.setupEvidence?.status, + criteria: feature.criteria?.map(criterion => criterion.evidence?.status) })))); +const publicJson = [runPath, join(WORK, ARTIFACT_FILE.backendLease), + join(evidenceDir, ARTIFACT_FILE.gradeBundle), join(evidenceDir, ARTIFACT_FILE.contractLint), + join(evidenceDir, ARTIFACT_FILE.actions), join(evidenceDir, 'grading-features.json')] + .map(path => readFileSync(path, 'utf8')).join('\n'); +check('public envelopes contain no secret or lease-token fields', + !/"(?:apiKey|leaseToken|ownershipToken|password|secret)"\s*:/i.test(publicJson)); +check('backend lease was released', + ['released', 'stopped'].includes(run.backendLease?.state ?? '') + && (run.backendLease?.resources?.locks?.every(lock => lock.releasedAt) ?? false), + JSON.stringify(run.backendLease?.state)); +check('a repair ran', level?.repairs === 1, `repairs=${level?.repairs}`); +check('successful repair is explicit', level?.repair?.status === 'corrected' + && level.repair.limit === 1 && level.repair.used === 1 + && level.repair.stopReason === 'passed', + JSON.stringify(level?.repair)); +const reportPath = join(APP, CODING_CONTAINER_BUG_REPORT_FILE); +const reportExists = existsSync(reportPath); +check('the bug report was written', reportExists); +// Behavioural findings must never reveal how they were detected, or a fix can +// target the check instead of the app. Missing-control findings are exempt: +// there the element id is the requirement. +const report = reportExists ? readFileSync(reportPath, 'utf8') : ''; +const behaviourSection = report.split('## Application interface')[0] ?? ''; +check('behavioural findings do not leak selectors or timings', + !/data-(?:role|testid)|locator|within \d+ms/.test(behaviourSection)); +check('missing interfaces are reported separately', /## Application interface/.test(report)); +check('build and fix costs are both recorded', + (level?.buildCostUsd ?? 0) > 0 && (level?.repairCostUsd ?? 0) > 0, + `build=${level?.buildCostUsd} fix=${level?.repairCostUsd}`); +check('build and fix sessions remain individually auditable', + level?.buildSessions?.length === 1 + && level.buildSessions[0]?.sessionId === 'stub-build' + && level?.repairSessions?.length === 1 + && level.repairSessions[0]?.sessionId === 'stub-fix', + JSON.stringify({ builds: level?.buildSessions, fixes: level?.repairSessions })); +check('level session totals include the build and fix', + level?.sessionTotals?.sessions === 2 + && level.sessionTotals.tokens === 2000 + && level.sessionTotals.turns === 5 + && level.sessionTotals.durationMs === 100, + JSON.stringify(level?.sessionTotals)); +check('grading produced a score out of a maximum', Number.isInteger(level?.score) && (level?.max ?? 0) > 0, + `${level?.score}/${level?.max}`); +check('code metrics captured', Boolean(level?.code) && typeof level?.code?.totalLoc === 'number', + JSON.stringify(level?.code)); +check('totals aggregate the levels', run.totals?.max === level?.max); +check('run totals aggregate every model session', + run.totals?.sessions === 2 && run.totals.tokens === 2000 + && run.totals.turns === 5 && run.totals.modelDurationMs === 100, + JSON.stringify(run.totals)); +check('wall time recorded', (run.totals?.durationSec ?? -1) >= 0); +check('the fix improved the contract lint', + /APPLICATION CONTRACT FAIL[\s\S]*APPLICATION CONTRACT PASS/.test(out) + || level?.contractPass === true, + 'expected the broken fixture to fail the lint and the fixed one to pass'); + +console.log('\nLoop test — zero repairs allowed'); +rmSync(WORK, { recursive: true, force: true }); +mkdirSync(APP, { recursive: true }); +runBench(['--repairs', '0']); +const capped = readArtifactPayload(runPath); +check('no fix ran when the cap is zero', capped.levels?.[0]?.repairs === 0); +check('no bug report was written when no fix is allowed', + !existsSync(join(APP, CODING_CONTAINER_BUG_REPORT_FILE))); + +console.log('\nLoop test - flat corrections exhaust their declared budget'); +rmSync(WORK, { recursive: true, force: true }); +mkdirSync(APP, { recursive: true }); +runBench(['--repairs', '2', '--model', 'deterministic-stall']); +const exhausted = readArtifactPayload(runPath); +const exhaustedLevel = exhausted.levels?.[0]; +check('both correction rounds ran after the first flat result', exhaustedLevel?.repairs === 2, + `repairs=${exhaustedLevel?.repairs}`); +check('an unresolved app records budget exhaustion', exhaustedLevel?.repair?.status === 'budget-exhausted' + && exhaustedLevel.repair.limit === 2 && exhaustedLevel.repair.used === 2 + && exhaustedLevel.repair.stopReason === 'budget-exhausted' + && exhaustedLevel.stalled === true && exhausted.outcome?.kind === 'app_failure', + JSON.stringify({ repair: exhaustedLevel?.repair, outcome: exhausted.outcome })); + +console.log('\nLoop test - a later finite grant continues the exact exhausted source'); +rmSync(WORK, { recursive: true, force: true }); +mkdirSync(APP, { recursive: true }); +runBench(['--repairs', '2', '--model', 'deterministic-deferred']); +const parentBefore = readFileSync(runPath, 'utf8'); +const deferred = readArtifactPayload(runPath); +check('the deferred parent exhausted its original two-round budget', + deferred.levels?.[0]?.repair?.status === 'budget-exhausted' + && deferred.levels[0].repair.used === 2, + JSON.stringify(deferred.levels?.[0]?.repair)); +let continuationOutput = ''; +try { + continuationOutput = execFileSync('node', [join(ROOT, 'dist', 'commands', 'repair-cli.js'), 'grant', WORK, + '--level', '1', '--repairs', '2', '--timeout-minutes', '10'], { + encoding: 'utf8', maxBuffer: 32 * 1024 * 1024, timeout: BENCH_TIMEOUT_MS, + }); +} catch (error: unknown) { + continuationOutput = processOutput(error); +} +const continuationRoot = join(WORK, 'continuations'); +const continuationDirectories = existsSync(continuationRoot) + ? readdirSync(continuationRoot, { withFileTypes: true }).filter(entry => entry.isDirectory()) : []; +const continuationDirectory = continuationDirectories.length === 1 + ? join(continuationRoot, continuationDirectories[0]?.name ?? '') : null; +const continuationPath = continuationDirectory + ? join(continuationDirectory, ARTIFACT_FILE.run) : null; +check('repair grant produced one linked continuation', + continuationPath !== null && existsSync(continuationPath), continuationOutput.slice(-2000)); +if (continuationDirectory && continuationPath && existsSync(continuationPath)) { + const continuationArtifact = readArtifact(continuationPath, { expectedKind: 'repair_continuation' }); + const continuation = continuationArtifact.payload; + const continuedLevel = continuation.levels?.[0]; + const continuationDetails = continuation.continuation; + const deferredLevel = deferred.levels?.[0]; + check('continuation reproduced the exact failed baseline before spending a repair', + continuationDetails?.baseline?.reproduced === true + && continuationDetails.baseline.score === deferredLevel?.score + && continuationDetails.baseline.sourceSha256 === deferredLevel?.checkpoint?.sha256, + JSON.stringify(continuationDetails?.baseline)); + check('continuation reached correctness inside its finite added budget', + continuation.outcome?.kind === 'passed' + && continuedLevel?.repair?.status === 'corrected' + && continuedLevel.repair.used === 1 + && continuationDetails?.cumulativeRepairsBefore === 2 + && continuationDetails?.cumulativeRepairsAfter === 3, + JSON.stringify({ repair: continuedLevel?.repair, continuation: continuationDetails })); + check('resume setup is visible, separately costed, and does not consume a repair', + continuationDetails?.resumeSetup?.sourceVerified === true + && continuedLevel?.resumeSession?.sessionId === 'stub-resume' + && (continuedLevel?.resumeCostUsd ?? 0) > 0 + && continuedLevel.repairs === 1, + JSON.stringify({ setup: continuationDetails?.resumeSetup, + resume: continuedLevel?.resumeSession, fixes: continuedLevel?.repairs })); + check('continuation process outcome is retained as a typed child artifact', + readArtifact(join(continuationDirectory, ARTIFACT_FILE.process), + { expectedKind: 'repair_process' }).attempt.parentId === deferred.id); +} +check('grant left the original run artifact byte-for-byte unchanged', + readFileSync(runPath, 'utf8') === parentBefore); + +rmSync(WORK, { recursive: true, force: true }); +console.log(`\n${failures === 0 ? 'loop OK' : `${failures} check(s) failed`}`); +process.exit(failures === 0 ? 0 : 1); diff --git a/tools/stack-bench/conditions/catalog.json b/tools/stack-bench/conditions/catalog.json new file mode 100644 index 00000000000..1c6ca9a1a18 --- /dev/null +++ b/tools/stack-bench/conditions/catalog.json @@ -0,0 +1,12 @@ +{ + "schemaVersion": 1, + "kind": "study-condition-catalog", + "guidanceProfiles": { + "model-free-stub@1.1.0": "guidance/model-free-stub.json", + "prescribed@1.2.0": "guidance/prescribed.json", + "neutral@1.8.0": "guidance/neutral-1.8.0.json" + }, + "repairPolicies": { + "scored-only@1.1.0": "repairs/scored-only.json" + } +} diff --git a/tools/stack-bench/conditions/guidance/model-free-stub.json b/tools/stack-bench/conditions/guidance/model-free-stub.json new file mode 100644 index 00000000000..5d3559cc5eb --- /dev/null +++ b/tools/stack-bench/conditions/guidance/model-free-stub.json @@ -0,0 +1,22 @@ +{ + "schemaVersion": 1, + "kind": "backend-guidance-profile", + "id": "model-free-stub", + "version": "1.1.0", + "state": "draft", + "mode": "neutral", + "material": { + "accessFacts": true, + "apiReference": false, + "designAdvice": false + }, + "documents": { + "stub": "backends/model-free-stub.md" + }, + "applicationInterfaces": { + "stub": "http" + }, + "skills": { + "stub": [] + } +} diff --git a/tools/stack-bench/conditions/guidance/neutral-1.8.0.json b/tools/stack-bench/conditions/guidance/neutral-1.8.0.json new file mode 100644 index 00000000000..c377ac67c45 --- /dev/null +++ b/tools/stack-bench/conditions/guidance/neutral-1.8.0.json @@ -0,0 +1,33 @@ +{ + "schemaVersion": 1, + "kind": "backend-guidance-profile", + "id": "neutral", + "version": "1.8.0", + "state": "qualified", + "mode": "neutral", + "material": { + "accessFacts": true, + "apiReference": true, + "designAdvice": false + }, + "documents": { + "mongodb": "backends/minimal/mongodb-1.5.md", + "postgres": "backends/minimal/postgres-1.5.md", + "spacetime": "backends/minimal/spacetime-1.6.md" + }, + "applicationInterfaces": { + "mongodb": "http", + "postgres": "http", + "spacetime": "reducer" + }, + "skills": { + "mongodb": [], + "postgres": [], + "spacetime": ["typescript-server", "typescript-client"] + }, + "credentialAliases": { + "stackbench-admin-2026": "store-admin-2026", + "stackbench-customer-2026": "store-customer-2026", + "stackbench-staff-2026": "store-staff-2026" + } +} diff --git a/tools/stack-bench/conditions/guidance/prescribed.json b/tools/stack-bench/conditions/guidance/prescribed.json new file mode 100644 index 00000000000..ad800f4cc83 --- /dev/null +++ b/tools/stack-bench/conditions/guidance/prescribed.json @@ -0,0 +1,33 @@ +{ + "schemaVersion": 1, + "kind": "backend-guidance-profile", + "id": "prescribed", + "version": "1.2.0", + "state": "qualified", + "mode": "prescribed", + "material": { + "accessFacts": true, + "apiReference": true, + "designAdvice": true + }, + "documents": { + "mongodb": "backends/mongodb.md", + "postgres": "backends/postgres.md", + "spacetime": "backends/spacetime.md" + }, + "applicationInterfaces": { + "mongodb": "http", + "postgres": "http", + "spacetime": "reducer" + }, + "skills": { + "mongodb": [], + "postgres": [], + "spacetime": ["typescript-server", "typescript-client"] + }, + "credentialAliases": { + "stackbench-admin-2026": "store-admin-2026", + "stackbench-customer-2026": "store-customer-2026", + "stackbench-staff-2026": "store-staff-2026" + } +} diff --git a/tools/stack-bench/conditions/repairs/scored-only.json b/tools/stack-bench/conditions/repairs/scored-only.json new file mode 100644 index 00000000000..d5edc402c57 --- /dev/null +++ b/tools/stack-bench/conditions/repairs/scored-only.json @@ -0,0 +1,10 @@ +{ + "schemaVersion": 1, + "kind": "repair-policy", + "id": "scored-only", + "version": "1.1.0", + "state": "qualified", + "scoredEvidence": true, + "observedEvidence": false, + "scenarioValues": "withheld" +} diff --git a/tools/stack-bench/container/Dockerfile b/tools/stack-bench/container/Dockerfile new file mode 100644 index 00000000000..f41c6073afd --- /dev/null +++ b/tools/stack-bench/container/Dockerfile @@ -0,0 +1,42 @@ +# The image a generated app is built in. +# +# The generated app must not see the harness, grader, or test definitions. +# +# No harness, grader, or test definition is copied in or mounted at run time. +# An adapter can mount only its selected stack artifacts, read-only. +# Keep the readable tag, but bind the base to an exact manifest. Campaigns use +# the digest of the completed image, so every attempt runs the same artifact. +FROM node:22-slim@sha256:f86be15afa9a8277608e141ce2a8aa55d3d9c40845921b8511f4fb7897be2554 + +# git: builds initialise repositories and some tooling shells out to it. +# curl: readiness probes against the app's own dev server. +# ca-certificates: TLS for npm and the API. +# procps: the build starts and stops its own dev servers. +# lsof: `kill-port` locates a listener with lsof on Linux, and finds nothing +# without it — it then prints "Process on port N killed" and exits 0 while the +# server keeps running. Every durability and deploy-window test would pass +# without restarting anything, which is worse than failing. None of lsof, fuser, +# ss or netstat is present in node:22-slim. +RUN apt-get update && apt-get install -y --no-install-recommends \ + git curl ca-certificates procps lsof \ + && rm -rf /var/lib/apt/lists/* + +# Pinned, and the auto-updater disabled: a CLI that updates itself mid-series +# changes the thing under test between one backend and the next. Override at +# build time to move deliberately rather than by drift. +ARG CLAUDE_VERSION=2.1.226 +ENV DISABLE_AUTOUPDATER=1 +RUN npm install -g @anthropic-ai/claude-code@${CLAUDE_VERSION} \ + && claude --version + +# The coding session can change the app and its own temporary state. It does +# not run as root, so it cannot change the system or harness control files. +RUN useradd --uid 10001 --create-home --shell /bin/bash developer \ + && chmod 0700 /home/developer + +# The app under construction. Everything the build writes lives here, and the +# host mounts its own work directory over it. +WORKDIR /app + +# No ENTRYPOINT: the run-build command supplies the whole command so the prompt can go +# in on stdin exactly as it does on the host. diff --git a/tools/stack-bench/container/binary-provenance.ts b/tools/stack-bench/container/binary-provenance.ts new file mode 100644 index 00000000000..a7cd6197562 --- /dev/null +++ b/tools/stack-bench/container/binary-provenance.ts @@ -0,0 +1,199 @@ +#!/usr/bin/env node + +import { createHash, randomBytes } from 'node:crypto'; +import { existsSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from 'node:fs'; +import { join, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; + +import { binarySourceIdentity, SOURCE_IDENTITY_SCHEME } + from '../src/releases/release-source.js'; +import { STACK_BENCH_RUNNER_PLATFORM } from '../src/runtime/runner-environment.js'; + +export const RUST_BUILDER_IMAGE = + 'rust:1.93-slim-bookworm@sha256:8f8609d448e821fbc0e44241bc5ca4ce49663cc6306ff1a17f655a0e2a7cd084'; +export const BINARY_NAMES = Object.freeze(['spacetimedb-cli', 'spacetimedb-standalone']); +const PROVENANCE_NAME = 'spacetimedb-binaries.json'; + +interface BinarySourceIdentity { + identityScheme: typeof SOURCE_IDENTITY_SCHEME; + revision: string; + sha256: string; + files: number; +} + +interface BinaryRecord { + sha256: string; + size: number; +} + +interface BinaryProvenance { + schemaVersion: 2; + platform: typeof STACK_BENCH_RUNNER_PLATFORM; + builderImage: string; + source: BinarySourceIdentity; + binaries: Record; +} + +function sha256File(path: string): string { + return createHash('sha256').update(readFileSync(path)).digest('hex'); +} + +function binaryPath(stackBenchRoot: string, name: string): string { + return join(stackBenchRoot, 'container', 'bin', name); +} + +function provenancePath(stackBenchRoot: string): string { + return join(stackBenchRoot, 'container', PROVENANCE_NAME); +} + +function assertSha256(value: unknown, label: string): asserts value is string { + if (typeof value !== 'string' || !/^[a-f0-9]{64}$/.test(value)) { + throw new Error(`${label} must be a SHA-256 digest`); + } +} + +function inspectBinary(path: string, name: string): BinaryRecord { + if (!existsSync(path)) { + throw new Error(`${name} is absent; run tools/stack-bench/container/build-linux-cli.sh`); + } + const stat = statSync(path); + if (!stat.isFile() || stat.size < 4) throw new Error(`${name} is not a non-empty file`); + const magic = readFileSync(path).subarray(0, 4); + if (!magic.equals(Buffer.from([0x7f, 0x45, 0x4c, 0x46]))) { + throw new Error(`${name} is not a Linux ELF binary`); + } + return { sha256: sha256File(path), size: stat.size }; +} + +export function createBinaryProvenance(stackBenchRoot: string, + source: BinarySourceIdentity): BinaryProvenance { + if (source?.identityScheme !== SOURCE_IDENTITY_SCHEME) { + throw new Error('binary source identity scheme is unsupported'); + } + assertSha256(source?.sha256, 'binary source identity'); + if (!/^[a-f0-9]{40}(?:[a-f0-9]{24})?$/.test(source?.revision ?? '')) { + throw new Error('binary source revision must be an exact commit id'); + } + if (!Number.isSafeInteger(source?.files) || source.files < 1) { + throw new Error('binary source file count must be a positive integer'); + } + const binaries: Record = {}; + for (const name of BINARY_NAMES) binaries[name] = inspectBinary(binaryPath(stackBenchRoot, name), name); + return { + schemaVersion: 2, + platform: STACK_BENCH_RUNNER_PLATFORM, + builderImage: RUST_BUILDER_IMAGE, + source: { identityScheme: source.identityScheme, + revision: source.revision, sha256: source.sha256, files: source.files }, + binaries, + }; +} + +export function assertBinarySourceUnchanged(before: BinarySourceIdentity, + after: BinarySourceIdentity): void { + if (before?.identityScheme !== after?.identityScheme + || before?.revision !== after?.revision || before?.sha256 !== after?.sha256 + || before?.files !== after?.files) { + throw new Error('binary source changed during the build'); + } +} + +function readProvenance(stackBenchRoot: string): BinaryProvenance { + const path = provenancePath(stackBenchRoot); + if (!existsSync(path)) { + throw new Error(`${PROVENANCE_NAME} is absent; run tools/stack-bench/container/build-linux-cli.sh`); + } + let manifest: BinaryProvenance & { status?: string }; + try { manifest = JSON.parse(readFileSync(path, 'utf8')); } + catch (error) { + throw new Error(`${PROVENANCE_NAME} is not valid JSON: ${error instanceof Error + ? error.message : String(error)}`); + } + if (manifest.status === 'unbuilt') { + throw new Error(`${PROVENANCE_NAME} has no verified binaries; run tools/stack-bench/container/build-linux-cli.sh`); + } + return manifest; +} + +export function verifyBinaryProvenance(stackBenchRoot: string, + { sourceSha256 }: { sourceSha256: string }): BinaryProvenance { + assertSha256(sourceSha256, 'expected binary source identity'); + const manifest = readProvenance(stackBenchRoot); + if (manifest.schemaVersion !== 2) throw new Error('unsupported binary provenance schema'); + if (manifest.platform !== STACK_BENCH_RUNNER_PLATFORM) { + throw new Error(`binary provenance platform must be ${STACK_BENCH_RUNNER_PLATFORM}`); + } + if (manifest.builderImage !== RUST_BUILDER_IMAGE) { + throw new Error('binary provenance does not use the pinned Rust builder image'); + } + assertSha256(manifest.source?.sha256, 'recorded binary source identity'); + if (manifest.source.identityScheme !== SOURCE_IDENTITY_SCHEME) { + throw new Error('recorded binary source identity scheme is unsupported'); + } + if (manifest.source.sha256 !== sourceSha256) { + throw new Error('SpacetimeDB binaries do not match the selected release source'); + } + if (!/^[a-f0-9]{40}(?:[a-f0-9]{24})?$/.test(manifest.source?.revision ?? '')) { + throw new Error('recorded binary source revision is invalid'); + } + if (!Number.isSafeInteger(manifest.source?.files) || manifest.source.files < 1) { + throw new Error('recorded binary source file count is invalid'); + } + for (const name of BINARY_NAMES) { + const expected = manifest.binaries?.[name]; + assertSha256(expected?.sha256, `${name} recorded checksum`); + if (!Number.isSafeInteger(expected.size) || expected.size < 4) { + throw new Error(`${name} recorded size is invalid`); + } + const actual = inspectBinary(binaryPath(stackBenchRoot, name), name); + if (actual.size !== expected.size) throw new Error(`${name} size does not match provenance`); + if (actual.sha256 !== expected.sha256) throw new Error(`${name} checksum does not match provenance`); + } + return manifest; +} + +function option(args: string[], name: string): string { + const index = args.indexOf(name); + const value = index === -1 ? undefined : args[index + 1]; + if (!value) throw new Error(`${name} is required`); + return value; +} + +function main(): void { + const [command, ...args] = process.argv.slice(2); + if (command === 'source') { + const repo = resolve(option(args, '--repo')); + console.log(JSON.stringify(binarySourceIdentity(repo), null, 2)); + return; + } + if (command === 'record') { + const repo = resolve(option(args, '--repo')); + const stackBenchRoot = join(repo, 'tools', 'stack-bench'); + const source = JSON.parse(readFileSync(resolve(option(args, '--source-file')), 'utf8')); + const current = binarySourceIdentity(repo); + assertBinarySourceUnchanged(source, current); + const manifest = createBinaryProvenance(stackBenchRoot, source); + const path = provenancePath(stackBenchRoot); + const temporary = `${path}.${process.pid}.${randomBytes(8).toString('hex')}.tmp`; + writeFileSync(temporary, `${JSON.stringify(manifest, null, 2)}\n`, { flag: 'wx' }); + try { renameSync(temporary, path); } + catch (error) { rmSync(temporary, { force: true }); throw error; } + console.log(`recorded ${path}`); + return; + } + if (command === 'verify') { + const stackBenchRoot = resolve(option(args, '--root')); + verifyBinaryProvenance(stackBenchRoot, { sourceSha256: option(args, '--source-sha256') }); + console.log('verified SpacetimeDB CLI and standalone binary provenance'); + return; + } + throw new Error('Usage: binary-provenance source --repo PATH | record --repo PATH --source-file PATH | verify --root PATH --source-sha256 SHA256'); +} + +if (process.argv[1] && pathToFileURL(resolve(process.argv[1])).href === import.meta.url) { + try { main(); } + catch (error) { + console.error(`binary provenance failed: ${error instanceof Error ? error.message : String(error)}`); + process.exit(1); + } +} diff --git a/tools/stack-bench/container/build-container-inspection.ts b/tools/stack-bench/container/build-container-inspection.ts new file mode 100644 index 00000000000..768be6980e7 --- /dev/null +++ b/tools/stack-bench/container/build-container-inspection.ts @@ -0,0 +1,188 @@ +import { spawnSync } from 'node:child_process'; +import type { SpawnSyncOptionsWithStringEncoding, SpawnSyncReturns } from 'node:child_process'; +import { resolve } from 'node:path'; + +import { LEGACY_SUBSCRIPTION_TOKEN_TARGET } from './container-auth.js'; +import type { ContainerMount } from '../src/runtime/container-mount.js'; + +type InspectedMount = { + type: string; + source: string; + name: string | null; + destination: string; + readOnly: boolean; +}; + +export type InspectedBuildContainer = { + id: string; + image: string; + running: boolean; + networkMode: string | null; + readonlyRootfs: boolean; + tmpfs: Record; + capAdd: string[]; + capDrop: string[]; + securityOpt: string[]; + pidsLimit: number | null; + nanoCpus: number | null; + memoryBytes: number | null; + memorySwapBytes: number | null; + mounts: InspectedMount[]; + unsafeCredentialExposure: boolean; +}; + +type DockerExecute = (command: string, args: readonly string[], + options: SpawnSyncOptionsWithStringEncoding) => SpawnSyncReturns; + +type JsonRecord = Record; + +function isRecord(value: unknown): value is JsonRecord { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +function stringArray(value: unknown): string[] { + return Array.isArray(value) ? value.filter((item): item is string => typeof item === 'string') : []; +} + +function numberOrNull(value: unknown): number | null { + return typeof value === 'number' ? value : null; +} + +function dockerDetail(result: SpawnSyncReturns): string { + return String(result.stderr || result.stdout || result.error?.message || `exit ${result.status}`).trim(); +} + +export function parseCgroupMemory(value: string) { + const bytes = (name: string): number | null => { + const match = value.match(new RegExp(`^\\[${name.replace('.', '\\.')}]\\r?\\n(\\d+)$`, 'm')); + if (!match) return null; + const parsed = Number(match[1]); + return Number.isSafeInteger(parsed) ? parsed : null; + }; + return { + currentBytes: bytes('memory.current'), + peakBytes: bytes('memory.peak'), + limitBytes: bytes('memory.max'), + }; +} + +export function inspectBuildContainer(name: string, { + env = process.env, + timeoutMs = 120_000, + execute = spawnSync as DockerExecute, +}: { env?: NodeJS.ProcessEnv; timeoutMs?: number; execute?: DockerExecute } = {}): InspectedBuildContainer | null { + const result = execute('docker', ['inspect', name], { encoding: 'utf8', env, timeout: timeoutMs }); + if (result.status !== 0) { + const detail = dockerDetail(result); + if (/no such (?:object|container)/i.test(detail)) return null; + throw new Error(`cannot inspect build container ${name}: ${detail}`); + } + + let parsed: unknown; + try { parsed = JSON.parse(result.stdout); } + catch (error) { + throw new Error(`Docker returned invalid inspection JSON for ${name}: ${error instanceof Error + ? error.message : String(error)}`); + } + if (!Array.isArray(parsed) || !isRecord(parsed[0])) { + throw new Error(`Docker returned an invalid container inspection for ${name}`); + } + + const inspected = parsed[0]; + const mounts = Array.isArray(inspected.Mounts) ? inspected.Mounts.filter(isRecord) : []; + const config = isRecord(inspected.Config) ? inspected.Config : {}; + const hostConfig = isRecord(inspected.HostConfig) ? inspected.HostConfig : {}; + const state = isRecord(inspected.State) ? inspected.State : {}; + const sensitiveTargets = new Set([LEGACY_SUBSCRIPTION_TOKEN_TARGET, '/root/.claude/.credentials.json']); + const capabilities = (values: unknown): string[] => stringArray(values).map(value => value.replace(/^CAP_/, '')); + const tmpfs = isRecord(hostConfig.Tmpfs) + ? Object.fromEntries(Object.entries(hostConfig.Tmpfs) + .filter((entry): entry is [string, string] => typeof entry[1] === 'string')) : {}; + + return { + id: String(inspected.Id), + image: String(inspected.Image), + running: state.Running === true, + networkMode: typeof hostConfig.NetworkMode === 'string' ? hostConfig.NetworkMode : null, + readonlyRootfs: hostConfig.ReadonlyRootfs === true, + tmpfs, + capAdd: capabilities(hostConfig.CapAdd), + capDrop: capabilities(hostConfig.CapDrop), + securityOpt: stringArray(hostConfig.SecurityOpt).map(option => option.replace(/:true$/, '')), + pidsLimit: numberOrNull(hostConfig.PidsLimit), + nanoCpus: numberOrNull(hostConfig.NanoCpus), + memoryBytes: numberOrNull(hostConfig.Memory), + memorySwapBytes: numberOrNull(hostConfig.MemorySwap), + mounts: mounts.map(mount => ({ + type: String(mount.Type), + source: String(mount.Source), + name: typeof mount.Name === 'string' ? mount.Name : null, + destination: String(mount.Destination), + readOnly: mount.RW !== true, + })), + unsafeCredentialExposure: mounts.some(mount => sensitiveTargets.has(String(mount.Destination))) + || stringArray(config.Env).some(value => /^(?:ANTHROPIC_API_KEY|CLAUDE_CODE_OAUTH_TOKEN)=/.test(value)), + }; +} + +export function sameHostPath(left: string, right: string, + platform: NodeJS.Platform = process.platform): boolean { + const normalize = (value: string): string => resolve(value).replaceAll('\\', '/'); + const normalizedLeft = normalize(left); + const normalizedRight = normalize(right); + return platform === 'win32' + ? normalizedLeft.toLowerCase() === normalizedRight.toLowerCase() + : normalizedLeft === normalizedRight; +} + +export function parsePublishedPorts(value: string | undefined): string[] { + if (!value) return []; + const ports = value.split(',').map(port => port.trim()).filter(Boolean); + if (ports.some(port => !/^\d+$/.test(port) || Number(port) < 1 || Number(port) > 65_535)) { + throw new Error('--ports must contain integers from 1 through 65535'); + } + if (new Set(ports).size !== ports.length) throw new Error('--ports must not contain duplicates'); + return ports; +} + +export function hasRequiredBuildContainerIsolation(container: InspectedBuildContainer, { + expectedMounts, + requiredTmpfs, + requiredCapabilities, + pidsLimit, + cpuCount, + memoryBytes, + memorySwapBytes, + image, +}: { + expectedMounts: ContainerMount[]; + requiredTmpfs: Readonly>; + requiredCapabilities: readonly string[]; + pidsLimit: number; + cpuCount: number; + memoryBytes: number; + memorySwapBytes: number; + image: string; +}): boolean { + const mountsMatch = container.mounts.length === expectedMounts.length + && expectedMounts.every(expected => container.mounts.some(actual => + actual.type === (expected.kind ?? 'bind') + && actual.destination === expected.target + && actual.readOnly === expected.readOnly + && (expected.kind === 'volume' + ? actual.name === expected.source + : sameHostPath(actual.source, expected.source)))); + return container.readonlyRootfs + && Object.entries(requiredTmpfs).every(([path, options]) => container.tmpfs[path] === options) + && Object.keys(container.tmpfs).length === Object.keys(requiredTmpfs).length + && requiredCapabilities.every(capability => container.capAdd.includes(capability)) + && container.capAdd.length === requiredCapabilities.length + && container.capDrop.includes('ALL') + && container.securityOpt.includes('no-new-privileges') + && container.pidsLimit === pidsLimit + && container.nanoCpus === cpuCount * 1_000_000_000 + && container.memoryBytes === memoryBytes + && container.memorySwapBytes === memorySwapBytes + && container.image === image + && mountsMatch; +} diff --git a/tools/stack-bench/container/build-linux-cli.sh b/tools/stack-bench/container/build-linux-cli.sh new file mode 100644 index 00000000000..5e9f45be8c7 --- /dev/null +++ b/tools/stack-bench/container/build-linux-cli.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +# Build the SpacetimeDB CLI for Linux, from THIS repository, so a containerised +# build can publish modules with the CLI actually under test. +# +# The benchmark's whole claim about SpacetimeDB rests on measuring the software +# in this checkout rather than a published release. `target/release/ +# spacetimedb-cli.exe` is a Windows PE binary and a Linux container cannot run +# it, so without this the SpacetimeDB backend falls back to running on the host +# and loses the isolation every other backend gets. +# +# Outputs: tools/stack-bench/container/bin/spacetimedb-cli and +# tools/stack-bench/container/bin/spacetimedb-standalone (Linux ELFs) +# +# Usage: bash tools/stack-bench/container/build-linux-cli.sh +set -euo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO="$(cd "$HERE/../../.." && pwd)" +OUT="$HERE/bin" + +# This digest is the linux/amd64 manifest for rust:1.93-slim-bookworm. Update it +# with rust-toolchain.toml and the expected image in the provenance command. +IMAGE="rust:1.93-slim-bookworm@sha256:8f8609d448e821fbc0e44241bc5ca4ce49663cc6306ff1a17f655a0e2a7cd084" + +# Cargo's target directory is a named volume, not a path in the repo. A Rust +# build against a Windows bind mount is many times slower, and it would also sit +# next to the Windows artifacts in target/ where the wrong one is easy to pick up +# so container runs cannot silently execute a stale or host-platform binary. +VOLUME="${STACK_BENCH_CARGO_VOLUME:-stack-bench-cargo-target}" + +SOURCE_RECORD="$(mktemp)" +trap 'rm -f "$SOURCE_RECORD"' EXIT +(cd "$HERE/.." && npm run build --silent) +PROVENANCE="$HERE/../dist/container/binary-provenance.js" +node "$PROVENANCE" source --repo "$REPO" >"$SOURCE_RECORD" + +mkdir -p "$OUT" +docker volume create "$VOLUME" >/dev/null + +echo "building spacetimedb-cli for linux (image $IMAGE, target volume $VOLUME)" +echo " first build compiles the whole workspace and takes a while; later ones reuse the volume" + +# Git Bash must not rewrite container-side paths in this Docker command. Keep +# the setting local so host-side Node paths still convert normally. +MSYS_NO_PATHCONV=1 docker run --rm --platform linux/amd64 \ + -v "$REPO:/src" \ + -v "$VOLUME:/target" \ + -v "$OUT:/out" \ + -w /src \ + -e CARGO_TARGET_DIR=/target \ + -e CARGO_TERM_COLOR=never \ + "$IMAGE" \ + bash -c ' + set -euo pipefail + # pkg-config/libssl: the CLI links openssl. clang/cmake: some transitive + # build scripts need them. Installed here rather than baked into an image so + # this script stays a single file with nothing to keep in sync. + apt-get update -qq + apt-get install -y -qq --no-install-recommends \ + pkg-config libssl-dev build-essential clang cmake perl git curl python3 >/dev/null + rustup show >/dev/null # honours rust-toolchain.toml + cargo build --release --locked \ + -p spacetimedb-cli --bin spacetimedb-cli \ + -p spacetimedb-standalone --bin spacetimedb-standalone + cp /target/release/spacetimedb-cli /out/spacetimedb-cli + cp /target/release/spacetimedb-standalone /out/spacetimedb-standalone + chmod +x /out/spacetimedb-cli /out/spacetimedb-standalone + ' + +echo "" +file "$OUT/spacetimedb-cli" 2>/dev/null || true +file "$OUT/spacetimedb-standalone" 2>/dev/null || true +MSYS_NO_PATHCONV=1 docker run --rm -v "$OUT:/deps:ro" "${STACK_BENCH_IMAGE:-stack-bench-build:2.1.226}" \ + sh -c 'test -x /deps/spacetimedb-standalone && /deps/spacetimedb-cli --version' + +node "$PROVENANCE" record --repo "$REPO" --source-file "$SOURCE_RECORD" diff --git a/tools/stack-bench/container/container-auth.ts b/tools/stack-bench/container/container-auth.ts new file mode 100644 index 00000000000..6bbe629593a --- /dev/null +++ b/tools/stack-bench/container/container-auth.ts @@ -0,0 +1,54 @@ +import { existsSync, readFileSync } from 'node:fs'; +import type { PathLike } from 'node:fs'; +import { isAbsolute, resolve } from 'node:path'; + +export const SUBSCRIPTION_TOKEN_ENVIRONMENT = 'CLAUDE_CODE_OAUTH_TOKEN'; +export const LEGACY_SUBSCRIPTION_TOKEN_TARGET = '/run/secrets/claude-code-oauth-token'; + +export type ContainerAuth = { + mode: 'api-key' | 'subscription-token'; + credential: string; +}; + +type ReadTextFile = (path: PathLike | number, encoding: BufferEncoding) => string; + +export interface ResolveContainerAuthOptions { + apiKey?: string; + env?: NodeJS.ProcessEnv; + credentialsPath?: string; + exists?: (path: PathLike) => boolean; + read?: ReadTextFile; +} + +export function resolveContainerAuth({ apiKey = '', env = process.env, credentialsPath, + exists = existsSync, read = readFileSync as ReadTextFile }: ResolveContainerAuthOptions = {}): ContainerAuth { + const token = String(env[SUBSCRIPTION_TOKEN_ENVIRONMENT] ?? '').trim(); + const tokenFileValue = String(env[`${SUBSCRIPTION_TOKEN_ENVIRONMENT}_FILE`] ?? '').trim(); + if (token && tokenFileValue) { + throw new Error(`use only one of ${SUBSCRIPTION_TOKEN_ENVIRONMENT} and ` + + `${SUBSCRIPTION_TOKEN_ENVIRONMENT}_FILE`); + } + if (apiKey && (token || tokenFileValue)) { + throw new Error('use only one of API-key and subscription-token authentication'); + } + if (apiKey) return { mode: 'api-key', credential: apiKey }; + if (token) return { mode: 'subscription-token', credential: token }; + if (tokenFileValue) { + if (!isAbsolute(tokenFileValue)) { + throw new Error(`${SUBSCRIPTION_TOKEN_ENVIRONMENT}_FILE must be an absolute path`); + } + const source = resolve(tokenFileValue); + if (!exists(source)) throw new Error(`subscription token file does not exist: ${source}`); + const credential = String(read(source, 'utf8')).trim(); + if (!credential) { + throw new Error(`subscription token file is empty: ${source}`); + } + return { mode: 'subscription-token', credential }; + } + if (credentialsPath && exists(credentialsPath)) { + throw new Error('rotating Claude credential files cannot be isolated from generated shell commands; ' + + 'select an API key or CLAUDE_CODE_OAUTH_TOKEN_FILE'); + } + throw new Error(`no API key, ${SUBSCRIPTION_TOKEN_ENVIRONMENT}, ` + + `${SUBSCRIPTION_TOKEN_ENVIRONMENT}_FILE, or credentials file is available`); +} diff --git a/tools/stack-bench/container/credential-broker-accounting.ts b/tools/stack-bench/container/credential-broker-accounting.ts new file mode 100644 index 00000000000..594b1287a3d --- /dev/null +++ b/tools/stack-bench/container/credential-broker-accounting.ts @@ -0,0 +1,310 @@ +import { randomBytes } from 'node:crypto'; +import { readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs'; +import { z } from 'zod'; + +import { normalizeClaudeUsage, priceClaudeUsage } from '../src/evidence/claude-usage-cost.js'; +import type { ClaudeUsage } from '../src/evidence/claude-usage-cost.js'; +import { validatePricingRates as validateSharedPricingRates } from '../src/evidence/pricing-authority.js'; +import { formatZodError } from '../src/zod-error.js'; + +export const BROKER_LEDGER_SCHEMA_VERSION = 4; +// Why a billable request was charged its cost ceiling instead of priced from +// the provider's usage: a 2xx response without complete usage (an aborted or +// errored stream, an oversized body), a response that broke off, or an +// upstream connection that failed. The ceiling makes spend an upper bound. +export const ESTIMATE_REASONS = ['no-usage', 'response-aborted', 'upstream-error'] as const; +export type EstimateReason = typeof ESTIMATE_REASONS[number]; +export type EstimateCounts = Record; +export const noEstimates = (): EstimateCounts => ({ 'no-usage': 0, 'response-aborted': 0, 'upstream-error': 0 }); +export const MAX_BROKER_OUTPUT_TOKENS = 128_000; +export const CLAUDE_USAGE_FIELDS = ['input', 'output', 'cacheRead', 'cacheWrite5m', 'cacheWrite1h'] as const; +const COST_TOLERANCE_USD = 0.0001; + +type JsonRecord = Record; +export type BrokerMode = 'api-key' | 'subscription-token'; +export type PricingRates = ReturnType; + +export type BrokerConfig = { + mode: BrokerMode; + credential: string; + sessionToken: string; + readyPath?: string; + parentPid?: number; + expiresAt?: number; + listenHost?: '127.0.0.1' | '0.0.0.0'; + ledgerPath?: string; + model: string; + maxOutputTokens: number; + maxBudgetUsd?: number | null; + pricingRates?: PricingRates; +}; + +export type BrokerLedger = { + schemaVersion: number; + model: string; + maxBudgetUsd: number | null; + acceptedRequests: number; + billableRequests: number; + completedBillableRequests: number; + estimatedBillableRequests: number; + estimatedByReason: EstimateCounts; + spentUsd: number; + reservedUsd: number; + usage: ClaudeUsage; + complete: boolean; + updatedAt: string; +}; + +// `costUsd` is what the broker charged: exact provider usage priced at the +// receipt's rates, plus the cost ceiling of every estimated request. With +// `exact` false it is an upper bound and `calculatedCostUsd`, priced from the +// exact usage alone, a lower bound. +export interface CredentialBrokerReceipt { + schemaVersion: 3; + source: 'credential-broker'; + model: string; + maxBudgetUsd: number; + costUsd: number; + cliCostUsd: number | null; + calculatedCostUsd: number | null; + usage: ClaudeUsage | null; + pricingRates: PricingRates | null; + exact: boolean; + estimatedRequests: number; + estimatedByReason: EstimateCounts; + complete: boolean; + reconciled: boolean; + error: string | null; +} + +export interface CredentialBrokerResult extends JsonRecord { + total_cost_usd: number; + usage?: ReturnType; + stack_bench_cost_receipt: CredentialBrokerReceipt; +} + +const positiveFinite = z.number().finite().positive(); +const nonNegativeFinite = z.number().finite().nonnegative(); +const nonNegativeSafeInteger = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER); +const usageSchema = z.strictObject({ + input: nonNegativeSafeInteger, + output: nonNegativeSafeInteger, + cacheRead: nonNegativeSafeInteger, + cacheWrite5m: nonNegativeSafeInteger, + cacheWrite1h: nonNegativeSafeInteger, +}); +const brokerConfigSchema = z.strictObject({ + mode: z.enum(['api-key', 'subscription-token']), + credential: z.string().min(16), + sessionToken: z.string().min(16), + readyPath: z.string().min(1).optional(), + parentPid: z.number().int().positive().max(Number.MAX_SAFE_INTEGER).optional(), + expiresAt: positiveFinite.optional(), + listenHost: z.enum(['127.0.0.1', '0.0.0.0']).optional(), + ledgerPath: z.string().min(1).optional(), + model: z.string().min(1), + maxOutputTokens: z.number().int().min(1).max(MAX_BROKER_OUTPUT_TOKENS), + maxBudgetUsd: positiveFinite.nullable().optional(), + pricingRates: z.unknown().optional(), +}).superRefine((value, context) => { + if (value.expiresAt !== undefined && value.expiresAt <= Date.now()) { + context.addIssue({ code: 'custom', path: ['expiresAt'], message: 'must be in the future' }); + } +}); +const brokerLedgerSchema = z.strictObject({ + schemaVersion: z.literal(BROKER_LEDGER_SCHEMA_VERSION), + model: z.string().min(1), + maxBudgetUsd: positiveFinite.nullable(), + acceptedRequests: nonNegativeSafeInteger, + billableRequests: nonNegativeSafeInteger, + completedBillableRequests: nonNegativeSafeInteger, + estimatedBillableRequests: nonNegativeSafeInteger, + estimatedByReason: z.strictObject({ + 'no-usage': nonNegativeSafeInteger, + 'response-aborted': nonNegativeSafeInteger, + 'upstream-error': nonNegativeSafeInteger, + }), + spentUsd: nonNegativeFinite, + reservedUsd: nonNegativeFinite, + usage: usageSchema, + complete: z.boolean(), + updatedAt: z.string().refine(value => !Number.isNaN(Date.parse(value)), 'must be a timestamp'), +}); + +function isRecord(value: unknown): value is JsonRecord { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +function isNumber(value: unknown): value is number { + return typeof value === 'number'; +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +function fail(message: string): never { + throw new Error(`credential broker: ${message}`); +} + +export function validatePricingRates(value: unknown): PricingRates { + try { return validateSharedPricingRates(value, { at: 'pricingRates' }); } + catch (error) { return fail(errorMessage(error)); } +} + +export function priceNormalizedClaudeUsage(usage: ClaudeUsage, rates: PricingRates): number { + return priceClaudeUsage({ + input_tokens: usage.input, + output_tokens: usage.output, + cache_read_input_tokens: usage.cacheRead, + cache_creation: { + ephemeral_5m_input_tokens: usage.cacheWrite5m, + ephemeral_1h_input_tokens: usage.cacheWrite1h, + }, + }, rates); +} + +function rawUsage(usage: ClaudeUsage): JsonRecord { + return { + input_tokens: usage.input, + output_tokens: usage.output, + cache_read_input_tokens: usage.cacheRead, + cache_creation_input_tokens: usage.cacheWrite5m + usage.cacheWrite1h, + cache_creation: { + ephemeral_5m_input_tokens: usage.cacheWrite5m, + ephemeral_1h_input_tokens: usage.cacheWrite1h, + }, + }; +} + +function brokerCoversCliUsage(broker: ClaudeUsage, cli: ClaudeUsage): boolean { + return broker.input >= cli.input + && broker.output >= cli.output + && broker.cacheRead >= cli.cacheRead + && broker.cacheWrite5m + broker.cacheWrite1h >= cli.cacheWrite5m + cli.cacheWrite1h; +} + +export function validateBrokerConfig(value: unknown): BrokerConfig { + const parsed = brokerConfigSchema.safeParse(value); + if (!parsed.success) fail(formatZodError(parsed.error, 'configuration')); + const { pricingRates, ...config } = parsed.data; + return config.maxBudgetUsd === null || config.maxBudgetUsd === undefined + ? config + : { ...config, pricingRates: validatePricingRates(pricingRates) }; +} + +function validateLedger(value: unknown, + { model = null, maxBudgetUsd = undefined }: { model?: string | null; maxBudgetUsd?: number | null } = {}): BrokerLedger { + const parsed = brokerLedgerSchema.safeParse(value); + if (!parsed.success) fail(formatZodError(parsed.error, 'spend ledger')); + const ledger = parsed.data; + if (model !== null && ledger.model !== model) fail('spend ledger model does not match'); + if (maxBudgetUsd !== undefined && ledger.maxBudgetUsd !== maxBudgetUsd) { + fail('spend ledger budget does not match'); + } + if (ledger.completedBillableRequests > ledger.billableRequests) { + fail('spend ledger completed request count is invalid'); + } + if (ledger.estimatedBillableRequests > ledger.completedBillableRequests) { + fail('spend ledger estimated request count is invalid'); + } + const reasons = ESTIMATE_REASONS.reduce((sum, reason) => sum + ledger.estimatedByReason[reason], 0); + if (reasons !== ledger.estimatedBillableRequests) fail('spend ledger estimate reasons do not add up'); + const complete = ledger.reservedUsd === 0 + && ledger.completedBillableRequests === ledger.billableRequests; + if (ledger.complete !== complete) fail('spend ledger completion state is invalid'); + return ledger; +} + +export function writeCredentialBrokerLedger(path: string | undefined, value: unknown): void { + if (!path) return; + const ledger = validateLedger(value); + const temporary = `${path}.${process.pid}.${randomBytes(8).toString('hex')}.tmp`; + writeFileSync(temporary, `${JSON.stringify(ledger)}\n`, { flag: 'wx', mode: 0o600 }); + try { renameSync(temporary, path); } + catch (error) { rmSync(temporary, { force: true }); throw error; } +} + +export function readCredentialBrokerLedger(path: string, + expected: { model?: string | null; maxBudgetUsd?: number | null } = {}): BrokerLedger { + return validateLedger(JSON.parse(readFileSync(path, 'utf8')), expected); +} + +export function reconcileCredentialBrokerReceipt({ ledger, cliResult, model, maxBudgetUsd, + pricingRates, brokerDiagnostics = null, toleranceUsd = COST_TOLERANCE_USD }: { + ledger: unknown; cliResult: unknown; model: unknown; maxBudgetUsd: unknown; pricingRates: unknown; + brokerDiagnostics?: unknown; toleranceUsd?: number; +}): { ok: boolean; result: CredentialBrokerResult; receipt: CredentialBrokerReceipt } { + if (typeof model !== 'string' || !model) fail('receipt model is invalid'); + if (!isNumber(maxBudgetUsd) || !Number.isFinite(maxBudgetUsd) || maxBudgetUsd <= 0) fail('receipt budget is invalid'); + if (!Number.isFinite(toleranceUsd) || toleranceUsd < 0) fail('receipt tolerance is invalid'); + const receiptBudget = maxBudgetUsd; + let verifiedLedger: BrokerLedger | null = null; + let verifiedRates: PricingRates | null = null; + let usage: ClaudeUsage | null = null; + let cliUsage: ClaudeUsage | null = null; + let calculatedCostUsd: number | null = null; + let issue: string | null = null; + try { verifiedLedger = validateLedger(ledger, { model, maxBudgetUsd: receiptBudget }); } + catch (error) { issue = errorMessage(error); } + if (!issue && verifiedLedger?.complete !== true) issue = 'credential broker spend ledger is incomplete'; + const estimatedRequests = verifiedLedger?.estimatedBillableRequests ?? 0; + const exact = verifiedLedger !== null && estimatedRequests === 0; + try { verifiedRates = validatePricingRates(pricingRates); } + catch (error) { if (!issue) issue = errorMessage(error); } + try { cliUsage = normalizeClaudeUsage(isRecord(cliResult) ? cliResult.usage : undefined); } + catch (error) { if (!issue) issue = errorMessage(error); } + if (verifiedLedger) usage = structuredClone(verifiedLedger.usage); + if (!issue && cliUsage && usage && !brokerCoversCliUsage(usage, cliUsage)) { + issue = 'credential broker usage is lower than CLI usage totals'; + } + try { if (verifiedRates && usage) calculatedCostUsd = priceNormalizedClaudeUsage(usage, verifiedRates); } + catch (error) { if (!issue) issue = errorMessage(error); } + const brokerCost = verifiedLedger + ? Math.min(receiptBudget, verifiedLedger.spentUsd + verifiedLedger.reservedUsd) : receiptBudget; + const cliCost = Number(isRecord(cliResult) ? cliResult.total_cost_usd : undefined); + if (!issue && (!Number.isFinite(cliCost) || cliCost < 0)) { + issue = 'coding session did not return a usable cost receipt'; + } + // Exact spend must price back to the broker's figure. Estimated requests + // add their ceilings on top of the priced usage, so the priced usage can + // only fall below the broker's figure, never above it. + if (!issue && calculatedCostUsd !== null && exact && Math.abs(calculatedCostUsd - brokerCost) > toleranceUsd) { + issue = `usage-priced spend $${calculatedCostUsd.toFixed(6)} does not match credential broker spend $${brokerCost.toFixed(6)}`; + } + if (!issue && calculatedCostUsd !== null && !exact && calculatedCostUsd - brokerCost > toleranceUsd) { + issue = `usage-priced spend $${calculatedCostUsd.toFixed(6)} exceeds credential broker spend $${brokerCost.toFixed(6)}`; + } + const receipt: CredentialBrokerReceipt = { + schemaVersion: 3, + source: 'credential-broker', + model, + maxBudgetUsd: receiptBudget, + costUsd: Number(brokerCost.toFixed(6)), + cliCostUsd: Number.isFinite(cliCost) && cliCost >= 0 ? Number(cliCost.toFixed(6)) : null, + calculatedCostUsd: calculatedCostUsd === null ? null : Number(calculatedCostUsd.toFixed(6)), + usage, + pricingRates: verifiedRates, + exact, + estimatedRequests, + estimatedByReason: verifiedLedger ? structuredClone(verifiedLedger.estimatedByReason) : noEstimates(), + complete: verifiedLedger?.complete === true, + reconciled: issue === null, + error: issue, + }; + const result: CredentialBrokerResult = { + ...(isRecord(cliResult) + ? structuredClone(cliResult) : { type: 'result', is_error: true, result: '' }), + total_cost_usd: receipt.costUsd, + stack_bench_cost_receipt: receipt, + }; + if (usage) result.usage = rawUsage(usage); + if (brokerDiagnostics) result.stack_bench_credential_broker = structuredClone(brokerDiagnostics); + if (issue) { + result.is_error = true; + result.terminal_reason = 'cost_receipt_error'; + result.result = [typeof result.result === 'string' ? result.result.trim() : '', issue] + .filter(Boolean).join('\n'); + } + return { ok: issue === null, result, receipt }; +} diff --git a/tools/stack-bench/container/credential-broker-process.ts b/tools/stack-bench/container/credential-broker-process.ts new file mode 100644 index 00000000000..f773269a6ee --- /dev/null +++ b/tools/stack-bench/container/credential-broker-process.ts @@ -0,0 +1,318 @@ +import { spawn } from 'node:child_process'; +import type { ChildProcess } from 'node:child_process'; +import { randomBytes } from 'node:crypto'; +import { chmodSync, existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { setTimeout as wait } from 'node:timers/promises'; + +import type { ContainerAuth } from './container-auth.js'; +import { MAX_BROKER_OUTPUT_TOKENS, readCredentialBrokerLedger, validateBrokerConfig } + from './credential-broker-accounting.js'; +import type { BrokerLedger, PricingRates } from './credential-broker-accounting.js'; +import { compiledEntrypoint } from '../src/package-root.js'; +import { killTree } from '../src/runtime/platform.js'; + +const BROKER_DRAIN_TIMEOUT_MS = 30_000; +const BROKER_DRAIN_POLL_MS = 100; +const BROKER_STDERR_LIMIT_BYTES = 16 * 1024; +const BROKER_STOP_GRACE_MS = 2_000; +const BROKER_STOP_FORCE_MS = 2_000; + +type JsonRecord = Record; +type Alive = (pid: number | undefined) => boolean; + +export type BrokerProcessState = { + exitCode: number | null; + signal: NodeJS.Signals | null; + exitedAt: string | null; + stderrTail: string; + stderrPending: string; + stderrTruncated: boolean; +}; + +export type BrokerError = { type: string; phase: string; message: string }; + +export type BrokerDiagnostics = { + schemaVersion: number; + endpointKind: string; + child: { pid: number | undefined | null; exitCode: number | null; signal: NodeJS.Signals | null; + exitedAt: string | null; stderrTail: string | null; stderrTruncated: boolean }; + drain: { timeoutMs: number; elapsedMs: number; timedOut: boolean; reason: string | null; + terminationRequested: boolean } | null; + termination: { gracefulRequested: boolean; forceRequested: boolean; exited: boolean; + gracefulTimeoutMs: number; forceTimeoutMs: number } | null; + ledger: BrokerLedger | null; + errors: BrokerError[]; +}; + +export interface CredentialBrokerChild { + pid?: number; + exitCode?: number | null; + signalCode?: NodeJS.Signals | null; + kill?: (signal?: NodeJS.Signals | number) => boolean; +} + +export interface CredentialBrokerHandle { + child: CredentialBrokerChild; + root: string; + ledgerPath: string; + model: string; + maxBudgetUsd: number | null; + sessionToken?: string; + baseUrl?: string; + listenHost?: string; + endpointKind?: string; + processState?: Partial; + diagnosticSecrets?: string[]; + finalDiagnostics?: BrokerDiagnostics | null; + finalLedger?: BrokerLedger | null; +} + +export interface CredentialBroker extends CredentialBrokerHandle { + child: ChildProcess; + sessionToken: string; + baseUrl: string; + listenHost: string; + endpointKind: string; + processState: BrokerProcessState; + diagnosticSecrets: string[]; + finalDiagnostics: BrokerDiagnostics | null; + finalLedger: BrokerLedger | null; +} + +function isRecord(value: unknown): value is JsonRecord { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +function fail(message: string): never { + throw new Error(`credential broker: ${message}`); +} + +function redactDiagnosticText(value: unknown, + broker: CredentialBrokerHandle | string[] | null | undefined): string { + let result = String(value ?? ''); + const secrets = Array.isArray(broker) ? broker : broker?.diagnosticSecrets ?? []; + for (const secret of secrets) { + if (typeof secret === 'string' && secret) result = result.replaceAll(secret, '[REDACTED]'); + } + return result; +} + +function appendDiagnosticStderr(state: BrokerProcessState, chunk: string, secrets: string[], flush = false): void { + const raw = state.stderrPending + chunk; + let redacted = redactDiagnosticText(raw, secrets); + let pendingLength = 0; + if (!flush) { + for (const secret of secrets) { + for (let length = Math.min(secret.length - 1, redacted.length); length > pendingLength; length -= 1) { + if (redacted.endsWith(secret.slice(0, length))) { pendingLength = length; break; } + } + } + } else if (raw && secrets.some(secret => secret.startsWith(raw))) redacted = '[REDACTED]'; + state.stderrPending = pendingLength ? redacted.slice(-pendingLength) : ''; + const safe = pendingLength ? redacted.slice(0, -pendingLength) : redacted; + const next = state.stderrTail + safe; + if (Buffer.byteLength(next) > BROKER_STDERR_LIMIT_BYTES) { + state.stderrTruncated = true; + state.stderrTail = Buffer.from(next).subarray(-BROKER_STDERR_LIMIT_BYTES).toString('utf8'); + } else state.stderrTail = next; +} + +export async function startCredentialBroker(selectedAuth: ContainerAuth, { networkMode, deadlineMs, + model, maxOutputTokens = MAX_BROKER_OUTPUT_TOKENS, maxBudgetUsd = null, pricingRates = null, + env = process.env }: { networkMode: 'bridge' | 'host'; deadlineMs: number; model: string; + maxOutputTokens?: number; maxBudgetUsd?: number | null; pricingRates?: PricingRates | null; + env?: NodeJS.ProcessEnv }): Promise { + if (!['bridge', 'host'].includes(networkMode)) fail('network mode is invalid'); + if (!Number.isFinite(deadlineMs) || deadlineMs <= 0) fail('deadline is invalid'); + const credential = selectedAuth.credential.trim(); + if (!credential) fail('selected authentication has no broker credential'); + const root = mkdtempSync(join(tmpdir(), 'stack-bench-credential-broker-')); + let child: ChildProcess | null = null; + const processState: BrokerProcessState = { exitCode: null, signal: null, exitedAt: null, + stderrTail: '', stderrPending: '', stderrTruncated: false }; + try { + chmodSync(root, 0o700); + const configPath = join(root, 'config.json'); + const readyPath = join(root, 'ready.json'); + const ledgerPath = join(root, 'spend-ledger.json'); + const sessionToken = randomBytes(32).toString('hex'); + const listenHost = networkMode === 'host' ? '127.0.0.1' : '0.0.0.0'; + const config = validateBrokerConfig({ mode: selectedAuth.mode, credential, sessionToken, readyPath, + parentPid: process.pid, expiresAt: Date.now() + deadlineMs + 60_000, listenHost, ledgerPath, + model, maxOutputTokens, maxBudgetUsd, pricingRates }); + writeFileSync(configPath, `${JSON.stringify(config)}\n`, { flag: 'wx', mode: 0o600 }); + child = spawn(process.execPath, [compiledEntrypoint('container', 'credential-broker.js'), + '--config', configPath], { + stdio: ['ignore', 'ignore', 'pipe'], + windowsHide: true, + env: Object.fromEntries(['PATH', 'Path', 'SystemRoot', 'WINDIR', 'SSL_CERT_FILE', + 'NODE_EXTRA_CA_CERTS', 'HTTPS_PROXY', 'HTTP_PROXY'] + .filter(name => env[name] !== undefined).map(name => [name, env[name]])), + }); + const diagnosticSecrets = [credential, sessionToken]; + child.stderr?.on('data', (chunk: Buffer) => appendDiagnosticStderr( + processState, chunk.toString('utf8'), diagnosticSecrets)); + child.once('exit', (code: number | null, signal: NodeJS.Signals | null) => { + appendDiagnosticStderr(processState, '', diagnosticSecrets, true); + processState.exitCode = code; + processState.signal = signal; + processState.exitedAt = new Date().toISOString(); + }); + let spawnError: Error | null = null; + child.once('error', (error: Error) => { spawnError = error; }); + const readyDeadline = Date.now() + 10_000; + while (!spawnError && child.exitCode === null && !existsSync(readyPath) && Date.now() < readyDeadline) { + await wait(100); + } + if (spawnError) throw spawnError; + if (!existsSync(readyPath)) throw new Error('credential broker did not become ready'); + const ready: unknown = JSON.parse(readFileSync(readyPath, 'utf8')); + if (!isRecord(ready) || typeof ready.port !== 'number' || !Number.isInteger(ready.port) + || ready.port < 1 || ready.port > 65_535) throw new Error('credential broker returned an invalid port'); + if (ready.host !== listenHost) throw new Error('credential broker returned an invalid host'); + const host = networkMode === 'host' ? '127.0.0.1' : 'host.docker.internal'; + return { child, root, ledgerPath, model, maxBudgetUsd: maxBudgetUsd ?? null, + sessionToken, baseUrl: `http://${host}:${ready.port}`, listenHost, + endpointKind: 'local-credential-broker', processState, + diagnosticSecrets, finalDiagnostics: null, finalLedger: null }; + } catch (error) { + if (child?.pid) killTree(child.pid); + rmSync(root, { recursive: true, force: true }); + throw error; + } +} + +function processAlive(pid: number | undefined): boolean { + if (typeof pid !== 'number' || !Number.isInteger(pid) || pid < 1) return false; + try { process.kill(pid, 0); return true; } + catch (error) { return !isRecord(error) || error.code !== 'ESRCH'; } +} + +function brokerExited(broker: CredentialBrokerHandle | null | undefined, alive: Alive): boolean { + const state = broker?.processState; + if (!state) return !alive(broker?.child?.pid); + if (state.exitedAt || state.exitCode !== null && state.exitCode !== undefined + || state.signal !== null && state.signal !== undefined + || broker?.child?.exitCode !== null && broker?.child?.exitCode !== undefined + || broker?.child?.signalCode !== null && broker?.child?.signalCode !== undefined) return true; + return !alive(broker?.child?.pid); +} + +async function waitForBrokerExit(broker: CredentialBrokerHandle, timeoutMs: number, + { sleep, now, alive }: { sleep: (ms: number) => Promise; now: () => number; alive: Alive }): Promise { + const deadline = now() + timeoutMs; + while (!brokerExited(broker, alive) && now() < deadline) await sleep(BROKER_DRAIN_POLL_MS); + return brokerExited(broker, alive); +} + +export function credentialBrokerDiagnostics(broker: CredentialBrokerHandle | null): BrokerDiagnostics | null { + if (!broker) return null; + if (broker.finalDiagnostics) return structuredClone(broker.finalDiagnostics); + const state = broker.processState ?? {}; + return { + schemaVersion: 1, + endpointKind: broker.endpointKind ?? 'local-credential-broker', + child: { pid: broker.child?.pid ?? null, + exitCode: state.exitCode ?? broker.child?.exitCode ?? null, + signal: state.signal ?? broker.child?.signalCode ?? null, + exitedAt: state.exitedAt ?? null, + stderrTail: state.stderrTail ? redactDiagnosticText(state.stderrTail, broker) : null, + stderrTruncated: state.stderrTruncated === true }, + drain: null, + termination: null, + ledger: null, + errors: [], + }; +} + +export async function stopCredentialBroker(broker: CredentialBrokerHandle | null, { + drainTimeoutMs = BROKER_DRAIN_TIMEOUT_MS, + pollMs = BROKER_DRAIN_POLL_MS, + gracefulTimeoutMs = BROKER_STOP_GRACE_MS, + forceTimeoutMs = BROKER_STOP_FORCE_MS, + readLedger = readCredentialBrokerLedger, + terminate = killTree, + requestStop = (child: CredentialBrokerChild) => child.kill?.('SIGTERM') ?? false, + alive = processAlive, + sleep = wait, + now = Date.now, +}: { drainTimeoutMs?: number; pollMs?: number; gracefulTimeoutMs?: number; forceTimeoutMs?: number; + readLedger?: typeof readCredentialBrokerLedger; terminate?: typeof killTree; + requestStop?: (child: CredentialBrokerChild) => boolean | void; alive?: Alive; + sleep?: (ms: number) => Promise; now?: () => number } = {}): Promise { + if (!broker) return null; + if (broker.finalDiagnostics) return structuredClone(broker.finalLedger ?? null); + let ledger: BrokerLedger | null = null; + const startedAt = now(); + let drainTimedOut = false; + let drainReason: string | null = null; + const errors: BrokerError[] = []; + const errorKeys = new Set(); + const recordError = (type: string, phase: string, error: unknown): void => { + const message = redactDiagnosticText(error instanceof Error ? error.message : error, broker) || 'unknown error'; + const key = `${type}:${phase}:${message}`; + if (errorKeys.has(key)) return; + errorKeys.add(key); + errors.push({ type, phase, message }); + }; + const read = (phase: string, expected: { model: string; maxBudgetUsd: number | null }): BrokerLedger | null => { + try { return readLedger(broker.ledgerPath, expected); } + catch (error) { recordError('ledger-read-error', phase, error); return null; } + }; + let gracefulRequested = false; + let forceRequested = false; + let exited = brokerExited(broker, alive); + const expected = { model: broker.model, maxBudgetUsd: broker.maxBudgetUsd }; + try { + const deadline = now() + drainTimeoutMs; + while (drainReason === null) { + ledger = read('drain', expected) ?? ledger; + if (ledger?.complete === true) { drainReason = 'ledger-complete'; break; } + exited = brokerExited(broker, alive); + if (exited) { drainReason = 'child-exited'; break; } + if (now() >= deadline) { drainTimedOut = true; drainReason = 'timeout'; break; } + await sleep(pollMs); + } + exited = brokerExited(broker, alive); + if (!exited) { + gracefulRequested = true; + try { requestStop(broker.child); } + catch (error) { recordError('termination-error', 'graceful-request', error); } + exited = await waitForBrokerExit(broker, gracefulTimeoutMs, { sleep, now, alive }); + } + if (!exited) { + forceRequested = true; + try { terminate(broker.child.pid); } + catch (error) { recordError('termination-error', 'force-request', error); } + exited = await waitForBrokerExit(broker, forceTimeoutMs, { sleep, now, alive }); + } + if (!exited) recordError('termination-error', 'exit-verification', + new Error('credential broker remained alive after forced termination')); + ledger = read('final', expected) ?? ledger; + } catch (error) { recordError('broker-stop-error', 'shutdown', error); } + finally { + const state = broker.processState ?? {}; + if (exited) { + try { rmSync(broker.root, { recursive: true, force: true }); } + catch (error) { recordError('cleanup-error', 'private-root', error); } + } + broker.finalLedger = ledger; + broker.finalDiagnostics = { + ...(credentialBrokerDiagnostics(broker) as BrokerDiagnostics), + child: { pid: broker.child?.pid ?? null, + exitCode: state.exitCode ?? broker.child?.exitCode ?? null, + signal: state.signal ?? broker.child?.signalCode ?? null, + exitedAt: state.exitedAt ?? null, + stderrTail: state.stderrTail ? redactDiagnosticText(state.stderrTail, broker) : null, + stderrTruncated: state.stderrTruncated === true }, + drain: { timeoutMs: drainTimeoutMs, elapsedMs: Math.max(0, now() - startedAt), + timedOut: drainTimedOut, reason: drainReason, terminationRequested: gracefulRequested }, + termination: { gracefulRequested, forceRequested, exited, gracefulTimeoutMs, forceTimeoutMs }, + ledger: ledger ? structuredClone(ledger) : null, + errors, + }; + } + return ledger; +} diff --git a/tools/stack-bench/container/credential-broker.ts b/tools/stack-bench/container/credential-broker.ts new file mode 100644 index 00000000000..9ad94469067 --- /dev/null +++ b/tools/stack-bench/container/credential-broker.ts @@ -0,0 +1,437 @@ +#!/usr/bin/env node +import { createServer } from 'node:http'; +import type { ClientRequest, IncomingMessage, OutgoingHttpHeaders, ServerResponse } from 'node:http'; +import { request as httpsRequest } from 'node:https'; +import type { RequestOptions } from 'node:https'; +import type { Socket } from 'node:net'; +import { readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { parseArgs as parseNodeArgs } from 'node:util'; +import type { AddressInfo } from 'node:net'; +import { brotliDecompressSync, gunzipSync, inflateSync } from 'node:zlib'; +import { createParser } from 'eventsource-parser'; + +import { normalizeClaudeUsage } from '../src/evidence/claude-usage-cost.js'; +import type { ClaudeUsage } from '../src/evidence/claude-usage-cost.js'; +import { BROKER_LEDGER_SCHEMA_VERSION, CLAUDE_USAGE_FIELDS, noEstimates, priceNormalizedClaudeUsage, + validateBrokerConfig, + writeCredentialBrokerLedger } from './credential-broker-accounting.js'; +import type { BrokerConfig, EstimateReason, PricingRates } + from './credential-broker-accounting.js'; + +const MAX_REQUEST_BYTES = 32 * 1024 * 1024; +const MAX_REQUESTS = 512; +const ALLOWED_PATHS = new Set(['/v1/messages', '/v1/messages/count_tokens']); +const BROKER_SERVER_CLOSE_GRACE_MS = 1_000; +export type { ClaudeUsage } from '../src/evidence/claude-usage-cost.js'; +type JsonRecord = Record; +export interface BrokerStats { + acceptedRequests: number; + billableRequests: number; + completedBillableRequests: number; + estimatedBillableRequests: number; + spentUsd: number; + reservedUsd: number; +} + +export interface CreatedCredentialBroker { + server: ReturnType; + stats: () => BrokerStats; +} + +type UpstreamRequest = (options: RequestOptions, + callback: (response: IncomingMessage) => void) => ClientRequest; + +function isRecord(value: unknown): value is JsonRecord { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +function isNumber(value: unknown): value is number { + return typeof value === 'number'; +} + +const roundUsd = (value: number): number => Number(value.toFixed(6)); +const reserveUsd = (value: number): number => Math.ceil(value * 1e6) / 1e6; + +function fail(message: string): never { + throw new Error(`credential broker: ${message}`); +} + +function clientAuthorized(request: IncomingMessage, sessionToken: string): boolean { + return request.headers.authorization === `Bearer ${sessionToken}` + || request.headers['x-api-key'] === sessionToken; +} + +function upstreamHeaders(request: IncomingMessage, config: BrokerConfig): OutgoingHttpHeaders { + const headers: OutgoingHttpHeaders = { ...request.headers }; + delete headers.host; + // Request identity encoding so accounting and the client read the same bytes. + delete headers['accept-encoding']; + delete headers.authorization; + delete headers['proxy-authorization']; + delete headers['x-api-key']; + if (config.mode === 'api-key') headers['x-api-key'] = config.credential; + else headers.authorization = `Bearer ${config.credential}`; + return headers; +} + +function requestPath(value: string | undefined): string | null { + try { return new URL(value ?? '', 'http://credential-broker.invalid').pathname; } + catch { return null; } +} + +function rejectRequest(request: IncomingMessage, response: ServerResponse, + status: number, message: string): void { + request.on('error', () => {}); + response.on('error', () => {}); + try { + response.shouldKeepAlive = false; + response.writeHead(status, { 'content-type': 'text/plain', connection: 'close' }); + response.end(message); + } catch { response.destroy(); } + request.resume(); +} + +function parseProviderRequest(body: Buffer, path: string, config: BrokerConfig): JsonRecord { + let payload: unknown; + try { payload = JSON.parse(body.toString('utf8')); } + catch { fail('request body must be valid JSON'); } + if (!isRecord(payload)) { + fail('request body must be an object'); + } + if (payload.model !== config.model) fail('request model does not match the selected model'); + if (path === '/v1/messages' + && (!isNumber(payload.max_tokens) || !Number.isInteger(payload.max_tokens) || payload.max_tokens < 1 + || payload.max_tokens > config.maxOutputTokens)) { + fail(`max_tokens must be from 1 through ${config.maxOutputTokens}`); + } + return payload; +} + +function requestCostCeiling(bodyBytes: number, maxTokens: number, rates: PricingRates): number { + const inputRate = Math.max(rates.input, rates.cacheWrite5m, rates.cacheWrite1h); + return bodyBytes * inputRate / 1e6 + maxTokens * rates.output / 1e6; +} + +function decodedResponseBody(body: Buffer, contentEncoding: string | string[] | undefined): Buffer { + const encodings = String(contentEncoding ?? '') + .split(',').map(value => value.trim().toLowerCase()).filter(Boolean); + let decoded = body; + for (const encoding of encodings.reverse()) { + if (encoding === 'identity') continue; + const options = { maxOutputLength: MAX_REQUEST_BYTES }; + if (encoding === 'gzip' || encoding === 'x-gzip') decoded = gunzipSync(decoded, options); + else if (encoding === 'deflate') decoded = inflateSync(decoded, options); + else if (encoding === 'br') decoded = brotliDecompressSync(decoded, options); + else throw new Error(`unsupported response encoding ${encoding}`); + } + return decoded; +} + +function responseUsage(body: Buffer, contentEncoding: string | string[] | undefined = undefined): JsonRecord | null { + const values: JsonRecord[] = []; + const add = (value: unknown): void => { + if (!isRecord(value)) return; + if (isRecord(value.usage)) values.push(value.usage); + if (isRecord(value.message) && isRecord(value.message.usage)) values.push(value.message.usage); + }; + let text: string; + try { text = decodedResponseBody(body, contentEncoding).toString('utf8'); } + catch { return null; } + try { + add(JSON.parse(text)); + } catch { + let sawError = false; + let sawFinalUsage = false; + let sawMessageStop = false; + let parseError = false; + const parser = createParser({ + maxBufferSize: MAX_REQUEST_BYTES, + onError: () => { parseError = true; }, + onEvent: ({ data }) => { + if (!data || data === '[DONE]') return; + try { + const event = JSON.parse(data); + if (isRecord(event) && event.type === 'error') sawError = true; + if (isRecord(event) && event.type === 'message_delta' && isRecord(event.usage)) { + sawFinalUsage = true; + } + if (isRecord(event) && event.type === 'message_stop') sawMessageStop = true; + add(event); + } catch { /* Ignore non-JSON event data. */ } + }, + }); + try { parser.feed(`${text}\n\n`); } + catch { parseError = true; } + if (parseError || sawError || !sawFinalUsage || !sawMessageStop) return null; + } + if (values.length === 0) return null; + const number = (field: string): number => Math.max(0, ...values.map(value => Number(value[field]) || 0)); + const cacheWrite = (field: string): number => Math.max(0, ...values.map(value => + isRecord(value.cache_creation) ? Number(value.cache_creation[field]) || 0 : 0)); + const cacheWrite5m = cacheWrite('ephemeral_5m_input_tokens'); + const cacheWrite1h = cacheWrite('ephemeral_1h_input_tokens'); + const flatCacheWrite = number('cache_creation_input_tokens'); + return { + input_tokens: number('input_tokens'), + output_tokens: number('output_tokens'), + cache_read_input_tokens: number('cache_read_input_tokens'), + cache_creation: { + ephemeral_5m_input_tokens: cacheWrite5m + cacheWrite1h > 0 ? cacheWrite5m : flatCacheWrite, + ephemeral_1h_input_tokens: cacheWrite1h, + }, + }; +} + +export function createCredentialBroker(configInput: unknown, { + requestUpstream = httpsRequest as UpstreamRequest, + upstream = { protocol: 'https:', hostname: 'api.anthropic.com', port: 443 }, + maxRequests = MAX_REQUESTS, + maxRequestBytes = MAX_REQUEST_BYTES, +}: { requestUpstream?: UpstreamRequest; + upstream?: { protocol: string; hostname: string; port: number }; + maxRequests?: number; maxRequestBytes?: number } = {}): CreatedCredentialBroker { + const config = validateBrokerConfig(configInput); + let acceptedRequests = 0; + let billableRequests = 0; + let completedBillableRequests = 0; + let estimatedBillableRequests = 0; + const estimatedByReason = noEstimates(); + let spentUsd = 0; + let reservedUsd = 0; + const usageTotals: ClaudeUsage = { input: 0, output: 0, cacheRead: 0, cacheWrite5m: 0, cacheWrite1h: 0 }; + const recordLedger = () => writeCredentialBrokerLedger(config.ledgerPath, { + schemaVersion: BROKER_LEDGER_SCHEMA_VERSION, + model: config.model, + maxBudgetUsd: config.maxBudgetUsd ?? null, + acceptedRequests, + billableRequests, + completedBillableRequests, + estimatedBillableRequests, + estimatedByReason, + spentUsd: Number(spentUsd.toFixed(6)), + reservedUsd: Number(reservedUsd.toFixed(6)), + usage: usageTotals, + complete: reservedUsd === 0 && completedBillableRequests === billableRequests, + updatedAt: new Date().toISOString(), + }); + recordLedger(); + const server = createServer((request, response) => { + // A client can disappear while the broker is still draining an upstream + // response. Socket errors must not terminate the broker and strand a paid + // request reservation in the ledger. + request.on('error', () => {}); + request.on('aborted', () => {}); + response.on('error', () => {}); + const responseOpen = (): boolean => !response.destroyed && !response.writableEnded; + const writeHead = (status: number, headers: OutgoingHttpHeaders): void => { + if (!responseOpen() || response.headersSent) return; + try { response.writeHead(status, headers); } + catch { response.destroy(); } + }; + const endResponse = (body?: string | Buffer): void => { + if (!responseOpen()) return; + try { response.end(body); } + catch { response.destroy(); } + }; + if (!clientAuthorized(request, config.sessionToken)) { + rejectRequest(request, response, 401, 'unauthorized'); + return; + } + const path = requestPath(request.url); + if (request.method !== 'POST' || path === null || !ALLOWED_PATHS.has(path)) { + rejectRequest(request, response, 404, 'not found'); + return; + } + acceptedRequests += 1; + recordLedger(); + if (acceptedRequests > maxRequests) { + rejectRequest(request, response, 429, 'session request limit reached'); + return; + } + + const chunks: Buffer[] = []; + let received = 0; + let tooLarge = false; + request.on('data', (chunk: Buffer) => { + if (tooLarge) return; + received += chunk.length; + if (received > maxRequestBytes) { + tooLarge = true; + writeHead(413, { 'content-type': 'text/plain' }); + endResponse('request is too large'); + return; + } + chunks.push(chunk); + }); + request.on('end', () => { + if (tooLarge) return; + const body = Buffer.concat(chunks); + let payload: JsonRecord; + try { payload = parseProviderRequest(body, path, config); } + catch { + writeHead(400, { 'content-type': 'text/plain' }); + endResponse('invalid provider request'); + return; + } + const billable = path === '/v1/messages' && config.maxBudgetUsd != null; + const costCeiling = billable + ? reserveUsd(requestCostCeiling(received, payload.max_tokens as number, + config.pricingRates as PricingRates)) : 0; + const budget = config.maxBudgetUsd; + if (billable && budget !== null && budget !== undefined + && spentUsd + reservedUsd + costCeiling > budget) { + writeHead(402, { 'content-type': 'text/plain' }); + endResponse('session cost limit reached'); + return; + } + if (billable) billableRequests += 1; + reservedUsd = roundUsd(reservedUsd + costCeiling); + recordLedger(); + let billableSettled = !billable; + const settleBillable = ({ usage = null, estimated = null }: + { usage?: ClaudeUsage | null; estimated?: EstimateReason | null } = {}): void => { + if (billableSettled) return; + billableSettled = true; + reservedUsd = roundUsd(reservedUsd - costCeiling); + completedBillableRequests += 1; + if (estimated) { + estimatedBillableRequests += 1; + estimatedByReason[estimated] += 1; + spentUsd = roundUsd(spentUsd + costCeiling); + } else if (usage) { + spentUsd = roundUsd(spentUsd + priceNormalizedClaudeUsage(usage, config.pricingRates as PricingRates)); + for (const field of CLAUDE_USAGE_FIELDS) usageTotals[field] += usage[field]; + } + recordLedger(); + }; + const headers = upstreamHeaders(request, config); + for (const name of ['connection', 'keep-alive', 'proxy-connection', 'te', 'trailer', + 'transfer-encoding', 'upgrade']) delete headers[name]; + headers['content-length'] = String(received); + const upstreamRequest = requestUpstream({ + protocol: upstream.protocol, + hostname: upstream.hostname, + port: upstream.port, + method: request.method, + path: request.url, + headers, + }, upstreamResponse => { + writeHead(upstreamResponse.statusCode ?? 502, upstreamResponse.headers); + const responseChunks: Buffer[] = []; + let responseBytes = 0; + upstreamResponse.on('data', (chunk: Buffer) => { + responseBytes += chunk.length; + if (responseBytes <= maxRequestBytes) responseChunks.push(chunk); + if (responseOpen()) { + try { response.write(chunk); } + catch { response.destroy(); } + } + }); + upstreamResponse.on('end', () => { + endResponse(); + if (!billable) return; + if ((upstreamResponse.statusCode ?? 502) >= 200 + && (upstreamResponse.statusCode ?? 502) < 300) { + const usage = responseBytes <= maxRequestBytes + ? responseUsage(Buffer.concat(responseChunks), upstreamResponse.headers['content-encoding']) + : null; + if (!usage) settleBillable({ estimated: 'no-usage' }); + else try { settleBillable({ usage: normalizeClaudeUsage(usage) }); } + catch { settleBillable({ estimated: 'no-usage' }); } + } else { + settleBillable(); + } + }); + const settleAbortedResponse = () => { + settleBillable({ estimated: 'response-aborted' }); + if (responseOpen()) response.destroy(); + }; + upstreamResponse.once('aborted', settleAbortedResponse); + upstreamResponse.once('error', settleAbortedResponse); + }); + upstreamRequest.on('error', () => { + settleBillable({ estimated: 'upstream-error' }); + writeHead(502, { 'content-type': 'text/plain' }); + endResponse('upstream request failed'); + }); + upstreamRequest.end(body); + }); + }); + server.on('clientError', (_error: Error, socket: Socket) => { + socket.on('error', () => {}); + if (socket.writable) socket.end('HTTP/1.1 400 Bad Request\r\nConnection: close\r\n\r\n'); + else socket.destroy(); + }); + return { server, stats: () => ({ acceptedRequests, + billableRequests, completedBillableRequests, estimatedBillableRequests, + estimatedByReason: { ...estimatedByReason }, + spentUsd: Number(spentUsd.toFixed(6)), reservedUsd: Number(reservedUsd.toFixed(6)) }) }; +} + +function parseArgs(argv: string[]): string { + const { values } = parseNodeArgs({ args: argv, options: { config: { type: 'string' } } }); + const configPath = values.config; + if (!configPath || argv.length !== 2) fail('use --config '); + return resolve(configPath); +} + +async function main() { + const configPath = parseArgs(process.argv.slice(2)); + let config: BrokerConfig; + try { config = validateBrokerConfig(JSON.parse(readFileSync(configPath, 'utf8'))); } + finally { rmSync(configPath, { force: true }); } + if (!config.readyPath) fail('readyPath is invalid'); + const { server } = createCredentialBroker(config); + const sockets = new Set(); + server.on('connection', (socket: Socket) => { + sockets.add(socket); + socket.once('close', () => sockets.delete(socket)); + }); + server.on('error', (error: Error) => { + process.stderr.write(`credential broker: ${error.message}\n`); + process.exitCode = 1; + }); + const readyPath = config.readyPath; + server.listen(0, config.listenHost ?? '127.0.0.1', () => { + const address: string | AddressInfo | null = server.address(); + if (!address || typeof address === 'string') fail('listener address is unavailable'); + writeFileSync(readyPath, `${JSON.stringify({ host: address.address, port: address.port })}\n`, + { flag: 'wx', mode: 0o600 }); + }); + let stopping = false; + const stop = () => { + if (stopping) return; + stopping = true; + const force = setTimeout(() => { + for (const socket of sockets) socket.destroy(); + server.closeAllConnections?.(); + process.exit(0); + }, BROKER_SERVER_CLOSE_GRACE_MS); + force.unref(); + server.close(() => { + clearTimeout(force); + process.exit(0); + }); + server.closeIdleConnections?.(); + }; + const parentPid = config.parentPid; + if (parentPid) { + setInterval(() => { + try { process.kill(parentPid, 0); } + catch { stop(); } + }, 1_000).unref(); + } + const expiresAt = config.expiresAt; + if (expiresAt) setTimeout(stop, Math.max(1, expiresAt - Date.now())).unref(); + process.on('SIGINT', stop); + process.on('SIGTERM', stop); +} + +if (process.argv[1] && pathToFileURL(resolve(process.argv[1])).href === import.meta.url) { + main().catch((error: unknown) => { + process.stderr.write(`${error instanceof Error ? error.stack ?? error.message : String(error)}\n`); + process.exit(1); + }); +} diff --git a/tools/stack-bench/container/reconcile-build-container.ts b/tools/stack-bench/container/reconcile-build-container.ts new file mode 100644 index 00000000000..68d2e1c4611 --- /dev/null +++ b/tools/stack-bench/container/reconcile-build-container.ts @@ -0,0 +1,85 @@ +import { spawnSync } from 'node:child_process'; +import type { SpawnSyncOptionsWithStringEncoding, SpawnSyncReturns } from 'node:child_process'; + +export const BUILD_CONTAINER_CREATION_LABEL = 'com.clockworklabs.stack-bench.creation'; + +const CONTAINER_ID = /^[a-f0-9]{64}$/i; + +type DockerResult = SpawnSyncReturns; +type DockerExecute = (command: string, args: readonly string[], + options: SpawnSyncOptionsWithStringEncoding) => DockerResult; + +export interface RemoveFailedBuildContainerOptions { + containerName: string; + creationToken: string; + createdId?: string | null; + dockerEnv?: NodeJS.ProcessEnv; + timeoutMs?: number; + execute?: DockerExecute; +} + +export function containerIdFromDockerOutput(output: unknown): string | null { + return String(output ?? '').split(/\r?\n/).map(line => line.trim()) + .find(line => CONTAINER_ID.test(line)) ?? null; +} + +function detail(result: DockerResult): string { + return String(result.stderr || result.stdout || result.error?.message + || `exit ${result.status}`).trim(); +} + +// The coding containers running on this Docker host, from any campaign. A +// stopped container keeps its label but holds no application or session. +export function listRunningCodingContainers({ dockerEnv = process.env, timeoutMs = 30_000, + execute = spawnSync as DockerExecute }: { + dockerEnv?: NodeJS.ProcessEnv; timeoutMs?: number; execute?: DockerExecute; + } = {}): string[] { + const result = execute('docker', ['ps', '--filter', `label=${BUILD_CONTAINER_CREATION_LABEL}`, + '--format', '{{.Names}}'], { encoding: 'utf8', env: dockerEnv, timeout: timeoutMs }); + if (result.error || result.status !== 0) { + throw new Error(`cannot list coding containers: ${detail(result)}`); + } + return String(result.stdout ?? '').split(/\r?\n/).map(line => line.trim()) + .filter(Boolean).sort(); +} + +export function removeFailedBuildContainer({ containerName, creationToken, createdId = null, + dockerEnv = process.env, timeoutMs = 120_000, + execute = spawnSync as DockerExecute }: RemoveFailedBuildContainerOptions): { + removed: boolean; absent: boolean; id?: string; +} { + if (typeof containerName !== 'string' || !containerName) { + throw new Error('failed build-container cleanup requires a container name'); + } + if (typeof creationToken !== 'string' || !creationToken) { + throw new Error('failed build-container cleanup requires a creation token'); + } + + let id = containerIdFromDockerOutput(createdId); + if (!id) { + const inspected = execute('docker', ['inspect', '--format', + `{{.Id}} {{index .Config.Labels "${BUILD_CONTAINER_CREATION_LABEL}"}}`, containerName], { + encoding: 'utf8', env: dockerEnv, timeout: timeoutMs, + }); + if (inspected.status !== 0) { + const reason = detail(inspected); + if (/no such (?:object|container)/i.test(reason)) return { removed: false, absent: true }; + throw new Error(`cannot prove cleanup of failed container ${containerName}: ${reason}`); + } + const [inspectedId, label, ...extra] = String(inspected.stdout ?? '').trim().split(/\s+/); + if (!CONTAINER_ID.test(inspectedId ?? '') || label !== creationToken || extra.length > 0) { + throw new Error(`refusing to remove ${containerName}: its creation identity does not match`); + } + id = inspectedId ?? null; + } + + if (!id) throw new Error(`cannot prove cleanup of failed container ${containerName}`); + + const removed = execute('docker', ['rm', '-f', id], { + encoding: 'utf8', env: dockerEnv, timeout: timeoutMs, + }); + if (removed.status !== 0) { + throw new Error(`could not remove failed build container ${id}: ${detail(removed)}`); + } + return { removed: true, absent: false, id }; +} diff --git a/tools/stack-bench/container/recover-build-container.ts b/tools/stack-bench/container/recover-build-container.ts new file mode 100644 index 00000000000..acefebcb228 --- /dev/null +++ b/tools/stack-bench/container/recover-build-container.ts @@ -0,0 +1,77 @@ +import { spawnSync } from 'node:child_process'; +import type { SpawnSyncOptionsWithStringEncoding } from 'node:child_process'; + +import { updateBackendLease } from '../src/runtime/backend-lease.js'; +import type { BackendLease } from '../src/runtime/backend-lease.js'; + +interface StoppedBuildContainer { + id: string; + running: false; +} + +interface LeaseContext { + path: string; + lease: BackendLease; +} + +export interface DockerExecuteResult { + status: number | null; + stdout?: string; + stderr?: string; + error?: Error; +} + +export type DockerExecute = (command: string, args: readonly string[], + options: SpawnSyncOptionsWithStringEncoding) => DockerExecuteResult; + +export interface RecoverStoppedBuildContainerOptions { + existing: StoppedBuildContainer; + containerName: string; + leaseContext: LeaseContext; + backend: string; + dockerEnv?: NodeJS.ProcessEnv; + timeoutMs?: number; + execute?: DockerExecute; +} + +function clearBuildContainerLease(leaseContext: LeaseContext, backend: string, + containerId: string, description: string): LeaseContext { + const lease = updateBackendLease(leaseContext.path, { + token: leaseContext.lease.ownershipToken, backend, runId: leaseContext.lease.runId, + }, next => { + if (next.resources.buildContainer?.id !== containerId) { + throw new Error(`${description} ownership changed before recovery`); + } + next.resources.buildContainer = null; + return next; + }); + return { path: leaseContext.path, lease }; +} + +export function recoverStoppedBuildContainer({ existing, containerName, leaseContext, backend, + dockerEnv = process.env, timeoutMs = 120_000, + execute = spawnSync }: RecoverStoppedBuildContainerOptions): LeaseContext { + const prior = leaseContext?.lease?.resources?.buildContainer ?? null; + if (!existing || existing.running) throw new Error('recovery requires a stopped container'); + if (!prior || prior.name !== containerName || prior.id !== existing.id) { + throw new Error('stopped container does not match the authenticated lease'); + } + const removed = execute('docker', ['rm', existing.id], { + encoding: 'utf8', env: dockerEnv, timeout: timeoutMs, + }); + if (removed.status !== 0) { + throw new Error(`could not remove exact stopped leased container ${existing.id}: ` + + String(removed.stderr || removed.stdout || removed.error?.message || `exit ${removed.status}`).trim()); + } + return clearBuildContainerLease(leaseContext, backend, existing.id, 'stopped container'); +} + +export function clearMissingBuildContainerLease({ containerName, leaseContext, backend }: { + containerName: string; leaseContext: LeaseContext; backend: string; +}): LeaseContext { + const prior = leaseContext?.lease?.resources?.buildContainer ?? null; + if (!prior || prior.name !== containerName) { + throw new Error('missing container does not match the authenticated lease'); + } + return clearBuildContainerLease(leaseContext, backend, prior.id, 'missing container'); +} diff --git a/tools/stack-bench/container/run-build.ts b/tools/stack-bench/container/run-build.ts new file mode 100644 index 00000000000..ae2501c0cca --- /dev/null +++ b/tools/stack-bench/container/run-build.ts @@ -0,0 +1,680 @@ +#!/usr/bin/env node +// The build container must not expose Stack Bench source or grading material. +import { spawnSync } from 'node:child_process'; +import { randomBytes } from 'node:crypto'; +import { chmodSync, existsSync, readFileSync, mkdirSync } from 'node:fs'; +import { join, resolve, basename, dirname } from 'node:path'; +import { homedir } from 'node:os'; +import { parseArgs } from 'node:util'; +import { leaseFromEnv, updateBackendLease } from '../src/runtime/backend-lease.js'; +import { resolveContainerImage } from '../src/runtime/container-image.js'; +import { leasedDatabaseEnvironment, STACK_ADAPTER_REGISTRY } from '../src/stacks/stack-adapters.js'; +import { BUILD_CONTAINER_RESOURCE_LIMITS, DEFAULT_BUILD_IMAGE } + from '../src/composition/product-config.js'; +import { dockerMountArguments } from '../src/runtime/container-mount.js'; +import type { ContainerMount } from '../src/runtime/container-mount.js'; +import { dockerHostGatewayArguments } from '../src/runtime/docker-network.js'; +import { resolveContainerAuth } from './container-auth.js'; +import { hasRequiredBuildContainerIsolation, inspectBuildContainer, parseCgroupMemory, + parsePublishedPorts } + from './build-container-inspection.js'; +import { reconcileCredentialBrokerReceipt } from './credential-broker-accounting.js'; +import { credentialBrokerDiagnostics, startCredentialBroker, stopCredentialBroker } + from './credential-broker-process.js'; +import { clearMissingBuildContainerLease, + recoverStoppedBuildContainer } from './recover-build-container.js'; +import { BUILD_CONTAINER_CREATION_LABEL, containerIdFromDockerOutput, + removeFailedBuildContainer } from './reconcile-build-container.js'; +import { CODING_SESSION_TIMEOUT_MS } from '../src/agents/coding-session-timeouts.js'; +import { CODING_CONTAINER_AGENT, CODING_CONTAINER_APP_ROOT, CODING_CONTAINER_CONTROL_DIR, + CODING_CONTAINER_PROCESS_IDENTITY, + codingContainerAgentEnvironment, codingContainerTranscriptHandoffCommands, + codingContainerWorkspaceHandoffCommands } + from '../src/runtime/coding-container-policy.js'; +import { runTranscriptAwareProcess, snapshotClaudeTranscripts } + from '../src/agents/claude-terminal-recovery.js'; +import { claudeRatesForModel } from '../src/evidence/claude-usage-cost.js'; +import { PRICING_UNIT, validatePricingAuthority } + from '../src/evidence/pricing-authority.js'; +import { REPOSITORY_ROOT } from '../src/package-root.js'; + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +const { values } = parseArgs({ args: process.argv.slice(2), options: { + app: { type: 'string' }, backend: { type: 'string' }, 'prepare-only': { type: 'boolean' }, + image: { type: 'string' }, effort: { type: 'string' }, model: { type: 'string' }, + 'max-budget-usd': { type: 'string' }, 'pricing-json': { type: 'string' }, + 'resume-session': { type: 'string' }, 'recover-stopped-container': { type: 'boolean' }, + 'completion-marker': { type: 'string' }, ports: { type: 'string' }, +} }); + +const appDir = values.app; +if (!appDir) { console.error('run-build.js: --app is required'); process.exit(2); } +const backend = values.backend; +if (!backend) { console.error('run-build.js: --backend is required'); process.exit(2); } +let adapter; +try { adapter = STACK_ADAPTER_REGISTRY.get(backend); } +catch (error) { console.error(`run-build.js: ${errorMessage(error)}`); process.exit(2); } +const prepareOnly = values['prepare-only'] ?? false; +const DOCKER_TIMEOUT_MS = 120_000; +const DOCKER_PROBE_TIMEOUT_MS = 10_000; +const { uid: AGENT_UID, gid: AGENT_GID, home: AGENT_HOME } = CODING_CONTAINER_AGENT; +const CONTROLLER_GID = process.getgid?.() ?? 0; +const AGENT_ENVIRONMENT = codingContainerAgentEnvironment(); +const CONTROL_DIR = CODING_CONTAINER_CONTROL_DIR; +const REQUIRED_CAPABILITIES = Object.freeze([ + 'CHOWN', 'DAC_OVERRIDE', 'FOWNER', 'KILL', 'SETGID', 'SETUID', +]); +const REQUIRED_TMPFS = Object.freeze({ + '/tmp': 'rw,nosuid,nodev,mode=1777', + [AGENT_HOME]: `rw,nosuid,nodev,uid=${AGENT_UID},gid=${AGENT_GID},mode=0700`, + [`${AGENT_HOME}/.claude`]: `rw,nosuid,nodev,uid=${AGENT_UID},gid=${AGENT_GID},mode=0700`, + '/deps': 'rw,exec,nosuid,nodev,mode=0755', + [CONTROL_DIR]: 'rw,nosuid,nodev,mode=0700', +}); + +const REPO = REPOSITORY_ROOT; +const imageReference = values.image ?? DEFAULT_BUILD_IMAGE; +let imageIdentity; +try { imageIdentity = resolveContainerImage(imageReference); } +catch (error) { + console.error(`run-build.js: cannot resolve image ${imageReference}: ${errorMessage(error)}`); + process.exit(2); +} +const image = imageIdentity.id; +const effort = values.effort ?? ''; +const model = values.model ?? ''; +if (!prepareOnly && (!effort || !model)) { + console.error('run-build.js: --effort and --model are required'); + process.exit(2); +} +const maxBudgetUsd = values['max-budget-usd'] ?? null; +if (maxBudgetUsd !== null && (!Number.isFinite(Number(maxBudgetUsd)) || Number(maxBudgetUsd) <= 0)) { + console.error('run-build.js: --max-budget-usd must be a positive number'); + process.exit(2); +} +let pricing = null; +try { + const supplied = values['pricing-json'] ?? null; + if (supplied !== null) { + pricing = validatePricingAuthority(JSON.parse(supplied), { at: '--pricing-json' }); + } else if (maxBudgetUsd !== null) { + const rates = claudeRatesForModel(model); + if (!rates) throw new Error(`no default pricing is recorded for model ${model}`); + pricing = validatePricingAuthority({ unit: PRICING_UNIT, rates }, + { at: 'default pricing' }); + } +} catch (error) { + console.error(`run-build.js: ${errorMessage(error)}`); + process.exit(2); +} +const resumeSession = values['resume-session'] ?? null; +if (resumeSession !== null + && !/^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(resumeSession)) { + console.error('run-build.js: --resume-session must be a UUID'); + process.exit(2); +} +const recoverStoppedContainer = values['recover-stopped-container'] ?? false; +const completionMarker = values['completion-marker'] ?? null; +if (!prepareOnly && !/^[A-Z][A-Z0-9_]*$/.test(completionMarker ?? '')) { + console.error('run-build.js: --completion-marker must be an uppercase marker'); + process.exit(2); +} +let ports: string[] = []; +try { ports = parsePublishedPorts(values.ports); } +catch (error) { console.error(`run-build.js: ${errorMessage(error)}`); process.exit(2); } + +const containerPlan = adapter.buildContainer.plan({ + repo: REPO, appDir, env: process.env, +}); + +// Auth is resolved in the controller. A short-lived broker forwards model API +// requests later. The coding container never receives the long-lived provider +// credential or a credential file. +const apiKey = process.env.STACK_BENCH_AGENT_API_KEY ?? process.env.ANTHROPIC_API_KEY ?? ''; +const creds = join(homedir(), '.claude', '.credentials.json'); +let auth = null; +if (!prepareOnly) { + try { auth = resolveContainerAuth({ apiKey, env: process.env, credentialsPath: creds }); } + catch (error) { console.error(`run-build.js: ${errorMessage(error)}`); process.exit(2); } +} + +// Persist this run's transcript without exposing other local sessions. +const projects = prepareOnly ? null : join(homedir(), '.claude', 'projects', + resolve(appDir).replace(/[\\/:]/g, '-').toLowerCase()); +function ensureAgentDirectory(directory: string): void { + mkdirSync(directory, { recursive: true, + mode: process.env.STACK_BENCH_APPLIANCE === '1' ? 0o700 : 0o777 }); + if (process.env.STACK_BENCH_APPLIANCE !== '1') chmodSync(directory, 0o777); +} + +ensureAgentDirectory(appDir); +if (projects) ensureAgentDirectory(projects); +for (const directory of containerPlan.ensureDirectories) ensureAgentDirectory(directory); + +// Grading and repair reuse this leased container. +const containerName = `stack-bench-${basename(dirname(resolve(appDir)))}`; +const dockerEnv: NodeJS.ProcessEnv = { ...process.env, MSYS_NO_PATHCONV: '1' }; + +function resolveNetworkMode(): 'bridge' | 'host' { + if (containerPlan!.networkNamespace !== 'host') return 'bridge'; + if (process.env.STACK_BENCH_APPLIANCE !== '1') { + throw new Error('the host network namespace is available only in appliance mode'); + } + return 'host'; +} + +let expectedNetworkMode: 'bridge' | 'host'; +try { expectedNetworkMode = resolveNetworkMode(); } +catch (error) { + console.error(`run-build.js: ${errorMessage(error)}`); + process.exit(2); +} + +const inspectContainer = (name: string) => inspectBuildContainer(name, + { env: dockerEnv, timeoutMs: DOCKER_TIMEOUT_MS }); + +const hasRequiredIsolation = (container: NonNullable>, + expectedMounts: ContainerMount[]): boolean => hasRequiredBuildContainerIsolation(container, { + expectedMounts, + requiredTmpfs: REQUIRED_TMPFS, + requiredCapabilities: REQUIRED_CAPABILITIES, + pidsLimit: BUILD_CONTAINER_RESOURCE_LIMITS.pids, + cpuCount: BUILD_CONTAINER_RESOURCE_LIMITS.cpuCount, + memoryBytes: BUILD_CONTAINER_RESOURCE_LIMITS.memoryBytes, + memorySwapBytes: BUILD_CONTAINER_RESOURCE_LIMITS.memorySwapBytes, + image, +}); + +const expectedMounts: ContainerMount[] = [ + { kind: 'bind' as const, source: resolve(appDir), target: CODING_CONTAINER_APP_ROOT, readOnly: false }, + ...(projects ? [{ kind: 'bind' as const, source: projects, + target: `${AGENT_HOME}/.claude/projects/-app`, readOnly: false }] : []), + ...containerPlan.mounts, +]; + +// Only the lease's immutable container id grants reuse or deletion authority. +let leaseContext; +try { leaseContext = leaseFromEnv(process.env, { backend, active: true }); } +catch (error) { + console.error(`run-build.js: an authenticated active backend lease is required: ${errorMessage(error)}`); + process.exit(3); +} + +let existing = inspectContainer(containerName); +const priorContainer = leaseContext.lease.resources.buildContainer ?? null; +if (existing) { + if (!priorContainer) { + console.error(`run-build.js: refusing to adopt existing unleased container ${containerName}`); + process.exit(3); + } + if (priorContainer.name !== containerName || priorContainer.id !== existing.id) { + console.error(`run-build.js: existing container ${containerName}/${existing.id} does not match lease ` + + `${priorContainer.name}/${priorContainer.id}`); + process.exit(3); + } + if (!existing.running) { + if (!recoverStoppedContainer) { + console.error(`run-build.js: leased container ${containerName} stopped unexpectedly; refusing to replace it`); + process.exit(3); + } + try { + leaseContext = recoverStoppedBuildContainer({ existing: { ...existing, running: false }, containerName, leaseContext, backend, + dockerEnv, timeoutMs: DOCKER_TIMEOUT_MS }); + existing = null; + } catch (error) { + console.error(`run-build.js: could not recover stopped container: ${errorMessage(error)}`); + process.exit(3); + } + } + if (existing && existing.networkMode !== expectedNetworkMode) { + console.error(`run-build.js: leased container ${containerName} uses network ${existing.networkMode}, ` + + `expected ${expectedNetworkMode}`); + process.exit(3); + } + if (existing?.unsafeCredentialExposure) { + console.error(`run-build.js: leased container ${containerName} was created with a provider credential; ` + + 'reconcile the run and start it with the isolated credential broker'); + process.exit(3); + } + if (existing && !hasRequiredIsolation(existing, expectedMounts)) { + console.error(`run-build.js: leased container ${containerName} does not have the required isolation`); + process.exit(3); + } +} else if (priorContainer) { + const leasedById = inspectContainer(priorContainer.id); + if (leasedById) { + console.error(`run-build.js: leased container ${priorContainer.id} still exists under an unexpected name`); + process.exit(3); + } + if (!recoverStoppedContainer) { + console.error(`run-build.js: leased container ${priorContainer.name}/${priorContainer.id} is missing`); + process.exit(3); + } + try { + leaseContext = clearMissingBuildContainerLease({ containerName, leaseContext, backend }); + } catch (error) { + console.error(`run-build.js: could not recover missing container lease: ${errorMessage(error)}`); + process.exit(3); + } +} + +// Create it if this is the first round of the run; reuse it for every round +// after, so a repair finds the app, its node_modules and its servers exactly +// where the build round left them. +let containerInspection = existing; +if (!existing) { + const creationToken = randomBytes(16).toString('hex'); + const create = [ + 'create', '--init', '--name', containerName, + '--label', `${BUILD_CONTAINER_CREATION_LABEL}=${creationToken}`, + '--cap-drop', 'ALL', '--security-opt', 'no-new-privileges:true', + '--pids-limit', String(BUILD_CONTAINER_RESOURCE_LIMITS.pids), + '--cpus', String(BUILD_CONTAINER_RESOURCE_LIMITS.cpuCount), + '--memory', String(BUILD_CONTAINER_RESOURCE_LIMITS.memoryBytes), + '--memory-swap', String(BUILD_CONTAINER_RESOURCE_LIMITS.memorySwapBytes), + // The agent may write the app, its own home directory, and temporary files. + // It must not replace system binaries or libraries used by later grading. + '--read-only', + '-v', `${resolve(appDir)}:${CODING_CONTAINER_APP_ROOT}`, + ]; + for (const capability of REQUIRED_CAPABILITIES) create.push('--cap-add', capability); + for (const [path, options] of Object.entries(REQUIRED_TMPFS)) { + create.push('--tmpfs', `${path}:${options}`); + } + create.push('--network', expectedNetworkMode); + create.push(...dockerHostGatewayArguments(expectedNetworkMode)); + if (projects) create.push('-v', `${projects}:${AGENT_HOME}/.claude/projects/-app`); + // The selected adapter owns every stack-specific mount. Giving a treatment + // another stack's artifacts would violate the "only artifacts under test" + // boundary. + for (const requiredPath of containerPlan.requiredPaths) { + if (!existsSync(requiredPath)) { + console.error(`run-build.js: ${backend} container artifact is missing: ${requiredPath}`); + process.exit(2); + } + } + for (const mount of containerPlan.mounts) { + try { create.push(...dockerMountArguments(mount)); } + catch (error) { + console.error(`run-build.js: ${backend} adapter returned an invalid container mount: ${errorMessage(error)}`); + process.exit(2); + } + } + + // Publish the track's ports for the host grader. + if (expectedNetworkMode === 'bridge') for (const p of ports) create.push('-p', `127.0.0.1:${p}:${p}`); + + // `--init` gives the container a real PID 1. Without it the dev servers the + // build leaves behind are reparented to `sleep`, which never reaps them. + const init = 'export HOME=/tmp npm_config_cache=/tmp/npm-cache; ' + + containerPlan.init; + create.push('-w', CODING_CONTAINER_APP_ROOT, image, 'sh', '-c', init); + + const made = spawnSync('docker', create, { + encoding: 'utf8', env: dockerEnv, timeout: DOCKER_TIMEOUT_MS, + }); + if (made.status !== 0) { + console.error(`run-build.js: could not create ${containerName}`); + console.error(made.stderr || made.stdout || made.error?.message || ''); + try { + removeFailedBuildContainer({ containerName, creationToken, + createdId: containerIdFromDockerOutput(made.stdout), dockerEnv, + timeoutMs: DOCKER_TIMEOUT_MS }); + } catch (cleanupError) { + console.error(`run-build.js: ${errorMessage(cleanupError)}`); + process.exit(3); + } + process.exit(2); + } + + const createdId = containerIdFromDockerOutput(made.stdout); + try { containerInspection = inspectContainer(containerName); } + catch (error) { + console.error(`run-build.js: cannot inspect ${containerName}: ${errorMessage(error)}`); + } + if (!containerInspection) { + try { + removeFailedBuildContainer({ containerName, creationToken, createdId, dockerEnv, + timeoutMs: DOCKER_TIMEOUT_MS }); + } catch (cleanupError) { + console.error(`run-build.js: ${errorMessage(cleanupError)}`); + process.exit(3); + } + console.error(`run-build.js: cannot inspect ${containerName}`); + process.exit(2); + } + if (containerInspection.unsafeCredentialExposure + || !hasRequiredIsolation(containerInspection, expectedMounts)) { + try { + removeFailedBuildContainer({ containerName, creationToken, createdId, dockerEnv, + timeoutMs: DOCKER_TIMEOUT_MS }); + } catch (cleanupError) { + console.error(`run-build.js: ${errorMessage(cleanupError)}`); + process.exit(3); + } + console.error(`run-build.js: created container ${containerName} does not have the required isolation`); + process.exit(2); + } +} + +if (!containerInspection) { + console.error(`run-build.js: cannot inspect ${containerName}`); + process.exit(2); +} +const { id: containerId, image: containerImage } = containerInspection; +try { + const { path, lease } = leaseContext; + const prior = lease.resources.buildContainer; + if (prior && (prior.name !== containerName || prior.id !== containerId)) { + throw new Error(`running container ${containerName}/${containerId} does not match lease ` + + `${prior.name}/${prior.id}`); + } + updateBackendLease(path, { token: lease.ownershipToken, backend, runId: lease.runId }, next => { + next.resources.buildContainer = { + name: containerName, id: containerId, image: containerImage, owned: true, running: existing !== null, + networkMode: expectedNetworkMode, + resourceLimits: structuredClone(BUILD_CONTAINER_RESOURCE_LIMITS), + }; + return next; + }); +} catch (error) { + // Creation succeeded but ownership recording did not. Remove only the exact + // id created by this invocation; leaving an unleased container is not safe. + if (!existing) { + spawnSync('docker', ['rm', '-f', containerId], { + stdio: 'ignore', env: dockerEnv, timeout: DOCKER_TIMEOUT_MS, + }); + } + console.error(`run-build.js: ${errorMessage(error)}`); + process.exit(3); +} + +if (!existing) { + const started = spawnSync('docker', ['start', containerId], { + encoding: 'utf8', env: dockerEnv, timeout: DOCKER_TIMEOUT_MS, + }); + if (started.status !== 0) { + console.error(`run-build.js: could not start leased container ${containerName}/${containerId}`); + console.error(started.stderr || started.stdout || started.error?.message || ''); + process.exit(2); + } + try { + const { path, lease } = leaseContext; + updateBackendLease(path, { token: lease.ownershipToken, backend, runId: lease.runId }, next => { + if (next.resources.buildContainer?.id !== containerId) { + throw new Error(`leased container changed before start: expected ${containerId}`); + } + next.resources.buildContainer.running = true; + return next; + }); + } catch (error) { + console.error(`run-build.js: started container ownership could not be recorded: ${errorMessage(error)}`); + process.exit(3); + } +} + +if (containerPlan.readyFile) { + // Wait until SDK staging finishes before starting the paid session. + let ready = false; + const readyDeadline = Date.now() + 90_000; + while (Date.now() < readyDeadline) { + const probe = spawnSync('docker', ['exec', containerName, 'test', '-f', containerPlan.readyFile], + { stdio: 'ignore', env: dockerEnv, timeout: DOCKER_PROBE_TIMEOUT_MS }); + if (probe.status === 0) { ready = true; break; } + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 500); + } + if (!ready) { + const logs = spawnSync('docker', ['logs', containerName], { + encoding: 'utf8', env: dockerEnv, timeout: DOCKER_TIMEOUT_MS, + }); + console.error(`run-build.js: timed out waiting for ${containerPlan.readyDescription ?? `${backend} setup`}`); + console.error(logs.stderr || logs.stdout || ''); + process.exit(2); + } +} + +if (process.env.STACK_BENCH_APPLIANCE === '1') { + const writableTargets = [AGENT_HOME, + ...expectedMounts.filter(mount => !mount.readOnly).map(mount => mount.target)]; + for (const [command, commandArgs] of [ + ['chown', ['-R', `${AGENT_UID}:${CONTROLLER_GID}`, '--', ...writableTargets]], + ['chmod', ['-R', 'u+rwX,g+rwX,o-rwx', '--', ...writableTargets]], + ] as const) { + const permissions = spawnSync('docker', ['exec', containerName, command, ...commandArgs], { + encoding: 'utf8', env: dockerEnv, timeout: DOCKER_PROBE_TIMEOUT_MS, + }); + if (permissions.status !== 0) { + console.error(`run-build.js: could not secure writable paths in ${containerName}`); + console.error(permissions.stderr || permissions.stdout || permissions.error?.message || ''); + process.exit(2); + } + } +} + +// A nested transcript mount makes Docker create its parent directories as +// root. Confirm that Claude can create its private session state before a +// provider request can spend money. +const homeProbe = spawnSync('docker', [ + 'exec', '--user', `${AGENT_UID}:${AGENT_GID}`, '-e', `HOME=${AGENT_HOME}`, + containerName, 'sh', '-c', + 'umask 077; mkdir -p "$HOME/.claude/session-env" && test -w "$HOME/.claude/session-env"', +], { encoding: 'utf8', env: dockerEnv, timeout: DOCKER_PROBE_TIMEOUT_MS }); +if (homeProbe.status !== 0) { + console.error(`run-build.js: agent home is not writable in ${containerName}`); + console.error(homeProbe.stderr || homeProbe.stdout || homeProbe.error?.message || ''); + process.exit(2); +} + +if (prepareOnly) { + process.stdout.write(`${JSON.stringify({ containerName, + identity: `${containerId} ${containerImage}`, + networkMode: expectedNetworkMode })}\n`); + process.exit(0); +} + +const args = ['exec', '-i', '--user', `${AGENT_UID}:${AGENT_GID}`, '-w', CODING_CONTAINER_APP_ROOT]; + +args.push('-e', `HOME=${AGENT_ENVIRONMENT.HOME}`, '-e', `USER=${AGENT_ENVIRONMENT.USER}`, + '-e', 'DISABLE_AUTOUPDATER=1', '-e', 'FORCE_PROMPT_CACHING_5M=1'); +const leasedEnvironment = leasedDatabaseEnvironment(adapter, { + database: leaseContext.lease.resources.database, networkMode: expectedNetworkMode, +}); +for (const [key, value] of Object.entries(leasedEnvironment)) args.push('-e', `${key}=${value}`); +const dockerExecEnv: NodeJS.ProcessEnv = { ...process.env, MSYS_NO_PATHCONV: '1' }; +// Forward only benchmark-owned environment settings. +if (process.env.MAX_THINKING_TOKENS) { + args.push('-e', `MAX_THINKING_TOKENS=${process.env.MAX_THINKING_TOKENS}`); +} + +const claudeArgs = [ + '--print', '--output-format', 'json', + // Isolate the session from project memory, plugins, and integrations. + '--bare', + '--permission-mode', 'acceptEdits', + '--settings', JSON.stringify({ permissions: { allow: ['Bash'] } }), + '--effort', effort, + '--model', model, + ...(maxBudgetUsd !== null ? ['--max-budget-usd', maxBudgetUsd] : []), + // The app is the only directory a session may reach; inside the container + // that is all there is, but the flag is kept so host and container runs are + // configured identically. + '--add-dir', CODING_CONTAINER_APP_ROOT, + ...(resumeSession !== null ? ['--resume', resumeSession] : []), +]; +// Record the exact remote PID. Killing the local `docker exec` client does not +// guarantee that Claude stops inside the long-lived build container. +const invocationToken = randomBytes(16).toString('hex'); +const processRecord = `${CODING_CONTAINER_PROCESS_IDENTITY.recordPrefix}${invocationToken}.pid`; +const claudeWrapper = 'umask 022; record="$1"; shift; ' + + 'start="$(awk \'{print $22}\' /proc/$$/stat)" || exit 1; ' + + 'printf \'%s %s\\n\' "$$" "$start" > "$record"; exec "$@"'; + +if (!auth) throw new Error('container authentication is unavailable'); +let credentialBroker: Awaited> | null = null; +try { + credentialBroker = await startCredentialBroker(auth, + { networkMode: expectedNetworkMode, deadlineMs: CODING_SESSION_TIMEOUT_MS, model, + maxBudgetUsd: maxBudgetUsd === null ? null : Number(maxBudgetUsd), + pricingRates: maxBudgetUsd === null ? null : pricing!.rates }); +} catch (error) { + console.error(`run-build.js: ${errorMessage(error)}`); + process.exit(2); +} +if (!credentialBroker) throw new Error('credential broker is unavailable'); +dockerExecEnv.ANTHROPIC_AUTH_TOKEN = credentialBroker.sessionToken; +args.push('-e', 'ANTHROPIC_AUTH_TOKEN', '-e', `ANTHROPIC_BASE_URL=${credentialBroker.baseUrl}`, + containerName, 'sh', '-c', claudeWrapper, CODING_CONTAINER_PROCESS_IDENTITY.sessionLabel, + processRecord, + 'claude', ...claudeArgs); + +// MSYS_NO_PATHCONV: Git Bash rewrites container-side paths like /app into +// Windows paths (C:/Program Files/Git/app) and every mount silently lands +// somewhere wrong. +if (!projects) throw new Error('transcript directory is unavailable'); +const transcriptSnapshot = snapshotClaudeTranscripts(projects); +const promptInput = process.stdin.isTTY ? '' : readFileSync(0, 'utf8'); +function signalClaude(signal: 'TERM' | 'KILL') { + const script = 'record="$1"; signal="$2"; test -r "$record" || exit 4; ' + + 'read -r pid expected < "$record"; ' + + 'current="$(awk \'{print $22}\' "/proc/$pid/stat" 2>/dev/null)" || exit 5; ' + + 'test "$current" = "$expected" || exit 3; kill "-$signal" "$pid"'; + return spawnSync('docker', ['exec', containerName, 'sh', '-c', script, + CODING_CONTAINER_PROCESS_IDENTITY.stopLabel, processRecord, signal], { + encoding: 'utf8', env: dockerExecEnv, timeout: DOCKER_PROBE_TIMEOUT_MS, + }); +} +function terminateClaude(child: { kill(signal?: NodeJS.Signals): boolean }): void { + const term = signalClaude('TERM'); + if (term.status !== 0) child.kill('SIGTERM'); + const force = setTimeout(() => { + signalClaude('KILL'); + child.kill('SIGKILL'); + }, 5_000); + force.unref(); +} + +let res: Awaited> | undefined; +let sessionError: unknown = null; +let brokerLedger = null; +let brokerDiagnostics = null; +const cleanupErrors: string[] = []; +const runCleanupCommand = (description: string, command: readonly string[]): void => { + const result = spawnSync('docker', ['exec', containerName, ...command], { + encoding: 'utf8', env: dockerExecEnv, timeout: DOCKER_PROBE_TIMEOUT_MS, + }); + if (result.status !== 0) cleanupErrors.push(`${description}: ${String(result.stderr || result.stdout + || result.error?.message || `exit ${result.status}`).trim()}`); +}; +try { + res = await runTranscriptAwareProcess({ command: 'docker', args, + input: promptInput, + maxBuffer: 256 * 1024 * 1024, + env: dockerExecEnv, + timeoutMs: CODING_SESSION_TIMEOUT_MS, + transcriptDirectory: projects, + transcriptSnapshot, + marker: completionMarker as string, + model, + pricingRates: pricing?.rates ?? null, + resumeSession: resumeSession ?? undefined, + terminate: terminateClaude, + }); +} catch (error) { + sessionError = error; +} finally { + brokerLedger = await stopCredentialBroker(credentialBroker); + brokerDiagnostics = credentialBrokerDiagnostics(credentialBroker); + for (const command of codingContainerTranscriptHandoffCommands(CONTROLLER_GID)) { + runCleanupCommand('transcript handoff', command); + } + const handoff = process.env.STACK_BENCH_APPLIANCE === '1' + ? codingContainerWorkspaceHandoffCommands(CONTROLLER_GID) + : [['chmod', '-R', 'a+rwX', CODING_CONTAINER_APP_ROOT]]; + for (const command of handoff) runCleanupCommand('workspace handoff', command); + runCleanupCommand('process-record cleanup', ['rm', '-f', processRecord]); +} + +if (sessionError) { + if (cleanupErrors.length) { + throw new AggregateError([sessionError, ...cleanupErrors.map(message => new Error(message))], + 'coding session and container cleanup failed'); + } + throw sessionError; +} +if (!res) throw new Error('coding session returned no process result'); +if (cleanupErrors.length) { + res.status = res.status === 0 ? 3 : res.status ?? 3; + res.stderr = `${res.stderr ?? ''}${res.stderr ? '\n' : ''}` + + `run-build.js: container cleanup failed: ${cleanupErrors.join('; ')}\n`; +} + +let cliResult = null; +const stdout = String(res.stdout ?? '').trim(); +try { cliResult = JSON.parse(stdout); } +catch { + for (const line of stdout.split(/\r?\n/).reverse()) { + try { cliResult = JSON.parse(line); break; } catch { /* Keep looking. */ } + } +} +const memory = spawnSync('docker', ['exec', containerName, 'sh', '-c', + 'for f in memory.events memory.current memory.peak memory.max; do ' + + 'p="/sys/fs/cgroup/$f"; if test -r "$p"; then echo "[$f]"; cat "$p"; fi; done'], { + encoding: 'utf8', env: dockerExecEnv, timeout: DOCKER_PROBE_TIMEOUT_MS, +}); +const resources = { + buildContainerMemory: memory.status === 0 ? parseCgroupMemory(memory.stdout) : null, + memoryProbeError: memory.status === 0 ? null + : memory.stderr?.trim() || (memory.error instanceof Error ? memory.error.message : null) + || `exit ${memory.status}`, +}; +if (maxBudgetUsd !== null) { + const reconciled = reconcileCredentialBrokerReceipt({ + ledger: brokerLedger, + cliResult, + model, + maxBudgetUsd: Number(maxBudgetUsd), + pricingRates: pricing!.rates, + brokerDiagnostics, + }); + reconciled.result.stack_bench_resources = resources; + res.stdout = `${JSON.stringify(reconciled.result)}\n`; + if (!reconciled.ok) { + res.status = res.status === 0 ? 3 : res.status ?? 3; + res.stderr = `${res.stderr ?? ''}${res.stderr ? '\n' : ''}` + + `run-build.js: ${reconciled.receipt.error}\n`; + } +} else if (cliResult && typeof cliResult === 'object' && !Array.isArray(cliResult)) { + cliResult.stack_bench_credential_broker = brokerDiagnostics; + cliResult.stack_bench_resources = resources; + res.stdout = `${JSON.stringify(cliResult)}\n`; +} + +if ((res.status ?? 1) !== 0) { + const state = spawnSync('docker', ['inspect', '--format', '{{json .State}}', containerName], { + encoding: 'utf8', env: dockerExecEnv, timeout: DOCKER_PROBE_TIMEOUT_MS, + }); + let containerState = null; + try { containerState = JSON.parse(state.stdout?.trim() || 'null'); } catch { /* retain raw text below */ } + const diagnostic = { + schemaVersion: 1, + kind: 'coding-process-exit', + status: res.status ?? null, + signal: res.signal ?? null, + error: res.error instanceof Error ? res.error.message : null, + container: containerState ?? { inspectError: state.stderr?.trim() + || (state.error instanceof Error ? state.error.message : null) }, + cgroupMemory: memory.stdout?.trim() || null, + cgroupProbeError: memory.status === 0 ? null + : memory.stderr?.trim() || (memory.error instanceof Error ? memory.error.message : null) + || `exit ${memory.status}`, + }; + process.stderr.write(`STACK_BENCH_CODING_PROCESS_DIAGNOSTIC ${JSON.stringify(diagnostic)}\n`); +} + +if (res.stdout) process.stdout.write(res.stdout); +if (res.stderr) process.stderr.write(res.stderr); +if (res.error) process.stderr.write(`run-build.js: coding session failed: ${errorMessage(res.error)}\n`); +process.exit(res.status ?? 1); diff --git a/tools/stack-bench/container/spacetimedb-binaries.json b/tools/stack-bench/container/spacetimedb-binaries.json new file mode 100644 index 00000000000..e4af3ced333 --- /dev/null +++ b/tools/stack-bench/container/spacetimedb-binaries.json @@ -0,0 +1,21 @@ +{ + "schemaVersion": 2, + "platform": "linux/amd64", + "builderImage": "rust:1.93-slim-bookworm@sha256:8f8609d448e821fbc0e44241bc5ca4ce49663cc6306ff1a17f655a0e2a7cd084", + "source": { + "identityScheme": "git-object-content-v1", + "revision": "64afb5fd35aa391ccbff423d0bec272cefbd6092", + "sha256": "c76824c57bbfee62d707b36c1c6a4bdf1fffb295770931cf0b67f7adb9f88ed3", + "files": 2586 + }, + "binaries": { + "spacetimedb-cli": { + "sha256": "376fbb1978533b75bc144a7b27cc202ceec5da4d39ad840dc4a88fff59049157", + "size": 46986456 + }, + "spacetimedb-standalone": { + "sha256": "d53a8cbc4325bb87024748b725d13d5984f85bb0600ace3cea646a7ff6c5dce2", + "size": 134334152 + } + } +} diff --git a/tools/stack-bench/dashboard/README.md b/tools/stack-bench/dashboard/README.md new file mode 100644 index 00000000000..1df142872f7 --- /dev/null +++ b/tools/stack-bench/dashboard/README.md @@ -0,0 +1,100 @@ +# Stack Bench dashboard + +The dashboard is an optional local view over Stack Bench results. It does not +schedule attempts, grade applications, operate Docker, or repair source itself. +Campaign plans, durable campaign state, and run artifacts remain the source of +truth; the dashboard only reads them and, in the appliance, asks the controller +to start or resume a run. + +## Pages + +- Campaigns (`/`) — a lane per running attempt, then one row per campaign with + its shape, status, and per-stack score. A campaign whose plan or state this + build cannot read appears with the status `unreadable` and the reason in + place of its title. +- Campaign (`/c/:key`) — one sheet: the plan's facts across the top, then + score, unaided, repairs, regressions, time, spend, climb, and attempt phase + per stack. Dependency campaigns add questline rows, which + `?questlines=grid|graph|replay` switches between; `&step=N` moves the replay + cursor. Sequential campaigns show one row pair per level instead. +- Attempt (`/c/:key/a/:attemptId`) — the attempt's figures, its climb, and + `?tab=checks|screenshots|files|log`. The log tab follows new bytes. +- Plans (`/plans`) — the frozen campaign plans found under the plans directory, + and the form that starts one. + +## Modes + +Inside the appliance (`STACK_BENCH_APPLIANCE=1`) the dashboard runs in +controller mode: the Plans form and the resume button post to the controller. +Elsewhere it runs read-only and those controls are unavailable; +`GET /api/health` reports which mode is active. + +For UI development, `npm run dashboard` starts a read-only host view over +`tools/stack-bench/results`. Pass `--port` to move it off 7331 and `--results` +to point it at another results directory. + +## Appliance + +```sh +docker compose --env-file /var/lib/stack-bench/operator.env \ + -f appliance/docker-compose.yaml --profile dashboard up -d dashboard +``` + +Open `http://127.0.0.1:7331`. Docker publishes that port only on the host's +loopback interface. Stop it with: + +```sh +docker compose --env-file /var/lib/stack-bench/operator.env \ + -f appliance/docker-compose.yaml --profile dashboard stop dashboard +``` + +Starting or resuming a run needs the separate dashboard control secret, typed +into the form. The server reads the expected value from the file configured by +`STACK_BENCH_DASHBOARD_CONTROL_SECRET_FILE` and no dashboard API returns it. A +wrong secret is answered with 403 and nothing is started. Starting a campaign +invokes the same `campaign run` command used by the CLI, so the CLI can inspect +or resume the result normally and CLI-started campaigns appear here. + +## Routes + +| route | returns | +| --- | --- | +| `GET /api/health` | `read-only` or `controller` | +| `GET /api/overview` | one summary per campaign | +| `GET /api/campaigns/:key` | the campaign sheet | +| `GET /api/campaigns/:key/progression` | the dependency graph and its replay | +| `GET /api/campaigns/:key/attempts/:id/checks` | per-check outcome and history | +| `GET /api/campaigns/:key/attempts/:id/package` | the evidence listing | +| `GET /api/campaigns/:key/attempts/:id/log?from=N` | log bytes after `N` | +| `GET /api/campaigns/:key/artifacts/:name` | one allowlisted artifact | +| `GET /api/events` | the change stream | +| `GET /api/plans` | the discovered plans | +| `POST /api/campaigns` | start a run | +| `POST /api/campaigns/:key/resume` | resume an interrupted dependency run | + +Each payload covers one question, so opening a campaign or a tab is what pays +for reading it. The overview and the sheet are cached against the size and +modification time of the evidence they read, including while a campaign runs. + +## The event stream + +`GET /api/events` is a server-sent event stream. A `campaign` event names a +campaign whose plan, state, run output, or progression state changed; a `log` +event names an attempt whose stdout grew. Changes are debounced for 500 ms and +the stream sends a comment every 25 seconds so an idle connection stays open. +The client loads the overview once, then refetches only what an event names. +While the stream is down it falls back to polling the overview every 15 +seconds. + +The watcher uses a recursive `fs.watch` per campaign directory. Where the +platform or the mount does not support one it polls the same file fingerprints +every 5 seconds instead. The server logs which mode it opened with when the +first client subscribes. + +## What it touches + +It reads campaign plans from `/plans` and campaigns from +`/campaigns`. It writes nothing under a campaign directory. The only +file it appends to is `/dashboard/operations.jsonl`, the record of +runs submitted through the dashboard, and the controller it starts writes its +own output under `/dashboard/operations`. diff --git a/tools/stack-bench/dashboard/dashboard-events.ts b/tools/stack-bench/dashboard/dashboard-events.ts new file mode 100644 index 00000000000..fb19a334e31 --- /dev/null +++ b/tools/stack-bench/dashboard/dashboard-events.ts @@ -0,0 +1,164 @@ +import { existsSync, readdirSync, statSync, watch } from 'node:fs'; +import type { FSWatcher } from 'node:fs'; +import { join } from 'node:path'; + +import { ARTIFACT_FILE } from '../src/evidence/artifacts.js'; +import { CAMPAIGN_FILE } from '../src/campaigns/campaign-path.js'; + +const DEBOUNCE_MS = 500; +const POLL_MS = 5000; +const LOG_FILE = 'process.stdout.log'; +const CAMPAIGN_FILES = [CAMPAIGN_FILE.plan, CAMPAIGN_FILE.state] as const; +const EXECUTION_FILES = [ARTIFACT_FILE.run, ARTIFACT_FILE.progressionState] as const; + +export interface CampaignChange { + type: 'campaign' | 'log'; + key: string; + attemptId?: string; +} + +export type WatchMode = 'watch' | 'poll'; + +export interface CampaignWatcher { + close(): void; +} + +interface CampaignFingerprint { + campaign: string; + logs: Map; +} + +function stamp(path: string): string | null { + if (!existsSync(path)) return null; + const stat = statSync(path); + return `${stat.size}:${stat.mtimeMs}`; +} + +function directories(root: string): string[] { + if (!existsSync(root)) return []; + return readdirSync(root, { withFileTypes: true }) + .filter(entry => entry.isDirectory()).map(entry => entry.name); +} + +// The evidence a view reads, in one pass: the campaign files and each +// execution's result, and the log sizes that tell a follower there are new +// bytes to fetch. +function fingerprintCampaign(directory: string): CampaignFingerprint { + const parts = CAMPAIGN_FILES.map(file => `${file}:${stamp(join(directory, file)) ?? 'missing'}`); + const logs = new Map(); + const attemptsRoot = join(directory, 'attempts'); + for (const attempt of directories(attemptsRoot)) { + const attemptDirectory = join(attemptsRoot, attempt); + let bytes = 0; + for (const execution of directories(attemptDirectory)) { + const executionDirectory = join(attemptDirectory, execution); + for (const file of EXECUTION_FILES) { + const value = stamp(join(executionDirectory, file)); + if (value) parts.push(`${attempt}/${execution}/${file}:${value}`); + } + const log = join(executionDirectory, LOG_FILE); + if (existsSync(log)) bytes += statSync(log).size; + } + logs.set(attempt, bytes); + } + return { campaign: parts.sort().join('|'), logs }; +} + +// One watcher for the whole server: a recursive watch per campaign directory +// where the platform supports it (Windows, macOS, and Linux on Node 20 and +// later), and a poll of the same fingerprints where it does not. +export function watchCampaigns(campaignsRoot: string, + emit: (change: CampaignChange) => void, + onMode: (mode: WatchMode) => void = () => {}): CampaignWatcher { + const fingerprints = new Map(); + const watchers = new Map(); + const timers = new Map(); + let rootWatcher: FSWatcher | null = null; + let poll: NodeJS.Timeout | null = null; + let closed = false; + + const check = (key: string): void => { + timers.delete(key); + const directory = join(campaignsRoot, key); + if (!existsSync(directory)) { + fingerprints.delete(key); + watchers.get(key)?.close(); + watchers.delete(key); + return; + } + const next = fingerprintCampaign(directory); + const previous = fingerprints.get(key); + fingerprints.set(key, next); + if (!previous) return; + if (previous.campaign !== next.campaign) emit({ type: 'campaign', key }); + for (const [attemptId, bytes] of next.logs) { + if ((previous.logs.get(attemptId) ?? 0) !== bytes) emit({ type: 'log', key, attemptId }); + } + }; + + const schedule = (key: string): void => { + if (closed || timers.has(key)) return; + timers.set(key, setTimeout(() => check(key), DEBOUNCE_MS).unref()); + }; + + const startPoll = (): void => { + if (closed || poll) return; + onMode('poll'); + poll = setInterval(() => { + attach(); + for (const key of directories(campaignsRoot)) check(key); + }, POLL_MS).unref(); + }; + + const attach = (): void => { + if (closed) return; + if (!rootWatcher && existsSync(campaignsRoot)) { + try { + rootWatcher = watch(campaignsRoot, { persistent: false }, (_event, name) => { + const key = String(name ?? '').split(/[/\\]/)[0]; + if (key) schedule(key); + attach(); + }); + rootWatcher.once('error', () => { rootWatcher = null; startPoll(); }); + } catch { startPoll(); } + } + for (const key of directories(campaignsRoot)) { + // A campaign that appeared since the last pass has everything to report. + if (!fingerprints.has(key)) { + fingerprints.set(key, { campaign: '', logs: new Map() }); + schedule(key); + } + if (watchers.has(key) || poll) continue; + try { + const watcher = watch(join(campaignsRoot, key), { persistent: false, recursive: true }, + () => schedule(key)); + watcher.once('error', () => { watchers.delete(key); startPoll(); }); + watchers.set(key, watcher); + } catch { + // No recursive watch on this platform: the poll reads the same files. + startPoll(); + return; + } + } + }; + + for (const key of directories(campaignsRoot)) { + fingerprints.set(key, fingerprintCampaign(join(campaignsRoot, key))); + } + attach(); + if (!rootWatcher) startPoll(); + else if (!poll) onMode('watch'); + return { + close() { + closed = true; + for (const timer of timers.values()) clearTimeout(timer); + timers.clear(); + if (poll) clearInterval(poll); + poll = null; + rootWatcher?.close(); + rootWatcher = null; + for (const watcher of watchers.values()) watcher.close(); + watchers.clear(); + }, + }; +} diff --git a/tools/stack-bench/dashboard/dashboard-model.ts b/tools/stack-bench/dashboard/dashboard-model.ts new file mode 100644 index 00000000000..5222b658c1d --- /dev/null +++ b/tools/stack-bench/dashboard/dashboard-model.ts @@ -0,0 +1,502 @@ +import { closeSync, existsSync, fstatSync, openSync, readFileSync, readSync, readdirSync, + lstatSync, realpathSync, statSync, +} from 'node:fs'; +import { basename, dirname, extname, join, relative, resolve, sep } from 'node:path'; + +import type { CompiledCampaignPlan } + from '../src/campaigns/campaign-compiler.js'; +import type { CampaignAttemptState } from '../src/campaigns/campaign-scheduler.js'; +import { ARTIFACT_FILE } from '../src/evidence/artifacts.js'; +import { compileCampaignFile } from '../src/campaigns/campaign-compiler.js'; +import { campaignLockIsActive } from '../src/campaigns/campaign-lock.js'; +import { readCampaignState } from '../src/campaigns/campaign-scheduler.js'; +import { campaignFacts, inspectCampaignAttempt } from '../src/campaigns/campaign-inspection.js'; +import { redactCredentials } from '../src/evidence/diagnostic-sanitizer.js'; +import { CAMPAIGN_FILE } from '../src/campaigns/campaign-path.js'; +import { repairBudgetLimit } from '../src/progression/repair-plan.js'; + +export const MAX_LOG_BYTES = 96 * 1024; +const MAX_PUBLIC_TEXT_BYTES = 8 * 1024 * 1024; +const MAX_ARTIFACTS_PER_EXECUTION = 512; +const IMAGE_TYPES = new Map([ + ['.png', 'image/png'], ['.jpg', 'image/jpeg'], ['.jpeg', 'image/jpeg'], ['.webp', 'image/webp'], +]); +const CAMPAIGN_ARTIFACT = /^(?:plan\.json|state\.json|report\/report\.(?:html|json))$/; +const EXECUTION_ARTIFACT = /^(?:run\.json|preflight\.json|recovery\.json|progression-state\.json|process\.json|process\.(?:stdout|stderr)\.log|backend\.log|level-l\d+-checkpoint\.json|progression\/attempt-\d+\/(?:bundle\.json|contract-lint\.json|actions\.json|grading-[^/]+\.json|failure-media\/[^/]+\.(?:png|jpe?g|webp))|(?:first-build-l\d+-grading|l\d+-fix\d+-grading|grading)\/(?:bundle\.json|contract-lint\.json|actions\.json|grading-[^/]+\.json|failure-media\/[^/]+\.(?:png|jpe?g|webp)))$/i; + +type ControllerActive = (directory: string, campaign: CompiledCampaignPlan) => boolean; + +export interface DashboardArtifact { + id: string; + path: string; + name: string; + kind: 'visual' | 'report' | 'log' | 'data'; + contentType: string; + size: number; +} + +export interface ResolvedDashboardArtifact extends DashboardArtifact { + absolute: string; +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +export function contained(root: string, path: string, label: string): string { + const absoluteRoot = resolve(root); + const absolute = resolve(absoluteRoot, path); + const rel = relative(absoluteRoot, absolute); + if (rel === '..' || rel.startsWith(`..${sep}`) || rel === '') { + throw new Error(`${label} is outside the configured dashboard root`); + } + return absolute; +} + +export function readTextTail(path: string, limit = MAX_LOG_BYTES): string { + if (!existsSync(path)) return ''; + const descriptor = openSync(path, 'r'); + try { + const size = fstatSync(descriptor).size; + const length = Math.min(size, limit); + const buffer = Buffer.alloc(length); + readSync(descriptor, buffer, 0, length, size - length); + return redactCredentials(buffer.toString('utf8')); + } finally { + closeSync(descriptor); + } +} + +function artifactId(relativePath: string): string { + return Buffer.from(relativePath, 'utf8').toString('base64url'); +} + +function artifactLabel(path: string): string { + const name = basename(path); + if (path === CAMPAIGN_FILE.plan) return 'Frozen plan'; + if (path === CAMPAIGN_FILE.state) return 'Campaign state'; + if (path === `report/${CAMPAIGN_FILE.reportHtml}`) return 'Campaign report'; + if (path === `report/${CAMPAIGN_FILE.reportJson}`) return 'Report data'; + if (name === ARTIFACT_FILE.run) return 'Run result'; + if (name === ARTIFACT_FILE.preflight) return 'Preflight result'; + if (name === ARTIFACT_FILE.recovery) return 'Recovery record'; + if (name === ARTIFACT_FILE.progressionState) return 'Dependency progress'; + if (name === 'process.stdout.log') return 'Run output'; + if (name === 'process.stderr.log') return 'Run errors'; + if (name === 'backend.log') return 'Backend output'; + if (name === ARTIFACT_FILE.gradeBundle) return `${basename(dirname(path))} bundle`; + if (name === ARTIFACT_FILE.actions) return `${basename(dirname(path))} actions`; + if (name === ARTIFACT_FILE.contractLint) return `${basename(dirname(path))} contract check`; + return name.replace(/[-_]/g, ' '); +} + +function artifactMetadata(campaignDirectory: string, path: string): DashboardArtifact { + const absolute = contained(campaignDirectory, path, 'campaign artifact'); + const size = statSync(absolute).size; + const extension = extname(path).toLowerCase(); + const kind = IMAGE_TYPES.has(extension) ? 'visual' + : path.endsWith('/report.html') ? 'report' + : path.endsWith('.log') ? 'log' : 'data'; + return { id: artifactId(path), path: path.replaceAll('\\', '/'), name: artifactLabel(path), + kind, contentType: IMAGE_TYPES.get(extension) ?? (kind === 'report' ? 'text/html' : 'text/plain'), + size }; +} + +function rejectSymlinkPath(root: string, path: string): void { + const rel = relative(resolve(root), resolve(path)); + let current = resolve(root); + for (const segment of rel.split(sep)) { + current = join(current, segment); + if (lstatSync(current).isSymbolicLink()) { + throw new Error('campaign artifact path contains a symbolic link'); + } + } +} + +export function walkPublicExecutionArtifacts(campaignDirectory: string, executionDirectory: string): { + artifacts: DashboardArtifact[]; + truncated: boolean; +} { + const found: DashboardArtifact[] = []; + let truncated = false; + const visit = (directory: string): void => { + const directoryRelative = relative(executionDirectory, directory).replaceAll('\\', '/'); + for (const entry of readdirSync(directory, { withFileTypes: true })) { + if (found.length >= MAX_ARTIFACTS_PER_EXECUTION) { + truncated = true; + return; + } + if (entry.isSymbolicLink()) continue; + const absolute = join(directory, entry.name); + if (entry.isDirectory()) { + const allowed = directoryRelative === '' + ? /^(?:first-build-l\d+-grading|l\d+-fix\d+-grading|grading|progression)$/i.test(entry.name) + : directoryRelative === 'progression' + ? /^attempt-\d+$/i.test(entry.name) + : (/^(?:progression\/attempt-\d+|(?:.*\/)?(?:first-build-l\d+-grading|l\d+-fix\d+-grading|grading))$/i + .test(directoryRelative) && entry.name === 'failure-media'); + if (allowed) visit(absolute); + if (truncated) return; + } + else if (entry.isFile()) { + const executionRelative = relative(executionDirectory, absolute).replaceAll('\\', '/'); + if (EXECUTION_ARTIFACT.test(executionRelative)) { + const campaignRelative = relative(campaignDirectory, absolute).replaceAll('\\', '/'); + found.push(artifactMetadata(campaignDirectory, campaignRelative)); + } + } + } + }; + if (existsSync(executionDirectory)) visit(executionDirectory); + return { artifacts: found.sort((left, right) => left.path.localeCompare(right.path)), truncated }; +} + +function campaignPackage(campaignDirectory: string, attempts: CampaignAttemptState[]) { + const campaign = [CAMPAIGN_FILE.plan, CAMPAIGN_FILE.state, + `report/${CAMPAIGN_FILE.reportHtml}`, `report/${CAMPAIGN_FILE.reportJson}`] + .filter(path => existsSync(join(campaignDirectory, path))) + .map(path => artifactMetadata(campaignDirectory, path)); + const executions: Array<{ + attemptId: string; + stack: string; + executionId: string; + ordinal: number; + status: string; + artifacts: DashboardArtifact[]; + visuals: DashboardArtifact[]; + truncated: boolean; + }> = []; + for (const attempt of attempts) { + for (const execution of attempt.executions) { + const directory = contained(campaignDirectory, execution.output, 'campaign execution'); + const scanned = walkPublicExecutionArtifacts(campaignDirectory, directory); + const artifacts = scanned.artifacts; + executions.push({ attemptId: attempt.plan.id, stack: attempt.plan.stack, + executionId: execution.id, ordinal: execution.ordinal, status: execution.status, + artifacts, visuals: artifacts.filter(artifact => artifact.kind === 'visual'), + truncated: scanned.truncated }); + } + } + return { campaign, executions }; +} + +export function resolveCampaignArtifact(resultsRoot: string, key: string, + id: string): ResolvedDashboardArtifact { + if (!/^[a-z0-9][a-z0-9.-]*$/.test(key)) throw new Error('campaign key is invalid'); + let path; + try { path = Buffer.from(id, 'base64url').toString('utf8'); } + catch { throw new Error('campaign artifact id is invalid'); } + if (!path || artifactId(path) !== id || path.includes('\\') || path.startsWith('/')) { + throw new Error('campaign artifact id is invalid'); + } + const executionMatch = path.match(/^attempts\/([^/]+)\/(execution-\d+)\/(.+)$/); + const allowed = CAMPAIGN_ARTIFACT.test(path) + || (executionMatch !== null && EXECUTION_ARTIFACT.test(executionMatch[3] ?? '')); + if (!allowed) { + throw new Error('campaign artifact is not available in the dashboard'); + } + const campaignsRoot = join(resolve(resultsRoot), 'campaigns'); + const campaignDirectory = contained(campaignsRoot, key, 'campaign'); + const absolute = contained(campaignDirectory, path, 'campaign artifact'); + if (!existsSync(absolute) || !statSync(absolute).isFile()) throw new Error('campaign artifact does not exist'); + rejectSymlinkPath(campaignsRoot, absolute); + const realCampaign = realpathSync(campaignDirectory); + const realArtifact = realpathSync(absolute); + contained(realCampaign, relative(realCampaign, realArtifact), 'campaign artifact'); + return { ...artifactMetadata(campaignDirectory, path), absolute }; +} + +export function readCampaignArtifactBody(artifact: ResolvedDashboardArtifact): Buffer { + if (artifact.kind === 'visual') return readFileSync(artifact.absolute); + if (artifact.size > MAX_PUBLIC_TEXT_BYTES) throw new Error('campaign artifact is too large to view'); + return Buffer.from(redactCredentials(readFileSync(artifact.absolute, 'utf8'))); +} + +function matches(text: string, pattern: RegExp): Array { + return [...text.matchAll(pattern)].map(match => Object.assign(match, { index: match.index ?? 0 })); +} + +export function parseRunProgress(log: string, { repairs = 0, running = true, status = null, + dependency = false }: { + repairs?: number; + running?: boolean; + status?: string | null; + dependency?: boolean; +} = {}) { + const totals = matches(log, /^\s*TOTAL\b.*?(\d+)\/(\d+)\s*$/gm) + .map(match => ({ index: match.index, score: Number(match[1]), max: Number(match[2]) })); + // A run-wide repair prints "repair N/M"; a feature repair prints + // "feature repair N: title" because its limit belongs to the feature. + const roundMarkers = matches(log, + /^--- (?:feature )?repair (\d+)(?:\/(\d+))?(?:: (.+))? ---$/gm) + .map(match => ({ index: match.index, round: Number(match[1]), + budget: match[2] === undefined ? null : Number(match[2]), + target: match[3] ?? null })); + const grading = matches(log, /^===\s+[^\n]*?-l(\d+)(?:-(?:first|fix(\d+)))?\s+\([^\n]+\)\s*===$/gm) + .map(match => ({ index: match.index, level: Number(match[1]), + round: match[2] ? Number(match[2]) : 0 })); + const latestTotal = totals.at(-1) ?? null; + const latestRound = roundMarkers.at(-1) ?? null; + const latestGrading = grading.at(-1) ?? null; + const latestIndex = Math.max(latestTotal?.index ?? -1, latestRound?.index ?? -1, + latestGrading?.index ?? -1); + let phase = status === 'pending' ? 'Waiting to start' + : running ? 'Building the generated app' : 'Finished'; + const level = latestGrading?.level ?? null; + const round = latestRound?.round ?? latestGrading?.round ?? 0; + const budget = latestRound ? latestRound.budget : repairs; + const target = latestRound?.target ? ` for ${latestRound.target}` : ''; + const of = (limit: number | null): string => limit === null ? '' : ` of ${limit}`; + const stage = (value: number): string => dependency ? `depth ${value}` : `L${value}`; + if (latestIndex === latestGrading?.index) { + phase = latestGrading.round + ? `Grading ${stage(latestGrading.level)} after repair ${round}${of(budget)}${target}` + : `Grading the first ${stage(latestGrading.level)} build`; + } else if (latestIndex === latestRound?.index) { + phase = latestRound.target + ? `Repairing ${latestRound.target} · ${latestRound.round}${of(latestRound.budget)}` + : `Repairing ${stage(latestGrading?.level ?? 1)} · round ${latestRound.round}${of(latestRound.budget)}`; + } else if (latestIndex === latestTotal?.index && running) { + phase = 'Preparing the next step'; + } + return { + phase, + level, + repair: { round, budget }, + firstScore: totals[0] ? { score: totals[0].score, max: totals[0].max } : null, + latestScore: latestTotal ? { score: latestTotal.score, max: latestTotal.max } : null, + completedGrades: totals.length, + // Every completed grade in order — the attempt's trajectory — carrying the + // level it graded and whether it was the unaided build of that level. A + // view can draw the climb with its bands, and a flat tail is the stall an + // operator otherwise discovers by diffing round logs. + series: totals.map(total => { + const mark = grading.findLast(entry => entry.index < total.index) ?? null; + return { score: total.score, max: total.max, level: mark?.level ?? null, + unaided: mark ? mark.round === 0 : false }; + }), + }; +} + +function summarizeAttempt(plan: CompiledCampaignPlan, attempt: CampaignAttemptState, + campaignDirectory: string, repairs: number, { includeLog = false }: { + includeLog?: boolean; + } = {}) { + const inspected = inspectCampaignAttempt(plan, attempt, campaignDirectory); + const execution = inspected.execution; + let executionDirectory = null; + let log = ''; + let logUpdatedAt = null; + if (execution) { + executionDirectory = contained(campaignDirectory, execution.output, 'campaign execution'); + const logPath = join(executionDirectory, 'process.stdout.log'); + log = readTextTail(logPath); + // When the run last wrote anything. A running attempt whose output has + // been silent for a long time is wedged in a way no score can show. + if (existsSync(logPath)) logUpdatedAt = new Date(statSync(logPath).mtimeMs).toISOString(); + } + const progress = parseRunProgress(log, { repairs, running: attempt.status === 'running', + status: attempt.status, dependency: plan.definition.mode.id === 'dependency' }); + if (inspected.result?.score) progress.latestScore = inspected.result.score; + return { + ...inspected, + progress, + logUpdatedAt, + ...(includeLog ? { log: log.split(/\r?\n/).slice(-160).join('\n') } : {}), + }; +} + +export function summarizeCampaign(directory: string, { + includeLogs = false, + includePackage = false, + includeAttempts = true, + controllerActive = null, +}: { + includeLogs?: boolean; + includePackage?: boolean; + includeAttempts?: boolean; + controllerActive?: ControllerActive | null; +} = {}) { + const { plan, state } = readCampaignState(directory, { requireCurrentInputs: false }); + let attempts = includeAttempts + ? state.attempts.map(attempt => summarizeAttempt(plan, attempt, directory, + repairBudgetLimit(plan.definition.repair, { + features: plan.featureCatalog?.definition.nodes.length ?? 1, + depths: plan.definition.levels.length, + }), { includeLog: includeLogs })) + : []; + const interrupted = state.status === 'running' && controllerActive !== null + && !controllerActive(directory, plan); + if (interrupted) { + attempts = attempts.map(attempt => attempt.status !== 'running' ? attempt : ({ + ...attempt, + status: 'interrupted', + execution: attempt.execution ? { ...attempt.execution, status: 'interrupted' } : null, + progress: { ...attempt.progress, phase: 'Controller stopped before completion' }, + })); + } + return { + key: basename(resolve(directory)), + id: plan.id, + version: plan.version, + sha256: plan.contentSha256, + title: plan.title, + state: plan.state, + mode: plan.definition.mode?.id ?? 'sequential', + status: interrupted ? 'attention-required' : state.status, + track: plan.definition.track, + levels: plan.definition.levels, + stacks: plan.stacks.map(stack => stack.id), + repetitions: plan.definition.repetitions, + maxParallel: state.maxParallel, + createdAt: state.createdAt, + updatedAt: state.updatedAt, + summary: interrupted ? { ...state.summary, interrupted: state.summary.running, running: 0 } + : state.summary, + interrupted, + ...(interrupted ? { statusReason: 'The campaign controller is no longer running.' } : {}), + budgets: plan.definition.budgets, + facts: campaignFacts(plan), + attempts, + ...(includePackage ? { package: campaignPackage(directory, state.attempts) } : {}), + }; +} + +export interface UnreadableDashboardCampaign { + key: string; + id: string; + title: string; + status: 'unreadable'; + error: string; + attempts: []; +} + +export type DashboardCampaign = ReturnType; +export type DashboardCampaignSummary = DashboardCampaign | UnreadableDashboardCampaign; + +const overviewCampaignCache = new Map(); + +function summarizeOverviewCampaign(directory: string, includeAttempts: boolean, + controllerActive: ControllerActive): DashboardCampaign { + const fingerprint = [CAMPAIGN_FILE.plan, CAMPAIGN_FILE.state] + .map(file => { + const stat = statSync(join(directory, file)); + return `${stat.size}:${stat.mtimeMs}`; + }).join('|'); + const key = `${includeAttempts ? 'attempts' : 'summary'}:${directory}`; + const cached = overviewCampaignCache.get(key); + if (cached?.fingerprint === fingerprint) return cached.campaign; + const campaign = summarizeCampaign(directory, { includeAttempts, controllerActive }); + if (campaign.summary.running === 0) { + overviewCampaignCache.set(key, { fingerprint, campaign }); + } + return campaign; +} + +export function discoverCampaigns(campaignsRoot: string, { + includeLogs = false, + controllerActive = campaignLockIsActive, +}: { includeLogs?: boolean; controllerActive?: ControllerActive } = {}) { + if (!existsSync(campaignsRoot)) return []; + const campaigns: DashboardCampaignSummary[] = []; + for (const entry of readdirSync(campaignsRoot, { withFileTypes: true })) { + if (!entry.isDirectory()) continue; + const directory = join(campaignsRoot, entry.name); + if (!existsSync(join(directory, CAMPAIGN_FILE.state)) + || !existsSync(join(directory, CAMPAIGN_FILE.plan))) continue; + try { + campaigns.push(includeLogs + ? summarizeCampaign(directory, { includeLogs, controllerActive }) + : summarizeOverviewCampaign(directory, false, controllerActive)); + } catch (error) { + campaigns.push({ key: entry.name, id: entry.name, title: entry.name, + status: 'unreadable', error: errorMessage(error), attempts: [] }); + } + } + campaigns.sort((left, right) => String('updatedAt' in right ? right.updatedAt ?? '' : '') + .localeCompare(String('updatedAt' in left ? left.updatedAt ?? '' : ''))); + if (includeLogs) return campaigns; + + const verdict = campaigns.find(campaign => campaign.status === 'completed' + && 'facts' in campaign && campaign.facts.grading.status === 'qualified'); + return campaigns.map(campaign => { + if (campaign.status !== 'running' && campaign !== verdict) return campaign; + try { + return summarizeOverviewCampaign(join(campaignsRoot, campaign.key), true, controllerActive); + } catch (error) { + const unreadable: UnreadableDashboardCampaign = { + key: campaign.key, + id: campaign.id, + title: campaign.title, + status: 'unreadable', + error: errorMessage(error), + attempts: [], + }; + return unreadable; + } + }); +} + +export interface DashboardPlan { + id: string; + version?: string; + title: string; + state: string; + mode?: string; + track?: string; + levels?: number[]; + stacks?: string[]; + attempts?: number; + parallelism?: number; + budgets?: CompiledCampaignPlan['definition']['budgets']; + repairBudget?: number; + sha256?: string; + file: string; + error?: string; +} + +export function discoverPlans(plansRoot: string): DashboardPlan[] { + if (!existsSync(plansRoot)) return []; + const plans: DashboardPlan[] = []; + for (const entry of readdirSync(plansRoot, { withFileTypes: true })) { + if (!entry.isFile() || !entry.name.endsWith('.json')) continue; + const path = join(plansRoot, entry.name); + try { + const plan = compileCampaignFile(path); + plans.push({ id: plan.id, version: plan.version, title: plan.title, state: plan.state, + mode: plan.definition.mode?.id ?? 'sequential', + track: plan.definition.track, levels: plan.definition.levels, + stacks: plan.stacks.map(stack => stack.id), attempts: plan.summary.attempts, + parallelism: plan.summary.parallelism, budgets: plan.definition.budgets, + repairBudget: repairBudgetLimit(plan.definition.repair, { + features: plan.featureCatalog?.definition.nodes.length ?? 1, + depths: plan.definition.levels.length, + }), + sha256: plan.contentSha256, file: entry.name }); + } catch (error) { + plans.push({ id: entry.name.slice(0, -5), title: entry.name, state: 'invalid', + error: errorMessage(error), file: entry.name }); + } + } + return plans.sort((left, right) => left.title.localeCompare(right.title)); +} + +export function readJsonLines(path: string): unknown[] { + if (!existsSync(path)) return []; + const lines = readFileSync(path, 'utf8').split(/\r?\n/); + const last = lines.findLastIndex(line => line.trim() !== ''); + const events: unknown[] = []; + for (let index = 0; index <= last; index += 1) { + const line = lines[index]; + if (!line?.trim()) continue; + try { events.push(JSON.parse(line)); } + catch { + if (index === last) break; + throw new Error(`dashboard operation feed line ${index + 1} is invalid JSON`); + } + } + return events; +} diff --git a/tools/stack-bench/dashboard/dashboard-server.ts b/tools/stack-bench/dashboard/dashboard-server.ts new file mode 100644 index 00000000000..2263361c44e --- /dev/null +++ b/tools/stack-bench/dashboard/dashboard-server.ts @@ -0,0 +1,504 @@ +#!/usr/bin/env node + +import { randomBytes, randomUUID, timingSafeEqual } from 'node:crypto'; +import { appendFileSync, closeSync, createReadStream, existsSync, mkdirSync, openSync, readFileSync, statSync } from 'node:fs'; +import { createServer } from 'node:http'; +import type { IncomingMessage, ServerResponse } from 'node:http'; +import { basename, dirname, join, resolve } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +import { spawn } from 'node:child_process'; +import { parseArgs as parseNodeArgs } from 'node:util'; + +import { contained, discoverPlans, readCampaignArtifactBody, + readJsonLines, resolveCampaignArtifact, summarizeCampaign, +} from './dashboard-model.js'; +import type { DashboardPlan } from './dashboard-model.js'; +import { attemptChecks, attemptLogSlice, attemptPackage, campaignProgression, campaignSheet, + overviewSummary } from './dashboard-views.js'; +import { watchCampaigns } from './dashboard-events.js'; +import type { CampaignChange, CampaignWatcher } from './dashboard-events.js'; +import { STACK_BENCH_ROOT } from '../src/package-root.js'; +import { stackBenchResultsRoot } from '../src/runtime/operational-paths.js'; + +const DASHBOARD_ROOT = dirname(fileURLToPath(import.meta.url)); +const RUNTIME_ROOT = dirname(DASHBOARD_ROOT); +const PUBLIC_ROOT = join(DASHBOARD_ROOT, 'public'); +const CAMPAIGN_CLI = join(RUNTIME_ROOT, 'commands', 'campaign-cli.js'); +const SAFE_NAME = /^[a-z0-9][a-z0-9.-]{2,119}$/; +const SPA_PATH = /^\/(?:plans|c\/[^/]+(?:\/a\/[^/]+)?)$/; +const HEARTBEAT_MS = 25_000; +const LOOPBACK = new Set(['127.0.0.1', '::1', 'localhost']); +const STATIC = new Map([ + ['/', ['index.html', 'text/html; charset=utf-8']], + ['/app.js', ['app.js', 'text/javascript; charset=utf-8']], + ['/climb.js', ['climb.js', 'text/javascript; charset=utf-8']], + ['/format.js', ['format.js', 'text/javascript; charset=utf-8']], + ['/graph.js', ['graph.js', 'text/javascript; charset=utf-8']], + ['/metrics.js', ['metrics.js', 'text/javascript; charset=utf-8']], + ['/views/attempt.js', ['views/attempt.js', 'text/javascript; charset=utf-8']], + ['/views/campaign.js', ['views/campaign.js', 'text/javascript; charset=utf-8']], + ['/views/campaigns.js', ['views/campaigns.js', 'text/javascript; charset=utf-8']], + ['/views/plans.js', ['views/plans.js', 'text/javascript; charset=utf-8']], + ['/styles.css', ['styles.css', 'text/css; charset=utf-8']], + ['/spacetimedb-mark.svg', ['spacetimedb-mark.svg', 'image/svg+xml']], + // The brand faces are served from here rather than a CDN: the dashboard's own + // content-security-policy allows 'self' only, and the appliance has no + // outbound access to fetch them at view time. + ['/fonts/inter-latin-variable.woff2', ['fonts/inter-latin-variable.woff2', 'font/woff2']], + ['/fonts/source-code-pro-latin-variable.woff2', ['fonts/source-code-pro-latin-variable.woff2', 'font/woff2']], +]); + +interface DashboardArgs { + host: string; + port: number; + resultsRoot: string; + plansRoot: string; + allowContainerBind: boolean; +} + +export interface DashboardOperation { + id: string; + updatedAt: string; + [key: string]: unknown; +} + +function dashboardOperation(value: unknown): DashboardOperation { + if (!value || typeof value !== 'object') throw new Error('dashboard operation must be an object'); + const id = 'id' in value ? value.id : undefined; + const updatedAt = 'updatedAt' in value ? value.updatedAt : undefined; + if (typeof id !== 'string' || !id) throw new Error('dashboard operation id is required'); + if (typeof updatedAt !== 'string' || !updatedAt) { + throw new Error('dashboard operation updatedAt is required'); + } + return { ...value, id, updatedAt }; +} + +export interface OperationFeed { + readonly path?: string; + append(event: DashboardOperation): void; + list(): DashboardOperation[]; +} + +export interface LaunchChild { + pid?: number; + once(event: 'error', listener: (error: Error) => void): unknown; + once(event: 'exit', listener: (code: number | null, signal: NodeJS.Signals | null) => void): unknown; +} + +export interface LaunchInput { + plan: DashboardPlan & { path: string }; + output: string; + operationId: string; + resultsRoot: string; + feed: OperationFeed; + env?: NodeJS.ProcessEnv; +} + +export interface DashboardServerOptions { + resultsRoot: string; + plansRoot: string; + allowLaunch?: boolean; + token?: string; + controlSecret?: string; + controlSecretFile?: string; + feed?: OperationFeed; + launch?: (input: LaunchInput) => LaunchChild; + plans?: () => DashboardPlan[]; +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +function loopbackHost(value: unknown): boolean { + return /^(?:localhost|127\.0\.0\.1|\[::1\])(?::\d+)?$/i.test(String(value ?? '')); +} + +function loadControlSecret(options: DashboardServerOptions, allowLaunch: boolean): string | null { + if (!allowLaunch) return null; + const file = options.controlSecretFile + ?? process.env.STACK_BENCH_DASHBOARD_CONTROL_SECRET_FILE; + let value = options.controlSecret; + if (value === undefined && file) { + try { value = readFileSync(resolve(file), 'utf8').trim(); } + catch { throw new Error('dashboard run controls require a readable operator control secret file'); } + } + if (typeof value !== 'string' || value.length < 32 || value.length > 4096 || /[\r\n]/.test(value)) { + throw new Error('dashboard run controls require a valid operator control secret file'); + } + return value; +} + +function sameSecret(actual: unknown, expected: unknown): boolean { + if (typeof actual !== 'string' || typeof expected !== 'string') return false; + const left = Buffer.from(actual); + const right = Buffer.from(expected); + return left.length === right.length && timingSafeEqual(left, right); +} + +function controlAuthorized(request: IncomingMessage, host: string | undefined, + csrfToken: string, controlSecret: string | null): boolean { + return request.headers.origin === `http://${host}` + && sameSecret(request.headers['x-stack-bench-token'], csrfToken) + && sameSecret(request.headers['x-stack-bench-control-secret'], controlSecret); +} + +export function parseDashboardArgs(argv: string[], env: NodeJS.ProcessEnv = process.env): DashboardArgs { + const { values } = parseNodeArgs({ args: argv.slice(2), options: { + host: { type: 'string' }, port: { type: 'string' }, results: { type: 'string' }, + plans: { type: 'string' }, 'allow-container-bind': { type: 'boolean' }, + } }); + const args: DashboardArgs = { host: values.host ?? '127.0.0.1', + port: values.port === undefined ? 7331 : Number(values.port), + resultsRoot: stackBenchResultsRoot(STACK_BENCH_ROOT, env), + plansRoot: '', allowContainerBind: values['allow-container-bind'] ?? false }; + if (values.results) args.resultsRoot = resolve(values.results); + if (values.plans) args.plansRoot = resolve(values.plans); + args.plansRoot ||= join(args.resultsRoot, 'plans'); + const applianceContainerBind = args.allowContainerBind + && env.STACK_BENCH_APPLIANCE === '1' && args.host === '0.0.0.0'; + if (!LOOPBACK.has(args.host) && !applianceContainerBind) { + throw new Error('dashboard must bind to localhost or a loopback address'); + } + if (!Number.isInteger(args.port) || args.port < 1 || args.port > 65535) { + throw new Error('dashboard port must be an integer from 1 through 65535'); + } + return args; +} + +function json(response: ServerResponse, status: number, value: unknown): void { + const body = Buffer.from(`${JSON.stringify(value)}\n`); + response.writeHead(status, { 'content-type': 'application/json; charset=utf-8', + 'content-length': body.length, 'cache-control': 'no-store' }); + response.end(body); +} + +function securityHeaders(response: ServerResponse): void { + response.setHeader('content-security-policy', "default-src 'self'; connect-src 'self'; img-src 'self' data:; script-src 'self'; style-src 'self'; frame-ancestors 'none'; base-uri 'none'; form-action 'self'"); + response.setHeader('x-content-type-options', 'nosniff'); + response.setHeader('x-frame-options', 'DENY'); + response.setHeader('referrer-policy', 'no-referrer'); +} + +async function body(request: IncomingMessage): Promise { + const chunks: Buffer[] = []; + let bytes = 0; + for await (const chunk of request) { + const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); + bytes += buffer.length; + if (bytes > 16 * 1024) throw new Error('request body is too large'); + chunks.push(buffer); + } + try { return JSON.parse(Buffer.concat(chunks).toString('utf8')); + } catch { throw new Error('request body must be valid JSON'); } +} + +function createOperationFeed(resultsRoot: string): OperationFeed { + const root = join(resolve(resultsRoot), 'dashboard'); + const path = join(root, 'operations.jsonl'); + mkdirSync(root, { recursive: true }); + return { + path, + append(event: DashboardOperation) { + appendFileSync(path, `${JSON.stringify(event)}\n`, { encoding: 'utf8', mode: 0o600 }); + }, + list() { + const latest = new Map(); + for (const value of readJsonLines(path)) { + const event = dashboardOperation(value); + latest.set(event.id, { ...(latest.get(event.id) ?? {}), ...event }); + } + return [...latest.values()].sort((left, right) => right.updatedAt.localeCompare(left.updatedAt)); + }, + }; +} + +function launchCampaign({ plan, output, operationId, resultsRoot, feed, + env = process.env }: LaunchInput): LaunchChild { + const operationsRoot = join(resolve(resultsRoot), 'dashboard', 'operations'); + mkdirSync(operationsRoot, { recursive: true }); + const stdoutPath = join(operationsRoot, `${operationId}.stdout.log`); + const stderrPath = join(operationsRoot, `${operationId}.stderr.log`); + const stdout = openSync(stdoutPath, 'a', 0o600); + const stderr = openSync(stderrPath, 'a', 0o600); + const child = spawn(process.execPath, [CAMPAIGN_CLI, 'run', plan.path, '--out', output], { + cwd: STACK_BENCH_ROOT, env, stdio: ['ignore', stdout, stderr], windowsHide: true, + }); + closeSync(stdout); + closeSync(stderr); + child.once('error', error => feed.append({ schemaVersion: 1, id: operationId, + status: 'failed', updatedAt: new Date().toISOString(), error: error.message })); + child.once('exit', (code, signal) => feed.append({ schemaVersion: 1, id: operationId, + status: code === 0 ? 'completed' : 'failed', updatedAt: new Date().toISOString(), + exitCode: code, signal })); + return child; +} + +export function createDashboardServer(options: DashboardServerOptions) { + const resultsRoot = resolve(options.resultsRoot); + const plansRoot = resolve(options.plansRoot); + const allowLaunch = options.allowLaunch ?? process.env.STACK_BENCH_APPLIANCE === '1'; + const token = options.token ?? randomBytes(24).toString('base64url'); + const controlSecret = loadControlSecret(options, allowLaunch); + const feed = options.feed ?? createOperationFeed(resultsRoot); + const launch = options.launch ?? launchCampaign; + const plans = options.plans ?? (() => discoverPlans(plansRoot)); + const launchReservations = new Set(); + const campaignsRoot = join(resultsRoot, 'campaigns'); + const listeners = new Set(); + let watcher: CampaignWatcher | null = null; + let heartbeat: NodeJS.Timeout | null = null; + const broadcast = (change: CampaignChange): void => { + const frame = `event: ${change.type}\ndata: ${JSON.stringify({ key: change.key, + ...(change.attemptId === undefined ? {} : { attemptId: change.attemptId }) })}\n\n`; + for (const listener of listeners) listener.write(frame); + }; + const stopEvents = (): void => { + watcher?.close(); + watcher = null; + if (heartbeat) clearInterval(heartbeat); + heartbeat = null; + }; + const server = createServer(async (request, response) => { + securityHeaders(response); + try { + if (!loopbackHost(request.headers.host)) { + return json(response, 421, { error: 'Dashboard requests must use a loopback host.' }); + } + const url = new URL(request.url ?? '/', `http://${request.headers.host ?? 'localhost'}`); + // The client routes are pages, not fragments: each serves the shell. + const staticFile = STATIC.get(url.pathname) + ?? (SPA_PATH.test(url.pathname) ? STATIC.get('/') : undefined); + if (request.method === 'GET' && staticFile) { + const [file, type] = staticFile; + const path = join(PUBLIC_ROOT, file); + const size = existsSync(path) ? statSync(path).size : 0; + if (!size) return json(response, 404, { error: 'Not found' }); + response.writeHead(200, { 'content-type': type, 'content-length': size, + 'cache-control': file === 'index.html' ? 'no-store' : 'public, max-age=300' }); + createReadStream(path).pipe(response); + return; + } + if (request.method === 'GET' && url.pathname === '/api/health') { + return json(response, 200, { ok: true, mode: allowLaunch ? 'controller' : 'read-only' }); + } + if (request.method === 'GET' && url.pathname === '/api/overview') { + return json(response, 200, { campaigns: overviewSummary(campaignsRoot), + canStart: allowLaunch, csrfToken: token }); + } + if (request.method === 'GET' && url.pathname === '/api/plans') { + return json(response, 200, plans()); + } + if (request.method === 'GET' && url.pathname === '/api/events') { + response.writeHead(200, { 'content-type': 'text/event-stream; charset=utf-8', + 'cache-control': 'no-store', connection: 'keep-alive' }); + response.write(': open\n\n'); + listeners.add(response); + watcher ??= watchCampaigns(campaignsRoot, broadcast, + mode => console.log(`Stack Bench dashboard: campaign watcher ${mode}`)); + // A silent connection is dropped by proxies long before a campaign + // writes anything. + heartbeat ??= setInterval(() => { + for (const listener of listeners) listener.write(': ping\n\n'); + }, HEARTBEAT_MS).unref(); + request.once('close', () => { + listeners.delete(response); + if (!listeners.size) stopEvents(); + }); + return; + } + const resumeRoute = url.pathname.match(/^\/api\/campaigns\/([^/]+)\/resume$/); + if (request.method === 'POST' && resumeRoute) { + if (!allowLaunch) return json(response, 503, { error: 'Run controls are available inside the Stack Bench appliance.' }); + if (!controlAuthorized(request, request.headers.host, token, controlSecret)) { + return json(response, 403, { error: 'The run request is not authorized.' }); + } + const key = decodeURIComponent(resumeRoute[1] ?? ''); + if (!SAFE_NAME.test(key)) return json(response, 400, { error: 'The campaign name is invalid.' }); + const campaign = summarizeCampaign( + contained(join(resultsRoot, 'campaigns'), key, 'campaign'), { includeAttempts: false }); + const priorExecutions = campaign.summary?.executions ?? 0; + if (campaign.mode !== 'dependency' || campaign.status !== 'prepared' || priorExecutions < 1) { + return json(response, 409, { error: 'Only an interrupted campaign that is ready can resume.' }); + } + const plan = plans().find(item => item.id === campaign.id && item.sha256 === campaign.sha256); + if (!plan || plan.state !== 'frozen') { + return json(response, 409, { error: 'The test plan used by this campaign is unavailable.' }); + } + const reservation = `${campaign.id}:${campaign.sha256}:${key}`; + if (launchReservations.has(reservation)) { + return json(response, 409, { error: 'This campaign already has an active controller.' }); + } + launchReservations.add(reservation); + const now = new Date().toISOString(); + const operation = { schemaVersion: 1, id: randomUUID(), type: 'campaign.resume', + status: 'running', createdAt: now, updatedAt: now, actor: 'local-operator', + campaignId: campaign.id, campaignSha256: campaign.sha256, outputName: key }; + feed.append(operation); + const output = join(resultsRoot, 'campaigns', key); + try { + const child = launch({ plan: { ...plan, path: join(plansRoot, plan.file) }, output, + operationId: operation.id, resultsRoot, feed, env: process.env }); + if (typeof child?.once === 'function') { + child.once('error', () => launchReservations.delete(reservation)); + child.once('exit', () => launchReservations.delete(reservation)); + } else { + launchReservations.delete(reservation); + } + feed.append({ ...operation, pid: child?.pid ?? null }); + } catch (error) { + launchReservations.delete(reservation); + feed.append({ schemaVersion: 1, id: operation.id, status: 'failed', + updatedAt: new Date().toISOString(), error: errorMessage(error) }); + throw error; + } + return json(response, 202, operation); + } + const artifactRoute = url.pathname.match(/^\/api\/campaigns\/([^/]+)\/artifacts\/([^/]+)$/); + if (request.method === 'GET' && artifactRoute) { + let artifact; + try { + artifact = resolveCampaignArtifact(resultsRoot, decodeURIComponent(artifactRoute[1] ?? ''), + decodeURIComponent(artifactRoute[2] ?? '')); + } catch { + return json(response, 404, { error: 'Campaign artifact not found.' }); + } + const body = readCampaignArtifactBody(artifact); + const download = url.searchParams.get('download') === '1'; + const type = artifact.kind === 'visual' ? artifact.contentType + : artifact.kind === 'report' && !download ? 'text/html; charset=utf-8' + : 'text/plain; charset=utf-8'; + if (artifact.kind === 'report' && !download) { + response.setHeader('content-security-policy', "sandbox; default-src 'none'; style-src 'unsafe-inline'; img-src data:"); + } + response.writeHead(200, { 'content-type': type, 'content-length': body.length, + 'cache-control': 'no-store', 'content-disposition': `${download ? 'attachment' : 'inline'}; filename*=UTF-8''${encodeURIComponent(basename(artifact.path))}` }); + response.end(body); + return; + } + const campaignRoute = url.pathname.match(/^\/api\/campaigns\/([^/]+)(?:\/(.*))?$/); + if (request.method === 'GET' && campaignRoute) { + const key = decodeURIComponent(campaignRoute[1] ?? ''); + const rest = campaignRoute[2] ?? ''; + if (!SAFE_NAME.test(key)) { + return json(response, 400, { error: 'The campaign name is invalid.' }); + } + const attemptRoute = rest.match(/^attempts\/([^/]+)\/(checks|package|log)$/); + const attemptId = attemptRoute ? decodeURIComponent(attemptRoute[1] ?? '') : ''; + if (attemptRoute && !SAFE_NAME.test(attemptId)) { + return json(response, 400, { error: 'The attempt name is invalid.' }); + } + const from = url.searchParams.get('from') ?? '0'; + if (attemptRoute?.[2] === 'log' && (!/^\d+$/.test(from) || !Number.isSafeInteger(Number(from)))) { + return json(response, 400, { error: 'The log offset must be a whole number of bytes.' }); + } + try { + if (!rest) return json(response, 200, campaignSheet(resultsRoot, key)); + if (rest === 'progression') { + const progression = campaignProgression(resultsRoot, key); + return progression + ? json(response, 200, progression) + : json(response, 404, { error: 'Progression is recorded for dependency campaigns only.' }); + } + if (attemptRoute?.[2] === 'checks') { + return json(response, 200, attemptChecks(resultsRoot, key, attemptId)); + } + if (attemptRoute?.[2] === 'package') { + return json(response, 200, attemptPackage(resultsRoot, key, attemptId)); + } + if (attemptRoute) { + const slice = attemptLogSlice(resultsRoot, key, attemptId, Number(from)); + const text = Buffer.from(slice.text); + response.writeHead(200, { 'content-type': 'text/plain; charset=utf-8', + 'content-length': text.length, 'cache-control': 'no-store', + 'x-stack-bench-log-offset': String(slice.offset) }); + response.end(text); + return; + } + } catch { + // The evidence reader names real paths; a campaign or attempt that + // cannot be read is a miss, not a message. + return json(response, 404, { error: 'Not found' }); + } + return json(response, 404, { error: 'Not found' }); + } + if (request.method === 'POST' && url.pathname === '/api/campaigns') { + if (!allowLaunch) return json(response, 503, { error: 'Run controls are available inside the Stack Bench appliance.' }); + if (!controlAuthorized(request, request.headers.host, token, controlSecret)) { + return json(response, 403, { error: 'The run request is not authorized.' }); + } + if (!String(request.headers['content-type'] ?? '').toLowerCase().startsWith('application/json')) { + return json(response, 415, { error: 'Run requests must use JSON.' }); + } + const input = await body(request); + const runRequest = input !== null && typeof input === 'object' + ? input as { planId?: unknown; outputName?: unknown } : {}; + if (typeof runRequest.planId !== 'string' || typeof runRequest.outputName !== 'string' + || !SAFE_NAME.test(runRequest.outputName)) { + return json(response, 400, { error: 'Choose a test plan and a simple run name.' }); + } + const plan = plans().find(item => item.id === runRequest.planId); + if (!plan || plan.state !== 'frozen') { + return json(response, 400, { error: 'The selected test plan is not ready to run.' }); + } + const path = join(plansRoot, plan.file); + const output = join(resultsRoot, 'campaigns', runRequest.outputName); + if (existsSync(output)) return json(response, 409, { error: 'That run output already exists.' }); + const reservation = `output:${runRequest.outputName}`; + if (launchReservations.has(reservation)) { + return json(response, 409, { error: 'That run output already has an active controller.' }); + } + launchReservations.add(reservation); + const now = new Date().toISOString(); + const operation = { schemaVersion: 1, id: randomUUID(), type: 'campaign.run', status: 'running', + createdAt: now, updatedAt: now, actor: 'local-operator', campaignId: plan.id, + campaignSha256: plan.sha256, outputName: runRequest.outputName }; + feed.append(operation); + try { + const child = launch({ plan: { ...plan, path }, output, operationId: operation.id, + resultsRoot, feed, env: process.env }); + if (typeof child?.once === 'function') { + child.once('error', () => launchReservations.delete(reservation)); + child.once('exit', () => launchReservations.delete(reservation)); + } else { + launchReservations.delete(reservation); + } + feed.append({ ...operation, pid: child?.pid ?? null }); + } catch (error) { + launchReservations.delete(reservation); + feed.append({ schemaVersion: 1, id: operation.id, status: 'failed', + updatedAt: new Date().toISOString(), error: errorMessage(error) }); + throw error; + } + return json(response, 202, operation); + } + return json(response, 404, { error: 'Not found' }); + } catch (error) { + return json(response, 500, { error: errorMessage(error) }); + } + }); + // An open event stream is not an idle connection: the watchers stop and the + // streams end as the server closes, not once it has. + const closeServer = server.close.bind(server); + server.close = ((callback?: (error?: Error) => void) => { + stopEvents(); + for (const listener of listeners) listener.end(); + listeners.clear(); + return closeServer(callback); + }) as typeof server.close; + return { server, token, allowLaunch }; +} + +async function main() { + const args = parseDashboardArgs(process.argv); + const { server, allowLaunch } = createDashboardServer(args); + await new Promise((resolveListen, reject) => { + server.once('error', reject); + server.listen(args.port, args.host, resolveListen); + }); + console.log(`Stack Bench dashboard: http://${args.host}:${args.port} (${allowLaunch ? 'controller' : 'read-only'})`); +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) { + main().catch(error => { console.error(`stack-bench-dashboard: ${errorMessage(error)}`); process.exitCode = 2; }); +} diff --git a/tools/stack-bench/dashboard/dashboard-views.ts b/tools/stack-bench/dashboard/dashboard-views.ts new file mode 100644 index 00000000000..06c9a646df8 --- /dev/null +++ b/tools/stack-bench/dashboard/dashboard-views.ts @@ -0,0 +1,794 @@ +import { closeSync, existsSync, fstatSync, openSync, readSync, readdirSync, statSync } + from 'node:fs'; +import { basename, join, resolve } from 'node:path'; + +import type { CampaignAttemptState } from '../src/campaigns/campaign-scheduler.js'; +import type { DependencyPromptSelection, DependencyState } + from '../src/progression/dependency-mode.js'; +import type { ProgressionState } from '../src/progression/progression-state.js'; +import type { CompiledCampaignPlan } from '../src/campaigns/campaign-compiler.js'; +import type { DependencyProgress } from '../src/campaigns/campaign-inspection.js'; +import type { GradeBundlePayload } from '../src/evidence/benchmark-run.js'; +import { ARTIFACT_FILE, readArtifactPayload } from '../src/evidence/artifacts.js'; +import { CAMPAIGN_FILE } from '../src/campaigns/campaign-path.js'; +import { campaignFacts, inspectCampaignAttempt } from '../src/campaigns/campaign-inspection.js'; +import { campaignLockIsActive } from '../src/campaigns/campaign-lock.js'; +import { campaignProgressionOwner } from '../src/campaigns/campaign-compiler.js'; +import { compileProgressionInput, dependencyRuntimeDefinition } + from '../src/progression/progression-definition.js'; +import { progressionEngine } from '../src/progression/progression-engine.js'; +import { readCampaignState } from '../src/campaigns/campaign-scheduler.js'; +import { readProgressionState } from '../src/progression/progression-state.js'; +import { redactCredentials } from '../src/evidence/diagnostic-sanitizer.js'; +import { repairBudgetLimit } from '../src/progression/repair-plan.js'; +import { MAX_LOG_BYTES, contained, parseRunProgress, readTextTail, + walkPublicExecutionArtifacts } from './dashboard-model.js'; +import type { DashboardArtifact } from './dashboard-model.js'; +import { attemptExcluded, attemptMetrics, attemptStalling, compareCampaign, median } + from './public/metrics.js'; + +const CAMPAIGN_KEY = /^[a-z0-9][a-z0-9.-]*$/; +const GRADE_DIRECTORY = /^(?:first-build-l(\d+)-grading|l(\d+)-fix(\d+)-grading|grading)$/i; +const PROGRESSION_ATTEMPT = /^attempt-(\d+)$/i; +const LOG_FILE = 'process.stdout.log'; + +type ControllerActive = (directory: string, campaign: CompiledCampaignPlan) => boolean; +type InspectedAttempt = ReturnType; + +interface ViewOptions { + controllerActive?: ControllerActive; +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +function percentage(value: number | null | undefined): number | null { + return value == null ? null : Math.round(value * 1000) / 10; +} + +function campaignDirectory(resultsRoot: string, key: string): string { + if (!CAMPAIGN_KEY.test(key)) throw new Error('campaign key is invalid'); + return contained(join(resolve(resultsRoot), 'campaigns'), key, 'campaign'); +} + +function fileFingerprint(path: string): string | null { + if (!existsSync(path)) return null; + const stat = statSync(path); + return `${stat.size}:${stat.mtimeMs}`; +} + +// Every file whose change can move a number in the view, and nothing else: a +// running campaign that has written nothing since the last read is unchanged. +function executionFingerprints(directory: string, files: readonly string[]): string[] { + const attemptsRoot = join(directory, 'attempts'); + if (!existsSync(attemptsRoot)) return []; + const parts: string[] = []; + for (const attempt of readdirSync(attemptsRoot, { withFileTypes: true })) { + if (!attempt.isDirectory()) continue; + const attemptDirectory = join(attemptsRoot, attempt.name); + for (const execution of readdirSync(attemptDirectory, { withFileTypes: true })) { + if (!execution.isDirectory()) continue; + for (const file of files) { + const stamp = fileFingerprint(join(attemptDirectory, execution.name, file)); + if (stamp) parts.push(`${attempt.name}/${execution.name}/${file}:${stamp}`); + } + } + } + return parts.sort(); +} + +function campaignFingerprint(directory: string, files: readonly string[], + executionFiles: readonly string[]): string { + return [...files.map(file => `${file}:${fileFingerprint(join(directory, file)) ?? 'missing'}`), + ...executionFingerprints(directory, executionFiles)].join('|'); +} + +// Overview + +export interface OverviewCampaign { + key: string; + id: string; + title: string; + status: string; + mode: string; + levels: number[]; + repetitions: number; + provisional: boolean; + updatedAt: string | null; + // The mode's official score per stack, as a percentage; null until a stack + // has a comparable result. + scores: Record; + attempts: { total: number; running: number; completed: number }; +} + +export interface UnreadableOverviewCampaign { + key: string; + id: string; + title: string; + status: 'unreadable'; + error: string; +} + +export type OverviewEntry = OverviewCampaign | UnreadableOverviewCampaign; + +const overviewCache = new Map(); + +function overviewCampaign(directory: string): { + plan: CompiledCampaignPlan; + campaign: OverviewCampaign; +} { + const { plan, state } = readCampaignState(directory, { requireCurrentInputs: false }); + const attempts = state.attempts.map(attempt => + inspectCampaignAttempt(plan, attempt, directory)); + const comparison = compareCampaign({ attempts }); + const scores = Object.fromEntries(plan.stacks.map(stack => + [stack.id, percentage(comparison.rows.find(row => row.stack === stack.id)?.final ?? null)])); + return { + plan, + campaign: { + key: basename(resolve(directory)), + id: plan.id, + title: plan.title, + status: state.status, + mode: plan.definition.mode?.id ?? 'sequential', + levels: plan.definition.levels, + repetitions: plan.definition.repetitions, + provisional: campaignFacts(plan).grading.status !== 'qualified', + updatedAt: state.updatedAt, + scores, + attempts: { total: state.summary.total, running: state.summary.running, + completed: state.summary.completed }, + }, + }; +} + +// Liveness is one more fact about a running campaign, not a condition of +// reading it: the read-only host view has no Docker socket to ask. +function controllerInterrupted(probe: ControllerActive, directory: string, + plan: CompiledCampaignPlan, status: string): boolean { + if (status !== 'running') return false; + try { return !probe(directory, plan); } catch { return false; } +} + +function withInterruption(campaign: OverviewCampaign, interrupted: boolean): OverviewCampaign { + if (!interrupted) return campaign; + return { ...campaign, status: 'attention-required', + attempts: { ...campaign.attempts, running: 0 } }; +} + +// Summaries only: no attempt list, no log, no plan. The fingerprint covers a +// running campaign too, so a poll that finds nothing changed costs one stat +// per evidence file instead of a full replay. +export function overviewSummary(campaignsRoot: string, + { controllerActive = campaignLockIsActive }: ViewOptions = {}): OverviewEntry[] { + if (!existsSync(campaignsRoot)) return []; + const campaigns: OverviewEntry[] = []; + for (const entry of readdirSync(campaignsRoot, { withFileTypes: true })) { + if (!entry.isDirectory()) continue; + const directory = join(campaignsRoot, entry.name); + if (!existsSync(join(directory, CAMPAIGN_FILE.state)) + || !existsSync(join(directory, CAMPAIGN_FILE.plan))) continue; + try { + const fingerprint = campaignFingerprint(directory, + [CAMPAIGN_FILE.plan, CAMPAIGN_FILE.state], [ARTIFACT_FILE.run]); + const cached = overviewCache.get(directory); + const fresh = cached?.fingerprint === fingerprint + ? { plan: cached.plan, campaign: cached.campaign } : overviewCampaign(directory); + overviewCache.set(directory, { fingerprint, ...fresh }); + campaigns.push(withInterruption(fresh.campaign, controllerInterrupted(controllerActive, + directory, fresh.plan, fresh.campaign.status))); + } catch (error) { + campaigns.push({ key: entry.name, id: entry.name, title: entry.name, + status: 'unreadable', error: errorMessage(error) }); + } + } + return campaigns.sort((left, right) => + String('updatedAt' in right ? right.updatedAt ?? '' : '') + .localeCompare(String('updatedAt' in left ? left.updatedAt ?? '' : ''))); +} + +// Campaign sheet + +export interface SheetFacts { + mode: string; + workSelection: string | null; + repairSelection: string | null; + repairBudget: number; + agent: string | null; + model: string | null; + guidance: string | null; + recipes: Array<{ level: number; id: string | null; version: string | null }>; + timeLimitMinutes: number; + spendLimitUsd: number | null; + controllerImage: string | null; + buildImage: string | null; + planSha256: string; + grading: string; + gradingReasons: string[]; +} + +export interface ClimbPoint { + score: number; + max: number; + level: number | null; + unaided: boolean; +} + +export interface SheetAttempt { + id: string; + repetition: number; + status: string; + phase: string; + stalling: boolean; + excluded: string | null; + continued: boolean; + logUpdatedAt: string | null; + score: number | null; + unaided: number | null; + repairs: { used: number; budget: number }; + timeSec: number | null; + spendUsd: number | null; + climb: ClimbPoint[]; +} + +export interface SheetLevel { + level: number; + unaided: { score: number; max: number } | null; + score: { score: number; max: number } | null; + repairs: number; +} + +export interface SheetQuestline { + id: string; + title: string; + score: number | null; + nodes: Array<{ id: string; status: string }>; +} + +export interface SheetStack { + stack: string; + score: number | null; + points: { score: number; max: number } | null; + unaided: number | null; + continued: boolean; + repairs: { used: number; budget: number }; + regressions: number; + timeSec: number | null; + spendUsd: number | null; + n: number; + climb: ClimbPoint[]; + attempts: SheetAttempt[]; + levels: SheetLevel[] | null; + questlines: SheetQuestline[] | null; +} + +export interface CampaignSheet { + key: string; + id: string; + title: string; + status: string; + mode: string; + levels: number[]; + repetitions: number; + provisional: boolean; + mixedScope: boolean; + executions: number; + // A dependency campaign that stopped between executions is the one thing an + // operator can restart; the server checks the same three facts again. + resumable: boolean; + createdAt: string; + updatedAt: string; + facts: SheetFacts; + stacks: SheetStack[]; +} + +interface SheetAttemptView { + inspected: InspectedAttempt; + attempt: SheetAttempt; + series: ClimbPoint[]; +} + +function sheetFacts(plan: CompiledCampaignPlan): SheetFacts { + const mode = plan.definition.mode; + const policy = plan.dependencyPolicy?.definition ?? null; + const agent = plan.agents[0] ?? null; + const facts = campaignFacts(plan); + return { + mode: mode.id, + workSelection: policy?.workSelection ?? mode.workSelection ?? null, + repairSelection: policy?.repair.selection ?? plan.definition.repair.selection, + repairBudget: repairBudgetLimit(plan.definition.repair, { + features: plan.featureCatalog?.definition.nodes.length ?? 1, + depths: plan.definition.levels.length, + }), + agent: agent?.adapter ?? null, + model: agent?.model ?? null, + guidance: plan.attempts[0]?.guidance ?? null, + recipes: facts.recipes, + timeLimitMinutes: plan.definition.budgets.attemptTimeoutMinutes, + spendLimitUsd: plan.definition.budgets.maxCostUsdPerAttempt, + controllerImage: facts.runtime.controllerImage, + buildImage: facts.runtime.buildImage, + planSha256: plan.contentSha256, + grading: gradingStatus(facts.grading), + gradingReasons: [...new Set(facts.grading.levels.flatMap(level => level.reasons ?? []))], + }; +} + +// A campaign whose levels disagree is partly publishable and says so. +function gradingStatus(grading: ReturnType['grading']): string { + const levels = new Set(grading.levels.map(level => level.status)); + return levels.size > 1 ? 'partial' : grading.status; +} + +function dependencyRepairs(plan: CompiledCampaignPlan, + dependency: DependencyProgress): { used: number; budget: number } { + return { + used: dependency.history?.repairAttempts ?? 0, + budget: repairBudgetLimit(plan.definition.repair, { + features: dependency.nodes.length, + depths: plan.definition.levels.length, + }), + }; +} + +function attemptRegressions(attempt: InspectedAttempt): number { + if (attempt.dependency) return attempt.dependency.regressions ?? 0; + return (attempt.result?.levels ?? []).reduce((total, level) => total + level.regressions, 0); +} + +// Continued: the attempt resumed on a repair grant, so its first grade is a +// checkpoint baseline rather than an unaided build. +function attemptContinued(attempt: InspectedAttempt): boolean { + if (attempt.dependency) { + return attempt.dependency.attempts.features.some(feature => + typeof feature.granted === 'number' && feature.granted > 0); + } + return (attempt.result?.levels ?? []).some(level => level.continued); +} + +function sheetLevels(attempt: InspectedAttempt | null): SheetLevel[] { + return (attempt?.result?.levels ?? []).map(level => ({ + level: level.level, + unaided: level.firstAbort ? null : level.firstScore, + score: level.finalScore, + repairs: level.used, + })); +} + +function sheetQuestlines(dependency: DependencyProgress): SheetQuestline[] { + const status = new Map(dependency.nodes.map(node => [node.id, node.status])); + const scored = new Map((dependency.score?.questlines ?? []) + .map(questline => [questline.id, questline.percentage ?? null])); + return (dependency.questlines ?? []).map(questline => ({ + id: questline.id, + title: questline.title, + score: scored.get(questline.id) ?? null, + nodes: questline.nodes.map(id => ({ id, status: status.get(id) ?? 'locked' })), + })); +} + +function sheetAttemptView(plan: CompiledCampaignPlan, state: CampaignAttemptState, + directory: string, interrupted: boolean): SheetAttemptView { + const inspected = inspectCampaignAttempt(plan, state, directory); + const execution = inspected.execution; + const logPath = execution + ? join(contained(directory, execution.output, 'campaign execution'), LOG_FILE) : null; + const log = logPath ? readTextTail(logPath) : ''; + const logUpdatedAt = logPath && existsSync(logPath) + ? new Date(statSync(logPath).mtimeMs).toISOString() : null; + const running = inspected.status === 'running' && !interrupted; + const repairLimit = repairBudgetLimit(plan.definition.repair, { + features: plan.featureCatalog?.definition.nodes.length ?? 1, + depths: plan.definition.levels.length, + }); + const progress = parseRunProgress(log, { repairs: repairLimit, + running, status: inspected.status, + dependency: plan.definition.mode?.id === 'dependency' }); + const metrics = attemptMetrics({ ...inspected, logUpdatedAt }); + const repairs = inspected.dependency ? dependencyRepairs(plan, inspected.dependency) : null; + return { + inspected, + series: progress.series, + attempt: { + id: inspected.id, + repetition: inspected.repetition, + status: interrupted && inspected.status === 'running' ? 'interrupted' : inspected.status, + phase: interrupted && inspected.status === 'running' + ? 'Controller stopped before completion' : progress.phase, + stalling: attemptStalling({ ...inspected, logUpdatedAt }, progress.series), + excluded: attemptExcluded(inspected), + continued: attemptContinued(inspected), + logUpdatedAt, + score: percentage(metrics?.final ?? null), + unaided: percentage(metrics?.first ?? null), + repairs: repairs ?? { used: metrics?.repairs ?? 0, budget: repairLimit }, + timeSec: metrics?.duration ?? null, + spendUsd: metrics?.spend ?? null, + climb: progress.series, + }, + }; +} + +const sheetCache = new Map(); + +// Facts and per-stack figures. No log text and no package walk: the climb and +// the phase come from the run output the controller already writes. +export function campaignSheet(resultsRoot: string, key: string, + { controllerActive = campaignLockIsActive }: ViewOptions = {}): CampaignSheet { + const directory = campaignDirectory(resultsRoot, key); + const fingerprint = campaignFingerprint(directory, [CAMPAIGN_FILE.plan, CAMPAIGN_FILE.state], + [ARTIFACT_FILE.run, ARTIFACT_FILE.progressionState, LOG_FILE]); + const { plan, state } = readCampaignState(directory, { requireCurrentInputs: false }); + const interrupted = controllerInterrupted(controllerActive, directory, plan, state.status); + const cacheKey = `${directory}:${interrupted ? 'interrupted' : 'live'}`; + const cached = sheetCache.get(cacheKey); + if (cached?.fingerprint === fingerprint) return cached.sheet; + const views = state.attempts.map(attempt => + sheetAttemptView(plan, attempt, directory, interrupted)); + const comparison = compareCampaign({ + attempts: views.map(view => view.inspected) }); + const dependency = plan.definition.mode?.id === 'dependency'; + const plannedRepairLimit = repairBudgetLimit(plan.definition.repair, { + features: plan.featureCatalog?.definition.nodes.length ?? 1, + depths: plan.definition.levels.length, + }); + const stacks = plan.stacks.map(stack => { + const owned = views.filter(view => view.inspected.stack === stack.id); + const row = comparison.rows.find(entry => entry.stack === stack.id); + // Figures a repetition cannot average — the climb, the questline board, the + // per-level rows — come from the newest attempt that actually ran. + const latest = owned.findLast(view => view.inspected.execution !== null) ?? null; + const lead = latest?.inspected ?? null; + const repairs = lead?.dependency ? dependencyRepairs(plan, lead.dependency) : null; + const metrics = lead ? attemptMetrics(lead) : null; + return { + stack: stack.id, + score: percentage(row?.final ?? null), + points: dependency ? uniquePoints(lead?.dependency ?? null) : metrics?.raw.final ?? null, + unaided: percentage(row?.first ?? null), + continued: owned.some(view => view.attempt.continued), + repairs: repairs ?? { used: Math.round(row?.repairs ?? 0), + budget: plannedRepairLimit }, + regressions: Math.round(median(owned.map(view => attemptRegressions(view.inspected))) ?? 0), + timeSec: row?.duration ?? null, + spendUsd: row?.spend ?? null, + n: row?.n ?? 0, + climb: latest?.series ?? [], + attempts: owned.map(view => view.attempt), + levels: dependency ? null : sheetLevels(lead), + questlines: lead?.dependency ? sheetQuestlines(lead.dependency) : null, + }; + }); + const sheet: CampaignSheet = { + key: basename(resolve(directory)), + id: plan.id, + title: plan.title, + status: interrupted ? 'attention-required' : state.status, + mode: plan.definition.mode?.id ?? 'sequential', + levels: plan.definition.levels, + repetitions: plan.definition.repetitions, + provisional: campaignFacts(plan).grading.status !== 'qualified', + mixedScope: comparison.mixedScope, + executions: state.summary.executions, + resumable: dependency && state.status === 'prepared' && state.summary.executions > 0, + createdAt: state.createdAt, + updatedAt: state.updatedAt, + facts: sheetFacts(plan), + stacks, + }; + sheetCache.set(cacheKey, { fingerprint, sheet }); + return sheet; +} + +function uniquePoints(dependency: DependencyProgress | null): { score: number; max: number } | null { + const unique = dependency?.score?.uniqueChecks; + if (!unique || unique.passedPoints == null || unique.availablePoints == null) return null; + return { score: unique.passedPoints, max: unique.availablePoints }; +} + +// Attempt sub-resources + +function attemptState(directory: string, attemptId: string): CampaignAttemptState { + const { state } = readCampaignState(directory, { requireCurrentInputs: false }); + const attempt = state.attempts.find(item => item.plan.id === attemptId); + if (!attempt) throw new Error('campaign attempt does not exist'); + return attempt; +} + +export interface AttemptCheckGrade { + id: string; + level: number | null; + round: number | null; + score: { score: number; max: number } | null; + error?: string; +} + +export interface AttemptCheck { + key: string; + id: string; + description: string; + points: number; + feature: string; + outcome: string; + regressed: boolean; + history: string[]; +} + +export interface AttemptChecks { + attemptId: string; + stack: string; + grades: AttemptCheckGrade[]; + checks: AttemptCheck[]; +} + +function checkOutcome(evidence: unknown): string { + const status = evidence !== null && typeof evidence === 'object' && 'status' in evidence + ? (evidence as { status?: unknown }).status : null; + if (status === 'passed') return 'pass'; + if (status === 'failed') return 'fail'; + return 'not-run'; +} + +function gradeDirectories(executionDirectory: string): AttemptCheckGrade[] { + if (!existsSync(executionDirectory)) return []; + const progression = join(executionDirectory, 'progression'); + if (existsSync(progression)) { + return readdirSync(progression, { withFileTypes: true }) + .filter(entry => entry.isDirectory() && PROGRESSION_ATTEMPT.test(entry.name)) + .map(entry => ({ id: `progression/${entry.name}`, + level: null, round: Number(PROGRESSION_ATTEMPT.exec(entry.name)?.[1] ?? 0), score: null })) + .sort((left, right) => (left.round ?? 0) - (right.round ?? 0)); + } + return readdirSync(executionDirectory, { withFileTypes: true }) + .filter(entry => entry.isDirectory() && GRADE_DIRECTORY.test(entry.name)) + .map(entry => { + const match = GRADE_DIRECTORY.exec(entry.name); + const level = match?.[1] ?? match?.[2] ?? null; + return { id: entry.name, level: level === null ? null : Number(level), + round: match?.[3] === undefined ? 0 : Number(match[3]), score: null }; + }) + .sort((left, right) => (left.level ?? 0) - (right.level ?? 0) + || (left.round ?? 0) - (right.round ?? 0)); +} + +// Per-check outcome and the history of every grade that reported it: the +// question "did this ever pass" has no other answer in the evidence. +export function attemptChecks(resultsRoot: string, key: string, attemptId: string): AttemptChecks { + const directory = campaignDirectory(resultsRoot, key); + const attempt = attemptState(directory, attemptId); + const execution = attempt.executions.at(-1) ?? null; + if (!execution) return { attemptId, stack: attempt.plan.stack, grades: [], checks: [] }; + const executionDirectory = contained(directory, execution.output, 'campaign execution'); + const grades = gradeDirectories(executionDirectory); + const checks = new Map(); + grades.forEach((grade, index) => { + const path = join(executionDirectory, grade.id, ARTIFACT_FILE.gradeBundle); + if (!existsSync(path)) { + grade.error = 'grade bundle is missing'; + return; + } + let payload; + try { + payload = readArtifactPayload(path, { expectedKind: 'grade_bundle' }); + } catch (error) { + grade.error = errorMessage(error); + return; + } + grade.score = payload.totals?.score == null || payload.totals.max == null + ? null : { score: payload.totals.score, max: payload.totals.max }; + for (const suite of Object.values(payload.suites ?? {})) { + for (const feature of suite.features ?? []) { + for (const criterion of feature.criteria ?? []) { + const stableKey = criterion.stableKey ?? `${feature.name ?? ''}.${criterion.id ?? ''}`; + const entry = checks.get(stableKey) ?? { key: stableKey, id: criterion.id ?? stableKey, + description: criterionDescription(criterion), points: criterion.points ?? 0, + feature: feature.name ?? '', outcome: 'not-run', regressed: false, + history: grades.map(() => 'not-run') }; + entry.history[index] = checkOutcome(criterion.evidence); + checks.set(stableKey, entry); + } + } + } + }); + for (const check of checks.values()) { + const conclusive = check.history.filter(outcome => outcome !== 'not-run'); + check.outcome = conclusive.at(-1) ?? 'not-run'; + check.regressed = conclusive.some((outcome, index) => + outcome === 'fail' && conclusive.slice(0, index).includes('pass')); + } + return { attemptId, stack: attempt.plan.stack, grades, checks: [...checks.values()] }; +} + +// The grade bundle names the criterion text `desc`. +function criterionDescription(criterion: object): string { + const record = criterion as { desc?: unknown; description?: unknown }; + if (typeof record.desc === 'string') return record.desc; + return typeof record.description === 'string' ? record.description : ''; +} + +export interface AttemptPackage { + attemptId: string; + stack: string; + executions: Array<{ + executionId: string; + ordinal: number; + status: string; + artifacts: DashboardArtifact[]; + visuals: DashboardArtifact[]; + truncated: boolean; + }>; +} + +export function attemptPackage(resultsRoot: string, key: string, + attemptId: string): AttemptPackage { + const directory = campaignDirectory(resultsRoot, key); + const attempt = attemptState(directory, attemptId); + return { + attemptId, + stack: attempt.plan.stack, + executions: attempt.executions.map(execution => { + const scanned = walkPublicExecutionArtifacts(directory, + contained(directory, execution.output, 'campaign execution')); + return { executionId: execution.id, ordinal: execution.ordinal, status: execution.status, + artifacts: scanned.artifacts, + visuals: scanned.artifacts.filter(artifact => artifact.kind === 'visual'), + truncated: scanned.truncated }; + }), + }; +} + +export interface AttemptLogSlice { + attemptId: string; + from: number; + offset: number; + size: number; + text: string; +} + +// Bytes after an offset, so a following view pays for growth rather than for +// the whole log on every poll. +export function attemptLogSlice(resultsRoot: string, key: string, attemptId: string, + fromOffset = 0): AttemptLogSlice { + const directory = campaignDirectory(resultsRoot, key); + const attempt = attemptState(directory, attemptId); + const execution = attempt.executions.at(-1) ?? null; + const path = execution + ? join(contained(directory, execution.output, 'campaign execution'), LOG_FILE) : null; + if (!path || !existsSync(path)) { + return { attemptId, from: fromOffset, offset: 0, size: 0, text: '' }; + } + const descriptor = openSync(path, 'r'); + try { + const size = fstatSync(descriptor).size; + // A rotated or truncated log invalidates the caller's offset. + const start = Math.min(Math.max(0, fromOffset), size); + const length = Math.min(size - start, MAX_LOG_BYTES); + const buffer = Buffer.alloc(length); + if (length) readSync(descriptor, buffer, 0, length, start); + return { attemptId, from: fromOffset, offset: start + length, size, + text: redactCredentials(buffer.toString('utf8')) }; + } finally { + closeSync(descriptor); + } +} + +// Campaign progression + +export interface ProgressionCatalogNode { + id: string; + title: string; + questline: string; + depth: number; + dependencies: string[]; +} + +export interface ProgressionStep { + sequence: number; + action: 'build' | 'repair' | 'grant'; + targets: string[]; + // Node status after the event, index-aligned with `nodes`. + statuses: string[]; + score: number | null; + repairs: number; +} + +export interface ProgressionTrack { + stack: string; + attemptId: string; + updatedAt: string; + steps: ProgressionStep[]; +} + +export interface CampaignProgression { + key: string; + depths: number[]; + questlines: Array<{ id: string; title: string; nodes: string[] }>; + nodes: ProgressionCatalogNode[]; + stacks: ProgressionTrack[]; +} + +function progressionSnapshot(state: ProgressionState, nodeIds: readonly string[]): { + statuses: string[]; + score: number | null; + repairs: number; +} { + const average = progressionEngine.score(state).questlineAveragePercentage; + return { + statuses: nodeIds.map(id => state.nodes[id]?.status ?? 'locked'), + score: average == null ? null : Math.round(average * 10) / 10, + repairs: state.attempts.filter(attempt => attempt.repair !== undefined).length, + }; +} + +function progressionSteps(state: DependencyState, nodeIds: readonly string[]): ProgressionStep[] { + let replay = progressionEngine.initialize(state.definition); + return state.events.map(event => { + const action = progressionEngine.nextAction(replay); + if (event.type === 'repairs-granted') { + replay = progressionEngine.grantRepairs(replay, event.grant); + return { sequence: event.sequence, action: 'grant' as const, + targets: [...event.grant.nodeIds], ...progressionSnapshot(replay, nodeIds) }; + } + const targets = action.type === 'terminal' + ? [] : [...(action.prompt as DependencyPromptSelection).nodeIds]; + const repair = action.type === 'repair'; + replay = progressionEngine.recordResult(replay, event.result); + return { sequence: event.sequence, action: repair ? 'repair' as const : 'build' as const, + targets, ...progressionSnapshot(replay, nodeIds) }; + }); +} + +const progressionCache = new Map(); + +// The catalog subgraph the campaign runs, plus one node-status snapshot per +// progression event: the graph and its replay come from the same read. +export function campaignProgression(resultsRoot: string, key: string): CampaignProgression | null { + const directory = campaignDirectory(resultsRoot, key); + const { plan, state } = readCampaignState(directory, { requireCurrentInputs: false }); + if (plan.definition.mode?.id !== 'dependency' || !plan.featureCatalog + || !plan.dependencyPolicy) return null; + const fingerprint = campaignFingerprint(directory, [CAMPAIGN_FILE.plan, CAMPAIGN_FILE.state], + [ARTIFACT_FILE.progressionState]); + const cached = progressionCache.get(directory); + if (cached?.fingerprint === fingerprint) return cached.view; + const progression = compileProgressionInput(dependencyRuntimeDefinition( + plan.featureCatalog, plan.dependencyPolicy)); + const definition = progression.definition; + const nodeIds = definition.nodes.map(node => node.id); + const owned = new Set(nodeIds); + const stacks: ProgressionTrack[] = []; + for (const attempt of state.attempts) { + const execution = attempt.executions.at(-1) ?? null; + if (!execution) continue; + const path = join(contained(directory, execution.output, 'campaign execution'), + ARTIFACT_FILE.progressionState); + if (!existsSync(path)) continue; + const stored = readProgressionState(path, { + progression, + featureCatalogIdentity: plan.featureCatalog.identity, + dependencyPolicyIdentity: plan.dependencyPolicy.identity, + owner: campaignProgressionOwner(plan, attempt.plan, { workspace: true }), + }); + stacks.push({ stack: attempt.plan.stack, attemptId: attempt.plan.id, + updatedAt: new Date(statSync(path).mtimeMs).toISOString(), + steps: progressionSteps(stored.state as DependencyState, nodeIds) }); + } + const view: CampaignProgression = { + key: basename(resolve(directory)), + depths: [...new Set(definition.nodes.map(node => node.level))].sort((a, b) => a - b), + questlines: definition.questlines.map(questline => ({ id: questline.id, + title: questline.title, nodes: [...questline.nodes] })), + nodes: definition.nodes.map(node => ({ id: node.id, title: node.title, + questline: node.questline, depth: node.level, + dependencies: node.dependencies.filter(id => owned.has(id)) })), + stacks, + }; + progressionCache.set(directory, { fingerprint, view }); + return view; +} diff --git a/tools/stack-bench/dashboard/public/app.ts b/tools/stack-bench/dashboard/public/app.ts new file mode 100644 index 00000000000..cdae43e0f16 --- /dev/null +++ b/tools/stack-bench/dashboard/public/app.ts @@ -0,0 +1,344 @@ +/// +/// + +// The client: real paths, one event stream, and keyed reconciliation so a +// refresh does not move what the pointer is on. Every view is a pure function +// of data; the only DOM work in the dashboard happens here. + +import type { AttemptChecks, AttemptPackage, CampaignProgression, CampaignSheet, OverviewEntry } + from '../dashboard-views.js'; +import type { DashboardPlan } from '../dashboard-model.js'; +import { type QuestlineView, campaignPage, replayTimeline } from './views/campaign.js'; +import { type AttemptTab, attemptPage } from './views/attempt.js'; +import { type CampaignFilter, campaignsPage } from './views/campaigns.js'; +import { type Page, type RunForm, afterRun, plansPage, runName, topbar } + from './views/plans.js'; +import { esc } from './format.js'; + +const FALLBACK_MS = 15_000; +const TABS: readonly AttemptTab[] = ['checks', 'screenshots', 'files', 'log']; +const VIEWS: readonly QuestlineView[] = ['grid', 'graph', 'replay']; +const FILTERS: readonly CampaignFilter[] = ['all', 'attention', 'completed', 'ready']; + +interface Route { + key: string; + attempt: string; + plans: boolean; + filter: CampaignFilter; + view: QuestlineView; + step: number; + tab: AttemptTab; +} + +const state = { + overview: [] as OverviewEntry[], + plans: [] as DashboardPlan[], + canStart: false, + csrfToken: '', + form: { planId: '', outputName: '', secret: '', error: '' } as RunForm, + sheets: new Map(), + progression: new Map(), + checks: new Map(), + evidence: new Map(), + log: { attempt: '', text: '', offset: 0 }, +}; +let fallback = 0; +let playing = 0; + +function route(): Route { + const url = new URL(location.href); + const parts = url.pathname.split('/').filter(Boolean); + const pick = (values: readonly Value[], name: string, fall: Value): Value => + values.find(value => value === url.searchParams.get(name)) ?? fall; + return { + key: parts[0] === 'c' ? parts[1] ?? '' : '', + attempt: parts[2] === 'a' ? parts[3] ?? '' : '', + plans: parts[0] === 'plans', + filter: pick(FILTERS, 'filter', 'all'), + view: pick(VIEWS, 'questlines', 'grid'), + step: Math.max(0, Number(url.searchParams.get('step') ?? 0)), + tab: pick(TABS, 'tab', 'checks'), + }; +} + +async function read(url: string): Promise { + try { + const response = await fetch(url, { headers: { accept: 'application/json' } }); + if (!response.ok) return null; + return await response.json() as Payload; + } catch { + return null; + } +} + +function attemptUrl(current: Route, suffix: string): string { + return `/api/campaigns/${encodeURIComponent(current.key)}` + + `/attempts/${encodeURIComponent(current.attempt)}/${suffix}`; +} + +async function readLog(current: Route): Promise { + if (state.log.attempt !== current.attempt) state.log = { attempt: current.attempt, text: '', offset: 0 }; + try { + const response = await fetch(attemptUrl(current, `log?from=${state.log.offset}`)); + if (!response.ok) return; + state.log.text += await response.text(); + state.log.offset = Number(response.headers.get('x-stack-bench-log-offset') ?? state.log.offset); + } catch { /* the stream reconnects and asks again */ } +} + +function chrome(current: Route): string { + const sheet = state.sheets.get(current.key) ?? null; + const page: Page = current.plans ? 'plans' + : current.key && !current.attempt ? 'campaign' : 'campaigns'; + return topbar({ page, key: current.key, canStart: state.canStart, error: state.form.error, + resumable: state.canStart && page === 'campaign' && (sheet?.resumable ?? false) }); +} + +function page(current: Route): string { + const sheet = state.sheets.get(current.key) ?? null; + if (current.plans) { + return plansPage({ plans: state.plans, canStart: state.canStart, form: state.form }); + } + if (!current.key) { + const running = state.overview.filter(campaign => campaign.status === 'running') + .map(campaign => state.sheets.get(campaign.key)) + .filter((entry): entry is CampaignSheet => entry !== undefined); + return campaignsPage({ campaigns: state.overview, sheets: running, filter: current.filter }); + } + if (!sheet) return `
Campaigns / ` + + `${esc(current.key)}
`; + if (current.attempt) { + return attemptPage({ sheet, attemptId: current.attempt, tab: current.tab, + checks: state.checks.get(current.attempt) ?? null, + evidence: state.evidence.get(current.attempt) ?? null, + log: state.log.attempt === current.attempt ? state.log.text : '' }); + } + return campaignPage({ sheet, progression: state.progression.get(current.key) ?? null, + view: current.view, step: current.step }); +} + +function sync(current: Element, next: Element): void { + for (const name of [...current.getAttributeNames()]) { + if (!next.hasAttribute(name)) current.removeAttribute(name); + } + for (const name of next.getAttributeNames()) { + if (current.getAttribute(name) !== next.getAttribute(name)) { + current.setAttribute(name, next.getAttribute(name) ?? ''); + } + } +} + +// Replace only what changed, matching children by position and data-key, so a +// row under the pointer keeps its hover across a refetch. +function patch(current: Element, next: Element): void { + const mine = [...current.children]; + const theirs = [...next.children]; + if (mine.length !== theirs.length || current.childNodes.length !== mine.length + || next.childNodes.length !== theirs.length) { + current.replaceChildren(...next.childNodes); + return; + } + mine.forEach((child, index) => { + const other = theirs[index]!; + if (child.tagName !== other.tagName + || child.getAttribute('data-key') !== other.getAttribute('data-key')) { + child.replaceWith(other); + return; + } + if (child.outerHTML === other.outerHTML) return; + if (!child.children.length || !other.children.length) { + child.replaceWith(other); + return; + } + sync(child, other); + patch(child, other); + }); +} + +function render(): void { + const current = route(); + const root = document.body; + const next = document.createElement('body'); + next.innerHTML = `${chrome(current)}
${page(current)}
`; + patch(root, next); + // The secret and the run name live in the tab, never in the markup. + for (const field of document.querySelectorAll('form[data-run] input')) { + const value = field.name === 'secret' ? state.form.secret : state.form.outputName; + if (field.value !== value) field.value = value; + } +} + +async function load(): Promise { + const current = route(); + if (!current.key || !state.csrfToken) { + const overview = await read<{ campaigns: OverviewEntry[]; canStart: boolean; + csrfToken: string; }>('/api/overview'); + if (overview) Object.assign(state, { overview: overview.campaigns, + canStart: overview.canStart, csrfToken: overview.csrfToken }); + render(); + } + if (current.plans) { + const plans = await read('/api/plans'); + if (plans) state.plans = plans; + const first = state.plans.find(plan => plan.state === 'frozen'); + if (first && !state.form.planId) { + state.form = { ...state.form, planId: first.id, outputName: runName(first.id, new Date()) }; + } + render(); + return; + } + if (!current.key) { + for (const campaign of state.overview.filter(entry => entry.status === 'running')) { + const sheet = await read(`/api/campaigns/${encodeURIComponent(campaign.key)}`); + if (sheet) state.sheets.set(campaign.key, sheet); + render(); + } + return; + } + const sheet = await read(`/api/campaigns/${encodeURIComponent(current.key)}`); + if (sheet) state.sheets.set(current.key, sheet); + render(); + if (sheet?.mode === 'dependency' && current.view !== 'grid' + && !state.progression.has(current.key)) { + state.progression.set(current.key, await read( + `/api/campaigns/${encodeURIComponent(current.key)}/progression`)); + render(); + } + if (!current.attempt) return; + if (current.tab === 'checks' && !state.checks.has(current.attempt)) { + const checks = await read(attemptUrl(current, 'checks')); + if (checks) state.checks.set(current.attempt, checks); + } else if ((current.tab === 'screenshots' || current.tab === 'files') + && !state.evidence.has(current.attempt)) { + const evidence = await read(attemptUrl(current, 'package')); + if (evidence) state.evidence.set(current.attempt, evidence); + } else if (current.tab === 'log') { + await readLog(current); + } + render(); +} + +function go(href: string): void { + history.pushState(null, '', href); + void load(); +} + +function stepTo(offset: number): void { + const current = route(); + const progression = state.progression.get(current.key) ?? null; + if (!progression) return; + const total = replayTimeline(progression).length; + const next = Math.min(Math.max(0, current.step + offset), Math.max(0, total - 1)); + const url = new URL(location.href); + url.searchParams.set('step', String(next)); + history.replaceState(null, '', `${url.pathname}${url.search}`); + render(); +} + +function subscribe(): void { + const source = new EventSource('/api/events'); + const changed = (event: MessageEvent): void => { + const current = route(); + const message = JSON.parse(event.data) as { key?: string; attemptId?: string }; + if (current.key && message.key !== current.key) return; + if (message.attemptId && message.attemptId !== current.attempt) return; + if (message.attemptId) state.checks.delete(message.attemptId); + void load(); + }; + source.addEventListener('campaign', changed); + source.addEventListener('log', changed); + source.addEventListener('open', () => { + if (fallback) clearInterval(fallback); + fallback = 0; + }); + // Only while the stream is down: a served dashboard that is up pays nothing. + source.addEventListener('error', () => { + fallback ||= window.setInterval(() => void load(), FALLBACK_MS); + }); +} + +document.addEventListener('click', event => { + if (event.defaultPrevented || event.button !== 0 || event.metaKey || event.ctrlKey + || event.shiftKey || event.altKey) return; + const shot = (event.target as Element | null)?.closest('[data-shot]'); + if (shot) { + const dialog = document.querySelector('.lightbox'); + const image = dialog?.querySelector('img'); + if (!dialog || !image) return; + image.src = shot.dataset.shot ?? ''; + image.alt = shot.dataset.shotName ?? ''; + dialog.showModal(); + return; + } + if (event.target instanceof HTMLDialogElement) event.target.close(); + const link = (event.target as Element | null)?.closest('a'); + const href = link?.getAttribute('href') ?? ''; + if (!href || href.startsWith('/api/') || !/^[/?]/.test(href)) return; + event.preventDefault(); + go(href.startsWith('?') ? `${location.pathname}${href}` : href); +}); + +// Start and resume are the same request twice: the browser token, the operator +// secret the operator just typed, and the plan the server re-reads itself. +async function post(form: HTMLFormElement): Promise { + const current = route(); + const data = new FormData(form); + const resume = form.dataset.run === 'resume'; + const output = resume ? current.key : String(data.get('output') ?? ''); + const response = await fetch(resume + ? `/api/campaigns/${encodeURIComponent(current.key)}/resume` : '/api/campaigns', { + method: 'POST', + headers: { 'content-type': 'application/json', 'x-stack-bench-token': state.csrfToken, + 'x-stack-bench-control-secret': String(data.get('secret') ?? '') }, + body: JSON.stringify(resume ? {} : { planId: String(data.get('plan') ?? ''), outputName: output }), + }); + if (response.ok) { + state.form = { ...state.form, secret: '', error: '' }; + if (resume) return void load(); + return go(`/c/${encodeURIComponent(output)}`); + } + const failure = await response.json().catch(() => ({})) as { error?: string }; + state.form = afterRun(state.form, response.status, failure.error ?? ''); + render(); +} + +document.addEventListener('submit', event => { + const form = event.target; + if (!(form instanceof HTMLFormElement) || !form.dataset.run) return; + event.preventDefault(); + void post(form); +}); + +// The form's fields are the state; picking a plan renames the output with it. +document.addEventListener('input', event => { + const field = event.target as HTMLInputElement; + if (field.name === 'secret') state.form = { ...state.form, secret: field.value }; + else if (field.name === 'output') state.form = { ...state.form, outputName: field.value }; + else if (field.name === 'plan') { + state.form = { ...state.form, planId: field.value, + outputName: runName(field.value, new Date()) }; + render(); + } +}); + +document.addEventListener('keydown', event => { + if (route().view !== 'replay') return; + if (event.key === 'ArrowRight') stepTo(1); + else if (event.key === 'ArrowLeft') stepTo(-1); + else if (event.key === ' ') { + event.preventDefault(); + if (playing) { + clearInterval(playing); + playing = 0; + } else playing = window.setInterval(() => stepTo(1), 600); + return; + } else return; + if (playing) { + clearInterval(playing); + playing = 0; + } +}); + +window.addEventListener('popstate', () => void load()); +subscribe(); +void load(); diff --git a/tools/stack-bench/dashboard/public/climb.ts b/tools/stack-bench/dashboard/public/climb.ts new file mode 100644 index 00000000000..dbaa24dc052 --- /dev/null +++ b/tools/stack-bench/dashboard/public/climb.ts @@ -0,0 +1,95 @@ +// The climb: one point per completed grade, unaided grades ringed, the current +// grade filled. Small in a lane or a sheet cell, large on the attempt page. + +import type { ClimbPoint } from '../dashboard-views.js'; +import { esc } from './format.js'; + +interface Plot { + x: number; + y: number; + point: ClimbPoint; +} + +function plot(series: readonly ClimbPoint[], left: number, right: number, + top: number, bottom: number): Plot[] { + const span = Math.max(1, series.length - 1); + return series.map((point, index) => ({ + x: series.length === 1 ? (left + right) / 2 : left + (right - left) * index / span, + y: bottom - (bottom - top) * (point.max ? point.score / point.max : 0), + point, + })); +} + +function stepPath(plots: readonly Plot[]): string { + const head = plots[0]; + if (!head) return ''; + return plots.slice(1).reduce((path, item, index) => + `${path} L${item.x} ${plots[index]!.y} L${item.x} ${item.y}`, `M${head.x} ${head.y}`); +} + +export function climb(series: readonly ClimbPoint[], { warn = false, height = 36 }: { + warn?: boolean; + height?: number; +} = {}): string { + if (!series.length) return ''; + const top = 4; + const bottom = height - 4; + const plots = plot(series, 8, 292, top, bottom); + const line = stepPath(plots); + const first = plots[0]!; + const last = plots.at(-1)!; + const tone = warn ? ' warn' : ''; + const rings = plots.filter(item => item.point.unaided || item === first) + .map(item => ``).join(''); + return `` + + `` + + `` + + `` + + `${rings}` + + ``; +} + +// Full size: the same points with a band per depth or level, and a number at +// the first, the best and the current grade. +export function bigClimb(series: readonly ClimbPoint[], stage: (level: number) => string): string { + if (!series.length) return ''; + const top = 10; + const bottom = 130; + const plots = plot(series, 100, 1010, top, bottom); + const bands: string[] = []; + let start = 0; + plots.forEach((item, index) => { + const next = plots[index + 1]; + if (next && next.point.level === item.point.level) return; + const level = item.point.level; + if (level !== null) { + const from = Math.max(60, plots[start]!.x - 40); + const width = Math.min(1050, item.x + 40) - from; + bands.push(`` + + `${esc(stage(level))}`); + } + start = index + 1; + }); + const line = stepPath(plots); + const first = plots[0]!; + const last = plots.at(-1)!; + const best = plots.reduce((top1, item) => item.y < top1.y ? item : top1, first); + const label = (item: Plot, tone: string): string => + `` + + `${Math.round(item.point.max ? 100 * item.point.score / item.point.max : 0)}`; + return `${bands.join('')}` + + [0, 50, 100].map(value => { + const y = bottom - (bottom - top) * value / 100; + return `` + + `${value}`; + }).join('') + + `` + + `` + + plots.map(item => ``).join('') + + label(first, '#b6c0cf') + (best === first || best === last ? '' : label(best, '#b6c0cf')) + + (last === first ? '' : label(last, '#e6e9f0')) + ''; +} diff --git a/tools/stack-bench/dashboard/public/fonts/inter-LICENSE.txt b/tools/stack-bench/dashboard/public/fonts/inter-LICENSE.txt new file mode 100644 index 00000000000..40589daa9de --- /dev/null +++ b/tools/stack-bench/dashboard/public/fonts/inter-LICENSE.txt @@ -0,0 +1,93 @@ +Copyright 2016 The Inter Project Authors (https://github.com/rsms/inter) Inter-Italic[opsz,wght].ttf: Copyright 2016 The Inter Project Authors (https://github.com/rsms/inter) + +This Font Software is licensed under the SIL Open Font License, Version 1.1. +This license is copied below, and is also available with a FAQ at: +http://scripts.sil.org/OFL + + +----------------------------------------------------------- +SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007 +----------------------------------------------------------- + +PREAMBLE +The goals of the Open Font License (OFL) are to stimulate worldwide +development of collaborative font projects, to support the font creation +efforts of academic and linguistic communities, and to provide a free and +open framework in which fonts may be shared and improved in partnership +with others. + +The OFL allows the licensed fonts to be used, studied, modified and +redistributed freely as long as they are not sold by themselves. The +fonts, including any derivative works, can be bundled, embedded, +redistributed and/or sold with any software provided that any reserved +names are not used by derivative works. The fonts and derivatives, +however, cannot be released under any other type of license. The +requirement for fonts to remain under this license does not apply +to any document created using the fonts or their derivatives. + +DEFINITIONS +"Font Software" refers to the set of files released by the Copyright +Holder(s) under this license and clearly marked as such. This may +include source files, build scripts and documentation. + +"Reserved Font Name" refers to any names specified as such after the +copyright statement(s). + +"Original Version" refers to the collection of Font Software components as +distributed by the Copyright Holder(s). + +"Modified Version" refers to any derivative made by adding to, deleting, +or substituting -- in part or in whole -- any of the components of the +Original Version, by changing formats or by porting the Font Software to a +new environment. + +"Author" refers to any designer, engineer, programmer, technical +writer or other person who contributed to the Font Software. + +PERMISSION & CONDITIONS +Permission is hereby granted, free of charge, to any person obtaining +a copy of the Font Software, to use, study, copy, merge, embed, modify, +redistribute, and sell modified and unmodified copies of the Font +Software, subject to the following conditions: + +1) Neither the Font Software nor any of its individual components, +in Original or Modified Versions, may be sold by itself. + +2) Original or Modified Versions of the Font Software may be bundled, +redistributed and/or sold with any software, provided that each copy +contains the above copyright notice and this license. These can be +included either as stand-alone text files, human-readable headers or +in the appropriate machine-readable metadata fields within text or +binary files as long as those fields can be easily viewed by the user. + +3) No Modified Version of the Font Software may use the Reserved Font +Name(s) unless explicit written permission is granted by the corresponding +Copyright Holder. This restriction only applies to the primary font name as +presented to the users. + +4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font +Software shall not be used to promote, endorse or advertise any +Modified Version, except to acknowledge the contribution(s) of the +Copyright Holder(s) and the Author(s) or with their explicit written +permission. + +5) The Font Software, modified or unmodified, in part or in whole, +must be distributed entirely under this license, and must not be +distributed under any other license. The requirement for fonts to +remain under this license does not apply to any document created +using the Font Software. + +TERMINATION +This license becomes null and void if any of the above conditions are +not met. + +DISCLAIMER +THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT +OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE +COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL +DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM +OTHER DEALINGS IN THE FONT SOFTWARE. diff --git a/tools/stack-bench/dashboard/public/fonts/inter-latin-variable.woff2 b/tools/stack-bench/dashboard/public/fonts/inter-latin-variable.woff2 new file mode 100644 index 00000000000..d15208de03c Binary files /dev/null and b/tools/stack-bench/dashboard/public/fonts/inter-latin-variable.woff2 differ diff --git a/tools/stack-bench/dashboard/public/fonts/source-code-pro-LICENSE.txt b/tools/stack-bench/dashboard/public/fonts/source-code-pro-LICENSE.txt new file mode 100644 index 00000000000..046fc664900 --- /dev/null +++ b/tools/stack-bench/dashboard/public/fonts/source-code-pro-LICENSE.txt @@ -0,0 +1,93 @@ +Google Inc. + +This Font Software is licensed under the SIL Open Font License, Version 1.1. +This license is copied below, and is also available with a FAQ at: +http://scripts.sil.org/OFL + + +----------------------------------------------------------- +SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007 +----------------------------------------------------------- + +PREAMBLE +The goals of the Open Font License (OFL) are to stimulate worldwide +development of collaborative font projects, to support the font creation +efforts of academic and linguistic communities, and to provide a free and +open framework in which fonts may be shared and improved in partnership +with others. + +The OFL allows the licensed fonts to be used, studied, modified and +redistributed freely as long as they are not sold by themselves. The +fonts, including any derivative works, can be bundled, embedded, +redistributed and/or sold with any software provided that any reserved +names are not used by derivative works. The fonts and derivatives, +however, cannot be released under any other type of license. The +requirement for fonts to remain under this license does not apply +to any document created using the fonts or their derivatives. + +DEFINITIONS +"Font Software" refers to the set of files released by the Copyright +Holder(s) under this license and clearly marked as such. This may +include source files, build scripts and documentation. + +"Reserved Font Name" refers to any names specified as such after the +copyright statement(s). + +"Original Version" refers to the collection of Font Software components as +distributed by the Copyright Holder(s). + +"Modified Version" refers to any derivative made by adding to, deleting, +or substituting -- in part or in whole -- any of the components of the +Original Version, by changing formats or by porting the Font Software to a +new environment. + +"Author" refers to any designer, engineer, programmer, technical +writer or other person who contributed to the Font Software. + +PERMISSION & CONDITIONS +Permission is hereby granted, free of charge, to any person obtaining +a copy of the Font Software, to use, study, copy, merge, embed, modify, +redistribute, and sell modified and unmodified copies of the Font +Software, subject to the following conditions: + +1) Neither the Font Software nor any of its individual components, +in Original or Modified Versions, may be sold by itself. + +2) Original or Modified Versions of the Font Software may be bundled, +redistributed and/or sold with any software, provided that each copy +contains the above copyright notice and this license. These can be +included either as stand-alone text files, human-readable headers or +in the appropriate machine-readable metadata fields within text or +binary files as long as those fields can be easily viewed by the user. + +3) No Modified Version of the Font Software may use the Reserved Font +Name(s) unless explicit written permission is granted by the corresponding +Copyright Holder. This restriction only applies to the primary font name as +presented to the users. + +4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font +Software shall not be used to promote, endorse or advertise any +Modified Version, except to acknowledge the contribution(s) of the +Copyright Holder(s) and the Author(s) or with their explicit written +permission. + +5) The Font Software, modified or unmodified, in part or in whole, +must be distributed entirely under this license, and must not be +distributed under any other license. The requirement for fonts to +remain under this license does not apply to any document created +using the Font Software. + +TERMINATION +This license becomes null and void if any of the above conditions are +not met. + +DISCLAIMER +THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT +OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE +COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL +DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM +OTHER DEALINGS IN THE FONT SOFTWARE. diff --git a/tools/stack-bench/dashboard/public/fonts/source-code-pro-latin-variable.woff2 b/tools/stack-bench/dashboard/public/fonts/source-code-pro-latin-variable.woff2 new file mode 100644 index 00000000000..bc303f50c5d Binary files /dev/null and b/tools/stack-bench/dashboard/public/fonts/source-code-pro-latin-variable.woff2 differ diff --git a/tools/stack-bench/dashboard/public/format.ts b/tools/stack-bench/dashboard/public/format.ts new file mode 100644 index 00000000000..9f8e0bb6eff --- /dev/null +++ b/tools/stack-bench/dashboard/public/format.ts @@ -0,0 +1,79 @@ +// One spelling per value. Every figure the dashboard prints goes through here, +// so a percentage, a duration and a dash look the same on every page. + +import type { SheetAttempt } from '../dashboard-views.js'; +import { stallRounds } from './metrics.js'; + +const SILENCE_MINUTES = 10; + +export const STACK_LABEL: Record = { spacetime: 'SpacetimeDB', + postgres: 'PostgreSQL', mongodb: 'MongoDB' }; +const STATUS_WORD: Record = { prepared: 'ready', 'attention-required': + 'needs attention', pending: 'queued', invalid: 'excluded', interrupted: 'interrupted' }; +export const DASH = '—'; + +export function esc(value: unknown): string { + return String(value ?? '').replace(/[&<>"']/g, character => + ({ '&': '&', '<': '<', '>': '>', '"': '"', "'": ''' })[character] ?? character); +} + +export function stackLabel(stack: string): string { + return STACK_LABEL[stack] ?? stack; +} + +export function statusWord(status: string): string { + return STATUS_WORD[status] ?? status; +} + +export function pct(value: number | null | undefined): string { + return value == null ? DASH : `${Math.round(value)}%`; +} + +export function num(value: number | null | undefined): string { + return value == null ? DASH : String(Math.round(value)); +} + +// One value: the count and the total it is out of. +export function ratio(used: number | null | undefined, budget: number | null | undefined): string { + if (used == null) return DASH; + return budget == null ? String(used) : `${used}/ ${budget}`; +} + +export function money(value: number | null | undefined): string { + if (value == null) return DASH; + return value >= 10 ? `$${Math.round(value)}` : `$${value.toFixed(2)}`; +} + +export function duration(seconds: number | null | undefined): string { + if (seconds == null) return DASH; + const minutes = Math.round(seconds / 60); + return minutes < 60 ? `${minutes}m` : `${Math.floor(minutes / 60)}h ${minutes % 60}m`; +} + +export function since(value: string | null | undefined, now = Date.now()): string { + if (!value) return DASH; + const minutes = Math.max(0, Math.floor((now - Date.parse(value)) / 60000)); + if (minutes < 60) return `${minutes}m`; + if (minutes < 60 * 48) return `${Math.floor(minutes / 60)}h`; + return `${Math.floor(minutes / 1440)}d`; +} + +// The phase, with the stall the operator would otherwise find by diffing round +// logs: three identical grades, or ten minutes without output. +export function phrase(attempt: SheetAttempt, now = Date.now()): string { + const parts = [attempt.phase]; + const flat = stallRounds(attempt.climb); + if (flat) parts.push(`same score for ${flat} grades`); + const silent = attempt.status === 'running' && attempt.logUpdatedAt + ? Math.floor((now - Date.parse(attempt.logUpdatedAt)) / 60000) : 0; + if (silent >= SILENCE_MINUTES) parts.push(`no output for ${silent}m`); + return parts.join(' · '); +} + +// depth 3 · 1× / L1–L3 · 3× +export function shape(mode: string, levels: readonly number[], repetitions: number): string { + const depth = levels.length ? Math.max(...levels) : 0; + const span = mode === 'dependency' ? `depth ${depth}` + : levels.length > 1 ? `L${Math.min(...levels)}–L${depth}` : `L${depth}`; + return `${span} · ${repetitions}×`; +} diff --git a/tools/stack-bench/dashboard/public/graph.ts b/tools/stack-bench/dashboard/public/graph.ts new file mode 100644 index 00000000000..c4f6510307a --- /dev/null +++ b/tools/stack-bench/dashboard/public/graph.ts @@ -0,0 +1,82 @@ +// One graph for the campaign: columns are depth, bands are questlines, edges +// are the catalog's own dependencies. Every stack builds the same catalog, so a +// node carries one dot per stack in fixed order. The renderer takes one +// node-status snapshot per stack, which is what the replay feeds it per step. + +import type { CampaignProgression } from '../dashboard-views.js'; +import { esc, stackLabel } from './format.js'; + +export interface GraphStack { + stack: string; + statuses: readonly string[]; +} + +const DOT: Record = { passed: 'p', active: 'a', working: 'a', failed: 'f', + blocked: 'b', locked: 'o' }; +const COLUMN = 260; +const NODE_W = 200; +const ROW = 30; + +interface Placed { + x: number; + y: number; + index: number; +} + +export function graph(view: CampaignProgression, stacks: readonly GraphStack[]): string { + const depths = view.depths; + const width = 150 + Math.max(1, depths.length) * COLUMN - 40; + const placed = new Map(); + const bands: string[] = []; + let top = 20; + for (const questline of view.questlines) { + const nodes = view.nodes.filter(node => node.questline === questline.id); + if (!nodes.length) continue; + const used = new Map(); + let rows = 0; + for (const node of nodes) { + const row = used.get(node.depth) ?? 0; + used.set(node.depth, row + 1); + rows = Math.max(rows, row + 1); + placed.set(node.id, { x: 150 + Math.max(0, depths.indexOf(node.depth)) * COLUMN, + y: top + 8 + row * ROW, index: view.nodes.indexOf(node) }); + } + const height = rows * ROW + 16; + bands.push(`${esc(questline.title)}`); + top += height; + bands.push(``); + } + const failed = (index: number): boolean => + stacks.some(entry => entry.statuses[index] === 'failed'); + const blocked = (index: number): boolean => + stacks.some(entry => entry.statuses[index] === 'blocked'); + const edges = view.nodes.flatMap(node => { + const target = placed.get(node.id); + if (!target) return []; + return node.dependencies.flatMap(id => { + const source = placed.get(id); + if (!source) return []; + const cut = failed(source.index) || blocked(target.index); + return [``]; + }); + }); + const nodes = view.nodes.map(node => { + const at = placed.get(node.id); + if (!at) return ''; + const dots = stacks.map((entry, column) => + ``).join(''); + const hover = stacks.map(entry => + `${stackLabel(entry.stack)} ${entry.statuses[at.index] ?? 'locked'}`).join(' · '); + return `${esc(`${node.title} · ${hover}`)}` + + `` + + `${esc(node.title.length > 22 + ? `${node.title.slice(0, 21)}…` : node.title)}${dots}`; + }); + const columns = depths.map((depth, index) => + `depth ${depth}`).join(''); + return `` + + `${bands.join('')}${columns}${edges.join('')}${nodes.join('')}`; +} diff --git a/tools/stack-bench/dashboard/public/index.html b/tools/stack-bench/dashboard/public/index.html new file mode 100644 index 00000000000..f07520b310b --- /dev/null +++ b/tools/stack-bench/dashboard/public/index.html @@ -0,0 +1,15 @@ + + + + + + + + Stack Bench + + + + + + + diff --git a/tools/stack-bench/dashboard/public/metrics.ts b/tools/stack-bench/dashboard/public/metrics.ts new file mode 100644 index 00000000000..2c8c77f723b --- /dev/null +++ b/tools/stack-bench/dashboard/public/metrics.ts @@ -0,0 +1,212 @@ +import type { CampaignRunLevelResult, CampaignRunResult, DependencyProgress } + from '../../src/campaigns/campaign-inspection.js'; + +// The dashboard's vocabulary in one place: Unaided, Score, Repairs, Regressions, +// Stalling and Excluded are defined here and nowhere else, so the server-rendered +// sheet and the browser read the same numbers from the same evidence. + +export const STACK_ORDER = ['spacetime', 'postgres', 'mongodb']; +const EXCLUDED_OUTCOMES = new Set(['harness_failure', 'inconclusive', 'ungraded', 'contaminated']); +const STALL_GRADES = 3; +const SILENCE_MINUTES = 10; + +export interface MetricExecution { + outcome: string | null; + reason: string | null; +} + +export interface MetricAttempt { + id: string; + stack: string; + status: string; + repetition?: number; + logUpdatedAt?: string | null; + execution: MetricExecution | null; + result: CampaignRunResult | null; + dependency: DependencyProgress | null; +} + +export interface AttemptMetrics { + first: number | null; + final: number; + repairs: number; + spend: number | null; + duration: number | null; + scope: string; + abortedFirst: number; + raw: { + first: { score: number; max: number } | null; + final: { score: number; max: number } | null; + }; +} + +export interface ComparisonEntry { + stack: string; + runs: Array<{ attempt: Attempt; metrics: AttemptMetrics }>; + excluded: Array<{ attempt: Attempt; reason: string }>; + pending: number; + spendSoFar: number | null; + abortedFirst: number; +} + +export type ComparisonRow = ComparisonEntry & { + n: number; scopes: string[]; first: number | null; final: number | null; + repairs: number | null; spend: number | null; duration: number | null; + firstRange: { min: number; max: number } | null; + spendRange: { min: number; max: number } | null; + durationRange: { min: number; max: number } | null; +}; + +export function median(values: readonly number[]): number | null { + if (!values.length) return null; + const sorted = [...values].sort((left, right) => left - right); + const middle = Math.floor(sorted.length / 2); + return sorted.length % 2 ? sorted[middle]! : (sorted[middle - 1]! + sorted[middle]!) / 2; +} + +// Live attempts record cost by level before final totals exist. +export function attemptSpend(attempt: MetricAttempt): number | null { + const run = attempt.result; + if (!run || run.unreadable) return null; + const levelled = (run.levels ?? []).reduce((total, level) => + level.costUsd == null ? total : (total ?? 0) + level.costUsd, null); + const spend = run.costUsd ?? levelled; + return spend != null && Number.isFinite(spend) && spend >= 0 ? spend : null; +} + +// An ungraded first build has no score; it is not a zero. +export function attemptMetrics(attempt: MetricAttempt): AttemptMetrics | null { + const run = attempt.result; + if (!run || run.unreadable) return null; + const dependency = attempt.dependency; + if (dependency) { + const score = dependency.score; + const unique = score?.uniqueChecks; + if (score?.status !== 'final' || unique?.percentage == null) return null; + const available = unique.availablePoints ?? 0; + return { + first: dependency.history ? dependency.history.firstTryPercentage / 100 : null, + // Passed points over every selected point in the graph, the same scale + // as the first build. The questline average is the sheet's secondary view. + final: unique.percentage / 100, + repairs: dependency.history?.repairAttempts ?? 0, + spend: run.costComplete === true ? attemptSpend(attempt) : null, + duration: run.durationSec ?? null, + scope: `dependency:${dependency.nodes.length}:${available}`, + abortedFirst: 0, + raw: { first: null, final: unique.passedPoints == null + ? null : { score: unique.passedPoints, max: available } }, + }; + } + type FinalLevel = CampaignRunLevelResult & { finalScore: { score: number; max: number } }; + type ScoredLevel = FinalLevel & { firstScore: { score: number; max: number } }; + const levels = (run.levels ?? []) + .filter((level): level is FinalLevel => level.finalScore !== null); + if (!levels.length) return null; + const sum = (list: readonly Level[], + pick: (level: Level) => number): number => list.reduce((total, item) => total + pick(item), 0); + const scored = levels.filter((level): level is ScoredLevel => + level.firstScore !== null && level.firstAbort === null); + const abortedFirst = levels.filter(level => level.firstAbort).length; + const firstMax = sum(scored, level => level.firstScore.max); + const finalMax = sum(levels, level => level.finalScore.max); + return { + first: firstMax ? sum(scored, level => level.firstScore.score) / firstMax : null, + final: sum(levels, level => level.finalScore.score) / finalMax, + repairs: sum(levels, level => level.used ?? 0), + spend: run.costComplete === true ? attemptSpend(attempt) : null, + duration: run.durationSec ?? null, + scope: `sequential:${levels.map(level => level.level).join(',')}`, + abortedFirst, + // Raw sums over the same set of levels, so a first and a final score shown + // side by side are always out of the same total. + raw: { first: firstMax ? { score: sum(scored, l => l.firstScore.score), max: firstMax } : null, + final: { score: sum(levels, l => l.finalScore.score), max: finalMax } }, + }; +} + +export function attemptExcluded(attempt: MetricAttempt): string | null { + const outcome = attempt.execution?.outcome ?? attempt.result?.outcome; + if (attempt.status === 'invalid') return attempt.execution?.reason ?? outcome ?? 'excluded'; + if (attempt.result?.unreadable && attempt.status !== 'running') return 'result could not be read'; + // 'ungraded' on an attempt still running means "not yet", not "thrown out". + if (outcome && EXCLUDED_OUTCOMES.has(outcome) && attempt.status === 'completed') return outcome; + return null; +} + +// Compare results only when they share the same recorded test plan. +export function compareCampaign(campaign: { + attempts?: readonly Attempt[]; +}): { rows: Array>; usable: Array>; + priced: Array>; burn: Map; + mixedScope: boolean; comparable: boolean } { + const byStack = new Map>(); + for (const attempt of campaign.attempts ?? []) { + const entry = byStack.get(attempt.stack) + ?? { stack: attempt.stack, runs: [], excluded: [], pending: 0, spendSoFar: null, abortedFirst: 0 }; + byStack.set(attempt.stack, entry); + // Excluded attempts still contribute to actual spend. + const incurred = attemptSpend(attempt); + if (incurred != null) entry.spendSoFar = (entry.spendSoFar ?? 0) + incurred; + const reason = attemptExcluded(attempt); + if (reason) { entry.excluded.push({ attempt, reason }); continue; } + const metrics = attempt.status === 'completed' ? attemptMetrics(attempt) : null; + if (metrics) { + entry.runs.push({ attempt, metrics }); + entry.abortedFirst += metrics.abortedFirst; + } else entry.pending += 1; + } + const rows = [...byStack.values()] + .sort((left, right) => STACK_ORDER.indexOf(left.stack) - STACK_ORDER.indexOf(right.stack)) + .map(entry => { + const pick = (key: 'first' | 'final' | 'repairs' | 'spend' | 'duration'): number[] => + entry.runs.map(run => run.metrics[key]).filter((value): value is number => value !== null); + const range = (values: readonly number[]): { min: number; max: number } | null => + values.length ? { min: Math.min(...values), max: Math.max(...values) } : null; + const spend = pick('spend'); + const duration = pick('duration'); + const first = pick('first'); + const scopes = [...new Set(entry.runs.map(run => run.metrics.scope))].sort(); + return { ...entry, n: entry.runs.length, scopes, + first: median(first), firstRange: range(first), + final: median(pick('final')), + repairs: median(pick('repairs')), + spend: median(spend), spendRange: range(spend), + duration: median(duration), durationRange: range(duration) }; + }); + const usable = rows.filter(row => row.n > 0); + const scopes = new Set(usable.flatMap(row => row.scopes)); + const priced = usable.filter(row => row.spend != null); + return { rows, usable, priced, + burn: new Map([...byStack.values()].map(entry => [entry.stack, entry.spendSoFar])), + mixedScope: scopes.size > 1, + comparable: priced.length > 1 && scopes.size === 1 }; +} + +// A trailing run of identical grades is the repair loop treading water. +export function stallRounds( + series: readonly { score: number; max: number }[] | null | undefined): number { + if (!series || series.length < STALL_GRADES + 1) return 0; + const last = series.at(-1); + if (!last) return 0; + let flat = 0; + for (let index = series.length - 2; index >= 0; index--) { + const item = series[index]; + if (item && item.score === last.score && item.max === last.max) flat += 1; + else break; + } + return flat >= STALL_GRADES ? flat : 0; +} + +export function outputSilentMinutes(attempt: MetricAttempt, now = Date.now()): number { + if (attempt.status !== 'running' || !attempt.logUpdatedAt) return 0; + return Math.floor((now - Date.parse(attempt.logUpdatedAt)) / 60000); +} + +// Stalling: three identical consecutive grades, or ten minutes of silence. +export function attemptStalling(attempt: MetricAttempt, + series: readonly { score: number; max: number }[] | null | undefined, + now = Date.now()): boolean { + if (attempt.status !== 'running') return false; + return stallRounds(series) > 0 || outputSilentMinutes(attempt, now) >= SILENCE_MINUTES; +} diff --git a/tools/stack-bench/dashboard/public/spacetimedb-mark.svg b/tools/stack-bench/dashboard/public/spacetimedb-mark.svg new file mode 100644 index 00000000000..f7957efa1ed --- /dev/null +++ b/tools/stack-bench/dashboard/public/spacetimedb-mark.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/tools/stack-bench/dashboard/public/styles.css b/tools/stack-bench/dashboard/public/styles.css new file mode 100644 index 00000000000..e6dfd517474 --- /dev/null +++ b/tools/stack-bench/dashboard/public/styles.css @@ -0,0 +1,241 @@ +/* Stack Bench dashboard. + * + * Tokens are the SpacetimeDB web palette under the names it uses in + * spacetimedb.com/app/styles/variables.css. Hue is state and nothing else; + * stacks are told apart by fixed order and label. 4px radius, no shadow. */ + +@font-face { + font-family: 'Inter Variable'; + font-style: normal; + font-display: swap; + font-weight: 100 900; + src: url(/fonts/inter-latin-variable.woff2) format('woff2-variations'); +} +@font-face { + font-family: 'Source Code Pro Variable'; + font-style: normal; + font-display: swap; + font-weight: 200 900; + src: url(/fonts/source-code-pro-latin-variable.woff2) format('woff2-variations'); +} + +:root { + --green: #4cf490; --green-25: #4cf49040; --green-10: #4cf4901a; + --blue: #02befa; --blue-25: #02befa40; + --yellow: #fbdc8e; --yellow-25: #fbdc8e40; --yellow-10: #fbdc8e1a; + --red: #ff4c4c; + --n1: #e6e9f0; --n2: #ced3e0; --n3: #b6c0cf; --n4: #6f7987; --n5: #363840; --n7: #050505; + --shade1: #162d38; --shade4: #121e24; --shade5: #0f191f; --shade6: #0e161a; + --shade7: #0b1114; --shade8: #0b0e12; + --sans: 'Inter Variable', Inter, ui-sans-serif, system-ui, -apple-system, 'Segoe UI', sans-serif; + --mono: 'Source Code Pro Variable', 'Source Code Pro', ui-monospace, SFMono-Regular, Consolas, monospace; + color-scheme: dark; +} + +* { box-sizing: border-box; } +body { margin: 0; background: var(--shade7); color: var(--n3); font: 14px/1.5 var(--sans); } +a { color: var(--n1); } +:focus-visible { outline: 2px solid var(--green); outline-offset: 2px; } +::selection { background: var(--green-25); } + +.topbar { display: flex; align-items: center; gap: 22px; height: 48px; padding: 0 24px; border-bottom: 1px solid var(--shade4); } +.brand { display: flex; align-items: center; gap: 10px; color: var(--n1); text-decoration: none; } +.brand b { font: 600 12px/1 var(--mono); letter-spacing: .1em; } +.btn { display: inline-flex; align-items: center; min-height: 30px; padding: 0 14px; border-radius: 4px; border: 1px solid var(--shade1); color: var(--n1); background: transparent; font: 500 11.7px/1 var(--mono); letter-spacing: .08em; text-transform: uppercase; cursor: pointer; list-style: none; } +.nav { display: flex; gap: 2px; } +.nav a { padding: 6px 10px; border-radius: 4px; color: var(--n4); text-decoration: none; font: 500 12.5px var(--sans); } +.nav a.on { color: var(--n1); background: var(--shade5); } +.btn.primary { background: var(--green); border-color: var(--green); color: var(--n7); text-decoration: none; } +.tools { display: flex; align-items: center; gap: 10px; margin-left: auto; } +.files { position: relative; } +.files summary::-webkit-details-marker { display: none; } +.files div { position: absolute; right: 0; top: 36px; display: grid; gap: 2px; padding: 8px 10px; background: var(--shade6); border: 1px solid var(--shade1); border-radius: 4px; } +.files div a { color: var(--n2); font: 12px var(--mono); text-decoration: none; } +.page { padding: 22px 24px 40px; } +.crumbs { color: var(--n4); font: 12px var(--mono); margin-bottom: 8px; } +.crumbs a { color: var(--n4); text-decoration: none; } +.crumbs b { color: var(--n2); font-weight: 500; } +.title { display: flex; align-items: center; gap: 12px; margin-bottom: 16px; } +.title h2 { margin: 0; color: var(--n1); font: 600 25px/28px var(--sans); letter-spacing: -.01em; } +.title h2 span { color: var(--n4); font-weight: 400; } +.label { color: var(--n4); font: 500 11.7px/1 var(--mono); letter-spacing: .1em; text-transform: uppercase; } +.state { color: var(--n4); font: 500 13px var(--sans); white-space: nowrap; } +.state.run { color: var(--green); } +.state.done { color: var(--blue); } +.state.warn { color: var(--yellow); } +.state.idle { color: var(--n4); } + +/* live lanes: stack, score, climb, phase */ +.live { border: 1px solid var(--shade4); border-radius: 4px; background: var(--shade6); margin-bottom: 18px; } +.live-head { display: flex; align-items: center; gap: 12px; height: 44px; padding: 0 16px; border-bottom: 1px solid var(--shade4); } +.live-head b { color: var(--n1); font-size: 15px; font-weight: 600; } +.lane { display: grid; grid-template-columns: 130px 96px minmax(200px, 1fr) minmax(260px, 1.1fr); gap: 20px; align-items: center; height: 60px; padding: 0 16px; border-top: 1px solid var(--shade4); } +.lane:first-of-type { border-top: 0; } +.lane .who { color: var(--n1); font-weight: 600; } +.lane .big { color: var(--n1); font: 600 26px/1 var(--sans); letter-spacing: -.02em; } +.lane .big.prov { color: var(--yellow); } +.lane .phase { font-size: 13px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +.lane .phase.warn { color: var(--yellow); } +.climb { display: block; width: 100%; height: 36px; } +.climb .area { fill: var(--green-10); } +.climb .area.warn { fill: var(--yellow-10); } +.climb .line { fill: none; stroke: var(--green); stroke-width: 1.6; stroke-linejoin: round; } +.climb .line.warn { stroke: var(--yellow); } +.climb .first { fill: var(--shade6); stroke: var(--n3); stroke-width: 1.4; } +.climb .now { fill: var(--green); } +.climb .now.warn { fill: var(--yellow); } +.climb .grid { stroke: var(--shade4); stroke-width: 1; } + +/* campaigns table */ +.tablewrap { border: 1px solid var(--shade4); border-radius: 4px; background: var(--shade6); overflow: hidden; } +.toolbar { display: flex; gap: 6px; align-items: center; height: 44px; padding: 0 12px; border-bottom: 1px solid var(--shade4); } +.chip { height: 24px; padding: 0 10px; border-radius: 4px; border: 1px solid var(--shade1); color: var(--n3); font: 500 12px/22px var(--sans); text-decoration: none; } +.chip.on { background: var(--shade1); color: var(--n1); } +.chip.sm { height: 20px; line-height: 18px; font-size: 10.5px; padding: 0 8px; } +.wrap { overflow-x: auto; } +table.runs { width: 100%; border-collapse: collapse; font-size: 13px; } +table.runs th, table.runs td { padding: 0 14px; height: 40px; text-align: left; vertical-align: middle; border-bottom: 1px solid var(--shade4); white-space: nowrap; } +table.runs thead th { color: var(--n4); font: 500 11.7px/1.2 var(--mono); letter-spacing: .1em; text-transform: uppercase; height: 36px; } +table.runs tbody tr:last-child td { border-bottom: 0; } +table.runs tbody tr:hover td { background: var(--shade5); } +table.runs td.name a { color: var(--n1); font-weight: 600; text-decoration: none; } +table.runs td.shape { color: var(--n4); font: 12px var(--mono); } +table.runs th.stack, table.runs td.stack { text-align: right; font: 13px var(--mono); font-variant-numeric: tabular-nums; color: var(--n1); width: 128px; } +table.runs th.stack:first-of-type, table.runs td.stack:first-of-type { border-left: 1px solid var(--shade4); } +table.runs td.stack.prov { color: var(--yellow); } +table.runs td.stack.na { color: var(--n5); } +table.runs td.stack u { text-decoration-color: var(--green); text-underline-offset: 5px; text-decoration-thickness: 2px; } +table.runs th.when, table.runs td.when { color: var(--n4); font: 12px var(--mono); text-align: right; } +table.runs thead th.when { color: var(--n4); font: 500 11.7px/1.2 var(--mono); letter-spacing: .1em; text-transform: uppercase; } + +/* plans and the run form */ +table.plans th.stack, table.plans td.stack { width: auto; } +table.plans td.name { color: var(--n1); font-weight: 600; } +.runform { display: flex; align-items: end; gap: 14px; flex-wrap: wrap; padding: 14px 16px; margin-bottom: 18px; border: 1px solid var(--shade4); border-radius: 4px; background: var(--shade6); } +.runform > div { display: grid; gap: 6px; } +.runform select, .runform input, .secret input { height: 30px; padding: 0 10px; border: 1px solid var(--shade1); border-radius: 4px; background: var(--shade7); color: var(--n1); font: 13px var(--mono); } +.runform select { min-width: 300px; } +.runform input[name=output] { min-width: 330px; } +.secret { display: flex; align-items: center; gap: 8px; } +.secret input { width: 168px; } +.err { color: var(--red); font: 12.5px var(--sans); align-self: center; } + +/* campaign sheet: stacks across, facts down */ +.facts { display: grid; grid-template-columns: repeat(auto-fill, minmax(168px, 1fr)); gap: 1px; margin: 0 0 16px; background: var(--shade4); border: 1px solid var(--shade4); border-radius: 4px; overflow: hidden; } +.facts div { display: grid; gap: 5px; min-width: 0; padding: 9px 12px; background: var(--shade6); } +.facts b { color: var(--n1); font: 500 12.5px var(--mono); white-space: nowrap; overflow: hidden; text-overflow: ellipsis; } +.facts b.warn { color: var(--yellow); } +.sheet { display: grid; grid-template-columns: 168px repeat(3, minmax(0, 1fr)); border: 1px solid var(--shade4); border-radius: 4px; background: var(--shade6); overflow: hidden; } +.sheet > div { display: flex; align-items: center; gap: 6px; min-height: 40px; padding: 0 16px; border-top: 1px solid var(--shade4); border-left: 1px solid var(--shade4); } +.sheet > div:nth-child(-n+4) { border-top: 0; } +.sheet > div:nth-child(4n+1) { border-left: 0; } +.sheet .k { color: var(--n4); font: 500 11.7px/1 var(--mono); letter-spacing: .1em; text-transform: uppercase; } +.sheet .k.views { gap: 8px; } +.sheet .h { min-height: 48px; color: var(--n1); font-size: 15px; font-weight: 600; gap: 10px; } +.sheet .h a { text-decoration: none; } +.sheet .v { color: var(--n1); font: 13px var(--mono); font-variant-numeric: tabular-nums; } +.sheet .v i, .checks .group i { color: var(--n4); font-style: normal; margin-left: 6px; } +.sheet .big { min-height: 64px; color: var(--n1); font: 600 30px/1 var(--sans); letter-spacing: -.02em; } +.sheet .big.prov { color: var(--yellow); } +.sheet .chart { min-height: 64px; padding: 10px 16px; } +.sheet .chart .climb { height: 44px; } +.sheet .phase { font-size: 13px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +.sheet .phase.warn { color: var(--yellow); } +.sheet .reasons { display: grid; gap: 5px; width: 100%; padding: 8px 0; } +.sheet .reasons div { display: flex; gap: 8px; min-width: 0; } +.sheet .reasons a { color: var(--yellow); white-space: nowrap; } +.sheet .reasons span { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +.sheet .q { min-height: 32px; } +.sheet .q.k { text-transform: none; letter-spacing: 0; font: 12.5px var(--sans); color: var(--n3); } +.sheet .q .pct { margin-left: auto; color: var(--n4); font: 11.5px var(--mono); font-variant-numeric: tabular-nums; } +.sheet .q .pct.full { color: var(--green); } +.sheet .sum .v { font-weight: 600; } +.sheet .band { background: var(--shade5); } +.sheet .links a { color: var(--n2); font: 12px var(--mono); text-decoration: none; margin-right: 14px; } +.sheet > .wide { grid-column: 1 / -1; display: block; padding: 8px 8px 4px; border-left: 0; } +.sheet > .span3 { grid-column: 2 / -1; } +.sheet > .evhead { display: flex; gap: 0; padding: 0; } +.sheet .ev { display: grid; gap: 5px; padding: 8px 16px; border-left: 1px solid var(--shade4); min-height: 48px; min-width: 130px; } +.sheet .ev:first-child { border-left: 0; } +.dot { width: 9px; height: 9px; border-radius: 50%; background: var(--shade1); flex: 0 0 auto; } +.dot.p { background: var(--green); } +.dot.a { background: var(--blue); box-shadow: 0 0 0 2px var(--blue-25); } +.dot.f { background: var(--red); } +.dot.b { background: transparent; border: 1.5px solid var(--red); } +.dot.o { background: transparent; border: 1.5px solid var(--shade1); } + +/* graph */ +.dag { display: block; width: 100%; height: auto; } +.dag .band { font: 11px var(--mono); fill: var(--n4); } +.dag .col { font: 500 10.5px var(--mono); fill: var(--n4); letter-spacing: .08em; text-transform: uppercase; } +.dag .sep { stroke: var(--shade4); } +.dag .e { fill: none; stroke: var(--shade1); stroke-width: 1.1; opacity: .8; } +.dag .e.cut { stroke: var(--red); stroke-dasharray: 3 4; opacity: .7; } +.dag .n rect { fill: var(--shade5); stroke: var(--shade1); } +.dag .n text { font: 10.5px var(--mono); fill: var(--n2); } +.dag .d { fill: var(--shade1); } +.dag .d.p { fill: var(--green); } +.dag .d.a { fill: var(--blue); } +.dag .d.f { fill: var(--red); } +.dag .d.b { fill: none; stroke: var(--red); stroke-width: 1.4; } +.dag .d.o { fill: none; stroke: var(--shade1); stroke-width: 1.4; } + +/* replay */ +.replay { min-height: 40px; padding: 6px 16px; } +.strip { display: block; width: 100%; height: 28px; } +.strip .st { fill: var(--n4); } +.strip .st.b { fill: var(--blue); } +.strip .st.r { fill: var(--yellow); } +.strip .st.g { fill: var(--n5); } +.strip .st.f { fill: var(--red); } +.strip .st.on { stroke: var(--yellow-25); stroke-width: 3; } +.strip .st.dim { opacity: .3; } +.strip .cur { stroke: var(--n1); stroke-width: 1; } + +/* attempt */ +.figs { display: flex; gap: 48px; margin: 4px 0 20px; flex-wrap: wrap; } +.figs div { display: grid; gap: 6px; } +.figs b { color: var(--n1); font: 600 30px/1 var(--sans); letter-spacing: -.02em; } +.figs b.prov { color: var(--yellow); } +.figs b.now { font: 500 15px/30px var(--sans); } +.figs b.now.warn { color: var(--yellow); } +.issue { margin: 0 0 20px; padding: 12px 14px; border-left: 2px solid var(--yellow); background: var(--yellow-10); } +.issue p { margin: 6px 0 0; color: var(--n2); } +.bigclimb { display: block; width: 100%; height: 170px; margin-bottom: 20px; } +.bigclimb text { font: 10.5px var(--mono); fill: var(--n4); } +.bigclimb .l { stroke: var(--green); stroke-width: 2; fill: none; stroke-linejoin: round; } +.bigclimb .a { fill: var(--green-10); } +.bigclimb .g { stroke: var(--shade4); } +.bigclimb .band { fill: var(--shade6); } +.bigclimb .ev { fill: var(--n1); } +.bigclimb .ev.first { fill: var(--shade7); stroke: var(--n3); stroke-width: 1.5; } +.bigclimb .ev.now { fill: var(--blue); } +.tabs { display: flex; gap: 2px; border-bottom: 1px solid var(--shade4); } +.tabs a { padding: 10px 12px; color: var(--n4); font: 500 13px var(--sans); border-bottom: 2px solid transparent; margin-bottom: -1px; text-decoration: none; } +.tabs a.on { color: var(--n1); border-bottom-color: var(--green); } +.tabs a i { color: var(--n4); font: 11.5px var(--mono); font-style: normal; margin-left: 6px; } +.checks { width: 100%; border-collapse: collapse; font-size: 13px; } +.checks th, .checks td { height: 34px; padding: 0 14px; text-align: left; border-bottom: 1px solid var(--shade4); white-space: nowrap; } +.checks thead th { color: var(--n4); font: 500 11.7px var(--mono); letter-spacing: .1em; text-transform: uppercase; height: 36px; } +.checks td.k { font-family: var(--mono); color: var(--n2); } +.checks td.d { color: var(--n3); white-space: normal; } +.checks td.h { color: var(--n4); font: 12px var(--mono); letter-spacing: .14em; } +.checks .h .p { color: var(--green); } +.checks .h .f { color: var(--red); } +.checks .h .x { color: var(--n5); } +.checks tr.group td { color: var(--n1); font-weight: 600; background: var(--shade6); } +.grade-key { display: flex; gap: 18px; padding: 10px 14px; color: var(--n4); font: 12px var(--mono); } +.grade-key .p { color: var(--green); } +.grade-key .f { color: var(--red); } +.grade-key .x { color: var(--n4); } +.shots { display: grid; grid-template-columns: repeat(auto-fill, minmax(240px, 1fr)); gap: 12px; padding: 14px 0; } +.shots button { padding: 0; border: 0; background: none; cursor: zoom-in; } +.shots img { width: 100%; border: 1px solid var(--shade4); border-radius: 4px; } +.lightbox { width: min(94vw, 1500px); max-height: 94vh; padding: 42px 12px 12px; border: 1px solid var(--shade1); border-radius: 4px; background: var(--shade8); } +.lightbox::backdrop { background: #000c; } +.lightbox form { position: absolute; top: 8px; right: 10px; } +.lightbox button { border: 0; background: none; color: var(--n2); cursor: pointer; } +.lightbox img { display: block; max-width: 100%; max-height: calc(94vh - 54px); margin: auto; } +.files-list { display: grid; gap: 6px; padding: 14px 0; } +.files-list a { color: var(--n2); font: 12px var(--mono); text-decoration: none; } +.log { margin: 14px 0 0; padding: 14px 16px; max-height: 520px; overflow: auto; background: var(--shade8); color: var(--n3); font: 12px/1.7 var(--mono); border-radius: 4px; } diff --git a/tools/stack-bench/dashboard/public/views/attempt.ts b/tools/stack-bench/dashboard/public/views/attempt.ts new file mode 100644 index 00000000000..5b9acf37101 --- /dev/null +++ b/tools/stack-bench/dashboard/public/views/attempt.ts @@ -0,0 +1,119 @@ +// One attempt: figures, the climb at full size, and the evidence behind tabs. +// Each tab is a link, so what is open survives a reload and a back button. + +import type { AttemptCheck, AttemptChecks, AttemptPackage, CampaignSheet, SheetAttempt, SheetStack } + from '../../dashboard-views.js'; +import { bigClimb } from '../climb.js'; +import { DASH, duration, esc, money, pct, phrase, ratio, stackLabel } from '../format.js'; + +export type AttemptTab = 'checks' | 'screenshots' | 'files' | 'log'; + +export interface AttemptPageInput { + sheet: CampaignSheet; + attemptId: string; + tab: AttemptTab; + checks: AttemptChecks | null; + evidence: AttemptPackage | null; + log: string; +} + +const GLYPH: Record = { pass: '', + fail: '', 'not-run': '·' }; + +function locate(sheet: CampaignSheet, attemptId: string): { + stack: SheetStack; + attempt: SheetAttempt; +} | null { + for (const stack of sheet.stacks) { + const attempt = stack.attempts.find(item => item.id === attemptId); + if (attempt) return { stack, attempt }; + } + return null; +} + +function checksTable(checks: AttemptChecks | null): string { + if (!checks) return ''; + const features = new Map(); + for (const check of checks.checks) { + features.set(check.feature, [...features.get(check.feature) ?? [], check]); + } + const groups = [...features.entries()].map(([feature, items]) => { + const points = items.reduce((total, check) => total + check.points, 0); + const passed = items.filter(check => check.outcome === 'pass') + .reduce((total, check) => total + check.points, 0); + return `${esc(feature)}` + + `${ratio(passed, points)}` + + items.map(check => `${esc(check.id)}` + + `${esc(check.description)}` + + `${check.history.map(outcome => GLYPH[outcome] ?? GLYPH['not-run']).join('')}` + + '').join(''); + }).join(''); + return '
Each mark is one grading pass, from left to right. ' + + '✓ Pass✕ Fail' + + '· Not run
' + + '
' + + `${groups}
CheckProvesGrades
`; +} + +function artifacts(evidence: AttemptPackage | null, key: string, visual: boolean): string { + const items = (evidence?.executions ?? []).flatMap(execution => + visual ? execution.visuals : execution.artifacts.filter(item => item.kind !== 'visual')); + const link = (id: string): string => + `/api/campaigns/${encodeURIComponent(key)}/artifacts/${encodeURIComponent(id)}`; + if (visual) { + return `
${items.map(item => { + const source = link(item.id); + return ``; + }).join('')}
` + + '
'; + } + return `
${items.map(item => + `${esc(item.path)}`).join('')}
`; +} + +export function attemptPage({ sheet, attemptId, tab, checks, evidence, log }: AttemptPageInput): string { + const found = locate(sheet, attemptId); + const crumbs = (tail: string): string => `
Campaigns / ` + + `${esc(sheet.title)} / ` + + `${esc(tail)}
`; + if (!found) { + return `
${crumbs(attemptId)}` + + '

no attempt

'; + } + const { stack, attempt } = found; + const name = `${stackLabel(stack.stack)} rep ${attempt.repetition}`; + const counts: Record = { + checks: checks ? String(checks.checks.length) : '', + screenshots: evidence + ? String(evidence.executions.reduce((total, item) => total + item.visuals.length, 0)) : '', + files: evidence ? String(evidence.executions.reduce((total, item) => + total + item.artifacts.filter(entry => entry.kind !== 'visual').length, 0)) : '', + log: attempt.status === 'running' ? 'live' : '', + }; + const tabs = (['checks', 'screenshots', 'files', 'log'] as const).map(entry => + `` + + `${entry[0]!.toUpperCase()}${entry.slice(1)}` + + `${counts[entry] ? `${esc(counts[entry])}` : ''}`).join(''); + const figure = (label: string, text: string, tone = ''): string => + `
${esc(label)}${text}
`; + const stage = (level: number): string => + sheet.mode === 'dependency' ? `depth ${level}` : `L${level}`; + const panel = tab === 'checks' ? checksTable(checks) + : tab === 'log' ? `
${esc(log)}
` + : artifacts(evidence, sheet.key, tab === 'screenshots'); + const issue = attempt.excluded + ? `
Why this run was excluded` + + `

${esc(attempt.excluded)}

` : ''; + return `
${crumbs(name)}` + + `

${esc(stackLabel(stack.stack))} ` + + `rep ${attempt.repetition}

` + + `
${figure('Score', pct(attempt.score), sheet.provisional ? 'prov' : '')}` + + figure('Unaided', pct(attempt.unaided)) + + figure('Repairs', ratio(attempt.repairs.used, attempt.repairs.budget)) + + figure('Time', duration(attempt.timeSec)) + + figure('Spend', money(attempt.spendUsd)) + + figure('Now', esc(phrase(attempt)), attempt.stalling ? 'now warn' : 'now') + + `
${issue}${bigClimb(attempt.climb, stage) || `

${DASH}

`}` + + `
${tabs}
${panel}
`; +} diff --git a/tools/stack-bench/dashboard/public/views/campaign.ts b/tools/stack-bench/dashboard/public/views/campaign.ts new file mode 100644 index 00000000000..37f7ac8a722 --- /dev/null +++ b/tools/stack-bench/dashboard/public/views/campaign.ts @@ -0,0 +1,241 @@ +// One campaign is one sheet: stacks across in fixed order, facts down, one +// value per cell. The questline rows are the grid; Graph and Replay replace +// them with a full-width cell drawn by the shared graph renderer. + +import type { CampaignProgression, CampaignSheet, ProgressionStep, SheetAttempt, SheetStack } + from '../../dashboard-views.js'; +import { climb } from '../climb.js'; +import { DASH, duration, esc, money, num, pct, phrase, ratio, stackLabel } from '../format.js'; +import { STACK_ORDER } from '../metrics.js'; +import { graph } from '../graph.js'; + +export type QuestlineView = 'grid' | 'graph' | 'replay'; + +export interface CampaignPageInput { + sheet: CampaignSheet; + progression: CampaignProgression | null; + view: QuestlineView; + step: number; +} + +export interface ReplayEvent { + stack: string; + ordinal: number; + step: ProgressionStep; +} + +const DOT: Record = { passed: 'p', active: 'a', working: 'a', failed: 'f', + blocked: 'b', locked: 'o' }; + +function short(value: string | null): string { + return value ? value.slice(0, 12) : DASH; +} + +function ordered(sheet: CampaignSheet): SheetStack[] { + return STACK_ORDER.flatMap(stack => sheet.stacks.filter(entry => entry.stack === stack)); +} + +function latest(stack: SheetStack): SheetAttempt | null { + return stack.attempts.findLast(attempt => attempt.status === 'running') + ?? stack.attempts.at(-1) ?? null; +} + +function facts(sheet: CampaignSheet): string { + const fact = sheet.facts; + const dependency = sheet.mode === 'dependency'; + const depth = sheet.levels.length ? Math.max(...sheet.levels) : 0; + const cells: Array<[string, string, string]> = [['Mode', fact.mode, '']]; + cells.push(dependency ? ['Depth', String(depth), ''] + : ['Levels', sheet.levels.map(level => `L${level}`).join('–'), '']); + if (dependency) { + cells.push(['Work', fact.workSelection ?? DASH, ''], + ['Repair', fact.repairSelection ?? DASH, ''], + ['Repair budget', String(fact.repairBudget), '']); + } else { + cells.push(['Repair budget', String(fact.repairBudget), '']); + } + cells.push(['Repetitions', String(sheet.repetitions), ''], + ['Agent', fact.agent ?? DASH, ''], ['Model', fact.model ?? DASH, ''], + ['Guidance', fact.guidance ?? DASH, ''], + [fact.recipes.length > 1 ? 'Recipes' : 'Recipe', fact.recipes.map(recipe => + [recipe.id, recipe.version].filter(Boolean).join(' ')).join(' · ') || DASH, ''], + ['Time limit', `${fact.timeLimitMinutes} min`, ''], + ['Spend limit', fact.spendLimitUsd === null ? DASH + : `$${fact.spendLimitUsd} per attempt`, ''], + ['Controller', short(fact.controllerImage), ''], ['Plan', short(fact.planSha256), ''], + ['Grading', fact.grading, fact.gradingReasons.join(' · ')]); + if (sheet.mixedScope) cells.push(['Scope', 'mixed', 'attempts do not share one test plan']); + const continued = sheet.stacks.filter(stack => stack.continued).length; + if (continued) cells.push(['Continued', String(continued), '']); + return `
${cells.map(([label, value, hover]) => + `
${esc(label)}` + + `${esc(value)}
`).join('')}
`; +} + +function questlineRows(sheet: CampaignSheet, stacks: readonly SheetStack[]): string { + const lead = stacks.find(stack => stack.questlines?.length)?.questlines ?? []; + const rows = lead.map(questline => { + const cells = stacks.map(stack => { + const owned = stack.questlines?.find(entry => entry.id === questline.id) ?? null; + const dots = (owned?.nodes ?? []).map(node => + ``).join(''); + const score = owned?.score ?? null; + return `
${dots}` + + `${pct(score)}
`; + }).join(''); + return `
${esc(questline.title)}
${cells}`; + }).join(''); + const average = stacks.map(stack => + `
${pct(stack.score)}
`).join(''); + if (sheet.mode !== 'dependency') return ''; + return `${rows}
Questline average
${average}`; +} + +function levelRows(stacks: readonly SheetStack[]): string { + const levels = stacks.find(stack => stack.levels?.length)?.levels ?? []; + return levels.map(level => ['unaided', 'score'].map(kind => { + const cells = stacks.map(stack => { + const owned = stack.levels?.find(entry => entry.level === level.level) ?? null; + const points = kind === 'unaided' ? owned?.unaided ?? null : owned?.score ?? null; + return `
${points + ? ratio(points.score, points.max) : DASH}
`; + }).join(''); + return `
L${level.level} ${kind}
${cells}`; + }).join('')).join(''); +} + +export function replayTimeline(progression: CampaignProgression): ReplayEvent[] { + const tracks = STACK_ORDER.flatMap(stack => + progression.stacks.filter(track => track.stack === stack)); + const depth = Math.max(0, ...tracks.map(track => track.steps.length)); + const events: ReplayEvent[] = []; + for (let ordinal = 0; ordinal < depth; ordinal += 1) { + for (const track of tracks) { + const step = track.steps[ordinal]; + if (step) events.push({ stack: track.stack, ordinal, step }); + } + } + return events; +} + +function marker(step: ProgressionStep, failed: boolean): string { + if (failed) return 'f'; + if (step.action === 'repair') return 'r'; + return step.action === 'grant' ? 'g' : 'b'; +} + +function replay(progression: CampaignProgression, cursor: number): string { + const events = replayTimeline(progression); + const span = Math.max(1, events.length - 1); + const selected = events[cursor] ?? events.at(-1) ?? null; + const failedAt = (step: ProgressionStep): boolean => step.targets.some(target => + step.statuses[progression.nodes.findIndex(node => node.id === target)] === 'failed'); + const title = (id: string): string => + progression.nodes.find(node => node.id === id)?.title ?? id; + const head = selected ? [['Step', ratio(cursor + 1, events.length)], + ['Stack', esc(stackLabel(selected.stack))], ['Action', esc(selected.step.action)], + ['Feature', selected.step.targets.length === 1 + ? esc(title(selected.step.targets[0] ?? '')) : `${selected.step.targets.length} features`], + ['Score', pct(selected.step.score)], ['Repairs', num(selected.step.repairs)]] + .map(([label, value]) => `
${label}` + + `${value}
`).join('') : ''; + // Drawn as one SVG per stack: the dashboard's policy allows no inline style, + // and a marker's position is geometry, not decoration. + const at = (index: number): number => 20 + 960 * index / span; + const rows = STACK_ORDER.flatMap(stack => progression.stacks + .filter(track => track.stack === stack) + .map(track => { + const marks = events.map((event, index) => event.stack !== track.stack ? '' : + ``).join(''); + return `
${esc(stackLabel(track.stack))}
` + + '' + + `${marks}
`; + })).join(''); + const snapshot = STACK_ORDER.flatMap(stack => progression.stacks + .filter(track => track.stack === stack).map(track => { + const step = events.filter((event, index) => + event.stack === track.stack && index <= cursor).at(-1)?.step ?? null; + return { stack: track.stack, + statuses: step?.statuses ?? progression.nodes.map(() => 'locked') }; + })); + return `
${head}
` + + `
${graph(progression, snapshot)}
${rows}`; +} + +function board({ sheet, progression, view, step }: CampaignPageInput, + stacks: readonly SheetStack[]): string { + const chips = (['grid', 'graph', 'replay'] as const).map(entry => + `` + + `${entry[0]!.toUpperCase()}${entry.slice(1)}`).join(''); + const switcher = `
${chips}
`; + if (sheet.mode !== 'dependency') return levelRows(stacks); + if (view === 'grid' || !progression) return switcher + questlineRows(sheet, stacks); + if (view === 'graph') { + const snapshot = STACK_ORDER.flatMap(stack => progression.stacks + .filter(track => track.stack === stack) + .map(track => ({ stack: track.stack, + statuses: track.steps.at(-1)?.statuses ?? progression.nodes.map(() => 'locked') }))); + return `${switcher}
${graph(progression, snapshot)}
`; + } + return switcher + replay(progression, step); +} + +export function campaignPage(input: CampaignPageInput): string { + const sheet = input.sheet; + const stacks = ordered(sheet); + const cell = (render: (stack: SheetStack) => string): string => + stacks.map(stack => render(stack)).join(''); + const row = (label: string, render: (stack: SheetStack) => string): string => + `
${esc(label)}
${cell(render)}`; + const value = (text: string): string => `
${text}
`; + const heads = stacks.map(stack => { + const attempt = latest(stack); + const label = esc(stackLabel(stack.stack)); + return `
${attempt + ? `` + + `${label}` : label}
`; + }).join(''); + const repetitions = sheet.repetitions > 1 + ? row('Completed', stack => value(num(stack.n))) + + row('Excluded', stack => + value(num(stack.attempts.filter(attempt => attempt.excluded).length))) + : ''; + const evidence = row('Evidence', stack => { + const attempt = latest(stack); + if (!attempt) return ``; + const base = `/c/${encodeURIComponent(sheet.key)}/a/${encodeURIComponent(attempt.id)}`; + return ``; + }); + const issues = stacks.some(stack => stack.attempts.some(attempt => attempt.excluded)) + ? row('Excluded because', stack => { + const excluded = stack.attempts.filter(attempt => attempt.excluded); + return excluded.length ? `
${excluded.map(attempt => { + const href = `/c/${encodeURIComponent(sheet.key)}/a/${encodeURIComponent(attempt.id)}`; + return `
rep ${attempt.repetition}` + + `${esc(attempt.excluded)}
`; + }).join('')}
` : value(DASH); + }) : ''; + return `
Campaigns / ` + + `${esc(sheet.key)}
` + + `

${esc(sheet.title)}

${facts(sheet)}` + + `
${heads}` + + row('Score', stack => `
` + + `${pct(stack.score)}
`) + + row('Unaided', stack => value(pct(stack.unaided))) + + row('Repairs', stack => value(ratio(stack.repairs.used, stack.repairs.budget))) + + row('Regressions', stack => value(num(stack.regressions))) + + row('Time', stack => value(duration(stack.timeSec))) + + row('Spend', stack => value(money(stack.spendUsd))) + + row('Climb', stack => `
${climb(stack.climb, { height: 44 })}
`) + + row('Attempt', stack => { + const attempt = latest(stack); + return `
` + + `${attempt ? esc(phrase(attempt)) : DASH}
`; + }) + + issues + repetitions + board(input, stacks) + evidence + '
'; +} diff --git a/tools/stack-bench/dashboard/public/views/campaigns.ts b/tools/stack-bench/dashboard/public/views/campaigns.ts new file mode 100644 index 00000000000..3ab2ab793b5 --- /dev/null +++ b/tools/stack-bench/dashboard/public/views/campaigns.ts @@ -0,0 +1,104 @@ +// Campaigns: a live block per running campaign, then one table of every +// campaign. Nothing here explains itself — a lane is four cells, a row is +// seven, and the status word lives in the Status column. + +import type { CampaignSheet, OverviewCampaign, OverviewEntry, SheetAttempt } + from '../../dashboard-views.js'; +import { climb } from '../climb.js'; +import { DASH, esc, pct, phrase, shape, since, stackLabel, statusWord } from '../format.js'; +import { STACK_ORDER } from '../metrics.js'; + +export type CampaignFilter = 'all' | 'attention' | 'completed' | 'ready'; + +const FILTERS: Array<{ id: CampaignFilter; label: string }> = [{ id: 'all', label: 'All' }, + { id: 'attention', label: 'Needs attention' }, { id: 'completed', label: 'Completed' }, + { id: 'ready', label: 'Ready' }]; + +function readable(campaign: OverviewEntry): campaign is OverviewCampaign { + return 'scores' in campaign; +} + +function matches(campaign: OverviewEntry, filter: CampaignFilter): boolean { + if (filter === 'all') return true; + if (filter === 'attention') { + return campaign.status === 'attention-required' || campaign.status === 'unreadable'; + } + if (filter === 'completed') return campaign.status === 'completed'; + return campaign.status === 'prepared'; +} + +// Percentage of the points the grades have offered so far, which is the only +// score a half-finished attempt has. +function running(attempt: SheetAttempt): number | null { + const last = attempt.climb.at(-1); + return last && last.max ? 100 * last.score / last.max : attempt.score; +} + +function lane(sheet: CampaignSheet, stack: string, attempt: SheetAttempt): string { + const warn = attempt.stalling; + return `
` + + `${esc(stackLabel(stack))}` + + `${pct(running(attempt))}` + + `${climb(attempt.climb, { warn })}` + + `${esc(phrase(attempt))}
`; +} + +function live(sheet: CampaignSheet): string { + const lanes = STACK_ORDER.flatMap(stack => { + const owner = sheet.stacks.find(entry => entry.stack === stack); + const attempt = owner?.attempts.findLast(item => item.status === 'running'); + return attempt ? [lane(sheet, stack, attempt)] : []; + }); + if (!lanes.length) return ''; + return `
` + + `${esc(sheet.title)}
${lanes.join('')}
`; +} + +function stackCell(campaign: OverviewEntry, stack: string, best: number | null): string { + const score = readable(campaign) ? campaign.scores[stack] ?? null : null; + if (score === null) return `${DASH}`; + const value = best !== null && score === best ? `${pct(score)}` : pct(score); + return `` + + `${value}`; +} + +function tone(status: string): string { + if (status === 'running') return 'run'; + if (status === 'completed') return 'done'; + if (status === 'attention-required' || status === 'unreadable') return 'warn'; + return 'idle'; +} + +function row(campaign: OverviewEntry): string { + const summary = readable(campaign) ? campaign : null; + const best = summary && summary.status === 'completed' && !summary.provisional + ? STACK_ORDER.reduce((top, stack) => { + const score = summary.scores[stack] ?? null; + return score !== null && (top === null || score > top) ? score : top; + }, null) : null; + return `` + + `${esc(campaign.title)}` + + `${summary + ? esc(shape(summary.mode, summary.levels, summary.repetitions)) : DASH}` + + `${esc(statusWord(campaign.status))}` + + STACK_ORDER.map(stack => stackCell(campaign, stack, best)).join('') + + `${summary ? esc(since(summary.updatedAt)) : DASH}`; +} + +export function campaignsPage({ campaigns, sheets, filter }: { + campaigns: readonly OverviewEntry[]; + sheets: readonly CampaignSheet[]; + filter: CampaignFilter; +}): string { + const shown = campaigns.filter(campaign => matches(campaign, filter)); + const chips = FILTERS.map(entry => + `` + + `${entry.label} ${campaigns.filter(campaign => matches(campaign, entry.id)).length}`).join(''); + const body = shown.length ? shown.map(row).join('') + : `no campaigns`; + return `
${sheets.map(live).join('')}` + + `
${chips}
` + + '' + + STACK_ORDER.map(stack => ``).join('') + + `${body}
CampaignShapeStatus${esc(stackLabel(stack))}Updated
`; +} diff --git a/tools/stack-bench/dashboard/public/views/plans.ts b/tools/stack-bench/dashboard/public/views/plans.ts new file mode 100644 index 00000000000..13f36db74ce --- /dev/null +++ b/tools/stack-bench/dashboard/public/views/plans.ts @@ -0,0 +1,112 @@ +// Plans, and the chrome that carries the run controls: one table of test +// plans, the form that starts a run, and the topbar whose Start a run and +// Resume exist only where the server accepts them. Only a frozen plan runs. + +import type { DashboardPlan } from '../../dashboard-model.js'; +import { DASH, esc, money, num } from '../format.js'; + +export type Page = 'campaigns' | 'plans' | 'campaign'; + +export interface RunForm { + planId: string; + outputName: string; + secret: string; + error: string; +} + +// The server's SAFE_NAME, spelled once here so the field cannot hold a name the +// route will reject. +export const RUN_NAME = '[a-z0-9][a-z0-9.-]{2,119}'; + +const HEADS: Array<[string, string]> = [['Plan', 'name'], ['Mode', 'shape'], ['Shape', 'shape'], + ['Stacks', 'stack'], ['Attempts', 'stack'], ['Parallel', 'stack'], ['Repairs', 'stack'], + ['Time limit', 'stack'], ['Spend limit', 'stack'], ['State', 'state']]; + +export function runName(planId: string, now: Date): string { + const pad = (value: number): string => String(value).padStart(2, '0'); + const stamp = `${now.getFullYear()}${pad(now.getMonth() + 1)}${pad(now.getDate())}` + + `-${pad(now.getHours())}${pad(now.getMinutes())}`; + return `${planId}-${stamp}`.toLowerCase().replace(/[^a-z0-9.-]+/g, '-') + .replace(/^[^a-z0-9]+/, '').slice(0, 120); +} + +// A 403 is the wrong operator secret: the tab forgets it and keeps the rest. +export function afterRun(form: RunForm, status: number, error: string): RunForm { + return { ...form, secret: status === 403 ? '' : form.secret, error }; +} + +export function topbar({ page, key, canStart, resumable, error }: { + page: Page; key: string; canStart: boolean; resumable: boolean; error: string; +}): string { + const artifact = (path: string): string => `/api/campaigns/${encodeURIComponent(key)}/artifacts/` + + btoa(path).replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/, ''); + const files = page === 'campaign' + ? '
Files
' + + `planstate` + + `report
` : ''; + const resume = resumable + ? '
' + + '' + + (error ? `${esc(error)}` : '') + '
' : ''; + const nav = (on: boolean, label: string, href: string): string => + `${label}`; + return '
' + + 'STACK BENCH' + + `
${resume}${files}` + + `${canStart && page !== 'plans' ? 'Start a run' : ''}` + + '
'; +} + +function shapeOf(plan: DashboardPlan): string { + const levels = plan.levels ?? []; + if (!levels.length) return DASH; + const depth = Math.max(...levels); + if (plan.mode === 'dependency') return `depth ${depth}`; + return levels.length > 1 ? `L${Math.min(...levels)}–L${depth}` : `L${depth}`; +} + +function planRow(plan: DashboardPlan): string { + const budgets = plan.budgets ?? null; + const stacks = plan.stacks ?? []; + const cell = (value: string, hover = ''): string => + `${value}`; + return `` + + `${esc(plan.title)}` + + `${esc(plan.mode ?? DASH)}` + + `${esc(shapeOf(plan))}` + + cell(stacks.length ? num(stacks.length) : DASH, stacks.join(' · ')) + + cell(num(plan.attempts)) + cell(num(plan.parallelism)) + + cell(plan.repairBudget === undefined ? DASH : num(plan.repairBudget)) + + cell(budgets ? `${budgets.attemptTimeoutMinutes} min` : DASH) + + cell(budgets ? money(budgets.maxCostUsdPerAttempt) : DASH) + + `${esc(plan.state)}`; +} + +function runForm(plans: readonly DashboardPlan[], form: RunForm): string { + const frozen = plans.filter(plan => plan.state === 'frozen'); + const options = [...new Set(frozen.map(plan => plan.mode ?? 'sequential'))] + .map(mode => `${frozen + .filter(plan => (plan.mode ?? 'sequential') === mode) + .map(plan => ``).join('')}`).join(''); + const field = (label: string, control: string): string => + `
${label}${control}
`; + return '
' + + field('Plan', ``) + + field('Run name', ``) + + field('Secret', '') + + '' + + (form.error ? `
${esc(form.error)}
` : '') + '
'; +} + +export function plansPage({ plans, canStart, form }: { + plans: readonly DashboardPlan[]; canStart: boolean; form: RunForm; +}): string { + return `
${canStart ? runForm(plans, form) : ''}` + + '
' + + HEADS.map(([label, kind]) => ``).join('') + + `${plans.length ? plans.map(planRow).join('') + : ``}
${label}
no plans
`; +} diff --git a/tools/stack-bench/docker-compose.yaml b/tools/stack-bench/docker-compose.yaml new file mode 100644 index 00000000000..82b36ef8630 --- /dev/null +++ b/tools/stack-bench/docker-compose.yaml @@ -0,0 +1,49 @@ +# Databases for the Postgres and MongoDB backends. +# +# Development-only ports, container names and volumes keep this stack separate +# from the appliance. The SpacetimeDB +# backend needs no service here; run `spacetime start` for it. +# +# docker compose -f tools/stack-bench/docker-compose.yaml up -d +# +name: stack-bench + +services: + postgres: + image: postgres:16@sha256:219341e4cedb06c8634f80af40851da3425b41b76603fd890272f58e37e139f7 + container_name: stack-bench-dev-postgres + ports: + - "127.0.0.1:6532:5432" + environment: + POSTGRES_USER: appuser + POSTGRES_PASSWORD: local-app-password + POSTGRES_DB: app + volumes: + - pgdata:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U appuser -d app"] + interval: 5s + timeout: 5s + retries: 12 + + # Single node, no replica set: change streams would give MongoDB a push + # mechanism the Postgres backend does not have, and the comparison is about + # what an agent builds on a standard stack. + mongodb: + image: mongo:7@sha256:554a9bb1ec6e00c40ba078a41974a834d1a9a8ab1772645b69142afecc87f082 + container_name: stack-bench-dev-mongodb + ports: + - "127.0.0.1:6537:27017" + volumes: + - mongodata:/data/db + healthcheck: + test: ["CMD", "mongosh", "--quiet", "--eval", "db.runCommand({ping:1})"] + interval: 5s + timeout: 5s + retries: 12 + +volumes: + pgdata: + name: stack-bench-dev-pgdata + mongodata: + name: stack-bench-dev-mongodata diff --git a/tools/stack-bench/docs/README.md b/tools/stack-bench/docs/README.md new file mode 100644 index 00000000000..8071c0e6200 --- /dev/null +++ b/tools/stack-bench/docs/README.md @@ -0,0 +1,41 @@ +# Stack Bench documentation + +Use the root [README](../README.md) for the product summary. + +## Run Stack Bench + +- [Development](development.md): local dependencies and source checks +- [Appliance operation](../appliance/README.md): configure and run campaigns +- [Dashboard](../dashboard/README.md): optional web interface +- [Recovery](../appliance/RECOVERY.md): interrupted runs and retained resources +- [Release](../appliance/RELEASE.md): assemble and verify a release + +## Understand the system + +- [System design](system-design.md): ownership, data flow, and operator loop +- [Prompting method](prompting.md): prompt inputs, specification treatments, + stack guidance, and repair examples +- [Appliance design](../appliance/DESIGN.md): security and container boundaries +- [Grader](../grader/README.md): scoring, evidence, and grader validation +- [Reference apps](../reference-apps/README.md): known-good grading fixtures + +## Define benchmark work + +- [Ecommerce composition](../tracks/ecommerce/composition/README.md): packs, + recipes, calibration, and specification treatment +- [Ecommerce levels](../tracks/ecommerce/LEVELS.md): cumulative and dependency + progression +- [Chat levels](../tracks/chat/LEVELS.md): current chat scope + +## Visuals + +- [Dependency graph](dependency-graph.html): generated ecommerce feature graph +- [Technical guide](technical-guide.html): current run path +- [Presentation](stack-bench.html): product presentation + +`dependency-graph.html` is generated from the versioned graph with +`npm run graph`. Do not edit it by hand. + +Markdown files under `backends/`, `conditions/`, `tracks/*/prompts`, and +`tracks/*/contracts` are executable benchmark inputs. They stay with their +owners and are not general documentation. diff --git a/tools/stack-bench/docs/dependency-graph.html b/tools/stack-bench/docs/dependency-graph.html new file mode 100644 index 00000000000..527fca8c706 --- /dev/null +++ b/tools/stack-bench/docs/dependency-graph.html @@ -0,0 +1,1301 @@ + + + + + +Stack Bench | Dependency Graph + + + +
+
+
+

Ecommerce dependency graph

+

Each row is a questline. Select a feature to see what it needs and what it unlocks.

+
+
+ +
+
+ Qualified feature pack + Draft feature pack +
+
+ +
+
+ +
+
+
+
+ + + + diff --git a/tools/stack-bench/docs/development.md b/tools/stack-bench/docs/development.md new file mode 100644 index 00000000000..500553bde0c --- /dev/null +++ b/tools/stack-bench/docs/development.md @@ -0,0 +1,91 @@ +# Stack Bench development + +This guide covers local source development. Use the +[appliance guide](../appliance/README.md) for runner configuration, credentials, +preflight, campaigns, and paid model work. + +## Requirements + +- Node.js 22 or newer +- Docker Engine with Compose v2 +- Chromium installed through the pinned Playwright dependency +- the repository Linux CLI and TypeScript bindings for SpacetimeDB work + +Install the locked dependencies and browser: + +```bash +cd tools/stack-bench +npm ci +npm run bootstrap:browsers +``` + +Build the local coding image: + +```bash +docker build -t stack-bench-build:2.1.226 container +``` + +On a Windows checkout, build the Linux SpacetimeDB CLI before testing the +SpacetimeDB adapter: + +```bash +bash container/build-linux-cli.sh +``` + +The local image tag is for development. The appliance uses image digests from +its release manifest. + +## Source checks + +Run the smallest check that covers the change: + +| Change | Check | +|---|---| +| TypeScript | `npm run typecheck` and the focused compiled test | +| Unit tests | `npm test` | +| Dashboard read model, routes, and pages | `npm run test:dashboard` | +| Repository contracts | `npm run test:contracts` | +| Mutation definitions and anchors | `npm run test:mutation-definitions` | +| Browser, process, and Docker integration | `npm run test:integration` | +| Prompt composition | `npm run check:prompts` | +| Track scenarios | `npm run check:scenarios` | +| Packs and recipes | `npm run check:composition` | +| Calibration | `npm run check:calibration` | +| Dependency graph | `npm run graph` | + +After a shared runtime, composition, grading, campaign, or release change is +stable, run the integrated source gate once: + +```bash +npm run lint +npm run typecheck +npm run test:all +``` + +Use `npm test` while changing code. Run `npm run test:dashboard` when the +dashboard read model, routes, or pages change; it writes thirty campaigns of +fixture evidence and stays out of the unit tier. Run `npm run test:contracts` +when tracks, prompts, reference applications, repository policies, or campaign +definitions change. `npm run test:all` runs the unit, dashboard, and contract +tiers after one build. Docker and qualification checks remain separate. +Mutation-definition tests are model-free. Run them when reference source, grading +checks, or mutation manifests change. They do not run during ordinary unit work. + +Documentation-only changes need link and formatting checks, not the harness. +Run Docker checks only when the changed code affects their boundary. Run +targeted mutations while developing checks and the complete mutation set only +for a release candidate. Integration files run sequentially because they can +own browsers, processes, ports, and Docker resources. + +A passing check stays valid until one of its inputs changes. Do not rerun it for +reassurance. Add a test only when it protects a distinct invariant that an +existing test does not cover. Pending qualification marks campaign scores as +provisional; it blocks publishing verified comparisons, not campaign execution. + +## Generated files + +Run `npm run graph` to rebuild `docs/dependency-graph.html` from the versioned +ecommerce graph. Do not edit generated output by hand. + +Build output, run artifacts, transcripts, local plans, and operational notes are +not product documentation and must remain untracked. diff --git a/tools/stack-bench/docs/prompting.md b/tools/stack-bench/docs/prompting.md new file mode 100644 index 00000000000..216d46acf1c --- /dev/null +++ b/tools/stack-bench/docs/prompting.md @@ -0,0 +1,239 @@ +# Prompting method + +Stack Bench gives the coding agent a normal software request. It does not tell +the agent that it is in a benchmark. What Stack Bench asks for and what Stack +Bench measures are separate choices. + +## Prompt inputs + +Each request is assembled from these owners: + +| Input | Purpose | Owner | +|---|---|---| +| Product framing | Says whether to build a new app or add work to an existing app | Recipe | +| Current features | Describes the product work to implement now | Feature packs | +| Requested production behavior | States production requirements that the campaign chose to disclose | Specification packs | +| Stack material | Gives required access details and the selected level of technical guidance | Guidance profile and backend document | +| API reference | Supplies selected SDK material, including SpacetimeDB skills | Guidance profile | +| Starting data | Gives the fixed catalog for a new app | Fixture | +| Application interface | Names the controls or operations needed for reliable use | Feature contracts | +| Repair report | Describes conclusive application failures from the last grade | Condition repair policy | + +The recipe and selected packs own the text. The prompt builder orders that text +and adds the small controller contract, such as the application directory, +listening address, start script, and completion response. + +## What the coding agent receives + +A new-build request has this shape: + +```text +Build the application described below and leave it running. + +Build the app in /app. +The web application must listen on 0.0.0.0. + +## Stack + + +## Selected API reference + + +## New application + + +## + + +## Starting catalog + + +## Application interface + +``` + +This is an abridged example. The exact request is composed from versioned files +and bound to the campaign by hashes. + +The new-build request does not include: + +- grader source or scenario files; +- check names, point values, expected scores, or comparison results; +- exact adversarial inputs chosen by a scenario; +- future dependency nodes that are not ready; +- production expectations assigned to the `expected` or `observed` treatments. + +## Features and production expectations + +A feature is product work. A production expectation describes how selected +features should behave under conditions such as reload, reconnect, concurrent +writes, authorization boundaries, or direct data changes. + +Each selected production expectation has one treatment: + +| Treatment | Included in request | Main score | Repair feedback | +|---|---:|---:|---:| +| `requested` | Yes | Yes | Yes | +| `expected` | No | Yes | Yes, after a conclusive failure | +| `observed` | No | No | No | + +`Expected` answers: "Does this stack produce sound production behavior when the +user did not prescribe the mechanism?" + +`Observed` is a separate first-build diagnostic. It cannot change the main +score or steer repairs. + +### Example: expected durability + +The campaign selects account creation as current work. It also selects session +durability as expected production behavior. + +The coding agent sees product text such as: + +```text +## Accounts + +Visitors can create an account with a username and password. Returning users +can sign in, see which account is active, and sign out. +``` + +The request does not mention reload behavior. Stack Bench can still verify that +the signed-in session survives a reload. A conclusive failure affects the main +score and can produce repair feedback. + +### Example: requested durability + +The campaign selects the same account feature and changes durability to +`requested`. The request now also includes text such as: + +```text +## State durability: accounts + +A signed-in session survives a page reload as the same account. +``` + +The scored behavior is the same. Only disclosure changed. This makes the two +conditions comparable without changing the feature itself. + +### Example: observed durability + +The campaign changes durability to `observed`. The request again omits reload +behavior. Stack Bench measures it after the first build, records the result as +a diagnostic, and does not include it in the score or repair report. + +## Stack guidance + +Stack selection and guidance selection are separate. + +- Neutral guidance gives the required stack, connection details, startup + contract, and selected API reference. The coding agent chooses libraries, + architecture, and project structure. +- Prescribed guidance can add design advice selected by the campaign. + +SpacetimeDB can include its TypeScript SDK skills in either condition. This is +API reference for a less familiar stack. It does not expose grading logic or +the implementation that a check expects. + +An abridged neutral PostgreSQL section is: + +```text +# PostgreSQL + +Use PostgreSQL for the application data. Choose the libraries, architecture, +and project structure. + +Use the supplied DATABASE_URL. Serve the application on the supplied port. +``` + +An abridged neutral SpacetimeDB section is: + +```text +# SpacetimeDB + +Use SpacetimeDB for the application data. Put the TypeScript module in the +required module directory. Choose the schema, libraries, architecture, and the +rest of the project structure. + +Use the supplied server URI, module name, CLI, SDK package, and web port. +``` + +## Application interfaces + +Feature contracts name stable controls or operations when deterministic use +requires them. They do not prescribe layout, data models, frameworks, or visual +design. The one data-shaped item a contract may name is an interoperability +surface that other systems write to directly, such as the stock tables; the +behavior expected around that surface stays in the specification. + +For example, the account contract names fields such as `signup-username` and +`signin-submit`. An HTTP stack also exposes the account operations through HTTP. +A reducer-based stack exposes the equivalent reducer operations. The product +behavior stays the same while the usable interface matches the selected stack. + +Scenario files own exact test data and edge-case values. Those values do not +belong in the product request or interface contract. + +## Dependency progression + +Dependency mode composes the request from features that are ready now. + +- A new app receives the framing, current root features, applicable requested + production expectations, starting data, and their interfaces. +- An upgrade receives only the newly ready feature work and its interfaces. +- Passed features are not repeated unless they fail again. +- Blocked descendants are not included until their dependencies pass. + +For example, if accounts and catalog items are ready, the request can include +those two features. Customer profile stays out until its account dependency +passes. A failure in the catalog path does not add or remove work from the +account path. + +## Repair requests + +A repair starts only after Stack Bench completes grading and records a +conclusive application failure. A repair report can name an expected +production behavior that the initial request withheld; that disclosure +happens only through the repair policy, after a conclusive failure, and under +the same rule for every stack. The coding agent receives a plain bug report: + +```text +Fix the reported application bugs. + +Expected: The signed-in account remains active after a reload. +Actual: The page returned to the signed-out state after reload. + +Change only what is needed. Do not alter behavior that is already correct. +``` + +The real report also identifies the affected product area and includes the +current feature text and application interface. Provider failures, harness +failures, and interrupted work do not become application bug reports. + +Every line of the report comes from one of three sources: the sentence the +request already gave the agent for that behavior, a finding from the +grader's closed catalog rendered as one sentence (a control that did not +appear, a number that read 9 instead of 12, a request that was accepted +when it had to be refused), and the application's own console errors. +The grader never writes prose into the report, and the values a scenario +chose to probe a behavior are never among a finding's fields. A repair +fixes the behavior, not the probe. + +## Authoring rules + +- Put product asks in feature prompts. +- Put optional production requirements in specification prompts. +- Put stable controls and operations in contracts. +- Put connection facts and API material in stack guidance. +- Put exact values and probes in scenarios. +- Never solve a check by adding its private input or expected implementation to + the request. +- Keep equivalent stacks equally informed about the product. +- Give every replay, forgery, or direct call a named application action that + declares both the HTTP route and the reducer. A campaign does not compile + while a selected check cannot be measured on a selected stack. +- Mark changed prompt inputs as draft until matching qualification is current. + +After a prompt change, run `npm run check:composition`, `npm run check:prompts`, +and the exact scenario check for the affected recipe. Inspect the rendered +request for every affected stack and depth. Do not run unrelated qualification +or paid work. diff --git a/tools/stack-bench/docs/stack-bench.html b/tools/stack-bench/docs/stack-bench.html new file mode 100644 index 00000000000..59954ae5ab5 --- /dev/null +++ b/tools/stack-bench/docs/stack-bench.html @@ -0,0 +1,1806 @@ + + + + + +Stack Bench + + + + +
+ +
← → or scroll
+ + +
+
+ + SpacetimeDB +
+

Stack Bench

+

Build the same product on different app stacks. Test what works, repair what fails, and compare the evidence.

+
+ + +
+
what it includes
+

Stack Bench manages the full workflow.

+

One system runs the build, tests the finished product, manages repairs, and preserves everything needed to compare app stacks.

+
+
+ +

AI build runner

+

Runs the selected model with the product brief, stack, and budget.

+
+
+ +

Modular product spec

+

Versions features, dependencies, prompts, checks, and scoring rules.

+
+
+ +

Isolated stack runtime

+

Starts the generated app and selected stack services in controlled containers.

+
+
+ +

Automated grader

+

Exercises real user flows, direct data changes, failures, and concurrency.

+
+
+ +

Repair controller

+

Returns failed checks to the agent and manages the repair budget.

+
+
+ +

Evidence and comparison

+

Packages source, prompts, scores, cost, screenshots, video, and traces.

+
+
+
+ + +
+
one controlled run
+

Stack Bench controls every step.

+

Each run keeps the request and environment fixed, isolates the build and services, and saves the results and evidence.

+ + + Stack Benchcontrols the run + preflightchecks setup first + agentbuilds or fixes + app stackselected for the run + buildisolated workspace + servicesisolated and reset + apprunning on the selected stack + graderruns the product tests + resultspass · fail · could not test + evidenceprompt · source · cost · visuals + + + + failed testsreturn to the agent + +
+ + +
+
the run plan
+

Each run starts with a fixed plan.

+

Choose the model, stack, features, tests, and repair limit. Stack Bench records the plan and builds the request the agent receives.

+ + + + + + + + + + + + Model + provider + exact model + + + + Stack + selected stack + tools + + + + Features + unlocked for this level + + + + Tests + checks + scoring + + + + Repairs + limit for each feature + + + + + + + + + + + BUILD, ADD, OR FIX + Work on the current features + + + Product brief + app + current features + Stack access + connection details + API reference + selected SDK material + Testing interface + hooks + lint command + + + On repair: one feature + its failed tests + +
+ + +
+
how testing works
+

Stack Bench tests the finished product.

+

The grader uses real browser sessions, direct data changes, service interruptions, and concurrent actions, then checks both visible and persisted state.

+ + + REAL USER FLOWSmultiple browser actors + ACCESS + OWNERSHIPprotected and cross-account actions + DURABILITY + RECOVERYreload · reconnect · restart + APP UNDER TEST + LIVE STATEdirect data changes reach open pages + CONCURRENCYoverlapping actions · exact totals + OPERATIONS + ACCOUNTINGshipping · pricing · revenue + +
+ + +
+
repair and retest
+

Failed tests return to the agent.

+ + + + + + + + + start level + dependencies passed + + + build + current features + + + test + unlocked features + + + all pass? + + + continue + next level opens + + + repairs left? + + + repair + one failed feature + + + that path stops + other paths continue + + + + + + yes + + + no + + yes + + + + none + + + continue with passed paths + +
+
levelL1L2
+
feature repairs0 of 31 of 3
+
features passed2 of 124 of 12
+
cost$0.00$3.20$6.40
+
+

Each failed feature has its own repair limit. A repair request contains one feature. A completed coding repair uses one repair, even if grading later fails. Provider errors and interrupted coding use none.

+
+ + +
+
dependency mode
+

Working features open the next work.

+

A feature can move forward when its product behavior works. Production checks still affect its score. A failed feature blocks only the paths that need it.

+ + DEPTH 1 + DEPTH 2 + DEPTH 3 + + + + + + + + + + + accountsOPENWORKINGPASS + catalogOPENWORKINGFAIL + cartOPENWORKINGPASS + warehouseOPENWORKINGPASS + + operator accessOPENWORKINGPASS + searchBLOCKED + checkoutOPENWORKINGFAIL + stock transfersOPENWORKINGPASS + + account recoveryOPEN + recommendationsBLOCKED + returnsBLOCKED + scheduled restocksOPEN + +
+ + + + + +
+
testing the benchmark
+

Stack Bench tests itself.

+

The same selected checks run against controlled apps. A correct app must pass, a planted defect must fail its target, and an empty app must score zero.

+ +
+ + +
+
results
+

Scores show where each stack works.

+

Each questline has its own score. Blocked and unfinished work stays in the denominator, so the overall score reflects the complete plan.

+ +
+ + +
+
comparison
+

Compare score, cost, duration, and repairs.

+

Each stack uses the same plan. The report shows what worked on the first build, what repairs improved, what remains, and what the run cost.

+ +
+ + +
+
evidence
+

Every run preserves its evidence.

+

Open the exact result, source, visuals, and run economics behind the score.

+ +
+ + + + diff --git a/tools/stack-bench/docs/system-design.md b/tools/stack-bench/docs/system-design.md new file mode 100644 index 00000000000..fa164ebe8eb --- /dev/null +++ b/tools/stack-bench/docs/system-design.md @@ -0,0 +1,117 @@ +# Stack Bench system design + +Stack Bench turns one versioned test plan into verified comparison data. The +system must make every decision, action, result, and cost traceable without +using chat history or operator memory. + +## One owner for each fact + +| Layer | Owns | Durable output | +|---|---|---| +| Definitions | Product work, prompt modules, checks, stacks, models, and budgets | Versioned source files | +| Compiler | The exact work matrix and all bound identities | `plan.json` | +| Admission | Whether the exact plan can run on this appliance | Admission artifact | +| Scheduler | Attempt order, concurrency, continuations, and terminal state | `state.json` | +| Run engine | Build, grade, repair, resource ownership, and cleanup | Attempt directory | +| Grader | Typed check results and evidence | Grade artifacts | +| Progression engine | Open, passed, failed, and blocked features | `progression-state.json` | +| Report | A reproducible view of retained evidence | `report.json` and `report.html` | + +No layer can silently replace a decision from a layer above it. A view can +summarize durable data, but it cannot create new run state. + +## Data flow + +```text +versioned definitions + | + v +compiled plan -> admission -> scheduler -> run engine -> grader + | | | + v v v + state.json run.json evidence + \ | / + \ v / + -> inspection -> report +``` + +The coding agent receives only the app request, current work, selected stack +material, and repair evidence allowed by the plan. It does not receive the +benchmark, grader, future work, expected implementation, or comparison data. + +## Operator loop + +An operator, human or agent, uses one loop: + +1. **Define.** Select one versioned campaign file. Do not rebuild the plan from + command flags. +2. **Validate.** Compile it and inspect the exact attempts, stacks, model, + prompt policy, checks, points, budgets, images, and parallelism. +3. **Admit.** Prove credentials, images, ports, resource capacity, and stack + access before model work starts. +4. **Run.** Start or resume the exact stored plan. A paid action is always + explicit. +5. **Observe.** Read durable campaign state first. Open logs only to diagnose a + live phase or failure. +6. **Decide.** Continue only through a legal state transition. Never hide an + invalid attempt or retry it automatically. +7. **Report.** Generate the result from retained run evidence. Publish it as + verified comparison data only when grading qualification is complete. +8. **Clean.** Remove temporary owned resources. Keep the campaign package. + +The CLI and dashboard use the same compiler, scheduler, state reader, and run +commands. The dashboard is a view and input surface. It is not another control +plane. + +## Agent interface + +The operator interface must answer these questions without source inspection: + +- What exact plan am I controlling? +- Can it start without spending model usage? +- What is running now, and in which phase? +- What has it cost and how long has it run? +- Which results are valid application results? +- Which failures belong to Stack Bench, the provider, the stack tools, the + host, or the operator? +- What evidence proves each answer? +- Which actions are legal now? + +Machine-facing commands return stable JSON. A compact response gives the plan +identity, campaign state, active work, cost, failures, and legal next actions. +Detailed responses add attempts and artifact paths. Logs and raw artifacts stay +available, but an operator does not need to parse them for normal control. + +Errors must name the failed subsystem, failure owner, retryability, retained +evidence, and next safe action. `inconclusive` is an intermediate measurement +state, not an accepted final explanation. + +## Resource rules + +- Compile and inspect before any model call. +- Run focused source checks after a change. Run the integrated source gate once + for the final source identity. +- Reuse qualification evidence only when all bound hashes match. +- Do not repeat reference, mutation, or null work for unchanged scope. +- Stop new paid attempts after a harness, provider, host, or operator failure. +- Do not retry or grant more repair work automatically. +- Run independent attempts in parallel only within the plan and admitted host + capacity. +- Preserve a failed package before a source or plan change. + +## Accumulated knowledge + +Operational knowledge belongs in typed artifacts, not chat transcripts or a +growing journal. Each completed action records its inputs, identity, outcome, +cost, duration, evidence paths, and owner. A later operator can reconstruct the +campaign from the retained package and continue from the last valid state. + +Local notes can explain an active investigation. They cannot authorize a run, +change a score, or replace a missing artifact. + +## Design test + +Every major structure must have one purpose, one owner, and one current +consumer. If its reason cannot be stated in one sentence, simplify or remove it. +Complexity is allowed only when it protects result validity, isolation, +security, recovery, or a current operator need. diff --git a/tools/stack-bench/docs/technical-guide.html b/tools/stack-bench/docs/technical-guide.html new file mode 100644 index 00000000000..2faafd5e678 --- /dev/null +++ b/tools/stack-bench/docs/technical-guide.html @@ -0,0 +1,617 @@ + + + + + + Stack Bench technical walkthrough + + + +
+
+

Stack Bench
technical walkthrough

+

Current implementation, data flow, code examples, and production boundaries.

+
+
+ + + +
+
+

What the system does

+

Stack Bench builds, tests, repairs, and records the same product work across technology stacks.

+
+
1
DefineTrack, feature catalog, recipe, stacks, model, budget
+
2
CompileResolve exact versions and hash the plan
+
3
PreflightCheck Docker, credentials, ports, files, and smoke paths
+
4
BuildGive the current request to the selected coding agent
+
5
Grade and repairRun selected checks and return application failures
+
6
RecordWrite typed artifacts, source hashes, cost, and reports
+
+
+ +
+

Code map

+

Main code boundaries. Source is TypeScript. The build emits ESM JavaScript into dist/.

+
Control +

Control plane

    +
  • commands/bench.tsRuns one benchmark attempt. It resolves scope, runs preflight, owns the stack lease, calls the coding agent, grades, repairs, and writes the run.
  • +
  • src/campaigns/campaign-compiler.tsTurns a campaign manifest into an exact, hashed plan.
  • +
  • src/campaigns/campaign-runner.tsAdmits, schedules, runs, retries, and reconciles campaign attempts.
  • +
  • src/campaigns/campaign-scheduler.tsStores the state machine for pending, running, completed, and invalid attempts.
  • +
+
+
Definitions +

Definition plane

    +
  • tracks/<track>/track.jsonDeclares levels, suites, actions, ports, restart probes, and validation status.
  • +
  • src/composition/*Compiles packs, recipes, task fragments, selected checks, calibration, and release identities.
  • +
  • src/progression/progression-definition.tsCompiles feature nodes from versioned packs. It assigns each scored group to one owner.
  • +
  • tracks/ecommerce/progression/ecommerce-2.0.2.jsonCurrent 43-node dependency catalog.
  • +
+
+
Execution +

Execution plane

    +
  • src/agents/*Validates agent adapters, credentials, supported modes, result shape, cost, and token usage.
  • +
  • src/stacks/*Defines the stack capability contract and routes work to MongoDB, PostgreSQL, SpacetimeDB, or stub adapters.
  • +
  • src/runtime/*Owns leases, resource locks, Docker runtime, checkpoints, recovery, and source snapshots.
  • +
  • src/progression/*Runs the dependency state machine and persists its event history.
  • +
+
+
Proof +

Proof plane

    +
  • grader/grade.tsRuns browser and API actions. It produces one structured result per selected check.
  • +
  • src/actions/*Defines action contracts and the actors used by scenarios.
  • +
  • src/evidence/*Validates evidence, writes artifacts, tracks provenance, and classifies mutations.
  • +
  • grader/mutation-test.tsBreaks a known-good app one defect at a time and proves that the expected check catches it.
  • +
  • src/references/*Imports, validates, runs, and qualifies known-good reference apps.
  • +
+
+
Interfaces +

Operator interfaces

    +
  • commands/campaign-cli.tsCLI access to prepare, run, resume, reconcile, inspect, audit, and report operations.
  • +
  • dashboard/Reads the same campaign state and artifacts. It does not define a second execution model.
  • +
  • src/campaigns/campaign-report.tsBuilds JSON and HTML reports from the recorded test plan and run state.
  • +
+
+
+ +
+

Definitions and composition

+

Versioned tracks, packs, recipes, and campaigns define each run.

+
+
Track

Product shape

Levels, suites, actions, ports, reset behavior, and validation status.

+
Pack

Reusable module

Prompt fragments, check groups, points, dependencies, and version.

+
Recipe

Selected test plan

Fixture, included modules, prompt composition, execution, and scoring.

+
+
Recipe excerpt: independent modules stay separately versioned
{
+  "id": "ecommerce.progression-catalog",
+  "version": "2.0.1",
+  "packs": [
+    { "id": "ecommerce.feature.accounts", "version": "1.2.0" },
+    { "id": "ecommerce.spec.state-durability", "version": "2.0.1" },
+    { "id": "ecommerce.progression.support-intake", "version": "1.0.0" }
+  ],
+  "execution": "all-selected-sources",
+  "scoring": { "mode": "source-points" }
+}
+
Real code: compose only the fragments selected for this task
const applies = fragment => fragment.ownerConditions
+  ? fragment.ownerConditions.some(entry => owners.has(entry.owner)
+    && entry.modes.includes(mode)
+    && entry.requiresFeatures.every(featureApplies))
+  : fragment.owners.some(owner => owners.has(owner))
+    && fragment.modes.includes(mode);
+
+const requirements = plan.recipe.task.requirements.filter(applies);
+const contracts = plan.recipe.task.contracts.filter(applies);
+
+

Unknown fields and unsupported schema versions fail during compilation.

+

Exact content hashes detect changed definitions between compile time and execution.

+

Exact references and the candidate catalog identify active versions. Older files cannot become launch inputs through filename discovery.

+
+
+ +
+

Dependency mode

+

Dependency depth sets the level. Passed parents open child features.

+

Open the current dependency graph to trace parents, children, and feature descriptions.

+
Work selection. feature sends one ready feature. progressive sends all ready features. all-at-once sends the full selected graph. Repair plan. Selection can target one failed feature or all current failures. The budget can limit the full run, each feature, each depth, or any combination. When limits are combined, the tightest remaining limit wins.
+
Real example: a node owns prompt modules, checks, and dependencies
{
+  "id": "purchasing",
+  "title": "Purchasing and orders",
+  "questline": "orders-fulfillment",
+  "featureRefs": ["ecommerce.feature.purchasing@1.2.1"],
+  "gradingGroups": [
+    "ecommerce.feature.purchasing@1.2.1#purchase-order",
+    "ecommerce.spec.access-control@2.0.0#order-ownership"
+  ],
+  "dependencies": [
+    { "id": "accounts", "reason": "A purchase needs a buyer." },
+    { "id": "catalog", "reason": "A purchase needs an item." }
+  ]
+}
+
Real code: prompt work and grading work are selected separately
function selectedPromptWork(state) {
+  const nodeIds = selectedPromptNodeIds(state);
+  return {
+    nodeIds,
+    featureRefs: [...new Set(selectionFor(state, nodeIds, 'featureRefs'))].sort(),
+    promptModules: [...new Set(selectionFor(state, nodeIds, 'promptModules'))].sort(),
+  };
+}
+
+function selectedGradingWork(state) {
+  const nodeIds = gradingNodeIds(state);
+  const selected = new Set(nodeIds);
+  const promptNodeIds = new Set(selectedPromptNodeIds(state));
+  return {
+    nodeIds,
+    checks: state.definition.nodes.filter(node => selected.has(node.id)).flatMap(node =>
+      node.gradingChecks.filter(check =>
+        state.definition.workSelection === 'all-at-once'
+          || checkRequirementsAvailable(state, check, promptNodeIds))
+        .map(check => ({ ...check, nodeId: node.id })))
+  };
+}
+
Real code: state changes append one event
export function recordDependencyResult(inputState, inputResult) {
+  const state = asDependencyState(inputState);
+  const event = {
+    sequence: state.events.length + 1,
+    type: 'attempt-recorded',
+    result: structuredClone(inputResult)
+  };
+  const next = applyDependencyResult(state, event.result);
+  next.events.push(event);
+  return next;
+}
+
+

Every scored group can have only one owner. Duplicate ownership fails compilation.

+

Feature pack dependencies must exist in the node or one of its ancestors.

+

The persisted event history is hash-checked and replayed. A contradictory or corrupted state cannot resume silently.

+
!

The full catalog is draft. Fresh qualification is still required for the current dependency definition.

+
+
+ +
+

Campaign compilation and scheduling

+

A compiled campaign fixes the scope, versions, budget, repetitions, and parallelism.

+
+

Compilation

  1. Validate the manifest schema.
  2. Resolve exact versions.
  3. Expand all stack, agent, condition, and repetition attempts.
  4. Hash the definition and engine identity.
+

Execution

  1. Run campaign-wide admission checks.
  2. Claim up to the configured parallel slots.
  3. Give each slot a distinct run index and resource scope.
  4. Write attempt state after each completion.
+
+
Example campaign excerpt
{
+  "mode": { "id": "dependency", "version": "4.0.0",
+    "workSelection": "feature" },
+  "repair": { "selection": "feature", "budget": { "perFeature": 1 } },
+  "levels": [1, 2, 3, 4, 5, 6],
+  "featureCatalog": "ecommerce.questlines@2.0.2",
+  "stacks": [
+    { "id": "mongodb", "adapterVersion": "1.4.0" },
+    { "id": "postgres", "adapterVersion": "1.5.0" },
+    { "id": "spacetime", "adapterVersion": "1.3.0" }
+  ],
+  "repetitions": 1,
+  "parallelism": 3,
+  "budgets": { "attemptTimeoutMinutes": 180 }
+}
+
Parallelism. Each active attempt gets separate ports, database names, containers, and resource locks.
+
Extension. campaign extend target.json --from prior-results --depth 2 --out extended-results copies each matching depth-2 source, regrades depths 1 and 2 on a fresh database, and starts depth 3 only after both rechecks pass.
+
+

Model-free drafts use the trial path. Paid campaigns require a complete test plan.

+

An unresolved running attempt must be reconciled before the controller resumes.

+
!

Single-attempt orchestration remains concentrated in commands/bench.ts. Focused tests must protect its boundaries with campaign, runtime, grading, and repair code.

+
+
+ +
+

Agent adapters and prompt construction

+

Agent adapters define model access, limits, credentials, and result format.

+
+

Agent receives

  • Product brief and current feature work
  • Selected stack access details
  • Selected SDK or skill material
  • Required application interface
  • On repair, failed checks and expected results
+

Controller retains

  • Checks selected only for grading
  • Expected and observed specification sets
  • Reference and mutation evidence
  • Campaign state and prior artifacts
  • Docker control authority
+
+
Adapter contract excerpt
export const AGENT_ADAPTER_SCHEMA_VERSION = 5;
+
+const MODES = new Set(['build', 'upgrade', 'resume', 'fix']);
+
+export function defineAgentAdapter(value) {
+  if (!object(value)) throw new Error('agent adapter must be an object');
+  if (value.schemaVersion !== AGENT_ADAPTER_SCHEMA_VERSION)
+    throw new Error('agent adapter schema is unsupported');
+  if (!Array.isArray(value.modes) || value.modes.some(mode => !MODES.has(mode)))
+    throw new Error(`agent adapter ${value.id}.modes is invalid`);
+  // Remaining identity, limit, credential, network, and executable fields
+  // are also validated before the adapter is frozen.
+}
+
Real code: remove grader-only selections from the agent request
const requested = resolved.selection.requested;
+const visible = createBoundRecipeTaskRequest(binding, {
+  featureIds: requested.features,
+  requestedSpecifications: requested.specifications?.requested ?? [],
+  expectedSpecifications: [],
+  observedSpecifications: [],
+  checkKeys: [],
+  dependencyExpansion: requested.dependencyExpansion,
+  taskMode: resolved.taskMode,
+});
+
+if (visible.task.sha256 !== resolved.task.sha256) {
+  throw new Error('undisclosed treatment removal changed the agent task');
+}
+
Selection boundary. Prompt selection and grading selection are independent.
+
+

The result validator checks app directory, mode, level, stack, track, model, tokens, cost, duration, session, and provider metadata.

+

Credentials and allowed outbound destinations are adapter data, not ad hoc shell arguments.

+
!

The code uses several terms for the same boundary: requested, expected, observed, selected, visible, and grading. Public copy needs a smaller vocabulary, even if internal schemas remain exact.

+
+
+ +
+

Stack adapters and runtime isolation

+

Stack adapters provide setup, access, reset, grading, lifecycle, and cleanup operations.

+
Real code: one capability call instead of stack conditionals
export function executeStackCapability(adapter, capabilityName, operation, input = {}) {
+  const provider = adapter.capabilities[capabilityName];
+  if (!provider) {
+    throw new StackCapabilityUnsupportedError(
+      `stack adapter ${adapter.id} does not support capability ${capabilityName}`);
+  }
+  if (!provider.operations.includes(operation)) {
+    throw new StackCapabilityUnsupportedError(
+      `stack adapter ${adapter.id} capability ${capabilityName} does not support operation ${operation}`);
+  }
+  return provider.execute(operation, input);
+}
+
+
Lease

Exact ownership

A generated run ID, ownership token, database identity, container identity, ports, and locks bind all destructive operations.

+
Container

Isolation

The coding process does not receive the controller, grader, scenarios, old results, or Docker socket in appliance mode.

+
Recovery

Cleanup proof

A stopped controller must prove owned resources are released before a campaign attempt can be retried.

+
+
Real code: run IDs and ownership-bound leases
export function newRunId({ track, backend, runIndex,
+  now = new Date(), nonce = randomUUID() }) {
+  const stamp = now.toISOString().replace(/[-:.TZ]/g, '').slice(0, 14);
+  const safe = value => String(value).toLowerCase().replace(/[^a-z0-9_-]+/g, '-');
+  return `${safe(track)}-${safe(backend)}-run${Number(runIndex)}-${stamp}-${safe(nonce).slice(0, 8)}`;
+}
+
+return {
+  version: LEASE_VERSION,
+  runId,
+  backend,
+  track,
+  runIndex,
+  ownerPid,
+  ownershipToken: randomUUID(),
+  createdAt: new Date().toISOString(),
+  state: 'created',
+  resources
+};
+
+

The adapter registry rejects a stack that does not implement every required engine operation.

+

Lease files are written atomically with restricted permissions.

+

Resource locks use exact keys. The controller does not select cleanup targets from generated code.

+
!

Host and appliance behavior still share some runtime modules. Every production path should be proven in the Docker appliance, which is the intended delivery shape.

+
+
+ +
+

Grading and check evidence

+

Selected scenarios produce typed check evidence. Only application failures are repairable.

+
+

Measured results

StatusMeaningRepair
passedThe app met the assertion.No
failedThe app did not meet the assertion.Yes
+

Invalid evidence

StatusMeaningScore
inconclusiveThe result cannot prove pass or fail.0 points; attempt invalid
harness_failureThe test system failed.0 points; attempt invalid
+
+
Core status policy
export const CHECK_EVIDENCE_DISPOSITIONS = Object.freeze({
+  passed: { outcomeKind: 'passed', passed: true, measured: true,
+    applicationFailure: false, repairable: false },
+  failed: { outcomeKind: 'app_failure', passed: false, measured: true,
+    applicationFailure: true, repairable: true },
+  inconclusive: { outcomeKind: 'inconclusive', passed: false, measured: false,
+    applicationFailure: false, repairable: false },
+  harness_failure: { outcomeKind: 'harness_failure', passed: false, measured: false,
+    applicationFailure: false, repairable: false },
+});
+
Status control. The status table alone controls scoring and repair eligibility.
+
+

A clean status boundary prevents a browser crash or setup fault from looking like an application defect.

+

Selected checks are mapped by stable recipe keys before the browser starts.

+
!

grader/grade.ts still coordinates browser setup, action execution, evidence, and aggregation. Focused action and evidence tests must protect those boundaries.

+
+
+ +
+

Repair loop and source checkpoints

+

Repairs use failed checks and preserve the last accepted source checkpoint.

+
+

Hash current source

The controller records the authored source before grading.

+

Grade selected checks

Passed, failed, inconclusive, and harness failure results remain distinct.

+

Send repairable failures

With feature selection, dependency mode sends one failed feature. Batch and sequential runs send their selected failing set.

+

Grade the changed source

A repair is useful only if the new result is usable and the source is intact.

+

Accept or restore

The checkpoint becomes the next source, or the controller restores the last accepted source.

+
+
Real code: source snapshots exclude runtime and harness files
const PRESERVED_DIRS = new Set(['node_modules']);
+const ROOT_PRESERVED_DIRS = new Set(['.git', 'stack-bench']);
+const TRANSIENT_DIRS = new Set([
+  'dist', '.vite', 'coverage',
+  '.apt', '.cache', '.debroot', '.libs', '.npm-cache', '.pw-browsers', '.pwcache'
+]);
+const TRANSIENT_PATHS = new Set(['client/src/module_bindings']);
+const ROOT_RUNTIME_FILES = new Set(['BUG_REPORT.md', 'client.log', 'server.log', 'vite.log']);
+
+export function hashAppSource(appDir) {
+  return hashDirectory(appDir, { exclude: rel => directoryDisposition(rel) !== 'source'
+    || preservedRuntimeFile(rel) || transientRuntimeFile(rel) });
+}
+
+

Source identity excludes dependencies, build output, logs, and harness-owned files.

+

Repair continuation checks the parent agent adapter, stack adapter, selection, build image, and source.

+
!

The repair loop must stop repeated failures that provide no new information. The run report should show repeated failure signatures and cost per changed result.

+
+
+ +
+

Mutation qualification

+

Mutation tests prove that known defects fail the intended checks.

+
+
1
Clean referenceHash and fully grade the baseline
+
2
Apply defectUse exact source edits from the manifest
+
3
Reset stateRestore database and app readiness
+
4
Grade targetRun the selected scenario and checks
+
5
ClassifyExpected catch, miss, collateral, or invalid evidence
+
+
Real code: mutations are distributed across isolated workers
function shardAssignments(mutations, count, defaultScenario) {
+  const workers = Array.from({ length: count }, () => []);
+  mutations.forEach((mutation, position) => {
+    const scenario = mutation.scenario ?? defaultScenario;
+    if (typeof scenario !== 'string' || !scenario.trim()) {
+      fail(`mutation ${mutation.id} has no scenario`);
+    }
+    workers[position % count].push(position);
+  });
+  return workers;
+}
+
Sharding. Individual defects are distributed evenly. Each worker has isolated ports, containers, databases, and evidence. Merge validation requires every mutation ID exactly once.
+
+

The mutation manifest binds the exact pristine fixture hash.

+

A mutation counts only when the intended criterion fails conclusively and no unrelated criterion regresses.

+

Parallel shard merge rejects missing, duplicate, or misassigned results.

+
!

Mutation coverage proves that known defects are detected. It does not prove that the grader detects every possible defect. Coverage must map to check ownership and risk.

+
+
+ +
+

Artifacts, provenance, and reports

+

Typed artifacts record the plan, run, evidence, identities, and source hashes.

+
+
Plan

What was selected

Track, versions, stacks, model, conditions, levels, pricing, budgets, repetitions, and engine identity.

+
Run

What happened

Preflight, build sessions, grades, repairs, source checkpoints, time, token usage, cost, and recovery.

+
Evidence

Why it scored

Check observations, expected values, actors, actions, screenshots, logs, source, and test identity.

+
+ + + + + + + + +
ArtifactPurposeMain validator
plan.jsonFrozen experiment and schedulevalidateCompiledCampaignPlan
state.jsonCampaign attempt statevalidateCampaignState
run.jsonOne complete attemptartifact schema and campaign run validator
bundle.jsonOne grading bundlegrade bundle artifact validator
progression-state.jsonDependency event history and bound identitieshash validation and event replay
recovery.jsonProof that exact-owned resources were cleanedrecovery artifact validator
+
+ +
+

CLI and dashboard

+

The CLI and dashboard use the same campaign state and artifacts.

+
Dashboard artifact allowlist
Campaign: plan, state, HTML report, JSON report
+Attempt: run, preflight, recovery, progression state, process metadata, logs, checkpoints
+Grade: bundle, contract lint, actions, check evidence, screenshots
+
+

Dashboard paths are resolved under the configured results root.

+

Credentials are redacted from log tails before display.

+

Visual evidence, JSON, reports, and logs are exposed through explicit patterns.

+
!

The dashboard should remain a controller client. Business rules must stay in shared source modules, not browser code.

+
+
+ +
+

Test layers

+

Use focused tests first. Use Docker and live qualification only when required.

+
+
Fast

Module and contract tests

Node tests cover compilers, state machines, adapters, evidence, scheduling, locks, prompts, and artifact validation.

npm run build --silent && node --test dist/tests/<focused-file>.test.js

+
Model free

Reference and control tests

Known-good fixtures, null controls, scenario checks, prompt snapshots, graph compilation, and mutation definition checks need no paid agent.

npm run check:scenarios

+
Integrated

Docker and live qualification

Container smoke, real stack reset, browser grading, reference qualification, fault injection, and campaign admission prove the deployed topology.

npm run test:container

+
+
Order. Focused test, affected model-free check, Docker smoke if needed, then scoped live qualification.
+ + + + + + + +
ChangeMinimum focused proofRelease proof
Graph dependencyProgression definition, engine, state, and graph testsAffected reference nodes and their mutations
Check or scenarioScenario compiler and exact grader testsKnown-good pass plus targeted mutation catch
Stack adapterAdapter contract and stack-specific unit testsDocker smoke, reset, recovery, and reference qualification
Campaign schedulerScheduler, locking, admission, and retry testsModel-free parallel campaign trial
Prompt compositionRecipe selection and prompt snapshot testsScope identity and agent-visible request review
+
+ +
+

Change examples

+

Common extension paths.

+
+
+

Move a feature to a later level

+
    +
  1. Change the node dependencies in progression/ecommerce-2.0.2.json.
  2. +
  3. Do not edit a numeric level. The compiler recalculates it from dependency depth.
  4. +
  5. Keep the same feature pack and grading group IDs if their behavior did not change.
  6. +
  7. Compile the graph. Check reachability, cycles, prompt mode, and ownership.
  8. +
  9. Run focused definition and progression tests.
  10. +
  11. Requalify only affected reference and mutation scopes.
  12. +
+

Expected result: the feature moves without rewriting its prompt module, grader scenario, or stable check identity.

+
+
+

Add a new technology stack

+
    +
  1. Implement every required capability in src/stacks/backends/<stack>.
  2. +
  3. Register the adapter with an exact semantic version.
  4. +
  5. Add connection, reset, lifecycle, grading, container, and cleanup operations.
  6. +
  7. Add focused contract tests with the stub path.
  8. +
  9. Add Docker admission and reference qualification for the real stack.
  10. +
  11. Add it to a campaign only after its reference checks pass.
  12. +
+

Expected result: campaign, agent, grader, evidence, and dashboard code do not need stack-specific branches.

+
+
+

Add a product feature

+
    +
  1. Create one feature pack with prompt requirements and application interface.
  2. +
  3. Create one or more scenario files with stable criteria and points.
  4. +
  5. Add the pack to the recipe catalog.
  6. +
  7. Add one graph node that owns its scored groups.
  8. +
  9. Declare only necessary parent features.
  10. +
  11. Add a known-good reference behavior and targeted mutation evidence.
  12. +
+
+
+

Continue a stopped repair

+
    +
  1. Select the exact parent run and level.
  2. +
  3. Create a repair grant with the number of repairs.
  4. +
  5. Verify source, adapter, selection, model, stack, and image identities.
  6. +
  7. Seed the accepted checkpoint.
  8. +
  9. Run only the added repairs and required regression checks.
  10. +
  11. Append the new cost and evidence to the lineage.
  12. +
+
+
+
+ +
+

Senior review checklist

+

Production comparison checklist.

+ + + + + + + + + + + + +
AreaWhat to proveCurrent signal
Definition integrityExact versions, no duplicate ownership, all nodes reachable, no cycles, prompt and test identities fixed.Strong compiler checks. The full six-depth catalog is still draft.
Prompt boundaryAgent receives only the selected product work and intended guidance. Grader-only selections do not change prompt text.Hash equality check exists.
Stack fairnessSame product work and scored checks. Only stack access and stack-specific setup differ.Adapter boundary exists. Needs qualified reference evidence per stack.
Grader correctnessKnown-good app passes. Known defects fail the intended criterion without collateral failures.Mutation framework exists. Qualification coverage is the gating evidence.
Failure attributionApplication failure, inconclusive evidence, provider failure, and harness failure remain separate.Central evidence disposition table exists.
Repair qualityFailures contain useful structured evidence. Repeated unchanged failures stop. Accepted source remains recoverable.Checkpoint and repair lineage exist. Cost efficiency still needs run evidence.
Parallel isolationEvery attempt gets unique ports, database, containers, paths, and locks.Run index, leases, and lock keys exist. Fresh qualification is required for the current definition.
ReproducibilityPlan, engine, definitions, adapters, image, model, source, pricing, and checks are all recorded.Identity model is broad and strict.
Code sizeLarge orchestration files have clear internal phases and focused tests.commands/bench.ts and grader/grade.ts remain the main responsibility concentrations.
Production dataOnly qualified, complete, comparable attempts enter stack conclusions.Current dependency qualification is pending. Paid comparison data must pass the same artifact validation.
+
Main risk. Large execution files can duplicate selection, scoring, cleanup, or retry policy.
+
+
+ + + diff --git a/tools/stack-bench/grader/README.md b/tools/stack-bench/grader/README.md new file mode 100644 index 00000000000..9689398f1d7 --- /dev/null +++ b/tools/stack-bench/grader/README.md @@ -0,0 +1,131 @@ +# Stack Bench grader + +The grader runs versioned scenarios against a generated app. It collects +browser, transport, lifecycle, and database evidence for each check. + +Each scenario actor receives a separate browser context. A live-update check +passes only when the page that was already open changes. The grader does not +reload a failed assertion and try again. + +## Outcomes and scoring + +Every check produces one outcome: + +- `passed`; +- `failed`; +- `inconclusive` when required evidence is unavailable; +- `harness_failure` when Stack Bench could not perform the measurement. + +Only a passed check adds its declared points. Other outcomes add zero and never +change the declared denominator. Console errors remain diagnostics and do not +change unrelated scores. + +Authorization and replay checks pass only when the requested call ran and +produced verifiable evidence. Visible UI behavior cannot replace missing server +evidence. + +An action never fails with a sentence. It fails with a finding from the closed +catalog in `src/actions/action-findings.ts`: a kind and its fields, where a +field is a contract control name, an action id, an actor label, a number, a +count, or an HTTP status. Every reader renders the finding from its one +template. Raw diagnostics travel in a `detail` field that is never rendered. + +## Scenario ownership + +Scenario JSON contains actors, setup steps, actions, and scored checks. The +action contracts are compiled and registered in `src/actions/`. Scenario prose +is not executable behavior. + +Actions run through capability-scoped executors. Browser, transport, +concurrency, lifecycle, and database actions use the same typed result contract. +Each stack adapter declares the capabilities it provides and whether named +application actions travel as HTTP routes or reducer calls. The campaign +compiler resolves every selected check against every selected stack and +refuses a campaign that a stack could not measure. + +When authoring assertions: + +- scope repeated elements to their owning row, room, message, or user; +- assert visible values, not the presence of an empty container; +- require the original open page for live-update behavior; +- use separate actors for identity boundaries; +- say in the criterion's `note` why it carries its points when they differ + from the feature's other criteria. + +Example: + +```json +{ + "do": "expect", + "actor": "bob", + "testid": "unread-badge", + "in": { "testid": "room-item", "contains": "{room:unread-main}" }, + "within": 5000 +} +``` + +## Run the grader + +Use `dist/commands/run-suite.js` for normal grading. It owns database reset, +provenance checks, contract linting, scenario execution, logs, and bundle +creation. + +Direct `dist/grader/grade.js` execution is for focused scenario authoring only: + +```bash +node dist/grader/grade.js --url http://localhost:6173 \ + --spec tracks/ecommerce/scenarios/01-account-create-2.4.0.json \ + --label spacetime-l1 --out report.json +``` + +If the grader exits before writing JSON, inspect the retained +`grader-.stdout.log` and `grader-.stderr.log` files. + +## Validate checks + +Reference apps prove that intended behavior passes. Null controls prove that a +blank app cannot earn points. Mutations prove that each scored check detects its +assigned defect. + +```bash +npm run test:null +npm run check:mutations -- --app --mutations +``` + +During development, run only mutations affected by the change. A full mutation +qualification is a release-candidate gate. + +The mutation runner requires: + +- a fully passing clean baseline; +- one exact source anchor for every edit; +- a conclusive failure at the intended check; +- no unrelated failures; +- successful source restoration and app reset. + +Setup, infrastructure, and inconclusive failures do not count as defect +detection. A surviving mutation can be equivalent, so confirm that its source +edit changes observable behavior before changing the check. + +## Media evidence + +`--media ` records videos and failure screenshots. `--trace` adds a +Playwright trace with DOM and network snapshots. + +```bash +npx playwright show-trace +``` + +Inspect the failing actor's evidence before attributing a failure. Media belongs +with run output and is not tracked in the repository. + +## Execution target + +Preflight binds the stack adapter, database or module name, ports, container +identity, and run lease. The suite runner verifies that exact target before +grading. A mismatch is a harness failure and cannot produce an application +score. + +When several stacks fail the same check, inspect the structured evidence. A +shared failure is useful diagnostic information, but it does not prove whether +the apps or the check are wrong. diff --git a/tools/stack-bench/grader/grade.ts b/tools/stack-bench/grader/grade.ts new file mode 100644 index 00000000000..bcf30b01c9b --- /dev/null +++ b/tools/stack-bench/grader/grade.ts @@ -0,0 +1,1048 @@ +#!/usr/bin/env node +/// +// Score declared criteria from one observed run in isolated actor contexts. +// +import { chromium } from 'playwright'; +import type { Browser, BrowserContext, Page } from 'playwright'; +import { randomUUID } from 'node:crypto'; +import { readFileSync, mkdirSync } from 'node:fs'; +import { basename, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { parseArgs as parseNodeArgs } from 'node:util'; +import { harnessBrowserFailure, harnessProcessFailure, + runBrowserInfrastructureOperation } from '../src/evidence/harness-errors.js'; +import { compileScenarioDefinition } from '../src/composition/definition-compiler.js'; +import { materializeScenarioCredentials } from '../src/composition/credential-aliases.js'; +import { loadTrack } from '../src/composition/tracks.js'; +import { isFinding } from '../src/actions/action-findings.js'; +import type { Finding } from '../src/actions/action-findings.js'; +import { recipeArtifactIdentities, writeArtifact } from '../src/evidence/artifacts.js'; +import { resolveCalibrationForRelease } from '../src/composition/calibration-compiler.js'; +import { resolveGradeRecipeArtifactBinding } from '../src/composition/recipe-release.js'; +import { selectScenarioChecks } from '../src/composition/recipe-selection.js'; +import { ACTION_REGISTRY } from '../src/actions/action-catalog.js'; +import { ActionApplicationFailure, executeAction } from '../src/actions/action-contract.js'; +import { createCheckEvidence, evidenceIsMeasured, evidencePassed } from '../src/evidence/check-evidence.js'; +import { evidenceNowMs } from '../src/evidence/evidence-timing.js'; +import { renderEvidenceConsoleLine } from '../src/evidence/evidence-presentation.js'; +import { measureGradePackRuntime } from '../src/composition/pack-runtime.js'; +import { STACK_ADAPTER_REGISTRY } from '../src/stacks/stack-adapters.js'; +import { stableElementSelector } from '../src/actions/element-selector.js'; +import { + createNamedActionsCapability, +} from '../src/actions/actor-transport-action-executors.js'; +import type { ConcurrentCallResult } + from '../src/actions/actor-transport-action-executors.js'; +import { + createDatabaseWriteCapability, + createLifecycleCapability, +} from '../src/actions/runtime-action-executors.js'; +import type { DatabaseWriteLease } from '../src/actions/runtime-action-executors.js'; +import { controlAppServer, controlBackendRuntime, parseRuntimeControlSpec } + from '../src/runtime/backend-control.js'; +import type { RuntimeControlSpec } from '../src/runtime/backend-control.js'; +import { leaseFromEnv } from '../src/runtime/backend-lease.js'; +import type { LeasedSpacetimeTarget } from '../src/runtime/spacetime-target.js'; + +import { STACK_BENCH_ROOT as ROOT } from '../src/package-root.js'; +import { transportFrameText } from './transport-frames.js'; +import type { ActionEvidence } from '../src/actions/action-contract.js'; +import type { CheckEvidence, CheckEvidenceAttachment, CheckEvidencePhase, + CheckEvidenceStatus } from '../src/evidence/check-evidence.js'; +import type { CompletedGradeFeatureResult, CompletedGradeReport, GradeCleanupFailure } + from '../src/evidence/grade-report.js'; +import type { CompiledFeature, CompiledScenarioDefinition, + CompiledStep } from '../src/composition/definition-compiler.js'; +import type { RecipeCheck, RecipeGradeRelease, RecipeRelease } from '../src/composition/recipe-release.js'; +import type { TrackAction } from '../src/composition/tracks.js'; + +type JsonRecord = Record; +type ActorWrite = { + url: string; + method: string; + headers: Record; + body: JsonRecord | null; +}; +type ActorWebSocketWrite = { event: unknown; body: JsonRecord }; +type ActorContextEntry = { context: BrowserContext; name: string; page: Page | null }; +type CleanupBrowserContext = { + tracing: { stop(options: { path: string }): Promise }; + close(): Promise; +}; +type CleanupVideo = { saveAs(path: string): Promise; delete(): Promise }; +type CleanupPage = { video(): CleanupVideo | null }; +type CleanupActorContextEntry = { + context: CleanupBrowserContext; + name: string; + page: CleanupPage | null; +}; +type FeatureResult = Omit & { + setupEvidence?: CheckEvidence; +}; +type GradeArgs = { + url?: string; + level: number; + headed: boolean; + selectedCheckKeys: string[]; + out?: string; + label?: string; + feature?: number; + spec?: string; + restartSpec?: RuntimeControlSpec; + backend?: string; + track?: string; + recipe?: string; + expectedRecipeSha256?: string; + credentialAliases?: unknown; + selectionSha256?: string; + parentAttemptId?: string; + dbName?: string; + app?: string; + media?: string; + failureMedia?: string; + trace?: boolean; + nullControl: boolean; + browserWsEndpoint?: string; +}; +type GradeRunContext = { + runId: string; + roomName: (base: string) => string; + restartSpec?: RuntimeControlSpec; + url: string; + backend?: string; + actions: TrackAction[]; + spacetime: LeasedSpacetimeTarget | null; + dbName?: string; + databaseLease?: DatabaseWriteLease | null; + appDir?: string; + scope?: string; + extraContexts?: ActorContextEntry[]; + recorded?: Record; + unverified?: string[]; + verified?: string[]; + actionEvidence?: Array<{ actor: string | null; evidence: ActionEvidence }>; + serverCheck?: string | null; + lastCalls?: ConcurrentCallResult | null; + defaultWithin?: number; + nullControl: boolean; + // True while a scenario step has stopped the application server and no + // later step or restore has started it again. + applicationStopped?: boolean; +}; +type ActionFailure = Error & { actionEvidence?: ActionEvidence; actionActor?: string | null }; +const APPLICATION_RESTORE_SETTLE_MS = 8000; +const APPLICATION_RESTORE_TIMEOUT_MS = 60_000; +class ApplicationNotRestored extends Error { + constructor(reason: string) { + super(`the application server stopped by the harness was not restored: ${reason}`); + } +} +type CheckFailure = { + status: CheckEvidenceStatus; + code: string; + actor: string | null; + summary: string | null; + finding: Finding | null; + observation: unknown; + expected: unknown; + retryable: boolean; +}; + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +function actionFailure(error: unknown): ActionFailure | null { + return error instanceof Error ? error as ActionFailure : null; +} +// The sentence the coding agent was given for this behaviour travels with the +// grade, so a repair report can repeat it instead of describing the check. +const authored = (criterion: { statedBy?: string }): { statedBy?: string } => + criterion.statedBy ? { statedBy: criterion.statedBy } : {}; +const DEFAULT_WITHIN = 5000; +const SETUP_WITHIN = 20000; +// Keep the cause when Playwright prefixes it with locator retry details. +function keepReason(detail: unknown, limit = 600): string { + const s = String(detail ?? ''); + if (s.length <= limit) return s; + const [head, ...rest] = s.split('\n'); + const reasons = rest + .map(l => l.trim()) + .filter(l => /^-\s/.test(l)) + .map(l => l.replace(/^-\s*/, '')) + .filter(l => !/^(waiting for|retrying|attempting|scrolling|done scrolling|locator resolved to|\d+ ×)/i.test(l)); + const kept = [...new Set(reasons)].slice(0, 4); + const out = kept.length ? `${head}\n - ${kept.join('\n - ')}` : s.slice(0, limit); + return out.length > limit ? out.slice(0, limit) : out; +} + +export function parseGradeArgs(argv: readonly string[]): GradeArgs { + const { values } = parseNodeArgs({ args: [...argv.slice(2)], options: { + url: { type: 'string' }, level: { type: 'string' }, out: { type: 'string' }, + label: { type: 'string' }, feature: { type: 'string' }, spec: { type: 'string' }, + 'restart-spec': { type: 'string' }, backend: { type: 'string' }, track: { type: 'string' }, + recipe: { type: 'string' }, 'expected-recipe-sha256': { type: 'string' }, + 'selected-check': { type: 'string', multiple: true }, + 'credential-aliases-json': { type: 'string' }, 'selection-sha256': { type: 'string' }, + 'parent-attempt-id': { type: 'string' }, 'db-name': { type: 'string' }, + app: { type: 'string' }, media: { type: 'string' }, 'failure-media': { type: 'string' }, + trace: { type: 'boolean' }, headed: { type: 'boolean' }, + 'null-control': { type: 'boolean' }, + 'browser-ws-endpoint': { type: 'string' }, + } }); + const args: GradeArgs = { url: values.url, level: values.level === undefined ? 1 : Number(values.level), + out: values.out, label: values.label, + feature: values.feature === undefined ? undefined : Number(values.feature), spec: values.spec, + restartSpec: values['restart-spec'] === undefined ? undefined + : parseRuntimeControlSpec(JSON.parse(values['restart-spec'])), + backend: values.backend, track: values.track, recipe: values.recipe, + expectedRecipeSha256: values['expected-recipe-sha256'], + selectedCheckKeys: values['selected-check'] ?? [], + credentialAliases: values['credential-aliases-json'] === undefined + ? undefined : JSON.parse(values['credential-aliases-json']), + selectionSha256: values['selection-sha256'], parentAttemptId: values['parent-attempt-id'], + dbName: values['db-name'], app: values.app, media: values.media, + failureMedia: values['failure-media'], trace: values.trace, headed: values.headed ?? false, + nullControl: values['null-control'] ?? false, + browserWsEndpoint: values['browser-ws-endpoint'] }; + if (!args.url || !args.spec) { + throw new Error('Usage: node dist/grader/grade.js --url --spec ' + + '--level [--out ] [--label ] [--feature ]'); + } + let url: URL; + try { url = new URL(args.url); } + catch { throw new Error('--url must be a valid HTTP or HTTPS URL'); } + if (url.protocol !== 'http:' && url.protocol !== 'https:') { + throw new Error('--url must use HTTP or HTTPS'); + } + if (!Number.isInteger(args.level) || args.level < 1) { + throw new Error('--level must be a positive integer'); + } + if (args.feature !== undefined && (!Number.isInteger(args.feature) || args.feature < 1)) { + throw new Error('--feature must be a positive integer'); + } + if (args.selectionSha256 && !/^[a-f0-9]{64}$/.test(args.selectionSha256)) { + throw new Error('--selection-sha256 must be 64 lowercase hexadecimal characters'); + } + if (args.browserWsEndpoint) { + let endpoint: URL; + try { endpoint = new URL(args.browserWsEndpoint); } + catch { throw new Error('--browser-ws-endpoint must be a valid WebSocket URL'); } + if (!['ws:', 'wss:'].includes(endpoint.protocol)) { + throw new Error('--browser-ws-endpoint must use ws or wss'); + } + } + return args; +} + +const tid = stableElementSelector; +const uniq = () => randomUUID().slice(0, 16); +const MAX_RECEIVED_BYTES = 8 * 1024 * 1024; +const MAX_CONSOLE_ERRORS = 200; + +// Isolated browser actor + +// Which requests count as writes worth capturing for replay and forgery. The +// default covers chat's routes; a scenario spec can widen it for an application +// whose endpoints are named differently (`writeUrlPattern`). +const DEFAULT_WRITE_URL = '\\/api\\/|\\/rooms|\\/messages'; +let WRITE_URL_RE = new RegExp(DEFAULT_WRITE_URL); + + +class Actor { + readonly name: string; + readonly context: BrowserContext; + page!: Page; + readonly consoleErrors: string[]; + readonly received: string[]; + receivedBytes = 0; + receivedOverflow = false; + lastWrite: ActorWrite | null = null; + lastWrites: Record = {}; + writes: ActorWrite[] = []; + lastWsWrite: ActorWebSocketWrite | null = null; + annotate = false; + + constructor(name: string, page: Page, context: BrowserContext) { + this.name = name; + this.context = context; + this.consoleErrors = []; + // Test privacy against delivered payloads, not rendered content. + this.received = []; + this.attach(page); + } + attach(page: Page): void { + this.page = page; + // Capture writes so checks can replay them with changed fields or actors. + this.lastWrite = null; + this.lastWrites = {}; + this.writes = []; + this.lastWsWrite = null; + page.on('dialog', dialog => { + void dialog.dismiss().catch(error => { + if (page.isClosed()) return; + this.consoleErrors.push(`dialog dismiss failed: ${errorMessage(error)}`); + if (this.consoleErrors.length > MAX_CONSOLE_ERRORS) this.consoleErrors.shift(); + }); + }); + // A missing client identity in a WebSocket write proves server-derived identity. + page.on('websocket', ws => { + ws.on('framesent', f => { + const p = typeof f.payload === 'string' ? f.payload : ''; + const m = p.match(/^\d+(\[.*\])$/s); + if (!m) return; + try { + const [event, arg] = JSON.parse(m[1] as string) as unknown[]; + if (arg && typeof arg === 'object' && !Array.isArray(arg)) { + this.lastWsWrite = { event, body: arg as JsonRecord }; + } + } catch { /* not a socket.io event frame */ } + }); + // Binary frames are decoded as UTF-8 too, after any SpacetimeDB frame + // compression: a binary wire format still carries message text as + // inline UTF-8 bytes, so a substring search finds it without the + // harness knowing the encoding. + ws.on('framereceived', f => this.record(f.payload)); + }); + page.on('request', req => { + if (req.method() === 'GET' || req.method() === 'OPTIONS') return; + const url = req.url(); + if (!WRITE_URL_RE.test(url)) return; + let body: JsonRecord | null = null; + try { + const candidate: unknown = JSON.parse(req.postData() ?? ''); + if (candidate && typeof candidate === 'object' && !Array.isArray(candidate)) { + body = candidate as JsonRecord; + } + } catch { /* bodyless, e.g. a DELETE */ } + // Forging needs a body to tamper with; replaying does not — a privileged + // action is often a bare DELETE whose meaning is entirely in the URL. + const write = { url, method: req.method(), headers: req.headers(), body }; + this.writes.push(write); + if (this.writes.length > 200) this.writes.shift(); + if (body && typeof body === 'object') { + this.lastWrite = write; + this.lastWrites[req.method()] = write; + } + }); + page.on('console', m => { + if (m.type() !== 'error') return; + const text = m.text(); + // Expected 4xx responses are not application console failures. + if (/Failed to load resource.*status of 4\d\d/.test(text)) return; + this.consoleErrors.push(text.slice(0, 200)); + if (this.consoleErrors.length > MAX_CONSOLE_ERRORS) this.consoleErrors.shift(); + }); + page.on('pageerror', e => { + this.consoleErrors.push(`pageerror: ${e.message.slice(0, 200)}`); + if (this.consoleErrors.length > MAX_CONSOLE_ERRORS) this.consoleErrors.shift(); + }); + page.on('response', async res => { + const type = res.headers()['content-type'] ?? ''; + // Data only. Scripts and markup are served as text/* too, and a Vite + // bundle would bury the buffer in megabytes of application source. + if (!/(application\/json|application\/x-ndjson|text\/event-stream|text\/plain)/.test(type)) return; + const length = Number(res.headers()['content-length']); + if (Number.isFinite(length) && length > MAX_RECEIVED_BYTES) { + this.receivedOverflow = true; + return; + } + try { this.record(await res.text()); } catch { /* body gone, or page closed */ } + }); + } + record(payload: string | Buffer): void { + const text = transportFrameText(payload); + if (!text) return; + const chunk = text.slice(0, 200_000); + this.received.push(chunk); + this.receivedBytes += Buffer.byteLength(chunk); + while (this.receivedBytes > MAX_RECEIVED_BYTES && this.received.length > 1) { + this.receivedOverflow = true; + this.receivedBytes -= Buffer.byteLength(this.received.shift()!); + } + } + wasSent(needle: string): boolean { + if (this.received.some(chunk => chunk.includes(needle))) return true; + if (this.receivedOverflow) throw new Error('transport evidence exceeded its memory limit'); + return false; + } + loc(testid: string, { contains, scope }: + { contains?: string; scope?: { testid: string; contains?: string } } = {}) { + // `scope` narrows the search to inside a specific container (e.g. the badge + // belonging to ONE room), so a stale element elsewhere can't satisfy it. + const root = scope + ? this.page.locator(tid(scope.testid), { hasText: scope.contains }).filter({ visible: true }).first() + : this.page; + return (contains + ? root.locator(tid(testid), { hasText: contains }) + : root.locator(tid(testid))).filter({ visible: true }).first(); + } +} + +// Expand scenario aliases to the run-scoped values used by the app. +const expand = (s: unknown, ctx: GradeRunContext): unknown => + typeof s === 'string' + ? s.replace(/\{room:([^}]+)\}/g, (_, b) => ctx.roomName(b)) + // Keep generated usernames alphanumeric so ordinary validators accept them. + .replace(/\{user:([^}]+)\}/g, (_, n) => `${n}${ctx.scope}`) + : s; + + +// Put test context in recordings without exposing it to scoped app selectors. + +const OVERLAY_ID = '__stackbench_overlay'; + +async function annotate(actor: Actor | undefined, { feature, criterion, step, status }: + { feature?: string; criterion?: string; step?: string; status?: 'fail' | 'pass' } = {}): Promise { + if (!actor?.annotate) return; + await actor.page.evaluate(({ id, feature, criterion, step, status, who }) => { + let el = document.getElementById(id); + if (!el) { + el = document.createElement('div'); + el.id = id; + el.style.cssText = [ + 'position:fixed', 'inset:0 0 auto 0', 'z-index:2147483647', + 'font:12px/1.5 ui-monospace,SFMono-Regular,Menlo,monospace', + 'padding:6px 10px', 'pointer-events:none', 'white-space:pre', + 'background:rgba(12,12,16,.92)', 'color:#e8e8ef', + 'border-bottom:2px solid #4c8dff', + ].join(';'); + document.documentElement.appendChild(el); + } + const colour = status === 'fail' ? '#ff5c5c' : status === 'pass' ? '#3ddc84' : '#4c8dff'; + el.style.borderBottomColor = colour; + el.textContent = [ + `${who} ${feature ?? ''}`, + criterion ? ` ${status === 'fail' ? 'FAILED' : 'checking'}: ${criterion}` : '', + step ? ` > ${step}` : '', + ].filter(Boolean).join(String.fromCharCode(10)); + }, { id: OVERLAY_ID, feature, criterion, step, status, who: actor.name }).catch(() => {}); +} + +// Step execution + +function abortableSleep(ms: number, signal: AbortSignal | null = null): Promise { + if (signal?.aborted) return Promise.reject(signal.reason ?? new Error('action cancelled')); + return new Promise((resolve, reject) => { + const timer = setTimeout(done, ms); + function done() { + signal?.removeEventListener('abort', cancelled); + resolve(); + } + function cancelled() { + clearTimeout(timer); + signal?.removeEventListener('abort', cancelled); + reject(signal?.reason ?? new Error('action cancelled')); + } + signal?.addEventListener('abort', cancelled, { once: true }); + }); +} + +function browserActionCapabilities(actors: Map, ctx: GradeRunContext): Readonly> { + const defaultWithin = ctx.defaultWithin ?? DEFAULT_WITHIN; + const actorAccess = Object.freeze({ get: (name: string) => actors.get(name) }); + const runtimeValues = Object.freeze({ + defaultWithin, + expand: (value: unknown) => expand(value, ctx), + hyphenatedScopedUser: (name: string) => `${name}-${ctx.scope}`, + roomName: (base: string) => ctx.roomName(base), + scopedUser: (name: string) => `${name}${ctx.scope}`, + recorded: Object.freeze({ + get: (key: string) => ctx.recorded?.[key], + set: (key: string, value: unknown) => { (ctx.recorded ??= {})[key] = value; }, + }), + sleep: abortableSleep, + testId: tid, + clients: Object.freeze({ + async open(actor: Actor, settleMs: number, signal: AbortSignal) { + const fresh = await actor.context.newPage(); + fresh.setDefaultTimeout(defaultWithin); + actor.attach(fresh); + await fresh.goto(ctx.url, { waitUntil: 'domcontentloaded', timeout: 20000 }); + await abortableSleep(settleMs, signal); + }, + async fresh(actor: Actor, sourceName: string) { + const browser = actor.page.context().browser(); + if (!browser) throw new Error('actor browser is unavailable'); + const context = await browser.newContext(); + const fresh = await context.newPage(); + const name = `${sourceName}-fresh`; + // Register teardown ownership before navigation. If goto fails, the + // partially opened context must still be closed with the feature. + ctx.extraContexts?.push({ context, name, page: fresh }); + fresh.setDefaultTimeout(defaultWithin); + const observer = new Actor(`${actor.name}-fresh`, fresh, context); + await fresh.goto(ctx.url, { waitUntil: 'domcontentloaded', timeout: 20000 }); + observer.annotate = actor.annotate; + actors.set(name, observer); + return name; + }, + }), + }); + const transportObservation = Object.freeze({ + defaultWithin, + expand: (value: unknown) => expand(value, ctx), + sleep: abortableSleep, + verification: Object.freeze({ + structural(message: string) { + ctx.verified?.push(message); + ctx.serverCheck = ctx.serverCheck ?? 'structural'; + }, + unverified(message: string) { + (ctx.unverified ??= []).push(message); + ctx.serverCheck = 'unverified'; + }, + verified(message: string) { + ctx.verified?.push(message); + ctx.serverCheck = 'verified'; + }, + }), + }); + const namedActions = createNamedActionsCapability({ + actions: ctx.actions, + backend: ctx.backend!, + url: ctx.url, + spacetime: ctx.spacetime, + lastCalls: Object.freeze({ + get: () => ctx.lastCalls ?? null, + set: value => { ctx.lastCalls = value; }, + }), + sleep: abortableSleep, + }); + const concurrency = Object.freeze({ + defaultWithin, + dispatch: (step: CompiledStep, signal: AbortSignal) => runRegisteredAction(step, actors, ctx, signal), + expand: (value: unknown) => expand(value, ctx), + sleep: abortableSleep, + testId: tid, + }); + return Object.freeze({ + actors: actorAccess, + 'application-files': Object.freeze({ root: ctx.appDir ?? null, expand: (value: unknown) => expand(value, ctx) }), + 'application-lifecycle': applicationLifecycle(ctx), + 'backend-lifecycle': createLifecycleCapability({ + restartSpec: ctx.restartSpec, + target: 'backend-runtime', + control: controlBackendRuntime, + sleep: abortableSleep, + }), + 'browser-interaction': runtimeValues, + 'browser-observation': runtimeValues, + clock: Object.freeze({ sleep: abortableSleep }), + concurrency, + 'database-write': createDatabaseWriteCapability({ + backend: ctx.backend, + spacetime: ctx.spacetime, + databaseLease: ctx.databaseLease, + skip: ctx.nullControl, + expand: (value: string) => String(expand(value, ctx)), + }), + 'named-actions': namedActions, + subprocess: Object.freeze({ sleep: abortableSleep }), + 'transport-observation': transportObservation, + }); +} + +function applicationLifecycle(ctx: GradeRunContext) { + return createLifecycleCapability({ + restartSpec: ctx.restartSpec, + target: 'app-server', + control: controlAppServer, + sleep: abortableSleep, + onOperated: mode => { ctx.applicationStopped = mode === 'stop'; }, + }); +} + +async function runRegisteredAction(step: CompiledStep, actors: Map, ctx: GradeRunContext, + signal: AbortSignal | null = null): Promise { + const actionEvidence = await executeAction(ACTION_REGISTRY, step.do, step, + { + capabilities: browserActionCapabilities(actors, ctx), + signal, + }); + ctx.actionEvidence?.push({ actor: step.actor ?? null, evidence: actionEvidence }); + if (actionEvidence.status === 'passed') return actionEvidence.observation; + const error = new Error(actionEvidence.summary ?? `${step.do} did not complete`); + Object.defineProperty(error, 'actionEvidence', { value: actionEvidence }); + Object.defineProperty(error, 'actionActor', { value: step.actor ?? null }); + throw error; +} + +function classifyCheckFailure(error: unknown, fallbackActor: string | null = null): CheckFailure { + const actionError = actionFailure(error); + const actionEvidence = actionError?.actionEvidence; + if (actionEvidence) { + return { + status: actionEvidence.status, + code: actionEvidence.code, + actor: actionError?.actionActor ?? fallbackActor, + summary: actionEvidence.summary ?? `${actionEvidence.action.id} did not complete`, + finding: actionEvidence.finding, + observation: actionEvidence.observation, + expected: actionEvidence.expected, + retryable: actionEvidence.retryable, + }; + } + if (error instanceof ApplicationNotRestored) { + return { status: 'harness_failure', code: 'application_not_restored', actor: fallbackActor, + summary: error.message, finding: null, observation: null, expected: null, retryable: false }; + } + const processFailure = harnessProcessFailure(error); + if (processFailure) return { status: 'harness_failure', code: 'process_failure', actor: fallbackActor, + summary: processFailure, finding: null, observation: null, expected: null, retryable: false }; + const browserFailure = harnessBrowserFailure(error); + if (browserFailure) return { status: 'harness_failure', code: 'browser_failure', actor: fallbackActor, + summary: browserFailure, finding: null, observation: null, expected: null, retryable: false }; + if (error instanceof ActionApplicationFailure) { + return { status: 'failed', code: 'application_failure', actor: fallbackActor, + summary: error.message, finding: isFinding(error.details.finding) ? error.details.finding : null, + observation: error.details.observation ?? null, + expected: error.details.expected ?? null, retryable: false }; + } + return { status: 'harness_failure', code: 'unclassified_exception', actor: fallbackActor, + summary: errorMessage(error ?? 'unknown grader failure'), + finding: null, observation: null, expected: null, retryable: false }; +} + +function buildCheckEvidence({ ctx, phase, startedAtMs, failure = null, actor = null, summary = null, + attachments = [], actions = ctx.actionEvidence ?? [], sensitivity = null }: { + ctx: GradeRunContext; phase: CheckEvidencePhase; startedAtMs: number; failure?: unknown; + actor?: string | null; summary?: string | null; attachments?: Array; + actions?: Array<{ actor: string | null; evidence: ActionEvidence }>; + sensitivity?: readonly string[] | null; + }): CheckEvidence { + const classified: CheckFailure = failure ? classifyCheckFailure(failure, actor) : { + status: 'passed', code: 'completed', actor: null, summary: null, finding: null, + observation: null, expected: null, retryable: false, + }; + const completedAtMs = Math.max(startedAtMs, evidenceNowMs()); + const evidenceSummary = summary ?? classified.summary; + return createCheckEvidence({ + ...classified, + phase, + summary: evidenceSummary == null ? null : keepReason(evidenceSummary), + startedAtMs, + completedAtMs, + actions, + attachments: attachments.map(attachment => typeof attachment === 'string' + ? { kind: 'screenshot', ref: basename(attachment) } : attachment), + sensitivity: sensitivity ?? actions.flatMap(entry => entry.evidence?.sensitivity ?? []), + }); +} + +async function runStep(step: CompiledStep, actors: Map, ctx: GradeRunContext): Promise { + return runRegisteredAction(step, actors, ctx); +} + +export async function closeActorContexts(entries: readonly CleanupActorContextEntry[], { + trace = false, media = null, slug = 'grade', +}: { trace?: boolean; media?: string | null; slug?: string } = {}): Promise { + const failures: GradeCleanupFailure[] = []; + const record = (name: string, stage: string, error: unknown): void => { failures.push({ + actor: name, + stage, + reason: keepReason(errorMessage(error)), + }); }; + for (const { context, name, page } of entries) { + if (trace) { + try { + await context.tracing.stop({ path: join(media ?? '.', `${slug}-${name}.trace.zip`) }); + } catch (error) { record(name, 'trace', error); } + } + let video = null; + if (media && page) { + try { video = page.video(); } + catch (error) { record(name, 'video-handle', error); } + } + try { await context.close(); } + catch (error) { record(name, 'context-close', error); } + if (video) { + try { await video.saveAs(join(media!, `${slug}-${name}.webm`)); } + catch (error) { record(name, 'video-save', error); } + try { await video.delete(); } + catch (error) { record(name, 'video-delete', error); } + } + } + return failures; +} + +// Feature grading + +function completedFeatureResult(result: FeatureResult): CompletedGradeFeatureResult { + if (!result.setupEvidence) { + throw new Error(`feature ${result.id} completed without setup evidence`); + } + return { ...result, setupEvidence: result.setupEvidence }; +} + +async function gradeFeature(browser: Browser, feature: CompiledFeature, args: GradeArgs, + runCtx: GradeRunContext): Promise { + // Features share the app's DATABASE even though each gets fresh browser + // contexts, so user and room names are scoped per feature — otherwise a + // defect in one feature (e.g. a hijacked account) corrupts later setups. + const scope = `${runCtx.runId}f${feature.id}`; + const extraContexts: ActorContextEntry[] = []; + const ctx: GradeRunContext = { ...runCtx, scope, roomName: (base: string) => `${base}-${scope}`, extraContexts, recorded: {}, + unverified: [], verified: [], actionEvidence: [] }; + const actors = new Map(); + const contexts: ActorContextEntry[] = []; + const slug = `${args.label ?? 'run'}-f${feature.id}`; + + // A feature is worth what its criteria are worth. An explicit `max` is only + // a consistency check enforced by check-scenarios, never a top-up. + const featureMax = feature.criteria.reduce((n, c) => n + (c.points ?? 1), 0); + const result: FeatureResult = { + id: feature.id, name: feature.name, score: 0, max: featureMax, + criteria: [], consoleErrors: [], + }; + const restoreFailures: GradeCleanupFailure[] = []; + const closeAll = async () => { + const failures = [...restoreFailures, ...await closeActorContexts([...contexts, ...extraContexts], { + trace: args.trace, media: args.media, slug, + })]; + if (failures.length) result.cleanupEvidence = { status: 'harness_failure', failures }; + return failures; + }; + // A criterion that stops the application server owns it only for its own + // steps. Whatever the outcome, the server is running again before the next + // criterion; a restore the harness cannot complete is the harness's failure + // and every later criterion in the feature is unmeasured, not failed. + const restoreApplicationServer = async () => { + if (!ctx.applicationStopped || restoreFailures.length) return; + try { + await applicationLifecycle(ctx) + .operate('start', APPLICATION_RESTORE_SETTLE_MS, AbortSignal.timeout(APPLICATION_RESTORE_TIMEOUT_MS)); + } catch (error) { + restoreFailures.push({ actor: null, stage: 'application-restore', + reason: keepReason(errorMessage(error)) }); + } + }; + const initializationStartedAtMs = evidenceNowMs(); + try { + for (const name of feature.actors!) { + // Isolated storage per actor. Video is per-context, so each actor gets its + // own recording — you can watch what every participant saw, side by side. + const context = await runBrowserInfrastructureOperation('context creation', () => + browser.newContext( + args.media ? { recordVideo: { dir: args.media, size: { width: 1280, height: 800 } } } : {} + )); + contexts.push({ context, name, page: null }); + if (args.trace) { + await runBrowserInfrastructureOperation('trace start', () => + context.tracing.start({ screenshots: true, snapshots: true })); + } + const page = await runBrowserInfrastructureOperation('page creation', () => context.newPage()); + contexts[contexts.length - 1]!.page = page; + page.setDefaultTimeout(SETUP_WITHIN); + const actor = new Actor(name, page, context); + actor.annotate = Boolean(args.media); + actors.set(name, actor); + try { + await page.goto(args.url!, { waitUntil: 'domcontentloaded', timeout: 20000 }); + } catch (cause) { + if (harnessBrowserFailure(cause)) throw cause; + throw new ActionApplicationFailure('application did not load during browser setup', { + observation: errorMessage(cause), expected: 'a reachable application page', + }); + } + } + } catch (error) { + const classified = classifyCheckFailure(error); + const reason = keepReason((classified.summary ?? '').trim()); + result.setupEvidence = buildCheckEvidence({ ctx, phase: 'setup', startedAtMs: initializationStartedAtMs, + failure: error, summary: reason }); + for (const criterion of feature.criteria) { + const points = criterion.points ?? 1; + const evidence = buildCheckEvidence({ ctx, phase: 'setup', startedAtMs: initializationStartedAtMs, + failure: error, summary: `browser setup failed: ${reason}`, actions: [] }); + result.criteria.push({ id: criterion.id, desc: criterion.desc, points, evidence, + ...authored(criterion) }); + result.inconclusive = [...(result.inconclusive ?? []), + { id: criterion.id, points, status: evidence.status, code: evidence.code, + phase: evidence.phase, summary: evidence.summary }]; + } + await closeAll(); + return completedFeatureResult(result); + } + + const captureFailureScreenshots = async (label: string): Promise => { + if (!args.failureMedia) return []; + mkdirSync(args.failureMedia, { recursive: true }); + const captured: string[] = []; + for (const { name, page } of [...contexts, ...extraContexts]) { + if (!page) continue; + const path = join(args.failureMedia, `${slug}-${label}-${name}.png`); + const ok = await page.screenshot({ path, fullPage: true, timeout: 5000 }) + .then(() => true, () => false); + if (ok) captured.push(path); + } + return captured; + }; + + const setupStartedAtMs = evidenceNowMs(); + ctx.defaultWithin = SETUP_WITHIN; + ctx.actionEvidence = []; + try { + // Setup is not scored, but a failure makes the feature untestable (0). + for (const step of feature.setup) { + await annotate(actors.get(step.actor), { feature: feature.name, criterion: 'setup', step: step.do }); + await runStep(step, actors, ctx); + } + } catch (err) { + // Preserve the typed setup failure on every affected criterion. + const classified = classifyCheckFailure(err); + const why = keepReason((classified.summary ?? '').trim()); + const screenshots = await captureFailureScreenshots('setup'); + result.setupEvidence = buildCheckEvidence({ ctx, phase: 'setup', startedAtMs: setupStartedAtMs, + failure: err, summary: why, attachments: screenshots }); + for (const c of feature.criteria) { + const base = why ? `setup failed: ${why}` : 'setup failed'; + const points = c.points ?? 1; + const evidence = buildCheckEvidence({ ctx, phase: 'setup', startedAtMs: setupStartedAtMs, + failure: err, summary: base, actions: [], sensitivity: result.setupEvidence.sensitivity, + attachments: [{ kind: 'check-evidence', ref: 'feature.setupEvidence' }, ...screenshots] }); + const recorded = { id: c.id, desc: c.desc, points, evidence, ...authored(c) }; + result.criteria.push(recorded); + if (!evidenceIsMeasured(evidence)) { + result.inconclusive = [...(result.inconclusive ?? []), + { id: c.id, points, status: evidence.status, code: evidence.code, + phase: evidence.phase, summary: evidence.summary }]; + } + } + if (screenshots.length) result.screenshots = screenshots; + await closeAll(); + return completedFeatureResult(result); + } + result.setupEvidence = buildCheckEvidence({ ctx, phase: 'setup', startedAtMs: setupStartedAtMs }); + ctx.defaultWithin = DEFAULT_WITHIN; + for (const actor of actors.values()) actor.page.setDefaultTimeout(DEFAULT_WITHIN); + + for (const criterion of feature.criteria) { + let failure: unknown = null, detail: string | null = null, activeActor: string | null = null; + let criterionScreenshots: string[] = []; + const criterionStartedAtMs = evidenceNowMs(); + ctx.actionEvidence = []; + ctx.serverCheck = null; + try { + if (restoreFailures.length) throw new ApplicationNotRestored(restoreFailures[0]!.reason); + for (const step of criterion.steps) { + activeActor = step.actor ?? activeActor; + await annotate(actors.get(step.actor) ?? actors.values().next().value, + { feature: feature.name, criterion: criterion.id, step: step.do }); + await runStep(step, actors, ctx); + } + for (const a of actors.values()) { + await annotate(a, { feature: feature.name, criterion: criterion.id, step: 'passed', status: 'pass' }); + } + } catch (err) { + failure = err; + const classified = classifyCheckFailure(err, activeActor); + detail = classified.summary; + if (args.media) { + for (const a of actors.values()) { + await annotate(a, { feature: feature.name, criterion: criterion.id, + step: errorMessage(err).slice(0, 120), status: 'fail' }); + } + const shotActor = actors.get(criterion.steps[criterion.steps.length - 1]?.actor) ?? actors.values().next().value; + const shot = join(args.media, `${slug}-${criterion.id}.png`); + const captured = await shotActor.page.screenshot({ path: shot, fullPage: true }) + .then(() => true, () => false); + if (captured) criterionScreenshots.push(shot); + } else { + criterionScreenshots = await captureFailureScreenshots(criterion.id); + } + if (criterionScreenshots.length) { + result.screenshots = [...(result.screenshots ?? []), ...criterionScreenshots]; + } + } + await restoreApplicationServer(); + const evidence = buildCheckEvidence({ ctx, phase: 'assertion', startedAtMs: criterionStartedAtMs, + failure, actor: activeActor, summary: detail, attachments: criterionScreenshots }); + result.criteria.push({ id: criterion.id, desc: criterion.desc, points: criterion.points, + evidence, ...authored(criterion), + ...(ctx.serverCheck ? { serverCheck: ctx.serverCheck } : {}) }); + if (evidencePassed(evidence)) result.score += criterion.points; + else if (!evidenceIsMeasured(evidence)) { + result.inconclusive = [...(result.inconclusive ?? []), + { id: criterion.id, points: criterion.points, status: evidence.status, code: evidence.code, + phase: evidence.phase, summary: evidence.summary }]; + } + } + + for (const actor of actors.values()) { + for (const e of actor.consoleErrors) result.consoleErrors.push(`[${actor.name}] ${e}`); + } + + // Retain diagnostics for server-side checks that could not execute. The + // action executor marks those criteria inconclusive, so they cannot score. + if (ctx.unverified?.length) result.unverified = ctx.unverified; + if (ctx.verified?.length) result.verified = ctx.verified; + + await closeAll(); + if (args.media) result.videos = contexts.map(c => join(args.media!, `${slug}-${c.name}.webm`)); + return completedFeatureResult(result); +} + +// Main + +async function main(): Promise { + const startedAt = new Date().toISOString(); + const args = parseGradeArgs(process.argv); + const specPath = args.spec!; + let spec: CompiledScenarioDefinition; + try { + const compiled = compileScenarioDefinition(JSON.parse(readFileSync(specPath, 'utf8')), + { source: specPath }); + spec = materializeScenarioCredentials(compiled, args.credentialAliases); + } catch (error) { + throw new Error(`cannot compile scenario ${specPath}: ${errorMessage(error)}`, { cause: error }); + } + + if (typeof spec.writeUrlPattern === 'string' && spec.writeUrlPattern) { + WRITE_URL_RE = new RegExp(spec.writeUrlPattern); + } + + const candidateFeatures = args.feature ? spec.features.filter(f => f.id === args.feature) : spec.features; + if (args.feature && candidateFeatures.length === 0) { + throw new Error(`scenario ${specPath} has no feature ${args.feature}`); + } + const runId = uniq(); + // Where the named actions live. The track declares their names; the + // authenticated backend lease—not generated application config—selects the + // SpacetimeDB host, module and exact build container used for direct SQL. + let actions: TrackAction[] = [], spacetime: LeasedSpacetimeTarget | null = null, + recipeRelease: RecipeGradeRelease | null = null, + recipeIdentityRelease: RecipeRelease | null = null, + calibration: ReturnType | null = null; + if (args.track) { + const track = loadTrack(args.track); + actions = track.actions; + const binding = resolveGradeRecipeArtifactBinding(track, args.level, specPath, + args.feature ?? null, args.recipe); + recipeRelease = binding?.release ?? null; + recipeIdentityRelease = binding?.sourceRelease ?? null; + } + if (args.expectedRecipeSha256 + && recipeRelease?.contentSha256 !== args.expectedRecipeSha256) { + throw new Error(`recipe changed before grading: expected ${args.expectedRecipeSha256}, ` + + `resolved ${recipeRelease?.contentSha256 ?? 'no recipe'}`); + } + const selectedScenario = selectScenarioChecks( + { ...spec, features: candidateFeatures }, recipeRelease, args.selectedCheckKeys); + const features = selectedScenario.features; + const selectedChecks = selectedScenario.checks; + if (!features.length) throw new Error(`scenario ${specPath} has no selected checks`); + if (args.track) { + const track = loadTrack(args.track); + calibration = resolveCalibrationForRelease(recipeIdentityRelease, { + trackRoot: track.dir, + stackBenchRoot: ROOT, + }); + } + spacetime = args.backend + ? STACK_ADAPTER_REGISTRY.get(args.backend).grading.context({ requireBuildContainer: true }) + : null; + const hostedBackend = args.backend === 'mongodb' || args.backend === 'postgres'; + const hasLeaseAuthority = Boolean(process.env.STACK_BENCH_LEASE + || process.env.STACK_BENCH_LEASE_TOKEN); + let databaseLease: DatabaseWriteLease | null = null; + if (hostedBackend && hasLeaseAuthority) { + const lease = leaseFromEnv(process.env, { backend: args.backend, active: true }).lease; + const { container, database } = lease.resources; + if (!container || !database) { + throw new Error(`active ${args.backend} lease has no complete database identity`); + } + databaseLease = { resources: { container, database } }; + } + + const ctx: GradeRunContext = { runId, roomName: (base: string) => `${base}-${runId}`, + restartSpec: args.restartSpec, url: args.url!, + backend: args.backend, actions, spacetime, dbName: args.dbName, + databaseLease, + nullControl: args.nullControl, + appDir: args.app }; + + const browser = args.browserWsEndpoint + ? await chromium.connect(args.browserWsEndpoint) + : await chromium.launch({ headless: !args.headed }); + const report: JsonRecord & CompletedGradeReport & { + inconclusive?: Array; + cleanupEvidence?: { status: 'harness_failure'; failures: GradeCleanupFailure[] }; + } = { + definitionSchemaVersion: spec.schemaVersion, + recipeRelease, + label: args.label ?? null, url: args.url, level: args.level, runId, + total: 0, max: features.reduce((n, f) => n + f.criteria.reduce((m, c) => m + (c.points ?? 1), 0), 0), features: [], + selection: recipeRelease ? { + ...(args.selectionSha256 ? { sha256: args.selectionSha256 } : {}), + checks: selectedChecks.map(({ stableKey, packId, checkGroupId, featureId, criterionId, + description, points }) => { + if (!packId) throw new Error(`selected check ${stableKey} has no pack id`); + return { stableKey, packId, checkGroupId, featureId, criterionId, description, points }; + }), + } : null, + }; + const checkByCriterion = new Map(selectedChecks.map(check => [ + `${String(check.featureId)}\0${String(check.criterionId)}`, check, + ])); + + try { + for (const feature of features) { + process.stdout.write(`Feature ${feature.id}: ${feature.name} ... `); + const r = await gradeFeature(browser, feature, args, ctx); + if (recipeRelease) { + for (const criterion of r.criteria) { + const check = checkByCriterion.get(`${String(feature.id)}\0${String(criterion.id)}`); + if (!check) throw new Error(`graded criterion ${feature.id}/${criterion.id} has no recipe check`); + criterion.stableKey = check.stableKey; + } + } + report.features.push(r); + report.total += r.score; + // The recipe owns the denominator. An unmeasured criterion earns zero and + // remains explicitly inconclusive; it must never change the contract. + if (r.inconclusive?.length) { + report.inconclusive = [...(report.inconclusive ?? []), + ...r.inconclusive.map(c => ({ feature: r.id, ...c }))]; + } + console.log(`${r.score}/${r.max}`); + for (const c of r.criteria.filter(c => !evidencePassed(c.evidence))) { + console.log(` ${renderEvidenceConsoleLine(c.evidence, c.id)}`); + } + } + } finally { + try { await browser.close(); } + catch (error) { + report.cleanupEvidence = { status: 'harness_failure', failures: [{ + actor: null, stage: 'browser-close', reason: keepReason(errorMessage(error)), + }] }; + } + } + + if (recipeRelease) report.packRuntime = measureGradePackRuntime(report); + + console.log(`\nTOTAL ${report.total}/${report.max}`); + if (args.out) { + const artifactId = `grade-${runId}`; + writeArtifact(args.out, { + kind: 'grade', + id: artifactId, + attempt: { id: artifactId, parentId: args.parentAttemptId ?? null }, + timestamps: { startedAt, completedAt: new Date().toISOString() }, + identities: recipeArtifactIdentities(recipeIdentityRelease, { + calibration: calibration ? { id: calibration.id, version: calibration.version, + sha256: calibration.contentSha256, state: calibration.state } : null, + stackAdapter: args.backend ? { id: args.backend } : null, + }), + payload: report, + }); + console.log(`Report written to ${args.out}`); + } +} + +if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) { + main().catch(error => { + console.error(error instanceof Error ? (error.stack ?? error.message) : String(error)); + process.exitCode = 2; + }); +} diff --git a/tools/stack-bench/grader/mutation-test.ts b/tools/stack-bench/grader/mutation-test.ts new file mode 100644 index 00000000000..09479de533e --- /dev/null +++ b/tools/stack-bench/grader/mutation-test.ts @@ -0,0 +1,743 @@ +#!/usr/bin/env node +// A valid mutation fails only its declared criterion against a passing baseline. +import { + copyFileSync, + cpSync, + existsSync, + mkdtempSync, + readFileSync, + rmSync, + unlinkSync, + writeFileSync, +} from "node:fs"; +import { createHash } from "node:crypto"; +import { dirname, join, relative, resolve, sep } from "node:path"; +import { tmpdir } from "node:os"; +import { fileURLToPath } from "node:url"; +import { execFileSync } from "node:child_process"; +import { parseArgs as parseNodeArgs } from "node:util"; +import { currentEngineIdentity, emptyArtifactIdentities, readArtifactPayload, + writeRunJson } from "../src/evidence/artifacts.js"; +import { controlBackendRuntime, parseRuntimeControlSpec } from "../src/runtime/backend-control.js"; +import type { RuntimeControlSpec } from "../src/runtime/backend-control.js"; +import { + classifyMutationResult, + groupMutationsByScenario, + isRetryableMutationBaseline, + isRetryableMutationResult, + mutationFileEdits, + mutationTargetKeys, + readMutationManifest, + releaseScenarioCheckKeys, + resolveMutationScenarioPath, + reusableMutationBaseline, + resolveMutationFile, + validateMutationBaseline, + validateMutationDefinitions, +} from "../src/evidence/mutation-analysis.js"; +import { dbName, loadTrack, TRACK_MANIFEST_FILE } from "../src/composition/tracks.js"; +import { resolveRecipeRelease } from "../src/composition/recipe-release.js"; +import { resetBackend } from "../src/stacks/backend-reset.js"; +import { STACK_ADAPTER_REGISTRY } from "../src/stacks/stack-adapters.js"; +import { mutationShard } from "../src/evidence/mutation-shards.js"; +import { reusableMutationEvidence } from "../src/evidence/mutation-checkpoint.js"; +import { MUTATION_GRADE_MAX_TIMEOUT_MS, mutationGradeTimeoutMs } + from "../src/evidence/mutation-control.js"; +import { assertAppSourceIdentity } from "../src/runtime/source-snapshot.js"; +import type { TextCommandExecutor } from '../src/runtime/command-executor.js'; +import type { LoadedMutationManifest, MutationDefinition } from '../src/evidence/mutation-analysis.js'; +import type { MutationCheckpointBaseline, MutationCheckpointIdentity, + MutationCheckpointResult } from '../src/evidence/mutation-checkpoint.js'; + +type JsonRecord = Record; +type MutationSpec = LoadedMutationManifest; +type MutationArgs = { + app?: string; url?: string; mutations?: string; level?: string; spec?: string; backend?: string; + track?: string; recipe?: string; selectedCheckKeys?: string[]; dbName?: string; runIndex?: string; + restartSpec?: RuntimeControlSpec; out?: string; parentAttemptId?: string; + mutationShardIndex?: number; mutationShardCount?: number; resumeFrom?: string; checkpointOut?: string; + baselineBundle?: string; expectedCalibrationIdentity?: JsonRecord; maxRuntimeMinutes?: number; + imageId?: string; mutationAttemptId?: string; expectedRecipeSha256?: string; + reseedOnReset?: boolean; +}; +type ParsedMutationArgs = MutationArgs & { + app: string; + url: string; + mutations: string; + level: string; + recipe: string; + maxRuntimeMinutes: number; + mutationAttemptId: string; +}; +type GradeReport = { total?: unknown; max?: unknown; + features?: Array<{ id?: unknown; score?: unknown; + criteria?: Array<{ id?: string; stableKey?: unknown; evidence?: unknown }> }>; + [key: string]: unknown }; +type MutationResult = ReturnType & { id: string; scenario: string; targets: string[] }; +type BaselineEntry = MutationCheckpointBaseline & { total: unknown; max: unknown }; +type MutationFile = { target: string; backup: string; original: string; edits: ReturnType }; + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +function jsonObject(value: unknown, label: string): JsonRecord { + if (value === null || typeof value !== 'object' || Array.isArray(value)) { + throw new Error(`${label} must be an object`); + } + return value as JsonRecord; +} + +const HERE = dirname(fileURLToPath(import.meta.url)); +const GRADER = join(HERE, "grade.js"); + +export function parseMutationArgs(argv: readonly string[]): ParsedMutationArgs { + const { values } = parseNodeArgs({ args: [...argv.slice(2)], options: { + app: { type: 'string' }, url: { type: 'string' }, mutations: { type: 'string' }, + level: { type: 'string' }, spec: { type: 'string' }, backend: { type: 'string' }, + track: { type: 'string' }, recipe: { type: 'string' }, + 'selected-check': { type: 'string', multiple: true }, 'db-name': { type: 'string' }, + 'run-index': { type: 'string' }, + 'restart-spec': { type: 'string' }, out: { type: 'string' }, 'parent-attempt-id': { type: 'string' }, + 'mutation-shard-index': { type: 'string' }, 'mutation-shard-count': { type: 'string' }, + 'resume-from': { type: 'string' }, 'checkpoint-out': { type: 'string' }, + 'baseline-bundle': { type: 'string' }, 'expected-calibration-json': { type: 'string' }, + 'max-runtime-minutes': { type: 'string' }, 'image-id': { type: 'string' }, + } }); + const a: MutationArgs = { app: values.app, url: values.url, mutations: values.mutations, + level: values.level, spec: values.spec, backend: values.backend, track: values.track, + recipe: values.recipe, selectedCheckKeys: values['selected-check'], dbName: values['db-name'], + runIndex: values['run-index'] ?? '0', + restartSpec: values['restart-spec'] === undefined ? undefined + : parseRuntimeControlSpec(JSON.parse(values['restart-spec'])), + out: values.out, parentAttemptId: values['parent-attempt-id'], + mutationShardIndex: values['mutation-shard-index'] === undefined + ? undefined : Number(values['mutation-shard-index']), + mutationShardCount: values['mutation-shard-count'] === undefined + ? undefined : Number(values['mutation-shard-count']), + resumeFrom: values['resume-from'] && resolve(values['resume-from']), + checkpointOut: values['checkpoint-out'] && resolve(values['checkpoint-out']), + baselineBundle: values['baseline-bundle'] && resolve(values['baseline-bundle']), + expectedCalibrationIdentity: values['expected-calibration-json'] === undefined + ? undefined : JSON.parse(values['expected-calibration-json']) as JsonRecord, + maxRuntimeMinutes: values['max-runtime-minutes'] === undefined + ? 60 : Number(values['max-runtime-minutes']), + imageId: values['image-id'] }; + if (!a.app || !a.url || !a.mutations || !a.level || !a.recipe) { + throw new Error( + "Usage: node dist/grader/mutation-test.js --app --url --mutations " + + "--level --recipe ", + ); + } + let url: URL; + try { url = new URL(a.url); } + catch { throw new Error('--url must be a valid HTTP or HTTPS URL'); } + if (url.protocol !== 'http:' && url.protocol !== 'https:') { + throw new Error('--url must use HTTP or HTTPS'); + } + if (!Number.isInteger(Number(a.level)) || Number(a.level) < 1) { + throw new Error('--level must be a positive integer'); + } + const shardFields = [a.mutationShardIndex, a.mutationShardCount] + .filter(value => value !== undefined); + if (shardFields.length === 1) { + throw new Error('--mutation-shard-index and --mutation-shard-count must be supplied together'); + } + if (a.mutationShardCount !== undefined + && (!Number.isInteger(a.mutationShardIndex) || !Number.isInteger(a.mutationShardCount) + || a.mutationShardIndex! < 0 || a.mutationShardCount < 1 + || a.mutationShardIndex! >= a.mutationShardCount)) { + throw new Error('--mutation-shard-index must be within the positive shard count'); + } + a.maxRuntimeMinutes ??= 60; + if (!Number.isFinite(a.maxRuntimeMinutes) || a.maxRuntimeMinutes < 1 + || a.maxRuntimeMinutes > 120) { + throw new Error('--max-runtime-minutes must be from 1 through 120'); + } + if (a.resumeFrom && !a.checkpointOut) a.checkpointOut = a.resumeFrom; + return { ...a, app: a.app, url: a.url, mutations: a.mutations, level: a.level, recipe: a.recipe, + maxRuntimeMinutes: a.maxRuntimeMinutes, + mutationAttemptId: `mutation-${new Date().toISOString().replace(/[:.]/g, "-")}` }; +} + +class MutationBatchDeadlineError extends Error {} + +export function remainingMutationBatchMs(deadlineMs: number, nowMs: number = Date.now()): number { + const remaining = Math.floor(deadlineMs - nowMs); + if (remaining <= 0) throw new MutationBatchDeadlineError('mutation batch deadline reached'); + return remaining; +} + +// Reset publishes SpacetimeDB source. Hosted stacks restart once to load source +// and seed the empty database. A separate source-change restart is redundant. +async function reset(a: MutationArgs, deadlineMs: number | null): Promise { + const exec: TextCommandExecutor = deadlineMs === null ? execFileSync : ((file, commandArgs, options) => + execFileSync(file, commandArgs, { ...options, + timeout: Math.min(options.timeout, remainingMutationBatchMs(deadlineMs)) })); + try { + resetBackend({ backend: a.backend!, app: a.app!, exec }); + const requiresReseed = STACK_ADAPTER_REGISTRY.get(a.backend!).reset.requiresReseed; + if (a.reseedOnReset && requiresReseed) { + if (!a.restartSpec) { + throw new Error(`track ${a.track} requires a lease-authenticated --restart-spec to reseed after reset`); + } + const signal = deadlineMs === null ? null + : AbortSignal.timeout(remainingMutationBatchMs(deadlineMs)); + await controlBackendRuntime(a.restartSpec, "restart", { signal, exec }); + } + } catch (error) { + if (deadlineMs !== null && Date.now() >= deadlineMs) { + throw new MutationBatchDeadlineError('mutation batch deadline reached', { cause: error }); + } + throw error; + } +} + +function rebuildClientAfterSourceChange(a: MutationArgs, deadlineMs: number): void { + const timeout = deadlineMs - Date.now(); + if (timeout <= 0) throw new MutationBatchDeadlineError(); + try { + execFileSync('npm', ['run', 'build'], { + cwd: join(a.app!, 'client'), stdio: 'pipe', timeout, + }); + } catch (cause) { + if (Date.now() >= deadlineMs) throw new MutationBatchDeadlineError(); + throw new Error('client build failed after source change', { cause }); + } +} + +async function grade(a: MutationArgs, reportPath: string, deadlineMs: number | null = null): Promise { + await reset(a, deadlineMs); + if (existsSync(reportPath)) unlinkSync(reportPath); + const gradeArgs: string[] = [ + GRADER, + "--url", + a.url!, + "--level", + a.level!, + "--out", + reportPath, + "--spec", + a.spec!, + "--backend", + a.backend!, + "--track", + a.track!, + "--app", + a.app!, + ]; + if (a.dbName) gradeArgs.push("--db-name", a.dbName); + if (a.restartSpec) gradeArgs.push("--restart-spec", JSON.stringify(a.restartSpec)); + if (a.mutationAttemptId) gradeArgs.push("--parent-attempt-id", a.mutationAttemptId); + if (a.recipe) gradeArgs.push("--recipe", a.recipe); + if (a.expectedRecipeSha256) { + gradeArgs.push("--expected-recipe-sha256", a.expectedRecipeSha256); + } + for (const stableKey of a.selectedCheckKeys ?? []) { + gradeArgs.push("--selected-check", stableKey); + } + const timeout = deadlineMs === null + ? MUTATION_GRADE_MAX_TIMEOUT_MS + : mutationGradeTimeoutMs(deadlineMs); + if (timeout === 0) throw new MutationBatchDeadlineError('mutation batch deadline reached'); + try { + execFileSync(process.execPath, gradeArgs, { + stdio: "pipe", + encoding: "utf8", + timeout, + }); + } catch (error) { + if (jsonObject(error, 'grader process error').code === 'ETIMEDOUT' && timeout < MUTATION_GRADE_MAX_TIMEOUT_MS) { + throw new MutationBatchDeadlineError('mutation grade reached the remaining batch deadline'); + } + throw error; + } + if (!existsSync(reportPath)) { + throw new Error("grader completed without producing its report"); + } + return readArtifactPayload(reportPath, { expectedKind: "grade" }); +} + +let args: ParsedMutationArgs; +let startedAt: number; +let startedIso: string; +const artifactPath = (id: string) => + resolve(args.out ?? join(HERE, "..", "results", `${id}.json`)); +let spec!: MutationSpec; + +function sha256(value: string | Buffer): string { + return createHash('sha256').update(value).digest('hex'); +} + +function scenarioKey(path: string): string { + return relative(resolve(HERE, '..'), path).replaceAll('\\', '/'); +} + +function checkpointGroup(path: string, mutations: MutationDefinition[], selectedCheckKeys: readonly string[]): + MutationCheckpointIdentity['groups'][number] { + const scenario = scenarioKey(path); + const scenarioSha256 = sha256(readFileSync(path)); + const mutationSha256 = sha256(JSON.stringify(mutations)); + const selectionSha256 = sha256(JSON.stringify([...selectedCheckKeys].sort())); + return { scenario, scenarioSha256, mutationSha256, selectionSha256, + identitySha256: sha256(JSON.stringify({ scenarioSha256, mutationSha256, selectionSha256 })), + mutationIds: mutations.map(mutation => mutation.id as string) }; +} + +function checkpointIdentity(groups: MutationCheckpointIdentity['groups'], shard: { index: number; count: number; + mutationIds: string[] }, track: ReturnType): MutationCheckpointIdentity { + return { + schemaVersion: 1, + engineSha256: currentEngineIdentity().sha256, + recipeSha256: args.expectedRecipeSha256, + fixtureSha256: spec.fixtureSha256, + calibrationSha256: args.expectedCalibrationIdentity?.sha256 ?? null, + imageId: args.imageId ?? null, + backend: args.backend, + track: args.track, + level: Number(args.level), + trackSha256: sha256(readFileSync(join(track.dir, TRACK_MANIFEST_FILE))), + shard: { index: shard.index, count: shard.count, mutationIds: shard.mutationIds }, + groups, + }; +} + +function resumableEvidence(path: string | undefined, identity: MutationCheckpointIdentity): + { results: MutationCheckpointResult[]; baselines: MutationCheckpointBaseline[] } { + if (!path || !existsSync(path)) return { results: [], baselines: [] }; + const prior = readArtifactPayload(path, { expectedKind: 'mutation_control' }); + const { results, baselines } = reusableMutationEvidence(prior, identity); + const shard = identity.shard as { mutationIds: string[] }; + console.log(`Resuming ${results.length}/${shard.mutationIds.length} completed mutations from ${path}`); + return { results, baselines }; +} + +function recordHarnessFailure(error: unknown): void { + const generatedAt = new Date().toISOString(); + const id = args.mutationAttemptId; + const artifact = { + id, + kind: "mutation_control", + startedAt: startedIso, + completedAt: generatedAt, + parentAttemptId: args.parentAttemptId ?? null, + identities: emptyArtifactIdentities({ + fixture: spec?.fixtureSha256 ? { id: "source-under-mutation", sha256: spec.fixtureSha256 } : null, + stackAdapter: (args.backend ?? spec?.backend) ? { id: args.backend ?? spec.backend } : null, + }), + durationMs: Date.now() - startedAt, + app: resolve(args.app), + mutations: resolve(args.mutations), + manifestStatus: spec?.status ?? null, + fixtureSha256: spec?.fixtureSha256 ?? null, + spec: args.spec ? resolve(args.spec) : null, + backend: args.backend ?? spec?.backend ?? null, + track: args.track ?? spec?.track ?? null, + ok: false, + outcome: { + kind: "harness_failure", + phase: "mutation-control", + reason: errorMessage(error), + }, + }; + try { + const outputPath = artifactPath(id); + writeRunJson(outputPath, artifact); + console.error( + `mutation harness failure: ${errorMessage(error)}\nartifact: ${outputPath}`, + ); + } catch (artifactError) { + console.error( + `mutation harness failure: ${errorMessage(error)}\nfailed to write failure artifact: ${errorMessage(artifactError)}`, + ); + } + process.exitCode = 2; +} + +async function main(): Promise { + spec = readMutationManifest(args.mutations!); + const fullMutations = spec.mutations; + const shard = args.mutationShardCount === undefined + ? { index: 0, count: 1, mutationIds: fullMutations.map(mutation => mutation.id as string), + mutations: fullMutations } + : mutationShard(fullMutations, + { index: args.mutationShardIndex!, count: args.mutationShardCount, + defaultScenario: spec.scenario }); + if (shard.mutations.length === 0) throw new Error('mutation shard has no assigned mutations'); + spec.mutations = shard.mutations; + if (args.backend && args.backend !== spec.backend) { + throw new Error( + `--backend conflicts with manifest backend ${spec.backend}`, + ); + } + if (args.track && args.track !== spec.track) { + throw new Error(`--track conflicts with manifest track ${spec.track}`); + } + args.backend = spec.backend; + args.track = spec.track; + const track = loadTrack(args.track); + const binding = resolveRecipeRelease(track, Number(args.level), args.recipe); + if (!binding) throw new Error(`${args.track} L${args.level} has no recipe release`); + args.recipe = `${binding.release.id}@${binding.release.version}`; + args.expectedRecipeSha256 = binding.release.contentSha256; + const recipeRelease = binding.release; + args.dbName ??= dbName(track, Number(args.runIndex)); + args.reseedOnReset = track.reseedOnReset; + const definitions = validateMutationDefinitions(spec.mutations, + { defaultScenario: spec.scenario, requireScenario: true }); + if (!definitions.ok) { + throw new Error( + `invalid mutation manifest: ${ + definitions.issues.map((issue) => + `${issue.mutation ?? ""}:${issue.kind}` + ).join(", ") + }`, + ); + } + const groups = new Map(); + for (const [scenario, mutations] of groupMutationsByScenario(spec)) { + const declaredSpec = resolveMutationScenarioPath(scenario); + groups.set(declaredSpec, mutations); + } + if (args.spec) { + const requested = resolve(args.spec); + if (groups.size !== 1 || !groups.has(requested)) { + throw new Error('--spec conflicts with the mutation manifest scenario selection'); + } + } + const work = mkdtempSync(join(tmpdir(), "stack-bench-mutation-")); + const reportPath = join(work, "grade.json"); + // Hosted apps serve this build; development servers compile client source on demand. + const clientDist = join(args.app, 'client', 'dist'); + const cleanClientDist = spec.mutations.some(mutation => + mutationFileEdits(mutation).some(edit => edit.file.replaceAll('\\', '/').startsWith('client/'))) + && existsSync(clientDist) ? join(work, 'client-dist') : null; + if (cleanClientDist) cpSync(clientDist, cleanClientDist, { recursive: true }); + process.once("exit", () => rmSync(work, { recursive: true, force: true })); + + // Reject backups left by an interrupted run before grading the baseline. + for (const m of spec.mutations) { + for (const file of new Set(mutationFileEdits(m).map(edit => edit.file))) { + const stale = resolveMutationFile(args.app, file) + ".mutation-backup"; + if (existsSync(stale)) { + throw new Error( + `${stale} exists; restore the interrupted mutation backup before running again`, + ); + } + } + } + + // Catch dirty source even when no backup file remains. + assertAppSourceIdentity(args.app, spec.fixtureSha256, 'mutation fixture'); + + // Reject missing or ambiguous edit anchors before baseline grading. + for (const m of spec.mutations) { + for (const edit of mutationFileEdits(m)) { + const source = readFileSync(resolveMutationFile(args.app, edit.file), "utf8"); + const matches = source.split(edit.find).length - 1; + if (matches !== 1) { + throw new Error( + `${m.id} anchor matched ${matches} times in ${edit.file}; expected exactly once`, + ); + } + } + } + + const plans = [...groups].map(([scenarioPath, mutations]) => { + const selectedCheckKeys = releaseScenarioCheckKeys(recipeRelease, track.dir, scenarioPath, + args.selectedCheckKeys ?? null); + return { scenarioPath, scenario: scenarioKey(scenarioPath), mutations, selectedCheckKeys, + checkpoint: checkpointGroup(scenarioPath, mutations, selectedCheckKeys) }; + }); + const cleanBaselineBundle = args.baselineBundle + ? readArtifactPayload(args.baselineBundle, { expectedKind: 'grade_bundle' }) + : null; + if (cleanBaselineBundle && !args.expectedCalibrationIdentity) { + throw new Error('a reusable clean baseline requires its expected calibration identity'); + } + const checkpoint = checkpointIdentity(plans.map(plan => plan.checkpoint), shard, track); + const resumed = resumableEvidence(args.resumeFrom, checkpoint); + const results: MutationResult[] = [...resumed.results] as MutationResult[]; + const baselines: BaselineEntry[] = [...resumed.baselines] as BaselineEntry[]; + const completedIds = new Set(results.map(result => result.id)); + if (completedIds.size !== results.length) { + throw new Error('mutation checkpoint contains duplicate results'); + } + const outputPath = artifactPath(args.mutationAttemptId); + const deadline = startedAt + args.maxRuntimeMinutes * 60_000; + + const createControlArtifact = (status: 'running' | 'incomplete' | 'complete', reason: string | null = null) => { + const ordered = [...results].sort((left, right) => + shard.mutationIds.indexOf(left.id) - shard.mutationIds.indexOf(right.id)); + const clean = ordered.filter(result => result.status === 'CAUGHT'); + const orderedBaselines = plans.map(plan => baselines.find(entry => + entry.scenario === plan.scenario)).filter((entry): entry is BaselineEntry => Boolean(entry)); + const remaining = shard.mutationIds.filter(id => !completedIds.has(id)); + return { + id: args.mutationAttemptId, + kind: 'mutation_control', + startedAt: startedIso, + completedAt: status === 'running' ? null : new Date().toISOString(), + parentAttemptId: args.parentAttemptId ?? null, + identities: emptyArtifactIdentities({ + fixture: { id: 'source-under-mutation', sha256: spec.fixtureSha256 }, + recipe: { id: recipeRelease.id, version: recipeRelease.version, + sha256: recipeRelease.contentSha256, state: recipeRelease.state }, + stackAdapter: { id: args.backend }, + }), + durationMs: Date.now() - startedAt, + app: resolve(args.app), + mutations: resolve(args.mutations), + manifestStatus: spec.status, + fixtureSha256: spec.fixtureSha256, + spec: plans.map(plan => plan.scenario), + backend: args.backend, + track: args.track, + shard: { index: shard.index, count: shard.count, mutationIds: shard.mutationIds }, + baseline: { + total: orderedBaselines.reduce((sum, entry) => sum + Number(entry.total), 0), + max: orderedBaselines.reduce((sum, entry) => sum + Number(entry.max), 0), + scenarios: orderedBaselines, + }, + ok: status === 'complete' && clean.length === ordered.length + && ordered.length === shard.mutationIds.length, + ...(status === 'complete' ? {} : { outcome: { kind: 'incomplete', + phase: 'mutation-control', reason: reason ?? 'mutation batch is in progress' } }), + summary: { caught: clean.length, completed: ordered.length, + total: shard.mutationIds.length, remaining: remaining.length }, + results: ordered, + checkpoint: { ...checkpoint, status, maxRuntimeMinutes: args.maxRuntimeMinutes, + updatedAt: new Date().toISOString() }, + }; + }; + const persist = (status: 'running' | 'incomplete' | 'complete', reason: string | null = null) => { + assertAppSourceIdentity(args.app, spec.fixtureSha256, + 'mutation fixture before checkpoint'); + const artifact = createControlArtifact(status, reason); + writeRunJson(outputPath, artifact); + if (args.checkpointOut && resolve(args.checkpointOut) !== outputPath) { + writeRunJson(args.checkpointOut, artifact); + } + return artifact; + }; + const stopAtBudget = () => { + const artifact = persist('incomplete', + `mutation batch reached its ${args.maxRuntimeMinutes} minute limit`); + console.log(`\n${artifact.summary.completed}/${artifact.summary.total} mutations completed; ` + + `${artifact.summary.remaining} remain`); + console.log(`checkpoint: ${args.checkpointOut ?? outputPath}`); + process.exitCode = 3; + }; + + for (const plan of plans) { + const { scenarioPath, scenario, mutations, selectedCheckKeys } = plan; + const pending = mutations.filter((mutation: MutationDefinition) => !completedIds.has(mutation.id as string)); + if (pending.length === 0) continue; + if (Date.now() >= deadline) return stopAtBudget(); + args.spec = scenarioPath; + args.selectedCheckKeys = selectedCheckKeys; + let baseline; + if (cleanBaselineBundle) { + const reused = reusableMutationBaseline(cleanBaselineBundle, { + backend: args.backend, + track: args.track, + level: Number(args.level), + fixtureSha256: spec.fixtureSha256, + recipe: { id: recipeRelease.id, version: recipeRelease.version, + sha256: recipeRelease.contentSha256 }, + identities: { + engine: currentEngineIdentity(), + calibration: args.expectedCalibrationIdentity, + stackAdapter: { id: args.backend }, + }, + selectedCheckKeys, + }); + if (!reused.ok) { + throw new Error(`cannot reuse clean baseline for ${scenarioPath}: ${reused.reason}`); + } + baseline = reused.report; + console.log(`Baseline (verified clean evidence, ${scenarioPath})...`); + } else { + console.log(`Baseline (unmutated app, ${scenarioPath})...`); + try { + baseline = await grade(args, reportPath, deadline); + } catch (error) { + if (error instanceof MutationBatchDeadlineError) return stopAtBudget(); + throw error; + } + const validation = validateMutationBaseline(baseline, mutations); + if (!validation.ok && isRetryableMutationBaseline(validation.issues)) { + console.log(' transient baseline failure; retrying once'); + try { + baseline = await grade(args, reportPath, deadline); + } catch (error) { + if (error instanceof MutationBatchDeadlineError) return stopAtBudget(); + throw error; + } + } + } + const baselineValidation = validateMutationBaseline(baseline, mutations); + if (!baselineValidation.ok) { + throw new Error( + `reference baseline is not known-good for ${scenarioPath}: ${ + JSON.stringify(baselineValidation.issues) + }`, + ); + } + console.log( + ` baseline: ${baseline.total}/${baseline.max} ${ + (baseline.features ?? []).map((f) => `F${f.id}:${(f as NonNullable[number]).score}`).join(" ") + }\n`, + ); + const baselineEntry = { scenario, identitySha256: plan.checkpoint.identitySha256, + total: baseline.total, max: baseline.max }; + const priorBaseline = baselines.findIndex(entry => entry.scenario === scenario); + if (priorBaseline === -1) baselines.push(baselineEntry); + else baselines[priorBaseline] = baselineEntry; + + for (const m of pending) { + if (Date.now() >= deadline) return stopAtBudget(); + const byFile = new Map(); + for (const edit of mutationFileEdits(m)) { + const target = resolveMutationFile(args.app, edit.file); + if (!byFile.has(target)) { + byFile.set(target, { + target, + backup: `${target}.mutation-backup`, + original: readFileSync(target, "utf8"), + edits: [], + }); + } + byFile.get(target)!.edits.push(edit); + } + const files = [...byFile.values()]; + const clientChanged = files.some(file => relative(args.app!, file.target) + .split(sep)[0] === 'client'); + const backedUp: MutationFile[] = []; + let r: GradeReport | undefined; + let classified: ReturnType | undefined; + let deadlineReached = false; + let mutationError: unknown = null; + try { + for (const file of files) { + copyFileSync(file.target, file.backup); + backedUp.push(file); + } + for (const file of files) { + writeFileSync(file.target, + file.edits.reduce((src, edit) => src.replace(edit.find, edit.replace), + file.original)); + } + if (clientChanged) rebuildClientAfterSourceChange(args, deadline); + r = await grade(args, reportPath, deadline); + classified = classifyMutationResult( + baseline as Parameters[0], + r as Parameters[1], + m, + ); + if (isRetryableMutationResult(classified.status)) { + console.log(` ${classified.status} result; retrying once`); + r = await grade(args, reportPath, deadline); + classified = classifyMutationResult( + baseline as Parameters[0], + r as Parameters[1], + m, + ); + } + } catch (error) { + if (error instanceof MutationBatchDeadlineError) deadlineReached = true; + else mutationError = error; + } + + const cleanupErrors: Error[] = []; + for (const file of backedUp) { + try { + copyFileSync(file.backup, file.target); + unlinkSync(file.backup); + } catch (error) { + cleanupErrors.push(new Error(`cannot restore ${file.target}: ${errorMessage(error)}`, + { cause: error })); + } + } + for (const file of files) { + try { + if (existsSync(file.backup) || readFileSync(file.target, 'utf8') !== file.original) { + cleanupErrors.push(new Error(`restore verification failed for ${file.target}`)); + } + } catch (error) { + cleanupErrors.push(new Error(`cannot verify restored source ${file.target}: ${errorMessage(error)}`, + { cause: error })); + } + } + if (clientChanged) { + try { + rmSync(clientDist, { recursive: true, force: true }); + if (cleanClientDist) cpSync(cleanClientDist, clientDist, { recursive: true }); + } catch (error) { + cleanupErrors.push(new Error(`cannot restore the built client: ${errorMessage(error)}`, + { cause: error })); + } + } + // A normal error leaves enough budget to restore the clean runtime. A + // deadline stop leaves clean source and lets the lease owner stop it. + if (mutationError !== null && cleanupErrors.length === 0) { + try { + await reset(args, deadline); + } catch (error) { + cleanupErrors.push(new Error(`cannot restore the clean runtime: ${errorMessage(error)}`, + { cause: error })); + } + } + if (cleanupErrors.length > 0) { + const errors = mutationError === null ? cleanupErrors : [mutationError, ...cleanupErrors]; + throw new AggregateError(errors, + 'mutation cleanup failed; do not reuse this app source'); + } + if (mutationError !== null) throw mutationError; + if (deadlineReached) return stopAtBudget(); + if (!r || !classified) throw new Error('mutation grade completed without a result'); + results.push({ id: m.id, scenario, + targets: mutationTargetKeys(m), ...classified }); + completedIds.add(m.id); + persist('running'); + console.log( + `${classified.status.padEnd(20)} ${m.id} — expected ${ + classified.targetKeys.join(", ") + }`, + ); + if (classified.regressions.length) { + console.log( + ` failed criteria: ${ + classified.regressions.map((item) => item.key).join(", ") + }`, + ); + } + } + } + + // Detect any source change outside the files restored above. + assertAppSourceIdentity(args.app, spec.fixtureSha256, 'mutation fixture after worker completion'); + // Restore the clean runtime and database before releasing the worker lease. + await reset(args, null); + + rmSync(work, { recursive: true, force: true }); + const artifact = persist('complete'); + console.log(`\n${artifact.summary.caught}/${artifact.summary.total} mutations cleanly caught`); + console.log(`artifact: ${outputPath}`); + if (!artifact.ok) process.exitCode = 1; +} + +function run(): void { + try { + args = parseMutationArgs(process.argv); + } catch (error) { + console.error(errorMessage(error)); + process.exitCode = 2; + return; + } + startedAt = Date.now(); + startedIso = new Date(startedAt).toISOString(); + main().catch(recordHarnessFailure); +} + +if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) run(); diff --git a/tools/stack-bench/grader/mutations/mongodb-ecommerce-2.0.1.json b/tools/stack-bench/grader/mutations/mongodb-ecommerce-2.0.1.json new file mode 100644 index 00000000000..4fffb722109 --- /dev/null +++ b/tools/stack-bench/grader/mutations/mongodb-ecommerce-2.0.1.json @@ -0,0 +1,1856 @@ +{ + "schemaVersion": 2, + "status": "candidate", + "fixtureSha256": "edb9732535b2273232a320b8e7b4ad6758991683a6bd06103f2ec4be07dafe20", + "backend": "mongodb", + "track": "ecommerce", + "note": "Mutation definitions for the MongoDB ecommerce reference.", + "mutations": [ + { + "id": "signup-does-not-expose-created-account", + "scenario": "tracks/ecommerce/scenarios/01-account-create-2.4.0.json", + "targets": [ + "ecommerce.feature.accounts.accounts.1a" + ], + "desc": "Signup succeeds but the client discards the created account identity from its current session view.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " saveSession(data.token, data.user);\n };\n\n const handleSignIn", + "replace": " saveSession(data.token, { ...data.user, username: \"\" });\n };\n\n const handleSignIn" + } + ] + }, + { + "id": "duplicate-signup-reports-success", + "scenario": "tracks/ecommerce/scenarios/01-account-duplicate-2.4.0.json", + "targets": [ + "ecommerce.feature.accounts.accounts.1b" + ], + "desc": "A duplicate username is reported as a successful empty signup response instead of a refusal.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " if (existing) return res.status(409).json({ error: \"Username is already taken\" });", + "replace": " if (existing) return res.json({}); // mutant: duplicate signup is falsely accepted" + } + ] + }, + { + "id": "signin-skips-password-verification", + "scenario": "tracks/ecommerce/scenarios/01-account-password-2.4.0.json", + "targets": [ + "ecommerce.feature.accounts.accounts.1c" + ], + "desc": "Signin accepts an existing account without requiring its password to match.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " if (!valid) return res.status(401).json({ error: \"Invalid username or password\" });", + "replace": " if (false && !valid) return res.status(401).json({ error: \"Invalid username or password\" });" + } + ] + }, + { + "id": "signout-keeps-current-account", + "scenario": "tracks/ecommerce/scenarios/01-account-signout-2.4.0.json", + "targets": [ + "ecommerce.feature.accounts.accounts.1d" + ], + "desc": "Signout disconnects the token state but leaves the current account and persisted credential in place.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " const handleSignOut = () => {\n clearSession();\n };", + "replace": " const handleSignOut = () => {\n setToken(null); // mutant: visible and persisted account state is not cleared\n };" + } + ] + }, + { + "id": "session-token-not-persisted", + "scenario": "tracks/ecommerce/scenarios/01-account-reload-2.4.0.json", + "targets": [ + "ecommerce.spec.state-durability.session-reload.1e" + ], + "desc": "The active session is kept only in React state and is unavailable after a page reload.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " localStorage.setItem(TOKEN_KEY, tok);\n setToken(tok);", + "replace": " void tok; // mutant: the session token is never persisted\n setToken(tok);" + } + ] + }, + { + "id": "purchase-counts-never-affect-ranking", + "scenario": "tracks/ecommerce/scenarios/01-core-2.4.0.json", + "targets": [ + "ecommerce.spec.live-state.ranking.2c" + ], + "desc": "The catalogue ranking ignores recorded purchases and therefore never promotes the bought item.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " purchaseCount: purchaseMap.get(id) || 0,", + "replace": " purchaseCount: 0, // mutant: ranking ignores durable purchase counts" + } + ] + }, + { + "id": "signed-out-visitors-see-purchase-controls", + "scenario": "tracks/ecommerce/scenarios/progression-signed-out-purchase-1.0.0.json", + "targets": [ + "ecommerce.spec.access-control.signed-out-purchase.3a" + ], + "desc": "The client treats every non-admin state, including a signed-out visitor, as a customer allowed to see purchase controls.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " const isCustomer = !!currentUser && !currentUser.isAdmin && !currentUser.isStaff;", + "replace": " const isCustomer = !currentUser?.isAdmin && !currentUser?.isStaff;" + } + ] + }, + { + "id": "espresso-stock-row-ignores-live-updates", + "scenario": "tracks/ecommerce/scenarios/01-buying-2.4.0.json", + "targets": [ + "ecommerce.spec.live-state.purchase-stock.3b" + ], + "desc": "The live catalogue handler preserves a stale Espresso Machine stock projection while applying all other item updates.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " socket.on(\"items:update\", (data: ItemT[]) => setItems(data));", + "replace": " socket.on(\"items:update\", (data: ItemT[]) => setItems((previous) => data.map((item) => item.name === \"Espresso Machine\" ? { ...item, stock: previous.find((old) => old.id === item.id)?.stock ?? item.stock } : item)));" + } + ] + }, + { + "id": "purchase-order-uses-zero-price", + "scenario": "tracks/ecommerce/scenarios/progression-purchasing-1.0.0.json", + "targets": [ + "ecommerce.feature.purchasing.purchase-order.3c" + ], + "desc": "A direct purchase records the item but stores a zero order total instead of the price paid.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " total: item.price,\n });", + "replace": " total: 0, // mutant: purchase receipt loses the authoritative price\n });" + } + ] + }, + { + "id": "reload-hydrates-an-empty-cart", + "scenario": "tracks/ecommerce/scenarios/01-cart-2.4.0.json", + "targets": [ + "ecommerce.spec.state-durability.cart-reload.4b" + ], + "desc": "Cart hydration discards the persisted server response after reload.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " const refreshCart = useCallback(async (tok: string) => {\n const data = await apiFetch(\"/api/cart\", tok);\n setCart(data);\n }, []);", + "replace": " const refreshCart = useCallback(async (tok: string) => {\n await apiFetch(\"/api/cart\", tok);\n setCart({ items: [], total: 0 }); // mutant: persisted cart response is discarded\n }, []);" + } + ] + }, + { + "id": "shared-cart-live-events-ignored", + "scenario": "tracks/ecommerce/scenarios/01-cart-2.4.0.json", + "targets": [ + "ecommerce.spec.live-state.shared-cart.4c" + ], + "desc": "An already-open second session ignores committed cart update events.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " socket.on(\"cart:update\", (data: CartT) => setCart(data));", + "replace": " socket.on(\"cart:update\", (data: CartT) => setCart(current => current.items.length === 0 ? current : data)); // mutant: an empty second-session cart ignores its first remote update" + } + ] + }, + { + "id": "review-comment-is-not-persisted", + "scenario": "tracks/ecommerce/scenarios/01-review-visibility-2.4.0.json", + "targets": [ + "ecommerce.feature.reviews.reviews.6a" + ], + "desc": "Review submission persists an empty comment rather than the customer's submitted text.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " { itemId, userId: user._id, username: user.username, rating: ratingNum,\n comment: typeof comment === \"string\" ? comment : \"\" },", + "replace": " { itemId, userId: user._id, username: user.username, rating: ratingNum,\n comment: \"\" }," + } + ] + }, + { + "id": "repeat-review-uses-a-new-owner-key", + "scenario": "tracks/ecommerce/scenarios/01-review-uniqueness-2.4.0.json", + "targets": [ + "ecommerce.spec.transactional-integrity.unique-review.6b" + ], + "desc": "Each review submission is stored under a fresh owner key, bypassing the one-review-per-customer constraint.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " { itemId, userId: user._id },\n { itemId, userId: user._id, username: user.username, rating: ratingNum,\n comment: typeof comment === \"string\" ? comment : \"\" },", + "replace": " { itemId, userId: new Types.ObjectId() },\n { itemId, userId: new Types.ObjectId(), username: user.username, rating: ratingNum,\n comment: typeof comment === \"string\" ? comment : \"\" }," + } + ] + }, + { + "id": "live-review-average-uses-an-extra-divisor", + "scenario": "tracks/ecommerce/scenarios/01-review-rating-live-2.4.0.json", + "targets": [ + "ecommerce.spec.live-state.rating.6c" + ], + "desc": "The live review event divides the rating sum by one more review than actually exists.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "async function broadcastReviews(itemId: string) {\n const reviews = await Review.find({ itemId }).sort({ createdAt: -1 });\n const average = reviews.length ? reviews.reduce((s, r) => s + r.rating, 0) / reviews.length : 0;", + "replace": "async function broadcastReviews(itemId: string) {\n const reviews = await Review.find({ itemId }).sort({ createdAt: -1 });\n const average = reviews.length ? reviews.reduce((s, r) => s + r.rating, 0) / (reviews.length + 1) : 0;" + } + ] + }, + { + "id": "warehouse-view-omits-one-location", + "scenario": "tracks/ecommerce/scenarios/01-warehouse-admin-staff-1.0.0.json", + "targets": [ + "ecommerce.feature.warehouse-admin.warehouse-view.7b" + ], + "desc": "The admin warehouse projection truncates the final item-location row.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " {overview.locations.map((loc) => (", + "replace": " {overview.locations.slice(0, -1).map((loc) => (" + } + ] + }, + { + "id": "unauthenticated-purchase-defaults-to-admin", + "scenario": "tracks/ecommerce/scenarios/01-purchase-session-2.4.0.json", + "targets": [ + "ecommerce.spec.access-control.purchase-session.101a" + ], + "desc": "The purchase endpoint drops authentication and assigns sessionless purchases to the seeded administrator.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "app.post(\"/api/items/:id/buy\", requireAuth, async (req, res) => {", + "replace": "app.post(\"/api/items/:id/buy\", async (req, res) => {" + }, + { + "find": " const user = (req as any).user;\n const order = await Order.create({", + "replace": " const user = (req as any).user || await User.findOne({ username: \"admin\" });\n const order = await Order.create({" + } + ] + }, + { + "id": "direct-purchase-total-ignores-store-price", + "scenario": "tracks/ecommerce/scenarios/01-server-price-2.4.0.json", + "targets": [ + "ecommerce.spec.transactional-integrity.server-price.104a" + ], + "desc": "The direct purchase creates one order but records a zero total rather than the store's current price.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " total: item.price,\n });", + "replace": " total: 0, // mutant: direct purchase ignores the authoritative price\n });" + } + ] + }, + { + "id": "cart-hydration-loses-account-state", + "scenario": "tracks/ecommerce/scenarios/progression-account-state-reload-1.0.1.json", + "targets": [ + "ecommerce.spec.state-durability.account-state-recovery.105a" + ], + "desc": "Reload hydration discards the account's persisted cart response.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " const refreshCart = useCallback(async (tok: string) => {\n const data = await apiFetch(\"/api/cart\", tok);\n setCart(data);\n }, []);", + "replace": " const refreshCart = useCallback(async (tok: string) => {\n await apiFetch(\"/api/cart\", tok);\n setCart({ items: [], total: 0 }); // mutant: account state is discarded on hydration\n }, []);" + } + ] + }, + { + "id": "reconnect-hydration-loses-account-state", + "scenario": "tracks/ecommerce/scenarios/progression-account-state-reconnect-1.0.0.json", + "targets": [ + "ecommerce.spec.state-durability.account-state-recovery.105b" + ], + "desc": "The initial account cart loads correctly, but after network restoration the client ignores both refreshed and pushed cart state.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " useEffect(() => {\n const socket = io({ auth: token ? { token } : {} });", + "replace": " useEffect(() => {\n const clearAccountOffline = () => {\n setCurrentUser(null);\n setCart({ items: [], total: 0 });\n };\n window.addEventListener(\"offline\", clearAccountOffline, { once: true });\n const socket = io({ auth: token ? { token } : {} });" + } + ] + }, + { + "id": "order-history-is-not-owner-scoped", + "scenario": "tracks/ecommerce/scenarios/01-order-ownership-2.4.0.json", + "targets": [ + "ecommerce.spec.access-control.order-ownership.106a" + ], + "desc": "Order history returns every customer's orders instead of filtering by the authenticated owner.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " const orders = await Order.find({ userId }).sort({ createdAt: -1 });", + "replace": " const orders = await Order.find({}).sort({ createdAt: -1 });" + } + ] + }, + { + "id": "revenue-aggregation-ignores-order-totals", + "scenario": "tracks/ecommerce/scenarios/progression-books-balance-1.0.0.json", + "targets": [ + "ecommerce.spec.transactional-integrity.books-balance.107a" + ], + "desc": "The admin revenue aggregation counts every order as zero regardless of its stored total.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " { $group: { _id: null, total: { $sum: { $multiply: [\"$items.price\", \"$items.quantity\"] } } } },", + "replace": " { $group: { _id: null, total: { $sum: 0 } } }," + } + ] + }, + { + "id": "unpurchased-review-is-accepted", + "scenario": "tracks/ecommerce/scenarios/01-review-eligibility-2.4.0.json", + "targets": [ + "ecommerce.spec.access-control.review-eligibility.108a" + ], + "desc": "The review endpoint bypasses its completed-purchase eligibility check.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " if (!hasPurchased) {\n return res.status(403).json({ error: \"You can only review items you have purchased\" });\n }", + "replace": " if (false && !hasPurchased) {\n return res.status(403).json({ error: \"You can only review items you have purchased\" });\n }" + } + ] + }, + { + "id": "purchased-review-eligibility-query-never-matches", + "scenario": "tracks/ecommerce/scenarios/01-review-eligibility-2.4.0.json", + "targets": [ + "ecommerce.spec.access-control.review-eligibility.108b" + ], + "desc": "The eligibility query requires an impossible negative order total, rejecting even a customer who bought the item.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " const hasPurchased = await Order.exists({ userId: user._id, \"items.itemId\": itemId });", + "replace": " const hasPurchased = await Order.exists({ userId: user._id, \"items.itemId\": itemId, total: -1 });" + } + ] + }, + { + "id": "external-stock-polling-disabled", + "scenario": "tracks/ecommerce/scenarios/01-external-live-sync-1.1.0.json", + "targets": [ + "ecommerce.spec.external-data-sync.external-stock.901a" + ], + "desc": "The server stops reconciling direct database stock writes into live catalogue events.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " broadcastItems().catch((err) => console.error(\"broadcastItems poll failed\", err));", + "replace": " // mutant: direct database stock changes are never reconciled" + } + ] + }, + { + "id": "server-restart-disables-catalog-recovery", + "scenario": "tracks/ecommerce/scenarios/01-external-server-restart-sync-1.1.0.json", + "targets": [ + "ecommerce.spec.external-data-sync.external-stock.901c" + ], + "desc": "After a socket disconnect, the existing page ignores both reconnect refreshes and later catalogue snapshots.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " const socketRef = useRef(null);\n\n const saveSession", + "replace": " const socketRef = useRef(null);\n const acceptCatalogRecovery = useRef(true);\n\n const saveSession" + }, + { + "find": " socket.on(\"connect\", () => {\n refreshItems().catch((err) => console.error(err));", + "replace": " socket.on(\"disconnect\", () => { acceptCatalogRecovery.current = false; });\n socket.on(\"connect\", () => {\n if (acceptCatalogRecovery.current) refreshItems().catch((err) => console.error(err));" + }, + { + "find": " socket.on(\"items:update\", (data: ItemT[]) => setItems(data));", + "replace": " socket.on(\"items:update\", (data: ItemT[]) => { if (acceptCatalogRecovery.current) setItems(data); });" + } + ] + }, + { + "id": "reconnect-generation-ignores-current-catalog", + "scenario": "tracks/ecommerce/scenarios/01-external-reconnect-sync-1.1.0.json", + "targets": [ + "ecommerce.spec.external-data-sync.external-stock.901d" + ], + "desc": "After the browser goes offline, the existing page ignores reconnect refreshes and subsequent catalogue events.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " const socketRef = useRef(null);\n\n const saveSession", + "replace": " const socketRef = useRef(null);\n const acceptCatalogUpdates = useRef(true);\n useEffect(() => {\n const stopCatalogRecovery = () => { acceptCatalogUpdates.current = false; };\n window.addEventListener(\"offline\", stopCatalogRecovery);\n return () => window.removeEventListener(\"offline\", stopCatalogRecovery);\n }, []);\n\n const saveSession" + }, + { + "find": " setItems(data.items);", + "replace": " if (acceptCatalogUpdates.current) setItems(data.items);" + }, + { + "find": " socket.on(\"items:update\", (data: ItemT[]) => setItems(data));", + "replace": " socket.on(\"items:update\", (data: ItemT[]) => {\n if (acceptCatalogUpdates.current) setItems(data);\n });" + } + ] + }, + { + "id": "open-review-list-ignores-live-update", + "scenario": "tracks/ecommerce/scenarios/progression-open-list-live-1.0.0.json", + "targets": [ + "ecommerce.spec.live-state.open-list.902a" + ], + "desc": "The already-open review list ignores a committed review update from another client.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " setItemDetail((prev) => (prev && prev.id === payload.itemId ? { ...prev, reviews: payload.reviews, average: payload.average } : prev));", + "replace": " void payload; // mutant: the already-open review list ignores committed updates" + } + ] + }, + { + "id": "cancel-does-not-restore-stock-feature", + "scenario": "tracks/ecommerce/scenarios/02-order-cancellation-core-1.0.0.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3a" + ], + "desc": "Cancellation changes order state but skips restoration of its recorded warehouse allocations.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " for (const line of order.items) {\n await restoreAllocations(line.allocations as any, line.itemId);\n }\n order.status = \"cancelled\";", + "replace": " for (const line of order.items) {\n void line; // mutant: cancellation does not restore its reserved stock\n }\n order.status = \"cancelled\";" + } + ] + }, + { + "id": "cancel-does-not-restore-stock-fresh-client", + "scenario": "tracks/ecommerce/scenarios/02-self-contained-1.4.0.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202b", + "ecommerce.inventory-operations.stock-conservation.202c" + ], + "desc": "Cancellation changes order state but skips restoration, so a fresh client reads the persisted shortfall.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " for (const line of order.items) {\n await restoreAllocations(line.allocations as any, line.itemId);\n }\n order.status = \"cancelled\";", + "replace": " for (const line of order.items) {\n void line; // mutant: cancellation does not restore its reserved stock\n }\n order.status = \"cancelled\";" + } + ] + }, + { + "id": "cancel-restores-stock-but-keeps-pending-status", + "scenario": "tracks/ecommerce/scenarios/02-order-cancellation-history-1.0.0.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3b" + ], + "desc": "Cancellation restores allocations but writes pending back to order history.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " order.status = \"cancelled\";\n await order.save();", + "replace": " order.status = \"pending\";\n await order.save();" + } + ] + }, + { + "id": "cancelled-order-remains-in-revenue-feature", + "scenario": "tracks/ecommerce/scenarios/02-order-cancellation-core-1.0.0.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3a" + ], + "desc": "Admin revenue includes cancelled orders even though cancellation otherwise succeeds.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "async function getRevenue(): Promise {\n const rows = await Order.aggregate([\n { $match: { status: { $ne: \"cancelled\" } } },", + "replace": "async function getRevenue(): Promise {\n const rows = await Order.aggregate([\n { $match: {} }," + } + ] + }, + { + "id": "cancelled-order-remains-in-revenue-invariant", + "scenario": "tracks/ecommerce/scenarios/02-invariants.json", + "targets": [ + "ecommerce.returns-pricing.refund-accounting.203a" + ], + "desc": "Admin revenue includes cancelled orders even though cancellation otherwise succeeds.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "async function getRevenue(): Promise {\n const rows = await Order.aggregate([\n { $match: { status: { $ne: \"cancelled\" } } },", + "replace": "async function getRevenue(): Promise {\n const rows = await Order.aggregate([\n { $match: {} }," + } + ] + }, + { + "id": "operator-authorization-allows-customer-transfer", + "scenario": "tracks/ecommerce/scenarios/02-strengthened-1.4.0.json", + "targets": [ + "ecommerce.operations-access.operator-authorization.201a" + ], + "desc": "The transfer route keeps authentication but drops its administrator role gate.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "app.post(\"/api/admin/transfer\", requireAuth, requireAdmin, async (req, res) => {", + "replace": "app.post(\"/api/admin/transfer\", requireAuth, async (req, res) => {" + } + ] + }, + { + "id": "customer-can-ship-order-direct-1-1", + "scenario": "tracks/ecommerce/scenarios/02-server-actions-1.1.0.json", + "targets": [ + "ecommerce.operations-access.operator-authorization.201c" + ], + "desc": "The shipping route keeps authentication but drops its staff role gate.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "app.post(\"/api/fulfilment/ship\", requireAuth, requireStaff, async (req, res) => {", + "replace": "app.post(\"/api/fulfilment/ship\", requireAuth, async (req, res) => {" + } + ] + }, + { + "id": "customer-can-cancel-foreign-order-1-1", + "scenario": "tracks/ecommerce/scenarios/02-server-actions-1.1.0.json", + "targets": [ + "ecommerce.operations-access.order-owner.204a" + ], + "desc": "Cancellation retains authentication and pending-state validation but drops order ownership.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "app.post(\"/api/orders/:id/cancel\", requireAuth, async (req, res) => {\n const user = (req as any).user;\n const orderId = objectId(req.params.id);\n const order = orderId ? await Order.findOne({ _id: orderId, userId: user._id }) : null;", + "replace": "app.post(\"/api/orders/:id/cancel\", requireAuth, async (req, res) => {\n const user = (req as any).user;\n const orderId = objectId(req.params.id);\n const order = orderId ? await Order.findOne({ _id: orderId }) : null;" + } + ] + }, + { + "id": "queue-depth-lags-one-order", + "scenario": "tracks/ecommerce/scenarios/02-fulfilment-live-1.0.0.json", + "targets": [ + "ecommerce.operations-access.fulfilment-queue.1a" + ], + "desc": "The queue renders every order but its visible depth remains one behind.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": "Orders waiting: {queue.depth}", + "replace": "Orders waiting: {Math.max(0, queue.depth - 1)}" + } + ] + }, + { + "id": "ship-acknowledges-without-changing-status", + "scenario": "tracks/ecommerce/scenarios/02-fulfilment-ship-1.0.0.json", + "targets": [ + "ecommerce.operations-access.fulfilment-queue.1c" + ], + "desc": "Shipping returns success but writes pending back to the order, leaving both live views unchanged.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " order.status = \"shipped\";\n await order.save();", + "replace": " order.status = \"pending\";\n await order.save();" + } + ] + }, + { + "id": "customer-sees-fulfilment-navigation", + "scenario": "tracks/ecommerce/scenarios/02-fulfilment-access-1.0.0.json", + "targets": [ + "ecommerce.operations-access.fulfilment-queue.1d" + ], + "desc": "The fulfilment navigation is shown to every signed-in account instead of only staff and administrators.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " {(currentUser?.isStaff || currentUser?.isAdmin) && (\n ", + "replace": "onClick={() => setSearchPage(page => page)}>Next" + } + ] + }, + { + "id": "managed-support-leaks-and-accepts-cross-account-replies", + "scenario": "tracks/ecommerce/scenarios/progression-managed-support-privacy-1.0.0.json", + "targets": [ + "ecommerce.progression.managed-support.managed-support-privacy.613b" + ], + "desc": "Managed support tickets are visible across accounts and replayed replies are accepted.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": ".filter(row => isTicketCreator(sender, row.creatorIdentity.toHexString()) ||\n !!actor && (actor.isAdmin || actor.isStaff || row.accountId === accountId))", + "replace": ".filter(() => true)" + }, + { + "file": "backend/spacetimedb/src/index.ts", + "find": "if (!actor.isAdmin && !actor.isStaff && ticket.accountId !== actor.id) {\n throw new SenderError('That support ticket is private.');\n }", + "replace": "// mutant: any signed-in account can access any support ticket" + } + ] + }, + { + "id": "managed-support-replies-are-empty", + "scenario": "tracks/ecommerce/scenarios/progression-managed-support-shared-1.0.0.json", + "targets": [ + "ecommerce.progression.managed-support.managed-support-shared.613a" + ], + "desc": "Managed support stores replies without their message body.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "body: body.trim(),\n createdMicros: nowMicros(ctx),", + "replace": "body: '',\n createdMicros: nowMicros(ctx)," + } + ] + }, + { + "id": "notification-preferences-are-not-saved", + "scenario": "tracks/ecommerce/scenarios/progression-notification-preferences-1.0.0.json", + "targets": [ + "ecommerce.progression.notification-preferences.notification-preferences-persistence.630a" + ], + "desc": "Saving notification preferences discards the selected values.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "if (existing) ctx.db.notificationPreference.accountId.update(row);\n else ctx.db.notificationPreference.insert(row);", + "replace": "void existing;\n void row; // mutant: notification preferences are discarded" + } + ] + }, + { + "id": "notification-preferences-leak-across-accounts", + "scenario": "tracks/ecommerce/scenarios/progression-notification-preferences-1.0.0.json", + "targets": [ + "ecommerce.progression.notification-preferences.notification-preferences-privacy.630b" + ], + "desc": "The preference view returns another account's first stored choice.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "const row = ctx.db.notificationPreference.accountId.find(accountId);\n return row ? { orderEnabled: row.orderEnabled, stockEnabled: row.stockEnabled } : undefined;", + "replace": "const row = [...ctx.db.notificationPreference.iter()][0];\n return row ? { orderEnabled: row.orderEnabled, stockEnabled: row.stockEnabled } : undefined;" + } + ] + }, + { + "id": "checkout-records-zero-payment", + "scenario": "tracks/ecommerce/scenarios/progression-core-business-1.0.0.json", + "targets": [ + "ecommerce.progression.payment-records.payment-records.623a" + ], + "desc": "Checkout records a paid payment with a zero amount.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "ctx.db.paymentRecord.insert({ id: 0n, orderId: order.id, amount: order.total, status: 'paid' });\n if (promo) {", + "replace": "ctx.db.paymentRecord.insert({ id: 0n, orderId: order.id, amount: 0, status: 'paid' });\n if (promo) {" + } + ] + }, + { + "id": "checkout-records-duplicate-payments", + "scenario": "tracks/ecommerce/scenarios/progression-core-business-1.0.0.json", + "targets": [ + "ecommerce.progression.payment-records.payment-records.623b" + ], + "desc": "Checkout inserts two payment records for one order.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "ctx.db.paymentRecord.insert({ id: 0n, orderId: order.id, amount: order.total, status: 'paid' });\n if (promo) {", + "replace": "ctx.db.paymentRecord.insert({ id: 0n, orderId: order.id, amount: order.total, status: 'paid' });\n ctx.db.paymentRecord.insert({ id: 0n, orderId: order.id, amount: order.total + 0.01, status: 'paid' });\n if (promo) {" + } + ] + }, + { + "id": "active-promotion-does-not-discount-checkout", + "scenario": "tracks/ecommerce/scenarios/progression-promotion-checkout-1.0.0.json", + "targets": [ + "ecommerce.progression.promotion-checkout.promotion-checkout-active.621a" + ], + "desc": "Checkout ignores an active promotion when it calculates the discount.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "const discount = promo ? total * (promo.discountPercent / 100) : 0;", + "replace": "const discount = 0;" + } + ] + }, + { + "id": "expired-promotion-is-accepted", + "scenario": "tracks/ecommerce/scenarios/progression-promotion-checkout-1.0.0.json", + "targets": [ + "ecommerce.progression.promotion-checkout.promotion-checkout-expired.621b" + ], + "desc": "Promotion application does not reject a promotion after its end time.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "if (!promo || promo.startMicros > now || promo.endMicros < now || promo.redemptions >= promo.usageLimit) {", + "replace": "if (!promo || promo.startMicros > now || promo.redemptions >= promo.usageLimit) {" + } + ] + }, + { + "id": "exhausted-promotion-is-accepted", + "scenario": "tracks/ecommerce/scenarios/progression-promotion-checkout-1.0.0.json", + "targets": [ + "ecommerce.progression.promotion-checkout.promotion-checkout-exhausted.621c" + ], + "desc": "Promotion application does not reject a promotion at its usage limit.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "if (!promo || promo.startMicros > now || promo.endMicros < now || promo.redemptions >= promo.usageLimit) {", + "replace": "if (!promo || promo.startMicros > now || promo.endMicros < now) {" + } + ] + }, + { + "id": "customers-can-create-promotions", + "scenario": "tracks/ecommerce/scenarios/progression-promotion-rules-1.0.0.json", + "targets": [ + "ecommerce.progression.promotion-rules.promotion-rule-access.620b" + ], + "desc": "Promotion creation does not require staff access.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " (ctx, input) => {\n requireStaffOrAdmin(ctx);\n if (input.discountPercent <= 0 || input.discountPercent > 100) {", + "replace": " (ctx, input) => {\n if (input.discountPercent <= 0 || input.discountPercent > 100) {" + } + ] + }, + { + "id": "promotion-rule-stores-the-wrong-discount", + "scenario": "tracks/ecommerce/scenarios/progression-promotion-rules-1.0.0.json", + "targets": [ + "ecommerce.progression.promotion-rules.promotion-rule-values.620a" + ], + "desc": "Promotion creation stores a one-percent discount instead of the submitted value.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "ctx.db.promotion.insert({ id: 0n, ...input, code: input.code.trim(), redemptions: 0 });", + "replace": "ctx.db.promotion.insert({ id: 0n, ...input, discountPercent: 1, code: input.code.trim(), redemptions: 0 });" + } + ] + }, + { + "id": "staff-cannot-open-staff-tools", + "scenario": "tracks/ecommerce/scenarios/progression-staff-access-1.0.0.json", + "targets": [ + "ecommerce.progression.staff-access.staff-access.601a" + ], + "desc": "The staff navigation link is shown to administrators but not staff members.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": "{(isStaff || isAdmin) && (\n ({", + "replace": "return [...ctx.db.notification.iter()].map(row => ({" + } + ] + }, + { + "id": "support-history-disappears-after-reload", + "scenario": "tracks/ecommerce/scenarios/progression-support-history-1.0.0.json", + "targets": [ + "ecommerce.progression.support-history.support-history-persistence.612a" + ], + "desc": "The customer support view hides every saved ticket.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "return [...ctx.db.supportTicket.iter()]\n .filter(row => isTicketCreator(sender, row.creatorIdentity.toHexString()) ||", + "replace": "return [...ctx.db.supportTicket.iter()]\n .filter(() => false)\n .filter(row => isTicketCreator(sender, row.creatorIdentity.toHexString()) ||" + } + ] + }, + { + "id": "support-history-leaks-across-customers", + "scenario": "tracks/ecommerce/scenarios/progression-support-history-1.0.0.json", + "targets": [ + "ecommerce.progression.support-history.support-history-privacy.612b" + ], + "desc": "The support history view returns tickets owned by other customers.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": ".filter(row => isTicketCreator(sender, row.creatorIdentity.toHexString()) ||\n !!actor && (actor.isAdmin || actor.isStaff || row.accountId === accountId))", + "replace": ".filter(() => true)" + } + ] + }, + { + "id": "visitor-support-reference-is-hidden", + "scenario": "tracks/ecommerce/scenarios/progression-support-intake-1.0.0.json", + "targets": [ + "ecommerce.progression.support-intake.support-intake.610a" + ], + "desc": "A visitor can create a support ticket but the returned reference is not rendered.", + "file": "client/src/components/ProgressionWorkbench.tsx", + "edits": [ + { + "find": "
{supportReference}
", + "replace": "
" + } + ] + }, + { + "id": "support-assignment-is-discarded", + "scenario": "tracks/ecommerce/scenarios/progression-support-triage-1.0.0.json", + "targets": [ + "ecommerce.progression.support-triage.support-assignment.611a" + ], + "desc": "Support triage saves status and priority but discards the assignee.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "ctx.db.supportTicket.id.update({ ...ticket, assigneeId: nextAssigneeId, priority, status });", + "replace": "ctx.db.supportTicket.id.update({ ...ticket, assigneeId: undefined, priority, status });" + } + ] + }, + { + "id": "support-priority-is-discarded", + "scenario": "tracks/ecommerce/scenarios/progression-support-triage-1.0.0.json", + "targets": [ + "ecommerce.progression.support-triage.support-priority.611b" + ], + "desc": "Support triage always saves normal priority instead of the submitted value.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "ctx.db.supportTicket.id.update({ ...ticket, assigneeId: nextAssigneeId, priority, status });", + "replace": "ctx.db.supportTicket.id.update({ ...ticket, assigneeId: nextAssigneeId, priority: 'normal', status });" + } + ] + }, + { + "id": "support-status-is-discarded", + "scenario": "tracks/ecommerce/scenarios/progression-support-triage-1.0.0.json", + "targets": [ + "ecommerce.progression.support-triage.support-status.611c" + ], + "desc": "Support triage preserves the old status instead of the submitted value.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "ctx.db.supportTicket.id.update({ ...ticket, assigneeId: nextAssigneeId, priority, status });", + "replace": "ctx.db.supportTicket.id.update({ ...ticket, assigneeId: nextAssigneeId, priority, status: ticket.status });" + } + ] + }, + { + "id": "nonpositive-cart-quantity-is-treated-as-removal", + "scenario": "tracks/ecommerce/scenarios/01-cart-boundary-2.4.0.json", + "targets": [ + "ecommerce.spec.access-control.cart-boundary.109b" + ], + "desc": "Accept a negative quantity and remove the cart line instead of refusing the request.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " if (quantity < 1) throw new SenderError('Quantity must be at least 1.');\n const existing = findCartLine(ctx, acc.id, itemId);\n if (!existing) throw new SenderError('That item is not in your cart.');\n replaceReservation(ctx, acc.id, itemId, quantity);\n ctx.db.cartItem.id.update({ ...existing, quantity });", + "replace": " const existing = findCartLine(ctx, acc.id, itemId);\n if (!existing) throw new SenderError('That item is not in your cart.');\n if (quantity < 1) {\n ctx.db.cartItem.id.delete(existing.id);\n return;\n }\n replaceReservation(ctx, acc.id, itemId, quantity);\n ctx.db.cartItem.id.update({ ...existing, quantity });" + } + ] + }, + { + "id": "catalog-core-seeds-wrong-air-purifier-price", + "scenario": "tracks/ecommerce/scenarios/01-core-2.4.0.json", + "targets": [ + "ecommerce.feature.catalog.catalog.2a" + ], + "desc": "Seed Air Purifier with an incorrect stored price while leaving the rest of the catalog intact.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " ['Air Purifier', 189.0, 60, 40, 'Home'],", + "replace": " ['Air Purifier', 999.0, 60, 40, 'Home']," + } + ] + }, + { + "id": "catalog-tie-breaks-in-reverse-alphabetical-order", + "scenario": "tracks/ecommerce/scenarios/01-core-2.4.0.json", + "targets": [ + "ecommerce.feature.catalog.catalog.2b" + ], + "desc": "Reverse the specified alphabetical tie-breaker. This necessarily breaks both the initial sequence and the post-purchase sequence in the same scenario.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " return a.name.localeCompare(b.name);", + "replace": " return b.name.localeCompare(a.name);" + } + ] + }, + { + "id": "search-only-examines-the-visible-top-ten", + "scenario": "tracks/ecommerce/scenarios/01-core-2.4.0.json", + "targets": [ + "ecommerce.feature.catalog.catalog.2d" + ], + "desc": "Search the storefront slice instead of the full catalog, hiding items outside the current top ten.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " return [...items]\n .filter(item => !q || item.name.toLowerCase().includes(q))", + "replace": " return rankedItems.slice(0, CATALOG_PAGE_SIZE)\n .filter(item => !q || item.name.toLowerCase().includes(q))" + } + ] + }, + { + "id": "admin-ui-allows-any-signed-in-account", + "scenario": "tracks/ecommerce/scenarios/01-warehouse-admin-2.4.0.json", + "targets": [ + "ecommerce.spec.access-control.admin-ui.7a" + ], + "desc": "Use account presence instead of the server-provided administrator flag to expose the admin area.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " const isAdmin = currentUser?.isAdmin ?? false;", + "replace": " const isAdmin = isSignedIn;" + } + ] + }, + { + "id": "admin-restock-preserves-existing-stock", + "scenario": "tracks/ecommerce/scenarios/01-warehouse-stock-live-2.4.0.json", + "targets": [ + "ecommerce.spec.live-state.warehouse-stock.7c" + ], + "desc": "Accept an administrator restock but write the existing quantity back unchanged.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " ctx.db.stock.insert({ ...existing, quantity: existing.quantity + quantity });", + "replace": " ctx.db.stock.insert({ ...existing, quantity: existing.quantity });" + } + ] + }, + { + "id": "admin-write-allows-customer-restock", + "scenario": "tracks/ecommerce/scenarios/01-admin-write-2.4.0.json", + "targets": [ + "ecommerce.spec.access-control.admin-write.103a" + ], + "desc": "Remove the server-side administrator check from the restock reducer.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "export const adminRestock = spacetimedb.reducer(\n { itemId: t.u64(), warehouseId: t.u64(), quantity: t.u32() },\n (ctx, { itemId, warehouseId, quantity }) => {\n requireAdmin(ctx);", + "replace": "export const adminRestock = spacetimedb.reducer(\n { itemId: t.u64(), warehouseId: t.u64(), quantity: t.u32() },\n (ctx, { itemId, warehouseId, quantity }) => {\n // mutant: no administrator check" + } + ] + }, + { + "id": "operator-authorization-allows-customer-price-change", + "scenario": "tracks/ecommerce/scenarios/02-strengthened-1.4.0.json", + "targets": [ + "ecommerce.operations-access.operator-authorization.201b" + ], + "desc": "The price reducer drops its administrator role gate.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " (ctx, { itemId, price }) => {\n requireAdmin(ctx);", + "replace": " (ctx, { itemId, price }) => {\n // mutant: no administrator role check" + } + ] + }, + { + "id": "fulfilment-queue-allows-customer-shipping", + "scenario": "tracks/ecommerce/scenarios/02-self-contained-1.4.0.json", + "targets": [ + "ecommerce.operations-access.fulfilment-queue.1e" + ], + "desc": "The shipping reducer drops the staff role check while retaining the pending-order guard.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "export const shipOrder = spacetimedb.reducer({ orderId: t.u64() }, (ctx, { orderId }) => {\n requireStaffOrAdmin(ctx);", + "replace": "export const shipOrder = spacetimedb.reducer({ orderId: t.u64() }, (ctx, { orderId }) => {\n // mutant: no staff role check" + } + ] + }, + { + "id": "catalog-search-keeps-pre-change-price", + "scenario": "tracks/ecommerce/scenarios/02-live-price-1.4.0.json", + "targets": [ + "ecommerce.returns-pricing.price-history.4b" + ], + "desc": "The search result renderer caches each item's first visible price and ignores later live price updates.", + "file": "client/src/components/ItemCard.tsx", + "edits": [ + { + "find": "import { ItemRow } from '../types';", + "replace": "import { useRef } from 'react';\nimport { ItemRow } from '../types';" + }, + { + "find": " const outOfStock = stock <= 0;\n const lowStock = !outOfStock && stock <= 5;", + "replace": " const outOfStock = stock <= 0;\n const lowStock = !outOfStock && stock <= 5;\n // mutant: the card retains the first price it renders\n const firstPrice = useRef(item.price);" + }, + { + "find": " {formatMoney(item.price)}", + "replace": " {formatMoney(firstPrice.current)}" + } + ] + }, + { + "id": "open-cart-keeps-pre-change-price", + "scenario": "tracks/ecommerce/scenarios/02-features.json", + "targets": [ + "ecommerce.returns-pricing.price-history.4c" + ], + "desc": "The open-cart memo ignores reactive item-table price updates while checkout still reads the current server price.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " [cartRows, items, stockByItem]", + "replace": " [cartRows, stockByItem]" + } + ] + }, + { + "id": "catalog-price-rewrites-receipts", + "scenario": "tracks/ecommerce/scenarios/02-paid-price-history-1.4.0.json", + "targets": [ + "ecommerce.returns-pricing.price-history.4a" + ], + "desc": "Changing a catalog price cascades into saved order lines and recomputes historical order totals inside the reducer transaction.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " ctx.db.item.id.update({ ...it, price });", + "replace": " ctx.db.item.id.update({ ...it, price });\n const repricedOrderIds = new Set();\n for (const line of ctx.db.orderItem.iter()) {\n if (line.itemId !== itemId) continue;\n ctx.db.orderItem.id.update({ ...line, unitPrice: price });\n repricedOrderIds.add(line.orderId);\n }\n for (const orderId of repricedOrderIds) {\n const historical = ctx.db.customerOrder.id.find(orderId);\n if (!historical) continue;\n let total = 0;\n for (const line of ctx.db.orderItem.orderId.filter(orderId)) total += line.unitPrice * line.quantity;\n ctx.db.customerOrder.id.update({ ...historical, total });\n }" + } + ] + }, + { + "id": "catalog-price-rewrites-earned-revenue", + "scenario": "tracks/ecommerce/scenarios/02-invariants.json", + "targets": [ + "ecommerce.returns-pricing.refund-accounting.203b" + ], + "desc": "Changing a catalog price cascades into saved order lines and recomputes historical order totals and revenue inside the reducer transaction.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " ctx.db.item.id.update({ ...it, price });", + "replace": " ctx.db.item.id.update({ ...it, price });\n const repricedOrderIds = new Set();\n for (const line of ctx.db.orderItem.iter()) {\n if (line.itemId !== itemId) continue;\n ctx.db.orderItem.id.update({ ...line, unitPrice: price });\n repricedOrderIds.add(line.orderId);\n }\n for (const orderId of repricedOrderIds) {\n const historical = ctx.db.customerOrder.id.find(orderId);\n if (!historical) continue;\n let total = 0;\n for (const line of ctx.db.orderItem.orderId.filter(orderId)) total += line.unitPrice * line.quantity;\n ctx.db.customerOrder.id.update({ ...historical, total });\n }" + } + ] + }, + { + "id": "returned-line-marker-omitted", + "scenario": "tracks/ecommerce/scenarios/02-strengthened-1.4.0.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3c" + ], + "desc": "A returned order line keeps its persisted returned state, restored stock, and adjusted revenue but omits the visible returned marker.", + "file": "client/src/components/OrdersPanel.tsx", + "edits": [ + { + "find": "{item.returned && Returned}", + "replace": "{false && Returned}" + } + ] + }, + { + "id": "direct-review-access-is-not-checked", + "scenario": "tracks/ecommerce/scenarios/progression-review-access-1.0.0.json", + "targets": [ + "ecommerce.progression.review-access-specifications.review-eligibility-direct.618a" + ], + "desc": "The direct review action accepts a review from a customer who did not buy the item.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " if (!bought) throw new SenderError('You can only review items you have purchased.');", + "replace": " // mutant: purchase eligibility is not checked" + } + ] + }, + { + "id": "support-messages-are-sent-to-other-customers", + "scenario": "tracks/ecommerce/scenarios/progression-support-read-privacy-1.0.0.json", + "targets": [ + "ecommerce.progression.support-privacy-specifications.private-message-delivery.619a" + ], + "desc": "A customer receives another customer's support message.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": ".filter(row => isTicketCreator(sender, row.creatorIdentity.toHexString()) ||\n !!actor && (actor.isAdmin || actor.isStaff || row.accountId === accountId))", + "replace": ".filter(() => true)" + } + ] + } + ] +} diff --git a/tools/stack-bench/grader/transport-frames.ts b/tools/stack-bench/grader/transport-frames.ts new file mode 100644 index 00000000000..0a3a6f596cd --- /dev/null +++ b/tools/stack-bench/grader/transport-frames.ts @@ -0,0 +1,18 @@ +import { brotliDecompressSync, gunzipSync } from 'node:zlib'; + +// A SpacetimeDB server frame carries a one-byte compression tag ahead of the +// message: 0 none, 1 brotli, 2 gzip, and the SDK compresses by default. The +// message text is inline UTF-8 once decoded, so a substring search finds it +// without the harness knowing the wire format. Any other frame is kept as it +// arrived. +export function transportFrameText(payload: string | Buffer): string { + if (typeof payload === 'string') return payload; + const bytes = Buffer.from(payload); + if (bytes.length > 1) { + try { + if (bytes[0] === 1) return brotliDecompressSync(bytes.subarray(1)).toString('utf8'); + if (bytes[0] === 2) return gunzipSync(bytes.subarray(1)).toString('utf8'); + } catch { /* not a compressed SpacetimeDB frame */ } + } + return bytes.toString('utf8'); +} diff --git a/tools/stack-bench/linter/fixtures/agreed.html b/tools/stack-bench/linter/fixtures/agreed.html new file mode 100644 index 00000000000..c462174caf2 --- /dev/null +++ b/tools/stack-bench/linter/fixtures/agreed.html @@ -0,0 +1,4 @@ +agreement fixture + +4 + diff --git a/tools/stack-bench/linter/fixtures/divergent.html b/tools/stack-bench/linter/fixtures/divergent.html new file mode 100644 index 00000000000..5d95d68cd79 --- /dev/null +++ b/tools/stack-bench/linter/fixtures/divergent.html @@ -0,0 +1,12 @@ +divergence fixture + +
+ + + diff --git a/tools/stack-bench/linter/fixtures/mock-chat.html b/tools/stack-bench/linter/fixtures/mock-chat.html new file mode 100644 index 00000000000..2677b6feb48 --- /dev/null +++ b/tools/stack-bench/linter/fixtures/mock-chat.html @@ -0,0 +1,71 @@ + + +Linter fixture — mock chat + + +
+ + +
+ + + + + + diff --git a/tools/stack-bench/linter/fixtures/mock-shop.html b/tools/stack-bench/linter/fixtures/mock-shop.html new file mode 100644 index 00000000000..5034271fdc9 --- /dev/null +++ b/tools/stack-bench/linter/fixtures/mock-shop.html @@ -0,0 +1,297 @@ + +

Mock Shop

+ +
+ + + + + +
+ + + + + +
+ + + + + + + + + + + + + + diff --git a/tools/stack-bench/linter/fixtures/spec-accounts.html b/tools/stack-bench/linter/fixtures/spec-accounts.html new file mode 100644 index 00000000000..ddbb1ad5630 --- /dev/null +++ b/tools/stack-bench/linter/fixtures/spec-accounts.html @@ -0,0 +1,29 @@ + +
+ + + + + +
+ + diff --git a/tools/stack-bench/linter/lint.ts b/tools/stack-bench/linter/lint.ts new file mode 100644 index 00000000000..1137466279b --- /dev/null +++ b/tools/stack-bench/linter/lint.ts @@ -0,0 +1,237 @@ +#!/usr/bin/env node +// Scenario-stage hooks require scenario setup and are not linted here. + +import { chromium } from 'playwright'; +import type { Page } from 'playwright'; +import { readFileSync, readdirSync } from 'node:fs'; +import { join } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +import { parseArgs as parseNodeArgs } from 'node:util'; +import { loadTrack, DEFAULT_TRACK } from '../src/composition/tracks.js'; +import { emptyArtifactIdentities, writeArtifact } from '../src/evidence/artifacts.js'; +import { stableElementSelector } from '../src/actions/element-selector.js'; + +const CHECK_TIMEOUT = 5000; + +export interface LintHook { + id: string; + element: string; + stage: string; + check: 'visible' | 'attached'; + note: string; + revealedBy?: string; +} + +export interface LintResult { + id: string; + status: 'PASS' | 'FAIL' | 'BLOCKED' | 'SCENARIO'; + detail?: string; +} + +export interface LintArgs { + url?: string; + track: string; + level: number; + json: boolean; + headed: boolean; + out?: string; + label?: string; + parentAttemptId?: string; + credentialAliases?: unknown; + hooks: string[]; +} + +export interface LintWalkContext { + page: Page; + args: LintArgs; + hooks: LintHook[]; + byStage(stage: string): LintHook[]; + blocked(stage: string): void; + checkHook(page: Page, hook: LintHook, results: LintResult[]): Promise; + results: LintResult[]; + uniq: string; + tid(id: string): string; + CHECK_TIMEOUT: number; +} + +function parseArgs(argv: string[]): LintArgs { + const { values } = parseNodeArgs({ args: argv.slice(2), options: { + url: { type: 'string' }, track: { type: 'string' }, level: { type: 'string' }, + json: { type: 'boolean' }, out: { type: 'string' }, label: { type: 'string' }, + 'parent-attempt-id': { type: 'string' }, 'credential-aliases-json': { type: 'string' }, + hook: { type: 'string', multiple: true }, headed: { type: 'boolean' }, + } }); + const args: LintArgs = { url: values.url, track: values.track ?? DEFAULT_TRACK, + level: values.level === undefined ? 1 : Number(values.level), json: values.json ?? false, + headed: values.headed ?? false, out: values.out, label: values.label, + parentAttemptId: values['parent-attempt-id'], + credentialAliases: values['credential-aliases-json'] === undefined + ? undefined : JSON.parse(values['credential-aliases-json']), + hooks: values.hook ?? [] }; + if (!args.url || !Number.isInteger(args.level) || args.level < 1) { + console.error('Usage: node dist/linter/lint.js --url --level [--json] [--headed]'); + process.exit(2); + } + return args; +} + +export function selectHooks(hooks: LintHook[], selectedIds: string[] = []): LintHook[] { + if (!selectedIds.length) return hooks; + const remaining = new Set(selectedIds); + const selected = hooks.filter(hook => remaining.delete(hook.id)); + const unknown: LintHook[] = [...remaining].sort().map(id => ({ + id, + element: `the selected application control ${id}`, + stage: 'scenario', + check: 'visible', + note: 'checked by the selected feature suite', + })); + return [...selected, ...unknown]; +} + +export function loadHooks(level: number, track: { contracts: string }, selectedIds: string[] = []): LintHook[] { + const CONTRACTS_DIR = track.contracts; + const files = readdirSync(CONTRACTS_DIR).filter(f => /^\d+-[a-z-]+\.json$/.test(f)).sort(); + const hooks = []; + for (const f of files) { + const contract = JSON.parse(readFileSync(join(CONTRACTS_DIR, f), 'utf8')) as { + level: number; hooks: LintHook[]; + }; + if (contract.level <= level) hooks.push(...contract.hooks); + } + if (hooks.length === 0 && selectedIds.length === 0) { + console.error(`No contracts found for level ${level} in ${CONTRACTS_DIR}`); + process.exit(2); + } + return selectHooks(hooks, selectedIds); +} + +const tid = stableElementSelector; +const uniq = Date.now().toString(36).slice(-5); + +async function checkHook(page: Page, hook: LintHook, results: LintResult[]): Promise { + const loc = page.locator(tid(hook.id)).first(); + try { + if (hook.revealedBy && !(await loc.count())) { + await page.locator(tid(hook.revealedBy)).first().click({ timeout: CHECK_TIMEOUT }); + } + await loc.waitFor({ + state: hook.check === 'visible' ? 'visible' : 'attached', + timeout: CHECK_TIMEOUT, + }); + results.push({ id: hook.id, status: 'PASS' }); + return true; + } catch { + results.push({ + id: hook.id, + status: 'FAIL', + detail: `no element matching ${tid(hook.id)} became ${hook.check} during contract stage ${JSON.stringify(hook.stage)}` + + (hook.revealedBy ? ` (after clicking ${tid(hook.revealedBy)})` : '') + + ` — expected: ${hook.element}`, + }); + return false; + } +} + +export function completeUnvisitedHooks(hooks: LintHook[], results: LintResult[]): LintResult[] { + const visited = new Set(results.map(result => result.id)); + for (const hook of hooks) { + if (visited.has(hook.id)) continue; + results.push(hook.stage === 'scenario' + ? { id: hook.id, status: 'SCENARIO', detail: hook.note } + : { id: hook.id, status: 'BLOCKED', + detail: `the core flow did not visit contract stage ${JSON.stringify(hook.stage)}` }); + } + return results; +} + +export function completeAbortedHooks(hooks: LintHook[], results: LintResult[], error: unknown): LintResult[] { + const visited = new Set(results.map(result => result.id)); + const detail = String(error instanceof Error ? error.message : error ?? 'unknown error') + .split(/\r?\n/).map(line => line.trim()).filter(Boolean).slice(0, 6).join(' ').slice(0, 800); + results.push({ id: 'core-flow', status: 'FAIL', detail: `core flow aborted: ${detail}` }); + for (const hook of hooks) { + if (visited.has(hook.id)) continue; + if (hook.stage === 'scenario') { + results.push({ id: hook.id, status: 'SCENARIO', detail: hook.note }); + } else { + results.push({ id: hook.id, status: 'BLOCKED', detail: 'core flow aborted' }); + } + } + return results; +} + +async function run() { + const args = parseArgs(process.argv); + const track = loadTrack(args.track); + const hooks = loadHooks(args.level, track, args.hooks); + const byStage = (stage: string): LintHook[] => hooks.filter(h => h.stage === stage); + const results: LintResult[] = []; + const blocked = (stage: string): void => { + for (const h of hooks.filter(x => x.stage === stage)) { + results.push({ id: h.id, status: 'BLOCKED', detail: 'earlier core flow step failed' }); + } + }; + + const browser = await chromium.launch({ headless: !args.headed }); + const page = await browser.newContext().then(c => c.newPage()); + page.setDefaultTimeout(CHECK_TIMEOUT); + + try { + // The core flow is the one part of linting that is entirely + // application-specific, so each track brings its own. + const { walk } = await import(pathToFileURL(track.walk).href) as { + walk(context: LintWalkContext): Promise; + }; + await walk({ page, args, hooks, byStage, blocked, checkHook, results, uniq, tid, CHECK_TIMEOUT }); + // Every lintable hook must record explicit evidence. + completeUnvisitedHooks(hooks, results); + } catch (err: unknown) { + console.error(`Core flow aborted: ${err instanceof Error ? err.message : String(err)}`); + completeAbortedHooks(hooks, results, err); + } finally { + await browser.close(); + } + + const failures = results.filter(r => r.status === 'FAIL' || r.status === 'BLOCKED'); + const report = { + label: args.label ?? null, + url: args.url, + level: args.level, + selectedHooks: args.hooks.length ? [...new Set(args.hooks)].sort() : null, + pass: failures.length === 0, + counts: { + lintable: results.filter(r => r.status !== 'SCENARIO').length, + pass: results.filter(r => r.status === 'PASS').length, + fail: results.filter(r => r.status === 'FAIL').length, + blocked: results.filter(r => r.status === 'BLOCKED').length, + scenario: results.filter(r => r.status === 'SCENARIO').length, + }, + results, + }; + if (args.out) { + const id = `${args.parentAttemptId ?? args.label ?? 'lint'}-contract-lint`; + writeArtifact(args.out, { + kind: 'contract_lint', id, + attempt: { id, parentId: args.parentAttemptId ?? null }, + identities: emptyArtifactIdentities(), + payload: report, + }); + if (!args.json) console.log(`\nLint report written to ${args.out}`); + } + if (args.json) { + console.log(JSON.stringify(report, null, 2)); + } else { + for (const r of results) { + console.log(`${r.status.padEnd(9)} ${r.id}${r.detail ? ` — ${r.detail}` : ''}`); + } + console.log(failures.length === 0 + ? report.counts.pass > 0 + ? `\nAPPLICATION CONTRACT PASS (${report.counts.pass} interfaces)` + : `\nAPPLICATION CONTRACT DEFERRED (${report.counts.scenario} interfaces checked during feature grading)` + : `\nAPPLICATION CONTRACT FAIL (${failures.length} interfaces missing or blocked)`); + } + process.exit(failures.length === 0 ? 0 : 1); +} + +if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) run(); diff --git a/tools/stack-bench/package-lock.json b/tools/stack-bench/package-lock.json new file mode 100644 index 00000000000..430532137f5 --- /dev/null +++ b/tools/stack-bench/package-lock.json @@ -0,0 +1,136 @@ +{ + "name": "@spacetimedb/stack-bench", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@spacetimedb/stack-bench", + "dependencies": { + "eventsource-parser": "3.1.1", + "playwright": "1.62.1", + "semver": "7.7.4", + "zod": "4.5.4" + }, + "devDependencies": { + "@types/node": "22.15.30", + "@types/semver": "7.8.0", + "typescript": "5.6.3" + }, + "engines": { + "node": ">=22" + } + }, + "node_modules/@types/node": { + "version": "22.15.30", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.15.30.tgz", + "integrity": "sha512-6Q7lr06bEHdlfplU6YRbgG1SFBdlsfNC4/lX+SkhiTs0cpJkOElmWls8PxDFv4yY/xKb8Y6SO0OmSX4wgqTZbA==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/@types/semver": { + "version": "7.8.0", + "resolved": "https://registry.npmjs.org/@types/semver/-/semver-7.8.0.tgz", + "integrity": "sha512-1mAINjtQCXXeLkJ9ehXkwOcBpqtLxiVtKhpUf83DdRNdQKV0iXZpaHYqRr7nj+wvxuJzoAmAwXI+sCNMv1CzLQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/eventsource-parser": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-3.1.1.tgz", + "integrity": "sha512-EKN1vKAMcZ8MlYMpaNuxN6R9yakzH6uajHcHVTqWJzvu5pWw9DyhbP35HH8MVBQ+dZjAfDxk+A8NiR9KWaXiyQ==", + "license": "MIT", + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/playwright": { + "version": "1.62.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.62.1.tgz", + "integrity": "sha512-0M+L3LAD8/nm554LOla9Ayx0j0tmFZ0FBcoQ7F1VuVHpM/XpiC8RcDzBQB8W5+hA8L22THxELzeF+2WcUzvcLg==", + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.62.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=20" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.62.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.62.1.tgz", + "integrity": "sha512-wPYSwEBJY9GHraISXqyqtx0na0LpO3XEX7jNDhntbex7tzUS7kLnZsOlFruFJB4Hi/rhDMjXGqHewDZ68nYZVw==", + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/semver": { + "version": "7.7.4", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", + "integrity": "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/typescript": { + "version": "5.6.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.6.3.tgz", + "integrity": "sha512-hjcS1mhfuyi4WW8IWtjP7brDrG2cuDZukyrYrSauoXGNgx0S7zceP07adYkJycEr56BOUTNPzbInooiN3fn1qw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/zod": { + "version": "4.5.4", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.5.4.tgz", + "integrity": "sha512-sC95tT5iHHH9gtpj6A81kh+NEaRAUFN+qlUPDUbRfOMvNf5QCBqsb3WgvnpVtK5Y+4UfA6KqufotuTvMGiTlsA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + } + } +} diff --git a/tools/stack-bench/package.json b/tools/stack-bench/package.json new file mode 100644 index 00000000000..5754cd5ad7b --- /dev/null +++ b/tools/stack-bench/package.json @@ -0,0 +1,85 @@ +{ + "name": "@spacetimedb/stack-bench", + "private": true, + "type": "module", + "engines": { + "node": ">=22" + }, + "scripts": { + "build": "npm run clean && tsc -p tsconfig.build.json && node dist/scripts/copy-dashboard-assets.js", + "clean": "node --eval \"require('node:fs').rmSync('dist', { recursive: true, force: true })\"", + "lint": "eslint appliance commands container dashboard grader linter scripts src tests", + "typecheck": "tsc -p tsconfig.json --noEmit", + "bootstrap:browsers": "playwright install chromium", + "prebench": "npm run build --silent", + "bench": "node dist/commands/bench.js", + "prepreflight": "npm run build", + "preflight": "node dist/commands/preflight.js", + "prerecover": "npm run build", + "recover": "node dist/commands/recovery.js recover", + "prerelease:bundle": "npm run build --silent", + "release:bundle": "node dist/src/releases/release-bundle.js", + "release:source": "npm run build --silent && node dist/src/releases/release-source.js --json", + "preverify:release": "npm run build --silent", + "verify:release": "node dist/src/releases/release-manifest.js verify", + "precheck:scenarios": "npm run build --silent", + "check:scenarios": "node dist/commands/check-scenarios.js --track chat && node dist/commands/check-scenarios.js --track ecommerce --recipe sequential-l1-2.5.0.json && node dist/commands/check-scenarios.js --track ecommerce --recipe sequential-l2-1.6.0.json && node dist/commands/check-scenarios.js --track ecommerce --recipe sequential-l3-1.0.0.json && node dist/commands/check-scenarios.js --track ecommerce --recipe progression-catalog-2.0.2.json && node dist/commands/check-scenarios.js --track ecommerce --recipe progression-depth3-2.0.2.json", + "precheck:mutations": "npm run build", + "check:mutations": "node dist/commands/check-mutations.js", + "precheck:definition-snapshots": "npm run build --silent", + "check:definition-snapshots": "node dist/commands/definition-snapshots.js", + "precheck:composition": "npm run build --silent", + "check:composition": "node dist/commands/check-composition.js", + "precheck:prompts": "npm run build --silent", + "check:prompts": "node --test dist/tests/dependency-neutral-prompt.contract.js", + "precheck:calibration": "npm run build --silent", + "check:calibration": "node dist/commands/check-calibration.js", + "precheck:references": "npm run build --silent", + "check:references": "node dist/src/references/reference-fixtures.js", + "pregraph": "npm run build", + "graph": "node dist/commands/progression-graph.js tracks/ecommerce/progression/ecommerce-2.0.2.json", + "pack": "npm run build --silent && node dist/commands/composition-cli.js pack", + "recipe": "npm run build --silent && node dist/commands/composition-cli.js recipe", + "precampaign": "npm run build --silent", + "campaign": "node dist/commands/campaign-cli.js", + "predashboard": "npm run build --silent", + "dashboard": "node dist/dashboard/dashboard-server.js", + "prerepair": "npm run build", + "repair": "node dist/commands/repair-cli.js", + "pretest": "npm run build --silent", + "test": "node --test --test-concurrency=4 dist/tests/*.test.js", + "pretest:dashboard": "npm run build --silent", + "test:dashboard": "node --test dist/tests/dashboard/*.test.js", + "pretest:all": "npm run build --silent", + "test:all": "node --test --test-concurrency=4 dist/tests/*.test.js dist/tests/dashboard/*.test.js dist/tests/*.contract.js", + "pretest:contracts": "npm run build --silent", + "test:contracts": "node --test --test-concurrency=4 dist/tests/*.contract.js", + "pretest:mutation-definitions": "npm run build --silent", + "test:mutation-definitions": "node --test --test-concurrency=4 dist/tests/*.mutation.js", + "pretest:integration": "npm run build --silent", + "test:integration": "node --test --test-concurrency=1 dist/tests/*.integration.js", + "pretest:container": "npm run build --silent", + "test:container": "node dist/commands/container-smoke.js", + "pretest:references": "npm run build --silent", + "test:references": "node dist/src/references/reference-build.js", + "prequalify:reference": "npm run build --silent", + "qualify:reference": "node dist/src/references/reference-live.js", + "pretest:faults": "npm run build --silent", + "test:faults": "node dist/commands/fault-injection.js", + "pretest:loop": "npm run build", + "test:loop": "node dist/commands/test-loop.js", + "pretest:null": "npm run build --silent", + "test:null": "node dist/commands/null-control.js" + }, + "dependencies": { + "eventsource-parser": "3.1.1", + "playwright": "1.62.1", + "semver": "7.7.4", + "zod": "4.5.4" + }, + "devDependencies": { + "@types/node": "22.15.30", + "@types/semver": "7.8.0", + "typescript": "5.6.3" + } +} diff --git a/tools/stack-bench/qualification-evidence/ecommerce-progression-depth3-2.0.1/mongodb-mutation.json b/tools/stack-bench/qualification-evidence/ecommerce-progression-depth3-2.0.1/mongodb-mutation.json new file mode 100644 index 00000000000..52af6dabd49 --- /dev/null +++ b/tools/stack-bench/qualification-evidence/ecommerce-progression-depth3-2.0.1/mongodb-mutation.json @@ -0,0 +1,375 @@ +{ + "artifactSchemaVersion": 2, + "kind": "reference_qualification", + "id": "reference-live-mongodb-20260831090827-14", + "attempt": { + "id": "reference-live-mongodb-20260831090827-14", + "parentId": null + }, + "timestamps": { + "startedAt": "2026-08-31T09:08:27.961Z", + "completedAt": "2026-08-31T09:33:10.960Z" + }, + "identities": { + "engine": { + "id": "stack-bench", + "version": null, + "sha256": "39da14d55d1ca82f41d18305dcb3e86aa7d5098c69a2bf6ba74533affbfbecc9", + "state": null + }, + "recipe": { + "id": "ecommerce.progression-depth3", + "version": "2.0.1", + "sha256": "4c808d5e5fb076ff0993937d7a9f940dad4191351c362518e1c667a34324bd49", + "state": "qualified" + }, + "fixture": { + "id": "ecommerce-reference-mongodb", + "version": null, + "sha256": "24d445f18cdcb25b9ab06dd4f4582003b348edaf0be0b97c27ec9fbb06751b1e", + "state": "active" + }, + "calibration": { + "id": "ecommerce.progression-depth3-calibration", + "version": "2.0.1", + "sha256": "dd91493c5066018cbfd949d6e14d64dc029deb634e93d796752caee3da97682c", + "state": "draft" + }, + "experiment": null, + "agentAdapter": null, + "stackAdapter": { + "id": "mongodb", + "version": null, + "sha256": null, + "state": null + }, + "packs": [] + }, + "payload": { + "fixture": "ecommerce-reference-mongodb", + "fixtureSha256": "24d445f18cdcb25b9ab06dd4f4582003b348edaf0be0b97c27ec9fbb06751b1e", + "requiredRepetitions": 1, + "isolation": "docker", + "runner": { + "schemaVersion": 1, + "mode": "appliance", + "platform": "linux", + "architecture": "x64", + "dockerEngineVersion": "29.6.2", + "dockerOs": "linux", + "dockerArchitecture": "x86_64", + "kernelVersion": "6.6.87.2-microsoft-standard-WSL2", + "cpuCount": 32, + "memoryBytes": 49232838656 + }, + "qualificationScope": { + "checksSha256": "a982217993be2d229e13b4006427ce15666263d1d6e333eb241c4431df59cdf7", + "executableSha256": "68ad187165516dd353c091557caeca8b243fee27321deaf5e3a8468be0cb1bf9", + "kind": "mutation", + "mutationSha256": "c0000b5eb59c74c6676d3b247e6eece4de43b7ce09fc8a4990423c971f3c14ab", + "recipe": { + "contentSha256": "4c808d5e5fb076ff0993937d7a9f940dad4191351c362518e1c667a34324bd49", + "id": "ecommerce.progression-depth3", + "version": "2.0.1" + }, + "schemaVersion": 2, + "stack": { + "id": "mongodb", + "reference": { + "id": "ecommerce-reference-mongodb", + "sourceSha256": "24d445f18cdcb25b9ab06dd4f4582003b348edaf0be0b97c27ec9fbb06751b1e" + }, + "version": "1.2.0" + }, + "sha256": "0c8285228b8787503a139c0bb7b88d8a57c15c524aa2ddfd2a20a8ee5b85d0f0" + }, + "mutationControl": true, + "qualifiedCheckKeys": [ + "ecommerce.feature.accounts.accounts.1a", + "ecommerce.feature.accounts.accounts.1b", + "ecommerce.feature.accounts.accounts.1c", + "ecommerce.feature.accounts.accounts.1d", + "ecommerce.feature.cart-checkout.cart.4a", + "ecommerce.feature.cart-checkout.cart.4d", + "ecommerce.feature.catalog.catalog-ranking.2b", + "ecommerce.feature.catalog.catalog-search.2d", + "ecommerce.feature.catalog.catalog-values.2a", + "ecommerce.feature.purchasing.purchase-order.3c", + "ecommerce.feature.reviews.reviews.6a", + "ecommerce.feature.warehouse-admin.access-boundary.7a", + "ecommerce.feature.warehouse-admin.admin-write.103a", + "ecommerce.feature.warehouse-admin.warehouse-stock.7c", + "ecommerce.feature.warehouse-admin.warehouse-view.7b", + "ecommerce.inventory-operations.operational-views.5a", + "ecommerce.inventory-operations.operational-views.5b", + "ecommerce.inventory-operations.operational-views.5c", + "ecommerce.inventory-operations.operational-views.5d", + "ecommerce.inventory-operations.stock-conservation.202a", + "ecommerce.inventory-operations.stock-conservation.202b", + "ecommerce.inventory-operations.stock-conservation.202c", + "ecommerce.inventory-operations.stock-conservation.202d", + "ecommerce.inventory-operations.warehouse-transfer.2a", + "ecommerce.inventory-operations.warehouse-transfer.2b", + "ecommerce.inventory-operations.warehouse-transfer.2c", + "ecommerce.l3.deferred-access.scheduled-work-access.317a", + "ecommerce.l3.deferred-durability.restart-survival.311a", + "ecommerce.l3.deferred-integrity.exactly-once.311a", + "ecommerce.l3.scheduled-restocks.scheduled-restocks.302a", + "ecommerce.l3.scheduled-restocks.scheduled-restocks.305a", + "ecommerce.l3.scheduled-restocks.scheduled-restocks.306a", + "ecommerce.l3.server-time.server-time.312a", + "ecommerce.operations-access.fulfilment-queue.1a", + "ecommerce.operations-access.fulfilment-queue.1b", + "ecommerce.operations-access.fulfilment-queue.1c", + "ecommerce.operations-access.fulfilment-queue.1d", + "ecommerce.operations-access.operator-authorization.201a", + "ecommerce.operations-access.operator-authorization.201c", + "ecommerce.operations-access.order-owner.204a", + "ecommerce.progression.catalog-management.catalog-management.622a", + "ecommerce.progression.catalog-management.catalog-management.622b", + "ecommerce.progression.customer-profile.customer-profile.620a", + "ecommerce.progression.customer-profile.customer-profile.620b", + "ecommerce.progression.faceted-search.faceted-search.401a", + "ecommerce.progression.faceted-search.faceted-search.402a", + "ecommerce.progression.managed-support.managed-support-privacy.613b", + "ecommerce.progression.managed-support.managed-support-shared.613a", + "ecommerce.progression.notification-preferences.notification-preferences-persistence.630a", + "ecommerce.progression.notification-preferences.notification-preferences-privacy.630b", + "ecommerce.progression.promotion-rules.promotion-rule-access.620b", + "ecommerce.progression.promotion-rules.promotion-rule-values.620a", + "ecommerce.progression.staff-access.staff-access.601a", + "ecommerce.progression.staff-access.staff-access.601b", + "ecommerce.progression.staff-roles.staff-roles.621a", + "ecommerce.progression.staff-roles.staff-roles.621b", + "ecommerce.progression.stock-alerts.stock-alert-delivery.631a", + "ecommerce.progression.stock-alerts.stock-alert-privacy.631b", + "ecommerce.progression.support-history.support-history-persistence.612a", + "ecommerce.progression.support-history.support-history-privacy.612b", + "ecommerce.progression.support-intake.support-intake.610a", + "ecommerce.progression.support-triage.support-assignment.611a", + "ecommerce.progression.support-triage.support-priority.611b", + "ecommerce.progression.support-triage.support-status.611c", + "ecommerce.returns-pricing.cancellation-and-return.3a", + "ecommerce.returns-pricing.cancellation-and-return.3b", + "ecommerce.returns-pricing.refund-accounting.203a", + "ecommerce.spec.access-control.cart-boundary.109a", + "ecommerce.spec.access-control.cart-boundary.109b", + "ecommerce.spec.access-control.order-ownership.106a", + "ecommerce.spec.access-control.purchase-attribution.102a", + "ecommerce.spec.access-control.purchase-session.101a", + "ecommerce.spec.access-control.review-eligibility.108a", + "ecommerce.spec.access-control.review-eligibility.108b", + "ecommerce.spec.access-control.signed-out-purchase.3a", + "ecommerce.spec.concurrency-safety.duplicate-checkout.203a", + "ecommerce.spec.concurrency-safety.duplicate-checkout.203b", + "ecommerce.spec.concurrency-safety.last-unit.201a", + "ecommerce.spec.concurrency-safety.last-unit.201b", + "ecommerce.spec.concurrency-safety.last-unit.201c", + "ecommerce.spec.concurrency-safety.restock-race.202a", + "ecommerce.spec.external-data-sync.external-stock.901a", + "ecommerce.spec.external-data-sync.external-stock.901c", + "ecommerce.spec.external-data-sync.external-stock.901d", + "ecommerce.spec.live-state.open-list.902a", + "ecommerce.spec.live-state.purchase-stock.3b", + "ecommerce.spec.live-state.ranking.2c", + "ecommerce.spec.live-state.rating.6c", + "ecommerce.spec.live-state.shared-cart.4c", + "ecommerce.spec.state-durability.account-state-recovery.105a", + "ecommerce.spec.state-durability.account-state-recovery.105b", + "ecommerce.spec.state-durability.cart-reload.4b", + "ecommerce.spec.state-durability.session-reload.1e", + "ecommerce.spec.transactional-integrity.books-balance.107a", + "ecommerce.spec.transactional-integrity.books-balance.107b", + "ecommerce.spec.transactional-integrity.server-price.104a", + "ecommerce.spec.transactional-integrity.unique-review.6b" + ], + "runs": [ + { + "repetition": 1, + "processError": null, + "harnessSha256Before": "55130157ba3c8b6364c6e8f0a9453479a4d8c7ecaa50cfaec92953bde488b13e", + "harnessSha256After": "55130157ba3c8b6364c6e8f0a9453479a4d8c7ecaa50cfaec92953bde488b13e", + "ok": true, + "failures": [], + "score": "162/162", + "imageId": "sha256:eccec6ea3861befc3ef0a05beb3f8ac13e0af2d0a1ef77c1a7beb3c0eb2aa075", + "criteria": 97, + "zeroPointCriteria": 0, + "outcome": "passed", + "packRuntime": { + "packs": [ + { + "id": "ecommerce.feature.accounts", + "exceeded": false + }, + { + "id": "ecommerce.feature.cart", + "exceeded": false + }, + { + "id": "ecommerce.feature.catalog-discovery", + "exceeded": false + }, + { + "id": "ecommerce.feature.catalog-items", + "exceeded": false + }, + { + "id": "ecommerce.feature.checkout", + "exceeded": false + }, + { + "id": "ecommerce.feature.purchasing", + "exceeded": false + }, + { + "id": "ecommerce.feature.reviews", + "exceeded": false + }, + { + "id": "ecommerce.feature.warehouse-admin", + "exceeded": false + }, + { + "id": "ecommerce.l2.inventory-dashboard", + "exceeded": false + }, + { + "id": "ecommerce.l2.order-cancellation-features", + "exceeded": false + }, + { + "id": "ecommerce.l2.recommendations", + "exceeded": false + }, + { + "id": "ecommerce.l2.sales-dashboard", + "exceeded": false + }, + { + "id": "ecommerce.l2.stock-transfers-features", + "exceeded": false + }, + { + "id": "ecommerce.l3.deferred-access-specifications", + "exceeded": false + }, + { + "id": "ecommerce.l3.deferred-durability-specifications", + "exceeded": false + }, + { + "id": "ecommerce.l3.deferred-integrity-specifications", + "exceeded": false + }, + { + "id": "ecommerce.l3.scheduled-restocks-features", + "exceeded": false + }, + { + "id": "ecommerce.l3.server-time-specifications", + "exceeded": false + }, + { + "id": "ecommerce.progression.cancellation-accounting-specifications", + "exceeded": false + }, + { + "id": "ecommerce.progression.catalog-management", + "exceeded": false + }, + { + "id": "ecommerce.progression.customer-profile", + "exceeded": false + }, + { + "id": "ecommerce.progression.faceted-search", + "exceeded": false + }, + { + "id": "ecommerce.progression.fulfilment-queue", + "exceeded": false + }, + { + "id": "ecommerce.progression.inventory-conservation-specifications", + "exceeded": false + }, + { + "id": "ecommerce.progression.managed-support", + "exceeded": false + }, + { + "id": "ecommerce.progression.notification-preferences", + "exceeded": false + }, + { + "id": "ecommerce.progression.operations-access-specifications", + "exceeded": false + }, + { + "id": "ecommerce.progression.promotion-rules", + "exceeded": false + }, + { + "id": "ecommerce.progression.staff-access", + "exceeded": false + }, + { + "id": "ecommerce.progression.staff-roles", + "exceeded": false + }, + { + "id": "ecommerce.progression.stock-alerts", + "exceeded": false + }, + { + "id": "ecommerce.progression.support-history", + "exceeded": false + }, + { + "id": "ecommerce.progression.support-intake", + "exceeded": false + }, + { + "id": "ecommerce.progression.support-triage", + "exceeded": false + }, + { + "id": "ecommerce.spec.access-control", + "exceeded": false + }, + { + "id": "ecommerce.spec.concurrency-safety", + "exceeded": false + }, + { + "id": "ecommerce.spec.external-data-sync", + "exceeded": false + }, + { + "id": "ecommerce.spec.live-state", + "exceeded": false + }, + { + "id": "ecommerce.spec.state-durability", + "exceeded": false + }, + { + "id": "ecommerce.spec.transactional-integrity", + "exceeded": false + } + ] + }, + "mutations": { + "caught": 92, + "total": 92 + } + } + ], + "stable": true, + "sameImage": true, + "sameHarness": true, + "harnessSha256": "55130157ba3c8b6364c6e8f0a9453479a4d8c7ecaa50cfaec92953bde488b13e", + "ok": true + } +} diff --git a/tools/stack-bench/qualification-evidence/ecommerce-progression-depth3-2.0.1/mongodb-reference.json b/tools/stack-bench/qualification-evidence/ecommerce-progression-depth3-2.0.1/mongodb-reference.json new file mode 100644 index 00000000000..12924373965 --- /dev/null +++ b/tools/stack-bench/qualification-evidence/ecommerce-progression-depth3-2.0.1/mongodb-reference.json @@ -0,0 +1,372 @@ +{ + "artifactSchemaVersion": 2, + "kind": "reference_qualification", + "id": "reference-live-mongodb-20260831090827-14-reference", + "attempt": { + "id": "reference-live-mongodb-20260831090827-14-reference", + "parentId": null + }, + "timestamps": { + "startedAt": "2026-08-31T09:08:27.961Z", + "completedAt": "2026-08-31T09:33:10.961Z" + }, + "identities": { + "engine": { + "id": "stack-bench", + "version": null, + "sha256": "39da14d55d1ca82f41d18305dcb3e86aa7d5098c69a2bf6ba74533affbfbecc9", + "state": null + }, + "recipe": { + "id": "ecommerce.progression-depth3", + "version": "2.0.1", + "sha256": "4c808d5e5fb076ff0993937d7a9f940dad4191351c362518e1c667a34324bd49", + "state": "qualified" + }, + "fixture": { + "id": "ecommerce-reference-mongodb", + "version": null, + "sha256": "24d445f18cdcb25b9ab06dd4f4582003b348edaf0be0b97c27ec9fbb06751b1e", + "state": "active" + }, + "calibration": { + "id": "ecommerce.progression-depth3-calibration", + "version": "2.0.1", + "sha256": "dd91493c5066018cbfd949d6e14d64dc029deb634e93d796752caee3da97682c", + "state": "draft" + }, + "experiment": null, + "agentAdapter": null, + "stackAdapter": { + "id": "mongodb", + "version": null, + "sha256": null, + "state": null + }, + "packs": [] + }, + "payload": { + "fixture": "ecommerce-reference-mongodb", + "fixtureSha256": "24d445f18cdcb25b9ab06dd4f4582003b348edaf0be0b97c27ec9fbb06751b1e", + "requiredRepetitions": 1, + "isolation": "docker", + "runner": { + "schemaVersion": 1, + "mode": "appliance", + "platform": "linux", + "architecture": "x64", + "dockerEngineVersion": "29.6.2", + "dockerOs": "linux", + "dockerArchitecture": "x86_64", + "kernelVersion": "6.6.87.2-microsoft-standard-WSL2", + "cpuCount": 32, + "memoryBytes": 49232838656 + }, + "qualificationScope": { + "checksSha256": "a982217993be2d229e13b4006427ce15666263d1d6e333eb241c4431df59cdf7", + "executableSha256": "b806484f7bb42170561e17fa8605b9fd3646d7299d2f2138fa0990e672d26759", + "kind": "reference", + "mutationSha256": null, + "recipe": { + "contentSha256": "4c808d5e5fb076ff0993937d7a9f940dad4191351c362518e1c667a34324bd49", + "id": "ecommerce.progression-depth3", + "version": "2.0.1" + }, + "schemaVersion": 2, + "stack": { + "id": "mongodb", + "reference": { + "id": "ecommerce-reference-mongodb", + "sourceSha256": "24d445f18cdcb25b9ab06dd4f4582003b348edaf0be0b97c27ec9fbb06751b1e" + }, + "version": "1.2.0" + }, + "sha256": "bb8635a5a87f917cd40d5efe01128a16627811d09837967d49b0e0df4af641e4" + }, + "mutationControl": false, + "qualifiedCheckKeys": [ + "ecommerce.feature.accounts.accounts.1a", + "ecommerce.feature.accounts.accounts.1b", + "ecommerce.feature.accounts.accounts.1c", + "ecommerce.feature.accounts.accounts.1d", + "ecommerce.feature.cart-checkout.cart.4a", + "ecommerce.feature.cart-checkout.cart.4d", + "ecommerce.feature.catalog.catalog-ranking.2b", + "ecommerce.feature.catalog.catalog-search.2d", + "ecommerce.feature.catalog.catalog-values.2a", + "ecommerce.feature.purchasing.purchase-order.3c", + "ecommerce.feature.reviews.reviews.6a", + "ecommerce.feature.warehouse-admin.access-boundary.7a", + "ecommerce.feature.warehouse-admin.admin-write.103a", + "ecommerce.feature.warehouse-admin.warehouse-stock.7c", + "ecommerce.feature.warehouse-admin.warehouse-view.7b", + "ecommerce.inventory-operations.operational-views.5a", + "ecommerce.inventory-operations.operational-views.5b", + "ecommerce.inventory-operations.operational-views.5c", + "ecommerce.inventory-operations.operational-views.5d", + "ecommerce.inventory-operations.stock-conservation.202a", + "ecommerce.inventory-operations.stock-conservation.202b", + "ecommerce.inventory-operations.stock-conservation.202c", + "ecommerce.inventory-operations.stock-conservation.202d", + "ecommerce.inventory-operations.warehouse-transfer.2a", + "ecommerce.inventory-operations.warehouse-transfer.2b", + "ecommerce.inventory-operations.warehouse-transfer.2c", + "ecommerce.l3.deferred-access.scheduled-work-access.317a", + "ecommerce.l3.deferred-durability.restart-survival.311a", + "ecommerce.l3.deferred-integrity.exactly-once.311a", + "ecommerce.l3.scheduled-restocks.scheduled-restocks.302a", + "ecommerce.l3.scheduled-restocks.scheduled-restocks.305a", + "ecommerce.l3.scheduled-restocks.scheduled-restocks.306a", + "ecommerce.l3.server-time.server-time.312a", + "ecommerce.operations-access.fulfilment-queue.1a", + "ecommerce.operations-access.fulfilment-queue.1b", + "ecommerce.operations-access.fulfilment-queue.1c", + "ecommerce.operations-access.fulfilment-queue.1d", + "ecommerce.operations-access.operator-authorization.201a", + "ecommerce.operations-access.operator-authorization.201c", + "ecommerce.operations-access.order-owner.204a", + "ecommerce.progression.catalog-management.catalog-management.622a", + "ecommerce.progression.catalog-management.catalog-management.622b", + "ecommerce.progression.customer-profile.customer-profile.620a", + "ecommerce.progression.customer-profile.customer-profile.620b", + "ecommerce.progression.faceted-search.faceted-search.401a", + "ecommerce.progression.faceted-search.faceted-search.402a", + "ecommerce.progression.managed-support.managed-support-privacy.613b", + "ecommerce.progression.managed-support.managed-support-shared.613a", + "ecommerce.progression.notification-preferences.notification-preferences-persistence.630a", + "ecommerce.progression.notification-preferences.notification-preferences-privacy.630b", + "ecommerce.progression.promotion-rules.promotion-rule-access.620b", + "ecommerce.progression.promotion-rules.promotion-rule-values.620a", + "ecommerce.progression.staff-access.staff-access.601a", + "ecommerce.progression.staff-access.staff-access.601b", + "ecommerce.progression.staff-roles.staff-roles.621a", + "ecommerce.progression.staff-roles.staff-roles.621b", + "ecommerce.progression.stock-alerts.stock-alert-delivery.631a", + "ecommerce.progression.stock-alerts.stock-alert-privacy.631b", + "ecommerce.progression.support-history.support-history-persistence.612a", + "ecommerce.progression.support-history.support-history-privacy.612b", + "ecommerce.progression.support-intake.support-intake.610a", + "ecommerce.progression.support-triage.support-assignment.611a", + "ecommerce.progression.support-triage.support-priority.611b", + "ecommerce.progression.support-triage.support-status.611c", + "ecommerce.returns-pricing.cancellation-and-return.3a", + "ecommerce.returns-pricing.cancellation-and-return.3b", + "ecommerce.returns-pricing.refund-accounting.203a", + "ecommerce.spec.access-control.cart-boundary.109a", + "ecommerce.spec.access-control.cart-boundary.109b", + "ecommerce.spec.access-control.order-ownership.106a", + "ecommerce.spec.access-control.purchase-attribution.102a", + "ecommerce.spec.access-control.purchase-session.101a", + "ecommerce.spec.access-control.review-eligibility.108a", + "ecommerce.spec.access-control.review-eligibility.108b", + "ecommerce.spec.access-control.signed-out-purchase.3a", + "ecommerce.spec.concurrency-safety.duplicate-checkout.203a", + "ecommerce.spec.concurrency-safety.duplicate-checkout.203b", + "ecommerce.spec.concurrency-safety.last-unit.201a", + "ecommerce.spec.concurrency-safety.last-unit.201b", + "ecommerce.spec.concurrency-safety.last-unit.201c", + "ecommerce.spec.concurrency-safety.restock-race.202a", + "ecommerce.spec.external-data-sync.external-stock.901a", + "ecommerce.spec.external-data-sync.external-stock.901c", + "ecommerce.spec.external-data-sync.external-stock.901d", + "ecommerce.spec.live-state.open-list.902a", + "ecommerce.spec.live-state.purchase-stock.3b", + "ecommerce.spec.live-state.ranking.2c", + "ecommerce.spec.live-state.rating.6c", + "ecommerce.spec.live-state.shared-cart.4c", + "ecommerce.spec.state-durability.account-state-recovery.105a", + "ecommerce.spec.state-durability.account-state-recovery.105b", + "ecommerce.spec.state-durability.cart-reload.4b", + "ecommerce.spec.state-durability.session-reload.1e", + "ecommerce.spec.transactional-integrity.books-balance.107a", + "ecommerce.spec.transactional-integrity.books-balance.107b", + "ecommerce.spec.transactional-integrity.server-price.104a", + "ecommerce.spec.transactional-integrity.unique-review.6b" + ], + "runs": [ + { + "repetition": 1, + "processError": null, + "harnessSha256Before": "55130157ba3c8b6364c6e8f0a9453479a4d8c7ecaa50cfaec92953bde488b13e", + "harnessSha256After": "55130157ba3c8b6364c6e8f0a9453479a4d8c7ecaa50cfaec92953bde488b13e", + "ok": true, + "failures": [], + "score": "162/162", + "imageId": "sha256:eccec6ea3861befc3ef0a05beb3f8ac13e0af2d0a1ef77c1a7beb3c0eb2aa075", + "criteria": 97, + "zeroPointCriteria": 0, + "outcome": "passed", + "packRuntime": { + "packs": [ + { + "id": "ecommerce.feature.accounts", + "exceeded": false + }, + { + "id": "ecommerce.feature.cart", + "exceeded": false + }, + { + "id": "ecommerce.feature.catalog-discovery", + "exceeded": false + }, + { + "id": "ecommerce.feature.catalog-items", + "exceeded": false + }, + { + "id": "ecommerce.feature.checkout", + "exceeded": false + }, + { + "id": "ecommerce.feature.purchasing", + "exceeded": false + }, + { + "id": "ecommerce.feature.reviews", + "exceeded": false + }, + { + "id": "ecommerce.feature.warehouse-admin", + "exceeded": false + }, + { + "id": "ecommerce.l2.inventory-dashboard", + "exceeded": false + }, + { + "id": "ecommerce.l2.order-cancellation-features", + "exceeded": false + }, + { + "id": "ecommerce.l2.recommendations", + "exceeded": false + }, + { + "id": "ecommerce.l2.sales-dashboard", + "exceeded": false + }, + { + "id": "ecommerce.l2.stock-transfers-features", + "exceeded": false + }, + { + "id": "ecommerce.l3.deferred-access-specifications", + "exceeded": false + }, + { + "id": "ecommerce.l3.deferred-durability-specifications", + "exceeded": false + }, + { + "id": "ecommerce.l3.deferred-integrity-specifications", + "exceeded": false + }, + { + "id": "ecommerce.l3.scheduled-restocks-features", + "exceeded": false + }, + { + "id": "ecommerce.l3.server-time-specifications", + "exceeded": false + }, + { + "id": "ecommerce.progression.cancellation-accounting-specifications", + "exceeded": false + }, + { + "id": "ecommerce.progression.catalog-management", + "exceeded": false + }, + { + "id": "ecommerce.progression.customer-profile", + "exceeded": false + }, + { + "id": "ecommerce.progression.faceted-search", + "exceeded": false + }, + { + "id": "ecommerce.progression.fulfilment-queue", + "exceeded": false + }, + { + "id": "ecommerce.progression.inventory-conservation-specifications", + "exceeded": false + }, + { + "id": "ecommerce.progression.managed-support", + "exceeded": false + }, + { + "id": "ecommerce.progression.notification-preferences", + "exceeded": false + }, + { + "id": "ecommerce.progression.operations-access-specifications", + "exceeded": false + }, + { + "id": "ecommerce.progression.promotion-rules", + "exceeded": false + }, + { + "id": "ecommerce.progression.staff-access", + "exceeded": false + }, + { + "id": "ecommerce.progression.staff-roles", + "exceeded": false + }, + { + "id": "ecommerce.progression.stock-alerts", + "exceeded": false + }, + { + "id": "ecommerce.progression.support-history", + "exceeded": false + }, + { + "id": "ecommerce.progression.support-intake", + "exceeded": false + }, + { + "id": "ecommerce.progression.support-triage", + "exceeded": false + }, + { + "id": "ecommerce.spec.access-control", + "exceeded": false + }, + { + "id": "ecommerce.spec.concurrency-safety", + "exceeded": false + }, + { + "id": "ecommerce.spec.external-data-sync", + "exceeded": false + }, + { + "id": "ecommerce.spec.live-state", + "exceeded": false + }, + { + "id": "ecommerce.spec.state-durability", + "exceeded": false + }, + { + "id": "ecommerce.spec.transactional-integrity", + "exceeded": false + } + ] + }, + "mutations": null + } + ], + "stable": true, + "sameImage": true, + "sameHarness": true, + "harnessSha256": "55130157ba3c8b6364c6e8f0a9453479a4d8c7ecaa50cfaec92953bde488b13e", + "ok": true + } +} diff --git a/tools/stack-bench/qualification-evidence/ecommerce-progression-depth3-2.0.1/null.json b/tools/stack-bench/qualification-evidence/ecommerce-progression-depth3-2.0.1/null.json new file mode 100644 index 00000000000..583729e54a7 --- /dev/null +++ b/tools/stack-bench/qualification-evidence/ecommerce-progression-depth3-2.0.1/null.json @@ -0,0 +1,1064 @@ +{ + "artifactSchemaVersion": 2, + "kind": "null_control", + "id": "null-control-2026-08-31T09-08-27-453Z", + "attempt": { + "id": "null-control-2026-08-31T09-08-27-453Z", + "parentId": null + }, + "timestamps": { + "startedAt": "2026-08-31T09:08:27.454Z", + "completedAt": "2026-08-31T09:31:55.804Z" + }, + "identities": { + "engine": { + "id": "stack-bench", + "version": null, + "sha256": "39da14d55d1ca82f41d18305dcb3e86aa7d5098c69a2bf6ba74533affbfbecc9", + "state": null + }, + "recipe": { + "id": "ecommerce.progression-depth3", + "version": "2.0.1", + "sha256": "4c808d5e5fb076ff0993937d7a9f940dad4191351c362518e1c667a34324bd49", + "state": "qualified" + }, + "fixture": null, + "calibration": { + "id": "ecommerce.progression-depth3-calibration", + "version": "2.0.1", + "sha256": "dd91493c5066018cbfd949d6e14d64dc029deb634e93d796752caee3da97682c", + "state": "draft" + }, + "experiment": null, + "agentAdapter": null, + "stackAdapter": null, + "packs": [] + }, + "payload": { + "runner": { + "schemaVersion": 1, + "mode": "appliance", + "platform": "linux", + "architecture": "x64", + "dockerEngineVersion": "29.6.2", + "dockerOs": "linux", + "dockerArchitecture": "x86_64", + "kernelVersion": "6.6.87.2-microsoft-standard-WSL2", + "cpuCount": 32, + "memoryBytes": 49232838656 + }, + "qualificationScope": { + "checksSha256": "a982217993be2d229e13b4006427ce15666263d1d6e333eb241c4431df59cdf7", + "executableSha256": "58b6187b453a4229b1ff3f403e4d38765468eabc487e137d6f789ee125cbc91a", + "kind": "null", + "mutationSha256": null, + "recipe": { + "contentSha256": "4c808d5e5fb076ff0993937d7a9f940dad4191351c362518e1c667a34324bd49", + "id": "ecommerce.progression-depth3", + "version": "2.0.1" + }, + "schemaVersion": 2, + "stack": null, + "sha256": "74b60e3dbd22ab3ecf33746db29354e5bd065ce8f0d644ddd390956d12ab802b" + }, + "tracks": [ + "ecommerce" + ], + "ok": true, + "summary": { + "criteria": 97, + "points": 162, + "expectedFailures": { + "criteria": 97, + "points": 162 + }, + "vacuousPasses": { + "criteria": 0, + "points": 0 + }, + "oracleGaps": { + "criteria": 0, + "points": 0 + }, + "unscored": { + "criteria": 0, + "passed": 0, + "failed": 0, + "inconclusive": 0 + } + }, + "criteria": [ + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/01-account-create-2.4.0.json", + "feature": 1, + "criterion": "1a", + "points": 1, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/01-account-duplicate-2.4.0.json", + "feature": 1, + "criterion": "1b", + "points": 1, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/01-account-password-2.4.0.json", + "feature": 1, + "criterion": "1c", + "points": 1, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/01-account-reload-2.4.0.json", + "feature": 1, + "criterion": "1e", + "points": 1, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/01-account-signout-2.4.0.json", + "feature": 1, + "criterion": "1d", + "points": 1, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/01-admin-write-staff-1.0.0.json", + "feature": 103, + "criterion": "103a", + "points": 2, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/01-buying-2.4.0.json", + "feature": 3, + "criterion": "3b", + "points": 2, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/01-cart-2.4.0.json", + "feature": 4, + "criterion": "4b", + "points": 1, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/01-cart-2.4.0.json", + "feature": 4, + "criterion": "4c", + "points": 2, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/01-cart-boundary-2.4.0.json", + "feature": 109, + "criterion": "109a", + "points": 2, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/01-cart-boundary-2.4.0.json", + "feature": 109, + "criterion": "109b", + "points": 1, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/01-catalog-ranking-1.0.0.json", + "feature": 2, + "criterion": "2b", + "points": 1, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/01-catalog-search-1.0.0.json", + "feature": 2, + "criterion": "2d", + "points": 1, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/01-catalog-values-1.0.0.json", + "feature": 2, + "criterion": "2a", + "points": 1, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/01-core-2.4.0.json", + "feature": 2, + "criterion": "2c", + "points": 1, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/01-duplicate-checkout-2.3.1.json", + "feature": 203, + "criterion": "203a", + "points": 1, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/01-duplicate-checkout-2.3.1.json", + "feature": 203, + "criterion": "203b", + "points": 1, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/01-external-live-sync-1.1.0.json", + "feature": 901, + "criterion": "901a", + "points": 1, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/01-external-reconnect-sync-1.1.0.json", + "feature": 901, + "criterion": "901d", + "points": 1, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/01-external-server-restart-sync-1.1.0.json", + "feature": 901, + "criterion": "901c", + "points": 1, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/01-last-unit-2.3.1.json", + "feature": 201, + "criterion": "201a", + "points": 1, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/01-last-unit-2.3.1.json", + "feature": 201, + "criterion": "201b", + "points": 1, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/01-last-unit-2.3.1.json", + "feature": 201, + "criterion": "201c", + "points": 1, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/01-order-ownership-2.4.0.json", + "feature": 106, + "criterion": "106a", + "points": 1, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/01-purchase-attribution-2.4.0.json", + "feature": 102, + "criterion": "102a", + "points": 2, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/01-purchase-session-2.4.0.json", + "feature": 101, + "criterion": "101a", + "points": 2, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/01-restock-race-2.3.0.json", + "feature": 202, + "criterion": "202a", + "points": 1, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/01-review-eligibility-2.4.0.json", + "feature": 108, + "criterion": "108a", + "points": 2, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/01-review-eligibility-2.4.0.json", + "feature": 108, + "criterion": "108b", + "points": 1, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/01-review-rating-live-2.4.0.json", + "feature": 6, + "criterion": "6c", + "points": 1, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/01-review-uniqueness-2.4.0.json", + "feature": 6, + "criterion": "6b", + "points": 1, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/01-review-visibility-2.4.0.json", + "feature": 6, + "criterion": "6a", + "points": 2, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/01-server-price-2.4.0.json", + "feature": 104, + "criterion": "104a", + "points": 2, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/01-warehouse-admin-staff-1.0.0.json", + "feature": 7, + "criterion": "7a", + "points": 1, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/01-warehouse-admin-staff-1.0.0.json", + "feature": 7, + "criterion": "7b", + "points": 1, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/01-warehouse-stock-live-staff-1.0.0.json", + "feature": 7, + "criterion": "7c", + "points": 2, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/02-fulfilment-access-1.0.0.json", + "feature": 1, + "criterion": "1d", + "points": 1, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/02-fulfilment-live-1.0.0.json", + "feature": 1, + "criterion": "1a", + "points": 2, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/02-fulfilment-ship-1.0.0.json", + "feature": 1, + "criterion": "1c", + "points": 2, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/02-invariants.json", + "feature": 203, + "criterion": "203a", + "points": 3, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/02-low-stock-1.4.0.json", + "feature": 5, + "criterion": "5a", + "points": 3, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/02-operational-best-sellers-1.0.0.json", + "feature": 5, + "criterion": "5d", + "points": 1, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/02-operational-category-totals-1.0.0.json", + "feature": 5, + "criterion": "5b", + "points": 3, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/02-operational-recommendations-1.0.0.json", + "feature": 5, + "criterion": "5c", + "points": 2, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/02-order-cancellation-core-1.0.0.json", + "feature": 3, + "criterion": "3a", + "points": 2, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/02-order-cancellation-history-1.0.0.json", + "feature": 3, + "criterion": "3b", + "points": 1, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/02-queue-warehouse-1.4.0.json", + "feature": 1, + "criterion": "1b", + "points": 1, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/02-self-contained-1.4.0.json", + "feature": 202, + "criterion": "202b", + "points": 2, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/02-self-contained-1.4.0.json", + "feature": 202, + "criterion": "202c", + "points": 1, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/02-server-actions-1.1.0.json", + "feature": 201, + "criterion": "201c", + "points": 2, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/02-server-actions-1.1.0.json", + "feature": 202, + "criterion": "202d", + "points": 2, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/02-server-actions-1.1.0.json", + "feature": 204, + "criterion": "204a", + "points": 2, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/02-strengthened-1.4.0.json", + "feature": 2, + "criterion": "2a", + "points": 3, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/02-strengthened-1.4.0.json", + "feature": 2, + "criterion": "2c", + "points": 2, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/02-strengthened-1.4.0.json", + "feature": 201, + "criterion": "201a", + "points": 2, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/02-strengthened-1.4.0.json", + "feature": 202, + "criterion": "202a", + "points": 2, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/02-transfer-totals-1.4.0.json", + "feature": 2, + "criterion": "2b", + "points": 2, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/03-deferred-access-1.0.0.json", + "feature": 317, + "criterion": "317a", + "points": 3, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/03-deferred-durability-1.0.0.json", + "feature": 311, + "criterion": "311a", + "points": 4, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/03-deferred-integrity-1.0.0.json", + "feature": 311, + "criterion": "311a", + "points": 3, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/03-scheduled-restocks-1.0.0.json", + "feature": 302, + "criterion": "302a", + "points": 2, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/03-scheduled-restocks-1.0.0.json", + "feature": 305, + "criterion": "305a", + "points": 3, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/03-scheduled-restocks-1.0.0.json", + "feature": 306, + "criterion": "306a", + "points": 2, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/03-server-time-1.0.0.json", + "feature": 312, + "criterion": "312a", + "points": 3, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/progression-account-state-reconnect-1.0.0.json", + "feature": 105, + "criterion": "105b", + "points": 1, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/progression-account-state-reload-1.0.1.json", + "feature": 105, + "criterion": "105a", + "points": 1, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/progression-books-balance-1.0.0.json", + "feature": 107, + "criterion": "107a", + "points": 1, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/progression-books-balance-1.0.0.json", + "feature": 107, + "criterion": "107b", + "points": 1, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/progression-cart-checkout-1.0.0.json", + "feature": 4, + "criterion": "4a", + "points": 1, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/progression-cart-checkout-1.0.0.json", + "feature": 4, + "criterion": "4d", + "points": 2, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/progression-catalog-management-1.0.1.json", + "feature": 622, + "criterion": "622a", + "points": 2, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/progression-catalog-management-1.0.1.json", + "feature": 622, + "criterion": "622b", + "points": 2, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/progression-customer-profile-1.0.0.json", + "feature": 620, + "criterion": "620a", + "points": 2, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/progression-customer-profile-1.0.0.json", + "feature": 620, + "criterion": "620b", + "points": 2, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/progression-faceted-filters-1.0.0.json", + "feature": 401, + "criterion": "401a", + "points": 3, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/progression-faceted-pagination-1.0.0.json", + "feature": 402, + "criterion": "402a", + "points": 3, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/progression-managed-support-privacy-1.0.0.json", + "feature": 613, + "criterion": "613b", + "points": 1, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/progression-managed-support-shared-1.0.0.json", + "feature": 613, + "criterion": "613a", + "points": 2, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/progression-notification-preferences-1.0.0.json", + "feature": 630, + "criterion": "630a", + "points": 2, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/progression-notification-preferences-1.0.0.json", + "feature": 630, + "criterion": "630b", + "points": 2, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/progression-open-list-live-1.0.0.json", + "feature": 902, + "criterion": "902a", + "points": 1, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/progression-promotion-rules-1.0.0.json", + "feature": 620, + "criterion": "620a", + "points": 2, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/progression-promotion-rules-1.0.0.json", + "feature": 620, + "criterion": "620b", + "points": 2, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/progression-purchasing-1.0.0.json", + "feature": 3, + "criterion": "3c", + "points": 1, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/progression-signed-out-purchase-1.0.0.json", + "feature": 3, + "criterion": "3a", + "points": 1, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/progression-staff-access-1.0.0.json", + "feature": 601, + "criterion": "601a", + "points": 2, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/progression-staff-access-1.0.0.json", + "feature": 601, + "criterion": "601b", + "points": 2, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/progression-staff-roles-1.0.0.json", + "feature": 621, + "criterion": "621a", + "points": 2, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/progression-staff-roles-1.0.0.json", + "feature": 621, + "criterion": "621b", + "points": 2, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/progression-stock-alerts-1.0.0.json", + "feature": 631, + "criterion": "631a", + "points": 2, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/progression-stock-alerts-1.0.0.json", + "feature": 631, + "criterion": "631b", + "points": 1, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/progression-support-history-1.0.0.json", + "feature": 612, + "criterion": "612a", + "points": 2, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/progression-support-history-1.0.0.json", + "feature": 612, + "criterion": "612b", + "points": 2, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/progression-support-intake-1.0.0.json", + "feature": 610, + "criterion": "610a", + "points": 2, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/progression-support-triage-1.0.0.json", + "feature": 611, + "criterion": "611a", + "points": 1, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/progression-support-triage-1.0.0.json", + "feature": 611, + "criterion": "611b", + "points": 1, + "status": "expected_fail", + "evidenceStatus": "failed" + }, + { + "track": "ecommerce", + "level": 3, + "scenario": "scenarios/progression-support-triage-1.0.0.json", + "feature": 611, + "criterion": "611c", + "points": 1, + "status": "expected_fail", + "evidenceStatus": "failed" + } + ] + } +} diff --git a/tools/stack-bench/qualification-evidence/ecommerce-progression-depth3-2.0.1/postgres-mutation.json b/tools/stack-bench/qualification-evidence/ecommerce-progression-depth3-2.0.1/postgres-mutation.json new file mode 100644 index 00000000000..7049527c39e --- /dev/null +++ b/tools/stack-bench/qualification-evidence/ecommerce-progression-depth3-2.0.1/postgres-mutation.json @@ -0,0 +1,375 @@ +{ + "artifactSchemaVersion": 2, + "kind": "reference_qualification", + "id": "reference-live-postgres-20260831090828-14", + "attempt": { + "id": "reference-live-postgres-20260831090828-14", + "parentId": null + }, + "timestamps": { + "startedAt": "2026-08-31T09:08:28.070Z", + "completedAt": "2026-08-31T09:32:37.415Z" + }, + "identities": { + "engine": { + "id": "stack-bench", + "version": null, + "sha256": "39da14d55d1ca82f41d18305dcb3e86aa7d5098c69a2bf6ba74533affbfbecc9", + "state": null + }, + "recipe": { + "id": "ecommerce.progression-depth3", + "version": "2.0.1", + "sha256": "4c808d5e5fb076ff0993937d7a9f940dad4191351c362518e1c667a34324bd49", + "state": "qualified" + }, + "fixture": { + "id": "ecommerce-reference-postgres", + "version": null, + "sha256": "f8f6152cebd68acc2af46a3e4cbf208664db17a7db58e0276d8e10ea6750aaf5", + "state": "active" + }, + "calibration": { + "id": "ecommerce.progression-depth3-calibration", + "version": "2.0.1", + "sha256": "dd91493c5066018cbfd949d6e14d64dc029deb634e93d796752caee3da97682c", + "state": "draft" + }, + "experiment": null, + "agentAdapter": null, + "stackAdapter": { + "id": "postgres", + "version": null, + "sha256": null, + "state": null + }, + "packs": [] + }, + "payload": { + "fixture": "ecommerce-reference-postgres", + "fixtureSha256": "f8f6152cebd68acc2af46a3e4cbf208664db17a7db58e0276d8e10ea6750aaf5", + "requiredRepetitions": 1, + "isolation": "docker", + "runner": { + "schemaVersion": 1, + "mode": "appliance", + "platform": "linux", + "architecture": "x64", + "dockerEngineVersion": "29.6.2", + "dockerOs": "linux", + "dockerArchitecture": "x86_64", + "kernelVersion": "6.6.87.2-microsoft-standard-WSL2", + "cpuCount": 32, + "memoryBytes": 49232838656 + }, + "qualificationScope": { + "checksSha256": "a982217993be2d229e13b4006427ce15666263d1d6e333eb241c4431df59cdf7", + "executableSha256": "d8d3a75d12a7b2fedcffdfd11ab76b1918005335ca8f6fb56f101b691d4002ad", + "kind": "mutation", + "mutationSha256": "dfc4af2d598882ef734492c81f4ff76972a249c11b762fd42e32eba93784267f", + "recipe": { + "contentSha256": "4c808d5e5fb076ff0993937d7a9f940dad4191351c362518e1c667a34324bd49", + "id": "ecommerce.progression-depth3", + "version": "2.0.1" + }, + "schemaVersion": 2, + "stack": { + "id": "postgres", + "reference": { + "id": "ecommerce-reference-postgres", + "sourceSha256": "f8f6152cebd68acc2af46a3e4cbf208664db17a7db58e0276d8e10ea6750aaf5" + }, + "version": "1.3.0" + }, + "sha256": "e53420120b3f74ee9267baaf4ac4e3714de205fb992b79af937b6424f941b709" + }, + "mutationControl": true, + "qualifiedCheckKeys": [ + "ecommerce.feature.accounts.accounts.1a", + "ecommerce.feature.accounts.accounts.1b", + "ecommerce.feature.accounts.accounts.1c", + "ecommerce.feature.accounts.accounts.1d", + "ecommerce.feature.cart-checkout.cart.4a", + "ecommerce.feature.cart-checkout.cart.4d", + "ecommerce.feature.catalog.catalog-ranking.2b", + "ecommerce.feature.catalog.catalog-search.2d", + "ecommerce.feature.catalog.catalog-values.2a", + "ecommerce.feature.purchasing.purchase-order.3c", + "ecommerce.feature.reviews.reviews.6a", + "ecommerce.feature.warehouse-admin.access-boundary.7a", + "ecommerce.feature.warehouse-admin.admin-write.103a", + "ecommerce.feature.warehouse-admin.warehouse-stock.7c", + "ecommerce.feature.warehouse-admin.warehouse-view.7b", + "ecommerce.inventory-operations.operational-views.5a", + "ecommerce.inventory-operations.operational-views.5b", + "ecommerce.inventory-operations.operational-views.5c", + "ecommerce.inventory-operations.operational-views.5d", + "ecommerce.inventory-operations.stock-conservation.202a", + "ecommerce.inventory-operations.stock-conservation.202b", + "ecommerce.inventory-operations.stock-conservation.202c", + "ecommerce.inventory-operations.stock-conservation.202d", + "ecommerce.inventory-operations.warehouse-transfer.2a", + "ecommerce.inventory-operations.warehouse-transfer.2b", + "ecommerce.inventory-operations.warehouse-transfer.2c", + "ecommerce.l3.deferred-access.scheduled-work-access.317a", + "ecommerce.l3.deferred-durability.restart-survival.311a", + "ecommerce.l3.deferred-integrity.exactly-once.311a", + "ecommerce.l3.scheduled-restocks.scheduled-restocks.302a", + "ecommerce.l3.scheduled-restocks.scheduled-restocks.305a", + "ecommerce.l3.scheduled-restocks.scheduled-restocks.306a", + "ecommerce.l3.server-time.server-time.312a", + "ecommerce.operations-access.fulfilment-queue.1a", + "ecommerce.operations-access.fulfilment-queue.1b", + "ecommerce.operations-access.fulfilment-queue.1c", + "ecommerce.operations-access.fulfilment-queue.1d", + "ecommerce.operations-access.operator-authorization.201a", + "ecommerce.operations-access.operator-authorization.201c", + "ecommerce.operations-access.order-owner.204a", + "ecommerce.progression.catalog-management.catalog-management.622a", + "ecommerce.progression.catalog-management.catalog-management.622b", + "ecommerce.progression.customer-profile.customer-profile.620a", + "ecommerce.progression.customer-profile.customer-profile.620b", + "ecommerce.progression.faceted-search.faceted-search.401a", + "ecommerce.progression.faceted-search.faceted-search.402a", + "ecommerce.progression.managed-support.managed-support-privacy.613b", + "ecommerce.progression.managed-support.managed-support-shared.613a", + "ecommerce.progression.notification-preferences.notification-preferences-persistence.630a", + "ecommerce.progression.notification-preferences.notification-preferences-privacy.630b", + "ecommerce.progression.promotion-rules.promotion-rule-access.620b", + "ecommerce.progression.promotion-rules.promotion-rule-values.620a", + "ecommerce.progression.staff-access.staff-access.601a", + "ecommerce.progression.staff-access.staff-access.601b", + "ecommerce.progression.staff-roles.staff-roles.621a", + "ecommerce.progression.staff-roles.staff-roles.621b", + "ecommerce.progression.stock-alerts.stock-alert-delivery.631a", + "ecommerce.progression.stock-alerts.stock-alert-privacy.631b", + "ecommerce.progression.support-history.support-history-persistence.612a", + "ecommerce.progression.support-history.support-history-privacy.612b", + "ecommerce.progression.support-intake.support-intake.610a", + "ecommerce.progression.support-triage.support-assignment.611a", + "ecommerce.progression.support-triage.support-priority.611b", + "ecommerce.progression.support-triage.support-status.611c", + "ecommerce.returns-pricing.cancellation-and-return.3a", + "ecommerce.returns-pricing.cancellation-and-return.3b", + "ecommerce.returns-pricing.refund-accounting.203a", + "ecommerce.spec.access-control.cart-boundary.109a", + "ecommerce.spec.access-control.cart-boundary.109b", + "ecommerce.spec.access-control.order-ownership.106a", + "ecommerce.spec.access-control.purchase-attribution.102a", + "ecommerce.spec.access-control.purchase-session.101a", + "ecommerce.spec.access-control.review-eligibility.108a", + "ecommerce.spec.access-control.review-eligibility.108b", + "ecommerce.spec.access-control.signed-out-purchase.3a", + "ecommerce.spec.concurrency-safety.duplicate-checkout.203a", + "ecommerce.spec.concurrency-safety.duplicate-checkout.203b", + "ecommerce.spec.concurrency-safety.last-unit.201a", + "ecommerce.spec.concurrency-safety.last-unit.201b", + "ecommerce.spec.concurrency-safety.last-unit.201c", + "ecommerce.spec.concurrency-safety.restock-race.202a", + "ecommerce.spec.external-data-sync.external-stock.901a", + "ecommerce.spec.external-data-sync.external-stock.901c", + "ecommerce.spec.external-data-sync.external-stock.901d", + "ecommerce.spec.live-state.open-list.902a", + "ecommerce.spec.live-state.purchase-stock.3b", + "ecommerce.spec.live-state.ranking.2c", + "ecommerce.spec.live-state.rating.6c", + "ecommerce.spec.live-state.shared-cart.4c", + "ecommerce.spec.state-durability.account-state-recovery.105a", + "ecommerce.spec.state-durability.account-state-recovery.105b", + "ecommerce.spec.state-durability.cart-reload.4b", + "ecommerce.spec.state-durability.session-reload.1e", + "ecommerce.spec.transactional-integrity.books-balance.107a", + "ecommerce.spec.transactional-integrity.books-balance.107b", + "ecommerce.spec.transactional-integrity.server-price.104a", + "ecommerce.spec.transactional-integrity.unique-review.6b" + ], + "runs": [ + { + "repetition": 1, + "processError": null, + "harnessSha256Before": "55130157ba3c8b6364c6e8f0a9453479a4d8c7ecaa50cfaec92953bde488b13e", + "harnessSha256After": "55130157ba3c8b6364c6e8f0a9453479a4d8c7ecaa50cfaec92953bde488b13e", + "ok": true, + "failures": [], + "score": "162/162", + "imageId": "sha256:eccec6ea3861befc3ef0a05beb3f8ac13e0af2d0a1ef77c1a7beb3c0eb2aa075", + "criteria": 97, + "zeroPointCriteria": 0, + "outcome": "passed", + "packRuntime": { + "packs": [ + { + "id": "ecommerce.feature.accounts", + "exceeded": false + }, + { + "id": "ecommerce.feature.cart", + "exceeded": false + }, + { + "id": "ecommerce.feature.catalog-discovery", + "exceeded": false + }, + { + "id": "ecommerce.feature.catalog-items", + "exceeded": false + }, + { + "id": "ecommerce.feature.checkout", + "exceeded": false + }, + { + "id": "ecommerce.feature.purchasing", + "exceeded": false + }, + { + "id": "ecommerce.feature.reviews", + "exceeded": false + }, + { + "id": "ecommerce.feature.warehouse-admin", + "exceeded": false + }, + { + "id": "ecommerce.l2.inventory-dashboard", + "exceeded": false + }, + { + "id": "ecommerce.l2.order-cancellation-features", + "exceeded": false + }, + { + "id": "ecommerce.l2.recommendations", + "exceeded": false + }, + { + "id": "ecommerce.l2.sales-dashboard", + "exceeded": false + }, + { + "id": "ecommerce.l2.stock-transfers-features", + "exceeded": false + }, + { + "id": "ecommerce.l3.deferred-access-specifications", + "exceeded": false + }, + { + "id": "ecommerce.l3.deferred-durability-specifications", + "exceeded": false + }, + { + "id": "ecommerce.l3.deferred-integrity-specifications", + "exceeded": false + }, + { + "id": "ecommerce.l3.scheduled-restocks-features", + "exceeded": false + }, + { + "id": "ecommerce.l3.server-time-specifications", + "exceeded": false + }, + { + "id": "ecommerce.progression.cancellation-accounting-specifications", + "exceeded": false + }, + { + "id": "ecommerce.progression.catalog-management", + "exceeded": false + }, + { + "id": "ecommerce.progression.customer-profile", + "exceeded": false + }, + { + "id": "ecommerce.progression.faceted-search", + "exceeded": false + }, + { + "id": "ecommerce.progression.fulfilment-queue", + "exceeded": false + }, + { + "id": "ecommerce.progression.inventory-conservation-specifications", + "exceeded": false + }, + { + "id": "ecommerce.progression.managed-support", + "exceeded": false + }, + { + "id": "ecommerce.progression.notification-preferences", + "exceeded": false + }, + { + "id": "ecommerce.progression.operations-access-specifications", + "exceeded": false + }, + { + "id": "ecommerce.progression.promotion-rules", + "exceeded": false + }, + { + "id": "ecommerce.progression.staff-access", + "exceeded": false + }, + { + "id": "ecommerce.progression.staff-roles", + "exceeded": false + }, + { + "id": "ecommerce.progression.stock-alerts", + "exceeded": false + }, + { + "id": "ecommerce.progression.support-history", + "exceeded": false + }, + { + "id": "ecommerce.progression.support-intake", + "exceeded": false + }, + { + "id": "ecommerce.progression.support-triage", + "exceeded": false + }, + { + "id": "ecommerce.spec.access-control", + "exceeded": false + }, + { + "id": "ecommerce.spec.concurrency-safety", + "exceeded": false + }, + { + "id": "ecommerce.spec.external-data-sync", + "exceeded": false + }, + { + "id": "ecommerce.spec.live-state", + "exceeded": false + }, + { + "id": "ecommerce.spec.state-durability", + "exceeded": false + }, + { + "id": "ecommerce.spec.transactional-integrity", + "exceeded": false + } + ] + }, + "mutations": { + "caught": 92, + "total": 92 + } + } + ], + "stable": true, + "sameImage": true, + "sameHarness": true, + "harnessSha256": "55130157ba3c8b6364c6e8f0a9453479a4d8c7ecaa50cfaec92953bde488b13e", + "ok": true + } +} diff --git a/tools/stack-bench/qualification-evidence/ecommerce-progression-depth3-2.0.1/postgres-reference.json b/tools/stack-bench/qualification-evidence/ecommerce-progression-depth3-2.0.1/postgres-reference.json new file mode 100644 index 00000000000..4815a24c19b --- /dev/null +++ b/tools/stack-bench/qualification-evidence/ecommerce-progression-depth3-2.0.1/postgres-reference.json @@ -0,0 +1,372 @@ +{ + "artifactSchemaVersion": 2, + "kind": "reference_qualification", + "id": "reference-live-postgres-20260831090828-14-reference", + "attempt": { + "id": "reference-live-postgres-20260831090828-14-reference", + "parentId": null + }, + "timestamps": { + "startedAt": "2026-08-31T09:08:28.070Z", + "completedAt": "2026-08-31T09:32:37.416Z" + }, + "identities": { + "engine": { + "id": "stack-bench", + "version": null, + "sha256": "39da14d55d1ca82f41d18305dcb3e86aa7d5098c69a2bf6ba74533affbfbecc9", + "state": null + }, + "recipe": { + "id": "ecommerce.progression-depth3", + "version": "2.0.1", + "sha256": "4c808d5e5fb076ff0993937d7a9f940dad4191351c362518e1c667a34324bd49", + "state": "qualified" + }, + "fixture": { + "id": "ecommerce-reference-postgres", + "version": null, + "sha256": "f8f6152cebd68acc2af46a3e4cbf208664db17a7db58e0276d8e10ea6750aaf5", + "state": "active" + }, + "calibration": { + "id": "ecommerce.progression-depth3-calibration", + "version": "2.0.1", + "sha256": "dd91493c5066018cbfd949d6e14d64dc029deb634e93d796752caee3da97682c", + "state": "draft" + }, + "experiment": null, + "agentAdapter": null, + "stackAdapter": { + "id": "postgres", + "version": null, + "sha256": null, + "state": null + }, + "packs": [] + }, + "payload": { + "fixture": "ecommerce-reference-postgres", + "fixtureSha256": "f8f6152cebd68acc2af46a3e4cbf208664db17a7db58e0276d8e10ea6750aaf5", + "requiredRepetitions": 1, + "isolation": "docker", + "runner": { + "schemaVersion": 1, + "mode": "appliance", + "platform": "linux", + "architecture": "x64", + "dockerEngineVersion": "29.6.2", + "dockerOs": "linux", + "dockerArchitecture": "x86_64", + "kernelVersion": "6.6.87.2-microsoft-standard-WSL2", + "cpuCount": 32, + "memoryBytes": 49232838656 + }, + "qualificationScope": { + "checksSha256": "a982217993be2d229e13b4006427ce15666263d1d6e333eb241c4431df59cdf7", + "executableSha256": "10f59cd1c82887afe4a34fe58383e2d195abf5e9850711f28d0e4ef3c7b2dec9", + "kind": "reference", + "mutationSha256": null, + "recipe": { + "contentSha256": "4c808d5e5fb076ff0993937d7a9f940dad4191351c362518e1c667a34324bd49", + "id": "ecommerce.progression-depth3", + "version": "2.0.1" + }, + "schemaVersion": 2, + "stack": { + "id": "postgres", + "reference": { + "id": "ecommerce-reference-postgres", + "sourceSha256": "f8f6152cebd68acc2af46a3e4cbf208664db17a7db58e0276d8e10ea6750aaf5" + }, + "version": "1.3.0" + }, + "sha256": "4ffd5ac32ec6379796867f311f9013178dfb0587c3242d2ba061d554abae2a01" + }, + "mutationControl": false, + "qualifiedCheckKeys": [ + "ecommerce.feature.accounts.accounts.1a", + "ecommerce.feature.accounts.accounts.1b", + "ecommerce.feature.accounts.accounts.1c", + "ecommerce.feature.accounts.accounts.1d", + "ecommerce.feature.cart-checkout.cart.4a", + "ecommerce.feature.cart-checkout.cart.4d", + "ecommerce.feature.catalog.catalog-ranking.2b", + "ecommerce.feature.catalog.catalog-search.2d", + "ecommerce.feature.catalog.catalog-values.2a", + "ecommerce.feature.purchasing.purchase-order.3c", + "ecommerce.feature.reviews.reviews.6a", + "ecommerce.feature.warehouse-admin.access-boundary.7a", + "ecommerce.feature.warehouse-admin.admin-write.103a", + "ecommerce.feature.warehouse-admin.warehouse-stock.7c", + "ecommerce.feature.warehouse-admin.warehouse-view.7b", + "ecommerce.inventory-operations.operational-views.5a", + "ecommerce.inventory-operations.operational-views.5b", + "ecommerce.inventory-operations.operational-views.5c", + "ecommerce.inventory-operations.operational-views.5d", + "ecommerce.inventory-operations.stock-conservation.202a", + "ecommerce.inventory-operations.stock-conservation.202b", + "ecommerce.inventory-operations.stock-conservation.202c", + "ecommerce.inventory-operations.stock-conservation.202d", + "ecommerce.inventory-operations.warehouse-transfer.2a", + "ecommerce.inventory-operations.warehouse-transfer.2b", + "ecommerce.inventory-operations.warehouse-transfer.2c", + "ecommerce.l3.deferred-access.scheduled-work-access.317a", + "ecommerce.l3.deferred-durability.restart-survival.311a", + "ecommerce.l3.deferred-integrity.exactly-once.311a", + "ecommerce.l3.scheduled-restocks.scheduled-restocks.302a", + "ecommerce.l3.scheduled-restocks.scheduled-restocks.305a", + "ecommerce.l3.scheduled-restocks.scheduled-restocks.306a", + "ecommerce.l3.server-time.server-time.312a", + "ecommerce.operations-access.fulfilment-queue.1a", + "ecommerce.operations-access.fulfilment-queue.1b", + "ecommerce.operations-access.fulfilment-queue.1c", + "ecommerce.operations-access.fulfilment-queue.1d", + "ecommerce.operations-access.operator-authorization.201a", + "ecommerce.operations-access.operator-authorization.201c", + "ecommerce.operations-access.order-owner.204a", + "ecommerce.progression.catalog-management.catalog-management.622a", + "ecommerce.progression.catalog-management.catalog-management.622b", + "ecommerce.progression.customer-profile.customer-profile.620a", + "ecommerce.progression.customer-profile.customer-profile.620b", + "ecommerce.progression.faceted-search.faceted-search.401a", + "ecommerce.progression.faceted-search.faceted-search.402a", + "ecommerce.progression.managed-support.managed-support-privacy.613b", + "ecommerce.progression.managed-support.managed-support-shared.613a", + "ecommerce.progression.notification-preferences.notification-preferences-persistence.630a", + "ecommerce.progression.notification-preferences.notification-preferences-privacy.630b", + "ecommerce.progression.promotion-rules.promotion-rule-access.620b", + "ecommerce.progression.promotion-rules.promotion-rule-values.620a", + "ecommerce.progression.staff-access.staff-access.601a", + "ecommerce.progression.staff-access.staff-access.601b", + "ecommerce.progression.staff-roles.staff-roles.621a", + "ecommerce.progression.staff-roles.staff-roles.621b", + "ecommerce.progression.stock-alerts.stock-alert-delivery.631a", + "ecommerce.progression.stock-alerts.stock-alert-privacy.631b", + "ecommerce.progression.support-history.support-history-persistence.612a", + "ecommerce.progression.support-history.support-history-privacy.612b", + "ecommerce.progression.support-intake.support-intake.610a", + "ecommerce.progression.support-triage.support-assignment.611a", + "ecommerce.progression.support-triage.support-priority.611b", + "ecommerce.progression.support-triage.support-status.611c", + "ecommerce.returns-pricing.cancellation-and-return.3a", + "ecommerce.returns-pricing.cancellation-and-return.3b", + "ecommerce.returns-pricing.refund-accounting.203a", + "ecommerce.spec.access-control.cart-boundary.109a", + "ecommerce.spec.access-control.cart-boundary.109b", + "ecommerce.spec.access-control.order-ownership.106a", + "ecommerce.spec.access-control.purchase-attribution.102a", + "ecommerce.spec.access-control.purchase-session.101a", + "ecommerce.spec.access-control.review-eligibility.108a", + "ecommerce.spec.access-control.review-eligibility.108b", + "ecommerce.spec.access-control.signed-out-purchase.3a", + "ecommerce.spec.concurrency-safety.duplicate-checkout.203a", + "ecommerce.spec.concurrency-safety.duplicate-checkout.203b", + "ecommerce.spec.concurrency-safety.last-unit.201a", + "ecommerce.spec.concurrency-safety.last-unit.201b", + "ecommerce.spec.concurrency-safety.last-unit.201c", + "ecommerce.spec.concurrency-safety.restock-race.202a", + "ecommerce.spec.external-data-sync.external-stock.901a", + "ecommerce.spec.external-data-sync.external-stock.901c", + "ecommerce.spec.external-data-sync.external-stock.901d", + "ecommerce.spec.live-state.open-list.902a", + "ecommerce.spec.live-state.purchase-stock.3b", + "ecommerce.spec.live-state.ranking.2c", + "ecommerce.spec.live-state.rating.6c", + "ecommerce.spec.live-state.shared-cart.4c", + "ecommerce.spec.state-durability.account-state-recovery.105a", + "ecommerce.spec.state-durability.account-state-recovery.105b", + "ecommerce.spec.state-durability.cart-reload.4b", + "ecommerce.spec.state-durability.session-reload.1e", + "ecommerce.spec.transactional-integrity.books-balance.107a", + "ecommerce.spec.transactional-integrity.books-balance.107b", + "ecommerce.spec.transactional-integrity.server-price.104a", + "ecommerce.spec.transactional-integrity.unique-review.6b" + ], + "runs": [ + { + "repetition": 1, + "processError": null, + "harnessSha256Before": "55130157ba3c8b6364c6e8f0a9453479a4d8c7ecaa50cfaec92953bde488b13e", + "harnessSha256After": "55130157ba3c8b6364c6e8f0a9453479a4d8c7ecaa50cfaec92953bde488b13e", + "ok": true, + "failures": [], + "score": "162/162", + "imageId": "sha256:eccec6ea3861befc3ef0a05beb3f8ac13e0af2d0a1ef77c1a7beb3c0eb2aa075", + "criteria": 97, + "zeroPointCriteria": 0, + "outcome": "passed", + "packRuntime": { + "packs": [ + { + "id": "ecommerce.feature.accounts", + "exceeded": false + }, + { + "id": "ecommerce.feature.cart", + "exceeded": false + }, + { + "id": "ecommerce.feature.catalog-discovery", + "exceeded": false + }, + { + "id": "ecommerce.feature.catalog-items", + "exceeded": false + }, + { + "id": "ecommerce.feature.checkout", + "exceeded": false + }, + { + "id": "ecommerce.feature.purchasing", + "exceeded": false + }, + { + "id": "ecommerce.feature.reviews", + "exceeded": false + }, + { + "id": "ecommerce.feature.warehouse-admin", + "exceeded": false + }, + { + "id": "ecommerce.l2.inventory-dashboard", + "exceeded": false + }, + { + "id": "ecommerce.l2.order-cancellation-features", + "exceeded": false + }, + { + "id": "ecommerce.l2.recommendations", + "exceeded": false + }, + { + "id": "ecommerce.l2.sales-dashboard", + "exceeded": false + }, + { + "id": "ecommerce.l2.stock-transfers-features", + "exceeded": false + }, + { + "id": "ecommerce.l3.deferred-access-specifications", + "exceeded": false + }, + { + "id": "ecommerce.l3.deferred-durability-specifications", + "exceeded": false + }, + { + "id": "ecommerce.l3.deferred-integrity-specifications", + "exceeded": false + }, + { + "id": "ecommerce.l3.scheduled-restocks-features", + "exceeded": false + }, + { + "id": "ecommerce.l3.server-time-specifications", + "exceeded": false + }, + { + "id": "ecommerce.progression.cancellation-accounting-specifications", + "exceeded": false + }, + { + "id": "ecommerce.progression.catalog-management", + "exceeded": false + }, + { + "id": "ecommerce.progression.customer-profile", + "exceeded": false + }, + { + "id": "ecommerce.progression.faceted-search", + "exceeded": false + }, + { + "id": "ecommerce.progression.fulfilment-queue", + "exceeded": false + }, + { + "id": "ecommerce.progression.inventory-conservation-specifications", + "exceeded": false + }, + { + "id": "ecommerce.progression.managed-support", + "exceeded": false + }, + { + "id": "ecommerce.progression.notification-preferences", + "exceeded": false + }, + { + "id": "ecommerce.progression.operations-access-specifications", + "exceeded": false + }, + { + "id": "ecommerce.progression.promotion-rules", + "exceeded": false + }, + { + "id": "ecommerce.progression.staff-access", + "exceeded": false + }, + { + "id": "ecommerce.progression.staff-roles", + "exceeded": false + }, + { + "id": "ecommerce.progression.stock-alerts", + "exceeded": false + }, + { + "id": "ecommerce.progression.support-history", + "exceeded": false + }, + { + "id": "ecommerce.progression.support-intake", + "exceeded": false + }, + { + "id": "ecommerce.progression.support-triage", + "exceeded": false + }, + { + "id": "ecommerce.spec.access-control", + "exceeded": false + }, + { + "id": "ecommerce.spec.concurrency-safety", + "exceeded": false + }, + { + "id": "ecommerce.spec.external-data-sync", + "exceeded": false + }, + { + "id": "ecommerce.spec.live-state", + "exceeded": false + }, + { + "id": "ecommerce.spec.state-durability", + "exceeded": false + }, + { + "id": "ecommerce.spec.transactional-integrity", + "exceeded": false + } + ] + }, + "mutations": null + } + ], + "stable": true, + "sameImage": true, + "sameHarness": true, + "harnessSha256": "55130157ba3c8b6364c6e8f0a9453479a4d8c7ecaa50cfaec92953bde488b13e", + "ok": true + } +} diff --git a/tools/stack-bench/qualification-evidence/ecommerce-progression-depth3-2.0.1/spacetime-mutation.json b/tools/stack-bench/qualification-evidence/ecommerce-progression-depth3-2.0.1/spacetime-mutation.json new file mode 100644 index 00000000000..6fde26cc811 --- /dev/null +++ b/tools/stack-bench/qualification-evidence/ecommerce-progression-depth3-2.0.1/spacetime-mutation.json @@ -0,0 +1,375 @@ +{ + "artifactSchemaVersion": 2, + "kind": "reference_qualification", + "id": "reference-live-spacetime-20260831090827-14", + "attempt": { + "id": "reference-live-spacetime-20260831090827-14", + "parentId": null + }, + "timestamps": { + "startedAt": "2026-08-31T09:08:27.717Z", + "completedAt": "2026-08-31T09:29:23.477Z" + }, + "identities": { + "engine": { + "id": "stack-bench", + "version": null, + "sha256": "39da14d55d1ca82f41d18305dcb3e86aa7d5098c69a2bf6ba74533affbfbecc9", + "state": null + }, + "recipe": { + "id": "ecommerce.progression-depth3", + "version": "2.0.1", + "sha256": "4c808d5e5fb076ff0993937d7a9f940dad4191351c362518e1c667a34324bd49", + "state": "qualified" + }, + "fixture": { + "id": "ecommerce-reference-spacetime", + "version": null, + "sha256": "8806e01bcd4d44fa7c7c491f722c2412d568605a9d666545dafcc0bdf2a2b4f5", + "state": "active" + }, + "calibration": { + "id": "ecommerce.progression-depth3-calibration", + "version": "2.0.1", + "sha256": "dd91493c5066018cbfd949d6e14d64dc029deb634e93d796752caee3da97682c", + "state": "draft" + }, + "experiment": null, + "agentAdapter": null, + "stackAdapter": { + "id": "spacetime", + "version": null, + "sha256": null, + "state": null + }, + "packs": [] + }, + "payload": { + "fixture": "ecommerce-reference-spacetime", + "fixtureSha256": "8806e01bcd4d44fa7c7c491f722c2412d568605a9d666545dafcc0bdf2a2b4f5", + "requiredRepetitions": 1, + "isolation": "docker", + "runner": { + "schemaVersion": 1, + "mode": "appliance", + "platform": "linux", + "architecture": "x64", + "dockerEngineVersion": "29.6.2", + "dockerOs": "linux", + "dockerArchitecture": "x86_64", + "kernelVersion": "6.6.87.2-microsoft-standard-WSL2", + "cpuCount": 32, + "memoryBytes": 49232838656 + }, + "qualificationScope": { + "checksSha256": "a982217993be2d229e13b4006427ce15666263d1d6e333eb241c4431df59cdf7", + "executableSha256": "3d512023e270c2b55c1c1eeeee70b1b6d70261814754e24b618f5c66bd086a0a", + "kind": "mutation", + "mutationSha256": "75829fe1839442ba6f4c4385ca89583ac7e287b077878850d463615b19752cb9", + "recipe": { + "contentSha256": "4c808d5e5fb076ff0993937d7a9f940dad4191351c362518e1c667a34324bd49", + "id": "ecommerce.progression-depth3", + "version": "2.0.1" + }, + "schemaVersion": 2, + "stack": { + "id": "spacetime", + "reference": { + "id": "ecommerce-reference-spacetime", + "sourceSha256": "8806e01bcd4d44fa7c7c491f722c2412d568605a9d666545dafcc0bdf2a2b4f5" + }, + "version": "1.0.0" + }, + "sha256": "9666859fecd693bca13467e59a649fc286ca42a1f9ed118073d32effd43129c5" + }, + "mutationControl": true, + "qualifiedCheckKeys": [ + "ecommerce.feature.accounts.accounts.1a", + "ecommerce.feature.accounts.accounts.1b", + "ecommerce.feature.accounts.accounts.1c", + "ecommerce.feature.accounts.accounts.1d", + "ecommerce.feature.cart-checkout.cart.4a", + "ecommerce.feature.cart-checkout.cart.4d", + "ecommerce.feature.catalog.catalog-ranking.2b", + "ecommerce.feature.catalog.catalog-search.2d", + "ecommerce.feature.catalog.catalog-values.2a", + "ecommerce.feature.purchasing.purchase-order.3c", + "ecommerce.feature.reviews.reviews.6a", + "ecommerce.feature.warehouse-admin.access-boundary.7a", + "ecommerce.feature.warehouse-admin.admin-write.103a", + "ecommerce.feature.warehouse-admin.warehouse-stock.7c", + "ecommerce.feature.warehouse-admin.warehouse-view.7b", + "ecommerce.inventory-operations.operational-views.5a", + "ecommerce.inventory-operations.operational-views.5b", + "ecommerce.inventory-operations.operational-views.5c", + "ecommerce.inventory-operations.operational-views.5d", + "ecommerce.inventory-operations.stock-conservation.202a", + "ecommerce.inventory-operations.stock-conservation.202b", + "ecommerce.inventory-operations.stock-conservation.202c", + "ecommerce.inventory-operations.stock-conservation.202d", + "ecommerce.inventory-operations.warehouse-transfer.2a", + "ecommerce.inventory-operations.warehouse-transfer.2b", + "ecommerce.inventory-operations.warehouse-transfer.2c", + "ecommerce.l3.deferred-access.scheduled-work-access.317a", + "ecommerce.l3.deferred-durability.restart-survival.311a", + "ecommerce.l3.deferred-integrity.exactly-once.311a", + "ecommerce.l3.scheduled-restocks.scheduled-restocks.302a", + "ecommerce.l3.scheduled-restocks.scheduled-restocks.305a", + "ecommerce.l3.scheduled-restocks.scheduled-restocks.306a", + "ecommerce.l3.server-time.server-time.312a", + "ecommerce.operations-access.fulfilment-queue.1a", + "ecommerce.operations-access.fulfilment-queue.1b", + "ecommerce.operations-access.fulfilment-queue.1c", + "ecommerce.operations-access.fulfilment-queue.1d", + "ecommerce.operations-access.operator-authorization.201a", + "ecommerce.operations-access.operator-authorization.201c", + "ecommerce.operations-access.order-owner.204a", + "ecommerce.progression.catalog-management.catalog-management.622a", + "ecommerce.progression.catalog-management.catalog-management.622b", + "ecommerce.progression.customer-profile.customer-profile.620a", + "ecommerce.progression.customer-profile.customer-profile.620b", + "ecommerce.progression.faceted-search.faceted-search.401a", + "ecommerce.progression.faceted-search.faceted-search.402a", + "ecommerce.progression.managed-support.managed-support-privacy.613b", + "ecommerce.progression.managed-support.managed-support-shared.613a", + "ecommerce.progression.notification-preferences.notification-preferences-persistence.630a", + "ecommerce.progression.notification-preferences.notification-preferences-privacy.630b", + "ecommerce.progression.promotion-rules.promotion-rule-access.620b", + "ecommerce.progression.promotion-rules.promotion-rule-values.620a", + "ecommerce.progression.staff-access.staff-access.601a", + "ecommerce.progression.staff-access.staff-access.601b", + "ecommerce.progression.staff-roles.staff-roles.621a", + "ecommerce.progression.staff-roles.staff-roles.621b", + "ecommerce.progression.stock-alerts.stock-alert-delivery.631a", + "ecommerce.progression.stock-alerts.stock-alert-privacy.631b", + "ecommerce.progression.support-history.support-history-persistence.612a", + "ecommerce.progression.support-history.support-history-privacy.612b", + "ecommerce.progression.support-intake.support-intake.610a", + "ecommerce.progression.support-triage.support-assignment.611a", + "ecommerce.progression.support-triage.support-priority.611b", + "ecommerce.progression.support-triage.support-status.611c", + "ecommerce.returns-pricing.cancellation-and-return.3a", + "ecommerce.returns-pricing.cancellation-and-return.3b", + "ecommerce.returns-pricing.refund-accounting.203a", + "ecommerce.spec.access-control.cart-boundary.109a", + "ecommerce.spec.access-control.cart-boundary.109b", + "ecommerce.spec.access-control.order-ownership.106a", + "ecommerce.spec.access-control.purchase-attribution.102a", + "ecommerce.spec.access-control.purchase-session.101a", + "ecommerce.spec.access-control.review-eligibility.108a", + "ecommerce.spec.access-control.review-eligibility.108b", + "ecommerce.spec.access-control.signed-out-purchase.3a", + "ecommerce.spec.concurrency-safety.duplicate-checkout.203a", + "ecommerce.spec.concurrency-safety.duplicate-checkout.203b", + "ecommerce.spec.concurrency-safety.last-unit.201a", + "ecommerce.spec.concurrency-safety.last-unit.201b", + "ecommerce.spec.concurrency-safety.last-unit.201c", + "ecommerce.spec.concurrency-safety.restock-race.202a", + "ecommerce.spec.external-data-sync.external-stock.901a", + "ecommerce.spec.external-data-sync.external-stock.901c", + "ecommerce.spec.external-data-sync.external-stock.901d", + "ecommerce.spec.live-state.open-list.902a", + "ecommerce.spec.live-state.purchase-stock.3b", + "ecommerce.spec.live-state.ranking.2c", + "ecommerce.spec.live-state.rating.6c", + "ecommerce.spec.live-state.shared-cart.4c", + "ecommerce.spec.state-durability.account-state-recovery.105a", + "ecommerce.spec.state-durability.account-state-recovery.105b", + "ecommerce.spec.state-durability.cart-reload.4b", + "ecommerce.spec.state-durability.session-reload.1e", + "ecommerce.spec.transactional-integrity.books-balance.107a", + "ecommerce.spec.transactional-integrity.books-balance.107b", + "ecommerce.spec.transactional-integrity.server-price.104a", + "ecommerce.spec.transactional-integrity.unique-review.6b" + ], + "runs": [ + { + "repetition": 1, + "processError": null, + "harnessSha256Before": "55130157ba3c8b6364c6e8f0a9453479a4d8c7ecaa50cfaec92953bde488b13e", + "harnessSha256After": "55130157ba3c8b6364c6e8f0a9453479a4d8c7ecaa50cfaec92953bde488b13e", + "ok": true, + "failures": [], + "score": "162/162", + "imageId": "sha256:eccec6ea3861befc3ef0a05beb3f8ac13e0af2d0a1ef77c1a7beb3c0eb2aa075", + "criteria": 97, + "zeroPointCriteria": 0, + "outcome": "passed", + "packRuntime": { + "packs": [ + { + "id": "ecommerce.feature.accounts", + "exceeded": false + }, + { + "id": "ecommerce.feature.cart", + "exceeded": false + }, + { + "id": "ecommerce.feature.catalog-discovery", + "exceeded": false + }, + { + "id": "ecommerce.feature.catalog-items", + "exceeded": false + }, + { + "id": "ecommerce.feature.checkout", + "exceeded": false + }, + { + "id": "ecommerce.feature.purchasing", + "exceeded": false + }, + { + "id": "ecommerce.feature.reviews", + "exceeded": false + }, + { + "id": "ecommerce.feature.warehouse-admin", + "exceeded": false + }, + { + "id": "ecommerce.l2.inventory-dashboard", + "exceeded": false + }, + { + "id": "ecommerce.l2.order-cancellation-features", + "exceeded": false + }, + { + "id": "ecommerce.l2.recommendations", + "exceeded": false + }, + { + "id": "ecommerce.l2.sales-dashboard", + "exceeded": false + }, + { + "id": "ecommerce.l2.stock-transfers-features", + "exceeded": false + }, + { + "id": "ecommerce.l3.deferred-access-specifications", + "exceeded": false + }, + { + "id": "ecommerce.l3.deferred-durability-specifications", + "exceeded": false + }, + { + "id": "ecommerce.l3.deferred-integrity-specifications", + "exceeded": false + }, + { + "id": "ecommerce.l3.scheduled-restocks-features", + "exceeded": false + }, + { + "id": "ecommerce.l3.server-time-specifications", + "exceeded": false + }, + { + "id": "ecommerce.progression.cancellation-accounting-specifications", + "exceeded": false + }, + { + "id": "ecommerce.progression.catalog-management", + "exceeded": false + }, + { + "id": "ecommerce.progression.customer-profile", + "exceeded": false + }, + { + "id": "ecommerce.progression.faceted-search", + "exceeded": false + }, + { + "id": "ecommerce.progression.fulfilment-queue", + "exceeded": false + }, + { + "id": "ecommerce.progression.inventory-conservation-specifications", + "exceeded": false + }, + { + "id": "ecommerce.progression.managed-support", + "exceeded": false + }, + { + "id": "ecommerce.progression.notification-preferences", + "exceeded": false + }, + { + "id": "ecommerce.progression.operations-access-specifications", + "exceeded": false + }, + { + "id": "ecommerce.progression.promotion-rules", + "exceeded": false + }, + { + "id": "ecommerce.progression.staff-access", + "exceeded": false + }, + { + "id": "ecommerce.progression.staff-roles", + "exceeded": false + }, + { + "id": "ecommerce.progression.stock-alerts", + "exceeded": false + }, + { + "id": "ecommerce.progression.support-history", + "exceeded": false + }, + { + "id": "ecommerce.progression.support-intake", + "exceeded": false + }, + { + "id": "ecommerce.progression.support-triage", + "exceeded": false + }, + { + "id": "ecommerce.spec.access-control", + "exceeded": false + }, + { + "id": "ecommerce.spec.concurrency-safety", + "exceeded": false + }, + { + "id": "ecommerce.spec.external-data-sync", + "exceeded": false + }, + { + "id": "ecommerce.spec.live-state", + "exceeded": false + }, + { + "id": "ecommerce.spec.state-durability", + "exceeded": false + }, + { + "id": "ecommerce.spec.transactional-integrity", + "exceeded": false + } + ] + }, + "mutations": { + "caught": 97, + "total": 97 + } + } + ], + "stable": true, + "sameImage": true, + "sameHarness": true, + "harnessSha256": "55130157ba3c8b6364c6e8f0a9453479a4d8c7ecaa50cfaec92953bde488b13e", + "ok": true + } +} diff --git a/tools/stack-bench/qualification-evidence/ecommerce-progression-depth3-2.0.1/spacetime-reference.json b/tools/stack-bench/qualification-evidence/ecommerce-progression-depth3-2.0.1/spacetime-reference.json new file mode 100644 index 00000000000..26cea0e8d56 --- /dev/null +++ b/tools/stack-bench/qualification-evidence/ecommerce-progression-depth3-2.0.1/spacetime-reference.json @@ -0,0 +1,372 @@ +{ + "artifactSchemaVersion": 2, + "kind": "reference_qualification", + "id": "reference-live-spacetime-20260831090827-14-reference", + "attempt": { + "id": "reference-live-spacetime-20260831090827-14-reference", + "parentId": null + }, + "timestamps": { + "startedAt": "2026-08-31T09:08:27.717Z", + "completedAt": "2026-08-31T09:29:23.478Z" + }, + "identities": { + "engine": { + "id": "stack-bench", + "version": null, + "sha256": "39da14d55d1ca82f41d18305dcb3e86aa7d5098c69a2bf6ba74533affbfbecc9", + "state": null + }, + "recipe": { + "id": "ecommerce.progression-depth3", + "version": "2.0.1", + "sha256": "4c808d5e5fb076ff0993937d7a9f940dad4191351c362518e1c667a34324bd49", + "state": "qualified" + }, + "fixture": { + "id": "ecommerce-reference-spacetime", + "version": null, + "sha256": "8806e01bcd4d44fa7c7c491f722c2412d568605a9d666545dafcc0bdf2a2b4f5", + "state": "active" + }, + "calibration": { + "id": "ecommerce.progression-depth3-calibration", + "version": "2.0.1", + "sha256": "dd91493c5066018cbfd949d6e14d64dc029deb634e93d796752caee3da97682c", + "state": "draft" + }, + "experiment": null, + "agentAdapter": null, + "stackAdapter": { + "id": "spacetime", + "version": null, + "sha256": null, + "state": null + }, + "packs": [] + }, + "payload": { + "fixture": "ecommerce-reference-spacetime", + "fixtureSha256": "8806e01bcd4d44fa7c7c491f722c2412d568605a9d666545dafcc0bdf2a2b4f5", + "requiredRepetitions": 1, + "isolation": "docker", + "runner": { + "schemaVersion": 1, + "mode": "appliance", + "platform": "linux", + "architecture": "x64", + "dockerEngineVersion": "29.6.2", + "dockerOs": "linux", + "dockerArchitecture": "x86_64", + "kernelVersion": "6.6.87.2-microsoft-standard-WSL2", + "cpuCount": 32, + "memoryBytes": 49232838656 + }, + "qualificationScope": { + "checksSha256": "a982217993be2d229e13b4006427ce15666263d1d6e333eb241c4431df59cdf7", + "executableSha256": "9cdd1c8cccaf7ecd5e5d3039b0a0cf5c41a42c355ab085d9816b2d064706c2b9", + "kind": "reference", + "mutationSha256": null, + "recipe": { + "contentSha256": "4c808d5e5fb076ff0993937d7a9f940dad4191351c362518e1c667a34324bd49", + "id": "ecommerce.progression-depth3", + "version": "2.0.1" + }, + "schemaVersion": 2, + "stack": { + "id": "spacetime", + "reference": { + "id": "ecommerce-reference-spacetime", + "sourceSha256": "8806e01bcd4d44fa7c7c491f722c2412d568605a9d666545dafcc0bdf2a2b4f5" + }, + "version": "1.0.0" + }, + "sha256": "571714687a2bd1202e4a97268094411a979a6376069826475f73755c980761db" + }, + "mutationControl": false, + "qualifiedCheckKeys": [ + "ecommerce.feature.accounts.accounts.1a", + "ecommerce.feature.accounts.accounts.1b", + "ecommerce.feature.accounts.accounts.1c", + "ecommerce.feature.accounts.accounts.1d", + "ecommerce.feature.cart-checkout.cart.4a", + "ecommerce.feature.cart-checkout.cart.4d", + "ecommerce.feature.catalog.catalog-ranking.2b", + "ecommerce.feature.catalog.catalog-search.2d", + "ecommerce.feature.catalog.catalog-values.2a", + "ecommerce.feature.purchasing.purchase-order.3c", + "ecommerce.feature.reviews.reviews.6a", + "ecommerce.feature.warehouse-admin.access-boundary.7a", + "ecommerce.feature.warehouse-admin.admin-write.103a", + "ecommerce.feature.warehouse-admin.warehouse-stock.7c", + "ecommerce.feature.warehouse-admin.warehouse-view.7b", + "ecommerce.inventory-operations.operational-views.5a", + "ecommerce.inventory-operations.operational-views.5b", + "ecommerce.inventory-operations.operational-views.5c", + "ecommerce.inventory-operations.operational-views.5d", + "ecommerce.inventory-operations.stock-conservation.202a", + "ecommerce.inventory-operations.stock-conservation.202b", + "ecommerce.inventory-operations.stock-conservation.202c", + "ecommerce.inventory-operations.stock-conservation.202d", + "ecommerce.inventory-operations.warehouse-transfer.2a", + "ecommerce.inventory-operations.warehouse-transfer.2b", + "ecommerce.inventory-operations.warehouse-transfer.2c", + "ecommerce.l3.deferred-access.scheduled-work-access.317a", + "ecommerce.l3.deferred-durability.restart-survival.311a", + "ecommerce.l3.deferred-integrity.exactly-once.311a", + "ecommerce.l3.scheduled-restocks.scheduled-restocks.302a", + "ecommerce.l3.scheduled-restocks.scheduled-restocks.305a", + "ecommerce.l3.scheduled-restocks.scheduled-restocks.306a", + "ecommerce.l3.server-time.server-time.312a", + "ecommerce.operations-access.fulfilment-queue.1a", + "ecommerce.operations-access.fulfilment-queue.1b", + "ecommerce.operations-access.fulfilment-queue.1c", + "ecommerce.operations-access.fulfilment-queue.1d", + "ecommerce.operations-access.operator-authorization.201a", + "ecommerce.operations-access.operator-authorization.201c", + "ecommerce.operations-access.order-owner.204a", + "ecommerce.progression.catalog-management.catalog-management.622a", + "ecommerce.progression.catalog-management.catalog-management.622b", + "ecommerce.progression.customer-profile.customer-profile.620a", + "ecommerce.progression.customer-profile.customer-profile.620b", + "ecommerce.progression.faceted-search.faceted-search.401a", + "ecommerce.progression.faceted-search.faceted-search.402a", + "ecommerce.progression.managed-support.managed-support-privacy.613b", + "ecommerce.progression.managed-support.managed-support-shared.613a", + "ecommerce.progression.notification-preferences.notification-preferences-persistence.630a", + "ecommerce.progression.notification-preferences.notification-preferences-privacy.630b", + "ecommerce.progression.promotion-rules.promotion-rule-access.620b", + "ecommerce.progression.promotion-rules.promotion-rule-values.620a", + "ecommerce.progression.staff-access.staff-access.601a", + "ecommerce.progression.staff-access.staff-access.601b", + "ecommerce.progression.staff-roles.staff-roles.621a", + "ecommerce.progression.staff-roles.staff-roles.621b", + "ecommerce.progression.stock-alerts.stock-alert-delivery.631a", + "ecommerce.progression.stock-alerts.stock-alert-privacy.631b", + "ecommerce.progression.support-history.support-history-persistence.612a", + "ecommerce.progression.support-history.support-history-privacy.612b", + "ecommerce.progression.support-intake.support-intake.610a", + "ecommerce.progression.support-triage.support-assignment.611a", + "ecommerce.progression.support-triage.support-priority.611b", + "ecommerce.progression.support-triage.support-status.611c", + "ecommerce.returns-pricing.cancellation-and-return.3a", + "ecommerce.returns-pricing.cancellation-and-return.3b", + "ecommerce.returns-pricing.refund-accounting.203a", + "ecommerce.spec.access-control.cart-boundary.109a", + "ecommerce.spec.access-control.cart-boundary.109b", + "ecommerce.spec.access-control.order-ownership.106a", + "ecommerce.spec.access-control.purchase-attribution.102a", + "ecommerce.spec.access-control.purchase-session.101a", + "ecommerce.spec.access-control.review-eligibility.108a", + "ecommerce.spec.access-control.review-eligibility.108b", + "ecommerce.spec.access-control.signed-out-purchase.3a", + "ecommerce.spec.concurrency-safety.duplicate-checkout.203a", + "ecommerce.spec.concurrency-safety.duplicate-checkout.203b", + "ecommerce.spec.concurrency-safety.last-unit.201a", + "ecommerce.spec.concurrency-safety.last-unit.201b", + "ecommerce.spec.concurrency-safety.last-unit.201c", + "ecommerce.spec.concurrency-safety.restock-race.202a", + "ecommerce.spec.external-data-sync.external-stock.901a", + "ecommerce.spec.external-data-sync.external-stock.901c", + "ecommerce.spec.external-data-sync.external-stock.901d", + "ecommerce.spec.live-state.open-list.902a", + "ecommerce.spec.live-state.purchase-stock.3b", + "ecommerce.spec.live-state.ranking.2c", + "ecommerce.spec.live-state.rating.6c", + "ecommerce.spec.live-state.shared-cart.4c", + "ecommerce.spec.state-durability.account-state-recovery.105a", + "ecommerce.spec.state-durability.account-state-recovery.105b", + "ecommerce.spec.state-durability.cart-reload.4b", + "ecommerce.spec.state-durability.session-reload.1e", + "ecommerce.spec.transactional-integrity.books-balance.107a", + "ecommerce.spec.transactional-integrity.books-balance.107b", + "ecommerce.spec.transactional-integrity.server-price.104a", + "ecommerce.spec.transactional-integrity.unique-review.6b" + ], + "runs": [ + { + "repetition": 1, + "processError": null, + "harnessSha256Before": "55130157ba3c8b6364c6e8f0a9453479a4d8c7ecaa50cfaec92953bde488b13e", + "harnessSha256After": "55130157ba3c8b6364c6e8f0a9453479a4d8c7ecaa50cfaec92953bde488b13e", + "ok": true, + "failures": [], + "score": "162/162", + "imageId": "sha256:eccec6ea3861befc3ef0a05beb3f8ac13e0af2d0a1ef77c1a7beb3c0eb2aa075", + "criteria": 97, + "zeroPointCriteria": 0, + "outcome": "passed", + "packRuntime": { + "packs": [ + { + "id": "ecommerce.feature.accounts", + "exceeded": false + }, + { + "id": "ecommerce.feature.cart", + "exceeded": false + }, + { + "id": "ecommerce.feature.catalog-discovery", + "exceeded": false + }, + { + "id": "ecommerce.feature.catalog-items", + "exceeded": false + }, + { + "id": "ecommerce.feature.checkout", + "exceeded": false + }, + { + "id": "ecommerce.feature.purchasing", + "exceeded": false + }, + { + "id": "ecommerce.feature.reviews", + "exceeded": false + }, + { + "id": "ecommerce.feature.warehouse-admin", + "exceeded": false + }, + { + "id": "ecommerce.l2.inventory-dashboard", + "exceeded": false + }, + { + "id": "ecommerce.l2.order-cancellation-features", + "exceeded": false + }, + { + "id": "ecommerce.l2.recommendations", + "exceeded": false + }, + { + "id": "ecommerce.l2.sales-dashboard", + "exceeded": false + }, + { + "id": "ecommerce.l2.stock-transfers-features", + "exceeded": false + }, + { + "id": "ecommerce.l3.deferred-access-specifications", + "exceeded": false + }, + { + "id": "ecommerce.l3.deferred-durability-specifications", + "exceeded": false + }, + { + "id": "ecommerce.l3.deferred-integrity-specifications", + "exceeded": false + }, + { + "id": "ecommerce.l3.scheduled-restocks-features", + "exceeded": false + }, + { + "id": "ecommerce.l3.server-time-specifications", + "exceeded": false + }, + { + "id": "ecommerce.progression.cancellation-accounting-specifications", + "exceeded": false + }, + { + "id": "ecommerce.progression.catalog-management", + "exceeded": false + }, + { + "id": "ecommerce.progression.customer-profile", + "exceeded": false + }, + { + "id": "ecommerce.progression.faceted-search", + "exceeded": false + }, + { + "id": "ecommerce.progression.fulfilment-queue", + "exceeded": false + }, + { + "id": "ecommerce.progression.inventory-conservation-specifications", + "exceeded": false + }, + { + "id": "ecommerce.progression.managed-support", + "exceeded": false + }, + { + "id": "ecommerce.progression.notification-preferences", + "exceeded": false + }, + { + "id": "ecommerce.progression.operations-access-specifications", + "exceeded": false + }, + { + "id": "ecommerce.progression.promotion-rules", + "exceeded": false + }, + { + "id": "ecommerce.progression.staff-access", + "exceeded": false + }, + { + "id": "ecommerce.progression.staff-roles", + "exceeded": false + }, + { + "id": "ecommerce.progression.stock-alerts", + "exceeded": false + }, + { + "id": "ecommerce.progression.support-history", + "exceeded": false + }, + { + "id": "ecommerce.progression.support-intake", + "exceeded": false + }, + { + "id": "ecommerce.progression.support-triage", + "exceeded": false + }, + { + "id": "ecommerce.spec.access-control", + "exceeded": false + }, + { + "id": "ecommerce.spec.concurrency-safety", + "exceeded": false + }, + { + "id": "ecommerce.spec.external-data-sync", + "exceeded": false + }, + { + "id": "ecommerce.spec.live-state", + "exceeded": false + }, + { + "id": "ecommerce.spec.state-durability", + "exceeded": false + }, + { + "id": "ecommerce.spec.transactional-integrity", + "exceeded": false + } + ] + }, + "mutations": null + } + ], + "stable": true, + "sameImage": true, + "sameHarness": true, + "harnessSha256": "55130157ba3c8b6364c6e8f0a9453479a4d8c7ecaa50cfaec92953bde488b13e", + "ok": true + } +} diff --git a/tools/stack-bench/reference-apps/README.md b/tools/stack-bench/reference-apps/README.md new file mode 100644 index 00000000000..7993b1c3d4d --- /dev/null +++ b/tools/stack-bench/reference-apps/README.md @@ -0,0 +1,68 @@ +# Reference applications + +Reference applications validate the grader. They are simple, auditable fixtures, +not product examples or recommended application designs. + +`registry.json` is the source of truth for fixture identity and status: + +- `blocked`: no acceptable source and check set exists; +- `candidate`: exact source is present, but qualification is incomplete; +- `active`: the exact source passed its required reference and mutation gates. + +One cumulative source tree can serve several recipes when each registry entry +binds the same source hash. Qualification evidence remains separate for each +recipe and calibration. + +## Promotion requirements + +An active fixture must satisfy all of these conditions: + +1. Dependencies install from committed lockfiles in the benchmark build image. +2. The app starts in Docker with run-specific ports and database or module names. +3. Every required scored and supporting check passes for the exact recipe. +4. The source contains no secrets, generated bindings, build output, + transcripts, grader output, or mutation backups. +5. Each mutation has an exact source anchor and produces the intended conclusive + failure without unrelated failures. +6. The registry records the qualified source hash. + +Compile success or an old full score does not promote a fixture. + +## Compile fixtures + +Run the model-free Docker compile check from `tools/stack-bench`: + +```bash +npm run test:references +``` + +Compile one changed fixture with: + +```bash +npm run test:references -- --fixture +``` + +The command copies source into a temporary workspace. It does not edit the +registered fixture. Compile success is not live grading evidence. + +## Live qualification + +Run the repetition plan declared by the selected calibration: + +```bash +npm run qualify:reference -- --backend +``` + +The qualifier binds the exact recipe, fixture, source, engine, image, stack, +runner, and check identities. It also verifies lease and resource cleanup. + +Add `--mutations` only when mutation evidence is required. The qualifier first +checks the clean baseline, then applies each selected defect through the same +isolated Docker lifecycle. + +During development, select only affected defects with `--mutation-id `. +Targeted output is diagnostic evidence. The complete set requires +`--release-candidate` and is used only for release qualification. + +Do not edit a registered reference during qualification. A changed source hash +requires new evidence. diff --git a/tools/stack-bench/reference-apps/ecommerce/mongodb/client/index.html b/tools/stack-bench/reference-apps/ecommerce/mongodb/client/index.html new file mode 100644 index 00000000000..8b77d8835cc --- /dev/null +++ b/tools/stack-bench/reference-apps/ecommerce/mongodb/client/index.html @@ -0,0 +1,12 @@ + + + + + + Storefront + + +
+ + + diff --git a/tools/stack-bench/reference-apps/ecommerce/mongodb/client/package-lock.json b/tools/stack-bench/reference-apps/ecommerce/mongodb/client/package-lock.json new file mode 100644 index 00000000000..1c05ce6c7c3 --- /dev/null +++ b/tools/stack-bench/reference-apps/ecommerce/mongodb/client/package-lock.json @@ -0,0 +1,1046 @@ +{ + "name": "client", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "client", + "version": "1.0.0", + "dependencies": { + "react": "^18.3.1", + "react-dom": "^18.3.1", + "socket.io-client": "^4.7.5" + }, + "devDependencies": { + "@types/react": "^18.3.3", + "@types/react-dom": "^18.3.0", + "@vitejs/plugin-react": "^6.1.0", + "typescript": "^5.5.3", + "vite": "^8.2.2" + } + }, + "node_modules/@oxc-project/types": { + "version": "0.146.0", + "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.146.0.tgz", + "integrity": "sha512-XC0QsnnhVe7sLIWmYmdPw7x5P0h4W8vUU3Nv1ySgWXtvCz8NizoAEpGXA0sOYoJQV2Rl13LgURAHQ5cI5ILCSA==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/Boshen" + } + }, + "node_modules/@rolldown/binding-android-arm-eabi": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm-eabi/-/binding-android-arm-eabi-1.2.5.tgz", + "integrity": "sha512-DLe/i+l8ynIBY7XEQ191TeZvCoowIGa18R+dIV30GW7DiOtp74i/xX8hs8GUjW5ARV7VZuie3d6AumSmCwbeRA==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-android-arm64": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.2.5.tgz", + "integrity": "sha512-zXcwKlQApYAOELHd8PwKDFkagYF9Wy4e0RJ+0qnzl9Pjnpj75TEG8ufv40p2J7kCEfwZAsNiuzRIyNNMWT38ig==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-darwin-arm64": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.2.5.tgz", + "integrity": "sha512-dK4QakI42nzWgJT5sm4y4y/O//D4OxM75/cH28RLV+nzIN9AY+YsbuUVrUTjlLjXR6vpyxFbSsbmNuJ6BP9sww==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-darwin-x64": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.2.5.tgz", + "integrity": "sha512-fqSALaUu1Wjd1nK2uW2kJDWdLCc8lx1IcY+MTY26Aurfdx19anlzhqXOgCFbBFQnlFDTn4TC1/7Nz4Bl2mLP3A==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-freebsd-x64": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.2.5.tgz", + "integrity": "sha512-/vCnNxlkxs9tKxNDcyWUePpJ/PgTzxIaVhoM5SmG8UV+GR/IcPam4VYxi7GIMo7PSDuNqlJqvprqii9NqqVCMw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm-gnueabihf": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.2.5.tgz", + "integrity": "sha512-abk0NLA519LxRCszmbE0jYKuQ9YPocOXTiOXOo6Yr+YAT95VH+PtqYAjOJvGKt3viEd/x4qzabAlwd5bHOOARg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm64-gnu": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.2.5.tgz", + "integrity": "sha512-Y7eALiJ8lr0M2HH103Js+g7V34wf6snlpZLAsHI90uLhr3PVlNsbFVAXJC9d/V6BnPyKtpSwI+NcB/RLxsQxuA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm64-musl": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.2.5.tgz", + "integrity": "sha512-xMvZgnbZg4YVnR/AX2b3oOPDTFYJvUVaJg5FedA/LuvexAtXibZQej4cnTkw3rjsJ/ggUROB64TdtETiim+FYA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-ppc64-gnu": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.2.5.tgz", + "integrity": "sha512-GRjeqTUDHTo5GwntsLaAMcBahG3nlpjftXWZLN73HiYQlhwEowvarFgQnRnQZtIp4keXX7quXFbG38uPZBa2EA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-s390x-gnu": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.2.5.tgz", + "integrity": "sha512-vLNTR45F2Uwc8AufkNXPmB4VliaXs+FvcheEogIzOXzO4l+LzieXF5A/TWxLy5HtqpsRCHUfd0lPVrrdgXdLHQ==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-x64-gnu": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.2.5.tgz", + "integrity": "sha512-Mgj59/HTuYeK9Gz2MA+mBWKnHsAgkBSec15ZMb1st3oIfFbX7gCjOae7GydHhzcyQi9Z/7M1QuN9bR3oFqF0jQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-x64-musl": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.2.5.tgz", + "integrity": "sha512-mY8AP0/ichsbhAxGnLa3d3+MwV0EfgrPND2bplI3Ym8T6R2pJ0N87bvrKVwNXmdy3jnr6eQBecdqx/HMknBmpA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-openharmony-arm64": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.2.5.tgz", + "integrity": "sha512-8SLssA2oweAxyRgDp789ACfRb/3P+zNRJpzZxSizxF9m8NUDQ4+3xjo8ttjhVGGw6Qxb70oZiEtIjaKikCO7Yw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-win32-arm64-msvc": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.2.5.tgz", + "integrity": "sha512-vGbruD5zquhoc8D9SViXgN2FBJtNdTyQ4DtG+SWiEGlJiAzoKcZ2xp+xuXCffhubVdt0NJlTZqkeRuERy7g8Cw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-win32-x64-msvc": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.2.5.tgz", + "integrity": "sha512-e/SXpgISz+IoqVcSSI0rx/d/he8zqLex+/rCWpnHpmVfmPIUjag9H6P7zotf0gJHwPUhQxZ/mF8tr6acebT9yw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/pluginutils": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.1.tgz", + "integrity": "sha512-2j9bGt5Jh8hj+vPtgzPtl72j0yRxHAyumoo6TNfAjsLB04UtpSvPbPcDcBMxz7n+9CYB0c1GxQFxYRg2jimqGw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@socket.io/component-emitter": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@socket.io/component-emitter/-/component-emitter-3.1.2.tgz", + "integrity": "sha512-9BCxFwvbGg/RsZK9tjXd8s4UcwR0MWeFQ1XEKIQVVvAGJyINdrqKMcTRyLoK8Rse1GjzLV9cwjWV1olXRWEXVA==", + "license": "MIT" + }, + "node_modules/@types/prop-types": { + "version": "15.7.15", + "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.15.tgz", + "integrity": "sha512-F6bEyamV9jKGAFBEmlQnesRPGOQqS2+Uwi0Em15xenOxHaf2hv6L8YCVn3rPdPJOiJfPiCnLIRyvwVaqMY3MIw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/react": { + "version": "18.3.31", + "resolved": "https://registry.npmjs.org/@types/react/-/react-18.3.31.tgz", + "integrity": "sha512-vfEqpXTvwT91yhmwdfouStN2hSKwTvyRs8qpLfADyrq/kxDw0hZM7Wk9Ug1FELj8hIby+S/+kQCSRFF32nv2Qw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/prop-types": "*", + "csstype": "^3.2.2" + } + }, + "node_modules/@types/react-dom": { + "version": "18.3.7", + "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-18.3.7.tgz", + "integrity": "sha512-MEe3UeoENYVFXzoXEWsvcpg6ZvlrFNlOQ7EOsvhI3CfAXwzPfO8Qwuxd40nepsYKqyyVQnTdEfv68q91yLcKrQ==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "@types/react": "^18.0.0" + } + }, + "node_modules/@vitejs/plugin-react": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/@vitejs/plugin-react/-/plugin-react-6.1.0.tgz", + "integrity": "sha512-qd2BzUBehkov86WFhg0JkEFEYyCLG9uPCe6qWTY/kRlss9OvJrOF2UbIWT7p+8IzZHkEu0DNGHc4HSv+JdDLsw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@rolldown/pluginutils": "^1.0.1" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "peerDependencies": { + "@rolldown/plugin-babel": "^0.1.7 || ^0.2.0", + "babel-plugin-react-compiler": "^1.0.0", + "oxc-transform-react": "^0.145.0", + "vite": "^8.0.0" + }, + "peerDependenciesMeta": { + "@rolldown/plugin-babel": { + "optional": true + }, + "babel-plugin-react-compiler": { + "optional": true + }, + "oxc-transform-react": { + "optional": true + } + } + }, + "node_modules/csstype": { + "version": "3.2.3", + "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.2.3.tgz", + "integrity": "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=8" + } + }, + "node_modules/engine.io-client": { + "version": "6.6.6", + "resolved": "https://registry.npmjs.org/engine.io-client/-/engine.io-client-6.6.6.tgz", + "integrity": "sha512-iY6QdftLQ9pyiPoX082bpf/u1UewnOaJrtJIF9T0++QB34lZrj0uP+Q/bj8AlUsAxqhnkTV2BS8SBZSxOmoV5Q==", + "license": "MIT", + "dependencies": { + "@socket.io/component-emitter": "~3.1.0", + "debug": "~4.4.1", + "engine.io-parser": "~5.2.1", + "ws": "~8.21.0", + "xmlhttprequest-ssl": "~2.1.1" + } + }, + "node_modules/engine.io-parser": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/engine.io-parser/-/engine.io-parser-5.2.3.tgz", + "integrity": "sha512-HqD3yTBfnBxIrbnM1DoD6Pcq8NECnh8d4As1Qgh0z5Gg3jRRIqijury0CL3ghu/edArpUYiYqQiDUQBIs4np3Q==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + } + }, + "node_modules/fdir": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", + "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "license": "MIT" + }, + "node_modules/lightningcss": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss/-/lightningcss-1.33.0.tgz", + "integrity": "sha512-WkUDrojuJs0xkgGf2udWxa3yGBRxPtxUkB79i6aCZLRgc7PM8fZe9TosfPDcvEpQZbuFASnHYmRLBLUbmLOIIA==", + "dev": true, + "license": "MPL-2.0", + "dependencies": { + "detect-libc": "^2.0.3" + }, + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + }, + "optionalDependencies": { + "lightningcss-android-arm64": "1.33.0", + "lightningcss-darwin-arm64": "1.33.0", + "lightningcss-darwin-x64": "1.33.0", + "lightningcss-freebsd-x64": "1.33.0", + "lightningcss-linux-arm-gnueabihf": "1.33.0", + "lightningcss-linux-arm64-gnu": "1.33.0", + "lightningcss-linux-arm64-musl": "1.33.0", + "lightningcss-linux-x64-gnu": "1.33.0", + "lightningcss-linux-x64-musl": "1.33.0", + "lightningcss-win32-arm64-msvc": "1.33.0", + "lightningcss-win32-x64-msvc": "1.33.0" + } + }, + "node_modules/lightningcss-android-arm64": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-android-arm64/-/lightningcss-android-arm64-1.33.0.tgz", + "integrity": "sha512-gEpRTalKdosp4Bb8qWtc2iOgE5SeIHlpS1up9bFq2wAyYhl1UdTObYiHe98zEM9SQvSoqQZ1IQD0JNpg3Ml5pg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-darwin-arm64": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-arm64/-/lightningcss-darwin-arm64-1.33.0.tgz", + "integrity": "sha512-Sciaz8eenNTKn9b3t7+xr0ipTp9YxKQY4npwQ3mrRuL0BAVHBLyZxofhaKBAVtzmtRZ/zTyo0/to4B1uWG/Djg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-darwin-x64": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-x64/-/lightningcss-darwin-x64-1.33.0.tgz", + "integrity": "sha512-Z5UPAxzrjlWNNyGy6i65cJzzvgJ5D3T6wMvs+gWpY9d7qRhANrxqAp6LhxIgZhWEw18RfJTGcRxjuLIBr+m8XQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-freebsd-x64": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-freebsd-x64/-/lightningcss-freebsd-x64-1.33.0.tgz", + "integrity": "sha512-QQM/Ti/hQajJwCY+RiWuCZ9sdtI/XQk7nDK5vC8kkdwixezOlDgvDx7+RT+QjK6FcFT4MpsuoBnHIo/O3StRRg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm-gnueabihf": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm-gnueabihf/-/lightningcss-linux-arm-gnueabihf-1.33.0.tgz", + "integrity": "sha512-N7FVBe6iS24MlM6R/4RBTxGhQheZGs7tiQ9U32UtF75NzP5Q7xWPRqLBCKxlRQRk3rY1jCIPLzx7WzOhuUIRLQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm64-gnu": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-gnu/-/lightningcss-linux-arm64-gnu-1.33.0.tgz", + "integrity": "sha512-j2v/itmy4HlNxlc6voKXYgBqNi0Ng2LShg4z7GufpEgs05P+2suBVyi9I6YHq5uoVFx9ETin3eCEhLVyXGQnKg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm64-musl": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-musl/-/lightningcss-linux-arm64-musl-1.33.0.tgz", + "integrity": "sha512-yiO5ROMuYQgXbC60yjZU5CYSFZGKXL0HFATXt9mHJn1+zW55oCtMI9NfcVhYLMFDL7gV7oBPon/EmMMGg2OvtQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-x64-gnu": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-gnu/-/lightningcss-linux-x64-gnu-1.33.0.tgz", + "integrity": "sha512-ar+Ju7LmcN0Jo4FpL4hpFybwNG9/3A/Br5KW2n2jyODg3MEZXaDYADdemoNS+BDNfMgKvylJLj4S5tyRActuAg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-x64-musl": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-musl/-/lightningcss-linux-x64-musl-1.33.0.tgz", + "integrity": "sha512-RYiYbkokw0trfKqqzfF55lginwEPrD3OJDfTuJzFs1MK6iFnDenaz1fqLLtX4ITG3OktJQXOeTaw1awrBAlZPw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-win32-arm64-msvc": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-win32-arm64-msvc/-/lightningcss-win32-arm64-msvc-1.33.0.tgz", + "integrity": "sha512-1K+MPfLSFVpphzpdbfkhlWk6wBrTObBzS2T6db10PNOZgR9GoVsAWzwNyuhUYYbTp23j+4RrncfujZ4uAzXvwA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-win32-x64-msvc": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-win32-x64-msvc/-/lightningcss-win32-x64-msvc-1.33.0.tgz", + "integrity": "sha512-OlEICDx/Xl0FqSp4bry8zFnCvGpig3Gl4gCquvYwHuqJKEC1+n9NgDniFvqHGmMv1ZkqDJrDqKKSykTDX+ehuA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/loose-envify": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz", + "integrity": "sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==", + "license": "MIT", + "dependencies": { + "js-tokens": "^3.0.0 || ^4.0.0" + }, + "bin": { + "loose-envify": "cli.js" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/nanoid": { + "version": "3.3.18", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.18.tgz", + "integrity": "sha512-DTg4MJbGMWkfi6VZFdNt2/caMbQy4Ou+Op/hJQvGEWcnVfoA1QA+xzRKAzw9jD6+GVOOeYr/mIcuDSdug6F6+w==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "dev": true, + "license": "ISC" + }, + "node_modules/picomatch": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.7.tgz", + "integrity": "sha512-qcJu88Q2IWqJsDD529JKMdwGm/dvInW4HvQnRwiH9JtihJvzGOscDtHE3x1pBKeUOTysQ8kVmLnJ2kJu7yhcGA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/postcss": { + "version": "8.5.26", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.26.tgz", + "integrity": "sha512-u82N74LFzG8ca+dD8puPnplTXoGH4fTPpVGuIbt36G3qvNlkvfD0lEAZSxaly3KX8TS/L1A1gsCEmvKmBcVbkQ==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "nanoid": "^3.3.17", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.1" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/react": { + "version": "18.3.1", + "resolved": "https://registry.npmjs.org/react/-/react-18.3.1.tgz", + "integrity": "sha512-wS+hAgJShR0KhEvPJArfuPVN1+Hz1t0Y6n5jLrGQbkb4urgPE/0Rve+1kMB1v/oWgHgm4WIcV+i7F2pTVj+2iQ==", + "license": "MIT", + "dependencies": { + "loose-envify": "^1.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/react-dom": { + "version": "18.3.1", + "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-18.3.1.tgz", + "integrity": "sha512-5m4nQKp+rZRb09LNH59GM4BxTh9251/ylbKIbpe7TpGxfJ+9kv6BLkLBXIjjspbgbnIBNqlI23tRnTWT0snUIw==", + "license": "MIT", + "dependencies": { + "loose-envify": "^1.1.0", + "scheduler": "^0.23.2" + }, + "peerDependencies": { + "react": "^18.3.1" + } + }, + "node_modules/rolldown": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.2.5.tgz", + "integrity": "sha512-VD2IE5PUG4Oj8zz2VGykiYd5wbnjdIiSsNQb8Qu5B+noEp+A78mu2iVvpp27g8es14Tk9rofNs5Tku9iQCS4fA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@oxc-project/types": "=0.146.0", + "@rolldown/pluginutils": "^1.0.0" + }, + "bin": { + "rolldown": "bin/cli.mjs" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "optionalDependencies": { + "@rolldown/binding-android-arm-eabi": "1.2.5", + "@rolldown/binding-android-arm64": "1.2.5", + "@rolldown/binding-darwin-arm64": "1.2.5", + "@rolldown/binding-darwin-x64": "1.2.5", + "@rolldown/binding-freebsd-x64": "1.2.5", + "@rolldown/binding-linux-arm-gnueabihf": "1.2.5", + "@rolldown/binding-linux-arm64-gnu": "1.2.5", + "@rolldown/binding-linux-arm64-musl": "1.2.5", + "@rolldown/binding-linux-ppc64-gnu": "1.2.5", + "@rolldown/binding-linux-s390x-gnu": "1.2.5", + "@rolldown/binding-linux-x64-gnu": "1.2.5", + "@rolldown/binding-linux-x64-musl": "1.2.5", + "@rolldown/binding-openharmony-arm64": "1.2.5", + "@rolldown/binding-win32-arm64-msvc": "1.2.5", + "@rolldown/binding-win32-x64-msvc": "1.2.5" + } + }, + "node_modules/scheduler": { + "version": "0.23.2", + "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.23.2.tgz", + "integrity": "sha512-UOShsPwz7NrMUqhR6t0hWjFduvOzbtv7toDH1/hIrfRNIDBnnBWd0CwJTGvTpngVlmwGCdP9/Zl/tVrDqcuYzQ==", + "license": "MIT", + "dependencies": { + "loose-envify": "^1.1.0" + } + }, + "node_modules/socket.io-client": { + "version": "4.8.3", + "resolved": "https://registry.npmjs.org/socket.io-client/-/socket.io-client-4.8.3.tgz", + "integrity": "sha512-uP0bpjWrjQmUt5DTHq9RuoCBdFJF10cdX9X+a368j/Ft0wmaVgxlrjvK3kjvgCODOMMOz9lcaRzxmso0bTWZ/g==", + "license": "MIT", + "dependencies": { + "@socket.io/component-emitter": "~3.1.0", + "debug": "~4.4.1", + "engine.io-client": "~6.6.1", + "socket.io-parser": "~4.2.4" + }, + "engines": { + "node": ">=10.0.0" + } + }, + "node_modules/socket.io-parser": { + "version": "4.2.7", + "resolved": "https://registry.npmjs.org/socket.io-parser/-/socket.io-parser-4.2.7.tgz", + "integrity": "sha512-IH/iSeO9T6gz1KkFleGDWkG9N3dl4jXVYUtMhIqH10Md0ttMer8nUNWiP1DKuNrybD2xBrixLJdCC9J6ECoYkg==", + "license": "MIT", + "dependencies": { + "@socket.io/component-emitter": "~3.1.0", + "debug": "~4.4.1" + }, + "engines": { + "node": ">=10.0.0" + } + }, + "node_modules/source-map-js": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/tinyglobby": { + "version": "0.2.17", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.17.tgz", + "integrity": "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==", + "dev": true, + "license": "MIT", + "dependencies": { + "fdir": "^6.5.0", + "picomatch": "^4.0.4" + }, + "engines": { + "node": ">=12.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/SuperchupuDev" + } + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/vite": { + "version": "8.2.2", + "resolved": "https://registry.npmjs.org/vite/-/vite-8.2.2.tgz", + "integrity": "sha512-cFKLV/PRgAUlIRm5WjMjJ86jrftzpqcgH+Us+DS8mI3CDNiH30Whrz8uHL3+MOLPAgqbMBAqWdAHAphOAM+z/Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "lightningcss": "^1.33.0", + "picomatch": "^4.0.5", + "postcss": "^8.5.26", + "rolldown": "~1.2.4", + "tinyglobby": "^0.2.17" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^20.19.0 || >=22.12.0", + "@vitejs/devtools": "^0.4.0 || ^0.5.0", + "esbuild": "^0.27.0 || ^0.28.0", + "jiti": ">=1.21.0", + "less": "^4.0.0", + "sass": "^1.70.0", + "sass-embedded": "^1.70.0", + "stylus": ">=0.54.8", + "sugarss": "^5.0.0", + "terser": "^5.16.0", + "tsx": "^4.8.1", + "yaml": "^2.4.2" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "@vitejs/devtools": { + "optional": true + }, + "esbuild": { + "optional": true + }, + "jiti": { + "optional": true + }, + "less": { + "optional": true + }, + "sass": { + "optional": true + }, + "sass-embedded": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + }, + "tsx": { + "optional": true + }, + "yaml": { + "optional": true + } + } + }, + "node_modules/ws": { + "version": "8.21.3", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.3.tgz", + "integrity": "sha512-201TZ/kPWxoPr/OKWjquZR1SWKXcvxdH+e1xrx89b3YbmzLMFCLfnaG1HFIgWzJOEWZ7MvpK++odZufgYR50Rw==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/xmlhttprequest-ssl": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/xmlhttprequest-ssl/-/xmlhttprequest-ssl-2.1.2.tgz", + "integrity": "sha512-TEU+nJVUUnA4CYJFLvK5X9AOeH4KvDvhIfm0vV1GaQRtchnG0hgK5p8hw/xjv8cunWYCsiPCSDzObPyhEwq3KQ==", + "engines": { + "node": ">=0.4.0" + } + } + } +} diff --git a/tools/stack-bench/reference-apps/ecommerce/mongodb/client/package.json b/tools/stack-bench/reference-apps/ecommerce/mongodb/client/package.json new file mode 100644 index 00000000000..4817027566c --- /dev/null +++ b/tools/stack-bench/reference-apps/ecommerce/mongodb/client/package.json @@ -0,0 +1,23 @@ +{ + "name": "client", + "version": "1.0.0", + "private": true, + "type": "module", + "scripts": { + "dev": "vite", + "build": "vite build", + "preview": "vite preview" + }, + "dependencies": { + "react": "^18.3.1", + "react-dom": "^18.3.1", + "socket.io-client": "^4.7.5" + }, + "devDependencies": { + "@types/react": "^18.3.3", + "@types/react-dom": "^18.3.0", + "@vitejs/plugin-react": "^6.1.0", + "typescript": "^5.5.3", + "vite": "^8.2.2" + } +} diff --git a/tools/stack-bench/reference-apps/ecommerce/mongodb/client/src/App.tsx b/tools/stack-bench/reference-apps/ecommerce/mongodb/client/src/App.tsx new file mode 100644 index 00000000000..31898e68cf9 --- /dev/null +++ b/tools/stack-bench/reference-apps/ecommerce/mongodb/client/src/App.tsx @@ -0,0 +1,1456 @@ +import React, { useEffect, useMemo, useRef, useState, useCallback } from "react"; +import { io, Socket } from "socket.io-client"; +import { ProgressionPanel } from "./ProgressionPanel"; + +const TOKEN_KEY = "mongodb_shop_token"; +const CATALOG_PAGE_SIZE = 10; + +interface ItemT { + id: string; + name: string; + price: number; + description?: string; + category: string; + stock: number; + purchaseCount: number; + variants?: string[]; +} + +interface ReviewT { + id: string; + itemId: string; + userId: string; + username: string; + rating: number; + comment: string; + createdAt: string; +} + +interface ItemDetailT { + id: string; + name: string; + price: number; + description: string; + stock: number; + reviews: ReviewT[]; + average: number; +} + +interface CartLineT { + itemId: string; + name: string; + price: number; + stock: number; + quantity: number; + reservationSeconds?: number; + expired?: boolean; +} + +interface CartT { + items: CartLineT[]; + total: number; + promotionCode?: string; + discount?: number; +} + +interface OrderLineT { + itemId: string; + name: string; + price: number; + quantity: number; + returned?: boolean; + warehouseNames?: string[]; +} + +interface OrderT { + id: string; + items: OrderLineT[]; + total: number; + status: "pending" | "shipped" | "delivered" | "cancelled" | "refunded"; + discount?: number; + refundTotal?: number; + createdAt: string; + payments?: Array<{ id: string; amount: number; status: string }>; +} + +interface UserT { + id: string; + username: string; + isAdmin: boolean; + isStaff: boolean; + roles?: string[]; +} + +interface AdminLocationT { + id: string; + itemId: string; + itemName: string; + warehouseId: string; + warehouseName: string; + quantity: number; +} + +interface CategoryTotalT { + category: string; + units: number; + revenue: number; +} + +interface AdminOverviewT { + items: Array<{ id: string; name: string; price: number; stock: number; category: string }>; + warehouses: Array<{ id: string; name: string; total: number }>; + locations: AdminLocationT[]; + revenue: number; + categories: CategoryTotalT[]; + lowStock: Array<{ id: string; name: string; stock: number }>; + queueDepth: number; +} + +interface FulfilmentQueueT { + orders: OrderT[]; + depth: number; +} + +function useTransientError(): [string, (msg: string) => void] { + const [message, setMessage] = useState(""); + const timer = useRef | null>(null); + const show = useCallback((msg: string) => { + setMessage(msg); + if (timer.current) clearTimeout(timer.current); + timer.current = setTimeout(() => setMessage(""), 5000); + }, []); + return [message, show]; +} + +async function apiFetch(path: string, token: string | null, options: RequestInit = {}) { + const headers: Record = { "Content-Type": "application/json" }; + if (token) headers.Authorization = `Bearer ${token}`; + const res = await fetch(path, { ...options, headers: { ...headers, ...(options.headers as any) } }); + const data = await res.json().catch(() => ({})); + if (!res.ok) throw new Error(data.error || "Request failed"); + return data; +} + +export default function App() { + const [token, setToken] = useState(() => localStorage.getItem(TOKEN_KEY)); + const [currentUser, setCurrentUser] = useState(null); + const [initializing, setInitializing] = useState(true); + const [items, setItems] = useState([]); + const [searchQuery, setSearchQuery] = useState(""); + const [categoryFilter, setCategoryFilter] = useState(""); + const [minimumPrice, setMinimumPrice] = useState(""); + const [maximumPrice, setMaximumPrice] = useState(""); + const [inStockOnly, setInStockOnly] = useState(false); + const [searchPage, setSearchPage] = useState(0); + const [cart, setCart] = useState({ items: [], total: 0 }); + // Keep floating panels mutually exclusive so navigation remains reachable. + const [activeView, setActiveView] = useState<"cart" | "orders" | "admin" | "fulfilment" | null>(null); + const cartOpen = activeView === "cart"; + const ordersOpen = activeView === "orders"; + const [orders, setOrders] = useState([]); + const adminOpen = activeView === "admin"; + const [adminOverview, setAdminOverview] = useState(null); + const fulfilmentOpen = activeView === "fulfilment"; + const [fulfilmentQueue, setFulfilmentQueue] = useState({ orders: [], depth: 0 }); + const [recommended, setRecommended] = useState([]); + const [selectedItemId, setSelectedItemId] = useState(null); + const [itemDetail, setItemDetail] = useState(null); + + const [buyError, showBuyError] = useTransientError(); + const [orderError, showOrderError] = useTransientError(); + const [promotionCode, setPromotionCode] = useState(""); + const [promotionError, setPromotionError] = useState(""); + + const socketRef = useRef(null); + + const saveSession = (tok: string, user: UserT) => { + localStorage.setItem(TOKEN_KEY, tok); + setToken(tok); + setCurrentUser(user); + }; + + const clearSession = () => { + localStorage.removeItem(TOKEN_KEY); + setToken(null); + setCurrentUser(null); + setCart({ items: [], total: 0 }); + setOrders([]); + setAdminOverview(null); + setFulfilmentQueue({ orders: [], depth: 0 }); + setActiveView(null); + }; + + const refreshItems = useCallback(async () => { + const data = await apiFetch("/api/items", null); + setItems(data.items); + }, []); + + const refreshCart = useCallback(async (tok: string) => { + const data = await apiFetch("/api/cart", tok); + setCart(data); + }, []); + + useEffect(() => { + if (!token || cart.items.length === 0) return; + const timer = setInterval(() => refreshCart(token).catch(() => undefined), 1000); + return () => clearInterval(timer); + }, [token, cart.items.length, refreshCart]); + + const refreshAdmin = useCallback(async (tok: string) => { + const data = await apiFetch("/api/admin/overview", tok); + setAdminOverview(data); + }, []); + + const refreshFulfilment = useCallback(async (tok: string) => { + const data = await apiFetch("/api/fulfilment/queue", tok); + setFulfilmentQueue(data); + }, []); + + const refreshRecommended = useCallback(async (tok: string | null) => { + const data = await apiFetch("/api/recommended", tok); + setRecommended(data.items); + }, []); + + // Initial load: restore session, fetch the live catalogue, and (if signed + // in) the account's cart. A page opened fresh always asks for current + // numbers rather than trusting anything cached. + useEffect(() => { + let cancelled = false; + (async () => { + try { + await refreshItems(); + } catch (err) { + console.error(err); + } + const tok = localStorage.getItem(TOKEN_KEY); + if (tok) { + try { + const me = await apiFetch("/api/auth/me", tok); + if (!cancelled) { + setCurrentUser(me.user); + await refreshCart(tok); + if (me.user.isAdmin) await refreshAdmin(tok); + if (me.user.isStaff || me.user.isAdmin) await refreshFulfilment(tok); + await refreshRecommended(tok); + } + } catch { + clearSession(); + } + } else { + await refreshRecommended(null).catch((err) => console.error(err)); + } + if (!cancelled) setInitializing(false); + })(); + return () => { + cancelled = true; + }; + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []); + + // Socket connection follows the current token. On every (re)connect — + // including after the server was down and the page never reloaded — pull a + // fresh snapshot instead of trusting whatever events were missed. + useEffect(() => { + const socket = io({ auth: token ? { token } : {} }); + socketRef.current = socket; + + socket.on("connect", () => { + refreshItems().catch((err) => console.error(err)); + if (token) { + refreshCart(token).catch((err) => console.error(err)); + } + }); + + socket.on("items:update", (data: ItemT[]) => setItems(data)); + socket.on("cart:update", (data: CartT) => setCart(data)); + socket.on("admin:update", (data: AdminOverviewT) => setAdminOverview(data)); + socket.on("orders:update", (data: OrderT[]) => setOrders(data)); + socket.on("fulfilment:update", (data: FulfilmentQueueT) => setFulfilmentQueue(data)); + socket.on("recommended:update", (data: ItemT[]) => setRecommended(data)); + socket.on("progression:update", () => { + refreshItems().catch((err) => console.error(err)); + refreshRecommended(token).catch((err) => console.error(err)); + if (token) { + refreshCart(token).catch((err) => console.error(err)); + apiFetch("/api/orders", token).then(data => setOrders(data.orders)).catch(() => undefined); + } + }); + socket.on("reviews:update", (payload: { itemId: string; reviews: ReviewT[]; average: number }) => { + setItemDetail((prev) => (prev && prev.id === payload.itemId ? { ...prev, reviews: payload.reviews, average: payload.average } : prev)); + }); + + return () => { + socket.disconnect(); + }; + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [token]); + + useEffect(() => { + if (currentUser?.isAdmin && token) { + refreshAdmin(token).catch((err) => console.error(err)); + } + if ((currentUser?.isStaff || currentUser?.isAdmin) && token) { + refreshFulfilment(token).catch((err) => console.error(err)); + } + }, [currentUser, token, refreshAdmin, refreshFulfilment]); + + useEffect(() => { + if (!selectedItemId) { + setItemDetail(null); + return; + } + let cancelled = false; + apiFetch(`/api/items/${selectedItemId}`, token) + .then((data) => { + if (!cancelled) setItemDetail(data.item); + }) + .catch((err) => console.error(err)); + return () => { + cancelled = true; + }; + }, [selectedItemId, token]); + + // Escape closes whichever overlay is open. + useEffect(() => { + const onKeyDown = (e: KeyboardEvent) => { + if (e.key !== "Escape") return; + if (selectedItemId) setSelectedItemId(null); + else if (activeView) setActiveView(null); + }; + window.addEventListener("keydown", onKeyDown); + return () => window.removeEventListener("keydown", onKeyDown); + }, [selectedItemId, activeView]); + + const handleSignUp = async (username: string, password: string) => { + const data = await apiFetch("/api/auth/signup", null, { method: "POST", body: JSON.stringify({ username, password }) }); + saveSession(data.token, data.user); + }; + + const handleSignIn = async (username: string, password: string) => { + const data = await apiFetch("/api/auth/signin", null, { method: "POST", body: JSON.stringify({ username, password }) }); + saveSession(data.token, data.user); + }; + + const handleSignOut = () => { + clearSession(); + }; + + const handleBuyNow = async (itemId: string) => { + try { + await apiFetch(`/api/items/${itemId}/buy`, token, { method: "POST" }); + } catch (err: any) { + showBuyError(err.message); + } + }; + + const handleAddToCart = async (itemId: string) => { + try { + const data = await apiFetch("/api/cart", token, { method: "POST", body: JSON.stringify({ itemId, quantity: 1 }) }); + setCart(data); + } catch (err: any) { + showBuyError(err.message); + } + }; + + const handleQuantityChange = async (itemId: string, quantity: number) => { + try { + const data = await apiFetch(`/api/cart/${itemId}`, token, { method: "PATCH", body: JSON.stringify({ quantity }) }); + setCart(data); + } catch (err: any) { + showBuyError(err.message); + } + }; + + const handleRemove = async (itemId: string) => { + try { + const data = await apiFetch(`/api/cart/${itemId}`, token, { method: "DELETE" }); + setCart(data); + } catch (err: any) { + showBuyError(err.message); + } + }; + + const handleCheckout = async () => { + try { + await apiFetch("/api/checkout", token, { method: "POST" }); + } catch (err: any) { + showBuyError(err.message); + } + }; + + const handleApplyPromotion = async () => { + setPromotionError(""); + try { + const data = await apiFetch("/api/progression/cart/promotion", token, { + method: "POST", body: JSON.stringify({ code: promotionCode }), + }); + setCart((value) => ({ ...value, promotionCode: data.promotion.code, + discount: data.promotion.discount })); + } catch (err: any) { + setPromotionError(err.message); + } + }; + + const openOrders = async () => { + setActiveView("orders"); + if (token) { + try { + const data = await apiFetch("/api/orders", token); + setOrders(data.orders); + } catch (err) { + console.error(err); + } + } + }; + + const openAdmin = async () => { + setActiveView("admin"); + if (token) { + try { + await refreshAdmin(token); + } catch (err) { + console.error(err); + } + } + }; + + const openFulfilment = async () => { + setActiveView("fulfilment"); + if (token) { + try { + await refreshFulfilment(token); + } catch (err) { + console.error(err); + } + } + }; + + const [reviewError, setReviewError] = useState(""); + const handleReviewSubmit = async (itemId: string, rating: number, comment: string) => { + setReviewError(""); + try { + const data = await apiFetch(`/api/items/${itemId}/reviews`, token, { + method: "POST", + body: JSON.stringify({ rating, comment }), + }); + setItemDetail(data.item); + } catch (err: any) { + setReviewError(err.message); + } + }; + + const handleRestock = async (itemId: string, warehouseId: string, quantity: number) => { + try { + const data = await apiFetch("/api/admin/restock", token, { + method: "POST", + body: JSON.stringify({ itemId, warehouseId, quantity }), + }); + setAdminOverview(data); + } catch (err: any) { + showBuyError(err.message); + } + }; + + const handleTransfer = async (itemId: string, fromWarehouseId: string, toWarehouseId: string, quantity: number) => { + try { + const data = await apiFetch("/api/admin/transfer", token, { + method: "POST", + body: JSON.stringify({ itemId, fromWarehouseId, toWarehouseId, quantity }), + }); + setAdminOverview(data); + } catch (err: any) { + showOrderError(err.message); + } + }; + + const handlePriceChange = async (itemId: string, price: number) => { + try { + const data = await apiFetch("/api/admin/price", token, { + method: "POST", + body: JSON.stringify({ itemId, price }), + }); + setAdminOverview(data); + } catch (err: any) { + showOrderError(err.message); + } + }; + + const handleShipOrder = async (orderId: string) => { + try { + await apiFetch("/api/fulfilment/ship", token, { + method: "POST", + body: JSON.stringify({ orderId }), + }); + } catch (err: any) { + showOrderError(err.message); + } + }; + + const handleCancelOrder = async (orderId: string) => { + try { + const data = await apiFetch(`/api/orders/${orderId}/cancel`, token, { method: "POST" }); + setOrders((prev) => prev.map((o) => (o.id === orderId ? data.order : o))); + } catch (err: any) { + showOrderError(err.message); + } + }; + + const handleReturnItem = async (orderId: string, itemId: string) => { + try { + const data = await apiFetch(`/api/orders/${orderId}/items/${itemId}/return`, token, { method: "POST" }); + setOrders((prev) => prev.map((o) => (o.id === orderId ? data.order : o))); + } catch (err: any) { + showOrderError(err.message); + } + }; + + const filteredItems = useMemo(() => { + const q = searchQuery.trim().toLowerCase(); + const min = minimumPrice === "" ? -Infinity : Number(minimumPrice); + const max = maximumPrice === "" ? Infinity : Number(maximumPrice); + return items.filter((it) => (!q || it.name.toLowerCase().includes(q)) + && (!categoryFilter || it.category === categoryFilter) + && it.price >= min && it.price <= max && (!inStockOnly || it.stock > 0)); + }, [items, searchQuery, categoryFilter, minimumPrice, maximumPrice, inStockOnly]); + const searchResults = filteredItems.slice(searchPage * CATALOG_PAGE_SIZE, + searchPage * CATALOG_PAGE_SIZE + CATALOG_PAGE_SIZE); + + const cartCount = cart.items.reduce((s, l) => s + l.quantity, 0); + const selectedItem = items.find((it) => it.id === selectedItemId) || null; + const isCustomer = !!currentUser && !currentUser.isAdmin && !currentUser.isStaff; + + return ( +
+ {initializing && ( +
+
+
Connecting to Storefront...
+
+ )} + +
+

+ Storefront +

+ + { setSearchQuery(e.target.value); setSearchPage(0); }} + onKeyDown={(e) => { + if (e.key === "Escape") setSearchQuery(""); + }} + /> +
+
+ + {currentUser && ( + + )} + {currentUser?.isAdmin && ( + + )} + {(currentUser?.isStaff || currentUser?.isAdmin) && ( + + )} + {currentUser ? ( + <> + + {currentUser.username} + + + + ) : ( + + )} +
+
+ +
+
+ {buyError && ( +
+ {buyError} +
+ )} + +
+
+ + { setMinimumPrice(e.target.value); setSearchPage(0); }} /> + { setMaximumPrice(e.target.value); setSearchPage(0); }} /> + +
+

Catalog

+
+
+ {searchResults.map((item) => ( + setSelectedItemId(item.id)} + onBuy={() => handleBuyNow(item.id)} + onAddToCart={() => handleAddToCart(item.id)} + /> + ))} +
+
+
+ + +
+
+ +
+

Recommended for you

+
+ {recommended.length === 0 ? ( +
Nothing recommended yet
+ ) : ( + recommended.map((item, index) => ( +
+ {index + 1} + setSelectedItemId(item.id)} onBuy={() => handleBuyNow(item.id)} + onAddToCart={() => handleAddToCart(item.id)} /> + {currentUser && } +
+ )) + )} +
+
+ token ? refreshCart(token) : Promise.resolve()} /> + {!activeView && (currentUser?.isStaff || currentUser?.isAdmin) && + token ? refreshCart(token) : Promise.resolve()} staffOnly />} +
+ + {selectedItem && ( + { + setSelectedItemId(null); + setReviewError(""); + }} + onBuy={() => handleBuyNow(selectedItem.id)} + onAddToCart={() => handleAddToCart(selectedItem.id)} + onSubmitReview={(rating, comment) => handleReviewSubmit(selectedItem.id, rating, comment)} + /> + )} +
+ +
setActiveView(null)} /> +
+
+

Cart

+ +
+
+ {cart.items.length === 0 ? ( +
+ Your cart is empty +
+ ) : ( + <> + {cart.items.map((line) => ( + handleQuantityChange(line.itemId, qty)} + onRemove={() => handleRemove(line.itemId)} + /> + ))} +
+ setPromotionCode(e.target.value)} placeholder="Promotion code" /> + +
+ {promotionError &&
{promotionError}
} +
+ Total + ${cart.total.toFixed(2)} +
+ + + )} +
+
+ +
setActiveView(null)} /> +
+
+

Order history

+ +
+ {orderError && ( +
+ {orderError} +
+ )} +
+ {orders.length === 0 ? ( +
You haven't placed any orders yet
+ ) : ( + orders.map((order) => ( +
+
+ {new Date(order.createdAt).toLocaleString()} + {order.status} +
+
{order.items.map((l) => `${l.name} ×${l.quantity}${l.returned ? " (returned)" : ""}`).join(", ")}
+
+ ${order.total.toFixed(2)} +
+
{Number(order.discount || 0).toFixed(2)}
+
{Number(order.refundTotal || 0).toFixed(2)}
+ {(order.payments ?? []).map((payment) => ( +
+ {payment.status} + {Number(payment.amount).toFixed(2)} +
+ ))} + {order.status === "refunded" && order.items.map(line => +
{line.name}
)} +
+ {order.status === "pending" && ( + + )} + {order.status === "shipped" && + order.items + .filter((l) => !l.returned) + .map((l) => ( + + ))} +
+
+ )) + )} +
+
+ + {fulfilmentOpen && (currentUser?.isStaff || currentUser?.isAdmin) && ( + setActiveView(null)} + onShip={handleShipOrder} orderError={orderError}> + token ? refreshCart(token) : Promise.resolve()} staffOnly /> + + )} + + {adminOpen && currentUser?.isAdmin && ( + setActiveView(null)} + onRestock={handleRestock} + onTransfer={handleTransfer} + onPriceChange={handlePriceChange} + orderError={orderError} + > + token ? refreshCart(token) : Promise.resolve()} staffOnly /> + + )} +
+ ); +} + +function ItemCard({ + item, + isCustomer, + onOpen, + onBuy, + onAddToCart, + testId = "item-card", +}: { + item: ItemT; + isCustomer: boolean; + onOpen: () => void; + onBuy: () => void; + onAddToCart: () => void; + testId?: string | null; +}) { + const outOfStock = item.stock === 0; + return ( +
+
+ {item.name} +
+
+ + ${item.price.toFixed(2)} + +
+
+ 0 && item.stock <= 5 ? " low" : ""}`} data-role="item-stock"> + {item.stock} + + {outOfStock && ( + + Out of stock + + )} +
+ {isCustomer && ( +
e.stopPropagation()}> + + +
+ )} + {(item.variants || []).map(variant => + {variant})} + {isCustomer && outOfStock && } +
+ ); +} + +function ItemDetailPanel({ + item, + detail, + isCustomer, + reviewError, + onClose, + onBuy, + onAddToCart, + onSubmitReview, +}: { + item: ItemT; + detail: ItemDetailT | null; + isCustomer: boolean; + reviewError: string; + onClose: () => void; + onBuy: () => void; + onAddToCart: () => void; + onSubmitReview: (rating: number, comment: string) => void; +}) { + const [rating, setRating] = useState(5); + const [comment, setComment] = useState(""); + const outOfStock = item.stock === 0; + + const submit = (e: React.FormEvent) => { + e.preventDefault(); + onSubmitReview(rating, comment); + setComment(""); + }; + + return ( +
+
+

{item.name}

+ +
+
+ + ${item.price.toFixed(2)} + + 0 && item.stock <= 5 ? "item-stock low" : "item-stock"}> + {item.stock} in stock + +
+ {outOfStock && ( + + Out of stock + + )} +

{detail?.description || "Loading description..."}

+ + {isCustomer && ( +
+ + +
+ )} + +

+ Reviews — average {(detail?.average ?? 0).toFixed(1)} +

+ {!detail || detail.reviews.length === 0 ? ( +
No reviews yet
+ ) : ( + detail.reviews.map((r) => ( +
+
+ {r.username} + {"★".repeat(r.rating)} +
+
{r.comment}
+
+ )) + )} + + {isCustomer && ( +
+ + setComment(e.target.value)} + /> + +
+ )} + {reviewError && ( +
+ {reviewError} +
+ )} +
+ ); +} + +function CartLineRow({ + line, + onQuantityChange, + onRemove, +}: { + line: CartLineT; + onQuantityChange: (qty: number) => void; + onRemove: () => void; +}) { + const [value, setValue] = useState(String(line.quantity)); + + useEffect(() => { + setValue(String(line.quantity)); + }, [line.quantity]); + + const commit = () => { + const qty = Number(value); + if (Number.isInteger(qty) && qty >= 1 && qty !== line.quantity) { + onQuantityChange(qty); + } else { + setValue(String(line.quantity)); + } + }; + + return ( +
+ {line.name} + {line.reservationSeconds || 0} + {line.expired && Expired} + setValue(e.target.value)} + onBlur={commit} + onKeyDown={(e) => { + if (e.key === "Enter") { + e.preventDefault(); + commit(); + } + }} + /> + ${(line.price * line.quantity).toFixed(2)} + +
+ ); +} + +function AuthBox({ + onSignUp, + onSignIn, +}: { + onSignUp: (username: string, password: string) => Promise; + onSignIn: (username: string, password: string) => Promise; +}) { + const [signUpUsername, setSignUpUsername] = useState(""); + const [signUpPassword, setSignUpPassword] = useState(""); + const [signUpError, setSignUpError] = useState(""); + + const [showSignIn, setShowSignIn] = useState(false); + const [signInUsername, setSignInUsername] = useState(""); + const [signInPassword, setSignInPassword] = useState(""); + const [signInError, setSignInError] = useState(""); + + const submitSignUp = async (e: React.FormEvent) => { + e.preventDefault(); + setSignUpError(""); + try { + await onSignUp(signUpUsername.trim(), signUpPassword); + } catch (err: any) { + setSignUpError(err.message); + } + }; + + const submitSignIn = async (e: React.FormEvent) => { + e.preventDefault(); + setSignInError(""); + try { + await onSignIn(signInUsername.trim(), signInPassword); + } catch (err: any) { + setSignInError(err.message); + } + }; + + return ( +
+
+ setSignUpUsername(e.target.value)} + /> + setSignUpPassword(e.target.value)} + /> + + {signUpError && ( +
+ {signUpError} +
+ )} +
+ + {showSignIn && ( +
+ setSignInUsername(e.target.value)} + /> + setSignInPassword(e.target.value)} + /> + + {signInError && ( +
+ {signInError} +
+ )} +
+ )} +
+ ); +} + +function AdminPanel({ + overview, + onClose, + onRestock, + onTransfer, + onPriceChange, + orderError, + children, +}: { + overview: AdminOverviewT | null; + onClose: () => void; + onRestock: (itemId: string, warehouseId: string, quantity: number) => void; + onTransfer: (itemId: string, fromWarehouseId: string, toWarehouseId: string, quantity: number) => void; + onPriceChange: (itemId: string, price: number) => void; + orderError: string; + children?: React.ReactNode; +}) { + const [restockValues, setRestockValues] = useState>({}); + const [globalRestock, setGlobalRestock] = useState({ item: "", warehouse: "", quantity: "" }); + const [priceValues, setPriceValues] = useState>({}); + const [transferValues, setTransferValues] = useState< + Record + >({}); + + if (!overview) { + return ( +
+
+

Admin

+ +
+
Loading admin data...
+
+ ); + } + + const warehouses = overview.warehouses; + + const transferFor = (itemId: string) => + transferValues[itemId] || { from: warehouses[0]?.id || "", to: warehouses[1]?.id || warehouses[0]?.id || "", qty: "" }; + + return ( +
+
+

Admin

+ +
+ + {orderError && ( +
+ {orderError} +
+ )} + +
+ Total revenue: ${overview.revenue.toFixed(2)} +
+ +
+ setGlobalRestock(value => ({ ...value, item: event.target.value }))} /> + setGlobalRestock(value => ({ ...value, warehouse: event.target.value }))} /> + setGlobalRestock(value => ({ ...value, quantity: event.target.value }))} /> + +
+ +
+
+

Items

+ {overview.items.map((it) => { + const transfer = transferFor(it.id); + return ( +
= 2 + ? JSON.stringify({ + itemId: it.id, + fromWarehouseId: warehouses.find((warehouse) => warehouse.name === "East")?.id + ?? warehouses[0].id, + toWarehouseId: warehouses.find((warehouse) => warehouse.name === "West")?.id + ?? warehouses[1].id, + quantity: 25, + }) + : undefined} key={it.id}> + {it.name} + {it.stock} +
+ setPriceValues((prev) => ({ ...prev, [it.id]: e.target.value }))} + /> + +
+
+ + + + setTransferValues((prev) => ({ ...prev, [it.id]: { ...transferFor(it.id), qty: e.target.value } })) + } + /> + +
+
+ ); + })} +
+
+

Warehouses

+
+ {overview.warehouses.map((w) => ( + + {w.name} — {w.total} + + ))} +
+

Stock by warehouse

+ {overview.locations.map((loc) => ( +
+ + {loc.itemName} @ {loc.warehouseName} + + {loc.quantity} +
+ setRestockValues((prev) => ({ ...prev, [loc.id]: e.target.value }))} + onKeyDown={(e) => { + if (e.key === "Enter") { + e.preventDefault(); + const qty = Number(restockValues[loc.id]); + if (Number.isInteger(qty) && qty >= 1) { + onRestock(loc.itemId, loc.warehouseId, qty); + setRestockValues((prev) => ({ ...prev, [loc.id]: "" })); + } + } + }} + /> + +
+
+ ))} +
+
+ +
+
+

Low stock

+
+ {overview.lowStock.length === 0 ? ( +
Nothing is running low
+ ) : ( + overview.lowStock.map((it) => ( +
+ {it.name} + {it.stock} +
+ )) + )} +
+
+
+

Category totals

+ {overview.categories.map((c) => ( +
+ {c.category} + {c.units} + ${c.revenue.toFixed(2)} +
+ ))} +
+
+ {children} +
+ ); +} + +function FulfilmentPanel({ + queue, + onClose, + onShip, + orderError, + children, +}: { + queue: FulfilmentQueueT; + onClose: () => void; + onShip: (orderId: string) => void; + orderError: string; + children?: React.ReactNode; +}) { + return ( +
+
+

Fulfilment queue

+ +
+ + {orderError && ( +
+ {orderError} +
+ )} + +
+ Orders waiting: {queue.depth} +
+ + {queue.orders.length === 0 ? ( +
Nothing waiting to ship
+ ) : ( + queue.orders.map((order) => ( +
+
+ {new Date(order.createdAt).toLocaleString()} +
+
{order.items.map((l) => `${l.name} ×${l.quantity}`).join(", ")}
+
+ {order.items.map((l, idx) => ( + + {(l.warehouseNames || []).join(", ") || "Unknown"} + + ))} +
+ +
+ )) + )} + {children} +
+ ); +} diff --git a/tools/stack-bench/reference-apps/ecommerce/mongodb/client/src/ProgressionPanel.tsx b/tools/stack-bench/reference-apps/ecommerce/mongodb/client/src/ProgressionPanel.tsx new file mode 100644 index 00000000000..8499198b1cb --- /dev/null +++ b/tools/stack-bench/reference-apps/ecommerce/mongodb/client/src/ProgressionPanel.tsx @@ -0,0 +1,331 @@ +import React, { useCallback, useEffect, useState } from "react"; + +type User = { username: string; isAdmin: boolean; isStaff: boolean; roles?: string[] }; +type Item = { id: string; name: string }; +type Order = { id: string; items: Array<{ name: string }>; total: number }; + +async function request(path: string, token: string | null, options: RequestInit = {}) { + const response = await fetch(path, { ...options, headers: { + "Content-Type": "application/json", ...(token ? { Authorization: `Bearer ${token}` } : {}), + } }); + const data = await response.json().catch(() => ({})); + if (!response.ok) throw new Error(data.error || "Request failed"); + return data; +} + +function nameFor(items: Item[], id: unknown) { + return items.find(item => item.id === String(id))?.name || String(id || "Unknown item"); +} + +export function ProgressionPanel({ token, user, items, orders, onSignIn, onRefreshItems, + onRefreshCart, staffOnly = false }: { + token: string | null; + user: User | null; + items: Item[]; + orders: Order[]; + onSignIn: (username: string, password: string) => Promise; + onRefreshItems: () => Promise; + onRefreshCart: () => Promise; + staffOnly?: boolean; +}) { + const [state, setState] = useState({ tickets: [], promotions: [], notifications: [], + scheduledRestocks: [], ledger: [], reorderRules: [], activities: [], staffUsers: [], orders: [] }); + const [error, setError] = useState(""); + const [staffName, setStaffName] = useState(""); + const [staffPassword, setStaffPassword] = useState(""); + const [profileOpen, setProfileOpen] = useState(false); + const [supportOpen, setSupportOpen] = useState(false); + const [notificationsOpen, setNotificationsOpen] = useState(false); + const [profileName, setProfileName] = useState(""); + const [profileAddress, setProfileAddress] = useState(""); + const [supportEmail, setSupportEmail] = useState(""); + const [supportSubject, setSupportSubject] = useState(""); + const [supportMessage, setSupportMessage] = useState(""); + const [supportReference, setSupportReference] = useState(""); + const [restoreWarning, setRestoreWarning] = useState(""); + + const refresh = useCallback(async () => { + try { + const next = await request("/api/progression/state", token); + setState(next); + if (next.profile) { + setProfileName(next.profile.name || ""); + setProfileAddress(next.profile.address || ""); + } + } catch (err: any) { + setError(err.message); + } + }, [token]); + + useEffect(() => { + refresh(); + const timer = setInterval(refresh, 1000); + return () => clearInterval(timer); + }, [refresh]); + + const act = async (path: string, options: RequestInit = {}) => { + setError(""); + try { + const result = await request(path, token, options); + await refresh(); + return result; + } catch (err: any) { + setError(err.message); + return null; + } + }; + + if (staffOnly) { + if (!(user?.isStaff || user?.isAdmin)) return null; + return ; + } + + const submitSupport = async () => { + const result = await act("/api/progression/support", { method: "POST", body: JSON.stringify({ + email: supportEmail, subject: supportSubject, message: supportMessage, + }) }); + if (result) setSupportReference(result.ticket.reference); + }; + + const saveProfile = () => act("/api/progression/profile", { method: "PUT", + body: JSON.stringify({ name: profileName, address: profileAddress }) }); + const preference = state.preference || { order: false, stock: false }; + + return
+ {!user ?
+

Staff sign in

+ setStaffName(event.target.value)} placeholder="Username" /> + setStaffPassword(event.target.value)} placeholder="Password" /> + +
: {user.username}} + + + + {profileOpen && user &&
+

Profile

+ setProfileName(event.target.value)} placeholder="Name" /> + setProfileAddress(event.target.value)} placeholder="Address" /> + +

{state.profile?.address || ""}

+
} + + {supportOpen &&
+

Support

+ setSupportEmail(event.target.value)} placeholder="Email" /> + setSupportSubject(event.target.value)} placeholder="Subject" /> +