diff --git a/.gitattributes b/.gitattributes
index 05b1a132398..1e9bf994ead 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -2,3 +2,5 @@
**/ModuleBindings/** linguist-generated=true eol=lf
/docs/llms/** linguist-generated=true
/docs/llms/*-details.json linguist-generated=false
+/tools/stack-bench/** text eol=lf
+/tools/stack-bench/**/*.woff2 -text -diff
diff --git a/codex-plugin/plugins/spacetimedb/skills/typescript-client/SKILL.md b/codex-plugin/plugins/spacetimedb/skills/typescript-client/SKILL.md
index 3a31183133f..62f86ff36fc 100644
--- a/codex-plugin/plugins/spacetimedb/skills/typescript-client/SKILL.md
+++ b/codex-plugin/plugins/spacetimedb/skills/typescript-client/SKILL.md
@@ -18,7 +18,7 @@ Generated bindings convert snake_case names to camelCase, including row fields:
## React: main.tsx
```typescript
-import React, { useEffect, useMemo } from 'react';
+import React, { useMemo } from 'react';
import ReactDOM from 'react-dom/client';
import { SpacetimeDBProvider } from 'spacetimedb/react';
import { DbConnection } from './module_bindings';
@@ -30,6 +30,7 @@ function Root() {
DbConnection.builder()
.withUri(SPACETIMEDB_URI)
.withDatabaseName(MODULE_NAME)
+ // Reuse the token issued on the previous connection.
.withToken(localStorage.getItem('auth_token') || undefined),
[]
);
@@ -46,6 +47,7 @@ ReactDOM.createRoot(document.getElementById('root')!).render();
## React: App.tsx
```typescript
+import { useEffect } from 'react';
import { useTable, useSpacetimeDB } from 'spacetimedb/react';
import { DbConnection, tables } from './module_bindings';
@@ -53,7 +55,7 @@ function App() {
const { isActive, identity: myIdentity, token, getConnection } = useSpacetimeDB();
const conn = getConnection() as DbConnection | null;
- // Save auth token
+ // Persist the issued token for the next page load.
useEffect(() => { if (token) localStorage.setItem('auth_token', token); }, [token]);
// Subscribe when connected. Prefer typed query builders over raw SQL
diff --git a/codex-plugin/plugins/spacetimedb/skills/typescript-server/SKILL.md b/codex-plugin/plugins/spacetimedb/skills/typescript-server/SKILL.md
index c9f2e7343fd..ba24bf781d5 100644
--- a/codex-plugin/plugins/spacetimedb/skills/typescript-server/SKILL.md
+++ b/codex-plugin/plugins/spacetimedb/skills/typescript-server/SKILL.md
@@ -98,6 +98,10 @@ Every column is a `t` builder value:
Modifiers: `.primaryKey()`, `.autoInc()`, `.unique()`, `.index('btree')`, `.default(value)`.
+`.primaryKey()` and `.unique()` apply to one column. For uniqueness across
+multiple columns, use a surrogate key, the multi-column index below, and a
+reducer that rejects an existing index match before inserting.
+
Use `.default(value)` only for a newly appended migration-safe field. Do not put defaults on primary-key, unique, or auto-increment columns.
Optional columns: `nickname: t.option(t.string())`
@@ -130,7 +134,9 @@ export { default } from './schema'; // re-export the schema for the module ent
## Reducers
-Reducers are created with `spacetimedb.reducer(...)`; the export name becomes the reducer name:
+Reducers are created with `spacetimedb.reducer(...)`. An exported `signUp`
+becomes `signUp` in generated clients and `sign_up` in `spacetime call` and
+`describe`:
```typescript
export const createEntity = spacetimedb.reducer(
@@ -278,9 +284,22 @@ const Shape = t.enum('Shape', {
A client subscribing to a view receives only the rows it returns. Use a per-user view
(keyed on `ctx.sender`) for per-viewer access control: deleting a row it depends on
(e.g. a membership row) automatically drops the rows it was exposing from that client.
+Use index accessors in views. Do not scan a whole table with `.iter()` when an
+indexed lookup can select the required rows.
`t.row(...)` and `t.object(...)` return schema builders, not TypeScript runtime row types. Let a view callback infer its result, or annotate a separately declared structural type such as `Array<{ sku: bigint; label: string }>`. A named output type must not reuse the generated PascalCase name of its view accessor (for example, reserve `DiscountedProduct` for a `discounted_product` view).
+A view context is `ViewCtx` (and `AnonymousViewCtx`), both exported from
+`spacetimedb/server`. It carries `sender`, a read-only `db`, and `from`; it is
+not a `ReducerCtx`, so a helper shared between a reducer and a view must accept
+either:
+
+```typescript
+import type { ReducerCtx, ViewCtx, InferSchema } from 'spacetimedb/server';
+type S = InferSchema;
+function stockOf(ctx: ReducerCtx | ViewCtx, itemId: bigint) { ... }
+```
+
Both `spacetimedb.view(...)` and `spacetimedb.anonymousView(...)` take three arguments: view options, the declared return schema, and the callback.
```typescript
@@ -288,7 +307,7 @@ Both `spacetimedb.view(...)` and `spacetimedb.anonymousView(...)` take three arg
export const activeUsers = spacetimedb.anonymousView(
{ name: 'active_users', public: true },
t.array(entity.rowType),
- (ctx) => [...ctx.db.entity.iter()].filter(e => e.active)
+ (ctx) => [...ctx.db.entity.active.filter(true)] // active: t.bool().index('btree')
);
// Per-user view (varies by ctx.sender):
diff --git a/crates/bindings-typescript/src/lib/query.ts b/crates/bindings-typescript/src/lib/query.ts
index bb93b0e6ce3..0c17e3d145a 100644
--- a/crates/bindings-typescript/src/lib/query.ts
+++ b/crates/bindings-typescript/src/lib/query.ts
@@ -248,19 +248,21 @@ export type NamespacedQueryBuilder =
* A runtime reference to a table. This materializes the RowExpr for us.
* TODO: Maybe add the full SchemaDef to the type signature depending on how joins will work.
*/
-export type TableRef = Readonly<{
- type: 'table';
- sourceName: TableDef['sourceName'];
- accessorName: string;
- cols: RowExpr;
- indexedCols: IndexedRowExpr;
- tableDef: TableDef;
+// Keep this named so TypeScript diagnostics show `TableRef` instead of its
+// expanded structure.
+export interface TableRef {
+ readonly type: 'table';
+ readonly sourceName: TableDef['sourceName'];
+ readonly accessorName: string;
+ readonly cols: RowExpr;
+ readonly indexedCols: IndexedRowExpr;
+ readonly tableDef: TableDef;
// Delegated UntypedTableDef properties for compatibility.
- columns: TableDef['columns'];
- indexes: TableDef['indexes'];
- rowType: TableDef['rowType'];
- constraints: any;
-}>;
+ readonly columns: TableDef['columns'];
+ readonly indexes: TableDef['indexes'];
+ readonly rowType: TableDef['rowType'];
+ readonly constraints: any;
+}
class TableRefImpl
implements TableRef, From
diff --git a/crates/bindings-typescript/src/sdk/connection_manager.ts b/crates/bindings-typescript/src/sdk/connection_manager.ts
index 211b01b5add..42febd5d221 100644
--- a/crates/bindings-typescript/src/sdk/connection_manager.ts
+++ b/crates/bindings-typescript/src/sdk/connection_manager.ts
@@ -145,9 +145,7 @@ class ConnectionManagerImpl {
clearTimeout(managed.reconnectTimer);
managed.reconnectTimer = null;
managed.reconnectAttempt = 0;
- if (managed.builder) {
- this.#buildManagedConnection(managed, managed.builder);
- }
+ this.#reconnectManagedConnection(managed);
continue;
}
@@ -179,9 +177,7 @@ class ConnectionManagerImpl {
connection.disconnect();
this.#updateState(managed, { isActive: false });
managed.reconnectAttempt = 0;
- if (managed.builder) {
- this.#buildManagedConnection(managed, managed.builder);
- }
+ this.#reconnectManagedConnection(managed);
}
/** Generates a unique key for a connection based on URI and module name. */
@@ -294,6 +290,14 @@ class ConnectionManagerImpl {
}
}
+ /** Reconnect with the issued token. Explicit rebuilds use the caller's token. */
+ #reconnectManagedConnection(managed: ManagedConnection): void {
+ if (!managed.builder) return;
+ const token = managed.state.token;
+ if (token) managed.builder.withToken(token);
+ this.#buildManagedConnection(managed, managed.builder);
+ }
+
#buildManagedConnection>(
managed: ManagedConnection,
builder: DbConnectionBuilder
@@ -349,7 +353,7 @@ class ConnectionManagerImpl {
return;
}
- this.#buildManagedConnection(managed, managed.builder);
+ this.#reconnectManagedConnection(managed);
}, delay);
}
diff --git a/crates/bindings-typescript/tests/connection_manager_liveness.test.ts b/crates/bindings-typescript/tests/connection_manager_liveness.test.ts
index ea24ea6887e..e9f76a909e9 100644
--- a/crates/bindings-typescript/tests/connection_manager_liveness.test.ts
+++ b/crates/bindings-typescript/tests/connection_manager_liveness.test.ts
@@ -12,7 +12,7 @@ type ErrorContextInterface = { isActive: boolean };
class MockConnection {
isActive = false;
identity = undefined;
- token = undefined;
+ token: string | undefined;
connectionId = ConnectionId.random();
isDisconnectRequested = false;
disconnected = false;
@@ -28,6 +28,10 @@ class MockConnection {
(ctx: ErrorContextInterface, error: Error) => void
>();
+ constructor(private readonly issuedToken?: string) {
+ this.token = undefined;
+ }
+
get isSocketClosed(): boolean {
return this.socketClosed;
}
@@ -63,6 +67,7 @@ class MockConnection {
simulateConnect(): void {
this.isActive = true;
+ this.token = this.issuedToken;
for (const cb of this.#onConnect) cb(this);
}
simulateDisconnect(error?: Error): void {
@@ -74,7 +79,9 @@ class MockConnection {
class MockBuilder {
buildCount = 0;
+ presentedTokens: Array = [];
connections: MockConnection[] = [];
+ #token: string | undefined;
#onConnect = new Set<(conn: MockConnection) => void>();
#onDisconnect = new Set<
@@ -84,9 +91,17 @@ class MockBuilder {
(ctx: ErrorContextInterface, error: Error) => void
>();
+ constructor(private readonly issuedToken?: string) {}
+
+ withToken(token?: string): MockBuilder {
+ this.#token = token;
+ return this;
+ }
+
build(): MockConnection {
- const connection = new MockConnection();
+ const connection = new MockConnection(this.issuedToken);
this.buildCount += 1;
+ this.presentedTokens.push(this.#token);
this.connections.push(connection);
for (const cb of this.#onConnect) connection.register('connect', cb);
for (const cb of this.#onDisconnect) connection.register('disconnect', cb);
@@ -201,6 +216,32 @@ describe('ConnectionManager liveness recovery', () => {
ConnectionManager.release(key);
});
+ test('reuses the issued token when reviving a dead socket', () => {
+ const key = nextKey();
+ const builder = new MockBuilder('issued-token');
+ const first = retain(key, builder);
+ expect(builder.presentedTokens).toEqual([undefined]);
+
+ first.simulateConnect();
+ first.socketClosed = true;
+ fire('win:online');
+
+ expect(builder.presentedTokens).toEqual([undefined, 'issued-token']);
+ ConnectionManager.release(key);
+ });
+
+ test('an explicit rebuild uses the caller token', () => {
+ const key = nextKey();
+ const firstBuilder = new MockBuilder('issued-token');
+ retain(key, firstBuilder).simulateConnect();
+
+ const replacement = new MockBuilder().withToken('caller-token');
+ ConnectionManager.rebuild(key, replacement as any);
+
+ expect(replacement.presentedTokens).toEqual(['caller-token']);
+ ConnectionManager.release(key);
+ });
+
test('does not rebuild a healthy connection on resume', () => {
const key = nextKey();
const builder = new MockBuilder();
diff --git a/crates/bindings-typescript/tests/table_ref_error_message.test.ts b/crates/bindings-typescript/tests/table_ref_error_message.test.ts
new file mode 100644
index 00000000000..009a9c192dd
--- /dev/null
+++ b/crates/bindings-typescript/tests/table_ref_error_message.test.ts
@@ -0,0 +1,79 @@
+import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import * as ts from 'typescript';
+import { describe, expect, it } from 'vitest';
+
+const bindingsRoot = path.resolve(
+ path.dirname(fileURLToPath(import.meta.url)),
+ '..'
+);
+
+function runTypecheck(source: string) {
+ const tmpDir = mkdtempSync(path.join(tmpdir(), 'stdb-tableref-diag-'));
+ const reproPath = path.join(tmpDir, 'repro.ts');
+ writeFileSync(reproPath, source);
+
+ try {
+ const options: ts.CompilerOptions = {
+ target: ts.ScriptTarget.ESNext,
+ module: ts.ModuleKind.ESNext,
+ strict: true,
+ noEmit: true,
+ skipLibCheck: true,
+ forceConsistentCasingInFileNames: true,
+ allowImportingTsExtensions: true,
+ noImplicitAny: true,
+ moduleResolution: ts.ModuleResolutionKind.Bundler,
+ useDefineForClassFields: true,
+ verbatimModuleSyntax: true,
+ isolatedModules: true,
+ };
+
+ const host = ts.createCompilerHost(options);
+ const program = ts.createProgram(
+ [reproPath, path.join(bindingsRoot, 'src/server/sys.d.ts')],
+ options,
+ host
+ );
+ const diagnostics = ts.getPreEmitDiagnostics(program);
+ return diagnostics.map(d =>
+ ts.flattenDiagnosticMessageText(d.messageText, '\n')
+ );
+ } finally {
+ rmSync(tmpDir, { recursive: true, force: true });
+ }
+}
+
+describe('TableRef diagnostics', () => {
+ const source = `
+import { t } from ${JSON.stringify(path.join(bindingsRoot, 'src/server/index.ts'))};
+import { table } from ${JSON.stringify(path.join(bindingsRoot, 'src/lib/table.ts'))};
+import { createTableRefFromDef } from ${JSON.stringify(path.join(bindingsRoot, 'src/lib/query.ts'))};
+import type { AllUnique } from ${JSON.stringify(path.join(bindingsRoot, 'src/lib/constraints.ts'))};
+
+const cartItem = table(
+ { name: 'cart_item' },
+ { id: t.u64().primaryKey().autoInc(), accountId: t.u64(), quantity: t.u32() }
+);
+
+const ref = createTableRefFromDef(cartItem as any, 'cartItem');
+type Boom = AllUnique;
+declare const b: Boom;
+`;
+
+ it('names the type instead of dumping its structure', () => {
+ const messages = runTypecheck(source);
+ const constraintError = messages.find(m =>
+ m.includes("does not satisfy the constraint 'UntypedTableDef'")
+ );
+
+ expect(constraintError).toBeDefined();
+ // The name, not the shape.
+ expect(constraintError).toContain('TableRef<');
+ expect(constraintError).not.toContain('type: "table"');
+ expect(constraintError).not.toContain('accessorName');
+ expect(constraintError.length).toBeLessThan(250);
+ }, 15000);
+});
diff --git a/crates/cli/build.rs b/crates/cli/build.rs
index c5bd4303464..90da9fa3bdd 100644
--- a/crates/cli/build.rs
+++ b/crates/cli/build.rs
@@ -110,6 +110,7 @@ fn generate_template_files() {
// Embed skill files from skills/*/SKILL.md
let skills_dir = repo_root.join("skills");
+ println!("cargo:rerun-if-changed={}", skills_dir.display());
let skill_names = discover_skill_names(&skills_dir);
generated_code.push_str("pub fn get_skill(name: &str) -> Option<&'static str> {\n");
diff --git a/crates/cli/src/subcommands/dev.rs b/crates/cli/src/subcommands/dev.rs
index a0588b6de50..a2622c8ea0c 100644
--- a/crates/cli/src/subcommands/dev.rs
+++ b/crates/cli/src/subcommands/dev.rs
@@ -738,7 +738,7 @@ pub async fn exec(mut config: Config, args: &ArgMatches) -> Result<(), anyhow::E
let loaded_config_dir = loaded_config.as_ref().map(|lc| lc.config_dir.clone());
generate_build_and_publish(
- &config,
+ &mut config,
&project_dir,
loaded_config_dir.as_deref(),
&spacetimedb_dir,
@@ -853,7 +853,7 @@ pub async fn exec(mut config: Config, args: &ArgMatches) -> Result<(), anyhow::E
println!("\n{}", "File change detected, rebuilding...".yellow());
match generate_build_and_publish(
- &config,
+ &mut config,
&project_dir,
loaded_config_dir.as_deref(),
&spacetimedb_dir,
@@ -1000,7 +1000,7 @@ fn upsert_env_db_names_and_hosts(env_path: &Path, server_host_url: &str, databas
#[allow(clippy::too_many_arguments)]
async fn generate_build_and_publish(
- config: &Config,
+ config: &mut Config,
project_dir: &Path,
config_dir: Option<&Path>,
spacetimedb_dir: &Path,
@@ -1146,7 +1146,8 @@ async fn generate_build_and_publish(
publish_entry.insert("break-clients".to_string(), json!(true));
}
- publish::exec_from_entry(config.clone(), publish_entry, config_dir, clear_database, yes).await?;
+ // Preserve a token created during publish for logs and later rebuilds.
+ publish::exec_from_entry(config, publish_entry, config_dir, clear_database, yes).await?;
}
println!("{}", "Published successfully!".green().bold());
diff --git a/crates/cli/src/subcommands/publish.rs b/crates/cli/src/subcommands/publish.rs
index 745664880f0..63d704d25af 100644
--- a/crates/cli/src/subcommands/publish.rs
+++ b/crates/cli/src/subcommands/publish.rs
@@ -450,7 +450,7 @@ pub async fn exec_with_options(
}
pub async fn exec_from_entry(
- mut config: Config,
+ config: &mut Config,
entry: HashMap,
config_dir: Option<&std::path::Path>,
clear_database: ClearMode,
@@ -465,7 +465,7 @@ pub async fn exec_from_entry(
let yes = if force { YesFlags::all() } else { YesFlags::default() };
- execute_publish_configs(&mut config, vec![command_config], true, config_dir, clear_database, yes).await
+ execute_publish_configs(config, vec![command_config], true, config_dir, clear_database, yes).await
}
async fn execute_publish_configs<'a>(
diff --git a/skills/spacetimedb-typescript-core/SKILL.md b/skills/spacetimedb-typescript-core/SKILL.md
new file mode 100644
index 00000000000..993a527a452
--- /dev/null
+++ b/skills/spacetimedb-typescript-core/SKILL.md
@@ -0,0 +1,101 @@
+---
+name: spacetimedb-typescript-core
+description: Core SpacetimeDB TypeScript server and client SDK syntax for building an application without framework or architecture guidance.
+license: Apache-2.0
+metadata:
+ author: clockworklabs
+ version: "1.0"
+ language: typescript
+---
+
+# SpacetimeDB TypeScript Core API
+
+## Server module
+
+Define tables with `table()`, bind them with `schema()`, and export the schema
+as the module default. Export reducers from the same module or its entry file.
+
+```typescript
+import { schema, table, t } from 'spacetimedb/server';
+
+const record = table(
+ { name: 'record', public: true },
+ {
+ id: t.u64().primaryKey().autoInc(),
+ label: t.string().index('btree'),
+ value: t.u32(),
+ },
+);
+
+const spacetimedb = schema({ record });
+export default spacetimedb;
+
+export const createRecord = spacetimedb.reducer(
+ { label: t.string(), value: t.u32() },
+ (ctx, { label, value }) => {
+ ctx.db.record.insert({ id: 0n, label, value });
+ },
+);
+```
+
+Table names must be snake_case. The keys passed to `schema({ ... })` are the
+server-side `ctx.db` accessor names. A split module must re-export the schema
+as the default export from its entry file.
+
+## Types and table access
+
+Common builders are `t.string()`, `t.bool()`, `t.u32()`, `t.i32()`,
+`t.u64()`, `t.i64()`, `t.identity()`, `t.timestamp()`, and
+`t.option(inner)`. The 64-bit integer builders use TypeScript `bigint` values.
+Use `0n` for an auto-increment `u64` or `i64` field during insertion.
+
+Column modifiers include `.primaryKey()`, `.autoInc()`, `.unique()`, and
+`.index('btree')`.
+
+```typescript
+const row = ctx.db.record.id.find(id); // row | null
+const inserted = ctx.db.record.insert(values); // inserted row
+if (row) ctx.db.record.id.update({ ...row, value: 2 }); // update by primary key
+ctx.db.record.id.delete(id); // delete by primary key
+const matching = [...ctx.db.record.label.filter(label)];
+const all = [...ctx.db.record.iter()];
+```
+
+`iter()` and `filter()` return iterators. Spread them before using array
+methods. Insert through the table accessor, not through an index accessor.
+
+## Generated client bindings
+
+Generated bindings convert snake_case table, reducer, and field names to
+camelCase. A server reducer named `createRecord` is called as `createRecord` in a
+TypeScript client.
+
+Create a connection with the generated `DbConnection`:
+
+```typescript
+import { DbConnection, tables } from './module_bindings';
+
+const connection = DbConnection.builder()
+ .withUri(serverUri)
+ .withDatabaseName(moduleName)
+ .onConnect(ctx => {
+ ctx.subscriptionBuilder()
+ .onApplied(() => console.log('ready'))
+ .subscribe([tables.record]);
+ })
+ .build();
+```
+
+Call reducers with an object argument:
+
+```typescript
+await connection.reducers.createRecord({ label: 'Example', value: 1 });
+```
+
+The generated database accessors support row callbacks:
+
+```typescript
+connection.db.record.onInsert((_ctx, row) => console.log(row.label));
+connection.db.record.onUpdate((_ctx, oldRow, newRow) => console.log(oldRow, newRow));
+connection.db.record.onDelete((_ctx, row) => console.log(row.id));
+```
diff --git a/skills/typescript-client/SKILL.md b/skills/typescript-client/SKILL.md
index 3a31183133f..62f86ff36fc 100644
--- a/skills/typescript-client/SKILL.md
+++ b/skills/typescript-client/SKILL.md
@@ -18,7 +18,7 @@ Generated bindings convert snake_case names to camelCase, including row fields:
## React: main.tsx
```typescript
-import React, { useEffect, useMemo } from 'react';
+import React, { useMemo } from 'react';
import ReactDOM from 'react-dom/client';
import { SpacetimeDBProvider } from 'spacetimedb/react';
import { DbConnection } from './module_bindings';
@@ -30,6 +30,7 @@ function Root() {
DbConnection.builder()
.withUri(SPACETIMEDB_URI)
.withDatabaseName(MODULE_NAME)
+ // Reuse the token issued on the previous connection.
.withToken(localStorage.getItem('auth_token') || undefined),
[]
);
@@ -46,6 +47,7 @@ ReactDOM.createRoot(document.getElementById('root')!).render();
## React: App.tsx
```typescript
+import { useEffect } from 'react';
import { useTable, useSpacetimeDB } from 'spacetimedb/react';
import { DbConnection, tables } from './module_bindings';
@@ -53,7 +55,7 @@ function App() {
const { isActive, identity: myIdentity, token, getConnection } = useSpacetimeDB();
const conn = getConnection() as DbConnection | null;
- // Save auth token
+ // Persist the issued token for the next page load.
useEffect(() => { if (token) localStorage.setItem('auth_token', token); }, [token]);
// Subscribe when connected. Prefer typed query builders over raw SQL
diff --git a/skills/typescript-server/SKILL.md b/skills/typescript-server/SKILL.md
index c9f2e7343fd..ba24bf781d5 100644
--- a/skills/typescript-server/SKILL.md
+++ b/skills/typescript-server/SKILL.md
@@ -98,6 +98,10 @@ Every column is a `t` builder value:
Modifiers: `.primaryKey()`, `.autoInc()`, `.unique()`, `.index('btree')`, `.default(value)`.
+`.primaryKey()` and `.unique()` apply to one column. For uniqueness across
+multiple columns, use a surrogate key, the multi-column index below, and a
+reducer that rejects an existing index match before inserting.
+
Use `.default(value)` only for a newly appended migration-safe field. Do not put defaults on primary-key, unique, or auto-increment columns.
Optional columns: `nickname: t.option(t.string())`
@@ -130,7 +134,9 @@ export { default } from './schema'; // re-export the schema for the module ent
## Reducers
-Reducers are created with `spacetimedb.reducer(...)`; the export name becomes the reducer name:
+Reducers are created with `spacetimedb.reducer(...)`. An exported `signUp`
+becomes `signUp` in generated clients and `sign_up` in `spacetime call` and
+`describe`:
```typescript
export const createEntity = spacetimedb.reducer(
@@ -278,9 +284,22 @@ const Shape = t.enum('Shape', {
A client subscribing to a view receives only the rows it returns. Use a per-user view
(keyed on `ctx.sender`) for per-viewer access control: deleting a row it depends on
(e.g. a membership row) automatically drops the rows it was exposing from that client.
+Use index accessors in views. Do not scan a whole table with `.iter()` when an
+indexed lookup can select the required rows.
`t.row(...)` and `t.object(...)` return schema builders, not TypeScript runtime row types. Let a view callback infer its result, or annotate a separately declared structural type such as `Array<{ sku: bigint; label: string }>`. A named output type must not reuse the generated PascalCase name of its view accessor (for example, reserve `DiscountedProduct` for a `discounted_product` view).
+A view context is `ViewCtx` (and `AnonymousViewCtx`), both exported from
+`spacetimedb/server`. It carries `sender`, a read-only `db`, and `from`; it is
+not a `ReducerCtx`, so a helper shared between a reducer and a view must accept
+either:
+
+```typescript
+import type { ReducerCtx, ViewCtx, InferSchema } from 'spacetimedb/server';
+type S = InferSchema;
+function stockOf(ctx: ReducerCtx | ViewCtx, itemId: bigint) { ... }
+```
+
Both `spacetimedb.view(...)` and `spacetimedb.anonymousView(...)` take three arguments: view options, the declared return schema, and the callback.
```typescript
@@ -288,7 +307,7 @@ Both `spacetimedb.view(...)` and `spacetimedb.anonymousView(...)` take three arg
export const activeUsers = spacetimedb.anonymousView(
{ name: 'active_users', public: true },
t.array(entity.rowType),
- (ctx) => [...ctx.db.entity.iter()].filter(e => e.active)
+ (ctx) => [...ctx.db.entity.active.filter(true)] // active: t.bool().index('btree')
);
// Per-user view (varies by ctx.sender):
diff --git a/tools/llm-sequential-upgrade/.gitignore b/tools/llm-sequential-upgrade/.gitignore
index 14aa619a63d..35223d12b8f 100644
--- a/tools/llm-sequential-upgrade/.gitignore
+++ b/tools/llm-sequential-upgrade/.gitignore
@@ -27,4 +27,4 @@ telemetry/metrics.jsonl
**/telemetry/**/metadata.json
# Sequential-upgrade run output lives in the external spacetimedb-ai-test-results repo
-sequential-upgrade/sequential-upgrade-*/
+sequential-upgrade/
diff --git a/tools/llm-sequential-upgrade/read-guard.sh b/tools/llm-sequential-upgrade/read-guard.sh
new file mode 100644
index 00000000000..314c52e7cde
--- /dev/null
+++ b/tools/llm-sequential-upgrade/read-guard.sh
@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+# Write Claude Code settings that deny direct Read tool access to benchmark
+# internals. Bash is allowed, so this is not filesystem isolation.
+
+write_read_guard() {
+ local app_dir="$1" backend="$2" out siblings=""
+ out="$app_dir/.read-guard-settings.json"
+
+ local b
+ for b in spacetime postgres mongodb; do
+ [[ "$b" == "$backend" ]] && continue
+ siblings+=" \"Read(**/$b/results/**)\",
+"
+ done
+
+ cat > "$out" <
+campaign inspect
+campaign report
+```
+
+- `status` is the compact normal view.
+- `inspect` adds score, cost, duration, cleanup, evidence, and feature progress.
+- `report` rebuilds `report/report.json` and `report/report.html` from retained
+ evidence.
+
+Do not infer state from logs. Use logs only to diagnose a reported phase or
+failure. The controller never retries, extends, or grants paid work
+automatically.
+
+## Resume and repair
+
+If the controller stopped while an attempt remained live, reconcile ownership
+before any resume:
+
+```sh
+campaign reconcile --out
+```
+
+Reconciliation changes state only when private supervisor evidence proves that
+the exact owned resources are clean.
+
+Dependency campaigns can grant more repairs to selected exhausted features:
+
+```sh
+campaign grant-repairs \
+ --attempt --grant-id --level \
+ --feature --repairs
+```
+
+The grant creates a linked continuation. It does not rewrite the completed
+execution. Use `campaign resume --out ` to
+run scheduled dependency work.
+
+## Model-free trials and qualification
+
+`campaign trial` accepts only registered non-billable adapters and zero pricing.
+It validates orchestration but does not produce comparative model data.
+
+Check qualification requirements without starting work:
+
+```sh
+docker compose --env-file /var/lib/stack-bench/operator.env \
+ -f appliance/docker-compose.yaml run --rm controller \
+ qualification status --track ecommerce --level
+```
+
+Run only evidence required by that exact status. Do not repeat reference,
+mutation, or null work when its bound inputs have not changed. See the
+[reference app guide](../reference-apps/README.md) and
+[grader guide](../grader/README.md) for qualification rules.
+
+## Dashboard
+
+Start the optional dashboard:
+
+```sh
+docker compose --env-file /var/lib/stack-bench/operator.env \
+ -f appliance/docker-compose.yaml --profile dashboard up -d dashboard
+```
+
+Open `http://127.0.0.1:7331`. The dashboard reads and controls the same campaign
+state as the CLI. See [dashboard/README.md](../dashboard/README.md).
+
+## Results and cleanup
+
+Results remain under `/var/lib/stack-bench/results` after the controller exits.
+Verify and copy the complete campaign package before deleting the runner.
+
+A run removes only resources whose private ownership evidence still matches.
+If cleanup cannot be proved, it preserves the evidence and quarantines the run.
+Follow [RECOVERY.md](RECOVERY.md). Do not delete same-name resources or clear the
+shared state root by guesswork.
diff --git a/tools/stack-bench/appliance/RECOVERY.md b/tools/stack-bench/appliance/RECOVERY.md
new file mode 100644
index 00000000000..ecf413d98af
--- /dev/null
+++ b/tools/stack-bench/appliance/RECOVERY.md
@@ -0,0 +1,67 @@
+# Interruption and recovery
+
+Stack Bench never guesses that a container, listener, lock, database, or data
+directory is safe to delete. Normal teardown authenticates the run's private
+lease, compares exact container IDs and listener PIDs, and releases only locks
+whose owner record still matches that lease.
+
+Every appliance run keeps two different records:
+
+- `results/.../recovery.json` is public, contains no ownership token, and says
+ whether cleanup is `clean`, intentionally `retained`, or `quarantined`;
+- `controller-home/supervisor/.json` is private recovery authority. It
+ contains the lease token and must remain readable only by the appliance
+ operator. Normal cleanup deletes it. Refused cleanup deliberately preserves
+ it.
+
+## If a run is interrupted
+
+1. Preserve the result directory and private supervisor-state file.
+2. Read `recovery.json`. Do not publish an attempt whose status is
+ `quarantined`.
+3. Do not start another run using any lock key listed in that artifact.
+4. Retry authenticated cleanup from the controller:
+
+```sh
+docker compose --env-file /var/lib/stack-bench/operator.env \
+ -f appliance/docker-compose.yaml run --rm controller \
+ recover /var/lib/stack-bench/controller-home/supervisor/.json
+```
+
+On success the command changes `recovery.json` to `clean`, releases the exact
+owned resources, and removes the private supervisor state. It is idempotent
+when public lease evidence already proves that an earlier cleanup completed.
+
+If the parent process ended before it retained a supervisor file, recover from
+the private runtime lease instead. Supply a durable output directory outside
+the private runtime directory:
+
+```sh
+docker compose --env-file /var/lib/stack-bench/operator.env \
+ -f appliance/docker-compose.yaml run --rm controller \
+ recover-lease /var/lib/stack-bench/controller-home/runtime//backend-lease.json \
+ --out /var/lib/stack-bench/results/recovery/
+```
+
+This path uses the same ownership token, container ID, listener PID, and lock
+checks. It refuses an output directory inside the runtime directory because a
+successful recovery removes that directory.
+
+## If recovery refuses
+
+Refusal is the safety behavior. It means a live resource does not match the
+lease or its identity could not be proven. The command leaves the private state,
+lease, lock records, and public quarantine artifact intact.
+
+Compare the live container ID and listener PIDs with `recovery.json` and the
+private lease before manual action. Never delete a same-name container, kill a
+port's current listener, remove another lock, or recursively clear the shared
+state root merely because its name resembles Stack Bench. Escalate with the
+complete result directory and private state stored separately from public
+artifacts.
+
+## Intentional retention
+
+`--retain-backend` is inspection mode, not successful cleanup. It writes
+`status: "retained"` and preserves private recovery authority. No other run may
+reuse the listed locks until the recovery command completes.
diff --git a/tools/stack-bench/appliance/RELEASE.md b/tools/stack-bench/appliance/RELEASE.md
new file mode 100644
index 00000000000..23b34502679
--- /dev/null
+++ b/tools/stack-bench/appliance/RELEASE.md
@@ -0,0 +1,121 @@
+# Release assembly and verification
+
+Stack Bench uses two deliberately different release states.
+
+- A `candidate` has exact image digests, checksummed files, and digest-bound
+ SPDX SBOMs. It is useful for inspecting and testing a proposed bundle, but it
+ is unsigned and cannot be called qualified.
+- A `qualified` release adds a bundled public key, a detached Sigstore bundle
+ covering `release.json`, and registry signatures for every image. Verification
+ must use a public key obtained outside the release bundle.
+
+Schema v2 is the only accepted release format.
+
+## Build the controller image
+
+Build the Linux/amd64 controller from a clean checkout. The source identity
+command refuses changed or untracked release inputs.
+
+```powershell
+$source = npm --prefix tools/stack-bench run release:source --silent | ConvertFrom-Json
+docker build --platform linux/amd64 `
+ -f tools/stack-bench/appliance/Controller.Dockerfile `
+ --build-arg SOURCE_REVISION=$($source.revision) `
+ --build-arg SOURCE_SHA256=$($source.sha256) `
+ --build-arg BINARY_SOURCE_SHA256=$($source.binarySourceSha256) `
+ -t stack-bench-controller:development .
+```
+
+The build accepts only Linux SpacetimeDB binaries recorded in
+`container/spacetimedb-binaries.json`. Rebuild them with
+`bash tools/stack-bench/container/build-linux-cli.sh` after a recorded binary
+source changes. Review and commit the updated provenance file. The binary files
+remain ignored.
+
+## Build a candidate
+
+Publish the first-party images, resolve every first- and third-party image to an
+exact single-platform `linux/amd64` manifest reference, then generate one SPDX
+SBOM for each exact reference. Do not use a multi-architecture index digest:
+Docker Scout correctly reports the selected child-manifest digest, so an index
+digest cannot satisfy the one-image/one-SBOM identity contract.
+
+```sh
+node dist/src/releases/release-bundle.js sbom registry.example/controller@sha256:DIGEST \
+ --output bundle/sbom/controller.spdx.json
+```
+
+The command uses registry resolution, refuses mutable references and existing
+output, and checks that Docker Scout's SPDX 2.3 document contains the requested
+image digest. A successful tool exit without that digest binding is rejected.
+
+Create a strict release specification with `state: "candidate"`,
+`signing: null`, and `files` entries containing only `path` and `role`. Place
+every input below the bundle root, then materialize immutable size and SHA-256
+metadata:
+
+```sh
+node dist/src/releases/release-bundle.js assemble release-spec.json \
+ --root bundle --output bundle/release.json
+node dist/src/releases/release-manifest.js verify bundle/release.json --root bundle
+```
+
+Candidate verification reports `candidate-file-integrity`. It validates all
+declared files and all four image-to-SBOM digest bindings. Candidate manifests
+must use `signing: null` and cannot include a public signing key.
+
+## Sign and qualify
+
+Signing keys are external CI inputs. Never copy a private key, registry token,
+or signing password into the source tree, image, bundle, Compose environment,
+or command transcript. Sign each exact registry image with Cosign. The
+authoritative image-signature evidence stays attached to the registry object
+and is checked directly during verification; the release does not preserve a
+redundant unverified export. Add the public half of the signing key as
+`signing/cosign.pub` with the `public-key` role.
+
+Change the specification to `state: "qualified"` and declare:
+
+```json
+{
+ "signing": {
+ "scheme": "cosign-public-key-v1",
+ "publicKeyPath": "signing/cosign.pub",
+ "manifestBundlePath": "signing/release-manifest.sigstore.json"
+ }
+}
+```
+
+Assemble `release.json` only after all other evidence exists, then sign that
+exact file with a detached Cosign bundle:
+
+```sh
+cosign sign-blob --yes --key "$COSIGN_KEY" \
+ --bundle bundle/signing/release-manifest.sigstore.json bundle/release.json
+```
+
+The detached bundle is intentionally not checksummed by `release.json`: a file
+cannot contain the hash of its own signature. Cosign authenticates it instead.
+
+Verify with the trusted public key copied to a path outside the downloaded
+bundle:
+
+```sh
+node dist/src/releases/release-manifest.js verify bundle/release.json --root bundle \
+ --trusted-key /operator/trust/stack-bench-cosign.pub
+```
+
+Qualified verification refuses an absent or bundle-local trust key, requires
+it to equal the public key bound by the signed manifest, verifies the detached
+manifest signature, and runs `cosign verify` against every exact registry image
+reference. A failed or unavailable Cosign invocation is a failed release; there
+is no downgrade to candidate verification. The controller image includes
+checksum-pinned Cosign 3.1.3 so this command is available in the delivered
+appliance rather than depending on an untracked host installation.
+
+## Trust distribution
+
+The release bundle cannot establish trust in its own key. Publish the expected
+public key and its SHA-256 fingerprint through a separately controlled channel.
+The operator must compare that fingerprint before verification. Key rotation
+requires a new release and an explicit trust-distribution update.
diff --git a/tools/stack-bench/appliance/campaign.ecommerce-progression-reference.json b/tools/stack-bench/appliance/campaign.ecommerce-progression-reference.json
new file mode 100644
index 00000000000..01a584ee447
--- /dev/null
+++ b/tools/stack-bench/appliance/campaign.ecommerce-progression-reference.json
@@ -0,0 +1,96 @@
+{
+ "schemaVersion": 7,
+ "kind": "campaign-manifest",
+ "id": "ecommerce-progression-reference",
+ "version": "2.0.1",
+ "state": "draft",
+ "title": "Ecommerce progression reference pilot",
+ "track": "ecommerce",
+ "mode": { "id": "dependency", "version": "4.3.0", "workSelection": "progressive" },
+ "repair": { "selection": "feature", "budget": { "total": 0 } },
+ "levels": [1, 2, 3, 4, 5, 6],
+ "featureCatalog": "ecommerce.questlines@2.0.2",
+ "selection": {
+ "levels": [
+ { "level": 1, "recipe": "ecommerce.progression-catalog@2.0.2" },
+ { "level": 2, "recipe": "ecommerce.progression-catalog@2.0.2" },
+ { "level": 3, "recipe": "ecommerce.progression-catalog@2.0.2" },
+ { "level": 4, "recipe": "ecommerce.progression-catalog@2.0.2" },
+ { "level": 5, "recipe": "ecommerce.progression-catalog@2.0.2" },
+ { "level": 6, "recipe": "ecommerce.progression-catalog@2.0.2" }
+ ]
+ },
+ "stacks": [
+ { "id": "mongodb", "adapterVersion": "1.4.0" },
+ { "id": "postgres", "adapterVersion": "1.5.0" },
+ { "id": "spacetime", "adapterVersion": "1.3.0" }
+ ],
+ "agents": [
+ {
+ "adapter": "reference-fixture",
+ "adapterVersion": "1.4.0",
+ "model": "reference-fixture"
+ }
+ ],
+ "conditions": [
+ {
+ "id": "reference-pilot",
+ "version": "1.0.0",
+ "guidanceProfile": "neutral@1.8.0",
+ "repairPolicy": "scored-only@1.1.0"
+ }
+ ],
+ "repetitions": 1,
+ "parallelism": 1,
+ "ordering": {
+ "method": "balanced-rotation",
+ "seed": "ecommerce-progression-reference-1"
+ },
+ "budgets": {
+ "attemptTimeoutMinutes": 180,
+ "maxCostUsdPerAttempt": null
+ },
+ "attemptPolicy": {
+ "retries": 0,
+ "retryOn": [],
+ "excludeFromAnalysis": [
+ "contaminated",
+ "harness_failure",
+ "inconclusive",
+ "ungraded"
+ ]
+ },
+ "runtime": {
+ "releaseManifestSha256": null,
+ "controllerImage": null,
+ "buildImage": null,
+ "platform": "linux/amd64"
+ },
+ "pricing": {
+ "currency": "USD",
+ "unit": "USD-per-million-tokens",
+ "capturedAt": "2026-08-25T00:00:00.000Z",
+ "source": "Reference fixtures make no provider calls.",
+ "models": {
+ "reference-fixture": {
+ "input": 0,
+ "output": 0,
+ "cacheWrite5m": 0,
+ "cacheWrite1h": 0,
+ "cacheRead": 0
+ }
+ }
+ },
+ "analysis": {
+ "primaryMetric": "finalScoreRate",
+ "secondaryMetrics": [
+ "firstBuildScoreRate",
+ "totalDurationMs",
+ "invalidAttemptRate"
+ ],
+ "dispersion": "median-iqr",
+ "invalidAttempts": "report-separately",
+ "missingData": "no-imputation",
+ "comparisonUnit": "stack-agent-condition-recipe"
+ }
+}
diff --git a/tools/stack-bench/appliance/campaign.example.json b/tools/stack-bench/appliance/campaign.example.json
new file mode 100644
index 00000000000..95d974b7002
--- /dev/null
+++ b/tools/stack-bench/appliance/campaign.example.json
@@ -0,0 +1,115 @@
+{
+ "schemaVersion": 7,
+ "kind": "campaign-manifest",
+ "id": "ecommerce-l1-example",
+ "version": "2.0.0",
+ "state": "draft",
+ "title": "Ecommerce L1 model-free example",
+ "track": "ecommerce",
+ "mode": { "id": "sequential", "version": "1.0.0" },
+ "repair": { "selection": "batch", "budget": { "total": 3 } },
+ "levels": [1],
+ "selection": {
+ "levels": [
+ {
+ "level": 1,
+ "recipe": "ecommerce.sequential-l1@2.5.0",
+ "features": [
+ "ecommerce.feature.accounts",
+ "ecommerce.feature.cart-checkout",
+ "ecommerce.feature.catalog",
+ "ecommerce.feature.purchasing",
+ "ecommerce.feature.reviews",
+ "ecommerce.feature.warehouse-admin"
+ ],
+ "checks": []
+ }
+ ]
+ },
+ "stacks": [
+ { "id": "spacetime", "adapterVersion": "1.3.0" },
+ { "id": "postgres", "adapterVersion": "1.5.0" },
+ { "id": "mongodb", "adapterVersion": "1.4.0" }
+ ],
+ "agents": [
+ {
+ "adapter": "deterministic",
+ "adapterVersion": "1.3.0",
+ "model": "deterministic"
+ }
+ ],
+ "conditions": [
+ {
+ "id": "prescribed",
+ "version": "1.1.0",
+ "guidanceProfile": "prescribed@1.2.0",
+ "repairPolicy": "scored-only@1.1.0",
+ "specifications": {
+ "levels": [
+ {
+ "level": 1,
+ "requested": [
+ "ecommerce.spec.access-control@1.2.0",
+ "ecommerce.spec.concurrency-safety@1.3.0",
+ "ecommerce.spec.external-data-sync@1.1.0",
+ "ecommerce.spec.live-state@1.2.0",
+ "ecommerce.spec.state-durability@1.1.0",
+ "ecommerce.spec.transactional-integrity@1.3.0"
+ ],
+ "expected": [],
+ "observed": []
+ }
+ ]
+ }
+ }
+ ],
+ "repetitions": 3,
+ "parallelism": 1,
+ "ordering": {
+ "method": "balanced-rotation",
+ "seed": "replace-before-measurement"
+ },
+ "budgets": {
+ "attemptTimeoutMinutes": 240,
+ "maxCostUsdPerAttempt": null
+ },
+ "attemptPolicy": {
+ "retries": 1,
+ "retryOn": ["provider_failure"],
+ "excludeFromAnalysis": ["contaminated", "harness_failure", "inconclusive", "ungraded"]
+ },
+ "runtime": {
+ "releaseManifestSha256": null,
+ "controllerImage": null,
+ "buildImage": null,
+ "platform": "linux/amd64"
+ },
+ "pricing": {
+ "currency": "USD",
+ "unit": "USD-per-million-tokens",
+ "capturedAt": "2026-08-12T00:00:00.000Z",
+ "source": "deterministic adapter makes no billable provider calls",
+ "models": {
+ "deterministic": {
+ "input": 0,
+ "output": 0,
+ "cacheWrite5m": 0,
+ "cacheWrite1h": 0,
+ "cacheRead": 0
+ }
+ }
+ },
+ "analysis": {
+ "primaryMetric": "firstBuildScoreRate",
+ "secondaryMetrics": [
+ "finalScoreRate",
+ "totalCostUsd",
+ "totalDurationMs",
+ "invalidAttemptRate"
+ ],
+ "dispersion": "median-iqr",
+ "invalidAttempts": "report-separately",
+ "missingData": "no-imputation",
+ "comparisonUnit": "stack-agent-condition-recipe"
+ }
+}
diff --git a/tools/stack-bench/appliance/campaign.product-brief-reference.json b/tools/stack-bench/appliance/campaign.product-brief-reference.json
new file mode 100644
index 00000000000..a8f7d37359c
--- /dev/null
+++ b/tools/stack-bench/appliance/campaign.product-brief-reference.json
@@ -0,0 +1,114 @@
+{
+ "schemaVersion": 7,
+ "kind": "campaign-manifest",
+ "id": "ecommerce-l1-product-brief-reference",
+ "version": "2.0.0",
+ "state": "draft",
+ "title": "Ecommerce L1 product brief and quality validation",
+ "track": "ecommerce",
+ "mode": { "id": "sequential", "version": "1.0.0" },
+ "repair": { "selection": "batch", "budget": { "total": 0 } },
+ "levels": [1],
+ "selection": {
+ "levels": [
+ {
+ "level": 1,
+ "recipe": "ecommerce.sequential-l1@2.5.0",
+ "features": [
+ "ecommerce.feature.accounts",
+ "ecommerce.feature.cart-checkout",
+ "ecommerce.feature.catalog",
+ "ecommerce.feature.purchasing",
+ "ecommerce.feature.reviews",
+ "ecommerce.feature.warehouse-admin"
+ ],
+ "checks": []
+ }
+ ]
+ },
+ "stacks": [
+ { "id": "spacetime", "adapterVersion": "1.3.0" },
+ { "id": "postgres", "adapterVersion": "1.5.0" },
+ { "id": "mongodb", "adapterVersion": "1.4.0" }
+ ],
+ "agents": [
+ {
+ "adapter": "reference-fixture",
+ "adapterVersion": "1.4.0",
+ "model": "reference-fixture"
+ }
+ ],
+ "conditions": [
+ {
+ "id": "product-brief-quality",
+ "version": "1.2.0",
+ "guidanceProfile": "neutral@1.8.0",
+ "repairPolicy": "scored-only@1.1.0",
+ "specifications": {
+ "levels": [
+ {
+ "level": 1,
+ "requested": [],
+ "expected": [
+ "ecommerce.spec.access-control@1.2.0",
+ "ecommerce.spec.concurrency-safety@1.3.0",
+ "ecommerce.spec.external-data-sync@1.1.0",
+ "ecommerce.spec.live-state@1.2.0",
+ "ecommerce.spec.state-durability@1.1.0",
+ "ecommerce.spec.transactional-integrity@1.3.0"
+ ],
+ "observed": []
+ }
+ ]
+ }
+ }
+ ],
+ "repetitions": 2,
+ "ordering": {
+ "method": "balanced-rotation",
+ "seed": "product-brief-quality-reference-1"
+ },
+ "budgets": {
+ "attemptTimeoutMinutes": 60,
+ "maxCostUsdPerAttempt": null
+ },
+ "attemptPolicy": {
+ "retries": 1,
+ "retryOn": ["harness_failure", "inconclusive"],
+ "excludeFromAnalysis": ["contaminated", "harness_failure", "inconclusive", "ungraded"]
+ },
+ "runtime": {
+ "releaseManifestSha256": null,
+ "controllerImage": null,
+ "buildImage": null,
+ "platform": "linux/amd64"
+ },
+ "pricing": {
+ "currency": "USD",
+ "unit": "USD-per-million-tokens",
+ "capturedAt": "2026-08-16T00:00:00.000Z",
+ "source": "reference fixture adapter makes no billable provider calls",
+ "models": {
+ "reference-fixture": {
+ "input": 0,
+ "output": 0,
+ "cacheWrite5m": 0,
+ "cacheWrite1h": 0,
+ "cacheRead": 0
+ }
+ }
+ },
+ "analysis": {
+ "primaryMetric": "firstBuildScoreRate",
+ "secondaryMetrics": [
+ "finalScoreRate",
+ "totalCostUsd",
+ "totalDurationMs",
+ "invalidAttemptRate"
+ ],
+ "dispersion": "median-iqr",
+ "invalidAttempts": "report-separately",
+ "missingData": "no-imputation",
+ "comparisonUnit": "stack-agent-condition-recipe"
+ }
+}
diff --git a/tools/stack-bench/appliance/controller.ts b/tools/stack-bench/appliance/controller.ts
new file mode 100644
index 00000000000..6ee5da5fa91
--- /dev/null
+++ b/tools/stack-bench/appliance/controller.ts
@@ -0,0 +1,151 @@
+#!/usr/bin/env node
+
+import { spawn } from 'node:child_process';
+import { join } from 'node:path';
+import { pathToFileURL } from 'node:url';
+
+import { STACK_BENCH_ROOT } from '../src/package-root.js';
+
+const RUNTIME_ROOT = join(STACK_BENCH_ROOT, 'dist');
+
+const COMMANDS = Object.freeze({
+ 'init-deps': [join(RUNTIME_ROOT, 'appliance', 'dependency-volume.js'), 'init'],
+ 'verify-deps': [join(RUNTIME_ROOT, 'appliance', 'dependency-volume.js'), 'verify'],
+ 'preflight': [join(RUNTIME_ROOT, 'commands', 'preflight.js')],
+ 'qualify-reference': [join(RUNTIME_ROOT, 'src', 'references', 'reference-live.js')],
+ 'qualify-null': [join(RUNTIME_ROOT, 'commands', 'null-control.js')],
+ 'qualification': [join(RUNTIME_ROOT, 'commands', 'qualification-cli.js')],
+ 'pack-budget': [join(RUNTIME_ROOT, 'commands', 'pack-budget.js')],
+ 'campaign': [join(RUNTIME_ROOT, 'commands', 'campaign-cli.js')],
+ 'dashboard': [join(RUNTIME_ROOT, 'dashboard', 'dashboard-server.js')],
+ 'repair': [join(RUNTIME_ROOT, 'commands', 'repair-cli.js')],
+ 'run': [join(RUNTIME_ROOT, 'commands', 'bench.js')],
+ 'verify-release': [join(RUNTIME_ROOT, 'src', 'releases', 'release-manifest.js'), 'verify'],
+ 'recover': [join(RUNTIME_ROOT, 'commands', 'recovery.js'), 'recover'],
+ 'recover-lease': [join(RUNTIME_ROOT, 'commands', 'recovery.js'), 'recover-lease'],
+} satisfies Record);
+
+const COMMANDS_REQUIRING_AGENT_AUTH = new Set(['preflight', 'dashboard', 'run']);
+
+export function controllerCommandRequiresAgentAuth(command: string | undefined,
+ args: string[] = []): boolean {
+ if (command && COMMANDS_REQUIRING_AGENT_AUTH.has(command)) return true;
+ return command === 'campaign' && args[0] === 'run';
+}
+
+export interface ResolvedControllerCommand {
+ executable: string;
+ args: string[];
+}
+
+export function resolveControllerCommand(argv: string[]): ResolvedControllerCommand | null {
+ const [command, ...rest] = argv;
+ if (!command || command === '--help' || command === 'help') return null;
+ if (!Object.hasOwn(COMMANDS, command)) {
+ throw new Error(`unknown controller command ${JSON.stringify(command)}`);
+ }
+ return { executable: process.execPath,
+ args: [...COMMANDS[command as keyof typeof COMMANDS], ...rest] };
+}
+
+export function controllerChildEnvironment(source: NodeJS.ProcessEnv = process.env,
+ { requireAgentAuth = true }: { requireAgentAuth?: boolean } = {}): NodeJS.ProcessEnv {
+ const env = { ...source };
+ delete env.ANTHROPIC_API_KEY;
+ delete env.ANTHROPIC_API_KEY_FILE;
+ delete env.CLAUDE_CODE_OAUTH_TOKEN;
+ delete env.CLAUDE_CODE_OAUTH_TOKEN_FILE;
+ if (!requireAgentAuth) return env;
+ const mode = source.STACK_BENCH_AGENT_AUTH ?? 'subscription-token';
+ if (!['subscription-token', 'api-key'].includes(mode)) {
+ throw new Error('STACK_BENCH_AGENT_AUTH must be subscription-token or api-key');
+ }
+ if (mode === 'api-key') {
+ const path = source.STACK_BENCH_ANTHROPIC_API_KEY_FILE?.trim();
+ if (!path) throw new Error('api-key auth requires STACK_BENCH_ANTHROPIC_API_KEY_FILE');
+ env.ANTHROPIC_API_KEY_FILE = path;
+ } else {
+ const path = source.STACK_BENCH_CLAUDE_OAUTH_TOKEN_FILE?.trim();
+ if (!path) {
+ throw new Error('subscription-token auth requires STACK_BENCH_CLAUDE_OAUTH_TOKEN_FILE');
+ }
+ env.CLAUDE_CODE_OAUTH_TOKEN_FILE = path;
+ }
+ return env;
+}
+
+interface SignalChild {
+ kill(signal: NodeJS.Signals): unknown;
+}
+
+interface SignalSource {
+ on(signal: NodeJS.Signals, listener: () => void): unknown;
+ off(signal: NodeJS.Signals, listener: () => void): unknown;
+}
+
+export function forwardControllerSignals(child: SignalChild,
+ source: SignalSource = process): () => void {
+ const signals: NodeJS.Signals[] = ['SIGINT', 'SIGTERM'];
+ const listeners = new Map void>(signals.map(signal =>
+ [signal, () => { child.kill(signal); }]));
+ for (const [signal, listener] of listeners) source.on(signal, listener);
+ return () => {
+ for (const [signal, listener] of listeners) source.off(signal, listener);
+ };
+}
+
+function help(): void {
+ process.stdout.write('Stack Bench controller\n\n'
+ + 'Commands:\n'
+ + ' preflight verify the runner without a model call\n'
+ + ' qualify-reference run a pristine or mutation reference gate\n'
+ + ' --mutation-workers N split one mutation gate across 1 to 8 isolated workers\n'
+ + ' qualify-null run the exact null-oracle gate\n'
+ + ' qualification status show exact launch and promotion blockers\n'
+ + ' pack-budget recommend derive reviewable bounds from exact reference evidence\n'
+ + ' campaign validate|show compile the exact comparison plan without running it\n'
+ + ' campaign trial --out exercise a model-free draft\n'
+ + ' campaign run --out start a campaign\n'
+ + ' campaign reconcile --out prove cleanup for interrupted work\n'
+ + ' campaign status inspect exact durable campaign state\n'
+ + ' campaign report regenerate deterministic JSON and static HTML\n'
+ + ' dashboard [--port N] serve the local operator dashboard\n'
+ + ' repair status --level N inspect whether a failed level can continue\n'
+ + ' repair grant --level N --repairs N add one finite repair budget\n'
+ + ' run execute and retain one requested run\n'
+ + ' verify-release verify candidate files or a qualified signed release\n'
+ + ' recover retry authenticated cleanup or retain quarantine\n'
+ + ' recover-lease --out recover when parent state was not retained\n'
+ + ' init-deps | verify-deps initialize or verify the release dependency volume\n');
+}
+
+interface ChildOutcome {
+ code: number | null;
+ signal: NodeJS.Signals | null;
+}
+
+async function main(argv: string[]): Promise {
+ const command = argv[2];
+ const resolved = resolveControllerCommand(argv.slice(2));
+ if (!resolved) { help(); return; }
+ const child = spawn(resolved.executable, resolved.args,
+ { stdio: 'inherit', env: controllerChildEnvironment(process.env,
+ { requireAgentAuth: controllerCommandRequiresAgentAuth(command, argv.slice(3)) }) });
+ const stopForwardingSignals = forwardControllerSignals(child);
+ let outcome: ChildOutcome;
+ try {
+ outcome = await new Promise((resolveExit, reject) => {
+ child.once('error', reject);
+ child.once('exit', (code, signal) => { resolveExit({ code, signal }); });
+ });
+ } finally { stopForwardingSignals(); }
+ if (outcome.signal) process.kill(process.pid, outcome.signal);
+ process.exitCode = outcome.code ?? 1;
+}
+
+if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) {
+ main(process.argv).catch((error: unknown) => {
+ console.error(`stack-bench-controller: ${error instanceof Error ? error.message : String(error)}`);
+ process.exitCode = 2;
+ });
+}
diff --git a/tools/stack-bench/appliance/dependency-volume.ts b/tools/stack-bench/appliance/dependency-volume.ts
new file mode 100644
index 00000000000..e6f49e55a86
--- /dev/null
+++ b/tools/stack-bench/appliance/dependency-volume.ts
@@ -0,0 +1,165 @@
+#!/usr/bin/env node
+
+import { createHash } from 'node:crypto';
+import {
+ chmodSync, copyFileSync, existsSync, lstatSync, mkdirSync, readFileSync, readdirSync,
+ renameSync, rmSync, writeFileSync,
+} from 'node:fs';
+import { dirname, join, relative, resolve, sep } from 'node:path';
+import { pathToFileURL } from 'node:url';
+import { parseArgs } from 'node:util';
+
+const MARKER = '.stack-bench-release-deps.json';
+
+export interface DependencyManifestFile {
+ path: string;
+ size: number;
+ mode: number;
+ sha256: string;
+}
+
+export interface DependencyManifest {
+ schemaVersion: 1;
+ files: DependencyManifestFile[];
+}
+
+interface DependencyVerification {
+ manifestSha256: string;
+ files: number;
+}
+
+interface DependencyInitialization extends DependencyVerification {
+ initialized: boolean;
+}
+
+function sha256Bytes(bytes: string | NodeJS.ArrayBufferView): string {
+ return createHash('sha256').update(bytes).digest('hex');
+}
+
+function normalizedRelative(root: string, path: string): string {
+ const value = relative(root, path).split(sep).join('/');
+ if (!value || value.startsWith('../') || value === '..') throw new Error(`path escapes dependency root: ${path}`);
+ return value;
+}
+
+function walk(root: string, current = root): string[] {
+ const files: string[] = [];
+ for (const entry of readdirSync(current, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))) {
+ const path = join(current, entry.name);
+ if (entry.isSymbolicLink()) throw new Error(`dependency tree cannot contain symlinks: ${normalizedRelative(root, path)}`);
+ if (entry.isDirectory()) files.push(...walk(root, path));
+ else if (entry.isFile()) files.push(path);
+ else throw new Error(`dependency tree contains unsupported entry: ${normalizedRelative(root, path)}`);
+ }
+ return files;
+}
+
+export function createDependencyManifest(root: string): DependencyManifest {
+ const absolute = resolve(root);
+ if (!existsSync(absolute) || !lstatSync(absolute).isDirectory()) {
+ throw new Error(`dependency source is not a directory: ${absolute}`);
+ }
+ const files = walk(absolute).map(path => {
+ const bytes = readFileSync(path);
+ return { path: normalizedRelative(absolute, path), size: bytes.length,
+ mode: lstatSync(path).mode & 0o777, sha256: sha256Bytes(bytes) };
+ });
+ if (!files.length) throw new Error('dependency source is empty');
+ return { schemaVersion: 1, files };
+}
+
+export function manifestSha256(manifest: DependencyManifest): string {
+ return sha256Bytes(`${JSON.stringify(manifest)}\n`);
+}
+
+export function verifyDependencyTree(root: string, manifest: DependencyManifest,
+ { allowMarker = false }: { allowMarker?: boolean } = {}): DependencyVerification {
+ if (!manifest || manifest.schemaVersion !== 1 || !Array.isArray(manifest.files) || !manifest.files.length) {
+ throw new Error('dependency manifest is invalid');
+ }
+ const absolute = resolve(root);
+ const actual = createDependencyManifest(absolute);
+ if (allowMarker) actual.files = actual.files.filter(file => file.path !== MARKER);
+ if (JSON.stringify(actual.files) !== JSON.stringify(manifest.files)) {
+ throw new Error(`dependency tree does not match manifest ${manifestSha256(manifest)}`);
+ }
+ return { manifestSha256: manifestSha256(manifest), files: manifest.files.length };
+}
+
+export function initializeDependencyVolume({ source, target, manifest }:
+ { source: string; target: string; manifest: DependencyManifest }): DependencyInitialization {
+ const sourceRoot = resolve(source);
+ const targetRoot = resolve(target);
+ const verified = verifyDependencyTree(sourceRoot, manifest);
+ mkdirSync(targetRoot, { recursive: true, mode: 0o755 });
+ const markerPath = join(targetRoot, MARKER);
+ const existing = readdirSync(targetRoot);
+ if (existing.length) {
+ if (!existsSync(markerPath)) throw new Error('dependency volume is non-empty but has no release marker');
+ const marker = JSON.parse(readFileSync(markerPath, 'utf8'));
+ if (marker.schemaVersion !== 1 || marker.manifestSha256 !== verified.manifestSha256) {
+ throw new Error('dependency volume belongs to a different release');
+ }
+ verifyDependencyTree(targetRoot, manifest, { allowMarker: true });
+ return { ...verified, initialized: false };
+ }
+
+ const staging = join(targetRoot, `.staging-${process.pid}`);
+ mkdirSync(staging, { mode: 0o700 });
+ try {
+ for (const file of manifest.files) {
+ const from = join(sourceRoot, ...file.path.split('/'));
+ const to = join(staging, ...file.path.split('/'));
+ mkdirSync(dirname(to), { recursive: true });
+ copyFileSync(from, to);
+ chmodSync(to, file.mode);
+ }
+ for (const entry of readdirSync(staging)) renameSync(join(staging, entry), join(targetRoot, entry));
+ rmSync(staging, { recursive: true, force: true });
+ writeFileSync(markerPath, `${JSON.stringify({ schemaVersion: 1,
+ manifestSha256: verified.manifestSha256 })}\n`, { flag: 'wx', mode: 0o444 });
+ verifyDependencyTree(targetRoot, manifest, { allowMarker: true });
+ return { ...verified, initialized: true };
+ } catch (error) {
+ rmSync(staging, { recursive: true, force: true });
+ throw error;
+ }
+}
+
+function main(argv: string[]): void {
+ const { values, positionals } = parseArgs({ args: argv.slice(2), allowPositionals: true,
+ options: {
+ source: { type: 'string', default: '/opt/stack-bench-embedded-deps' },
+ target: { type: 'string', default: '/opt/stack-bench-release-deps' },
+ manifest: { type: 'string', default: '/opt/stack-bench/dependency-manifest.json' },
+ out: { type: 'string' },
+ } });
+ const [command] = positionals;
+ const source = values.source;
+ const target = values.target;
+ const manifestPath = values.manifest;
+ if (command === 'manifest') {
+ const output = values.out;
+ if (!output) throw new Error('manifest requires --out');
+ writeFileSync(resolve(output), `${JSON.stringify(createDependencyManifest(source), null, 2)}\n`, { flag: 'wx' });
+ return;
+ }
+ const manifest: DependencyManifest = JSON.parse(readFileSync(resolve(manifestPath), 'utf8'));
+ if (command === 'init') {
+ process.stdout.write(`${JSON.stringify(initializeDependencyVolume({ source, target, manifest }))}\n`);
+ return;
+ }
+ if (command === 'verify') {
+ process.stdout.write(`${JSON.stringify(verifyDependencyTree(target, manifest, { allowMarker: true }))}\n`);
+ return;
+ }
+ throw new Error('usage: dependency-volume manifest|init|verify [options]');
+}
+
+if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) {
+ try { main(process.argv); }
+ catch (error) {
+ console.error(`dependency-volume: ${error instanceof Error ? error.message : String(error)}`);
+ process.exit(2);
+ }
+}
diff --git a/tools/stack-bench/appliance/docker-compose.yaml b/tools/stack-bench/appliance/docker-compose.yaml
new file mode 100644
index 00000000000..1edcead0ca5
--- /dev/null
+++ b/tools/stack-bench/appliance/docker-compose.yaml
@@ -0,0 +1,114 @@
+name: stack-bench-appliance
+
+services:
+ deps-init:
+ image: ${STACK_BENCH_CONTROLLER_IMAGE:?set STACK_BENCH_CONTROLLER_IMAGE to the manifest digest reference}
+ platform: linux/amd64
+ command: ["init-deps"]
+ read_only: true
+ cap_drop: ["ALL"]
+ security_opt: ["no-new-privileges:true"]
+ volumes:
+ - type: volume
+ source: release-deps
+ target: /opt/stack-bench-release-deps
+ tmpfs:
+ - /tmp:size=64m,mode=1777
+
+ controller:
+ image: ${STACK_BENCH_CONTROLLER_IMAGE:?set STACK_BENCH_CONTROLLER_IMAGE to the manifest digest reference}
+ platform: linux/amd64
+ init: true
+ network_mode: host
+ read_only: true
+ cap_drop: ["ALL"]
+ security_opt: ["no-new-privileges:true"]
+ depends_on:
+ deps-init:
+ condition: service_completed_successfully
+ postgres:
+ condition: service_healthy
+ mongodb:
+ condition: service_healthy
+ environment:
+ STACK_BENCH_CONTROLLER_IMAGE: ${STACK_BENCH_CONTROLLER_IMAGE:?set STACK_BENCH_CONTROLLER_IMAGE to the manifest digest reference}
+ STACK_BENCH_IMAGE: ${STACK_BENCH_BUILD_IMAGE:?set STACK_BENCH_BUILD_IMAGE to the manifest digest reference}
+ STACK_BENCH_RELEASE_MANIFEST: ${STACK_BENCH_RELEASE_MANIFEST:-}
+ STACK_BENCH_APPLIANCE: "1"
+ STACK_BENCH_COMPOSE_FILE: /opt/stack-bench/appliance/docker-compose.yaml
+ STACK_BENCH_WORK_DIR: /var/lib/stack-bench/work
+ STACK_BENCH_RESULTS_DIR: /var/lib/stack-bench/results
+ STACK_BENCH_SUPERVISOR_DIR: /var/lib/stack-bench/controller-home/supervisor
+ STACK_BENCH_RUNTIME_DIR: /var/lib/stack-bench/controller-home/runtime
+ STACK_BENCH_RESOURCE_LOCK_DIR: /var/lib/stack-bench/controller-home/resource-locks
+ STACK_BENCH_RELEASE_DEPS_VOLUME: stack-bench-release-deps
+ STACK_BENCH_LINUX_CLI: /opt/stack-bench-release-deps/spacetimedb-cli
+ STDB_PACKAGE: /opt/stack-bench-release-deps/bindings-typescript
+ SPACETIME_BIN: /opt/stack-bench-release-deps/spacetimedb-cli
+ STACK_BENCH_AGENT_AUTH: ${STACK_BENCH_AGENT_AUTH:-subscription-token}
+ STACK_BENCH_CLAUDE_OAUTH_TOKEN_FILE: ${STACK_BENCH_CLAUDE_OAUTH_TOKEN_FILE:-/var/lib/stack-bench/secrets/claude_subscription_token}
+ STACK_BENCH_ANTHROPIC_API_KEY_FILE: ${STACK_BENCH_ANTHROPIC_API_KEY_FILE:-}
+ HOME: /var/lib/stack-bench/controller-home
+ volumes:
+ - type: bind
+ source: /var/run/docker.sock
+ target: /var/run/docker.sock
+ - type: bind
+ source: /var/lib/stack-bench
+ target: /var/lib/stack-bench
+ - type: volume
+ source: release-deps
+ target: /opt/stack-bench-release-deps
+ read_only: true
+ tmpfs:
+ - /tmp:size=1g,mode=1777
+ command: ["--help"]
+
+ dashboard:
+ extends:
+ service: controller
+ profiles: ["dashboard"]
+ network_mode: bridge
+ environment:
+ STACK_BENCH_DASHBOARD_CONTROL_SECRET_FILE: ${STACK_BENCH_DASHBOARD_CONTROL_SECRET_FILE:-/var/lib/stack-bench/secrets/dashboard_control_secret}
+ ports:
+ - "127.0.0.1:7331:7331"
+ command: ["dashboard", "--host", "0.0.0.0", "--port", "7331", "--allow-container-bind"]
+
+ postgres:
+ image: postgres:16@sha256:219341e4cedb06c8634f80af40851da3425b41b76603fd890272f58e37e139f7
+ platform: linux/amd64
+ container_name: stack-bench-postgres
+ ports: ["127.0.0.1:6532:5432"]
+ environment:
+ POSTGRES_USER: appuser
+ POSTGRES_PASSWORD: local-app-password
+ POSTGRES_DB: app
+ volumes:
+ - pgdata:/var/lib/postgresql/data
+ healthcheck:
+ test: ["CMD-SHELL", "pg_isready -U appuser -d app"]
+ interval: 5s
+ timeout: 5s
+ retries: 12
+
+ mongodb:
+ image: mongo:7@sha256:554a9bb1ec6e00c40ba078a41974a834d1a9a8ab1772645b69142afecc87f082
+ platform: linux/amd64
+ container_name: stack-bench-mongodb
+ ports: ["127.0.0.1:6537:27017"]
+ volumes:
+ - mongodata:/data/db
+ healthcheck:
+ test: ["CMD", "mongosh", "--quiet", "--eval", "db.runCommand({ping:1})"]
+ interval: 5s
+ timeout: 5s
+ retries: 12
+
+volumes:
+ release-deps:
+ name: stack-bench-release-deps
+ pgdata:
+ name: stack-bench-appliance-pgdata
+ mongodata:
+ name: stack-bench-appliance-mongodata
diff --git a/tools/stack-bench/appliance/operator.env.example b/tools/stack-bench/appliance/operator.env.example
new file mode 100644
index 00000000000..ad900f63759
--- /dev/null
+++ b/tools/stack-bench/appliance/operator.env.example
@@ -0,0 +1,20 @@
+# Subscription billing is the default. Generate a dedicated long-lived Claude
+# setup token, write only the token to this mode-0600 file, and never commit it.
+STACK_BENCH_AGENT_AUTH=subscription-token
+STACK_BENCH_CLAUDE_OAUTH_TOKEN_FILE=/var/lib/stack-bench/secrets/claude_subscription_token
+
+# To bill through an API key instead, set the mode to api-key and provide an
+# absolute path below /var/lib/stack-bench containing only that key.
+# STACK_BENCH_ANTHROPIC_API_KEY_FILE=/var/lib/stack-bench/secrets/anthropic_api_key
+
+# The dashboard reads this separate operator secret from a mode-0600 file.
+# The file must contain at least 32 random characters on one line.
+STACK_BENCH_DASHBOARD_CONTROL_SECRET_FILE=/var/lib/stack-bench/secrets/dashboard_control_secret
+
+# Both image values must be registry references ending in @sha256:<64 hex chars>.
+STACK_BENCH_CONTROLLER_IMAGE=registry.example/stack-bench-controller@sha256:replace-with-release-digest
+STACK_BENCH_BUILD_IMAGE=registry.example/stack-bench-build@sha256:replace-with-release-digest
+
+# Optional for internal campaigns and required for a distributed release
+# campaign. The file must be below the appliance state root.
+STACK_BENCH_RELEASE_MANIFEST=/var/lib/stack-bench/release/release.json
diff --git a/tools/stack-bench/backends/minimal/mongodb-1.5.md b/tools/stack-bench/backends/minimal/mongodb-1.5.md
new file mode 100644
index 00000000000..8b9e91fbdba
--- /dev/null
+++ b/tools/stack-bench/backends/minimal/mongodb-1.5.md
@@ -0,0 +1,19 @@
+# MongoDB
+
+Use MongoDB for the application data. Choose the libraries, architecture, and
+project structure.
+
+## Connection
+
+| Setting | Value |
+|---|---|
+| `DATABASE_URL` | `` |
+| Web application | `http://localhost:` |
+
+The MongoDB service is already running. Use the exact `DATABASE_URL`. Do not
+start another MongoDB server, connect to another instance, or create another
+database. Serve the complete application on ``.
+Create `/app/start.sh`. From a clean source checkout, it must install
+dependencies, build the complete application, and start it on ``.
+The script must not change source files. Leave the application running when the
+work is complete.
diff --git a/tools/stack-bench/backends/minimal/postgres-1.5.md b/tools/stack-bench/backends/minimal/postgres-1.5.md
new file mode 100644
index 00000000000..4a9124a88d2
--- /dev/null
+++ b/tools/stack-bench/backends/minimal/postgres-1.5.md
@@ -0,0 +1,19 @@
+# PostgreSQL
+
+Use PostgreSQL for the application data. Choose the libraries, architecture,
+and project structure.
+
+## Connection
+
+| Setting | Value |
+|---|---|
+| `DATABASE_URL` | `` |
+| Web application | `http://localhost:` |
+
+The PostgreSQL service is already running. Use the exact `DATABASE_URL`. Do not
+start another PostgreSQL server, connect to another instance, or create another
+database. Serve the complete application on ``.
+Create `/app/start.sh`. From a clean source checkout, it must install
+dependencies, build the complete application, and start it on ``.
+The script must not change source files. Leave the application running when the
+work is complete.
diff --git a/tools/stack-bench/backends/minimal/spacetime-1.6.md b/tools/stack-bench/backends/minimal/spacetime-1.6.md
new file mode 100644
index 00000000000..4d686b4d547
--- /dev/null
+++ b/tools/stack-bench/backends/minimal/spacetime-1.6.md
@@ -0,0 +1,28 @@
+# SpacetimeDB
+
+Use SpacetimeDB for the application data. Put the TypeScript module in the
+required directory below. Choose the schema, libraries, architecture, and the
+rest of the project structure.
+
+## Connection
+
+Use the connection settings below.
+
+| Setting | Value |
+|---|---|
+| Server URI | `` |
+| Module name | `` |
+| SpacetimeDB CLI | `` |
+| TypeScript SDK package | `` |
+| Module source directory | `/app/backend/spacetimedb` |
+| Web application | `http://localhost:` |
+
+Publish only the named module to the exact server URI. Local publish and
+development commands must use `--yes`. Do not pipe confirmation input, publish
+anonymously, or use the hosted service. Create `/app/start.sh`. From a clean
+source checkout, it must install dependencies, build the complete application,
+and start it on ``. The script must not change source files. Leave
+the application running when the work is complete.
+
+The included TypeScript core SDK reference describes the available core API syntax.
+CLI `--help` is available for command syntax.
diff --git a/tools/stack-bench/backends/model-free-stub.md b/tools/stack-bench/backends/model-free-stub.md
new file mode 100644
index 00000000000..7a83d0e4bc2
--- /dev/null
+++ b/tools/stack-bench/backends/model-free-stub.md
@@ -0,0 +1,4 @@
+# Model-free service
+
+Use the supplied service. Leave the app running on the assigned client port
+when the work is complete.
diff --git a/tools/stack-bench/backends/mongodb.md b/tools/stack-bench/backends/mongodb.md
new file mode 100644
index 00000000000..4671ddbb19e
--- /dev/null
+++ b/tools/stack-bench/backends/mongodb.md
@@ -0,0 +1,46 @@
+# Backend: MongoDB
+
+An Express API server with Socket.io for live updates, Mongoose over MongoDB,
+and a React client.
+
+## Layout
+
+```
+/
+ server/
+ package.json express, socket.io, mongoose, dotenv, tsx
+ .env DATABASE_URL and PORT
+ src/models.ts Mongoose schemas and models
+ src/index.ts Express routes, Socket.io handlers
+ client/
+ package.json react, react-dom, vite, socket.io-client
+ vite.config.ts server.port , proxy /api and /socket.io to
+ index.html
+ src/main.tsx
+ src/App.tsx
+```
+
+## Deploy
+
+```bash
+cd server && npm install && npm run dev # on
+cd client && npm install && npm run dev # on
+```
+
+Mongoose creates collections on first write; there is no migration step.
+
+The server prints to the terminal running `npm run dev`; it restarts on save,
+so a code change is live without redeploying.
+
+Keep existing application data when you change the schema. Do not drop
+collections during upgrades or repairs.
+
+## Configuration
+
+| Setting | Value |
+|---|---|
+| `DATABASE_URL` | `` |
+| API server port | `` |
+| Client dev server | `` |
+
+Use this exact `DATABASE_URL`. Do not point at another MongoDB instance.
diff --git a/tools/stack-bench/backends/postgres.md b/tools/stack-bench/backends/postgres.md
new file mode 100644
index 00000000000..32ee3ac4afd
--- /dev/null
+++ b/tools/stack-bench/backends/postgres.md
@@ -0,0 +1,48 @@
+# Backend: PostgreSQL
+
+An Express API server with Socket.io for live updates, Drizzle ORM over
+PostgreSQL, and a React client.
+
+## Layout
+
+```
+/
+ server/
+ package.json express, socket.io, drizzle-orm, pg, dotenv, tsx
+ .env DATABASE_URL and PORT
+ drizzle.config.ts
+ src/schema.ts Drizzle table definitions
+ src/index.ts Express routes, Socket.io handlers
+ client/
+ package.json react, react-dom, vite, socket.io-client
+ vite.config.ts server.port , proxy /api and /socket.io to
+ index.html
+ src/main.tsx
+ src/App.tsx
+```
+
+## Deploy
+
+```bash
+cd server && npm install && npx drizzle-kit push && npm run dev # on
+cd client && npm install && npm run dev # on
+```
+
+Re-run `npx drizzle-kit push` after any schema change.
+
+The server prints to the terminal running `npm run dev`; it restarts on save,
+so a code change is live without redeploying.
+
+Keep existing application data when you change the schema. Do not drop or
+recreate tables during upgrades or repairs.
+
+## Configuration
+
+| Setting | Value |
+|---|---|
+| `DATABASE_URL` | `` |
+| API server port | `` |
+| Client dev server | `` |
+
+Use this exact `DATABASE_URL`. Do not point at another PostgreSQL instance and do
+not create databases outside it.
diff --git a/tools/stack-bench/backends/spacetime.md b/tools/stack-bench/backends/spacetime.md
new file mode 100644
index 00000000000..250587f88fe
--- /dev/null
+++ b/tools/stack-bench/backends/spacetime.md
@@ -0,0 +1,80 @@
+# Backend: SpacetimeDB
+
+The database runs your server logic. There is no separate API server and no ORM:
+tables and reducers are a WASM module you publish, and the client subscribes to
+tables and calls reducers over a live connection.
+
+## Layout
+
+```
+/
+ backend/spacetimedb/
+ package.json { "type": "module", dependencies: { "spacetimedb": "" },
+ devDependencies: { "typescript": "~5.6.2" } } ← required; the build runs tsc from node_modules
+ tsconfig.json
+ src/schema.ts tables and indexes
+ src/index.ts reducers and lifecycle hooks
+ client/
+ package.json react, react-dom, vite, and "spacetimedb": ""
+ vite.config.ts server.port must be
+ index.html
+ src/config.ts MODULE_NAME and SPACETIMEDB_URI
+ src/main.tsx React entry
+ src/App.tsx
+ src/module_bindings/ generated; never edit by hand
+```
+
+## Deploy
+
+Publish the module, then regenerate the client bindings from it:
+
+```bash
+ publish --module-path backend/spacetimedb -s --yes
+ generate --lang typescript --out-dir client/src/module_bindings --module-path backend/spacetimedb
+```
+
+**While iterating, run development mode instead of republishing by hand.** It
+watches the module and automatically rebuilds, publishes, and regenerates the
+client bindings on every save:
+
+```bash
+ dev --module-path backend/spacetimedb -s --yes
+```
+
+Leave it running in the background while you work. The manual commands below
+are for one-off publishes and for the first deploy.
+
+Republish after any server change, and regenerate after any schema change.
+
+Keep existing application data when you change the schema.
+
+Always use `--yes` for local publish and development commands. It selects the
+CLI's non-interactive authentication flow for the target server. Do not pipe
+`y` into the command and do not publish anonymously. Use the same local
+identity for every publish to the named module.
+
+Then start the client:
+
+```bash
+cd client && npm install && npm run dev
+```
+
+` logs -s ` shows module output, including reducer errors.
+
+To inspect stored data while debugging:
+
+```bash
+ sql "SELECT * FROM item LIMIT 5" -s
+```
+
+## Configuration
+
+| Setting | Value |
+|---|---|
+| Server URI | `` |
+| Module name | `` |
+| Client dev server | `` |
+
+The SDK reference for writing modules and clients is in the skill documents
+included with these instructions. Follow them for API specifics: import paths,
+type builders, accessors and context typing.
diff --git a/tools/stack-bench/commands/agent.ts b/tools/stack-bench/commands/agent.ts
new file mode 100644
index 00000000000..5fe8372dcba
--- /dev/null
+++ b/tools/stack-bench/commands/agent.ts
@@ -0,0 +1,897 @@
+#!/usr/bin/env node
+
+import { execFileSync } from 'node:child_process';
+import type { ExecFileSyncOptionsWithStringEncoding } from 'node:child_process';
+import { readFileSync, writeFileSync, mkdirSync, existsSync, realpathSync,
+ openSync, readSync, closeSync, readdirSync } from 'node:fs';
+import { join, dirname, resolve, relative, isAbsolute, sep } from 'node:path';
+import { homedir } from 'node:os';
+import { fileURLToPath } from 'node:url';
+import { parseArgs as parseNodeArgs } from 'node:util';
+import { loadTrack, levelPrompt, appendix, suitesFor, dbName, moduleName, portsFor,
+ DEFAULT_TRACK, TRACK_MANIFEST_FILE } from '../src/composition/tracks.js';
+import type { Track, TrackDefinition } from '../src/composition/tracks.js';
+import type { BackendLease } from '../src/runtime/backend-lease.js';
+import { resolveRecipeRelease } from '../src/composition/recipe-release.js';
+import { parseGuidanceMode, resolveDefaultGuidanceForStack, type GuidanceMode,
+ type ResolvedGuidanceDocument, type ResolvedSkills }
+ from '../src/campaigns/condition-compiler.js';
+import type { ExactRecipeRequest, RecipeBinding } from '../src/composition/recipe-release.js';
+import { createBoundRecipeTaskRequest, resolveBoundRecipeTaskRequest } from '../src/composition/recipe-selection.js';
+import { agentVisibleContractText, assertAgentVisibleText }
+ from '../src/composition/agent-visible-contract.js';
+import { DEFAULT_SPACETIME_SERVER_URI, leaseFromEnv } from '../src/runtime/backend-lease.js';
+import { CODING_CONTAINER_APP_ROOT, CODING_CONTAINER_BUG_REPORT_FILE,
+ CODING_CONTAINER_RELEASE_DEPS_ROOT, CODING_CONTAINER_SPACETIME_CLI,
+ CODING_CONTAINER_SPACETIME_PACKAGE }
+ from '../src/runtime/coding-container-policy.js';
+import { resolveContainerImage } from '../src/runtime/container-image.js';
+import { hashDirectory, sessionProvenance, sha256 } from '../src/evidence/provenance.js';
+import type { StackRunPorts } from '../src/stacks/stack-adapter-contract.js';
+import { STACK_ADAPTER_REGISTRY } from '../src/stacks/stack-adapters.js';
+import { requireLeasedDatabase, requireLeasedSpacetime }
+ from '../src/stacks/backend-reset-guard.js';
+import { DEFAULT_BUILD_IMAGE } from '../src/composition/product-config.js';
+import { dockerMountArguments } from '../src/runtime/container-mount.js';
+import { normalizePromptText, readAgentSkillDocuments, selectAgentSkills } from '../src/agents/agent-materials.js';
+import { codingSessionFailure, DEFAULT_THROTTLE_MAX_WAIT_MS, providerSessionFailure,
+ runCodingSessionWithRetries } from '../src/agents/coding-session-retry.js';
+import type { CodingSessionRetryResult } from '../src/agents/coding-session-retry.js';
+import { AGENT_PROCESS_TIMEOUT_MS } from '../src/agents/coding-session-timeouts.js';
+import { assertNewOrEmptyDirectory } from '../src/runtime/path-safety.js';
+import { claudeRatesForModel } from '../src/evidence/claude-usage-cost.js';
+import { PRICING_UNIT, validatePricingAuthority }
+ from '../src/evidence/pricing-authority.js';
+import type { PricingAuthority } from '../src/evidence/pricing-authority.js';
+
+import { STACK_BENCH_ROOT as ROOT, compiledEntrypoint } from '../src/package-root.js';
+const REPO = resolve(ROOT, '..', '..');
+const CONTROL_COMMAND_TIMEOUT_MS = 120_000;
+const DEFAULT_CODING_INTERRUPTION_RETRIES = 2;
+
+type UnknownRecord = Record;
+interface PromptMaterials {
+ skillsText?: string;
+ requirementText?: string;
+ contractText?: string;
+ startingCatalog?: string;
+}
+
+type RecipeTaskRequest = Parameters[1] & {
+ recipe?: Exclude;
+};
+
+type AgentMode = 'build' | 'upgrade' | 'fix' | 'resume';
+
+interface AgentArgs {
+ mode: AgentMode;
+ backend: string;
+ app: string;
+ level: number;
+ runIndex: number;
+ model: string;
+ guidance: GuidanceMode;
+ track: string;
+ pricing: Readonly | null;
+ guidanceDocument?: ResolvedGuidanceDocument;
+ credentialAliases?: Readonly>;
+ recipe?: string;
+ recipeTask?: RecipeTaskRequest;
+ thinking?: string;
+ maxBudgetUsd?: number;
+ skills?: string[];
+ skillIdentity?: ResolvedSkills;
+ apiKey?: string;
+ printPrompt?: boolean;
+}
+
+interface ThinkingVolume {
+ blocks: number;
+ signatureBytes: number;
+ bytesPerBlock: number;
+}
+
+interface SessionUsage {
+ input_tokens?: number;
+ output_tokens?: number;
+ cache_creation_input_tokens?: number;
+ cache_read_input_tokens?: number;
+}
+
+const isRecord = (value: unknown): value is UnknownRecord =>
+ value !== null && typeof value === 'object' && !Array.isArray(value);
+
+const stringValue = (value: unknown): string | null => typeof value === 'string' ? value : null;
+
+function stringArray(value: string, option: string): string[] {
+ const parsed: unknown = JSON.parse(value);
+ if (!Array.isArray(parsed) || parsed.some(item => typeof item !== 'string')) {
+ throw new Error(`${option} must be an array of strings`);
+ }
+ return parsed;
+}
+
+function sessionUsage(value: unknown): SessionUsage {
+ return isRecord(value) ? {
+ input_tokens: typeof value.input_tokens === 'number' ? value.input_tokens : undefined,
+ output_tokens: typeof value.output_tokens === 'number' ? value.output_tokens : undefined,
+ cache_creation_input_tokens: typeof value.cache_creation_input_tokens === 'number'
+ ? value.cache_creation_input_tokens : undefined,
+ cache_read_input_tokens: typeof value.cache_read_input_tokens === 'number'
+ ? value.cache_read_input_tokens : undefined,
+ } : {};
+}
+
+// Use only the benchmark-owned SpacetimeDB host.
+const STDB_URI = process.env.STACK_BENCH_STDB_URI ?? DEFAULT_SPACETIME_SERVER_URI;
+
+// Test the CLI and SDK from this checkout.
+const LOCAL_CLI = join(REPO, 'target', 'release', 'spacetimedb-cli.exe');
+const STDB_BIN = process.env.SPACETIME_BIN ?? (existsSync(LOCAL_CLI) ? LOCAL_CLI : 'spacetime');
+const LOCAL_PKG = process.env.STDB_PACKAGE ?? join(REPO, 'crates', 'bindings-typescript');
+
+const fwd = (path: string): string => path.split('\\').join('/');
+
+// Keep the provider's default thinking budget unless an experiment selects one
+// explicitly. The run records observed reasoning volume so default changes are
+// visible in the evidence.
+const THINKING_TOKENS = process.env.STACK_BENCH_THINKING ?? null;
+
+const EFFORT = process.env.STACK_BENCH_EFFORT ?? 'high';
+
+const IMAGE = process.env.STACK_BENCH_IMAGE ?? DEFAULT_BUILD_IMAGE;
+
+// Containers reach host services through this address. App ports remain local.
+export function hostServiceAddress(env: NodeJS.ProcessEnv = process.env): string {
+ return env.STACK_BENCH_HOST_ALIAS
+ ?? (env.STACK_BENCH_APPLIANCE === '1' ? '127.0.0.1' : 'host.docker.internal');
+}
+
+const HOST_ADDR = hostServiceAddress();
+const hostUrl = (url: string): string => url.replace(/127\.0\.0\.1|localhost/g, HOST_ADDR);
+
+const C_BIN = CODING_CONTAINER_SPACETIME_CLI;
+
+// The container requires the Linux CLI from this checkout.
+const LINUX_CLI = process.env.STACK_BENCH_LINUX_CLI
+ ?? join(ROOT, 'container', 'bin', 'spacetimedb-cli');
+
+// The provider CLI keeps one JSONL transcript per session under its project
+// directory for the application path.
+function transcriptFile(appDir: string, sessionId: string): string | null {
+ const store = join(homedir(), '.claude', 'projects');
+ if (!existsSync(store)) return null;
+ const want = resolve(appDir).replace(/[\\/:]/g, '-').toLowerCase();
+ const dir = readdirSync(store).find(d => {
+ const n = d.toLowerCase();
+ return n === want || n === want.replace(/^-+/, '');
+ });
+ const file = dir && join(store, dir, `${sessionId}.jsonl`);
+ return file && existsSync(file) ? file : null;
+}
+
+// The model ids the provider actually served. The requested name is an alias
+// that can resolve to different snapshots over time; the transcript records
+// what answered each request.
+function transcriptModels(appDir: string, sessionIds: readonly (string | null | undefined)[]): string[] {
+ const models = new Set();
+ for (const sessionId of new Set(sessionIds.filter((id): id is string => Boolean(id)))) {
+ try {
+ const file = transcriptFile(appDir, sessionId);
+ if (!file) continue;
+ for (const line of readFileSync(file, 'utf8').split('\n')) {
+ if (!line.includes('"model"')) continue;
+ let record: unknown;
+ try { record = JSON.parse(line); } catch { continue; }
+ if (!isRecord(record) || !isRecord(record.message)) continue;
+ const model = stringValue(record.message.model);
+ if (model) models.add(model);
+ }
+ } catch { /* an unreadable transcript leaves the list shorter, never wrong */ }
+ }
+ return [...models].sort();
+}
+
+// The transcript exposes reasoning blocks and signature bytes, not reasoning tokens.
+function thinkingVolume(appDir: string, sessionId: string | null | undefined): ThinkingVolume | null {
+ if (!sessionId) return null;
+ try {
+ const file = transcriptFile(appDir, sessionId);
+ if (!file) return null;
+
+ let blocks = 0, bytes = 0;
+ for (const line of readFileSync(file, 'utf8').split('\n')) {
+ if (!line.includes('"thinking"')) continue; // cheap filter before parsing
+ let record: unknown;
+ try { record = JSON.parse(line); } catch { continue; }
+ if (!isRecord(record) || !isRecord(record.message)
+ || !Array.isArray(record.message.content)) continue;
+ for (const content of record.message.content) {
+ if (!isRecord(content) || content.type !== 'thinking') continue;
+ blocks++;
+ bytes += stringValue(content.signature)?.length ?? 0;
+ }
+ }
+ return { blocks, signatureBytes: bytes,
+ bytesPerBlock: blocks ? Math.round(bytes / blocks) : 0 };
+ } catch { return null; }
+}
+
+function combinedThinkingVolume(appDir: string, sessionIds: readonly (string | null | undefined)[]): ThinkingVolume | null {
+ const volumes: ThinkingVolume[] = [...new Set(sessionIds.filter((id): id is string => Boolean(id)))]
+ .map(id => thinkingVolume(appDir, id)).filter((item): item is ThinkingVolume => item !== null);
+ if (!volumes.length) return null;
+ const blocks = volumes.reduce((sum, item) => sum + item.blocks, 0);
+ const signatureBytes = volumes.reduce((sum, item) => sum + item.signatureBytes, 0);
+ return { blocks, signatureBytes,
+ bytesPerBlock: blocks ? Math.round(signatureBytes / blocks) : 0 };
+}
+
+
+// Record the Linux CLI executed by the container. The host and container
+// binaries can change independently and must not share an identity.
+function linuxSpacetimeVersion(image: string): { commit: string | null; binarySha256: string | null; raw: string } {
+ try {
+ const releaseVolume = process.env.STACK_BENCH_RELEASE_DEPS_VOLUME?.trim() || null;
+ const mountArgs = releaseVolume
+ ? dockerMountArguments({ kind: 'volume', source: releaseVolume,
+ target: CODING_CONTAINER_RELEASE_DEPS_ROOT, readOnly: true })
+ : ['-v', `${LINUX_CLI}:${CODING_CONTAINER_SPACETIME_CLI}:ro`];
+ const entrypoint = releaseVolume
+ ? `${CODING_CONTAINER_RELEASE_DEPS_ROOT}/spacetimedb-cli`
+ : CODING_CONTAINER_SPACETIME_CLI;
+ const out = execFileSync('docker',
+ ['run', '--rm', ...mountArgs, '--entrypoint', entrypoint, image, '--version'],
+ { encoding: 'utf8', stdio: 'pipe', env: { ...process.env, MSYS_NO_PATHCONV: '1' },
+ timeout: CONTROL_COMMAND_TIMEOUT_MS });
+ const commit = out.match(/Commit:\s*([0-9a-f]+)/i)?.[1] ?? null;
+ return { commit, binarySha256: sha256(readFileSync(LINUX_CLI)),
+ raw: out.trim().split(/\r?\n/).slice(0, 2).join(' ') };
+ } catch { return { commit: null, binarySha256: null, raw: 'unknown' }; }
+}
+
+function bindingsIdentity(pkgDir: string): { package: string; sourceSha256: string | null; sourceFiles: number } {
+ try {
+ const p = JSON.parse(readFileSync(join(pkgDir, 'package.json'), 'utf8'));
+ const source = hashDirectory(pkgDir, { exclude: name =>
+ /(^|\/)(node_modules|dist|target)(\/|$)/.test(name) });
+ return { package: `${p.name}@${p.version}`, sourceSha256: source.sha256,
+ sourceFiles: source.files.length };
+ } catch { return { package: 'unknown', sourceSha256: null, sourceFiles: 0 }; }
+}
+
+// The CLI version inside the build image. Read by running it, not by trusting
+// the tag: the image is pinned by ARG and a tag can be moved.
+function imageCliVersion(image: string): string {
+ try {
+ return execFileSync('docker', ['run', '--rm', '--entrypoint', 'claude', image, '--version'],
+ { encoding: 'utf8', stdio: 'pipe', env: { ...process.env, MSYS_NO_PATHCONV: '1' },
+ timeout: CONTROL_COMMAND_TIMEOUT_MS }).trim();
+ } catch { return 'unknown'; }
+}
+
+function imageNodeVersion(image: string): string {
+ try {
+ return execFileSync('docker', ['run', '--rm', '--entrypoint', 'node', image, '--version'],
+ { encoding: 'utf8', stdio: 'pipe', env: { ...process.env, MSYS_NO_PATHCONV: '1' },
+ timeout: CONTROL_COMMAND_TIMEOUT_MS }).trim();
+ } catch { return 'unknown'; }
+}
+
+function containerImage(name: string): { reference: string; imageId: string | null | undefined } {
+ try {
+ const out = execFileSync('docker', ['inspect', '-f', '{{.Config.Image}} {{.Image}}', name],
+ { encoding: 'utf8', stdio: 'pipe', timeout: CONTROL_COMMAND_TIMEOUT_MS }).trim();
+ const [reference, imageId] = out.split(/\s+/, 2);
+ return { reference: reference ?? '', imageId };
+ } catch { return { reference: 'unknown', imageId: null }; }
+}
+
+// Record ambient provider configuration that can change model behaviour while
+// replacing credential values with presence markers.
+function ambientEnv(): Record {
+ const seen: Record = {};
+ for (const [k, v] of Object.entries(process.env)) {
+ if (!/^(CLAUDE|ANTHROPIC|MAX_THINKING|DISABLE_AUTOUPDATER|FORCE_PROMPT)/.test(k)) continue;
+ // Never record a credential, only that one was present.
+ seen[k] = /KEY|TOKEN|SECRET/i.test(k) ? '' : v;
+ }
+ return seen;
+}
+
+export function parseAgentArgs(argv: readonly string[]): AgentArgs {
+ const strings = ['mode', 'track', 'backend', 'level', 'app', 'run-index', 'model',
+ 'pricing-json', 'guidance', 'guidance-document-json', 'credential-aliases-json',
+ 'recipe', 'recipe-task-json', 'thinking', 'max-budget-usd', 'skills', 'skills-json',
+ 'skill-identity-json', 'api-key'] as const;
+ const { values: rawValues } = parseNodeArgs({ args: [...argv.slice(2)], options: Object.fromEntries([
+ ...strings.map(name => [name, { type: 'string' as const }]),
+ ['print-prompt', { type: 'boolean' as const }],
+ ]), strict: true, allowPositionals: false });
+ const values = rawValues as Partial>
+ & { 'print-prompt'?: boolean };
+ const mode = values.mode;
+ if (mode !== 'build' && mode !== 'upgrade' && mode !== 'fix' && mode !== 'resume') {
+ throw new Error('--mode must be build, upgrade, fix, or resume');
+ }
+ const backend = values.backend;
+ const app = values.app;
+ if (!backend || !app) {
+ throw new Error('usage: node dist/commands/agent.js --mode build|upgrade|fix|resume '
+ + '--backend --app [--level ]');
+ }
+ const level = values.level === undefined ? 1 : Number(values.level);
+ if (!Number.isSafeInteger(level) || level < 1) {
+ throw new Error('--level must be a positive integer');
+ }
+ const runIndex = values['run-index'] === undefined ? 0 : Number(values['run-index']);
+ if (!Number.isSafeInteger(runIndex) || runIndex < 0) {
+ throw new Error('--run-index must be a non-negative integer');
+ }
+ const model = values.model ?? 'claude-sonnet-5';
+ const maxBudgetUsd = values['max-budget-usd'] === undefined
+ ? undefined : Number(values['max-budget-usd']);
+ if (maxBudgetUsd !== undefined && (!Number.isFinite(maxBudgetUsd) || maxBudgetUsd <= 0)) {
+ throw new Error('--max-budget-usd must be a positive number');
+ }
+ if (values.skills !== undefined && values['skills-json'] !== undefined) {
+ throw new Error('--skills and --skills-json cannot be used together');
+ }
+ const skills = values.skills?.split(',').map(skill => skill.trim()).filter(Boolean)
+ ?? (values['skills-json'] === undefined ? undefined
+ : stringArray(values['skills-json'], '--skills-json'));
+ let pricing = values['pricing-json'] === undefined
+ ? undefined : validatePricingAuthority(JSON.parse(values['pricing-json']), { at: '--pricing-json' });
+ if (pricing === undefined && maxBudgetUsd !== undefined) {
+ const rates = claudeRatesForModel(model);
+ if (!rates) throw new Error(`no default pricing is recorded for model ${model}`);
+ pricing = validatePricingAuthority({ unit: PRICING_UNIT, rates },
+ { at: 'default pricing' });
+ }
+ return { mode, backend, app, level, runIndex, model,
+ guidance: parseGuidanceMode(values.guidance ?? 'prescribed'),
+ track: values.track ?? DEFAULT_TRACK, pricing: pricing ?? null,
+ ...(values['guidance-document-json'] ? {
+ guidanceDocument: JSON.parse(values['guidance-document-json']) as ResolvedGuidanceDocument,
+ } : {}),
+ ...(values['credential-aliases-json'] ? {
+ credentialAliases: JSON.parse(values['credential-aliases-json']) as Record,
+ } : {}),
+ ...(values.recipe ? { recipe: values.recipe } : {}),
+ ...(values['recipe-task-json'] ? {
+ recipeTask: JSON.parse(values['recipe-task-json']) as RecipeTaskRequest,
+ } : {}),
+ ...(values.thinking ? { thinking: values.thinking } : {}),
+ ...(maxBudgetUsd !== undefined ? { maxBudgetUsd } : {}),
+ ...(skills ? { skills } : {}),
+ ...(values['skill-identity-json'] ? {
+ skillIdentity: validateSkillIdentity(JSON.parse(values['skill-identity-json'])),
+ } : {}),
+ ...(values['api-key'] ? { apiKey: values['api-key'] } : {}),
+ ...(values['print-prompt'] ? { printPrompt: true } : {}) };
+}
+
+const dbUrl = (backend: string, runIndex: number, dbPort: number | null, track: Track): string | null => {
+ const adapter = STACK_ADAPTER_REGISTRY.get(backend);
+ if (adapter.id === 'spacetime' || adapter.id === 'stub') return null;
+ if (!dbPort) throw new Error(`${backend} has no assigned database port`);
+ return adapter.agent.connectionUrl({ dbPort, database: dbName(track, runIndex), hostUrl });
+};
+
+// Create the leased database before the app connects. A build clears its schema.
+// A reset between suites preserves the schema required by the running app.
+type DatabasePreparationLease = BackendLease;
+
+type DatabaseCommandOptions = Pick;
+
+type DatabaseCommandExecutor = (command: string, args: readonly string[],
+ options: DatabaseCommandOptions) => string;
+
+const databaseCommandExecutor: DatabaseCommandExecutor = (command, args, options) =>
+ String(execFileSync(command, args, { ...options, encoding: 'utf8' }));
+
+interface DatabasePreparationOptions {
+ exec?: DatabaseCommandExecutor;
+ stdbBin?: string;
+ lease?: DatabasePreparationLease;
+}
+
+export function ensureDatabase(backend: string, runIndex: number, dbPort: number | null,
+ track: Pick, wipe = false,
+ { exec = databaseCommandExecutor, stdbBin = STDB_BIN, lease: suppliedLease }: DatabasePreparationOptions = {}) {
+ const lease = suppliedLease ?? leaseFromEnv(process.env, { backend, active: true }).lease;
+ if (lease.runIndex !== runIndex || lease.track !== track.name) {
+ throw new Error(`backend lease ${lease.runId} belongs to ${lease.track}/run${lease.runIndex}, `
+ + `not ${track.name}/run${runIndex}`);
+ }
+ const expectedName = dbName(track, runIndex);
+ const name = lease.resources.database ?? expectedName;
+ const input = { name, expectedName, wipe, exec, cli: stdbBin,
+ expectedServerUri: STDB_URI, expectedModule: moduleName(track, runIndex), dbPort };
+ const adapter = STACK_ADAPTER_REGISTRY.get(backend);
+ if (adapter.id === 'postgres' || adapter.id === 'mongodb') {
+ return adapter.database.prepare({ ...input, lease: requireLeasedDatabase(lease) });
+ }
+ if (adapter.id === 'spacetime') {
+ return adapter.database.prepare({ ...input, lease: requireLeasedSpacetime(lease) });
+ }
+ return adapter.database.prepare({ name });
+}
+
+// Prescribed guidance chooses an implementation stack. Neutral guidance gives
+// only stack access facts and the selected API references.
+export function readBackendGuidanceDocument(
+ document: ResolvedGuidanceDocument | undefined,
+ fallbackRelativePath: string,
+): string {
+ if (typeof fallbackRelativePath !== 'string' || !fallbackRelativePath) {
+ throw new Error('backend guidance fallback path is required');
+ }
+ if (document !== undefined) {
+ const fields = new Set(['path', 'sha256', 'bytes', 'applicationInterface']);
+ if (!document || typeof document !== 'object' || Array.isArray(document)
+ || Object.keys(document).some(field => !fields.has(field))
+ || typeof document.path !== 'string' || !document.path || isAbsolute(document.path)
+ || document.path.includes('\\')
+ || !/^[a-f0-9]{64}$/.test(document.sha256)
+ || !Number.isSafeInteger(document.bytes) || document.bytes < 0
+ || !['http', 'reducer'].includes(document.applicationInterface)) {
+ throw new Error('campaign guidance document identity is invalid');
+ }
+ }
+ const root = realpathSync(ROOT);
+ const candidate = resolve(root, document?.path ?? fallbackRelativePath);
+ const candidateRel = relative(root, candidate);
+ if (candidateRel === '..' || candidateRel.startsWith(`..${sep}`) || isAbsolute(candidateRel)) {
+ throw new Error('campaign guidance document escapes the Stack Bench root');
+ }
+ const selectedPath = realpathSync(candidate);
+ const resolvedRel = relative(root, selectedPath);
+ if (resolvedRel === '..' || resolvedRel.startsWith(`..${sep}`) || isAbsolute(resolvedRel)) {
+ throw new Error('campaign guidance document resolves outside the Stack Bench root');
+ }
+ const bytes = Buffer.from(normalizePromptText(readFileSync(selectedPath, 'utf8')), 'utf8');
+ if (document && (sha256(bytes) !== document.sha256 || bytes.length !== document.bytes)) {
+ throw new Error(`campaign guidance document changed after compilation: ${document.path}`);
+ }
+ return bytes.toString('utf8');
+}
+
+function validateSkillIdentity(value: unknown): ResolvedSkills {
+ const fields = new Set(['ids', 'sha256', 'bytes']);
+ if (!isRecord(value) || Object.keys(value).some(field => !fields.has(field))
+ || !Array.isArray(value.ids) || new Set(value.ids).size !== value.ids.length
+ || value.ids.some(id => typeof id !== 'string' || !/^[a-z][a-z0-9-]*$/.test(id))
+ || typeof value.sha256 !== 'string' || !/^[a-f0-9]{64}$/.test(value.sha256)
+ || !Number.isSafeInteger(value.bytes) || Number(value.bytes) < 0) {
+ throw new Error('campaign skill identity is invalid');
+ }
+ return { ids: value.ids as string[], sha256: value.sha256, bytes: Number(value.bytes) };
+}
+
+function backendDoc(args: AgentArgs, p: StackRunPorts, track: Track): string {
+ const defaultGuidance = resolveDefaultGuidanceForStack(args.guidance, args.backend);
+ let defaultPath = defaultGuidance?.documents[args.backend]?.path;
+ if (!defaultPath && args.guidance === 'neutral') {
+ throw new Error(`neutral guidance has no document for ${args.backend}`);
+ }
+ defaultPath ??= join('backends', `${args.backend}.md`);
+ const raw = readBackendGuidanceDocument(args.guidanceDocument, defaultPath);
+ return raw
+ .replaceAll('', String(p.vite))
+ .replaceAll('', String(p.express ?? ''))
+ .replaceAll('', track.title)
+ .replaceAll('', moduleName(track, args.runIndex))
+ .replaceAll('', p.dbPort ? dbUrl(args.backend, args.runIndex, p.dbPort, track) ?? '' : '')
+ .replaceAll('', hostUrl(STDB_URI))
+ .replaceAll('', C_BIN)
+ .replaceAll('', `file:${CODING_CONTAINER_SPACETIME_PACKAGE}`);
+}
+
+// Fail before a paid session when the selected container cannot run this checkout.
+function containerBlocker(backend: string): string | null {
+ try {
+ execFileSync('docker', ['image', 'inspect', IMAGE],
+ { stdio: 'pipe', timeout: CONTROL_COMMAND_TIMEOUT_MS });
+ } catch (error: unknown) {
+ const detail = error instanceof Error ? error.message.split('\n')[0] : String(error).split('\n')[0];
+ return `cannot verify isolation image ${IMAGE}: ${detail} — `
+ + `build it with docker build -t ${IMAGE} ${fwd(join(ROOT, 'container'))}`;
+ }
+ if (!STACK_ADAPTER_REGISTRY.get(backend).agent.linuxCliRequired) return null;
+ if (!existsSync(LINUX_CLI)) {
+ return `no Linux SpacetimeDB CLI at ${fwd(LINUX_CLI)} — `
+ + 'bash tools/stack-bench/container/build-linux-cli.sh';
+ }
+ // A file at this path must be a Linux executable, not the Windows build.
+ const magic = Buffer.alloc(4);
+ try {
+ const fd = openSync(LINUX_CLI, 'r');
+ try { readSync(fd, magic, 0, 4, 0); } finally { closeSync(fd); }
+ } catch {
+ return `cannot read the Linux SpacetimeDB CLI at ${fwd(LINUX_CLI)}`;
+ }
+ if (magic.toString('binary') !== '\x7fELF') {
+ return `${fwd(LINUX_CLI)} is not a Linux binary; rebuild it with `
+ + 'container/build-linux-cli.sh';
+ }
+ return null;
+}
+
+function decideIsolation(args: AgentArgs): { container: true; reason: null } {
+ const blocker = containerBlocker(args.backend);
+ if (!blocker) return { container: true, reason: null };
+ console.error(`agent.js: isolated build unavailable: ${blocker}`);
+ console.error(' benchmark coding sessions require the isolation container');
+ process.exit(2);
+}
+
+// Pin every round to the build's recorded container topology.
+function resolveIsolation(args: AgentArgs): { container: true; reason: null } {
+ const marker = resolve(args.app, '..', '.stack-bench-isolation');
+ const backendMarker = resolve(args.app, '..', '.stack-bench-backend');
+
+ if (args.mode === 'build') {
+ const decided = decideIsolation(args);
+ if (!args.printPrompt) {
+ mkdirSync(dirname(marker), { recursive: true });
+ writeFileSync(marker, 'container');
+ }
+ return decided;
+ }
+
+ if (existsSync(marker)) {
+ const pinned = readFileSync(marker, 'utf8').trim();
+ if (pinned !== 'container') {
+ console.error(`agent.js: unsupported isolation marker ${JSON.stringify(pinned)}; expected "container"`);
+ process.exit(2);
+ }
+ const blocker = containerBlocker(args.backend);
+ if (blocker) {
+ console.error(`agent.js: this run's build ran in a container, but ${blocker}`);
+ console.error(' refusing to run this round in a different environment');
+ process.exit(2);
+ }
+ return { container: true, reason: null };
+ }
+
+ // A backend marker without an isolation marker is ambiguous prior state.
+ if (existsSync(backendMarker)) {
+ console.error('agent.js: app has prior benchmark state but no isolation marker');
+ console.error(' refusing to guess where earlier rounds ran; start a clean run');
+ process.exit(2);
+ }
+ const decided = decideIsolation(args);
+ if (!args.printPrompt) {
+ mkdirSync(dirname(marker), { recursive: true });
+ writeFileSync(marker, 'container');
+ }
+ return decided;
+}
+
+export function buildPrompt(args: AgentArgs, p: StackRunPorts, track: Track,
+ materials: PromptMaterials = {}): string {
+ const prompt = (lines: string[]): string => assertAgentVisibleText(lines.join('\n'));
+ const applicationInterface = args.guidanceDocument?.applicationInterface
+ ?? resolveDefaultGuidanceForStack(args.guidance, args.backend)
+ ?.documents[args.backend]?.applicationInterface;
+ if (applicationInterface !== 'http' && applicationInterface !== 'reducer') {
+ throw new Error(`stack ${args.backend} has no application interface`);
+ }
+ const common = [
+ `Build the app in ${CODING_CONTAINER_APP_ROOT}.`,
+ // Published container ports require the app to bind to all interfaces.
+ '',
+ 'The web application must listen on 0.0.0.0, not localhost, so it is reachable '
+ + 'outside its process.',
+ 'The environment can run /app/start.sh again with APP_WARM_START=1. '
+ + 'When dependencies are current, reuse them instead of installing them again.',
+ '',
+ '## Stack',
+ '',
+ agentVisibleContractText(backendDoc(args, p, track), args.credentialAliases,
+ applicationInterface),
+ ];
+ const skills = materials.skillsText ?? readAgentSkillDocuments(REPO, args.skills ?? []);
+ if (skills) common.push('', '## Selected API reference', '', skills);
+
+ if (args.mode === 'resume') {
+ return prompt([
+ 'Restore the existing application to a runnable state.',
+ '',
+ 'This is a saved application from an earlier completed run. Install its',
+ 'dependencies and start its existing database module, server, and web client',
+ 'as needed. Do not implement features or fix application behavior. Do not',
+ 'change source files. The saved source must remain byte-for-byte identical.',
+ '',
+ 'Output RESUME_COMPLETE when the existing app is running.',
+ '',
+ ...common,
+ ]);
+ }
+
+ if (args.mode === 'fix') {
+ return prompt([
+ 'Fix the reported application bugs.',
+ '',
+ `Read ${CODING_CONTAINER_BUG_REPORT_FILE} in the app directory. Each entry says what was expected`,
+ 'and what actually happened. Fix the app so the behaviour matches, redeploy,',
+ 'and make sure the dev server is running.',
+ '',
+ 'Change only what is needed. Do not alter behaviour that is already correct.',
+ '',
+ 'Output FIX_COMPLETE when done.',
+ '',
+ ...common,
+ '',
+ agentVisibleContractText(materials.requirementText ?? levelPrompt(track, args.level),
+ args.credentialAliases, applicationInterface),
+ '',
+ '## Application interface',
+ '',
+ agentVisibleContractText(materials.contractText ?? appendix(track, args.level),
+ args.credentialAliases, applicationInterface),
+ ]);
+ }
+
+ const verb = args.mode === 'upgrade'
+ ? [
+ 'Add the features below to the existing app.',
+ '',
+ 'Keep completed features working. Add only the current features below.',
+ ]
+ : [`Build the application described below and leave it running.`];
+ const startingCatalog = args.mode === 'build' && materials.startingCatalog
+ ? ['', '## Starting catalog', '', 'Use exactly this starting data:', '',
+ '```json', materials.startingCatalog, '```'] : [];
+
+ return prompt([
+ ...verb,
+ '',
+ `After the web application is running, reply with ${args.mode === 'upgrade'
+ ? 'UPGRADE_COMPLETE' : 'DEPLOY_COMPLETE'}.`,
+ '',
+ ...common,
+ '',
+ agentVisibleContractText(materials.requirementText ?? levelPrompt(track, args.level),
+ args.credentialAliases, applicationInterface),
+ ...startingCatalog,
+ '',
+ '## Application interface',
+ '',
+ agentVisibleContractText(materials.contractText ?? appendix(track, args.level),
+ args.credentialAliases, applicationInterface),
+ ]);
+}
+
+export function agentScenarioPaths(track: Track, level: number,
+ recipeBinding: RecipeBinding | null = null): string[] {
+ const execution = recipeBinding?.execution;
+ if (execution) return execution.map(entry => resolve(track.dir, entry.source ?? ''));
+ return suitesFor(track, level).map(suite => suite.spec);
+}
+
+export function agentRecipeRequest(explicitRecipe: string | null = null,
+ recipeTask: RecipeTaskRequest | null = null): ExactRecipeRequest | null {
+ const bound = recipeTask?.recipe;
+ if (!bound) return explicitRecipe;
+ const identity = `${bound.id}@${bound.version}`;
+ if (explicitRecipe && explicitRecipe !== identity) {
+ throw new Error(`agent recipe ${explicitRecipe} does not match bound task ${identity}`);
+ }
+ return bound;
+}
+
+// The coding container must not contain the controller or grading inputs.
+
+async function main() {
+ const args = parseAgentArgs(process.argv);
+ const track = loadTrack(args.track);
+ const p = portsFor(track, args.backend, args.runIndex);
+ const adapter = STACK_ADAPTER_REGISTRY.get(args.backend);
+ const defaultGuidance = resolveDefaultGuidanceForStack(args.guidance, args.backend);
+ args.credentialAliases ??= defaultGuidance?.credentialAliases ?? {};
+ const profileSkills = defaultGuidance?.skills[args.backend]?.ids;
+ const defaultSkills = profileSkills ?? [...adapter.agent.defaultSkills];
+ const selectedSkills = selectAgentSkills(defaultSkills,
+ args.skillIdentity?.ids ?? args.skills ?? null);
+ const skillsText = readAgentSkillDocuments(REPO, selectedSkills);
+ if (args.skillIdentity && (sha256(skillsText) !== args.skillIdentity.sha256
+ || Buffer.byteLength(skillsText) !== args.skillIdentity.bytes)) {
+ throw new Error('campaign skill material changed after compilation');
+ }
+ const recipeBinding = resolveRecipeRelease(track, args.level,
+ agentRecipeRequest(args.recipe ?? null, args.recipeTask ?? null));
+ if (args.recipeTask && !recipeBinding) {
+ throw new Error(`L${args.level} has no recipe release for the requested task`);
+ }
+ const selectedTask = recipeBinding
+ ? (args.recipeTask
+ ? resolveBoundRecipeTaskRequest(recipeBinding, args.recipeTask)
+ : createBoundRecipeTaskRequest(recipeBinding))
+ : null;
+ const requirementText = selectedTask?.task.requirementText ?? levelPrompt(track, args.level);
+ const contractText = selectedTask?.task.contractText ?? appendix(track, args.level);
+ const taskMode = isRecord(args.recipeTask?.task) ? args.recipeTask.task.mode : null;
+ const startingCatalog = recipeBinding && taskMode === 'fresh' ? JSON.stringify({
+ warehouses: recipeBinding.plan.fixture.warehouses,
+ items: recipeBinding.plan.fixture.items,
+ }, null, 2) : undefined;
+
+ // Print the exact prompt without starting a session or changing the app.
+ if (args.printPrompt) {
+ process.stdout.write(buildPrompt(args, p, track,
+ { skillsText, requirementText, contractText, startingCatalog }));
+ return;
+ }
+ if (args.mode === 'build') {
+ assertNewOrEmptyDirectory(args.app, 'build application directory');
+ }
+ resolveIsolation(args);
+ const imageIdentity = resolveContainerImage(IMAGE);
+ // Build wipes all backend state. Later rounds preserve it.
+ ensureDatabase(args.backend, args.runIndex, p.dbPort, track, args.mode === 'build');
+ // Never erase a caller-supplied application tree.
+ mkdirSync(args.app, { recursive: true });
+ writeFileSync(resolve(args.app, '..', '.stack-bench-backend'), args.backend);
+
+ const prompt = buildPrompt(args, p, track,
+ { skillsText, requirementText, contractText, startingCatalog });
+ const bugReportPath = join(args.app, CODING_CONTAINER_BUG_REPORT_FILE);
+ const bugReportText = args.mode === 'fix' && existsSync(bugReportPath)
+ ? readFileSync(bugReportPath, 'utf8') : null;
+ const provenance = sessionProvenance({ prompt, skillsText, contractText, bugReportText,
+ scenarioPaths: agentScenarioPaths(track, args.level, recipeBinding),
+ trackDir: track.dir, trackManifestPath: join(track.dir, TRACK_MANIFEST_FILE) });
+ const started = Date.now();
+ const retryLimitRaw = process.env.STACK_BENCH_CODING_INTERRUPTION_RETRIES
+ ?? String(DEFAULT_CODING_INTERRUPTION_RETRIES);
+ const retryLimit = Number(retryLimitRaw);
+ if (!Number.isInteger(retryLimit) || retryLimit < 0 || retryLimit > 3) {
+ throw new Error('STACK_BENCH_CODING_INTERRUPTION_RETRIES must be an integer from 0 to 3');
+ }
+ // The throttle wait must fit inside the adapter deadline.
+ const throttleWaitRaw = process.env.STACK_BENCH_PROVIDER_THROTTLE_MAX_WAIT_MINUTES
+ ?? String(DEFAULT_THROTTLE_MAX_WAIT_MS / 60_000);
+ const throttleMaxWaitMinutes = Number(throttleWaitRaw);
+ if (!Number.isInteger(throttleMaxWaitMinutes) || throttleMaxWaitMinutes < 0
+ || throttleMaxWaitMinutes > DEFAULT_THROTTLE_MAX_WAIT_MS / 60_000) {
+ throw new Error('STACK_BENCH_PROVIDER_THROTTLE_MAX_WAIT_MINUTES must be an integer from 0 to '
+ + `${DEFAULT_THROTTLE_MAX_WAIT_MS / 60_000}`);
+ }
+ // Concurrent campaign slots must not wake and retry as one burst. This
+ // stable offset keeps retries reproducible while spreading them over 45s.
+ const throttleJitterMs = parseInt(sha256(Buffer.from(
+ `${args.backend}:${args.runIndex}:${args.level}:${args.mode}`)).slice(0, 8), 16) % 45_001;
+ let coding: CodingSessionRetryResult;
+ try {
+ // Send prompts through stdin to avoid the Windows command-line limit.
+ const cliEnv = { ...process.env,
+ // Absent unless deliberately overridden — see THINKING_TOKENS above.
+ ...((args.thinking ?? THINKING_TOKENS)
+ ? { MAX_THINKING_TOKENS: String(args.thinking ?? THINKING_TOKENS) }
+ : {}),
+ // Keep the CLI fixed across the campaign.
+ DISABLE_AUTOUPDATER: '1',
+ // Pin cache lifetime so run order cannot change cost.
+ FORCE_PROMPT_CACHING_5M: '1' };
+
+ coding = runCodingSessionWithRetries({ prompt, model: args.model, retryLimit,
+ maxBudgetUsd: args.maxBudgetUsd,
+ throttleMaxWaitMs: throttleMaxWaitMinutes * 60_000,
+ throttleJitterMs,
+ invoke: ({ input, maxBudgetUsd, resumeSession, recoverStoppedContainer }) =>
+ execFileSync(process.execPath, [
+ compiledEntrypoint('container', 'run-build.js'),
+ '--app', args.app,
+ '--backend', args.backend,
+ '--image', imageIdentity.id,
+ '--effort', EFFORT,
+ '--model', args.model,
+ ...(args.pricing ? ['--pricing-json', JSON.stringify(args.pricing)] : []),
+ '--completion-marker', args.mode === 'fix' ? 'FIX_COMPLETE'
+ : args.mode === 'upgrade' ? 'UPGRADE_COMPLETE'
+ : args.mode === 'resume' ? 'RESUME_COMPLETE' : 'DEPLOY_COMPLETE',
+ ...(maxBudgetUsd != null ? ['--max-budget-usd', String(maxBudgetUsd)] : []),
+ '--ports', [p.vite, p.express].filter(Boolean).join(','),
+ ...(resumeSession ? ['--resume-session', resumeSession] : []),
+ ...(recoverStoppedContainer ? ['--recover-stopped-container'] : []),
+ ], { input, encoding: 'utf8', maxBuffer: 256 * 1024 * 1024,
+ env: { ...cliEnv, ...(args.apiKey ? { STACK_BENCH_AGENT_API_KEY: args.apiKey } : {}) },
+ timeout: AGENT_PROCESS_TIMEOUT_MS }),
+ });
+ } catch (err: unknown) {
+ coding = { raw: '', spawnError: codingSessionFailure(isRecord(err) ? err : {}), sessionResults: [],
+ interruptions: [], result: { total_cost_usd: 0, num_turns: 0,
+ usage: { input_tokens: 0, output_tokens: 0, cache_creation_input_tokens: 0,
+ cache_read_input_tokens: 0 }, stack_bench_cost_receipts: [] },
+ throttle: { waits: 0, waitedMs: 0, maxWaitMs: throttleMaxWaitMinutes * 60_000,
+ jitterMs: throttleJitterMs } };
+ }
+
+ const { raw, spawnError, sessionResults, interruptions, result, throttle } = coding;
+ const noOutput = !result.session_id && !raw.trim();
+ const failed = Boolean(spawnError || noOutput);
+ const providerFailure = providerSessionFailure(result);
+ const usage = sessionUsage(result.usage);
+ const input = usage.input_tokens ?? 0;
+ const output = usage.output_tokens ?? 0;
+ const cacheWrite = usage.cache_creation_input_tokens ?? 0;
+ const cacheRead = usage.cache_read_input_tokens ?? 0;
+ const turns = result.num_turns ?? 0;
+
+ // Preserve the cost inputs needed to explain stack differences.
+ const setupMetadata = adapter.agent.setupMetadata({
+ imageId: imageIdentity.id,
+ localPackage: LOCAL_PKG,
+ env: process.env,
+ helpers: { linuxSpacetimeVersion, bindingsIdentity, containerImage },
+ });
+ const out = {
+ appDir: args.app,
+ mode: args.mode,
+ level: args.level,
+ track: args.track,
+ backend: args.backend,
+ model: args.model,
+ guidance: args.guidance,
+ setup: {
+ thinkingTokens: (args.thinking ?? THINKING_TOKENS) ? Number(args.thinking ?? THINKING_TOKENS) : 'cli default',
+ permissionMode: 'acceptEdits',
+ effort: EFFORT,
+ skills: selectedSkills,
+ cacheTier: '5m',
+ autoUpdater: 'disabled',
+ codingInterruptionRetries: { limit: retryLimit,
+ used: interruptions.filter(item => item.kind !== 'provider-throttled').length },
+ providerThrottle: { maxWaitMinutes: throttleMaxWaitMinutes,
+ waits: throttle?.waits ?? 0, waitedMs: throttle?.waitedMs ?? 0,
+ jitterMs: throttle?.jitterMs ?? throttleJitterMs },
+ cliVersion: imageCliVersion(imageIdentity.id),
+ isolation: { mode: 'container', image: imageIdentity.reference,
+ imageId: imageIdentity.id, hostAlias: HOST_ADDR },
+ auth: (args.apiKey ?? process.env.ANTHROPIC_API_KEY) ? 'api-key'
+ : (process.env.CLAUDE_CODE_OAUTH_TOKEN || process.env.CLAUDE_CODE_OAUTH_TOKEN_FILE)
+ ? 'subscription-token' : 'not-selected',
+ ...(isRecord(setupMetadata) ? setupMetadata : {}),
+ env: ambientEnv(),
+ node: { orchestrator: process.version, codingContainer: imageNodeVersion(imageIdentity.id) },
+ platform: process.platform,
+ resources: result.stack_bench_resources ?? null,
+ },
+ costUsd: Number((result.total_cost_usd ?? 0).toFixed(6)),
+ costReceipts: result.stack_bench_cost_receipts ?? [],
+ tokens: input + output + cacheWrite + cacheRead,
+ outputTokens: output,
+ usage: { input, output, cacheWrite, cacheRead },
+ provenance,
+ turns,
+ promptBytes: Buffer.byteLength(prompt),
+ tokensPerTurn: turns ? Math.round((input + output + cacheWrite + cacheRead) / turns) : null,
+ thinking: combinedThinkingVolume(args.app, sessionResults.map(item => item.session_id)),
+ durationMs: Date.now() - started,
+ sessionId: result.session_id ?? null,
+ ok: !failed && result.is_error === false,
+ providerMetadata: { failureCode: failed
+ ? String(spawnError ?? '').startsWith('provider stayed throttled')
+ ? 'provider-throttle-exhausted'
+ : providerFailure?.code ?? (noOutput ? 'coding-session-no-output' : 'coding-session-failed')
+ : result.is_error === true ? 'provider-session-error' : null,
+ diagnostic: spawnError,
+ failure: failed ? {
+ providerStatus: result.api_error_status ?? null,
+ waitedMs: throttle?.waitedMs ?? 0,
+ waits: throttle?.waits ?? 0,
+ } : null,
+ interruptions, invocations: sessionResults.length,
+ terminalRecovery: isRecord(result) ? result.terminal_recovery ?? null : null,
+ credentialBroker: result.stack_bench_credential_broker ?? null,
+ sessionIds: [...new Set(sessionResults.map(item => item.session_id).filter(Boolean))],
+ models: transcriptModels(args.app, sessionResults.map(item => item.session_id)) },
+ };
+ console.log(JSON.stringify(out));
+}
+
+if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
+ main().catch(err => { console.error(err); process.exit(1); });
+}
diff --git a/tools/stack-bench/commands/bench-arguments.ts b/tools/stack-bench/commands/bench-arguments.ts
new file mode 100644
index 00000000000..ac4f486e5e1
--- /dev/null
+++ b/tools/stack-bench/commands/bench-arguments.ts
@@ -0,0 +1,344 @@
+import { dirname, resolve } from 'node:path';
+import { parseArgs } from 'node:util';
+import { readArtifact } from '../src/evidence/artifacts.js';
+import { DEFAULT_BUILD_IMAGE } from '../src/composition/product-config.js';
+import { DEFAULT_TRACK, RUN_INDEX_CAP } from '../src/composition/tracks.js';
+import { validateCompiledCampaignPlan } from '../src/campaigns/campaign-compiler.js';
+import type { CampaignAttemptPlan, CampaignSelection }
+ from '../src/campaigns/campaign-compiler.js';
+import { campaignAdmissionSmokeReuse, readCampaignAdmission }
+ from '../src/campaigns/campaign-admission.js';
+import type { CampaignAdmissionSmokeReuse } from '../src/campaigns/campaign-admission.js';
+import { compileProgressionInput, dependencyRuntimeDefinition, progressionLevels,
+ validateFeatureCatalogInput, validateProgressionInput }
+ from '../src/progression/progression-definition.js';
+import type { CompiledDependencyPolicyDefinition, CompiledProgressionDefinition,
+ ProgressionInput } from '../src/progression/progression-definition.js';
+import { validatePricingAuthority } from '../src/evidence/pricing-authority.js';
+import type { PricingAuthority } from '../src/evidence/pricing-authority.js';
+import { parseGuidanceMode } from '../src/campaigns/condition-compiler.js';
+import type { GuidanceMode } from '../src/campaigns/condition-compiler.js';
+import { validateCampaignExtensionSeed } from '../src/campaigns/campaign-scheduler.js';
+import type { CampaignExtensionSeed } from '../src/campaigns/campaign-scheduler.js';
+import { repairBudgetLimit } from '../src/progression/repair-plan.js';
+
+type StudyCondition = CampaignAttemptPlan['condition'];
+type UnknownRecord = Record;
+
+export interface BenchArguments {
+ backend?: string;
+ track: string;
+ levels: string;
+ levelsProvided: boolean;
+ levelList: number[];
+ model: string | null;
+ agentAdapter: string;
+ pricing?: PricingAuthority | null;
+ repairs: number;
+ maxStalledRepairs: number;
+ maxBudgetUsd?: number;
+ runIndex: number;
+ out?: string;
+ app?: string;
+ url?: string;
+ media: boolean;
+ retainBackend?: boolean;
+ guidance: GuidanceMode;
+ guidanceDocument?: unknown;
+ condition?: StudyCondition;
+ selectionRequest?: CampaignSelection;
+ taskMode?: string;
+ packIds: string[];
+ checkKeys: string[];
+ featureIds: string[];
+ requestedSpecifications: string[];
+ expectedSpecifications: string[];
+ observedSpecifications: string[];
+ skills?: string[];
+ apiKey?: string;
+ apiKeyFile?: string;
+ mutations?: string;
+ mutationShardIndex?: number;
+ mutationShardCount?: number;
+ mutationResumeFrom?: string;
+ mutationCheckpointOut?: string;
+ mutationBaselineBundle?: string;
+ expectedMutationCalibration?: unknown;
+ mutationMaxRuntimeMinutes: number;
+ referenceMutationOnly?: boolean;
+ seedFrom?: string;
+ seedThrough?: number;
+ progressionSeed?: CampaignExtensionSeed;
+ parentAttemptId?: string;
+ repairFrom?: string;
+ repairLevel?: number;
+ recipe?: string;
+ campaignFile?: string;
+ campaignAttemptId?: string;
+ campaignAdmissionId?: string;
+ progressionResumeFrom?: string;
+ experimentIdentity?: { id: string; version: string; sha256: string; state: string };
+ campaignAdmission?: { id: string } & CampaignAdmissionSmokeReuse;
+ runMode?: CampaignAttemptPlan['mode'];
+ featureCatalog?: ProgressionInput;
+ dependencyPolicy?: ProgressionInput;
+ progression?: ProgressionInput;
+ progressionOwner?: UnknownRecord;
+}
+
+interface BenchCliOptions extends Partial {
+ pack?: string[];
+ check?: string[];
+ pricingJson?: unknown;
+ featureModule?: string[];
+ requestSpec?: string[];
+ expectSpec?: string[];
+ observeSpec?: string[];
+ expectedMutationCalibrationJson?: unknown;
+ progressionSeedJson?: unknown;
+}
+
+function parseCli(argv: readonly string[]): BenchCliOptions {
+ const strings = ['backend', 'track', 'levels', 'campaign-file', 'campaign-attempt-id',
+ 'campaign-admission-id', 'progression-resume-from', 'recipe', 'model', 'pricing-json',
+ 'repairs', 'max-stalled-repairs', 'max-budget-usd', 'run-index', 'out', 'app', 'url',
+ 'agent-adapter', 'guidance', 'task-mode', 'skills', 'mutations',
+ 'mutation-shard-index', 'mutation-shard-count', 'mutation-resume-from',
+ 'mutation-checkpoint-out', 'mutation-baseline-bundle',
+ 'expected-mutation-calibration-json', 'mutation-max-runtime-minutes', 'seed-from',
+ 'seed-through', 'progression-seed-json',
+ 'parent-attempt-id', 'repair-from', 'repair-level'] as const;
+ const multiple = ['pack', 'check', 'feature-module', 'request-spec', 'expect-spec',
+ 'observe-spec'] as const;
+ const options = Object.fromEntries([
+ ...strings.map(name => [name, { type: 'string' as const }]),
+ ...multiple.map(name => [name, { type: 'string' as const, multiple: true }]),
+ ...['no-media', 'retain-backend', 'reference-mutation-only'].map(name =>
+ [name, { type: 'boolean' as const }]),
+ ]);
+ const { values } = parseArgs({ args: [...argv.slice(2)], options, strict: true,
+ allowPositionals: false });
+ const parsed: Record = {};
+ for (const [key, value] of Object.entries(values)) {
+ parsed[key.replace(/-([a-z])/g, (_, letter: string) => letter.toUpperCase())] = value;
+ }
+ for (const key of ['pack', 'check', 'featureModule', 'requestSpec', 'expectSpec',
+ 'observeSpec']) {
+ const value = parsed[key] as string[] | undefined;
+ if (value) parsed[key] = value.flatMap(item => item.split(',').filter(Boolean));
+ }
+ for (const key of ['repairs', 'maxStalledRepairs', 'maxBudgetUsd', 'mutationShardIndex',
+ 'mutationShardCount', 'mutationMaxRuntimeMinutes', 'repairLevel', 'seedThrough']) {
+ if (typeof parsed[key] === 'string') parsed[key] = Number(parsed[key]);
+ }
+ if (typeof parsed.runIndex === 'string') parsed.runIndex = Number(parsed.runIndex);
+ for (const key of ['campaignFile', 'progressionResumeFrom', 'mutations',
+ 'mutationResumeFrom', 'mutationCheckpointOut', 'mutationBaselineBundle', 'repairFrom']) {
+ if (typeof parsed[key] === 'string') parsed[key] = resolve(parsed[key]);
+ }
+ for (const key of ['pricingJson', 'expectedMutationCalibrationJson', 'progressionSeedJson']) {
+ if (typeof parsed[key] === 'string') parsed[key] = JSON.parse(parsed[key]);
+ }
+ if (typeof parsed.guidance === 'string') parsed.guidance = parseGuidanceMode(parsed.guidance);
+ if (typeof parsed.skills === 'string') parsed.skills = parsed.skills.split(',').filter(Boolean);
+ if (parsed.noMedia === true) parsed.media = false;
+ delete parsed.noMedia;
+ return parsed as BenchCliOptions;
+}
+
+export function parseBenchArguments(argv: readonly string[]): BenchArguments {
+ const args: BenchArguments = { model: null, agentAdapter: 'claude-code',
+ repairs: 10, runIndex: 0, levels: '1', levelsProvided: false, media: true,
+ levelList: [], maxStalledRepairs: 3, guidance: 'prescribed', track: DEFAULT_TRACK,
+ packIds: [], checkKeys: [], featureIds: [], requestedSpecifications: [],
+ expectedSpecifications: [], observedSpecifications: [],
+ mutationMaxRuntimeMinutes: 60 };
+ const { pack, check, pricingJson, featureModule, requestSpec, expectSpec, observeSpec,
+ expectedMutationCalibrationJson, progressionSeedJson, ...options } = parseCli(argv);
+ Object.assign(args, options);
+ if (pack) args.packIds = pack;
+ if (check) args.checkKeys = check;
+ if (pricingJson !== undefined) {
+ args.pricing = validatePricingAuthority(pricingJson, { at: '--pricing-json' });
+ }
+ if (featureModule) args.featureIds = featureModule;
+ if (requestSpec) args.requestedSpecifications = requestSpec;
+ if (expectSpec) args.expectedSpecifications = expectSpec;
+ if (observeSpec) args.observedSpecifications = observeSpec;
+ args.levelsProvided = options.levels !== undefined;
+ if (expectedMutationCalibrationJson !== undefined) {
+ args.expectedMutationCalibration = expectedMutationCalibrationJson;
+ }
+ if (progressionSeedJson !== undefined) {
+ args.progressionSeed = validateCampaignExtensionSeed(progressionSeedJson);
+ }
+ if ((args.mutationResumeFrom || args.mutationCheckpointOut || args.mutationBaselineBundle)
+ && !args.mutations) {
+ throw new Error('mutation control options require --mutations');
+ }
+ if (args.expectedMutationCalibration && !args.mutations) {
+ throw new Error('--expected-mutation-calibration-json requires --mutations');
+ }
+ if (!Number.isFinite(args.mutationMaxRuntimeMinutes) || args.mutationMaxRuntimeMinutes < 1
+ || args.mutationMaxRuntimeMinutes > 120) {
+ throw new Error('--mutation-max-runtime-minutes must be from 1 through 120');
+ }
+ if (args.referenceMutationOnly && (!args.mutations || args.agentAdapter !== 'reference-fixture'
+ || args.repairs !== 0 || !args.app || args.campaignFile)) {
+ throw new Error('--reference-mutation-only requires a mutation-bound reference fixture run');
+ }
+ if (args.mutationBaselineBundle && !args.referenceMutationOnly) {
+ throw new Error('--mutation-baseline-bundle is an internal reference mutation option');
+ }
+ if (args.repairFrom && (args.repairLevel === undefined
+ || !Number.isSafeInteger(args.repairLevel) || args.repairLevel < 1)) {
+ throw new Error('--repair-from requires --repair-level with a positive integer');
+ }
+ if (args.campaignFile && !args.campaignAttemptId) {
+ throw new Error('--campaign-file requires --campaign-attempt-id');
+ }
+ if (!args.campaignFile && (args.campaignAttemptId || args.campaignAdmissionId)) {
+ throw new Error('campaign binding requires --campaign-file');
+ }
+ if (args.progressionResumeFrom && !args.campaignFile) {
+ throw new Error('--progression-resume-from requires a compiled campaign');
+ }
+ if (args.seedThrough !== undefined && (!args.seedFrom || !args.campaignFile)) {
+ throw new Error('--seed-through requires --seed-from and a compiled campaign');
+ }
+ if (args.seedThrough !== undefined && !args.progressionSeed) {
+ throw new Error('--seed-through requires --progression-seed-json');
+ }
+ if (args.progressionSeed !== undefined && args.seedThrough === undefined) {
+ throw new Error('--progression-seed-json requires --seed-through');
+ }
+ if (args.progressionSeed && args.progressionSeed.fromDepth !== args.seedThrough) {
+ throw new Error('--progression-seed-json does not match --seed-through');
+ }
+ if (args.campaignFile) {
+ const allowed = new Set(['--campaign-file', '--campaign-attempt-id',
+ '--campaign-admission-id', '--progression-resume-from', '--run-index', '--out',
+ '--max-budget-usd', '--seed-from', '--seed-through', '--progression-seed-json']);
+ const override = argv.slice(2).find(value => value.startsWith('--')
+ && !allowed.has(value.split('=', 1)[0]!));
+ if (override) throw new Error(`campaign attempts cannot override ${override}`);
+ bindCampaign(args);
+ }
+ if (!args.backend && !args.repairFrom) {
+ throw new Error('--backend is required unless --repair-from or --campaign-file is supplied');
+ }
+ if (args.progression) {
+ if (args.levelsProvided) throw new Error('--levels cannot be combined with progression input');
+ args.progression = validateProgressionInput(args.progression);
+ args.levelList = progressionLevels(args.progression);
+ args.levels = `${args.levelList[0]}-${args.levelList.at(-1)}`;
+ const seedThrough = args.seedThrough;
+ if (seedThrough !== undefined && (!args.levelList.includes(seedThrough)
+ || !args.levelList.some(level => level > seedThrough))) {
+ throw new Error('--seed-through must precede another planned dependency depth');
+ }
+ if (seedThrough !== undefined
+ && args.dependencyPolicy?.definition.workSelection !== 'progressive') {
+ throw new Error('--seed-through requires progressive dependency work selection');
+ }
+ } else {
+ const [fromText, toText] = args.levels.split('-');
+ const from = Number(fromText);
+ const to = toText === undefined ? from : Number(toText);
+ if (!Number.isSafeInteger(from) || from < 1 || !Number.isSafeInteger(to) || to < from) {
+ throw new Error('--levels must be one positive level or an ascending range');
+ }
+ args.levelList = Array.from({ length: (to ?? from) - from + 1 }, (_, index) => from + index);
+ if (args.seedThrough !== undefined) {
+ throw new Error('--seed-through requires dependency mode');
+ }
+ }
+ if (args.recipe && args.levelList.length !== 1) {
+ throw new Error('--recipe requires exactly one requested level');
+ }
+ if (!Number.isSafeInteger(args.repairs) || args.repairs < 0) {
+ throw new Error('--repairs must be a non-negative safe integer');
+ }
+ if (!Number.isInteger(args.maxStalledRepairs) || args.maxStalledRepairs < 0
+ || args.maxStalledRepairs > 20) {
+ throw new Error('--max-stalled-repairs must be an integer from 0 through 20');
+ }
+ if (args.maxBudgetUsd !== undefined
+ && (!Number.isFinite(args.maxBudgetUsd) || args.maxBudgetUsd <= 0)) {
+ throw new Error('--max-budget-usd must be a positive number');
+ }
+ if (!Number.isSafeInteger(args.runIndex) || args.runIndex < 0 || args.runIndex > RUN_INDEX_CAP) {
+ throw new Error(`--run-index must be an integer from 0 through ${RUN_INDEX_CAP}`);
+ }
+ if ((args.mutationShardIndex === undefined) !== (args.mutationShardCount === undefined)) {
+ throw new Error('--mutation-shard-index and --mutation-shard-count must be supplied together');
+ }
+ return args;
+}
+
+function bindCampaign(args: BenchArguments): void {
+ if (!args.campaignFile) throw new Error('campaign file is required');
+ const artifact = readArtifact(args.campaignFile, { expectedKind: 'campaign_plan' });
+ const plan = validateCompiledCampaignPlan(artifact.payload);
+ const attempt = plan.attempts.find(item => item.id === args.campaignAttemptId);
+ if (!attempt) throw new Error('--campaign-attempt-id is not in the compiled campaign plan');
+ const plannedBudget = plan.definition.budgets.maxCostUsdPerAttempt;
+ if (args.maxBudgetUsd !== undefined
+ && (plannedBudget === null || args.maxBudgetUsd > plannedBudget)) {
+ throw new Error('--max-budget-usd exceeds the compiled campaign budget');
+ }
+ args.backend = attempt.stack;
+ args.track = plan.definition.track;
+ args.model = attempt.model;
+ args.agentAdapter = attempt.agentAdapter;
+ args.pricing = validatePricingAuthority(attempt.pricing, { at: 'compiled campaign pricing' });
+ args.guidance = parseGuidanceMode(attempt.guidance);
+ args.condition = structuredClone(attempt.condition);
+ args.skills = structuredClone(attempt.skills);
+ args.selectionRequest = structuredClone(plan.definition.selection);
+ args.guidanceDocument = structuredClone(
+ attempt.condition.guidance.documents[attempt.stack]);
+ args.packIds = structuredClone(plan.definition.selection.packs ?? []);
+ args.checkKeys = structuredClone(plan.definition.selection.checks ?? []);
+ args.repairs = attempt.mode.id === 'dependency'
+ ? 0 : repairBudgetLimit(plan.definition.repair);
+ args.maxBudgetUsd ??= plannedBudget ?? undefined;
+ args.parentAttemptId = attempt.id;
+ args.media = false;
+ args.levels = `${Math.min(...attempt.levels)}-${Math.max(...attempt.levels)}`;
+ args.experimentIdentity = {
+ id: plan.id, version: plan.version, sha256: plan.contentSha256, state: plan.state,
+ };
+ if (args.campaignAdmissionId) {
+ const admission = readCampaignAdmission(dirname(args.campaignFile),
+ args.campaignAdmissionId, plan);
+ const image = plan.definition.runtime.buildImage
+ ?? process.env.STACK_BENCH_IMAGE ?? DEFAULT_BUILD_IMAGE;
+ args.campaignAdmission = {
+ id: args.campaignAdmissionId,
+ ...campaignAdmissionSmokeReuse(admission, {
+ agentAdapter: args.agentAdapter,
+ runIndex: args.runIndex,
+ backend: args.backend,
+ image,
+ }),
+ };
+ }
+ args.runMode = structuredClone(attempt.mode);
+ if (plan.featureCatalog) {
+ args.featureCatalog = validateFeatureCatalogInput(plan.featureCatalog);
+ }
+ if (attempt.mode.id === 'dependency') {
+ if (!plan.dependencyPolicy || !args.featureCatalog) {
+ throw new Error('dependency campaign requires a feature catalog and dependency policy');
+ }
+ args.dependencyPolicy = plan.dependencyPolicy;
+ args.progression = compileProgressionInput(dependencyRuntimeDefinition(
+ args.featureCatalog, args.dependencyPolicy));
+ args.progressionOwner = { schemaVersion: 1,
+ campaign: { id: plan.id, version: plan.version, sha256: plan.contentSha256 },
+ attempt: { id: attempt.id, track: plan.definition.track, stack: attempt.stack,
+ agentAdapter: attempt.agentAdapter, model: attempt.model,
+ conditionSha256: attempt.condition.sha256 } };
+ }
+}
diff --git a/tools/stack-bench/commands/bench.ts b/tools/stack-bench/commands/bench.ts
new file mode 100644
index 00000000000..7ca98080553
--- /dev/null
+++ b/tools/stack-bench/commands/bench.ts
@@ -0,0 +1,2678 @@
+#!/usr/bin/env node
+
+import { execFile, execFileSync } from 'node:child_process';
+import type { ChildProcess, ExecFileSyncOptionsWithStringEncoding } from 'node:child_process';
+import { readFileSync, writeFileSync, mkdirSync, existsSync, cpSync, rmSync } from 'node:fs';
+import { join, dirname, resolve } from 'node:path';
+import { tmpdir } from 'node:os';
+import { pathToFileURL } from 'node:url';
+import { loadTrack, resultsName, portsFor, workDirFor, assertNoPortCollisions,
+ moduleName, dbName, suitesFor } from '../src/composition/tracks.js';
+import { parseBenchArguments } from './bench-arguments.js';
+import type { BenchArguments } from './bench-arguments.js';
+import { killTree } from '../src/runtime/platform.js';
+import { formatRepairProgress } from '../src/evidence/scoring.js';
+import { ARTIFACT_FILE, emptyArtifactIdentities, readArtifact, readArtifactPayload,
+ writeArtifact, writeRunJson } from '../src/evidence/artifacts.js';
+import { aggregateRunOutcome, classifyBundle, ladderMayAdvance, ladderMayContinue,
+ mutationControlEligible, runExitCode, runOutcomeKind } from '../src/evidence/outcomes.js';
+import { summarizeSessions } from '../src/evidence/session-metrics.js';
+import { hashDirectory, sha256 } from '../src/evidence/provenance.js';
+import { createBackendLease, newRunId, publicBackendLease, readBackendLease,
+ acquireResourceLocks, backendResourceLockKeys, releaseResourceLocks, resourceLockScope,
+ writeBackendLease } from '../src/runtime/backend-lease.js';
+import { captureApplicationDiagnostics } from '../src/runtime/backend-control.js';
+import type { RuntimeControlSpec } from '../src/runtime/backend-control.js';
+import { releaseBackendLease } from '../src/runtime/backend-teardown.js';
+import { resolveRecipeRelease } from '../src/composition/recipe-release.js';
+import { createAgentVisibleTaskRequest, createBoundRecipeTaskRequest }
+ from '../src/composition/recipe-selection.js';
+import { criterionEvidence, evidencePassed } from '../src/evidence/check-evidence.js';
+import { STACK_ADAPTER_REGISTRY } from '../src/stacks/stack-adapters.js';
+import { agentRecipeIdentity, agentRequestArgv } from '../src/agents/agent-adapter-contract.js';
+import { agentSessionFailure, validateAgentResult }
+ from '../src/agents/agent-result-contract.js';
+import { AGENT_ADAPTER_REGISTRY, agentAdapterIdentity } from '../src/agents/agent-adapters.js';
+import { archiveTranscripts } from '../src/agents/transcript-archive.js';
+import { runPreflight } from '../src/runtime/preflight.js';
+import { DEFAULT_BUILD_IMAGE } from '../src/composition/product-config.js';
+import { SUPERVISOR_STATE_VERSION, writeRecoveryArtifact } from '../src/runtime/recovery.js';
+import { applyAgentCredential } from '../src/agents/agent-credentials.js';
+import { hashAppSource, resetAppToSource, seedAppSource, snapshotAppSource } from '../src/runtime/source-snapshot.js';
+import { finalPackageEvidenceRequired, preserveFinalPackageEvidence, preserveLevelCheckpoint,
+ sourceBoundFirstBuildOutcome } from '../src/runtime/source-checkpoint.js';
+import { materializationAppFailure, materializeAcceptedSource }
+ from '../src/runtime/source-materialization.js';
+import { compareRepairBaseline, createRepairGrant } from '../src/runtime/repair-grant.js';
+import { canonicalDefinitionJson } from '../src/composition/definition-plan.js';
+import { contractInterfaceNames } from '../src/composition/agent-visible-contract.js';
+import { clearPrivateGradingEvidence, levelGradeIsUsable, repairEvidenceDecision,
+ repairHistoryEntry, repairProgressState, repairRegressionDecision,
+ restorePrivateGradingEvidence }
+ from '../src/evidence/repair-evidence.js';
+import { mutationControlArgv, mutationControlTimeoutMs, pristineMutationBaselinePath }
+ from '../src/evidence/mutation-control.js';
+import type { MutationControlArgs } from '../src/evidence/mutation-control.js';
+import { progressionEngine } from '../src/progression/progression-engine.js';
+import { dependencyRepairBudget, dependencyRepairRecords }
+ from '../src/progression/dependency-mode.js';
+import { DEPENDENCY_MODE_VERSION } from '../src/progression/dependency-definition.js';
+import { resolveProgressionRecipeAction, resolveProgressionRecipeLevelSelection,
+ resolveProgressionRepairTarget, validateProgressionCampaignLevelScope }
+ from '../src/progression/progression-recipe-selection.js';
+import { createLiveProgressionExecution }
+ from '../src/progression/live-progression.js';
+import type { CampaignSelection } from '../src/campaigns/campaign-compiler.js';
+import { gradingRunTimeoutMs, selectedGradingSourceCount }
+ from '../src/runtime/grading-timeout.js';
+import { claudeRatesForModel } from '../src/evidence/claude-usage-cost.js';
+import { PRICING_UNIT, validatePricingAuthority }
+ from '../src/evidence/pricing-authority.js';
+import type { BoundRecipeTaskRequestResult } from '../src/composition/recipe-selection.js';
+import type { RecipeBinding } from '../src/composition/recipe-release.js';
+import type { RepairGrantResolution, RepairOutcome } from '../src/runtime/repair-grant.js';
+import type { AgentAdapter, AgentMode, AgentRequest }
+ from '../src/agents/agent-adapter-contract.js';
+import type { ValidatedAgentResult } from '../src/agents/agent-result-contract.js';
+import type { Track } from '../src/composition/tracks.js';
+import type { RunOutcome } from '../src/evidence/outcomes.js';
+import type { GradeBundlePayload, BenchmarkRunRecord, RunLevelRecord,
+ RunContinuation, RunRepairCandidate, RunSessionRecord, RunTotals }
+ from '../src/evidence/benchmark-run.js';
+import { addCostUsd, finalizeRunTotals, runSessionRecord }
+ from '../src/evidence/benchmark-run.js';
+import { formatLevelSummary } from '../src/evidence/evidence-presentation.js';
+import type { ProgressionAction } from '../src/progression/progression-engine.js';
+import type { ProgressionRepairRegression, ProgressionState }
+ from '../src/progression/progression-state.js';
+import type { ProgressionRecipeAction, ProgressionRecipeSelections }
+ from '../src/progression/progression-recipe-selection.js';
+import type { BackendLease } from '../src/runtime/backend-lease.js';
+
+import { STACK_BENCH_ROOT as ROOT, compiledEntrypoint } from '../src/package-root.js';
+import { stackBenchResultsRoot } from '../src/runtime/operational-paths.js';
+import { runningContainerIdentity } from '../src/runtime/container-identity.js';
+const COMMAND_TIMEOUT_MS = 20 * 60_000;
+
+type UnknownRecord = Record;
+type ContaminationAudit = { kind: 'contaminated' | 'harness_failure'; evidence: string[];
+ verdict: string };
+type LeakAuditEntry = { hits: Array<{ kind: string; path: string }> };
+type CommandFailure = Error & { stdout?: string | Buffer; stderr?: string | Buffer;
+ status?: number | null; signal?: NodeJS.Signals | null };
+type GradeOptions = { observation?: 'scored' | 'observed'; out?: string | null;
+ sourceSha256?: string | null; applicationFailure?: RunOutcome | null;
+ recipeTask?: GradeRecipeTask };
+type MutationControlResult = UnknownRecord & { ok: boolean; artifact?: string;
+ skipped?: boolean; processError?: string | null; outcome: RunOutcome | null };
+type RecipeTask = (BoundRecipeTaskRequestResult | ProgressionRecipeSelections['grader']) & { agentRequest?: UnknownRecord;
+ progressionAction?: ProgressionAction };
+type BenchArgs = BenchArguments & {
+ recipeTasks: Map;
+ recipeBindings: Map;
+ repairGrant?: RepairGrantResolution;
+ mutationImageId?: string;
+ spentBudgetUsd?: number;
+};
+type ProgressionWorkRecipeAction = ProgressionRecipeSelections & {
+ action: Exclude;
+};
+type FirstBuildRecord = {
+ score: number | null;
+ max: number | null;
+ regression: NonNullable['regression'] | null;
+ contractPass: boolean | null;
+ outcome: RunOutcome;
+ source: { sha256: string; files: number } | null;
+ missed: string[];
+ observations?: UnknownRecord;
+};
+type RepairStatus = 'not-needed' | 'corrected' | 'budget-exhausted' | 'incomplete' | 'ungraded';
+type ProgressionFailure = { kind?: string; reason?: string };
+
+const object = (value: unknown): value is UnknownRecord =>
+ value !== null && typeof value === 'object' && !Array.isArray(value);
+
+const errorMessage = (error: unknown): string =>
+ error instanceof Error ? error.message : String(error);
+
+function commandFailure(error: unknown): CommandFailure {
+ if (error instanceof Error) return error;
+ throw error;
+}
+
+function parseLeakAudit(value: string): LeakAuditEntry[] {
+ const parsed: unknown = JSON.parse(value);
+ if (!Array.isArray(parsed)) throw new Error('contamination audit output must be an array');
+ return parsed.map((entry, index) => {
+ if (!object(entry) || !Array.isArray(entry.hits)) {
+ throw new Error(`contamination audit output[${index}] is invalid`);
+ }
+ const hits = entry.hits.map((hit, hitIndex) => {
+ if (!object(hit) || typeof hit.kind !== 'string' || typeof hit.path !== 'string') {
+ throw new Error(`contamination audit output[${index}].hits[${hitIndex}] is invalid`);
+ }
+ return { kind: hit.kind, path: hit.path };
+ });
+ return { hits };
+ });
+}
+
+function stringArray(value: unknown, at: string): string[] {
+ if (!Array.isArray(value) || value.some(entry => typeof entry !== 'string')) {
+ throw new Error(`${at} must be an array of strings`);
+ }
+ return [...value];
+}
+
+function campaignSelection(value: unknown, at: string): CampaignSelection {
+ if (!object(value)) throw new Error(`${at} must be an object`);
+ const optionalStrings = (field: 'packs' | 'checks'): string[] | undefined => {
+ const entry = value[field];
+ if (entry === undefined) return undefined;
+ return stringArray(entry, `${at}.${field}`);
+ };
+ let levels: CampaignSelection['levels'];
+ if (value.levels !== undefined) {
+ if (!Array.isArray(value.levels)) throw new Error(`${at}.levels must be an array`);
+ levels = value.levels.map((entry, index) => {
+ if (!object(entry)) throw new Error(`${at}.levels[${index}] is invalid`);
+ const level = entry.level;
+ const recipe = entry.recipe;
+ if (typeof level !== 'number' || !Number.isSafeInteger(level) || typeof recipe !== 'string') {
+ throw new Error(`${at}.levels[${index}] is invalid`);
+ }
+ return { level, recipe,
+ ...(entry.features === undefined ? {} : { features: stringArray(entry.features,
+ `${at}.levels[${index}].features`) }),
+ ...(entry.checks === undefined ? {} : { checks: stringArray(entry.checks,
+ `${at}.levels[${index}].checks`) }) };
+ });
+ }
+ return { ...(optionalStrings('packs') === undefined ? {} : { packs: optionalStrings('packs') }),
+ ...(optionalStrings('checks') === undefined ? {} : { checks: optionalStrings('checks') }),
+ ...(levels === undefined ? {} : { levels }) };
+}
+
+function isProgressionWorkRecipeAction(value: ProgressionRecipeAction):
+ value is ProgressionWorkRecipeAction {
+ return value.action.type !== 'terminal';
+}
+
+function repairCheckKeys(value: ProgressionRecipeAction | null): string[] {
+ if (!value || !isProgressionWorkRecipeAction(value) || value.action.type !== 'repair') return [];
+ if (!object(value.action.prompt) || !Array.isArray(value.action.prompt.nodeIds)
+ || !object(value.action.grading) || !Array.isArray(value.action.grading.checks)) {
+ throw new Error('dependency repair action has invalid prompt or grading selections');
+ }
+ const promptNodeIds = new Set(value.action.prompt.nodeIds.map(nodeId => {
+ if (typeof nodeId !== 'string' || !nodeId) {
+ throw new Error('dependency repair action has an invalid prompt node');
+ }
+ return nodeId;
+ }));
+ const checks = value.action.grading.checks.flatMap(check => {
+ if (!object(check) || typeof check.id !== 'string' || !check.id
+ || typeof check.nodeId !== 'string' || !check.nodeId) {
+ throw new Error('dependency repair action has an invalid grading check');
+ }
+ return promptNodeIds.has(check.nodeId) ? [check.id] : [];
+ });
+ if (checks.length === 0) throw new Error('dependency repair action selects no repair checks');
+ return checks;
+}
+
+function repairReportArgs(value: ProgressionRecipeAction | null): string[] {
+ const checks = repairCheckKeys(value);
+ if (!value || !isProgressionWorkRecipeAction(value) || checks.length === 0) return [];
+ const interfaces = contractInterfaceNames(value.agent.task.contractText);
+ return ['--checks-json', JSON.stringify(checks),
+ '--controls-json', JSON.stringify(interfaces)];
+}
+
+function repairOwnerNodeIds(value: ProgressionRecipeAction | null): string[] {
+ if (!value || !isProgressionWorkRecipeAction(value) || value.action.type !== 'repair') return [];
+ return [...value.action.repair.nodeIds].sort();
+}
+
+function savedRepairRegression(state: ProgressionState | null,
+ selected: ProgressionRecipeAction | null): ProgressionRepairRegression | null {
+ const saved = state?.attempts.at(-1)?.repairRegression;
+ if (!saved) return null;
+ const owners = repairOwnerNodeIds(selected);
+ return JSON.stringify([...saved.ownerNodeIds].sort()) === JSON.stringify(owners)
+ ? structuredClone(saved) : null;
+}
+
+function requireProgressionState(state: ProgressionState | null): ProgressionState {
+ if (!state) throw new Error('live dependency progression has no active state');
+ return state;
+}
+
+function requireContinuation(run: BenchmarkRunRecord): RunContinuation {
+ if (!run.continuation) throw new Error('repair continuation has no continuation record');
+ return run.continuation;
+}
+
+function requireRunTotals(run: BenchmarkRunRecord): RunTotals {
+ if (!run.totals) throw new Error('benchmark run totals are not available');
+ return run.totals;
+}
+
+function repairOutcome(outcome: RunOutcome): RepairOutcome {
+ return { kind: outcome.kind, appFailures: [...(outcome.appFailures ?? [])],
+ inconclusive: [...(outcome.inconclusive ?? [])],
+ harnessFailures: [...(outcome.harnessFailures ?? [])] };
+}
+
+function progressionFailure(outcome: RunOutcome): ProgressionFailure {
+ return { kind: outcome.kind, ...(outcome.reason === null || outcome.reason === undefined
+ ? {} : { reason: outcome.reason }) };
+}
+
+function featureCheckKeys(selected: ProgressionWorkRecipeAction,
+ state: ProgressionState): string[] {
+ if (!object(selected.action.prompt) || !Array.isArray(selected.action.prompt.nodeIds)) {
+ throw new Error('feature action has no prompt nodes');
+ }
+ const selectedNodes = new Set(selected.action.prompt.nodeIds);
+ return state.definition.nodes
+ .filter(node => selectedNodes.has(node.id))
+ .flatMap(node => node.gradingChecks
+ .filter(check => check.role === 'feature')
+ .map(check => check.id));
+}
+
+function bundlePassedChecks(bundle: GradeBundlePayload | null): Set {
+ return new Set(Object.values(bundle?.suites ?? {}).flatMap(suite =>
+ (suite?.features ?? []).flatMap(feature =>
+ (feature.criteria ?? []).flatMap(criterion =>
+ typeof criterion.stableKey === 'string'
+ && evidencePassed(criterionEvidence(criterion)) ? [criterion.stableKey] : []))));
+}
+
+function featureCandidateAccepted(selected: ProgressionWorkRecipeAction,
+ state: ProgressionState, candidate: GradeBundlePayload | null): boolean {
+ if (!levelGradeIsUsable(classifyBundle(candidate))) return false;
+ const passed = bundlePassedChecks(candidate);
+ if (!featureCheckKeys(selected, state).every(check => passed.has(check))) return false;
+ return state.definition.nodes.every(node => node.gradingChecks.every(check =>
+ state.nodes[node.id]?.checks[check.id] !== 'pass' || passed.has(check.id)));
+}
+
+function featureActionNeedsCoding(selected: ProgressionWorkRecipeAction,
+ state: ProgressionState): boolean {
+ if (selected.action.type === 'repair') return true;
+ if (!object(selected.action.prompt) || !Array.isArray(selected.action.prompt.nodeIds)) {
+ throw new Error('feature action has no prompt nodes');
+ }
+ return selected.action.prompt.nodeIds.some(nodeId => {
+ const node = state.nodes[String(nodeId)];
+ return node?.status === 'active'
+ && Object.values(node.checks).every(outcome => outcome === null);
+ });
+}
+
+function mergeFeatureLevelRecord(previous: RunLevelRecord | null,
+ current: RunLevelRecord): RunLevelRecord {
+ if (!previous) return current;
+ const buildSessions: RunSessionRecord[] = [
+ ...(previous.buildSessions ?? []),
+ ...(current.buildSessions ?? []),
+ ];
+ const repairSessions = [...(previous.repairSessions ?? []), ...(current.repairSessions ?? [])];
+ const sessionTotals = summarizeSessions([...buildSessions, ...repairSessions]);
+ const repairs = (previous.repairs ?? 0) + (current.repairs ?? 0);
+ const repair = current.repair ? {
+ ...current.repair,
+ limit: Math.max(previous.repair?.limit ?? 0,
+ (previous.repairs ?? 0) + current.repair.limit),
+ used: repairs,
+ } : previous.repair;
+ const merged: RunLevelRecord = {
+ ...previous,
+ ...current,
+ buildSessions,
+ buildCostUsd: addCostUsd(previous.buildCostUsd, current.buildCostUsd),
+ repairSessions,
+ repairCostUsd: addCostUsd(previous.repairCostUsd, current.repairCostUsd),
+ repairHistory: [...(previous.repairHistory ?? []), ...(current.repairHistory ?? [])],
+ repairs,
+ repair,
+ sessionTotals,
+ tokens: sessionTotals.tokens,
+ usage: sessionTotals.usage,
+ turns: sessionTotals.turns,
+ promptBytes: sessionTotals.promptBytes,
+ tokensPerTurn: sessionTotals.turns
+ ? Math.round(sessionTotals.tokens / sessionTotals.turns) : null,
+ thinking: sessionTotals.thinking,
+ costUsd: addCostUsd(previous.costUsd, current.costUsd),
+ durationSec: (previous.durationSec ?? 0) + (current.durationSec ?? 0),
+ };
+ return merged;
+}
+
+function mutationControlArgs(args: BenchArgs): MutationControlArgs {
+ if (!args.out || !args.mutations || !args.backend || !args.parentAttemptId) {
+ throw new Error('mutation control has incomplete run identity');
+ }
+ return { levelList: args.levelList, out: args.out, recipe: args.recipe,
+ recipeTasks: args.recipeTasks, mutations: args.mutations, backend: args.backend,
+ track: args.track, runIndex: args.runIndex, parentAttemptId: args.parentAttemptId,
+ mutationShardIndex: args.mutationShardIndex, mutationShardCount: args.mutationShardCount,
+ mutationResumeFrom: args.mutationResumeFrom, mutationCheckpointOut: args.mutationCheckpointOut,
+ mutationBaselineBundle: args.mutationBaselineBundle,
+ expectedMutationCalibration: args.expectedMutationCalibration,
+ mutationMaxRuntimeMinutes: args.mutationMaxRuntimeMinutes,
+ mutationImageId: args.mutationImageId };
+}
+
+function mutationOutcome(value: unknown): RunOutcome | null {
+ if (value === null || value === undefined) return null;
+ if (!object(value)) {
+ throw new Error('mutation control artifact outcome is invalid');
+ }
+ return { kind: runOutcomeKind(value.kind),
+ ...(typeof value.phase === 'string' ? { phase: value.phase } : {}),
+ ...(typeof value.reason === 'string' ? { reason: value.reason } : {}) };
+}
+
+function recipeRequestIdentity(value: unknown): { recipeSha256: string; selectionSha256: string;
+ taskPacks: unknown; taskSha256: string } {
+ if (!object(value) || !object(value.recipe) || !object(value.selection) || !object(value.task)
+ || typeof value.recipe.contentSha256 !== 'string' || typeof value.selection.sha256 !== 'string'
+ || typeof value.task.sha256 !== 'string') {
+ throw new Error('recipe task request has no complete identity');
+ }
+ return { recipeSha256: value.recipe.contentSha256, selectionSha256: value.selection.sha256,
+ taskPacks: value.selection.taskPacks, taskSha256: value.task.sha256 };
+}
+
+function snapshotSource(appDir: string, to: string): void {
+ snapshotAppSource(appDir, to);
+}
+
+// The ports a build may legitimately reach: its own web, database, and
+// SpacetimeDB listeners. Every other local port belongs to another run, the
+// controller, or the dashboard.
+function runOwnPorts(track: Parameters[0],
+ args: { backend: string; runIndex: number }): number[] {
+ const ports = portsFor(track, args.backend, args.runIndex);
+ const stdb = process.env.STACK_BENCH_STDB_URI
+ ? Number(new URL(process.env.STACK_BENCH_STDB_URI).port) : null;
+ return [ports.vite, ports.express, ports.dbPort, stdb].filter((port): port is number =>
+ typeof port === 'number' && Number.isInteger(port) && port > 0);
+}
+
+// Check contamination after every coding session. File-tool permissions do not
+// govern shell reads, so the transcript audit remains a separate hard gate.
+function auditContamination(appDir: string, ownPorts: readonly number[],
+ expectTranscripts: boolean): ContaminationAudit | null {
+ // A non-billable adapter runs no provider session and leaves no transcript;
+ // there is nothing to audit and nothing that could have been read.
+ if (!expectTranscripts) return null;
+ const args = [join(ROOT, 'dist', 'commands', 'leak-audit.js'), '--app', appDir, '--json',
+ '--own-ports', ownPorts.join(',')];
+ let firstFailure: unknown = null;
+ for (let attempt = 1; attempt <= 2; attempt++) {
+ try {
+ const audit = sh('node', args, { stdio: 'pipe' });
+ const entries = parseLeakAudit(audit);
+ if (entries.length === 0) {
+ return { kind: 'harness_failure',
+ evidence: ['no session transcript was found to audit'],
+ verdict: 'SCORES NOT USABLE — nothing verified this build stayed inside its directory.' };
+ }
+ const escapes = entries.flatMap(entry => entry.hits);
+ const serious = escapes.filter(h => /GRADER|CONTRACT|BENCHMARK NOTES|PROMPTS|NETWORK/.test(h.kind));
+ if (firstFailure) {
+ console.error(` warning: contamination audit passed on retry after: ${auditFailureSummary(firstFailure)}`);
+ }
+ if (!serious.length) return null;
+ return { kind: 'contaminated',
+ evidence: [...new Set(serious.map(h => `${h.kind}: ${h.path.split('/').slice(-2).join('/')}`))].slice(0, 8),
+ verdict: 'SCORES NOT USABLE — the build read the harness that grades it.' };
+ } catch (error) {
+ firstFailure ??= error;
+ if (attempt === 2) {
+ // An audit that could not run is not a pass. Keep the process details so
+ // the failure can be repaired without another paid reproduction.
+ return { kind: 'harness_failure',
+ evidence: [`audit did not run after retry: ${auditFailureSummary(error)}`],
+ verdict: 'SCORES NOT USABLE — nothing verified this build stayed inside its directory.' };
+ }
+ }
+ }
+ return null;
+}
+
+export function auditFailureSummary(error: unknown): string {
+ const failure = object(error) ? error : {};
+ const message = errorMessage(error).split(/\r?\n/)[0] ?? '';
+ const stderrLines = String(failure.stderr ?? '').trim().split(/\r?\n/).filter(Boolean);
+ const stderr = stderrLines.find(line => /(?:error|eacces|permission denied|failed)/i.test(line))
+ ?? stderrLines[0];
+ const details = [
+ Number.isInteger(failure.status) ? `exit ${String(failure.status)}` : null,
+ failure.signal ? `signal ${String(failure.signal)}` : null,
+ stderr ? `stderr: ${stderr}` : null,
+ ].filter(Boolean);
+ return details.length ? `${message} (${details.join('; ')})` : message;
+}
+
+const sh = (cmd: string, args: readonly string[],
+ opts: Omit = {}): string =>
+ execFileSync(cmd, [...args], {
+ encoding: 'utf8', maxBuffer: 64 * 1024 * 1024, timeout: COMMAND_TIMEOUT_MS, ...opts,
+ });
+
+let activeAgentChild: ChildProcess | null = null;
+// Set once a run owns resources. The top-level rejection handler invokes this
+// directly; relying only on process 'exit' made cleanup best-effort precisely
+// when an awaited build rejected unexpectedly.
+let emergencyTeardown: (() => void) | null = null;
+
+export function parseAgentProcessResult(stdout: string, stderr: string, processError: unknown,
+ request: AgentRequest): ValidatedAgentResult {
+ const resultLine = stdout.trim().split('\n').pop();
+ let result: ValidatedAgentResult;
+ try {
+ if (!resultLine) throw new Error('agent returned no result line');
+ result = validateAgentResult(JSON.parse(resultLine), request);
+ } catch (resultError) {
+ const stdoutTail = stdout.trim().slice(-2000) || '';
+ const stderrTail = stderr.trim().slice(-4000) || '';
+ const processDetail = processError ? `agent process failed: ${errorMessage(processError)}\n` : '';
+ throw new Error(`${processDetail}agent returned an invalid result: ${errorMessage(resultError)}\n`
+ + `agent stdout tail:\n${stdoutTail}\nagent stderr tail:\n${stderrTail}`);
+ }
+ if (processError && result.ok) {
+ throw new Error(`agent process failed after reporting success: ${errorMessage(processError)}`);
+ }
+ return result;
+}
+
+function runAgent(
+ args: BenchArgs,
+ adapter: AgentAdapter,
+ mode: AgentMode,
+ level: number,
+ appDir: string,
+): Promise {
+ if (!args.backend || !args.model) throw new Error('agent run requires backend and model');
+ const remainingBudget = args.maxBudgetUsd == null ? null
+ : addCostUsd(args.maxBudgetUsd, -(args.spentBudgetUsd ?? 0));
+ if (remainingBudget !== null && remainingBudget <= 0) {
+ throw new Error(`attempt cost cap of $${args.maxBudgetUsd} was exhausted before ${mode} L${level}`);
+ }
+ if (remainingBudget !== null && adapter.costLimit === 'unsupported') {
+ throw new Error(`agent adapter ${adapter.id} cannot enforce --max-budget-usd`);
+ }
+ const recipeTask = args.recipeTasks?.get(level)?.agentRequest
+ ?? args.recipeTasks?.get(level)?.request ?? null;
+ const request: AgentRequest = { mode, level, app: appDir, backend: args.backend, track: args.track,
+ runIndex: args.runIndex, model: args.model, guidance: args.guidance, skills: args.skills,
+ ...(adapter.usesStackSkills
+ ? { skillIdentity: args.condition?.guidance.skills[args.backend] } : {}),
+ recipe: agentRecipeIdentity(args.recipe, recipeTask),
+ guidanceDocument: args.guidanceDocument,
+ credentialAliases: args.condition?.guidance?.credentialAliases ?? {},
+ recipeTask, pricing: args.pricing,
+ maxBudgetUsd: remainingBudget, adapterCostLimit: adapter.costLimit };
+ const argv = agentRequestArgv(adapter, request);
+ if (args.apiKey && !adapter.apiKeyEnvironmentVariable) {
+ throw new Error(`agent adapter ${adapter.id} does not accept an API key`);
+ }
+ const env = { ...process.env };
+ if (args.apiKey) {
+ const credentialName = adapter.apiKeyEnvironmentVariable;
+ if (!credentialName) throw new Error(`agent adapter ${adapter.id} does not accept an API key`);
+ env[credentialName] = args.apiKey;
+ }
+ return new Promise((resolveRun, rejectRun) => {
+ const child = execFile('node', argv, {
+ encoding: 'utf8', maxBuffer: 64 * 1024 * 1024, timeout: adapter.deadlineMs,
+ env,
+ },
+ (error, stdout, stderr) => {
+ if (activeAgentChild === child) activeAgentChild = null;
+ try {
+ const result = parseAgentProcessResult(stdout, stderr, error, request);
+ args.spentBudgetUsd = addCostUsd(args.spentBudgetUsd, result.costUsd);
+ resolveRun(result);
+ }
+ catch (parseError) {
+ rejectRun(parseError);
+ }
+ });
+ activeAgentChild = child;
+ });
+}
+
+interface GradeCheck {
+ stableKey: string;
+ executionId?: string;
+ source?: string;
+}
+
+interface GradeRecipeTask {
+ request: UnknownRecord;
+ selection: { checks: readonly GradeCheck[] }
+ | { scoredChecks: readonly GradeCheck[]; observedChecks?: readonly GradeCheck[] };
+}
+
+function checksForGrade(task: GradeRecipeTask | undefined, observation: GradeOptions['observation']):
+ readonly GradeCheck[] {
+ if (!task) return [];
+ if ('scoredChecks' in task.selection) {
+ return observation === 'observed'
+ ? task.selection.observedChecks ?? [] : task.selection.scoredChecks;
+ }
+ return task.selection.checks;
+}
+
+type GradeArguments = Pick & {
+ recipeTasks?: ReadonlyMap;
+ progression?: { identity: { policy?: string } };
+ condition?: { guidance?: { credentialAliases?: Record } };
+};
+
+export function gradeArgv(
+ args: GradeArguments,
+ appDir: string,
+ url: string,
+ label: string,
+ level: number,
+ track: Track,
+ parentAttemptId: string,
+ options: GradeOptions = {},
+): string[] {
+ const { observation = 'scored', out = null, sourceSha256 = null,
+ applicationFailure = null } = options;
+ if (!args.backend) throw new Error('grading requires a backend');
+ const restartSpec = restartSpecFor(args, appDir, track);
+ const task = options.recipeTask ?? args.recipeTasks?.get(level);
+ return [compiledEntrypoint('commands', 'run-suite.js'), '--app', appDir, '--url', url,
+ '--backend', args.backend, '--label', label, '--level', String(level),
+ '--track', args.track,
+ '--run-index', String(args.runIndex),
+ '--parent-attempt-id', parentAttemptId,
+ '--observation', observation,
+ ...(out ? ['--out', out] : []),
+ ...(sourceSha256 ? ['--source-sha256', sourceSha256] : []),
+ ...(args.recipe ? ['--recipe', args.recipe] : []),
+ ...(task ? ['--recipe-task-json', JSON.stringify(task.request)] : []),
+ ...(args.condition?.guidance?.credentialAliases
+ ? ['--credential-aliases-json', JSON.stringify(
+ args.condition.guidance.credentialAliases)] : []),
+ ...(applicationFailure
+ ? ['--application-failure-json', JSON.stringify(applicationFailure)] : []),
+ ...(observation === 'scored' && args.recipeTasks && !args.progression
+ ? ['--regression-checks-json', JSON.stringify([...args.recipeTasks.entries()]
+ .filter(([priorLevel]) => priorLevel < level)
+ .flatMap(([, priorTask]) => checksForGrade(priorTask, 'scored')
+ .map(check => check.stableKey)))] : []),
+ ...(args.media && observation === 'scored' ? [] : ['--no-media']),
+ ...(!STACK_ADAPTER_REGISTRY.get(args.backend).runPolicy.resetEnabled
+ ? ['--no-reset']
+ : ['--restart-spec', JSON.stringify(restartSpec)])];
+}
+
+function grade(
+ args: BenchArgs,
+ appDir: string,
+ url: string,
+ label: string,
+ level: number,
+ track: Track,
+ parentAttemptId: string,
+ options: GradeOptions = {},
+): GradeBundlePayload | null {
+ const { out = null } = options;
+ const source = hashAppSource(appDir);
+ const argv = gradeArgv(args, appDir, url, label, level, track, parentAttemptId, {
+ ...options, sourceSha256: options.sourceSha256 ?? source.sha256,
+ });
+ const bundle = join(out ?? join(appDir, 'stack-bench'), ARTIFACT_FILE.gradeBundle);
+ rmSync(bundle, { force: true });
+ const task = options.recipeTask ?? args.recipeTasks?.get(level);
+ const currentChecks = checksForGrade(task, options.observation);
+ const regressionChecks = options.observation === 'observed' || args.progression
+ ? []
+ : [...(args.recipeTasks?.entries() ?? [])]
+ .filter(([priorLevel]) => priorLevel < level)
+ .flatMap(([, priorTask]) => checksForGrade(priorTask, 'scored'));
+ const sourceCount = task
+ ? selectedGradingSourceCount(currentChecks, regressionChecks)
+ : suitesFor(track, level).length;
+ try {
+ sh('node', argv, { stdio: 'inherit', timeout: gradingRunTimeoutMs(sourceCount) });
+ } catch { /* a current bundle may still explain a scored failure */ }
+ return existsSync(bundle)
+ ? readArtifactPayload(bundle, { expectedKind: 'grade_bundle' }) : null;
+}
+
+function restartSpecFor(args: Pick,
+ appDir: string, track: Track): RuntimeControlSpec {
+ if (!args.backend) throw new Error('restart specification requires a backend');
+ const port = portsFor(track, args.backend, args.runIndex).vite ?? null;
+ if (port == null) throw new Error(`stack ${args.backend} has no application port`);
+ return { backend: args.backend, app: appDir, port: Number(port), probe: '' };
+}
+
+function runMutationControl(
+ args: BenchArgs,
+ appDir: string,
+ url: string,
+ track: Track,
+ imageId: string | null,
+): MutationControlResult {
+ if (!args.out || !args.mutations) throw new Error('mutation control requires output and manifest paths');
+ const output = join(args.out, ARTIFACT_FILE.mutationControl);
+ if (!args.mutationResumeFrom || resolve(args.mutationResumeFrom) !== resolve(output)) {
+ rmSync(output, { force: true });
+ }
+ if (imageId) args.mutationImageId = imageId;
+ else delete args.mutationImageId;
+ const argv = mutationControlArgv(mutationControlArgs(args), appDir, url, track);
+ let processError = null;
+ try { sh(process.execPath, argv, {
+ stdio: 'inherit', timeout: mutationControlTimeoutMs(args.mutationMaxRuntimeMinutes),
+ }); }
+ catch (error) { processError = errorMessage(error).split('\n')[0] ?? null; }
+ if (!existsSync(output)) {
+ return { ok: false, artifact: output, processError,
+ outcome: { kind: 'harness_failure', phase: 'mutation-control',
+ reason: processError ?? 'mutation runner produced no artifact' } };
+ }
+ const artifact = readArtifactPayload(output, { expectedKind: 'mutation_control' });
+ return { ok: artifact.ok === true && !processError, artifact: output,
+ processError, summary: artifact.summary ?? null, outcome: mutationOutcome(artifact.outcome),
+ results: artifact.results ?? [] };
+}
+
+function validateMutationInput(args: BenchArgs): void {
+ if (!args.mutations) return;
+ if (!args.app) throw new Error('--mutations requires an explicit pristine --app');
+ const manifest = JSON.parse(readFileSync(args.mutations, 'utf8'));
+ if (!/^[a-f0-9]{64}$/.test(manifest.fixtureSha256 ?? '')) {
+ throw new Error('mutation manifest has no valid fixtureSha256');
+ }
+ const fixture = hashDirectory(args.app);
+ if (fixture.sha256 !== manifest.fixtureSha256) {
+ throw new Error(`mutation manifest targets fixture ${manifest.fixtureSha256}, not ${fixture.sha256}`);
+ }
+}
+
+async function main() {
+ const args: BenchArgs = {
+ ...parseBenchArguments(process.argv),
+ recipeTasks: new Map(),
+ recipeBindings: new Map(),
+ };
+ let repairGrant = null;
+ if (args.repairFrom) {
+ const repairLevel = args.repairLevel;
+ if (typeof repairLevel !== 'number' || !Number.isSafeInteger(repairLevel) || repairLevel < 1) {
+ throw new Error('--repair-from requires a positive --repair-level');
+ }
+ repairGrant = createRepairGrant(args.repairFrom,
+ { level: repairLevel, repairs: args.repairs });
+ const config = repairGrant.configuration;
+ if (config.buildImage && process.env.STACK_BENCH_IMAGE
+ && config.buildImage !== process.env.STACK_BENCH_IMAGE) {
+ throw new Error('repair continuation build image differs from its parent run');
+ }
+ if (config.buildImage) process.env.STACK_BENCH_IMAGE = config.buildImage;
+ Object.assign(args, {
+ backend: config.backend,
+ track: config.track,
+ recipe: config.recipe,
+ levels: String(config.level),
+ levelList: [config.level],
+ runIndex: config.runIndex,
+ agentAdapter: config.agentAdapter,
+ model: config.model,
+ guidance: config.guidance,
+ guidanceDocument: config.guidanceDocument,
+ condition: config.condition,
+ selectionRequest: campaignSelection(config.selectionRequest, 'repair configuration.selectionRequest'),
+ skills: config.skills,
+ packIds: [...(campaignSelection(config.selectionRequest,
+ 'repair configuration.selectionRequest').packs ?? [])],
+ checkKeys: [...(campaignSelection(config.selectionRequest,
+ 'repair configuration.selectionRequest').checks ?? [])],
+ featureIds: [],
+ requestedSpecifications: [],
+ expectedSpecifications: [],
+ observedSpecifications: [],
+ seedFrom: repairGrant.sourcePath,
+ url: config.url,
+ parentAttemptId: repairGrant.parent.id,
+ repairGrant,
+ });
+ }
+ if (!args.backend) throw new Error('benchmark run requires a backend');
+ const stackAdapter = STACK_ADAPTER_REGISTRY.get(args.backend);
+ const materializeCodingOutput = stackAdapter.id !== 'stub';
+ const agentAdapter = AGENT_ADAPTER_REGISTRY.get(args.agentAdapter);
+ if (process.env.STACK_BENCH_APPLIANCE !== '1' && agentAdapter.costLimit !== 'non-billable') {
+ throw new Error(`agent adapter ${agentAdapter.id} requires the Docker appliance`);
+ }
+ if (repairGrant) {
+ const currentAgent = agentAdapterIdentity(agentAdapter);
+ const parentAgent = repairGrant.parentArtifact.identities.agentAdapter;
+ if (currentAgent.id !== parentAgent?.id || currentAgent.version !== parentAgent?.version
+ || currentAgent.sha256 !== parentAgent?.sha256) {
+ throw new Error('repair continuation agent adapter differs from its parent run');
+ }
+ if (stackAdapter.id !== repairGrant.parentArtifact.identities.stackAdapter?.id
+ || stackAdapter.version !== repairGrant.parentArtifact.identities.stackAdapter?.version) {
+ throw new Error('repair continuation stack adapter differs from its parent run');
+ }
+ }
+ applyAgentCredential(args, agentAdapter);
+ args.model ??= agentAdapter.defaultModel;
+ if (!args.model) throw new Error(`agent adapter ${agentAdapter.id} has no default model`);
+ if (args.pricing !== undefined) {
+ args.pricing = validatePricingAuthority(args.pricing, { at: '--pricing-json' });
+ } else if (args.maxBudgetUsd != null && agentAdapter.costLimit === 'native') {
+ const rates = claudeRatesForModel(args.model);
+ if (!rates) throw new Error(`no default pricing is recorded for model ${args.model}`);
+ args.pricing = validatePricingAuthority({ unit: PRICING_UNIT, rates },
+ { at: 'default pricing' });
+ } else {
+ args.pricing = null;
+ }
+ if (args.retainBackend && !stackAdapter.runPolicy.retainHostSupported) {
+ throw new Error(`stack adapter ${args.backend} does not support --retain-backend`);
+ }
+ const stackRuntime = stackAdapter.orchestrator.config(
+ { root: ROOT, env: process.env, helpers: { exists: existsSync } });
+ Object.assign(process.env, stackRuntime.environment);
+ process.env.STACK_BENCH_NODE_BIN = process.platform === 'win32' ? 'node.exe' : process.execPath;
+ const track = loadTrack(args.track);
+ const ownPorts = runOwnPorts(track, { backend: stackAdapter.id, runIndex: args.runIndex });
+ const auditsTranscripts = agentAdapter.costLimit !== 'non-billable';
+ // Resolve the requested scope for every level before probing the sandbox,
+ // acquiring a backend lease or paying for a build. A pack that exists at L2
+ // but not L1 is not a late grading surprise; it is an invalid run request.
+ args.selectionRequest ??= { packs: [...args.packIds], checks: [...args.checkKeys] };
+ for (const level of args.levelList) {
+ const declared = args.condition?.requested?.levels?.find(entry => entry.level === level) ?? null;
+ const modularSelection = args.selectionRequest.levels?.find(entry => entry.level === level) ?? null;
+ if (declared?.selection?.schemaVersion === 3) {
+ const expected = args.featureCatalog
+ ? { level, recipe: `${declared.recipe.id}@${declared.recipe.version}` }
+ : { level, recipe: `${declared.recipe.id}@${declared.recipe.version}`,
+ features: declared.selection.requested.features,
+ checks: declared.selection.requested.checks };
+ if (canonicalDefinitionJson(modularSelection) !== canonicalDefinitionJson(expected)) {
+ throw new Error(`campaign selection changed before L${level}`);
+ }
+ } else if (modularSelection) {
+ throw new Error(`campaign selection declares modular L${level} without a modular condition`);
+ }
+ const declaredRecipe = declared
+ ? `${declared.recipe.id}@${declared.recipe.version}` : null;
+ const binding = resolveRecipeRelease(track, level, declaredRecipe ?? args.recipe);
+ if (!binding && (args.packIds.length || args.checkKeys.length)) {
+ throw new Error(`L${level} has no recipe release, so --pack/--check cannot be resolved`);
+ }
+ if (binding) {
+ args.recipeBindings.set(level, binding);
+ if (args.featureCatalog) {
+ validateProgressionCampaignLevelScope(binding, args.featureCatalog, declared, level);
+ }
+ const requested = declared?.selection?.requested;
+ const progressionSelection = args.featureCatalog
+ ? resolveProgressionRecipeLevelSelection(binding, args.featureCatalog, level,
+ { cumulative: Boolean(args.progression) }) : null;
+ const resolved = progressionSelection === null
+ ? createBoundRecipeTaskRequest(binding, requested?.features
+ ? { featureIds: requested.features,
+ requestedSpecifications: requested.specifications?.requested,
+ expectedSpecifications: requested.specifications?.expected,
+ observedSpecifications: requested.specifications?.observed,
+ checkKeys: requested.checks }
+ : args) : null;
+ const grader = progressionSelection?.grader ?? resolved;
+ if (!grader) throw new Error(`L${level} has no recipe task request`);
+ if (args.condition && !declared) {
+ throw new Error(`study condition does not bind requested L${level}`);
+ }
+ const graderIdentity = recipeRequestIdentity(grader.request);
+ if (declared && (declared.recipe.contentSha256 !== graderIdentity.recipeSha256
+ || declared.selection.sha256 !== graderIdentity.selectionSha256
+ || JSON.stringify(declared.selection.taskPacks) !== JSON.stringify(graderIdentity.taskPacks)
+ || declared.task.sha256 !== graderIdentity.taskSha256)) {
+ throw new Error(`study condition requested scope changed before L${level}`);
+ }
+ if (progressionSelection) {
+ const progressionGrader = progressionSelection.grader;
+ args.recipeTasks.set(level, {
+ request: progressionGrader.request,
+ selection: progressionGrader.selection,
+ task: progressionGrader.task,
+ agentRequest: progressionSelection.agent.request,
+ });
+ } else if (resolved) {
+ args.recipeTasks.set(level, {
+ ...resolved,
+ agentRequest: createAgentVisibleTaskRequest(binding, resolved),
+ });
+ }
+ }
+ }
+ if (args.progression) {
+ const state = progressionEngine.initialize(args.progression.definition);
+ const declared = args.condition?.requested?.levels
+ ?.find(entry => entry.level === state.level) ?? null;
+ const binding = resolveRecipeRelease(track, state.level,
+ declared ? `${declared.recipe.id}@${declared.recipe.version}` : null);
+ if (!binding) throw new Error(`L${state.level} has no recipe release`);
+ resolveProgressionRecipeAction(binding, state);
+ if (!args.progressionOwner) {
+ throw new Error('live dependency progression requires an exact compiled campaign attempt');
+ }
+ }
+ if (repairGrant) {
+ const expectedSelection = repairGrant.level.selection?.sha256 ?? null;
+ const repairTask = args.recipeTasks.get(repairGrant.level.level);
+ const resolvedSelection = repairTask ? recipeRequestIdentity(repairTask.request).selectionSha256 : null;
+ if (resolvedSelection !== expectedSelection) {
+ throw new Error('repair continuation test selection differs from its parent run');
+ }
+ }
+ if (!args.selectionRequest.levels && (JSON.stringify(args.selectionRequest.packs) !== JSON.stringify(args.packIds)
+ || JSON.stringify(args.selectionRequest.checks) !== JSON.stringify(args.checkKeys))) {
+ throw new Error('campaign pack/check selection changed before execution');
+ }
+ // Caller-owned mutation inputs are pure request data. Reject them before
+ // checking credentials, Docker, ports, or any other ambient runner state so
+ // an invalid experiment can never be masked by an unrelated preflight error.
+ validateMutationInput(args);
+ assertNoPortCollisions();
+ // The deterministic adapter/stack is the model-free unit loop. Real runs
+ // prove the exact requested scope, engine, image, credentials, storage and
+ // ports before any paid coding session begins.
+ const admittedSmoke = args.campaignAdmission?.reusable === true
+ ? { id: args.campaignAdmission.id, createdAt: args.campaignAdmission.createdAt }
+ : null;
+ const preflight = args.backend === 'stub' ? null : runPreflight({
+ backends: [args.backend], track: args.track, levels: args.levels,
+ levelList: args.levelList, runIndex: args.runIndex, agentAdapter: args.agentAdapter,
+ guidance: args.guidance,
+ recipe: args.recipe,
+ ...(args.condition?.requested ? { requestedScopes: [args.condition.requested] } : {}),
+ ...(args.featureCatalog ? { featureCatalog: args.featureCatalog } : {}),
+ ...(args.runMode ? { mode: args.runMode } : {}),
+ agentSkills: args.skills ?? null,
+ packIds: args.packIds, checkKeys: args.checkKeys, smoke: admittedSmoke === null,
+ ...(admittedSmoke ? { admittedSmoke } : {}),
+ ...(process.env.STACK_BENCH_SUPERVISOR_STATE
+ ? { supervisorState: process.env.STACK_BENCH_SUPERVISOR_STATE } : {}),
+ image: process.env.STACK_BENCH_IMAGE ?? DEFAULT_BUILD_IMAGE,
+ resultsDir: resolve(args.out ?? stackBenchResultsRoot(ROOT)),
+ }, { env: args.apiKey && agentAdapter.apiKeyEnvironmentVariable
+ ? { ...process.env, [agentAdapter.apiKeyEnvironmentVariable]: '' }
+ : process.env });
+ if (preflight && !preflight.ok) {
+ const failures = preflight.checks.filter(check => check.status === 'fail');
+ console.error('\nPREFLIGHT FAILED — no model session was started.');
+ for (const failure of failures) {
+ console.error(` ${failure.id}: ${failure.summary}`);
+ if (failure.remediation) console.error(` fix: ${failure.remediation}`);
+ }
+ process.exit(2);
+ }
+ if (preflight) console.log(` preflight ... ${preflight.summary.passed} checks passed`
+ + `${preflight.summary.warnings ? `, ${preflight.summary.warnings} warning(s)` : ''}`);
+ if (process.env.STACK_BENCH_APPLIANCE === '1') {
+ console.log(' sandbox ... coding container is isolated from the controller and grading files');
+ }
+ let url = args.url ?? `http://localhost:${portsFor(track, args.backend, args.runIndex).vite}`;
+ const runDir = resultsName(track, args.backend, args.runIndex);
+ const runId = newRunId({ track: args.track, backend: args.backend, runIndex: args.runIndex });
+ const artifactLabel = `${runDir}-${runId}`;
+ // Default results never reuse a directory. The stable backend/run name is a
+ // grouping directory only; every artifact beneath it belongs to one run id.
+ args.out ??= join(stackBenchResultsRoot(ROOT), runDir, runId);
+ if (!args.out) throw new Error('benchmark run has no results directory');
+ const outputDir = args.out;
+ mkdirSync(args.out, { recursive: true });
+ if (existsSync(join(args.out, ARTIFACT_FILE.run))) {
+ throw new Error(`refusing to reuse result directory containing ${ARTIFACT_FILE.run}: ${args.out}`);
+ }
+ if (preflight) writeArtifact(join(args.out, ARTIFACT_FILE.preflight), {
+ kind: 'preflight', id: `${runId}-preflight`,
+ attempt: { id: `${runId}-preflight`, parentId: runId },
+ identities: emptyArtifactIdentities({
+ agentAdapter: agentAdapterIdentity(agentAdapter),
+ stackAdapter: { id: stackAdapter.id, version: stackAdapter.version },
+ }),
+ payload: preflight,
+ });
+
+ // Validate caller-owned source before acquiring a backend slot so a bad
+ // fixture cannot leave leased resources behind.
+ const ownWorkDir = !args.app;
+ const appDir = args.app ?? join(workDirFor(track, args.backend, args.runIndex, runId), 'app');
+ if (args.repairGrant && url.startsWith('file:')) {
+ url = pathToFileURL(join(appDir, 'index.html')).href;
+ }
+
+ // Bind destructive and lifecycle operations to exact resource identities and
+ // an ownership token. Targets come only from the lease, never generated code.
+ const runtimeRoot = resolve(process.env.STACK_BENCH_RUNTIME_DIR
+ ?? join(tmpdir(), 'stack-bench-runtime'));
+ const runtimeDir = join(runtimeRoot, runId);
+ const leasePath = join(runtimeDir, ARTIFACT_FILE.backendLease);
+ const preparedLease = stackAdapter.lease.prepare({
+ track,
+ runIndex: args.runIndex,
+ runtimeDir,
+ serverUri: stackRuntime.lease.serverUri,
+ env: process.env,
+ helpers: { containerIdentity: runningContainerIdentity, dbName, moduleName },
+ });
+ const initialLease = createBackendLease({
+ runId,
+ backend: args.backend,
+ track: args.track,
+ runIndex: args.runIndex,
+ ...preparedLease.lease,
+ });
+ const lockScope = resourceLockScope();
+ const lockKeys = backendResourceLockKeys(initialLease, preparedLease.lockKeys);
+ let privateSupervisorStatePath = null;
+ try {
+ initialLease.resources.locks.push(...acquireResourceLocks({
+ ...lockScope, keys: lockKeys, lease: initialLease,
+ }));
+ writeBackendLease(leasePath, initialLease);
+ const supervisorState = process.env.STACK_BENCH_SUPERVISOR_STATE
+ ?? (process.env.STACK_BENCH_SUPERVISOR_DIR
+ ? join(resolve(process.env.STACK_BENCH_SUPERVISOR_DIR), `${runId}.json`) : null);
+ if (supervisorState) {
+ // Private handoff to an outer timeout supervisor. It contains the lease
+ // token, so create it once with owner-only permissions and never place it
+ // in the results tree.
+ privateSupervisorStatePath = resolve(supervisorState);
+ mkdirSync(dirname(privateSupervisorStatePath), { recursive: true, mode: 0o700 });
+ writeFileSync(privateSupervisorStatePath, `${JSON.stringify({
+ version: SUPERVISOR_STATE_VERSION, runId, backend: args.backend, runtimeDir, leasePath,
+ ownershipToken: initialLease.ownershipToken, output: resolve(args.out),
+ })}\n`, { flag: 'wx', mode: 0o600 });
+ }
+ } catch (error) {
+ releaseResourceLocks(initialLease);
+ rmSync(runtimeDir, { recursive: true, force: true });
+ throw error;
+ }
+ process.env.STACK_BENCH_LEASE = leasePath;
+ process.env.STACK_BENCH_LEASE_TOKEN = initialLease.ownershipToken;
+ if (process.platform === 'win32') {
+ // When Windows resolves `bash` through WSL, WSLENV must carry lease paths
+ // and tokens into lifecycle scripts with path translation.
+ const bridge = ['STACK_BENCH_LEASE/p', 'STACK_BENCH_LEASE_TOKEN',
+ 'STACK_BENCH_NODE_BIN', ...stackRuntime.windowsEnvironmentBridge];
+ const existing = (process.env.WSLENV ?? '').split(':').filter(Boolean);
+ process.env.WSLENV = [...new Set([...existing, ...bridge])].join(':');
+ }
+
+ let tornDown = false;
+ let activeRun: BenchmarkRunRecord | null = null;
+ const recoveryPath = join(outputDir, ARTIFACT_FILE.recovery);
+ const writeLeaseEvidence = (knownLease: BackendLease | null = null) => {
+ const lease = knownLease ?? readBackendLease(leasePath,
+ { token: initialLease.ownershipToken, backend: args.backend, runId });
+ const out = join(outputDir, ARTIFACT_FILE.backendLease);
+ const evidence = publicBackendLease(lease);
+ const id = `${runId}-backend-lease`;
+ writeArtifact(out, {
+ kind: 'backend_lease_evidence', id,
+ attempt: { id, parentId: runId },
+ timestamps: { startedAt: evidence.createdAt, completedAt: new Date().toISOString() },
+ identities: emptyArtifactIdentities({ stackAdapter: { id: args.backend } }),
+ payload: evidence,
+ });
+ return evidence;
+ };
+ const teardown = ({ reason = null, retainBackend = args.retainBackend }:
+ { reason?: string | null; retainBackend?: boolean } = {}) => {
+ if (tornDown) return;
+ if (activeAgentChild?.pid) {
+ killTree(activeAgentChild.pid);
+ activeAgentChild = null;
+ }
+ // Preserve restart failures before removing the only filesystem that holds
+ // their stderr. A 500 after restart is otherwise impossible to distinguish
+ // from an application defect, a dead dependency, or host pressure.
+ if (activeRun) {
+ try {
+ activeRun.backendDiagnostics = captureApplicationDiagnostics(join(outputDir, 'backend.log'));
+ } catch (error) {
+ activeRun.backendDiagnostics = { captured: false,
+ reason: errorMessage(error).split(/\r?\n/)[0] };
+ }
+ }
+ let released = false;
+ let cleanupError: unknown = null;
+ try {
+ released = releaseBackendLease(leasePath, initialLease.ownershipToken,
+ { retainBackend });
+ } catch (error) { cleanupError = error; }
+ let finalLease = initialLease;
+ try {
+ finalLease = readBackendLease(leasePath,
+ { token: initialLease.ownershipToken, backend: args.backend, runId });
+ } catch (error) { cleanupError ??= error; released = false; }
+ const evidence = writeLeaseEvidence(finalLease);
+ writeRecoveryArtifact(recoveryPath, finalLease, { cleanupSucceeded: released,
+ retained: Boolean(retainBackend),
+ reason: cleanupError === null ? reason ?? (released ? null : 'authenticated cleanup refused')
+ : errorMessage(cleanupError) });
+ if (activeRun) {
+ activeRun.backendLease = evidence;
+ activeRun.outcome ??= aggregateRunOutcome(activeRun.levels);
+ writeRunJson(join(outputDir, ARTIFACT_FILE.run), activeRun);
+ }
+ tornDown = released;
+ if (released && !retainBackend) {
+ rmSync(runtimeDir, { recursive: true, force: true });
+ if (privateSupervisorStatePath) rmSync(privateSupervisorStatePath, { force: true });
+ }
+ if (cleanupError) throw cleanupError;
+ if (!released) throw new Error(`backend teardown refused: listener no longer matches lease ${runId}`);
+ };
+ emergencyTeardown = teardown;
+
+ try {
+ stackAdapter.lifecycle.activate({
+ leasePath, leaseToken: initialLease.ownershipToken, lease: initialLease,
+ ...stackRuntime.lifecycle,
+ });
+ } catch (error) {
+ try { teardown({ reason: `backend activation failed: ${errorMessage(error)}`, retainBackend: false }); }
+ catch (cleanupError) {
+ console.error(` activation cleanup quarantined: ${errorMessage(cleanupError).split(/\r?\n/)[0]}`);
+ }
+ throw error;
+ }
+
+ // Teardown stops only resources recorded in this run's lease.
+ const interrupt = (signal: NodeJS.Signals, exitCode: number) => {
+ console.log(`interrupted by ${signal} — stopping exact owned resources`);
+ try { teardown({ reason: `interrupted by ${signal}` }); }
+ catch (error) { console.error(` cleanup quarantined: ${errorMessage(error).split(/\r?\n/)[0]}`); }
+ process.exit(exitCode);
+ };
+ process.on('SIGINT', () => interrupt('SIGINT', 130));
+ process.on('SIGTERM', () => interrupt('SIGTERM', 143));
+ process.on('exit', () => {
+ if (!tornDown) {
+ try { teardown(); } catch (error) {
+ console.error(` cleanup failed: ${errorMessage(error).split('\n')[0]}`);
+ }
+ }
+ });
+
+ // Seed source only; the upgrade session installs its own dependencies.
+ if (args.seedFrom) {
+ const from = resolve(args.seedFrom);
+ if (!existsSync(from)) { console.error(`--seed-from path does not exist: ${from}`); process.exit(2); }
+ seedAppSource(from, appDir);
+ if (args.progressionSeed) {
+ const seeded = hashAppSource(appDir);
+ if (seeded.sha256 !== args.progressionSeed.sourceSha256
+ || seeded.files.length !== args.progressionSeed.sourceFiles) {
+ throw new Error('extension source does not match its recorded identity');
+ }
+ }
+ console.log(args.repairGrant
+ ? ` restored L${args.levelList[0]} checkpoint from ${from} for a bounded repair continuation`
+ : ` seeded from ${from} — level ${args.levelList[0]} will UPGRADE it, not rebuild`);
+ }
+
+ const started = Date.now();
+ const run: BenchmarkRunRecord = { id: runId,
+ ...(args.repairGrant ? { kind: 'repair_continuation',
+ continuation: structuredClone(args.repairGrant.grant) } : {}),
+ startedAt: new Date(started).toISOString(),
+ parentAttemptId: args.parentAttemptId ?? null,
+ identities: emptyArtifactIdentities({
+ experiment: args.experimentIdentity ?? null,
+ agentAdapter: agentAdapterIdentity(agentAdapter),
+ stackAdapter: { id: stackAdapter.id, version: stackAdapter.version },
+ }),
+ mode: args.runMode ?? { id: args.progression ? 'dependency' : 'sequential',
+ version: args.progression ? DEPENDENCY_MODE_VERSION : '1.0.0' },
+ track: args.track, backend: args.backend, model: args.model,
+ pricing: args.pricing,
+ guidance: args.guidance, condition: args.condition ?? null,
+ skills: args.skills ?? [],
+ runtime: { buildImage: process.env.STACK_BENCH_IMAGE ?? DEFAULT_BUILD_IMAGE, url },
+ selectionRequest: args.selectionRequest,
+ featureCatalog: args.featureCatalog?.identity ?? null,
+ dependencyPolicy: args.dependencyPolicy?.identity ?? null,
+ ...(args.progressionOwner ? { progressionOwner: args.progressionOwner } : {}),
+ ...(args.progressionSeed ? { progressionSeed: {
+ fromDepth: args.progressionSeed.fromDepth,
+ sourceSha256: args.progressionSeed.sourceSha256,
+ sourceFiles: args.progressionSeed.sourceFiles,
+ parent: structuredClone(args.progressionSeed.parent),
+ validatedDepths: [],
+ } } : {}),
+ backendLease: publicBackendLease(readBackendLease(leasePath,
+ { token: initialLease.ownershipToken, backend: args.backend, runId })),
+ validation: {
+ ladder: { policy: args.progression ? args.progression.identity.policy : 'pass-before-next-level',
+ requestedLevels: [...args.levelList],
+ completedLevels: [], stoppedAfterLevel: null, blockedLevels: [] } }, levels: [] };
+ activeRun = run;
+
+ const progressionOwner = args.progression ? {
+ ...args.progressionOwner,
+ workspace: { appDirectory: 'source' },
+ } : null;
+ const progressionExecution = args.progression ? createLiveProgressionExecution({
+ progression: args.progression,
+ featureCatalogIdentity: args.featureCatalog?.identity,
+ dependencyPolicyIdentity: args.dependencyPolicy?.identity,
+ owner: progressionOwner,
+ statePath: join(args.out, ARTIFACT_FILE.progressionState),
+ runId,
+ outputDir: args.out,
+ appDir,
+ track: args.track,
+ backend: args.backend,
+ identities: run.identities,
+ recipeBindings: args.recipeBindings,
+ resumeFrom: args.progressionResumeFrom ?? null,
+ getRunArtifact: () => {
+ writeRunJson(join(outputDir, ARTIFACT_FILE.run), run);
+ return readArtifact(join(outputDir, ARTIFACT_FILE.run));
+ },
+ onState: status => {
+ run.progressionStatus = status;
+ writeRunJson(join(outputDir, ARTIFACT_FILE.run), run);
+ },
+ }) : null;
+ const progressionStart = progressionExecution?.initialize() ?? null;
+ if (progressionStart?.resumed) {
+ const prior = progressionStart.priorRun;
+ if (!prior) throw new Error('resumed dependency progression has no prior run artifact');
+ const actionLevel = progressionStart.action.type === 'terminal'
+ ? Number.MAX_SAFE_INTEGER : progressionStart.action.level;
+ const inheritedLevels = (prior.payload.levels ?? [])
+ .filter(level => level.level < actionLevel).map(level => level.level);
+ run.levels = (prior.payload.levels ?? [])
+ .filter(level => inheritedLevels.includes(level.level)).map(level => structuredClone(level));
+ run.validation.ladder.completedLevels = [...inheritedLevels];
+ run.progressionResume = {
+ priorRunId: prior.id,
+ priorRunSha256: sha256(canonicalDefinitionJson(prior)),
+ stateSha256: progressionStart.stateSha256,
+ action: progressionStart.action.type === 'terminal'
+ ? { type: 'terminal' }
+ : { type: progressionStart.action.type, level: progressionStart.action.level },
+ inheritedLevels,
+ priorTotals: prior.payload.totals ?? null,
+ };
+ run.progressionStatus = progressionStart.status;
+ writeRunJson(join(args.out, ARTIFACT_FILE.run), run);
+ }
+
+ const bindProgressionAction = (level: number): ProgressionRecipeAction | null => {
+ if (!progressionExecution) return null;
+ const selected = progressionExecution.bind();
+ if (!isProgressionWorkRecipeAction(selected)) return selected;
+ if (!args.recipeTasks) throw new Error('recipe task map is unavailable');
+ args.recipeTasks.set(level, {
+ request: selected.grader.request,
+ selection: selected.grader.selection,
+ task: selected.grader.task,
+ agentRequest: selected.agent.request,
+ progressionAction: selected.action,
+ });
+ return selected;
+ };
+
+ const progressionBundles = new Map();
+ const recordProgressionGrade = (input: Parameters['record']>[0]) => {
+ const next = progressionExecution?.record(input) ?? null;
+ const last = progressionExecution?.state?.attempts.at(-1) ?? null;
+ if (input.bundle && input.selected && isProgressionWorkRecipeAction(input.selected)
+ && last?.outcome === 'conclusive') {
+ progressionBundles.set(input.selected.action.level, input.bundle as GradeBundlePayload);
+ }
+ return next;
+ };
+
+ const appendLevelRecord = (record: RunLevelRecord): void => {
+ if (args.dependencyPolicy?.definition.workSelection !== 'feature') {
+ run.levels.push(record);
+ return;
+ }
+ const index = run.levels.findIndex(candidate => candidate.level === record.level);
+ if (index < 0) run.levels.push(record);
+ else run.levels[index] = mergeFeatureLevelRecord(run.levels[index] ?? null, record);
+ };
+
+ let runCostComplete = true;
+
+ const runAgentForLevel = async (mode: AgentMode, level: number,
+ onFailure?: () => Promise): Promise => {
+ try {
+ clearPrivateGradingEvidence(appDir);
+ const result = await runAgent(args, agentAdapter, mode, level, appDir);
+ if (result.costComplete !== true) runCostComplete = false;
+ return result;
+ } catch (error) {
+ await onFailure?.();
+ const reason = errorMessage(error).split(/\r?\n/)[0] ?? 'agent execution failed';
+ run.outcome = { kind: 'harness_failure', phase: `agent-${mode}`,
+ reason, appFailures: [], inconclusive: [], harnessFailures: [reason] };
+ run.validation.ladder.stoppedAfterLevel = run.levels.at(-1)?.level ?? null;
+ run.validation.ladder.blockedLevels = args.levelList.filter(candidate => candidate >= level);
+ if (progressionExecution) {
+ run.progressionStatus = progressionExecution.status();
+ run.validation.ladder.completedLevels = [...new Set(requireProgressionState(progressionExecution.state).attempts
+ .filter(attempt => attempt.outcome === 'conclusive')
+ .map(attempt => attempt.level))];
+ }
+ finalizeRunTotals(run, started, { costComplete: false });
+ run.completedAt = new Date().toISOString();
+ writeRunJson(join(outputDir, ARTIFACT_FILE.run), run);
+ throw error;
+ }
+ };
+
+ // Stop before grading if a coding session read protected material or if the
+ // audit itself failed. Keep the paid session and exact cost in the run artifact even
+ // though no score may be used.
+ const abortUnusableSession = (whichSession: string, audit: ContaminationAudit,
+ levelRecord: UnknownRecord & { level: number },
+ selected: ProgressionRecipeAction | null, completedRepair = false) => {
+ const reason = audit.evidence.join('; ');
+ const outcome: RunOutcome = { kind: audit.kind === 'harness_failure' ? 'harness_failure' : 'ungraded',
+ phase: 'contamination-audit', reason,
+ appFailures: [], inconclusive: [],
+ harnessFailures: audit.kind === 'harness_failure' ? [reason] : [] };
+ run.contaminated = audit.kind === 'contaminated';
+ run.contamination = { evidence: audit.evidence, verdict: audit.verdict,
+ detectedAt: whichSession };
+ const record: RunLevelRecord = { ...levelRecord, error: reason, outcome,
+ level: levelRecord.level, graded: false, score: null, max: null, selection: null };
+ appendLevelRecord(record);
+ if (progressionExecution) {
+ recordProgressionGrade({ selected, bundle: null, level: levelRecord.level,
+ failure: progressionFailure(outcome),
+ completedRepair });
+ run.progressionStatus = progressionExecution!.status();
+ }
+ run.validation.ladder.stoppedAfterLevel = run.levels.at(-2)?.level ?? null;
+ run.validation.ladder.blockedLevels = args.levelList
+ .filter(candidate => candidate >= levelRecord.level);
+ finalizeRunTotals(run, started, { costComplete: runCostComplete });
+ run.outcome = outcome;
+ run.completedAt = new Date().toISOString();
+ if (run.contaminated) {
+ console.log(`\n !! CONTAMINATED at ${whichSession}:`);
+ for (const evidence of audit.evidence) console.log(` ${evidence}`);
+ console.log(' Scores from this run must not be quoted.');
+ } else {
+ console.log(`\n !! HARNESS FAILURE at ${whichSession}:`);
+ for (const evidence of audit.evidence) console.log(` ${evidence}`);
+ console.log(' The audit did not establish a usable result.');
+ }
+ try { writeRunJson(join(outputDir, ARTIFACT_FILE.run), run); } catch { /* best effort */ }
+ try { archiveTranscripts(appDir, artifactLabel); } catch { /* best effort */ }
+ teardown();
+ process.exit(4);
+ };
+
+ for (let levelIndex = 0; levelIndex < args.levelList.length; levelIndex += 1) {
+ const level = args.levelList[levelIndex]!;
+ const t0 = Date.now();
+ const continuing = Boolean(args.repairGrant);
+ console.log(`\n================ ${args.backend} — level ${level} ================`);
+
+ let progressionSelection = bindProgressionAction(level);
+ if (progressionSelection?.action.type === 'terminal') break;
+ if (args.dependencyPolicy?.definition.workSelection === 'all-at-once'
+ && progressionSelection?.action.level !== level) continue;
+ const applicationControl = materializeCodingOutput
+ ? restartSpecFor(args, appDir, track) : null;
+ const featureActionSequence = args.dependencyPolicy?.definition.workSelection === 'feature'
+ ? requireProgressionState(progressionExecution?.state ?? null).attempts.length + 1 : null;
+ const featureActionSuffix = featureActionSequence === null
+ ? '' : `-action${String(featureActionSequence).padStart(3, '0')}`;
+ // A clean-source start that fails voids a grade. Its launch log is the
+ // only account of why, so it stays beside the run.
+ const keepStartLog = (error: unknown, label: string): void => {
+ const startLog = error !== null && typeof error === 'object' && 'startLog' in error
+ ? error.startLog : null;
+ if (typeof startLog !== 'string' || !startLog) return;
+ writeFileSync(join(outputDir, `${label}-start.log`), `${startLog}\n`);
+ };
+ const restoreFeatureAcceptedSource = async (): Promise => {
+ if (featureActionSequence === null) return;
+ const source = join(outputDir, 'source');
+ if (applicationControl) {
+ try {
+ await materializeAcceptedSource(source, appDir, applicationControl);
+ } catch {
+ resetAppToSource(source, appDir);
+ }
+ } else resetAppToSource(source, appDir);
+ };
+ if (featureActionSequence !== null && progressionSelection
+ && isProgressionWorkRecipeAction(progressionSelection)
+ && !featureActionNeedsCoding(progressionSelection,
+ requireProgressionState(progressionExecution?.state ?? null))) {
+ await restoreFeatureAcceptedSource();
+ const bundle = grade(args, appDir, url,
+ `${args.backend}-l${level}${featureActionSuffix}-regrade`, level, track, runId);
+ const outcome = classifyBundle(bundle);
+ const repair = { status: levelGradeIsUsable(outcome) ? 'not-needed' as const : 'ungraded' as const,
+ limit: 0, used: 0, stopReason: 'accepted-source-regrade' };
+ let next = recordProgressionGrade({ selected: progressionSelection, bundle,
+ level });
+ let finalOutcome = outcome;
+ const currentState = requireProgressionState(progressionExecution?.state ?? null);
+ if (next?.type === 'build' && next.level === level
+ && !featureActionNeedsCoding(progressionSelection, currentState)) {
+ const reason = 'accepted source still has ungraded checks after regrade';
+ finalOutcome = { kind: 'harness_failure', phase: 'feature-regrade', reason,
+ appFailures: [], inconclusive: [], harnessFailures: [reason] };
+ next = recordProgressionGrade({ selected: progressionSelection, bundle: null,
+ level, failure: progressionFailure(finalOutcome) });
+ }
+ const state = requireProgressionState(progressionExecution?.state ?? null);
+ const graded = levelGradeIsUsable(finalOutcome)
+ && state.attempts.at(-1)?.outcome === 'conclusive';
+ let checkpoint = null;
+ if (graded && (state.phase === 'terminal' || state.level > level)) {
+ checkpoint = preserveLevelCheckpoint({ appDir, outputDir, runId,
+ identities: run.identities, track: args.track, backend: args.backend, level,
+ repair, outcome: finalOutcome,
+ selectionSha256: bundle?.selection?.sha256 ?? null });
+ }
+ appendLevelRecord({ level, graded,
+ score: graded ? bundle?.totals?.score ?? null : null,
+ max: graded ? bundle?.totals?.max ?? null : null,
+ selection: graded ? bundle?.selection ?? null : null,
+ regression: bundle?.totals?.regression ?? null,
+ contractPass: bundle?.totals?.contractPass ?? null,
+ code: bundle?.code ?? null,
+ repair,
+ checkpoint,
+ sessionTotals: summarizeSessions([]),
+ costUsd: 0,
+ repairs: 0,
+ durationSec: Math.round((Date.now() - t0) / 1000),
+ outcome: finalOutcome });
+ if (graded && (state.phase === 'terminal' || state.level > level)
+ && !run.validation.ladder.completedLevels.includes(level)) {
+ run.validation.ladder.completedLevels.push(level);
+ }
+ run.progressionStatus = progressionExecution!.status();
+ writeRunJson(join(outputDir, ARTIFACT_FILE.run), run);
+ if (state.phase === 'terminal') break;
+ if (state.attempts.at(-1)?.outcome === 'inconclusive') {
+ run.validation.ladder.stoppedAfterLevel = level;
+ run.validation.ladder.blockedLevels = args.levelList.filter(candidate => candidate >= level);
+ break;
+ }
+ if (next?.type !== 'terminal' && next?.level === level) {
+ levelIndex -= 1;
+ continue;
+ }
+ if (next?.type !== 'terminal' && next && next.level < level) {
+ throw new Error(`dependency progression moved backward from L${level}`);
+ }
+ continue;
+ }
+ if (args.seedThrough !== undefined && level <= args.seedThrough) {
+ if (!progressionSelection || !isProgressionWorkRecipeAction(progressionSelection)
+ || progressionSelection.action.level !== level || !args.seedFrom || !run.progressionSeed) {
+ throw new Error(`extension cannot validate depth ${level}`);
+ }
+ let applicationFailure: RunOutcome | null = null;
+ if (applicationControl) {
+ try {
+ await materializeAcceptedSource(args.seedFrom, appDir, applicationControl);
+ } catch (error) {
+ applicationFailure = materializationAppFailure(error);
+ keepStartLog(error, `${args.backend}-extension-l${level}`);
+ }
+ } else {
+ resetAppToSource(args.seedFrom, appDir);
+ }
+ const bundle = grade(args, appDir, url, `${args.backend}-extension-l${level}`,
+ level, track, runId, { applicationFailure });
+ const outcome = applicationFailure ?? classifyBundle(bundle);
+ const next = recordProgressionGrade({ selected: progressionSelection, bundle, level });
+ const progressionState = requireProgressionState(progressionExecution?.state ?? null);
+ const progressionAttempt = progressionState.attempts.at(-1) ?? null;
+ const graded = levelGradeIsUsable(outcome, progressionAttempt);
+ const passed = graded && outcome.kind === 'passed';
+ const repair = { status: passed ? 'not-needed' as const : 'incomplete' as const,
+ limit: 0, used: 0,
+ stopReason: passed ? 'not-needed' : 'extension-validation-failed' };
+ let checkpoint = null;
+ try {
+ checkpoint = preserveLevelCheckpoint({ appDir, outputDir: args.out, runId,
+ identities: run.identities, track: args.track, backend: args.backend, level,
+ repair, outcome, selectionSha256: bundle?.selection?.sha256 ?? null });
+ } catch (error) {
+ throw new Error(`could not preserve extension depth ${level}: ${errorMessage(error)}`);
+ }
+ const source = hashAppSource(appDir);
+ appendLevelRecord({ level, graded,
+ score: graded ? bundle?.totals?.score ?? null : null,
+ max: graded ? bundle?.totals?.max ?? null : null,
+ selection: bundle?.selection ?? null,
+ baseline: { kind: 'extension-validation', source: {
+ sha256: source.sha256, files: source.files.length } },
+ regression: bundle?.totals?.regression ?? null,
+ contractPass: bundle?.totals?.contractPass ?? null,
+ code: bundle?.code ?? null,
+ repair,
+ checkpoint,
+ sessionTotals: summarizeSessions([]),
+ costUsd: 0,
+ repairs: 0,
+ durationSec: Math.round((Date.now() - t0) / 1000),
+ outcome });
+ if (progressionAttempt?.outcome === 'conclusive') {
+ if (!run.validation.ladder.completedLevels.includes(level)) {
+ run.validation.ladder.completedLevels.push(level);
+ }
+ }
+ run.progressionStatus = progressionExecution!.status();
+ if (passed) run.progressionSeed.validatedDepths.push(level);
+ if (!passed) {
+ run.validation.ladder.stoppedAfterLevel = level > 1 ? level - 1 : null;
+ run.validation.ladder.blockedLevels = args.levelList.filter(candidate => candidate >= level);
+ writeRunJson(join(args.out, ARTIFACT_FILE.run), run);
+ break;
+ }
+ writeRunJson(join(args.out, ARTIFACT_FILE.run), run);
+ if (!next || next.type === 'terminal' || next.level === undefined || next.level <= level) {
+ throw new Error(`extension did not advance after depth ${level}`);
+ }
+ continue;
+ }
+ const resumedRepair = progressionStart?.resumed === true
+ && progressionStart.action.type === 'repair';
+ // The interrupted repair was charged but never graded. Grade its preserved
+ // source before any coding session.
+ const resumedGrade = resumedRepair && progressionStart?.action.type === 'repair'
+ && progressionStart.action.repair.awaitingGrade === true;
+ const resumedRegression = resumedRepair
+ ? savedRepairRegression(progressionExecution?.state ?? null, progressionSelection) : null;
+ const resumedRegressionReport = resumedRegression
+ ? join(args.out, 'repair-reports',
+ `rejected-regression-l${level}${featureActionSuffix}-resume.md`) : null;
+ const priorRepairs = resumedRepair
+ ? progressionStart.priorRun?.payload.levels?.find(item => item.level === level)
+ ?.repair?.used ?? 0
+ : args.progression ? 0 : run.levels.reduce((sum, item) => sum + (item.repairs ?? 0), 0);
+ const repairBudgetFor = (selected: ProgressionRecipeAction | null,
+ completedRepairs: number) => selected
+ && isProgressionWorkRecipeAction(selected)
+ ? dependencyRepairBudget(selected.action, completedRepairs)
+ : args.repairs;
+ const levelRepairNodeIds = new Set(progressionSelection?.action.repair.nodeIds ?? []);
+ let progressionRepairLimit = repairBudgetFor(
+ progressionSelection, priorRepairs);
+ const trackProgressionBudget = (selected: ProgressionRecipeAction | null,
+ completedRepairs: number) => {
+ if (!selected || !isProgressionWorkRecipeAction(selected)) return;
+ selected.action.repair.nodeIds.forEach(nodeId => levelRepairNodeIds.add(nodeId));
+ progressionRepairLimit = Math.max(
+ progressionRepairLimit, repairBudgetFor(selected, completedRepairs));
+ };
+ if (resumedRepair && !resumedGrade) {
+ let reportFailure: string | null = null;
+ try {
+ if (resumedRegressionReport && resumedRegression) {
+ mkdirSync(dirname(resumedRegressionReport), { recursive: true });
+ writeFileSync(resumedRegressionReport, resumedRegression.report);
+ }
+ sh('node', [join(ROOT, 'dist', 'commands', 'report-bugs.js'), '--app', appDir,
+ '--history-json', '[]', '--archive', join(args.out, 'repair-reports',
+ `bug-report-l${level}${featureActionSuffix}-resume.md`),
+ ...(resumedRegressionReport ? ['--prior-regression', resumedRegressionReport] : []),
+ ...repairReportArgs(progressionSelection)],
+ { stdio: 'pipe' });
+ } catch (error) {
+ reportFailure = errorMessage(error).split(/\r?\n/)[0]
+ ?? 'repair report generation failed';
+ }
+ if (reportFailure) {
+ const outcome: RunOutcome = {
+ kind: 'harness_failure', phase: 'repair-report', reason: reportFailure,
+ appFailures: [], inconclusive: [], harnessFailures: [reportFailure],
+ };
+ recordProgressionGrade({ selected: progressionSelection, bundle: null, level,
+ failure: progressionFailure(outcome) });
+ appendLevelRecord({ level, graded: false, score: null, max: null,
+ selection: null, error: reportFailure, outcome,
+ repair: { status: 'ungraded', limit: progressionRepairLimit,
+ used: priorRepairs, stopReason: 'repair-report' },
+ repairCostUsd: 0, repairSessions: [], repairs: 0, priorRepairs,
+ cumulativeRepairs: priorRepairs, sessionTotals: summarizeSessions([]),
+ costUsd: 0, durationSec: Math.round((Date.now() - t0) / 1000) });
+ run.validation.ladder.stoppedAfterLevel = level;
+ run.validation.ladder.blockedLevels = args.levelList.filter(candidate => candidate >= level);
+ writeRunJson(join(args.out, ARTIFACT_FILE.run), run);
+ break;
+ }
+ }
+
+ const firstMode = resumedRepair ? 'fix'
+ : continuing ? 'resume' : args.seedFrom ? 'upgrade' : 'build';
+ const build = resumedGrade ? null : await runAgentForLevel(
+ resumedRepair || run.levels.length === 0 ? firstMode : 'upgrade', level,
+ featureActionSequence === null ? undefined : restoreFeatureAcceptedSource);
+ // Only a resumed grade runs a level without a coding session.
+ const requireBuild = (): NonNullable => {
+ if (!build) throw new Error(`level ${level} has no coding session`);
+ return build;
+ };
+ const buildLeak = build ? auditContamination(appDir, ownPorts, auditsTranscripts) : null;
+ if (buildLeak) {
+ const session = requireBuild();
+ const buildSession = runSessionRecord(session,
+ resumedRepair ? priorRepairs + 1 : null);
+ const sessionTotals = summarizeSessions([buildSession]);
+ abortUnusableSession(`level ${level} ${firstMode}`, buildLeak, {
+ level, graded: false, score: null, max: null, selection: null,
+ ...(resumedRepair
+ ? { repairCostUsd: session.costUsd, repairSessions: [buildSession], repairs: 1,
+ priorRepairs, cumulativeRepairs: priorRepairs + 1 }
+ : continuing
+ ? { resumeCostUsd: session.costUsd, resumeSession: buildSession }
+ : { buildCostUsd: session.costUsd, buildSessions: [buildSession] }),
+ sessionTotals, costUsd: session.costUsd, durationMs: Date.now() - t0,
+ }, progressionSelection, resumedRepair);
+ }
+ // Record the session setup needed to compare runs.
+ if (build) run.setup ??= build.setup;
+ if (continuing) {
+ const session = requireBuild();
+ requireContinuation(run).resumeSetup = {
+ sessionId: session.sessionId ?? null,
+ costUsd: session.costUsd,
+ durationMs: session.durationMs,
+ sourceVerified: false,
+ };
+ }
+ // No session, no app. Grading an empty directory yields a real-looking zero
+ // that is a harness failure, not a result for this backend.
+ const buildFailure = build ? agentSessionFailure(build) : null;
+ if (buildFailure) {
+ const session = requireBuild();
+ await restoreFeatureAcceptedSource();
+ console.log(` ABORTED: ${buildFailure.reason}. Details will be kept in ${join(args.out, ARTIFACT_FILE.run)}`);
+ const failedSession = runSessionRecord(session);
+ if (progressionExecution) {
+ recordProgressionGrade({ selected: progressionSelection, bundle: null, level,
+ failure: buildFailure });
+ }
+ appendLevelRecord({ level, graded: false, score: null, max: null,
+ selection: null, error: buildFailure.reason,
+ outcome: buildFailure,
+ ...(continuing
+ ? { resumeSession: failedSession, resumeCostUsd: session.costUsd }
+ : { buildSessions: [failedSession], buildCostUsd: session.costUsd }),
+ sessionTotals: summarizeSessions([session]),
+ costUsd: session.costUsd, durationMs: Date.now() - t0 });
+ break;
+ }
+ const gradeAcceptedSource = async (sourcePath: string,
+ label: string): Promise => {
+ let failure: RunOutcome | null = null;
+ if (applicationControl) {
+ try {
+ await materializeAcceptedSource(sourcePath, appDir, applicationControl);
+ } catch (error) {
+ failure = materializationAppFailure(error);
+ keepStartLog(error, label);
+ }
+ } else {
+ resetAppToSource(sourcePath, appDir);
+ }
+ return grade(args, appDir, url, label, level, track, runId,
+ { applicationFailure: failure });
+ };
+ const archiveCandidateGrade = (label: string): void => {
+ const gradingDirectory = join(appDir, 'stack-bench');
+ if (!existsSync(gradingDirectory)) return;
+ cpSync(gradingDirectory, join(outputDir, 'candidate-grades', label), {
+ recursive: true,
+ filter: source => !/[\\/]media([\\/]|$)/.test(source),
+ });
+ };
+ const restoreAcceptedRepair = async (sourcePath: string, gradingPath: string): Promise => {
+ if (applicationControl) await materializeAcceptedSource(sourcePath, appDir, applicationControl);
+ else resetAppToSource(sourcePath, appDir);
+ restorePrivateGradingEvidence(appDir, gradingPath);
+ };
+ // Keep a repaired source whose grade did not finish beside the run, bound
+ // by hash, so a resume can grade exactly what the paid session produced.
+ const preserveRepairCandidate = (directory: string): RunRepairCandidate => {
+ const path = join(outputDir, directory);
+ rmSync(path, { recursive: true, force: true });
+ const live = hashAppSource(appDir);
+ snapshotSource(appDir, path);
+ const preserved = hashDirectory(path);
+ if (live.sha256 !== preserved.sha256 || live.files.length !== preserved.files.length) {
+ throw new Error('preserved repair source differs from the live application source');
+ }
+ return { directory, sha256: preserved.sha256, files: preserved.files.length };
+ };
+ if (resumedGrade) {
+ const candidate = progressionStart?.priorRun?.payload.levels
+ ?.find(item => item.level === level)?.repair?.candidate;
+ const candidateRoot = args.progressionResumeFrom ?? outputDir;
+ let candidateFailure: string | null = null;
+ if (!candidate || !/^[A-Za-z0-9._-]+$/.test(candidate.directory)
+ || !existsSync(join(candidateRoot, candidate.directory))) {
+ candidateFailure = 'the interrupted repair left no source to grade';
+ } else {
+ const candidatePath = join(candidateRoot, candidate.directory);
+ const preserved = hashDirectory(candidatePath);
+ if (preserved.sha256 !== candidate.sha256
+ || preserved.files.length !== candidate.files) {
+ candidateFailure = 'the interrupted repair source does not match its record';
+ } else {
+ resetAppToSource(candidatePath, appDir);
+ console.log(` restored the interrupted repair source from ${candidatePath}`);
+ }
+ }
+ if (candidateFailure) {
+ const outcome: RunOutcome = {
+ kind: 'harness_failure', phase: 'repair-candidate', reason: candidateFailure,
+ appFailures: [], inconclusive: [], harnessFailures: [candidateFailure],
+ };
+ recordProgressionGrade({ selected: progressionSelection, bundle: null, level,
+ failure: progressionFailure(outcome) });
+ appendLevelRecord({ level, graded: false, score: null, max: null,
+ selection: null, error: candidateFailure, outcome,
+ repair: { status: 'ungraded', limit: progressionRepairLimit,
+ used: priorRepairs, stopReason: 'repair-candidate' },
+ repairCostUsd: 0, repairSessions: [], repairs: 0, priorRepairs,
+ cumulativeRepairs: priorRepairs, sessionTotals: summarizeSessions([]),
+ costUsd: 0, durationSec: Math.round((Date.now() - t0) / 1000) });
+ run.validation.ladder.stoppedAfterLevel = level;
+ run.validation.ladder.blockedLevels = args.levelList.filter(candidate => candidate >= level);
+ writeRunJson(join(args.out, ARTIFACT_FILE.run), run);
+ break;
+ }
+ }
+ if (continuing) {
+ // A resume may restore runtime state but cannot change checkpoint source.
+ const resumed = hashAppSource(appDir);
+ const repairGrant = args.repairGrant;
+ if (!repairGrant) throw new Error('repair continuation has no grant');
+ if (resumed.sha256 !== repairGrant.checkpoint.payload.source.sha256
+ || resumed.files.length !== repairGrant.checkpoint.payload.source.files) {
+ throw new Error('resume setup changed the parent checkpoint source');
+ }
+ const continuation = requireContinuation(run);
+ if (!continuation.resumeSetup) throw new Error('repair continuation did not record resume setup');
+ continuation.resumeSetup.sourceVerified = true;
+ }
+ if (args.referenceMutationOnly) {
+ const session = requireBuild();
+ appendLevelRecord({ level, score: null, max: null, graded: false, contractPass: null,
+ selection: null,
+ outcome: { kind: 'ungraded', phase: 'reference-mutation-only',
+ reason: 'the parent qualification owns the full clean grade',
+ appFailures: [], inconclusive: [], harnessFailures: [] },
+ buildSessions: [runSessionRecord(session)],
+ buildCostUsd: session.costUsd, sessionTotals: summarizeSessions([session]),
+ costUsd: session.costUsd, durationMs: Date.now() - t0 });
+ break;
+ }
+ const firstBuildDirectory = continuing
+ ? `baseline-l${level}${featureActionSuffix}`
+ : `first-build-l${level}${featureActionSuffix}`;
+ const firstBuildPath = join(args.out, firstBuildDirectory);
+ let firstBuildSource = null;
+ let materializationOutcome: RunOutcome | null = null;
+ try {
+ const liveSource = hashAppSource(appDir);
+ snapshotSource(appDir, firstBuildPath);
+ const preservedSource = hashDirectory(firstBuildPath);
+ if (liveSource.sha256 !== preservedSource.sha256) {
+ throw new Error('preserved first-build source differs from the live application source');
+ }
+ firstBuildSource = { sha256: liveSource.sha256, files: liveSource.files.length };
+ if (applicationControl) {
+ await materializeAcceptedSource(firstBuildPath, appDir, applicationControl);
+ }
+ console.log(` kept the ${continuing ? 'continuation baseline' : 'unaided'} source at ${firstBuildPath}`);
+ } catch (error) {
+ if (firstBuildSource) {
+ materializationOutcome = materializationAppFailure(error);
+ keepStartLog(error, `${args.backend}-l${level}${featureActionSuffix}`);
+ }
+ console.log(materializationOutcome
+ ? ` !! ${materializationOutcome.reason}`
+ : ` !! could not bind the first-build source: ${errorMessage(error).split('\n')[0]}`);
+ }
+ const firstBuildLabel = `${args.backend}-l${level}${featureActionSuffix}`;
+ let bundle = firstBuildSource
+ ? grade(args, appDir, url, firstBuildLabel, level, track, runId,
+ { applicationFailure: materializationOutcome }) : null;
+ // A grader failure on unchanged source is retried once, as a repair grade is.
+ if (firstBuildSource && !materializationOutcome
+ && !levelGradeIsUsable(classifyBundle(bundle))) {
+ console.log(' grade did not complete; retrying the same source once');
+ bundle = grade(args, appDir, url, `${firstBuildLabel}-retry`, level, track, runId);
+ }
+ let reusableRepairEvidence: {
+ bundle: GradeBundlePayload;
+ results: string;
+ } | null = null;
+
+ // What the model built BEFORE being handed the answers. Every backend can
+ // reach the same total given enough repairs, so the post-fix score stops
+ // discriminating — what it got right unaided is the comparison that survives.
+ const firstBuild: FirstBuildRecord = {
+ score: bundle?.totals?.score ?? null,
+ max: bundle?.totals?.max ?? null,
+ regression: bundle?.totals?.regression ?? null,
+ contractPass: bundle?.totals?.contractPass ?? null,
+ outcome: materializationOutcome ?? sourceBoundFirstBuildOutcome(bundle, firstBuildSource),
+ source: firstBuildSource,
+ missed: Object.values(bundle?.suites ?? {}).flatMap(s =>
+ (s?.features ?? []).flatMap(f =>
+ (f.criteria ?? []).filter(c => !evidencePassed(criterionEvidence(c)))
+ .map(c => `${f.name}/${c.id}`))),
+ };
+
+ if (continuing) {
+ const repairGrant = args.repairGrant;
+ if (!repairGrant) throw new Error('repair continuation has no grant');
+ if (firstBuild.score === null || firstBuild.max === null || firstBuildSource === null) {
+ throw new Error('repair continuation did not produce a source-bound baseline score');
+ }
+ const reproduction = compareRepairBaseline(repairGrant.level, {
+ score: firstBuild.score,
+ max: firstBuild.max,
+ selectionSha256: bundle?.selection?.sha256 ?? null,
+ sourceSha256: firstBuildSource.sha256,
+ expectedSourceSha256: repairGrant.checkpoint.payload.source.sha256,
+ outcome: repairOutcome(firstBuild.outcome),
+ });
+ requireContinuation(run).baseline = {
+ score: firstBuild.score,
+ max: firstBuild.max,
+ selectionSha256: bundle?.selection?.sha256 ?? null,
+ sourceSha256: firstBuildSource?.sha256 ?? null,
+ outcome: firstBuild.outcome,
+ ...reproduction,
+ };
+ if (!reproduction.reproduced) {
+ const reason = `restored checkpoint did not reproduce its parent: ${reproduction.mismatches.join(', ')}`;
+ console.log(` CONTINUATION STOPPED: ${reason}`);
+ const failure = { kind: 'harness_failure', phase: 'continuation-baseline', reason,
+ appFailures: [], inconclusive: [], harnessFailures: [] };
+ firstBuild.outcome = failure;
+ bundle = { ...bundle, outcome: failure };
+ }
+ }
+
+ const selectedObservedChecks = checksForGrade(args.recipeTasks?.get(level), 'observed');
+ if (!continuing && !resumedRepair && selectedObservedChecks.length) {
+ const observationOut = join(args.out,
+ `first-build-l${level}${featureActionSuffix}-observed`);
+ let observationBundle = null;
+ let observationOutcome;
+ if (!firstBuildSource) {
+ observationOutcome = { kind: 'harness_failure', phase: 'first-build-source',
+ reason: 'observed specifications require a source-bound first build' };
+ } else if (!ladderMayContinue(firstBuild.outcome)) {
+ observationOutcome = { kind: 'ungraded', phase: 'first-build-observation',
+ reason: 'scored first-build grading did not establish a usable environment' };
+ } else {
+ observationBundle = grade(args, appDir, url, `${args.backend}-l${level}-observed`, level,
+ track, runId, { observation: 'observed', out: observationOut,
+ sourceSha256: firstBuildSource.sha256 });
+ observationOutcome = classifyBundle(observationBundle);
+ }
+ firstBuild.observations = {
+ sourceSha256: firstBuildSource?.sha256 ?? null,
+ selectionSha256: args.recipeTasks?.get(level)?.selection.sha256 ?? null,
+ selectedChecks: selectedObservedChecks.map(check => check.stableKey),
+ reportedChecks: observationBundle?.selection?.reportedChecks ?? [],
+ passedPoints: observationBundle?.totals?.score ?? null,
+ observedPoints: observationBundle?.totals?.max ?? null,
+ scoreContribution: false,
+ repairVisible: false,
+ artifact: observationBundle
+ ? `first-build-l${level}${featureActionSuffix}-observed/${ARTIFACT_FILE.gradeBundle}` : null,
+ outcome: observationOutcome,
+ };
+ }
+
+ let initialProgressionFailure: ProgressionFailure | null = null;
+ if (featureActionSequence !== null && progressionSelection
+ && isProgressionWorkRecipeAction(progressionSelection)) {
+ const candidateOutcome = classifyBundle(bundle);
+ archiveCandidateGrade(`l${level}${featureActionSuffix}`);
+ if (!levelGradeIsUsable(candidateOutcome)) {
+ await restoreFeatureAcceptedSource();
+ initialProgressionFailure = progressionFailure(candidateOutcome);
+ } else if (!featureCandidateAccepted(progressionSelection,
+ requireProgressionState(progressionExecution?.state ?? null),
+ bundle)) {
+ bundle = await gradeAcceptedSource(join(outputDir, 'source'),
+ `${args.backend}-l${level}${featureActionSuffix}-restored`,
+ );
+ const restoredOutcome = classifyBundle(bundle);
+ if (!levelGradeIsUsable(restoredOutcome)) {
+ initialProgressionFailure = progressionFailure(restoredOutcome);
+ }
+ }
+ }
+
+ // Preserve the first source and scored grading before repair overwrites the
+ // app. Observed evidence remains in its own source-bound result directory.
+ const acceptedGradingDirectory = continuing
+ ? `baseline-l${level}${featureActionSuffix}-grading`
+ : `first-build-l${level}${featureActionSuffix}-grading`;
+ try {
+ const gradingFrom = join(appDir, 'stack-bench');
+ if (existsSync(gradingFrom)) {
+ const gradingTo = join(args.out, acceptedGradingDirectory);
+ cpSync(gradingFrom, gradingTo, {
+ recursive: true,
+ filter: src => !/[\\/]media([\\/]|$)/.test(src),
+ });
+ if (bundle) reusableRepairEvidence = { bundle, results: gradingTo };
+ console.log(` kept the ${continuing ? 'continuation baseline' : 'unaided'} grading at ${join(args.out, acceptedGradingDirectory)}`);
+ }
+ } catch (e) {
+ // Never worth losing a run over: the score is already recorded.
+ console.log(` !! could not keep the first build: ${errorMessage(e).split('\n')[0]}`);
+ }
+
+ // A resumed grade settles a repair that was already charged; only a
+ // resumed coding session is a new repair.
+ let repairs = resumedRepair && !resumedGrade ? 1 : 0;
+ let repairCost = resumedRepair && build ? build.costUsd : 0;
+ const repairSessions = resumedRepair && build
+ ? [runSessionRecord(build, priorRepairs + 1)] : [];
+ const repairHistory: ReturnType[] = [];
+ let repairCandidate: RunRepairCandidate | null = null;
+ let priorRegressionReport = resumedRegressionReport;
+ let priorRegressionOwner = repairOwnerNodeIds(progressionSelection).join('\n') || null;
+ let regressed = false;
+ let repairStopReason: string | null = null;
+ let repairProgress = repairProgressState(null, bundle);
+ const pauseForRepeatedFindings = () => {
+ if (args.progression) return false;
+ repairProgress = repairProgressState(repairProgress, bundle);
+ if (args.maxStalledRepairs === 0
+ || repairProgress.stalledRounds < args.maxStalledRepairs) return false;
+ repairStopReason = 'repeated-findings';
+ console.log(` pausing after ${repairProgress.stalledRounds} repairs `
+ + 'with the same failed checks and no score gain');
+ return true;
+ };
+ const initialBundleOutcome = classifyBundle(bundle);
+ const initialProgressionAttempt = args.progression
+ ? requireProgressionState(progressionExecution?.state ?? null).attempts.at(-1) ?? null
+ : null;
+ const initialGradeUsable = levelGradeIsUsable(initialBundleOutcome,
+ initialProgressionAttempt);
+ if (!initialGradeUsable) {
+ repairStopReason = 'initial-grading-failed';
+ console.log(' repairs skipped: the initial grade did not complete, so there are no reliable findings to fix');
+ }
+
+ let progressionNext = recordProgressionGrade({
+ selected: progressionSelection,
+ bundle: initialProgressionFailure ? null : bundle,
+ level,
+ failure: initialProgressionFailure,
+ completedRepair: resumedRepair && !resumedGrade,
+ });
+ // Never start a paid repair with nothing left to spend or while a charged
+ // repair still waits for its grade.
+ const progressionMayRepair = () => !args.progression
+ || (progressionNext?.type === 'repair' && progressionNext.repair.remaining > 0
+ && progressionNext.repair.awaitingGrade !== true);
+ // One allowance for both modes: the engine's remaining repairs in
+ // dependency mode, the run-wide total less every repair so far otherwise.
+ const repairsRemaining = (): number => args.progression
+ ? (progressionNext?.type === 'repair' ? progressionNext.repair.remaining : 0)
+ : args.repairs - priorRepairs - repairs;
+ const mayRepair = (): boolean => args.progression
+ ? progressionMayRepair() : repairsRemaining() > 0;
+ const recordRepairProgression = ({ failure = null, repairRegression = null,
+ completedRepair = false }: {
+ failure?: ProgressionFailure | null;
+ repairRegression?: ProgressionRepairRegression | null;
+ completedRepair?: boolean;
+ } = {}) => {
+ progressionNext = recordProgressionGrade({
+ selected: progressionSelection,
+ bundle: failure ? null : bundle,
+ level,
+ failure,
+ repairRegression,
+ completedRepair,
+ });
+ return progressionMayRepair();
+ };
+ const recordRepairHarnessFailure = (phase: string, reason: string,
+ failedBundle: GradeBundlePayload | null = null, completedRepair = false): void => {
+ const failure: RunOutcome = {
+ kind: 'harness_failure', phase, reason,
+ appFailures: [], inconclusive: [], harnessFailures: [reason],
+ };
+ bundle = failedBundle ? { ...failedBundle, outcome: failure } : { outcome: failure };
+ repairStopReason = phase;
+ if (args.progression) recordRepairProgression({
+ failure: progressionFailure(failure), completedRepair,
+ });
+ };
+ const restoreProgressionGrade = (accepted: GradeBundlePayload | null,
+ label: string): boolean => {
+ const expected = progressionSelection && isProgressionWorkRecipeAction(progressionSelection)
+ ? progressionSelection.grader.selectionSha256 : null;
+ if (!expected || accepted?.selection?.sha256 === expected) {
+ bundle = accepted;
+ return true;
+ }
+ bundle = grade(args, appDir, url, label, level, track, runId);
+ const outcome = classifyBundle(bundle);
+ if (levelGradeIsUsable(outcome)) return true;
+ recordRepairHarnessFailure('repair-restore-grading',
+ outcome.reason ?? 'restored source did not produce a reliable grade', bundle, true);
+ return false;
+ };
+ const writeRepairReport = (results: string | null = null):
+ { status: 0 | 3 | 4 } | { status: 'failed'; reason: string } => {
+ try {
+ sh('node', [join(ROOT, 'dist', 'commands', 'report-bugs.js'), '--app', appDir,
+ ...(results ? ['--results', results] : []),
+ '--history-json', JSON.stringify(repairHistory),
+ '--archive', join(outputDir, 'repair-reports',
+ `bug-report-l${level}${featureActionSuffix}-round${repairs + 1}.md`),
+ ...(priorRegressionReport ? ['--prior-regression', priorRegressionReport] : []),
+ ...repairReportArgs(progressionSelection)], { stdio: 'pipe' });
+ return { status: 0 };
+ } catch (error) {
+ const failure = commandFailure(error);
+ if (failure.status === 3 || failure.status === 4) return { status: failure.status };
+ return { status: 'failed', reason: errorMessage(failure).split(/\r?\n/)[0]
+ ?? 'repair report generation failed' };
+ }
+ };
+ const recordMissingRepairFeedback = (status: 3 | 4): void => {
+ repairStopReason = 'no-actionable-findings';
+ if (!args.progression) return;
+ const reason = status === 3
+ ? 'selected repair checks contain no failures'
+ : 'selected repair checks produced no actionable findings';
+ const failure: RunOutcome = {
+ kind: 'harness_failure', phase: 'repair-report', reason,
+ appFailures: [], inconclusive: [], harnessFailures: [reason],
+ };
+ bundle = { outcome: failure };
+ recordRepairProgression({ failure: progressionFailure(failure) });
+ };
+
+ // Hand back findings and let the agent fix, until clean or out of rounds.
+ while (levelGradeIsUsable(classifyBundle(bundle), args.progression
+ ? requireProgressionState(progressionExecution?.state ?? null).attempts.at(-1) ?? null
+ : null) && mayRepair()) {
+ let reportReady = false;
+ const acceptedBundle = bundle;
+ let repairBaselineBundle = bundle;
+ if (args.progression) {
+ progressionSelection = bindProgressionAction(level);
+ const repairOwner = repairOwnerNodeIds(progressionSelection).join('\n') || null;
+ if (repairOwner !== priorRegressionOwner) {
+ priorRegressionReport = null;
+ priorRegressionOwner = repairOwner;
+ }
+ trackProgressionBudget(progressionSelection, priorRepairs + repairs);
+ if (progressionSelection && isProgressionWorkRecipeAction(progressionSelection)
+ && bundle?.selection?.sha256 !== progressionSelection.grader.selectionSha256) {
+ const sequence = requireProgressionState(progressionExecution?.state ?? null).attempts.length + 1;
+ const targetChecks = repairCheckKeys(progressionSelection);
+ const sourceSha256 = hashAppSource(appDir).sha256;
+ const hasCurrentTargetEvidence = bundle?.source?.sha256 === sourceSha256
+ && targetChecks.every(check => bundle?.selection?.reportedChecks?.includes(check));
+ const reusable = reusableRepairEvidence?.bundle.source?.sha256 === sourceSha256
+ && targetChecks.every(check =>
+ reusableRepairEvidence?.bundle.selection?.reportedChecks?.includes(check))
+ ? reusableRepairEvidence : null;
+ let repairResults = hasCurrentTargetEvidence ? null : reusable?.results ?? null;
+ if (reusable && !hasCurrentTargetEvidence) repairBaselineBundle = reusable.bundle;
+ if (!hasCurrentTargetEvidence && !reusable) {
+ const binding = args.recipeBindings.get(level);
+ if (!binding) throw new Error(`L${level} has no recipe binding`);
+ const targetTask = resolveProgressionRepairTarget(binding,
+ requireProgressionState(progressionExecution?.state ?? null));
+ repairResults = join(outputDir, 'repair-grades',
+ `l${level}${featureActionSuffix}-round-${repairs + 1}`);
+ repairBaselineBundle = grade(args, appDir, url,
+ `${args.backend}-l${level}-repair-target${sequence}`,
+ level, track, runId, { out: repairResults, recipeTask: targetTask });
+ }
+ const refreshOutcome = classifyBundle(repairBaselineBundle);
+ const refreshUsable = levelGradeIsUsable(refreshOutcome);
+ if (!refreshUsable) {
+ recordRepairHarnessFailure('refresh-grading-failed',
+ refreshOutcome.reason ?? 'repair target did not produce a reliable grade',
+ repairBaselineBundle);
+ break;
+ }
+ const refreshReport = writeRepairReport(repairResults);
+ if (refreshReport.status === 'failed') {
+ recordRepairHarnessFailure('repair-report', refreshReport.reason);
+ break;
+ }
+ if (refreshReport.status === 3) {
+ bundle = grade(args, appDir, url,
+ `${args.backend}-l${level}-repair-refresh${sequence}`,
+ level, track, runId);
+ if (!levelGradeIsUsable(classifyBundle(bundle))) {
+ recordRepairHarnessFailure('refresh-grading-failed',
+ classifyBundle(bundle).reason ?? 'repair refresh did not produce a reliable grade',
+ bundle);
+ break;
+ }
+ recordRepairProgression();
+ continue;
+ }
+ if (refreshReport.status === 4) {
+ recordMissingRepairFeedback(refreshReport.status);
+ break;
+ }
+ reportReady = true;
+ }
+ }
+ const report = reportReady ? { status: 0 as const } : writeRepairReport();
+ if (report.status === 'failed') {
+ recordRepairHarnessFailure('repair-report', report.reason);
+ break;
+ }
+ if (report.status !== 0) {
+ recordMissingRepairFeedback(report.status);
+ break;
+ }
+
+ const before = repairBaselineBundle?.totals?.score ?? 0;
+ const beforeMax = repairBaselineBundle?.totals?.max ?? 0;
+ const beforeBundle = repairBaselineBundle;
+ // Keep the accepted source outside paths visible to the coding session.
+ const snapshot = join(tmpdir(), `stack-bench-snapshot-${args.backend}-${args.track}-run${args.runIndex}-l${level}`);
+ const gradingSnapshot = `${snapshot}-grading`;
+ const acceptedSource = hashAppSource(appDir);
+ snapshotSource(appDir, snapshot);
+ rmSync(gradingSnapshot, { recursive: true, force: true });
+ if (existsSync(join(appDir, 'stack-bench'))) {
+ cpSync(join(appDir, 'stack-bench'), gradingSnapshot, { recursive: true });
+ }
+ const cleanupRepairSnapshots = () => {
+ rmSync(snapshot, { recursive: true, force: true });
+ rmSync(gradingSnapshot, { recursive: true, force: true });
+ };
+ try {
+ const displayedRepairBudget = args.progression
+ ? progressionRepairLimit
+ : args.repairs;
+ if (args.dependencyPolicy?.definition.repair.selection === 'feature'
+ && progressionSelection && isProgressionWorkRecipeAction(progressionSelection)) {
+ const [nodeId] = progressionSelection.action.repair.nodeIds;
+ const node = requireProgressionState(progressionExecution?.state ?? null).definition.nodes
+ .find(candidate => candidate.id === nodeId);
+ if (!nodeId || !node) {
+ throw new Error('feature repair has no selected feature');
+ }
+ const used = requireProgressionState(progressionExecution?.state ?? null)
+ .nodes[nodeId]?.repairs.used ?? 0;
+ console.log(`--- feature repair ${used + 1}: ${node.title} ---`);
+ } else {
+ console.log(`--- repair ${priorRepairs + repairs + 1}/${displayedRepairBudget} ---`);
+ }
+ const fix = await runAgentForLevel('fix', level,
+ featureActionSequence === null ? undefined : restoreFeatureAcceptedSource);
+ repairCost += fix.costUsd;
+ repairSessions.push(runSessionRecord(fix, priorRepairs + repairs + 1));
+
+ const fixFailure = agentSessionFailure(fix);
+ if (fixFailure) {
+ if (featureActionSequence !== null) {
+ await restoreAcceptedRepair(snapshot, gradingSnapshot);
+ }
+ console.log(` coding session failed: ${fixFailure.reason}; stopping repairs`);
+ bundle = { outcome: fixFailure };
+ repairHistory.push(repairHistoryEntry(repairs + 1, beforeBundle, bundle,
+ 'agent session failed'));
+ repairStopReason = 'agent-session-failure';
+ recordRepairProgression({ failure: progressionFailure(fixFailure) });
+ break;
+ }
+ repairs += 1;
+
+ // Reject contaminated repairs before spending time on grading.
+ const fixLeak = auditContamination(appDir, ownPorts, auditsTranscripts);
+ if (fixLeak) {
+ const buildSession = build ? runSessionRecord(build) : null;
+ const sessions = resumedRepair || !buildSession
+ ? repairSessions : [buildSession, ...repairSessions];
+ const sessionTotals = summarizeSessions(sessions);
+ cleanupRepairSnapshots();
+ abortUnusableSession(`repair ${repairs}`, fixLeak, {
+ level, graded: false, score: null, max: null,
+ selection: bundle?.selection ?? null,
+ ...(resumedRepair
+ ? { resumedRepair: firstBuild }
+ : continuing
+ ? { baseline: firstBuild, resumeCostUsd: requireBuild().costUsd,
+ resumeSession: runSessionRecord(requireBuild()) }
+ : { firstBuild, buildCostUsd: requireBuild().costUsd,
+ buildSessions: [runSessionRecord(requireBuild())] }),
+ repairCostUsd: addCostUsd(repairCost), repairSessions, repairs,
+ ...(resumedRepair ? { priorRepairs,
+ cumulativeRepairs: priorRepairs + repairs } : {}),
+ repair: { status: 'ungraded', limit: displayedRepairBudget,
+ used: priorRepairs + repairs,
+ stopReason: fixLeak.kind === 'harness_failure' ? 'audit-failure' : 'contaminated' },
+ sessionTotals,
+ costUsd: resumedRepair ? addCostUsd(repairCost)
+ : addCostUsd(requireBuild().costUsd, repairCost),
+ durationMs: Date.now() - t0,
+ }, progressionSelection, true);
+ }
+ if (hashAppSource(appDir).sha256 === acceptedSource.sha256) {
+ clearPrivateGradingEvidence(appDir);
+ if (existsSync(gradingSnapshot)) {
+ cpSync(gradingSnapshot, join(appDir, 'stack-bench'), { recursive: true });
+ }
+ const reason = 'repair made no source change';
+ console.log(` ${reason}; ${args.progression
+ ? 'counting the failed attempt'
+ : 'pausing before another paid round'}`);
+ repairHistory.push(repairHistoryEntry(repairs, beforeBundle, beforeBundle, reason));
+ if (args.progression) {
+ if (!restoreProgressionGrade(acceptedBundle,
+ `${args.backend}-l${level}-unchanged${repairs}`)) break;
+ if (!recordRepairProgression({ completedRepair: true })) break;
+ continue;
+ }
+ repairStopReason = 'no-source-change';
+ break;
+ }
+ const repairedSource = `${snapshot}-accepted`;
+ snapshotSource(appDir, repairedSource);
+ try {
+ bundle = await gradeAcceptedSource(repairedSource,
+ `${args.backend}-l${level}-fix${repairs}`);
+ if (!levelGradeIsUsable(classifyBundle(bundle))) {
+ console.log(' repair grade did not complete; retrying the same source once');
+ bundle = await gradeAcceptedSource(repairedSource,
+ `${args.backend}-l${level}-fix${repairs}-retry`);
+ }
+ } finally {
+ rmSync(repairedSource, { recursive: true, force: true });
+ }
+
+ const repairedOutcome = classifyBundle(bundle);
+ if (!levelGradeIsUsable(repairedOutcome)) {
+ const reason = repairedOutcome.reason
+ ?? 'the repaired source did not produce a reliable grade';
+ // The session is paid for and charged. Keep what it produced so a
+ // resume grades it instead of buying another repair; the feature's
+ // status waits for that grade.
+ const candidateDirectory = `repair-candidate-l${level}${featureActionSuffix}`;
+ let preserveFailure: string | null = null;
+ try {
+ repairCandidate = preserveRepairCandidate(candidateDirectory);
+ console.log(` repair grade failed: ${reason}; kept the repaired source at `
+ + `${join(outputDir, candidateDirectory)} for grading on resume`);
+ } catch (error) {
+ preserveFailure = errorMessage(error).split(/\r?\n/)[0]
+ ?? 'could not keep the repaired source';
+ console.log(` repair grade failed: ${reason}; ${preserveFailure}; restoring the accepted source`);
+ await restoreAcceptedRepair(snapshot, gradingSnapshot);
+ }
+ repairHistory.push(repairHistoryEntry(repairs, beforeBundle, bundle,
+ 'repair completed; grading failed'));
+ recordRepairHarnessFailure('repair-grading',
+ preserveFailure ? `${reason}; ${preserveFailure}` : reason, bundle, true);
+ break;
+ }
+
+ if (featureActionSequence !== null && progressionSelection
+ && isProgressionWorkRecipeAction(progressionSelection)
+ && !featureCandidateAccepted(progressionSelection,
+ requireProgressionState(progressionExecution?.state ?? null), bundle)) {
+ const rejectedBundle = bundle;
+ archiveCandidateGrade(`l${level}${featureActionSuffix}-repair${repairs}`);
+ await restoreAcceptedRepair(snapshot, gradingSnapshot);
+ if (!restoreProgressionGrade(acceptedBundle,
+ `${args.backend}-l${level}${featureActionSuffix}-rollback${repairs}`)) break;
+ repairHistory.push(repairHistoryEntry(repairs, beforeBundle, rejectedBundle,
+ 'rolled back because the feature still failed or earlier behavior regressed'));
+ if (!recordRepairProgression({ completedRepair: true })) break;
+ if (pauseForRepeatedFindings()) break;
+ continue;
+ }
+
+ const after = bundle?.totals?.score ?? 0;
+ const afterMax = bundle?.totals?.max ?? 0;
+ // Lost or inconclusive evidence cannot hide a repair regression.
+ let decision = repairEvidenceDecision(beforeBundle, bundle);
+ const regressionDecision = repairRegressionDecision(acceptedBundle, bundle);
+ if (regressionDecision.action === 'rollback-regression') decision = regressionDecision;
+ const shared = decision.shared;
+ if (decision.action === 'keep-setup-repair') {
+ console.log(afterMax > 0
+ ? ` application setup is now gradeable (${after}/${afterMax}); keeping this repair`
+ : ' application setup is still failing; keeping the attempted repair for the next round');
+ repairHistory.push(repairHistoryEntry(repairs, beforeBundle, bundle,
+ afterMax > 0
+ ? 'kept because the app became gradeable'
+ : 'kept to continue repairing application setup'));
+ if (!recordRepairProgression({ completedRepair: true })) break;
+ if (pauseForRepeatedFindings()) break;
+ continue;
+ }
+ if (decision.action === 'rollback-no-comparison') {
+ console.log(' no criteria were conclusively scored in both rounds; rolling back this fix');
+ await restoreAcceptedRepair(snapshot, gradingSnapshot);
+ if (!restoreProgressionGrade(acceptedBundle,
+ `${args.backend}-l${level}-rollback${repairs}`)) break;
+ repairHistory.push(repairHistoryEntry(repairs, beforeBundle, bundle,
+ 'rolled back because the result could not be compared'));
+ if (!recordRepairProgression({ completedRepair: true })) break;
+ if (pauseForRepeatedFindings()) break;
+ continue;
+ }
+ if (shared.points < Math.min(beforeMax, afterMax)) {
+ console.log(` comparing ${shared.points} point(s) across ${shared.count} criteria scored in both rounds`
+ + ` (${before}/${beforeMax} -> ${after}/${afterMax} overall)`);
+ }
+ if (decision.action === 'rollback-regression') {
+ if (shared.regressions.length) {
+ console.log(` broke ${shared.regressions.length} earlier passing check(s); rolling back this fix`);
+ } else if (shared.lostEvidence.length) {
+ console.log(` lost conclusive evidence for ${shared.lostEvidence.length} criterion/criteria; rolling back this fix`);
+ } else if (shared.definitionChanges.length) {
+ console.log(' rubric points changed between grades; rolling back this fix');
+ } else {
+ console.log(` regressed (${shared.before} -> ${shared.after} on shared criteria); rolling back this fix`);
+ }
+ let repairRegression: ProgressionRepairRegression | null = null;
+ let regressionReportFailure: string | null = null;
+ try {
+ if (shared.regressions.length) {
+ const path = join(outputDir, 'repair-reports',
+ `rejected-regression-l${level}${featureActionSuffix}-round${repairs}.md`);
+ sh('node', [join(ROOT, 'dist', 'commands', 'report-bugs.js'), '--app', appDir,
+ '--out', path, '--checks-json', JSON.stringify(shared.regressions),
+ '--regression-context'], { stdio: 'pipe' });
+ repairRegression = {
+ ownerNodeIds: repairOwnerNodeIds(progressionSelection),
+ report: readFileSync(path, 'utf8'),
+ };
+ priorRegressionReport = path;
+ }
+ } catch (error) {
+ regressionReportFailure = errorMessage(error).split(/\r?\n/)[0]
+ ?? 'regression report generation failed';
+ } finally {
+ await restoreAcceptedRepair(snapshot, gradingSnapshot);
+ }
+ if (regressionReportFailure) {
+ repairHistory.push(repairHistoryEntry(repairs, beforeBundle, bundle,
+ 'repair completed; regression reporting failed'));
+ recordRepairHarnessFailure('repair-regression-report', regressionReportFailure,
+ null, true);
+ break;
+ }
+ if (!restoreProgressionGrade(acceptedBundle,
+ `${args.backend}-l${level}-rollback${repairs}`)) break;
+ regressed = true;
+ repairHistory.push(repairHistoryEntry(repairs, beforeBundle, bundle,
+ 'rolled back because earlier behavior regressed'));
+ if (!recordRepairProgression({ repairRegression, completedRepair: true })) break;
+ if (pauseForRepeatedFindings()) break;
+ continue;
+ }
+ priorRegressionReport = null;
+ if (shared.after === shared.before) {
+ const remaining = displayedRepairBudget - priorRepairs - repairs;
+ console.log(` ${formatRepairProgress(shared, { before, beforeMax, after, afterMax })}; `
+ + (remaining > 0 ? `${remaining} repair(s) remain` : 'repair budget exhausted'));
+ }
+ repairHistory.push(repairHistoryEntry(repairs, beforeBundle, bundle,
+ shared.after === shared.before ? 'kept with no score gain' : 'kept'));
+ if (!recordRepairProgression({ completedRepair: true })) break;
+ if (pauseForRepeatedFindings()) break;
+ } finally {
+ cleanupRepairSnapshots();
+ }
+ }
+
+ // Missing grade evidence is not a zero score.
+ const progressionState = progressionExecution
+ ? requireProgressionState(progressionExecution.state) : null;
+ const progressionAttempt = progressionState
+ ? progressionState.attempts.findLast(attempt => attempt.level === level) ?? null
+ : null;
+ const levelBundle = progressionAttempt?.outcome === 'inconclusive'
+ ? bundle : progressionBundles.get(level) ?? bundle;
+ const finalBundleOutcome = classifyBundle(levelBundle);
+ // Progression uses stricter evidence rules than a regular scored bundle.
+ // Store one answer when a selected check is not measured: the raw bundle
+ // remains available for diagnosis, but the level is not a usable grade.
+ const graded = levelGradeIsUsable(finalBundleOutcome,
+ args.progression ? progressionAttempt : null);
+ const finalTotals = graded ? levelBundle?.totals ?? null : null;
+ const nodeRepairs = progressionState
+ ? dependencyRepairRecords(progressionState, level, levelRepairNodeIds)
+ : null;
+ const repairLimit = progressionExecution
+ ? Math.max(priorRepairs + repairs, progressionRepairLimit)
+ : args.repairs;
+ const repairBudgetExhausted = progressionExecution
+ ? finalBundleOutcome.kind === 'app_failure'
+ && progressionNext?.type !== 'repair'
+ : priorRepairs + repairs >= args.repairs;
+ const repairStatus: RepairStatus = repairStopReason === 'no-source-change' ? 'incomplete'
+ : !graded ? 'ungraded'
+ : finalBundleOutcome.kind === 'passed' ? (repairs > 0 ? 'corrected' : 'not-needed')
+ : repairBudgetExhausted ? 'budget-exhausted' : 'incomplete';
+ const stopReasons: Record = {
+ 'not-needed': 'not-needed',
+ corrected: 'passed',
+ 'budget-exhausted': 'budget-exhausted',
+ incomplete: null,
+ ungraded: null,
+ };
+ const stopReason = repairStopReason ?? stopReasons[repairStatus];
+ const repair = {
+ status: repairStatus,
+ limit: repairLimit,
+ used: priorRepairs + repairs,
+ ...(!args.progression ? { stallLimitRounds: args.maxStalledRepairs } : {}),
+ stopReason,
+ ...(nodeRepairs ? { nodeRepairs } : {}),
+ ...(repairCandidate ? { candidate: repairCandidate } : {}),
+ };
+ const latestProgressionAttempt = progressionState?.attempts.at(-1) ?? null;
+ const featureDepthContinues = featureActionSequence !== null
+ && progressionState?.phase === 'active' && progressionState.level === level;
+ if (progressionState && latestProgressionAttempt && latestProgressionAttempt.level !== level) {
+ const prior = run.levels.find(item => item.level === latestProgressionAttempt.level);
+ const latestBundle = progressionBundles.get(latestProgressionAttempt.level);
+ if (prior && latestBundle) {
+ const latestOutcome = classifyBundle(latestBundle);
+ const latestGraded = levelGradeIsUsable(latestOutcome, latestProgressionAttempt);
+ prior.graded = latestGraded;
+ prior.score = latestGraded ? latestBundle.totals?.score ?? null : null;
+ prior.max = latestGraded ? latestBundle.totals?.max ?? null : null;
+ prior.regression = latestBundle.totals?.regression ?? null;
+ prior.selection = latestBundle.selection ?? null;
+ prior.contractPass = latestBundle.totals?.contractPass ?? null;
+ prior.code = latestBundle.code ?? null;
+ prior.repair = { ...repair,
+ nodeRepairs: dependencyRepairRecords(
+ progressionState, latestProgressionAttempt.level, levelRepairNodeIds),
+ };
+ prior.stalled = repairStatus === 'budget-exhausted';
+ prior.outcome = latestOutcome;
+ }
+ }
+ if (continuing) {
+ const continuation = requireContinuation(run);
+ continuation.cumulativeRepairsAfter = continuation.cumulativeRepairsBefore + repairs;
+ }
+ let checkpoint = null;
+ if (graded && !featureDepthContinues
+ && (!latestProgressionAttempt || latestProgressionAttempt.level === level)) {
+ try {
+ checkpoint = preserveLevelCheckpoint({
+ appDir,
+ outputDir: args.out,
+ runId,
+ identities: run.identities,
+ track: args.track,
+ backend: args.backend,
+ level,
+ repair,
+ outcome: finalBundleOutcome,
+ selectionSha256: levelBundle?.selection?.sha256 ?? null,
+ });
+ console.log(` kept the L${level} source checkpoint at ${join(args.out, checkpoint.directory)}`);
+ } catch (error) {
+ console.log(` !! could not keep the L${level} source checkpoint: ${errorMessage(error).split('\n')[0]}`);
+ }
+ }
+ if (!graded) {
+ console.log(` L${level}: GRADING DID NOT COMPLETE — no usable bundle. ` +
+ `Score is unknown, not zero; re-grade this level before using the run.`);
+ }
+ const buildSession = build ? runSessionRecord(build) : null;
+ const requireBuildSession = (): RunSessionRecord => {
+ if (!buildSession) throw new Error(`level ${level} has no coding session`);
+ return buildSession;
+ };
+ const sessionTotals = summarizeSessions(resumedRepair || !buildSession ? repairSessions
+ : [buildSession, ...repairSessions]);
+ appendLevelRecord({
+ level,
+ graded,
+ score: finalTotals?.score ?? null,
+ max: finalTotals?.max ?? null,
+ // Preserve earlier-level guarantees in the durable result.
+ regression: levelBundle?.totals?.regression ?? null,
+ selection: levelBundle?.selection ?? null,
+ ...(resumedRepair
+ ? { resumedRepair: firstBuild }
+ : continuing
+ ? { baseline: firstBuild, resumeCostUsd: requireBuild().costUsd,
+ resumeSession: requireBuildSession() }
+ : featureActionSequence !== null
+ ? { buildCostUsd: requireBuild().costUsd, buildSessions: [requireBuildSession()] }
+ : { firstBuild, buildCostUsd: requireBuild().costUsd,
+ buildSessions: [requireBuildSession()] }),
+ contractPass: levelBundle?.totals?.contractPass ?? null,
+ code: levelBundle?.code ?? null,
+ repairCostUsd: addCostUsd(repairCost),
+ repairSessions,
+ repairHistory,
+ sessionTotals,
+ tokens: sessionTotals.tokens,
+ usage: sessionTotals.usage,
+ turns: sessionTotals.turns,
+ promptBytes: sessionTotals.promptBytes,
+ tokensPerTurn: sessionTotals.turns
+ ? Math.round(sessionTotals.tokens / sessionTotals.turns) : null,
+ // Record actual reasoning because the provider default is not pinned.
+ thinking: sessionTotals.thinking,
+ repairs,
+ ...(resumedRepair ? { priorRepairs,
+ cumulativeRepairs: priorRepairs + repairs } : {}),
+ repair,
+ checkpoint,
+ // Keep the summary flag derived from the typed status so the two cannot drift.
+ stalled: repairStatus === 'budget-exhausted'
+ || ['repeated-findings', 'no-source-change'].includes(repairStopReason ?? ''),
+ regressed,
+ outcome: finalBundleOutcome,
+ durationSec: Math.round((Date.now() - t0) / 1000),
+ });
+ if (!args.progression || requireProgressionState(progressionExecution?.state ?? null).attempts
+ .some(attempt => attempt.level === level && attempt.outcome === 'conclusive')) {
+ if (!featureDepthContinues && !run.validation.ladder.completedLevels.includes(level)) {
+ run.validation.ladder.completedLevels.push(level);
+ }
+ }
+ writeRunJson(join(args.out, ARTIFACT_FILE.run), run);
+ const blockedLevels = args.levelList.filter(candidate => candidate > level);
+ if (args.progression) {
+ const progressionState = requireProgressionState(progressionExecution?.state ?? null);
+ if (progressionState.phase === 'terminal') {
+ if (blockedLevels.length) run.validation.ladder.stoppedAfterLevel = level;
+ run.validation.ladder.blockedLevels = blockedLevels;
+ writeRunJson(join(args.out, ARTIFACT_FILE.run), run);
+ break;
+ }
+ if (progressionState.level <= level) {
+ if (progressionState.attempts.at(-1)?.outcome === 'inconclusive') {
+ run.validation.ladder.stoppedAfterLevel = level;
+ run.validation.ladder.blockedLevels = [level, ...blockedLevels];
+ writeRunJson(join(args.out, ARTIFACT_FILE.run), run);
+ break;
+ }
+ if (featureActionSequence !== null && progressionState.level === level) {
+ levelIndex -= 1;
+ continue;
+ }
+ throw new Error(`dependency progression did not leave L${level} after its repair budget`);
+ }
+ continue;
+ }
+ if (blockedLevels.length && !ladderMayAdvance(finalBundleOutcome)) {
+ run.validation.ladder.stoppedAfterLevel = level;
+ run.validation.ladder.blockedLevels = blockedLevels;
+ writeRunJson(join(args.out, ARTIFACT_FILE.run), run);
+ console.log(` ladder paused after L${level}: L${level} must pass before `
+ + `${blockedLevels.map(candidate => `L${candidate}`).join(', ')} can start`);
+ console.log(' inspect the failures, then explicitly grant more repairs or correct the benchmark');
+ break;
+ }
+ }
+
+ if (args.mutations) {
+ console.log(`\n================ ${args.backend} mutation control ================`);
+ const pristineOutcome = aggregateRunOutcome(run.levels);
+ if (args.referenceMutationOnly || mutationControlEligible(pristineOutcome)) {
+ args.parentAttemptId = runId;
+ const baselineBundle = pristineMutationBaselinePath(args);
+ if (baselineBundle) args.mutationBaselineBundle = baselineBundle;
+ else delete args.mutationBaselineBundle;
+ run.mutationControl = runMutationControl(args, appDir, url, track,
+ run.setup?.isolation?.imageId ?? null);
+ } else {
+ console.log(` skipped: pristine outcome is ${pristineOutcome.kind}`);
+ run.mutationControl = { ok: false, skipped: true,
+ outcome: { kind: pristineOutcome.kind, phase: 'mutation-control-prerequisite',
+ reason: `pristine outcome is ${pristineOutcome.kind}` } };
+ }
+ writeRunJson(join(args.out, ARTIFACT_FILE.run), run);
+ }
+
+ // Record a final transcript audit in addition to the per-session hard gates.
+ // The same retry and diagnostic path is used at both gates.
+ let finalAuditFailure = null;
+ const finalAudit = auditContamination(appDir, ownPorts, auditsTranscripts);
+ if (!finalAudit) {
+ run.contaminated = false;
+ run.contamination = { evidence: 'no agent access to private benchmark files detected',
+ verdict: 'private-access audit passed' };
+ } else if (finalAudit.kind === 'contaminated') {
+ run.contaminated = true;
+ run.contamination = { evidence: finalAudit.evidence, verdict: finalAudit.verdict };
+ console.log('\n !! CONTAMINATED: this build read the harness that grades it:');
+ for (const evidence of finalAudit.evidence) console.log(` ${evidence}`);
+ console.log(' Scores from this run must not be quoted.');
+ } else {
+ run.contaminated = false;
+ run.contamination = { evidence: finalAudit.evidence, verdict: finalAudit.verdict };
+ const reason = finalAudit.evidence.join('; ');
+ finalAuditFailure = { kind: 'harness_failure', phase: 'contamination-audit', reason,
+ appFailures: [], inconclusive: [], harnessFailures: [reason] };
+ console.log('\n !! AUDIT DID NOT COMPLETE. Scores from this run must not be quoted.');
+ }
+
+ // Keep the transcript evidence outside the provider CLI's prunable store.
+ try { archiveTranscripts(appDir, artifactLabel); }
+ catch { console.log(' (transcript archiving failed — evidence is on a 30-day timer)'); }
+
+ run.outcome = finalAuditFailure ?? (args.referenceMutationOnly && run.mutationControl?.ok
+ ? { kind: 'passed', phase: 'mutation-control', reason: null,
+ appFailures: [], inconclusive: [], harnessFailures: [] }
+ : aggregateRunOutcome(run.levels));
+ if (args.mutations && !run.mutationControl?.ok && !run.mutationControl?.skipped) {
+ run.outcome = { kind: run.mutationControl?.outcome?.kind === 'incomplete'
+ ? 'incomplete' : 'harness_failure', phase: 'mutation-control',
+ reason: run.mutationControl?.outcome?.reason
+ ?? run.mutationControl?.processError
+ ?? 'one or more declared mutations were not cleanly caught',
+ appFailures: [], inconclusive: [] };
+ }
+
+ if (finalPackageEvidenceRequired(run.outcome, run.levels)) {
+ try {
+ preserveFinalPackageEvidence({ appDir, outputDir });
+ console.log(` source kept at ${join(outputDir, 'source')}`);
+ console.log(` grading detail kept at ${join(outputDir, 'grading')}`);
+ } catch (error) {
+ const reason = errorMessage(error).split(/\r?\n/)[0] ?? 'evidence preservation failed';
+ run.outcome = { kind: 'harness_failure', phase: 'evidence-preservation', reason,
+ appFailures: [], inconclusive: [], harnessFailures: [reason] };
+ console.log(` !! ${reason}`);
+ }
+ }
+
+ finalizeRunTotals(run, started, { costComplete: runCostComplete });
+ if (args.repairGrant) {
+ const continuation = requireContinuation(run);
+ const totals = requireRunTotals(run);
+ continuation.cumulativeCostAfterUsd = addCostUsd(continuation.cumulativeCostBeforeUsd, totals.costUsd);
+ continuation.cumulativeDurationAfterSec = continuation.cumulativeDurationBeforeSec + totals.durationSec;
+ }
+ run.completedAt = new Date().toISOString();
+ writeRunJson(join(args.out, ARTIFACT_FILE.run), run);
+
+ console.log(`\n================ ${args.backend} summary ================`);
+ for (const l of run.levels) {
+ console.log(` ${formatLevelSummary(l)}`);
+ }
+ const totals = requireRunTotals(run);
+ console.log(` TOTAL ${totals.score}/${totals.max} ` +
+ `$${totals.costUsd} ${totals.repairs} repair(s) ${totals.durationSec}s`);
+ console.log(` ${join(outputDir, ARTIFACT_FILE.run)}`);
+
+ teardown();
+
+ // Remove only the temporary directory created by this run.
+ if (ownWorkDir) {
+ try {
+ rmSync(dirname(appDir), { recursive: true, force: true });
+ } catch {
+ console.log(` (work dir still held: ${dirname(appDir)} — the next sweep will take it)`);
+ }
+ }
+ process.exitCode = runExitCode(run.outcome);
+}
+
+if (process.argv[1] && pathToFileURL(resolve(process.argv[1])).href === import.meta.url) {
+ main().catch(error => {
+ console.error(error instanceof Error ? error.stack ?? error.message : errorMessage(error));
+ try { emergencyTeardown?.(); }
+ catch (cleanupError) {
+ console.error(`cleanup after failure also failed: ${errorMessage(cleanupError).split(/\r?\n/)[0]}`);
+ }
+ process.exitCode = 1;
+ });
+}
diff --git a/tools/stack-bench/commands/campaign-cli.ts b/tools/stack-bench/commands/campaign-cli.ts
new file mode 100644
index 00000000000..38c927d4ac0
--- /dev/null
+++ b/tools/stack-bench/commands/campaign-cli.ts
@@ -0,0 +1,288 @@
+#!/usr/bin/env node
+
+import { resolve } from 'node:path';
+import { pathToFileURL } from 'node:url';
+
+import { compileCampaignFile } from '../src/campaigns/campaign-compiler.js';
+import { CAMPAIGN_MODE_REGISTRY } from '../src/campaigns/campaign-mode.js';
+import { executeCampaign, inspectCampaign, reconcileCampaign }
+ from '../src/campaigns/campaign-runner.js';
+import { inspectCampaignSummary } from '../src/campaigns/campaign-inspection.js';
+import { generateCampaignReport } from '../src/campaigns/campaign-report.js';
+import { grantCampaignDependencyRepairs }
+ from '../src/campaigns/campaign-progression-grant.js';
+import { auditProgressionReferenceCampaign, formatProgressionReferenceCampaignAudit }
+ from '../src/campaigns/progression-reference-campaign-audit.js';
+import type { ReferenceCampaignAudit }
+ from '../src/campaigns/progression-reference-campaign-audit.js';
+import { prepareCampaignExtension } from '../src/campaigns/campaign-extension.js';
+
+interface CampaignSummaryPlan {
+ id: string;
+ version: string;
+ contentSha256: string;
+}
+
+interface CampaignSummaryState {
+ status: string;
+ summary: unknown;
+ attempts: Array<{
+ plan: { id: string };
+ status: string;
+ executions: Array<{
+ id: string;
+ outcome: unknown;
+ reason: string | null;
+ }>;
+ }>;
+}
+
+interface ReferenceCampaignPlan {
+ attempts: Array<{
+ mode?: { id?: string };
+ agentAdapter?: string;
+ }>;
+}
+
+interface ReferenceCampaignState {
+ status: string;
+}
+
+interface ResumeCampaign {
+ plan: {
+ contentSha256: string;
+ definition: { mode?: { id?: string } };
+ };
+ state: {
+ status: string;
+ attempts: Array<{ executions: readonly unknown[] }>;
+ };
+}
+
+type ReferenceCampaignAuditFunction = (directory: string) => ReferenceCampaignAudit | null;
+
+export type CampaignArgs =
+ | { command: 'modes' }
+ | { command: 'validate'; path: string }
+ | { command: 'show'; path: string }
+ | { command: 'status'; directory: string; full: boolean }
+ | { command: 'inspect'; directory: string }
+ | { command: 'report'; directory: string }
+ | { command: 'audit'; directory: string }
+ | { command: 'grant-repairs'; directory: string; attemptId: string; grantId: string;
+ level: number; nodeIds: string[]; repairs: number }
+ | { command: 'extend'; path: string; parentDirectory: string; fromDepth: number;
+ directory: string }
+ | { command: 'trial'; path: string; directory: string }
+ | { command: 'run'; path: string; directory: string }
+ | { command: 'resume'; path: string; directory: string }
+ | { command: 'reconcile'; path: string; directory: string };
+
+function isOneOf(value: string | undefined,
+ values: readonly T[]): value is T {
+ return value !== undefined && values.some(candidate => candidate === value);
+}
+
+export function campaignStateSummary(plan: CampaignSummaryPlan, state: CampaignSummaryState) {
+ const failures = state.attempts.flatMap(attempt => {
+ const execution = attempt.executions.at(-1);
+ if (!execution || execution.outcome === null || execution.outcome === 'passed') return [];
+ return [{
+ attempt: attempt.plan.id,
+ status: attempt.status,
+ execution: execution.id,
+ outcome: execution.outcome,
+ reason: execution.reason,
+ }];
+ });
+ return {
+ campaign: { id: plan.id, version: plan.version, sha256: plan.contentSha256 },
+ status: state.status,
+ summary: state.summary,
+ failures,
+ };
+}
+
+export function auditCompletedReferenceCampaign(directory: string, plan: ReferenceCampaignPlan,
+ state: ReferenceCampaignState, {
+ audit = auditProgressionReferenceCampaign,
+}: { audit?: ReferenceCampaignAuditFunction } = {}): ReferenceCampaignAudit | null {
+ const hasReferenceProgression = plan.attempts.some(attempt =>
+ attempt.mode?.id === 'dependency' && attempt.agentAdapter === 'reference-fixture');
+ return state.status === 'completed' && hasReferenceProgression ? audit(directory) : null;
+}
+
+export function validateResumeCampaignState(
+ requested: { contentSha256: string }, existing: T): T {
+ if (requested.contentSha256 !== existing.plan.contentSha256) {
+ throw new Error('resume requires the exact campaign plan already stored in the output directory');
+ }
+ if (existing.plan.definition.mode?.id !== 'dependency') {
+ throw new Error('resume is available only for dependency campaigns');
+ }
+ const executions = existing.state.attempts.reduce((total, attempt) =>
+ total + attempt.executions.length, 0);
+ if (existing.state.status !== 'prepared' || executions < 1) {
+ throw new Error('resume requires a dependency campaign with scheduled work');
+ }
+ return existing;
+}
+
+export function validateResumeCampaign(path: string, directory: string): ResumeCampaign {
+ return validateResumeCampaignState(compileCampaignFile(path), inspectCampaign(directory));
+}
+
+export function parseCampaignArgs(argv: string[]): CampaignArgs {
+ const [command, path, ...rest] = argv.slice(2);
+ if (command === 'modes' && path === undefined) return { command };
+ if (isOneOf(command, ['validate', 'show']) && path && rest.length === 0) {
+ return { command, path: resolve(path) };
+ }
+ if (command === 'status' && path
+ && (rest.length === 0 || (rest.length === 1 && rest[0] === '--full'))) {
+ return { command, directory: resolve(path), full: rest.length === 1 };
+ }
+ if (isOneOf(command, ['inspect', 'report', 'audit']) && path && rest.length === 0) {
+ return { command, directory: resolve(path) };
+ }
+ if (command === 'grant-repairs' && path) {
+ const values: { attemptId?: string; grantId?: string; level?: number; repairs?: number;
+ nodeIds: string[] } = { nodeIds: [] };
+ const seen = new Set();
+ for (let index = 0; index < rest.length; index += 2) {
+ const flag = rest[index];
+ const value = rest[index + 1];
+ if (flag === undefined || value === undefined
+ || !['--attempt', '--grant-id', '--level', '--feature', '--repairs'].includes(flag)
+ || (flag !== '--feature' && seen.has(flag))) {
+ throw new Error(`invalid or duplicate grant-repairs option ${String(flag)}`);
+ }
+ seen.add(flag);
+ if (flag === '--attempt') values.attemptId = value;
+ else if (flag === '--grant-id') values.grantId = value;
+ else if (flag === '--level') values.level = Number(value);
+ else if (flag === '--repairs') values.repairs = Number(value);
+ else values.nodeIds.push(value);
+ }
+ if (!values.attemptId || !values.grantId || typeof values.level !== 'number'
+ || !Number.isSafeInteger(values.level) || typeof values.repairs !== 'number'
+ || !Number.isSafeInteger(values.repairs) || values.nodeIds.length === 0) {
+ throw new Error('grant-repairs requires --attempt, --grant-id, --level, '
+ + 'one or more --feature values, and --repairs');
+ }
+ return { command, directory: resolve(path), attemptId: values.attemptId,
+ grantId: values.grantId, level: values.level, nodeIds: values.nodeIds,
+ repairs: values.repairs };
+ }
+ if (command === 'extend' && path && rest.length === 6
+ && rest[0] === '--from' && rest[2] === '--depth' && rest[4] === '--out') {
+ const fromDepth = Number(rest[3]);
+ if (!Number.isSafeInteger(fromDepth) || fromDepth < 1) {
+ throw new Error('extend --depth must be a positive integer');
+ }
+ return { command, path: resolve(path), parentDirectory: resolve(rest[1]!),
+ fromDepth, directory: resolve(rest[5]!) };
+ }
+ if (isOneOf(command, ['trial', 'run', 'resume', 'reconcile'])
+ && path && rest.length === 2 && rest[0] === '--out') {
+ return { command, path: resolve(path), directory: resolve(rest[1]!) };
+ }
+ throw new Error('usage: campaign-cli.js modes | validate|show '
+ + '| trial|run|resume|reconcile --out '
+ + '| extend --from --depth --out '
+ + '| status [--full] | inspect|report|audit '
+ + '| grant-repairs --attempt --grant-id --level '
+ + '--feature [--feature ...] --repairs ');
+}
+
+async function main() {
+ const args = parseCampaignArgs(process.argv);
+ if (args.command === 'modes') {
+ console.log(JSON.stringify(CAMPAIGN_MODE_REGISTRY.ids.map(value => {
+ const [id, version] = value.split('@');
+ return { id, version };
+ }), null, 2));
+ return;
+ }
+ if (args.command === 'status') {
+ const campaign = inspectCampaign(args.directory, { requireCurrentInputs: false });
+ console.log(JSON.stringify(args.full
+ ? campaign.state
+ : campaignStateSummary(campaign.plan, campaign.state), null, 2));
+ return;
+ }
+ if (args.command === 'inspect') {
+ console.log(JSON.stringify(inspectCampaignSummary(args.directory), null, 2));
+ return;
+ }
+ if (args.command === 'report') {
+ const generated = generateCampaignReport(args.directory);
+ console.log(`${generated.reportPath}\n${generated.htmlPath}\n${generated.report.contentSha256}`);
+ return;
+ }
+ if (args.command === 'audit') {
+ const report = auditProgressionReferenceCampaign(args.directory);
+ if (report === null) throw new Error('campaign has no dependency reference attempts to audit');
+ console.log(formatProgressionReferenceCampaignAudit(report));
+ if (!report.ok) process.exitCode = 1;
+ return;
+ }
+ if (args.command === 'grant-repairs') {
+ console.log(JSON.stringify(grantCampaignDependencyRepairs(args.directory, {
+ attemptId: args.attemptId,
+ grantId: args.grantId,
+ level: args.level,
+ nodeIds: args.nodeIds,
+ repairs: args.repairs,
+ }), null, 2));
+ return;
+ }
+ if (args.command === 'extend') {
+ prepareCampaignExtension(args.path, args.parentDirectory, args.directory, args.fromDepth);
+ const plan = compileCampaignFile(args.path);
+ const state = await executeCampaign(args.path, args.directory, { mode: 'frozen' });
+ console.log(JSON.stringify(campaignStateSummary(plan, state), null, 2));
+ if (state.status !== 'completed') process.exitCode = 1;
+ return;
+ }
+ const plan = compileCampaignFile(args.path);
+ if (args.command === 'reconcile') {
+ const state = reconcileCampaign(args.path, args.directory);
+ console.log(JSON.stringify(campaignStateSummary(plan, state), null, 2));
+ return;
+ }
+ if (args.command === 'trial' || args.command === 'run' || args.command === 'resume') {
+ if (args.command === 'resume') validateResumeCampaign(args.path, args.directory);
+ const cancellation = new AbortController();
+ const cancel = () => cancellation.abort();
+ process.on('SIGINT', cancel);
+ process.on('SIGTERM', cancel);
+ let state;
+ try {
+ const executionMode = args.command === 'trial'
+ || (args.command === 'resume' && plan.state === 'draft')
+ ? 'model-free-trial' : 'frozen';
+ state = await executeCampaign(args.path, args.directory, {
+ mode: executionMode,
+ signal: cancellation.signal,
+ });
+ } finally {
+ process.off('SIGINT', cancel);
+ process.off('SIGTERM', cancel);
+ }
+ console.log(JSON.stringify(campaignStateSummary(plan, state), null, 2));
+ const audit = auditCompletedReferenceCampaign(args.directory, plan, state);
+ if (audit !== null) console.log(formatProgressionReferenceCampaignAudit(audit));
+ if (state.status !== 'completed' || audit?.ok === false) process.exitCode = 1;
+ return;
+ }
+ if (args.command === 'show') console.log(JSON.stringify(plan, null, 2));
+ else console.log(`${plan.id}@${plan.version} ${plan.state}: ${plan.summary.attempts} attempts, ${plan.contentSha256}`);
+}
+
+if (process.argv[1] && pathToFileURL(resolve(process.argv[1])).href === import.meta.url) {
+ main().catch((error: unknown) => {
+ console.error(error instanceof Error ? error.message : String(error));
+ process.exitCode = 2;
+ });
+}
diff --git a/tools/stack-bench/commands/check-actions.ts b/tools/stack-bench/commands/check-actions.ts
new file mode 100644
index 00000000000..b213a3fc8a6
--- /dev/null
+++ b/tools/stack-bench/commands/check-actions.ts
@@ -0,0 +1,113 @@
+#!/usr/bin/env node
+// Probes are unauthenticated or malformed and must never mutate data.
+
+import { parseArgs as parseNodeArgs } from 'node:util';
+
+import { emptyArtifactIdentities, writeArtifact } from '../src/evidence/artifacts.js';
+import { loadTrack } from '../src/composition/tracks.js';
+import { STACK_ADAPTER_REGISTRY } from '../src/stacks/stack-adapters.js';
+
+interface CheckActionsArgs {
+ backend?: string;
+ url?: string;
+ app?: string;
+ out?: string;
+ track?: string;
+ quiet?: boolean;
+ parentAttemptId?: string;
+}
+
+import type { NamedAction } from '../src/composition/tracks.js';
+
+interface ActionResult {
+ id: string;
+ ok: boolean;
+ status: number;
+ note: string;
+}
+
+function parseArgs(argv: string[]): CheckActionsArgs {
+ const { values } = parseNodeArgs({ args: argv.slice(2), options: {
+ backend: { type: 'string' }, url: { type: 'string' }, app: { type: 'string' },
+ out: { type: 'string' }, track: { type: 'string' }, quiet: { type: 'boolean' },
+ 'parent-attempt-id': { type: 'string' },
+ } });
+ const a: CheckActionsArgs = { backend: values.backend, url: values.url, app: values.app,
+ out: values.out, track: values.track, quiet: values.quiet,
+ parentAttemptId: values['parent-attempt-id'] };
+ if (!a.backend) { console.error('--backend is required'); process.exit(2); }
+ return a;
+}
+
+const args = parseArgs(process.argv);
+const backend = args.backend;
+if (!backend) throw new Error('--backend is required');
+
+// Use non-writing probes declared by the selected track.
+const track = args.track ? loadTrack(args.track) : null;
+const ACTIONS = (track?.actions ?? []).map(action => ({ ...action,
+ http: { method: 'POST', path: action.path } }));
+if (!ACTIONS.length) {
+ if (!args.quiet) console.log(` no named actions declared for track "${args.track ?? '(none)'}" — nothing to check`);
+ if (args.out) {
+ const id = `${args.parentAttemptId ?? 'actions'}-action-check`;
+ writeArtifact(args.out, { kind: 'action_check', id,
+ attempt: { id, parentId: args.parentAttemptId ?? null },
+ identities: emptyArtifactIdentities({ stackAdapter: { id: backend } }),
+ payload: { backend, results: [], missing: [] } });
+ }
+ process.exit(0);
+}
+
+// SpacetimeDB control targets come from the authenticated lease. Client config
+// is app-controlled input and may use environment expressions rather than
+// literals; it is neither authoritative nor safe for harness operations.
+const adapter = STACK_ADAPTER_REGISTRY.get(backend);
+const spacetime = adapter.grading.context({ requireBuildContainer: false });
+
+async function probe(action: NamedAction): Promise> {
+ try {
+ const request = adapter.namedAction.request(
+ { action, input: { args: action.args }, spacetime, url: args.url });
+ if (!request.url) return { ok: false, status: 0, note: 'no --url given for a server-based backend' };
+ const r = await fetch(request.url, {
+ method: request.method ?? 'POST',
+ headers: { 'Content-Type': 'application/json' },
+ body: request.body,
+ });
+ const rejectedByApplication = 'applicationRejectionStatuses' in request
+ && request.applicationRejectionStatuses.includes(r.status);
+ const recognizedWithoutRunning = r.status >= 400 && r.status < 500
+ && ![404, 405, 429].includes(r.status);
+ const ok = r.ok || rejectedByApplication || recognizedWithoutRunning;
+ return { ok, status: r.status,
+ note: r.status === 404 ? request.missingNote
+ : ok ? '' : `action probe returned HTTP ${r.status}` };
+ } catch (error: unknown) {
+ const message = error instanceof Error ? error.message : String(error);
+ return { ok: false, status: 0, note: (message.split('\n')[0] ?? '').slice(0, 90) };
+ }
+}
+
+const results: ActionResult[] = await Promise.all(ACTIONS.map(async action => ({
+ id: action.id,
+ ...(await probe(action)),
+})));
+
+const missing = results.filter(r => !r.ok);
+if (!args.quiet) {
+ for (const r of results) {
+ console.log(` ${r.ok ? 'ready' : 'UNUSABLE'} ${r.id.padEnd(11)} ${r.status ? `HTTP ${r.status}` : ''} ${r.note}`);
+ }
+ console.log(missing.length
+ ? `\n${missing.length} named action(s) unusable — contention and volume tests cannot be issued against this app.`
+ : '\nall named actions are ready.');
+}
+if (args.out) {
+ const id = `${args.parentAttemptId ?? 'actions'}-action-check`;
+ writeArtifact(args.out, { kind: 'action_check', id,
+ attempt: { id, parentId: args.parentAttemptId ?? null },
+ identities: emptyArtifactIdentities({ stackAdapter: { id: backend } }),
+ payload: { backend, results, missing: missing.map(m => m.id) } });
+}
+process.exit(missing.length ? 1 : 0);
diff --git a/tools/stack-bench/commands/check-calibration.ts b/tools/stack-bench/commands/check-calibration.ts
new file mode 100644
index 00000000000..188a72d9833
--- /dev/null
+++ b/tools/stack-bench/commands/check-calibration.ts
@@ -0,0 +1,71 @@
+#!/usr/bin/env node
+
+import { existsSync, readFileSync, readdirSync } from 'node:fs';
+import { dirname, join, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { parseArgs } from 'node:util';
+
+import { compileCalibrationDefinition, compileCalibrationFile } from '../src/composition/calibration-compiler.js';
+import { buildRecipeRelease } from '../src/composition/recipe-release.js';
+import { listTracks, TRACKS_DIR } from '../src/composition/tracks.js';
+
+import { STACK_BENCH_ROOT as ROOT } from '../src/package-root.js';
+
+export interface CalibrationCheckResult {
+ track: string;
+ id: string;
+ version: string;
+ state: string;
+ recipe: string;
+ controls: number;
+ stacks: number;
+ contentSha256: string;
+}
+
+export function checkCalibrations(
+ { trackName = null }: { trackName?: string | null } = {},
+): CalibrationCheckResult[] {
+ const availableTracks = listTracks({ includeInternal: true });
+ if (trackName && !availableTracks.includes(trackName)) {
+ throw new Error(`unknown calibration track ${trackName}`);
+ }
+ const tracks = trackName ? [trackName] : availableTracks;
+ const results: CalibrationCheckResult[] = [];
+ for (const name of tracks) {
+ const trackRoot = join(TRACKS_DIR, name);
+ const directory = join(trackRoot, 'composition', 'calibrations');
+ if (!existsSync(directory)) continue;
+ for (const file of readdirSync(directory).filter(candidate => candidate.endsWith('.json')).sort()) {
+ const path = join(directory, file);
+ const source = `composition/calibrations/${file}`;
+ const input = JSON.parse(readFileSync(path, 'utf8'));
+ const definition = compileCalibrationDefinition(input, { source });
+ const recipePath = resolve(dirname(path), definition.recipe.path);
+ const release = buildRecipeRelease(recipePath, { trackRoot });
+ const plan = compileCalibrationFile(path, { trackRoot, stackBenchRoot: ROOT, release });
+ results.push({ track: name, id: plan.id, version: plan.version, state: plan.state,
+ recipe: plan.recipe.id, controls: plan.controls.length, stacks: plan.qualification.stacks.length,
+ contentSha256: plan.contentSha256 });
+ }
+ }
+ return results;
+}
+
+function main() {
+ const { values } = parseArgs({ args: process.argv.slice(2), options: {
+ track: { type: 'string' },
+ }, strict: true, allowPositionals: false });
+ const results = checkCalibrations({ trackName: values.track ?? null });
+ for (const result of results) {
+ console.log(`${result.track}: ${result.id}@${result.version} ${result.state}; ` +
+ `${result.controls} controls, ${result.stacks} stacks, ${result.contentSha256.slice(0, 12)}`);
+ }
+}
+
+if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
+ try { main(); }
+ catch (error: unknown) {
+ console.error(error instanceof Error ? error.stack ?? error.message : String(error));
+ process.exitCode = 1;
+ }
+}
diff --git a/tools/stack-bench/commands/check-composition.ts b/tools/stack-bench/commands/check-composition.ts
new file mode 100644
index 00000000000..4548a0730d5
--- /dev/null
+++ b/tools/stack-bench/commands/check-composition.ts
@@ -0,0 +1,87 @@
+import { existsSync, readFileSync, readdirSync } from 'node:fs';
+import { join } from 'node:path';
+import { pathToFileURL } from 'node:url';
+
+import {
+ compileFixtureDefinition,
+ compilePackDefinition,
+ compilePromotionFile,
+ compileRecipeFile,
+} from '../src/composition/composition-compiler.js';
+import { TRACKS_DIR, listTracks } from '../src/composition/tracks.js';
+
+function json(path: string): unknown {
+ try { return JSON.parse(readFileSync(path, 'utf8')); }
+ catch (error: unknown) {
+ const message = error instanceof Error ? error.message : String(error);
+ throw new Error(`cannot read composition source ${path}: ${message}`, { cause: error });
+ }
+}
+
+export interface CompositionSummary {
+ track: string;
+ packs: number;
+ fixtures: number;
+ recipes: number;
+ checks: number;
+ aliases: number;
+}
+
+export function checkCompositions(
+ { trackName = null }: { trackName?: string | null } = {},
+): CompositionSummary[] {
+ const names = trackName ? [trackName] : listTracks({ includeInternal: true });
+ const summary = [];
+ for (const name of names) {
+ const trackRoot = join(TRACKS_DIR, name);
+ const root = join(trackRoot, 'composition');
+ if (!existsSync(root)) {
+ if (trackName) throw new Error(`track ${name} has no composition directory`);
+ continue;
+ }
+ const packs = join(root, 'packs');
+ const fixtures = join(root, 'fixtures');
+ const recipes = join(root, 'recipes');
+ const packFiles = readdirSync(packs).filter(file => file.endsWith('.json')).sort();
+ const fixtureFiles = readdirSync(fixtures).filter(file => file.endsWith('.json')).sort();
+ const recipeFiles = readdirSync(recipes).filter(file => file.endsWith('.json')).sort();
+ if (!packFiles.length || !fixtureFiles.length || !recipeFiles.length) {
+ throw new Error(`track ${name} composition must contain packs, fixtures, and recipes`);
+ }
+ for (const file of packFiles) {
+ const path = join(packs, file);
+ compilePackDefinition(json(path), { source: path });
+ }
+ for (const file of fixtureFiles) {
+ const path = join(fixtures, file);
+ compileFixtureDefinition(json(path), { source: path });
+ }
+ const plans = recipeFiles.map(file => compileRecipeFile(join(recipes, file), { trackRoot }));
+ const promotionPath = join(root, 'promotions.json');
+ const promotion = existsSync(promotionPath)
+ ? compilePromotionFile(promotionPath, { trackRoot }) : null;
+ summary.push({ track: name, packs: packFiles.length, fixtures: fixtureFiles.length,
+ recipes: plans.length, checks: plans.reduce((total, plan) => total + plan.checks.length, 0),
+ aliases: promotion?.entries.length ?? 0 });
+ }
+ return summary;
+}
+
+function main(): void {
+ const args = process.argv.slice(2);
+ let trackName: string | null = null;
+ for (let index = 0; index < args.length; index += 1) {
+ const value = args[index + 1];
+ if (args[index] === '--track' && value) {
+ trackName = value;
+ index += 1;
+ } else throw new Error(`unknown or incomplete argument ${args[index]}`);
+ }
+ const summary = checkCompositions({ trackName });
+ if (!summary.length) throw new Error('no composition sources found');
+ for (const row of summary) {
+ console.log(`${row.track}: ${row.packs} packs, ${row.fixtures} fixtures, ${row.recipes} recipes, ${row.checks} selected checks, ${row.aliases} promotion entries`);
+ }
+}
+
+if (process.argv[1] && pathToFileURL(process.argv[1]).href === import.meta.url) main();
diff --git a/tools/stack-bench/commands/check-mutations.ts b/tools/stack-bench/commands/check-mutations.ts
new file mode 100644
index 00000000000..042a6d9770b
--- /dev/null
+++ b/tools/stack-bench/commands/check-mutations.ts
@@ -0,0 +1,81 @@
+#!/usr/bin/env node
+
+import { existsSync, readFileSync } from 'node:fs';
+import { parseArgs as parseNodeArgs } from 'node:util';
+
+import { mutationFileEdits, resolveMutationFile, validateMutationDefinitions }
+ from '../src/evidence/mutation-analysis.js';
+import type { MutationDefinition } from '../src/evidence/mutation-analysis.js';
+
+interface CliArgs {
+ app: string;
+ mutations: string;
+ quiet: boolean;
+}
+
+interface MutationSpec {
+ anchoredTo?: unknown;
+ mutations?: MutationDefinition[];
+}
+
+function parseArgs(argv: string[]): CliArgs {
+ const { values: { app, mutations, quiet = false } } = parseNodeArgs({ args: argv.slice(2),
+ options: { app: { type: 'string' }, mutations: { type: 'string' }, quiet: { type: 'boolean' } } });
+ if (!app || !mutations) {
+ console.error('Usage: node dist/commands/check-mutations.js --app --mutations ');
+ process.exit(2);
+ }
+ return { app, mutations, quiet };
+}
+
+const args = parseArgs(process.argv);
+const parsed: unknown = JSON.parse(readFileSync(args.mutations, 'utf8'));
+if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) {
+ throw new Error('mutation manifest must be an object');
+}
+const spec = parsed as MutationSpec;
+const say = (...message: unknown[]): void => { if (!args.quiet) console.log(...message); };
+
+say(`mutations : ${args.mutations}`);
+say(`app : ${args.app}`);
+if (spec.anchoredTo) say(`anchored : ${String(spec.anchoredTo).split('.')[0]}`);
+say('');
+
+let bad = 0;
+const definitions = validateMutationDefinitions(spec.mutations);
+for (const issue of definitions.issues) {
+ console.log(` BAD MANIFEST ${issue.mutation ?? ''} -> ${issue.kind}`);
+ bad += 1;
+}
+for (const mutation of spec.mutations ?? []) {
+ const mutationId = String(mutation.id ?? '');
+ for (const edit of mutationFileEdits(mutation)) {
+ let file: string;
+ try { file = resolveMutationFile(args.app, edit.file); }
+ catch {
+ console.log(` UNSAFE FILE ${mutationId} -> ${edit.file} escapes the app directory`);
+ bad += 1;
+ continue;
+ }
+ if (!existsSync(file)) {
+ console.log(` DEAD FILE ${mutationId} -> ${edit.file} does not exist in this app`);
+ bad += 1;
+ continue;
+ }
+ const source = readFileSync(file, 'utf8');
+ const matches = source.split(edit.find).length - 1;
+ if (matches === 1) {
+ say(` ok ${mutationId} -> ${edit.file}`);
+ continue;
+ }
+ console.log(matches === 0
+ ? ` DEAD ANCHOR ${mutationId} -> not found in ${edit.file}`
+ : ` AMBIGUOUS ${mutationId} -> matches ${matches}x in ${edit.file}; the edit would land in more than one place`);
+ bad += 1;
+ }
+}
+
+console.log(bad
+ ? `\n${bad} problem(s) — these mutations cannot validate anything against this app.`
+ : '\nall anchors present and unique — this file can validate against this app.');
+process.exit(bad ? 1 : 0);
diff --git a/tools/stack-bench/commands/check-scenarios.ts b/tools/stack-bench/commands/check-scenarios.ts
new file mode 100644
index 00000000000..7fd01f2c378
--- /dev/null
+++ b/tools/stack-bench/commands/check-scenarios.ts
@@ -0,0 +1,304 @@
+#!/usr/bin/env node
+// Check scenario action names, actors, UI hooks, and score totals without an app.
+
+import { existsSync, readFileSync, readdirSync } from 'node:fs';
+import { join } from 'node:path';
+import { parseArgs } from 'node:util';
+
+import { ACTION_REGISTRY } from '../src/actions/action-catalog.js';
+import { compileRecipeFile, type CompiledOwnedTaskFragment, type CompiledRecipeRelease }
+ from '../src/composition/composition-compiler.js';
+import { compileScenarioDefinition, type CompiledStep }
+ from '../src/composition/definition-compiler.js';
+import { DEFAULT_TRACK, listTracks, loadTrack, type Track }
+ from '../src/composition/tracks.js';
+
+interface ScenarioScope {
+ features: Map>;
+ contractOwners: Set;
+ requirementOwners: Set;
+ contractText: string;
+ requirementText: string;
+}
+
+interface RecipeSource {
+ baseRecipe: string | null;
+ isolatesSelectedSources: boolean;
+}
+
+type HooksByLevel = Map>;
+
+function isRecord(value: unknown): value is Record {
+ return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+
+function readJson(path: string): unknown {
+ return JSON.parse(readFileSync(path, 'utf8')) as unknown;
+}
+
+function readRecipeSource(path: string): RecipeSource {
+ const source = readJson(path);
+ if (!isRecord(source)) throw new Error(`${path}: recipe must be an object`);
+ const task = source.task;
+ if (!isRecord(task)) throw new Error(`${path}: recipe task must be an object`);
+ const baseRecipe = task.baseRecipe;
+ let baseRecipePath: string | null = null;
+ if (baseRecipe !== undefined) {
+ if (!isRecord(baseRecipe) || typeof baseRecipe.path !== 'string') {
+ throw new Error(`${path}: task.baseRecipe.path must be a string`);
+ }
+ baseRecipePath = baseRecipe.path;
+ }
+ return {
+ baseRecipe: baseRecipePath,
+ isolatesSelectedSources: source.execution === 'all-selected-sources',
+ };
+}
+
+function contractHookIds(path: string): string[] {
+ const contract = readJson(path);
+ if (!isRecord(contract) || !Array.isArray(contract.hooks)) {
+ throw new Error(`${path}: hooks must be an array`);
+ }
+ return contract.hooks.map((hook, index) => {
+ if (!isRecord(hook) || typeof hook.id !== 'string') {
+ throw new Error(`${path}: hooks[${index}].id must be a string`);
+ }
+ return hook.id;
+ });
+}
+
+// Contract levels are cumulative. A level can use hooks introduced earlier.
+function hooksByLevel(track: Track): HooksByLevel {
+ const perFile = new Map();
+ for (const file of readdirSync(track.contracts).filter(name => /^\d\d-.*\.json$/.test(name))) {
+ perFile.set(file.slice(0, 2), contractHookIds(join(track.contracts, file)));
+ }
+ const byLevel: HooksByLevel = new Map();
+ for (const level of perFile.keys()) {
+ const ids = [...perFile.entries()]
+ .filter(([candidate]) => candidate <= level)
+ .flatMap(([, hookIds]) => hookIds);
+ byLevel.set(level, new Set(ids));
+ }
+ return byLevel;
+}
+
+function packId(reference: string): string {
+ return reference.slice(0, reference.lastIndexOf('@'));
+}
+
+function ownedFragment(
+ fragment: CompiledOwnedTaskFragment,
+ owners: ReadonlySet,
+): boolean {
+ return fragment.owners.some(owner => owners.has(owner));
+}
+
+function recipeScenarioScopes(track: Track, recipeFile: string): Map {
+ const recipeDir = join(track.dir, 'composition', 'recipes');
+ const chain: CompiledRecipeRelease[] = [];
+ const seen = new Set();
+ let currentFile: string | null = recipeFile;
+ let isolatesSelectedSources = false;
+ while (currentFile !== null) {
+ if (seen.has(currentFile)) throw new Error(`recipe base cycle at ${currentFile}`);
+ seen.add(currentFile);
+ const path = join(recipeDir, currentFile);
+ chain.push(compileRecipeFile(path, { trackRoot: track.dir }));
+ const source = readRecipeSource(path);
+ if (chain.length === 1) isolatesSelectedSources = source.isolatesSelectedSources;
+ currentFile = source.baseRecipe;
+ }
+
+ const recipe = chain[0];
+ if (recipe === undefined) throw new Error(`recipe chain is empty for ${recipeFile}`);
+ const packs = new Map(recipe.packs.map(pack => [pack.id, pack]));
+ const contracts = isolatesSelectedSources
+ ? recipe.recipe.task.contracts
+ : chain.flatMap(release => release.recipe.task.contracts);
+ const requirements = isolatesSelectedSources
+ ? recipe.recipe.task.requirements
+ : chain.flatMap(release => release.recipe.task.requirements);
+ const scopes = new Map();
+
+ const ownersFor = (check: CompiledRecipeRelease['checks'][number]): Set => {
+ const found = new Set([check.packId, ...(check.requiresFeatures ?? [])]);
+ const visit = (id: string): void => {
+ const pack = packs.get(id);
+ if (pack === undefined) return;
+ for (const reference of pack.requiresPacks) {
+ const dependency = packId(reference);
+ if (found.has(dependency)) continue;
+ found.add(dependency);
+ visit(dependency);
+ }
+ };
+ [...found].forEach(visit);
+ return found;
+ };
+
+ for (const check of recipe.checks) {
+ const source = check.source.replace(/^scenarios\//, '');
+ const scope = scopes.get(source) ?? {
+ features: new Map>(),
+ contractOwners: new Set(),
+ requirementOwners: new Set(),
+ contractText: '',
+ requirementText: '',
+ };
+ const criteria = scope.features.get(check.featureId) ?? new Set();
+ criteria.add(check.criterionId);
+ scope.features.set(check.featureId, criteria);
+ for (const owner of ownersFor(check)) {
+ scope.contractOwners.add(owner);
+ scope.requirementOwners.add(owner);
+ }
+ scopes.set(source, scope);
+ }
+
+ for (const scope of scopes.values()) {
+ const selectedContracts = isolatesSelectedSources
+ ? contracts.filter(fragment => ownedFragment(fragment, scope.contractOwners))
+ : contracts;
+ const selectedRequirements = isolatesSelectedSources
+ ? requirements.filter(fragment => ownedFragment(fragment, scope.requirementOwners))
+ : requirements;
+ scope.contractText = selectedContracts.map(fragment => fragment.text).join('\n');
+ scope.requirementText = selectedRequirements.map(fragment => fragment.text).join('\n');
+ }
+ return scopes;
+}
+
+function normalizeText(text: string): string {
+ return text.replace(/\*\*/g, '').replace(/—/g, '-').toLowerCase().replace(/\s+/g, ' ').trim();
+}
+
+function promptFor(track: Track, level: string): string | null {
+ const dir = join(track.dir, 'prompts');
+ if (!existsSync(dir)) return null;
+ const file = readdirSync(dir).find(name => name.startsWith(`${level}-`) && name.endsWith('.md'));
+ return file === undefined ? null : normalizeText(readFileSync(join(dir, file), 'utf8'));
+}
+
+function referencedActors(step: CompiledStep): string[] {
+ return [step.from, step.fromActor].filter((actor): actor is string => actor !== undefined);
+}
+
+function referencedTestIds(step: CompiledStep): string[] {
+ return [step.testid, step.in?.testid].filter((id): id is string => id !== undefined);
+}
+
+function main(args: readonly string[]): number {
+ const { values } = parseArgs({ args: [...args], options: {
+ track: { type: 'string' },
+ recipe: { type: 'string' },
+ }, strict: true, allowPositionals: false });
+ const trackArg = values.track ?? null;
+ const recipeArg = values.recipe ?? null;
+ const availableTracks = listTracks();
+ const trackNames = trackArg === null
+ ? (availableTracks.length > 0 ? availableTracks : [DEFAULT_TRACK])
+ : [trackArg];
+ if (recipeArg !== null && trackNames.length !== 1) {
+ throw new Error('--recipe requires one --track');
+ }
+
+ const knownActions = new Set(ACTION_REGISTRY.ids);
+ let problems = 0;
+ let unstatedWarnings = 0;
+ let staleStatementWarnings = 0;
+ const fail = (where: string, message: string): void => {
+ console.log(` ${where}: ${message}`);
+ problems += 1;
+ };
+
+ for (const name of trackNames) {
+ const track = loadTrack(name);
+ console.log(`# track: ${name}`);
+ const contracts = hooksByLevel(track);
+ const recipeScopes = recipeArg === null ? null : recipeScenarioScopes(track, recipeArg);
+ for (const file of readdirSync(track.scenarios).filter(candidate => candidate.endsWith('.json'))) {
+ const recipeScope = recipeScopes?.get(file);
+ if (recipeScopes !== null && recipeScope === undefined) continue;
+ const scenarioPath = join(track.scenarios, file);
+ let spec;
+ try {
+ spec = compileScenarioDefinition(readJson(scenarioPath), { source: scenarioPath });
+ } catch (error: unknown) {
+ fail(file, error instanceof Error ? error.message : String(error));
+ continue;
+ }
+ const level = String(spec.level).padStart(2, '0');
+ const hooks = recipeScope === undefined ? (contracts.get(level) ?? null) : null;
+
+ console.log(file);
+ const prompt = recipeScope === undefined
+ ? promptFor(track, level)
+ : normalizeText(recipeScope.requirementText);
+ for (const feature of spec.features) {
+ const selectedCriteria = recipeScope?.features.get(feature.id);
+ if (recipeScope !== undefined && selectedCriteria === undefined) continue;
+ const criteria = selectedCriteria === undefined
+ ? feature.criteria
+ : feature.criteria.filter(criterion => selectedCriteria.has(criterion.id));
+ for (const criterion of criteria) {
+ if (criterion.statedBy !== undefined) {
+ if (recipeScope === undefined && prompt !== null
+ && !prompt.includes(normalizeText(criterion.statedBy))) {
+ staleStatementWarnings += 1;
+ console.log(` warn F${feature.id} ${criterion.id}: statedBy text is not in the level ${level} prompt`);
+ }
+ } else if (recipeScope === undefined && criterion.points > 0) {
+ unstatedWarnings += 1;
+ console.log(` warn F${feature.id} ${criterion.id}: carries ${criterion.points} point(s) with no statedBy - the requirement may be unstated`);
+ }
+ }
+
+ const actors = new Set(feature.actors ?? []);
+ const steps = [...feature.setup, ...criteria.flatMap(criterion => criterion.steps)];
+ const declared = (actor: string): boolean => actors.has(actor)
+ || [...actors].some(candidate => actor.startsWith(`${candidate}-`));
+ for (const step of steps) {
+ const at = `F${feature.id} ${step.do}`;
+ if (!knownActions.has(step.do)) fail(at, `unknown step type "${step.do}"`);
+ if (step.actor !== undefined && actors.size > 0 && !declared(step.actor)) {
+ fail(at, `actor "${step.actor}" is not in the feature's actor list`);
+ }
+ for (const actor of referencedActors(step)) {
+ if (actors.size > 0 && !declared(actor)) {
+ fail(at, `actor "${actor}" is not in the feature's actor list`);
+ }
+ }
+ if (hooks !== null) {
+ for (const id of referencedTestIds(step)) {
+ if (!hooks.has(id)) fail(at, `testid "${id}" is not in the contract`);
+ }
+ }
+ if (recipeScope !== undefined) {
+ for (const id of referencedTestIds(step)) {
+ if (!recipeScope.contractText.includes(`\`${id}\``)) {
+ fail(at, `testid "${id}" is not in the selected recipe contracts`);
+ }
+ }
+ }
+ }
+
+ const points = criteria.reduce((total, criterion) => total + criterion.points, 0);
+ if (recipeScope === undefined && feature.max !== undefined && points !== feature.max) {
+ fail(`F${feature.id}`, `criteria total ${points} but max says ${feature.max}`);
+ }
+ }
+ }
+ }
+
+ const warnings = unstatedWarnings + staleStatementWarnings;
+ console.log(problems > 0
+ ? `\n${problems} error(s); ${warnings} warning(s)`
+ : warnings > 0
+ ? `\n0 errors; ${warnings} warning(s) (${unstatedWarnings} point-carrying criteria lack statedBy; ${staleStatementWarnings} statedBy references are outside level prompts)`
+ : '\n0 errors; 0 warnings');
+ return problems > 0 ? 1 : 0;
+}
+
+process.exitCode = main(process.argv.slice(2));
diff --git a/tools/stack-bench/commands/composition-cli.ts b/tools/stack-bench/commands/composition-cli.ts
new file mode 100644
index 00000000000..820698e7ea0
--- /dev/null
+++ b/tools/stack-bench/commands/composition-cli.ts
@@ -0,0 +1,321 @@
+#!/usr/bin/env node
+
+import { existsSync, readdirSync, realpathSync } from 'node:fs';
+import { join, relative, resolve, sep } from 'node:path';
+import { pathToFileURL } from 'node:url';
+import { parseArgs } from 'node:util';
+
+import { compilePackDefinition, compileRecipeFile, resolveTaskFragment } from '../src/composition/composition-compiler.js';
+import { compileScenarioDefinition } from '../src/composition/definition-compiler.js';
+import { canonicalDefinitionJson, readDefinitionJson }
+ from '../src/composition/definition-plan.js';
+import { buildRecipeRelease } from '../src/composition/recipe-release.js';
+import { composeSelectedRecipeTask, selectRecipeRelease } from '../src/composition/recipe-selection.js';
+import { TRACKS_DIR } from '../src/composition/tracks.js';
+import type { CompiledPackDefinition, CompiledRecipePlan } from '../src/composition/composition-compiler.js';
+import type { RecipeRelease } from '../src/composition/recipe-release.js';
+import type { RecipeSelectionOptions, SelectedRecipeRelease } from '../src/composition/recipe-selection.js';
+
+export { selectRecipeRelease } from '../src/composition/recipe-selection.js';
+
+interface TrackRootOptions {
+ trackRoot: string;
+}
+
+interface PackIndexEntry {
+ pack: CompiledPackDefinition;
+ path: string;
+}
+
+interface CalibrationValue {
+ id: string;
+ version: string;
+ recipe?: { id?: string; version?: string; contentSha256?: string };
+}
+
+type RecipeOptions = TrackRootOptions & RecipeSelectionOptions;
+type RecipeTaskKind = 'requirements' | 'contracts';
+
+function contained(root: string, path: string, label: string): string {
+ const absoluteRoot = realpathSync(resolve(root));
+ const candidate = resolve(path);
+ const lexical = relative(absoluteRoot, candidate);
+ if (lexical === '..' || lexical.startsWith(`..${sep}`)) throw new Error(`${label} escapes ${absoluteRoot}`);
+ if (!existsSync(candidate)) throw new Error(`${label} does not exist: ${candidate}`);
+ const absolute = realpathSync(candidate);
+ const physical = relative(absoluteRoot, absolute);
+ if (physical === '..' || physical.startsWith(`..${sep}`)) throw new Error(`${label} escapes ${absoluteRoot}`);
+ return absolute;
+}
+
+function packIndex(trackRoot: string): Map {
+ const directory = join(trackRoot, 'composition', 'packs');
+ const byRef = new Map();
+ for (const name of readdirSync(directory).filter(file => file.endsWith('.json')).sort()) {
+ const path = join(directory, name);
+ const pack = compilePackDefinition(readDefinitionJson(path, 'pack'), {
+ source: relative(trackRoot, path).replaceAll('\\', '/'),
+ });
+ const ref = `${pack.id}@${pack.version}`;
+ if (byRef.has(ref)) throw new Error(`duplicate pack release ${ref}`);
+ byRef.set(ref, { pack, path: realpathSync(path) });
+ }
+ for (const [ref, { pack }] of byRef) {
+ for (const dependency of [...pack.requiresPacks, ...pack.conflictsWith]) {
+ if (!byRef.has(dependency)) throw new Error(`${ref} references missing pack ${dependency}`);
+ }
+ }
+ return byRef;
+}
+
+export function validatePackFile(path: string, options: Partial = {}) {
+ const { trackRoot } = options;
+ if (trackRoot === undefined) throw new Error('pack validation requires trackRoot');
+ const root = realpathSync(resolve(trackRoot));
+ const absolute = contained(join(root, 'composition'), path, 'pack path');
+ const pack = compilePackDefinition(readDefinitionJson(absolute, 'pack'), {
+ source: relative(root, absolute).replaceAll('\\', '/'),
+ });
+ const packs = packIndex(root);
+ const ownRef = `${pack.id}@${pack.version}`;
+ const indexed = packs.get(ownRef);
+ if (!indexed || indexed.path !== absolute) throw new Error(`${ownRef} is not the indexed source ${absolute}`);
+ for (const ref of [...pack.requiresPacks, ...pack.conflictsWith]) {
+ if (!packs.has(ref)) throw new Error(`${ownRef} references missing pack ${ref}`);
+ }
+ const sourceCache = new Map();
+ for (const kind of ['requirements', 'contracts'] satisfies RecipeTaskKind[]) {
+ for (const fragment of pack.task[kind]) {
+ resolveTaskFragment(fragment, { trackRoot: root,
+ source: `${relative(root, absolute).replaceAll('\\', '/')}.task.${kind}.${fragment.id}`,
+ sourceCache });
+ }
+ }
+ const state = new Map();
+ const visit = (ref: string, chain: string[] = []): void => {
+ if (state.get(ref) === 'done') return;
+ if (state.get(ref) === 'visiting') throw new Error(`pack dependency cycle: ${[...chain, ref].join(' -> ')}`);
+ state.set(ref, 'visiting');
+ const entry = packs.get(ref);
+ if (!entry) throw new Error(`missing pack release ${ref}`);
+ for (const dependency of entry.pack.requiresPacks) {
+ if (!packs.has(dependency)) throw new Error(`${ref} references missing pack ${dependency}`);
+ visit(dependency, [...chain, ref]);
+ }
+ state.set(ref, 'done');
+ };
+ visit(ownRef);
+ let criteria = 0;
+ for (const check of pack.checks) {
+ const scenarioPath = contained(root, join(root, check.source), `${pack.id}.${check.id}.source`);
+ const scenario = compileScenarioDefinition(readDefinitionJson(scenarioPath, 'scenario'), {
+ source: relative(root, scenarioPath).replaceAll('\\', '/'),
+ });
+ const feature = scenario.features.find(candidate => candidate.id === check.feature);
+ if (!feature) throw new Error(`${pack.id}.${check.id} references missing feature ${check.feature}`);
+ criteria += feature.criteria.length;
+ }
+ return { id: pack.id, version: pack.version, state: pack.state, path: absolute,
+ checkGroups: pack.checks.length, criteria, requiresPacks: pack.requiresPacks };
+}
+
+export function validateRecipeFile(path: string, options: Partial = {}): {
+ plan: CompiledRecipePlan;
+ release: RecipeRelease;
+} {
+ const { trackRoot } = options;
+ if (trackRoot === undefined) throw new Error('recipe validation requires trackRoot');
+ const absolute = contained(join(trackRoot, 'composition'), path, 'recipe path');
+ const plan = compileRecipeFile(absolute, { trackRoot });
+ const release = buildRecipeRelease(absolute, { trackRoot });
+ return { plan, release };
+}
+
+export function showRecipeFile(path: string, options: RecipeOptions): SelectedRecipeRelease & {
+ builderTask: ReturnType & { note: string };
+} {
+ const compiled = validateRecipeFile(path, options);
+ const selected = selectRecipeRelease(compiled.release, options);
+ const builderTask = composeSelectedRecipeTask(compiled.plan, selected.selection);
+ return {
+ ...selected,
+ builderTask: {
+ ...builderTask,
+ note: 'Pack selection defines the requested task; a check-only filter narrows measurement inside it.',
+ },
+ };
+}
+
+const same = (left: unknown, right: unknown): boolean =>
+ canonicalDefinitionJson(left) === canonicalDefinitionJson(right);
+
+function meaningView(release: RecipeRelease) {
+ return {
+ track: release.track,
+ task: release.task,
+ checks: release.checkCatalog.map(({ stableKey, packId, checkGroupId, role, source,
+ featureId, criterionId, description }) => ({ stableKey, packId, checkGroupId, role,
+ source, featureId, criterionId, description })),
+ };
+}
+
+function scoringView(release: RecipeRelease) {
+ return { scoring: release.scoring,
+ checks: release.checkCatalog.map(({ stableKey, points }) => ({ stableKey, points })) };
+}
+
+function metadataView(release: RecipeRelease) {
+ return { id: release.id, version: release.version, state: release.state, title: release.title,
+ sequence: release.sequence, sourceManifestSha256: release.sourceManifestSha256 };
+}
+
+function matchingCalibrations(trackRoot: string, release: RecipeRelease): Array<{
+ path: string;
+ value: CalibrationValue;
+}> {
+ const directory = join(trackRoot, 'composition', 'calibrations');
+ if (!existsSync(directory)) return [];
+ return readdirSync(directory).filter(name => name.endsWith('.json')).sort()
+ .map(name => ({ path: join(directory, name),
+ value: readDefinitionJson(join(directory, name), 'calibration') }))
+ .filter(({ value }) => value.recipe?.id === release.id
+ && value.recipe?.version === release.version
+ && value.recipe?.contentSha256 === release.contentSha256);
+}
+
+export function diffRecipeFiles(fromPath: string, toPath: string, options: Partial = {}) {
+ const { trackRoot } = options;
+ if (trackRoot === undefined) throw new Error('recipe diff requires trackRoot');
+ const from = validateRecipeFile(fromPath, { trackRoot }).release;
+ const to = validateRecipeFile(toPath, { trackRoot }).release;
+ const categories = {
+ meaning: !same(meaningView(from), meaningView(to)),
+ scoring: !same(scoringView(from), scoringView(to)),
+ fixtures: !same(from.components.fixture, to.components.fixture),
+ execution: from.executionSha256 !== to.executionSha256,
+ metadata: !same(metadataView(from), metadataView(to)),
+ };
+ const recipeBindingChanged = from.id !== to.id || from.version !== to.version
+ || from.meaningSha256 !== to.meaningSha256 || from.executionSha256 !== to.executionSha256
+ || from.contentSha256 !== to.contentSha256;
+ const calibrations = matchingCalibrations(trackRoot, from).map(({ path, value }) => {
+ const invalidated = [];
+ const stateChanged = from.state !== to.state;
+ if (recipeBindingChanged) invalidated.push('recipe binding');
+ if (stateChanged) invalidated.push('recipe qualification state');
+ if (categories.fixtures) invalidated.push('fixture binding');
+ if (categories.scoring) invalidated.push('zero-point control policy');
+ if (categories.meaning || categories.scoring || categories.execution || categories.fixtures) {
+ invalidated.push('reference repetitions', 'mutation repetitions');
+ }
+ if (categories.meaning || categories.scoring || categories.fixtures) invalidated.push('null repetitions');
+ if (recipeBindingChanged || stateChanged) invalidated.push('promotion decision');
+ return { id: value.id, version: value.version,
+ path: relative(trackRoot, path).replaceAll('\\', '/'), invalidated: [...new Set(invalidated)] };
+ });
+ const fragmentDiff = (kind: RecipeTaskKind) => {
+ const before = new Map(from.task[kind].map(fragment => [fragment.id, fragment]));
+ const after = new Map(to.task[kind].map(fragment => [fragment.id, fragment]));
+ return {
+ added: [...after.keys()].filter(key => !before.has(key)).sort(),
+ removed: [...before.keys()].filter(key => !after.has(key)).sort(),
+ changed: [...after.keys()].filter(key => before.has(key)
+ && !same(before.get(key), after.get(key))).sort(),
+ };
+ };
+ return {
+ from: { id: from.id, version: from.version, state: from.state, meaningSha256: from.meaningSha256,
+ executionSha256: from.executionSha256, contentSha256: from.contentSha256 },
+ to: { id: to.id, version: to.version, state: to.state, meaningSha256: to.meaningSha256,
+ executionSha256: to.executionSha256, contentSha256: to.contentSha256 },
+ categories,
+ taskFragments: {
+ requirements: fragmentDiff('requirements'),
+ contracts: fragmentDiff('contracts'),
+ composedTaskChanged: from.task.composedSha256 !== to.task.composedSha256,
+ },
+ calibrations,
+ };
+}
+
+type CliSubject = 'pack' | 'recipe';
+type CliCommand = 'validate' | 'show' | 'diff';
+
+interface ParsedArgs extends RecipeSelectionOptions {
+ json: boolean;
+ positional: string[];
+ packIds: string[];
+ checkKeys: string[];
+ track?: string;
+ trackRoot?: string;
+}
+
+interface CliArgs extends ParsedArgs {
+ subject: CliSubject;
+ command: CliCommand;
+ paths: string[];
+ trackRoot: string;
+}
+
+function parse(argv: string[]): CliArgs {
+ const { positionals, values } = parseArgs({ args: argv.slice(2), allowPositionals: true,
+ options: { track: { type: 'string' }, 'track-root': { type: 'string' },
+ pack: { type: 'string', multiple: true }, check: { type: 'string', multiple: true },
+ json: { type: 'boolean' } } });
+ const args: ParsedArgs = { json: values.json ?? false, positional: positionals,
+ packIds: (values.pack ?? []).flatMap(value => value.split(',').filter(Boolean)),
+ checkKeys: (values.check ?? []).flatMap(value => value.split(',').filter(Boolean)),
+ track: values.track,
+ trackRoot: values['track-root'] === undefined ? undefined : resolve(values['track-root']) };
+ const [subject, command, ...paths] = args.positional;
+ if (subject !== 'pack' && subject !== 'recipe') {
+ throw new Error('usage: npm run pack -- validate --track | npm run recipe -- validate|show --track | npm run recipe -- diff --track ');
+ }
+ if (command !== 'validate' && command !== 'show' && command !== 'diff') {
+ throw new Error('usage: npm run pack -- validate --track | npm run recipe -- validate|show --track | npm run recipe -- diff --track ');
+ }
+ if (subject === 'pack' && command !== 'validate') throw new Error(`pack ${command} is not supported`);
+ if ((command === 'diff' ? paths.length !== 2 : paths.length !== 1)) throw new Error(`${subject} ${command} received the wrong number of paths`);
+ if (!args.trackRoot && !args.track) throw new Error('--track or --track-root is required');
+ if ((args.packIds.length || args.checkKeys.length) && !(subject === 'recipe' && command === 'show')) {
+ throw new Error('--pack and --check are allowed only with recipe show');
+ }
+ const trackRoot = args.trackRoot ?? join(TRACKS_DIR, args.track ?? '');
+ return { ...args, subject, command, paths, trackRoot };
+}
+
+function main() {
+ const args = parse(process.argv);
+ const firstPath = args.paths[0];
+ if (firstPath === undefined) throw new Error('command requires a source path');
+ let result: object;
+ if (args.subject === 'pack') result = validatePackFile(firstPath, args);
+ else if (args.command === 'diff') {
+ const secondPath = args.paths[1];
+ if (secondPath === undefined) throw new Error('recipe diff requires two source paths');
+ result = diffRecipeFiles(firstPath, secondPath, args);
+ } else if (args.command === 'show') result = showRecipeFile(firstPath, args);
+ else {
+ const compiled = validateRecipeFile(firstPath, args);
+ result = {
+ id: compiled.release.id, version: compiled.release.version, state: compiled.release.state,
+ packs: compiled.release.components.packs.length, checks: compiled.release.checkCatalog.length,
+ points: compiled.release.checkCatalog.reduce((total, check) => total + check.points, 0),
+ meaningSha256: compiled.release.meaningSha256,
+ executionSha256: compiled.release.executionSha256,
+ contentSha256: compiled.release.contentSha256,
+ };
+ }
+ if (args.json || args.command === 'show' || args.command === 'diff') console.log(JSON.stringify(result, null, 2));
+ else if ('id' in result && 'version' in result && 'state' in result) {
+ console.log(`${String(result.id)}@${String(result.version)} ${String(result.state)}: valid`);
+ } else throw new Error('validation result has no release identity');
+}
+
+if (process.argv[1] && pathToFileURL(process.argv[1]).href === import.meta.url) {
+ try { main(); }
+ catch (error) {
+ console.error(error instanceof Error ? error.message : String(error));
+ process.exitCode = 2;
+ }
+}
diff --git a/tools/stack-bench/commands/container-smoke.ts b/tools/stack-bench/commands/container-smoke.ts
new file mode 100644
index 00000000000..9ea5f297de3
--- /dev/null
+++ b/tools/stack-bench/commands/container-smoke.ts
@@ -0,0 +1,200 @@
+#!/usr/bin/env node
+// Use only ephemeral resources owned by this smoke run.
+
+import { spawn, execFileSync } from 'node:child_process';
+import type { ChildProcess } from 'node:child_process';
+import { cpSync, existsSync, mkdirSync, mkdtempSync, rmSync } from 'node:fs';
+import { createServer } from 'node:net';
+import type { AddressInfo } from 'node:net';
+import { basename, join, resolve } from 'node:path';
+import { tmpdir } from 'node:os';
+import { setTimeout as delay } from 'node:timers/promises';
+
+import { killTree, pidsOnPort, processIdentity } from '../src/runtime/platform.js';
+import { createBackendLease, readBackendLease, writeBackendLease } from '../src/runtime/backend-lease.js';
+import { fetchStatus } from '../src/runtime/readiness.js';
+import { DEFAULT_BUILD_IMAGE } from '../src/composition/product-config.js';
+import { containerReachableSpacetimeUri } from '../src/runtime/spacetime-target.js';
+import { CODING_CONTAINER_APP_ROOT, CODING_CONTAINER_SPACETIME_CLI,
+ codingContainerAgentExecOptions } from '../src/runtime/coding-container-policy.js';
+import { ARTIFACT_FILE } from '../src/evidence/artifacts.js';
+
+import { STACK_BENCH_ROOT as ROOT, compiledEntrypoint } from '../src/package-root.js';
+const REPO = resolve(ROOT, '..', '..');
+const IMAGE = process.env.STACK_BENCH_IMAGE ?? DEFAULT_BUILD_IMAGE;
+const CLI = process.env.SPACETIME_BIN ?? join(REPO, 'target', 'release',
+ process.platform === 'win32' ? 'spacetimedb-cli.exe' : 'spacetimedb-cli');
+const RUN_BUILD = compiledEntrypoint('container', 'run-build.js');
+const FIXTURE = join(ROOT, 'tests', 'fixtures', 'spacetime-module');
+
+interface PreparedContainerIdentity {
+ containerName: string;
+ identity: string;
+ networkMode: string | null;
+}
+
+function isRecord(value: unknown): value is Record {
+ return value !== null && typeof value === 'object' && !Array.isArray(value);
+}
+
+function parsePreparedContainerIdentity(text: string): PreparedContainerIdentity {
+ const value: unknown = JSON.parse(text.trim().split(/\r?\n/).pop() ?? '');
+ if (!isRecord(value)) throw new Error('prepared container identity is invalid');
+ const record = value;
+ if (typeof record.containerName !== 'string' || typeof record.identity !== 'string'
+ || (record.networkMode !== null && typeof record.networkMode !== 'string')) {
+ throw new Error('prepared container identity is invalid');
+ }
+ return { containerName: record.containerName, identity: record.identity, networkMode: record.networkMode };
+}
+
+async function freePort() {
+ const server = createServer();
+ await new Promise((ok, fail) => server.listen({ port: 0, host: '127.0.0.1' }, ok).once('error', fail));
+ const address = server.address();
+ if (!address || typeof address === 'string') throw new Error('could not allocate a TCP port');
+ const port: AddressInfo['port'] = address.port;
+ await new Promise(ok => server.close(ok));
+ return port;
+}
+
+async function waitFor(check: () => boolean | Promise, timeoutMs: number, description: string): Promise {
+ const deadline = Date.now() + timeoutMs;
+ while (Date.now() < deadline) {
+ if (await check()) return;
+ await delay(250);
+ }
+ throw new Error(`timed out waiting for ${description}`);
+}
+
+async function main() {
+ if (!existsSync(CLI)) throw new Error(`local SpacetimeDB CLI is missing: ${CLI}`);
+ execFileSync('docker', ['image', 'inspect', IMAGE], { stdio: 'pipe' });
+
+ const root = mkdtempSync(join(tmpdir(), 'stack-bench-container-smoke-'));
+ const app = join(root, 'app');
+ const dataDir = join(root, 'spacetime-data');
+ const port = await freePort();
+ const uri = `http://127.0.0.1:${port}`;
+ const module = `stackbench-container-smoke-${process.pid}`;
+ const containerName = `stack-bench-${basename(root)}`;
+ const leasePath = join(root, ARTIFACT_FILE.backendLease);
+ let host: ChildProcess | null = null;
+ let dev: ChildProcess | null = null;
+ let output = '';
+
+ try {
+ mkdirSync(app, { recursive: true });
+ host = spawn(CLI, ['start', '--listen-addr', `127.0.0.1:${port}`, '--data-dir', dataDir],
+ { stdio: 'ignore', windowsHide: true });
+ await waitFor(async () => {
+ const status = await fetchStatus(`${uri}/v1/ping`, { timeoutMs: 5000 });
+ return status !== null && status >= 200 && status < 300;
+ }, 120_000, `dedicated SpacetimeDB host on :${port}`);
+
+ const lease = createBackendLease({ runId: basename(root), backend: 'spacetime',
+ track: 'container-smoke', runIndex: 0, serverUri: uri, module, dataDir });
+ lease.state = 'active';
+ lease.resources.launchedProcess = host.pid ? processIdentity(host.pid) : null;
+ lease.resources.listenerProcesses = pidsOnPort(port).map(pid => processIdentity(pid))
+ .filter((identity): identity is NonNullable => identity !== null);
+ writeBackendLease(leasePath, lease);
+
+ const prepared = execFileSync(process.execPath,
+ [RUN_BUILD, '--app', app, '--backend', 'spacetime', '--image', IMAGE, '--prepare-only'],
+ { encoding: 'utf8', stdio: 'pipe', maxBuffer: 16 * 1024 * 1024,
+ env: { ...process.env, STACK_BENCH_LEASE: leasePath,
+ STACK_BENCH_LEASE_TOKEN: lease.ownershipToken, STACK_BENCH_STDB_URI: uri } });
+ const identity = parsePreparedContainerIdentity(prepared);
+ if (identity.containerName !== containerName) {
+ throw new Error(`prepared unexpected container ${identity.containerName}`);
+ }
+ const leasedContainer = readBackendLease(leasePath,
+ { token: lease.ownershipToken, backend: 'spacetime', active: true }).resources.buildContainer;
+ if (!leasedContainer || identity.identity.split(' ')[0] !== leasedContainer.id) {
+ throw new Error('prepared container identity was not recorded in the backend lease');
+ }
+ if (!leasedContainer.image || !/^sha256:[0-9a-f]{64}$/.test(leasedContainer.image)) {
+ throw new Error(`prepared container did not record an immutable image id: ${leasedContainer.image}`);
+ }
+
+ cpSync(FIXTURE, join(app, 'spacetimedb'), { recursive: true });
+ const agentExec = ['exec', ...codingContainerAgentExecOptions()];
+ const cliAccess = execFileSync('docker', [...agentExec, containerName, 'sh', '-c',
+ `stat -c '%a %U %G' ${CODING_CONTAINER_SPACETIME_CLI}; test -x ${CODING_CONTAINER_SPACETIME_CLI}`],
+ { encoding: 'utf8', stdio: 'pipe' });
+ if (!/^755 root root/m.test(cliAccess)) throw new Error(`unexpected CLI access: ${cliAccess.trim()}`);
+ execFileSync('docker', [...agentExec, containerName, 'sh', '-c',
+ `umask 000; cd ${CODING_CONTAINER_APP_ROOT}/spacetimedb && npm install --no-audit --no-fund`],
+ { stdio: 'pipe' });
+
+ const startedDev = spawn('docker', [...agentExec, '-i', containerName, 'sh', '-c',
+ `umask 000; cd ${CODING_CONTAINER_APP_ROOT}/spacetimedb && ${CODING_CONTAINER_SPACETIME_CLI} dev ${module} `
+ + `--project-path ${CODING_CONTAINER_APP_ROOT}/spacetimedb --module-path . `
+ + '--server-only --skip-generate '
+ + `-s ${containerReachableSpacetimeUri({ resources: { serverUri: uri,
+ buildContainer: lease.resources.buildContainer } }, identity.networkMode)} -y`],
+ { stdio: ['ignore', 'pipe', 'pipe'], windowsHide: true });
+ dev = startedDev;
+ const collect = (chunk: Buffer): void => { output = (output + chunk.toString()).slice(-128 * 1024); };
+ startedDev.stdout?.on('data', collect);
+ startedDev.stderr?.on('data', collect);
+
+ await waitFor(() => {
+ if (/Published successfully!/.test(output)) return true;
+ if (startedDev.exitCode !== null) throw new Error(`spacetime dev exited ${startedDev.exitCode}:\n${output}`);
+ return false;
+ }, 240_000, 'containerized module publish');
+
+ const sql = execFileSync(CLI, ['sql', module, 'SELECT * FROM smoke_item', '-s', uri],
+ { encoding: 'utf8', stdio: 'pipe' });
+ if (!/\bid\s*\|\s*value\b/.test(sql)) throw new Error(`SQL verification failed:\n${sql}`);
+ if (startedDev.exitCode !== null) throw new Error('spacetime dev did not remain alive as a watcher');
+
+ // Publishing and log streaming must retain one authenticated identity. A
+ // prior dev bug published with a token stored only in a Config clone, then
+ // directly logged in again for logs and received an authorization error.
+ await delay(2_000);
+ const logStreamingAuthorized = !/Log streaming error:.*not authorized/s.test(output);
+ console.log(JSON.stringify({ ok: true, image: IMAGE, container: identity.identity,
+ host: { uri, listenerPids: pidsOnPort(port) }, published: true, sqlVerified: true,
+ watcherAlive: true, leasedContainer: true, immutableImagePinned: true,
+ logStreamingAuthorized }, null, 2));
+ if (!logStreamingAuthorized) {
+ throw new Error('`spacetime dev` published successfully but its log stream was not authorized');
+ }
+ // The grader resets by republishing the same named database from this exact
+ // leased container. Prove that `-y` retained a reusable local identity,
+ // rather than merely proving that the first anonymous-looking publish ran.
+ execFileSync('docker', [...agentExec, containerName, 'sh', '-c',
+ 'for process in /proc/[0-9]*; do '
+ + 'test "$(readlink "$process/exe" 2>/dev/null)" = /deps/.spacetimedb-cli '
+ + '&& kill -TERM "${process##*/}" || true; done'], { stdio: 'pipe' });
+ await waitFor(() => startedDev.exitCode !== null, 15_000, 'spacetime dev to stop before reset publish');
+ const targetUri = containerReachableSpacetimeUri({ resources: { serverUri: uri,
+ buildContainer: lease.resources.buildContainer } }, identity.networkMode);
+ execFileSync('docker', [...agentExec, containerName, 'sh', '-c',
+ `umask 000; cd ${CODING_CONTAINER_APP_ROOT}/spacetimedb && ${CODING_CONTAINER_SPACETIME_CLI} publish ${module} `
+ + `--module-path . -s ${targetUri} --delete-data -y`],
+ { stdio: 'pipe', timeout: 240_000 });
+ const afterReset = execFileSync(CLI, ['sql', module, 'SELECT * FROM smoke_item', '-s', uri],
+ { encoding: 'utf8', stdio: 'pipe' });
+ if (!/\bid\s*\|\s*value\b/.test(afterReset)) {
+ throw new Error(`SQL verification after reset publish failed:\n${afterReset}`);
+ }
+ console.log(JSON.stringify({ resetRepublished: true, resetSqlVerified: true }));
+ } finally {
+ if (dev && dev.exitCode === null) dev.kill('SIGTERM');
+ try { execFileSync('docker', ['rm', '-f', containerName], { stdio: 'ignore' }); } catch { /* absent */ }
+ // The port was proven unused before this script started the host. Kill only
+ // listeners on that exact ephemeral port, then the wrapper if it remains.
+ for (const pid of pidsOnPort(port)) killTree(pid);
+ if (host && host.exitCode === null) killTree(host.pid);
+ rmSync(root, { recursive: true, force: true });
+ }
+}
+
+main().catch(error => {
+ console.error(error.stack ?? error.message);
+ process.exitCode = 1;
+});
diff --git a/tools/stack-bench/commands/definition-snapshots.ts b/tools/stack-bench/commands/definition-snapshots.ts
new file mode 100644
index 00000000000..9f227e19d30
--- /dev/null
+++ b/tools/stack-bench/commands/definition-snapshots.ts
@@ -0,0 +1,80 @@
+import { mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+import { pathToFileURL } from 'node:url';
+
+import { compileScenarioDefinition } from '../src/composition/definition-compiler.js';
+import { canonicalDefinitionJson, compileTrackPlan } from '../src/composition/definition-plan.js';
+import { listTracks } from '../src/composition/tracks.js';
+import { STACK_BENCH_ROOT } from '../src/package-root.js';
+
+const SNAPSHOT_DIR = join(STACK_BENCH_ROOT, 'tests', 'snapshots', 'definitions');
+const ALL_ACTIONS = join(STACK_BENCH_ROOT, 'tests', 'fixtures', 'definitions', 'all-actions.json');
+
+function atomicWrite(path: string, contents: string): void {
+ mkdirSync(dirname(path), { recursive: true });
+ const temporary = `${path}.tmp-${process.pid}`;
+ writeFileSync(temporary, contents);
+ renameSync(temporary, path);
+}
+
+interface DefinitionSnapshot {
+ name: string;
+ value: unknown;
+}
+
+export function currentDefinitionSnapshots(): DefinitionSnapshot[] {
+ const entries: DefinitionSnapshot[] = listTracks({ includeInternal: true }).map(name => ({
+ name: `${name}.snapshot.json`,
+ value: compileTrackPlan(name),
+ }));
+ entries.push({
+ name: 'all-actions.snapshot.json',
+ value: compileScenarioDefinition(JSON.parse(readFileSync(ALL_ACTIONS, 'utf8')), {
+ source: ALL_ACTIONS,
+ }),
+ });
+ return entries.sort((a, b) => a.name.localeCompare(b.name));
+}
+
+export interface DefinitionSnapshotResult {
+ checked: number;
+ changed: string[];
+}
+
+export function checkDefinitionSnapshots(
+ { update = false }: { update?: boolean } = {},
+): DefinitionSnapshotResult {
+ const entries = currentDefinitionSnapshots();
+ const changed: string[] = [];
+ for (const entry of entries) {
+ const path = join(SNAPSHOT_DIR, entry.name);
+ const actual = canonicalDefinitionJson(entry.value);
+ let expected: string | null = null;
+ try {
+ expected = readFileSync(path, 'utf8').replaceAll('\r\n', '\n');
+ } catch (error: unknown) {
+ if (!(error instanceof Error && 'code' in error && error.code === 'ENOENT')) throw error;
+ }
+ if (expected === actual) continue;
+ changed.push(entry.name);
+ if (update) atomicWrite(path, actual);
+ }
+ if (changed.length > 0 && !update) {
+ throw new Error(
+ `definition snapshot drift: ${changed.join(', ')}; inspect the semantic change, then run npm run check:definition-snapshots -- --update`,
+ );
+ }
+ return { checked: entries.length, changed };
+}
+
+function main(): void {
+ const args = new Set(process.argv.slice(2));
+ for (const arg of args) {
+ if (arg !== '--update') throw new Error(`unknown argument ${arg}`);
+ }
+ const result = checkDefinitionSnapshots({ update: args.has('--update') });
+ console.log(`${result.checked} definition snapshots checked${
+ result.changed.length > 0 ? `; ${result.changed.length} updated` : '; no drift'}`);
+}
+
+if (process.argv[1] && pathToFileURL(process.argv[1]).href === import.meta.url) main();
diff --git a/tools/stack-bench/commands/fault-injection.ts b/tools/stack-bench/commands/fault-injection.ts
new file mode 100644
index 00000000000..fc8f324d6be
--- /dev/null
+++ b/tools/stack-bench/commands/fault-injection.ts
@@ -0,0 +1,250 @@
+#!/usr/bin/env node
+// Fault injection may remove only resources owned by its lease.
+
+import assert from 'node:assert/strict';
+import { execFileSync, spawn } from 'node:child_process';
+import type { ChildProcess } from 'node:child_process';
+import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync } from 'node:fs';
+import { createServer } from 'node:http';
+import type { Server } from 'node:http';
+import type { AddressInfo } from 'node:net';
+import { basename, join, resolve } from 'node:path';
+import { tmpdir } from 'node:os';
+import { setTimeout as delay } from 'node:timers/promises';
+
+import { createBackendLease, writeBackendLease } from '../src/runtime/backend-lease.js';
+import { killTree, pidsOnPort } from '../src/runtime/platform.js';
+import { ARTIFACT_FILE, readArtifact, readArtifactPayload } from '../src/evidence/artifacts.js';
+import { DEFAULT_BUILD_IMAGE } from '../src/composition/product-config.js';
+
+import { STACK_BENCH_ROOT as ROOT, compiledEntrypoint } from '../src/package-root.js';
+const REPO = resolve(ROOT, '..', '..');
+const IMAGE = process.env.STACK_BENCH_IMAGE ?? DEFAULT_BUILD_IMAGE;
+const CLI = process.env.SPACETIME_BIN ?? join(REPO, 'target', 'release',
+ process.platform === 'win32' ? 'spacetimedb-cli.exe' : 'spacetimedb-cli');
+const RUN_BUILD = compiledEntrypoint('container', 'run-build.js');
+interface ContainerIdentity { id: string; running: boolean; }
+interface ExitResult { code: number | null; signal: NodeJS.Signals | null; }
+interface FaultLeaseResources {
+ listenerProcesses: Array<{ pid: number; startMarker: string }>;
+ buildContainer: { id: string; image: string; running: boolean; removedAt?: string };
+ locks: { releasedAt?: string }[];
+}
+interface FaultLeaseEvidence {
+ runId: string;
+ state: string;
+ stoppedAt?: string;
+ releasedAt?: string;
+ resources: FaultLeaseResources;
+}
+
+function isRecord(value: unknown): value is Record {
+ return value !== null && typeof value === 'object' && !Array.isArray(value);
+}
+
+async function freePort() {
+ const server = createServer((_request, response) => response.end('foreign'));
+ await new Promise((ok, fail) => server.listen({ port: 0, host: '127.0.0.1' }, ok).once('error', fail));
+ const address = server.address();
+ if (!address || typeof address === 'string') throw new Error('could not allocate a TCP port');
+ const port: AddressInfo['port'] = address.port;
+ await new Promise(ok => server.close(ok));
+ return port;
+}
+
+function inspectContainer(target: string): ContainerIdentity | null {
+ try {
+ const output = execFileSync('docker', ['inspect', '--format',
+ '{{.Id}} {{.State.Running}}', target], { encoding: 'utf8', stdio: 'pipe' }).trim();
+ const [id, running] = output.split(/\s+/, 2);
+ if (!id) return null;
+ return { id, running: running === 'true' };
+ } catch { return null; }
+}
+
+function startContainer(name: string): ContainerIdentity {
+ const id = execFileSync('docker', ['run', '-d', '--init', '--name', name,
+ IMAGE, 'sleep', 'infinity'], { encoding: 'utf8', stdio: 'pipe' }).trim();
+ assert.ok(id, `Docker did not return an id for ${name}`);
+ const container = inspectContainer(id);
+ if (!container) throw new Error(`Docker did not return a running container for ${name}`);
+ return container;
+}
+
+function removeExactContainer(identity: ContainerIdentity | { id: string } | null): void {
+ if (!identity) return;
+ const current = inspectContainer(identity.id);
+ if (!current || current.id !== identity.id) return;
+ execFileSync('docker', ['rm', '-f', identity.id], { stdio: 'ignore' });
+}
+
+async function waitForExit(child: ChildProcess, timeoutMs: number): Promise {
+ return new Promise((resolveExit, reject) => {
+ const timeout = setTimeout(() => reject(new Error(
+ `benchmark runner did not exit after injected failure within ${timeoutMs}ms`)), timeoutMs);
+ child.once('exit', (code: number | null, signal: NodeJS.Signals | null) => {
+ clearTimeout(timeout);
+ resolveExit({ code, signal });
+ });
+ });
+}
+
+async function assertRefusesUnleasedCollision() {
+ const root = mkdtempSync(join(tmpdir(), 'stack-bench-container-collision-'));
+ const app = join(root, 'app');
+ const name = `stack-bench-${basename(root)}`;
+ const leasePath = join(root, ARTIFACT_FILE.backendLease);
+ let foreign = null;
+ try {
+ mkdirSync(app, { recursive: true });
+ foreign = startContainer(name);
+ const lease = createBackendLease({ runId: `collision-${process.pid}`, backend: 'spacetime',
+ track: 'fault-injection', runIndex: 0, serverUri: 'http://127.0.0.1:1',
+ module: `collision-${process.pid}`, dataDir: join(root, 'data') });
+ lease.state = 'active';
+ writeBackendLease(leasePath, lease);
+
+ let refused = false;
+ try {
+ execFileSync(process.execPath,
+ [RUN_BUILD, '--app', app, '--backend', 'spacetime', '--prepare-only'],
+ { stdio: 'pipe', env: { ...process.env, STACK_BENCH_LEASE: leasePath,
+ STACK_BENCH_LEASE_TOKEN: lease.ownershipToken } });
+ } catch (error: unknown) {
+ const childError = error instanceof Error && isRecord(error) ? error : null;
+ refused = childError?.status === 3
+ && /refusing to adopt existing unleased container/.test(String(childError?.stderr));
+ }
+ assert.equal(refused, true, 'launcher did not explicitly refuse an unleased same-name container');
+ assert.deepEqual(inspectContainer(foreign.id), foreign,
+ 'collision refusal changed or stopped the foreign container');
+ } finally {
+ removeExactContainer(foreign);
+ rmSync(root, { recursive: true, force: true });
+ }
+}
+
+async function main() {
+ assert.ok(existsSync(CLI), `local SpacetimeDB CLI is missing: ${CLI}`);
+ execFileSync('docker', ['image', 'inspect', IMAGE], { stdio: 'pipe' });
+ await assertRefusesUnleasedCollision();
+
+ const root = mkdtempSync(join(tmpdir(), 'stack-bench-fault-'));
+ const app = join(root, 'app');
+ const out = join(root, 'out');
+ const markerPath = join(app, '.fault-ready.json');
+ const port = await freePort();
+ const uri = `http://127.0.0.1:${port}`;
+ const foreignName = `stack-bench-foreign-${process.pid}-${Date.now()}`;
+ let foreignContainer: ContainerIdentity | null = null;
+ let foreignServer: Server | null = null;
+ let bench: ChildProcess | null = null;
+ let marker: { lease: { runId: string; state: string; resources: FaultLeaseResources }; leasePath: string; phase: string } | null = null;
+ let output = '';
+
+ try {
+ mkdirSync(app, { recursive: true });
+ mkdirSync(out, { recursive: true });
+ foreignContainer = startContainer(foreignName);
+ const startedForeignServer = createServer((_request, response) => response.end('foreign'));
+ foreignServer = startedForeignServer;
+ await new Promise((ok, fail) => startedForeignServer.listen({ port: 0, host: '127.0.0.1' }, ok).once('error', fail));
+ const foreignAddress = startedForeignServer.address();
+ if (!foreignAddress || typeof foreignAddress === 'string') throw new Error('could not allocate foreign TCP port');
+ const foreignUri = `http://127.0.0.1:${foreignAddress.port}`;
+
+ bench = spawn(process.execPath,
+ [compiledEntrypoint('commands', 'bench.js'), '--backend', 'spacetime', '--track', 'loop',
+ '--levels', '1', '--agent-adapter', 'fault-injection', '--app', app, '--out', out,
+ '--url', `file:///${app.replace(/\\/g, '/')}/index.html`],
+ { env: { ...process.env, STACK_BENCH_STDB_URI: uri, STACK_BENCH_IMAGE: IMAGE,
+ SPACETIME_BIN: CLI },
+ stdio: ['ignore', 'pipe', 'pipe'], windowsHide: true });
+ const collect = (chunk: Buffer): void => { output = (output + chunk.toString()).slice(-256 * 1024); };
+ bench.stdout?.on('data', collect);
+ bench.stderr?.on('data', collect);
+ const exited = await waitForExit(bench, 300_000);
+ assert.notEqual(exited.code, 0, 'injected coding-agent failure unexpectedly exited zero');
+ assert.ok(existsSync(markerPath), `fault marker was not written before failure:\n${output}`);
+ const markerValue: unknown = JSON.parse(readFileSync(markerPath, 'utf8'));
+ if (!isRecord(markerValue) || !isRecord(markerValue.lease) || !isRecord(markerValue.lease.resources)
+ || typeof markerValue.leasePath !== 'string' || typeof markerValue.phase !== 'string'
+ || typeof markerValue.lease.runId !== 'string' || typeof markerValue.lease.state !== 'string') {
+ throw new Error('fault marker is invalid');
+ }
+ const markerResources = markerValue.lease.resources;
+ if (!Array.isArray(markerResources.listenerProcesses) || !isRecord(markerResources.buildContainer)
+ || typeof markerResources.buildContainer.id !== 'string') throw new Error('fault marker resources are invalid');
+ marker = { phase: markerValue.phase, leasePath: markerValue.leasePath,
+ lease: { runId: markerValue.lease.runId, state: markerValue.lease.state,
+ resources: { listenerProcesses: markerResources.listenerProcesses.filter((item): item is {
+ pid: number; startMarker: string } => isRecord(item) && typeof item.pid === 'number'
+ && typeof item.startMarker === 'string'), buildContainer: {
+ id: markerResources.buildContainer.id, image: String(markerResources.buildContainer.image ?? ''),
+ running: markerResources.buildContainer.running === true }, locks: [] } } };
+ assert.equal(marker.phase, 'restart-stopped',
+ 'fault was not injected inside the backend restart window');
+ assert.equal(marker.lease.state, 'restarting');
+ assert.match(marker.lease.resources.buildContainer.image, /^sha256:[0-9a-f]{64}$/,
+ 'build container lease did not record an immutable image id');
+
+ const evidencePath = join(out, ARTIFACT_FILE.backendLease);
+ assert.ok(existsSync(evidencePath), `teardown did not preserve lease evidence:\n${output}`);
+ const evidence = readArtifactPayload(evidencePath, { expectedKind: 'backend_lease_evidence' });
+ const preflight = readArtifact(join(out, ARTIFACT_FILE.preflight),
+ { expectedKind: 'preflight' });
+ assert.equal(preflight.payload.ok, true, 'paid-run preflight did not pass');
+ assert.equal(preflight.attempt.parentId, marker.lease.runId,
+ 'preflight evidence is not attached to the run it admitted');
+ assert.equal(evidence.runId, marker.lease.runId);
+ assert.equal(evidence.state, 'released', 'benchmark lease did not reach its terminal state');
+ assert.ok(evidence.stoppedAt, 'benchmark-owned SpacetimeDB host has no stop evidence');
+ assert.ok(evidence.releasedAt, 'benchmark lease has no release evidence');
+ assert.deepEqual(evidence.resources.listenerProcesses, []);
+ assert.equal(evidence.resources.buildContainer.running, false,
+ 'benchmark-owned build container was not marked removed');
+ assert.ok(evidence.resources.buildContainer.removedAt);
+ assert.ok(evidence.resources.locks.every(lock => lock.releasedAt),
+ 'one or more resource locks were not released');
+ assert.equal(inspectContainer(marker.lease.resources.buildContainer.id), null,
+ 'benchmark-owned build container survived fatal cleanup');
+ assert.equal(pidsOnPort(port).length, 0, 'benchmark-owned listener survived fatal cleanup');
+ assert.equal(existsSync(marker.leasePath), false, 'private runtime lease was not removed');
+
+ assert.equal((await fetch(foreignUri)).status, 200,
+ 'foreign listener was disturbed by benchmark cleanup');
+ assert.deepEqual(inspectContainer(foreignContainer.id), foreignContainer,
+ 'foreign container was changed or removed by benchmark cleanup');
+
+ console.log(JSON.stringify({ ok: true, injectedAt: 'restart-stopped-before-replacement',
+ benchmarkHostStopped: true, benchmarkContainerRemoved: true, locksReleased: true,
+ privateLeaseRemoved: true, foreignListenerSurvived: true,
+ foreignContainerSurvived: true, unleasedCollisionRefused: true,
+ immutableImagePinned: true }, null, 2));
+ } finally {
+ if (bench?.exitCode === null) {
+ killTree(bench.pid);
+ await delay(500);
+ }
+ if (marker?.lease?.resources?.buildContainer) {
+ removeExactContainer(marker.lease.resources.buildContainer);
+ }
+ for (const identity of marker?.lease?.resources?.listenerProcesses ?? []) {
+ if (pidsOnPort(port).includes(String(identity.pid))) killTree(identity.pid);
+ }
+ if (foreignServer) {
+ const server = foreignServer;
+ // The verification fetch uses a keep-alive connection. Waiting on
+ // close() alone can hold CI open until Undici retires that socket.
+ server.closeAllConnections();
+ await new Promise((ok, fail) => server.close(error => error ? fail(error) : ok()));
+ }
+ removeExactContainer(foreignContainer);
+ rmSync(root, { recursive: true, force: true });
+ }
+}
+
+main().catch(error => {
+ console.error(error.stack ?? error.message);
+ process.exitCode = 1;
+});
diff --git a/tools/stack-bench/commands/leak-audit.ts b/tools/stack-bench/commands/leak-audit.ts
new file mode 100644
index 00000000000..11e156302a2
--- /dev/null
+++ b/tools/stack-bench/commands/leak-audit.ts
@@ -0,0 +1,287 @@
+#!/usr/bin/env node
+// Use the recorded cwd as the app boundary; transcript folder names are not authority.
+
+import { readdirSync, readFileSync, existsSync, statSync } from 'node:fs';
+import { join, resolve } from 'node:path';
+import { homedir } from 'node:os';
+import { pathToFileURL } from 'node:url';
+import { parseArgs } from 'node:util';
+import { CODING_CONTAINER_APP_ROOT } from '../src/runtime/coding-container-policy.js';
+
+// --dir is exclusive. --app resolves its matching CLI transcript directory.
+function transcriptsFor(appDir: string): string[] {
+ const base = join(homedir(), '.claude', 'projects');
+ if (!existsSync(base)) return [];
+ const want = resolve(appDir).replace(/[\\/:]/g, '-').toLowerCase();
+ return readdirSync(base)
+ .filter(d => d.toLowerCase() === want || d.toLowerCase() === want.replace(/^-+/, ''))
+ .map(d => join(base, d));
+}
+
+const norm = (value: unknown): string => String(value ?? '')
+ .replace(/\\/g, '/').replace(/^["']|["']$/g, '').toLowerCase();
+
+// Ignore dependencies, build output, and this session's CLI task output.
+const IGNORE = /node_modules|\.git[/\\]|package-lock\.json|\/dist\/|\.map$|[/\\]temp[/\\]claude[/\\].*[/\\]tasks[/\\]/;
+
+// Commands that pull file contents into context.
+const READER = /(?:^|[;&|]\s*)(?:cat|head|tail|less|more|type|grep|rg|ack|find|ls\s+-\w*l|sed\s+-n|awk)\s+([^;&|]+)/g;
+
+// Network targets in a shell command: any URL, and a raw socket target. A
+// build legitimately reaches its own web, database, and SpacetimeDB ports;
+// every other local port belongs to another run, the controller, or the
+// dashboard. Internet targets are recorded, not judged.
+const URL_TARGET = /https?:\/\/([^\s/'"`]+)/gi;
+const SOCKET_TARGET = /(?:^|[;&|]\s*)(?:nc|ncat|netcat)\s+(?:-\S+\s+)*([\w.-]+)\s+(\d{2,5})\b/g;
+const LOCAL_HOST = /^(?:127\.\d+\.\d+\.\d+|localhost|0\.0\.0\.0|\[::1\]|host\.docker\.internal|10\.\d+\.\d+\.\d+|172\.(?:1[6-9]|2\d|3[01])\.\d+\.\d+|192\.168\.\d+\.\d+)$/i;
+
+export interface NetworkTarget {
+ host: string;
+ port: number | null;
+}
+
+export function networkTargetsFromBash(command: unknown): NetworkTarget[] {
+ const targets: NetworkTarget[] = [];
+ const text = String(command ?? '');
+ for (const match of text.matchAll(URL_TARGET)) {
+ const authority = (match[1] ?? '').replace(/^[^@]*@/, '');
+ const port = authority.match(/:(\d{1,5})$/)?.[1];
+ targets.push({ host: authority.replace(/:\d{1,5}$/, ''), port: port ? Number(port) : null });
+ }
+ for (const match of text.matchAll(SOCKET_TARGET)) {
+ targets.push({ host: match[1] ?? '', port: Number(match[2]) });
+ }
+ return targets;
+}
+
+const networkKind = (target: NetworkTarget, ownPorts: ReadonlySet): string | null => {
+ if (!LOCAL_HOST.test(target.host)) return 'network (internet)';
+ if (target.port !== null && ownPorts.has(target.port)) return null;
+ return 'NETWORK / OTHER RUN';
+};
+
+const CLASSES: Array = [
+ [/[/\\]stack-bench(?:[/\\]|$)/, 'GRADER / TEST SPECS'],
+ [/\.claude[/\\]projects.*memory|[/\\]memory[/\\].*\.md$/, 'BENCHMARK NOTES'],
+ [/scenarios[/\\].*\.json|grade\.(?:js|ts)|mutation|check-scenarios/, 'GRADER / TEST SPECS'],
+ [/contracts[/\\].*\.json|appendix-\d+\.md|walk\.(?:js|ts)|lint\.(?:js|ts)/, 'CONTRACT / LINTER'],
+ [/prompts[/\\]|test-plans[/\\]|GRADING|RUBRIC/, 'PROMPTS / RUBRIC'],
+ [/[/\\]skills[/\\]/, 'skill docs (intended)'],
+ [/backends[/\\].*\.md|CLAUDE\.md|README/, 'setup docs (intended)'],
+];
+const classify = (path: string): string => CLASSES.find(([pattern]) => pattern.test(path))?.[1]
+ ?? 'other';
+
+// The shallowest recorded cwd is the app boundary when --app is absent.
+function sessionCwd(lines: string[]): string | null {
+ const seen = new Set();
+ for (const l of lines) {
+ const m = l.match(/"cwd":"((?:[^"\\]|\\.)*)"/);
+ if (m?.[1]) seen.add(norm(m[1].replace(/\\\\/g, '/')));
+ }
+ if (!seen.size) return null;
+ return [...seen].sort((a, b) => a.split('/').length - b.split('/').length || a.length - b.length)[0]
+ ?? null;
+}
+
+export function pathsFromBash(command: unknown): string[] {
+ const out: string[] = [];
+ for (const match of String(command).matchAll(READER)) {
+ const argumentsText = match[1];
+ if (!argumentsText) continue;
+ for (const tokRaw of argumentsText.split(/\s+/)) {
+ const t = tokRaw.replace(/^["']|["']$/g, '');
+ if (!t || t.startsWith('-')) continue;
+ if (/[*?]/.test(t) || /\//.test(t) || /\\/.test(t) || /\.\w+$/.test(t)) out.push(t);
+ }
+ }
+ return out;
+}
+
+// Count file-tool reads only after their result confirms success.
+// Bash reads count unless the command fails.
+interface AuditHit {
+ path: string;
+ via: string;
+ kind: string;
+ unresolved?: boolean;
+}
+
+interface PendingRead {
+ paths: string[];
+ network: Array<{ path: string; kind: string }>;
+ via: string;
+}
+
+interface TranscriptAudit {
+ file: string;
+ cwd: string | null;
+ fileTool: number;
+ bashReads: number;
+ hits: AuditHit[];
+ refused: AuditHit[];
+}
+
+interface AuditResult extends TranscriptAudit {
+ root: string;
+}
+
+interface TranscriptContent {
+ type?: string;
+ name?: string;
+ id?: string;
+ tool_use_id?: string;
+ is_error?: boolean;
+ input?: { file_path?: string; path?: string; pattern?: string; command?: string };
+}
+
+export function auditTranscript(file: string, boundary: string | null,
+ { ownPorts = [] }: { ownPorts?: readonly number[] } = {}): TranscriptAudit {
+ const own = new Set(ownPorts);
+ const lines = readFileSync(file, 'utf8').split('\n').filter(Boolean);
+ // Container transcripts use /app; --app is its host path.
+ const recorded = sessionCwd(lines);
+ const cwd = recorded === CODING_CONTAINER_APP_ROOT ? recorded : (boundary ?? recorded);
+ const hits: AuditHit[] = [];
+ const refused: AuditHit[] = [];
+ const pending = new Map();
+ let fileTool = 0, bashReads = 0;
+
+ for (const line of lines) {
+ let event: { message?: { content?: TranscriptContent[] } };
+ try { event = JSON.parse(line) as typeof event; } catch { continue; }
+ const c = event.message?.content;
+ if (!Array.isArray(c)) continue;
+ for (const p of c) {
+ if (p.type === 'tool_result' && p.tool_use_id && pending.has(p.tool_use_id)) {
+ const completed = pending.get(p.tool_use_id ?? '');
+ if (!completed) continue;
+ const { paths, network, via } = completed;
+ pending.delete(p.tool_use_id ?? '');
+ const blocked = p.is_error === true;
+ for (const n of paths) (blocked ? refused : hits).push({ path: n, via, kind: classify(n) });
+ for (const target of network) (blocked ? refused : hits).push({ ...target, via });
+ continue;
+ }
+ if (p.type !== 'tool_use') continue;
+ const cand = [];
+ if (/^(Read|Grep|Glob|NotebookRead)$/.test(p.name ?? '')) {
+ fileTool++;
+ cand.push(p.input?.file_path ?? p.input?.path ?? p.input?.pattern ?? '');
+ } else if (p.name === 'Bash') {
+ const found = pathsFromBash(p.input?.command ?? '');
+ bashReads += found.length;
+ cand.push(...found);
+ }
+ const paths = [];
+ const network: Array<{ path: string; kind: string }> = [];
+ if (p.name === 'Bash') {
+ for (const target of networkTargetsFromBash(p.input?.command ?? '')) {
+ const kind = networkKind(target, own);
+ if (kind) network.push({ path: `${target.host}${target.port === null ? '' : `:${target.port}`}`, kind });
+ }
+ }
+ for (const raw of cand) {
+ let n = norm(raw);
+ if (!n || IGNORE.test(n)) continue;
+ // The CLI keeps auto-memory for the session's OWN project dir. A build
+ // A session may read its own memory, never another project's memory.
+ if (cwd && /[/\\]projects[/\\][^/\\]+[/\\]memory[/\\]/.test(n)
+ && n.includes(cwd.replace(/[\\/:]/g, '-'))) continue;
+ const absolute = /^[a-z]:/.test(n) || n.startsWith('/');
+ if (!absolute && cwd) n = `${cwd}/${n.replace(/^\.\//, '')}`;
+ const privateHarnessPath = cwd
+ && (n === `${cwd}/stack-bench` || n.startsWith(`${cwd}/stack-bench/`));
+ if (!privateHarnessPath && !absolute) continue;
+ if (!privateHarnessPath && cwd && (n === cwd || n.startsWith(`${cwd}/`))) continue;
+ paths.push(n);
+ }
+ if ((paths.length || network.length) && p.id && p.name) {
+ pending.set(p.id, { paths, network, via: p.name });
+ }
+ }
+ }
+ // A call whose result never arrived (session cut short) is unresolved, and
+ // unresolved is not innocent: count it.
+ for (const { paths, network, via } of pending.values()) {
+ for (const n of paths) hits.push({ path: n, via, kind: classify(n), unresolved: true });
+ for (const target of network) hits.push({ ...target, via, unresolved: true });
+ }
+
+ return { file, cwd, fileTool, bashReads, hits, refused };
+}
+
+function main(): void {
+ const { values } = parseArgs({ args: process.argv.slice(2), options: {
+ app: { type: 'string' }, dir: { type: 'string' }, json: { type: 'boolean' },
+ 'own-ports': { type: 'string' },
+ } });
+ const ownPorts = (values['own-ports'] ?? '').split(',').filter(Boolean).map(Number);
+ if (ownPorts.some(port => !Number.isInteger(port) || port <= 0)) {
+ throw new Error('--own-ports must list positive integers');
+ }
+ const requestedApp = values.app;
+ const requestedDirectory = values.dir;
+ if (requestedApp && requestedDirectory) throw new Error('--app and --dir cannot be used together');
+ const roots = requestedApp ? transcriptsFor(requestedApp)
+ : requestedDirectory ? [resolve(requestedDirectory)]
+ : [join(homedir(), '.claude', 'projects')];
+ // When the caller names the app directory, that is the boundary. Do not
+ // infer it from a transcript folder name.
+ const appBoundary = requestedApp ? norm(resolve(requestedApp)) : null;
+ const results: AuditResult[] = [];
+for (const root of roots) {
+ if (!existsSync(root)) continue;
+ const stack = [root];
+ while (stack.length) {
+ const d = stack.pop();
+ if (!d) continue;
+ for (const e of readdirSync(d, { withFileTypes: true })) {
+ const p = join(d, e.name);
+ if (e.isDirectory()) { if (!/node_modules/.test(p)) stack.push(p); continue; }
+ if (!/\.jsonl$/.test(e.name)) continue;
+ // Include transcripts from the main session and its subagents.
+ if (!/transcript|^agent-|^[0-9a-f-]{36}\.jsonl$/.test(e.name)) continue;
+ if (statSync(p).size < 2000) continue;
+ results.push({ ...auditTranscript(p, appBoundary, { ownPorts }), root });
+ }
+ }
+}
+
+if (values.json) {
+ console.log(JSON.stringify(results, null, 2));
+ return;
+}
+
+const label = (file: string): string => file.replace(/\\/g, '/')
+ .split('/').slice(-3).join('/').slice(0, 62);
+console.log('\nBuilds that read outside their own directory');
+console.log('(counts BOTH file tools and Bash cat/grep/find; boundary = the session\'s own cwd)\n');
+
+let clean = 0;
+for (const r of results.sort((a, b) => b.hits.length - a.hits.length)) {
+ if (!r.cwd) { console.log(` ?? ${label(r.file)} — no cwd recorded, cannot judge`); continue; }
+ if (!r.hits.length) {
+ clean++;
+ // Blocked attempts are worth printing: they are the sandbox doing its job,
+ // and they say which paths a build still goes looking for.
+ if (r.refused?.length) {
+ const kinds = [...new Set(r.refused.map(h => h.kind))].join(', ');
+ console.log(` ${label(r.file)}\n clean — ${r.refused.length} attempt(s) BLOCKED by the sandbox (${kinds})`);
+ }
+ continue;
+ }
+ const byKind: Record = {};
+ for (const h of r.hits) (byKind[h.kind] ??= []).push(h.path);
+ console.log(` ${label(r.file)}`);
+ console.log(` cwd: ...${r.cwd.slice(-52)} (${r.fileTool} file-tool, ${r.bashReads} bash reads)`);
+ for (const [k, v] of Object.entries(byKind).sort((a, b) => b[1].length - a[1].length)) {
+ const example = [...new Set(v)][0] ?? '';
+ console.log(` ${String(v.length).padStart(3)}x ${k.padEnd(22)} ${example.split('/').slice(-2).join('/')}`);
+ }
+}
+console.log(`\n ${clean} transcript(s) read nothing outside their directory.`);
+console.log(` ${results.length} transcript(s) examined.\n`);
+}
+
+if (process.argv[1] && pathToFileURL(process.argv[1]).href === import.meta.url) main();
diff --git a/tools/stack-bench/commands/null-control.ts b/tools/stack-bench/commands/null-control.ts
new file mode 100644
index 00000000000..4716e66e1ea
--- /dev/null
+++ b/tools/stack-bench/commands/null-control.ts
@@ -0,0 +1,260 @@
+#!/usr/bin/env node
+// Grade the real validated production scenarios against a reachable app that
+// implements nothing. Every point-bearing criterion must conclusively fail.
+
+import { execFile } from 'node:child_process';
+import { createServer } from 'node:http';
+import type { Server } from 'node:http';
+import type { AddressInfo } from 'node:net';
+import { mkdirSync, mkdtempSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { basename, join, relative, resolve } from 'node:path';
+import { pathToFileURL } from 'node:url';
+import { parseArgs as parseNodeArgs } from 'node:util';
+import { readArtifactPayload, writeRunJson } from '../src/evidence/artifacts.js';
+import { calibrationQualificationIdentity, calibrationQualificationRelease,
+ resolveCalibrationForRelease } from '../src/composition/calibration-compiler.js';
+import { qualificationScopeIdentity } from '../src/composition/qualification-scope.js';
+import { analyseNullReports } from '../src/evidence/null-control-analysis.js';
+import { resolveRecipeRelease } from '../src/composition/recipe-release.js';
+import { resolveRecipeSelection } from '../src/composition/recipe-selection.js';
+import { isDeclaredLevel, listTracks, loadTrack, suitesFor } from '../src/composition/tracks.js';
+import { controllerRunner } from '../src/runtime/runner-environment.js';
+import type { CalibrationPlan } from '../src/composition/calibration-compiler.js';
+import type { RecipeBinding } from '../src/composition/recipe-release.js';
+import type { Track } from '../src/composition/tracks.js';
+
+import { STACK_BENCH_ROOT as ROOT, compiledEntrypoint } from '../src/package-root.js';
+const GRADE = compiledEntrypoint('grader', 'grade.js');
+const NULL_CONTROL_WORKERS = 4;
+
+interface NullControlArgs {
+ tracks: string[];
+ level: number | null;
+ recipe?: string;
+ out?: string;
+ audit: boolean;
+ parentAttemptId?: string;
+}
+
+export function parseNullControlArgs(argv: string[]): NullControlArgs {
+ const { values } = parseNodeArgs({ args: argv.slice(2), options: {
+ track: { type: 'string' }, level: { type: 'string' }, recipe: { type: 'string' },
+ out: { type: 'string' }, audit: { type: 'boolean' }, 'parent-attempt-id': { type: 'string' },
+ } });
+ const args: NullControlArgs = {
+ tracks: values.track?.split(',').filter(Boolean) ?? listTracks(),
+ level: values.level === undefined ? null : Number(values.level), audit: values.audit ?? false,
+ recipe: values.recipe, out: values.out, parentAttemptId: values['parent-attempt-id'],
+ };
+ if (args.level !== null && (!Number.isInteger(args.level) || args.level < 1)) {
+ throw new Error('--level must be a positive integer');
+ }
+ if (args.level !== null && args.tracks.length !== 1) {
+ throw new Error('--level requires exactly one --track');
+ }
+ if (args.recipe && args.level === null) throw new Error('--recipe requires --level');
+ return args;
+}
+
+function runGrade(argv: string[], timeoutMs = 300_000): Promise<{ stdout: string; stderr: string }> {
+ return new Promise((resolve, reject) => {
+ execFile(process.execPath, [GRADE, ...argv], {
+ encoding: 'utf8', maxBuffer: 32 * 1024 * 1024, timeout: timeoutMs,
+ }, (error, stdout, stderr) => {
+ if (error) {
+ error.message = `grader failed: ${error.message}\n${stdout}\n${stderr}`;
+ reject(error);
+ } else resolve({ stdout, stderr });
+ });
+ });
+}
+
+export function nullControlSuites(track: Track, selectedLevel: number | null = null,
+ binding: RecipeBinding | null = null) {
+ if (selectedLevel !== null && !isDeclaredLevel(track, selectedLevel)) {
+ throw new Error(`L${selectedLevel} is not declared for ${track.name}`);
+ }
+ if (binding) {
+ if (selectedLevel === null) throw new Error('recipe-bound null control requires one level');
+ if (!Array.isArray(binding.execution) || !binding.execution.length) {
+ throw new Error('recipe-bound null control requires a typed execution plan');
+ }
+ const executionIds = new Set();
+ const mappedKeys = new Set();
+ const suites = binding.execution.map(execution => {
+ if (executionIds.has(execution.id)) {
+ throw new Error(`recipe-bound null control repeats execution ${execution.id}`);
+ }
+ executionIds.add(execution.id);
+ const checks = binding.release.checkCatalog.filter(check => check.executionId === execution.id);
+ if (!checks.length) {
+ throw new Error(`recipe-bound null control execution ${execution.id} maps no checks`);
+ }
+ for (const check of checks) {
+ if (mappedKeys.has(check.stableKey)) {
+ throw new Error(`recipe-bound null control maps check ${check.stableKey} more than once`);
+ }
+ mappedKeys.add(check.stableKey);
+ }
+ return { id: execution.id, spec: resolve(track.dir, execution.source ?? ''),
+ level: selectedLevel, checks };
+ });
+ const missing = binding.release.checkCatalog
+ .filter(check => !mappedKeys.has(check.stableKey)).map(check => check.stableKey);
+ if (missing.length) {
+ throw new Error(`recipe-bound null control leaves checks unmapped: ${missing.join(', ')}`);
+ }
+ return suites;
+ }
+ const seen = new Set();
+ const suites = [];
+ const levels = selectedLevel === null
+ ? Array.from({ length: track.validatedThrough }, (_, index) => index + 1)
+ : [selectedLevel];
+ for (const level of levels) {
+ for (const suite of suitesFor(track, level)) {
+ if (seen.has(suite.spec)) continue;
+ seen.add(suite.spec);
+ suites.push({ ...suite, level });
+ }
+ }
+ return suites;
+}
+
+export function selectNullQualificationBinding(binding: RecipeBinding, calibration: CalibrationPlan): RecipeBinding {
+ const selected = calibrationQualificationRelease(calibration, binding.release, binding.execution);
+ return { ...binding, release: selected.release, execution: selected.execution };
+}
+
+export function createNullQualification(binding: RecipeBinding, calibration: CalibrationPlan) {
+ const selectedBinding = selectNullQualificationBinding(binding, calibration);
+ const selection = resolveRecipeSelection(selectedBinding.release, {
+ checkKeys: selectedBinding.release.checkCatalog.map(check => check.stableKey),
+ });
+ return {
+ binding: selectedBinding,
+ calibration,
+ identity: calibrationQualificationIdentity(calibration),
+ selectionSha256: selection.sha256,
+ };
+}
+
+async function listen(server: Server): Promise {
+ await new Promise