From f95032da5eb2d98a47c463beab7fd69ac9059c8c Mon Sep 17 00:00:00 2001 From: Moritz Hoffmann Date: Wed, 8 Jul 2026 22:42:06 +0200 Subject: [PATCH 01/45] console: design doc for dataflow visualizer rebuild Co-Authored-By: Claude Sonnet 5 --- .../20260706_dataflow_visualizer_rebuild.md | 156 ++++++++++++++++++ 1 file changed, 156 insertions(+) create mode 100644 console/doc/design/20260706_dataflow_visualizer_rebuild.md diff --git a/console/doc/design/20260706_dataflow_visualizer_rebuild.md b/console/doc/design/20260706_dataflow_visualizer_rebuild.md new file mode 100644 index 0000000000000..f02a4d0601261 --- /dev/null +++ b/console/doc/design/20260706_dataflow_visualizer_rebuild.md @@ -0,0 +1,156 @@ +# Dataflow visualizer rebuild + +- Associated: [CNS-108](https://linear.app/materializeinc/issue/CNS-108), [CNS-109](https://linear.app/materializeinc/issue/CNS-109) + +## The Problem + +The dataflow visualizer (`src/platform/clusters/DataflowVisualizer.tsx`) builds a Graphviz DOT string by hand and renders it with `d3-graphviz`, which pulls in a WASM Graphviz build. +This design has produced two open bugs. +CNS-108: `scopeToGv` concatenates catalog strings into DOT and only escapes double quotes, allowing XSS through crafted object names. +CNS-109: `useSqlApiRequest` never clears previous results, so a failed refetch leaves the stale graph on screen. +Beyond the bugs, the visualizer lacks features operators need: no dataflow selection (one object, one dataflow), no whole-graph overview (drill-down replaces the canvas), no filtering, and no way to find hotspots in large dataflows. + +## Success Criteria + +* A user can pick any dataflow running on a replica, including transient ones, and see its operator graph. +* Regions expand and collapse in place, with aggregated stats and rerouted edges on collapsed regions. +* Nodes show scheduling time and arrangement size, edges show message counts and container type. +* Filter tools let a user locate operators by name, hide idle elements, heat-color by metric, and filter edges by container type. +* Rendering strings from the catalog cannot inject markup (closes CNS-108 by construction). +* A failed refetch always replaces the graph with an error state (closes CNS-109, with a regression test). +* No `d3-graphviz` or WASM dependency remains. + +## Out of Scope + +* Backend changes. + All required data exists in `mz_introspection` relations already queried today. +* Live-updating stats via `SUBSCRIBE`. + The data layer separates structure from stats so this can be added later, but v1 ships manual refresh only. +* Per-worker breakdowns and skew analysis. + V1 uses the worker-aggregated introspection views, as today. +* URL-persisted filter and collapse state. +* New end-to-end tests. + +## Solution Proposal + +Rebuild the visualizer on React Flow (`@xyflow/react`) for rendering and interaction, with elkjs (`elk.layered`) for hierarchical layout in a web worker. +Nodes become React components styled with Chakra, so all catalog strings render as text nodes and the DOT injection class disappears. +React Flow provides pan, zoom, minimap, selection, nested parent nodes, and viewport culling (`onlyRenderVisibleElements`), which matters for dataflows with over a thousand operators. +elkjs is the only mainstream JS layout engine with real compound-graph support: nested regions and edges routed across hierarchy boundaries. +elkjs is GWT-compiled JavaScript, not WASM. + +### Product surface + +Two entry points share one graph implementation. + +* A new "Dataflows" tab on cluster detail: replica selector, then a dataflow list from `mz_dataflows` joined with `mz_records_per_dataflow` and summed `mz_scheduling_elapsed` (name, records, size, elapsed). Clicking a row opens the graph view. This covers transient dataflows such as peeks and subscribes. +* The existing "Visualize" tab on index and materialized view detail deep-links into the cluster page with the object's dataflow preselected, resolved via `mz_compute_exports`. + +Both remain behind the `visualization-features` LaunchDarkly flag. +The old `DataflowVisualizer.tsx` and the `d3-graphviz` dependency are deleted once the new view lands, since the visualizer is its only consumer. + +### Data layer + +The three structure queries from `useDataflowStructure.ts` (operators, channels, LIR operators) are kept, but the operator and channel queries are keyed by dataflow id instead of export id so transient dataflows work. +`mz_lir_mapping` is keyed by `global_id`, so LIR data exists only for dataflows backing an export. +For transient dataflows the LIR panel and badges are hidden. +A new cheap query feeds the dataflow selector list. + +A new hook returns results tagged with the parameters that produced them. +The render layer discards results whose tag does not match the current selection, and an error always wins over retained data. +This closes CNS-109 without depending on the broken `requestIdRef` logic in `useSqlApiRequest`. + +Refresh is manual: a refresh button plus a "last fetched" timestamp. +Layout, collapse state, and viewport survive a refresh. +Stats update in place, and relayout happens only if the structure changed. +The structure/stats split makes a later `SUBSCRIBE`-driven live mode a drop-in: stats stream into node badges without relayout. + +### Graph model and collapse + +Pure functions in a new `dataflowGraph.ts` build the region tree from `mz_dataflow_addresses` and `mz_dataflow_operator_parents`, replacing `collateOperators`. +Regions map to React Flow parent (group) nodes, operators to leaf nodes. +Every node carries own and transitive stats: arrangement records, size, elapsed. + +Each region is expanded or collapsed in place. +A collapsed region renders as a single node with aggregated stats and a child count. +Edges crossing a collapsed boundary are remapped to the region node and aggregated: one edge per direction pair, summed records and batches, and the set of container types. +The derivation is a pure function from `(structure, collapseState)` to visible nodes and edges. +The default state shows the root's direct children with everything below collapsed. + +Regions and LIR spans are two different hierarchies, so only regions get the nesting. +LIR information appears as a badge and operator text on each node, plus a side panel listing the dataflow's LIR operators from `mz_lir_mapping`. +Hovering or clicking a panel entry highlights the member operators on the canvas. +Scope inputs and outputs stay as small port nodes on the region boundary, as today. + +```mermaid +flowchart LR + sql[SQL results] --> structure[dataflowGraph.ts: region tree + stats] + structure --> visible["(structure, collapseState, filters) -> visible nodes/edges"] + visible --> elk[elkjs worker layout] + elk --> rf[React Flow canvas] + rf -- expand/collapse, filter --> visible +``` + +### Rendering and layout + +elkjs runs `elk.layered` in a web worker, direction left to right, over exactly the visible post-collapse graph. +Toggling collapse recomputes layout, memoized per collapse state so toggling back is instant. +A small "layouting" overlay shows during computation while the canvas stays interactive. + +The node component shows name, arranged records, size, and elapsed time. +Default colors keep today's two-by-two palette (region versus operator, has arrangement versus not). +Heatmap mode overrides node color. +Edges are labeled with records and batches, dashed when zero messages were sent, with the container type as an edge badge and tooltip. + +Perf guards: collapse-by-default keeps the initial node count small, viewport culling handles large expanded graphs, and if the visible node count would exceed roughly 1500 the UI warns and refuses expand-all. + +### Filters and interactions + +A toolbar above the canvas offers: + +* Name search: matching operators highlighted, others dimmed, next/previous jump that auto-expands ancestor regions of a match. +* Hide idle: hides zero-message edges and zero-elapsed operators, with region aggregates recomputed over the visible set. +* Heatmap: mode select (off, elapsed, arrangement size), sequential color scale, threshold slider that dims nodes below the cutoff, with a legend. +* Channel type: multi-select over the container types present, non-matching edges dimmed. + +Filters are pure derivations over the visible graph and compose. +Filter state lives in component state. +Clicking a node opens a detail side panel (full name, all stats, LIR operator, address). +Double-clicking a region toggles collapse. + +### Error handling + +An error always replaces the graph. +The CNS-109 regression test from the issue lands, adapted to the new component. +`INSUFFICIENT_PRIVILEGE` keeps the existing USAGE-privilege alert. +A timeout shows an error box with retry. +A replica or dataflow that disappeared on refresh shows a "dataflow no longer exists" empty state and refreshes the selector. + +## Minimal Viable Prototype + +The riskiest assumption is elkjs layout quality and speed on real dataflow shapes. +Phase 1 of the implementation plan is a thin vertical slice: fetch one dataflow, build the region tree, lay out the default collapsed view with elkjs, and render it in React Flow with expand/collapse only. +That slice runs against a local environmentd with a non-trivial dataflow (for example a multi-way join with reductions) and validates layout, before filters and the selector page are built. + +## Alternatives + +* Extend the in-house `src/components/Graph` Canvas plus dagre stack (used by WorkflowGraph, RoleGraph, CriticalPathGraph). + No new dependencies, but dagre's compound-graph support is weak, in-place collapse and boundary edge rerouting would be hand-built, and the plain SVG canvas has no viewport culling for graphs with over a thousand nodes. + The hard part of this feature is exactly the part dagre is bad at. +* In-house Canvas rendering with elkjs layout. + Keeps console rendering conventions and solves hierarchy, but rebuilds interaction machinery React Flow ships (nested nodes, culling, minimap, selection). +* Keep Graphviz but sanitize DOT properly. + Fixes CNS-108 but keeps the WASM dependency, string-based pipeline, and none of the requested interaction features. + +## Open questions + +* None blocking. + The `SUBSCRIBE` live mode and per-worker views are deliberate follow-ups, not open design points. + +## Testing + +* Unit (vitest): `dataflowGraph.ts` pure functions. + Region tree construction, collapse edge remapping and aggregation, filter derivations, LIR membership. + Most logic lives here, so most coverage lands here. +* Component (vitest plus msw): selector flow, error-on-refetch (the CNS-109 case), permission alert, empty states. + React Flow renders with the elk worker stubbed to deterministic positions. From 065ea7e66ea2395c93d97994c5eb317afbf69705 Mon Sep 17 00:00:00 2001 From: Moritz Hoffmann Date: Wed, 8 Jul 2026 22:42:06 +0200 Subject: [PATCH 02/45] console: address spec review for dataflow visualizer rebuild Co-Authored-By: Claude Sonnet 5 --- .../20260706_dataflow_visualizer_rebuild.md | 103 ++++++++++++++++-- 1 file changed, 96 insertions(+), 7 deletions(-) diff --git a/console/doc/design/20260706_dataflow_visualizer_rebuild.md b/console/doc/design/20260706_dataflow_visualizer_rebuild.md index f02a4d0601261..a6cb7c49735c9 100644 --- a/console/doc/design/20260706_dataflow_visualizer_rebuild.md +++ b/console/doc/design/20260706_dataflow_visualizer_rebuild.md @@ -27,7 +27,8 @@ Beyond the bugs, the visualizer lacks features operators need: no dataflow selec * Live-updating stats via `SUBSCRIBE`. The data layer separates structure from stats so this can be added later, but v1 ships manual refresh only. * Per-worker breakdowns and skew analysis. - V1 uses the worker-aggregated introspection views, as today. + V1 uses the non-per-worker introspection views, as today. + Note their semantics differ: structural views (operators, addresses, channels) restrict to worker 0, stats views (elapsed, arrangement sizes, message counts) sum across workers. * URL-persisted filter and collapse state. * New end-to-end tests. @@ -48,14 +49,20 @@ Two entry points share one graph implementation. Both remain behind the `visualization-features` LaunchDarkly flag. The old `DataflowVisualizer.tsx` and the `d3-graphviz` dependency are deleted once the new view lands, since the visualizer is its only consumer. +No e2e test references `DataflowVisualizer`, so deletion needs no test migration. ### Data layer -The three structure queries from `useDataflowStructure.ts` (operators, channels, LIR operators) are kept, but the operator and channel queries are keyed by dataflow id instead of export id so transient dataflows work. -`mz_lir_mapping` is keyed by `global_id`, so LIR data exists only for dataflows backing an export. -For transient dataflows the LIR panel and badges are hidden. +The three structure queries from `useDataflowStructure.ts` (operators, channels, LIR operators) are kept, but keyed by dataflow id instead of export id so transient dataflows work. +Channels need no extra join for this: the first element of an operator address is the dataflow id, so the channel query filters on `from_operator_address[1]`. +`mz_lir_mapping` is keyed by `global_id`, so the LIR query joins `mz_compute_exports` on `dataflow_id` to find the dataflow's export ids. +Transient dataflows (peeks, subscribes) are logged in both relations while alive (`CollectionLogging::new` runs for every export id in `compute_state.rs`, `log_lir_mapping` for every built object in `render.rs`), so LIR data works for them too. +A dataflow can have multiple exports, so the LIR panel groups entries by export id. A new cheap query feeds the dataflow selector list. +The full pre-collapse structure is fetched client-side in one shot. +Assumption: dataflows stay within tens of thousands of operators and channels, which fits comfortably in memory and within the existing 30 second query timeout, which v1 keeps. + A new hook returns results tagged with the parameters that produced them. The render layer discards results whose tag does not match the current selection, and an error always wins over retained data. This closes CNS-109 without depending on the broken `requestIdRef` logic in `useSqlApiRequest`. @@ -67,9 +74,11 @@ The structure/stats split makes a later `SUBSCRIBE`-driven live mode a drop-in: ### Graph model and collapse -Pure functions in a new `dataflowGraph.ts` build the region tree from `mz_dataflow_addresses` and `mz_dataflow_operator_parents`, replacing `collateOperators`. +Pure functions in a new `dataflowGraph.ts` build the region tree from `mz_dataflow_addresses` alone, replacing `collateOperators`. +An operator's parent is the operator whose address is its own minus the last element, so `mz_dataflow_operator_parents` (itself derived from addresses) is dropped to avoid two sources of truth that can disagree across query times. Regions map to React Flow parent (group) nodes, operators to leaf nodes. Every node carries own and transitive stats: arrangement records, size, elapsed. +Transitive stats are computed once when the structure is built, not per collapse toggle. Each region is expanded or collapsed in place. A collapsed region renders as a single node with aggregated stats and a child count. @@ -91,9 +100,73 @@ flowchart LR rf -- expand/collapse, filter --> visible ``` +### Type contracts + +The units communicate through these shapes, pinned up front so the graph model, the layout worker, and the rendering layer can be built independently. + +```typescript +type Address = number[]; +type NodeId = string; // JSON.stringify(address), as today + +interface NodeStats { + arrangementRecords: bigint; + arrangementSize: bigint; + elapsedNs: bigint; +} + +interface DataflowNode { + id: NodeId; + address: Address; + name: string; + parent: NodeId | null; + children: NodeId[]; // empty for leaf operators + own: NodeStats; + transitive: NodeStats; // own + subtree, precomputed + lir: { exportId: string; lirId: string; operator: string } | null; +} + +interface DataflowStructure { + nodes: Map; + root: NodeId; + channels: Channel[]; // as fetched, operator-level +} + +type CollapseState = ReadonlySet; // collapsed region ids + +interface VisibleEdge { + id: string; + source: NodeId; // visible node after remapping + target: NodeId; + messagesSent: bigint; + batchesSent: bigint; + channelTypes: string[]; // set union when aggregated +} + +interface VisibleGraph { + nodes: DataflowNode[]; // only visible ones, regions included + edges: VisibleEdge[]; // directed; A->B and B->A stay separate +} + +// Worker protocol. One in-flight request, stale responses dropped by id. +interface LayoutRequest { + requestId: number; + graph: VisibleGraph; +} +interface LayoutResponse { + requestId: number; + positions: Map; +} +``` + +Filter and collapse state are owned by the top-level visualizer page component and passed down. +The visible-graph derivation is `deriveVisibleGraph(structure, collapseState, filters): VisibleGraph`, a pure function. + ### Rendering and layout elkjs runs `elk.layered` in a web worker, direction left to right, over exactly the visible post-collapse graph. +The console builds with Vite, so the worker is loaded via Vite's worker import (`elkjs/lib/elk-worker.min.js?worker`) passed as `workerFactory` to the ELK constructor. +Phase 1 verifies this in both `vite dev` and the production build. +If the prebuilt worker script does not survive Vite bundling, the fallback is `elk.bundled.js` on the main thread in a lazily loaded chunk, accepting UI stalls during layout. Toggling collapse recomputes layout, memoized per collapse state so toggling back is instant. A small "layouting" overlay shows during computation while the canvas stays interactive. @@ -102,7 +175,9 @@ Default colors keep today's two-by-two palette (region versus operator, has arra Heatmap mode overrides node color. Edges are labeled with records and batches, dashed when zero messages were sent, with the container type as an edge badge and tooltip. -Perf guards: collapse-by-default keeps the initial node count small, viewport culling handles large expanded graphs, and if the visible node count would exceed roughly 1500 the UI warns and refuses expand-all. +Perf guards: collapse-by-default keeps the initial node count small, and viewport culling handles large expanded graphs. +A constant `MAX_VISIBLE_NODES = 1500` bounds the visible graph, counting leaf and region nodes but not edges. +An expand that would exceed it is refused with a warning toast. ### Filters and interactions @@ -111,6 +186,8 @@ A toolbar above the canvas offers: * Name search: matching operators highlighted, others dimmed, next/previous jump that auto-expands ancestor regions of a match. * Hide idle: hides zero-message edges and zero-elapsed operators, with region aggregates recomputed over the visible set. * Heatmap: mode select (off, elapsed, arrangement size), sequential color scale, threshold slider that dims nodes below the cutoff, with a legend. + The scale domain is `[0, max]` over the visible nodes' transitive metric. + If the max is zero, all nodes get the neutral base color and the slider is disabled. * Channel type: multi-select over the container types present, non-matching edges dimmed. Filters are pure derivations over the visible graph and compose. @@ -120,6 +197,7 @@ Double-clicking a region toggles collapse. ### Error handling +The first fetch for a selection shows a centered spinner, as today. An error always replaces the graph. The CNS-109 regression test from the issue lands, adapted to the new component. `INSUFFICIENT_PRIVILEGE` keeps the existing USAGE-privilege alert. @@ -130,7 +208,18 @@ A replica or dataflow that disappeared on refresh shows a "dataflow no longer ex The riskiest assumption is elkjs layout quality and speed on real dataflow shapes. Phase 1 of the implementation plan is a thin vertical slice: fetch one dataflow, build the region tree, lay out the default collapsed view with elkjs, and render it in React Flow with expand/collapse only. -That slice runs against a local environmentd with a non-trivial dataflow (for example a multi-way join with reductions) and validates layout, before filters and the selector page are built. + +The slice runs against a local environmentd with a reference dataflow: an index on a five-way join with at least two aggregations, producing at least 300 operators. +The exact SQL is fixed in the implementation plan. +The gate, all parts required to proceed: + +* Worker loading works in `vite dev` and the production build (see rendering section for the pinned mechanism and fallback). +* Default collapsed view lays out in under 500 ms. +* A fully expanded view of 1500 visible nodes lays out in under 5 s in the worker, UI responsive throughout. +* No overlapping nodes, edges follow left-to-right flow. +* Manual sign-off on visual readability by the requester, from screenshots of the reference dataflow collapsed and expanded. + +If the gate fails, work stops and the decision returns to this document: tune ELK options, or fall back to the in-house Canvas alternative, decided with the reviewer rather than unilaterally. ## Alternatives From 50f59132b43992db94eedb28cb597cd02d0629a0 Mon Sep 17 00:00:00 2001 From: Moritz Hoffmann Date: Wed, 8 Jul 2026 22:42:06 +0200 Subject: [PATCH 03/45] console: implementation plan for dataflow visualizer rebuild Co-Authored-By: Claude Sonnet 5 --- .../2026-07-06-dataflow-visualizer-rebuild.md | 2758 +++++++++++++++++ 1 file changed, 2758 insertions(+) create mode 100644 docs/superpowers/plans/2026-07-06-dataflow-visualizer-rebuild.md diff --git a/docs/superpowers/plans/2026-07-06-dataflow-visualizer-rebuild.md b/docs/superpowers/plans/2026-07-06-dataflow-visualizer-rebuild.md new file mode 100644 index 0000000000000..795d8a57a40ef --- /dev/null +++ b/docs/superpowers/plans/2026-07-06-dataflow-visualizer-rebuild.md @@ -0,0 +1,2758 @@ +# Dataflow visualizer rebuild implementation plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Replace the console's graphviz-WASM dataflow visualizer with a React Flow + elkjs graph view with dataflow selection, in-place collapsible regions, filters, and stats, closing CNS-108 and CNS-109. + +**Architecture:** Pure graph-model functions (`dataflowGraph.ts`) turn introspection SQL rows into a region tree, then derive a visible graph from collapse state. elkjs lays out the visible graph in a web worker. React Flow renders Chakra-styled nodes. Data hooks tag results with the params that produced them so stale data never renders. + +**Tech Stack:** React 18, TypeScript, Chakra UI v2, kysely (SQL text), vitest + msw, `@xyflow/react` (new), `elkjs` (new), Vite. + +**Spec:** `console/doc/design/20260706_dataflow_visualizer_rebuild.md` (approved at 1aa33ec74f). Two review items folded in: `Channel` is defined in Task 2, and `DataflowNode.lir` is an array (one entry per export span covering the operator, no disjointness assumed). + +## Global constraints + +* Repo: worktree `/home/moritz/dev/repos/materialize/.claude/worktrees/dataflow-viz-rebuild`, branch `moritz/dataflow-visualizer-rebuild`. All paths below relative to repo root. +* All commands run from `console/` unless stated. Test: `yarn test --run `. Typecheck: `yarn typecheck`. Lint: `yarn lint`. +* Every new file starts with the BSL copyright header (copy verbatim from any existing `console/src` file). +* Commit messages: `console: `. Commit after every task. +* No polling. No `d3-graphviz` or WASM. No new deps beyond `@xyflow/react` and `elkjs`. +* Catalog strings (operator names, channel types, LIR operators) must only ever render as React text nodes, never concatenated into any markup or DSL string. +* No em-dashes or structuring semicolons in comments. Comments state contracts, not narration. +* `MAX_VISIBLE_NODES = 1500` (counts nodes, not edges). +* Everything stays behind the `visualization-features` LaunchDarkly flag. +* SQL identifiers interpolated into kysely `sql` templates must go through `escapedLiteral` (`lit`); numeric ids are validated with `/^\d+$/` and cast (`::uint8`) after `lit`. + +## File structure + +``` +console/src/platform/dataflows/ + dataflowGraph.ts types + buildDataflowStructure + deriveVisibleGraph + decorateGraph (pure) + dataflowGraph.test.ts + elkGraph.ts VisibleGraph -> ELK JSON, ELK result -> positions (pure) + elkGraph.test.ts + layout.worker.ts module worker: elk.bundled.js + LayoutRequest/LayoutResponse + useElkLayout.ts hook owning the worker, request ids, position cache + nodes.tsx OperatorNode, RegionNode, CollapsedRegionNode, PortNode + dimensions + ChannelEdge.tsx custom edge with label + channel-type tooltip + DataflowGraphView.tsx React Flow canvas wiring + DataflowsPage.tsx cluster-level list page + export deep-link resolution + DataflowDetailPage.tsx graph page: selectors, toolbar, panels, error states + DataflowToolbar.tsx search / hide-idle / heatmap / channel-type controls + NodeDetailPanel.tsx click-a-node side panel + LirPanel.tsx LIR operator list + hover highlight + DataflowDetailPage.test.tsx +console/src/api/materialize/dataflow/ + useDataflowGraphData.ts structure queries keyed by dataflow id, params-tagged results + useDataflowList.ts selector list query + useDataflowIdForExport.ts export id -> dataflow id +console/src/test/mockReactFlow.tsx shared @xyflow/react mock for jsdom tests +``` + +Modified: `console/package.json`, `console/src/platform/clusters/ClusterDetail.tsx`, `console/src/platform/object-explorer/SimpleObjectDetailRoutes.tsx`, spec doc. +Deleted at the end: `console/src/platform/clusters/DataflowVisualizer.tsx`, `console/src/api/materialize/useDataflowStructure.ts`, `d3-graphviz` dep. + +--- + +## Phase 1: gated vertical slice + +### Task 1: Dependencies and spec amendment + +**Files:** +- Modify: `console/package.json` +- Modify: `console/doc/design/20260706_dataflow_visualizer_rebuild.md` + +**Interfaces:** +- Produces: `@xyflow/react` and `elkjs` installable, importable. + +- [ ] **Step 1: Add deps** + +Run: `cd console && yarn add @xyflow/react elkjs` +Expected: `package.json` gains both under `dependencies`, `yarn.lock` updated. No other dep changes (`git diff yarn.lock | grep '^+version' | wc -l` small). + +- [ ] **Step 2: Amend spec worker mechanism** + +The spec pinned `elkjs/lib/elk-worker.min.js?worker`. We use the strictly more Vite-native mechanism instead: our own module worker (`layout.worker.ts`) that imports `elkjs/lib/elk.bundled.js` and speaks the spec's `LayoutRequest`/`LayoutResponse` protocol directly. Same code off the main thread, first-class Vite dev + build support, and the protocol wrapper is needed anyway for stale-response dropping. In the spec's rendering section, replace the two sentences pinning the mechanism with: + +``` +The console builds with Vite, so layout runs in a Vite module worker (`new Worker(new URL("./layout.worker.ts", import.meta.url), { type: "module" })`) that imports `elkjs/lib/elk.bundled.js` and implements the layout request protocol below. +Phase 1 verifies this in both `vite dev` and the production build. +``` + +Also replace the spec's fallback sentence (it references the prebuilt worker script that no longer exists under this mechanism) with: + +``` +If the module worker fails to bundle or run `elk.bundled.js`, the fallback is running elkjs on the main thread in a lazily loaded chunk, accepting UI stalls during layout. +``` + +- [ ] **Step 3: Typecheck + commit** + +Run: `yarn typecheck` +Expected: PASS (no source changes yet). + +```bash +git add console/package.json console/yarn.lock console/doc/design/20260706_dataflow_visualizer_rebuild.md +git commit -m "console: add @xyflow/react and elkjs for dataflow visualizer rebuild" +``` + +### Task 2: Graph model types and buildDataflowStructure + +**Files:** +- Create: `console/src/platform/dataflows/dataflowGraph.ts` +- Test: `console/src/platform/dataflows/dataflowGraph.test.ts` + +**Interfaces:** +- Produces (exact, later tasks depend on these): + +```typescript +export type Address = number[]; +export type NodeId = string; +export function nodeIdOf(address: Address): NodeId; // JSON.stringify(address) + +export interface NodeStats { + arrangementRecords: bigint; + arrangementSize: bigint; + elapsedNs: bigint; +} + +export interface LirInfo { exportId: string; lirId: string; operator: string; } + +export interface DataflowNode { + id: NodeId; + address: Address; + name: string; + parent: NodeId | null; + children: NodeId[]; + own: NodeStats; + transitive: NodeStats; // own + subtree, precomputed + lir: LirInfo[]; // one entry per export span covering this operator id +} + +export interface Channel { + id: number; + fromAddress: Address; + fromPort: number; + toAddress: Address; + toPort: number; + messagesSent: bigint; + batchesSent: bigint; + channelType: string | null; +} + +export interface DataflowStructure { + nodes: Map; + root: NodeId; + channels: Channel[]; +} + +// SQL row shapes (values arrive as strings or numbers, normalized here) +export interface OperatorRow { + id: bigint | number | string; + address: string[]; + name: string; + arrangementRecords: bigint | number | string | null; + arrangementSize: bigint | number | string | null; + elapsedNs: bigint | number | string | null; +} +export interface ChannelRow { + id: number | string; + fromOperatorAddress: string[]; + fromPort: number | string; + toOperatorAddress: string[]; + toPort: number | string; + messagesSent: bigint | number | string; + batchesSent: bigint | number | string; + channelType: string | null; +} +export interface LirSpanRow { + exportId: string; + lirId: string; + operator: string; + operatorIdStart: bigint | number | string; + operatorIdEnd: bigint | number | string; +} + +export function buildDataflowStructure( + operators: OperatorRow[], + channels: ChannelRow[], + lirSpans: LirSpanRow[], +): DataflowStructure; // throws if not exactly one root (address length 1) +``` + +- [ ] **Step 1: Write the failing tests** + +`dataflowGraph.test.ts` (header + imports elided in later snippets, always include them): + +```typescript +import { describe, expect, it } from "vitest"; + +import { + buildDataflowStructure, + nodeIdOf, + type ChannelRow, + type LirSpanRow, + type OperatorRow, +} from "./dataflowGraph"; + +// Dataflow 5: root [5], region [5,1] with children [5,1,1], [5,1,2], leaf [5,2]. +export const OPS: OperatorRow[] = [ + { id: "10", address: ["5"], name: "Dataflow", arrangementRecords: null, arrangementSize: null, elapsedNs: "0" }, + { id: "11", address: ["5", "1"], name: "Region", arrangementRecords: "0", arrangementSize: "0", elapsedNs: "5" }, + { id: "12", address: ["5", "1", "1"], name: "Join", arrangementRecords: "100", arrangementSize: "4096", elapsedNs: "7" }, + { id: "13", address: ["5", "1", "2"], name: "Map", arrangementRecords: "0", arrangementSize: "0", elapsedNs: "1" }, + { id: "14", address: ["5", "2"], name: "Sink", arrangementRecords: "0", arrangementSize: "0", elapsedNs: "2" }, +]; +export const CHANNELS: ChannelRow[] = [ + // into the region: leaf [5,2] <- region output; region input port -> child + { id: "1", fromOperatorAddress: ["5", "1", "0"], fromPort: "0", toOperatorAddress: ["5", "1", "1"], toPort: "0", messagesSent: "3", batchesSent: "1", channelType: "rows" }, + { id: "2", fromOperatorAddress: ["5", "1", "1"], fromPort: "0", toOperatorAddress: ["5", "1", "2"], toPort: "0", messagesSent: "5", batchesSent: "2", channelType: "rows" }, + { id: "3", fromOperatorAddress: ["5", "1"], fromPort: "0", toOperatorAddress: ["5", "2"], toPort: "0", messagesSent: "5", batchesSent: "2", channelType: "batches" }, +]; +export const LIR_SPANS: LirSpanRow[] = [ + { exportId: "u42", lirId: "1", operator: "Join::Differential", operatorIdStart: "11", operatorIdEnd: "13" }, +]; + +describe("buildDataflowStructure", () => { + it("builds the region tree from address prefixes", () => { + const s = buildDataflowStructure(OPS, CHANNELS, LIR_SPANS); + expect(s.root).toEqual(nodeIdOf([5])); + const root = s.nodes.get(s.root)!; + expect(root.children).toEqual([nodeIdOf([5, 1]), nodeIdOf([5, 2])]); + const region = s.nodes.get(nodeIdOf([5, 1]))!; + expect(region.parent).toEqual(s.root); + expect(region.children).toEqual([nodeIdOf([5, 1, 1]), nodeIdOf([5, 1, 2])]); + }); + + it("precomputes transitive stats", () => { + const s = buildDataflowStructure(OPS, CHANNELS, LIR_SPANS); + const region = s.nodes.get(nodeIdOf([5, 1]))!; + expect(region.own.elapsedNs).toEqual(5n); + expect(region.transitive.elapsedNs).toEqual(13n); + expect(region.transitive.arrangementRecords).toEqual(100n); + expect(region.transitive.arrangementSize).toEqual(4096n); + }); + + it("maps LIR spans to operators by id range, as an array", () => { + const s = buildDataflowStructure(OPS, CHANNELS, LIR_SPANS); + expect(s.nodes.get(nodeIdOf([5, 1, 1]))!.lir).toEqual([ + { exportId: "u42", lirId: "1", operator: "Join::Differential" }, + ]); + // end is exclusive + expect(s.nodes.get(nodeIdOf([5, 2]))!.lir).toEqual([]); + }); + + it("throws without exactly one root", () => { + expect(() => buildDataflowStructure([], [], [])).toThrow(); + }); + + it("normalizes channels", () => { + const s = buildDataflowStructure(OPS, CHANNELS, []); + expect(s.channels[0]).toEqual({ + id: 1, fromAddress: [5, 1, 0], fromPort: 0, toAddress: [5, 1, 1], toPort: 0, + messagesSent: 3n, batchesSent: 1n, channelType: "rows", + }); + }); +}); +``` + +- [ ] **Step 2: Run tests, verify failure** + +Run: `yarn test --run src/platform/dataflows/dataflowGraph.test.ts` +Expected: FAIL, cannot resolve `./dataflowGraph`. + +- [ ] **Step 3: Implement** + +```typescript +export function nodeIdOf(address: Address): NodeId { + return JSON.stringify(address); +} + +const toBigInt = (v: bigint | number | string | null | undefined): bigint => + v == null ? 0n : BigInt(v); + +export function buildDataflowStructure( + operators: OperatorRow[], + channels: ChannelRow[], + lirSpans: LirSpanRow[], +): DataflowStructure { + const spans = lirSpans.map((s) => ({ + exportId: s.exportId, + lirId: s.lirId, + operator: s.operator, + start: toBigInt(s.operatorIdStart), + end: toBigInt(s.operatorIdEnd), + })); + const nodes = new Map(); + for (const row of operators) { + const address = row.address.map(Number); + const id = nodeIdOf(address); + const opId = toBigInt(row.id); + const own: NodeStats = { + arrangementRecords: toBigInt(row.arrangementRecords), + arrangementSize: toBigInt(row.arrangementSize), + elapsedNs: toBigInt(row.elapsedNs), + }; + nodes.set(id, { + id, + address, + name: row.name, + parent: address.length > 1 ? nodeIdOf(address.slice(0, -1)) : null, + children: [], + own, + transitive: own, // replaced below + lir: spans + .filter((s) => s.start <= opId && opId < s.end) + .map(({ exportId, lirId, operator }) => ({ exportId, lirId, operator })), + }); + } + const roots: NodeId[] = []; + for (const node of nodes.values()) { + if (node.parent === null) roots.push(node.id); + else nodes.get(node.parent)?.children.push(node.id); + } + if (roots.length !== 1) { + throw new Error(`expected exactly one root operator, found ${roots.length}`); + } + // Children in address order so layout and tests are deterministic. + for (const node of nodes.values()) { + node.children.sort((a, b) => { + const [x, y] = [nodes.get(a)!.address, nodes.get(b)!.address]; + return x[x.length - 1] - y[y.length - 1]; + }); + } + const fillTransitive = (id: NodeId): NodeStats => { + const node = nodes.get(id)!; + const t = { ...node.own }; + for (const c of node.children) { + const ct = fillTransitive(c); + t.arrangementRecords += ct.arrangementRecords; + t.arrangementSize += ct.arrangementSize; + t.elapsedNs += ct.elapsedNs; + } + node.transitive = t; + return t; + }; + fillTransitive(roots[0]); + return { + nodes, + root: roots[0], + channels: channels.map((c) => ({ + id: Number(c.id), + fromAddress: c.fromOperatorAddress.map(Number), + fromPort: Number(c.fromPort), + toAddress: c.toOperatorAddress.map(Number), + toPort: Number(c.toPort), + messagesSent: toBigInt(c.messagesSent), + batchesSent: toBigInt(c.batchesSent), + channelType: c.channelType, + })), + }; +} +``` + +- [ ] **Step 4: Run tests, verify pass** + +Run: `yarn test --run src/platform/dataflows/dataflowGraph.test.ts` +Expected: PASS (5 tests). + +- [ ] **Step 5: Commit** + +```bash +git add console/src/platform/dataflows/ +git commit -m "console: add dataflow graph model and structure builder" +``` + +### Task 3: deriveVisibleGraph (collapse, ports, edge aggregation) + +**Files:** +- Modify: `console/src/platform/dataflows/dataflowGraph.ts` +- Test: `console/src/platform/dataflows/dataflowGraph.test.ts` + +**Interfaces:** +- Produces: + +```typescript +export type CollapseState = ReadonlySet; // collapsed region ids + +export interface VisibleNode { + id: string; // NodeId, or `${scopeId}:in:${port}` / `${scopeId}:out:${port}` for ports + kind: "operator" | "region" | "collapsedRegion" | "port"; + label: string; + parent: string | null; // enclosing visible region, null when direct child of root + stats: NodeStats | null; // own for operator/region, transitive for collapsedRegion, null for port + transitive: NodeStats | null; // null for port + childCount: number; + lir: LirInfo[]; + address: Address | null; // null for ports +} + +export interface VisibleEdge { + id: string; // `${source}=>${target}` + source: string; + target: string; + messagesSent: bigint; + batchesSent: bigint; + channelTypes: string[]; // sorted set union of aggregated channels +} + +export interface VisibleGraph { nodes: VisibleNode[]; edges: VisibleEdge[]; } + +// Nodes are emitted parents-before-children (React Flow requirement). +// The root is the canvas itself, never a node. Collapsed descendants of a +// collapsed region are absorbed into it. Edges between endpoints that map to +// the same visible node are dropped. +export function deriveVisibleGraph( + structure: DataflowStructure, + collapsed: CollapseState, +): VisibleGraph; + +export function defaultCollapseState(structure: DataflowStructure): CollapseState; +// every region except the root collapsed + +export function visibleNodeCount(structure: DataflowStructure, collapsed: CollapseState): number; +export const MAX_VISIBLE_NODES = 1500; +``` + +- [ ] **Step 1: Write the failing tests** + +Append to `dataflowGraph.test.ts`: + +```typescript +import { + defaultCollapseState, + deriveVisibleGraph, + MAX_VISIBLE_NODES, + visibleNodeCount, +} from "./dataflowGraph"; + +describe("deriveVisibleGraph", () => { + const s = buildDataflowStructure(OPS, CHANNELS, LIR_SPANS); + const regionId = nodeIdOf([5, 1]); + + it("collapses regions to a single node with remapped edges", () => { + const g = deriveVisibleGraph(s, new Set([regionId])); + expect(g.nodes.map((n) => [n.id, n.kind])).toEqual([ + [regionId, "collapsedRegion"], + [nodeIdOf([5, 2]), "operator"], + ]); + // channel 3 region -> sink survives, channels 1 and 2 are internal + expect(g.edges).toEqual([ + { + id: `${regionId}=>${nodeIdOf([5, 2])}`, + source: regionId, + target: nodeIdOf([5, 2]), + messagesSent: 5n, + batchesSent: 2n, + channelTypes: ["batches"], + }, + ]); + const collapsed = g.nodes[0]; + expect(collapsed.stats).toEqual(s.nodes.get(regionId)!.transitive); + expect(collapsed.childCount).toEqual(2); + }); + + it("expands regions with port pseudo-nodes, parents before children", () => { + const g = deriveVisibleGraph(s, new Set()); + const ids = g.nodes.map((n) => n.id); + expect(ids.indexOf(regionId)).toBeLessThan(ids.indexOf(nodeIdOf([5, 1, 1]))); + const port = g.nodes.find((n) => n.kind === "port")!; + expect(port.id).toEqual(`${regionId}:in:0`); + expect(port.parent).toEqual(regionId); + expect(port.label).toEqual("input 0"); + // port -> Join edge preserved + expect(g.edges.map((e) => e.id)).toContain(`${regionId}:in:0=>${nodeIdOf([5, 1, 1])}`); + }); + + it("aggregates parallel channels between the same visible pair", () => { + const extra = buildDataflowStructure(OPS, [ + ...CHANNELS, + { id: "4", fromOperatorAddress: ["5", "1"], fromPort: "1", toOperatorAddress: ["5", "2"], toPort: "1", messagesSent: "7", batchesSent: "1", channelType: "rows" }, + ], []); + const g = deriveVisibleGraph(extra, new Set([regionId])); + const e = g.edges.find((e) => e.target === nodeIdOf([5, 2]))!; + expect(e.messagesSent).toEqual(12n); + expect(e.channelTypes).toEqual(["batches", "rows"]); + }); + + it("defaultCollapseState collapses all non-root regions", () => { + expect(defaultCollapseState(s)).toEqual(new Set([regionId])); + expect(visibleNodeCount(s, defaultCollapseState(s))).toEqual(2); + expect(MAX_VISIBLE_NODES).toEqual(1500); + }); +}); +``` + +- [ ] **Step 2: Run tests, verify failure** + +Run: `yarn test --run src/platform/dataflows/dataflowGraph.test.ts` +Expected: FAIL, `deriveVisibleGraph` not exported. + +- [ ] **Step 3: Implement** + +```typescript +export const MAX_VISIBLE_NODES = 1500; + +// Shallowest collapsed ancestor absorbs everything beneath it. +function representative(address: Address, collapsed: CollapseState): NodeId { + for (let len = 1; len < address.length; len++) { + const prefix = nodeIdOf(address.slice(0, len)); + if (collapsed.has(prefix)) return prefix; + } + return nodeIdOf(address); +} + +// A channel endpoint address ending in 0 denotes the enclosing scope's +// input (from side) or output (to side) port. +function endpointId( + address: Address, + port: number, + side: "from" | "to", + collapsed: CollapseState, +): { id: string; port?: { scope: NodeId; direction: "input" | "output"; port: number } } { + if (address[address.length - 1] === 0) { + const scope = address.slice(0, -1); + const scopeId = nodeIdOf(scope); + const rep = scope.length === 0 ? scopeId : representative(scope, collapsed); + if (rep !== scopeId || collapsed.has(scopeId)) return { id: rep }; + const direction = side === "from" ? "input" : "output"; + return { + id: `${scopeId}:${direction === "input" ? "in" : "out"}:${port}`, + port: { scope: scopeId, direction, port }, + }; + } + return { id: representative(address, collapsed) }; +} + +export function deriveVisibleGraph( + structure: DataflowStructure, + collapsed: CollapseState, +): VisibleGraph { + const nodes: VisibleNode[] = []; + const emit = (id: NodeId, parent: string | null) => { + const node = structure.nodes.get(id)!; + const isCollapsed = collapsed.has(id) && node.children.length > 0; + const kind = + node.children.length === 0 ? "operator" : isCollapsed ? "collapsedRegion" : "region"; + nodes.push({ + id, + kind, + label: node.name, + parent, + stats: kind === "collapsedRegion" ? node.transitive : node.own, + transitive: node.transitive, + childCount: node.children.length, + lir: node.lir, + address: node.address, + }); + if (!isCollapsed) for (const c of node.children) emit(c, id); + }; + for (const c of structure.nodes.get(structure.root)!.children) emit(c, null); + + const edgesById = new Map }>(); + const ports = new Map(); + for (const ch of structure.channels) { + const from = endpointId(ch.fromAddress, ch.fromPort, "from", collapsed); + const to = endpointId(ch.toAddress, ch.toPort, "to", collapsed); + if (from.id === to.id) continue; + for (const p of [from.port, to.port]) { + if (p && !ports.has(`${p.scope}:${p.direction === "input" ? "in" : "out"}:${p.port}`)) { + const id = `${p.scope}:${p.direction === "input" ? "in" : "out"}:${p.port}`; + ports.set(id, { + id, + kind: "port", + label: `${p.direction} ${p.port}`, + parent: p.scope === structure.root ? null : p.scope, + stats: null, + transitive: null, + childCount: 0, + lir: [], + address: null, + }); + } + } + const id = `${from.id}=>${to.id}`; + const existing = edgesById.get(id); + if (existing) { + existing.messagesSent += ch.messagesSent; + existing.batchesSent += ch.batchesSent; + if (ch.channelType) existing.typeSet.add(ch.channelType); + } else { + edgesById.set(id, { + id, + source: from.id, + target: to.id, + messagesSent: ch.messagesSent, + batchesSent: ch.batchesSent, + channelTypes: [], + typeSet: new Set(ch.channelType ? [ch.channelType] : []), + }); + } + } + const edges = [...edgesById.values()].map(({ typeSet, ...e }) => ({ + ...e, + channelTypes: [...typeSet].sort(), + })); + return { nodes: [...nodes, ...ports.values()], edges }; +} + +export function defaultCollapseState(structure: DataflowStructure): CollapseState { + const collapsed = new Set(); + for (const node of structure.nodes.values()) { + if (node.children.length > 0 && node.id !== structure.root) collapsed.add(node.id); + } + return collapsed; +} + +export function visibleNodeCount( + structure: DataflowStructure, + collapsed: CollapseState, +): number { + return deriveVisibleGraph(structure, collapsed).nodes.length; +} +``` + +Note: edge endpoints can reference a node id that is not visible when a channel endpoint lies inside an expanded region that the other endpoint's collapsed ancestor absorbed. `representative` guarantees both endpoints are visible ids by construction (any collapsed prefix wins). Port nodes are added to the node list after regions, and their `parent` region always precedes them because region nodes were emitted first. + +- [ ] **Step 4: Run tests, verify pass** + +Run: `yarn test --run src/platform/dataflows/dataflowGraph.test.ts` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add console/src/platform/dataflows/ +git commit -m "console: derive visible dataflow graph from collapse state" +``` + +### Task 4: ELK graph conversion and layout worker + +**Files:** +- Create: `console/src/platform/dataflows/elkGraph.ts` +- Create: `console/src/platform/dataflows/layout.worker.ts` +- Create: `console/src/platform/dataflows/useElkLayout.ts` +- Test: `console/src/platform/dataflows/elkGraph.test.ts` + +**Interfaces:** +- Consumes: `VisibleGraph`, `VisibleNode` from Task 3. +- Produces: + +```typescript +// elkGraph.ts +export interface NodePosition { x: number; y: number; width: number; height: number; } +export type Positions = Record; // key: visible node id, x/y relative to parent +export const NODE_DIMENSIONS: Record; +export function toElkGraph(graph: VisibleGraph): ElkNode; // nested by parent +export function extractPositions(layouted: ElkNode): Positions; + +// layout.worker.ts protocol +export interface LayoutRequest { requestId: number; graph: VisibleGraph; } +export interface LayoutResponse { requestId: number; positions?: Positions; error?: string; } + +// useElkLayout.ts +export function useElkLayout( + graph: VisibleGraph | null, + cacheKey: string, // caller-provided, e.g. structure key + sorted collapsed ids +): { positions: Positions | null; layouting: boolean; error: string | null }; +``` + +- [ ] **Step 1: Write the failing tests** + +`elkGraph.test.ts`: + +```typescript +import { describe, expect, it } from "vitest"; + +import { buildDataflowStructure, deriveVisibleGraph, nodeIdOf } from "./dataflowGraph"; +import { CHANNELS, LIR_SPANS, OPS } from "./dataflowGraph.test"; +import { extractPositions, NODE_DIMENSIONS, toElkGraph } from "./elkGraph"; + +describe("toElkGraph", () => { + const s = buildDataflowStructure(OPS, CHANNELS, LIR_SPANS); + + it("nests elk children by visible parent", () => { + const elk = toElkGraph(deriveVisibleGraph(s, new Set())); + const region = elk.children!.find((c) => c.id === nodeIdOf([5, 1]))!; + expect(region.children!.map((c) => c.id)).toEqual( + expect.arrayContaining([ + nodeIdOf([5, 1, 1]), + nodeIdOf([5, 1, 2]), + `${nodeIdOf([5, 1])}:in:0`, + ]), + ); + const leaf = region.children!.find((c) => c.id === nodeIdOf([5, 1, 1]))!; + expect(leaf.width).toEqual(NODE_DIMENSIONS.operator.width); + }); + + it("round-trips positions relative to parent", () => { + const elk = toElkGraph(deriveVisibleGraph(s, new Set())); + // simulate a layout result + elk.children![0].x = 10; + elk.children![0].y = 20; + const positions = extractPositions(elk); + expect(positions[elk.children![0].id]).toMatchObject({ x: 10, y: 20 }); + }); +}); +``` + +Export `OPS`, `CHANNELS`, `LIR_SPANS` from `dataflowGraph.test.ts` (they already are, per Task 2). + +- [ ] **Step 2: Run tests, verify failure** + +Run: `yarn test --run src/platform/dataflows/elkGraph.test.ts` +Expected: FAIL, module missing. + +- [ ] **Step 3: Implement elkGraph.ts** + +```typescript +import type { ElkNode } from "elkjs/lib/elk-api"; + +import type { VisibleGraph, VisibleNode } from "./dataflowGraph"; + +export const NODE_DIMENSIONS: Record = { + operator: { width: 240, height: 72 }, + collapsedRegion: { width: 260, height: 88 }, + region: { width: 0, height: 0 }, // sized by elk from children + port: { width: 90, height: 24 }, +}; + +const LAYOUT_OPTIONS: Record = { + "elk.algorithm": "layered", + "elk.direction": "RIGHT", + "elk.hierarchyHandling": "INCLUDE_CHILDREN", + "elk.padding": "[top=48,left=16,bottom=16,right=16]", + "elk.spacing.nodeNode": "24", + "elk.layered.spacing.nodeNodeBetweenLayers": "48", +}; + +export function toElkGraph(graph: VisibleGraph): ElkNode { + const elkNodes = new Map(); + const root: ElkNode = { id: "__root__", layoutOptions: LAYOUT_OPTIONS, children: [], edges: [] }; + for (const node of graph.nodes) { + const dims = NODE_DIMENSIONS[node.kind]; + const elkNode: ElkNode = { + id: node.id, + children: [], + ...(node.kind === "region" ? { layoutOptions: LAYOUT_OPTIONS } : dims), + }; + elkNodes.set(node.id, elkNode); + const parent = node.parent ? elkNodes.get(node.parent)! : root; + parent.children!.push(elkNode); + } + root.edges = graph.edges.map((e) => ({ + id: e.id, + sources: [e.source], + targets: [e.target], + })); + return root; +} + +export function extractPositions(layouted: ElkNode): Positions { + const positions: Positions = {}; + const walk = (node: ElkNode) => { + for (const child of node.children ?? []) { + positions[child.id] = { + x: child.x ?? 0, + y: child.y ?? 0, + width: child.width ?? 0, + height: child.height ?? 0, + }; + walk(child); + } + }; + walk(layouted); + return positions; +} +``` + +Note: `graph.nodes` is parents-before-children except ports, which come last but whose parent regions exist by then, so the single pass with `elkNodes.get(node.parent)` is safe. Hierarchical edges at the root level with `INCLUDE_CHILDREN` is the documented elk pattern for cross-hierarchy edges. + +- [ ] **Step 4: Run tests, verify pass** + +Run: `yarn test --run src/platform/dataflows/elkGraph.test.ts` +Expected: PASS. + +- [ ] **Step 5: Implement worker + hook (no unit test, covered by phase 1 gate and stubbed in component tests)** + +`layout.worker.ts`: + +```typescript +import ELK from "elkjs/lib/elk.bundled.js"; + +import type { VisibleGraph } from "./dataflowGraph"; +import { extractPositions, toElkGraph, type Positions } from "./elkGraph"; + +export interface LayoutRequest { requestId: number; graph: VisibleGraph; } +export interface LayoutResponse { requestId: number; positions?: Positions; error?: string; } + +const elk = new ELK(); + +self.onmessage = async (event: MessageEvent) => { + const { requestId, graph } = event.data; + try { + const layouted = await elk.layout(toElkGraph(graph)); + const response: LayoutResponse = { requestId, positions: extractPositions(layouted) }; + self.postMessage(response); + } catch (error) { + self.postMessage({ requestId, error: String(error) } satisfies LayoutResponse); + } +}; +``` + +`useElkLayout.ts`: + +```typescript +import React from "react"; + +import type { VisibleGraph } from "./dataflowGraph"; +import type { LayoutRequest, LayoutResponse } from "./layout.worker"; +import type { Positions } from "./elkGraph"; + +export function useElkLayout(graph: VisibleGraph | null, cacheKey: string) { + const workerRef = React.useRef(null); + const requestIdRef = React.useRef(0); + const cacheRef = React.useRef(new Map()); + const [state, setState] = React.useState<{ + key: string | null; + positions: Positions | null; + error: string | null; + }>({ key: null, positions: null, error: null }); + + React.useEffect(() => { + const worker = new Worker(new URL("./layout.worker.ts", import.meta.url), { + type: "module", + }); + workerRef.current = worker; + return () => worker.terminate(); + }, []); + + React.useEffect(() => { + if (!graph) return; + const cached = cacheRef.current.get(cacheKey); + if (cached) { + setState({ key: cacheKey, positions: cached, error: null }); + return; + } + const requestId = ++requestIdRef.current; + const worker = workerRef.current; + if (!worker) return; + const onMessage = (event: MessageEvent) => { + // Drop responses for superseded requests. + if (event.data.requestId !== requestIdRef.current) return; + if (event.data.positions) { + cacheRef.current.set(cacheKey, event.data.positions); + setState({ key: cacheKey, positions: event.data.positions, error: null }); + } else { + setState({ key: cacheKey, positions: null, error: event.data.error ?? "layout failed" }); + } + }; + worker.addEventListener("message", onMessage); + worker.postMessage({ requestId, graph } satisfies LayoutRequest); + return () => worker.removeEventListener("message", onMessage); + }, [graph, cacheKey]); + + return { + positions: state.key === cacheKey ? state.positions : null, + layouting: graph !== null && state.key !== cacheKey, + error: state.key === cacheKey ? state.error : null, + }; +} +``` + +- [ ] **Step 6: Typecheck + commit** + +Run: `yarn typecheck` +Expected: PASS. If `elkjs/lib/elk.bundled.js` lacks types, add `console/src/types/elkjs-bundled.d.ts` with `declare module "elkjs/lib/elk.bundled.js" { export { default } from "elkjs/lib/elk-api"; }` and include it in the commit. + +```bash +git add console/src/platform/dataflows/ console/src/types/ 2>/dev/null || git add console/src/platform/dataflows/ +git commit -m "console: elk layout worker and hook for dataflow graphs" +``` + +### Task 5: Data hook keyed by dataflow id with params tagging + +**Files:** +- Create: `console/src/api/materialize/dataflow/useDataflowGraphData.ts` +- Test: `console/src/api/materialize/dataflow/useDataflowGraphData.test.ts` + +**Interfaces:** +- Consumes: `buildDataflowStructure`, `DataflowStructure` (Task 2), `useSqlManyTyped`, `escapedLiteral as lit`, `queryBuilder` (existing, see `console/src/api/materialize/useDataflowStructure.ts` for import shapes). +- Produces: + +```typescript +export interface DataflowGraphParams { + clusterName: string; + replicaName: string; + dataflowId: string; // numeric string, mz_dataflows.id on the selected replica +} +export function useDataflowGraphData(params?: DataflowGraphParams): { + data: { structure: DataflowStructure; fetchedAt: Date } | null; + error: unknown; + databaseError: { code?: string } | null; // same shape useDataflowStructure exposes today + loading: boolean; + refetch: () => void; +}; +``` + +Contract: `data` is non-null only when the last successful result was produced by exactly the current `params` and there is no error. Error always wins (CNS-109). A same-params `refetch` keeps `data` while `loading` is true so the graph refreshes in place. + +- [ ] **Step 1: Write the failing test** + +Use msw like the CNS-109 issue test. `useDataflowGraphData.test.ts`, using `renderHook` from `@testing-library/react` and the msw `server` from `~/api/mocks/server`: + +```typescript +import { renderHook, waitFor } from "@testing-library/react"; +import { http, HttpResponse } from "msw"; + +import { ErrorCode } from "~/api/materialize/types"; +import server from "~/api/mocks/server"; + +import { useDataflowGraphData } from "./useDataflowGraphData"; + +const FAILING_REPLICA = "r2"; +const okResult = { + desc: { columns: [] }, + rows: [], +}; +// operators result with one root row so buildDataflowStructure succeeds +const operatorsResult = { + desc: { + columns: [ + { name: "id" }, { name: "address" }, { name: "name" }, + { name: "arrangementRecords" }, { name: "arrangementSize" }, { name: "elapsedNs" }, + ], + }, + rows: [["10", ["7"], "Dataflow", "0", "0", "0"]], +}; + +beforeEach(() => { + server.use( + http.post("*/api/sql", async ({ request }) => { + const options = JSON.parse( + new URL(request.url).searchParams.get("options") ?? "{}", + ); + if (options.cluster_replica === FAILING_REPLICA) { + return HttpResponse.json({ + results: [ + { error: { message: "no such replica", code: ErrorCode.INTERNAL_ERROR } }, + ], + }); + } + return HttpResponse.json({ results: [operatorsResult, okResult, okResult] }); + }), + ); +}); + +describe("useDataflowGraphData", () => { + it("clears data when a refetch for new params fails", async () => { + const { result, rerender } = renderHook( + (params) => useDataflowGraphData(params), + { + initialProps: { clusterName: "c", replicaName: "r1", dataflowId: "7" } as + | { clusterName: string; replicaName: string; dataflowId: string } + | undefined, + }, + ); + await waitFor(() => expect(result.current.data).not.toBeNull()); + + rerender({ clusterName: "c", replicaName: FAILING_REPLICA, dataflowId: "7" }); + await waitFor(() => expect(result.current.error).toBeTruthy()); + // The retained previous result must not surface for the new params. + expect(result.current.data).toBeNull(); + }); + + it("rejects a non-numeric dataflow id", () => { + expect(() => + renderHook(() => + useDataflowGraphData({ clusterName: "c", replicaName: "r1", dataflowId: "7; DROP" }), + ), + ).toThrow(); + }); +}); +``` + +`mapRowToObject` (`console/src/api/materialize/index.ts`) maps column aliases to object keys verbatim, so these mocked shapes match the hook's row types as written. + +- [ ] **Step 2: Run test, verify failure** + +Run: `yarn test --run src/api/materialize/dataflow/useDataflowGraphData.test.ts` +Expected: FAIL, module missing. + +- [ ] **Step 3: Implement** + +```typescript +import { sql } from "kysely"; +import React from "react"; + +import { escapedLiteral as lit, useSqlManyTyped } from "~/api/materialize"; +import { + buildDataflowStructure, + type ChannelRow, + type DataflowStructure, + type LirSpanRow, + type OperatorRow, +} from "~/platform/dataflows/dataflowGraph"; + +import { queryBuilder } from "../db"; + +export interface DataflowGraphParams { + clusterName: string; + replicaName: string; + dataflowId: string; +} + +function dataflowIdLiteral(dataflowId: string) { + if (!/^\d+$/.test(dataflowId)) { + throw new Error(`invalid dataflow id: ${dataflowId}`); + } + return sql`${lit(dataflowId)}::uint8`; +} + +export function useDataflowGraphData(params?: DataflowGraphParams) { + const queries = React.useMemo(() => { + if (!params) return null; + const id = dataflowIdLiteral(params.dataflowId); + return { + operators: sql` + SELECT + mdod.id, + mda.address, + mdod.name, + coalesce(mas.records, 0) AS "arrangementRecords", + coalesce(mas.size, 0) AS "arrangementSize", + coalesce(mse.elapsed_ns, 0) AS "elapsedNs" + FROM mz_dataflow_operator_dataflows AS mdod + JOIN mz_dataflow_addresses AS mda ON mda.id = mdod.id + LEFT JOIN mz_arrangement_sizes AS mas ON mas.operator_id = mdod.id + LEFT JOIN mz_scheduling_elapsed AS mse ON mse.id = mdod.id + WHERE mdod.dataflow_id = ${id}` + .$castTo() + .compile(queryBuilder), + channels: sql` + SELECT + mdco.id, + from_operator_address AS "fromOperatorAddress", + from_port AS "fromPort", + to_operator_address AS "toOperatorAddress", + to_port AS "toPort", + COALESCE(sum(mmc.sent), 0) AS "messagesSent", + COALESCE(sum(mmc.batch_sent), 0) AS "batchesSent", + mdco.type AS "channelType" + FROM mz_dataflow_channel_operators AS mdco + LEFT JOIN mz_message_counts AS mmc ON mdco.id = mmc.channel_id + WHERE from_operator_address[1] = ${id} + GROUP BY + mdco.id, "fromOperatorAddress", "fromPort", + "toOperatorAddress", "toPort", mdco.type` + .$castTo() + .compile(queryBuilder), + lirSpans: sql` + SELECT + mce.export_id AS "exportId", + mlm.lir_id::text AS "lirId", + mlm.operator, + mlm.operator_id_start AS "operatorIdStart", + mlm.operator_id_end AS "operatorIdEnd" + FROM mz_lir_mapping AS mlm + JOIN mz_compute_exports AS mce ON mlm.global_id = mce.export_id + WHERE mce.dataflow_id = ${id}` + .$castTo() + .compile(queryBuilder), + }; + }, [params]); + + const { results, error, databaseError, loading, refetch } = useSqlManyTyped(queries, { + cluster: params?.clusterName, + replica: params?.replicaName, + // This query can be slow for large dataflows. + timeout: 30_000, + }); + + const key = params + ? `${params.clusterName}|${params.replicaName}|${params.dataflowId}` + : null; + const [lastGood, setLastGood] = React.useState<{ + key: string; + data: { structure: DataflowStructure; fetchedAt: Date }; + } | null>(null); + + // Tag successful results with the params key they were fetched for. Stale + // in-flight responses for older params are aborted by useSqlMany, so a + // result observed here belongs to the current key. + React.useEffect(() => { + if (key && results && !error && !loading) { + setLastGood({ + key, + data: { + structure: buildDataflowStructure( + results.operators ?? [], + results.channels ?? [], + results.lirSpans ?? [], + ), + fetchedAt: new Date(), + }, + }); + } + }, [key, results, error, loading]); + + // Error always wins, and data for other params never renders. + const data = !error && lastGood && lastGood.key === key ? lastGood.data : null; + return { data, error, databaseError, loading, refetch }; +} +``` + +- [ ] **Step 4: Run test, verify pass** + +Run: `yarn test --run src/api/materialize/dataflow/useDataflowGraphData.test.ts` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add console/src/api/materialize/dataflow/ +git commit -m "console: dataflow graph data hook keyed by dataflow id" +``` + +### Task 6: Nodes, edges, and DataflowGraphView + +**Files:** +- Create: `console/src/platform/dataflows/nodes.tsx` +- Create: `console/src/platform/dataflows/ChannelEdge.tsx` +- Create: `console/src/platform/dataflows/DataflowGraphView.tsx` +- Create: `console/src/test/mockReactFlow.tsx` +- Modify: `console/src/platform/dataflows/dataflowGraph.ts` (add `GraphDecorations`, Step 0) + +**Interfaces:** +- Consumes: `VisibleGraph`, `Positions`, `useElkLayout`, `NODE_DIMENSIONS`, `MAX_VISIBLE_NODES`, `visibleNodeCount`. +- Produces: + +```typescript +export interface DataflowGraphViewProps { + structure: DataflowStructure; + collapsed: CollapseState; + onCollapsedChange: (next: CollapseState) => void; + cacheKey: string; // structure identity, owner appends collapse signature internally + decorations?: GraphDecorations; // added in Task 13, optional from the start + onNodeClick?: (node: VisibleNode) => void; +} +// Declared in dataflowGraph.ts (single source; DataflowGraphView imports it). +// Task 13 adds dimmedEdgeIds and searchMatches and makes decorateGraph return +// all fields; until then the view treats every field as optional. +export interface GraphDecorations { + dimmedNodeIds?: ReadonlySet; + hiddenNodeIds?: ReadonlySet; + hiddenEdgeIds?: ReadonlySet; + dimmedEdgeIds?: ReadonlySet; + nodeColors?: ReadonlyMap; // heatmap override + searchMatches?: string[]; +} +``` + +- [ ] **Step 0: Add GraphDecorations to dataflowGraph.ts** + +Append to `console/src/platform/dataflows/dataflowGraph.ts` (all-optional here; Task 13's `decorateGraph` returns all fields set): + +```typescript +export interface GraphDecorations { + dimmedNodeIds?: ReadonlySet; + hiddenNodeIds?: ReadonlySet; + hiddenEdgeIds?: ReadonlySet; + dimmedEdgeIds?: ReadonlySet; + nodeColors?: ReadonlyMap; // heatmap override + searchMatches?: string[]; +} +``` + +- [ ] **Step 1: Implement nodes.tsx** + +Colors keep the current palette constants. All text via React, `formatBytesShort` from `~/utils/format`. + +```tsx +import { Badge, Box, Text, Tooltip } from "@chakra-ui/react"; +import { Handle, Position, type NodeProps } from "@xyflow/react"; +import React from "react"; + +import { formatBytesShort } from "~/utils/format"; + +import type { VisibleNode } from "./dataflowGraph"; + +export const COLORS = { + noArrangementRegion: "#12b886", + noArrangementOperator: "#ffffff", + arrangementRegion: "#7950f2", + arrangementOperator: "#fab005", +}; + +export function nodeFillColor(node: VisibleNode): string { + const arranged = (node.transitive?.arrangementRecords ?? 0n) > 0n; + const region = node.kind !== "operator" && node.kind !== "port"; + if (region) return arranged ? COLORS.arrangementRegion : COLORS.noArrangementRegion; + return arranged ? COLORS.arrangementOperator : COLORS.noArrangementOperator; +} + +export function formatElapsed(ns: bigint): string { + return `scheduled ${Math.round(Number(ns) / 1e9)}s`; +} + +export type FlowNodeData = { node: VisibleNode; dimmed: boolean; color: string }; + +const statLines = (node: VisibleNode) => { + const lines: string[] = []; + const stats = node.stats; + if (!stats) return lines; + if (stats.arrangementRecords > 0n) { + lines.push(`${stats.arrangementRecords} arranged records`); + lines.push(formatBytesShort(stats.arrangementSize)); + } + if (stats.elapsedNs > 0n) lines.push(formatElapsed(stats.elapsedNs)); + return lines; +}; + +const CardShell = ({ + data, + children, +}: { + data: FlowNodeData; + children: React.ReactNode; +}) => ( + + {children} + + + +); + +export const OperatorNode = ({ data }: NodeProps & { data: FlowNodeData }) => ( + + `LIR ${l.lirId}: ${l.operator}`).join(", ")} + isDisabled={data.node.lir.length === 0} + > + + + {data.node.label} + + {statLines(data.node).map((line) => ( + + {line} + + ))} + + + +); + +export const CollapsedRegionNode = ({ data }: NodeProps & { data: FlowNodeData }) => ( + + + {data.node.label} + + {data.node.childCount} children + {statLines(data.node).map((line) => ( + + {line} + + ))} + +); + +export const RegionNode = ({ data }: NodeProps & { data: FlowNodeData }) => ( + + + {data.node.label} + + + + +); + +export const PortNode = ({ data }: NodeProps & { data: FlowNodeData }) => ( + + {data.node.label} + + + +); + +export const nodeTypes = { + operator: OperatorNode, + collapsedRegion: CollapsedRegionNode, + region: RegionNode, + port: PortNode, +}; +``` + +- [ ] **Step 2: Implement ChannelEdge.tsx** + +```tsx +import { Text, Tooltip } from "@chakra-ui/react"; +import { + BaseEdge, + EdgeLabelRenderer, + getBezierPath, + type EdgeProps, +} from "@xyflow/react"; +import React from "react"; + +export type ChannelEdgeData = { + messagesSent: bigint; + batchesSent: bigint; + channelTypes: string[]; + dimmed: boolean; +}; + +export const ChannelEdge = (props: EdgeProps & { data: ChannelEdgeData }) => { + const [path, labelX, labelY] = getBezierPath(props); + const { messagesSent, batchesSent, channelTypes, dimmed } = props.data; + const idle = messagesSent === 0n; + const label = idle ? "" : `${messagesSent} records / ${batchesSent} batches`; + return ( + <> + + {label && ( + + + + {label} + {channelTypes.length > 0 && ` · ${channelTypes.join(", ")}`} + + + + )} + + ); +}; +``` + +- [ ] **Step 3: Implement DataflowGraphView.tsx** + +```tsx +import { Box, Spinner, useToast } from "@chakra-ui/react"; +import { + Background, + Controls, + MiniMap, + ReactFlow, + type Edge, + type Node, +} from "@xyflow/react"; +import React from "react"; + +import "@xyflow/react/dist/style.css"; + +import { + deriveVisibleGraph, + MAX_VISIBLE_NODES, + visibleNodeCount, + type CollapseState, + type DataflowStructure, + type GraphDecorations, + type VisibleNode, +} from "./dataflowGraph"; +import { ChannelEdge } from "./ChannelEdge"; +import { NODE_DIMENSIONS } from "./elkGraph"; +import { nodeFillColor, nodeTypes } from "./nodes"; +import { useElkLayout } from "./useElkLayout"; + +const edgeTypes = { channel: ChannelEdge }; + +export interface DataflowGraphViewProps { + structure: DataflowStructure; + collapsed: CollapseState; + onCollapsedChange: (next: CollapseState) => void; + cacheKey: string; + decorations?: GraphDecorations; + onNodeClick?: (node: VisibleNode) => void; +} + +export const DataflowGraphView = ({ + structure, + collapsed, + onCollapsedChange, + cacheKey, + decorations, + onNodeClick, +}: DataflowGraphViewProps) => { + const toast = useToast(); + const visible = React.useMemo(() => { + const graph = deriveVisibleGraph(structure, collapsed); + if (!decorations?.hiddenNodeIds && !decorations?.hiddenEdgeIds) return graph; + const hiddenNodes = decorations.hiddenNodeIds ?? new Set(); + const hiddenEdges = decorations.hiddenEdgeIds ?? new Set(); + return { + nodes: graph.nodes.filter((n) => !hiddenNodes.has(n.id)), + edges: graph.edges.filter( + (e) => + !hiddenEdges.has(e.id) && !hiddenNodes.has(e.source) && !hiddenNodes.has(e.target), + ), + }; + }, [structure, collapsed, decorations?.hiddenNodeIds, decorations?.hiddenEdgeIds]); + + const layoutKey = `${cacheKey}|${[...collapsed].sort().join(",")}|${ + decorations?.hiddenNodeIds ? [...decorations.hiddenNodeIds].sort().join(",") : "" + }`; + const { positions, layouting, error } = useElkLayout(visible, layoutKey); + + const toggleRegion = React.useCallback( + (node: VisibleNode) => { + const next = new Set(collapsed); + if (next.has(node.id)) { + next.delete(node.id); + if (visibleNodeCount(structure, next) > MAX_VISIBLE_NODES) { + toast({ + status: "warning", + title: `Expanding would show more than ${MAX_VISIBLE_NODES} nodes.`, + }); + return; + } + } else { + next.add(node.id); + } + onCollapsedChange(next); + }, + [collapsed, onCollapsedChange, structure, toast], + ); + + const nodes: Node[] = React.useMemo(() => { + if (!positions) return []; + return visible.nodes.map((n) => { + const pos = positions[n.id] ?? { x: 0, y: 0, ...NODE_DIMENSIONS[n.kind] }; + return { + id: n.id, + type: n.kind, + position: { x: pos.x, y: pos.y }, + parentId: n.parent ?? undefined, + extent: n.parent ? ("parent" as const) : undefined, + style: { width: pos.width, height: pos.height }, + draggable: false, + connectable: false, + data: { + node: n, + dimmed: decorations?.dimmedNodeIds?.has(n.id) ?? false, + color: decorations?.nodeColors?.get(n.id) ?? nodeFillColor(n), + }, + }; + }); + }, [visible, positions, decorations?.dimmedNodeIds, decorations?.nodeColors]); + + const edges: Edge[] = React.useMemo( + () => + visible.edges.map((e) => ({ + id: e.id, + source: e.source, + target: e.target, + type: "channel", + data: { + messagesSent: e.messagesSent, + batchesSent: e.batchesSent, + channelTypes: e.channelTypes, + dimmed: + (decorations?.dimmedNodeIds?.has(e.source) ?? false) || + (decorations?.dimmedNodeIds?.has(e.target) ?? false) || + (decorations?.dimmedEdgeIds?.has(e.id) ?? false), + }, + })), + [visible, decorations?.dimmedNodeIds], + ); + + if (error) throw new Error(error); + return ( + + {layouting && ( + + + + )} + onNodeClick?.((node.data as { node: VisibleNode }).node)} + onNodeDoubleClick={(_, node) => { + const visibleNode = (node.data as { node: VisibleNode }).node; + if (visibleNode.kind === "region" || visibleNode.kind === "collapsedRegion") { + toggleRegion(visibleNode); + } + }} + proOptions={{ hideAttribution: true }} + > + + + + + + ); +}; +``` + +- [ ] **Step 4: Create the shared React Flow test mock** + +`console/src/test/mockReactFlow.tsx`. jsdom lacks `ResizeObserver`/`DOMMatrixReadOnly`, so component tests mock `@xyflow/react` to a flat renderer that preserves the decision logic under test (which nodes and labels exist): + +```tsx +import { vi } from "vitest"; +import React from "react"; + +export function mockReactFlow() { + vi.mock("@xyflow/react", () => ({ + ReactFlow: ({ nodes, children }: { nodes: { id: string; data: { node: { label: string } } }[]; children?: React.ReactNode }) => ( +
+ {nodes.map((n) => ( +
+ {n.data.node.label} +
+ ))} + {children} +
+ ), + Background: () => null, + Controls: () => null, + MiniMap: () => null, + Handle: () => null, + Position: { Left: "left", Right: "right" }, + BaseEdge: () => null, + EdgeLabelRenderer: ({ children }: { children: React.ReactNode }) => <>{children}, + getBezierPath: () => ["", 0, 0], + })); +} +``` + +Also mock `./useElkLayout` in component tests to return deterministic positions synchronously: + +```typescript +vi.mock("~/platform/dataflows/useElkLayout", () => ({ + useElkLayout: (graph: { nodes: { id: string }[] } | null) => ({ + positions: graph + ? Object.fromEntries(graph.nodes.map((n, i) => [n.id, { x: i * 100, y: 0, width: 100, height: 50 }])) + : null, + layouting: false, + error: null, + }), +})); +``` + +- [ ] **Step 5: Typecheck, lint, commit** + +Run: `yarn typecheck && yarn lint` +Expected: PASS. + +```bash +git add console/src/platform/dataflows/ console/src/test/mockReactFlow.tsx +git commit -m "console: React Flow dataflow graph view with collapsible regions" +``` + +### Task 7: Minimal detail page and route + +**Files:** +- Create: `console/src/platform/dataflows/DataflowDetailPage.tsx` +- Modify: `console/src/platform/clusters/ClusterDetail.tsx` + +**Interfaces:** +- Consumes: `useDataflowGraphData`, `DataflowGraphView`, `defaultCollapseState`, cluster store (`useAllClusters` pattern from `DataflowVisualizer.tsx:261-267`), `LabeledSelect`, `ErrorBox`, `MainContentContainer`. +- Produces: route `dataflows/:dataflowId` under cluster detail, replica via `?replica=` search param. Page component signature: `const DataflowDetailPage: () => JSX.Element` (default export). + +- [ ] **Step 1: Implement DataflowDetailPage (minimal: replica select, graph, spinner, error box)** + +```tsx +import { Alert, AlertIcon, Spinner, Text, VStack } from "@chakra-ui/react"; +import React from "react"; +import { useParams, useSearchParams } from "react-router-dom"; + +import { ErrorCode } from "~/api/materialize/types"; +import { useDataflowGraphData } from "~/api/materialize/dataflow/useDataflowGraphData"; +import ErrorBox from "~/components/ErrorBox"; +import LabeledSelect from "~/components/LabeledSelect"; +import { MainContentContainer } from "~/layouts/BaseLayout"; +import { useAllClusters } from "~/store/allClusters"; + +import { + defaultCollapseState, + type CollapseState, +} from "./dataflowGraph"; +import { DataflowGraphView } from "./DataflowGraphView"; + +const DataflowDetailPage = () => { + const { clusterId, dataflowId } = useParams(); + const { getClusterById } = useAllClusters(); + const cluster = clusterId ? getClusterById(clusterId) : undefined; + const [searchParams, setSearchParams] = useSearchParams(); + const replicaName = searchParams.get("replica") ?? cluster?.replicas[0]?.name; + + const params = React.useMemo( + () => + cluster && replicaName && dataflowId + ? { clusterName: cluster.name, replicaName, dataflowId } + : undefined, + [cluster, replicaName, dataflowId], + ); + const { data, error, databaseError, loading } = useDataflowGraphData(params); + + const [collapsed, setCollapsed] = React.useState(null); + const structureKey = data + ? `${params?.dataflowId}/${params?.replicaName}/${data.structure.nodes.size}` + : null; + React.useEffect(() => { + if (data) setCollapsed(defaultCollapseState(data.structure)); + // Reset collapse state when the structure identity changes. + }, [structureKey]); // eslint-disable-line react-hooks/exhaustive-deps + + if (!cluster) return null; + const permissionError = + databaseError && + "code" in databaseError && + databaseError.code === ErrorCode.INSUFFICIENT_PRIVILEGE; + + return ( + + + setSearchParams({ replica: e.target.value })} + flexShrink={0} + > + {cluster.replicas.map((r) => ( + + ))} + + {permissionError ? ( + + + + You'll need{" "} + + USAGE + {" "} + privilege on this cluster to visualize this dataflow. + + + ) : error ? ( + + ) : !data || !collapsed ? ( + + ) : data.structure.nodes.size <= 1 ? ( + This dataflow contains no operators. + ) : ( + + )} + + + ); +}; + +export default DataflowDetailPage; +``` + +- [ ] **Step 2: Wire the route** + +In `console/src/platform/clusters/ClusterDetail.tsx`: lazy-import the page, add the route inside `SentryRoutes` (`ClusterDetail.tsx:149-162`), and add the tab to `subnavItems` (`ClusterDetail.tsx:127-137`) gated on the flag. Use the same `useFlags` import that `SimpleObjectDetailRoutes.tsx` uses (check its import block and copy it). + +```tsx +const DataflowDetailPage = React.lazy( + () => import("~/platform/dataflows/DataflowDetailPage"), +); +``` + +```tsx +const flags = useFlags(); +const subnavItems: Tab[] = React.useMemo(() => { + const tabs: Tab[] = [ + { label: "Overview", href: "..", end: true }, + { label: "Replicas", href: "../replicas" }, + { label: "Materialized Views", href: "../materialized-views" }, + { label: "Indexes", href: "../indexes" }, + { label: "Sources", href: "../sources" }, + { label: "Sinks", href: "../sinks" }, + ]; + if (flags["visualization-features"]) { + tabs.push({ label: "Dataflows", href: "../dataflows" }); + } + return tabs; +}, [flags]); +``` + +```tsx +} +/> +``` + +(The list route at `path="dataflows"` comes in Task 9. Until then, navigate by URL directly.) + +- [ ] **Step 3: Manual smoke test** + +Start a local Materialize (`bin/environmentd` from repo root; see `mz-run` skill if it fails) and the console dev server (`cd console && yarn start`). Create a table plus an index, find its dataflow id (`SELECT id, name FROM mz_introspection.mz_dataflows`, run with `SET cluster = ...`), and open `https://local.dev.materialize.com:3000/.../clusters///dataflows/`. Expected: graph renders, regions expand and collapse on double click. + +- [ ] **Step 4: Typecheck, lint, commit** + +```bash +yarn typecheck && yarn lint +git add console/src +git commit -m "console: dataflow detail page with graph view behind flag" +``` + +### Task 8: CHECKPOINT, phase 1 gate + +**Execution mode:** driver-executed. This task needs a browser (devtools performance panel, screenshots) and a local environmentd, so the main session or a human runs it. Do NOT dispatch it to a code subagent. + +**Files:** none committed. Temporary `performance.mark` instrumentation in `useElkLayout.ts` around postMessage/response is permitted for Step 2 and must be reverted before Task 9 (`git status` clean afterwards). + +STOP after this task and report to the user. + +- [ ] **Step 1: Create the reference dataflow** + +Against local environmentd (`psql -h localhost -p 6875 -U materialize`): + +```sql +CREATE TABLE orders (id int, cust int, amt int); +CREATE TABLE customers (id int, region int); +CREATE TABLE regions (id int, name text); +CREATE TABLE items (order_id int, sku int, qty int); +CREATE TABLE skus (id int, price int); +CREATE MATERIALIZED VIEW gate_mv AS +SELECT r.name, count(*) AS orders, sum(i.qty * s.price) AS revenue +FROM orders o +JOIN customers c ON o.cust = c.id +JOIN regions r ON c.region = r.id +JOIN items i ON i.order_id = o.id +JOIN skus s ON s.id = i.sku +GROUP BY r.name +HAVING sum(i.qty * s.price) > 100; +INSERT INTO regions VALUES (1, 'emea'), (2, 'amer'); +INSERT INTO customers SELECT g, 1 + g % 2 FROM generate_series(1, 100) g; +INSERT INTO orders SELECT g, 1 + g % 100, g FROM generate_series(1, 1000) g; +INSERT INTO skus SELECT g, g FROM generate_series(1, 50) g; +INSERT INTO items SELECT g % 1000 + 1, g % 50 + 1, g FROM generate_series(1, 5000) g; +``` + +Verify operator count: `SET cluster = quickstart; SELECT count(*) FROM mz_introspection.mz_dataflow_operator_dataflows WHERE dataflow_name LIKE '%gate_mv%';` +Expected: >= 300. If under 300, add a sixth join (`JOIN customers c2 ON c2.id = o.cust`) and re-check. + +- [ ] **Step 2: Measure** + +Open the gate_mv dataflow in the visualizer (dev server). In browser devtools performance panel or via `performance.mark` temporarily added around the `useElkLayout` postMessage/response, record: + +* Default collapsed view layout time. Gate: < 500 ms. +* Expand regions until visible node count approaches 1500 (temporarily raise expansion via expand-all in console, or expand largest regions). Layout time. Gate: < 5 s, UI responsive (canvas pans during layout). +* Repeat both in a production build: `yarn build:local && yarn preview`, same URL on port 3000. Gate: worker loads and lays out in the built bundle. + +- [ ] **Step 3: Visual check** + +Screenshots of collapsed and expanded gate_mv. Gate: no overlapping nodes, edges flow left to right. + +- [ ] **Step 4: STOP, report** + +Report measurements + screenshots to the user for sign-off (spec MVP section). If any gate fails: do not proceed, return to the design doc per spec. + +--- + +## Phase 2: selection and refresh + +### Task 9: Dataflow list hook and DataflowsPage + +**Files:** +- Create: `console/src/api/materialize/dataflow/useDataflowList.ts` +- Create: `console/src/platform/dataflows/DataflowsPage.tsx` +- Modify: `console/src/platform/clusters/ClusterDetail.tsx` +- Test: `console/src/api/materialize/dataflow/useDataflowList.test.ts` + +**Interfaces:** +- Produces: + +```typescript +export interface DataflowListEntry { + id: string; // uint8 as string + name: string; + records: bigint; + size: bigint; + elapsedNs: bigint; +} +export function useDataflowList(params?: { clusterName: string; replicaName: string }): { + data: DataflowListEntry[] | null; + error: unknown; databaseError: unknown; loading: boolean; refetch: () => void; +}; +``` + +- [ ] **Step 1: Write failing hook test** (msw, same harness as Task 5): mock one row, assert mapping to `DataflowListEntry` with bigint fields normalized via `BigInt()`, assert `data` is null while `error` set. + +```typescript +const listResult = { + desc: { columns: [{ name: "id" }, { name: "name" }, { name: "records" }, { name: "size" }, { name: "elapsedNs" }] }, + rows: [["7", "Dataflow: mv", "100", "4096", "12345"]], +}; +// handler returns { results: [listResult] }; assert +// result.current.data?.[0] === { id: "7", name: "Dataflow: mv", records: 100n, size: 4096n, elapsedNs: 12345n } +``` + +- [ ] **Step 2: Verify failure, then implement** + +Query (kysely `sql`, `.$castTo<...>()`, same pattern as Task 5, no literals so no escaping): + +```sql +SELECT + md.id::text AS id, + md.name, + COALESCE(mrpd.records, 0) AS records, + COALESCE(mrpd.size, 0) AS size, + COALESCE(el.elapsed_ns, 0) AS "elapsedNs" +FROM mz_dataflows AS md +LEFT JOIN mz_records_per_dataflow AS mrpd ON mrpd.id = md.id +LEFT JOIN ( + SELECT mdod.dataflow_id, sum(mse.elapsed_ns) AS elapsed_ns + FROM mz_dataflow_operator_dataflows AS mdod + JOIN mz_scheduling_elapsed AS mse ON mse.id = mdod.id + GROUP BY mdod.dataflow_id +) AS el ON el.dataflow_id = md.id +ORDER BY size DESC +``` + +Hook body mirrors `useDataflowGraphData` minus tagging (a list going momentarily stale across replica switch is masked by `loading`, and errors clear `data` the same way: `const data = !error && results ? normalize(results.list) : null`). + +- [ ] **Step 3: Implement DataflowsPage** + +Loading spinner, `ErrorBox` on error, permission alert exactly as in Task 7. Export-deep-link resolution is Task 10, leave a plain list here. Core: + +```tsx +const DataflowsPage = () => { + const { clusterId } = useParams(); + const { getClusterById } = useAllClusters(); + const cluster = clusterId ? getClusterById(clusterId) : undefined; + const [searchParams, setSearchParams] = useSearchParams(); + const replicaName = searchParams.get("replica") ?? cluster?.replicas[0]?.name; + const params = React.useMemo( + () => + cluster && replicaName ? { clusterName: cluster.name, replicaName } : undefined, + [cluster, replicaName], + ); + const { data, error, databaseError, loading } = useDataflowList(params); + if (!cluster) return null; + const permissionError = + databaseError && + "code" in databaseError && + databaseError.code === ErrorCode.INSUFFICIENT_PRIVILEGE; + return ( + + + setSearchParams({ replica: e.target.value })} + > + {cluster.replicas.map((r) => ( + + ))} + + {permissionError ? ( + + + + You'll need{" "} + + USAGE + {" "} + privilege on this cluster to list its dataflows. + + + ) : error ? ( + + ) : !data && loading ? ( + + ) : ( + + + + + + {(data ?? []).map((d) => ( + + + + + + + ))} + +
NameRecordsSizeScheduled
+ {d.name} + {d.records.toString()}{formatBytesShort(d.size)}{Math.round(Number(d.elapsedNs) / 1e9)}s
+ )} + + + ); +}; +export default DataflowsPage; +``` + +- [ ] **Step 4: Wire route** + +`ClusterDetail.tsx`: `} />` next to the Task 7 route (lazy import, flag-gated same as tab). + +- [ ] **Step 5: Test, typecheck, manual check list renders, commit** + +```bash +yarn test --run src/api/materialize/dataflow/ && yarn typecheck && yarn lint +git add console/src +git commit -m "console: cluster dataflows list page" +``` + +### Task 10: Export deep-link and object tab rewire + +**Files:** +- Create: `console/src/api/materialize/dataflow/useDataflowIdForExport.ts` +- Modify: `console/src/platform/dataflows/DataflowsPage.tsx` +- Modify: `console/src/platform/object-explorer/SimpleObjectDetailRoutes.tsx` + +**Interfaces:** +- Produces: `useDataflowIdForExport(params?: { clusterName: string; replicaName: string; exportId: string }): { dataflowId: string | null; loading: boolean; error: unknown }`. Query: `SELECT dataflow_id::text AS "dataflowId" FROM mz_compute_exports WHERE export_id = ${lit(exportId)}` (exportId validated `/^[stu]\d+$/`). + +- [ ] **Step 1: Implement hook** (pattern of Task 9, single query, `dataflowId = results?.rows?.[0]?.dataflowId ?? null`). + +- [ ] **Step 2: DataflowsPage export resolution** + +When `?export=` is present and a replica is chosen: call the hook, on `dataflowId` render ``. On resolved-but-null (export gone): show `ErrorBox` message "This object has no running dataflow on the selected replica." with the list rendered below. + +- [ ] **Step 3: Rewire the object "Visualize" tab** + +In `SimpleObjectDetailRoutes.tsx`: keep `shouldShowDataflowVisualizer` logic (`:136-139`). Replace the tab href (`:160-165`) and the route (`:203-205`): the tab now links to the cluster page. The object row has `clusterId`; get the cluster name via `useAllClusters().getClusterById`. Href: `/clusters/${object.clusterId}/${clusterName}/dataflows?export=${object.id}`. Verify the absolute-path shape against how `ClustersList.tsx` builds cluster links and match it (react-router basename handles the prefix). Delete the `dataflow-visualizer` Route and the `DataflowVisualizer` lazy import (`:37-39`). The old component file itself is deleted in Task 18. + +- [ ] **Step 4: Manual check** Object detail → Visualize → lands on graph of that object's dataflow. Typecheck, lint, commit. + +```bash +git add console/src +git commit -m "console: deep-link object visualize tab into dataflows page" +``` + +### Task 11: Refresh in place + +**Files:** +- Modify: `console/src/platform/dataflows/DataflowDetailPage.tsx` + +**Interfaces:** +- Consumes: `refetch` and `data.fetchedAt` from `useDataflowGraphData`. + +- [ ] **Step 1: Add refresh UI** + +Toolbar row above the graph: `Button` "Refresh" calling `refetch`, disabled while `loading`. `Text` "Last fetched {fetchedAt.toLocaleTimeString()}". While a same-params refetch is loading, `data` stays non-null (Task 5 contract), so the graph stays mounted and stats update in place when the new result lands. + +- [ ] **Step 2: Preserve layout across refresh** + +`structureKey` in Task 7 already encodes `nodes.size`. Strengthen it so pure stats refreshes reuse layout and structure changes relayout: `structureKey = `${dataflowId}/${replicaName}/` + a stable digest of sorted node ids` (join sorted `[...structure.nodes.keys()]` with "," and use its length + a simple 32-bit string hash, implemented inline as `hashString`). Collapse state resets only when `structureKey` changes, which was already the effect dependency. + +```typescript +function hashString(s: string): number { + let h = 0; + for (let i = 0; i < s.length; i++) h = (Math.imul(h, 31) + s.charCodeAt(i)) | 0; + return h; +} +``` + +- [ ] **Step 3: Manual check** Refresh keeps viewport + expansion, stats badges update. Typecheck, lint, commit. + +```bash +git add console/src +git commit -m "console: manual refresh preserving dataflow graph layout" +``` + +--- + +## Phase 3: interactions + +### Task 12: Node detail panel + +**Files:** +- Create: `console/src/platform/dataflows/NodeDetailPanel.tsx` +- Modify: `console/src/platform/dataflows/DataflowDetailPage.tsx` +- Modify: `console/src/platform/dataflows/DataflowGraphView.tsx` (add `onPaneClick` pass-through prop) + +- [ ] **Step 1: Implement panel** + +```tsx +export const NodeDetailPanel = ({ + node, + onClose, +}: { + node: VisibleNode; + onClose: () => void; +}) => { + const row = (label: string, value: string) => ( + + {label} + {value} + + ); + return ( + + + {node.label} + + + {row("Kind", node.kind)} + {node.address && row("Address", node.address.join("."))} + {node.childCount > 0 && row("Children", String(node.childCount))} + {node.stats && ( + <> + {row("Arranged records", node.stats.arrangementRecords.toString())} + {row("Arrangement size", formatBytesShort(node.stats.arrangementSize))} + {row("Elapsed", `${Math.round(Number(node.stats.elapsedNs) / 1e9)}s`)} + + )} + {node.transitive && node.childCount > 0 && ( + <> + {row("Subtree records", node.transitive.arrangementRecords.toString())} + {row("Subtree size", formatBytesShort(node.transitive.arrangementSize))} + {row("Subtree elapsed", `${Math.round(Number(node.transitive.elapsedNs) / 1e9)}s`)} + + )} + {node.lir.map((l) => ( + + LIR {l.lirId} ({l.exportId}): {l.operator} + + ))} + + ); +}; +``` + +- [ ] **Step 2: Wire** `onNodeClick` in `DataflowDetailPage` sets `selectedNode` state; render panel beside the graph in an `HStack`. Clicking the canvas background clears it: `DataflowGraphViewProps` gains `onPaneClick?: () => void;`, forwarded as ``. + +- [ ] **Step 3: Typecheck, lint, manual check, commit** + +```bash +git add console/src +git commit -m "console: node detail panel for dataflow visualizer" +``` + +### Task 13: Filters model + toolbar (search, hide idle, channel types) + +**Files:** +- Modify: `console/src/platform/dataflows/dataflowGraph.ts` (decorations, pure) +- Create: `console/src/platform/dataflows/DataflowToolbar.tsx` +- Modify: `console/src/platform/dataflows/DataflowDetailPage.tsx` +- Modify: `console/src/platform/dataflows/DataflowGraphView.tsx` (`centerRef` prop, `CenterHelper`) +- Test: `console/src/platform/dataflows/dataflowGraph.test.ts` + +**Interfaces:** +- Produces (in `dataflowGraph.ts`): + +```typescript +export interface Filters { + search: string; + hideIdle: boolean; + heatmap: "off" | "elapsed" | "size"; + heatmapThreshold: number; // 0..1 fraction of max + channelTypes: string[] | null; // null = all +} +export const DEFAULT_FILTERS: Filters; + +// Replaces the all-optional Task 6 declaration in dataflowGraph.ts. decorateGraph +// returns every field set. DataflowGraphView's optional-chaining reads and its +// optional `decorations` prop remain valid against the narrowed type. +export interface GraphDecorations { + dimmedNodeIds: Set; + hiddenNodeIds: Set; + hiddenEdgeIds: Set; + dimmedEdgeIds: Set; + nodeColors: Map; + searchMatches: string[]; // visible node ids, document order +} +export function decorateGraph( + graph: VisibleGraph, + filters: Filters, + heatColor: (t: number) => string, // injected so the pure module stays d3-free +): GraphDecorations; + +export function allChannelTypes(structure: DataflowStructure): string[]; + +// Search may match inside collapsed regions. Returns a collapse state with +// every ancestor of every matching node expanded (respecting nothing else). +export function expandForSearch( + structure: DataflowStructure, + collapsed: CollapseState, + search: string, +): CollapseState; +``` + +Semantics (each a test): +* `search` non-empty: case-insensitive substring on `label`. Non-matching operator/collapsedRegion nodes → `dimmedNodeIds`. Matches listed in `searchMatches`. +* `hideIdle`: operators with `stats.elapsedNs === 0n` and no arranged records → `hiddenNodeIds`; edges with `messagesSent === 0n` → `hiddenEdgeIds`. Regions and ports never hidden. +* `channelTypes` non-null: edges whose `channelTypes` don't intersect → `hiddenEdgeIds` additions? No: dimmed, per spec. Put them in a dim path: edges get dimmed via node dimming today, so extend `ChannelEdgeData.dimmed` computation in `DataflowGraphView` with `hiddenEdgeIds`-independent `dimmedEdgeIds: Set` added to `GraphDecorations` and populated here. +* `heatmap` != off: for every non-port node, `t = Number(metric(node)) / Number(max over visible non-port nodes)` with `metric = transitive.elapsedNs | transitive.arrangementSize`; `nodeColors.set(id, heatColor(t))`; nodes with `t < heatmapThreshold` → `dimmedNodeIds`. If max is 0, no colors set (spec: neutral base, slider disabled). + +- [ ] **Step 1: Write failing tests** + +Append to `dataflowGraph.test.ts`: + +```typescript +import { + allChannelTypes, + decorateGraph, + DEFAULT_FILTERS, + expandForSearch, +} from "./dataflowGraph"; + +describe("decorateGraph", () => { + const s = buildDataflowStructure(OPS, CHANNELS, LIR_SPANS); + const expanded = deriveVisibleGraph(s, new Set()); + const heat = (t: number) => `heat(${t.toFixed(2)})`; + + it("search dims non-matching leaf nodes and lists matches", () => { + const d = decorateGraph(expanded, { ...DEFAULT_FILTERS, search: "join" }, heat); + expect(d.searchMatches).toEqual([nodeIdOf([5, 1, 1])]); + expect(d.dimmedNodeIds.has(nodeIdOf([5, 2]))).toBe(true); + expect(d.dimmedNodeIds.has(nodeIdOf([5, 1, 1]))).toBe(false); + }); + + it("hideIdle hides zero-activity operators and zero-message edges, never regions or ports", () => { + const d = decorateGraph(expanded, { ...DEFAULT_FILTERS, hideIdle: true }, heat); + // every fixture operator has elapsed > 0 except none; hide check via a synthetic idle op + const idle = buildDataflowStructure( + [...OPS, { id: "15", address: ["5", "3"], name: "Idle", arrangementRecords: "0", arrangementSize: "0", elapsedNs: "0" }], + CHANNELS, [], + ); + const d2 = decorateGraph(deriveVisibleGraph(idle, new Set()), { ...DEFAULT_FILTERS, hideIdle: true }, heat); + expect(d2.hiddenNodeIds.has(nodeIdOf([5, 3]))).toBe(true); + expect(d2.hiddenNodeIds.has(nodeIdOf([5, 1]))).toBe(false); + expect(d.hiddenEdgeIds.size).toBe(0); // all fixture edges have messages + }); + + it("channel type filter dims non-matching edges", () => { + const d = decorateGraph(expanded, { ...DEFAULT_FILTERS, channelTypes: ["batches"] }, heat); + const rowsEdge = expanded.edges.find((e) => e.channelTypes.includes("rows"))!; + const batchesEdge = expanded.edges.find((e) => e.channelTypes.includes("batches"))!; + expect(d.dimmedEdgeIds.has(rowsEdge.id)).toBe(true); + expect(d.dimmedEdgeIds.has(batchesEdge.id)).toBe(false); + }); + + it("heatmap colors by transitive metric and dims below threshold", () => { + const d = decorateGraph( + expanded, + { ...DEFAULT_FILTERS, heatmap: "elapsed", heatmapThreshold: 0.5 }, + heat, + ); + // max transitive elapsed among visible non-ports is the region (13n) + expect(d.nodeColors.get(nodeIdOf([5, 1]))).toEqual("heat(1.00)"); + expect(d.dimmedNodeIds.has(nodeIdOf([5, 2]))).toBe(true); // 2/13 < 0.5 + }); + + it("heatmap with all-zero metric sets no colors", () => { + const zero = buildDataflowStructure( + OPS.map((o) => ({ ...o, elapsedNs: "0" })), CHANNELS, [], + ); + const d = decorateGraph( + deriveVisibleGraph(zero, new Set()), + { ...DEFAULT_FILTERS, heatmap: "elapsed" }, + heat, + ); + expect(d.nodeColors.size).toBe(0); + }); +}); + +describe("expandForSearch / allChannelTypes", () => { + const s = buildDataflowStructure(OPS, CHANNELS, LIR_SPANS); + it("expands ancestors of matches", () => { + const next = expandForSearch(s, defaultCollapseState(s), "join"); + expect(next.has(nodeIdOf([5, 1]))).toBe(false); + }); + it("collects channel types", () => { + expect(allChannelTypes(s)).toEqual(["batches", "rows"]); + }); +}); +``` + +- [ ] **Step 2: Implement** + +```typescript +export const DEFAULT_FILTERS: Filters = { + search: "", + hideIdle: false, + heatmap: "off", + heatmapThreshold: 0, + channelTypes: null, +}; + +export function allChannelTypes(structure: DataflowStructure): string[] { + const types = new Set(); + for (const c of structure.channels) if (c.channelType) types.add(c.channelType); + return [...types].sort(); +} + +export function decorateGraph( + graph: VisibleGraph, + filters: Filters, + heatColor: (t: number) => string, +): GraphDecorations { + const d: GraphDecorations = { + dimmedNodeIds: new Set(), + hiddenNodeIds: new Set(), + hiddenEdgeIds: new Set(), + dimmedEdgeIds: new Set(), + nodeColors: new Map(), + searchMatches: [], + }; + const needle = filters.search.trim().toLowerCase(); + for (const n of graph.nodes) { + if (needle && n.kind !== "port" && n.kind !== "region") { + if (n.label.toLowerCase().includes(needle)) d.searchMatches.push(n.id); + else d.dimmedNodeIds.add(n.id); + } + if ( + filters.hideIdle && + n.kind === "operator" && + n.stats !== null && + n.stats.elapsedNs === 0n && + n.stats.arrangementRecords === 0n + ) { + d.hiddenNodeIds.add(n.id); + } + } + for (const e of graph.edges) { + if (filters.hideIdle && e.messagesSent === 0n) d.hiddenEdgeIds.add(e.id); + if ( + filters.channelTypes !== null && + !e.channelTypes.some((t) => filters.channelTypes!.includes(t)) + ) { + d.dimmedEdgeIds.add(e.id); + } + } + if (filters.heatmap !== "off") { + const metric = (n: VisibleNode): bigint => + filters.heatmap === "elapsed" + ? n.transitive?.elapsedNs ?? 0n + : n.transitive?.arrangementSize ?? 0n; + const candidates = graph.nodes.filter((n) => n.kind !== "port"); + const max = candidates.reduce((m, n) => (metric(n) > m ? metric(n) : m), 0n); + if (max > 0n) { + for (const n of candidates) { + const t = Number(metric(n)) / Number(max); + d.nodeColors.set(n.id, heatColor(t)); + if (t < filters.heatmapThreshold) d.dimmedNodeIds.add(n.id); + } + } + } + return d; +} + +export function expandForSearch( + structure: DataflowStructure, + collapsed: CollapseState, + search: string, +): CollapseState { + const needle = search.trim().toLowerCase(); + if (!needle) return collapsed; + const next = new Set(collapsed); + for (const node of structure.nodes.values()) { + if (!node.name.toLowerCase().includes(needle)) continue; + for (let len = 1; len < node.address.length; len++) { + next.delete(nodeIdOf(node.address.slice(0, len))); + } + } + return next; +} +``` + + +- [ ] **Step 3: Verify tests pass** + +Run: `yarn test --run src/platform/dataflows/dataflowGraph.test.ts` + +- [ ] **Step 4: Toolbar UI** + +Pure controlled component: + +```tsx +export interface DataflowToolbarProps { + filters: Filters; + onFiltersChange: (next: Filters) => void; + channelTypes: string[]; + matchCount: number; + matchIndex: number; + onJump: (delta: 1 | -1) => void; +} + +export const DataflowToolbar = ({ + filters, + onFiltersChange, + channelTypes, + matchCount, + matchIndex, + onJump, +}: DataflowToolbarProps) => { + const [search, setSearch] = React.useState(filters.search); + // Debounce search input into the filters object. + React.useEffect(() => { + if (search === filters.search) return; + const timeout = setTimeout(() => onFiltersChange({ ...filters, search }), 300); + return () => clearTimeout(timeout); + }, [search, filters, onFiltersChange]); + return ( + + setSearch(e.target.value)} + /> + {filters.search && ( + + + + + {matchCount === 0 ? "0/0" : `${matchIndex + 1}/${matchCount}`} + + + )} + + + Hide idle + + onFiltersChange({ ...filters, hideIdle: e.target.checked })} + /> + + + onFiltersChange({ ...filters, heatmapThreshold: v })} + > + + + + + + + + Channel types + + + {channelTypes.map((t) => ( + + { + const current = filters.channelTypes ?? channelTypes; + const next = e.target.checked + ? [...current, t] + : current.filter((x) => x !== t); + onFiltersChange({ + ...filters, + channelTypes: next.length === channelTypes.length ? null : next, + }); + }} + > + {t} + + + ))} + + + + ); +}; +``` + +- [ ] **Step 5: Wire into DataflowDetailPage** + +`filters` state, `decorations = React.useMemo(() => decorateGraph(deriveVisibleGraph(structure, collapsed), filters, heatColor), ...)` where `heatColor = (t) => interpolateYlOrRd(0.15 + 0.85 * t)` from `d3-scale-chromatic` (existing dep). Heat legend: small gradient `Box` + min/max labels next to the heatmap select. + +Centering needs React Flow context, so `DataflowGraphView` gains a `centerRef` prop filled by a helper rendered inside ``: + +```tsx +// DataflowGraphViewProps gains: +// centerRef?: React.MutableRefObject<((id: string) => void) | null>; + +const CenterHelper = ({ + centerRef, +}: { + centerRef: React.MutableRefObject<((id: string) => void) | null>; +}) => { + const reactFlow = useReactFlow(); + React.useEffect(() => { + centerRef.current = (id: string) => { + const internal = reactFlow.getInternalNode(id); + if (!internal) return; + const { x, y } = internal.internals.positionAbsolute; + const width = internal.measured?.width ?? 0; + const height = internal.measured?.height ?? 0; + reactFlow.setCenter(x + width / 2, y + height / 2, { zoom: 1, duration: 300 }); + }; + return () => { + centerRef.current = null; + }; + }, [reactFlow, centerRef]); + return null; +}; +// In DataflowGraphView's JSX, inside : +// {centerRef && } +``` + +Search jump wiring in `DataflowDetailPage`: + +```tsx +const centerRef = React.useRef<((id: string) => void) | null>(null); +const [matchIndex, setMatchIndex] = React.useState(0); +// New search: expand ancestors of matches once, reset the cursor. +React.useEffect(() => { + setMatchIndex(0); + if (filters.search && data) { + setCollapsed((c) => (c ? expandForSearch(data.structure, c, filters.search) : c)); + } +}, [filters.search]); // eslint-disable-line react-hooks/exhaustive-deps +const onJump = React.useCallback( + (delta: 1 | -1) => { + const matches = decorations.searchMatches; + if (matches.length === 0) return; + const next = (matchIndex + delta + matches.length) % matches.length; + setMatchIndex(next); + centerRef.current?.(matches[next]); + }, + [decorations.searchMatches, matchIndex], +); +``` + +- [ ] **Step 6: Typecheck, lint, tests, manual check, commit** + +```bash +yarn test --run src/platform/dataflows/ && yarn typecheck && yarn lint +git add console/src +git commit -m "console: dataflow visualizer filters and toolbar" +``` + +### Task 14: LIR panel + +**Files:** +- Create: `console/src/platform/dataflows/LirPanel.tsx` +- Modify: `console/src/platform/dataflows/DataflowDetailPage.tsx` +- Modify: `console/src/platform/dataflows/dataflowGraph.ts` (one pure helper + test) + +**Interfaces:** +- Produces: `lirIndex(structure: DataflowStructure): Map` keyed by `${exportId}/${lirId}` (test: fixture yields one entry with members `[5,1]`, `[5,1,1]` per span 11..13 covering ids 11 and 12). + +- [ ] **Step 1: Failing test + helper** + +Test (append to `dataflowGraph.test.ts`): + +```typescript +import { lirIndex } from "./dataflowGraph"; + +describe("lirIndex", () => { + it("buckets member nodes by export/lir id", () => { + const s = buildDataflowStructure(OPS, CHANNELS, LIR_SPANS); + const index = lirIndex(s); + expect([...index.keys()]).toEqual(["u42/1"]); + // span 11..13 covers operator ids 11 ([5,1]) and 12 ([5,1,1]) + expect(index.get("u42/1")!.memberIds.sort()).toEqual( + [nodeIdOf([5, 1]), nodeIdOf([5, 1, 1])].sort(), + ); + }); +}); +``` + +Implementation: + +```typescript +export function lirIndex( + structure: DataflowStructure, +): Map { + const index = new Map(); + for (const node of structure.nodes.values()) { + for (const info of node.lir) { + const key = `${info.exportId}/${info.lirId}`; + const entry = index.get(key) ?? { info, memberIds: [] }; + entry.memberIds.push(node.id); + index.set(key, entry); + } + } + return index; +} +``` + +- [ ] **Step 2: Panel** + +```tsx +export const LirPanel = ({ + structure, + onHighlight, +}: { + structure: DataflowStructure; + onHighlight: (ids: ReadonlySet | null) => void; +}) => { + const index = React.useMemo(() => lirIndex(structure), [structure]); + const byExport = React.useMemo(() => { + const groups = new Map(); + for (const [key, entry] of index) { + const list = groups.get(entry.info.exportId) ?? []; + list.push({ key, ...entry }); + groups.set(entry.info.exportId, list); + } + return groups; + }, [index]); + if (index.size === 0) return null; + return ( + + {[...byExport.entries()].map(([exportId, entries]) => ( + + + {exportId} + + {entries.map((e) => ( + onHighlight(new Set(e.memberIds))} + onMouseLeave={() => onHighlight(null)} + > + LIR {e.info.lirId}: {e.info.operator} + + ))} + + ))} + + ); +}; +``` + +- [ ] **Step 3: Wire** highlight set in `DataflowDetailPage`: when non-null, everything not in the set joins `decorations.dimmedNodeIds` (merge before passing to the view). Note collapsed ancestors: map each member id through the current collapse state via `representative`-style logic already available (`deriveVisibleGraph` output ids), simplest: dim nothing that is not currently visible, i.e. intersect with visible node ids. + +- [ ] **Step 4: Typecheck, lint, manual check (hover LIR row highlights members), commit** + +```bash +git add console/src +git commit -m "console: LIR operator panel with member highlighting" +``` + +--- + +## Phase 4: hardening and cleanup + +### Task 15: Error and empty states + CNS-109 regression test + +**Files:** +- Modify: `console/src/platform/dataflows/DataflowDetailPage.tsx` +- Test: `console/src/platform/dataflows/DataflowDetailPage.test.tsx` + +- [ ] **Step 1: Empty/gone states** + +In `DataflowDetailPage`: when `data` loads and `structure.nodes.size <= 1` (root only or empty), render `Text` "This dataflow no longer exists on this replica." with a `Link` back to `..` (the list). Timeout errors already surface through `error` → `ErrorBox`; add a "Retry" `Button` calling `refetch` under the `ErrorBox`. + +- [ ] **Step 2: Write the CNS-109 regression component test** + +`DataflowDetailPage.test.tsx`: `mockReactFlow()` + the `useElkLayout` mock from Task 6, msw handler that fails for `cluster_replica === "r2"` and returns a small successful structure otherwise (reuse the row shapes Task 5's test settled on). Setup: + +```tsx +import { screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { http, HttpResponse } from "msw"; +import React from "react"; +import { Route, Routes } from "react-router-dom"; + +import { Cluster } from "~/api/materialize/cluster/clusterList"; +import { ErrorCode } from "~/api/materialize/types"; +import server from "~/api/mocks/server"; +import { getStore } from "~/jotai"; +import { allClusters } from "~/store/allClusters"; +import { mockReactFlow } from "~/test/mockReactFlow"; +import { mockSubscribeState } from "~/test/mockSubscribe"; +import { renderComponent } from "~/test/utils"; + +import DataflowDetailPage from "./DataflowDetailPage"; + +mockReactFlow(); +// plus the useElkLayout vi.mock block from Task 6 + +const CLUSTER_ID = "u5"; +const FAILING_REPLICA = "r2"; +const testCluster = { + id: CLUSTER_ID, + name: "test_cluster", + replicas: [{ name: "r1" }, { name: FAILING_REPLICA }], +} as Cluster; + +const renderVisualizer = () => + renderComponent( + + } + /> + , + { initialRouterEntries: [`/clusters/${CLUSTER_ID}/test_cluster/dataflows/7`] }, + ); + +const okResult = { desc: { columns: [] }, rows: [] }; +const operatorsResult = { + desc: { + columns: [ + { name: "id" }, { name: "address" }, { name: "name" }, + { name: "arrangementRecords" }, { name: "arrangementSize" }, { name: "elapsedNs" }, + ], + }, + rows: [ + ["10", ["7"], "Dataflow", "0", "0", "0"], + ["11", ["7", "1"], "Map", "0", "0", "1"], + ], +}; +const handler = http.post("*/api/sql", async ({ request }) => { + const options = JSON.parse( + new URL(request.url).searchParams.get("options") ?? "{}", + ); + if (options.cluster_replica === FAILING_REPLICA) { + return HttpResponse.json({ + results: [ + { error: { message: "no such replica", code: ErrorCode.INTERNAL_ERROR } }, + ], + }); + } + return HttpResponse.json({ results: [operatorsResult, okResult, okResult] }); +}); + +beforeEach(() => { + server.use(handler); + getStore().set(allClusters, mockSubscribeState({ data: [testCluster] })); +}); +``` + +If `renderComponent`'s option name differs, match `console/src/test/utils.tsx`. Assertions: + +```tsx +it("shows an error instead of the stale graph when a refetch fails", async () => { + renderVisualizer(); + const replicaSelect = await screen.findByRole("combobox"); + // graph rendered from r1 data first + expect(await screen.findByTestId("react-flow")).toBeVisible(); + await userEvent.selectOptions(replicaSelect, "r2"); + expect( + await screen.findByText("There was an error visualizing your dataflow"), + ).toBeVisible(); + expect(screen.queryByTestId("react-flow")).toBeNull(); +}); +``` + +- [ ] **Step 3: Run test, verify pass** (the Task 5 contract makes it pass; if it fails, the bug is real, fix the hook not the test). + +Run: `yarn test --run src/platform/dataflows/DataflowDetailPage.test.tsx` + +- [ ] **Step 4: Commit** + +```bash +git add console/src +git commit -m "console: dataflow visualizer error states and CNS-109 regression test" +``` + +### Task 16: Delete the old visualizer and d3-graphviz + +**Files:** +- Delete: `console/src/platform/clusters/DataflowVisualizer.tsx` +- Delete: `console/src/api/materialize/useDataflowStructure.ts` +- Modify: `console/package.json` + +- [ ] **Step 1: Verify no remaining references** + +Run: `grep -rn "DataflowVisualizer\|useDataflowStructure\|d3-graphviz" console/src console/e2e-tests` +Expected: no hits outside the two files being deleted (Task 10 removed the route + import). If hits remain, fix them first. + +- [ ] **Step 2: Delete** + +```bash +git rm console/src/platform/clusters/DataflowVisualizer.tsx console/src/api/materialize/useDataflowStructure.ts +cd console && yarn remove d3-graphviz +``` + +`d3` stays (used elsewhere). Confirm `@hpcc-js/wasm` left the lockfile: `grep hpcc console/yarn.lock` → no hits. + +- [ ] **Step 3: Full sweep** + +Run: `yarn test --run && yarn typecheck && yarn lint && yarn build:local` +Expected: all PASS, build succeeds with no WASM chunk for graphviz. + +- [ ] **Step 4: Commit** + +```bash +git add console/package.json console/yarn.lock +git commit -m "console: delete graphviz dataflow visualizer and d3-graphviz dep" +``` + +--- + +## Self-review notes + +* Spec coverage: surface (Tasks 7, 9, 10), data layer + CNS-109 (5, 15), graph model/collapse/LIR (2, 3, 14), rendering/layout/perf guard (4, 6), filters (13), refresh (11), errors (15), deletion/CNS-108-by-construction (6, 16), MVP gate (8). Review items: `Channel` defined (Task 2), `lir` widened to array with test (Task 2). +* Detail panel is spec section "Filters and interactions" (Task 12). +* Type names match across tasks: `VisibleNode`, `VisibleGraph`, `Positions`, `GraphDecorations` (defined once in Task 13, consumed by Task 6 via optional prop declared there with matching shape). +* Known judgment calls delegated to implementers: exact msw row shapes (Task 5 step 1 note), `useFlags` import source (copy from `SimpleObjectDetailRoutes.tsx`), absolute cluster link shape (verify against `ClustersList.tsx` in Task 10). From de1a83e250999fc3af8441e5001aeadd6aa677e9 Mon Sep 17 00:00:00 2001 From: Moritz Hoffmann Date: Wed, 8 Jul 2026 22:42:07 +0200 Subject: [PATCH 04/45] Add @xyflow/react and elkjs, and the dataflow graph model Co-Authored-By: Claude Sonnet 5 --- .../20260706_dataflow_visualizer_rebuild.md | 4 +- console/package.json | 2 + .../platform/dataflows/dataflowGraph.test.ts | 152 ++++++++++++++++ .../src/platform/dataflows/dataflowGraph.ts | 169 ++++++++++++++++++ console/yarn.lock | 122 ++++++++++++- 5 files changed, 445 insertions(+), 4 deletions(-) create mode 100644 console/src/platform/dataflows/dataflowGraph.test.ts create mode 100644 console/src/platform/dataflows/dataflowGraph.ts diff --git a/console/doc/design/20260706_dataflow_visualizer_rebuild.md b/console/doc/design/20260706_dataflow_visualizer_rebuild.md index a6cb7c49735c9..48f91a45e5219 100644 --- a/console/doc/design/20260706_dataflow_visualizer_rebuild.md +++ b/console/doc/design/20260706_dataflow_visualizer_rebuild.md @@ -164,9 +164,9 @@ The visible-graph derivation is `deriveVisibleGraph(structure, collapseState, fi ### Rendering and layout elkjs runs `elk.layered` in a web worker, direction left to right, over exactly the visible post-collapse graph. -The console builds with Vite, so the worker is loaded via Vite's worker import (`elkjs/lib/elk-worker.min.js?worker`) passed as `workerFactory` to the ELK constructor. +The console builds with Vite, so layout runs in a Vite module worker (`new Worker(new URL("./layout.worker.ts", import.meta.url), { type: "module" })`) that imports `elkjs/lib/elk.bundled.js` and implements the layout request protocol below. Phase 1 verifies this in both `vite dev` and the production build. -If the prebuilt worker script does not survive Vite bundling, the fallback is `elk.bundled.js` on the main thread in a lazily loaded chunk, accepting UI stalls during layout. +If the module worker fails to bundle or run `elk.bundled.js`, the fallback is running elkjs on the main thread in a lazily loaded chunk, accepting UI stalls during layout. Toggling collapse recomputes layout, memoized per collapse state so toggling back is instant. A small "layouting" overlay shows during computation while the canvas stays interactive. diff --git a/console/package.json b/console/package.json index 4b2a72ae49490..633c209842e32 100644 --- a/console/package.json +++ b/console/package.json @@ -72,6 +72,7 @@ "@visx/shape": "^3.12.0", "@visx/tooltip": "^3.12.0", "@xstate/fsm": "^2.1.0", + "@xyflow/react": "^12.11.1", "base32-encoding": "^1.0.0", "buffer": "^6.0.3", "codemirror": "^6.0.1", @@ -82,6 +83,7 @@ "date-fns": "^4.1.0", "date-fns-tz": "^3.2.0", "downshift": "^9.0.8", + "elkjs": "^0.11.1", "fast-deep-equal": "^3.1.3", "framer-motion": "^12.38.0", "jotai": "^2.9.3", diff --git a/console/src/platform/dataflows/dataflowGraph.test.ts b/console/src/platform/dataflows/dataflowGraph.test.ts new file mode 100644 index 0000000000000..4ab9c8b83c22b --- /dev/null +++ b/console/src/platform/dataflows/dataflowGraph.test.ts @@ -0,0 +1,152 @@ +// Copyright Materialize, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +import { describe, expect, it } from "vitest"; + +import { + buildDataflowStructure, + type ChannelRow, + type LirSpanRow, + nodeIdOf, + type OperatorRow, +} from "./dataflowGraph"; + +// Dataflow 5: root [5], region [5,1] with children [5,1,1], [5,1,2], leaf [5,2]. +export const OPS: OperatorRow[] = [ + { + id: "10", + address: ["5"], + name: "Dataflow", + arrangementRecords: null, + arrangementSize: null, + elapsedNs: "0", + }, + { + id: "11", + address: ["5", "1"], + name: "Region", + arrangementRecords: "0", + arrangementSize: "0", + elapsedNs: "5", + }, + { + id: "12", + address: ["5", "1", "1"], + name: "Join", + arrangementRecords: "100", + arrangementSize: "4096", + elapsedNs: "7", + }, + { + id: "13", + address: ["5", "1", "2"], + name: "Map", + arrangementRecords: "0", + arrangementSize: "0", + elapsedNs: "1", + }, + { + id: "14", + address: ["5", "2"], + name: "Sink", + arrangementRecords: "0", + arrangementSize: "0", + elapsedNs: "2", + }, +]; +export const CHANNELS: ChannelRow[] = [ + // into the region: leaf [5,2] <- region output; region input port -> child + { + id: "1", + fromOperatorAddress: ["5", "1", "0"], + fromPort: "0", + toOperatorAddress: ["5", "1", "1"], + toPort: "0", + messagesSent: "3", + batchesSent: "1", + channelType: "rows", + }, + { + id: "2", + fromOperatorAddress: ["5", "1", "1"], + fromPort: "0", + toOperatorAddress: ["5", "1", "2"], + toPort: "0", + messagesSent: "5", + batchesSent: "2", + channelType: "rows", + }, + { + id: "3", + fromOperatorAddress: ["5", "1"], + fromPort: "0", + toOperatorAddress: ["5", "2"], + toPort: "0", + messagesSent: "5", + batchesSent: "2", + channelType: "batches", + }, +]; +export const LIR_SPANS: LirSpanRow[] = [ + { + exportId: "u42", + lirId: "1", + operator: "Join::Differential", + operatorIdStart: "11", + operatorIdEnd: "13", + }, +]; + +describe("buildDataflowStructure", () => { + it("builds the region tree from address prefixes", () => { + const s = buildDataflowStructure(OPS, CHANNELS, LIR_SPANS); + expect(s.root).toEqual(nodeIdOf([5])); + const root = s.nodes.get(s.root)!; + expect(root.children).toEqual([nodeIdOf([5, 1]), nodeIdOf([5, 2])]); + const region = s.nodes.get(nodeIdOf([5, 1]))!; + expect(region.parent).toEqual(s.root); + expect(region.children).toEqual([nodeIdOf([5, 1, 1]), nodeIdOf([5, 1, 2])]); + }); + + it("precomputes transitive stats", () => { + const s = buildDataflowStructure(OPS, CHANNELS, LIR_SPANS); + const region = s.nodes.get(nodeIdOf([5, 1]))!; + expect(region.own.elapsedNs).toEqual(5n); + expect(region.transitive.elapsedNs).toEqual(13n); + expect(region.transitive.arrangementRecords).toEqual(100n); + expect(region.transitive.arrangementSize).toEqual(4096n); + }); + + it("maps LIR spans to operators by id range, as an array", () => { + const s = buildDataflowStructure(OPS, CHANNELS, LIR_SPANS); + expect(s.nodes.get(nodeIdOf([5, 1, 1]))!.lir).toEqual([ + { exportId: "u42", lirId: "1", operator: "Join::Differential" }, + ]); + // end is exclusive + expect(s.nodes.get(nodeIdOf([5, 2]))!.lir).toEqual([]); + }); + + it("throws without exactly one root", () => { + expect(() => buildDataflowStructure([], [], [])).toThrow(); + }); + + it("normalizes channels", () => { + const s = buildDataflowStructure(OPS, CHANNELS, []); + expect(s.channels[0]).toEqual({ + id: 1, + fromAddress: [5, 1, 0], + fromPort: 0, + toAddress: [5, 1, 1], + toPort: 0, + messagesSent: 3n, + batchesSent: 1n, + channelType: "rows", + }); + }); +}); diff --git a/console/src/platform/dataflows/dataflowGraph.ts b/console/src/platform/dataflows/dataflowGraph.ts new file mode 100644 index 0000000000000..c575dfac6ca12 --- /dev/null +++ b/console/src/platform/dataflows/dataflowGraph.ts @@ -0,0 +1,169 @@ +// Copyright Materialize, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +export type Address = number[]; +export type NodeId = string; +export function nodeIdOf(address: Address): NodeId { + return JSON.stringify(address); +} + +export interface NodeStats { + arrangementRecords: bigint; + arrangementSize: bigint; + elapsedNs: bigint; +} + +export interface LirInfo { + exportId: string; + lirId: string; + operator: string; +} + +export interface DataflowNode { + id: NodeId; + address: Address; + name: string; + parent: NodeId | null; + children: NodeId[]; + own: NodeStats; + transitive: NodeStats; // own + subtree, precomputed + lir: LirInfo[]; // one entry per export span covering this operator id +} + +export interface Channel { + id: number; + fromAddress: Address; + fromPort: number; + toAddress: Address; + toPort: number; + messagesSent: bigint; + batchesSent: bigint; + channelType: string | null; +} + +export interface DataflowStructure { + nodes: Map; + root: NodeId; + channels: Channel[]; +} + +// SQL row shapes (values arrive as strings or numbers, normalized here) +export interface OperatorRow { + id: bigint | number | string; + address: string[]; + name: string; + arrangementRecords: bigint | number | string | null; + arrangementSize: bigint | number | string | null; + elapsedNs: bigint | number | string | null; +} +export interface ChannelRow { + id: number | string; + fromOperatorAddress: string[]; + fromPort: number | string; + toOperatorAddress: string[]; + toPort: number | string; + messagesSent: bigint | number | string; + batchesSent: bigint | number | string; + channelType: string | null; +} +export interface LirSpanRow { + exportId: string; + lirId: string; + operator: string; + operatorIdStart: bigint | number | string; + operatorIdEnd: bigint | number | string; +} + +const toBigInt = (v: bigint | number | string | null | undefined): bigint => + v == null ? 0n : BigInt(v); + +export function buildDataflowStructure( + operators: OperatorRow[], + channels: ChannelRow[], + lirSpans: LirSpanRow[], +): DataflowStructure { + const spans = lirSpans.map((s) => ({ + exportId: s.exportId, + lirId: s.lirId, + operator: s.operator, + start: toBigInt(s.operatorIdStart), + end: toBigInt(s.operatorIdEnd), + })); + const nodes = new Map(); + for (const row of operators) { + const address = row.address.map(Number); + const id = nodeIdOf(address); + const opId = toBigInt(row.id); + const own: NodeStats = { + arrangementRecords: toBigInt(row.arrangementRecords), + arrangementSize: toBigInt(row.arrangementSize), + elapsedNs: toBigInt(row.elapsedNs), + }; + nodes.set(id, { + id, + address, + name: row.name, + parent: address.length > 1 ? nodeIdOf(address.slice(0, -1)) : null, + children: [], + own, + transitive: own, // replaced below + lir: spans + .filter((s) => s.start <= opId && opId < s.end) + .map(({ exportId, lirId, operator }) => ({ + exportId, + lirId, + operator, + })), + }); + } + const roots: NodeId[] = []; + for (const node of nodes.values()) { + if (node.parent === null) roots.push(node.id); + else nodes.get(node.parent)?.children.push(node.id); + } + if (roots.length !== 1) { + throw new Error( + `expected exactly one root operator, found ${roots.length}`, + ); + } + // Children in address order so layout and tests are deterministic. + for (const node of nodes.values()) { + node.children.sort((a, b) => { + const [x, y] = [nodes.get(a)!.address, nodes.get(b)!.address]; + return x[x.length - 1] - y[y.length - 1]; + }); + } + const fillTransitive = (id: NodeId): NodeStats => { + const node = nodes.get(id)!; + const t = { ...node.own }; + for (const c of node.children) { + const ct = fillTransitive(c); + t.arrangementRecords += ct.arrangementRecords; + t.arrangementSize += ct.arrangementSize; + t.elapsedNs += ct.elapsedNs; + } + node.transitive = t; + return t; + }; + fillTransitive(roots[0]); + return { + nodes, + root: roots[0], + channels: channels.map((c) => ({ + id: Number(c.id), + fromAddress: c.fromOperatorAddress.map(Number), + fromPort: Number(c.fromPort), + toAddress: c.toOperatorAddress.map(Number), + toPort: Number(c.toPort), + messagesSent: toBigInt(c.messagesSent), + batchesSent: toBigInt(c.batchesSent), + channelType: c.channelType, + })), + }; +} diff --git a/console/yarn.lock b/console/yarn.lock index bda1f83709b4f..ee09c48b756e5 100644 --- a/console/yarn.lock +++ b/console/yarn.lock @@ -4491,6 +4491,15 @@ __metadata: languageName: node linkType: hard +"@types/d3-drag@npm:^3.0.7": + version: 3.0.7 + resolution: "@types/d3-drag@npm:3.0.7" + dependencies: + "@types/d3-selection": "npm:*" + checksum: 10c0/65e29fa32a87c72d26c44b5e2df3bf15af21cd128386bcc05bcacca255927c0397d0cd7e6062aed5f0abd623490544a9d061c195f5ed9f018fe0b698d99c079d + languageName: node + linkType: hard + "@types/d3-dsv@npm:*": version: 3.0.1 resolution: "@types/d3-dsv@npm:3.0.1" @@ -4582,6 +4591,15 @@ __metadata: languageName: node linkType: hard +"@types/d3-interpolate@npm:^3.0.4": + version: 3.0.4 + resolution: "@types/d3-interpolate@npm:3.0.4" + dependencies: + "@types/d3-color": "npm:*" + checksum: 10c0/066ebb8da570b518dd332df6b12ae3b1eaa0a7f4f0c702e3c57f812cf529cc3500ec2aac8dc094f31897790346c6b1ebd8cd7a077176727f4860c2b181a65ca4 + languageName: node + linkType: hard + "@types/d3-path@npm:*": version: 3.0.0 resolution: "@types/d3-path@npm:3.0.0" @@ -4656,6 +4674,13 @@ __metadata: languageName: node linkType: hard +"@types/d3-selection@npm:^3.0.10": + version: 3.0.11 + resolution: "@types/d3-selection@npm:3.0.11" + checksum: 10c0/0c512956c7503ff5def4bb32e0c568cc757b9a2cc400a104fc0f4cfe5e56d83ebde2a97821b6f2cb26a7148079d3b86a2f28e11d68324ed311cf35c2ed980d1d + languageName: node + linkType: hard + "@types/d3-shape@npm:*": version: 3.1.1 resolution: "@types/d3-shape@npm:3.1.1" @@ -4720,6 +4745,15 @@ __metadata: languageName: node linkType: hard +"@types/d3-transition@npm:^3.0.8": + version: 3.0.9 + resolution: "@types/d3-transition@npm:3.0.9" + dependencies: + "@types/d3-selection": "npm:*" + checksum: 10c0/4f68b9df7ac745b3491216c54203cbbfa0f117ae4c60e2609cdef2db963582152035407fdff995b10ee383bae2f05b7743493f48e1b8e46df54faa836a8fb7b5 + languageName: node + linkType: hard + "@types/d3-zoom@npm:*": version: 3.0.2 resolution: "@types/d3-zoom@npm:3.0.2" @@ -4740,6 +4774,16 @@ __metadata: languageName: node linkType: hard +"@types/d3-zoom@npm:^3.0.8": + version: 3.0.8 + resolution: "@types/d3-zoom@npm:3.0.8" + dependencies: + "@types/d3-interpolate": "npm:*" + "@types/d3-selection": "npm:*" + checksum: 10c0/1dbdbcafddcae12efb5beb6948546963f29599e18bc7f2a91fb69cc617c2299a65354f2d47e282dfb86fec0968406cd4fb7f76ba2d2fb67baa8e8d146eb4a547 + languageName: node + linkType: hard + "@types/d3@npm:^7.4.3": version: 7.4.3 resolution: "@types/d3@npm:7.4.3" @@ -5556,6 +5600,44 @@ __metadata: languageName: node linkType: hard +"@xyflow/react@npm:^12.11.1": + version: 12.11.1 + resolution: "@xyflow/react@npm:12.11.1" + dependencies: + "@xyflow/system": "npm:0.0.78" + classcat: "npm:^5.0.3" + zustand: "npm:^4.4.0" + peerDependencies: + "@types/react": ">=17" + "@types/react-dom": ">=17" + react: ">=17" + react-dom: ">=17" + peerDependenciesMeta: + "@types/react": + optional: true + "@types/react-dom": + optional: true + checksum: 10c0/5774a01a28289effd8f5f3427265e1ddb34911557efbd2bc82d3648641369e5a68d56d992596980bfc2c95ea64252ac983b82412dad1cb2f2175a38b9cefeb68 + languageName: node + linkType: hard + +"@xyflow/system@npm:0.0.78": + version: 0.0.78 + resolution: "@xyflow/system@npm:0.0.78" + dependencies: + "@types/d3-drag": "npm:^3.0.7" + "@types/d3-interpolate": "npm:^3.0.4" + "@types/d3-selection": "npm:^3.0.10" + "@types/d3-transition": "npm:^3.0.8" + "@types/d3-zoom": "npm:^3.0.8" + d3-drag: "npm:^3.0.0" + d3-interpolate: "npm:^3.0.1" + d3-selection: "npm:^3.0.0" + d3-zoom: "npm:^3.0.0" + checksum: 10c0/1c4b497b5fcc86457a5b5120371223131036bbc3aa0ed0b0de5dac7f5db9f6198312839f2363a4a18c2bb0ba0dfe6ea4bf92701913a2a289d74da59da07b6a96 + languageName: node + linkType: hard + "@zag-js/dom-query@npm:0.16.0": version: 0.16.0 resolution: "@zag-js/dom-query@npm:0.16.0" @@ -6352,6 +6434,13 @@ __metadata: languageName: node linkType: hard +"classcat@npm:^5.0.3": + version: 5.0.5 + resolution: "classcat@npm:5.0.5" + checksum: 10c0/ff8d273055ef9b518529cfe80fd0486f7057a9917373807ff802d75ceb46e8f8e148f41fa094ee7625c8f34642cfaa98395ff182d9519898da7cbf383d4a210d + languageName: node + linkType: hard + "classnames@npm:^2.3.1": version: 2.3.2 resolution: "classnames@npm:2.3.2" @@ -6862,7 +6951,7 @@ __metadata: languageName: node linkType: hard -"d3-drag@npm:2 - 3, d3-drag@npm:3": +"d3-drag@npm:2 - 3, d3-drag@npm:3, d3-drag@npm:^3.0.0": version: 3.0.0 resolution: "d3-drag@npm:3.0.0" dependencies: @@ -7028,7 +7117,7 @@ __metadata: languageName: node linkType: hard -"d3-selection@npm:2 - 3, d3-selection@npm:3": +"d3-selection@npm:2 - 3, d3-selection@npm:3, d3-selection@npm:^3.0.0": version: 3.0.0 resolution: "d3-selection@npm:3.0.0" checksum: 10c0/e59096bbe8f0cb0daa1001d9bdd6dbc93a688019abc97d1d8b37f85cd3c286a6875b22adea0931b0c88410d025563e1643019161a883c516acf50c190a11b56b @@ -7591,6 +7680,13 @@ __metadata: languageName: node linkType: hard +"elkjs@npm:^0.11.1": + version: 0.11.1 + resolution: "elkjs@npm:0.11.1" + checksum: 10c0/e2216a7439c9dc65413e5f8d9e46bf829c32dacbfd869585bae8f8775b3cd5802d0ac9ccf0f45ad14decfa622244a092533263cc3a596e6d82e8247bf6bf84d7 + languageName: node + linkType: hard + "emoji-regex@npm:^10.3.0": version: 10.3.0 resolution: "emoji-regex@npm:10.3.0" @@ -10986,6 +11082,7 @@ __metadata: "@visx/tooltip": "npm:^3.12.0" "@vitejs/plugin-react": "npm:^4.5.0" "@xstate/fsm": "npm:^2.1.0" + "@xyflow/react": "npm:^12.11.1" base32-encoding: "npm:^1.0.0" browserslist-to-esbuild: "npm:^2.1.1" buffer: "npm:^6.0.3" @@ -11001,6 +11098,7 @@ __metadata: debug: "npm:^4.3.7" dedent: "npm:^1.7.2" downshift: "npm:^9.0.8" + elkjs: "npm:^0.11.1" eslint: "npm:^8.57.0" eslint-config-prettier: "npm:^9.1.0" eslint-plugin-import: "npm:^2.31.0" @@ -15681,3 +15779,23 @@ __metadata: checksum: 10c0/8f14c87d6b1b53c944c25ce7a28616896319d95bc46a9660fe441adc0ed0a81253b02b5abdaeffedbeb23bdd25a0bf1c29d2c12dd919aef6447652dd295e3e69 languageName: node linkType: hard + +"zustand@npm:^4.4.0": + version: 4.5.7 + resolution: "zustand@npm:4.5.7" + dependencies: + use-sync-external-store: "npm:^1.2.2" + peerDependencies: + "@types/react": ">=16.8" + immer: ">=9.0.6" + react: ">=16.8" + peerDependenciesMeta: + "@types/react": + optional: true + immer: + optional: true + react: + optional: true + checksum: 10c0/55559e37a82f0c06cadc61cb08f08314c0fe05d6a93815e41e3376130c13db22a5017cbb0cd1f018c82f2dad0051afe3592561d40f980bd4082e32005e8a950c + languageName: node + linkType: hard From 67f6b715cf13575647165589e20070707ada2310 Mon Sep 17 00:00:00 2001 From: Moritz Hoffmann Date: Wed, 8 Jul 2026 22:42:07 +0200 Subject: [PATCH 05/45] Derive the visible dataflow graph from collapse state, add the elk layout worker/hook Co-Authored-By: Claude Sonnet 5 --- .../platform/dataflows/dataflowGraph.test.ts | 77 +++++++++ .../src/platform/dataflows/dataflowGraph.ts | 163 ++++++++++++++++++ .../src/platform/dataflows/elkGraph.test.ts | 45 +++++ console/src/platform/dataflows/elkGraph.ts | 84 +++++++++ .../src/platform/dataflows/layout.worker.ts | 43 +++++ .../src/platform/dataflows/useElkLayout.ts | 72 ++++++++ 6 files changed, 484 insertions(+) create mode 100644 console/src/platform/dataflows/elkGraph.test.ts create mode 100644 console/src/platform/dataflows/elkGraph.ts create mode 100644 console/src/platform/dataflows/layout.worker.ts create mode 100644 console/src/platform/dataflows/useElkLayout.ts diff --git a/console/src/platform/dataflows/dataflowGraph.test.ts b/console/src/platform/dataflows/dataflowGraph.test.ts index 4ab9c8b83c22b..b053db1ec5d54 100644 --- a/console/src/platform/dataflows/dataflowGraph.test.ts +++ b/console/src/platform/dataflows/dataflowGraph.test.ts @@ -12,9 +12,13 @@ import { describe, expect, it } from "vitest"; import { buildDataflowStructure, type ChannelRow, + defaultCollapseState, + deriveVisibleGraph, type LirSpanRow, + MAX_VISIBLE_NODES, nodeIdOf, type OperatorRow, + visibleNodeCount, } from "./dataflowGraph"; // Dataflow 5: root [5], region [5,1] with children [5,1,1], [5,1,2], leaf [5,2]. @@ -150,3 +154,76 @@ describe("buildDataflowStructure", () => { }); }); }); + +describe("deriveVisibleGraph", () => { + const s = buildDataflowStructure(OPS, CHANNELS, LIR_SPANS); + const regionId = nodeIdOf([5, 1]); + + it("collapses regions to a single node with remapped edges", () => { + const g = deriveVisibleGraph(s, new Set([regionId])); + expect(g.nodes.map((n) => [n.id, n.kind])).toEqual([ + [regionId, "collapsedRegion"], + [nodeIdOf([5, 2]), "operator"], + ]); + // channel 3 region -> sink survives, channels 1 and 2 are internal + expect(g.edges).toEqual([ + { + id: `${regionId}=>${nodeIdOf([5, 2])}`, + source: regionId, + target: nodeIdOf([5, 2]), + messagesSent: 5n, + batchesSent: 2n, + channelTypes: ["batches"], + }, + ]); + const collapsed = g.nodes[0]; + expect(collapsed.stats).toEqual(s.nodes.get(regionId)!.transitive); + expect(collapsed.childCount).toEqual(2); + }); + + it("expands regions with port pseudo-nodes, parents before children", () => { + const g = deriveVisibleGraph(s, new Set()); + const ids = g.nodes.map((n) => n.id); + expect(ids.indexOf(regionId)).toBeLessThan( + ids.indexOf(nodeIdOf([5, 1, 1])), + ); + const port = g.nodes.find((n) => n.kind === "port")!; + expect(port.id).toEqual(`${regionId}:in:0`); + expect(port.parent).toEqual(regionId); + expect(port.label).toEqual("input 0"); + // port -> Join edge preserved + expect(g.edges.map((e) => e.id)).toContain( + `${regionId}:in:0=>${nodeIdOf([5, 1, 1])}`, + ); + }); + + it("aggregates parallel channels between the same visible pair", () => { + const extra = buildDataflowStructure( + OPS, + [ + ...CHANNELS, + { + id: "4", + fromOperatorAddress: ["5", "1"], + fromPort: "1", + toOperatorAddress: ["5", "2"], + toPort: "1", + messagesSent: "7", + batchesSent: "1", + channelType: "rows", + }, + ], + [], + ); + const g = deriveVisibleGraph(extra, new Set([regionId])); + const e = g.edges.find((edge) => edge.target === nodeIdOf([5, 2]))!; + expect(e.messagesSent).toEqual(12n); + expect(e.channelTypes).toEqual(["batches", "rows"]); + }); + + it("defaultCollapseState collapses all non-root regions", () => { + expect(defaultCollapseState(s)).toEqual(new Set([regionId])); + expect(visibleNodeCount(s, defaultCollapseState(s))).toEqual(2); + expect(MAX_VISIBLE_NODES).toEqual(1500); + }); +}); diff --git a/console/src/platform/dataflows/dataflowGraph.ts b/console/src/platform/dataflows/dataflowGraph.ts index c575dfac6ca12..790761dddeb26 100644 --- a/console/src/platform/dataflows/dataflowGraph.ts +++ b/console/src/platform/dataflows/dataflowGraph.ts @@ -53,6 +53,34 @@ export interface DataflowStructure { channels: Channel[]; } +export type CollapseState = ReadonlySet; + +export interface VisibleNode { + id: string; + kind: "operator" | "region" | "collapsedRegion" | "port"; + label: string; + parent: string | null; + stats: NodeStats | null; + transitive: NodeStats | null; + childCount: number; + lir: LirInfo[]; + address: Address | null; +} + +export interface VisibleEdge { + id: string; + source: string; + target: string; + messagesSent: bigint; + batchesSent: bigint; + channelTypes: string[]; +} + +export interface VisibleGraph { + nodes: VisibleNode[]; + edges: VisibleEdge[]; +} + // SQL row shapes (values arrive as strings or numbers, normalized here) export interface OperatorRow { id: bigint | number | string; @@ -167,3 +195,138 @@ export function buildDataflowStructure( })), }; } + +export const MAX_VISIBLE_NODES = 1500; + +// Shallowest collapsed ancestor absorbs everything beneath it. +function representative(address: Address, collapsed: CollapseState): NodeId { + for (let len = 1; len < address.length; len++) { + const prefix = nodeIdOf(address.slice(0, len)); + if (collapsed.has(prefix)) return prefix; + } + return nodeIdOf(address); +} + +// A channel endpoint address ending in 0 denotes the enclosing scope's +// input (from side) or output (to side) port. +function endpointId( + address: Address, + port: number, + side: "from" | "to", + collapsed: CollapseState, +): { + id: string; + port?: { scope: NodeId; direction: "input" | "output"; port: number }; +} { + if (address[address.length - 1] === 0) { + const scope = address.slice(0, -1); + const scopeId = nodeIdOf(scope); + const rep = scope.length === 0 ? scopeId : representative(scope, collapsed); + if (rep !== scopeId || collapsed.has(scopeId)) return { id: rep }; + const direction = side === "from" ? "input" : "output"; + return { + id: `${scopeId}:${direction === "input" ? "in" : "out"}:${port}`, + port: { scope: scopeId, direction, port }, + }; + } + return { id: representative(address, collapsed) }; +} + +export function deriveVisibleGraph( + structure: DataflowStructure, + collapsed: CollapseState, +): VisibleGraph { + const nodes: VisibleNode[] = []; + const emit = (id: NodeId, parent: string | null) => { + const node = structure.nodes.get(id)!; + const isCollapsed = collapsed.has(id) && node.children.length > 0; + const kind = + node.children.length === 0 + ? "operator" + : isCollapsed + ? "collapsedRegion" + : "region"; + nodes.push({ + id, + kind, + label: node.name, + parent, + stats: kind === "collapsedRegion" ? node.transitive : node.own, + transitive: node.transitive, + childCount: node.children.length, + lir: node.lir, + address: node.address, + }); + if (!isCollapsed) for (const c of node.children) emit(c, id); + }; + for (const c of structure.nodes.get(structure.root)!.children) emit(c, null); + + const edgesById = new Map }>(); + const ports = new Map(); + for (const ch of structure.channels) { + const from = endpointId(ch.fromAddress, ch.fromPort, "from", collapsed); + const to = endpointId(ch.toAddress, ch.toPort, "to", collapsed); + if (from.id === to.id) continue; + for (const p of [from.port, to.port]) { + if ( + p && + !ports.has( + `${p.scope}:${p.direction === "input" ? "in" : "out"}:${p.port}`, + ) + ) { + const id = `${p.scope}:${p.direction === "input" ? "in" : "out"}:${p.port}`; + ports.set(id, { + id, + kind: "port", + label: `${p.direction} ${p.port}`, + parent: p.scope === structure.root ? null : p.scope, + stats: null, + transitive: null, + childCount: 0, + lir: [], + address: null, + }); + } + } + const id = `${from.id}=>${to.id}`; + const existing = edgesById.get(id); + if (existing) { + existing.messagesSent += ch.messagesSent; + existing.batchesSent += ch.batchesSent; + if (ch.channelType) existing.typeSet.add(ch.channelType); + } else { + edgesById.set(id, { + id, + source: from.id, + target: to.id, + messagesSent: ch.messagesSent, + batchesSent: ch.batchesSent, + channelTypes: [], + typeSet: new Set(ch.channelType ? [ch.channelType] : []), + }); + } + } + const edges = [...edgesById.values()].map(({ typeSet, ...e }) => ({ + ...e, + channelTypes: [...typeSet].sort(), + })); + return { nodes: [...nodes, ...ports.values()], edges }; +} + +export function defaultCollapseState( + structure: DataflowStructure, +): CollapseState { + const collapsed = new Set(); + for (const node of structure.nodes.values()) { + if (node.children.length > 0 && node.id !== structure.root) + collapsed.add(node.id); + } + return collapsed; +} + +export function visibleNodeCount( + structure: DataflowStructure, + collapsed: CollapseState, +): number { + return deriveVisibleGraph(structure, collapsed).nodes.length; +} diff --git a/console/src/platform/dataflows/elkGraph.test.ts b/console/src/platform/dataflows/elkGraph.test.ts new file mode 100644 index 0000000000000..4f6881d3edbb2 --- /dev/null +++ b/console/src/platform/dataflows/elkGraph.test.ts @@ -0,0 +1,45 @@ +// Copyright Materialize, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +import { describe, expect, it } from "vitest"; + +import { + buildDataflowStructure, + deriveVisibleGraph, + nodeIdOf, +} from "./dataflowGraph"; +import { CHANNELS, LIR_SPANS, OPS } from "./dataflowGraph.test"; +import { extractPositions, NODE_DIMENSIONS, toElkGraph } from "./elkGraph"; + +describe("toElkGraph", () => { + const s = buildDataflowStructure(OPS, CHANNELS, LIR_SPANS); + + it("nests elk children by visible parent", () => { + const elk = toElkGraph(deriveVisibleGraph(s, new Set())); + const region = elk.children!.find((c) => c.id === nodeIdOf([5, 1]))!; + expect(region.children!.map((c) => c.id)).toEqual( + expect.arrayContaining([ + nodeIdOf([5, 1, 1]), + nodeIdOf([5, 1, 2]), + `${nodeIdOf([5, 1])}:in:0`, + ]), + ); + const leaf = region.children!.find((c) => c.id === nodeIdOf([5, 1, 1]))!; + expect(leaf.width).toEqual(NODE_DIMENSIONS.operator.width); + }); + + it("round-trips positions relative to parent", () => { + const elk = toElkGraph(deriveVisibleGraph(s, new Set())); + // simulate a layout result + elk.children![0].x = 10; + elk.children![0].y = 20; + const positions = extractPositions(elk); + expect(positions[elk.children![0].id]).toMatchObject({ x: 10, y: 20 }); + }); +}); diff --git a/console/src/platform/dataflows/elkGraph.ts b/console/src/platform/dataflows/elkGraph.ts new file mode 100644 index 0000000000000..6979962f55f35 --- /dev/null +++ b/console/src/platform/dataflows/elkGraph.ts @@ -0,0 +1,84 @@ +// Copyright Materialize, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +import type { ElkNode } from "elkjs/lib/elk-api"; + +import type { VisibleGraph, VisibleNode } from "./dataflowGraph"; + +export interface NodePosition { + x: number; + y: number; + width: number; + height: number; +} + +export type Positions = Record; + +export const NODE_DIMENSIONS: Record< + VisibleNode["kind"], + { width: number; height: number } +> = { + operator: { width: 240, height: 72 }, + collapsedRegion: { width: 260, height: 88 }, + region: { width: 0, height: 0 }, + port: { width: 90, height: 24 }, +}; + +const LAYOUT_OPTIONS: Record = { + "elk.algorithm": "layered", + "elk.direction": "RIGHT", + "elk.hierarchyHandling": "INCLUDE_CHILDREN", + "elk.padding": "[top=48,left=16,bottom=16,right=16]", + "elk.spacing.nodeNode": "24", + "elk.layered.spacing.nodeNodeBetweenLayers": "48", +}; + +export function toElkGraph(graph: VisibleGraph): ElkNode { + const elkNodes = new Map(); + const root: ElkNode = { + id: "__root__", + layoutOptions: LAYOUT_OPTIONS, + children: [], + edges: [], + }; + for (const node of graph.nodes) { + const dims = NODE_DIMENSIONS[node.kind]; + const elkNode: ElkNode = { + id: node.id, + children: [], + ...(node.kind === "region" ? { layoutOptions: LAYOUT_OPTIONS } : dims), + }; + elkNodes.set(node.id, elkNode); + const parent = node.parent ? elkNodes.get(node.parent)! : root; + parent.children!.push(elkNode); + } + root.edges = graph.edges.map((e) => ({ + id: e.id, + sources: [e.source], + targets: [e.target], + })); + return root; +} + +export function extractPositions(layouted: ElkNode): Positions { + const positions: Positions = {}; + const walk = (node: ElkNode) => { + for (const child of node.children ?? []) { + positions[child.id] = { + x: child.x ?? 0, + y: child.y ?? 0, + width: child.width ?? 0, + height: child.height ?? 0, + }; + walk(child); + } + }; + walk(layouted); + return positions; +} diff --git a/console/src/platform/dataflows/layout.worker.ts b/console/src/platform/dataflows/layout.worker.ts new file mode 100644 index 0000000000000..f89e4998ea339 --- /dev/null +++ b/console/src/platform/dataflows/layout.worker.ts @@ -0,0 +1,43 @@ +// Copyright Materialize, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +import ELK from "elkjs/lib/elk.bundled.js"; + +import type { VisibleGraph } from "./dataflowGraph"; +import { extractPositions, type Positions, toElkGraph } from "./elkGraph"; + +export interface LayoutRequest { + requestId: number; + graph: VisibleGraph; +} + +export interface LayoutResponse { + requestId: number; + positions?: Positions; + error?: string; +} + +const elk = new ELK(); + +self.onmessage = async (event: MessageEvent) => { + const { requestId, graph } = event.data; + try { + const layouted = await elk.layout(toElkGraph(graph)); + const response: LayoutResponse = { + requestId, + positions: extractPositions(layouted), + }; + self.postMessage(response); + } catch (error) { + self.postMessage({ + requestId, + error: String(error), + } satisfies LayoutResponse); + } +}; diff --git a/console/src/platform/dataflows/useElkLayout.ts b/console/src/platform/dataflows/useElkLayout.ts new file mode 100644 index 0000000000000..1baa4326521fa --- /dev/null +++ b/console/src/platform/dataflows/useElkLayout.ts @@ -0,0 +1,72 @@ +// Copyright Materialize, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +import React from "react"; + +import type { VisibleGraph } from "./dataflowGraph"; +import type { Positions } from "./elkGraph"; +import type { LayoutRequest, LayoutResponse } from "./layout.worker"; + +export function useElkLayout(graph: VisibleGraph | null, cacheKey: string) { + const workerRef = React.useRef(null); + const requestIdRef = React.useRef(0); + const cacheRef = React.useRef(new Map()); + const [state, setState] = React.useState<{ + key: string | null; + positions: Positions | null; + error: string | null; + }>({ key: null, positions: null, error: null }); + + React.useEffect(() => { + const worker = new Worker(new URL("./layout.worker.ts", import.meta.url), { + type: "module", + }); + workerRef.current = worker; + return () => worker.terminate(); + }, []); + + React.useEffect(() => { + if (!graph) return; + const cached = cacheRef.current.get(cacheKey); + if (cached) { + setState({ key: cacheKey, positions: cached, error: null }); + return; + } + const requestId = ++requestIdRef.current; + const worker = workerRef.current; + if (!worker) return; + const onMessage = (event: MessageEvent) => { + // Drop responses for superseded requests. + if (event.data.requestId !== requestIdRef.current) return; + if (event.data.positions) { + cacheRef.current.set(cacheKey, event.data.positions); + setState({ + key: cacheKey, + positions: event.data.positions, + error: null, + }); + } else { + setState({ + key: cacheKey, + positions: null, + error: event.data.error ?? "layout failed", + }); + } + }; + worker.addEventListener("message", onMessage); + worker.postMessage({ requestId, graph } satisfies LayoutRequest); + return () => worker.removeEventListener("message", onMessage); + }, [graph, cacheKey]); + + return { + positions: state.key === cacheKey ? state.positions : null, + layouting: graph !== null && state.key !== cacheKey, + error: state.key === cacheKey ? state.error : null, + }; +} From 0981d7dd1c4a70a7198db61be4f390d01e340200 Mon Sep 17 00:00:00 2001 From: Moritz Hoffmann Date: Wed, 8 Jul 2026 22:42:07 +0200 Subject: [PATCH 06/45] console: add dataflow graph data hook keyed by dataflow id Co-Authored-By: Claude Sonnet 5 --- .../dataflow/useDataflowGraphData.test.ts | 139 +++++++++++++++ .../dataflow/useDataflowGraphData.ts | 158 ++++++++++++++++++ 2 files changed, 297 insertions(+) create mode 100644 console/src/api/materialize/dataflow/useDataflowGraphData.test.ts create mode 100644 console/src/api/materialize/dataflow/useDataflowGraphData.ts diff --git a/console/src/api/materialize/dataflow/useDataflowGraphData.test.ts b/console/src/api/materialize/dataflow/useDataflowGraphData.test.ts new file mode 100644 index 0000000000000..4d80de8c5d459 --- /dev/null +++ b/console/src/api/materialize/dataflow/useDataflowGraphData.test.ts @@ -0,0 +1,139 @@ +// Copyright Materialize, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +import { renderHook, waitFor } from "@testing-library/react"; +import { http, HttpResponse } from "msw"; + +import { ErrorCode } from "~/api/materialize/types"; +import server from "~/api/mocks/server"; +import { createProviderWrapper } from "~/test/utils"; + +import { useDataflowGraphData } from "./useDataflowGraphData"; + +const FAILING_REPLICA = "r2"; +const okResult = { + desc: { columns: [] }, + rows: [], +}; +// operators result with one root row so buildDataflowStructure succeeds +const operatorsResult = { + desc: { + columns: [ + { name: "id" }, + { name: "address" }, + { name: "name" }, + { name: "arrangementRecords" }, + { name: "arrangementSize" }, + { name: "elapsedNs" }, + ], + }, + rows: [["10", ["7"], "Dataflow", "0", "0", "0"]], +}; + +beforeEach(() => { + server.use( + http.post("*/api/sql", async ({ request }) => { + const options = JSON.parse( + new URL(request.url).searchParams.get("options") ?? "{}", + ); + if (options.cluster_replica === FAILING_REPLICA) { + return HttpResponse.json({ + results: [ + { + error: { + message: "no such replica", + code: ErrorCode.INTERNAL_ERROR, + }, + }, + ], + }); + } + return HttpResponse.json({ + results: [operatorsResult, okResult, okResult], + }); + }), + ); +}); + +describe("useDataflowGraphData", () => { + it("clears data when a refetch for new params fails", async () => { + // The hook reads the current environment off a jotai atom that resolves + // asynchronously, so it needs the same Suspense + JotaiProvider wrapper + // the app renders under. Without it the atom never resolves against the + // fake environment set up in vitest.setup.ts and the query never fires. + const Wrapper = await createProviderWrapper(); + const { result, rerender } = renderHook( + (params) => useDataflowGraphData(params), + { + wrapper: Wrapper, + initialProps: { + clusterName: "c", + replicaName: "r1", + dataflowId: "7", + } as + | { clusterName: string; replicaName: string; dataflowId: string } + | undefined, + }, + ); + await waitFor(() => expect(result.current.data).not.toBeNull()); + + rerender({ + clusterName: "c", + replicaName: FAILING_REPLICA, + dataflowId: "7", + }); + await waitFor(() => expect(result.current.error).toBeTruthy()); + // The retained previous result must not surface for the new params. + expect(result.current.data).toBeNull(); + }); + + it("hides the previous graph while a new selection is loading", async () => { + const Wrapper = await createProviderWrapper(); + const { result, rerender } = renderHook( + (params) => useDataflowGraphData(params), + { + wrapper: Wrapper, + initialProps: { + clusterName: "c", + replicaName: "r1", + dataflowId: "7", + } as + | { clusterName: string; replicaName: string; dataflowId: string } + | undefined, + }, + ); + await waitFor(() => expect(result.current.data).not.toBeNull()); + const firstStructure = result.current.data?.structure; + + // Switch to a different valid dataflow whose fetch also succeeds. The + // previous fetch's results are still resident, so a naive guard would tag + // them with the new key and render the old graph. React flushes the + // render-phase reset synchronously, so data must already be null here. + rerender({ clusterName: "c", replicaName: "r1", dataflowId: "8" }); + expect(result.current.data).toBeNull(); + + await waitFor(() => expect(result.current.data).not.toBeNull()); + expect(result.current.data?.structure).not.toBe(firstStructure); + }); + + it("rejects a non-numeric dataflow id", async () => { + const Wrapper = await createProviderWrapper(); + expect(() => + renderHook( + () => + useDataflowGraphData({ + clusterName: "c", + replicaName: "r1", + dataflowId: "7; DROP", + }), + { wrapper: Wrapper }, + ), + ).toThrow(); + }); +}); diff --git a/console/src/api/materialize/dataflow/useDataflowGraphData.ts b/console/src/api/materialize/dataflow/useDataflowGraphData.ts new file mode 100644 index 0000000000000..c9c08cf301304 --- /dev/null +++ b/console/src/api/materialize/dataflow/useDataflowGraphData.ts @@ -0,0 +1,158 @@ +// Copyright Materialize, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +import { sql } from "kysely"; +import React from "react"; + +import { escapedLiteral as lit, useSqlManyTyped } from "~/api/materialize"; +import { + buildDataflowStructure, + type ChannelRow, + type DataflowStructure, + type LirSpanRow, + type OperatorRow, +} from "~/platform/dataflows/dataflowGraph"; + +import { queryBuilder } from "../db"; + +export interface DataflowGraphParams { + clusterName: string; + replicaName: string; + dataflowId: string; +} + +function dataflowIdLiteral(dataflowId: string) { + if (!/^\d+$/.test(dataflowId)) { + throw new Error(`invalid dataflow id: ${dataflowId}`); + } + return sql`${lit(dataflowId)}::uint8`; +} + +export function useDataflowGraphData(params?: DataflowGraphParams) { + // Read primitives so an inline params object from a caller does not + // recompile the queries (and refetch) on every render. Only the dataflow id + // is interpolated into the SQL. Cluster and replica reach the request + // through the options below, so the query text does not depend on them. + const dataflowId = params?.dataflowId; + const queries = React.useMemo(() => { + if (dataflowId === undefined) return null; + const id = dataflowIdLiteral(dataflowId); + return { + operators: sql` + SELECT + mdod.id, + mda.address, + mdod.name, + coalesce(mas.records, 0) AS "arrangementRecords", + coalesce(mas.size, 0) AS "arrangementSize", + coalesce(mse.elapsed_ns, 0) AS "elapsedNs" + FROM mz_dataflow_operator_dataflows AS mdod + JOIN mz_dataflow_addresses AS mda ON mda.id = mdod.id + LEFT JOIN mz_arrangement_sizes AS mas ON mas.operator_id = mdod.id + LEFT JOIN mz_scheduling_elapsed AS mse ON mse.id = mdod.id + WHERE mdod.dataflow_id = ${id}` + .$castTo() + .compile(queryBuilder), + channels: sql` + SELECT + mdco.id, + from_operator_address AS "fromOperatorAddress", + from_port AS "fromPort", + to_operator_address AS "toOperatorAddress", + to_port AS "toPort", + COALESCE(sum(mmc.sent), 0) AS "messagesSent", + COALESCE(sum(mmc.batch_sent), 0) AS "batchesSent", + mdco.type AS "channelType" + FROM mz_dataflow_channel_operators AS mdco + LEFT JOIN mz_message_counts AS mmc ON mdco.id = mmc.channel_id + WHERE from_operator_address[1] = ${id} + GROUP BY + mdco.id, "fromOperatorAddress", "fromPort", + "toOperatorAddress", "toPort", mdco.type` + .$castTo() + .compile(queryBuilder), + lirSpans: sql` + SELECT + mce.export_id AS "exportId", + mlm.lir_id::text AS "lirId", + mlm.operator, + mlm.operator_id_start AS "operatorIdStart", + mlm.operator_id_end AS "operatorIdEnd" + FROM mz_lir_mapping AS mlm + JOIN mz_compute_exports AS mce ON mlm.global_id = mce.export_id + WHERE mce.dataflow_id = ${id}` + .$castTo() + .compile(queryBuilder), + }; + }, [dataflowId]); + + const { results, error, databaseError, loading, refetch } = useSqlManyTyped( + queries, + { + cluster: params?.clusterName, + replica: params?.replicaName, + // This query can be slow for large dataflows. + timeout: 30_000, + }, + ); + + // JSON tuple, not a delimiter-joined string. Cluster and replica names are + // identifiers that can themselves contain any delimiter we might pick. + const key = params + ? JSON.stringify([ + params.clusterName, + params.replicaName, + params.dataflowId, + ]) + : null; + const [lastGood, setLastGood] = React.useState<{ + key: string; + data: { structure: DataflowStructure; fetchedAt: Date }; + } | null>(null); + + // Whether a fetch has actually started under the current key. Guards against + // tagging the previous fetch's still-resident results with the new key in + // the render after the selection changes but before useSqlMany's runSql + // effect flips loading. Without it the old graph would show under the new + // selection until the new fetch resolves (CNS-109). + const sawLoadingRef = React.useRef(false); + + // Reset acceptance synchronously when the selection changes. Using the + // render-phase state-adjustment pattern so the reset lands before any effect + // runs, not one render later. + const [trackedKey, setTrackedKey] = React.useState(key); + if (key !== trackedKey) { + setTrackedKey(key); + sawLoadingRef.current = false; + } + + // Tag successful results with the key they were fetched for, but only once a + // fetch has started under that key. + React.useEffect(() => { + if (loading) sawLoadingRef.current = true; + if (key && results && !error && !loading && sawLoadingRef.current) { + setLastGood({ + key, + data: { + structure: buildDataflowStructure( + results.operators ?? [], + results.channels ?? [], + results.lirSpans ?? [], + ), + fetchedAt: new Date(), + }, + }); + } + }, [key, results, error, loading]); + + // Error always wins, and data for other params never renders. + const data = + !error && lastGood && lastGood.key === key ? lastGood.data : null; + return { data, error, databaseError, loading, refetch }; +} From b498ac5b5a556713deed60d5f3ada93596720cc3 Mon Sep 17 00:00:00 2001 From: Moritz Hoffmann Date: Wed, 8 Jul 2026 22:42:07 +0200 Subject: [PATCH 07/45] console: React Flow dataflow graph view with collapsible regions Co-Authored-By: Claude Sonnet 5 --- .../src/platform/dataflows/ChannelEdge.tsx | 62 ++++++ .../platform/dataflows/DataflowGraphView.tsx | 200 ++++++++++++++++++ .../src/platform/dataflows/dataflowGraph.ts | 10 + console/src/platform/dataflows/nodeStyle.ts | 35 +++ console/src/platform/dataflows/nodes.tsx | 123 +++++++++++ console/src/test/mockReactFlow.tsx | 45 ++++ 6 files changed, 475 insertions(+) create mode 100644 console/src/platform/dataflows/ChannelEdge.tsx create mode 100644 console/src/platform/dataflows/DataflowGraphView.tsx create mode 100644 console/src/platform/dataflows/nodeStyle.ts create mode 100644 console/src/platform/dataflows/nodes.tsx create mode 100644 console/src/test/mockReactFlow.tsx diff --git a/console/src/platform/dataflows/ChannelEdge.tsx b/console/src/platform/dataflows/ChannelEdge.tsx new file mode 100644 index 0000000000000..2d63f8fb4c5d6 --- /dev/null +++ b/console/src/platform/dataflows/ChannelEdge.tsx @@ -0,0 +1,62 @@ +// Copyright Materialize, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +import { Text, Tooltip } from "@chakra-ui/react"; +import { + BaseEdge, + EdgeLabelRenderer, + type EdgeProps, + getBezierPath, +} from "@xyflow/react"; +import React from "react"; + +export type ChannelEdgeData = { + messagesSent: bigint; + batchesSent: bigint; + channelTypes: string[]; + dimmed: boolean; +}; + +export const ChannelEdge = (props: EdgeProps & { data: ChannelEdgeData }) => { + const [path, labelX, labelY] = getBezierPath(props); + const { messagesSent, batchesSent, channelTypes, dimmed } = props.data; + const idle = messagesSent === 0n; + const label = idle ? "" : `${messagesSent} records / ${batchesSent} batches`; + return ( + <> + + {label && ( + + + + {label} + {channelTypes.length > 0 && ` · ${channelTypes.join(", ")}`} + + + + )} + + ); +}; diff --git a/console/src/platform/dataflows/DataflowGraphView.tsx b/console/src/platform/dataflows/DataflowGraphView.tsx new file mode 100644 index 0000000000000..80240d69118b2 --- /dev/null +++ b/console/src/platform/dataflows/DataflowGraphView.tsx @@ -0,0 +1,200 @@ +// Copyright Materialize, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +import "@xyflow/react/dist/style.css"; + +import { Box, Spinner, useToast } from "@chakra-ui/react"; +import { + Background, + Controls, + type Edge, + MiniMap, + type Node, + ReactFlow, +} from "@xyflow/react"; +import React from "react"; + +import { ChannelEdge } from "./ChannelEdge"; +import { + type CollapseState, + type DataflowStructure, + deriveVisibleGraph, + type GraphDecorations, + MAX_VISIBLE_NODES, + type VisibleNode, + visibleNodeCount, +} from "./dataflowGraph"; +import { NODE_DIMENSIONS } from "./elkGraph"; +import { + CollapsedRegionNode, + OperatorNode, + PortNode, + RegionNode, +} from "./nodes"; +import { nodeFillColor } from "./nodeStyle"; +import { useElkLayout } from "./useElkLayout"; + +const edgeTypes = { channel: ChannelEdge }; +const nodeTypes = { + operator: OperatorNode, + collapsedRegion: CollapsedRegionNode, + region: RegionNode, + port: PortNode, +}; + +export interface DataflowGraphViewProps { + structure: DataflowStructure; + collapsed: CollapseState; + onCollapsedChange: (next: CollapseState) => void; + cacheKey: string; + decorations?: GraphDecorations; + onNodeClick?: (node: VisibleNode) => void; +} + +export const DataflowGraphView = ({ + structure, + collapsed, + onCollapsedChange, + cacheKey, + decorations, + onNodeClick, +}: DataflowGraphViewProps) => { + const toast = useToast(); + const visible = React.useMemo(() => { + const graph = deriveVisibleGraph(structure, collapsed); + if (!decorations?.hiddenNodeIds && !decorations?.hiddenEdgeIds) + return graph; + const hiddenNodes = decorations.hiddenNodeIds ?? new Set(); + const hiddenEdges = decorations.hiddenEdgeIds ?? new Set(); + return { + nodes: graph.nodes.filter((n) => !hiddenNodes.has(n.id)), + edges: graph.edges.filter( + (e) => + !hiddenEdges.has(e.id) && + !hiddenNodes.has(e.source) && + !hiddenNodes.has(e.target), + ), + }; + }, [ + structure, + collapsed, + decorations?.hiddenNodeIds, + decorations?.hiddenEdgeIds, + ]); + + const layoutKey = `${cacheKey}|${[...collapsed].sort().join(",")}|${ + decorations?.hiddenNodeIds + ? [...decorations.hiddenNodeIds].sort().join(",") + : "" + }|${ + decorations?.hiddenEdgeIds + ? [...decorations.hiddenEdgeIds].sort().join(",") + : "" + }`; + const { positions, layouting, error } = useElkLayout(visible, layoutKey); + + const toggleRegion = React.useCallback( + (node: VisibleNode) => { + const next = new Set(collapsed); + if (next.has(node.id)) { + next.delete(node.id); + if (visibleNodeCount(structure, next) > MAX_VISIBLE_NODES) { + toast({ + status: "warning", + title: `Expanding would show more than ${MAX_VISIBLE_NODES} nodes.`, + }); + return; + } + } else { + next.add(node.id); + } + onCollapsedChange(next); + }, + [collapsed, onCollapsedChange, structure, toast], + ); + + const nodes: Node[] = React.useMemo(() => { + if (!positions) return []; + return visible.nodes.map((n) => { + const pos = positions[n.id] ?? { x: 0, y: 0, ...NODE_DIMENSIONS[n.kind] }; + return { + id: n.id, + type: n.kind, + position: { x: pos.x, y: pos.y }, + parentId: n.parent ?? undefined, + extent: n.parent ? ("parent" as const) : undefined, + style: { width: pos.width, height: pos.height }, + draggable: false, + connectable: false, + data: { + node: n, + dimmed: decorations?.dimmedNodeIds?.has(n.id) ?? false, + color: decorations?.nodeColors?.get(n.id) ?? nodeFillColor(n), + }, + }; + }); + }, [visible, positions, decorations?.dimmedNodeIds, decorations?.nodeColors]); + + const edges: Edge[] = React.useMemo( + () => + visible.edges.map((e) => ({ + id: e.id, + source: e.source, + target: e.target, + type: "channel", + data: { + messagesSent: e.messagesSent, + batchesSent: e.batchesSent, + channelTypes: e.channelTypes, + dimmed: + (decorations?.dimmedNodeIds?.has(e.source) ?? false) || + (decorations?.dimmedNodeIds?.has(e.target) ?? false) || + (decorations?.dimmedEdgeIds?.has(e.id) ?? false), + }, + })), + [visible, decorations?.dimmedNodeIds, decorations?.dimmedEdgeIds], + ); + + if (error) throw new Error(error); + return ( + + {layouting && ( + + + + )} + + onNodeClick?.((node.data as { node: VisibleNode }).node) + } + onNodeDoubleClick={(_, node) => { + const visibleNode = (node.data as { node: VisibleNode }).node; + if ( + visibleNode.kind === "region" || + visibleNode.kind === "collapsedRegion" + ) { + toggleRegion(visibleNode); + } + }} + proOptions={{ hideAttribution: true }} + > + + + + + + ); +}; diff --git a/console/src/platform/dataflows/dataflowGraph.ts b/console/src/platform/dataflows/dataflowGraph.ts index 790761dddeb26..58327b409b849 100644 --- a/console/src/platform/dataflows/dataflowGraph.ts +++ b/console/src/platform/dataflows/dataflowGraph.ts @@ -330,3 +330,13 @@ export function visibleNodeCount( ): number { return deriveVisibleGraph(structure, collapsed).nodes.length; } + +// All-optional here. Task 13's decorateGraph returns all fields set. +export interface GraphDecorations { + dimmedNodeIds?: ReadonlySet; + hiddenNodeIds?: ReadonlySet; + hiddenEdgeIds?: ReadonlySet; + dimmedEdgeIds?: ReadonlySet; + nodeColors?: ReadonlyMap; // heatmap override + searchMatches?: string[]; +} diff --git a/console/src/platform/dataflows/nodeStyle.ts b/console/src/platform/dataflows/nodeStyle.ts new file mode 100644 index 0000000000000..3a4db57cdf16c --- /dev/null +++ b/console/src/platform/dataflows/nodeStyle.ts @@ -0,0 +1,35 @@ +// Copyright Materialize, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +import type { VisibleNode } from "./dataflowGraph"; + +export const COLORS = { + noArrangementRegion: "#12b886", + noArrangementOperator: "#ffffff", + arrangementRegion: "#7950f2", + arrangementOperator: "#fab005", +}; + +export function nodeFillColor(node: VisibleNode): string { + const arranged = (node.transitive?.arrangementRecords ?? 0n) > 0n; + const region = node.kind !== "operator" && node.kind !== "port"; + if (region) + return arranged ? COLORS.arrangementRegion : COLORS.noArrangementRegion; + return arranged ? COLORS.arrangementOperator : COLORS.noArrangementOperator; +} + +export function formatElapsed(ns: bigint): string { + return `scheduled ${Math.round(Number(ns) / 1e9)}s`; +} + +export type FlowNodeData = { + node: VisibleNode; + dimmed: boolean; + color: string; +}; diff --git a/console/src/platform/dataflows/nodes.tsx b/console/src/platform/dataflows/nodes.tsx new file mode 100644 index 0000000000000..78205c6f19d0b --- /dev/null +++ b/console/src/platform/dataflows/nodes.tsx @@ -0,0 +1,123 @@ +// Copyright Materialize, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +import { Badge, Box, Text, Tooltip } from "@chakra-ui/react"; +import { Handle, type NodeProps, Position } from "@xyflow/react"; +import React from "react"; + +import { formatBytesShort } from "~/utils/format"; + +import type { VisibleNode } from "./dataflowGraph"; +import { type FlowNodeData, formatElapsed } from "./nodeStyle"; + +const statLines = (node: VisibleNode) => { + const lines: string[] = []; + const stats = node.stats; + if (!stats) return lines; + if (stats.arrangementRecords > 0n) { + lines.push(`${stats.arrangementRecords} arranged records`); + lines.push(formatBytesShort(stats.arrangementSize)); + } + if (stats.elapsedNs > 0n) lines.push(formatElapsed(stats.elapsedNs)); + return lines; +}; + +const CardShell = ({ + data, + children, +}: { + data: FlowNodeData; + children: React.ReactNode; +}) => ( + + {children} + + + +); + +export const OperatorNode = ({ data }: NodeProps & { data: FlowNodeData }) => ( + + `LIR ${l.lirId}: ${l.operator}`) + .join(", ")} + isDisabled={data.node.lir.length === 0} + > + + + {data.node.label} + + {statLines(data.node).map((line) => ( + + {line} + + ))} + + + +); + +export const CollapsedRegionNode = ({ + data, +}: NodeProps & { data: FlowNodeData }) => ( + + + {data.node.label} + + {data.node.childCount} children + {statLines(data.node).map((line) => ( + + {line} + + ))} + +); + +export const RegionNode = ({ data }: NodeProps & { data: FlowNodeData }) => ( + + + {data.node.label} + + + + +); + +export const PortNode = ({ data }: NodeProps & { data: FlowNodeData }) => ( + + {data.node.label} + + + +); diff --git a/console/src/test/mockReactFlow.tsx b/console/src/test/mockReactFlow.tsx new file mode 100644 index 0000000000000..e1d3343503fbb --- /dev/null +++ b/console/src/test/mockReactFlow.tsx @@ -0,0 +1,45 @@ +// Copyright Materialize, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +import React from "react"; +import { vi } from "vitest"; + +// jsdom lacks ResizeObserver/DOMMatrixReadOnly, so component tests mock +// @xyflow/react to a flat renderer that preserves the decision logic under +// test (which nodes and labels exist) instead of exercising real layout. +export function mockReactFlow() { + vi.mock("@xyflow/react", () => ({ + ReactFlow: ({ + nodes, + children, + }: { + nodes: { id: string; data: { node: { label: string } } }[]; + children?: React.ReactNode; + }) => ( +
+ {nodes.map((n) => ( +
+ {n.data.node.label} +
+ ))} + {children} +
+ ), + Background: () => null, + Controls: () => null, + MiniMap: () => null, + Handle: () => null, + Position: { Left: "left", Right: "right" }, + BaseEdge: () => null, + EdgeLabelRenderer: ({ children }: { children: React.ReactNode }) => ( + <>{children} + ), + getBezierPath: () => ["", 0, 0], + })); +} From b8d6cc07cc8003e1689dfcbba518f053d2a04e2f Mon Sep 17 00:00:00 2001 From: Moritz Hoffmann Date: Wed, 8 Jul 2026 22:42:22 +0200 Subject: [PATCH 08/45] Add the dataflow detail page behind a flag, with phase-1 gate fixes Co-Authored-By: Claude Sonnet 5 --- .../dataflow/useDataflowGraphData.ts | 9 +- .../src/platform/clusters/ClusterDetail.tsx | 25 +++- .../src/platform/dataflows/ChannelEdge.tsx | 21 +++- .../platform/dataflows/DataflowDetailPage.tsx | 107 ++++++++++++++++++ .../platform/dataflows/DataflowGraphView.tsx | 8 ++ console/src/platform/dataflows/elkGraph.ts | 2 +- .../src/platform/dataflows/layout.worker.ts | 43 ------- console/src/platform/dataflows/nodes.tsx | 12 +- .../src/platform/dataflows/useElkLayout.ts | 61 +++++----- 9 files changed, 195 insertions(+), 93 deletions(-) create mode 100644 console/src/platform/dataflows/DataflowDetailPage.tsx delete mode 100644 console/src/platform/dataflows/layout.worker.ts diff --git a/console/src/api/materialize/dataflow/useDataflowGraphData.ts b/console/src/api/materialize/dataflow/useDataflowGraphData.ts index c9c08cf301304..f6ba58fe12d0b 100644 --- a/console/src/api/materialize/dataflow/useDataflowGraphData.ts +++ b/console/src/api/materialize/dataflow/useDataflowGraphData.ts @@ -63,18 +63,19 @@ export function useDataflowGraphData(params?: DataflowGraphParams) { SELECT mdco.id, from_operator_address AS "fromOperatorAddress", - from_port AS "fromPort", + mdc.from_port AS "fromPort", to_operator_address AS "toOperatorAddress", - to_port AS "toPort", + mdc.to_port AS "toPort", COALESCE(sum(mmc.sent), 0) AS "messagesSent", COALESCE(sum(mmc.batch_sent), 0) AS "batchesSent", mdco.type AS "channelType" FROM mz_dataflow_channel_operators AS mdco + JOIN mz_dataflow_channels AS mdc ON mdc.id = mdco.id LEFT JOIN mz_message_counts AS mmc ON mdco.id = mmc.channel_id WHERE from_operator_address[1] = ${id} GROUP BY - mdco.id, "fromOperatorAddress", "fromPort", - "toOperatorAddress", "toPort", mdco.type` + mdco.id, "fromOperatorAddress", mdc.from_port, + "toOperatorAddress", mdc.to_port, mdco.type` .$castTo() .compile(queryBuilder), lirSpans: sql` diff --git a/console/src/platform/clusters/ClusterDetail.tsx b/console/src/platform/clusters/ClusterDetail.tsx index a729379cb17d3..93c21343ddd13 100644 --- a/console/src/platform/clusters/ClusterDetail.tsx +++ b/console/src/platform/clusters/ClusterDetail.tsx @@ -15,6 +15,7 @@ import { isSystemCluster } from "~/api/materialize"; import { ClusterWithOwnership } from "~/api/materialize/cluster/clusterList"; import DeleteObjectMenuItem from "~/components/DeleteObjectMenuItem"; import OverflowMenu from "~/components/OverflowMenu"; +import { useFlags } from "~/hooks/useFlags"; import { Breadcrumb, PageBreadcrumbs, @@ -40,6 +41,10 @@ import Sinks from "./Sinks"; import Sources from "./Sources"; import { useShowSystemObjects } from "./useShowSystemObjects"; +const DataflowDetailPage = React.lazy( + () => import("~/platform/dataflows/DataflowDetailPage"), +); + const ClusterDetailBreadcrumbs = (props: { crumbs: Breadcrumb[] }) => { const [showSystemObjects] = useShowSystemObjects(); const { clusterId, clusterName } = useParams(); @@ -124,17 +129,21 @@ const ClusterDetailPage = () => { ], [clusterName], ); - const subnavItems: Tab[] = React.useMemo( - () => [ + const flags = useFlags(); + const subnavItems: Tab[] = React.useMemo(() => { + const tabs: Tab[] = [ { label: "Overview", href: "..", end: true }, { label: "Replicas", href: "../replicas" }, { label: "Materialized Views", href: "../materialized-views" }, { label: "Indexes", href: "../indexes" }, { label: "Sources", href: "../sources" }, { label: "Sinks", href: "../sinks" }, - ], - [], - ); + ]; + if (flags["visualization-features"]) { + tabs.push({ label: "Dataflows", href: "../dataflows" }); + } + return tabs; + }, [flags]); // Setting key on the route elements prevents any weird jank when you use the context // menu to switch between clusters. @@ -159,6 +168,12 @@ const ClusterDetailPage = () => { } /> } /> } /> + {flags["visualization-features"] && ( + } + /> + )} ); diff --git a/console/src/platform/dataflows/ChannelEdge.tsx b/console/src/platform/dataflows/ChannelEdge.tsx index 2d63f8fb4c5d6..ad932de2316f0 100644 --- a/console/src/platform/dataflows/ChannelEdge.tsx +++ b/console/src/platform/dataflows/ChannelEdge.tsx @@ -23,11 +23,27 @@ export type ChannelEdgeData = { dimmed: boolean; }; +const compactCount = Intl.NumberFormat("default", { + notation: "compact", + maximumFractionDigits: 1, +}); + +const compact = (n: bigint) => compactCount.format(n); + export const ChannelEdge = (props: EdgeProps & { data: ChannelEdgeData }) => { const [path, labelX, labelY] = getBezierPath(props); const { messagesSent, batchesSent, channelTypes, dimmed } = props.data; const idle = messagesSent === 0n; - const label = idle ? "" : `${messagesSent} records / ${batchesSent} batches`; + // Inline label stays terse: compact record/batch counts only. Exact figures + // and the full container type names live in the tooltip, since those Rust + // type strings are long enough to swamp the canvas. + const label = idle + ? "" + : `${compact(messagesSent)} / ${compact(batchesSent)}`; + const tooltip = idle + ? channelTypes.join(", ") || "unknown channel type" + : `${messagesSent} records / ${batchesSent} batches` + + (channelTypes.length > 0 ? ` · ${channelTypes.join(", ")}` : ""); return ( <> { /> {label && ( - + { className="nopan" > {label} - {channelTypes.length > 0 && ` · ${channelTypes.join(", ")}`} diff --git a/console/src/platform/dataflows/DataflowDetailPage.tsx b/console/src/platform/dataflows/DataflowDetailPage.tsx new file mode 100644 index 0000000000000..7025f71164cc3 --- /dev/null +++ b/console/src/platform/dataflows/DataflowDetailPage.tsx @@ -0,0 +1,107 @@ +// Copyright Materialize, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +import { Alert, AlertIcon, Spinner, Text, VStack } from "@chakra-ui/react"; +import React from "react"; +import { useParams, useSearchParams } from "react-router-dom"; + +import { useDataflowGraphData } from "~/api/materialize/dataflow/useDataflowGraphData"; +import { ErrorCode } from "~/api/materialize/types"; +import ErrorBox from "~/components/ErrorBox"; +import LabeledSelect from "~/components/LabeledSelect"; +import { MainContentContainer } from "~/layouts/BaseLayout"; +import { useAllClusters } from "~/store/allClusters"; + +import { type CollapseState, defaultCollapseState } from "./dataflowGraph"; +import { DataflowGraphView } from "./DataflowGraphView"; + +const DataflowDetailPage = () => { + const { clusterId, dataflowId } = useParams(); + const { getClusterById } = useAllClusters(); + const cluster = clusterId ? getClusterById(clusterId) : undefined; + const [searchParams, setSearchParams] = useSearchParams(); + const replicaName = searchParams.get("replica") ?? cluster?.replicas[0]?.name; + + const params = React.useMemo( + () => + cluster && replicaName && dataflowId + ? { clusterName: cluster.name, replicaName, dataflowId } + : undefined, + [cluster, replicaName, dataflowId], + ); + const { data, error, databaseError } = useDataflowGraphData(params); + + const [collapsed, setCollapsed] = React.useState(null); + const structureKey = data + ? `${params?.dataflowId}/${params?.replicaName}/${data.structure.nodes.size}` + : null; + React.useEffect(() => { + if (data) setCollapsed(defaultCollapseState(data.structure)); + // Reset collapse state when the structure identity changes. + }, [structureKey]); // eslint-disable-line react-hooks/exhaustive-deps + + if (!cluster) return null; + if (cluster.replicas.length === 0) { + return ( + + This cluster has no replicas. + + ); + } + const permissionError = + databaseError && + "code" in databaseError && + databaseError.code === ErrorCode.INSUFFICIENT_PRIVILEGE; + + return ( + + + setSearchParams({ replica: e.target.value })} + flexShrink={0} + > + {cluster.replicas.map((r) => ( + + ))} + + {permissionError ? ( + + + + You'll need{" "} + + USAGE + {" "} + privilege on this cluster to visualize this dataflow. + + + ) : error ? ( + + ) : !data || !collapsed ? ( + + ) : data.structure.nodes.size <= 1 ? ( + This dataflow contains no operators. + ) : ( + + )} + + + ); +}; + +export default DataflowDetailPage; diff --git a/console/src/platform/dataflows/DataflowGraphView.tsx b/console/src/platform/dataflows/DataflowGraphView.tsx index 80240d69118b2..d5a94b0aef4fc 100644 --- a/console/src/platform/dataflows/DataflowGraphView.tsx +++ b/console/src/platform/dataflows/DataflowGraphView.tsx @@ -16,6 +16,7 @@ import { type Edge, MiniMap, type Node, + Position, ReactFlow, } from "@xyflow/react"; import React from "react"; @@ -129,6 +130,11 @@ export const DataflowGraphView = ({ position: { x: pos.x, y: pos.y }, parentId: n.parent ?? undefined, extent: n.parent ? ("parent" as const) : undefined, + // Match the Top/Bottom handles so bezier control points meet the + // node edges. Without this the edge curves toward the default + // Bottom/Top and appears detached across region boundaries. + targetPosition: Position.Top, + sourcePosition: Position.Bottom, style: { width: pos.width, height: pos.height }, draggable: false, connectable: false, @@ -177,6 +183,8 @@ export const DataflowGraphView = ({ onlyRenderVisibleElements fitView minZoom={0.05} + // Double-click is the collapse toggle, so it must not also zoom. + zoomOnDoubleClick={false} onNodeClick={(_, node) => onNodeClick?.((node.data as { node: VisibleNode }).node) } diff --git a/console/src/platform/dataflows/elkGraph.ts b/console/src/platform/dataflows/elkGraph.ts index 6979962f55f35..78233d7b7509f 100644 --- a/console/src/platform/dataflows/elkGraph.ts +++ b/console/src/platform/dataflows/elkGraph.ts @@ -32,7 +32,7 @@ export const NODE_DIMENSIONS: Record< const LAYOUT_OPTIONS: Record = { "elk.algorithm": "layered", - "elk.direction": "RIGHT", + "elk.direction": "DOWN", "elk.hierarchyHandling": "INCLUDE_CHILDREN", "elk.padding": "[top=48,left=16,bottom=16,right=16]", "elk.spacing.nodeNode": "24", diff --git a/console/src/platform/dataflows/layout.worker.ts b/console/src/platform/dataflows/layout.worker.ts deleted file mode 100644 index f89e4998ea339..0000000000000 --- a/console/src/platform/dataflows/layout.worker.ts +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright Materialize, Inc. and contributors. All rights reserved. -// -// Use of this software is governed by the Business Source License -// included in the LICENSE file. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0. - -import ELK from "elkjs/lib/elk.bundled.js"; - -import type { VisibleGraph } from "./dataflowGraph"; -import { extractPositions, type Positions, toElkGraph } from "./elkGraph"; - -export interface LayoutRequest { - requestId: number; - graph: VisibleGraph; -} - -export interface LayoutResponse { - requestId: number; - positions?: Positions; - error?: string; -} - -const elk = new ELK(); - -self.onmessage = async (event: MessageEvent) => { - const { requestId, graph } = event.data; - try { - const layouted = await elk.layout(toElkGraph(graph)); - const response: LayoutResponse = { - requestId, - positions: extractPositions(layouted), - }; - self.postMessage(response); - } catch (error) { - self.postMessage({ - requestId, - error: String(error), - } satisfies LayoutResponse); - } -}; diff --git a/console/src/platform/dataflows/nodes.tsx b/console/src/platform/dataflows/nodes.tsx index 78205c6f19d0b..df98ddc0d0bc7 100644 --- a/console/src/platform/dataflows/nodes.tsx +++ b/console/src/platform/dataflows/nodes.tsx @@ -47,8 +47,8 @@ const CardShell = ({ opacity={data.dimmed ? 0.25 : 1} > {children} - - + + ); @@ -103,8 +103,8 @@ export const RegionNode = ({ data }: NodeProps & { data: FlowNodeData }) => ( {data.node.label} - - + + ); @@ -117,7 +117,7 @@ export const PortNode = ({ data }: NodeProps & { data: FlowNodeData }) => ( opacity={data.dimmed ? 0.25 : 1} > {data.node.label} - - + + ); diff --git a/console/src/platform/dataflows/useElkLayout.ts b/console/src/platform/dataflows/useElkLayout.ts index 1baa4326521fa..8a1845bf9e498 100644 --- a/console/src/platform/dataflows/useElkLayout.ts +++ b/console/src/platform/dataflows/useElkLayout.ts @@ -7,14 +7,21 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. +import ELK, { type ElkNode } from "elkjs/lib/elk-api.js"; +// The elk engine detects a worker context at load time and registers itself +// as the worker's message handler, so it must be the worker entry point +// itself. A hand-written worker wrapping it cannot work: imported into +// another worker the engine hijacks that worker's onmessage and exports +// nothing. elk-api on the main thread speaks to it and correlates +// request/response pairs internally. +import ElkWorker from "elkjs/lib/elk-worker.min.js?worker"; import React from "react"; import type { VisibleGraph } from "./dataflowGraph"; -import type { Positions } from "./elkGraph"; -import type { LayoutRequest, LayoutResponse } from "./layout.worker"; +import { extractPositions, type Positions, toElkGraph } from "./elkGraph"; export function useElkLayout(graph: VisibleGraph | null, cacheKey: string) { - const workerRef = React.useRef(null); + const elkRef = React.useRef | null>(null); const requestIdRef = React.useRef(0); const cacheRef = React.useRef(new Map()); const [state, setState] = React.useState<{ @@ -24,11 +31,11 @@ export function useElkLayout(graph: VisibleGraph | null, cacheKey: string) { }>({ key: null, positions: null, error: null }); React.useEffect(() => { - const worker = new Worker(new URL("./layout.worker.ts", import.meta.url), { - type: "module", - }); - workerRef.current = worker; - return () => worker.terminate(); + const elk = new ELK({ workerFactory: () => new ElkWorker() }); + elkRef.current = elk; + return () => { + elk.terminateWorker(); + }; }, []); React.useEffect(() => { @@ -38,30 +45,22 @@ export function useElkLayout(graph: VisibleGraph | null, cacheKey: string) { setState({ key: cacheKey, positions: cached, error: null }); return; } + const elk = elkRef.current; + if (!elk) return; const requestId = ++requestIdRef.current; - const worker = workerRef.current; - if (!worker) return; - const onMessage = (event: MessageEvent) => { - // Drop responses for superseded requests. - if (event.data.requestId !== requestIdRef.current) return; - if (event.data.positions) { - cacheRef.current.set(cacheKey, event.data.positions); - setState({ - key: cacheKey, - positions: event.data.positions, - error: null, - }); - } else { - setState({ - key: cacheKey, - positions: null, - error: event.data.error ?? "layout failed", - }); - } - }; - worker.addEventListener("message", onMessage); - worker.postMessage({ requestId, graph } satisfies LayoutRequest); - return () => worker.removeEventListener("message", onMessage); + elk.layout(toElkGraph(graph)).then( + (layouted: ElkNode) => { + // Drop responses for superseded requests. + if (requestId !== requestIdRef.current) return; + const positions = extractPositions(layouted); + cacheRef.current.set(cacheKey, positions); + setState({ key: cacheKey, positions, error: null }); + }, + (error: unknown) => { + if (requestId !== requestIdRef.current) return; + setState({ key: cacheKey, positions: null, error: String(error) }); + }, + ); }, [graph, cacheKey]); return { From f5b0c6eaed2df3c860139703aaa5ba53dc8a3571 Mon Sep 17 00:00:00 2001 From: Moritz Hoffmann Date: Wed, 8 Jul 2026 22:42:22 +0200 Subject: [PATCH 09/45] Add the cluster dataflows list page, deep-link, and manual refresh Co-Authored-By: Claude Sonnet 5 --- .../dataflow/useDataflowIdForExport.ts | 51 ++++++ .../dataflow/useDataflowList.test.ts | 85 ++++++++++ .../materialize/dataflow/useDataflowList.ts | 90 +++++++++++ .../src/platform/clusters/ClusterDetail.tsx | 17 +- .../platform/dataflows/DataflowDetailPage.tsx | 40 ++++- .../src/platform/dataflows/DataflowsPage.tsx | 149 ++++++++++++++++++ .../SimpleObjectDetailRoutes.tsx | 31 ++-- 7 files changed, 445 insertions(+), 18 deletions(-) create mode 100644 console/src/api/materialize/dataflow/useDataflowIdForExport.ts create mode 100644 console/src/api/materialize/dataflow/useDataflowList.test.ts create mode 100644 console/src/api/materialize/dataflow/useDataflowList.ts create mode 100644 console/src/platform/dataflows/DataflowsPage.tsx diff --git a/console/src/api/materialize/dataflow/useDataflowIdForExport.ts b/console/src/api/materialize/dataflow/useDataflowIdForExport.ts new file mode 100644 index 0000000000000..9df8af53533b5 --- /dev/null +++ b/console/src/api/materialize/dataflow/useDataflowIdForExport.ts @@ -0,0 +1,51 @@ +// Copyright Materialize, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +import { sql } from "kysely"; +import React from "react"; + +import { escapedLiteral as lit, useSqlManyTyped } from "~/api/materialize"; + +import { queryBuilder } from "../db"; + +export interface DataflowIdForExportParams { + clusterName: string; + replicaName: string; + exportId: string; +} + +interface DataflowIdRow { + dataflowId: string; +} + +export function useDataflowIdForExport(params?: DataflowIdForExportParams) { + const exportId = params?.exportId; + const queries = React.useMemo(() => { + // Export ids are of the form s/t/u. Reject anything else before + // interpolating so a malformed param can never reach the query. + if (exportId === undefined || !/^[stu]\d+$/.test(exportId)) return null; + return { + row: sql` + SELECT dataflow_id::text AS "dataflowId" + FROM mz_compute_exports + WHERE export_id = ${lit(exportId)}` + .$castTo() + .compile(queryBuilder), + }; + }, [exportId]); + + const { results, error, loading } = useSqlManyTyped(queries, { + cluster: params?.clusterName, + replica: params?.replicaName, + }); + + const dataflowId = + !error && results ? (results.row?.[0]?.dataflowId ?? null) : null; + return { dataflowId, loading, error }; +} diff --git a/console/src/api/materialize/dataflow/useDataflowList.test.ts b/console/src/api/materialize/dataflow/useDataflowList.test.ts new file mode 100644 index 0000000000000..60e90dd3cfe4b --- /dev/null +++ b/console/src/api/materialize/dataflow/useDataflowList.test.ts @@ -0,0 +1,85 @@ +// Copyright Materialize, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +import { renderHook, waitFor } from "@testing-library/react"; +import { http, HttpResponse } from "msw"; + +import { ErrorCode } from "~/api/materialize/types"; +import server from "~/api/mocks/server"; +import { createProviderWrapper } from "~/test/utils"; + +import { useDataflowList } from "./useDataflowList"; + +const FAILING_REPLICA = "r2"; +const listResult = { + desc: { + columns: [ + { name: "id" }, + { name: "name" }, + { name: "records" }, + { name: "size" }, + { name: "elapsedNs" }, + ], + }, + rows: [["7", "Dataflow: mv", "100", "4096", "12345"]], +}; + +beforeEach(() => { + server.use( + http.post("*/api/sql", async ({ request }) => { + const options = JSON.parse( + new URL(request.url).searchParams.get("options") ?? "{}", + ); + if (options.cluster_replica === FAILING_REPLICA) { + return HttpResponse.json({ + results: [ + { + error: { + message: "no such replica", + code: ErrorCode.INTERNAL_ERROR, + }, + }, + ], + }); + } + return HttpResponse.json({ results: [listResult] }); + }), + ); +}); + +describe("useDataflowList", () => { + it("maps rows to DataflowListEntry with bigint fields", async () => { + const Wrapper = await createProviderWrapper(); + const { result } = renderHook( + () => useDataflowList({ clusterName: "c", replicaName: "r1" }), + { wrapper: Wrapper }, + ); + await waitFor(() => expect(result.current.data).not.toBeNull()); + expect(result.current.data?.[0]).toEqual({ + id: "7", + name: "Dataflow: mv", + records: 100n, + size: 4096n, + elapsedNs: 12345n, + }); + }); + + it("clears data when the fetch fails", async () => { + // The hook reads the current environment off a jotai atom that resolves + // asynchronously, so it needs the same Suspense + JotaiProvider wrapper the + // app renders under. + const Wrapper = await createProviderWrapper(); + const { result } = renderHook( + () => useDataflowList({ clusterName: "c", replicaName: FAILING_REPLICA }), + { wrapper: Wrapper }, + ); + await waitFor(() => expect(result.current.error).toBeTruthy()); + expect(result.current.data).toBeNull(); + }); +}); diff --git a/console/src/api/materialize/dataflow/useDataflowList.ts b/console/src/api/materialize/dataflow/useDataflowList.ts new file mode 100644 index 0000000000000..f2c0ab7df48e8 --- /dev/null +++ b/console/src/api/materialize/dataflow/useDataflowList.ts @@ -0,0 +1,90 @@ +// Copyright Materialize, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +import { sql } from "kysely"; +import React from "react"; + +import { useSqlManyTyped } from "~/api/materialize"; + +import { queryBuilder } from "../db"; + +export interface DataflowListParams { + clusterName: string; + replicaName: string; +} + +export interface DataflowListEntry { + id: string; // uint8 as string + name: string; + records: bigint; + size: bigint; + elapsedNs: bigint; +} + +// Raw row shape as returned by the query. The numeric columns arrive as +// strings and are widened to bigint in normalize. +interface DataflowListRow { + id: string; + name: string; + records: string; + size: string; + elapsedNs: string; +} + +function normalize(rows: DataflowListRow[]): DataflowListEntry[] { + return rows.map((row) => ({ + id: row.id, + name: row.name, + records: BigInt(row.records), + size: BigInt(row.size), + elapsedNs: BigInt(row.elapsedNs), + })); +} + +export function useDataflowList(params?: DataflowListParams) { + // The query text is constant. Cluster and replica reach the request through + // the options below, so recompiling only depends on whether params exist. + const enabled = params !== undefined; + const queries = React.useMemo(() => { + if (!enabled) return null; + return { + list: sql` + SELECT + md.id::text AS id, + md.name, + COALESCE(mrpd.records, 0) AS records, + COALESCE(mrpd.size, 0) AS size, + COALESCE(el.elapsed_ns, 0) AS "elapsedNs" + FROM mz_dataflows AS md + LEFT JOIN mz_records_per_dataflow AS mrpd ON mrpd.id = md.id + LEFT JOIN ( + SELECT mdod.dataflow_id, sum(mse.elapsed_ns) AS elapsed_ns + FROM mz_dataflow_operator_dataflows AS mdod + JOIN mz_scheduling_elapsed AS mse ON mse.id = mdod.id + GROUP BY mdod.dataflow_id + ) AS el ON el.dataflow_id = md.id + ORDER BY size DESC` + .$castTo() + .compile(queryBuilder), + }; + }, [enabled]); + + const { results, error, databaseError, loading, refetch } = useSqlManyTyped( + queries, + { + cluster: params?.clusterName, + replica: params?.replicaName, + }, + ); + + // Error always wins, and a list going momentarily stale across a replica + // switch is masked by loading. + const data = !error && results ? normalize(results.list ?? []) : null; + return { data, error, databaseError, loading, refetch }; +} diff --git a/console/src/platform/clusters/ClusterDetail.tsx b/console/src/platform/clusters/ClusterDetail.tsx index 93c21343ddd13..a8dfb25188851 100644 --- a/console/src/platform/clusters/ClusterDetail.tsx +++ b/console/src/platform/clusters/ClusterDetail.tsx @@ -44,6 +44,9 @@ import { useShowSystemObjects } from "./useShowSystemObjects"; const DataflowDetailPage = React.lazy( () => import("~/platform/dataflows/DataflowDetailPage"), ); +const DataflowsPage = React.lazy( + () => import("~/platform/dataflows/DataflowsPage"), +); const ClusterDetailBreadcrumbs = (props: { crumbs: Breadcrumb[] }) => { const [showSystemObjects] = useShowSystemObjects(); @@ -169,10 +172,16 @@ const ClusterDetailPage = () => { } /> } /> {flags["visualization-features"] && ( - } - /> + <> + } + /> + } + /> + )} diff --git a/console/src/platform/dataflows/DataflowDetailPage.tsx b/console/src/platform/dataflows/DataflowDetailPage.tsx index 7025f71164cc3..b40db6f0c5eba 100644 --- a/console/src/platform/dataflows/DataflowDetailPage.tsx +++ b/console/src/platform/dataflows/DataflowDetailPage.tsx @@ -7,7 +7,15 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -import { Alert, AlertIcon, Spinner, Text, VStack } from "@chakra-ui/react"; +import { + Alert, + AlertIcon, + Button, + HStack, + Spinner, + Text, + VStack, +} from "@chakra-ui/react"; import React from "react"; import { useParams, useSearchParams } from "react-router-dom"; @@ -21,6 +29,16 @@ import { useAllClusters } from "~/store/allClusters"; import { type CollapseState, defaultCollapseState } from "./dataflowGraph"; import { DataflowGraphView } from "./DataflowGraphView"; +// Stable 32-bit hash so a pure stats refresh (identical node ids) keeps the +// same structure key and reuses the layout, while any structural change +// produces a new key and relayouts. +function hashString(s: string): number { + let h = 0; + for (let i = 0; i < s.length; i++) + h = (Math.imul(h, 31) + s.charCodeAt(i)) | 0; + return h; +} + const DataflowDetailPage = () => { const { clusterId, dataflowId } = useParams(); const { getClusterById } = useAllClusters(); @@ -35,11 +53,17 @@ const DataflowDetailPage = () => { : undefined, [cluster, replicaName, dataflowId], ); - const { data, error, databaseError } = useDataflowGraphData(params); + const { data, error, databaseError, loading, refetch } = + useDataflowGraphData(params); const [collapsed, setCollapsed] = React.useState(null); + // Digest of the sorted node ids: identical structure across a stats-only + // refresh yields the same key, so layout and collapse state are preserved. const structureKey = data - ? `${params?.dataflowId}/${params?.replicaName}/${data.structure.nodes.size}` + ? (() => { + const ids = [...data.structure.nodes.keys()].sort(); + return `${params?.dataflowId}/${params?.replicaName}/${ids.length}-${hashString(ids.join(","))}`; + })() : null; React.useEffect(() => { if (data) setCollapsed(defaultCollapseState(data.structure)); @@ -74,6 +98,16 @@ const DataflowDetailPage = () => { ))} + {data && ( + + + + Last fetched {data.fetchedAt.toLocaleTimeString()} + + + )} {permissionError ? ( diff --git a/console/src/platform/dataflows/DataflowsPage.tsx b/console/src/platform/dataflows/DataflowsPage.tsx new file mode 100644 index 0000000000000..5bf3aaf2e8706 --- /dev/null +++ b/console/src/platform/dataflows/DataflowsPage.tsx @@ -0,0 +1,149 @@ +// Copyright Materialize, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +import { + Alert, + AlertIcon, + Spinner, + Table, + Tbody, + Td, + Text, + Th, + Thead, + Tr, + VStack, +} from "@chakra-ui/react"; +import React from "react"; +import { Link, Navigate, useParams, useSearchParams } from "react-router-dom"; + +import { useDataflowIdForExport } from "~/api/materialize/dataflow/useDataflowIdForExport"; +import { useDataflowList } from "~/api/materialize/dataflow/useDataflowList"; +import { ErrorCode } from "~/api/materialize/types"; +import ErrorBox from "~/components/ErrorBox"; +import LabeledSelect from "~/components/LabeledSelect"; +import { MainContentContainer } from "~/layouts/BaseLayout"; +import { useAllClusters } from "~/store/allClusters"; +import { formatBytesShort } from "~/utils/format"; + +const DataflowsPage = () => { + const { clusterId } = useParams(); + const { getClusterById } = useAllClusters(); + const cluster = clusterId ? getClusterById(clusterId) : undefined; + const [searchParams, setSearchParams] = useSearchParams(); + const replicaName = searchParams.get("replica") ?? cluster?.replicas[0]?.name; + const params = React.useMemo( + () => + cluster && replicaName + ? { clusterName: cluster.name, replicaName } + : undefined, + [cluster, replicaName], + ); + const { data, error, databaseError, loading } = useDataflowList(params); + + // When an object is deep-linked via ?export=, resolve its running + // dataflow on the chosen replica and redirect to that dataflow's page. The + // hook is called unconditionally and no-ops when there is no export to + // resolve. + const exportId = searchParams.get("export") ?? undefined; + const exportParams = React.useMemo( + () => + cluster && replicaName && exportId + ? { clusterName: cluster.name, replicaName, exportId } + : undefined, + [cluster, replicaName, exportId], + ); + const { + dataflowId, + loading: exportLoading, + error: exportError, + } = useDataflowIdForExport(exportParams); + + if (!cluster) return null; + + if (exportParams) { + if (exportLoading) return ; + if (dataflowId !== null) { + return ; + } + } + const permissionError = + databaseError && + "code" in databaseError && + databaseError.code === ErrorCode.INSUFFICIENT_PRIVILEGE; + + // The export resolved cleanly but no dataflow is running for it on this + // replica. Surface that above the list, which still renders below. + const exportHasNoDataflow = + exportParams !== undefined && + !exportLoading && + !exportError && + dataflowId === null; + return ( + + + {exportHasNoDataflow && ( + + )} + setSearchParams({ replica: e.target.value })} + > + {cluster.replicas.map((r) => ( + + ))} + + {permissionError ? ( + + + + You'll need{" "} + + USAGE + {" "} + privilege on this cluster to list its dataflows. + + + ) : error ? ( + + ) : !data && loading ? ( + + ) : ( + + + + + + + + + + + {(data ?? []).map((d) => ( + + + + + + + ))} + +
NameRecordsSizeScheduled
+ {d.name} + {d.records.toString()}{formatBytesShort(d.size)}{Math.round(Number(d.elapsedNs) / 1e9)}s
+ )} + + + ); +}; + +export default DataflowsPage; diff --git a/console/src/platform/object-explorer/SimpleObjectDetailRoutes.tsx b/console/src/platform/object-explorer/SimpleObjectDetailRoutes.tsx index c93baafffb064..74653e3427ec4 100644 --- a/console/src/platform/object-explorer/SimpleObjectDetailRoutes.tsx +++ b/console/src/platform/object-explorer/SimpleObjectDetailRoutes.tsx @@ -22,8 +22,11 @@ import { ShowCreateBlock } from "~/components/ShowCreateBlock"; import WorkflowGraph from "~/components/WorkflowGraph/WorkflowGraph"; import { useFlags } from "~/hooks/useFlags"; import { MainContentContainer, Tab } from "~/layouts/BaseLayout"; +import { absoluteClusterPath } from "~/platform/routeHelpers"; import { SentryRoutes } from "~/sentry"; +import { useAllClusters } from "~/store/allClusters"; import { useAllObjects } from "~/store/allObjects"; +import { useRegionSlug } from "~/store/environments"; import { ObjectDetailsContainer, ObjectDetailsStrip } from "./detailComponents"; import { ObjectColumns } from "./ObjectColumns"; @@ -34,10 +37,6 @@ import { SourceOverviewContainer } from "./SourceDetailRoutes"; import { useSchemaObjectParams } from "./useSchemaObjectParams"; import { useToastIfObjectNotExtant } from "./useToastIfObjectNotExtant"; -const DataflowVisualizer = React.lazy( - () => import("~/platform/clusters/DataflowVisualizer"), -); - const SIMPLE_OBJECTS_WITH_INDEXES = ["materialized-view", "view", "table"]; const DATAFLOW_VISUALIZER_OBJECTS = ["materialized-view", "index"]; @@ -102,6 +101,8 @@ export const SimpleObjectDetailRoutes = () => { useToastIfObjectNotExtant(); const { data: objects } = useAllObjects(); const object = objects.find((o) => o.id === id); + const regionSlug = useRegionSlug(); + const { getClusterById } = useAllClusters(); const dataflowVisualizerEnabled = flags["visualization-features"]; @@ -158,10 +159,21 @@ export const SimpleObjectDetailRoutes = () => { }); } if (shouldShowDataflowVisualizer) { - tabStripItems.push({ - label: "Visualize", - href: `../dataflow-visualizer`, - }); + // The visualizer lives on the owning cluster's dataflows page, which + // resolves the object's running dataflow from the ?export param. Skip the + // tab when the cluster is not resolvable rather than link somewhere broken. + const cluster = object.clusterId + ? getClusterById(object.clusterId) + : undefined; + if (cluster) { + tabStripItems.push({ + label: "Visualize", + href: `${absoluteClusterPath(regionSlug, { + id: cluster.id, + name: cluster.name, + })}/dataflows?export=${object.id}`, + }); + } } return ( @@ -200,9 +212,6 @@ export const SimpleObjectDetailRoutes = () => { {shouldShowIndexes && ( } /> )} - {shouldShowDataflowVisualizer && ( - } /> - )} } /> From 7e961093405450d8039581c90653448521d982d3 Mon Sep 17 00:00:00 2001 From: Moritz Hoffmann Date: Wed, 8 Jul 2026 22:42:22 +0200 Subject: [PATCH 10/45] console: node detail panel for dataflow visualizer Co-Authored-By: Claude Sonnet 5 --- .../platform/dataflows/DataflowDetailPage.tsx | 32 +++++-- .../platform/dataflows/DataflowGraphView.tsx | 3 + .../platform/dataflows/NodeDetailPanel.tsx | 84 +++++++++++++++++++ 3 files changed, 112 insertions(+), 7 deletions(-) create mode 100644 console/src/platform/dataflows/NodeDetailPanel.tsx diff --git a/console/src/platform/dataflows/DataflowDetailPage.tsx b/console/src/platform/dataflows/DataflowDetailPage.tsx index b40db6f0c5eba..a53aee727c1fb 100644 --- a/console/src/platform/dataflows/DataflowDetailPage.tsx +++ b/console/src/platform/dataflows/DataflowDetailPage.tsx @@ -26,8 +26,13 @@ import LabeledSelect from "~/components/LabeledSelect"; import { MainContentContainer } from "~/layouts/BaseLayout"; import { useAllClusters } from "~/store/allClusters"; -import { type CollapseState, defaultCollapseState } from "./dataflowGraph"; +import { + type CollapseState, + defaultCollapseState, + type VisibleNode, +} from "./dataflowGraph"; import { DataflowGraphView } from "./DataflowGraphView"; +import { NodeDetailPanel } from "./NodeDetailPanel"; // Stable 32-bit hash so a pure stats refresh (identical node ids) keeps the // same structure key and reuses the layout, while any structural change @@ -57,6 +62,9 @@ const DataflowDetailPage = () => { useDataflowGraphData(params); const [collapsed, setCollapsed] = React.useState(null); + const [selectedNode, setSelectedNode] = React.useState( + null, + ); // Digest of the sorted node ids: identical structure across a stats-only // refresh yields the same key, so layout and collapse state are preserved. const structureKey = data @@ -126,12 +134,22 @@ const DataflowDetailPage = () => { ) : data.structure.nodes.size <= 1 ? ( This dataflow contains no operators. ) : ( - + + setSelectedNode(null)} + /> + {selectedNode && ( + setSelectedNode(null)} + /> + )} + )} diff --git a/console/src/platform/dataflows/DataflowGraphView.tsx b/console/src/platform/dataflows/DataflowGraphView.tsx index d5a94b0aef4fc..d9ddccb408dc6 100644 --- a/console/src/platform/dataflows/DataflowGraphView.tsx +++ b/console/src/platform/dataflows/DataflowGraphView.tsx @@ -56,6 +56,7 @@ export interface DataflowGraphViewProps { cacheKey: string; decorations?: GraphDecorations; onNodeClick?: (node: VisibleNode) => void; + onPaneClick?: () => void; } export const DataflowGraphView = ({ @@ -65,6 +66,7 @@ export const DataflowGraphView = ({ cacheKey, decorations, onNodeClick, + onPaneClick, }: DataflowGraphViewProps) => { const toast = useToast(); const visible = React.useMemo(() => { @@ -188,6 +190,7 @@ export const DataflowGraphView = ({ onNodeClick={(_, node) => onNodeClick?.((node.data as { node: VisibleNode }).node) } + onPaneClick={onPaneClick} onNodeDoubleClick={(_, node) => { const visibleNode = (node.data as { node: VisibleNode }).node; if ( diff --git a/console/src/platform/dataflows/NodeDetailPanel.tsx b/console/src/platform/dataflows/NodeDetailPanel.tsx new file mode 100644 index 0000000000000..a9cb498f4ac8a --- /dev/null +++ b/console/src/platform/dataflows/NodeDetailPanel.tsx @@ -0,0 +1,84 @@ +// Copyright Materialize, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +import { Box, CloseButton, HStack, Text } from "@chakra-ui/react"; +import React from "react"; + +import { formatBytesShort } from "~/utils/format"; + +import type { VisibleNode } from "./dataflowGraph"; + +export const NodeDetailPanel = ({ + node, + onClose, +}: { + node: VisibleNode; + onClose: () => void; +}) => { + const row = (label: string, value: string) => ( + + + {label} + + + {value} + + + ); + return ( + + + + {node.label} + + + + {row("Kind", node.kind)} + {node.address && row("Address", node.address.join("."))} + {node.childCount > 0 && row("Children", String(node.childCount))} + {node.stats && ( + <> + {row("Arranged records", node.stats.arrangementRecords.toString())} + {row( + "Arrangement size", + formatBytesShort(node.stats.arrangementSize), + )} + {row("Elapsed", `${Math.round(Number(node.stats.elapsedNs) / 1e9)}s`)} + + )} + {node.transitive && node.childCount > 0 && ( + <> + {row( + "Subtree records", + node.transitive.arrangementRecords.toString(), + )} + {row( + "Subtree size", + formatBytesShort(node.transitive.arrangementSize), + )} + {row( + "Subtree elapsed", + `${Math.round(Number(node.transitive.elapsedNs) / 1e9)}s`, + )} + + )} + {node.lir.map((l) => ( + + LIR {l.lirId} ({l.exportId}): {l.operator} + + ))} + + ); +}; From 4d9c4f637df8ff3a91830238e11dd7c4aca4e677 Mon Sep 17 00:00:00 2001 From: Moritz Hoffmann Date: Wed, 8 Jul 2026 22:42:22 +0200 Subject: [PATCH 11/45] console: dataflow visualizer filters and toolbar Co-Authored-By: Claude Sonnet 5 --- .../platform/dataflows/DataflowDetailPage.tsx | 108 ++++++++++-- .../platform/dataflows/DataflowGraphView.tsx | 33 ++++ .../platform/dataflows/DataflowToolbar.tsx | 163 ++++++++++++++++++ .../platform/dataflows/dataflowGraph.test.ts | 107 ++++++++++++ .../src/platform/dataflows/dataflowGraph.ts | 115 +++++++++++- 5 files changed, 504 insertions(+), 22 deletions(-) create mode 100644 console/src/platform/dataflows/DataflowToolbar.tsx diff --git a/console/src/platform/dataflows/DataflowDetailPage.tsx b/console/src/platform/dataflows/DataflowDetailPage.tsx index a53aee727c1fb..9fab1f1390772 100644 --- a/console/src/platform/dataflows/DataflowDetailPage.tsx +++ b/console/src/platform/dataflows/DataflowDetailPage.tsx @@ -10,12 +10,14 @@ import { Alert, AlertIcon, + Box, Button, HStack, Spinner, Text, VStack, } from "@chakra-ui/react"; +import { interpolateYlOrRd } from "d3-scale-chromatic"; import React from "react"; import { useParams, useSearchParams } from "react-router-dom"; @@ -27,13 +29,23 @@ import { MainContentContainer } from "~/layouts/BaseLayout"; import { useAllClusters } from "~/store/allClusters"; import { + allChannelTypes, type CollapseState, + decorateGraph, + DEFAULT_FILTERS, defaultCollapseState, + deriveVisibleGraph, + expandForSearch, + type Filters, type VisibleNode, } from "./dataflowGraph"; import { DataflowGraphView } from "./DataflowGraphView"; +import { DataflowToolbar } from "./DataflowToolbar"; import { NodeDetailPanel } from "./NodeDetailPanel"; +// Injected into decorateGraph so the pure graph module stays d3-free. +const heatColor = (t: number) => interpolateYlOrRd(0.15 + 0.85 * t); + // Stable 32-bit hash so a pure stats refresh (identical node ids) keeps the // same structure key and reuses the layout, while any structural change // produces a new key and relayouts. @@ -65,6 +77,42 @@ const DataflowDetailPage = () => { const [selectedNode, setSelectedNode] = React.useState( null, ); + const [filters, setFilters] = React.useState(DEFAULT_FILTERS); + const [matchIndex, setMatchIndex] = React.useState(0); + const centerRef = React.useRef<((id: string) => void) | null>(null); + + const decorations = React.useMemo( + () => + data && collapsed + ? decorateGraph( + deriveVisibleGraph(data.structure, collapsed), + filters, + heatColor, + ) + : undefined, + [data, collapsed, filters], + ); + + // New search: expand ancestors of matches once, reset the cursor. + React.useEffect(() => { + setMatchIndex(0); + if (filters.search && data) { + setCollapsed((c) => + c ? expandForSearch(data.structure, c, filters.search) : c, + ); + } + }, [filters.search]); // eslint-disable-line react-hooks/exhaustive-deps + + const onJump = React.useCallback( + (delta: 1 | -1) => { + const matches = decorations?.searchMatches ?? []; + if (matches.length === 0) return; + const next = (matchIndex + delta + matches.length) % matches.length; + setMatchIndex(next); + centerRef.current?.(matches[next]); + }, + [decorations?.searchMatches, matchIndex], + ); // Digest of the sorted node ids: identical structure across a stats-only // refresh yields the same key, so layout and collapse state are preserved. const structureKey = data @@ -134,22 +182,52 @@ const DataflowDetailPage = () => { ) : data.structure.nodes.size <= 1 ? ( This dataflow contains no operators. ) : ( - - setSelectedNode(null)} - /> - {selectedNode && ( - setSelectedNode(null)} + + + - )} - + {filters.heatmap !== "off" && ( + + + low + + + + high + + + )} + + + setSelectedNode(null)} + /> + {selectedNode && ( + setSelectedNode(null)} + /> + )} + + )} diff --git a/console/src/platform/dataflows/DataflowGraphView.tsx b/console/src/platform/dataflows/DataflowGraphView.tsx index d9ddccb408dc6..864bf01eb1069 100644 --- a/console/src/platform/dataflows/DataflowGraphView.tsx +++ b/console/src/platform/dataflows/DataflowGraphView.tsx @@ -18,6 +18,7 @@ import { type Node, Position, ReactFlow, + useReactFlow, } from "@xyflow/react"; import React from "react"; @@ -57,8 +58,38 @@ export interface DataflowGraphViewProps { decorations?: GraphDecorations; onNodeClick?: (node: VisibleNode) => void; onPaneClick?: () => void; + centerRef?: React.MutableRefObject<((id: string) => void) | null>; } +// Exposes a centering callback through the ref once React Flow context exists. +const CenterHelper = ({ + centerRef, +}: { + centerRef: React.MutableRefObject<((id: string) => void) | null>; +}) => { + const reactFlow = useReactFlow(); + React.useEffect(() => { + // Assigning to the ref inside an effect is safe: it never runs during + // render, so the parent's ref just receives the latest centering callback. + // eslint-disable-next-line react-compiler/react-compiler + centerRef.current = (id: string) => { + const internal = reactFlow.getInternalNode(id); + if (!internal) return; + const { x, y } = internal.internals.positionAbsolute; + const width = internal.measured?.width ?? 0; + const height = internal.measured?.height ?? 0; + reactFlow.setCenter(x + width / 2, y + height / 2, { + zoom: 1, + duration: 300, + }); + }; + return () => { + centerRef.current = null; + }; + }, [reactFlow, centerRef]); + return null; +}; + export const DataflowGraphView = ({ structure, collapsed, @@ -67,6 +98,7 @@ export const DataflowGraphView = ({ decorations, onNodeClick, onPaneClick, + centerRef, }: DataflowGraphViewProps) => { const toast = useToast(); const visible = React.useMemo(() => { @@ -205,6 +237,7 @@ export const DataflowGraphView = ({ + {centerRef && } ); diff --git a/console/src/platform/dataflows/DataflowToolbar.tsx b/console/src/platform/dataflows/DataflowToolbar.tsx new file mode 100644 index 0000000000000..b1ea2d216ccfc --- /dev/null +++ b/console/src/platform/dataflows/DataflowToolbar.tsx @@ -0,0 +1,163 @@ +// Copyright Materialize, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +import { + Button, + Checkbox, + FormControl, + FormLabel, + HStack, + Input, + Menu, + MenuButton, + MenuItem, + MenuList, + Select, + Slider, + SliderFilledTrack, + SliderThumb, + SliderTrack, + Switch, + Text, +} from "@chakra-ui/react"; +import React from "react"; + +import { type Filters } from "./dataflowGraph"; + +export interface DataflowToolbarProps { + filters: Filters; + onFiltersChange: (next: Filters) => void; + channelTypes: string[]; + matchCount: number; + matchIndex: number; + onJump: (delta: 1 | -1) => void; +} + +export const DataflowToolbar = ({ + filters, + onFiltersChange, + channelTypes, + matchCount, + matchIndex, + onJump, +}: DataflowToolbarProps) => { + const [search, setSearch] = React.useState(filters.search); + // Debounce search input into the filters object. + React.useEffect(() => { + if (search === filters.search) return; + const timeout = setTimeout( + () => onFiltersChange({ ...filters, search }), + 300, + ); + return () => clearTimeout(timeout); + }, [search, filters, onFiltersChange]); + return ( + + setSearch(e.target.value)} + /> + {filters.search && ( + + + + + {matchCount === 0 ? "0/0" : `${matchIndex + 1}/${matchCount}`} + + + )} + + + Hide idle + + + onFiltersChange({ ...filters, hideIdle: e.target.checked }) + } + /> + + + onFiltersChange({ ...filters, heatmapThreshold: v })} + > + + + + + + + + Channel types + + + {channelTypes.map((t) => ( + + { + const current = filters.channelTypes ?? channelTypes; + const next = e.target.checked + ? [...current, t] + : current.filter((x) => x !== t); + onFiltersChange({ + ...filters, + channelTypes: + next.length === channelTypes.length ? null : next, + }); + }} + > + {t} + + + ))} + + + + ); +}; diff --git a/console/src/platform/dataflows/dataflowGraph.test.ts b/console/src/platform/dataflows/dataflowGraph.test.ts index b053db1ec5d54..e9094d2d2a944 100644 --- a/console/src/platform/dataflows/dataflowGraph.test.ts +++ b/console/src/platform/dataflows/dataflowGraph.test.ts @@ -10,10 +10,14 @@ import { describe, expect, it } from "vitest"; import { + allChannelTypes, buildDataflowStructure, type ChannelRow, + decorateGraph, + DEFAULT_FILTERS, defaultCollapseState, deriveVisibleGraph, + expandForSearch, type LirSpanRow, MAX_VISIBLE_NODES, nodeIdOf, @@ -227,3 +231,106 @@ describe("deriveVisibleGraph", () => { expect(MAX_VISIBLE_NODES).toEqual(1500); }); }); + +describe("decorateGraph", () => { + const s = buildDataflowStructure(OPS, CHANNELS, LIR_SPANS); + const expanded = deriveVisibleGraph(s, new Set()); + const heat = (t: number) => `heat(${t.toFixed(2)})`; + + it("search dims non-matching leaf nodes and lists matches", () => { + const d = decorateGraph( + expanded, + { ...DEFAULT_FILTERS, search: "join" }, + heat, + ); + expect(d.searchMatches).toEqual([nodeIdOf([5, 1, 1])]); + expect(d.dimmedNodeIds.has(nodeIdOf([5, 2]))).toBe(true); + expect(d.dimmedNodeIds.has(nodeIdOf([5, 1, 1]))).toBe(false); + }); + + it("hideIdle hides zero-activity operators and zero-message edges, never regions or ports", () => { + const d = decorateGraph( + expanded, + { ...DEFAULT_FILTERS, hideIdle: true }, + heat, + ); + // every fixture operator has elapsed > 0 except none; hide check via a synthetic idle op + const idle = buildDataflowStructure( + [ + ...OPS, + { + id: "15", + address: ["5", "3"], + name: "Idle", + arrangementRecords: "0", + arrangementSize: "0", + elapsedNs: "0", + }, + ], + CHANNELS, + [], + ); + const d2 = decorateGraph( + deriveVisibleGraph(idle, new Set()), + { ...DEFAULT_FILTERS, hideIdle: true }, + heat, + ); + expect(d2.hiddenNodeIds.has(nodeIdOf([5, 3]))).toBe(true); + expect(d2.hiddenNodeIds.has(nodeIdOf([5, 1]))).toBe(false); + expect(d.hiddenEdgeIds.size).toBe(0); // all fixture edges have messages + }); + + it("channel type filter dims non-matching edges", () => { + const d = decorateGraph( + expanded, + { ...DEFAULT_FILTERS, channelTypes: ["batches"] }, + heat, + ); + const rowsEdge = expanded.edges.find((e) => + e.channelTypes.includes("rows"), + )!; + const batchesEdge = expanded.edges.find((e) => + e.channelTypes.includes("batches"), + )!; + expect(d.dimmedEdgeIds.has(rowsEdge.id)).toBe(true); + expect(d.dimmedEdgeIds.has(batchesEdge.id)).toBe(false); + }); + + it("heatmap colors by transitive metric and dims below threshold", () => { + const d = decorateGraph( + expanded, + { ...DEFAULT_FILTERS, heatmap: "elapsed", heatmapThreshold: 0.5 }, + heat, + ); + // max transitive elapsed among visible non-ports is the region (13n) + expect(d.nodeColors.get(nodeIdOf([5, 1]))).toEqual("heat(1.00)"); + expect(d.dimmedNodeIds.has(nodeIdOf([5, 2]))).toBe(true); // 2/13 < 0.5 + }); + + it("heatmap with all-zero metric sets no colors", () => { + const zero = buildDataflowStructure( + OPS.map((o) => ({ ...o, elapsedNs: "0" })), + CHANNELS, + [], + ); + const d = decorateGraph( + deriveVisibleGraph(zero, new Set()), + { ...DEFAULT_FILTERS, heatmap: "elapsed" }, + heat, + ); + expect(d.nodeColors.size).toBe(0); + }); +}); + +describe("expandForSearch / allChannelTypes", () => { + const s = buildDataflowStructure(OPS, CHANNELS, LIR_SPANS); + + it("expands ancestors of matches", () => { + const next = expandForSearch(s, defaultCollapseState(s), "join"); + expect(next.has(nodeIdOf([5, 1]))).toBe(false); + }); + + it("collects channel types", () => { + expect(allChannelTypes(s)).toEqual(["batches", "rows"]); + }); +}); diff --git a/console/src/platform/dataflows/dataflowGraph.ts b/console/src/platform/dataflows/dataflowGraph.ts index 58327b409b849..18b17f849df50 100644 --- a/console/src/platform/dataflows/dataflowGraph.ts +++ b/console/src/platform/dataflows/dataflowGraph.ts @@ -331,12 +331,113 @@ export function visibleNodeCount( return deriveVisibleGraph(structure, collapsed).nodes.length; } -// All-optional here. Task 13's decorateGraph returns all fields set. +export interface Filters { + search: string; + hideIdle: boolean; + heatmap: "off" | "elapsed" | "size"; + heatmapThreshold: number; // 0..1 fraction of max + channelTypes: string[] | null; // null = all +} + +export const DEFAULT_FILTERS: Filters = { + search: "", + hideIdle: false, + heatmap: "off", + heatmapThreshold: 0, + channelTypes: null, +}; + +// decorateGraph returns every field set. export interface GraphDecorations { - dimmedNodeIds?: ReadonlySet; - hiddenNodeIds?: ReadonlySet; - hiddenEdgeIds?: ReadonlySet; - dimmedEdgeIds?: ReadonlySet; - nodeColors?: ReadonlyMap; // heatmap override - searchMatches?: string[]; + dimmedNodeIds: Set; + hiddenNodeIds: Set; + hiddenEdgeIds: Set; + dimmedEdgeIds: Set; + nodeColors: Map; + searchMatches: string[]; // visible node ids, document order +} + +export function allChannelTypes(structure: DataflowStructure): string[] { + const types = new Set(); + for (const c of structure.channels) + if (c.channelType) types.add(c.channelType); + return [...types].sort(); +} + +export function decorateGraph( + graph: VisibleGraph, + filters: Filters, + heatColor: (t: number) => string, +): GraphDecorations { + const d: GraphDecorations = { + dimmedNodeIds: new Set(), + hiddenNodeIds: new Set(), + hiddenEdgeIds: new Set(), + dimmedEdgeIds: new Set(), + nodeColors: new Map(), + searchMatches: [], + }; + const needle = filters.search.trim().toLowerCase(); + for (const n of graph.nodes) { + if (needle && n.kind !== "port" && n.kind !== "region") { + if (n.label.toLowerCase().includes(needle)) d.searchMatches.push(n.id); + else d.dimmedNodeIds.add(n.id); + } + if ( + filters.hideIdle && + n.kind === "operator" && + n.stats !== null && + n.stats.elapsedNs === 0n && + n.stats.arrangementRecords === 0n + ) { + d.hiddenNodeIds.add(n.id); + } + } + for (const e of graph.edges) { + if (filters.hideIdle && e.messagesSent === 0n) d.hiddenEdgeIds.add(e.id); + if ( + filters.channelTypes !== null && + !e.channelTypes.some((t) => filters.channelTypes!.includes(t)) + ) { + d.dimmedEdgeIds.add(e.id); + } + } + if (filters.heatmap !== "off") { + const metric = (n: VisibleNode): bigint => + filters.heatmap === "elapsed" + ? (n.transitive?.elapsedNs ?? 0n) + : (n.transitive?.arrangementSize ?? 0n); + const candidates = graph.nodes.filter((n) => n.kind !== "port"); + const max = candidates.reduce( + (m, n) => (metric(n) > m ? metric(n) : m), + 0n, + ); + if (max > 0n) { + for (const n of candidates) { + const t = Number(metric(n)) / Number(max); + d.nodeColors.set(n.id, heatColor(t)); + if (t < filters.heatmapThreshold) d.dimmedNodeIds.add(n.id); + } + } + } + return d; +} + +// Search may match inside collapsed regions. Returns a collapse state with +// every ancestor of every matching node expanded (respecting nothing else). +export function expandForSearch( + structure: DataflowStructure, + collapsed: CollapseState, + search: string, +): CollapseState { + const needle = search.trim().toLowerCase(); + if (!needle) return collapsed; + const next = new Set(collapsed); + for (const node of structure.nodes.values()) { + if (!node.name.toLowerCase().includes(needle)) continue; + for (let len = 1; len < node.address.length; len++) { + next.delete(nodeIdOf(node.address.slice(0, len))); + } + } + return next; } From be28c7aaf2aba3b65c6ffa005714c64bb958225e Mon Sep 17 00:00:00 2001 From: Moritz Hoffmann Date: Wed, 8 Jul 2026 22:42:22 +0200 Subject: [PATCH 12/45] Add the LIR operator panel, error states, and CNS-109 regression test Co-Authored-By: Claude Sonnet 5 --- .../dataflow/useDataflowGraphData.ts | 17 +- .../dataflows/DataflowDetailPage.test.tsx | 173 ++++++++++++++++++ .../platform/dataflows/DataflowDetailPage.tsx | 46 +++-- console/src/platform/dataflows/LirPanel.tsx | 70 +++++++ .../platform/dataflows/dataflowGraph.test.ts | 13 ++ .../src/platform/dataflows/dataflowGraph.ts | 18 ++ 6 files changed, 319 insertions(+), 18 deletions(-) create mode 100644 console/src/platform/dataflows/DataflowDetailPage.test.tsx create mode 100644 console/src/platform/dataflows/LirPanel.tsx diff --git a/console/src/api/materialize/dataflow/useDataflowGraphData.ts b/console/src/api/materialize/dataflow/useDataflowGraphData.ts index f6ba58fe12d0b..6bad322aff698 100644 --- a/console/src/api/materialize/dataflow/useDataflowGraphData.ts +++ b/console/src/api/materialize/dataflow/useDataflowGraphData.ts @@ -78,16 +78,25 @@ export function useDataflowGraphData(params?: DataflowGraphParams) { "toOperatorAddress", mdc.to_port, mdco.type` .$castTo() .compile(queryBuilder), + // mz_lir_mapping is keyed by the built object's global id (often a + // transient id), which mz_compute_exports does not bridge to a dataflow. + // Operator id ranges do: a span belongs to this dataflow when one of the + // dataflow's operators falls inside its [start, end) range. lirSpans: sql` - SELECT - mce.export_id AS "exportId", + SELECT DISTINCT + mlm.global_id AS "exportId", mlm.lir_id::text AS "lirId", mlm.operator, mlm.operator_id_start AS "operatorIdStart", mlm.operator_id_end AS "operatorIdEnd" FROM mz_lir_mapping AS mlm - JOIN mz_compute_exports AS mce ON mlm.global_id = mce.export_id - WHERE mce.dataflow_id = ${id}` + WHERE EXISTS ( + SELECT 1 + FROM mz_dataflow_operator_dataflows AS dod + WHERE dod.dataflow_id = ${id} + AND dod.id >= mlm.operator_id_start + AND dod.id < mlm.operator_id_end + )` .$castTo() .compile(queryBuilder), }; diff --git a/console/src/platform/dataflows/DataflowDetailPage.test.tsx b/console/src/platform/dataflows/DataflowDetailPage.test.tsx new file mode 100644 index 0000000000000..64f3f24cf4be9 --- /dev/null +++ b/console/src/platform/dataflows/DataflowDetailPage.test.tsx @@ -0,0 +1,173 @@ +// Copyright Materialize, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +import { screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { http, HttpResponse } from "msw"; +import React from "react"; +import { Route, Routes } from "react-router-dom"; + +import type { Cluster } from "~/api/materialize/cluster/clusterList"; +import { ErrorCode } from "~/api/materialize/types"; +import server from "~/api/mocks/server"; +import { getStore } from "~/jotai"; +import { allClusters } from "~/store/allClusters"; +import { mockSubscribeState } from "~/test/mockSubscribe"; +import { renderComponent } from "~/test/utils"; + +import DataflowDetailPage from "./DataflowDetailPage"; + +// jsdom lacks ResizeObserver/DOMMatrixReadOnly, so replace @xyflow/react with a +// flat renderer that preserves the decision under test (whether the canvas and +// its nodes exist). This mirrors src/test/mockReactFlow.tsx but also stubs +// useReactFlow, which DataflowDetailPage's centering helper reads and the +// shared helper does not provide. +vi.mock("@xyflow/react", () => ({ + ReactFlow: ({ + nodes, + children, + }: { + nodes: { id: string; data: { node: { label: string } } }[]; + children?: React.ReactNode; + }) => ( +
+ {nodes.map((n) => ( +
+ {n.data.node.label} +
+ ))} + {children} +
+ ), + Background: () => null, + Controls: () => null, + MiniMap: () => null, + Handle: () => null, + Position: { Left: "left", Right: "right", Top: "top", Bottom: "bottom" }, + BaseEdge: () => null, + EdgeLabelRenderer: ({ children }: { children: React.ReactNode }) => ( + <>{children} + ), + getBezierPath: () => ["", 0, 0], + useReactFlow: () => ({ + getInternalNode: () => undefined, + setCenter: () => {}, + }), +})); + +// elkjs runs in a web worker that jsdom cannot load, so replace the layout hook +// with a synchronous stub. The Proxy resolves any node id to a fixed box, so +// every visible node lands on the mocked React Flow canvas. +vi.mock("./useElkLayout", () => ({ + useElkLayout: () => ({ + positions: new Proxy( + {}, + { get: () => ({ x: 0, y: 0, width: 100, height: 40 }) }, + ), + layouting: false, + error: null, + }), +})); + +const FAILING_REPLICA = "r2"; + +const cluster = { + id: "u5", + name: "test_cluster", + replicas: [{ name: "r1" }, { name: "r2" }], +} as unknown as Cluster; + +const okResult = { + desc: { columns: [] }, + rows: [], +}; +// Two operators (a root and one child) so buildDataflowStructure yields a +// graph with more than one node and DataflowDetailPage renders the canvas. +const operatorsResult = { + desc: { + columns: [ + { name: "id" }, + { name: "address" }, + { name: "name" }, + { name: "arrangementRecords" }, + { name: "arrangementSize" }, + { name: "elapsedNs" }, + ], + }, + rows: [ + ["10", ["7"], "Dataflow", "0", "0", "0"], + ["11", ["7", "1"], "Map", "0", "0", "1"], + ], +}; + +beforeEach(() => { + const store = getStore(); + store.set(allClusters, mockSubscribeState({ data: [cluster] })); + server.use( + http.post("*/api/sql", async ({ request }) => { + const options = JSON.parse( + new URL(request.url).searchParams.get("options") ?? "{}", + ); + if (options.cluster_replica === FAILING_REPLICA) { + return HttpResponse.json({ + results: [ + { + error: { + message: "no such replica", + code: ErrorCode.INTERNAL_ERROR, + }, + }, + ], + }); + } + return HttpResponse.json({ + results: [operatorsResult, okResult, okResult], + }); + }), + ); +}); + +describe("DataflowDetailPage", () => { + // CNS-109: switching to a replica whose query fails must surface the error + // state and drop the graph from the previously successful replica. + it("shows the error state and hides the stale graph when the new replica fails", async () => { + await renderComponent( + + } + /> + , + { + initialRouterEntries: ["/clusters/u5/test_cluster/dataflows/7"], + }, + ); + + // r1 succeeds, so the graph renders first. + expect(await screen.findByTestId("react-flow")).toBeVisible(); + + // Switch to r2, whose query fails. The toolbar renders its own selects, so + // pick the replica select by its options rather than the ambiguous role. + const replicaSelect = screen + .getAllByRole("combobox") + .find((select) => + Array.from((select as HTMLSelectElement).options).some( + (option) => option.value === FAILING_REPLICA, + ), + ); + expect(replicaSelect).toBeDefined(); + await userEvent.selectOptions(replicaSelect!, FAILING_REPLICA); + + expect( + await screen.findByText("There was an error visualizing your dataflow"), + ).toBeVisible(); + // The stale graph from r1 must be gone. + expect(screen.queryByTestId("react-flow")).toBeNull(); + }); +}); diff --git a/console/src/platform/dataflows/DataflowDetailPage.tsx b/console/src/platform/dataflows/DataflowDetailPage.tsx index 9fab1f1390772..a1c424adab443 100644 --- a/console/src/platform/dataflows/DataflowDetailPage.tsx +++ b/console/src/platform/dataflows/DataflowDetailPage.tsx @@ -19,7 +19,7 @@ import { } from "@chakra-ui/react"; import { interpolateYlOrRd } from "d3-scale-chromatic"; import React from "react"; -import { useParams, useSearchParams } from "react-router-dom"; +import { Link, useParams, useSearchParams } from "react-router-dom"; import { useDataflowGraphData } from "~/api/materialize/dataflow/useDataflowGraphData"; import { ErrorCode } from "~/api/materialize/types"; @@ -41,6 +41,7 @@ import { } from "./dataflowGraph"; import { DataflowGraphView } from "./DataflowGraphView"; import { DataflowToolbar } from "./DataflowToolbar"; +import { LirPanel } from "./LirPanel"; import { NodeDetailPanel } from "./NodeDetailPanel"; // Injected into decorateGraph so the pure graph module stays d3-free. @@ -79,19 +80,24 @@ const DataflowDetailPage = () => { ); const [filters, setFilters] = React.useState(DEFAULT_FILTERS); const [matchIndex, setMatchIndex] = React.useState(0); + const [lirHighlight, setLirHighlight] = + React.useState | null>(null); const centerRef = React.useRef<((id: string) => void) | null>(null); - const decorations = React.useMemo( - () => - data && collapsed - ? decorateGraph( - deriveVisibleGraph(data.structure, collapsed), - filters, - heatColor, - ) - : undefined, - [data, collapsed, filters], - ); + const decorations = React.useMemo(() => { + if (!data || !collapsed) return undefined; + const visible = deriveVisibleGraph(data.structure, collapsed); + const d = decorateGraph(visible, filters, heatColor); + // A hovered LIR row dims every visible node outside its member set, so the + // members stand out. Members that are collapsed away simply have no visible + // node to keep lit. + if (lirHighlight) { + for (const n of visible.nodes) { + if (!lirHighlight.has(n.id)) d.dimmedNodeIds.add(n.id); + } + } + return d; + }, [data, collapsed, filters, lirHighlight]); // New search: expand ancestors of matches once, reset the cursor. React.useEffect(() => { @@ -176,11 +182,19 @@ const DataflowDetailPage = () => { ) : error ? ( - + + + + ) : !data || !collapsed ? ( ) : data.structure.nodes.size <= 1 ? ( - This dataflow contains no operators. + + This dataflow no longer exists on this replica.{" "} + Back to dataflows + ) : ( @@ -210,6 +224,10 @@ const DataflowDetailPage = () => { )} + | null) => void; +}) => { + const index = React.useMemo(() => lirIndex(structure), [structure]); + const byExport = React.useMemo(() => { + const groups = new Map< + string, + { key: string; info: LirInfo; memberIds: NodeId[] }[] + >(); + for (const [key, entry] of index) { + const list = groups.get(entry.info.exportId) ?? []; + list.push({ key, ...entry }); + groups.set(entry.info.exportId, list); + } + return groups; + }, [index]); + if (index.size === 0) return null; + return ( + + {[...byExport.entries()].map(([exportId, entries]) => ( + + + {exportId} + + {entries.map((e) => ( + onHighlight(new Set(e.memberIds))} + onMouseLeave={() => onHighlight(null)} + > + LIR {e.info.lirId}: {e.info.operator} + + ))} + + ))} + + ); +}; diff --git a/console/src/platform/dataflows/dataflowGraph.test.ts b/console/src/platform/dataflows/dataflowGraph.test.ts index e9094d2d2a944..44c4922f81684 100644 --- a/console/src/platform/dataflows/dataflowGraph.test.ts +++ b/console/src/platform/dataflows/dataflowGraph.test.ts @@ -18,6 +18,7 @@ import { defaultCollapseState, deriveVisibleGraph, expandForSearch, + lirIndex, type LirSpanRow, MAX_VISIBLE_NODES, nodeIdOf, @@ -334,3 +335,15 @@ describe("expandForSearch / allChannelTypes", () => { expect(allChannelTypes(s)).toEqual(["batches", "rows"]); }); }); + +describe("lirIndex", () => { + it("buckets member nodes by export/lir id", () => { + const s = buildDataflowStructure(OPS, CHANNELS, LIR_SPANS); + const index = lirIndex(s); + expect([...index.keys()]).toEqual(["u42/1"]); + // span 11..13 covers operator ids 11 ([5,1]) and 12 ([5,1,1]) + expect(index.get("u42/1")!.memberIds.sort()).toEqual( + [nodeIdOf([5, 1]), nodeIdOf([5, 1, 1])].sort(), + ); + }); +}); diff --git a/console/src/platform/dataflows/dataflowGraph.ts b/console/src/platform/dataflows/dataflowGraph.ts index 18b17f849df50..8d68ba4c08db3 100644 --- a/console/src/platform/dataflows/dataflowGraph.ts +++ b/console/src/platform/dataflows/dataflowGraph.ts @@ -441,3 +441,21 @@ export function expandForSearch( } return next; } + +// Groups operator nodes by the LIR span covering them, keyed +// `${exportId}/${lirId}`. One dataflow can back several exports, so entries +// are not unique per lir id across exports. +export function lirIndex( + structure: DataflowStructure, +): Map { + const index = new Map(); + for (const node of structure.nodes.values()) { + for (const info of node.lir) { + const key = `${info.exportId}/${info.lirId}`; + const entry = index.get(key) ?? { info, memberIds: [] }; + entry.memberIds.push(node.id); + index.set(key, entry); + } + } + return index; +} From b9eb34065d85e15cf6cd963bb3b84225a940b99b Mon Sep 17 00:00:00 2001 From: Moritz Hoffmann Date: Wed, 8 Jul 2026 22:42:23 +0200 Subject: [PATCH 13/45] console: delete graphviz dataflow visualizer and d3-graphviz dep Co-Authored-By: Claude Sonnet 5 --- console/package.json | 2 - .../api/materialize/useDataflowStructure.ts | 162 -------- .../platform/clusters/DataflowVisualizer.tsx | 389 ------------------ console/yarn.lock | 94 +---- 4 files changed, 5 insertions(+), 642 deletions(-) delete mode 100644 console/src/api/materialize/useDataflowStructure.ts delete mode 100644 console/src/platform/clusters/DataflowVisualizer.tsx diff --git a/console/package.json b/console/package.json index 633c209842e32..69cec039294f7 100644 --- a/console/package.json +++ b/console/package.json @@ -59,7 +59,6 @@ "@tanstack/react-query-devtools": "^5.95.0", "@tanstack/react-table": "^8.21.3", "@types/d3": "^7.4.3", - "@types/d3-graphviz": "^2.6.10", "@types/debug": "^4.1.12", "@types/react-table": "^7.7.20", "@visx/axis": "^3.12.0", @@ -78,7 +77,6 @@ "codemirror": "^6.0.1", "copy-to-clipboard": "^3.3.3", "d3": "^7.9.0", - "d3-graphviz": "^5.6.0", "d3-scale-chromatic": "^3.1.0", "date-fns": "^4.1.0", "date-fns-tz": "^3.2.0", diff --git a/console/src/api/materialize/useDataflowStructure.ts b/console/src/api/materialize/useDataflowStructure.ts deleted file mode 100644 index a5cc9fe897f86..0000000000000 --- a/console/src/api/materialize/useDataflowStructure.ts +++ /dev/null @@ -1,162 +0,0 @@ -// Copyright Materialize, Inc. and contributors. All rights reserved. -// -// Use of this software is governed by the Business Source License -// included in the LICENSE file. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0. - -import { sql } from "kysely"; -import React from "react"; - -import { escapedLiteral as lit, useSqlManyTyped } from "~/api/materialize"; - -import { queryBuilder } from "./db"; - -export interface DataflowStructureParams { - clusterName: string; - replicaName: string; - objectId: string; -} - -export type DataflowStructureResult = ReturnType< - typeof useDataflowStructure ->["results"]; - -export type Operator = { - id: bigint; - address: string[]; - name: string; - parentId: bigint | null; - arrangementRecords: bigint | null; - arrangementSizes: bigint | null; - elapsedNs: number; - lirId: string | null; - lirOperator: string | null; -}; - -export type Channel = { - id: number; - fromOperatorId: number; - fromOperatorAddress: string[]; - fromPort: number; - toOperatorId: number; - toOperatorAddress: string[]; - toPort: number; - messagesSent: number; - batchesSent: number; - channelType: string; -}; - -export type LirOperator = { - lir_id: string; - operator: string; - addresses: string[][]; -}; - -export function useDataflowStructure(params?: DataflowStructureParams) { - const queries = React.useMemo(() => { - if (!params) { - return null; - } - const { objectId } = params; - - return { - channels: sql` - SELECT - mdco.id, - from_operator_id AS "fromOperatorId", - from_operator_address AS "fromOperatorAddress", - from_port AS "fromPort", - to_operator_id AS "toOperatorId", - to_operator_address AS "toOperatorAddress", - to_port AS "toPort", - COALESCE(sum(sent), 0) AS "messagesSent", - COALESCE(sum(batch_sent), 0) AS "batchesSent", - mdc.type as "channelType" - FROM - mz_dataflow_channel_operators AS mdco - JOIN mz_dataflow_channels AS mdc ON - mdc.id = mdco.id - LEFT JOIN mz_message_counts AS mmc ON - mdco.id = mmc.channel_id - JOIN mz_compute_exports mce ON mce.dataflow_id = from_operator_address[1] - WHERE mce.export_id = ${lit(objectId)} - GROUP BY - mdco.id, - "fromOperatorId", - "fromOperatorAddress", - "toOperatorId", - "toOperatorAddress", - "fromPort", - "toPort", - mdc."type"` - .$castTo() - .compile(queryBuilder), - - operators: sql` - WITH - export_to_dataflow AS ( - SELECT export_id, id FROM mz_compute_exports AS mce JOIN mz_dataflows AS md ON mce.dataflow_id = md.id - ), - all_ops AS ( - SELECT - e2d.export_id, mdod.id, mda.address, mdod.name, mdop.parent_id AS "parentId", - coalesce(mas.records, 0) AS "arrangementRecords", - coalesce(mas.size, 0) AS "arrangementSizes", - coalesce(mse.elapsed_ns, 0) AS "elapsedNs", - mlm.lir_id::text AS "lirId", - mlm.operator AS "lirOperator" - FROM - export_to_dataflow AS e2d - JOIN mz_dataflow_operator_dataflows - AS mdod ON e2d.id = mdod.dataflow_id - LEFT JOIN mz_scheduling_elapsed - AS mse ON mdod.id = mse.id - LEFT JOIN mz_arrangement_sizes - AS mas ON mdod.id = mas.operator_id - LEFT JOIN mz_dataflow_operator_parents - AS mdop ON mdod.id = mdop.id - LEFT JOIN mz_dataflow_addresses - AS mda ON mdod.id = mda.id - LEFT JOIN mz_lir_mapping - AS mlm ON mlm.operator_id_start <= mdod.id AND mdod.id < mlm.operator_id_end AND mlm.global_id = export_id - ) - SELECT - id, - address, - name, - "parentId", - "arrangementRecords", - "arrangementSizes", - "elapsedNs", - "lirId", - "lirOperator" - FROM all_ops WHERE export_id = ${lit(objectId)}` - .$castTo() - .compile(queryBuilder), - - lir_operators: sql` - SELECT lir_id, operator, list_agg(mda.address) AS addresses - FROM - mz_lir_mapping AS mlm - LEFT JOIN mz_dataflow_operators AS mdo - ON (mlm.operator_id_start <= mdo.id - AND mdo.id < mlm.operator_id_end) - JOIN mz_dataflow_addresses AS mda - ON mdo.id = mda.id - WHERE mlm.global_id = ${lit(objectId)} - GROUP BY lir_id, operator` - .$castTo() - .compile(queryBuilder), - }; - }, [params]); - const { clusterName, replicaName } = params ?? {}; - return useSqlManyTyped(queries, { - cluster: clusterName, - replica: replicaName, - // This query can be slow for large dataflows - timeout: 30_000, - }); -} diff --git a/console/src/platform/clusters/DataflowVisualizer.tsx b/console/src/platform/clusters/DataflowVisualizer.tsx deleted file mode 100644 index a228be7e4885b..0000000000000 --- a/console/src/platform/clusters/DataflowVisualizer.tsx +++ /dev/null @@ -1,389 +0,0 @@ -// Copyright Materialize, Inc. and contributors. All rights reserved. -// -// Use of this software is governed by the Business Source License -// included in the LICENSE file. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0. - -import { - Alert, - AlertIcon, - Box, - Button, - HStack, - Spinner, - Text, - VStack, -} from "@chakra-ui/react"; -import * as d3 from "d3"; -import { graphviz } from "d3-graphviz"; -import React from "react"; -import { useParams } from "react-router-dom"; - -import { Replica } from "~/api/materialize/cluster/clusterList"; -import { ErrorCode } from "~/api/materialize/types"; -import { - Channel, - LirOperator, - Operator, - useDataflowStructure, -} from "~/api/materialize/useDataflowStructure"; -import ErrorBox from "~/components/ErrorBox"; -import LabeledSelect from "~/components/LabeledSelect"; -import { MainContentContainer } from "~/layouts/BaseLayout"; -import { useAllClusters } from "~/store/allClusters"; -import { useAllObjects } from "~/store/allObjects"; -import { assert } from "~/util"; -import { formatBytesShort } from "~/utils/format"; - -interface EnrichedOperator extends Operator { - // First-level. - channelsInScope: Channel[]; - children: EnrichedOperator[]; - transitiveArrangementRecords: bigint | null; - transitiveArrangementSizes: bigint | null; -} - -type EnrichedLirOperator = { - lir_id: string; - operator: string; - addresses: string[]; -}; - -function groupBy(values: T[], group: (item: T) => K): Map { - const output = new Map(); - for (const v of values) { - const k = group(v); - if (!output.has(k)) { - output.set(k, []); - } - output.get(k)!.push(v); - } - return output; -} - -// Returns a map of (stringified) operator address to corresponding operators, -// as well as a designated root. -function collateOperators( - operators: Operator[], - channels: Channel[], - lirOperators: LirOperator[], -): [Map, EnrichedOperator, EnrichedLirOperator[]] { - const scopes = groupBy(operators, (o) => o.parentId); - const channelsByParentScope = groupBy(channels, (ch) => - stringifyAddress(ch.fromOperatorAddress.slice(0, -1)), - ); - - const roots = scopes.get(null) || []; - - function walk( - op: Operator, - m: Map, - ): EnrichedOperator { - assert(!m.has(stringifyAddress(op.address))); - const children = (scopes.get(op.id) || []).map((ch) => walk(ch, m)); - const channelsInScope = - channelsByParentScope.get(stringifyAddress(op.address)) || []; - const ret = { - ...op, - children, - channelsInScope, - transitiveArrangementRecords: - children - .map((child) => child.transitiveArrangementRecords || 0n) - .reduce((a, b) => a + b, 0n) + (op.arrangementRecords || 0n), - transitiveArrangementSizes: - children - .map((child) => child.transitiveArrangementSizes || 0n) - .reduce((a, b) => a + b, 0n) + (op.arrangementSizes || 0n), - }; - m.set(stringifyAddress(ret.address), ret); - return ret; - } - - const enrichedLirOperators: EnrichedLirOperator[] = lirOperators.map( - (lirOp) => ({ - lir_id: lirOp.lir_id, - operator: lirOp.operator, - addresses: lirOp.addresses.map((addr) => stringifyAddress(addr)), - }), - ); - - const m = new Map(); - - const enrichedRoots = roots.map((r) => walk(r, m)); - assert(enrichedRoots.length == 1); - return [m, enrichedRoots[0], enrichedLirOperators]; -} - -const noArrangementRegionColor = "#12b886"; -const noArrangementOperatorColor = "#ffffff"; -const arrangementRegionColor = "#7950f2"; -const arrangementOperatorColor = "#fab005"; - -function stringifyAddress(address: string[]) { - return JSON.stringify(address.map((val) => parseInt(val))); -} - -function scopeToGv( - scope: EnrichedOperator, - lir_operators: EnrichedLirOperator[], -): string { - const chunks = ["digraph {", 'node [style="filled",shape=box];']; - const addresses = new Set(); - for (const op of scope.children) { - const isRegion = op.children.length !== 0; - const hasArrangedData = (op.transitiveArrangementRecords || 0n) > 0n; - let fillColor; - if (isRegion) { - if (hasArrangedData) { - fillColor = arrangementRegionColor; - } else { - fillColor = noArrangementRegionColor; - } - } else { - if (hasArrangedData) { - fillColor = arrangementOperatorColor; - } else { - fillColor = noArrangementOperatorColor; - } - } - const nodeLabelFields = [op.name]; - if (hasArrangedData) { - nodeLabelFields.push( - `${op.transitiveArrangementRecords} arranged records`, - ); - nodeLabelFields.push( - formatBytesShort(BigInt(op.transitiveArrangementSizes || 0)), - ); - } - if (op.elapsedNs > 0) { - nodeLabelFields.push( - `scheduled ${Math.round(op.elapsedNs / 1_000_000_000)}s`, - ); - } - - const tooltip = `Lir ID ${op.lirId}: ${op.lirOperator}`.replace( - /"/g, - '\\"', - ); - - const opAddressString = stringifyAddress(op.address); - addresses.add(opAddressString); - - const nodeGv = `"${opAddressString}" [fillcolor="${fillColor}",tooltip="${tooltip}",id="${opAddressString}",label="${nodeLabelFields.join("\n").replace(/"/g, '\\"')}",class="${isRegion ? "region" : ""}"];`; - chunks.push(nodeGv); - } - const pseudoOperators = new Map(); - for (const ch of scope.channelsInScope) { - let fromAddressKey = stringifyAddress(ch.fromOperatorAddress); - let toAddressKey = stringifyAddress(ch.toOperatorAddress); - - if (ch.fromOperatorAddress[ch.fromOperatorAddress.length - 1] === "0") { - fromAddressKey = `${fromAddressKey}:${ch.fromPort}:FROM`; - pseudoOperators.set(fromAddressKey, `input ${ch.fromPort}`); - } - if (ch.toOperatorAddress[ch.toOperatorAddress.length - 1] === "0") { - toAddressKey = `${toAddressKey}:${ch.toPort}:TO`; - pseudoOperators.set(toAddressKey, `output ${ch.toPort}`); - } - const chanAttr = ch.messagesSent === 0 ? `,style="dashed"` : ""; - const chanLabel = `${ch.messagesSent > 0 ? `${ch.messagesSent} records` : ""}${ch.batchesSent > 0 ? `\n${ch.batchesSent} batches` : ""}`; - const tooltip = ch.channelType || "unknown channel type"; - const chanGv = `"${fromAddressKey}" -> "${toAddressKey}" [label="${chanLabel}",tooltip="${tooltip.replace(/"/g, '\\"')}"${chanAttr}];`; - chunks.push(chanGv); - } - - for (const [k, v] of pseudoOperators) { - chunks.push( - `"${k}" [fillcolor="lightgrey",id="${k}",label="${v.replace(/"/g, '\\"')}"]`, - ); - } - - for (const lir_operator of lir_operators) { - if (!lir_operator.addresses.some((addr) => addresses.has(addr))) continue; - chunks.push("subgraph cluster_" + lir_operator.lir_id + " {"); - chunks.push( - `label="LIR ID ${lir_operator.lir_id}: ${lir_operator.operator.replace(/"/g, '\\"')}";`, - ); - for (const addr of lir_operator.addresses) { - chunks.push(`"${addr}";`); - } - chunks.push("}"); - } - - chunks.push("}"); - const ret = chunks.join("\n"); - return ret; -} - -interface DotVizProps { - dot?: string; - onClickedNode: (id: string) => void; -} - -const DotViz = ({ dot, onClickedNode }: DotVizProps) => { - const d3Container = React.useRef(null); - React.useEffect(() => { - if (d3Container.current && dot) { - const gv = graphviz(d3Container.current) - .scale(0.5) - .attributer(function (d) { - if (d.tag === "svg") { - d.attributes.width = "100%"; - d.attributes.height = "100%"; - } - }); - gv.on("initEnd", () => { - gv.renderDot(dot, function () { - gv.resetZoom(); - - const regions = d3.selectAll(".region"); - regions.on("dblclick", function (event) { - const clickedId = event.currentTarget.getAttribute("id")!; - if (clickedId) { - event.stopPropagation(); - onClickedNode(clickedId); - } - }); - }); - }); - } - }, [dot, onClickedNode]); - return ; -}; - -const defaultReplicas: Replica[] = []; - -const DataflowVisualizer = () => { - const { getClusterById } = useAllClusters(); - const params = useParams(); - const { data: allObjects } = useAllObjects(); - const object = allObjects.find((o) => o.id === params.id); - assert(object && object.clusterId); - const cluster = getClusterById(object.clusterId); - const replicas = cluster?.replicas ?? defaultReplicas; - const [scopeBreadcrumb, setScopeBreadcrumb] = React.useState([]); - const [replicaName, setReplicaName] = React.useState(null); - - React.useEffect(() => { - if (replicas.length > 0 && replicaName === null) { - setReplicaName(replicas[0].name); - } - }, [replicaName, replicas]); - - // Reset scope if props changed. - // TODO - If we track scopes by - // address, rather than by operator ID, we can avoid resetting it - // when replica name changes (addresses other than the initial component - // are the same across replicas, whereas operator IDs aren't - React.useEffect(() => { - setScopeBreadcrumb([]); - }, [params.id, replicaName]); - const dfStructureParams = React.useMemo( - () => - cluster && replicaName - ? { clusterName: cluster.name, replicaName, objectId: object.id } - : undefined, - [cluster, object.id, replicaName], - ); - const { - results: structure, - failedToLoad, - databaseError, - loading, - } = useDataflowStructure(dfStructureParams); - const [allEnriched, root, lirOperators] = React.useMemo(() => { - return structure && structure.operators.length > 0 - ? collateOperators( - structure.operators, - structure.channels, - structure.lir_operators, - ) - : [null, null, []]; - }, [structure]); - const dot = React.useMemo(() => { - const scopeOperator = - scopeBreadcrumb.length > 0 && allEnriched - ? allEnriched.get(scopeBreadcrumb[scopeBreadcrumb.length - 1])! - : root; - return scopeOperator ? scopeToGv(scopeOperator, lirOperators) : undefined; - }, [allEnriched, root, lirOperators, scopeBreadcrumb]); - - const pushScope = React.useCallback( - (s: string) => { - const newBreadcrumb = [...scopeBreadcrumb, s]; - setScopeBreadcrumb(newBreadcrumb); - }, - [scopeBreadcrumb, setScopeBreadcrumb], - ); - - if (!cluster) return null; - - const permissionError = - databaseError && - "code" in databaseError && - databaseError.code === ErrorCode.INSUFFICIENT_PRIVILEGE; - return ( - - {permissionError ? ( - - - - You'll need{" "} - - USAGE - {" "} - privilege on this cluster to visualize this dataflow. - - - ) : failedToLoad ? ( - - ) : loading ? ( - - ) : ( - - {replicaName && ( - setReplicaName(e.target.value)} - flexShrink={0} - > - {cluster.replicas.map((r) => ( - - ))} - - )} - - {" "} - - - {dot === undefined ? ( - This dataflow contains no operators. - ) : ( - - )} - - )} - - ); -}; - -export default DataflowVisualizer; diff --git a/console/yarn.lock b/console/yarn.lock index ee09c48b756e5..eb488e48eb09e 100644 --- a/console/yarn.lock +++ b/console/yarn.lock @@ -2769,17 +2769,6 @@ __metadata: languageName: node linkType: hard -"@hpcc-js/wasm@npm:^2.20.0": - version: 2.20.0 - resolution: "@hpcc-js/wasm@npm:2.20.0" - dependencies: - yargs: "npm:17.7.2" - bin: - dot-wasm: node ./node_modules/@hpcc-js/wasm-graphviz-cli/bin/index.js - checksum: 10c0/925ce14d47389a99837b4fc7d97d270cd401a72e089c55d5fac62212ae652830c4f3f54475e46685a4fed0658c39598a82dd182468bfa603d29d647c419f1ea5 - languageName: node - linkType: hard - "@humanwhocodes/config-array@npm:^0.11.14": version: 0.11.14 resolution: "@humanwhocodes/config-array@npm:0.11.14" @@ -4451,13 +4440,6 @@ __metadata: languageName: node linkType: hard -"@types/d3-color@npm:^1": - version: 1.4.2 - resolution: "@types/d3-color@npm:1.4.2" - checksum: 10c0/f1c70d7deabe2b30e337361c3fa26b7ce4e9c875dfd1ad75e17379c6a5596f116fb21ca0e595c2f2220e04866bca6bdb2623e772cfb5f8bf4d20ffe6ba5c72ab - languageName: node - linkType: hard - "@types/d3-contour@npm:*": version: 3.0.2 resolution: "@types/d3-contour@npm:3.0.2" @@ -4555,17 +4537,6 @@ __metadata: languageName: node linkType: hard -"@types/d3-graphviz@npm:^2.6.10": - version: 2.6.10 - resolution: "@types/d3-graphviz@npm:2.6.10" - dependencies: - "@types/d3-selection": "npm:^1" - "@types/d3-transition": "npm:^1" - "@types/d3-zoom": "npm:^1" - checksum: 10c0/bbb4143dcef1ebac612f93e448a53291cec3f708489261dc54c657db0441ed0a87c3bdf5e902e6d695918b7a39ace15bcf07e2c828d6568611eee8b5c15fe6c4 - languageName: node - linkType: hard - "@types/d3-hierarchy@npm:*": version: 3.1.2 resolution: "@types/d3-hierarchy@npm:3.1.2" @@ -4582,15 +4553,6 @@ __metadata: languageName: node linkType: hard -"@types/d3-interpolate@npm:^1": - version: 1.4.2 - resolution: "@types/d3-interpolate@npm:1.4.2" - dependencies: - "@types/d3-color": "npm:^1" - checksum: 10c0/98bff93ce4d94485a4f6117e554854ec69072382910008e785d2c960b50e643093a8cfa2e0875b3d1dff19f3603b6e16a4eea8122c7c8ead3623daf3044cd22e - languageName: node - linkType: hard - "@types/d3-interpolate@npm:^3.0.4": version: 3.0.4 resolution: "@types/d3-interpolate@npm:3.0.4" @@ -4667,13 +4629,6 @@ __metadata: languageName: node linkType: hard -"@types/d3-selection@npm:^1": - version: 1.4.3 - resolution: "@types/d3-selection@npm:1.4.3" - checksum: 10c0/47c181f8362ade4df151e01737816356c939bc5728ff87c3a29bd43aaa0413296170119949eb3aa0ef8d9c10fac4463eb1d154b6fd0e89617b45eeb06bdefb8b - languageName: node - linkType: hard - "@types/d3-selection@npm:^3.0.10": version: 3.0.11 resolution: "@types/d3-selection@npm:3.0.11" @@ -4736,15 +4691,6 @@ __metadata: languageName: node linkType: hard -"@types/d3-transition@npm:^1": - version: 1.3.2 - resolution: "@types/d3-transition@npm:1.3.2" - dependencies: - "@types/d3-selection": "npm:^1" - checksum: 10c0/9e1340c2840fde63f224550cbde5531b8b2493218d421f97de138c1e7ca9b1f481dfc9e4c91cb5ce84df9df671e2d95468556be1f8cf1e28f0c33929322115a4 - languageName: node - linkType: hard - "@types/d3-transition@npm:^3.0.8": version: 3.0.9 resolution: "@types/d3-transition@npm:3.0.9" @@ -4764,16 +4710,6 @@ __metadata: languageName: node linkType: hard -"@types/d3-zoom@npm:^1": - version: 1.8.3 - resolution: "@types/d3-zoom@npm:1.8.3" - dependencies: - "@types/d3-interpolate": "npm:^1" - "@types/d3-selection": "npm:^1" - checksum: 10c0/d93ab0b610a68c02926c8388ed839fcc31ac0ee2ed19c69c59b5ea0cb87afd70515d507fc10c063eba5fc400389b3216f1bcdfe97ad84b75eac6c30e67cbccdf - languageName: node - linkType: hard - "@types/d3-zoom@npm:^3.0.8": version: 3.0.8 resolution: "@types/d3-zoom@npm:3.0.8" @@ -6944,7 +6880,7 @@ __metadata: languageName: node linkType: hard -"d3-dispatch@npm:1 - 3, d3-dispatch@npm:3, d3-dispatch@npm:^3.0.1": +"d3-dispatch@npm:1 - 3, d3-dispatch@npm:3": version: 3.0.1 resolution: "d3-dispatch@npm:3.0.1" checksum: 10c0/6eca77008ce2dc33380e45d4410c67d150941df7ab45b91d116dbe6d0a3092c0f6ac184dd4602c796dc9e790222bad3ff7142025f5fd22694efe088d1d941753 @@ -7009,7 +6945,7 @@ __metadata: languageName: node linkType: hard -"d3-format@npm:1 - 3, d3-format@npm:3, d3-format@npm:3.1.0, d3-format@npm:^3.1.0": +"d3-format@npm:1 - 3, d3-format@npm:3, d3-format@npm:3.1.0": version: 3.1.0 resolution: "d3-format@npm:3.1.0" checksum: 10c0/049f5c0871ebce9859fc5e2f07f336b3c5bfff52a2540e0bac7e703fce567cd9346f4ad1079dd18d6f1e0eaa0599941c1810898926f10ac21a31fd0a34b4aa75 @@ -7025,24 +6961,6 @@ __metadata: languageName: node linkType: hard -"d3-graphviz@npm:^5.6.0": - version: 5.6.0 - resolution: "d3-graphviz@npm:5.6.0" - dependencies: - "@hpcc-js/wasm": "npm:^2.20.0" - d3-dispatch: "npm:^3.0.1" - d3-format: "npm:^3.1.0" - d3-interpolate: "npm:^3.0.1" - d3-path: "npm:^3.1.0" - d3-timer: "npm:^3.0.1" - d3-transition: "npm:^3.0.1" - d3-zoom: "npm:^3.0.0" - peerDependencies: - d3-selection: ^3.0.0 - checksum: 10c0/ffbe5facb35cfe80b9bc1985acbd1facdab180c0eb0cfaccaf6f662e9f1d6be6bd8d20539a5fa04f555075b0c511154ebb567e96335b9d3283874f9f3b5b2744 - languageName: node - linkType: hard - "d3-hierarchy@npm:3": version: 3.1.2 resolution: "d3-hierarchy@npm:3.1.2" @@ -7160,14 +7078,14 @@ __metadata: languageName: node linkType: hard -"d3-timer@npm:1 - 3, d3-timer@npm:3, d3-timer@npm:^3.0.1": +"d3-timer@npm:1 - 3, d3-timer@npm:3": version: 3.0.1 resolution: "d3-timer@npm:3.0.1" checksum: 10c0/d4c63cb4bb5461d7038aac561b097cd1c5673969b27cbdd0e87fa48d9300a538b9e6f39b4a7f0e3592ef4f963d858c8a9f0e92754db73116770856f2fc04561a languageName: node linkType: hard -"d3-transition@npm:2 - 3, d3-transition@npm:3, d3-transition@npm:^3.0.1": +"d3-transition@npm:2 - 3, d3-transition@npm:3": version: 3.0.1 resolution: "d3-transition@npm:3.0.1" dependencies: @@ -11054,7 +10972,6 @@ __metadata: "@testing-library/user-event": "npm:^14.5.2" "@types/base32-encoding": "npm:^1.0.2" "@types/d3": "npm:^7.4.3" - "@types/d3-graphviz": "npm:^2.6.10" "@types/dagre": "npm:^0.7.54" "@types/debug": "npm:^4.1.12" "@types/jsonwebtoken": "npm:^9" @@ -11091,7 +11008,6 @@ __metadata: copy-to-clipboard: "npm:^3.3.3" core-js: "npm:^3.38.1" d3: "npm:^7.9.0" - d3-graphviz: "npm:^5.6.0" d3-scale-chromatic: "npm:^3.1.0" date-fns: "npm:^4.1.0" date-fns-tz: "npm:^3.2.0" @@ -15718,7 +15634,7 @@ __metadata: languageName: node linkType: hard -"yargs@npm:17.7.2, yargs@npm:^17.7.2": +"yargs@npm:^17.7.2": version: 17.7.2 resolution: "yargs@npm:17.7.2" dependencies: From e22e3435bd4b4e3e442e06faa1b2277525d48c7d Mon Sep 17 00:00:00 2001 From: Moritz Hoffmann Date: Wed, 8 Jul 2026 22:42:23 +0200 Subject: [PATCH 14/45] Drop dangling channels and route external-to-region channels through ports Co-Authored-By: Claude Sonnet 5 --- .../platform/dataflows/DataflowGraphView.tsx | 13 ++- .../platform/dataflows/dataflowGraph.test.ts | 83 +++++++++++++++++++ .../src/platform/dataflows/dataflowGraph.ts | 49 ++++++++++- console/src/platform/dataflows/elkGraph.ts | 2 +- console/src/platform/dataflows/nodes.tsx | 21 +++-- 5 files changed, 155 insertions(+), 13 deletions(-) diff --git a/console/src/platform/dataflows/DataflowGraphView.tsx b/console/src/platform/dataflows/DataflowGraphView.tsx index 864bf01eb1069..29a87ffbaf75b 100644 --- a/console/src/platform/dataflows/DataflowGraphView.tsx +++ b/console/src/platform/dataflows/DataflowGraphView.tsx @@ -169,6 +169,11 @@ export const DataflowGraphView = ({ // Bottom/Top and appears detached across region boundaries. targetPosition: Position.Top, sourcePosition: Position.Bottom, + // Set as explicit node fields, not just CSS style: the MiniMap and + // culled (onlyRenderVisibleElements) off-screen nodes both need a + // known size without waiting for DOM measurement. + width: pos.width, + height: pos.height, style: { width: pos.width, height: pos.height }, draggable: false, connectable: false, @@ -236,7 +241,13 @@ export const DataflowGraphView = ({ > - + + (node.data as { color?: string }).color ?? "#ccc" + } + /> {centerRef && } diff --git a/console/src/platform/dataflows/dataflowGraph.test.ts b/console/src/platform/dataflows/dataflowGraph.test.ts index 44c4922f81684..7fdb7fffb8d4e 100644 --- a/console/src/platform/dataflows/dataflowGraph.test.ts +++ b/console/src/platform/dataflows/dataflowGraph.test.ts @@ -231,6 +231,89 @@ describe("deriveVisibleGraph", () => { expect(visibleNodeCount(s, defaultCollapseState(s))).toEqual(2); expect(MAX_VISIBLE_NODES).toEqual(1500); }); + + it("routes an external-to-region channel through a port node, keyed by port number", () => { + // Materialize logs a scope boundary crossing as two channels sharing a + // port number but addressed differently: the outer half (external + // producer/consumer <-> scope) targets the scope's OWN operator address + // directly (no [..., 0] suffix), the same address the region's own + // operator row uses. Verified against live introspection data: an + // external channel `{5,4}->{5,2}` (to_port=0) pairs with the internal + // fan-out channel `{5,2,0}->{5,2,1}` (from_port=0). + const withExternal = buildDataflowStructure( + [ + ...OPS, + { + id: "20", + address: ["5", "3"], + name: "External", + arrangementRecords: "0", + arrangementSize: "0", + elapsedNs: "1", + }, + ], + [ + ...CHANNELS, + { + id: "98", + fromOperatorAddress: ["5", "3"], + fromPort: "0", + toOperatorAddress: ["5", "1"], + toPort: "0", + messagesSent: "9", + batchesSent: "3", + channelType: "rows", + }, + ], + LIR_SPANS, + ); + const externalId = nodeIdOf([5, 3]); + const portId = `${regionId}:in:0`; + + const expanded = deriveVisibleGraph(withExternal, new Set()); + expect( + expanded.nodes.some((n) => n.id === portId && n.kind === "port"), + ).toBe(true); + const edge = expanded.edges.find((e) => e.source === externalId)!; + expect(edge.target).toEqual(portId); + + // Collapsed, the crossing lands on the region's own (collapsed) node, + // matching how every other boundary crossing into a collapsed region + // behaves: the whole region is one box. + const collapsed = deriveVisibleGraph(withExternal, new Set([regionId])); + const collapsedEdge = collapsed.edges.find((e) => e.source === externalId)!; + expect(collapsedEdge.target).toEqual(regionId); + }); + + it("drops channels that reference an address with no operator row", () => { + // Real dataflows can log a channel touching a scope address that never + // got its own operator row (e.g. an elided region). [5, 5] does not + // appear in OPS. + const withDangling = buildDataflowStructure( + OPS, + [ + ...CHANNELS, + { + id: "99", + fromOperatorAddress: ["5", "5"], + fromPort: "0", + toOperatorAddress: ["5", "2"], + toPort: "0", + messagesSent: "1", + batchesSent: "1", + channelType: "rows", + }, + ], + LIR_SPANS, + ); + expect(() => deriveVisibleGraph(withDangling, new Set())).not.toThrow(); + const g = deriveVisibleGraph(withDangling, new Set()); + const nodeIds = new Set(g.nodes.map((n) => n.id)); + for (const e of g.edges) { + expect(nodeIds.has(e.source)).toBe(true); + expect(nodeIds.has(e.target)).toBe(true); + } + }); }); describe("decorateGraph", () => { diff --git a/console/src/platform/dataflows/dataflowGraph.ts b/console/src/platform/dataflows/dataflowGraph.ts index 8d68ba4c08db3..c8427dd7442fe 100644 --- a/console/src/platform/dataflows/dataflowGraph.ts +++ b/console/src/platform/dataflows/dataflowGraph.ts @@ -207,9 +207,20 @@ function representative(address: Address, collapsed: CollapseState): NodeId { return nodeIdOf(address); } -// A channel endpoint address ending in 0 denotes the enclosing scope's -// input (from side) or output (to side) port. +// A scope boundary crossing is logged as two channels that share a port +// number but use different addressing for each half: +// - the outer half (external <-> scope) addresses the scope by its own +// operator address, e.g. an external producer's channel row targets +// address [5,2] directly, the same address BuildRegion's own operator +// row uses; +// - the inner half (scope <-> child) addresses the scope's internal +// fan-out point one level deeper, e.g. [5,2,0]. +// Both halves are mapped to the same synthetic port id, keyed by (scope, +// direction, port number), so they visually chain through one port node +// instead of the outer half landing on the region's own node (indistinguishable +// from every other port sharing that scope) or the inner half dangling. function endpointId( + structure: DataflowStructure, address: Address, port: number, side: "from" | "to", @@ -229,6 +240,19 @@ function endpointId( port: { scope: scopeId, direction, port }, }; } + const scopeId = nodeIdOf(address); + const scopeNode = structure.nodes.get(scopeId); + if (scopeNode && scopeNode.children.length > 0) { + const rep = representative(address, collapsed); + if (rep === scopeId && !collapsed.has(scopeId)) { + const direction = side === "to" ? "input" : "output"; + return { + id: `${scopeId}:${direction === "input" ? "in" : "out"}:${port}`, + port: { scope: scopeId, direction, port }, + }; + } + return { id: rep }; + } return { id: representative(address, collapsed) }; } @@ -264,8 +288,25 @@ export function deriveVisibleGraph( const edgesById = new Map }>(); const ports = new Map(); for (const ch of structure.channels) { - const from = endpointId(ch.fromAddress, ch.fromPort, "from", collapsed); - const to = endpointId(ch.toAddress, ch.toPort, "to", collapsed); + const from = endpointId( + structure, + ch.fromAddress, + ch.fromPort, + "from", + collapsed, + ); + const to = endpointId(structure, ch.toAddress, ch.toPort, "to", collapsed); + // A non-port endpoint id names an operator or region address, which must + // exist in the structure. Real dataflows can log channels touching an + // address with no corresponding operator row (e.g. an elided scope), so + // drop the channel rather than hand elk a reference to a node that will + // never be emitted. + if ( + (!from.port && !structure.nodes.has(from.id)) || + (!to.port && !structure.nodes.has(to.id)) + ) { + continue; + } if (from.id === to.id) continue; for (const p of [from.port, to.port]) { if ( diff --git a/console/src/platform/dataflows/elkGraph.ts b/console/src/platform/dataflows/elkGraph.ts index 78233d7b7509f..5f81aeea48d62 100644 --- a/console/src/platform/dataflows/elkGraph.ts +++ b/console/src/platform/dataflows/elkGraph.ts @@ -25,7 +25,7 @@ export const NODE_DIMENSIONS: Record< { width: number; height: number } > = { operator: { width: 240, height: 72 }, - collapsedRegion: { width: 260, height: 88 }, + collapsedRegion: { width: 260, height: 96 }, region: { width: 0, height: 0 }, port: { width: 90, height: 24 }, }; diff --git a/console/src/platform/dataflows/nodes.tsx b/console/src/platform/dataflows/nodes.tsx index df98ddc0d0bc7..cf4492e48a5e2 100644 --- a/console/src/platform/dataflows/nodes.tsx +++ b/console/src/platform/dataflows/nodes.tsx @@ -7,7 +7,7 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -import { Badge, Box, Text, Tooltip } from "@chakra-ui/react"; +import { Badge, Box, HStack, Text, Tooltip } from "@chakra-ui/react"; import { Handle, type NodeProps, Position } from "@xyflow/react"; import React from "react"; @@ -16,13 +16,16 @@ import { formatBytesShort } from "~/utils/format"; import type { VisibleNode } from "./dataflowGraph"; import { type FlowNodeData, formatElapsed } from "./nodeStyle"; +// Records and size share one line, not two: the fixed node height only has +// room for name + a couple of stat lines before text spills out of the box. const statLines = (node: VisibleNode) => { const lines: string[] = []; const stats = node.stats; if (!stats) return lines; if (stats.arrangementRecords > 0n) { - lines.push(`${stats.arrangementRecords} arranged records`); - lines.push(formatBytesShort(stats.arrangementSize)); + lines.push( + `${stats.arrangementRecords} rec · ${formatBytesShort(stats.arrangementSize)}`, + ); } if (stats.elapsedNs > 0n) lines.push(formatElapsed(stats.elapsedNs)); return lines; @@ -78,10 +81,14 @@ export const CollapsedRegionNode = ({ data, }: NodeProps & { data: FlowNodeData }) => ( - - {data.node.label} - - {data.node.childCount} children + + + {data.node.label} + + + {data.node.childCount} + + {statLines(data.node).map((line) => ( {line} From 5d996c02d0375ebe3ec890e3d86f7748106484c7 Mon Sep 17 00:00:00 2001 From: Moritz Hoffmann Date: Wed, 8 Jul 2026 22:42:23 +0200 Subject: [PATCH 15/45] console: reroute connectivity around hidden idle runs Co-Authored-By: Claude Sonnet 5 --- .../platform/dataflows/DataflowGraphView.tsx | 22 +-- .../platform/dataflows/dataflowGraph.test.ts | 127 ++++++++++++++++++ .../src/platform/dataflows/dataflowGraph.ts | 114 ++++++++++++++++ 3 files changed, 253 insertions(+), 10 deletions(-) diff --git a/console/src/platform/dataflows/DataflowGraphView.tsx b/console/src/platform/dataflows/DataflowGraphView.tsx index 29a87ffbaf75b..25554e7e0fb44 100644 --- a/console/src/platform/dataflows/DataflowGraphView.tsx +++ b/console/src/platform/dataflows/DataflowGraphView.tsx @@ -29,6 +29,7 @@ import { deriveVisibleGraph, type GraphDecorations, MAX_VISIBLE_NODES, + rerouteHiddenNodes, type VisibleNode, visibleNodeCount, } from "./dataflowGraph"; @@ -103,17 +104,18 @@ export const DataflowGraphView = ({ const toast = useToast(); const visible = React.useMemo(() => { const graph = deriveVisibleGraph(structure, collapsed); - if (!decorations?.hiddenNodeIds && !decorations?.hiddenEdgeIds) - return graph; - const hiddenNodes = decorations.hiddenNodeIds ?? new Set(); - const hiddenEdges = decorations.hiddenEdgeIds ?? new Set(); + // Hidden nodes get spliced out with connectivity preserved (a hidden idle + // run still shows a pass-through edge). Hidden edges (independently + // zero-message) are a plain removal: nothing to reroute since both + // endpoints stay visible. + const rerouted = decorations?.hiddenNodeIds + ? rerouteHiddenNodes(graph, decorations.hiddenNodeIds) + : graph; + if (!decorations?.hiddenEdgeIds?.size) return rerouted; return { - nodes: graph.nodes.filter((n) => !hiddenNodes.has(n.id)), - edges: graph.edges.filter( - (e) => - !hiddenEdges.has(e.id) && - !hiddenNodes.has(e.source) && - !hiddenNodes.has(e.target), + nodes: rerouted.nodes, + edges: rerouted.edges.filter( + (e) => !decorations.hiddenEdgeIds!.has(e.id), ), }; }, [ diff --git a/console/src/platform/dataflows/dataflowGraph.test.ts b/console/src/platform/dataflows/dataflowGraph.test.ts index 7fdb7fffb8d4e..28c611b363643 100644 --- a/console/src/platform/dataflows/dataflowGraph.test.ts +++ b/console/src/platform/dataflows/dataflowGraph.test.ts @@ -23,6 +23,9 @@ import { MAX_VISIBLE_NODES, nodeIdOf, type OperatorRow, + rerouteHiddenNodes, + type VisibleEdge, + type VisibleNode, visibleNodeCount, } from "./dataflowGraph"; @@ -430,3 +433,127 @@ describe("lirIndex", () => { ); }); }); + +describe("rerouteHiddenNodes", () => { + const node = (id: string): VisibleNode => ({ + id, + kind: "operator", + label: id, + parent: null, + stats: null, + transitive: null, + childCount: 0, + lir: [], + address: null, + }); + const edge = ( + id: string, + source: string, + target: string, + messagesSent = 0n, + batchesSent = 0n, + channelTypes: string[] = [], + ): VisibleEdge => ({ + id, + source, + target, + messagesSent, + batchesSent, + channelTypes, + }); + + it("returns the graph unchanged when nothing is hidden", () => { + const g = { + nodes: [node("a"), node("b")], + edges: [edge("a=>b", "a", "b")], + }; + expect(rerouteHiddenNodes(g, new Set())).toBe(g); + }); + + it("splices a pass-through edge across a single hidden node", () => { + const g = { + nodes: [node("a"), node("b"), node("c")], + edges: [ + edge("a=>b", "a", "b", 3n, 1n, ["rows"]), + edge("b=>c", "b", "c", 3n, 1n, ["rows"]), + ], + }; + const r = rerouteHiddenNodes(g, new Set(["b"])); + expect(r.nodes.map((n) => n.id)).toEqual(["a", "c"]); + expect(r.edges).toEqual([ + { + id: "a=>c", + source: "a", + target: "c", + messagesSent: 6n, + batchesSent: 2n, + channelTypes: ["rows"], + }, + ]); + }); + + it("fans out through a hidden node with multiple successors", () => { + const g = { + nodes: [node("a"), node("b"), node("c"), node("d")], + edges: [ + edge("a=>b", "a", "b"), + edge("b=>c", "b", "c"), + edge("b=>d", "b", "d"), + ], + }; + const r = rerouteHiddenNodes(g, new Set(["b"])); + expect(new Set(r.edges.map((e) => e.id))).toEqual( + new Set(["a=>c", "a=>d"]), + ); + }); + + it("fans in from multiple sources through a hidden node", () => { + const g = { + nodes: [node("a1"), node("a2"), node("b"), node("c")], + edges: [ + edge("a1=>b", "a1", "b"), + edge("a2=>b", "a2", "b"), + edge("b=>c", "b", "c"), + ], + }; + const r = rerouteHiddenNodes(g, new Set(["b"])); + expect(new Set(r.edges.map((e) => e.id))).toEqual( + new Set(["a1=>c", "a2=>c"]), + ); + }); + + it("does not hang on a hidden feedback cycle and still bridges past it", () => { + // Timely feedback loops mean two hidden nodes can point at each other. + const g = { + nodes: [node("a"), node("l"), node("m"), node("c")], + edges: [ + edge("a=>l", "a", "l"), + edge("l=>m", "l", "m"), + edge("m=>l", "m", "l"), // cycle + edge("m=>c", "m", "c"), + ], + }; + const r = rerouteHiddenNodes(g, new Set(["l", "m"])); + expect(r.nodes.map((n) => n.id)).toEqual(["a", "c"]); + expect(r.edges.map((e) => e.id)).toEqual(["a=>c"]); + }); + + it("drops an entirely hidden component with no visible endpoint", () => { + const g = { + nodes: [node("a"), node("b")], + edges: [edge("a=>b", "a", "b")], + }; + const r = rerouteHiddenNodes(g, new Set(["a", "b"])); + expect(r.nodes).toEqual([]); + expect(r.edges).toEqual([]); + }); + + it("keeps ordinary edges between two surviving visible nodes untouched", () => { + const g = { + nodes: [node("a"), node("b"), node("hidden")], + edges: [edge("a=>b", "a", "b", 5n, 1n, ["rows"])], + }; + const r = rerouteHiddenNodes(g, new Set(["hidden"])); + expect(r.edges).toEqual(g.edges); + }); +}); diff --git a/console/src/platform/dataflows/dataflowGraph.ts b/console/src/platform/dataflows/dataflowGraph.ts index c8427dd7442fe..f312f0143ff75 100644 --- a/console/src/platform/dataflows/dataflowGraph.ts +++ b/console/src/platform/dataflows/dataflowGraph.ts @@ -354,6 +354,120 @@ export function deriveVisibleGraph( return { nodes: [...nodes, ...ports.values()], edges }; } +// Hiding a run of idle operators would otherwise sever every path through +// them, splitting the graph into disconnected islands even though a real +// (idle) path exists. Splices a pass-through edge across each hidden run, +// so hiding idle nodes never removes connectivity, only the boxes in +// between. The spliced edge sums messages/batches/types along the run, +// which is almost always 0/0 (that is why the run was hidden) and so +// renders with the same dashed "idle" styling as an ordinary quiet edge. +// +// Cycles (timely feedback loops) are real and must not hang this: `path` +// tracks nodes on the current walk, and re-entering one simply stops that +// branch rather than looping forever. +export function rerouteHiddenNodes( + graph: VisibleGraph, + hiddenNodeIds: ReadonlySet, +): VisibleGraph { + if (hiddenNodeIds.size === 0) return graph; + const nodes = graph.nodes.filter((n) => !hiddenNodeIds.has(n.id)); + + const outAdj = new Map(); + for (const e of graph.edges) { + const list = outAdj.get(e.source); + if (list) list.push(e); + else outAdj.set(e.source, [e]); + } + + interface Agg { + messagesSent: bigint; + batchesSent: bigint; + types: Set; + } + + // Every visible node reachable by walking forward from `from` through only + // hidden nodes, with edge stats accumulated along the way. + function reachableVisible(from: string, path: Set): Map { + const result = new Map(); + if (path.has(from)) return result; + path.add(from); + for (const e of outAdj.get(from) ?? []) { + if (!hiddenNodeIds.has(e.target)) { + merge(result, e.target, e.messagesSent, e.batchesSent, e.channelTypes); + } else { + for (const [target, agg] of reachableVisible(e.target, path)) { + merge( + result, + target, + agg.messagesSent + e.messagesSent, + agg.batchesSent + e.batchesSent, + [...agg.types, ...e.channelTypes], + ); + } + } + } + path.delete(from); + return result; + } + + function merge( + into: Map, + target: string, + messagesSent: bigint, + batchesSent: bigint, + types: string[], + ) { + const existing = into.get(target); + if (existing) { + existing.messagesSent += messagesSent; + existing.batchesSent += batchesSent; + for (const t of types) existing.types.add(t); + } else { + into.set(target, { messagesSent, batchesSent, types: new Set(types) }); + } + } + + const edgesById = new Map(); + for (const e of graph.edges) { + if (!hiddenNodeIds.has(e.source) && !hiddenNodeIds.has(e.target)) { + edgesById.set(e.id, e); + } + } + for (const e of graph.edges) { + if (hiddenNodeIds.has(e.source) || !hiddenNodeIds.has(e.target)) continue; + for (const [target, agg] of reachableVisible( + e.target, + new Set([e.source]), + )) { + const id = `${e.source}=>${target}`; + const messagesSent = e.messagesSent + agg.messagesSent; + const batchesSent = e.batchesSent + agg.batchesSent; + const types = new Set([...e.channelTypes, ...agg.types]); + const existing = edgesById.get(id); + if (existing) { + edgesById.set(id, { + ...existing, + messagesSent: existing.messagesSent + messagesSent, + batchesSent: existing.batchesSent + batchesSent, + channelTypes: [ + ...new Set([...existing.channelTypes, ...types]), + ].sort(), + }); + } else { + edgesById.set(id, { + id, + source: e.source, + target, + messagesSent, + batchesSent, + channelTypes: [...types].sort(), + }); + } + } + } + return { nodes, edges: [...edgesById.values()] }; +} + export function defaultCollapseState( structure: DataflowStructure, ): CollapseState { From a774b104ce2d9c0e09fb01257548b8be03a97518 Mon Sep 17 00:00:00 2001 From: Moritz Hoffmann Date: Wed, 8 Jul 2026 22:42:23 +0200 Subject: [PATCH 16/45] Add click-to-inspect for edges/ports and a dataflow switcher Co-Authored-By: Claude Sonnet 5 --- .../src/platform/dataflows/ChannelEdge.tsx | 18 +- .../dataflows/DataflowDetailPage.test.tsx | 20 ++ .../platform/dataflows/DataflowDetailPage.tsx | 105 ++++++++--- .../platform/dataflows/DataflowGraphView.tsx | 66 ++++++- .../platform/dataflows/NodeDetailPanel.tsx | 175 +++++++++++++----- .../platform/dataflows/dataflowGraph.test.ts | 5 + .../src/platform/dataflows/dataflowGraph.ts | 6 + console/src/platform/dataflows/nodeStyle.ts | 11 ++ console/src/platform/dataflows/nodes.tsx | 17 +- 9 files changed, 342 insertions(+), 81 deletions(-) diff --git a/console/src/platform/dataflows/ChannelEdge.tsx b/console/src/platform/dataflows/ChannelEdge.tsx index ad932de2316f0..583feee7035c3 100644 --- a/console/src/platform/dataflows/ChannelEdge.tsx +++ b/console/src/platform/dataflows/ChannelEdge.tsx @@ -16,11 +16,14 @@ import { } from "@xyflow/react"; import React from "react"; +import { HIGHLIGHT_COLORS } from "./nodeStyle"; + export type ChannelEdgeData = { messagesSent: bigint; batchesSent: bigint; channelTypes: string[]; dimmed: boolean; + selected: boolean; }; const compactCount = Intl.NumberFormat("default", { @@ -32,14 +35,17 @@ const compact = (n: bigint) => compactCount.format(n); export const ChannelEdge = (props: EdgeProps & { data: ChannelEdgeData }) => { const [path, labelX, labelY] = getBezierPath(props); - const { messagesSent, batchesSent, channelTypes, dimmed } = props.data; + const { messagesSent, batchesSent, channelTypes, dimmed, selected } = + props.data; const idle = messagesSent === 0n; // Inline label stays terse: compact record/batch counts only. Exact figures // and the full container type names live in the tooltip, since those Rust - // type strings are long enough to swamp the canvas. - const label = idle - ? "" - : `${compact(messagesSent)} / ${compact(batchesSent)}`; + // type strings are long enough to swamp the canvas. Selecting an idle edge + // still shows its (zero) counts, confirming the click landed. + const label = + idle && !selected + ? "" + : `${compact(messagesSent)} / ${compact(batchesSent)}`; const tooltip = idle ? channelTypes.join(", ") || "unknown channel type" : `${messagesSent} records / ${batchesSent} batches` + @@ -52,6 +58,8 @@ export const ChannelEdge = (props: EdgeProps & { data: ChannelEdgeData }) => { style={{ strokeDasharray: idle ? "6 4" : undefined, opacity: dimmed ? 0.15 : 1, + stroke: selected ? HIGHLIGHT_COLORS.selected : undefined, + strokeWidth: selected ? 2.5 : undefined, }} /> {label && ( diff --git a/console/src/platform/dataflows/DataflowDetailPage.test.tsx b/console/src/platform/dataflows/DataflowDetailPage.test.tsx index 64f3f24cf4be9..e0da0a0913851 100644 --- a/console/src/platform/dataflows/DataflowDetailPage.test.tsx +++ b/console/src/platform/dataflows/DataflowDetailPage.test.tsx @@ -105,6 +105,20 @@ const operatorsResult = { ["11", ["7", "1"], "Map", "0", "0", "1"], ], }; +// The page also renders a dataflow switcher (useDataflowList), a separate +// request from the graph data's three queries above. +const dataflowListResult = { + desc: { + columns: [ + { name: "id" }, + { name: "name" }, + { name: "records" }, + { name: "size" }, + { name: "elapsedNs" }, + ], + }, + rows: [["7", "Dataflow: mv", "0", "0", "0"]], +}; beforeEach(() => { const store = getStore(); @@ -126,6 +140,12 @@ beforeEach(() => { ], }); } + const body = await request.text(); + // useDataflowList's query targets mz_dataflows directly; none of + // useDataflowGraphData's three queries reference that exact table name. + if (body.includes("mz_dataflows")) { + return HttpResponse.json({ results: [dataflowListResult] }); + } return HttpResponse.json({ results: [operatorsResult, okResult, okResult], }); diff --git a/console/src/platform/dataflows/DataflowDetailPage.tsx b/console/src/platform/dataflows/DataflowDetailPage.tsx index a1c424adab443..278bf865b6079 100644 --- a/console/src/platform/dataflows/DataflowDetailPage.tsx +++ b/console/src/platform/dataflows/DataflowDetailPage.tsx @@ -19,14 +19,22 @@ import { } from "@chakra-ui/react"; import { interpolateYlOrRd } from "d3-scale-chromatic"; import React from "react"; -import { Link, useParams, useSearchParams } from "react-router-dom"; +import { + Link, + useNavigate, + useParams, + useSearchParams, +} from "react-router-dom"; import { useDataflowGraphData } from "~/api/materialize/dataflow/useDataflowGraphData"; +import { useDataflowList } from "~/api/materialize/dataflow/useDataflowList"; import { ErrorCode } from "~/api/materialize/types"; import ErrorBox from "~/components/ErrorBox"; import LabeledSelect from "~/components/LabeledSelect"; import { MainContentContainer } from "~/layouts/BaseLayout"; +import { absoluteClusterPath } from "~/platform/routeHelpers"; import { useAllClusters } from "~/store/allClusters"; +import { useRegionSlug } from "~/store/environments"; import { allChannelTypes, @@ -37,12 +45,11 @@ import { deriveVisibleGraph, expandForSearch, type Filters, - type VisibleNode, } from "./dataflowGraph"; import { DataflowGraphView } from "./DataflowGraphView"; import { DataflowToolbar } from "./DataflowToolbar"; import { LirPanel } from "./LirPanel"; -import { NodeDetailPanel } from "./NodeDetailPanel"; +import { NodeDetailPanel, type Selection } from "./NodeDetailPanel"; // Injected into decorateGraph so the pure graph module stays d3-free. const heatColor = (t: number) => interpolateYlOrRd(0.15 + 0.85 * t); @@ -59,6 +66,8 @@ function hashString(s: string): number { const DataflowDetailPage = () => { const { clusterId, dataflowId } = useParams(); + const navigate = useNavigate(); + const regionSlug = useRegionSlug(); const { getClusterById } = useAllClusters(); const cluster = clusterId ? getClusterById(clusterId) : undefined; const [searchParams, setSearchParams] = useSearchParams(); @@ -74,10 +83,19 @@ const DataflowDetailPage = () => { const { data, error, databaseError, loading, refetch } = useDataflowGraphData(params); - const [collapsed, setCollapsed] = React.useState(null); - const [selectedNode, setSelectedNode] = React.useState( - null, + // Lets the toolbar switch dataflows in place instead of forcing a trip + // back to the list page. + const listParams = React.useMemo( + () => + cluster && replicaName + ? { clusterName: cluster.name, replicaName } + : undefined, + [cluster, replicaName], ); + const { data: dataflowList } = useDataflowList(listParams); + + const [collapsed, setCollapsed] = React.useState(null); + const [selection, setSelection] = React.useState(null); const [filters, setFilters] = React.useState(DEFAULT_FILTERS); const [matchIndex, setMatchIndex] = React.useState(0); const [lirHighlight, setLirHighlight] = @@ -148,18 +166,43 @@ const DataflowDetailPage = () => { return ( - setSearchParams({ replica: e.target.value })} - flexShrink={0} - > - {cluster.replicas.map((r) => ( - - ))} - + + setSearchParams({ replica: e.target.value })} + flexShrink={0} + > + {cluster.replicas.map((r) => ( + + ))} + + + navigate( + `${absoluteClusterPath(regionSlug, cluster)}/dataflows/${e.target.value}?replica=${replicaName}`, + ) + } + flexShrink={0} + width="320px" + > + {dataflowId && !dataflowList?.some((d) => d.id === dataflowId) && ( + // The list query hasn't resolved yet (or this dataflow just + // disappeared from it); keep the select populated with + // something so it isn't blank while data is in flight. + + )} + {(dataflowList ?? []).map((d) => ( + + ))} + + {data && ( - ) : !data || !collapsed ? ( + ) : !data || !focusedScope ? ( ) : data.structure.nodes.size <= 1 ? ( @@ -288,12 +373,16 @@ const DataflowDetailPage = () => { ) : ( + @@ -324,8 +413,8 @@ const DataflowDetailPage = () => { /> { ? selection.edge.id : undefined } - activeMatchId={decorations?.searchMatches[matchIndex]} + activeMatchId={ + allMatches.length > 0 + ? nodeIdOf(allMatches[matchIndex].address) + : undefined + } onNodeClick={(node, connectedEdges) => setSelection({ kind: "node", @@ -350,11 +443,13 @@ const DataflowDetailPage = () => { } onEdgeClick={(edge) => setSelection({ kind: "edge", edge })} onPaneClick={() => setSelection(null)} + onJumpToPeer={onJumpToPeer} /> {selection && ( setSelection(null)} + onJumpTo={onJumpToPeer} /> )} diff --git a/console/src/platform/dataflows/DataflowGraphView.css b/console/src/platform/dataflows/DataflowGraphView.css new file mode 100644 index 0000000000000..56dbe19e0464b --- /dev/null +++ b/console/src/platform/dataflows/DataflowGraphView.css @@ -0,0 +1,20 @@ +/* Copyright Materialize, Inc. and contributors. All rights reserved. + * + * Use of this software is governed by the Business Source License + * included in the LICENSE file. + * + * As of the Change Date specified in that file, in accordance with + * the Business Source License, use of this software will be governed + * by the Apache License, Version 2.0. + */ + +/* Expanding/collapsing a region re-runs elk layout from scratch, moving + * every node to a new position. Without a transition this reads as a jump + * cut; nodes are non-draggable, so animating transform/size here never + * fights with drag interaction. */ +.react-flow__node { + transition: + transform 200ms ease, + width 200ms ease, + height 200ms ease; +} diff --git a/console/src/platform/dataflows/DataflowGraphView.tsx b/console/src/platform/dataflows/DataflowGraphView.tsx index 48897395d0832..5ed1d6424b80d 100644 --- a/console/src/platform/dataflows/DataflowGraphView.tsx +++ b/console/src/platform/dataflows/DataflowGraphView.tsx @@ -8,8 +8,9 @@ // by the Apache License, Version 2.0. import "@xyflow/react/dist/style.css"; +import "./DataflowGraphView.css"; -import { Box, Spinner, useToast } from "@chakra-ui/react"; +import { Box, Spinner } from "@chakra-ui/react"; import { Background, Controls, @@ -24,30 +25,23 @@ import React from "react"; import { ChannelEdge } from "./ChannelEdge"; import { - type CollapseState, type DataflowStructure, deriveVisibleGraph, type GraphDecorations, - MAX_VISIBLE_NODES, + type NodeId, + type PortPeer, rerouteHiddenNodes, type VisibleEdge, type VisibleNode, - visibleNodeCount, } from "./dataflowGraph"; import { NODE_DIMENSIONS } from "./elkGraph"; -import { - CollapsedRegionNode, - OperatorNode, - PortNode, - RegionNode, -} from "./nodes"; +import { OperatorNode, PortNode, RegionNode } from "./nodes"; import { nodeFillColor } from "./nodeStyle"; import { useElkLayout } from "./useElkLayout"; const edgeTypes = { channel: ChannelEdge }; const nodeTypes = { operator: OperatorNode, - collapsedRegion: CollapsedRegionNode, region: RegionNode, port: PortNode, }; @@ -60,8 +54,11 @@ export type SelectedEdge = VisibleEdge & { export interface DataflowGraphViewProps { structure: DataflowStructure; - collapsed: CollapseState; - onCollapsedChange: (next: CollapseState) => void; + // The scope whose direct children this view renders; double-clicking a + // region box navigates to a new view rooted there rather than expanding it + // in place. + focusedScope: NodeId; + onNavigate: (scope: NodeId) => void; cacheKey: string; decorations?: GraphDecorations; selectedId?: string; @@ -72,6 +69,11 @@ export interface DataflowGraphViewProps { onNodeClick?: (node: VisibleNode, connectedEdges: SelectedEdge[]) => void; onEdgeClick?: (edge: SelectedEdge) => void; onPaneClick?: () => void; + // Double-clicking a port with exactly one peer jumps straight there; with + // zero or several peers it's ambiguous (or there's nothing to jump to), so + // the preceding click/click of the double-click has already opened the + // port's own detail panel instead, same as a single click would. + onJumpToPeer?: (peer: PortPeer) => void; centerRef?: React.MutableRefObject<((id: string) => void) | null>; // A node to center on once its layout position exists (e.g. right after an // expand triggered by jumping to it). Centering an id whose ancestors were @@ -132,8 +134,8 @@ const CenterHelper = ({ export const DataflowGraphView = ({ structure, - collapsed, - onCollapsedChange, + focusedScope, + onNavigate, cacheKey, decorations, selectedId, @@ -141,6 +143,7 @@ export const DataflowGraphView = ({ onNodeClick, onEdgeClick, onPaneClick, + onJumpToPeer, centerRef, centerOnId, onCentered, @@ -148,9 +151,8 @@ export const DataflowGraphView = ({ fitOnIds, onFit, }: DataflowGraphViewProps) => { - const toast = useToast(); const visible = React.useMemo(() => { - const graph = deriveVisibleGraph(structure, collapsed); + const graph = deriveVisibleGraph(structure, focusedScope); // Hidden nodes get spliced out with connectivity preserved (a hidden idle // run still shows a pass-through edge). Hidden edges (independently // zero-message) are a plain removal: nothing to reroute since both @@ -167,12 +169,12 @@ export const DataflowGraphView = ({ }; }, [ structure, - collapsed, + focusedScope, decorations?.hiddenNodeIds, decorations?.hiddenEdgeIds, ]); - const layoutKey = `${cacheKey}|${[...collapsed].sort().join(",")}|${ + const layoutKey = `${cacheKey}|${focusedScope}|${ decorations?.hiddenNodeIds ? [...decorations.hiddenNodeIds].sort().join(",") : "" @@ -200,26 +202,6 @@ export const DataflowGraphView = ({ onFit?.(); }, [fitOnIds, positions, fitRef, onFit]); - const toggleRegion = React.useCallback( - (node: VisibleNode) => { - const next = new Set(collapsed); - if (next.has(node.id)) { - next.delete(node.id); - if (visibleNodeCount(structure, next) > MAX_VISIBLE_NODES) { - toast({ - status: "warning", - title: `Expanding would show more than ${MAX_VISIBLE_NODES} nodes.`, - }); - return; - } - } else { - next.add(node.id); - } - onCollapsedChange(next); - }, - [collapsed, onCollapsedChange, structure, toast], - ); - const nodes: Node[] = React.useMemo(() => { if (!positions) return []; return visible.nodes.map((n) => { @@ -228,8 +210,6 @@ export const DataflowGraphView = ({ id: n.id, type: n.kind, position: { x: pos.x, y: pos.y }, - parentId: n.parent ?? undefined, - extent: n.parent ? ("parent" as const) : undefined, // Match the Top/Bottom handles so bezier control points meet the // node edges. Without this the edge curves toward the default // Bottom/Top and appears detached across region boundaries. @@ -274,17 +254,11 @@ export const DataflowGraphView = ({ channelTypes: e.channelTypes, dimmed: (decorations?.dimmedNodeIds?.has(e.source) ?? false) || - (decorations?.dimmedNodeIds?.has(e.target) ?? false) || - (decorations?.dimmedEdgeIds?.has(e.id) ?? false), + (decorations?.dimmedNodeIds?.has(e.target) ?? false), selected: e.id === selectedId, }, })), - [ - visible, - decorations?.dimmedNodeIds, - decorations?.dimmedEdgeIds, - selectedId, - ], + [visible, decorations?.dimmedNodeIds, selectedId], ); if (error) throw new Error(error); @@ -303,7 +277,7 @@ export const DataflowGraphView = ({ onlyRenderVisibleElements fitView minZoom={0.05} - // Double-click is the collapse toggle, so it must not also zoom. + // Double-click navigates into a region, so it must not also zoom. zoomOnDoubleClick={false} onNodeClick={(_, node) => { const visibleNode = (node.data as { node: VisibleNode }).node; @@ -334,11 +308,13 @@ export const DataflowGraphView = ({ onPaneClick={onPaneClick} onNodeDoubleClick={(_, node) => { const visibleNode = (node.data as { node: VisibleNode }).node; - if ( - visibleNode.kind === "region" || - visibleNode.kind === "collapsedRegion" + if (visibleNode.kind === "region") { + onNavigate(visibleNode.id); + } else if ( + visibleNode.kind === "port" && + visibleNode.peers.length === 1 ) { - toggleRegion(visibleNode); + onJumpToPeer?.(visibleNode.peers[0]); } }} proOptions={{ hideAttribution: true }} diff --git a/console/src/platform/dataflows/DataflowToolbar.tsx b/console/src/platform/dataflows/DataflowToolbar.tsx index b1ea2d216ccfc..d7314f8f6c2c3 100644 --- a/console/src/platform/dataflows/DataflowToolbar.tsx +++ b/console/src/platform/dataflows/DataflowToolbar.tsx @@ -9,15 +9,10 @@ import { Button, - Checkbox, FormControl, FormLabel, HStack, Input, - Menu, - MenuButton, - MenuItem, - MenuList, Select, Slider, SliderFilledTrack, @@ -33,7 +28,6 @@ import { type Filters } from "./dataflowGraph"; export interface DataflowToolbarProps { filters: Filters; onFiltersChange: (next: Filters) => void; - channelTypes: string[]; matchCount: number; matchIndex: number; onJump: (delta: 1 | -1) => void; @@ -42,7 +36,6 @@ export interface DataflowToolbarProps { export const DataflowToolbar = ({ filters, onFiltersChange, - channelTypes, matchCount, matchIndex, onJump, @@ -113,6 +106,8 @@ export const DataflowToolbar = ({ + + - - - Channel types - - - {channelTypes.map((t) => ( - - { - const current = filters.channelTypes ?? channelTypes; - const next = e.target.checked - ? [...current, t] - : current.filter((x) => x !== t); - onFiltersChange({ - ...filters, - channelTypes: - next.length === channelTypes.length ? null : next, - }); - }} - > - {t} - - - ))} - - ); }; diff --git a/console/src/platform/dataflows/NodeDetailPanel.tsx b/console/src/platform/dataflows/NodeDetailPanel.tsx index 00bd3a1880bd5..72a92a598cf25 100644 --- a/console/src/platform/dataflows/NodeDetailPanel.tsx +++ b/console/src/platform/dataflows/NodeDetailPanel.tsx @@ -7,13 +7,14 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -import { Box, CloseButton, HStack, Text } from "@chakra-ui/react"; +import { Box, Button, CloseButton, HStack, Text } from "@chakra-ui/react"; import React from "react"; import { formatBytesShort } from "~/utils/format"; -import type { VisibleNode } from "./dataflowGraph"; +import type { PortPeer, VisibleNode } from "./dataflowGraph"; import type { SelectedEdge } from "./DataflowGraphView"; +import { prettyPrintChannelType } from "./nodeStyle"; export type Selection = | { kind: "node"; node: VisibleNode; connectedEdges?: SelectedEdge[] } @@ -30,16 +31,17 @@ const Row = ({ label, value }: { label: string; value: string }) => ( ); -// Channel types are raw Rust container type signatures (e.g. nested Vec<...> -// generics) with no spaces to wrap on, so they get their own full-width -// stacked row instead of Row's space-between line. +// Channel types are Rust container type signatures (pretty-printed, but +// still nested-generic shaped, e.g. "[Rc>]") long enough +// that they get their own full-width stacked row instead of Row's +// space-between line. const TypeRow = ({ channelTypes }: { channelTypes: string[] }) => ( Type - {channelTypes.join(", ") || "unknown"} + {channelTypes.map(prettyPrintChannelType).join(", ") || "unknown"} ); @@ -52,12 +54,39 @@ const EdgeRows = ({ edge }: { edge: SelectedEdge }) => ( ); +// A port's peer lives outside the current view (that's why it's a port and +// not a drawn edge), so its only affordance is a jump: navigate to wherever +// the peer actually is instead of tracing a line to it. +const PeerRow = ({ + peer, + onJumpTo, +}: { + peer: PortPeer; + onJumpTo: (peer: PortPeer) => void; +}) => ( + + + + → {peer.label} + + + + + + + +); + const NodeDetail = ({ node, connectedEdges, + onJumpTo, }: { node: VisibleNode; connectedEdges?: SelectedEdge[]; + onJumpTo: (peer: PortPeer) => void; }) => ( <> @@ -126,15 +155,27 @@ const NodeDetail = ({ ))} )} + {node.peers.length > 0 && ( + + + Connects outside this view + + {node.peers.map((p) => ( + + ))} + + )} ); export const NodeDetailPanel = ({ selection, onClose, + onJumpTo, }: { selection: Selection; onClose: () => void; + onJumpTo: (peer: PortPeer) => void; }) => { const title = selection.kind === "node" @@ -158,6 +199,7 @@ export const NodeDetailPanel = ({ ) : ( diff --git a/console/src/platform/dataflows/dataflowGraph.test.ts b/console/src/platform/dataflows/dataflowGraph.test.ts index fdeb33323e343..cb69934e5b3a1 100644 --- a/console/src/platform/dataflows/dataflowGraph.test.ts +++ b/console/src/platform/dataflows/dataflowGraph.test.ts @@ -10,25 +10,24 @@ import { describe, expect, it } from "vitest"; import { - allChannelTypes, + allSearchMatches, buildDataflowStructure, type ChannelRow, + commonAncestorScope, decorateGraph, DEFAULT_FILTERS, - defaultCollapseState, deriveVisibleGraph, - expandAncestorsOf, - expandForSearch, lirIndex, type LirSpanRow, lirTree, - MAX_VISIBLE_NODES, nodeIdOf, type OperatorRow, + type PerWorkerStatRow, + representativeInView, rerouteHiddenNodes, + subtreeSearchMatches, type VisibleEdge, type VisibleNode, - visibleNodeCount, } from "./dataflowGraph"; // Dataflow 5: root [5], region [5,1] with children [5,1,1], [5,1,2], leaf [5,2]. @@ -172,19 +171,56 @@ describe("buildDataflowStructure", () => { channelType: "rows", }); }); + + describe("skew", () => { + // Join (id 12, [5,1,1]): worker 0 does 10ns/1000B, worker 1 does + // 30ns/3000B -> 2x every direction. Map (id 13, [5,1,2]): only worker 0 + // ever scheduled it (5ns) and it never arranges anything -> perfectly + // even CPU (a single data point can't be skewed), no memory data at all. + const PER_WORKER: PerWorkerStatRow[] = [ + { id: "12", workerId: "0", elapsedNs: "10", arrangementSize: "1000" }, + { id: "12", workerId: "1", elapsedNs: "30", arrangementSize: "3000" }, + { id: "13", workerId: "0", elapsedNs: "5", arrangementSize: null }, + ]; + + it("computes own skew as worst-worker over average, over participating workers only", () => { + const s = buildDataflowStructure(OPS, CHANNELS, LIR_SPANS, PER_WORKER); + const join = s.nodes.get(nodeIdOf([5, 1, 1]))!; + expect(join.ownSkew.cpuSkew).toBeCloseTo(1.5); // 30 / ((10+30)/2) + expect(join.ownSkew.memorySkew).toBeCloseTo(1.5); // 3000 / 2000 + + const map = s.nodes.get(nodeIdOf([5, 1, 2]))!; + expect(map.ownSkew.cpuSkew).toBeCloseTo(1); // one worker, one data point + expect(map.ownSkew.memorySkew).toEqual(0); // no memory data at all + + const sink = s.nodes.get(nodeIdOf([5, 2]))!; + expect(sink.ownSkew).toEqual({ cpuSkew: 0, memorySkew: 0 }); + }); + + it("computes transitive skew from the subtree's summed per-worker vectors, not from children's own skew", () => { + const s = buildDataflowStructure(OPS, CHANNELS, LIR_SPANS, PER_WORKER); + const region = s.nodes.get(nodeIdOf([5, 1]))!; + // cpu: worker0 = 10 (Join) + 5 (Map) = 15, worker1 = 30 (Join only, + // Map never touched worker 1) -> avg 22.5, skew 30/22.5. + expect(region.transitiveSkew.cpuSkew).toBeCloseTo(30 / 22.5); + // memory: Map contributes nothing, so this is just Join's own vector. + expect(region.transitiveSkew.memorySkew).toBeCloseTo(1.5); + }); + }); }); describe("deriveVisibleGraph", () => { const s = buildDataflowStructure(OPS, CHANNELS, LIR_SPANS); const regionId = nodeIdOf([5, 1]); - it("collapses regions to a single node with remapped edges", () => { - const g = deriveVisibleGraph(s, new Set([regionId])); + it("shows the region collapsed and the sibling leaf from the root view", () => { + const g = deriveVisibleGraph(s, s.root); expect(g.nodes.map((n) => [n.id, n.kind])).toEqual([ - [regionId, "collapsedRegion"], + [regionId, "region"], [nodeIdOf([5, 2]), "operator"], ]); - // channel 3 region -> sink survives, channels 1 and 2 are internal + // channel 3 region -> sink survives, channels 1 and 2 are internal to + // the region and never surface at this level. expect(g.edges).toEqual([ { id: `${regionId}=>${nodeIdOf([5, 2])}`, @@ -195,28 +231,58 @@ describe("deriveVisibleGraph", () => { channelTypes: ["batches"], }, ]); - const collapsed = g.nodes[0]; - expect(collapsed.stats).toEqual(s.nodes.get(regionId)!.transitive); - expect(collapsed.childCount).toEqual(2); + const region = g.nodes[0]; + expect(region.stats).toEqual(s.nodes.get(regionId)!.transitive); + expect(region.childCount).toEqual(2); }); - it("expands regions with port pseudo-nodes, parents before children", () => { - const g = deriveVisibleGraph(s, new Set()); - const ids = g.nodes.map((n) => n.id); - expect(ids.indexOf(regionId)).toBeLessThan( - ids.indexOf(nodeIdOf([5, 1, 1])), + it("drilling into the region shows its children and an inbound port", () => { + const g = deriveVisibleGraph(s, regionId); + expect(g.nodes.map((n) => [n.id, n.kind]).sort()).toEqual( + [ + [nodeIdOf([5, 1, 1]), "operator"], + [nodeIdOf([5, 1, 2]), "operator"], + [`${regionId}:in:0`, "port"], + [`${regionId}:out:0`, "port"], + ].sort(), ); - const port = g.nodes.find((n) => n.kind === "port")!; - expect(port.id).toEqual(`${regionId}:in:0`); - expect(port.parent).toEqual(regionId); - expect(port.label).toEqual("input 0"); - expect(port.operatorId).toBeNull(); - // port -> Join edge preserved - expect(g.edges.map((e) => e.id)).toContain( - `${regionId}:in:0=>${nodeIdOf([5, 1, 1])}`, + const inPort = g.nodes.find((n) => n.id === `${regionId}:in:0`)!; + expect(inPort.label).toEqual("input 0"); + expect(inPort.operatorId).toBeNull(); + // channel 1 (port fan-in) and channel 2 (internal) both survive. + expect(g.edges.map((e) => e.id).sort()).toEqual( + [ + `${regionId}:in:0=>${nodeIdOf([5, 1, 1])}`, + `${nodeIdOf([5, 1, 1])}=>${nodeIdOf([5, 1, 2])}`, + ].sort(), ); - const regionNode = g.nodes.find((n) => n.id === regionId)!; - expect(regionNode.operatorId).toEqual(11n); + // channel 3's outer half (region's own address -> sibling [5,2]) has no + // representable target from inside the region (the sibling isn't part + // of this view), so its edge is dropped, but the "out" port it resolved + // to registers anyway: the box's boundary traffic isn't silently erased. + expect( + g.edges.some( + (e) => + e.source === `${regionId}:out:0` || e.target === `${regionId}:out:0`, + ), + ).toBe(false); + // ...and the sibling it can't show directly is recorded as a jump target. + const outPort = g.nodes.find((n) => n.id === `${regionId}:out:0`)!; + expect(outPort.peers).toEqual([ + { + address: [5, 2], + label: "Sink", + messagesSent: 5n, + batchesSent: 2n, + channelTypes: ["batches"], + // Sink is a leaf: nothing to drill into, so the peer box itself is + // the target. + peerPortId: null, + }, + ]); + // channel 1's other end (Join) is already visible in this same view, so + // the "in" port needs no jump link for it. + expect(inPort.peers).toEqual([]); }); it("aggregates parallel channels between the same visible pair", () => { @@ -237,18 +303,12 @@ describe("deriveVisibleGraph", () => { ], [], ); - const g = deriveVisibleGraph(extra, new Set([regionId])); + const g = deriveVisibleGraph(extra, extra.root); const e = g.edges.find((edge) => edge.target === nodeIdOf([5, 2]))!; expect(e.messagesSent).toEqual(12n); expect(e.channelTypes).toEqual(["batches", "rows"]); }); - it("defaultCollapseState collapses all non-root regions", () => { - expect(defaultCollapseState(s)).toEqual(new Set([regionId])); - expect(visibleNodeCount(s, defaultCollapseState(s))).toEqual(2); - expect(MAX_VISIBLE_NODES).toEqual(1500); - }); - it("routes an external-to-region channel through a port node, keyed by port number", () => { // Materialize logs a scope boundary crossing as two channels sharing a // port number but addressed differently: the outer half (external @@ -287,19 +347,121 @@ describe("deriveVisibleGraph", () => { const externalId = nodeIdOf([5, 3]); const portId = `${regionId}:in:0`; - const expanded = deriveVisibleGraph(withExternal, new Set()); + // Viewed from inside the region, the crossing's inner half (channel 1) + // already produced this port; the outer half's external source isn't + // part of this view, so it contributes no new edge or node. + const drilledIn = deriveVisibleGraph(withExternal, regionId); + const inPort = drilledIn.nodes.find((n) => n.id === portId)!; + expect(inPort.kind).toEqual("port"); expect( - expanded.nodes.some((n) => n.id === portId && n.kind === "port"), - ).toBe(true); - const edge = expanded.edges.find((e) => e.source === externalId)!; - expect(edge.target).toEqual(portId); + drilledIn.edges.some( + (e) => e.source === externalId || e.target === externalId, + ), + ).toBe(false); + // ...but the external source is still reachable as a jump target off + // the port itself. + expect(inPort.peers).toEqual([ + { + address: [5, 3], + label: "External", + messagesSent: 9n, + batchesSent: 3n, + channelTypes: ["rows"], + // External is a leaf too. + peerPortId: null, + }, + ]); - // Collapsed, the crossing lands on the region's own (collapsed) node, - // matching how every other boundary crossing into a collapsed region - // behaves: the whole region is one box. - const collapsed = deriveVisibleGraph(withExternal, new Set([regionId])); - const collapsedEdge = collapsed.edges.find((e) => e.source === externalId)!; - expect(collapsedEdge.target).toEqual(regionId); + // From the root, the crossing lands on the region's own (collapsed) + // node, matching how every other boundary crossing into a collapsed + // region behaves: the whole region is one box. + const root = deriveVisibleGraph(withExternal, withExternal.root); + const rootEdge = root.edges.find((e) => e.source === externalId)!; + expect(rootEdge.target).toEqual(regionId); + }); + + it("fans a port out to multiple peers (one output can feed several inputs)", () => { + const fannedOut = buildDataflowStructure( + [ + ...OPS, + { + id: "21", + address: ["5", "4"], + name: "Sink2", + arrangementRecords: "0", + arrangementSize: "0", + elapsedNs: "1", + }, + ], + [ + ...CHANNELS, + // Same output (region's port 0) also feeds a second sibling. + { + id: "5", + fromOperatorAddress: ["5", "1"], + fromPort: "0", + toOperatorAddress: ["5", "4"], + toPort: "0", + messagesSent: "2", + batchesSent: "1", + channelType: "rows", + }, + ], + [], + ); + const g = deriveVisibleGraph(fannedOut, regionId); + const outPort = g.nodes.find((n) => n.id === `${regionId}:out:0`)!; + expect(outPort.peers.map((p) => p.label).sort()).toEqual(["Sink", "Sink2"]); + expect(outPort.peers.find((p) => p.label === "Sink2")).toEqual({ + address: [5, 4], + label: "Sink2", + messagesSent: 2n, + batchesSent: 1n, + channelTypes: ["rows"], + peerPortId: null, + }); + }); + + it("resolves a region peer's own port, so a jump can drill straight to it", () => { + // Two sibling regions, A's output feeding B's input directly (not + // nested in each other): from inside A, B is a region peer with a + // real port to land on inside B; from inside B, A is the same. + const s2 = buildDataflowStructure( + [ + { ...OPS[0], id: "1", address: ["9"], name: "Dataflow" }, + { ...OPS[0], id: "2", address: ["9", "1"], name: "RegionA" }, + { ...OPS[0], id: "3", address: ["9", "1", "1"], name: "LeafA" }, + { ...OPS[0], id: "4", address: ["9", "2"], name: "RegionB" }, + { ...OPS[0], id: "5", address: ["9", "2", "1"], name: "LeafB" }, + ], + [ + { + id: "1", + fromOperatorAddress: ["9", "1"], + fromPort: "3", + toOperatorAddress: ["9", "2"], + toPort: "7", + messagesSent: "1", + batchesSent: "1", + channelType: "rows", + }, + ], + [], + ); + const aId = nodeIdOf([9, 1]); + const bId = nodeIdOf([9, 2]); + + const fromA = deriveVisibleGraph(s2, aId); + const outPort = fromA.nodes.find((n) => n.id === `${aId}:out:3`)!; + expect(outPort.peers).toEqual([ + expect.objectContaining({ address: [9, 2], peerPortId: `${bId}:in:7` }), + ]); + + const fromB = deriveVisibleGraph(s2, bId); + const inPort = fromB.nodes.find((n) => n.id === `${bId}:in:7`)!; + expect(inPort.peers).toEqual([ + expect.objectContaining({ address: [9, 1], peerPortId: `${aId}:out:3` }), + ]); }); it("drops channels that reference an address with no operator row", () => { @@ -323,8 +485,10 @@ describe("deriveVisibleGraph", () => { ], LIR_SPANS, ); - expect(() => deriveVisibleGraph(withDangling, new Set())).not.toThrow(); - const g = deriveVisibleGraph(withDangling, new Set()); + expect(() => + deriveVisibleGraph(withDangling, withDangling.root), + ).not.toThrow(); + const g = deriveVisibleGraph(withDangling, withDangling.root); const nodeIds = new Set(g.nodes.map((n) => n.id)); for (const e of g.edges) { expect(nodeIds.has(e.source)).toBe(true); @@ -333,27 +497,102 @@ describe("deriveVisibleGraph", () => { }); }); +describe("commonAncestorScope / representativeInView", () => { + const s = buildDataflowStructure(OPS, CHANNELS, LIR_SPANS); + const regionId = nodeIdOf([5, 1]); + + it("a single leaf address navigates to its immediate parent scope", () => { + expect(commonAncestorScope(s, [[5, 1, 1]])).toEqual(regionId); + expect(representativeInView([5, 1, 1], [5, 1])).toEqual( + nodeIdOf([5, 1, 1]), + ); + }); + + it("backs off past an address that IS the raw common prefix", () => { + // [5,1] is itself a prefix of [5,1,1], so it can't be the view root + // (nothing could highlight it as its own child); back off to its parent. + expect( + commonAncestorScope(s, [ + [5, 1], + [5, 1, 1], + ]), + ).toEqual(s.root); + // From the root, [5,1] represents itself and [5,1,1] rolls up into it. + expect(representativeInView([5, 1], [5])).toEqual(regionId); + expect(representativeInView([5, 1, 1], [5])).toEqual(regionId); + }); + + it("returns null for an address outside the view root's subtree", () => { + expect(representativeInView([5, 2], [5, 1])).toBeNull(); + }); +}); + +describe("allSearchMatches / subtreeSearchMatches", () => { + const s = buildDataflowStructure(OPS, CHANNELS, LIR_SPANS); + const regionId = nodeIdOf([5, 1]); + + it("allSearchMatches finds matches anywhere, in address order, excluding the root", () => { + expect(allSearchMatches(s, "i").map((n) => n.id)).toEqual([ + regionId, // "Region" + nodeIdOf([5, 1, 1]), // "Join" + nodeIdOf([5, 2]), // "Sink" + ]); + expect(allSearchMatches(s, "nonexistent")).toEqual([]); + expect(allSearchMatches(s, "")).toEqual([]); + }); + + it("subtreeSearchMatches flags a region as containing a match without matching itself", () => { + const { matches, containsMatch } = subtreeSearchMatches(s, "join"); + expect(matches.has(nodeIdOf([5, 1, 1]))).toBe(true); + expect(matches.has(regionId)).toBe(false); + // The region doesn't match "join" itself, but a descendant does. + expect(containsMatch.has(regionId)).toBe(true); + expect(containsMatch.has(nodeIdOf([5, 2]))).toBe(false); + }); +}); + describe("decorateGraph", () => { const s = buildDataflowStructure(OPS, CHANNELS, LIR_SPANS); - const expanded = deriveVisibleGraph(s, new Set()); + const regionId = nodeIdOf([5, 1]); + const root = deriveVisibleGraph(s, s.root); + const drilledIn = deriveVisibleGraph(s, regionId); const heat = (t: number) => `heat(${t.toFixed(2)})`; - it("search dims non-matching leaf nodes and lists matches", () => { + it("a region containing a match (but not matching itself) stays undimmed and uncounted", () => { + const searchInfo = subtreeSearchMatches(s, "join"); const d = decorateGraph( - expanded, + root, { ...DEFAULT_FILTERS, search: "join" }, heat, + searchInfo, ); - expect(d.searchMatches).toEqual([nodeIdOf([5, 1, 1])]); + // "Join" is inside the region, not visible at the root; the region + // doesn't match "join" by name either, so it's neither a listed match + // nor dimmed, while the sibling sink (no match anywhere inside it) is. + expect(d.searchMatches).toEqual([]); + expect(d.dimmedNodeIds.has(regionId)).toBe(false); expect(d.dimmedNodeIds.has(nodeIdOf([5, 2]))).toBe(true); + }); + + it("a directly visible match is listed and left undimmed", () => { + const searchInfo = subtreeSearchMatches(s, "join"); + const d = decorateGraph( + drilledIn, + { ...DEFAULT_FILTERS, search: "join" }, + heat, + searchInfo, + ); + expect(d.searchMatches).toEqual([nodeIdOf([5, 1, 1])]); expect(d.dimmedNodeIds.has(nodeIdOf([5, 1, 1]))).toBe(false); + expect(d.dimmedNodeIds.has(nodeIdOf([5, 1, 2]))).toBe(true); }); it("hideIdle hides zero-activity operators and zero-message edges, never regions or ports", () => { const d = decorateGraph( - expanded, + root, { ...DEFAULT_FILTERS, hideIdle: true }, heat, + null, ); // every fixture operator has elapsed > 0 except none; hide check via a synthetic idle op const idle = buildDataflowStructure( @@ -372,39 +611,25 @@ describe("decorateGraph", () => { [], ); const d2 = decorateGraph( - deriveVisibleGraph(idle, new Set()), + deriveVisibleGraph(idle, idle.root), { ...DEFAULT_FILTERS, hideIdle: true }, heat, + null, ); expect(d2.hiddenNodeIds.has(nodeIdOf([5, 3]))).toBe(true); expect(d2.hiddenNodeIds.has(nodeIdOf([5, 1]))).toBe(false); expect(d.hiddenEdgeIds.size).toBe(0); // all fixture edges have messages }); - it("channel type filter dims non-matching edges", () => { - const d = decorateGraph( - expanded, - { ...DEFAULT_FILTERS, channelTypes: ["batches"] }, - heat, - ); - const rowsEdge = expanded.edges.find((e) => - e.channelTypes.includes("rows"), - )!; - const batchesEdge = expanded.edges.find((e) => - e.channelTypes.includes("batches"), - )!; - expect(d.dimmedEdgeIds.has(rowsEdge.id)).toBe(true); - expect(d.dimmedEdgeIds.has(batchesEdge.id)).toBe(false); - }); - it("heatmap colors by transitive metric and dims below threshold", () => { const d = decorateGraph( - expanded, + root, { ...DEFAULT_FILTERS, heatmap: "elapsed", heatmapThreshold: 0.5 }, heat, + null, ); // max transitive elapsed among visible non-ports is the region (13n) - expect(d.nodeColors.get(nodeIdOf([5, 1]))).toEqual("heat(1.00)"); + expect(d.nodeColors.get(regionId)).toEqual("heat(1.00)"); expect(d.dimmedNodeIds.has(nodeIdOf([5, 2]))).toBe(true); // 2/13 < 0.5 }); @@ -415,31 +640,30 @@ describe("decorateGraph", () => { [], ); const d = decorateGraph( - deriveVisibleGraph(zero, new Set()), + deriveVisibleGraph(zero, zero.root), { ...DEFAULT_FILTERS, heatmap: "elapsed" }, heat, + null, ); expect(d.nodeColors.size).toBe(0); }); -}); - -describe("expandForSearch / allChannelTypes", () => { - const s = buildDataflowStructure(OPS, CHANNELS, LIR_SPANS); - - it("expands ancestors of matches", () => { - const next = expandForSearch(s, defaultCollapseState(s), "join"); - expect(next.has(nodeIdOf([5, 1]))).toBe(false); - }); - it("expandAncestorsOf expands ancestors of given addresses directly", () => { - const next = expandAncestorsOf(defaultCollapseState(s), [[5, 1, 1]]); - expect(next.has(nodeIdOf([5, 1]))).toBe(false); - // unrelated collapsed regions are untouched - expect(next.size).toEqual(defaultCollapseState(s).size - 1); - }); - - it("collects channel types", () => { - expect(allChannelTypes(s)).toEqual(["batches", "rows"]); + it("cpuSkew/memorySkew heatmap colors by transitive skew", () => { + const skewed = buildDataflowStructure(OPS, CHANNELS, LIR_SPANS, [ + { id: "12", workerId: "0", elapsedNs: "10", arrangementSize: "1000" }, + { id: "12", workerId: "1", elapsedNs: "30", arrangementSize: "3000" }, + ]); + const skewedRoot = deriveVisibleGraph(skewed, skewed.root); + const d = decorateGraph( + skewedRoot, + { ...DEFAULT_FILTERS, heatmap: "cpuSkew" }, + heat, + null, + ); + // The region's subtree (containing Join) is the only skewed thing; the + // sibling sink has no per-worker data at all, so it's the coolest. + expect(d.nodeColors.get(regionId)).toEqual("heat(1.00)"); + expect(d.nodeColors.get(nodeIdOf([5, 2]))).toEqual("heat(0.00)"); }); }); @@ -593,13 +817,14 @@ describe("rerouteHiddenNodes", () => { id, kind: "operator", label: id, - parent: null, stats: null, transitive: null, + transitiveSkew: null, childCount: 0, lir: [], address: null, operatorId: null, + peers: [], }); const edge = ( id: string, diff --git a/console/src/platform/dataflows/dataflowGraph.ts b/console/src/platform/dataflows/dataflowGraph.ts index 5d5de2f25f803..db95e5b86d721 100644 --- a/console/src/platform/dataflows/dataflowGraph.ts +++ b/console/src/platform/dataflows/dataflowGraph.ts @@ -19,6 +19,17 @@ export interface NodeStats { elapsedNs: bigint; } +// How unevenly work for this node is spread across workers: worst worker's +// value over the average, matching EXPLAIN ANALYZE ... WITH SKEW (avg is +// over workers that show up at all for this node, not the full replica +// worker count; a worker with zero rows anywhere for a node is invisible to +// both, same convention). 1 means perfectly even; higher means more skewed; +// 0 means no data. +export interface SkewStats { + cpuSkew: number; + memorySkew: number; +} + export interface LirInfo { exportId: string; lirId: string; @@ -36,6 +47,8 @@ export interface DataflowNode { children: NodeId[]; own: NodeStats; transitive: NodeStats; // own + subtree, precomputed + ownSkew: SkewStats; + transitiveSkew: SkewStats; // recomputed from the subtree's summed per-worker vectors, not from children's own skew (skew doesn't sum) lir: LirInfo[]; // one entry per export span covering this operator id } @@ -56,20 +69,37 @@ export interface DataflowStructure { channels: Channel[]; } -export type CollapseState = ReadonlySet; +// A boundary crossing this view can't show directly (the far side isn't part +// of it): kept so a port can still be jumped to, even fanning out to several +// peers (one output can feed multiple inputs). +export interface PortPeer { + address: Address; + label: string; + messagesSent: bigint; + batchesSent: bigint; + channelTypes: string[]; + // The exact port this crossing lands on from inside the peer, so jumping + // can drill straight into it rather than parking outside as an unlabeled + // box; null when the peer is a leaf (nothing to drill into, so the peer + // itself is the target). + peerPortId: NodeId | null; +} export interface VisibleNode { id: string; - kind: "operator" | "region" | "collapsedRegion" | "port"; + kind: "operator" | "region" | "port"; label: string; - parent: string | null; stats: NodeStats | null; transitive: NodeStats | null; + transitiveSkew: SkewStats | null; childCount: number; lir: LirInfo[]; address: Address | null; // null for synthetic port nodes, which have no operator row of their own. operatorId: bigint | null; + // Non-empty only for ports: the out-of-view side of every crossing that + // resolved to this port and wasn't already reachable some other way. + peers: PortPeer[]; } export interface VisibleEdge { @@ -114,14 +144,48 @@ export interface LirSpanRow { operatorIdStart: bigint | number | string; operatorIdEnd: bigint | number | string; } +export interface PerWorkerStatRow { + id: bigint | number | string; + workerId: bigint | number | string; + elapsedNs: bigint | number | string | null; + arrangementSize: bigint | number | string | null; +} const toBigInt = (v: bigint | number | string | null | undefined): bigint => v == null ? 0n : BigInt(v); +// workerId -> value. A worker absent from a node's vector never scheduled or +// arranged anything for it (no row at all, per the introspection tables' +// convention), which the caller treats as "doesn't count towards average" +// rather than "counts as zero" (matching EXPLAIN ANALYZE ... WITH SKEW). +type WorkerVector = Map; + +function mergeWorkerVectors(a: WorkerVector, b: WorkerVector): WorkerVector { + if (b.size === 0) return a; + const merged = new Map(a); + for (const [workerId, v] of b) { + merged.set(workerId, (merged.get(workerId) ?? 0n) + v); + } + return merged; +} + +function skewRatio(vector: WorkerVector): number { + if (vector.size === 0) return 0; + let sum = 0n; + let max = 0n; + for (const v of vector.values()) { + sum += v; + if (v > max) max = v; + } + const avg = Number(sum) / vector.size; + return avg === 0 ? 0 : Number(max) / avg; +} + export function buildDataflowStructure( operators: OperatorRow[], channels: ChannelRow[], lirSpans: LirSpanRow[], + perWorkerStats: PerWorkerStatRow[] = [], ): DataflowStructure { const spans = lirSpans.map((s) => ({ exportId: s.exportId, @@ -133,6 +197,7 @@ export function buildDataflowStructure( end: toBigInt(s.operatorIdEnd), })); const nodes = new Map(); + const nodeIdByOperatorId = new Map(); for (const row of operators) { const address = row.address.map(Number); const id = nodeIdOf(address); @@ -142,6 +207,7 @@ export function buildDataflowStructure( arrangementSize: toBigInt(row.arrangementSize), elapsedNs: toBigInt(row.elapsedNs), }; + nodeIdByOperatorId.set(opId.toString(), id); nodes.set(id, { id, operatorId: opId, @@ -151,6 +217,8 @@ export function buildDataflowStructure( children: [], own, transitive: own, // replaced below + ownSkew: { cpuSkew: 0, memorySkew: 0 }, // replaced below + transitiveSkew: { cpuSkew: 0, memorySkew: 0 }, // replaced below lir: spans .filter((s) => s.start <= opId && opId < s.end) .map(({ exportId, lirId, parentLirId, nesting, operator }) => ({ @@ -179,17 +247,55 @@ export function buildDataflowStructure( return x[x.length - 1] - y[y.length - 1]; }); } - const fillTransitive = (id: NodeId): NodeStats => { + + const ownCpuByNode = new Map(); + const ownMemoryByNode = new Map(); + for (const row of perWorkerStats) { + const id = nodeIdByOperatorId.get(toBigInt(row.id).toString()); + if (!id) continue; // operator outside this dataflow's own address tree + const workerId = Number(row.workerId); + if (row.elapsedNs != null) { + const vec = ownCpuByNode.get(id) ?? new Map(); + vec.set(workerId, (vec.get(workerId) ?? 0n) + toBigInt(row.elapsedNs)); + ownCpuByNode.set(id, vec); + } + if (row.arrangementSize != null) { + const vec = ownMemoryByNode.get(id) ?? new Map(); + vec.set( + workerId, + (vec.get(workerId) ?? 0n) + toBigInt(row.arrangementSize), + ); + ownMemoryByNode.set(id, vec); + } + } + + const fillTransitive = ( + id: NodeId, + ): { stats: NodeStats; cpuVec: WorkerVector; memoryVec: WorkerVector } => { const node = nodes.get(id)!; + const ownCpuVec = ownCpuByNode.get(id) ?? new Map(); + const ownMemoryVec = ownMemoryByNode.get(id) ?? new Map(); + node.ownSkew = { + cpuSkew: skewRatio(ownCpuVec), + memorySkew: skewRatio(ownMemoryVec), + }; const t = { ...node.own }; + let cpuVec = ownCpuVec; + let memoryVec = ownMemoryVec; for (const c of node.children) { - const ct = fillTransitive(c); - t.arrangementRecords += ct.arrangementRecords; - t.arrangementSize += ct.arrangementSize; - t.elapsedNs += ct.elapsedNs; + const child = fillTransitive(c); + t.arrangementRecords += child.stats.arrangementRecords; + t.arrangementSize += child.stats.arrangementSize; + t.elapsedNs += child.stats.elapsedNs; + cpuVec = mergeWorkerVectors(cpuVec, child.cpuVec); + memoryVec = mergeWorkerVectors(memoryVec, child.memoryVec); } node.transitive = t; - return t; + node.transitiveSkew = { + cpuSkew: skewRatio(cpuVec), + memorySkew: skewRatio(memoryVec), + }; + return { stats: t, cpuVec, memoryVec }; }; fillTransitive(roots[0]); return { @@ -208,15 +314,27 @@ export function buildDataflowStructure( }; } -export const MAX_VISIBLE_NODES = 1500; - -// Shallowest collapsed ancestor absorbs everything beneath it. -function representative(address: Address, collapsed: CollapseState): NodeId { - for (let len = 1; len < address.length; len++) { - const prefix = nodeIdOf(address.slice(0, len)); - if (collapsed.has(prefix)) return prefix; +// A view shows exactly one scope's direct children; anything deeper is +// rolled up into its child's box, addressed at that box's depth (viewRoot's +// depth + 1). representativeInView is the general form of that projection, +// exported for translating arbitrary structure addresses (search matches, +// LIR members) into the box that would represent them in a given view, or +// null if the address isn't inside viewRootAddress's subtree at all. +export function representativeInView( + address: Address, + viewRootAddress: Address, +): NodeId | null { + if ( + address.length > viewRootAddress.length && + viewRootAddress.every((v, i) => address[i] === v) + ) { + return nodeIdOf(address.slice(0, viewRootAddress.length + 1)); } - return nodeIdOf(address); + return null; +} + +function representative(address: Address, viewRootAddress: Address): NodeId { + return representativeInView(address, viewRootAddress) ?? nodeIdOf(address); } // A scope boundary crossing is logged as two channels that share a port @@ -231,12 +349,18 @@ function representative(address: Address, collapsed: CollapseState): NodeId { // direction, port number), so they visually chain through one port node // instead of the outer half landing on the region's own node (indistinguishable // from every other port sharing that scope) or the inner half dangling. +// +// A port only ever renders for viewRoot's own boundary: every other scope in +// a view is shown as a collapsed box (never "expanded"), so any crossing at +// its boundary rolls up into that box instead, exactly like a normal channel +// endpoint landing inside it would. function endpointId( structure: DataflowStructure, address: Address, port: number, side: "from" | "to", - collapsed: CollapseState, + viewRoot: NodeId, + viewRootAddress: Address, ): { id: string; port?: { scope: NodeId; direction: "input" | "output"; port: number }; @@ -244,8 +368,9 @@ function endpointId( if (address[address.length - 1] === 0) { const scope = address.slice(0, -1); const scopeId = nodeIdOf(scope); - const rep = scope.length === 0 ? scopeId : representative(scope, collapsed); - if (rep !== scopeId || collapsed.has(scopeId)) return { id: rep }; + const rep = + scope.length === 0 ? scopeId : representative(scope, viewRootAddress); + if (rep !== scopeId || scopeId !== viewRoot) return { id: rep }; const direction = side === "from" ? "input" : "output"; return { id: `${scopeId}:${direction === "input" ? "in" : "out"}:${port}`, @@ -255,8 +380,8 @@ function endpointId( const scopeId = nodeIdOf(address); const scopeNode = structure.nodes.get(scopeId); if (scopeNode && scopeNode.children.length > 0) { - const rep = representative(address, collapsed); - if (rep === scopeId && !collapsed.has(scopeId)) { + const rep = representative(address, viewRootAddress); + if (rep === scopeId && scopeId === viewRoot) { const direction = side === "to" ? "input" : "output"; return { id: `${scopeId}:${direction === "input" ? "in" : "out"}:${port}`, @@ -265,84 +390,170 @@ function endpointId( } return { id: rep }; } - return { id: representative(address, collapsed) }; + return { id: representative(address, viewRootAddress) }; } +// Renders exactly one scope's direct children, each either a leaf operator or +// a box summarizing a whole nested subtree (never expanded in place); +// double-clicking a box navigates to a new view rooted there instead. export function deriveVisibleGraph( structure: DataflowStructure, - collapsed: CollapseState, + viewRoot: NodeId, ): VisibleGraph { - const nodes: VisibleNode[] = []; - const emit = (id: NodeId, parent: string | null) => { + const viewRootNode = structure.nodes.get(viewRoot)!; + const viewRootAddress = viewRootNode.address; + const nodes: VisibleNode[] = viewRootNode.children.map((id) => { const node = structure.nodes.get(id)!; - const isCollapsed = collapsed.has(id) && node.children.length > 0; - const kind = - node.children.length === 0 - ? "operator" - : isCollapsed - ? "collapsedRegion" - : "region"; - nodes.push({ + const kind = node.children.length === 0 ? "operator" : "region"; + return { id, kind, label: node.name, - parent, - stats: kind === "collapsedRegion" ? node.transitive : node.own, + stats: kind === "region" ? node.transitive : node.own, transitive: node.transitive, + transitiveSkew: node.transitiveSkew, childCount: node.children.length, lir: node.lir, address: node.address, operatorId: node.operatorId, - }); - if (!isCollapsed) for (const c of node.children) emit(c, id); - }; - for (const c of structure.nodes.get(structure.root)!.children) emit(c, null); + peers: [], + }; + }); + const visibleIds = new Set(nodes.map((n) => n.id)); const edgesById = new Map }>(); const ports = new Map(); + const portId = (p: { + scope: NodeId; + direction: "input" | "output"; + port: number; + }) => `${p.scope}:${p.direction === "input" ? "in" : "out"}:${p.port}`; + // Records the out-of-view side of a crossing on its port, so a port that + // can't show its peer directly can still be jumped to. One output can feed + // several inputs (and vice versa), so this can add up across channels + // rather than replace; the same peer address showing up twice (e.g. two + // channel rows for the same logical pair) just accumulates its stats. + const addPeer = ( + p: { scope: NodeId; direction: "input" | "output"; port: number }, + peerAddress: Address, + peerPort: number, + peerSide: "from" | "to", + messagesSent: bigint, + batchesSent: bigint, + channelType: string | null, + ) => { + const peerId = nodeIdOf(peerAddress); + if (!structure.nodes.has(peerId)) return; // elided scope, nothing to jump to + const port = ports.get(portId(p))!; + const existing = port.peers.find( + (peer) => nodeIdOf(peer.address) === peerId, + ); + if (existing) { + existing.messagesSent += messagesSent; + existing.batchesSent += batchesSent; + if (channelType && !existing.channelTypes.includes(channelType)) { + existing.channelTypes = [...existing.channelTypes, channelType].sort(); + } + } else { + // Re-resolve this same crossing as if viewRoot were the peer itself, + // to find the exact port a jump should land on inside it. A leaf peer + // has nothing to drill into, so this naturally comes back port-less. + const fromPeer = endpointId( + structure, + peerAddress, + peerPort, + peerSide, + peerId, + peerAddress, + ); + port.peers.push({ + address: peerAddress, + label: structure.nodes.get(peerId)!.name, + messagesSent, + batchesSent, + channelTypes: channelType ? [channelType] : [], + peerPortId: fromPeer.port ? fromPeer.id : null, + }); + } + }; for (const ch of structure.channels) { const from = endpointId( structure, ch.fromAddress, ch.fromPort, "from", - collapsed, + viewRoot, + viewRootAddress, ); - const to = endpointId(structure, ch.toAddress, ch.toPort, "to", collapsed); - // A non-port endpoint id names an operator or region address, which must - // exist in the structure. Real dataflows can log channels touching an - // address with no corresponding operator row (e.g. an elided scope), so - // drop the channel rather than hand elk a reference to a node that will - // never be emitted. - if ( - (!from.port && !structure.nodes.has(from.id)) || - (!to.port && !structure.nodes.has(to.id)) - ) { - continue; - } - if (from.id === to.id) continue; + const to = endpointId( + structure, + ch.toAddress, + ch.toPort, + "to", + viewRoot, + viewRootAddress, + ); + // A port node is registered whenever either side resolves to one, even + // if the edge itself is about to be dropped: viewRoot's own boundary can + // have traffic to a sibling entirely outside this view (nothing on the + // far side to draw an edge to), and the port stub is the only visible + // trace that the crossing exists at all. for (const p of [from.port, to.port]) { - if ( - p && - !ports.has( - `${p.scope}:${p.direction === "input" ? "in" : "out"}:${p.port}`, - ) - ) { - const id = `${p.scope}:${p.direction === "input" ? "in" : "out"}:${p.port}`; - ports.set(id, { - id, + if (p && !ports.has(portId(p))) { + ports.set(portId(p), { + id: portId(p), kind: "port", label: `${p.direction} ${p.port}`, - parent: p.scope === structure.root ? null : p.scope, stats: null, transitive: null, + transitiveSkew: null, childCount: 0, lir: [], address: null, operatorId: null, + peers: [], }); } } + // The far side of a port crossing is worth recording as a jump target + // only when this view doesn't already show it some other way (e.g. the + // inner half of a boundary pairing lands on a node already visible here, + // which needs no jump link since there's already a drawn edge to it). + if (from.port && !to.port && !visibleIds.has(to.id)) { + addPeer( + from.port, + ch.toAddress, + ch.toPort, + "to", + ch.messagesSent, + ch.batchesSent, + ch.channelType, + ); + } + if (to.port && !from.port && !visibleIds.has(from.id)) { + addPeer( + to.port, + ch.fromAddress, + ch.fromPort, + "from", + ch.messagesSent, + ch.batchesSent, + ch.channelType, + ); + } + // A non-port endpoint id must name one of this view's own boxes. Real + // dataflows can log channels touching an address with no corresponding + // operator row (e.g. an elided scope), and any address outside viewRoot's + // subtree resolves to an id this view never emits either; drop the + // edge (but keep any port registered above) in both cases rather than + // hand elk a dangling reference. + if ( + (!from.port && !visibleIds.has(from.id)) || + (!to.port && !visibleIds.has(to.id)) + ) { + continue; + } + if (from.id === to.id) continue; const id = `${from.id}=>${to.id}`; const existing = edgesById.get(id); if (existing) { @@ -368,6 +579,32 @@ export function deriveVisibleGraph( return { nodes: [...nodes, ...ports.values()], edges }; } +// The scope to navigate to so that every given address is directly visible, +// each as either itself (if it's already a direct child of the result) or +// the box that rolls it up. Backs off past any address that IS the raw +// common prefix (so every input lands strictly below the result, never +// equal to it) and past any prefix with no operator row of its own (an +// elided scope can't be navigated to, since it never appears in a view). +export function commonAncestorScope( + structure: DataflowStructure, + addresses: readonly Address[], +): NodeId { + let prefix = addresses[0]; + for (const a of addresses.slice(1)) { + let len = 0; + while (len < prefix.length && len < a.length && prefix[len] === a[len]) { + len++; + } + prefix = prefix.slice(0, len); + } + const shortest = Math.min(...addresses.map((a) => a.length)); + while (prefix.length >= shortest || !structure.nodes.has(nodeIdOf(prefix))) { + if (prefix.length === 0) return structure.root; + prefix = prefix.slice(0, -1); + } + return nodeIdOf(prefix); +} + // Hiding a run of idle operators would otherwise sever every path through // them, splitting the graph into disconnected islands even though a real // (idle) path exists. Splices a pass-through edge across each hidden run, @@ -482,30 +719,11 @@ export function rerouteHiddenNodes( return { nodes, edges: [...edgesById.values()] }; } -export function defaultCollapseState( - structure: DataflowStructure, -): CollapseState { - const collapsed = new Set(); - for (const node of structure.nodes.values()) { - if (node.children.length > 0 && node.id !== structure.root) - collapsed.add(node.id); - } - return collapsed; -} - -export function visibleNodeCount( - structure: DataflowStructure, - collapsed: CollapseState, -): number { - return deriveVisibleGraph(structure, collapsed).nodes.length; -} - export interface Filters { search: string; hideIdle: boolean; - heatmap: "off" | "elapsed" | "size"; + heatmap: "off" | "elapsed" | "size" | "cpuSkew" | "memorySkew"; heatmapThreshold: number; // 0..1 fraction of max - channelTypes: string[] | null; // null = all } export const DEFAULT_FILTERS: Filters = { @@ -513,7 +731,6 @@ export const DEFAULT_FILTERS: Filters = { hideIdle: false, heatmap: "off", heatmapThreshold: 0, - channelTypes: null, }; // decorateGraph returns every field set. @@ -521,36 +738,34 @@ export interface GraphDecorations { dimmedNodeIds: Set; hiddenNodeIds: Set; hiddenEdgeIds: Set; - dimmedEdgeIds: Set; nodeColors: Map; searchMatches: string[]; // visible node ids, document order } -export function allChannelTypes(structure: DataflowStructure): string[] { - const types = new Set(); - for (const c of structure.channels) - if (c.channelType) types.add(c.channelType); - return [...types].sort(); -} - +// Search matches anywhere in the whole dataflow, not just the current view +// (search is structure-wide so it stays useful once a graph is too big to +// show in one screen); searchInfo carries that lookup in, precomputed once +// per search string rather than re-walked per view. A box that doesn't +// itself match but contains a match deeper in its rolled-up subtree is left +// undimmed rather than excluded, so the match's path stays visible without +// forcing a navigation on every keystroke. export function decorateGraph( graph: VisibleGraph, filters: Filters, heatColor: (t: number) => string, + searchInfo: SubtreeSearchMatches | null, ): GraphDecorations { const d: GraphDecorations = { dimmedNodeIds: new Set(), hiddenNodeIds: new Set(), hiddenEdgeIds: new Set(), - dimmedEdgeIds: new Set(), nodeColors: new Map(), searchMatches: [], }; - const needle = filters.search.trim().toLowerCase(); for (const n of graph.nodes) { - if (needle && n.kind !== "port" && n.kind !== "region") { - if (n.label.toLowerCase().includes(needle)) d.searchMatches.push(n.id); - else d.dimmedNodeIds.add(n.id); + if (searchInfo && n.kind !== "port") { + if (searchInfo.matches.has(n.id)) d.searchMatches.push(n.id); + else if (!searchInfo.containsMatch.has(n.id)) d.dimmedNodeIds.add(n.id); } if ( filters.hideIdle && @@ -564,26 +779,26 @@ export function decorateGraph( } for (const e of graph.edges) { if (filters.hideIdle && e.messagesSent === 0n) d.hiddenEdgeIds.add(e.id); - if ( - filters.channelTypes !== null && - !e.channelTypes.some((t) => filters.channelTypes!.includes(t)) - ) { - d.dimmedEdgeIds.add(e.id); - } } if (filters.heatmap !== "off") { - const metric = (n: VisibleNode): bigint => - filters.heatmap === "elapsed" - ? (n.transitive?.elapsedNs ?? 0n) - : (n.transitive?.arrangementSize ?? 0n); + const heatmap = filters.heatmap; + const metric = (n: VisibleNode): number => { + switch (heatmap) { + case "elapsed": + return Number(n.transitive?.elapsedNs ?? 0n); + case "size": + return Number(n.transitive?.arrangementSize ?? 0n); + case "cpuSkew": + return n.transitiveSkew?.cpuSkew ?? 0; + case "memorySkew": + return n.transitiveSkew?.memorySkew ?? 0; + } + }; const candidates = graph.nodes.filter((n) => n.kind !== "port"); - const max = candidates.reduce( - (m, n) => (metric(n) > m ? metric(n) : m), - 0n, - ); - if (max > 0n) { + const max = candidates.reduce((m, n) => Math.max(m, metric(n)), 0); + if (max > 0) { for (const n of candidates) { - const t = Number(metric(n)) / Number(max); + const t = metric(n) / max; d.nodeColors.set(n.id, heatColor(t)); if (t < filters.heatmapThreshold) d.dimmedNodeIds.add(n.id); } @@ -592,37 +807,58 @@ export function decorateGraph( return d; } -// Search may match inside collapsed regions. Returns a collapse state with -// every ancestor of every matching node expanded (respecting nothing else). -// Expands every ancestor region of the given addresses, so their operators -// are guaranteed visible regardless of current collapse state. -export function expandAncestorsOf( - collapsed: CollapseState, - addresses: Iterable
, -): CollapseState { - const next = new Set(collapsed); - for (const address of addresses) { - for (let len = 1; len < address.length; len++) { - next.delete(nodeIdOf(address.slice(0, len))); +export interface SubtreeSearchMatches { + matches: Set; // nodes whose own name matches + containsMatch: Set; // matches, plus any ancestor of one +} + +// Computed once over the whole structure (not the current view), so a box is +// never dimmed just because the current scope hasn't drilled down to the +// match nested inside it yet. +export function subtreeSearchMatches( + structure: DataflowStructure, + search: string, +): SubtreeSearchMatches { + const needle = search.trim().toLowerCase(); + const matches = new Set(); + const containsMatch = new Set(); + if (!needle) return { matches, containsMatch }; + const visit = (id: NodeId): boolean => { + const node = structure.nodes.get(id)!; + let hit = node.name.toLowerCase().includes(needle); + if (hit) matches.add(id); + for (const c of node.children) { + if (visit(c)) hit = true; } + if (hit) containsMatch.add(id); + return hit; + }; + for (const id of structure.nodes.get(structure.root)!.children) visit(id); + return { matches, containsMatch }; +} + +function addressCompare(a: Address, b: Address): number { + for (let i = 0; i < Math.min(a.length, b.length); i++) { + if (a[i] !== b[i]) return a[i] - b[i]; } - return next; + return a.length - b.length; } -export function expandForSearch( +// The ordered, structure-wide match list prev/next cycles through: jumping +// to a match may navigate the view (unlike decorateGraph's searchMatches, +// which only ever lists what's already visible in the current scope). +export function allSearchMatches( structure: DataflowStructure, - collapsed: CollapseState, search: string, -): CollapseState { +): DataflowNode[] { const needle = search.trim().toLowerCase(); - if (!needle) return collapsed; - const matches = [...structure.nodes.values()].filter((node) => - node.name.toLowerCase().includes(needle), - ); - return expandAncestorsOf( - collapsed, - matches.map((node) => node.address), - ); + if (!needle) return []; + return [...structure.nodes.values()] + .filter( + (node) => + node.id !== structure.root && node.name.toLowerCase().includes(needle), + ) + .sort((a, b) => addressCompare(a.address, b.address)); } // Groups operator nodes by the LIR span covering them, keyed diff --git a/console/src/platform/dataflows/elkGraph.test.ts b/console/src/platform/dataflows/elkGraph.test.ts index 4f6881d3edbb2..430874eeb68c0 100644 --- a/console/src/platform/dataflows/elkGraph.test.ts +++ b/console/src/platform/dataflows/elkGraph.test.ts @@ -20,22 +20,24 @@ import { extractPositions, NODE_DIMENSIONS, toElkGraph } from "./elkGraph"; describe("toElkGraph", () => { const s = buildDataflowStructure(OPS, CHANNELS, LIR_SPANS); - it("nests elk children by visible parent", () => { - const elk = toElkGraph(deriveVisibleGraph(s, new Set())); - const region = elk.children!.find((c) => c.id === nodeIdOf([5, 1]))!; - expect(region.children!.map((c) => c.id)).toEqual( + it("lists every visible node flat, at its kind's dimensions", () => { + const elk = toElkGraph(deriveVisibleGraph(s, nodeIdOf([5, 1]))); + const ids = elk.children!.map((c) => c.id); + expect(ids).toEqual( expect.arrayContaining([ nodeIdOf([5, 1, 1]), nodeIdOf([5, 1, 2]), `${nodeIdOf([5, 1])}:in:0`, ]), ); - const leaf = region.children!.find((c) => c.id === nodeIdOf([5, 1, 1]))!; + const leaf = elk.children!.find((c) => c.id === nodeIdOf([5, 1, 1]))!; expect(leaf.width).toEqual(NODE_DIMENSIONS.operator.width); + // Flat: no node carries children of its own. + for (const c of elk.children!) expect(c.children).toBeUndefined(); }); - it("round-trips positions relative to parent", () => { - const elk = toElkGraph(deriveVisibleGraph(s, new Set())); + it("round-trips positions", () => { + const elk = toElkGraph(deriveVisibleGraph(s, s.root)); // simulate a layout result elk.children![0].x = 10; elk.children![0].y = 20; diff --git a/console/src/platform/dataflows/elkGraph.ts b/console/src/platform/dataflows/elkGraph.ts index 5f81aeea48d62..41f86e80478a8 100644 --- a/console/src/platform/dataflows/elkGraph.ts +++ b/console/src/platform/dataflows/elkGraph.ts @@ -25,60 +25,45 @@ export const NODE_DIMENSIONS: Record< { width: number; height: number } > = { operator: { width: 240, height: 72 }, - collapsedRegion: { width: 260, height: 96 }, - region: { width: 0, height: 0 }, + region: { width: 260, height: 96 }, port: { width: 90, height: 24 }, }; const LAYOUT_OPTIONS: Record = { "elk.algorithm": "layered", "elk.direction": "DOWN", - "elk.hierarchyHandling": "INCLUDE_CHILDREN", "elk.padding": "[top=48,left=16,bottom=16,right=16]", "elk.spacing.nodeNode": "24", "elk.layered.spacing.nodeNodeBetweenLayers": "48", }; +// A view is always one scope's direct children, so this is always a flat, +// single-level graph: nothing is ever nested inside anything else. export function toElkGraph(graph: VisibleGraph): ElkNode { - const elkNodes = new Map(); - const root: ElkNode = { + return { id: "__root__", layoutOptions: LAYOUT_OPTIONS, - children: [], - edges: [], + children: graph.nodes.map((n) => ({ + id: n.id, + ...NODE_DIMENSIONS[n.kind], + })), + edges: graph.edges.map((e) => ({ + id: e.id, + sources: [e.source], + targets: [e.target], + })), }; - for (const node of graph.nodes) { - const dims = NODE_DIMENSIONS[node.kind]; - const elkNode: ElkNode = { - id: node.id, - children: [], - ...(node.kind === "region" ? { layoutOptions: LAYOUT_OPTIONS } : dims), - }; - elkNodes.set(node.id, elkNode); - const parent = node.parent ? elkNodes.get(node.parent)! : root; - parent.children!.push(elkNode); - } - root.edges = graph.edges.map((e) => ({ - id: e.id, - sources: [e.source], - targets: [e.target], - })); - return root; } export function extractPositions(layouted: ElkNode): Positions { const positions: Positions = {}; - const walk = (node: ElkNode) => { - for (const child of node.children ?? []) { - positions[child.id] = { - x: child.x ?? 0, - y: child.y ?? 0, - width: child.width ?? 0, - height: child.height ?? 0, - }; - walk(child); - } - }; - walk(layouted); + for (const child of layouted.children ?? []) { + positions[child.id] = { + x: child.x ?? 0, + y: child.y ?? 0, + width: child.width ?? 0, + height: child.height ?? 0, + }; + } return positions; } diff --git a/console/src/platform/dataflows/nodeStyle.test.ts b/console/src/platform/dataflows/nodeStyle.test.ts new file mode 100644 index 0000000000000..44c59605e03f9 --- /dev/null +++ b/console/src/platform/dataflows/nodeStyle.test.ts @@ -0,0 +1,38 @@ +// Copyright Materialize, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +import { describe, expect, it } from "vitest"; + +import { prettyPrintChannelType } from "./nodeStyle"; + +describe("prettyPrintChannelType", () => { + it("strips module paths, aliases Diff/Error, and brackets Vec", () => { + expect( + prettyPrintChannelType( + "alloc::vec::Vec<(mz_repr::row::Row, mz_repr::timestamp::Timestamp, mz_ore::overflowing::Overflowing)>", + ), + ).toEqual("[(Row, Timestamp, Diff)]"); + }); + + it("handles the error-channel variant", () => { + expect( + prettyPrintChannelType( + "alloc::vec::Vec<(mz_compute::render::errors::DataflowErrorSer, mz_repr::timestamp::Timestamp, mz_ore::overflowing::Overflowing)>", + ), + ).toEqual("[(Error, Timestamp, Diff)]"); + }); + + it("unwraps nested Vec and unrecognized generics without mangling them", () => { + expect( + prettyPrintChannelType( + "alloc::vec::Vec)>>>>", + ), + ).toEqual("[Rc>>]"); + }); +}); diff --git a/console/src/platform/dataflows/nodeStyle.ts b/console/src/platform/dataflows/nodeStyle.ts index f505d0899963a..2b6698f020dd1 100644 --- a/console/src/platform/dataflows/nodeStyle.ts +++ b/console/src/platform/dataflows/nodeStyle.ts @@ -37,6 +37,40 @@ export function formatElapsed(ns: bigint): string { return `scheduled ${Math.round(Number(ns) / 1e9)}s`; } +// A channel's `type` column is the raw monomorphized Rust container type +// (e.g. "alloc::vec::Vec<(mz_repr::row::Row, mz_repr::timestamp::Timestamp, +// mz_ore::overflowing::Overflowing)>"). Path-qualified and generics-heavy, +// so this drops module paths down to the final segment, aliases the two +// generics that always mean the same thing in this position (the diff/count +// column, the decoded error variant), and turns `Vec` into `[X]` to match +// how we'd write the type by hand. Anything it doesn't recognize (batch/trace +// internals) still gets the path-stripping and Vec treatment, which is +// already most of the noise, even without a dedicated alias. +export function prettyPrintChannelType(raw: string): string { + let s = raw.replace(/(?:[A-Za-z_][A-Za-z0-9_]*::)+/g, ""); + s = s.replace(/\bOverflowing/g, "Diff"); + s = s.replace(/\bDataflowErrorSer\b/g, "Error"); + // `Vec<...>` -> `[...]`, repeatedly: each pass strips one level of nesting, + // so a Vec inside a Vec's contents surfaces on the next iteration. + for (;;) { + const start = s.indexOf("Vec<"); + if (start === -1) break; + let depth = 0; + let end = start + 3; + for (; end < s.length; end++) { + if (s[end] === "<") depth++; + else if (s[end] === ">" && --depth === 0) break; + } + s = + s.slice(0, start) + + "[" + + s.slice(start + 4, end) + + "]" + + s.slice(end + 1); + } + return s; +} + export type FlowNodeData = { node: VisibleNode; dimmed: boolean; diff --git a/console/src/platform/dataflows/nodes.tsx b/console/src/platform/dataflows/nodes.tsx index b7a8c1914ca98..7a5c2ca0ffdea 100644 --- a/console/src/platform/dataflows/nodes.tsx +++ b/console/src/platform/dataflows/nodes.tsx @@ -90,9 +90,9 @@ export const OperatorNode = ({ data }: NodeProps & { data: FlowNodeData }) => ( ); -export const CollapsedRegionNode = ({ - data, -}: NodeProps & { data: FlowNodeData }) => ( +// Always shown collapsed (a scope is never expanded in place); double-click +// navigates into it instead of unfolding it here. +export const RegionNode = ({ data }: NodeProps & { data: FlowNodeData }) => ( @@ -110,25 +110,6 @@ export const CollapsedRegionNode = ({ ); -export const RegionNode = ({ data }: NodeProps & { data: FlowNodeData }) => ( - - - {data.node.label} - - - - -); - export const PortNode = ({ data }: NodeProps & { data: FlowNodeData }) => ( Date: Wed, 8 Jul 2026 22:42:38 +0200 Subject: [PATCH 20/45] console: drop dataflow visualizer implementation plan doc Co-Authored-By: Claude Sonnet 5 --- .../2026-07-06-dataflow-visualizer-rebuild.md | 2758 ----------------- 1 file changed, 2758 deletions(-) delete mode 100644 docs/superpowers/plans/2026-07-06-dataflow-visualizer-rebuild.md diff --git a/docs/superpowers/plans/2026-07-06-dataflow-visualizer-rebuild.md b/docs/superpowers/plans/2026-07-06-dataflow-visualizer-rebuild.md deleted file mode 100644 index 795d8a57a40ef..0000000000000 --- a/docs/superpowers/plans/2026-07-06-dataflow-visualizer-rebuild.md +++ /dev/null @@ -1,2758 +0,0 @@ -# Dataflow visualizer rebuild implementation plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Replace the console's graphviz-WASM dataflow visualizer with a React Flow + elkjs graph view with dataflow selection, in-place collapsible regions, filters, and stats, closing CNS-108 and CNS-109. - -**Architecture:** Pure graph-model functions (`dataflowGraph.ts`) turn introspection SQL rows into a region tree, then derive a visible graph from collapse state. elkjs lays out the visible graph in a web worker. React Flow renders Chakra-styled nodes. Data hooks tag results with the params that produced them so stale data never renders. - -**Tech Stack:** React 18, TypeScript, Chakra UI v2, kysely (SQL text), vitest + msw, `@xyflow/react` (new), `elkjs` (new), Vite. - -**Spec:** `console/doc/design/20260706_dataflow_visualizer_rebuild.md` (approved at 1aa33ec74f). Two review items folded in: `Channel` is defined in Task 2, and `DataflowNode.lir` is an array (one entry per export span covering the operator, no disjointness assumed). - -## Global constraints - -* Repo: worktree `/home/moritz/dev/repos/materialize/.claude/worktrees/dataflow-viz-rebuild`, branch `moritz/dataflow-visualizer-rebuild`. All paths below relative to repo root. -* All commands run from `console/` unless stated. Test: `yarn test --run `. Typecheck: `yarn typecheck`. Lint: `yarn lint`. -* Every new file starts with the BSL copyright header (copy verbatim from any existing `console/src` file). -* Commit messages: `console: `. Commit after every task. -* No polling. No `d3-graphviz` or WASM. No new deps beyond `@xyflow/react` and `elkjs`. -* Catalog strings (operator names, channel types, LIR operators) must only ever render as React text nodes, never concatenated into any markup or DSL string. -* No em-dashes or structuring semicolons in comments. Comments state contracts, not narration. -* `MAX_VISIBLE_NODES = 1500` (counts nodes, not edges). -* Everything stays behind the `visualization-features` LaunchDarkly flag. -* SQL identifiers interpolated into kysely `sql` templates must go through `escapedLiteral` (`lit`); numeric ids are validated with `/^\d+$/` and cast (`::uint8`) after `lit`. - -## File structure - -``` -console/src/platform/dataflows/ - dataflowGraph.ts types + buildDataflowStructure + deriveVisibleGraph + decorateGraph (pure) - dataflowGraph.test.ts - elkGraph.ts VisibleGraph -> ELK JSON, ELK result -> positions (pure) - elkGraph.test.ts - layout.worker.ts module worker: elk.bundled.js + LayoutRequest/LayoutResponse - useElkLayout.ts hook owning the worker, request ids, position cache - nodes.tsx OperatorNode, RegionNode, CollapsedRegionNode, PortNode + dimensions - ChannelEdge.tsx custom edge with label + channel-type tooltip - DataflowGraphView.tsx React Flow canvas wiring - DataflowsPage.tsx cluster-level list page + export deep-link resolution - DataflowDetailPage.tsx graph page: selectors, toolbar, panels, error states - DataflowToolbar.tsx search / hide-idle / heatmap / channel-type controls - NodeDetailPanel.tsx click-a-node side panel - LirPanel.tsx LIR operator list + hover highlight - DataflowDetailPage.test.tsx -console/src/api/materialize/dataflow/ - useDataflowGraphData.ts structure queries keyed by dataflow id, params-tagged results - useDataflowList.ts selector list query - useDataflowIdForExport.ts export id -> dataflow id -console/src/test/mockReactFlow.tsx shared @xyflow/react mock for jsdom tests -``` - -Modified: `console/package.json`, `console/src/platform/clusters/ClusterDetail.tsx`, `console/src/platform/object-explorer/SimpleObjectDetailRoutes.tsx`, spec doc. -Deleted at the end: `console/src/platform/clusters/DataflowVisualizer.tsx`, `console/src/api/materialize/useDataflowStructure.ts`, `d3-graphviz` dep. - ---- - -## Phase 1: gated vertical slice - -### Task 1: Dependencies and spec amendment - -**Files:** -- Modify: `console/package.json` -- Modify: `console/doc/design/20260706_dataflow_visualizer_rebuild.md` - -**Interfaces:** -- Produces: `@xyflow/react` and `elkjs` installable, importable. - -- [ ] **Step 1: Add deps** - -Run: `cd console && yarn add @xyflow/react elkjs` -Expected: `package.json` gains both under `dependencies`, `yarn.lock` updated. No other dep changes (`git diff yarn.lock | grep '^+version' | wc -l` small). - -- [ ] **Step 2: Amend spec worker mechanism** - -The spec pinned `elkjs/lib/elk-worker.min.js?worker`. We use the strictly more Vite-native mechanism instead: our own module worker (`layout.worker.ts`) that imports `elkjs/lib/elk.bundled.js` and speaks the spec's `LayoutRequest`/`LayoutResponse` protocol directly. Same code off the main thread, first-class Vite dev + build support, and the protocol wrapper is needed anyway for stale-response dropping. In the spec's rendering section, replace the two sentences pinning the mechanism with: - -``` -The console builds with Vite, so layout runs in a Vite module worker (`new Worker(new URL("./layout.worker.ts", import.meta.url), { type: "module" })`) that imports `elkjs/lib/elk.bundled.js` and implements the layout request protocol below. -Phase 1 verifies this in both `vite dev` and the production build. -``` - -Also replace the spec's fallback sentence (it references the prebuilt worker script that no longer exists under this mechanism) with: - -``` -If the module worker fails to bundle or run `elk.bundled.js`, the fallback is running elkjs on the main thread in a lazily loaded chunk, accepting UI stalls during layout. -``` - -- [ ] **Step 3: Typecheck + commit** - -Run: `yarn typecheck` -Expected: PASS (no source changes yet). - -```bash -git add console/package.json console/yarn.lock console/doc/design/20260706_dataflow_visualizer_rebuild.md -git commit -m "console: add @xyflow/react and elkjs for dataflow visualizer rebuild" -``` - -### Task 2: Graph model types and buildDataflowStructure - -**Files:** -- Create: `console/src/platform/dataflows/dataflowGraph.ts` -- Test: `console/src/platform/dataflows/dataflowGraph.test.ts` - -**Interfaces:** -- Produces (exact, later tasks depend on these): - -```typescript -export type Address = number[]; -export type NodeId = string; -export function nodeIdOf(address: Address): NodeId; // JSON.stringify(address) - -export interface NodeStats { - arrangementRecords: bigint; - arrangementSize: bigint; - elapsedNs: bigint; -} - -export interface LirInfo { exportId: string; lirId: string; operator: string; } - -export interface DataflowNode { - id: NodeId; - address: Address; - name: string; - parent: NodeId | null; - children: NodeId[]; - own: NodeStats; - transitive: NodeStats; // own + subtree, precomputed - lir: LirInfo[]; // one entry per export span covering this operator id -} - -export interface Channel { - id: number; - fromAddress: Address; - fromPort: number; - toAddress: Address; - toPort: number; - messagesSent: bigint; - batchesSent: bigint; - channelType: string | null; -} - -export interface DataflowStructure { - nodes: Map; - root: NodeId; - channels: Channel[]; -} - -// SQL row shapes (values arrive as strings or numbers, normalized here) -export interface OperatorRow { - id: bigint | number | string; - address: string[]; - name: string; - arrangementRecords: bigint | number | string | null; - arrangementSize: bigint | number | string | null; - elapsedNs: bigint | number | string | null; -} -export interface ChannelRow { - id: number | string; - fromOperatorAddress: string[]; - fromPort: number | string; - toOperatorAddress: string[]; - toPort: number | string; - messagesSent: bigint | number | string; - batchesSent: bigint | number | string; - channelType: string | null; -} -export interface LirSpanRow { - exportId: string; - lirId: string; - operator: string; - operatorIdStart: bigint | number | string; - operatorIdEnd: bigint | number | string; -} - -export function buildDataflowStructure( - operators: OperatorRow[], - channels: ChannelRow[], - lirSpans: LirSpanRow[], -): DataflowStructure; // throws if not exactly one root (address length 1) -``` - -- [ ] **Step 1: Write the failing tests** - -`dataflowGraph.test.ts` (header + imports elided in later snippets, always include them): - -```typescript -import { describe, expect, it } from "vitest"; - -import { - buildDataflowStructure, - nodeIdOf, - type ChannelRow, - type LirSpanRow, - type OperatorRow, -} from "./dataflowGraph"; - -// Dataflow 5: root [5], region [5,1] with children [5,1,1], [5,1,2], leaf [5,2]. -export const OPS: OperatorRow[] = [ - { id: "10", address: ["5"], name: "Dataflow", arrangementRecords: null, arrangementSize: null, elapsedNs: "0" }, - { id: "11", address: ["5", "1"], name: "Region", arrangementRecords: "0", arrangementSize: "0", elapsedNs: "5" }, - { id: "12", address: ["5", "1", "1"], name: "Join", arrangementRecords: "100", arrangementSize: "4096", elapsedNs: "7" }, - { id: "13", address: ["5", "1", "2"], name: "Map", arrangementRecords: "0", arrangementSize: "0", elapsedNs: "1" }, - { id: "14", address: ["5", "2"], name: "Sink", arrangementRecords: "0", arrangementSize: "0", elapsedNs: "2" }, -]; -export const CHANNELS: ChannelRow[] = [ - // into the region: leaf [5,2] <- region output; region input port -> child - { id: "1", fromOperatorAddress: ["5", "1", "0"], fromPort: "0", toOperatorAddress: ["5", "1", "1"], toPort: "0", messagesSent: "3", batchesSent: "1", channelType: "rows" }, - { id: "2", fromOperatorAddress: ["5", "1", "1"], fromPort: "0", toOperatorAddress: ["5", "1", "2"], toPort: "0", messagesSent: "5", batchesSent: "2", channelType: "rows" }, - { id: "3", fromOperatorAddress: ["5", "1"], fromPort: "0", toOperatorAddress: ["5", "2"], toPort: "0", messagesSent: "5", batchesSent: "2", channelType: "batches" }, -]; -export const LIR_SPANS: LirSpanRow[] = [ - { exportId: "u42", lirId: "1", operator: "Join::Differential", operatorIdStart: "11", operatorIdEnd: "13" }, -]; - -describe("buildDataflowStructure", () => { - it("builds the region tree from address prefixes", () => { - const s = buildDataflowStructure(OPS, CHANNELS, LIR_SPANS); - expect(s.root).toEqual(nodeIdOf([5])); - const root = s.nodes.get(s.root)!; - expect(root.children).toEqual([nodeIdOf([5, 1]), nodeIdOf([5, 2])]); - const region = s.nodes.get(nodeIdOf([5, 1]))!; - expect(region.parent).toEqual(s.root); - expect(region.children).toEqual([nodeIdOf([5, 1, 1]), nodeIdOf([5, 1, 2])]); - }); - - it("precomputes transitive stats", () => { - const s = buildDataflowStructure(OPS, CHANNELS, LIR_SPANS); - const region = s.nodes.get(nodeIdOf([5, 1]))!; - expect(region.own.elapsedNs).toEqual(5n); - expect(region.transitive.elapsedNs).toEqual(13n); - expect(region.transitive.arrangementRecords).toEqual(100n); - expect(region.transitive.arrangementSize).toEqual(4096n); - }); - - it("maps LIR spans to operators by id range, as an array", () => { - const s = buildDataflowStructure(OPS, CHANNELS, LIR_SPANS); - expect(s.nodes.get(nodeIdOf([5, 1, 1]))!.lir).toEqual([ - { exportId: "u42", lirId: "1", operator: "Join::Differential" }, - ]); - // end is exclusive - expect(s.nodes.get(nodeIdOf([5, 2]))!.lir).toEqual([]); - }); - - it("throws without exactly one root", () => { - expect(() => buildDataflowStructure([], [], [])).toThrow(); - }); - - it("normalizes channels", () => { - const s = buildDataflowStructure(OPS, CHANNELS, []); - expect(s.channels[0]).toEqual({ - id: 1, fromAddress: [5, 1, 0], fromPort: 0, toAddress: [5, 1, 1], toPort: 0, - messagesSent: 3n, batchesSent: 1n, channelType: "rows", - }); - }); -}); -``` - -- [ ] **Step 2: Run tests, verify failure** - -Run: `yarn test --run src/platform/dataflows/dataflowGraph.test.ts` -Expected: FAIL, cannot resolve `./dataflowGraph`. - -- [ ] **Step 3: Implement** - -```typescript -export function nodeIdOf(address: Address): NodeId { - return JSON.stringify(address); -} - -const toBigInt = (v: bigint | number | string | null | undefined): bigint => - v == null ? 0n : BigInt(v); - -export function buildDataflowStructure( - operators: OperatorRow[], - channels: ChannelRow[], - lirSpans: LirSpanRow[], -): DataflowStructure { - const spans = lirSpans.map((s) => ({ - exportId: s.exportId, - lirId: s.lirId, - operator: s.operator, - start: toBigInt(s.operatorIdStart), - end: toBigInt(s.operatorIdEnd), - })); - const nodes = new Map(); - for (const row of operators) { - const address = row.address.map(Number); - const id = nodeIdOf(address); - const opId = toBigInt(row.id); - const own: NodeStats = { - arrangementRecords: toBigInt(row.arrangementRecords), - arrangementSize: toBigInt(row.arrangementSize), - elapsedNs: toBigInt(row.elapsedNs), - }; - nodes.set(id, { - id, - address, - name: row.name, - parent: address.length > 1 ? nodeIdOf(address.slice(0, -1)) : null, - children: [], - own, - transitive: own, // replaced below - lir: spans - .filter((s) => s.start <= opId && opId < s.end) - .map(({ exportId, lirId, operator }) => ({ exportId, lirId, operator })), - }); - } - const roots: NodeId[] = []; - for (const node of nodes.values()) { - if (node.parent === null) roots.push(node.id); - else nodes.get(node.parent)?.children.push(node.id); - } - if (roots.length !== 1) { - throw new Error(`expected exactly one root operator, found ${roots.length}`); - } - // Children in address order so layout and tests are deterministic. - for (const node of nodes.values()) { - node.children.sort((a, b) => { - const [x, y] = [nodes.get(a)!.address, nodes.get(b)!.address]; - return x[x.length - 1] - y[y.length - 1]; - }); - } - const fillTransitive = (id: NodeId): NodeStats => { - const node = nodes.get(id)!; - const t = { ...node.own }; - for (const c of node.children) { - const ct = fillTransitive(c); - t.arrangementRecords += ct.arrangementRecords; - t.arrangementSize += ct.arrangementSize; - t.elapsedNs += ct.elapsedNs; - } - node.transitive = t; - return t; - }; - fillTransitive(roots[0]); - return { - nodes, - root: roots[0], - channels: channels.map((c) => ({ - id: Number(c.id), - fromAddress: c.fromOperatorAddress.map(Number), - fromPort: Number(c.fromPort), - toAddress: c.toOperatorAddress.map(Number), - toPort: Number(c.toPort), - messagesSent: toBigInt(c.messagesSent), - batchesSent: toBigInt(c.batchesSent), - channelType: c.channelType, - })), - }; -} -``` - -- [ ] **Step 4: Run tests, verify pass** - -Run: `yarn test --run src/platform/dataflows/dataflowGraph.test.ts` -Expected: PASS (5 tests). - -- [ ] **Step 5: Commit** - -```bash -git add console/src/platform/dataflows/ -git commit -m "console: add dataflow graph model and structure builder" -``` - -### Task 3: deriveVisibleGraph (collapse, ports, edge aggregation) - -**Files:** -- Modify: `console/src/platform/dataflows/dataflowGraph.ts` -- Test: `console/src/platform/dataflows/dataflowGraph.test.ts` - -**Interfaces:** -- Produces: - -```typescript -export type CollapseState = ReadonlySet; // collapsed region ids - -export interface VisibleNode { - id: string; // NodeId, or `${scopeId}:in:${port}` / `${scopeId}:out:${port}` for ports - kind: "operator" | "region" | "collapsedRegion" | "port"; - label: string; - parent: string | null; // enclosing visible region, null when direct child of root - stats: NodeStats | null; // own for operator/region, transitive for collapsedRegion, null for port - transitive: NodeStats | null; // null for port - childCount: number; - lir: LirInfo[]; - address: Address | null; // null for ports -} - -export interface VisibleEdge { - id: string; // `${source}=>${target}` - source: string; - target: string; - messagesSent: bigint; - batchesSent: bigint; - channelTypes: string[]; // sorted set union of aggregated channels -} - -export interface VisibleGraph { nodes: VisibleNode[]; edges: VisibleEdge[]; } - -// Nodes are emitted parents-before-children (React Flow requirement). -// The root is the canvas itself, never a node. Collapsed descendants of a -// collapsed region are absorbed into it. Edges between endpoints that map to -// the same visible node are dropped. -export function deriveVisibleGraph( - structure: DataflowStructure, - collapsed: CollapseState, -): VisibleGraph; - -export function defaultCollapseState(structure: DataflowStructure): CollapseState; -// every region except the root collapsed - -export function visibleNodeCount(structure: DataflowStructure, collapsed: CollapseState): number; -export const MAX_VISIBLE_NODES = 1500; -``` - -- [ ] **Step 1: Write the failing tests** - -Append to `dataflowGraph.test.ts`: - -```typescript -import { - defaultCollapseState, - deriveVisibleGraph, - MAX_VISIBLE_NODES, - visibleNodeCount, -} from "./dataflowGraph"; - -describe("deriveVisibleGraph", () => { - const s = buildDataflowStructure(OPS, CHANNELS, LIR_SPANS); - const regionId = nodeIdOf([5, 1]); - - it("collapses regions to a single node with remapped edges", () => { - const g = deriveVisibleGraph(s, new Set([regionId])); - expect(g.nodes.map((n) => [n.id, n.kind])).toEqual([ - [regionId, "collapsedRegion"], - [nodeIdOf([5, 2]), "operator"], - ]); - // channel 3 region -> sink survives, channels 1 and 2 are internal - expect(g.edges).toEqual([ - { - id: `${regionId}=>${nodeIdOf([5, 2])}`, - source: regionId, - target: nodeIdOf([5, 2]), - messagesSent: 5n, - batchesSent: 2n, - channelTypes: ["batches"], - }, - ]); - const collapsed = g.nodes[0]; - expect(collapsed.stats).toEqual(s.nodes.get(regionId)!.transitive); - expect(collapsed.childCount).toEqual(2); - }); - - it("expands regions with port pseudo-nodes, parents before children", () => { - const g = deriveVisibleGraph(s, new Set()); - const ids = g.nodes.map((n) => n.id); - expect(ids.indexOf(regionId)).toBeLessThan(ids.indexOf(nodeIdOf([5, 1, 1]))); - const port = g.nodes.find((n) => n.kind === "port")!; - expect(port.id).toEqual(`${regionId}:in:0`); - expect(port.parent).toEqual(regionId); - expect(port.label).toEqual("input 0"); - // port -> Join edge preserved - expect(g.edges.map((e) => e.id)).toContain(`${regionId}:in:0=>${nodeIdOf([5, 1, 1])}`); - }); - - it("aggregates parallel channels between the same visible pair", () => { - const extra = buildDataflowStructure(OPS, [ - ...CHANNELS, - { id: "4", fromOperatorAddress: ["5", "1"], fromPort: "1", toOperatorAddress: ["5", "2"], toPort: "1", messagesSent: "7", batchesSent: "1", channelType: "rows" }, - ], []); - const g = deriveVisibleGraph(extra, new Set([regionId])); - const e = g.edges.find((e) => e.target === nodeIdOf([5, 2]))!; - expect(e.messagesSent).toEqual(12n); - expect(e.channelTypes).toEqual(["batches", "rows"]); - }); - - it("defaultCollapseState collapses all non-root regions", () => { - expect(defaultCollapseState(s)).toEqual(new Set([regionId])); - expect(visibleNodeCount(s, defaultCollapseState(s))).toEqual(2); - expect(MAX_VISIBLE_NODES).toEqual(1500); - }); -}); -``` - -- [ ] **Step 2: Run tests, verify failure** - -Run: `yarn test --run src/platform/dataflows/dataflowGraph.test.ts` -Expected: FAIL, `deriveVisibleGraph` not exported. - -- [ ] **Step 3: Implement** - -```typescript -export const MAX_VISIBLE_NODES = 1500; - -// Shallowest collapsed ancestor absorbs everything beneath it. -function representative(address: Address, collapsed: CollapseState): NodeId { - for (let len = 1; len < address.length; len++) { - const prefix = nodeIdOf(address.slice(0, len)); - if (collapsed.has(prefix)) return prefix; - } - return nodeIdOf(address); -} - -// A channel endpoint address ending in 0 denotes the enclosing scope's -// input (from side) or output (to side) port. -function endpointId( - address: Address, - port: number, - side: "from" | "to", - collapsed: CollapseState, -): { id: string; port?: { scope: NodeId; direction: "input" | "output"; port: number } } { - if (address[address.length - 1] === 0) { - const scope = address.slice(0, -1); - const scopeId = nodeIdOf(scope); - const rep = scope.length === 0 ? scopeId : representative(scope, collapsed); - if (rep !== scopeId || collapsed.has(scopeId)) return { id: rep }; - const direction = side === "from" ? "input" : "output"; - return { - id: `${scopeId}:${direction === "input" ? "in" : "out"}:${port}`, - port: { scope: scopeId, direction, port }, - }; - } - return { id: representative(address, collapsed) }; -} - -export function deriveVisibleGraph( - structure: DataflowStructure, - collapsed: CollapseState, -): VisibleGraph { - const nodes: VisibleNode[] = []; - const emit = (id: NodeId, parent: string | null) => { - const node = structure.nodes.get(id)!; - const isCollapsed = collapsed.has(id) && node.children.length > 0; - const kind = - node.children.length === 0 ? "operator" : isCollapsed ? "collapsedRegion" : "region"; - nodes.push({ - id, - kind, - label: node.name, - parent, - stats: kind === "collapsedRegion" ? node.transitive : node.own, - transitive: node.transitive, - childCount: node.children.length, - lir: node.lir, - address: node.address, - }); - if (!isCollapsed) for (const c of node.children) emit(c, id); - }; - for (const c of structure.nodes.get(structure.root)!.children) emit(c, null); - - const edgesById = new Map }>(); - const ports = new Map(); - for (const ch of structure.channels) { - const from = endpointId(ch.fromAddress, ch.fromPort, "from", collapsed); - const to = endpointId(ch.toAddress, ch.toPort, "to", collapsed); - if (from.id === to.id) continue; - for (const p of [from.port, to.port]) { - if (p && !ports.has(`${p.scope}:${p.direction === "input" ? "in" : "out"}:${p.port}`)) { - const id = `${p.scope}:${p.direction === "input" ? "in" : "out"}:${p.port}`; - ports.set(id, { - id, - kind: "port", - label: `${p.direction} ${p.port}`, - parent: p.scope === structure.root ? null : p.scope, - stats: null, - transitive: null, - childCount: 0, - lir: [], - address: null, - }); - } - } - const id = `${from.id}=>${to.id}`; - const existing = edgesById.get(id); - if (existing) { - existing.messagesSent += ch.messagesSent; - existing.batchesSent += ch.batchesSent; - if (ch.channelType) existing.typeSet.add(ch.channelType); - } else { - edgesById.set(id, { - id, - source: from.id, - target: to.id, - messagesSent: ch.messagesSent, - batchesSent: ch.batchesSent, - channelTypes: [], - typeSet: new Set(ch.channelType ? [ch.channelType] : []), - }); - } - } - const edges = [...edgesById.values()].map(({ typeSet, ...e }) => ({ - ...e, - channelTypes: [...typeSet].sort(), - })); - return { nodes: [...nodes, ...ports.values()], edges }; -} - -export function defaultCollapseState(structure: DataflowStructure): CollapseState { - const collapsed = new Set(); - for (const node of structure.nodes.values()) { - if (node.children.length > 0 && node.id !== structure.root) collapsed.add(node.id); - } - return collapsed; -} - -export function visibleNodeCount( - structure: DataflowStructure, - collapsed: CollapseState, -): number { - return deriveVisibleGraph(structure, collapsed).nodes.length; -} -``` - -Note: edge endpoints can reference a node id that is not visible when a channel endpoint lies inside an expanded region that the other endpoint's collapsed ancestor absorbed. `representative` guarantees both endpoints are visible ids by construction (any collapsed prefix wins). Port nodes are added to the node list after regions, and their `parent` region always precedes them because region nodes were emitted first. - -- [ ] **Step 4: Run tests, verify pass** - -Run: `yarn test --run src/platform/dataflows/dataflowGraph.test.ts` -Expected: PASS. - -- [ ] **Step 5: Commit** - -```bash -git add console/src/platform/dataflows/ -git commit -m "console: derive visible dataflow graph from collapse state" -``` - -### Task 4: ELK graph conversion and layout worker - -**Files:** -- Create: `console/src/platform/dataflows/elkGraph.ts` -- Create: `console/src/platform/dataflows/layout.worker.ts` -- Create: `console/src/platform/dataflows/useElkLayout.ts` -- Test: `console/src/platform/dataflows/elkGraph.test.ts` - -**Interfaces:** -- Consumes: `VisibleGraph`, `VisibleNode` from Task 3. -- Produces: - -```typescript -// elkGraph.ts -export interface NodePosition { x: number; y: number; width: number; height: number; } -export type Positions = Record; // key: visible node id, x/y relative to parent -export const NODE_DIMENSIONS: Record; -export function toElkGraph(graph: VisibleGraph): ElkNode; // nested by parent -export function extractPositions(layouted: ElkNode): Positions; - -// layout.worker.ts protocol -export interface LayoutRequest { requestId: number; graph: VisibleGraph; } -export interface LayoutResponse { requestId: number; positions?: Positions; error?: string; } - -// useElkLayout.ts -export function useElkLayout( - graph: VisibleGraph | null, - cacheKey: string, // caller-provided, e.g. structure key + sorted collapsed ids -): { positions: Positions | null; layouting: boolean; error: string | null }; -``` - -- [ ] **Step 1: Write the failing tests** - -`elkGraph.test.ts`: - -```typescript -import { describe, expect, it } from "vitest"; - -import { buildDataflowStructure, deriveVisibleGraph, nodeIdOf } from "./dataflowGraph"; -import { CHANNELS, LIR_SPANS, OPS } from "./dataflowGraph.test"; -import { extractPositions, NODE_DIMENSIONS, toElkGraph } from "./elkGraph"; - -describe("toElkGraph", () => { - const s = buildDataflowStructure(OPS, CHANNELS, LIR_SPANS); - - it("nests elk children by visible parent", () => { - const elk = toElkGraph(deriveVisibleGraph(s, new Set())); - const region = elk.children!.find((c) => c.id === nodeIdOf([5, 1]))!; - expect(region.children!.map((c) => c.id)).toEqual( - expect.arrayContaining([ - nodeIdOf([5, 1, 1]), - nodeIdOf([5, 1, 2]), - `${nodeIdOf([5, 1])}:in:0`, - ]), - ); - const leaf = region.children!.find((c) => c.id === nodeIdOf([5, 1, 1]))!; - expect(leaf.width).toEqual(NODE_DIMENSIONS.operator.width); - }); - - it("round-trips positions relative to parent", () => { - const elk = toElkGraph(deriveVisibleGraph(s, new Set())); - // simulate a layout result - elk.children![0].x = 10; - elk.children![0].y = 20; - const positions = extractPositions(elk); - expect(positions[elk.children![0].id]).toMatchObject({ x: 10, y: 20 }); - }); -}); -``` - -Export `OPS`, `CHANNELS`, `LIR_SPANS` from `dataflowGraph.test.ts` (they already are, per Task 2). - -- [ ] **Step 2: Run tests, verify failure** - -Run: `yarn test --run src/platform/dataflows/elkGraph.test.ts` -Expected: FAIL, module missing. - -- [ ] **Step 3: Implement elkGraph.ts** - -```typescript -import type { ElkNode } from "elkjs/lib/elk-api"; - -import type { VisibleGraph, VisibleNode } from "./dataflowGraph"; - -export const NODE_DIMENSIONS: Record = { - operator: { width: 240, height: 72 }, - collapsedRegion: { width: 260, height: 88 }, - region: { width: 0, height: 0 }, // sized by elk from children - port: { width: 90, height: 24 }, -}; - -const LAYOUT_OPTIONS: Record = { - "elk.algorithm": "layered", - "elk.direction": "RIGHT", - "elk.hierarchyHandling": "INCLUDE_CHILDREN", - "elk.padding": "[top=48,left=16,bottom=16,right=16]", - "elk.spacing.nodeNode": "24", - "elk.layered.spacing.nodeNodeBetweenLayers": "48", -}; - -export function toElkGraph(graph: VisibleGraph): ElkNode { - const elkNodes = new Map(); - const root: ElkNode = { id: "__root__", layoutOptions: LAYOUT_OPTIONS, children: [], edges: [] }; - for (const node of graph.nodes) { - const dims = NODE_DIMENSIONS[node.kind]; - const elkNode: ElkNode = { - id: node.id, - children: [], - ...(node.kind === "region" ? { layoutOptions: LAYOUT_OPTIONS } : dims), - }; - elkNodes.set(node.id, elkNode); - const parent = node.parent ? elkNodes.get(node.parent)! : root; - parent.children!.push(elkNode); - } - root.edges = graph.edges.map((e) => ({ - id: e.id, - sources: [e.source], - targets: [e.target], - })); - return root; -} - -export function extractPositions(layouted: ElkNode): Positions { - const positions: Positions = {}; - const walk = (node: ElkNode) => { - for (const child of node.children ?? []) { - positions[child.id] = { - x: child.x ?? 0, - y: child.y ?? 0, - width: child.width ?? 0, - height: child.height ?? 0, - }; - walk(child); - } - }; - walk(layouted); - return positions; -} -``` - -Note: `graph.nodes` is parents-before-children except ports, which come last but whose parent regions exist by then, so the single pass with `elkNodes.get(node.parent)` is safe. Hierarchical edges at the root level with `INCLUDE_CHILDREN` is the documented elk pattern for cross-hierarchy edges. - -- [ ] **Step 4: Run tests, verify pass** - -Run: `yarn test --run src/platform/dataflows/elkGraph.test.ts` -Expected: PASS. - -- [ ] **Step 5: Implement worker + hook (no unit test, covered by phase 1 gate and stubbed in component tests)** - -`layout.worker.ts`: - -```typescript -import ELK from "elkjs/lib/elk.bundled.js"; - -import type { VisibleGraph } from "./dataflowGraph"; -import { extractPositions, toElkGraph, type Positions } from "./elkGraph"; - -export interface LayoutRequest { requestId: number; graph: VisibleGraph; } -export interface LayoutResponse { requestId: number; positions?: Positions; error?: string; } - -const elk = new ELK(); - -self.onmessage = async (event: MessageEvent) => { - const { requestId, graph } = event.data; - try { - const layouted = await elk.layout(toElkGraph(graph)); - const response: LayoutResponse = { requestId, positions: extractPositions(layouted) }; - self.postMessage(response); - } catch (error) { - self.postMessage({ requestId, error: String(error) } satisfies LayoutResponse); - } -}; -``` - -`useElkLayout.ts`: - -```typescript -import React from "react"; - -import type { VisibleGraph } from "./dataflowGraph"; -import type { LayoutRequest, LayoutResponse } from "./layout.worker"; -import type { Positions } from "./elkGraph"; - -export function useElkLayout(graph: VisibleGraph | null, cacheKey: string) { - const workerRef = React.useRef(null); - const requestIdRef = React.useRef(0); - const cacheRef = React.useRef(new Map()); - const [state, setState] = React.useState<{ - key: string | null; - positions: Positions | null; - error: string | null; - }>({ key: null, positions: null, error: null }); - - React.useEffect(() => { - const worker = new Worker(new URL("./layout.worker.ts", import.meta.url), { - type: "module", - }); - workerRef.current = worker; - return () => worker.terminate(); - }, []); - - React.useEffect(() => { - if (!graph) return; - const cached = cacheRef.current.get(cacheKey); - if (cached) { - setState({ key: cacheKey, positions: cached, error: null }); - return; - } - const requestId = ++requestIdRef.current; - const worker = workerRef.current; - if (!worker) return; - const onMessage = (event: MessageEvent) => { - // Drop responses for superseded requests. - if (event.data.requestId !== requestIdRef.current) return; - if (event.data.positions) { - cacheRef.current.set(cacheKey, event.data.positions); - setState({ key: cacheKey, positions: event.data.positions, error: null }); - } else { - setState({ key: cacheKey, positions: null, error: event.data.error ?? "layout failed" }); - } - }; - worker.addEventListener("message", onMessage); - worker.postMessage({ requestId, graph } satisfies LayoutRequest); - return () => worker.removeEventListener("message", onMessage); - }, [graph, cacheKey]); - - return { - positions: state.key === cacheKey ? state.positions : null, - layouting: graph !== null && state.key !== cacheKey, - error: state.key === cacheKey ? state.error : null, - }; -} -``` - -- [ ] **Step 6: Typecheck + commit** - -Run: `yarn typecheck` -Expected: PASS. If `elkjs/lib/elk.bundled.js` lacks types, add `console/src/types/elkjs-bundled.d.ts` with `declare module "elkjs/lib/elk.bundled.js" { export { default } from "elkjs/lib/elk-api"; }` and include it in the commit. - -```bash -git add console/src/platform/dataflows/ console/src/types/ 2>/dev/null || git add console/src/platform/dataflows/ -git commit -m "console: elk layout worker and hook for dataflow graphs" -``` - -### Task 5: Data hook keyed by dataflow id with params tagging - -**Files:** -- Create: `console/src/api/materialize/dataflow/useDataflowGraphData.ts` -- Test: `console/src/api/materialize/dataflow/useDataflowGraphData.test.ts` - -**Interfaces:** -- Consumes: `buildDataflowStructure`, `DataflowStructure` (Task 2), `useSqlManyTyped`, `escapedLiteral as lit`, `queryBuilder` (existing, see `console/src/api/materialize/useDataflowStructure.ts` for import shapes). -- Produces: - -```typescript -export interface DataflowGraphParams { - clusterName: string; - replicaName: string; - dataflowId: string; // numeric string, mz_dataflows.id on the selected replica -} -export function useDataflowGraphData(params?: DataflowGraphParams): { - data: { structure: DataflowStructure; fetchedAt: Date } | null; - error: unknown; - databaseError: { code?: string } | null; // same shape useDataflowStructure exposes today - loading: boolean; - refetch: () => void; -}; -``` - -Contract: `data` is non-null only when the last successful result was produced by exactly the current `params` and there is no error. Error always wins (CNS-109). A same-params `refetch` keeps `data` while `loading` is true so the graph refreshes in place. - -- [ ] **Step 1: Write the failing test** - -Use msw like the CNS-109 issue test. `useDataflowGraphData.test.ts`, using `renderHook` from `@testing-library/react` and the msw `server` from `~/api/mocks/server`: - -```typescript -import { renderHook, waitFor } from "@testing-library/react"; -import { http, HttpResponse } from "msw"; - -import { ErrorCode } from "~/api/materialize/types"; -import server from "~/api/mocks/server"; - -import { useDataflowGraphData } from "./useDataflowGraphData"; - -const FAILING_REPLICA = "r2"; -const okResult = { - desc: { columns: [] }, - rows: [], -}; -// operators result with one root row so buildDataflowStructure succeeds -const operatorsResult = { - desc: { - columns: [ - { name: "id" }, { name: "address" }, { name: "name" }, - { name: "arrangementRecords" }, { name: "arrangementSize" }, { name: "elapsedNs" }, - ], - }, - rows: [["10", ["7"], "Dataflow", "0", "0", "0"]], -}; - -beforeEach(() => { - server.use( - http.post("*/api/sql", async ({ request }) => { - const options = JSON.parse( - new URL(request.url).searchParams.get("options") ?? "{}", - ); - if (options.cluster_replica === FAILING_REPLICA) { - return HttpResponse.json({ - results: [ - { error: { message: "no such replica", code: ErrorCode.INTERNAL_ERROR } }, - ], - }); - } - return HttpResponse.json({ results: [operatorsResult, okResult, okResult] }); - }), - ); -}); - -describe("useDataflowGraphData", () => { - it("clears data when a refetch for new params fails", async () => { - const { result, rerender } = renderHook( - (params) => useDataflowGraphData(params), - { - initialProps: { clusterName: "c", replicaName: "r1", dataflowId: "7" } as - | { clusterName: string; replicaName: string; dataflowId: string } - | undefined, - }, - ); - await waitFor(() => expect(result.current.data).not.toBeNull()); - - rerender({ clusterName: "c", replicaName: FAILING_REPLICA, dataflowId: "7" }); - await waitFor(() => expect(result.current.error).toBeTruthy()); - // The retained previous result must not surface for the new params. - expect(result.current.data).toBeNull(); - }); - - it("rejects a non-numeric dataflow id", () => { - expect(() => - renderHook(() => - useDataflowGraphData({ clusterName: "c", replicaName: "r1", dataflowId: "7; DROP" }), - ), - ).toThrow(); - }); -}); -``` - -`mapRowToObject` (`console/src/api/materialize/index.ts`) maps column aliases to object keys verbatim, so these mocked shapes match the hook's row types as written. - -- [ ] **Step 2: Run test, verify failure** - -Run: `yarn test --run src/api/materialize/dataflow/useDataflowGraphData.test.ts` -Expected: FAIL, module missing. - -- [ ] **Step 3: Implement** - -```typescript -import { sql } from "kysely"; -import React from "react"; - -import { escapedLiteral as lit, useSqlManyTyped } from "~/api/materialize"; -import { - buildDataflowStructure, - type ChannelRow, - type DataflowStructure, - type LirSpanRow, - type OperatorRow, -} from "~/platform/dataflows/dataflowGraph"; - -import { queryBuilder } from "../db"; - -export interface DataflowGraphParams { - clusterName: string; - replicaName: string; - dataflowId: string; -} - -function dataflowIdLiteral(dataflowId: string) { - if (!/^\d+$/.test(dataflowId)) { - throw new Error(`invalid dataflow id: ${dataflowId}`); - } - return sql`${lit(dataflowId)}::uint8`; -} - -export function useDataflowGraphData(params?: DataflowGraphParams) { - const queries = React.useMemo(() => { - if (!params) return null; - const id = dataflowIdLiteral(params.dataflowId); - return { - operators: sql` - SELECT - mdod.id, - mda.address, - mdod.name, - coalesce(mas.records, 0) AS "arrangementRecords", - coalesce(mas.size, 0) AS "arrangementSize", - coalesce(mse.elapsed_ns, 0) AS "elapsedNs" - FROM mz_dataflow_operator_dataflows AS mdod - JOIN mz_dataflow_addresses AS mda ON mda.id = mdod.id - LEFT JOIN mz_arrangement_sizes AS mas ON mas.operator_id = mdod.id - LEFT JOIN mz_scheduling_elapsed AS mse ON mse.id = mdod.id - WHERE mdod.dataflow_id = ${id}` - .$castTo() - .compile(queryBuilder), - channels: sql` - SELECT - mdco.id, - from_operator_address AS "fromOperatorAddress", - from_port AS "fromPort", - to_operator_address AS "toOperatorAddress", - to_port AS "toPort", - COALESCE(sum(mmc.sent), 0) AS "messagesSent", - COALESCE(sum(mmc.batch_sent), 0) AS "batchesSent", - mdco.type AS "channelType" - FROM mz_dataflow_channel_operators AS mdco - LEFT JOIN mz_message_counts AS mmc ON mdco.id = mmc.channel_id - WHERE from_operator_address[1] = ${id} - GROUP BY - mdco.id, "fromOperatorAddress", "fromPort", - "toOperatorAddress", "toPort", mdco.type` - .$castTo() - .compile(queryBuilder), - lirSpans: sql` - SELECT - mce.export_id AS "exportId", - mlm.lir_id::text AS "lirId", - mlm.operator, - mlm.operator_id_start AS "operatorIdStart", - mlm.operator_id_end AS "operatorIdEnd" - FROM mz_lir_mapping AS mlm - JOIN mz_compute_exports AS mce ON mlm.global_id = mce.export_id - WHERE mce.dataflow_id = ${id}` - .$castTo() - .compile(queryBuilder), - }; - }, [params]); - - const { results, error, databaseError, loading, refetch } = useSqlManyTyped(queries, { - cluster: params?.clusterName, - replica: params?.replicaName, - // This query can be slow for large dataflows. - timeout: 30_000, - }); - - const key = params - ? `${params.clusterName}|${params.replicaName}|${params.dataflowId}` - : null; - const [lastGood, setLastGood] = React.useState<{ - key: string; - data: { structure: DataflowStructure; fetchedAt: Date }; - } | null>(null); - - // Tag successful results with the params key they were fetched for. Stale - // in-flight responses for older params are aborted by useSqlMany, so a - // result observed here belongs to the current key. - React.useEffect(() => { - if (key && results && !error && !loading) { - setLastGood({ - key, - data: { - structure: buildDataflowStructure( - results.operators ?? [], - results.channels ?? [], - results.lirSpans ?? [], - ), - fetchedAt: new Date(), - }, - }); - } - }, [key, results, error, loading]); - - // Error always wins, and data for other params never renders. - const data = !error && lastGood && lastGood.key === key ? lastGood.data : null; - return { data, error, databaseError, loading, refetch }; -} -``` - -- [ ] **Step 4: Run test, verify pass** - -Run: `yarn test --run src/api/materialize/dataflow/useDataflowGraphData.test.ts` -Expected: PASS. - -- [ ] **Step 5: Commit** - -```bash -git add console/src/api/materialize/dataflow/ -git commit -m "console: dataflow graph data hook keyed by dataflow id" -``` - -### Task 6: Nodes, edges, and DataflowGraphView - -**Files:** -- Create: `console/src/platform/dataflows/nodes.tsx` -- Create: `console/src/platform/dataflows/ChannelEdge.tsx` -- Create: `console/src/platform/dataflows/DataflowGraphView.tsx` -- Create: `console/src/test/mockReactFlow.tsx` -- Modify: `console/src/platform/dataflows/dataflowGraph.ts` (add `GraphDecorations`, Step 0) - -**Interfaces:** -- Consumes: `VisibleGraph`, `Positions`, `useElkLayout`, `NODE_DIMENSIONS`, `MAX_VISIBLE_NODES`, `visibleNodeCount`. -- Produces: - -```typescript -export interface DataflowGraphViewProps { - structure: DataflowStructure; - collapsed: CollapseState; - onCollapsedChange: (next: CollapseState) => void; - cacheKey: string; // structure identity, owner appends collapse signature internally - decorations?: GraphDecorations; // added in Task 13, optional from the start - onNodeClick?: (node: VisibleNode) => void; -} -// Declared in dataflowGraph.ts (single source; DataflowGraphView imports it). -// Task 13 adds dimmedEdgeIds and searchMatches and makes decorateGraph return -// all fields; until then the view treats every field as optional. -export interface GraphDecorations { - dimmedNodeIds?: ReadonlySet; - hiddenNodeIds?: ReadonlySet; - hiddenEdgeIds?: ReadonlySet; - dimmedEdgeIds?: ReadonlySet; - nodeColors?: ReadonlyMap; // heatmap override - searchMatches?: string[]; -} -``` - -- [ ] **Step 0: Add GraphDecorations to dataflowGraph.ts** - -Append to `console/src/platform/dataflows/dataflowGraph.ts` (all-optional here; Task 13's `decorateGraph` returns all fields set): - -```typescript -export interface GraphDecorations { - dimmedNodeIds?: ReadonlySet; - hiddenNodeIds?: ReadonlySet; - hiddenEdgeIds?: ReadonlySet; - dimmedEdgeIds?: ReadonlySet; - nodeColors?: ReadonlyMap; // heatmap override - searchMatches?: string[]; -} -``` - -- [ ] **Step 1: Implement nodes.tsx** - -Colors keep the current palette constants. All text via React, `formatBytesShort` from `~/utils/format`. - -```tsx -import { Badge, Box, Text, Tooltip } from "@chakra-ui/react"; -import { Handle, Position, type NodeProps } from "@xyflow/react"; -import React from "react"; - -import { formatBytesShort } from "~/utils/format"; - -import type { VisibleNode } from "./dataflowGraph"; - -export const COLORS = { - noArrangementRegion: "#12b886", - noArrangementOperator: "#ffffff", - arrangementRegion: "#7950f2", - arrangementOperator: "#fab005", -}; - -export function nodeFillColor(node: VisibleNode): string { - const arranged = (node.transitive?.arrangementRecords ?? 0n) > 0n; - const region = node.kind !== "operator" && node.kind !== "port"; - if (region) return arranged ? COLORS.arrangementRegion : COLORS.noArrangementRegion; - return arranged ? COLORS.arrangementOperator : COLORS.noArrangementOperator; -} - -export function formatElapsed(ns: bigint): string { - return `scheduled ${Math.round(Number(ns) / 1e9)}s`; -} - -export type FlowNodeData = { node: VisibleNode; dimmed: boolean; color: string }; - -const statLines = (node: VisibleNode) => { - const lines: string[] = []; - const stats = node.stats; - if (!stats) return lines; - if (stats.arrangementRecords > 0n) { - lines.push(`${stats.arrangementRecords} arranged records`); - lines.push(formatBytesShort(stats.arrangementSize)); - } - if (stats.elapsedNs > 0n) lines.push(formatElapsed(stats.elapsedNs)); - return lines; -}; - -const CardShell = ({ - data, - children, -}: { - data: FlowNodeData; - children: React.ReactNode; -}) => ( - - {children} - - - -); - -export const OperatorNode = ({ data }: NodeProps & { data: FlowNodeData }) => ( - - `LIR ${l.lirId}: ${l.operator}`).join(", ")} - isDisabled={data.node.lir.length === 0} - > - - - {data.node.label} - - {statLines(data.node).map((line) => ( - - {line} - - ))} - - - -); - -export const CollapsedRegionNode = ({ data }: NodeProps & { data: FlowNodeData }) => ( - - - {data.node.label} - - {data.node.childCount} children - {statLines(data.node).map((line) => ( - - {line} - - ))} - -); - -export const RegionNode = ({ data }: NodeProps & { data: FlowNodeData }) => ( - - - {data.node.label} - - - - -); - -export const PortNode = ({ data }: NodeProps & { data: FlowNodeData }) => ( - - {data.node.label} - - - -); - -export const nodeTypes = { - operator: OperatorNode, - collapsedRegion: CollapsedRegionNode, - region: RegionNode, - port: PortNode, -}; -``` - -- [ ] **Step 2: Implement ChannelEdge.tsx** - -```tsx -import { Text, Tooltip } from "@chakra-ui/react"; -import { - BaseEdge, - EdgeLabelRenderer, - getBezierPath, - type EdgeProps, -} from "@xyflow/react"; -import React from "react"; - -export type ChannelEdgeData = { - messagesSent: bigint; - batchesSent: bigint; - channelTypes: string[]; - dimmed: boolean; -}; - -export const ChannelEdge = (props: EdgeProps & { data: ChannelEdgeData }) => { - const [path, labelX, labelY] = getBezierPath(props); - const { messagesSent, batchesSent, channelTypes, dimmed } = props.data; - const idle = messagesSent === 0n; - const label = idle ? "" : `${messagesSent} records / ${batchesSent} batches`; - return ( - <> - - {label && ( - - - - {label} - {channelTypes.length > 0 && ` · ${channelTypes.join(", ")}`} - - - - )} - - ); -}; -``` - -- [ ] **Step 3: Implement DataflowGraphView.tsx** - -```tsx -import { Box, Spinner, useToast } from "@chakra-ui/react"; -import { - Background, - Controls, - MiniMap, - ReactFlow, - type Edge, - type Node, -} from "@xyflow/react"; -import React from "react"; - -import "@xyflow/react/dist/style.css"; - -import { - deriveVisibleGraph, - MAX_VISIBLE_NODES, - visibleNodeCount, - type CollapseState, - type DataflowStructure, - type GraphDecorations, - type VisibleNode, -} from "./dataflowGraph"; -import { ChannelEdge } from "./ChannelEdge"; -import { NODE_DIMENSIONS } from "./elkGraph"; -import { nodeFillColor, nodeTypes } from "./nodes"; -import { useElkLayout } from "./useElkLayout"; - -const edgeTypes = { channel: ChannelEdge }; - -export interface DataflowGraphViewProps { - structure: DataflowStructure; - collapsed: CollapseState; - onCollapsedChange: (next: CollapseState) => void; - cacheKey: string; - decorations?: GraphDecorations; - onNodeClick?: (node: VisibleNode) => void; -} - -export const DataflowGraphView = ({ - structure, - collapsed, - onCollapsedChange, - cacheKey, - decorations, - onNodeClick, -}: DataflowGraphViewProps) => { - const toast = useToast(); - const visible = React.useMemo(() => { - const graph = deriveVisibleGraph(structure, collapsed); - if (!decorations?.hiddenNodeIds && !decorations?.hiddenEdgeIds) return graph; - const hiddenNodes = decorations.hiddenNodeIds ?? new Set(); - const hiddenEdges = decorations.hiddenEdgeIds ?? new Set(); - return { - nodes: graph.nodes.filter((n) => !hiddenNodes.has(n.id)), - edges: graph.edges.filter( - (e) => - !hiddenEdges.has(e.id) && !hiddenNodes.has(e.source) && !hiddenNodes.has(e.target), - ), - }; - }, [structure, collapsed, decorations?.hiddenNodeIds, decorations?.hiddenEdgeIds]); - - const layoutKey = `${cacheKey}|${[...collapsed].sort().join(",")}|${ - decorations?.hiddenNodeIds ? [...decorations.hiddenNodeIds].sort().join(",") : "" - }`; - const { positions, layouting, error } = useElkLayout(visible, layoutKey); - - const toggleRegion = React.useCallback( - (node: VisibleNode) => { - const next = new Set(collapsed); - if (next.has(node.id)) { - next.delete(node.id); - if (visibleNodeCount(structure, next) > MAX_VISIBLE_NODES) { - toast({ - status: "warning", - title: `Expanding would show more than ${MAX_VISIBLE_NODES} nodes.`, - }); - return; - } - } else { - next.add(node.id); - } - onCollapsedChange(next); - }, - [collapsed, onCollapsedChange, structure, toast], - ); - - const nodes: Node[] = React.useMemo(() => { - if (!positions) return []; - return visible.nodes.map((n) => { - const pos = positions[n.id] ?? { x: 0, y: 0, ...NODE_DIMENSIONS[n.kind] }; - return { - id: n.id, - type: n.kind, - position: { x: pos.x, y: pos.y }, - parentId: n.parent ?? undefined, - extent: n.parent ? ("parent" as const) : undefined, - style: { width: pos.width, height: pos.height }, - draggable: false, - connectable: false, - data: { - node: n, - dimmed: decorations?.dimmedNodeIds?.has(n.id) ?? false, - color: decorations?.nodeColors?.get(n.id) ?? nodeFillColor(n), - }, - }; - }); - }, [visible, positions, decorations?.dimmedNodeIds, decorations?.nodeColors]); - - const edges: Edge[] = React.useMemo( - () => - visible.edges.map((e) => ({ - id: e.id, - source: e.source, - target: e.target, - type: "channel", - data: { - messagesSent: e.messagesSent, - batchesSent: e.batchesSent, - channelTypes: e.channelTypes, - dimmed: - (decorations?.dimmedNodeIds?.has(e.source) ?? false) || - (decorations?.dimmedNodeIds?.has(e.target) ?? false) || - (decorations?.dimmedEdgeIds?.has(e.id) ?? false), - }, - })), - [visible, decorations?.dimmedNodeIds], - ); - - if (error) throw new Error(error); - return ( - - {layouting && ( - - - - )} - onNodeClick?.((node.data as { node: VisibleNode }).node)} - onNodeDoubleClick={(_, node) => { - const visibleNode = (node.data as { node: VisibleNode }).node; - if (visibleNode.kind === "region" || visibleNode.kind === "collapsedRegion") { - toggleRegion(visibleNode); - } - }} - proOptions={{ hideAttribution: true }} - > - - - - - - ); -}; -``` - -- [ ] **Step 4: Create the shared React Flow test mock** - -`console/src/test/mockReactFlow.tsx`. jsdom lacks `ResizeObserver`/`DOMMatrixReadOnly`, so component tests mock `@xyflow/react` to a flat renderer that preserves the decision logic under test (which nodes and labels exist): - -```tsx -import { vi } from "vitest"; -import React from "react"; - -export function mockReactFlow() { - vi.mock("@xyflow/react", () => ({ - ReactFlow: ({ nodes, children }: { nodes: { id: string; data: { node: { label: string } } }[]; children?: React.ReactNode }) => ( -
- {nodes.map((n) => ( -
- {n.data.node.label} -
- ))} - {children} -
- ), - Background: () => null, - Controls: () => null, - MiniMap: () => null, - Handle: () => null, - Position: { Left: "left", Right: "right" }, - BaseEdge: () => null, - EdgeLabelRenderer: ({ children }: { children: React.ReactNode }) => <>{children}, - getBezierPath: () => ["", 0, 0], - })); -} -``` - -Also mock `./useElkLayout` in component tests to return deterministic positions synchronously: - -```typescript -vi.mock("~/platform/dataflows/useElkLayout", () => ({ - useElkLayout: (graph: { nodes: { id: string }[] } | null) => ({ - positions: graph - ? Object.fromEntries(graph.nodes.map((n, i) => [n.id, { x: i * 100, y: 0, width: 100, height: 50 }])) - : null, - layouting: false, - error: null, - }), -})); -``` - -- [ ] **Step 5: Typecheck, lint, commit** - -Run: `yarn typecheck && yarn lint` -Expected: PASS. - -```bash -git add console/src/platform/dataflows/ console/src/test/mockReactFlow.tsx -git commit -m "console: React Flow dataflow graph view with collapsible regions" -``` - -### Task 7: Minimal detail page and route - -**Files:** -- Create: `console/src/platform/dataflows/DataflowDetailPage.tsx` -- Modify: `console/src/platform/clusters/ClusterDetail.tsx` - -**Interfaces:** -- Consumes: `useDataflowGraphData`, `DataflowGraphView`, `defaultCollapseState`, cluster store (`useAllClusters` pattern from `DataflowVisualizer.tsx:261-267`), `LabeledSelect`, `ErrorBox`, `MainContentContainer`. -- Produces: route `dataflows/:dataflowId` under cluster detail, replica via `?replica=` search param. Page component signature: `const DataflowDetailPage: () => JSX.Element` (default export). - -- [ ] **Step 1: Implement DataflowDetailPage (minimal: replica select, graph, spinner, error box)** - -```tsx -import { Alert, AlertIcon, Spinner, Text, VStack } from "@chakra-ui/react"; -import React from "react"; -import { useParams, useSearchParams } from "react-router-dom"; - -import { ErrorCode } from "~/api/materialize/types"; -import { useDataflowGraphData } from "~/api/materialize/dataflow/useDataflowGraphData"; -import ErrorBox from "~/components/ErrorBox"; -import LabeledSelect from "~/components/LabeledSelect"; -import { MainContentContainer } from "~/layouts/BaseLayout"; -import { useAllClusters } from "~/store/allClusters"; - -import { - defaultCollapseState, - type CollapseState, -} from "./dataflowGraph"; -import { DataflowGraphView } from "./DataflowGraphView"; - -const DataflowDetailPage = () => { - const { clusterId, dataflowId } = useParams(); - const { getClusterById } = useAllClusters(); - const cluster = clusterId ? getClusterById(clusterId) : undefined; - const [searchParams, setSearchParams] = useSearchParams(); - const replicaName = searchParams.get("replica") ?? cluster?.replicas[0]?.name; - - const params = React.useMemo( - () => - cluster && replicaName && dataflowId - ? { clusterName: cluster.name, replicaName, dataflowId } - : undefined, - [cluster, replicaName, dataflowId], - ); - const { data, error, databaseError, loading } = useDataflowGraphData(params); - - const [collapsed, setCollapsed] = React.useState(null); - const structureKey = data - ? `${params?.dataflowId}/${params?.replicaName}/${data.structure.nodes.size}` - : null; - React.useEffect(() => { - if (data) setCollapsed(defaultCollapseState(data.structure)); - // Reset collapse state when the structure identity changes. - }, [structureKey]); // eslint-disable-line react-hooks/exhaustive-deps - - if (!cluster) return null; - const permissionError = - databaseError && - "code" in databaseError && - databaseError.code === ErrorCode.INSUFFICIENT_PRIVILEGE; - - return ( - - - setSearchParams({ replica: e.target.value })} - flexShrink={0} - > - {cluster.replicas.map((r) => ( - - ))} - - {permissionError ? ( - - - - You'll need{" "} - - USAGE - {" "} - privilege on this cluster to visualize this dataflow. - - - ) : error ? ( - - ) : !data || !collapsed ? ( - - ) : data.structure.nodes.size <= 1 ? ( - This dataflow contains no operators. - ) : ( - - )} - - - ); -}; - -export default DataflowDetailPage; -``` - -- [ ] **Step 2: Wire the route** - -In `console/src/platform/clusters/ClusterDetail.tsx`: lazy-import the page, add the route inside `SentryRoutes` (`ClusterDetail.tsx:149-162`), and add the tab to `subnavItems` (`ClusterDetail.tsx:127-137`) gated on the flag. Use the same `useFlags` import that `SimpleObjectDetailRoutes.tsx` uses (check its import block and copy it). - -```tsx -const DataflowDetailPage = React.lazy( - () => import("~/platform/dataflows/DataflowDetailPage"), -); -``` - -```tsx -const flags = useFlags(); -const subnavItems: Tab[] = React.useMemo(() => { - const tabs: Tab[] = [ - { label: "Overview", href: "..", end: true }, - { label: "Replicas", href: "../replicas" }, - { label: "Materialized Views", href: "../materialized-views" }, - { label: "Indexes", href: "../indexes" }, - { label: "Sources", href: "../sources" }, - { label: "Sinks", href: "../sinks" }, - ]; - if (flags["visualization-features"]) { - tabs.push({ label: "Dataflows", href: "../dataflows" }); - } - return tabs; -}, [flags]); -``` - -```tsx -} -/> -``` - -(The list route at `path="dataflows"` comes in Task 9. Until then, navigate by URL directly.) - -- [ ] **Step 3: Manual smoke test** - -Start a local Materialize (`bin/environmentd` from repo root; see `mz-run` skill if it fails) and the console dev server (`cd console && yarn start`). Create a table plus an index, find its dataflow id (`SELECT id, name FROM mz_introspection.mz_dataflows`, run with `SET cluster = ...`), and open `https://local.dev.materialize.com:3000/.../clusters///dataflows/`. Expected: graph renders, regions expand and collapse on double click. - -- [ ] **Step 4: Typecheck, lint, commit** - -```bash -yarn typecheck && yarn lint -git add console/src -git commit -m "console: dataflow detail page with graph view behind flag" -``` - -### Task 8: CHECKPOINT, phase 1 gate - -**Execution mode:** driver-executed. This task needs a browser (devtools performance panel, screenshots) and a local environmentd, so the main session or a human runs it. Do NOT dispatch it to a code subagent. - -**Files:** none committed. Temporary `performance.mark` instrumentation in `useElkLayout.ts` around postMessage/response is permitted for Step 2 and must be reverted before Task 9 (`git status` clean afterwards). - -STOP after this task and report to the user. - -- [ ] **Step 1: Create the reference dataflow** - -Against local environmentd (`psql -h localhost -p 6875 -U materialize`): - -```sql -CREATE TABLE orders (id int, cust int, amt int); -CREATE TABLE customers (id int, region int); -CREATE TABLE regions (id int, name text); -CREATE TABLE items (order_id int, sku int, qty int); -CREATE TABLE skus (id int, price int); -CREATE MATERIALIZED VIEW gate_mv AS -SELECT r.name, count(*) AS orders, sum(i.qty * s.price) AS revenue -FROM orders o -JOIN customers c ON o.cust = c.id -JOIN regions r ON c.region = r.id -JOIN items i ON i.order_id = o.id -JOIN skus s ON s.id = i.sku -GROUP BY r.name -HAVING sum(i.qty * s.price) > 100; -INSERT INTO regions VALUES (1, 'emea'), (2, 'amer'); -INSERT INTO customers SELECT g, 1 + g % 2 FROM generate_series(1, 100) g; -INSERT INTO orders SELECT g, 1 + g % 100, g FROM generate_series(1, 1000) g; -INSERT INTO skus SELECT g, g FROM generate_series(1, 50) g; -INSERT INTO items SELECT g % 1000 + 1, g % 50 + 1, g FROM generate_series(1, 5000) g; -``` - -Verify operator count: `SET cluster = quickstart; SELECT count(*) FROM mz_introspection.mz_dataflow_operator_dataflows WHERE dataflow_name LIKE '%gate_mv%';` -Expected: >= 300. If under 300, add a sixth join (`JOIN customers c2 ON c2.id = o.cust`) and re-check. - -- [ ] **Step 2: Measure** - -Open the gate_mv dataflow in the visualizer (dev server). In browser devtools performance panel or via `performance.mark` temporarily added around the `useElkLayout` postMessage/response, record: - -* Default collapsed view layout time. Gate: < 500 ms. -* Expand regions until visible node count approaches 1500 (temporarily raise expansion via expand-all in console, or expand largest regions). Layout time. Gate: < 5 s, UI responsive (canvas pans during layout). -* Repeat both in a production build: `yarn build:local && yarn preview`, same URL on port 3000. Gate: worker loads and lays out in the built bundle. - -- [ ] **Step 3: Visual check** - -Screenshots of collapsed and expanded gate_mv. Gate: no overlapping nodes, edges flow left to right. - -- [ ] **Step 4: STOP, report** - -Report measurements + screenshots to the user for sign-off (spec MVP section). If any gate fails: do not proceed, return to the design doc per spec. - ---- - -## Phase 2: selection and refresh - -### Task 9: Dataflow list hook and DataflowsPage - -**Files:** -- Create: `console/src/api/materialize/dataflow/useDataflowList.ts` -- Create: `console/src/platform/dataflows/DataflowsPage.tsx` -- Modify: `console/src/platform/clusters/ClusterDetail.tsx` -- Test: `console/src/api/materialize/dataflow/useDataflowList.test.ts` - -**Interfaces:** -- Produces: - -```typescript -export interface DataflowListEntry { - id: string; // uint8 as string - name: string; - records: bigint; - size: bigint; - elapsedNs: bigint; -} -export function useDataflowList(params?: { clusterName: string; replicaName: string }): { - data: DataflowListEntry[] | null; - error: unknown; databaseError: unknown; loading: boolean; refetch: () => void; -}; -``` - -- [ ] **Step 1: Write failing hook test** (msw, same harness as Task 5): mock one row, assert mapping to `DataflowListEntry` with bigint fields normalized via `BigInt()`, assert `data` is null while `error` set. - -```typescript -const listResult = { - desc: { columns: [{ name: "id" }, { name: "name" }, { name: "records" }, { name: "size" }, { name: "elapsedNs" }] }, - rows: [["7", "Dataflow: mv", "100", "4096", "12345"]], -}; -// handler returns { results: [listResult] }; assert -// result.current.data?.[0] === { id: "7", name: "Dataflow: mv", records: 100n, size: 4096n, elapsedNs: 12345n } -``` - -- [ ] **Step 2: Verify failure, then implement** - -Query (kysely `sql`, `.$castTo<...>()`, same pattern as Task 5, no literals so no escaping): - -```sql -SELECT - md.id::text AS id, - md.name, - COALESCE(mrpd.records, 0) AS records, - COALESCE(mrpd.size, 0) AS size, - COALESCE(el.elapsed_ns, 0) AS "elapsedNs" -FROM mz_dataflows AS md -LEFT JOIN mz_records_per_dataflow AS mrpd ON mrpd.id = md.id -LEFT JOIN ( - SELECT mdod.dataflow_id, sum(mse.elapsed_ns) AS elapsed_ns - FROM mz_dataflow_operator_dataflows AS mdod - JOIN mz_scheduling_elapsed AS mse ON mse.id = mdod.id - GROUP BY mdod.dataflow_id -) AS el ON el.dataflow_id = md.id -ORDER BY size DESC -``` - -Hook body mirrors `useDataflowGraphData` minus tagging (a list going momentarily stale across replica switch is masked by `loading`, and errors clear `data` the same way: `const data = !error && results ? normalize(results.list) : null`). - -- [ ] **Step 3: Implement DataflowsPage** - -Loading spinner, `ErrorBox` on error, permission alert exactly as in Task 7. Export-deep-link resolution is Task 10, leave a plain list here. Core: - -```tsx -const DataflowsPage = () => { - const { clusterId } = useParams(); - const { getClusterById } = useAllClusters(); - const cluster = clusterId ? getClusterById(clusterId) : undefined; - const [searchParams, setSearchParams] = useSearchParams(); - const replicaName = searchParams.get("replica") ?? cluster?.replicas[0]?.name; - const params = React.useMemo( - () => - cluster && replicaName ? { clusterName: cluster.name, replicaName } : undefined, - [cluster, replicaName], - ); - const { data, error, databaseError, loading } = useDataflowList(params); - if (!cluster) return null; - const permissionError = - databaseError && - "code" in databaseError && - databaseError.code === ErrorCode.INSUFFICIENT_PRIVILEGE; - return ( - - - setSearchParams({ replica: e.target.value })} - > - {cluster.replicas.map((r) => ( - - ))} - - {permissionError ? ( - - - - You'll need{" "} - - USAGE - {" "} - privilege on this cluster to list its dataflows. - - - ) : error ? ( - - ) : !data && loading ? ( - - ) : ( - - - - - - {(data ?? []).map((d) => ( - - - - - - - ))} - -
NameRecordsSizeScheduled
- {d.name} - {d.records.toString()}{formatBytesShort(d.size)}{Math.round(Number(d.elapsedNs) / 1e9)}s
- )} - - - ); -}; -export default DataflowsPage; -``` - -- [ ] **Step 4: Wire route** - -`ClusterDetail.tsx`: `} />` next to the Task 7 route (lazy import, flag-gated same as tab). - -- [ ] **Step 5: Test, typecheck, manual check list renders, commit** - -```bash -yarn test --run src/api/materialize/dataflow/ && yarn typecheck && yarn lint -git add console/src -git commit -m "console: cluster dataflows list page" -``` - -### Task 10: Export deep-link and object tab rewire - -**Files:** -- Create: `console/src/api/materialize/dataflow/useDataflowIdForExport.ts` -- Modify: `console/src/platform/dataflows/DataflowsPage.tsx` -- Modify: `console/src/platform/object-explorer/SimpleObjectDetailRoutes.tsx` - -**Interfaces:** -- Produces: `useDataflowIdForExport(params?: { clusterName: string; replicaName: string; exportId: string }): { dataflowId: string | null; loading: boolean; error: unknown }`. Query: `SELECT dataflow_id::text AS "dataflowId" FROM mz_compute_exports WHERE export_id = ${lit(exportId)}` (exportId validated `/^[stu]\d+$/`). - -- [ ] **Step 1: Implement hook** (pattern of Task 9, single query, `dataflowId = results?.rows?.[0]?.dataflowId ?? null`). - -- [ ] **Step 2: DataflowsPage export resolution** - -When `?export=` is present and a replica is chosen: call the hook, on `dataflowId` render ``. On resolved-but-null (export gone): show `ErrorBox` message "This object has no running dataflow on the selected replica." with the list rendered below. - -- [ ] **Step 3: Rewire the object "Visualize" tab** - -In `SimpleObjectDetailRoutes.tsx`: keep `shouldShowDataflowVisualizer` logic (`:136-139`). Replace the tab href (`:160-165`) and the route (`:203-205`): the tab now links to the cluster page. The object row has `clusterId`; get the cluster name via `useAllClusters().getClusterById`. Href: `/clusters/${object.clusterId}/${clusterName}/dataflows?export=${object.id}`. Verify the absolute-path shape against how `ClustersList.tsx` builds cluster links and match it (react-router basename handles the prefix). Delete the `dataflow-visualizer` Route and the `DataflowVisualizer` lazy import (`:37-39`). The old component file itself is deleted in Task 18. - -- [ ] **Step 4: Manual check** Object detail → Visualize → lands on graph of that object's dataflow. Typecheck, lint, commit. - -```bash -git add console/src -git commit -m "console: deep-link object visualize tab into dataflows page" -``` - -### Task 11: Refresh in place - -**Files:** -- Modify: `console/src/platform/dataflows/DataflowDetailPage.tsx` - -**Interfaces:** -- Consumes: `refetch` and `data.fetchedAt` from `useDataflowGraphData`. - -- [ ] **Step 1: Add refresh UI** - -Toolbar row above the graph: `Button` "Refresh" calling `refetch`, disabled while `loading`. `Text` "Last fetched {fetchedAt.toLocaleTimeString()}". While a same-params refetch is loading, `data` stays non-null (Task 5 contract), so the graph stays mounted and stats update in place when the new result lands. - -- [ ] **Step 2: Preserve layout across refresh** - -`structureKey` in Task 7 already encodes `nodes.size`. Strengthen it so pure stats refreshes reuse layout and structure changes relayout: `structureKey = `${dataflowId}/${replicaName}/` + a stable digest of sorted node ids` (join sorted `[...structure.nodes.keys()]` with "," and use its length + a simple 32-bit string hash, implemented inline as `hashString`). Collapse state resets only when `structureKey` changes, which was already the effect dependency. - -```typescript -function hashString(s: string): number { - let h = 0; - for (let i = 0; i < s.length; i++) h = (Math.imul(h, 31) + s.charCodeAt(i)) | 0; - return h; -} -``` - -- [ ] **Step 3: Manual check** Refresh keeps viewport + expansion, stats badges update. Typecheck, lint, commit. - -```bash -git add console/src -git commit -m "console: manual refresh preserving dataflow graph layout" -``` - ---- - -## Phase 3: interactions - -### Task 12: Node detail panel - -**Files:** -- Create: `console/src/platform/dataflows/NodeDetailPanel.tsx` -- Modify: `console/src/platform/dataflows/DataflowDetailPage.tsx` -- Modify: `console/src/platform/dataflows/DataflowGraphView.tsx` (add `onPaneClick` pass-through prop) - -- [ ] **Step 1: Implement panel** - -```tsx -export const NodeDetailPanel = ({ - node, - onClose, -}: { - node: VisibleNode; - onClose: () => void; -}) => { - const row = (label: string, value: string) => ( - - {label} - {value} - - ); - return ( - - - {node.label} - - - {row("Kind", node.kind)} - {node.address && row("Address", node.address.join("."))} - {node.childCount > 0 && row("Children", String(node.childCount))} - {node.stats && ( - <> - {row("Arranged records", node.stats.arrangementRecords.toString())} - {row("Arrangement size", formatBytesShort(node.stats.arrangementSize))} - {row("Elapsed", `${Math.round(Number(node.stats.elapsedNs) / 1e9)}s`)} - - )} - {node.transitive && node.childCount > 0 && ( - <> - {row("Subtree records", node.transitive.arrangementRecords.toString())} - {row("Subtree size", formatBytesShort(node.transitive.arrangementSize))} - {row("Subtree elapsed", `${Math.round(Number(node.transitive.elapsedNs) / 1e9)}s`)} - - )} - {node.lir.map((l) => ( - - LIR {l.lirId} ({l.exportId}): {l.operator} - - ))} - - ); -}; -``` - -- [ ] **Step 2: Wire** `onNodeClick` in `DataflowDetailPage` sets `selectedNode` state; render panel beside the graph in an `HStack`. Clicking the canvas background clears it: `DataflowGraphViewProps` gains `onPaneClick?: () => void;`, forwarded as ``. - -- [ ] **Step 3: Typecheck, lint, manual check, commit** - -```bash -git add console/src -git commit -m "console: node detail panel for dataflow visualizer" -``` - -### Task 13: Filters model + toolbar (search, hide idle, channel types) - -**Files:** -- Modify: `console/src/platform/dataflows/dataflowGraph.ts` (decorations, pure) -- Create: `console/src/platform/dataflows/DataflowToolbar.tsx` -- Modify: `console/src/platform/dataflows/DataflowDetailPage.tsx` -- Modify: `console/src/platform/dataflows/DataflowGraphView.tsx` (`centerRef` prop, `CenterHelper`) -- Test: `console/src/platform/dataflows/dataflowGraph.test.ts` - -**Interfaces:** -- Produces (in `dataflowGraph.ts`): - -```typescript -export interface Filters { - search: string; - hideIdle: boolean; - heatmap: "off" | "elapsed" | "size"; - heatmapThreshold: number; // 0..1 fraction of max - channelTypes: string[] | null; // null = all -} -export const DEFAULT_FILTERS: Filters; - -// Replaces the all-optional Task 6 declaration in dataflowGraph.ts. decorateGraph -// returns every field set. DataflowGraphView's optional-chaining reads and its -// optional `decorations` prop remain valid against the narrowed type. -export interface GraphDecorations { - dimmedNodeIds: Set; - hiddenNodeIds: Set; - hiddenEdgeIds: Set; - dimmedEdgeIds: Set; - nodeColors: Map; - searchMatches: string[]; // visible node ids, document order -} -export function decorateGraph( - graph: VisibleGraph, - filters: Filters, - heatColor: (t: number) => string, // injected so the pure module stays d3-free -): GraphDecorations; - -export function allChannelTypes(structure: DataflowStructure): string[]; - -// Search may match inside collapsed regions. Returns a collapse state with -// every ancestor of every matching node expanded (respecting nothing else). -export function expandForSearch( - structure: DataflowStructure, - collapsed: CollapseState, - search: string, -): CollapseState; -``` - -Semantics (each a test): -* `search` non-empty: case-insensitive substring on `label`. Non-matching operator/collapsedRegion nodes → `dimmedNodeIds`. Matches listed in `searchMatches`. -* `hideIdle`: operators with `stats.elapsedNs === 0n` and no arranged records → `hiddenNodeIds`; edges with `messagesSent === 0n` → `hiddenEdgeIds`. Regions and ports never hidden. -* `channelTypes` non-null: edges whose `channelTypes` don't intersect → `hiddenEdgeIds` additions? No: dimmed, per spec. Put them in a dim path: edges get dimmed via node dimming today, so extend `ChannelEdgeData.dimmed` computation in `DataflowGraphView` with `hiddenEdgeIds`-independent `dimmedEdgeIds: Set` added to `GraphDecorations` and populated here. -* `heatmap` != off: for every non-port node, `t = Number(metric(node)) / Number(max over visible non-port nodes)` with `metric = transitive.elapsedNs | transitive.arrangementSize`; `nodeColors.set(id, heatColor(t))`; nodes with `t < heatmapThreshold` → `dimmedNodeIds`. If max is 0, no colors set (spec: neutral base, slider disabled). - -- [ ] **Step 1: Write failing tests** - -Append to `dataflowGraph.test.ts`: - -```typescript -import { - allChannelTypes, - decorateGraph, - DEFAULT_FILTERS, - expandForSearch, -} from "./dataflowGraph"; - -describe("decorateGraph", () => { - const s = buildDataflowStructure(OPS, CHANNELS, LIR_SPANS); - const expanded = deriveVisibleGraph(s, new Set()); - const heat = (t: number) => `heat(${t.toFixed(2)})`; - - it("search dims non-matching leaf nodes and lists matches", () => { - const d = decorateGraph(expanded, { ...DEFAULT_FILTERS, search: "join" }, heat); - expect(d.searchMatches).toEqual([nodeIdOf([5, 1, 1])]); - expect(d.dimmedNodeIds.has(nodeIdOf([5, 2]))).toBe(true); - expect(d.dimmedNodeIds.has(nodeIdOf([5, 1, 1]))).toBe(false); - }); - - it("hideIdle hides zero-activity operators and zero-message edges, never regions or ports", () => { - const d = decorateGraph(expanded, { ...DEFAULT_FILTERS, hideIdle: true }, heat); - // every fixture operator has elapsed > 0 except none; hide check via a synthetic idle op - const idle = buildDataflowStructure( - [...OPS, { id: "15", address: ["5", "3"], name: "Idle", arrangementRecords: "0", arrangementSize: "0", elapsedNs: "0" }], - CHANNELS, [], - ); - const d2 = decorateGraph(deriveVisibleGraph(idle, new Set()), { ...DEFAULT_FILTERS, hideIdle: true }, heat); - expect(d2.hiddenNodeIds.has(nodeIdOf([5, 3]))).toBe(true); - expect(d2.hiddenNodeIds.has(nodeIdOf([5, 1]))).toBe(false); - expect(d.hiddenEdgeIds.size).toBe(0); // all fixture edges have messages - }); - - it("channel type filter dims non-matching edges", () => { - const d = decorateGraph(expanded, { ...DEFAULT_FILTERS, channelTypes: ["batches"] }, heat); - const rowsEdge = expanded.edges.find((e) => e.channelTypes.includes("rows"))!; - const batchesEdge = expanded.edges.find((e) => e.channelTypes.includes("batches"))!; - expect(d.dimmedEdgeIds.has(rowsEdge.id)).toBe(true); - expect(d.dimmedEdgeIds.has(batchesEdge.id)).toBe(false); - }); - - it("heatmap colors by transitive metric and dims below threshold", () => { - const d = decorateGraph( - expanded, - { ...DEFAULT_FILTERS, heatmap: "elapsed", heatmapThreshold: 0.5 }, - heat, - ); - // max transitive elapsed among visible non-ports is the region (13n) - expect(d.nodeColors.get(nodeIdOf([5, 1]))).toEqual("heat(1.00)"); - expect(d.dimmedNodeIds.has(nodeIdOf([5, 2]))).toBe(true); // 2/13 < 0.5 - }); - - it("heatmap with all-zero metric sets no colors", () => { - const zero = buildDataflowStructure( - OPS.map((o) => ({ ...o, elapsedNs: "0" })), CHANNELS, [], - ); - const d = decorateGraph( - deriveVisibleGraph(zero, new Set()), - { ...DEFAULT_FILTERS, heatmap: "elapsed" }, - heat, - ); - expect(d.nodeColors.size).toBe(0); - }); -}); - -describe("expandForSearch / allChannelTypes", () => { - const s = buildDataflowStructure(OPS, CHANNELS, LIR_SPANS); - it("expands ancestors of matches", () => { - const next = expandForSearch(s, defaultCollapseState(s), "join"); - expect(next.has(nodeIdOf([5, 1]))).toBe(false); - }); - it("collects channel types", () => { - expect(allChannelTypes(s)).toEqual(["batches", "rows"]); - }); -}); -``` - -- [ ] **Step 2: Implement** - -```typescript -export const DEFAULT_FILTERS: Filters = { - search: "", - hideIdle: false, - heatmap: "off", - heatmapThreshold: 0, - channelTypes: null, -}; - -export function allChannelTypes(structure: DataflowStructure): string[] { - const types = new Set(); - for (const c of structure.channels) if (c.channelType) types.add(c.channelType); - return [...types].sort(); -} - -export function decorateGraph( - graph: VisibleGraph, - filters: Filters, - heatColor: (t: number) => string, -): GraphDecorations { - const d: GraphDecorations = { - dimmedNodeIds: new Set(), - hiddenNodeIds: new Set(), - hiddenEdgeIds: new Set(), - dimmedEdgeIds: new Set(), - nodeColors: new Map(), - searchMatches: [], - }; - const needle = filters.search.trim().toLowerCase(); - for (const n of graph.nodes) { - if (needle && n.kind !== "port" && n.kind !== "region") { - if (n.label.toLowerCase().includes(needle)) d.searchMatches.push(n.id); - else d.dimmedNodeIds.add(n.id); - } - if ( - filters.hideIdle && - n.kind === "operator" && - n.stats !== null && - n.stats.elapsedNs === 0n && - n.stats.arrangementRecords === 0n - ) { - d.hiddenNodeIds.add(n.id); - } - } - for (const e of graph.edges) { - if (filters.hideIdle && e.messagesSent === 0n) d.hiddenEdgeIds.add(e.id); - if ( - filters.channelTypes !== null && - !e.channelTypes.some((t) => filters.channelTypes!.includes(t)) - ) { - d.dimmedEdgeIds.add(e.id); - } - } - if (filters.heatmap !== "off") { - const metric = (n: VisibleNode): bigint => - filters.heatmap === "elapsed" - ? n.transitive?.elapsedNs ?? 0n - : n.transitive?.arrangementSize ?? 0n; - const candidates = graph.nodes.filter((n) => n.kind !== "port"); - const max = candidates.reduce((m, n) => (metric(n) > m ? metric(n) : m), 0n); - if (max > 0n) { - for (const n of candidates) { - const t = Number(metric(n)) / Number(max); - d.nodeColors.set(n.id, heatColor(t)); - if (t < filters.heatmapThreshold) d.dimmedNodeIds.add(n.id); - } - } - } - return d; -} - -export function expandForSearch( - structure: DataflowStructure, - collapsed: CollapseState, - search: string, -): CollapseState { - const needle = search.trim().toLowerCase(); - if (!needle) return collapsed; - const next = new Set(collapsed); - for (const node of structure.nodes.values()) { - if (!node.name.toLowerCase().includes(needle)) continue; - for (let len = 1; len < node.address.length; len++) { - next.delete(nodeIdOf(node.address.slice(0, len))); - } - } - return next; -} -``` - - -- [ ] **Step 3: Verify tests pass** - -Run: `yarn test --run src/platform/dataflows/dataflowGraph.test.ts` - -- [ ] **Step 4: Toolbar UI** - -Pure controlled component: - -```tsx -export interface DataflowToolbarProps { - filters: Filters; - onFiltersChange: (next: Filters) => void; - channelTypes: string[]; - matchCount: number; - matchIndex: number; - onJump: (delta: 1 | -1) => void; -} - -export const DataflowToolbar = ({ - filters, - onFiltersChange, - channelTypes, - matchCount, - matchIndex, - onJump, -}: DataflowToolbarProps) => { - const [search, setSearch] = React.useState(filters.search); - // Debounce search input into the filters object. - React.useEffect(() => { - if (search === filters.search) return; - const timeout = setTimeout(() => onFiltersChange({ ...filters, search }), 300); - return () => clearTimeout(timeout); - }, [search, filters, onFiltersChange]); - return ( - - setSearch(e.target.value)} - /> - {filters.search && ( - - - - - {matchCount === 0 ? "0/0" : `${matchIndex + 1}/${matchCount}`} - - - )} - - - Hide idle - - onFiltersChange({ ...filters, hideIdle: e.target.checked })} - /> - - - onFiltersChange({ ...filters, heatmapThreshold: v })} - > - - - - - - - - Channel types - - - {channelTypes.map((t) => ( - - { - const current = filters.channelTypes ?? channelTypes; - const next = e.target.checked - ? [...current, t] - : current.filter((x) => x !== t); - onFiltersChange({ - ...filters, - channelTypes: next.length === channelTypes.length ? null : next, - }); - }} - > - {t} - - - ))} - - - - ); -}; -``` - -- [ ] **Step 5: Wire into DataflowDetailPage** - -`filters` state, `decorations = React.useMemo(() => decorateGraph(deriveVisibleGraph(structure, collapsed), filters, heatColor), ...)` where `heatColor = (t) => interpolateYlOrRd(0.15 + 0.85 * t)` from `d3-scale-chromatic` (existing dep). Heat legend: small gradient `Box` + min/max labels next to the heatmap select. - -Centering needs React Flow context, so `DataflowGraphView` gains a `centerRef` prop filled by a helper rendered inside ``: - -```tsx -// DataflowGraphViewProps gains: -// centerRef?: React.MutableRefObject<((id: string) => void) | null>; - -const CenterHelper = ({ - centerRef, -}: { - centerRef: React.MutableRefObject<((id: string) => void) | null>; -}) => { - const reactFlow = useReactFlow(); - React.useEffect(() => { - centerRef.current = (id: string) => { - const internal = reactFlow.getInternalNode(id); - if (!internal) return; - const { x, y } = internal.internals.positionAbsolute; - const width = internal.measured?.width ?? 0; - const height = internal.measured?.height ?? 0; - reactFlow.setCenter(x + width / 2, y + height / 2, { zoom: 1, duration: 300 }); - }; - return () => { - centerRef.current = null; - }; - }, [reactFlow, centerRef]); - return null; -}; -// In DataflowGraphView's JSX, inside : -// {centerRef && } -``` - -Search jump wiring in `DataflowDetailPage`: - -```tsx -const centerRef = React.useRef<((id: string) => void) | null>(null); -const [matchIndex, setMatchIndex] = React.useState(0); -// New search: expand ancestors of matches once, reset the cursor. -React.useEffect(() => { - setMatchIndex(0); - if (filters.search && data) { - setCollapsed((c) => (c ? expandForSearch(data.structure, c, filters.search) : c)); - } -}, [filters.search]); // eslint-disable-line react-hooks/exhaustive-deps -const onJump = React.useCallback( - (delta: 1 | -1) => { - const matches = decorations.searchMatches; - if (matches.length === 0) return; - const next = (matchIndex + delta + matches.length) % matches.length; - setMatchIndex(next); - centerRef.current?.(matches[next]); - }, - [decorations.searchMatches, matchIndex], -); -``` - -- [ ] **Step 6: Typecheck, lint, tests, manual check, commit** - -```bash -yarn test --run src/platform/dataflows/ && yarn typecheck && yarn lint -git add console/src -git commit -m "console: dataflow visualizer filters and toolbar" -``` - -### Task 14: LIR panel - -**Files:** -- Create: `console/src/platform/dataflows/LirPanel.tsx` -- Modify: `console/src/platform/dataflows/DataflowDetailPage.tsx` -- Modify: `console/src/platform/dataflows/dataflowGraph.ts` (one pure helper + test) - -**Interfaces:** -- Produces: `lirIndex(structure: DataflowStructure): Map` keyed by `${exportId}/${lirId}` (test: fixture yields one entry with members `[5,1]`, `[5,1,1]` per span 11..13 covering ids 11 and 12). - -- [ ] **Step 1: Failing test + helper** - -Test (append to `dataflowGraph.test.ts`): - -```typescript -import { lirIndex } from "./dataflowGraph"; - -describe("lirIndex", () => { - it("buckets member nodes by export/lir id", () => { - const s = buildDataflowStructure(OPS, CHANNELS, LIR_SPANS); - const index = lirIndex(s); - expect([...index.keys()]).toEqual(["u42/1"]); - // span 11..13 covers operator ids 11 ([5,1]) and 12 ([5,1,1]) - expect(index.get("u42/1")!.memberIds.sort()).toEqual( - [nodeIdOf([5, 1]), nodeIdOf([5, 1, 1])].sort(), - ); - }); -}); -``` - -Implementation: - -```typescript -export function lirIndex( - structure: DataflowStructure, -): Map { - const index = new Map(); - for (const node of structure.nodes.values()) { - for (const info of node.lir) { - const key = `${info.exportId}/${info.lirId}`; - const entry = index.get(key) ?? { info, memberIds: [] }; - entry.memberIds.push(node.id); - index.set(key, entry); - } - } - return index; -} -``` - -- [ ] **Step 2: Panel** - -```tsx -export const LirPanel = ({ - structure, - onHighlight, -}: { - structure: DataflowStructure; - onHighlight: (ids: ReadonlySet | null) => void; -}) => { - const index = React.useMemo(() => lirIndex(structure), [structure]); - const byExport = React.useMemo(() => { - const groups = new Map(); - for (const [key, entry] of index) { - const list = groups.get(entry.info.exportId) ?? []; - list.push({ key, ...entry }); - groups.set(entry.info.exportId, list); - } - return groups; - }, [index]); - if (index.size === 0) return null; - return ( - - {[...byExport.entries()].map(([exportId, entries]) => ( - - - {exportId} - - {entries.map((e) => ( - onHighlight(new Set(e.memberIds))} - onMouseLeave={() => onHighlight(null)} - > - LIR {e.info.lirId}: {e.info.operator} - - ))} - - ))} - - ); -}; -``` - -- [ ] **Step 3: Wire** highlight set in `DataflowDetailPage`: when non-null, everything not in the set joins `decorations.dimmedNodeIds` (merge before passing to the view). Note collapsed ancestors: map each member id through the current collapse state via `representative`-style logic already available (`deriveVisibleGraph` output ids), simplest: dim nothing that is not currently visible, i.e. intersect with visible node ids. - -- [ ] **Step 4: Typecheck, lint, manual check (hover LIR row highlights members), commit** - -```bash -git add console/src -git commit -m "console: LIR operator panel with member highlighting" -``` - ---- - -## Phase 4: hardening and cleanup - -### Task 15: Error and empty states + CNS-109 regression test - -**Files:** -- Modify: `console/src/platform/dataflows/DataflowDetailPage.tsx` -- Test: `console/src/platform/dataflows/DataflowDetailPage.test.tsx` - -- [ ] **Step 1: Empty/gone states** - -In `DataflowDetailPage`: when `data` loads and `structure.nodes.size <= 1` (root only or empty), render `Text` "This dataflow no longer exists on this replica." with a `Link` back to `..` (the list). Timeout errors already surface through `error` → `ErrorBox`; add a "Retry" `Button` calling `refetch` under the `ErrorBox`. - -- [ ] **Step 2: Write the CNS-109 regression component test** - -`DataflowDetailPage.test.tsx`: `mockReactFlow()` + the `useElkLayout` mock from Task 6, msw handler that fails for `cluster_replica === "r2"` and returns a small successful structure otherwise (reuse the row shapes Task 5's test settled on). Setup: - -```tsx -import { screen } from "@testing-library/react"; -import userEvent from "@testing-library/user-event"; -import { http, HttpResponse } from "msw"; -import React from "react"; -import { Route, Routes } from "react-router-dom"; - -import { Cluster } from "~/api/materialize/cluster/clusterList"; -import { ErrorCode } from "~/api/materialize/types"; -import server from "~/api/mocks/server"; -import { getStore } from "~/jotai"; -import { allClusters } from "~/store/allClusters"; -import { mockReactFlow } from "~/test/mockReactFlow"; -import { mockSubscribeState } from "~/test/mockSubscribe"; -import { renderComponent } from "~/test/utils"; - -import DataflowDetailPage from "./DataflowDetailPage"; - -mockReactFlow(); -// plus the useElkLayout vi.mock block from Task 6 - -const CLUSTER_ID = "u5"; -const FAILING_REPLICA = "r2"; -const testCluster = { - id: CLUSTER_ID, - name: "test_cluster", - replicas: [{ name: "r1" }, { name: FAILING_REPLICA }], -} as Cluster; - -const renderVisualizer = () => - renderComponent( - - } - /> - , - { initialRouterEntries: [`/clusters/${CLUSTER_ID}/test_cluster/dataflows/7`] }, - ); - -const okResult = { desc: { columns: [] }, rows: [] }; -const operatorsResult = { - desc: { - columns: [ - { name: "id" }, { name: "address" }, { name: "name" }, - { name: "arrangementRecords" }, { name: "arrangementSize" }, { name: "elapsedNs" }, - ], - }, - rows: [ - ["10", ["7"], "Dataflow", "0", "0", "0"], - ["11", ["7", "1"], "Map", "0", "0", "1"], - ], -}; -const handler = http.post("*/api/sql", async ({ request }) => { - const options = JSON.parse( - new URL(request.url).searchParams.get("options") ?? "{}", - ); - if (options.cluster_replica === FAILING_REPLICA) { - return HttpResponse.json({ - results: [ - { error: { message: "no such replica", code: ErrorCode.INTERNAL_ERROR } }, - ], - }); - } - return HttpResponse.json({ results: [operatorsResult, okResult, okResult] }); -}); - -beforeEach(() => { - server.use(handler); - getStore().set(allClusters, mockSubscribeState({ data: [testCluster] })); -}); -``` - -If `renderComponent`'s option name differs, match `console/src/test/utils.tsx`. Assertions: - -```tsx -it("shows an error instead of the stale graph when a refetch fails", async () => { - renderVisualizer(); - const replicaSelect = await screen.findByRole("combobox"); - // graph rendered from r1 data first - expect(await screen.findByTestId("react-flow")).toBeVisible(); - await userEvent.selectOptions(replicaSelect, "r2"); - expect( - await screen.findByText("There was an error visualizing your dataflow"), - ).toBeVisible(); - expect(screen.queryByTestId("react-flow")).toBeNull(); -}); -``` - -- [ ] **Step 3: Run test, verify pass** (the Task 5 contract makes it pass; if it fails, the bug is real, fix the hook not the test). - -Run: `yarn test --run src/platform/dataflows/DataflowDetailPage.test.tsx` - -- [ ] **Step 4: Commit** - -```bash -git add console/src -git commit -m "console: dataflow visualizer error states and CNS-109 regression test" -``` - -### Task 16: Delete the old visualizer and d3-graphviz - -**Files:** -- Delete: `console/src/platform/clusters/DataflowVisualizer.tsx` -- Delete: `console/src/api/materialize/useDataflowStructure.ts` -- Modify: `console/package.json` - -- [ ] **Step 1: Verify no remaining references** - -Run: `grep -rn "DataflowVisualizer\|useDataflowStructure\|d3-graphviz" console/src console/e2e-tests` -Expected: no hits outside the two files being deleted (Task 10 removed the route + import). If hits remain, fix them first. - -- [ ] **Step 2: Delete** - -```bash -git rm console/src/platform/clusters/DataflowVisualizer.tsx console/src/api/materialize/useDataflowStructure.ts -cd console && yarn remove d3-graphviz -``` - -`d3` stays (used elsewhere). Confirm `@hpcc-js/wasm` left the lockfile: `grep hpcc console/yarn.lock` → no hits. - -- [ ] **Step 3: Full sweep** - -Run: `yarn test --run && yarn typecheck && yarn lint && yarn build:local` -Expected: all PASS, build succeeds with no WASM chunk for graphviz. - -- [ ] **Step 4: Commit** - -```bash -git add console/package.json console/yarn.lock -git commit -m "console: delete graphviz dataflow visualizer and d3-graphviz dep" -``` - ---- - -## Self-review notes - -* Spec coverage: surface (Tasks 7, 9, 10), data layer + CNS-109 (5, 15), graph model/collapse/LIR (2, 3, 14), rendering/layout/perf guard (4, 6), filters (13), refresh (11), errors (15), deletion/CNS-108-by-construction (6, 16), MVP gate (8). Review items: `Channel` defined (Task 2), `lir` widened to array with test (Task 2). -* Detail panel is spec section "Filters and interactions" (Task 12). -* Type names match across tasks: `VisibleNode`, `VisibleGraph`, `Positions`, `GraphDecorations` (defined once in Task 13, consumed by Task 6 via optional prop declared there with matching shape). -* Known judgment calls delegated to implementers: exact msw row shapes (Task 5 step 1 note), `useFlags` import source (copy from `SimpleObjectDetailRoutes.tsx`), absolute cluster link shape (verify against `ClustersList.tsx` in Task 10). From 090d0506bbfb70c9a7321e73e31cbfb3914dc12e Mon Sep 17 00:00:00 2001 From: Moritz Hoffmann Date: Wed, 8 Jul 2026 22:42:38 +0200 Subject: [PATCH 21/45] console: update dataflow visualizer design doc for drill-down + skew Co-Authored-By: Claude Sonnet 5 --- .../20260706_dataflow_visualizer_rebuild.md | 118 ++++++++++-------- 1 file changed, 64 insertions(+), 54 deletions(-) diff --git a/console/doc/design/20260706_dataflow_visualizer_rebuild.md b/console/doc/design/20260706_dataflow_visualizer_rebuild.md index 48f91a45e5219..9d9932c234f77 100644 --- a/console/doc/design/20260706_dataflow_visualizer_rebuild.md +++ b/console/doc/design/20260706_dataflow_visualizer_rebuild.md @@ -13,9 +13,9 @@ Beyond the bugs, the visualizer lacks features operators need: no dataflow selec ## Success Criteria * A user can pick any dataflow running on a replica, including transient ones, and see its operator graph. -* Regions expand and collapse in place, with aggregated stats and rerouted edges on collapsed regions. +* A user drills into one region at a time, with aggregated stats on regions and jump navigation across scope boundaries. * Nodes show scheduling time and arrangement size, edges show message counts and container type. -* Filter tools let a user locate operators by name, hide idle elements, heat-color by metric, and filter edges by container type. +* Filter tools let a user locate operators by name, hide idle elements, heat-color by metric (including per-worker skew), and filter edges by container type. * Rendering strings from the catalog cannot inject markup (closes CNS-108 by construction). * A failed refetch always replaces the graph with an error state (closes CNS-109, with a regression test). * No `d3-graphviz` or WASM dependency remains. @@ -26,10 +26,7 @@ Beyond the bugs, the visualizer lacks features operators need: no dataflow selec All required data exists in `mz_introspection` relations already queried today. * Live-updating stats via `SUBSCRIBE`. The data layer separates structure from stats so this can be added later, but v1 ships manual refresh only. -* Per-worker breakdowns and skew analysis. - V1 uses the non-per-worker introspection views, as today. - Note their semantics differ: structural views (operators, addresses, channels) restrict to worker 0, stats views (elapsed, arrangement sizes, message counts) sum across workers. -* URL-persisted filter and collapse state. +* URL-persisted filter and drill-down state. * New end-to-end tests. ## Solution Proposal @@ -59,8 +56,9 @@ Channels need no extra join for this: the first element of an operator address i Transient dataflows (peeks, subscribes) are logged in both relations while alive (`CollectionLogging::new` runs for every export id in `compute_state.rs`, `log_lir_mapping` for every built object in `render.rs`), so LIR data works for them too. A dataflow can have multiple exports, so the LIR panel groups entries by export id. A new cheap query feeds the dataflow selector list. +A fourth query, `perWorkerStats`, feeds the skew heatmaps: per-worker elapsed time from `mz_scheduling_elapsed_per_worker` and per-worker arrangement size from `mz_arrangement_sizes_per_worker`, full-outer-joined on `(id, worker_id)` since an operator can appear in one source and not the other, and a worker absent from a source never touched that side of it (must not be coalesced to a false zero, since that would understate skew). -The full pre-collapse structure is fetched client-side in one shot. +The full structure is fetched client-side in one shot. Assumption: dataflows stay within tens of thousands of operators and channels, which fits comfortably in memory and within the existing 30 second query timeout, which v1 keeps. A new hook returns results tagged with the parameters that produced them. @@ -68,36 +66,35 @@ The render layer discards results whose tag does not match the current selection This closes CNS-109 without depending on the broken `requestIdRef` logic in `useSqlApiRequest`. Refresh is manual: a refresh button plus a "last fetched" timestamp. -Layout, collapse state, and viewport survive a refresh. +Layout, `focusedScope`, and viewport survive a refresh. Stats update in place, and relayout happens only if the structure changed. The structure/stats split makes a later `SUBSCRIBE`-driven live mode a drop-in: stats stream into node badges without relayout. -### Graph model and collapse +### Graph model and drill-down Pure functions in a new `dataflowGraph.ts` build the region tree from `mz_dataflow_addresses` alone, replacing `collateOperators`. An operator's parent is the operator whose address is its own minus the last element, so `mz_dataflow_operator_parents` (itself derived from addresses) is dropped to avoid two sources of truth that can disagree across query times. -Regions map to React Flow parent (group) nodes, operators to leaf nodes. -Every node carries own and transitive stats: arrangement records, size, elapsed. -Transitive stats are computed once when the structure is built, not per collapse toggle. +Every node carries own and transitive stats: arrangement records, size, elapsed, and CPU/memory skew (worst worker over the average, matching `EXPLAIN ANALYZE ... WITH SKEW`). +Transitive stats and skew are computed once when the structure is built, from per-worker vectors merged up the tree; skew itself is recomputed from the merged vector at each level rather than summed. -Each region is expanded or collapsed in place. -A collapsed region renders as a single node with aggregated stats and a child count. -Edges crossing a collapsed boundary are remapped to the region node and aggregated: one edge per direction pair, summed records and batches, and the set of container types. -The derivation is a pure function from `(structure, collapseState)` to visible nodes and edges. -The default state shows the root's direct children with everything below collapsed. +Rather than expanding and collapsing regions in place, the view shows exactly one scope's direct children at a time, addressed by a single `focusedScope: NodeId`. +Double-clicking a region navigates into it (`focusedScope` becomes that region); a breadcrumb trail navigates back out. +This avoids the in-place model's edge-remapping and aggregation machinery entirely: a scope's boundary-crossing channels become synthetic port nodes (`kind: "port"`) on the boundary, each carrying a `peers: PortPeer[]` list describing where the crossing goes outside the current view. +A peer records the address on the far side and, when that peer is itself a region, the exact port inside it the crossing lands on (`peerPortId`), so a jump can drill straight to the matching port instead of parking on the peer's outer box. +Jumping is click-to-select-then-jump, or double-click direct when a port has exactly one peer; a fanned-out port (one output feeding several inputs) has no single unambiguous target, so double-click just selects it. +The derivation `deriveVisibleGraph(structure, focusedScope)` is a pure function from a structure and a scope to the visible nodes, edges, and ports; there is no persisted collapse state, so a stale scope from a previous structure resolves at render time by falling back to the new structure's root rather than crashing. -Regions and LIR spans are two different hierarchies, so only regions get the nesting. -LIR information appears as a badge and operator text on each node, plus a side panel listing the dataflow's LIR operators from `mz_lir_mapping`. -Hovering or clicking a panel entry highlights the member operators on the canvas. -Scope inputs and outputs stay as small port nodes on the region boundary, as today. +Regions and LIR spans are two different hierarchies. +LIR information appears as a badge and operator text on each node, plus a side panel listing the dataflow's LIR operators from `mz_lir_mapping` as a tree, with member highlighting and multi-pin. +Clicking a panel entry navigates to the scope containing its operators and highlights them. ```mermaid flowchart LR - sql[SQL results] --> structure[dataflowGraph.ts: region tree + stats] - structure --> visible["(structure, collapseState, filters) -> visible nodes/edges"] + sql[SQL results] --> structure[dataflowGraph.ts: region tree + stats + skew] + structure --> visible["(structure, focusedScope) -> visible nodes/edges/ports"] visible --> elk[elkjs worker layout] elk --> rf[React Flow canvas] - rf -- expand/collapse, filter --> visible + rf -- navigate into region, jump to peer, filter --> visible ``` ### Type contracts @@ -114,6 +111,12 @@ interface NodeStats { elapsedNs: bigint; } +// Worst worker's value over the average, matching EXPLAIN ANALYZE ... WITH SKEW. +interface SkewStats { + cpuSkew: number; + memorySkew: number; +} + interface DataflowNode { id: NodeId; address: Address; @@ -122,7 +125,9 @@ interface DataflowNode { children: NodeId[]; // empty for leaf operators own: NodeStats; transitive: NodeStats; // own + subtree, precomputed - lir: { exportId: string; lirId: string; operator: string } | null; + ownSkew: SkewStats; + transitiveSkew: SkewStats; // recomputed from merged per-worker vectors, not summed + lir: { exportId: string; lirId: string; operator: string }[]; } interface DataflowStructure { @@ -131,20 +136,28 @@ interface DataflowStructure { channels: Channel[]; // as fetched, operator-level } -type CollapseState = ReadonlySet; // collapsed region ids - -interface VisibleEdge { - id: string; - source: NodeId; // visible node after remapping - target: NodeId; +// A boundary crossing out of the current view. peerPortId names the exact +// port inside the peer this crossing lands on, when the peer is a region; +// null for a leaf peer, which has no inside to drill into. +interface PortPeer { + address: Address; + label: string; messagesSent: bigint; batchesSent: bigint; - channelTypes: string[]; // set union when aggregated + channelTypes: string[]; + peerPortId: NodeId | null; +} + +interface VisibleNode { + id: NodeId; + kind: "operator" | "region" | "port"; + // ... name, stats, transitiveSkew, address, etc. + peers: PortPeer[]; // non-empty only for kind: "port" } interface VisibleGraph { - nodes: DataflowNode[]; // only visible ones, regions included - edges: VisibleEdge[]; // directed; A->B and B->A stay separate + nodes: VisibleNode[]; // exactly focusedScope's direct children, plus ports + edges: VisibleEdge[]; // directed, scoped to the current view only } // Worker protocol. One in-flight request, stale responses dropped by id. @@ -158,42 +171,39 @@ interface LayoutResponse { } ``` -Filter and collapse state are owned by the top-level visualizer page component and passed down. -The visible-graph derivation is `deriveVisibleGraph(structure, collapseState, filters): VisibleGraph`, a pure function. +Filter state and `focusedScope` are owned by the top-level visualizer page component and passed down. +The visible-graph derivation is `deriveVisibleGraph(structure, focusedScope): VisibleGraph`; filters are a separate `decorateGraph(graph, filters)` pass over the result (dimming, heat coloring, highlighting) rather than part of the derivation itself. ### Rendering and layout -elkjs runs `elk.layered` in a web worker, direction left to right, over exactly the visible post-collapse graph. +elkjs runs `elk.layered` in a web worker, direction top to bottom, over exactly the current scope's visible graph. The console builds with Vite, so layout runs in a Vite module worker (`new Worker(new URL("./layout.worker.ts", import.meta.url), { type: "module" })`) that imports `elkjs/lib/elk.bundled.js` and implements the layout request protocol below. -Phase 1 verifies this in both `vite dev` and the production build. +Phase 1 verified this in both `vite dev` and the production build. If the module worker fails to bundle or run `elk.bundled.js`, the fallback is running elkjs on the main thread in a lazily loaded chunk, accepting UI stalls during layout. -Toggling collapse recomputes layout, memoized per collapse state so toggling back is instant. +Navigating into or out of a region recomputes layout for the new scope. A small "layouting" overlay shows during computation while the canvas stays interactive. The node component shows name, arranged records, size, and elapsed time. -Default colors keep today's two-by-two palette (region versus operator, has arrangement versus not). +Default colors keep a two-by-two palette (region versus operator, has arrangement versus not). Heatmap mode overrides node color. Edges are labeled with records and batches, dashed when zero messages were sent, with the container type as an edge badge and tooltip. -Perf guards: collapse-by-default keeps the initial node count small, and viewport culling handles large expanded graphs. -A constant `MAX_VISIBLE_NODES = 1500` bounds the visible graph, counting leaf and region nodes but not edges. -An expand that would exceed it is refused with a warning toast. +Perf guard: drill-down bounds the visible node count structurally, since a view only ever renders one scope's direct children, not the whole graph; viewport culling handles scopes with many children. ### Filters and interactions A toolbar above the canvas offers: -* Name search: matching operators highlighted, others dimmed, next/previous jump that auto-expands ancestor regions of a match. -* Hide idle: hides zero-message edges and zero-elapsed operators, with region aggregates recomputed over the visible set. -* Heatmap: mode select (off, elapsed, arrangement size), sequential color scale, threshold slider that dims nodes below the cutoff, with a legend. +* Name search: matching operators highlighted, others dimmed, next/previous jump that navigates to the scope containing the current match. +* Hide idle: hides zero-message edges and zero-elapsed operators. +* Heatmap: mode select (off, elapsed, arrangement size, CPU skew, memory skew), sequential color scale, threshold slider that dims nodes below the cutoff, with a legend. The scale domain is `[0, max]` over the visible nodes' transitive metric. If the max is zero, all nodes get the neutral base color and the slider is disabled. -* Channel type: multi-select over the container types present, non-matching edges dimmed. -Filters are pure derivations over the visible graph and compose. +Filters are pure derivations over the visible graph (`decorateGraph`) and compose. Filter state lives in component state. -Clicking a node opens a detail side panel (full name, all stats, LIR operator, address). -Double-clicking a region toggles collapse. +Clicking a node or port opens a detail side panel (full name, all stats, LIR operator, address, and for ports, the peers on the far side of the boundary with a jump-to-peer action). +Double-clicking a region navigates into it; double-clicking a port with exactly one peer jumps directly to it, drilling into the peer's matching port when the peer is a region. ### Error handling @@ -216,7 +226,7 @@ The gate, all parts required to proceed: * Worker loading works in `vite dev` and the production build (see rendering section for the pinned mechanism and fallback). * Default collapsed view lays out in under 500 ms. * A fully expanded view of 1500 visible nodes lays out in under 5 s in the worker, UI responsive throughout. -* No overlapping nodes, edges follow left-to-right flow. +* No overlapping nodes, edges follow top-to-bottom flow. * Manual sign-off on visual readability by the requester, from screenshots of the reference dataflow collapsed and expanded. If the gate fails, work stops and the decision returns to this document: tune ELK options, or fall back to the in-house Canvas alternative, decided with the reviewer rather than unilaterally. @@ -234,12 +244,12 @@ If the gate fails, work stops and the decision returns to this document: tune EL ## Open questions * None blocking. - The `SUBSCRIBE` live mode and per-worker views are deliberate follow-ups, not open design points. + The `SUBSCRIBE` live mode is a deliberate follow-up, not an open design point. ## Testing * Unit (vitest): `dataflowGraph.ts` pure functions. - Region tree construction, collapse edge remapping and aggregation, filter derivations, LIR membership. + Region tree construction, skew computation, drill-down derivation, peer-port resolution, filter derivations, LIR membership. Most logic lives here, so most coverage lands here. -* Component (vitest plus msw): selector flow, error-on-refetch (the CNS-109 case), permission alert, empty states. +* Component (vitest plus msw): selector flow, error-on-refetch (the CNS-109 case), the stale-scope crash regression, port jump (click and double-click, 1:1 and fan-out), permission alert, empty states. React Flow renders with the elk worker stubbed to deterministic positions. From 9338185eab9068ed6d5ded8fa88e2a6a63dfcfc0 Mon Sep 17 00:00:00 2001 From: Moritz Hoffmann Date: Wed, 8 Jul 2026 22:42:39 +0200 Subject: [PATCH 22/45] Fix review findings: empty-dataflow crash, stale search index, dead Visualize links, stale pins Co-Authored-By: Claude Sonnet 5 --- .../dataflow/useDataflowList.test.ts | 47 +++++++-- .../materialize/dataflow/useDataflowList.ts | 35 ++++++- console/src/platform/clusters/IndexList.tsx | 15 ++- .../clusters/MaterializedViewsList.tsx | 15 ++- .../platform/dataflows/DataflowDetailPage.tsx | 96 ++++++++++++------- .../platform/dataflows/DataflowGraphView.tsx | 77 +++++++++------ .../src/platform/dataflows/DataflowsPage.tsx | 19 +--- console/src/platform/dataflows/LirPanel.tsx | 3 +- .../platform/dataflows/NodeDetailPanel.tsx | 13 +-- .../dataflows/UsagePrivilegeAlert.tsx | 24 +++++ .../platform/dataflows/dataflowGraph.test.ts | 18 +++- .../src/platform/dataflows/dataflowGraph.ts | 58 +++++++++-- .../src/platform/dataflows/useElkLayout.ts | 8 +- 13 files changed, 311 insertions(+), 117 deletions(-) create mode 100644 console/src/platform/dataflows/UsagePrivilegeAlert.tsx diff --git a/console/src/api/materialize/dataflow/useDataflowList.test.ts b/console/src/api/materialize/dataflow/useDataflowList.test.ts index 60e90dd3cfe4b..250315c2c1d24 100644 --- a/console/src/api/materialize/dataflow/useDataflowList.test.ts +++ b/console/src/api/materialize/dataflow/useDataflowList.test.ts @@ -17,18 +17,24 @@ import { createProviderWrapper } from "~/test/utils"; import { useDataflowList } from "./useDataflowList"; const FAILING_REPLICA = "r2"; +const COLUMNS = [ + { name: "id" }, + { name: "name" }, + { name: "records" }, + { name: "size" }, + { name: "elapsedNs" }, +]; const listResult = { - desc: { - columns: [ - { name: "id" }, - { name: "name" }, - { name: "records" }, - { name: "size" }, - { name: "elapsedNs" }, - ], - }, + desc: { columns: COLUMNS }, rows: [["7", "Dataflow: mv", "100", "4096", "12345"]], }; +// Distinct ids from listResult: dataflow ids are per-replica, so a stale row +// from the wrong replica reads as an entirely different, meaningless entry +// rather than merely out of date. +const otherReplicaResult = { + desc: { columns: COLUMNS }, + rows: [["9", "Dataflow: other_mv", "200", "8192", "54321"]], +}; beforeEach(() => { server.use( @@ -48,6 +54,9 @@ beforeEach(() => { ], }); } + if (options.cluster_replica === "r3") { + return HttpResponse.json({ results: [otherReplicaResult] }); + } return HttpResponse.json({ results: [listResult] }); }), ); @@ -70,6 +79,26 @@ describe("useDataflowList", () => { }); }); + it("hides the previous replica's list while the new one is loading", async () => { + const Wrapper = await createProviderWrapper(); + const { result, rerender } = renderHook( + (replicaName: string) => + useDataflowList({ clusterName: "c", replicaName }), + { wrapper: Wrapper, initialProps: "r1" }, + ); + await waitFor(() => expect(result.current.data).not.toBeNull()); + expect(result.current.data?.[0].id).toBe("7"); + + // Switching replicas must not show r1's still-resident rows tagged as + // r3's: dataflow ids are per-replica, so navigating one would be + // meaningless, and a transient dataflow gone on r3 would crash instead. + rerender("r3"); + expect(result.current.data).toBeNull(); + + await waitFor(() => expect(result.current.data).not.toBeNull()); + expect(result.current.data?.[0].id).toBe("9"); + }); + it("clears data when the fetch fails", async () => { // The hook reads the current environment off a jotai atom that resolves // asynchronously, so it needs the same Suspense + JotaiProvider wrapper the diff --git a/console/src/api/materialize/dataflow/useDataflowList.ts b/console/src/api/materialize/dataflow/useDataflowList.ts index f2c0ab7df48e8..4feb55ff4c99f 100644 --- a/console/src/api/materialize/dataflow/useDataflowList.ts +++ b/console/src/api/materialize/dataflow/useDataflowList.ts @@ -83,8 +83,37 @@ export function useDataflowList(params?: DataflowListParams) { }, ); - // Error always wins, and a list going momentarily stale across a replica - // switch is masked by loading. - const data = !error && results ? normalize(results.list ?? []) : null; + // JSON tuple, not a delimiter-joined string: identifiers can themselves + // contain any delimiter picked. + const key = params + ? JSON.stringify([params.clusterName, params.replicaName]) + : null; + const [lastGood, setLastGood] = React.useState<{ + key: string; + data: DataflowListEntry[]; + } | null>(null); + + // The key for which a fetch has actually started. Without this, the + // previous replica's still-resident results get tagged with the new key in + // the render after the selection changes but before useSqlMany's effect + // flips loading, showing the old replica's dataflow ids under the new + // replica (they're per-replica, so navigating one is meaningless, and for + // a transient dataflow that's gone on the new replica, a crash). Since a + // key change alone can't equal a not-yet-updated sawLoadingForKey, this + // doubles as the reset: no separate render-phase state adjustment needed. + const [sawLoadingForKey, setSawLoadingForKey] = React.useState( + null, + ); + + React.useEffect(() => { + if (loading) setSawLoadingForKey(key); + if (key && results && !error && !loading && sawLoadingForKey === key) { + setLastGood({ key, data: normalize(results.list ?? []) }); + } + }, [key, results, error, loading, sawLoadingForKey]); + + // Error always wins, and data for other params never renders. + const data = + !error && lastGood && lastGood.key === key ? lastGood.data : null; return { data, error, databaseError, loading, refetch }; } diff --git a/console/src/platform/clusters/IndexList.tsx b/console/src/platform/clusters/IndexList.tsx index 70daf6e7a4354..9e31accc1183a 100644 --- a/console/src/platform/clusters/IndexList.tsx +++ b/console/src/platform/clusters/IndexList.tsx @@ -31,7 +31,7 @@ import { isSystemCluster, isSystemId, } from "~/api/materialize"; -import { Replica } from "~/api/materialize/cluster/clusterList"; +import { Cluster, Replica } from "~/api/materialize/cluster/clusterList"; import { Index } from "~/api/materialize/cluster/indexesList"; import { AppErrorBoundary } from "~/components/AppErrorBoundary"; import IndexListEmptyState from "~/components/IndexListEmptyState"; @@ -50,8 +50,12 @@ import { useFlags } from "~/hooks/useFlags"; import useLocalStorage from "~/hooks/useLocalStorage"; import { InfoIcon } from "~/icons"; import { MainContentContainer } from "~/layouts/BaseLayout"; -import { useBuildIndexPath } from "~/platform/routeHelpers"; +import { + absoluteClusterPath, + useBuildIndexPath, +} from "~/platform/routeHelpers"; import { useAllClusters } from "~/store/allClusters"; +import { useRegionSlug } from "~/store/environments"; import { MaterializeTheme } from "~/theme"; import { truncateMaxWidth } from "~/theme/components/Table"; import { assert } from "~/util"; @@ -187,6 +191,7 @@ const IndexListInner = ({ replicas={cluster?.replicas ?? []} memoryUsageMap={memoryUsageById} lagMap={lagMap} + cluster={cluster} /> ); }; @@ -196,12 +201,14 @@ interface IndexTableProps { replicas: Replica[]; memoryUsageMap: ArrangmentsMemoryUsageMap | undefined; lagMap?: LagMap; + cluster: Cluster | undefined; } const IndexTable = (props: IndexTableProps) => { const navigate = useNavigate(); const flags = useFlags(); const indexPath = useBuildIndexPath(); + const regionSlug = useRegionSlug(); const dataflowVisualizerEnabled = flags["visualization-features"]; const { colors } = useTheme(); @@ -275,7 +282,7 @@ const IndexTable = (props: IndexTableProps) => { )} {formattedLag} - {dataflowVisualizerEnabled && ( + {dataflowVisualizerEnabled && props.cluster && ( { { e.stopPropagation(); }} diff --git a/console/src/platform/clusters/MaterializedViewsList.tsx b/console/src/platform/clusters/MaterializedViewsList.tsx index 4f5697fcc7d85..1dd0f1b5ae6cf 100644 --- a/console/src/platform/clusters/MaterializedViewsList.tsx +++ b/console/src/platform/clusters/MaterializedViewsList.tsx @@ -25,7 +25,7 @@ import React from "react"; import { Link, useNavigate, useParams } from "react-router-dom"; import { createNamespace } from "~/api/materialize"; -import { Replica } from "~/api/materialize/cluster/clusterList"; +import { Cluster, Replica } from "~/api/materialize/cluster/clusterList"; import { MaterializedView, MaterializedViewsResponse, @@ -51,8 +51,12 @@ import { SampleCodeBoxWrapper, } from "~/layouts/listPageComponents"; import docUrls from "~/mz-doc-urls.json"; -import { useBuildMaterializedViewPath } from "~/platform/routeHelpers"; +import { + absoluteClusterPath, + useBuildMaterializedViewPath, +} from "~/platform/routeHelpers"; import { useAllClusters } from "~/store/allClusters"; +import { useRegionSlug } from "~/store/environments"; import { MaterializeTheme } from "~/theme"; import { truncateMaxWidth } from "~/theme/components/Table"; import { assert } from "~/util"; @@ -205,6 +209,7 @@ const MaterializedViewListInner = ({ replicas={cluster?.replicas ?? []} memoryUsageMap={memoryUsageById} lagMap={lagMap} + cluster={cluster} /> @@ -216,12 +221,14 @@ interface MaterializedViewTableProps { replicas: Replica[]; memoryUsageMap: ArrangmentsMemoryUsageMap | undefined; lagMap?: LagMap; + cluster: Cluster | undefined; } const MaterializedViewTable = (props: MaterializedViewTableProps) => { const navigate = useNavigate(); const flags = useFlags(); const materializedViewPath = useBuildMaterializedViewPath(); + const regionSlug = useRegionSlug(); const dataflowVisualizerEnabled = flags["visualization-features"]; const { colors } = useTheme(); @@ -279,7 +286,7 @@ const MaterializedViewTable = (props: MaterializedViewTableProps) => { )} {formattedLag} - {dataflowVisualizerEnabled && ( + {dataflowVisualizerEnabled && props.cluster && ( { { e.stopPropagation(); }} diff --git a/console/src/platform/dataflows/DataflowDetailPage.tsx b/console/src/platform/dataflows/DataflowDetailPage.tsx index 517cdcff93048..3ce08a18d028b 100644 --- a/console/src/platform/dataflows/DataflowDetailPage.tsx +++ b/console/src/platform/dataflows/DataflowDetailPage.tsx @@ -7,16 +7,7 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -import { - Alert, - AlertIcon, - Box, - Button, - HStack, - Spinner, - Text, - VStack, -} from "@chakra-ui/react"; +import { Box, Button, HStack, Spinner, Text, VStack } from "@chakra-ui/react"; import { interpolateYlOrRd } from "d3-scale-chromatic"; import React from "react"; import { @@ -55,6 +46,7 @@ import { DataflowGraphView } from "./DataflowGraphView"; import { DataflowToolbar } from "./DataflowToolbar"; import { LirPanel } from "./LirPanel"; import { NodeDetailPanel, type Selection } from "./NodeDetailPanel"; +import { UsagePrivilegeAlert } from "./UsagePrivilegeAlert"; // Injected into decorateGraph so the pure graph module stays d3-free. const heatColor = (t: number) => interpolateYlOrRd(0.15 + 0.85 * t); @@ -126,6 +118,27 @@ const DataflowDetailPage = () => { const [pendingFitIds, setPendingFitIds] = React.useState( null, ); + + // The route only remounts on a clusterId change, so switching dataflow or + // replica in place (via the dropdowns) keeps this component instance, its + // selection, filters, and pins alive. Those reference node ids from the + // old structure. On the new one they silently miss instead of crashing + // (see focusedScope above), which for a pin means every node reads as + // "not in the highlighted set" and the whole graph dims with no way to + // un-pin. Reset render-phase, not via effect, so there is no render in + // between showing the new graph under the old pins. + const resetKey = JSON.stringify([replicaName, dataflowId]); + const [trackedResetKey, setTrackedResetKey] = React.useState(resetKey); + if (resetKey !== trackedResetKey) { + setTrackedResetKey(resetKey); + setFocusedScope(null); + setSelection(null); + setFilters(DEFAULT_FILTERS); + setMatchIndex(0); + setLirHighlight(null); + setPinnedLir(new Map()); + } + const centerRef = React.useRef<((id: string) => void) | null>(null); const fitRef = React.useRef<((ids: string[]) => void) | null>(null); @@ -186,7 +199,7 @@ const DataflowDetailPage = () => { // A port's peer lives outside the current view. When the peer is itself a // region, peerPortId names the exact port this crossing lands on from // inside it, so the jump drills straight there instead of parking outside - // as an unlabeled box; a leaf peer has no inside to drill into, so the + // as an unlabeled box. A leaf peer has no inside to drill into, so the // fallback lands on the peer itself, in its own containing scope. const onJumpToPeer = React.useCallback( (peer: PortPeer) => { @@ -219,10 +232,26 @@ const DataflowDetailPage = () => { () => (data ? allSearchMatches(data.structure, filters.search) : []), [data, filters.search], ); + // matchIndex resets to 0 only when the search text changes (see the effect + // below). A refetch that shrinks the match set for the same search text, + // or the render between a search-text change and that effect running, + // otherwise leaves it pointing past the end of allMatches. + const activeMatchIndex = Math.min(matchIndex, allMatches.length - 1); + + // Shared with DataflowGraphView via the visible prop below, so the current + // scope's graph is derived once per render instead of once here and again + // there. + const visibleGraph = React.useMemo( + () => + data && focusedScope + ? deriveVisibleGraph(data.structure, focusedScope) + : null, + [data, focusedScope], + ); const decorations = React.useMemo(() => { - if (!data || !focusedScope) return undefined; - const visible = deriveVisibleGraph(data.structure, focusedScope); + if (!data || !focusedScope || !visibleGraph) return undefined; + const visible = visibleGraph; const searchInfo = filters.search ? subtreeSearchMatches(data.structure, filters.search) : null; @@ -253,7 +282,7 @@ const DataflowDetailPage = () => { } } return d; - }, [data, focusedScope, filters, lirHighlight, pinnedLir]); + }, [data, focusedScope, visibleGraph, filters, lirHighlight, pinnedLir]); React.useEffect(() => { setMatchIndex(0); @@ -262,20 +291,23 @@ const DataflowDetailPage = () => { const onJump = React.useCallback( (delta: 1 | -1) => { if (allMatches.length === 0) return; - const next = (matchIndex + delta + allMatches.length) % allMatches.length; + const next = + (activeMatchIndex + delta + allMatches.length) % allMatches.length; setMatchIndex(next); navigateAndFit([allMatches[next].address]); }, - [allMatches, matchIndex, navigateAndFit], + [allMatches, activeMatchIndex, navigateAndFit], ); // Digest of the sorted node ids: identical structure across a stats-only // refresh yields the same key, so layout and collapse state are preserved. - const structureKey = data - ? (() => { - const ids = [...data.structure.nodes.keys()].sort(); - return `${params?.dataflowId}/${params?.replicaName}/${ids.length}-${hashString(ids.join(","))}`; - })() - : null; + // Memoized since sorting and hashing every node id is real work on a + // dataflow with tens of thousands of operators, and this otherwise reran + // on every render, not just on a new structure. + const structureKey = React.useMemo(() => { + if (!data) return null; + const ids = [...data.structure.nodes.keys()].sort(); + return `${params?.dataflowId}/${params?.replicaName}/${ids.length}-${hashString(ids.join(","))}`; + }, [data, params?.dataflowId, params?.replicaName]); if (!cluster) return null; if (cluster.replicas.length === 0) { return ( @@ -347,16 +379,7 @@ const DataflowDetailPage = () => { )} {permissionError ? ( - - - - You'll need{" "} - - USAGE - {" "} - privilege on this cluster to visualize this dataflow. - - + ) : error ? ( @@ -383,7 +406,7 @@ const DataflowDetailPage = () => { filters={filters} onFiltersChange={setFilters} matchCount={allMatches.length} - matchIndex={matchIndex} + matchIndex={activeMatchIndex} onJump={onJump} /> {filters.heatmap !== "off" && ( @@ -412,7 +435,10 @@ const DataflowDetailPage = () => { onTogglePin={onTogglePinLir} /> { } activeMatchId={ allMatches.length > 0 - ? nodeIdOf(allMatches[matchIndex].address) + ? nodeIdOf(allMatches[activeMatchIndex].address) : undefined } onNodeClick={(node, connectedEdges) => diff --git a/console/src/platform/dataflows/DataflowGraphView.tsx b/console/src/platform/dataflows/DataflowGraphView.tsx index 5ed1d6424b80d..875549c0d08a5 100644 --- a/console/src/platform/dataflows/DataflowGraphView.tsx +++ b/console/src/platform/dataflows/DataflowGraphView.tsx @@ -10,7 +10,7 @@ import "@xyflow/react/dist/style.css"; import "./DataflowGraphView.css"; -import { Box, Spinner } from "@chakra-ui/react"; +import { Box, Button, Spinner, VStack } from "@chakra-ui/react"; import { Background, Controls, @@ -23,15 +23,16 @@ import { } from "@xyflow/react"; import React from "react"; +import ErrorBox from "~/components/ErrorBox"; + import { ChannelEdge } from "./ChannelEdge"; import { - type DataflowStructure, - deriveVisibleGraph, type GraphDecorations, type NodeId, type PortPeer, rerouteHiddenNodes, type VisibleEdge, + type VisibleGraph, type VisibleNode, } from "./dataflowGraph"; import { NODE_DIMENSIONS } from "./elkGraph"; @@ -53,8 +54,11 @@ export type SelectedEdge = VisibleEdge & { }; export interface DataflowGraphViewProps { - structure: DataflowStructure; - // The scope whose direct children this view renders; double-clicking a + // The current scope's graph, already derived by the caller (which also + // needs it, to decorate). Computing it again here would repeat the same + // work every render. + visible: VisibleGraph; + // The scope whose direct children this view renders. Double-clicking a // region box navigates to a new view rooted there rather than expanding it // in place. focusedScope: NodeId; @@ -69,7 +73,7 @@ export interface DataflowGraphViewProps { onNodeClick?: (node: VisibleNode, connectedEdges: SelectedEdge[]) => void; onEdgeClick?: (edge: SelectedEdge) => void; onPaneClick?: () => void; - // Double-clicking a port with exactly one peer jumps straight there; with + // Double-clicking a port with exactly one peer jumps straight there. With // zero or several peers it's ambiguous (or there's nothing to jump to), so // the preceding click/click of the double-click has already opened the // port's own detail panel instead, same as a single click would. @@ -133,7 +137,7 @@ const CenterHelper = ({ }; export const DataflowGraphView = ({ - structure, + visible: rawVisible, focusedScope, onNavigate, cacheKey, @@ -152,14 +156,13 @@ export const DataflowGraphView = ({ onFit, }: DataflowGraphViewProps) => { const visible = React.useMemo(() => { - const graph = deriveVisibleGraph(structure, focusedScope); // Hidden nodes get spliced out with connectivity preserved (a hidden idle // run still shows a pass-through edge). Hidden edges (independently // zero-message) are a plain removal: nothing to reroute since both // endpoints stay visible. const rerouted = decorations?.hiddenNodeIds - ? rerouteHiddenNodes(graph, decorations.hiddenNodeIds) - : graph; + ? rerouteHiddenNodes(rawVisible, decorations.hiddenNodeIds) + : rawVisible; if (!decorations?.hiddenEdgeIds?.size) return rerouted; return { nodes: rerouted.nodes, @@ -167,12 +170,7 @@ export const DataflowGraphView = ({ (e) => !decorations.hiddenEdgeIds!.has(e.id), ), }; - }, [ - structure, - focusedScope, - decorations?.hiddenNodeIds, - decorations?.hiddenEdgeIds, - ]); + }, [rawVisible, decorations?.hiddenNodeIds, decorations?.hiddenEdgeIds]); const layoutKey = `${cacheKey}|${focusedScope}|${ decorations?.hiddenNodeIds @@ -183,11 +181,20 @@ export const DataflowGraphView = ({ ? [...decorations.hiddenEdgeIds].sort().join(",") : "" }`; - const { positions, layouting, error } = useElkLayout(visible, layoutKey); + const { positions, layouting, error, retry } = useElkLayout( + visible, + layoutKey, + ); React.useEffect(() => { if (!centerOnId || !positions?.[centerOnId]) return; - centerRef?.current?.(centerOnId); + // CenterHelper assigns centerRef.current in its own mount effect; on the + // rare commit where that hasn't run yet, skip without marking the + // request consumed (onCentered stays uncalled) so the next positions or + // centerOnId change gets another chance, rather than silently dropping + // the jump forever. + if (!centerRef?.current) return; + centerRef.current(centerOnId); onCentered?.(); }, [centerOnId, positions, centerRef, onCentered]); @@ -198,7 +205,8 @@ export const DataflowGraphView = ({ // behind a filter) doesn't block the fit indefinitely. const present = fitOnIds.filter((id) => positions?.[id]); if (present.length === 0) return; - fitRef?.current?.(present); + if (!fitRef?.current) return; + fitRef.current(present); onFit?.(); }, [fitOnIds, positions, fitRef, onFit]); @@ -261,7 +269,24 @@ export const DataflowGraphView = ({ [visible, decorations?.dimmedNodeIds, selectedId], ); - if (error) throw new Error(error); + // Built once per visible-graph change instead of a .find() per connected + // edge in the click handlers below, which is O(nodes) per edge and adds up + // for a hub node in a large scope. + const labelById = React.useMemo( + () => new Map(visible.nodes.map((n) => [n.id, n.label])), + [visible], + ); + + if (error) { + return ( + + + + + ); + } return ( {layouting && ( @@ -281,28 +306,24 @@ export const DataflowGraphView = ({ zoomOnDoubleClick={false} onNodeClick={(_, node) => { const visibleNode = (node.data as { node: VisibleNode }).node; - const resolveLabel = (id: string) => - visible.nodes.find((n) => n.id === id)?.label ?? id; const connectedEdges = visible.edges .filter( (e) => e.source === visibleNode.id || e.target === visibleNode.id, ) .map((e) => ({ ...e, - sourceLabel: resolveLabel(e.source), - targetLabel: resolveLabel(e.target), + sourceLabel: labelById.get(e.source) ?? e.source, + targetLabel: labelById.get(e.target) ?? e.target, })); onNodeClick?.(visibleNode, connectedEdges); }} onEdgeClick={(_, edge) => { const e = visible.edges.find((ve) => ve.id === edge.id); if (!e) return; - const resolveLabel = (id: string) => - visible.nodes.find((n) => n.id === id)?.label ?? id; onEdgeClick?.({ ...e, - sourceLabel: resolveLabel(e.source), - targetLabel: resolveLabel(e.target), + sourceLabel: labelById.get(e.source) ?? e.source, + targetLabel: labelById.get(e.target) ?? e.target, }); }} onPaneClick={onPaneClick} diff --git a/console/src/platform/dataflows/DataflowsPage.tsx b/console/src/platform/dataflows/DataflowsPage.tsx index 5bf3aaf2e8706..4cdcb517a196e 100644 --- a/console/src/platform/dataflows/DataflowsPage.tsx +++ b/console/src/platform/dataflows/DataflowsPage.tsx @@ -8,13 +8,10 @@ // by the Apache License, Version 2.0. import { - Alert, - AlertIcon, Spinner, Table, Tbody, Td, - Text, Th, Thead, Tr, @@ -32,6 +29,9 @@ import { MainContentContainer } from "~/layouts/BaseLayout"; import { useAllClusters } from "~/store/allClusters"; import { formatBytesShort } from "~/utils/format"; +import { formatElapsedNs } from "./dataflowGraph"; +import { UsagePrivilegeAlert } from "./UsagePrivilegeAlert"; + const DataflowsPage = () => { const { clusterId } = useParams(); const { getClusterById } = useAllClusters(); @@ -103,16 +103,7 @@ const DataflowsPage = () => { ))} {permissionError ? ( - - - - You'll need{" "} - - USAGE - {" "} - privilege on this cluster to list its dataflows. - - + ) : error ? ( ) : !data && loading ? ( @@ -135,7 +126,7 @@ const DataflowsPage = () => { {d.records.toString()} {formatBytesShort(d.size)} - {Math.round(Number(d.elapsedNs) / 1e9)}s + {formatElapsedNs(d.elapsedNs)} ))} diff --git a/console/src/platform/dataflows/LirPanel.tsx b/console/src/platform/dataflows/LirPanel.tsx index fd869e86a40f2..6f2adb7321f15 100644 --- a/console/src/platform/dataflows/LirPanel.tsx +++ b/console/src/platform/dataflows/LirPanel.tsx @@ -14,6 +14,7 @@ import { formatBytesShort } from "~/utils/format"; import { type DataflowStructure, + formatElapsedNs, lirIndex, lirTree, type LirTreeNode, @@ -86,7 +87,7 @@ const LirSummaryCard = ({ node }: { node: LirTreeNode }) => ( Elapsed - {Math.round(Number(node.summary.elapsedNs) / 1e9)}s + {formatElapsedNs(node.summary.elapsedNs)} diff --git a/console/src/platform/dataflows/NodeDetailPanel.tsx b/console/src/platform/dataflows/NodeDetailPanel.tsx index 72a92a598cf25..d3957c50dfbb8 100644 --- a/console/src/platform/dataflows/NodeDetailPanel.tsx +++ b/console/src/platform/dataflows/NodeDetailPanel.tsx @@ -12,7 +12,11 @@ import React from "react"; import { formatBytesShort } from "~/utils/format"; -import type { PortPeer, VisibleNode } from "./dataflowGraph"; +import { + formatElapsedNs, + type PortPeer, + type VisibleNode, +} from "./dataflowGraph"; import type { SelectedEdge } from "./DataflowGraphView"; import { prettyPrintChannelType } from "./nodeStyle"; @@ -113,10 +117,7 @@ const NodeDetail = ({ label="Arrangement size" value={formatBytesShort(node.stats.arrangementSize)} /> - + )} {node.transitive && node.childCount > 0 && ( @@ -131,7 +132,7 @@ const NodeDetail = ({ /> )} diff --git a/console/src/platform/dataflows/UsagePrivilegeAlert.tsx b/console/src/platform/dataflows/UsagePrivilegeAlert.tsx new file mode 100644 index 0000000000000..35c0241325e53 --- /dev/null +++ b/console/src/platform/dataflows/UsagePrivilegeAlert.tsx @@ -0,0 +1,24 @@ +// Copyright Materialize, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +import { Alert, AlertIcon, Text } from "@chakra-ui/react"; +import React from "react"; + +export const UsagePrivilegeAlert = ({ action }: { action: string }) => ( + + + + You'll need{" "} + + USAGE + {" "} + privilege on this cluster to {action}. + + +); diff --git a/console/src/platform/dataflows/dataflowGraph.test.ts b/console/src/platform/dataflows/dataflowGraph.test.ts index cb69934e5b3a1..9caebf99e6257 100644 --- a/console/src/platform/dataflows/dataflowGraph.test.ts +++ b/console/src/platform/dataflows/dataflowGraph.test.ts @@ -154,8 +154,22 @@ describe("buildDataflowStructure", () => { expect(s.nodes.get(nodeIdOf([5, 2]))!.lir).toEqual([]); }); - it("throws without exactly one root", () => { - expect(() => buildDataflowStructure([], [], [])).toThrow(); + it("throws with more than one root", () => { + const twoRoots: OperatorRow[] = [ + { ...OPS[0], id: "10", address: ["5"] }, + { ...OPS[0], id: "20", address: ["6"] }, + ]; + expect(() => buildDataflowStructure(twoRoots, [], [])).toThrow(); + }); + + it("returns a single placeholder node for a dropped dataflow, instead of throwing on zero operators", () => { + // A dropped or transient dataflow returns zero operator rows: a valid + // empty result, not an error. nodes.size === 1 here is what routes it + // through the same "no longer exists" branch as any other empty + // dataflow, rather than crashing mid-fetch. + const s = buildDataflowStructure([], [], []); + expect(s.nodes.size).toBe(1); + expect(s.nodes.get(s.root)).toBeDefined(); }); it("normalizes channels", () => { diff --git a/console/src/platform/dataflows/dataflowGraph.ts b/console/src/platform/dataflows/dataflowGraph.ts index db95e5b86d721..f7bc74dd4f1e9 100644 --- a/console/src/platform/dataflows/dataflowGraph.ts +++ b/console/src/platform/dataflows/dataflowGraph.ts @@ -13,6 +13,10 @@ export function nodeIdOf(address: Address): NodeId { return JSON.stringify(address); } +export function formatElapsedNs(ns: bigint): string { + return `${Math.round(Number(ns) / 1e9)}s`; +} + export interface NodeStats { arrangementRecords: bigint; arrangementSize: bigint; @@ -20,10 +24,10 @@ export interface NodeStats { } // How unevenly work for this node is spread across workers: worst worker's -// value over the average, matching EXPLAIN ANALYZE ... WITH SKEW (avg is +// value over the average, matching EXPLAIN ANALYZE ... WITH SKEW. Avg is // over workers that show up at all for this node, not the full replica -// worker count; a worker with zero rows anywhere for a node is invisible to -// both, same convention). 1 means perfectly even; higher means more skewed; +// worker count. A worker with zero rows anywhere for a node is invisible to +// both, same convention. 1 means perfectly even, higher means more skewed, // 0 means no data. export interface SkewStats { cpuSkew: number; @@ -80,7 +84,7 @@ export interface PortPeer { channelTypes: string[]; // The exact port this crossing lands on from inside the peer, so jumping // can drill straight into it rather than parking outside as an unlabeled - // box; null when the peer is a leaf (nothing to drill into, so the peer + // box. Null when the peer is a leaf (nothing to drill into, so the peer // itself is the target). peerPortId: NodeId | null; } @@ -187,6 +191,40 @@ export function buildDataflowStructure( lirSpans: LirSpanRow[], perWorkerStats: PerWorkerStatRow[] = [], ): DataflowStructure { + // A dropped or transient dataflow that has already gone away returns zero + // operator rows, a valid query result rather than an error. Represent it + // as a single-node placeholder structure instead of throwing below, so it + // renders through the same nodes.size <= 1 "no longer exists" branch as + // any other empty dataflow. + if (operators.length === 0) { + const placeholderId = nodeIdOf([]); + return { + nodes: new Map([ + [ + placeholderId, + { + id: placeholderId, + operatorId: 0n, + address: [], + name: "", + parent: null, + children: [], + own: { arrangementRecords: 0n, arrangementSize: 0n, elapsedNs: 0n }, + transitive: { + arrangementRecords: 0n, + arrangementSize: 0n, + elapsedNs: 0n, + }, + ownSkew: { cpuSkew: 0, memorySkew: 0 }, + transitiveSkew: { cpuSkew: 0, memorySkew: 0 }, + lir: [], + }, + ], + ]), + root: placeholderId, + channels: [], + }; + } const spans = lirSpans.map((s) => ({ exportId: s.exportId, lirId: s.lirId, @@ -314,7 +352,7 @@ export function buildDataflowStructure( }; } -// A view shows exactly one scope's direct children; anything deeper is +// A view shows exactly one scope's direct children. Anything deeper is // rolled up into its child's box, addressed at that box's depth (viewRoot's // depth + 1). representativeInView is the general form of that projection, // exported for translating arbitrary structure addresses (search matches, @@ -394,8 +432,8 @@ function endpointId( } // Renders exactly one scope's direct children, each either a leaf operator or -// a box summarizing a whole nested subtree (never expanded in place); -// double-clicking a box navigates to a new view rooted there instead. +// a box summarizing a whole nested subtree (never expanded in place). +// Double-clicking a box navigates to a new view rooted there instead. export function deriveVisibleGraph( structure: DataflowStructure, viewRoot: NodeId, @@ -431,7 +469,7 @@ export function deriveVisibleGraph( // Records the out-of-view side of a crossing on its port, so a port that // can't show its peer directly can still be jumped to. One output can feed // several inputs (and vice versa), so this can add up across channels - // rather than replace; the same peer address showing up twice (e.g. two + // rather than replace. The same peer address showing up twice (e.g. two // channel rows for the same logical pair) just accumulates its stats. const addPeer = ( p: { scope: NodeId; direction: "input" | "output"; port: number }, @@ -744,7 +782,7 @@ export interface GraphDecorations { // Search matches anywhere in the whole dataflow, not just the current view // (search is structure-wide so it stays useful once a graph is too big to -// show in one screen); searchInfo carries that lookup in, precomputed once +// show in one screen). searchInfo carries that lookup in, precomputed once // per search string rather than re-walked per view. A box that doesn't // itself match but contains a match deeper in its rolled-up subtree is left // undimmed rather than excluded, so the match's path stays visible without @@ -915,7 +953,7 @@ export function lirSummary( // Arranges lirIndex's flat entries into a tree per export, following // parentLirId. A lir id is assigned once its own build (and so every // descendant's) completes, so within one export a higher lir id is never an -// ancestor of a lower one; sorting each level descending by lir id therefore +// ancestor of a lower one. Sorting each level descending by lir id therefore // matches construction order without needing it as an explicit input, the // same convention EXPLAIN ANALYZE's own tree rendering uses. export function lirTree( diff --git a/console/src/platform/dataflows/useElkLayout.ts b/console/src/platform/dataflows/useElkLayout.ts index 8a1845bf9e498..d9b1a2a7103e2 100644 --- a/console/src/platform/dataflows/useElkLayout.ts +++ b/console/src/platform/dataflows/useElkLayout.ts @@ -29,6 +29,11 @@ export function useElkLayout(graph: VisibleGraph | null, cacheKey: string) { positions: Positions | null; error: string | null; }>({ key: null, positions: null, error: null }); + // Bumped by retry() to force the layout effect to run again for the same + // (graph, cacheKey): a failed layout is never cached, so re-running it is + // enough to retry, but the effect's own dependencies wouldn't otherwise + // change. + const [retryNonce, setRetryNonce] = React.useState(0); React.useEffect(() => { const elk = new ELK({ workerFactory: () => new ElkWorker() }); @@ -61,11 +66,12 @@ export function useElkLayout(graph: VisibleGraph | null, cacheKey: string) { setState({ key: cacheKey, positions: null, error: String(error) }); }, ); - }, [graph, cacheKey]); + }, [graph, cacheKey, retryNonce]); return { positions: state.key === cacheKey ? state.positions : null, layouting: graph !== null && state.key !== cacheKey, error: state.key === cacheKey ? state.error : null, + retry: () => setRetryNonce((n) => n + 1), }; } From 90a433fb6c46621f6ef91bfe768bc958a0dc3941 Mon Sep 17 00:00:00 2001 From: Moritz Hoffmann Date: Wed, 8 Jul 2026 22:42:39 +0200 Subject: [PATCH 23/45] Put drill-down scope in the URL, recenter on empty canvas, minimap stroke, edge highlight Co-Authored-By: Claude Sonnet 5 --- .../20260706_dataflow_visualizer_rebuild.md | 9 +- .../platform/dataflows/ChannelEdge.test.tsx | 70 ++++++++++++ .../src/platform/dataflows/ChannelEdge.tsx | 22 +++- .../dataflows/DataflowDetailPage.test.tsx | 101 +++++++++++++++++- .../platform/dataflows/DataflowDetailPage.tsx | 54 +++++++--- .../platform/dataflows/DataflowGraphView.tsx | 70 +++++++++++- console/src/platform/dataflows/nodeStyle.ts | 5 + 7 files changed, 308 insertions(+), 23 deletions(-) create mode 100644 console/src/platform/dataflows/ChannelEdge.test.tsx diff --git a/console/doc/design/20260706_dataflow_visualizer_rebuild.md b/console/doc/design/20260706_dataflow_visualizer_rebuild.md index 9d9932c234f77..1ba63653ce507 100644 --- a/console/doc/design/20260706_dataflow_visualizer_rebuild.md +++ b/console/doc/design/20260706_dataflow_visualizer_rebuild.md @@ -26,7 +26,8 @@ Beyond the bugs, the visualizer lacks features operators need: no dataflow selec All required data exists in `mz_introspection` relations already queried today. * Live-updating stats via `SUBSCRIBE`. The data layer separates structure from stats so this can be added later, but v1 ships manual refresh only. -* URL-persisted filter and drill-down state. +* URL-persisted filters (search, heatmap mode, hide-idle). + The drill-down scope itself is URL-persisted (see Graph model and drill-down), since back/forward and copyable links depend on it; filters are lower-value to persist and stay in component state. * New end-to-end tests. ## Solution Proposal @@ -84,6 +85,10 @@ A peer records the address on the far side and, when that peer is itself a regio Jumping is click-to-select-then-jump, or double-click direct when a port has exactly one peer; a fanned-out port (one output feeding several inputs) has no single unambiguous target, so double-click just selects it. The derivation `deriveVisibleGraph(structure, focusedScope)` is a pure function from a structure and a scope to the visible nodes, edges, and ports; there is no persisted collapse state, so a stale scope from a previous structure resolves at render time by falling back to the new structure's root rather than crashing. +`focusedScope` lives in the URL, not component state, as the dataflow address it resolves to (a `scope` search param, e.g. `?scope=5.1.2`; omitted at the root). +Every navigation goes through `setSearchParams`, which pushes rather than replaces, so browser back/forward walks the drill-down history and a copied link reopens at the same scope. +The same stale-scope tolerance above covers a scope param that doesn't resolve in the current structure, whether from a stale link, a reshaped dataflow, or no param at all. + Regions and LIR spans are two different hierarchies. LIR information appears as a badge and operator text on each node, plus a side panel listing the dataflow's LIR operators from `mz_lir_mapping` as a tree, with member highlighting and multi-pin. Clicking a panel entry navigates to the scope containing its operators and highlights them. @@ -182,6 +187,8 @@ Phase 1 verified this in both `vite dev` and the production build. If the module worker fails to bundle or run `elk.bundled.js`, the fallback is running elkjs on the main thread in a lazily loaded chunk, accepting UI stalls during layout. Navigating into or out of a region recomputes layout for the new scope. A small "layouting" overlay shows during computation while the canvas stays interactive. +Since elk lays each scope out independently near its own origin, the viewport's pan/zoom from a previous, unrelated scope can land somewhere with nothing from the new one in it. +Once a scope's layout lands, if none of its nodes are actually on screen, the view refits; this is checked once per scope rather than on every render, so it doesn't fight deliberate panning once landed. The node component shows name, arranged records, size, and elapsed time. Default colors keep a two-by-two palette (region versus operator, has arrangement versus not). diff --git a/console/src/platform/dataflows/ChannelEdge.test.tsx b/console/src/platform/dataflows/ChannelEdge.test.tsx new file mode 100644 index 0000000000000..9f69ee1d1812a --- /dev/null +++ b/console/src/platform/dataflows/ChannelEdge.test.tsx @@ -0,0 +1,70 @@ +// Copyright Materialize, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +import { render } from "@testing-library/react"; +import { Position, ReactFlowProvider } from "@xyflow/react"; +import React from "react"; + +import { ChannelEdge, type ChannelEdgeData } from "./ChannelEdge"; +import { HIGHLIGHT_COLORS } from "./nodeStyle"; + +const BASE_POSITION = { + id: "e1", + source: "a", + target: "b", + sourceX: 0, + sourceY: 0, + targetX: 100, + targetY: 100, + sourcePosition: Position.Bottom, + targetPosition: Position.Top, +}; + +function renderEdge(data: ChannelEdgeData) { + // The non-idle label renders through EdgeLabelRenderer, which reads React + // Flow's store context even outside an actual canvas. + const { container } = render( + + + + + , + ); + return container.querySelector("path")!; +} + +const DATA: ChannelEdgeData = { + messagesSent: 1n, + batchesSent: 1n, + channelTypes: [], + dimmed: false, + selected: false, + connected: false, +}; + +describe("ChannelEdge", () => { + it("plain edge has no highlight stroke", () => { + const path = renderEdge(DATA); + expect(path.style.stroke).toBe(""); + }); + + // Touching the selected node (but not being the clicked thing itself) + // gets a lighter highlight than the directly selected edge does. + it("a connected edge gets the lighter highlight", () => { + const path = renderEdge({ ...DATA, connected: true }); + expect(path.style.stroke).toBe(HIGHLIGHT_COLORS.connected); + expect(path.style.strokeWidth).toBe("1.75"); + }); + + it("the directly selected edge gets the stronger highlight, not the connected one", () => { + const path = renderEdge({ ...DATA, selected: true, connected: true }); + expect(path.style.stroke).toBe(HIGHLIGHT_COLORS.selected); + expect(path.style.strokeWidth).toBe("2.5"); + }); +}); diff --git a/console/src/platform/dataflows/ChannelEdge.tsx b/console/src/platform/dataflows/ChannelEdge.tsx index 78b2e7adbaada..5da7339807077 100644 --- a/console/src/platform/dataflows/ChannelEdge.tsx +++ b/console/src/platform/dataflows/ChannelEdge.tsx @@ -24,6 +24,10 @@ export type ChannelEdgeData = { channelTypes: string[]; dimmed: boolean; selected: boolean; + // Touches the selected node without being the clicked thing itself (an + // edge whose own id is selected gets `selected` instead), so it reads as + // a lighter, secondary emphasis. + connected: boolean; }; const compactCount = Intl.NumberFormat("default", { @@ -35,8 +39,14 @@ const compact = (n: bigint) => compactCount.format(n); export const ChannelEdge = (props: EdgeProps & { data: ChannelEdgeData }) => { const [path, labelX, labelY] = getBezierPath(props); - const { messagesSent, batchesSent, channelTypes, dimmed, selected } = - props.data; + const { + messagesSent, + batchesSent, + channelTypes, + dimmed, + selected, + connected, + } = props.data; const idle = messagesSent === 0n; // Inline label stays terse: compact record/batch counts only. Exact figures // and the full container type names live in the tooltip, since those Rust @@ -59,8 +69,12 @@ export const ChannelEdge = (props: EdgeProps & { data: ChannelEdgeData }) => { style={{ strokeDasharray: idle ? "6 4" : undefined, opacity: dimmed ? 0.15 : 1, - stroke: selected ? HIGHLIGHT_COLORS.selected : undefined, - strokeWidth: selected ? 2.5 : undefined, + stroke: selected + ? HIGHLIGHT_COLORS.selected + : connected + ? HIGHLIGHT_COLORS.connected + : undefined, + strokeWidth: selected ? 2.5 : connected ? 1.75 : undefined, }} /> {label && ( diff --git a/console/src/platform/dataflows/DataflowDetailPage.test.tsx b/console/src/platform/dataflows/DataflowDetailPage.test.tsx index 717e9f3daac0e..b146eba0dd4ea 100644 --- a/console/src/platform/dataflows/DataflowDetailPage.test.tsx +++ b/console/src/platform/dataflows/DataflowDetailPage.test.tsx @@ -11,7 +11,7 @@ import { screen } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import { http, HttpResponse } from "msw"; import React from "react"; -import { Route, Routes } from "react-router-dom"; +import { Route, Routes, useLocation } from "react-router-dom"; import type { Cluster } from "~/api/materialize/cluster/clusterList"; import { ErrorCode } from "~/api/materialize/types"; @@ -24,6 +24,10 @@ import { renderComponent } from "~/test/utils"; import DataflowDetailPage from "./DataflowDetailPage"; import { nodeIdOf } from "./dataflowGraph"; +// Declared via vi.hoisted so the vi.mock factory below (itself hoisted +// above regular imports) can close over it. +const fitViewSpy = vi.hoisted(() => vi.fn()); + // jsdom lacks ResizeObserver/DOMMatrixReadOnly, so replace @xyflow/react with a // flat renderer that preserves the decision under test (whether the canvas and // its nodes exist). This mirrors src/test/mockReactFlow.tsx but also stubs @@ -69,7 +73,8 @@ vi.mock("@xyflow/react", () => ({ useReactFlow: () => ({ getInternalNode: () => undefined, setCenter: () => {}, - fitView: () => {}, + fitView: fitViewSpy, + getViewport: () => ({ x: 0, y: 0, zoom: 1 }), }), })); @@ -184,6 +189,7 @@ const fanOutChannelsResult = { }; beforeEach(() => { + fitViewSpy.mockClear(); const store = getStore(); store.set(allClusters, mockSubscribeState({ data: [cluster] })); server.use( @@ -238,6 +244,13 @@ beforeEach(() => { ); }); +// Exposes the current URL search string so a test can assert the +// drill-down scope actually landed in the URL, not just in the rendered +// graph (which would still look right if it were only in component state). +const LocationProbe = () => ( +
{useLocation().search}
+); + describe("DataflowDetailPage", () => { // CNS-109: switching to a replica whose query fails must surface the error // state and drop the graph from the previously successful replica. @@ -352,6 +365,90 @@ describe("DataflowDetailPage", () => { expect(await screen.findAllByText("input 0")).toHaveLength(2); }); + // The drill-down scope has to actually be in the URL, not just reflected + // in the rendered graph, for a copied link or the back button to work. + it("puts the drill-down scope in the URL, so a link into it reopens there directly", async () => { + const { unmount } = await renderComponent( + + + + + + } + /> + , + { + initialRouterEntries: ["/clusters/u5/test_cluster/dataflows/8"], + }, + ); + + const regionAId = nodeIdOf([8, 1]); + await screen.findByTestId(`node-${regionAId}`); + // A fresh, undrilled link carries no scope param at all. + expect(screen.getByTestId("location-search").textContent).toBe(""); + + await userEvent.dblClick(screen.getByTestId(`node-${regionAId}`)); + await screen.findByTestId(`node-${nodeIdOf([8, 1, 1])}`); + const search = screen.getByTestId("location-search").textContent; + expect(search).toBe("?scope=8.1"); + + unmount(); + + // Reopening at that exact URL, with no clicks, lands drilled in already. + await renderComponent( + + } + /> + , + { + initialRouterEntries: [ + `/clusters/u5/test_cluster/dataflows/8${search}`, + ], + }, + ); + expect( + await screen.findByTestId(`node-${nodeIdOf([8, 1, 1])}`), + ).toBeVisible(); + }); + + // elk lays each scope out from scratch, so a viewport left over from + // wherever the user was in the previous scope can easily miss the new + // one entirely. Navigating into a scope should recenter for that reason, + // but an unrelated re-render within the same scope (e.g. selecting a + // node) must not refit again, or it would fight the user's own panning. + it("recenters after navigating into a scope, but not again within it", async () => { + await renderComponent( + + } + /> + , + { + initialRouterEntries: ["/clusters/u5/test_cluster/dataflows/8"], + }, + ); + + const regionAId = nodeIdOf([8, 1]); + await screen.findByTestId(`node-${regionAId}`); + // The initial scope is skipped: React Flow's own `fitView` prop already + // covers first load. + expect(fitViewSpy).not.toHaveBeenCalled(); + + await userEvent.dblClick(screen.getByTestId(`node-${regionAId}`)); + const leafAId = nodeIdOf([8, 1, 1]); + await screen.findByTestId(`node-${leafAId}`); + expect(fitViewSpy).toHaveBeenCalledTimes(1); + + await userEvent.click(screen.getByTestId(`node-${leafAId}`)); + expect(fitViewSpy).toHaveBeenCalledTimes(1); + }); + // A 1:n port (RegionA's single output feeding both RegionB and RegionC) // can't pick a jump target on its own; double-clicking it should behave // like a plain click (open its own detail panel) rather than guess. diff --git a/console/src/platform/dataflows/DataflowDetailPage.tsx b/console/src/platform/dataflows/DataflowDetailPage.tsx index 3ce08a18d028b..0d0213a4ff6aa 100644 --- a/console/src/platform/dataflows/DataflowDetailPage.tsx +++ b/console/src/platform/dataflows/DataflowDetailPage.tsx @@ -91,18 +91,42 @@ const DataflowDetailPage = () => { ); const { data: dataflowList } = useDataflowList(listParams); - const [rawFocusedScope, setFocusedScope] = React.useState( - null, - ); - // Derived, not synchronized via effect: a scope from a previous structure - // (a stale drill-down target after a refetch or a dataflow/replica switch - // changes the node id set) falls back to the new structure's root rather - // than crashing every direct `nodes.get(focusedScope)!` lookup downstream. + // The drill-down scope lives in the URL (as the dataflow address it + // resolves to, e.g. "5.1.2"), not component state: back/forward then walk + // the drill-down history, and a copied link reopens at the same scope. + // Read fresh every render rather than synchronized via effect, so a scope + // that doesn't resolve in the current structure (a stale link, a dataflow + // whose shape changed, or simply no scope param) falls back to the root + // rather than crashing every direct `nodes.get(focusedScope)!` lookup + // downstream. + const scopeParam = searchParams.get("scope"); + const rawFocusedScope = scopeParam + ? nodeIdOf(scopeParam.split(".").map(Number)) + : null; const focusedScope = data ? rawFocusedScope && data.structure.nodes.has(rawFocusedScope) ? rawFocusedScope : data.structure.root : null; + const setFocusedScope = React.useCallback( + (scope: NodeId) => { + const address = data?.structure.nodes.get(scope)?.address; + if (!address) return; + setSearchParams((prev) => { + const next = new URLSearchParams(prev); + // The root has no scope of its own (it's exactly a link's default), + // so leave the URL as the plain dataflow link instead of carrying a + // redundant, always-implied param. + if (address.length > 1) { + next.set("scope", address.join(".")); + } else { + next.delete("scope"); + } + return next; + }); + }, + [data, setSearchParams], + ); const [selection, setSelection] = React.useState(null); const [filters, setFilters] = React.useState(DEFAULT_FILTERS); const [matchIndex, setMatchIndex] = React.useState(0); @@ -123,15 +147,17 @@ const DataflowDetailPage = () => { // replica in place (via the dropdowns) keeps this component instance, its // selection, filters, and pins alive. Those reference node ids from the // old structure. On the new one they silently miss instead of crashing - // (see focusedScope above), which for a pin means every node reads as - // "not in the highlighted set" and the whole graph dims with no way to - // un-pin. Reset render-phase, not via effect, so there is no render in - // between showing the new graph under the old pins. + // (same tolerance as focusedScope above), which for a pin means every node + // reads as "not in the highlighted set" and the whole graph dims with no + // way to un-pin. focusedScope itself needs no entry here: switching + // dataflow or replica always lands on a fresh URL with no scope param (the + // dropdowns build one from scratch), so it already reads back as the new + // structure's root. Reset render-phase, not via effect, so there is no + // render in between showing the new graph under the old pins. const resetKey = JSON.stringify([replicaName, dataflowId]); const [trackedResetKey, setTrackedResetKey] = React.useState(resetKey); if (resetKey !== trackedResetKey) { setTrackedResetKey(resetKey); - setFocusedScope(null); setSelection(null); setFilters(DEFAULT_FILTERS); setMatchIndex(0); @@ -160,7 +186,7 @@ const DataflowDetailPage = () => { ]; setPendingFitIds(targets); }, - [data], + [data, setFocusedScope], ); const onLirSelect = React.useCallback( @@ -193,7 +219,7 @@ const DataflowDetailPage = () => { ); setSelection(node ? { kind: "node", node } : null); }, - [data], + [data, setFocusedScope], ); // A port's peer lives outside the current view. When the peer is itself a diff --git a/console/src/platform/dataflows/DataflowGraphView.tsx b/console/src/platform/dataflows/DataflowGraphView.tsx index 875549c0d08a5..897f006fcb071 100644 --- a/console/src/platform/dataflows/DataflowGraphView.tsx +++ b/console/src/platform/dataflows/DataflowGraphView.tsx @@ -35,7 +35,7 @@ import { type VisibleGraph, type VisibleNode, } from "./dataflowGraph"; -import { NODE_DIMENSIONS } from "./elkGraph"; +import { NODE_DIMENSIONS, type Positions } from "./elkGraph"; import { OperatorNode, PortNode, RegionNode } from "./nodes"; import { nodeFillColor } from "./nodeStyle"; import { useElkLayout } from "./useElkLayout"; @@ -136,6 +136,54 @@ const CenterHelper = ({ return null; }; +// elk lays each scope out independently, starting near its own origin, so a +// viewport pan/zoom left over from a previous, unrelated scope (or from the +// user having panned away) can land on a spot with nothing from the new +// scope in it at all. Once this scope's layout lands, if none of its nodes +// are actually on screen, refit rather than leaving the canvas looking +// empty. Checked once per scope (skipping the initial one, which React +// Flow's own `fitView` prop already covers), not on every render, so +// panning away deliberately after landing doesn't get fought. +const ViewportGuard = ({ + focusedScope, + nodes, + positions, + paneRef, +}: { + focusedScope: NodeId; + nodes: VisibleNode[]; + positions: Positions | null; + paneRef: React.RefObject; +}) => { + const reactFlow = useReactFlow(); + const checkedScopeRef = React.useRef(focusedScope); + React.useEffect(() => { + if (!positions || nodes.length === 0) return; + if (checkedScopeRef.current === focusedScope) return; + checkedScopeRef.current = focusedScope; + const pane = paneRef.current; + if (!pane) return; + const rect = pane.getBoundingClientRect(); + const { x, y, zoom } = reactFlow.getViewport(); + const visMinX = -x / zoom; + const visMinY = -y / zoom; + const visMaxX = visMinX + rect.width / zoom; + const visMaxY = visMinY + rect.height / zoom; + const anyVisible = nodes.some((n) => { + const p = positions[n.id]; + return ( + p && + p.x < visMaxX && + p.x + p.width > visMinX && + p.y < visMaxY && + p.y + p.height > visMinY + ); + }); + if (!anyVisible) void reactFlow.fitView({ padding: 0.3, duration: 300 }); + }, [focusedScope, nodes, positions, paneRef, reactFlow]); + return null; +}; + export const DataflowGraphView = ({ visible: rawVisible, focusedScope, @@ -264,6 +312,11 @@ export const DataflowGraphView = ({ (decorations?.dimmedNodeIds?.has(e.source) ?? false) || (decorations?.dimmedNodeIds?.has(e.target) ?? false), selected: e.id === selectedId, + // selectedId is a node id when a node (rather than an edge) is + // selected; an edge's source/target are always node ids too, so + // this naturally never matches while an edge itself is selected, + // with no need to separately track which kind selectedId is. + connected: e.source === selectedId || e.target === selectedId, }, })), [visible, decorations?.dimmedNodeIds, selectedId], @@ -277,6 +330,8 @@ export const DataflowGraphView = ({ [visible], ); + const paneRef = React.useRef(null); + if (error) { return ( @@ -288,7 +343,7 @@ export const DataflowGraphView = ({ ); } return ( - + {layouting && ( @@ -348,8 +403,19 @@ export const DataflowGraphView = ({ nodeColor={(node) => (node.data as { color?: string }).color ?? "#ccc" } + // A scope of plain, unarranged operators fills every node white + // (COLORS.noArrangementOperator), and the minimap's default node + // stroke is transparent on a white background, so without an + // explicit stroke that scope's minimap renders entirely blank. + nodeStrokeColor="#9ca3af" /> {centerRef && } + ); diff --git a/console/src/platform/dataflows/nodeStyle.ts b/console/src/platform/dataflows/nodeStyle.ts index 2b6698f020dd1..6839ad6a1927a 100644 --- a/console/src/platform/dataflows/nodeStyle.ts +++ b/console/src/platform/dataflows/nodeStyle.ts @@ -23,6 +23,11 @@ export const COLORS = { export const HIGHLIGHT_COLORS = { selected: "#0093e6", activeMatch: "#fe581d", + // Same hue as `selected`, for an edge that merely touches the selected + // node rather than being the clicked thing itself; ChannelEdge pairs this + // with a thinner stroke than `selected` gets, so the two read as a clear + // emphasis/de-emphasis pair rather than two unrelated colors. + connected: "#8fd2f5", }; export function nodeFillColor(node: VisibleNode): string { From d268cb7674ba64c1b2b45cb9e72d2a4c7b6a3c05 Mon Sep 17 00:00:00 2001 From: Moritz Hoffmann Date: Wed, 8 Jul 2026 22:42:39 +0200 Subject: [PATCH 24/45] console: design doc for LIR grouping boxes Co-Authored-By: Claude Sonnet 5 --- .../20260707_dataflow_lir_grouping_boxes.md | 102 ++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 console/doc/design/20260707_dataflow_lir_grouping_boxes.md diff --git a/console/doc/design/20260707_dataflow_lir_grouping_boxes.md b/console/doc/design/20260707_dataflow_lir_grouping_boxes.md new file mode 100644 index 0000000000000..02ef10ca6191c --- /dev/null +++ b/console/doc/design/20260707_dataflow_lir_grouping_boxes.md @@ -0,0 +1,102 @@ +# Dataflow visualizer: LIR grouping boxes + +- Associated: [CNS-108](https://linear.app/materializeinc/issue/CNS-108), [CNS-109](https://linear.app/materializeinc/issue/CNS-109), builds on `20260706_dataflow_visualizer_rebuild.md` + +## The Problem + +The rebuilt dataflow visualizer (`20260706_dataflow_visualizer_rebuild.md`) shows LIR membership only as a badge, a tooltip on hover, and a side-panel tree. +Regions and LIR spans were deliberately kept as two different hierarchies: a region is a drill-down scope, a LIR span is an unrelated grouping over the same operators. +A user looking at a scope's operator graph cannot see at a glance which operators came from the same LIR node, which is exactly the grouping `EXPLAIN` output already draws (dashed boxes, one per LIR node, nested when LIR spans nest). +Matching that visual on the canvas would let a user correlate the rendered graph directly with the plan they already read. + +## Success Criteria + +* A user can toggle "Show LIR groups" in the toolbar and see, for the current scope, a dashed box around each LIR node's member operators, nested to match LIR span nesting. +* A box shows the LIR operator string as a header label. +* Clicking a box's header opens the same detail info the LIR side panel already shows for that LIR node (id, operator, member count, aggregated stats). +* Toggling off returns to exactly today's flat layout, byte-identical to the current behavior. +* Search, dimming, and heatmap decoration keep working unchanged on individual operators inside a box. + +## Out of Scope + +* Chains: gluing linear (in-degree 1, out-degree 1) operator runs into one visually merged box, a separate idea explored and shelved during this design's brainstorming as added complexity without a clean elk-native way to render it. +* Any change to how LIR data is queried or computed (`useDataflowGraphData.ts`, `mz_lir_mapping`) — grouping is a pure derivation over data already fetched. +* Heat-coloring or aggregate stats displayed directly on a group box; a box is a label-only wrapper, its members carry all stats as they do today. +* Drill-down or double-click behavior on a group box. LIR groups are not scopes and do not navigate. + +## Solution Proposal + +Add a pure grouping step between the existing decoration pass and the elk layout step, and teach the elk/React Flow rendering path to lay out and draw nested compound nodes, which it does not do today. + +### Data model + +A dataflow's direct children in a scope are already ordered by operator id, and each carries `lir: LirInfo[]`, an outer-to-inner nesting stack. +LIR spans are contiguous ranges over operator ids and properly nested: a region is always fully enclosed by a single LIR node's range, and a single region can contain operators from multiple LIR nodes, but always consecutive by operator id. +A new pure function in `dataflowGraph.ts`, `groupByLir(nodes: VisibleNode[]): LirGrouping`, exploits this with a single left-to-right scan and an explicit stack of open groups: for each node, pop groups whose `lirId` is no longer a prefix of the node's stack, push newly opened ones. +This is a bracket-matching walk, valid because of the contiguity and proper-nesting invariant above; it does not need to handle malformed input, since the invariant is guaranteed by how LIR spans are constructed. + +Rather than replacing the flat node list with a nested tree, grouping is a separate layer over it, the same way `decorateGraph` layers dimming and color over an unchanged `VisibleGraph`: every downstream consumer (search, dimming, click handling) keeps working against the flat list untouched, and only the layout/render step reads the grouping layer. +Some children stay ungrouped (empty `lir`, e.g. ports, which carry no LIR data and are never absorbed into a group); every other child, and every group itself, has an immediate parent recorded in `parentOf`: + +```typescript +interface LirGroupNode { + id: NodeId; // `${exportId}/${lirId}`, matching lirIndex's existing key + exportId: string; + lirId: string; + operator: string; + nesting: number; +} + +interface LirGrouping { + groups: LirGroupNode[]; // outer groups before the inner groups nested in them + parentOf: Map; // a VisibleNode or LirGroupNode id -> its immediate enclosing group's id +} +``` + +A dataflow can back several exports, and an operator's `lir: LirInfo[]` can hold entries from more than one of them at once, when a shared subplan feeds multiple exports. +A compound-node tree needs a single parent chain per node, so `groupByLir` groups by one export only: the lowest `exportId` (string-compared) present among the scope's direct children. +A child whose only `lir` entries belong to a different export renders ungrouped in that scope, alongside children with no LIR data at all. +This can under-group a multi-export scope, but never draws two overlapping box hierarchies over the same operators. + +### Rendering and layout + +`elkGraph.ts`'s `toElkGraph` is flat-only today, by its own comment: a view is always one scope's direct children, nothing nests. +LIR groups are the first use of elk's native compound-node support in this codebase. +`toElkGraph` grows a case for `LirGroupNode`: it becomes an elk parent node with its members as elk children (recursing for nested groups), no fixed width or height, since elk auto-sizes a parent from its children plus `elk.padding`, reserving a header strip for the label. +Ungrouped siblings and group roots sit together as top-level elk children, exactly like today's flat list when there are no groups. +Edges can still cross group boundaries (an edge's endpoints are unrelated to which LIR group either side belongs to), so the root layout options need `elk.hierarchyHandling: INCLUDE_CHILDREN`, harmless when there are no groups at all. + +React Flow's `parentId` mechanism (nested parent nodes, already used as one of the reasons the project chose React Flow) maps onto this directly, and needs no coordinate conversion: elk reports a child's `x`/`y` relative to its immediate parent's origin, the same convention React Flow uses for a node with `parentId` set. +A new `LirGroupNode` component renders the dashed border and a small header label (from `LirInfo.operator`, truncated with ellipsis like the existing LIR panel), colored by `lirId`, sized from elk's computed group bounds. +It carries no stats and is unaffected by heatmap coloring. + +Pipeline order: `deriveVisibleGraph` → `decorateGraph` → (if the toggle is on) `groupByLir` → `toElkGraph` → layout → render. +Grouping is the last, purely structural step over already-decorated nodes, so search highlighting, dimming, and heatmap coloring apply to members exactly as they do without groups; a group box just visually contains whichever of its members are dimmed or highlighted. +With the toggle off, `deriveVisibleGraph`'s output goes straight to `toElkGraph`, unrouted through `groupByLir`, identical to today's behavior. + +### Interactions + +Clicking a group's header selects that LIR node and opens `NodeDetailPanel` with the same info a `LirPanel` tree entry already produces (`lirIndex`/`lirSummary`): id, operator string, member count, aggregated stats. +No double-click or drill-down action on a group; LIR groups are not scopes. + +### Toolbar + +A new "Show LIR groups" checkbox in `DataflowToolbar`, off by default, alongside the existing search/hide-idle/heatmap controls. +Off, the view is exactly today's flat grid. + +## Minimal Viable Prototype + +The risk worth de-risking before full build is elk's compound-node layout quality and cost on a real, multi-level LIR-nested dataflow, since this codebase has never exercised that path. +A thin slice: run `groupByLir` and the elk-nesting change against one scope of the reference dataflow from the original design doc (or another with visibly nested LIR spans), confirm cross-group edge routing looks reasonable and layout time stays acceptable, before building out the toolbar toggle, `LirGroupNode` styling, and the click-to-detail wiring. + +## Alternatives + +* Render-layer-only grouping: keep elk flat, compute each group's bounding box in the render layer from its members' already-laid-out positions, and draw an absolutely-positioned dashed `` behind them without going through elk's compound-node support or React Flow's `parentId`. + Avoids touching `elkGraph.ts`, but duplicates layout knowledge elk already has (padding, header reservation, nested spacing) in a second place, and elk has no reason to leave room for a label strip or extra padding around a group it doesn't know exists, so boxes would collide with sibling groups or overlap edges routed through the gap. +* Always-on, no toggle. + Simpler UI, but permanently changes the default look of every scope that has LIR data and adds elk layout cost (nested compound-node layout is slower than flat) to every view, including ones where the operator/LIR correlation isn't what the user is looking for. + +## Open questions + +* None blocking. + Chains (gluing linear operator runs) were explored in the same brainstorming session and shelved; if revisited, see the discussion of why render-layer-only glue was rejected there, which mirrors the render-layer alternative rejected above. From 2389f9949d7b45a1025d56b08d68c1723b713809 Mon Sep 17 00:00:00 2001 From: Moritz Hoffmann Date: Wed, 8 Jul 2026 22:42:39 +0200 Subject: [PATCH 25/45] console: implementation plan for LIR grouping boxes Co-Authored-By: Claude Sonnet 5 --- .../2026-07-07-dataflow-lir-grouping-boxes.md | 1292 +++++++++++++++++ 1 file changed, 1292 insertions(+) create mode 100644 docs/superpowers/plans/2026-07-07-dataflow-lir-grouping-boxes.md diff --git a/docs/superpowers/plans/2026-07-07-dataflow-lir-grouping-boxes.md b/docs/superpowers/plans/2026-07-07-dataflow-lir-grouping-boxes.md new file mode 100644 index 0000000000000..ead92ad0e7389 --- /dev/null +++ b/docs/superpowers/plans/2026-07-07-dataflow-lir-grouping-boxes.md @@ -0,0 +1,1292 @@ +# Dataflow LIR Grouping Boxes Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Draw nested dashed boxes around a scope's operators that share a LIR node, toggled from the dataflow visualizer toolbar, matching `EXPLAIN`'s own LIR grouping. + +**Architecture:** A new pure function `groupByLir` derives a grouping layer (which group each `VisibleNode` sits in, if any) over the already-flat `VisibleGraph`, the same way `decorateGraph` layers dimming/color over it without replacing it. `elkGraph.ts` turns that layer into elk's native compound-node nesting (the first use of it in this codebase); `DataflowGraphView.tsx` turns elk's result into React Flow `parentId` nesting and a new `LirGroupNode` render component. A toolbar switch and a `NodeDetailPanel` case round out the feature. + +**Tech Stack:** TypeScript, React, `@xyflow/react` (React Flow), `elkjs`, Chakra UI, Vitest. + +## Global Constraints + +* Design doc: `console/doc/design/20260707_dataflow_lir_grouping_boxes.md`. Every task below implements a specific section of it; read it first if anything here is ambiguous. +* Toggle defaults to off (`Filters.showLirGroups: false`); with it off, output must be byte-identical to today's flat layout. +* A node whose `lir` entries all belong to an export other than the scope's chosen export (lowest `exportId` present) renders ungrouped, never in two overlapping group hierarchies. +* No drill-down/double-click behavior on a group box; groups are label-only wrappers, never scopes. +* Run `cd console && yarn typecheck && yarn lint && yarn test ` before every commit in this plan (see `mz-run`/`mz-test` conventions); this plan's own commands below are the minimum, not a substitute. +* Copy comments verbatim where this plan quotes them from the design doc or existing code; don't invent new phrasing for an already-settled explanation. + +--- + +### Task 1: `groupByLir` pure function + +**Files:** +- Modify: `console/src/platform/dataflows/dataflowGraph.ts` +- Test: `console/src/platform/dataflows/dataflowGraph.test.ts` + +**Interfaces:** +- Consumes: `VisibleNode` (already defined in this file: `id`, `operatorId: bigint | null`, `lir: LirInfo[]`). +- Produces: + ```typescript + export interface LirGroupNode { + id: NodeId; // `${exportId}/${lirId}`, matching lirIndex's existing key + exportId: string; + lirId: string; + operator: string; + nesting: number; + } + export interface LirGrouping { + groups: LirGroupNode[]; // outer groups appear before groups nested inside them + parentOf: Map; // a VisibleNode or LirGroupNode id -> its immediate enclosing group's id + } + export function groupByLir(nodes: readonly VisibleNode[]): LirGrouping + ``` + Later tasks import `groupByLir`, `LirGrouping`, `LirGroupNode` from this file. + +- [ ] **Step 1: Write the failing tests** + +Add to `console/src/platform/dataflows/dataflowGraph.test.ts`, after the existing `import` block add `type LirInfo` and `groupByLir` to the named imports from `./dataflowGraph`, then append this new `describe` block at the end of the file: + +```typescript +describe("groupByLir", () => { + const node = ( + id: string, + operatorId: number, + lir: LirInfo[] = [], + ): VisibleNode => ({ + id, + kind: "operator", + label: id, + stats: null, + transitive: null, + transitiveSkew: null, + childCount: 0, + lir, + address: null, + operatorId: BigInt(operatorId), + peers: [], + }); + const port = (id: string): VisibleNode => ({ + id, + kind: "port", + label: id, + stats: null, + transitive: null, + transitiveSkew: null, + childCount: 0, + lir: [], + address: null, + operatorId: null, + peers: [], + }); + const span = ( + exportId: string, + lirId: string, + nesting: number, + parentLirId: string | null, + ): LirInfo => ({ exportId, lirId, parentLirId, nesting, operator: lirId }); + + it("groups nothing when no node has lir data", () => { + const g = groupByLir([node("a", 1), node("b", 2)]); + expect(g.groups).toEqual([]); + expect(g.parentOf.size).toEqual(0); + }); + + it("groups a flat run under one lir id", () => { + const s = span("u1", "10", 0, null); + const g = groupByLir([node("a", 1, [s]), node("b", 2, [s]), node("c", 3, [s])]); + expect(g.groups.map((x) => x.id)).toEqual(["u1/10"]); + expect(g.parentOf.get("a")).toEqual("u1/10"); + expect(g.parentOf.get("b")).toEqual("u1/10"); + expect(g.parentOf.get("c")).toEqual("u1/10"); + }); + + it("nests an inner span inside an outer span", () => { + const outer = span("u1", "10", 0, null); + const inner = span("u1", "11", 1, "10"); + const g = groupByLir([ + node("a", 1, [outer]), + node("b", 2, [outer, inner]), + node("c", 3, [outer, inner]), + node("d", 4, [outer]), + ]); + expect(g.groups.map((x) => x.id)).toEqual(["u1/10", "u1/11"]); + expect(g.parentOf.get("u1/11")).toEqual("u1/10"); + expect(g.parentOf.get("a")).toEqual("u1/10"); + expect(g.parentOf.get("b")).toEqual("u1/11"); + expect(g.parentOf.get("c")).toEqual("u1/11"); + expect(g.parentOf.get("d")).toEqual("u1/10"); + }); + + it("leaves ports and lir-less nodes ungrouped alongside a group", () => { + const s = span("u1", "10", 0, null); + const g = groupByLir([node("a", 1, [s]), port("p"), node("b", 2)]); + expect(g.groups.map((x) => x.id)).toEqual(["u1/10"]); + expect(g.parentOf.get("a")).toEqual("u1/10"); + expect(g.parentOf.has("p")).toEqual(false); + expect(g.parentOf.has("b")).toEqual(false); + }); + + it("picks the lowest exportId, leaving other-export-only nodes ungrouped", () => { + const sA = span("u1", "5", 0, null); + const sB = span("u2", "3", 0, null); + const g = groupByLir([node("a", 1, [sA]), node("b", 2, [sB]), node("c", 3, [sA, sB])]); + expect(g.groups.map((x) => x.id)).toEqual(["u1/5"]); + expect(g.parentOf.get("a")).toEqual("u1/5"); + expect(g.parentOf.has("b")).toEqual(false); + expect(g.parentOf.get("c")).toEqual("u1/5"); + }); + + it("sorts input by operatorId regardless of array order", () => { + const outer = span("u1", "10", 0, null); + const inner = span("u1", "11", 1, "10"); + // Deliberately out of operatorId order. + const g = groupByLir([ + node("d", 4, [outer]), + node("b", 2, [outer, inner]), + node("a", 1, [outer]), + node("c", 3, [outer, inner]), + ]); + expect(g.groups.map((x) => x.id)).toEqual(["u1/10", "u1/11"]); + expect(g.parentOf.get("b")).toEqual("u1/11"); + expect(g.parentOf.get("c")).toEqual("u1/11"); + }); +}); +``` + +- [ ] **Step 2: Run the tests to verify they fail** + +Run: `cd console && yarn vitest run src/platform/dataflows/dataflowGraph.test.ts` +Expected: FAIL with `groupByLir is not exported` / `groupByLir is not a function`. + +- [ ] **Step 3: Implement `groupByLir`** + +Add to `console/src/platform/dataflows/dataflowGraph.ts`, after `lirTree` (end of file): + +```typescript +export interface LirGroupNode { + id: NodeId; // `${exportId}/${lirId}`, matching lirIndex's key + exportId: string; + lirId: string; + operator: string; + nesting: number; +} + +// A grouping layer over an already-derived VisibleGraph, the same way +// decorateGraph layers dimming and color over it: nothing here replaces +// `nodes`, so search, dimming, and click handling keep working against the +// flat list untouched. +export interface LirGrouping { + groups: LirGroupNode[]; // outer groups appear before groups nested inside them + parentOf: Map; // a VisibleNode or LirGroupNode id -> its immediate enclosing group's id +} + +// Groups a scope's direct children into the LIR tree that encloses them, for +// drawing nested boxes on the canvas. LIR spans are contiguous ranges over +// operator ids and properly nested: a region is always fully enclosed by a +// single LIR node's range. A dataflow can back several exports, and a shared +// subplan can carry lir entries from more than one of them at once; a +// compound-node tree needs one parent chain per node, so this groups by the +// lowest exportId (string-compared) present among the given nodes only. A +// node whose only lir entries belong to a different export renders +// ungrouped, alongside nodes with no lir data at all (e.g. ports). +export function groupByLir(nodes: readonly VisibleNode[]): LirGrouping { + const groups: LirGroupNode[] = []; + const parentOf = new Map(); + const ordered = nodes + .filter((n) => n.operatorId !== null) + .slice() + .sort((a, b) => { + const x = a.operatorId!; + const y = b.operatorId!; + return x < y ? -1 : x > y ? 1 : 0; + }); + if (ordered.length === 0) return { groups, parentOf }; + + let chosenExport: string | null = null; + for (const n of ordered) { + for (const l of n.lir) { + if (chosenExport === null || l.exportId < chosenExport) { + chosenExport = l.exportId; + } + } + } + if (chosenExport === null) return { groups, parentOf }; + + const groupIndex = new Map(); + const stack: LirGroupNode[] = []; // open groups, outer to inner + for (const node of ordered) { + const stackKeys = node.lir + .filter((l) => l.exportId === chosenExport) + .sort((a, b) => a.nesting - b.nesting) + .map((l) => `${l.exportId}/${l.lirId}`); + let commonLen = 0; + while ( + commonLen < stack.length && + commonLen < stackKeys.length && + stack[commonLen].id === stackKeys[commonLen] + ) { + commonLen++; + } + stack.length = commonLen; // pop back to the shared prefix + for (let depth = commonLen; depth < stackKeys.length; depth++) { + const key = stackKeys[depth]; + let group = groupIndex.get(key); + if (!group) { + const info = node.lir.find((l) => `${l.exportId}/${l.lirId}` === key)!; + group = { + id: key, + exportId: info.exportId, + lirId: info.lirId, + operator: info.operator, + nesting: info.nesting, + }; + groupIndex.set(key, group); + groups.push(group); + } + if (depth > 0) parentOf.set(group.id, stack[depth - 1].id); + stack.push(group); + } + if (stack.length > 0) parentOf.set(node.id, stack[stack.length - 1].id); + } + return { groups, parentOf }; +} +``` + +- [ ] **Step 4: Run the tests to verify they pass** + +Run: `cd console && yarn vitest run src/platform/dataflows/dataflowGraph.test.ts` +Expected: PASS, all tests including the six new `groupByLir` cases. + +- [ ] **Step 5: Typecheck and lint** + +Run: `cd console && yarn typecheck && yarn lint` +Expected: no errors. + +- [ ] **Step 6: Commit** + +```bash +git add console/src/platform/dataflows/dataflowGraph.ts console/src/platform/dataflows/dataflowGraph.test.ts +git commit -m "$(cat <<'EOF' +console: add groupByLir for LIR-node grouping boxes + +Pure derivation over a scope's already-derived VisibleGraph, mirroring +decorateGraph's layering rather than replacing the flat node list, so +downstream search/dimming/click handling is untouched. +EOF +)" +``` + +--- + +### Task 2: elk nested compound-node layout + +**Files:** +- Modify: `console/src/platform/dataflows/elkGraph.ts` +- Modify: `console/src/platform/dataflows/useElkLayout.ts` +- Test: `console/src/platform/dataflows/elkGraph.test.ts` + +**Interfaces:** +- Consumes: `LirGrouping`, `LirGroupNode` from Task 1's `dataflowGraph.ts`. +- Produces: `toElkGraph(graph: VisibleGraph, grouping?: LirGrouping): ElkNode` (grouping now optional third-party consumers can omit), `extractPositions` unchanged in signature but now recurses into nested `children`. `useElkLayout(graph, cacheKey, grouping?)` gains an optional third parameter. + +- [ ] **Step 1: Write the failing tests** + +Add to `console/src/platform/dataflows/elkGraph.test.ts`. First, add `groupByLir` to the import from `./dataflowGraph`, and add this import at the top (a real, synchronous elk instance — not the worker-backed `elk-api` used at runtime — for the empirical round-trip test): + +```typescript +import ELK from "elkjs"; +``` + +Then append: + +```typescript +describe("toElkGraph with a LIR grouping", () => { + const s = buildDataflowStructure(OPS, CHANNELS, LIR_SPANS); + + it("nests a group's members as elk children, sized only by content", () => { + const visible = deriveVisibleGraph(s, s.root); + const grouping = groupByLir(visible.nodes); + const elk = toElkGraph(visible, grouping); + const group = elk.children!.find((c) => c.id === grouping.groups[0].id)!; + expect(group.children!.map((c) => c.id)).toEqual( + expect.arrayContaining([nodeIdOf([5, 1])]), + ); + // Auto-sized by elk from its content, not one of NODE_DIMENSIONS's fixed sizes. + expect(group.width).toBeUndefined(); + expect(group.height).toBeUndefined(); + // Flat sibling stays a direct child of the root, not absorbed into the group. + expect(elk.children!.map((c) => c.id)).toEqual( + expect.arrayContaining([nodeIdOf([5, 2])]), + ); + }); + + it("keeps every edge at the root level regardless of nesting depth", () => { + const visible = deriveVisibleGraph(s, s.root); + const grouping = groupByLir(visible.nodes); + const elk = toElkGraph(visible, grouping); + expect(elk.edges!.map((e) => e.id)).toEqual(visible.edges.map((e) => e.id)); + for (const child of elk.children!) expect(child.edges).toBeUndefined(); + }); + + it("round-trips nested positions unchanged (parent-relative, no conversion)", () => { + const visible = deriveVisibleGraph(s, s.root); + const grouping = groupByLir(visible.nodes); + const elk = toElkGraph(visible, grouping); + const group = elk.children!.find((c) => c.id === grouping.groups[0].id)!; + group.x = 12; + group.y = 87; + group.width = 132; + group.height = 259; + const member = group.children!.find((c) => c.id === nodeIdOf([5, 1]))!; + member.x = 16; + member.y = 24; + const positions = extractPositions(elk); + // The member's extracted position is exactly its own x/y, not offset by + // the group's: elk already reports it relative to the group. + expect(positions[member.id]).toMatchObject({ x: 16, y: 24 }); + expect(positions[group.id]).toMatchObject({ x: 12, y: 87, width: 132, height: 259 }); + }); + + it("real elk lays out nested groups with auto-sized bounds and cross-boundary edges", async () => { + // Empirical check of the assumption the design doc relies on: elk + // auto-sizes a compound node from its children, reports child positions + // relative to their own parent (matching React Flow's parentId + // convention with no coordinate conversion needed), and relocates edges + // that cross group boundaries to the right container even when every + // edge is declared at the root, given hierarchyHandling: INCLUDE_CHILDREN. + const elk = new ELK(); + const graph = { + id: "root", + layoutOptions: { + "elk.algorithm": "layered", + "elk.direction": "DOWN", + "elk.hierarchyHandling": "INCLUDE_CHILDREN", + }, + children: [ + { id: "ungrouped", width: 100, height: 50 }, + { + id: "groupA", + layoutOptions: { "elk.padding": "[top=24,left=8,bottom=8,right=8]" }, + children: [ + { id: "a1", width: 100, height: 50 }, + { + id: "groupA_inner", + layoutOptions: { "elk.padding": "[top=24,left=8,bottom=8,right=8]" }, + children: [ + { id: "a2", width: 100, height: 50 }, + { id: "a3", width: 100, height: 50 }, + ], + }, + ], + }, + ], + edges: [ + { id: "e1", sources: ["ungrouped"], targets: ["a1"] }, + { id: "e2", sources: ["a1"], targets: ["a2"] }, + { id: "e3", sources: ["a2"], targets: ["a3"] }, + ], + }; + const result = await elk.layout(graph); + const groupA = result.children!.find((c) => c.id === "groupA")!; + const groupAInner = groupA.children!.find((c) => c.id === "groupA_inner")!; + // Auto-sized larger than any single member: real content-driven sizing. + expect(groupA.width!).toBeGreaterThan(100); + expect(groupA.height!).toBeGreaterThan(50); + // e3 (both endpoints inside groupA_inner) is relocated there even though + // it was declared at the root. + expect((result.edges ?? []).some((e) => e.id === "e3")).toBe(false); + const findEdge = (node: typeof groupAInner, id: string) => + (node.edges ?? []).some((e) => e.id === id); + expect(findEdge(groupAInner, "e3")).toBe(true); + }); +}); +``` + +- [ ] **Step 2: Run the tests to verify they fail** + +Run: `cd console && yarn vitest run src/platform/dataflows/elkGraph.test.ts` +Expected: FAIL — `toElkGraph` doesn't accept a second argument yet, and the flat-only implementation never nests anything (`group.children` is `undefined`, and the "real elk" test itself should already pass in isolation since it doesn't touch this codebase's functions — confirm it passes standalone before moving on, since it is the risk-gate check, not a test of code you're about to write). + +- [ ] **Step 3: Implement nested `toElkGraph` and recursive `extractPositions`** + +Replace the full contents of `console/src/platform/dataflows/elkGraph.ts` from the `LAYOUT_OPTIONS` constant to the end of the file with: + +```typescript +const LAYOUT_OPTIONS: Record = { + "elk.algorithm": "layered", + "elk.direction": "DOWN", + "elk.padding": "[top=48,left=16,bottom=16,right=16]", + "elk.spacing.nodeNode": "24", + "elk.layered.spacing.nodeNodeBetweenLayers": "48", + // Lets an edge crossing a LIR group's boundary be declared uniformly at + // the root: elk relocates it to the correct container itself. Harmless + // when there's no grouping at all (today's flat views). + "elk.hierarchyHandling": "INCLUDE_CHILDREN", +}; + +// A LIR group is a label-only wrapper, sized purely from its content (no +// fixed width/height), with top padding reserved for the header strip +// LirGroupNode renders. +const GROUP_LAYOUT_OPTIONS: Record = { + "elk.algorithm": "layered", + "elk.direction": "DOWN", + "elk.padding": "[top=24,left=8,bottom=8,right=8]", + "elk.spacing.nodeNode": "24", + "elk.layered.spacing.nodeNodeBetweenLayers": "48", +}; + +// A view is always one scope's direct children, so without a grouping this +// is flat: nothing is nested inside anything else. With a grouping, a +// group's members (VisibleNodes or other groups) become its elk children; +// everything else stays a direct child of the root, exactly like today. +export function toElkGraph(graph: VisibleGraph, grouping?: LirGrouping): ElkNode { + const nodeById = new Map(graph.nodes.map((n) => [n.id, n])); + const groupById = new Map((grouping?.groups ?? []).map((g) => [g.id, g])); + const childrenOf = new Map(); // parent id ("" = root) -> ordered child ids + const addChild = (parentKey: string, id: string) => { + const list = childrenOf.get(parentKey); + if (list) list.push(id); + else childrenOf.set(parentKey, [id]); + }; + for (const n of graph.nodes) addChild(grouping?.parentOf.get(n.id) ?? "", n.id); + for (const g of groupById.values()) addChild(grouping?.parentOf.get(g.id) ?? "", g.id); + + const buildNode = (id: string): ElkNode => { + const group = groupById.get(id); + if (group) { + return { + id, + layoutOptions: GROUP_LAYOUT_OPTIONS, + children: (childrenOf.get(id) ?? []).map(buildNode), + }; + } + return { id, ...NODE_DIMENSIONS[nodeById.get(id)!.kind] }; + }; + + return { + id: "__root__", + layoutOptions: LAYOUT_OPTIONS, + children: (childrenOf.get("") ?? []).map(buildNode), + edges: graph.edges.map((e) => ({ + id: e.id, + sources: [e.source], + targets: [e.target], + })), + }; +} + +// Recurses into nested children: elk reports every node's x/y already +// relative to its own immediate parent, the same convention React Flow uses +// for a node with parentId set, so no coordinate conversion happens here — +// this only flattens the tree into one lookup map, keyed by id at any depth. +export function extractPositions(layouted: ElkNode): Positions { + const positions: Positions = {}; + const visit = (node: ElkNode) => { + for (const child of node.children ?? []) { + positions[child.id] = { + x: child.x ?? 0, + y: child.y ?? 0, + width: child.width ?? 0, + height: child.height ?? 0, + }; + visit(child); + } + }; + visit(layouted); + return positions; +} +``` + +Add `LirGrouping` to the existing `import type { VisibleGraph, VisibleNode } from "./dataflowGraph";` line at the top of the file (change to `import type { LirGrouping, VisibleGraph, VisibleNode } from "./dataflowGraph";`). + +- [ ] **Step 4: Run the tests to verify they pass** + +Run: `cd console && yarn vitest run src/platform/dataflows/elkGraph.test.ts` +Expected: PASS, all tests including the two pre-existing flat-graph tests (unaffected) and the four new ones. + +- [ ] **Step 5: Thread `grouping` through `useElkLayout`** + +In `console/src/platform/dataflows/useElkLayout.ts`: + +Change the import line to: +```typescript +import type { LirGrouping, VisibleGraph } from "./dataflowGraph"; +``` + +Change the function signature: +```typescript +export function useElkLayout( + graph: VisibleGraph | null, + cacheKey: string, + grouping?: LirGrouping, +) { +``` + +Change the layout call inside the second effect: +```typescript + elk.layout(toElkGraph(graph, grouping)).then( +``` + +Add `grouping` to that effect's dependency array: +```typescript + }, [graph, cacheKey, retryNonce, grouping]); +``` + +- [ ] **Step 6: Typecheck and lint** + +Run: `cd console && yarn typecheck && yarn lint` +Expected: no errors. (`useElkLayout.ts` has no dedicated test file today; its behavior is exercised through `DataflowGraphView`/`DataflowDetailPage` tests in Task 4.) + +- [ ] **Step 7: Commit** + +```bash +git add console/src/platform/dataflows/elkGraph.ts console/src/platform/dataflows/elkGraph.test.ts console/src/platform/dataflows/useElkLayout.ts +git commit -m "$(cat <<'EOF' +console: nest LIR groups as elk compound nodes + +First use of elk's native compound-node layout in this codebase. +Verified against the real elk engine (not the worker-backed elk-api +used at runtime): groups auto-size from content, child positions come +back already relative to their own parent, and hierarchyHandling: +INCLUDE_CHILDREN relocates a boundary-crossing edge to its correct +container even when every edge is declared uniformly at the root. +EOF +)" +``` + +--- + +### Task 3: `LirGroupNode` render component and color helper + +**Files:** +- Modify: `console/src/platform/dataflows/nodeStyle.ts` +- Modify: `console/src/platform/dataflows/nodes.tsx` +- Test: `console/src/platform/dataflows/nodeStyle.test.ts` + +**Interfaces:** +- Consumes: nothing new from earlier tasks (this is a leaf UI unit; Task 4 wires it in). +- Produces: `lirGroupColor(lirId: string): string`, `type FlowGroupData = { group: LirGroupNode; label: string; color: string }`, `LirGroupNode` (the React Flow component, named the same as the data-layer type from Task 1 — imported under an alias wherever both are needed, see Task 4). + +- [ ] **Step 1: Write the failing test** + +Add to `console/src/platform/dataflows/nodeStyle.test.ts`: + +```typescript +import { lirGroupColor } from "./nodeStyle"; + +describe("lirGroupColor", () => { + it("is deterministic for the same lirId", () => { + expect(lirGroupColor("42")).toEqual(lirGroupColor("42")); + }); + + it("differs for different lirIds often enough to be useful", () => { + const colors = new Set(["1", "2", "3", "4", "5", "6"].map(lirGroupColor)); + expect(colors.size).toBeGreaterThan(1); + }); +}); +``` + +- [ ] **Step 2: Run the test to verify it fails** + +Run: `cd console && yarn vitest run src/platform/dataflows/nodeStyle.test.ts` +Expected: FAIL with `lirGroupColor is not exported`. + +- [ ] **Step 3: Implement `lirGroupColor` and `FlowGroupData`** + +Add to `console/src/platform/dataflows/nodeStyle.ts`, after the `HIGHLIGHT_COLORS` constant: + +```typescript +// A small, visually distinct palette for LIR group borders/headers, cycled +// by a deterministic hash so the same lirId always gets the same color +// within one render (and across re-renders, since lirId is stable). +const GROUP_PALETTE = [ + "#e8590c", + "#5f3dc4", + "#0b7285", + "#2f9e44", + "#c2255c", + "#1971c2", + "#e67700", + "#7048e8", +]; + +export function lirGroupColor(lirId: string): string { + let hash = 0; + for (let i = 0; i < lirId.length; i++) { + hash = (hash * 31 + lirId.charCodeAt(i)) | 0; + } + return GROUP_PALETTE[Math.abs(hash) % GROUP_PALETTE.length]; +} +``` + +Add to the bottom of `console/src/platform/dataflows/nodeStyle.ts`, and add `LirGroupNode` to its existing `import type { VisibleNode } from "./dataflowGraph";` line (making it `import type { LirGroupNode, VisibleNode } from "./dataflowGraph";`): + +```typescript +export type FlowGroupData = { + group: LirGroupNode; + label: string; + color: string; +}; +``` + +- [ ] **Step 4: Run the test to verify it passes** + +Run: `cd console && yarn vitest run src/platform/dataflows/nodeStyle.test.ts` +Expected: PASS. + +- [ ] **Step 5: Implement the `LirGroupNode` component** + +Add to `console/src/platform/dataflows/nodes.tsx`, after the existing `PortNode` component, and add `FlowGroupData` to the existing `import { ... } from "./nodeStyle";` line: + +```typescript +// A label-only wrapper around its members, never itself a click target +// except its header (the body is pointerEvents="none" so clicks pass +// through to whatever member is underneath). Its width/height come from +// elk's auto-sized bounds, same as every other node in this file. +export const LirGroupNode = ({ + data, +}: NodeProps & { data: FlowGroupData }) => ( + + + + + {data.label} + + + +); +``` + +- [ ] **Step 6: Typecheck and lint** + +Run: `cd console && yarn typecheck && yarn lint` +Expected: no errors. (No isolated component test for this one, matching `OperatorNode`/`RegionNode`/`PortNode`, none of which have dedicated test files either; Task 4's integration test covers its rendering.) + +- [ ] **Step 7: Commit** + +```bash +git add console/src/platform/dataflows/nodeStyle.ts console/src/platform/dataflows/nodeStyle.test.ts console/src/platform/dataflows/nodes.tsx +git commit -m "console: add LirGroupNode component and lirGroupColor helper" +``` + +--- + +### Task 4: Wire grouping into `DataflowGraphView` + +**Files:** +- Modify: `console/src/platform/dataflows/DataflowGraphView.tsx` +- Modify: `console/src/platform/dataflows/DataflowDetailPage.test.tsx` (fix the inline React Flow mock) + +**Interfaces:** +- Consumes: `groupByLir`, `LirGrouping`, `LirGroupNode` (data type, Task 1); `LirGroupNode` (component, Task 3, imported as `LirGroupNodeComponent` to avoid the name clash); `lirGroupColor` (Task 3); `useElkLayout`'s new third parameter (Task 2). +- Produces: two new `DataflowGraphViewProps`: `showLirGroups?: boolean` and `onLirGroupClick?: (group: LirGroupNode) => void`, consumed by `DataflowDetailPage.tsx` in Tasks 5 and 6. + +- [ ] **Step 1: Fix the test mock's node-rendering assumption** + +The mock `ReactFlow` in `DataflowDetailPage.test.tsx` renders each node's label by reading `n.data.node.label`, which every operator/region/port node has but a group node (whose `data` is `{ group, label, color }`, no nested `.node`) does not. Before adding any group node to what gets rendered, fix the mock to handle both shapes. + +In `console/src/platform/dataflows/DataflowDetailPage.test.tsx`, change the `vi.mock("@xyflow/react", ...)` block's `ReactFlow` implementation: + +```typescript + ReactFlow: ({ + nodes, + children, + onNodeClick, + onNodeDoubleClick, + }: { + nodes: { + id: string; + type?: string; + data: { node?: { label: string }; label?: string }; + }[]; + children?: React.ReactNode; + onNodeClick?: (e: unknown, node: unknown) => void; + onNodeDoubleClick?: (e: unknown, node: unknown) => void; + }) => ( +
+ {nodes.map((n) => ( +
onNodeClick?.(null, n)} + onDoubleClick={() => onNodeDoubleClick?.(null, n)} + > + {n.data.node ? n.data.node.label : n.data.label} +
+ ))} + {children} +
+ ), +``` + +- [ ] **Step 2: Run the full existing suite to confirm the mock fix alone changes nothing** + +Run: `cd console && yarn vitest run src/platform/dataflows/DataflowDetailPage.test.tsx` +Expected: PASS, same as before this step (this is a pure widening of the mock's type, not a behavior change). + +- [ ] **Step 3: Add a dedicated LIR fixture and the failing test for group rendering** + +The file's existing default dataflow ("7") fixture has zero LIR spans (its mocked SQL response sends `okResult`, an empty result, for the `lirSpans` query), and every other existing test relies on that staying empty. Rather than touching it, add a new dataflow id, following the same pattern already used for dataflow 8 (`portJumpOperatorsResult`) and 9 (`fanOutOperatorsResult`). + +Add these constants to `console/src/platform/dataflows/DataflowDetailPage.test.tsx`, near the other dataflow fixtures (after `fanOutChannelsResult`): + +```typescript +// Dataflow 40: root [40], child [40,1] "Join" (operator id 41), covered by +// one LIR span, to exercise the "Show LIR groups" toggle in isolation from +// every other test's dataflow 7 fixture, which has no LIR spans at all. +const lirOperatorsResult = { + ...operatorsResult, + rows: [ + ["40", ["40"], "Dataflow", "0", "0", "0"], + ["41", ["40", "1"], "Join", "0", "0", "0"], + ], +}; +const lirSpansResult = { + desc: { + columns: [ + { name: "exportId" }, + { name: "lirId" }, + { name: "parentLirId" }, + { name: "nesting" }, + { name: "operator" }, + { name: "operatorIdStart" }, + { name: "operatorIdEnd" }, + ], + }, + rows: [["u7", "1", null, "0", "Join::Differential", "41", "42"]], +}; +``` + +Add a new branch to the `http.post("*/api/sql", ...)` handler inside `beforeEach`, alongside the existing `'8'`/`'9'` branches (order among these branches doesn't matter, but put it before the final unconditional fallback): + +```typescript + if (body.includes("'40'")) { + return HttpResponse.json({ + results: [lirOperatorsResult, okResult, lirSpansResult, okResult], + }); + } +``` + +Then append this test to the `describe("DataflowDetailPage", ...)` block, following the same `renderComponent`/`initialRouterEntries` pattern the dataflow-8 tests already use (e.g. `"jumps to an output port's peer, same as an input port's peer"`): + +```typescript + it("shows a LIR group box when the toggle is on, and hides it when off", async () => { + await renderComponent( + + } + /> + , + { + initialRouterEntries: ["/clusters/u5/test_cluster/dataflows/40"], + }, + ); + + expect( + await screen.findByTestId(`node-${nodeIdOf([40, 1])}`), + ).toBeInTheDocument(); + expect(screen.queryByText(/Join::Differential/)).not.toBeInTheDocument(); + + await userEvent.click(screen.getByLabelText("Show LIR groups")); + + const groupNode = screen.getByTestId("node-u7/1"); + expect(groupNode).toHaveTextContent("Join::Differential"); + + // Regression check for the onNodeDoubleClick guard added in Step 4: + // double-clicking a group must not throw (it read `.data.node.kind` + // unconditionally before the guard, which crashes on a group node) and + // must not navigate anywhere, since groups aren't scopes. + await userEvent.dblClick(groupNode); + expect(screen.getByTestId("node-u7/1")).toBeInTheDocument(); + }); +``` + +This references `screen.getByLabelText("Show LIR groups")`, which doesn't exist until Task 5. Run it now to confirm it fails on the missing label (not a crash), then move on — it is expected to stay red until Task 5. + +- [ ] **Step 4: Implement grouping in `DataflowGraphView.tsx`** + +In `console/src/platform/dataflows/DataflowGraphView.tsx`: + +Change the import block to add the new pieces: +```typescript +import { ChannelEdge } from "./ChannelEdge"; +import { + type GraphDecorations, + groupByLir, + type LirGroupNode as LirGroupNodeData, + type NodeId, + type PortPeer, + rerouteHiddenNodes, + type VisibleEdge, + type VisibleGraph, + type VisibleNode, +} from "./dataflowGraph"; +import { NODE_DIMENSIONS, type Positions } from "./elkGraph"; +import { + LirGroupNode as LirGroupNodeComponent, + OperatorNode, + PortNode, + RegionNode, +} from "./nodes"; +import { lirGroupColor, nodeFillColor } from "./nodeStyle"; +import { useElkLayout } from "./useElkLayout"; +``` + +Add `lirGroup: LirGroupNodeComponent` to `nodeTypes`: +```typescript +const nodeTypes = { + operator: OperatorNode, + region: RegionNode, + port: PortNode, + lirGroup: LirGroupNodeComponent, +}; +``` + +Add two new props to `DataflowGraphViewProps` (near `onJumpToPeer`): +```typescript + // Toggled from the toolbar; grouping is computed here (from the same + // post-hideIdle, post-reroute node list elk actually lays out) rather than + // by the caller, so a hidden node can never end up as a group's only + // member. + showLirGroups?: boolean; + onLirGroupClick?: (group: LirGroupNodeData) => void; +``` + +Destructure them in the component signature (add `showLirGroups, onLirGroupClick,` to the existing destructured props list). + +After the existing `const visible = React.useMemo(...)` block (which computes the rerouted/filtered graph), add: +```typescript + const grouping = React.useMemo( + () => (showLirGroups ? groupByLir(visible.nodes) : null), + [showLirGroups, visible.nodes], + ); +``` + +Change `layoutKey` to include the toggle, so turning it on/off invalidates the layout cache: +```typescript + const layoutKey = `${cacheKey}|${focusedScope}|${showLirGroups ?? false}|${ + decorations?.hiddenNodeIds + ? [...decorations.hiddenNodeIds].sort().join(",") + : "" + }|${ + decorations?.hiddenEdgeIds + ? [...decorations.hiddenEdgeIds].sort().join(",") + : "" + }`; + const { positions, layouting, error, retry } = useElkLayout( + visible, + layoutKey, + grouping ?? undefined, + ); +``` + +Replace the `nodes: Node[] = React.useMemo(...)` block with one that emits group nodes before member nodes, and sets `parentId` on anything grouping places inside a group: +```typescript + const nodes: Node[] = React.useMemo(() => { + if (!positions) return []; + const groupNodes: Node[] = (grouping?.groups ?? []).map((g) => { + const pos = positions[g.id] ?? { x: 0, y: 0, width: 0, height: 0 }; + return { + id: g.id, + type: "lirGroup", + position: { x: pos.x, y: pos.y }, + parentId: grouping!.parentOf.get(g.id), + width: pos.width, + height: pos.height, + style: { width: pos.width, height: pos.height }, + draggable: false, + connectable: false, + // Below its members, which draw over it; React Flow requires a + // group node to precede its children in this array, which the + // groupNodes-then-memberNodes concat below already guarantees, but + // an explicit zIndex also protects against member nodes elsewhere + // in the array being reordered by a future change. + zIndex: -1, + data: { group: g, label: g.operator, color: lirGroupColor(g.lirId) }, + }; + }); + const memberNodes: Node[] = visible.nodes.map((n) => { + const pos = positions[n.id] ?? { x: 0, y: 0, ...NODE_DIMENSIONS[n.kind] }; + return { + id: n.id, + type: n.kind, + position: { x: pos.x, y: pos.y }, + parentId: grouping?.parentOf.get(n.id), + // Match the Top/Bottom handles so bezier control points meet the + // node edges. Without this the edge curves toward the default + // Bottom/Top and appears detached across region boundaries. + targetPosition: Position.Top, + sourcePosition: Position.Bottom, + // Set as explicit node fields, not just CSS style: the MiniMap and + // culled (onlyRenderVisibleElements) off-screen nodes both need a + // known size without waiting for DOM measurement. + width: pos.width, + height: pos.height, + style: { width: pos.width, height: pos.height }, + draggable: false, + connectable: false, + data: { + node: n, + dimmed: decorations?.dimmedNodeIds?.has(n.id) ?? false, + color: decorations?.nodeColors?.get(n.id) ?? nodeFillColor(n), + selected: n.id === selectedId, + activeMatch: n.id === activeMatchId, + }, + }; + }); + return [...groupNodes, ...memberNodes]; + }, [ + visible, + positions, + grouping, + decorations?.dimmedNodeIds, + decorations?.nodeColors, + selectedId, + activeMatchId, + ]); +``` + +Change the `onNodeClick` handler passed to `` to branch on group clicks first: +```typescript + onNodeClick={(_, node) => { + if (node.type === "lirGroup") { + onLirGroupClick?.((node.data as { group: LirGroupNodeData }).group); + return; + } + const visibleNode = (node.data as { node: VisibleNode }).node; + const connectedEdges = visible.edges + .filter( + (e) => e.source === visibleNode.id || e.target === visibleNode.id, + ) + .map((e) => ({ + ...e, + sourceLabel: labelById.get(e.source) ?? e.source, + targetLabel: labelById.get(e.target) ?? e.target, + })); + onNodeClick?.(visibleNode, connectedEdges); + }} +``` + +Also guard the existing `onNodeDoubleClick` handler, a few lines below `onNodeClick` in the same `` element: it currently reads `(node.data as { node: VisibleNode }).node.kind` unconditionally, which throws on a group node (`data` has no `.node`). Add the same type check at the top, before that line: +```typescript + onNodeDoubleClick={(_, node) => { + // Groups aren't scopes: no drill-down, no jump, nothing happens. + if (node.type === "lirGroup") return; + const visibleNode = (node.data as { node: VisibleNode }).node; + if (visibleNode.kind === "region") { + onNavigate(visibleNode.id); + } else if ( + visibleNode.kind === "port" && + visibleNode.peers.length === 1 + ) { + onJumpToPeer?.(visibleNode.peers[0]); + } + }} +``` + +- [ ] **Step 5: Run typecheck to confirm wiring compiles** + +Run: `cd console && yarn typecheck` +Expected: no errors. (The Task 3 test written in Step 3 is still expected to fail — it depends on Task 5's toolbar switch, which doesn't exist yet. Confirm it fails with "Unable to find a label with the text: Show LIR groups", not a crash, so you know the rest of the wiring is otherwise sound.) + +Run: `cd console && yarn vitest run src/platform/dataflows/DataflowDetailPage.test.tsx` +Expected: every test PASSES except the new one from Step 3, which fails on the missing label (not a crash/type error). + +- [ ] **Step 6: Lint** + +Run: `cd console && yarn lint` +Expected: no errors. + +- [ ] **Step 7: Commit** + +```bash +git add console/src/platform/dataflows/DataflowGraphView.tsx console/src/platform/dataflows/DataflowDetailPage.test.tsx +git commit -m "$(cat <<'EOF' +console: wire LIR grouping into DataflowGraphView + +Grouping is computed from the same post-hideIdle, post-reroute node +list elk actually lays out, so a hidden node never ends up as a +group's only member. Group nodes render behind their members via +React Flow's parentId nesting, one new nodeTypes entry, and an +onNodeClick branch for group-header clicks (wired to a real handler +in Task 6). + +This task's new toggle test doesn't pass yet — it depends on Task 5's +toolbar switch, which doesn't exist. +EOF +)" +``` + +--- + +### Task 5: Toolbar toggle + +**Files:** +- Modify: `console/src/platform/dataflows/dataflowGraph.ts` +- Modify: `console/src/platform/dataflows/DataflowToolbar.tsx` +- Modify: `console/src/platform/dataflows/DataflowDetailPage.tsx` +- Test: existing tests in `dataflowGraph.test.ts` and `DataflowDetailPage.test.tsx` (the one from Task 4, Step 3) + +**Interfaces:** +- Consumes: `DataflowGraphView`'s `showLirGroups` prop (Task 4). +- Produces: `Filters.showLirGroups: boolean`, `DEFAULT_FILTERS.showLirGroups: false`. + +- [ ] **Step 1: Update `DEFAULT_FILTERS` assertions if any exist, then add the field** + +Search for existing assertions against `DEFAULT_FILTERS`'s exact shape: + +Run: `cd console && grep -rn "DEFAULT_FILTERS" src/platform/dataflows/*.test.ts src/platform/dataflows/*.test.tsx` + +If any test asserts `DEFAULT_FILTERS` equals an object literal without `showLirGroups`, update it to include `showLirGroups: false`. (As of this plan being written, no such assertion exists — `DEFAULT_FILTERS` is only referenced by identity, e.g. `React.useState(DEFAULT_FILTERS)` — but confirm before proceeding, since the plan can't see future changes to this file.) + +In `console/src/platform/dataflows/dataflowGraph.ts`, change the `Filters` interface and `DEFAULT_FILTERS` constant: + +```typescript +export interface Filters { + search: string; + hideIdle: boolean; + heatmap: "off" | "elapsed" | "size" | "cpuSkew" | "memorySkew"; + heatmapThreshold: number; // 0..1 fraction of max + showLirGroups: boolean; +} + +export const DEFAULT_FILTERS: Filters = { + search: "", + hideIdle: false, + heatmap: "off", + heatmapThreshold: 0, + showLirGroups: false, +}; +``` + +- [ ] **Step 2: Add the toolbar switch** + +In `console/src/platform/dataflows/DataflowToolbar.tsx`, add a new `FormControl`/`Switch` pair after the existing "Hide idle" one: + +```typescript + + + Show LIR groups + + + onFiltersChange({ ...filters, showLirGroups: e.target.checked }) + } + /> + +``` + +The existing "Hide idle" switch has no `aria-label`, relying on its `FormLabel` for accessible naming via Chakra's `FormControl` association; do the same here (Chakra's `FormControl` context automatically associates the `FormLabel` with the `Switch`, so `getByLabelText("Show LIR groups")` in tests resolves without an explicit `aria-label`). Remove the `aria-label` line above if Chakra's automatic association already covers it — check by running the Task 4 test after this step; add `aria-label="Show LIR groups"` back only if the test can't find it by label text otherwise. + +- [ ] **Step 3: Wire the prop through `DataflowDetailPage.tsx`** + +In `console/src/platform/dataflows/DataflowDetailPage.tsx`, add `showLirGroups={filters.showLirGroups}` to the `` element's props (alongside the existing `decorations={decorations}` line). + +- [ ] **Step 4: Run the Task 4 toggle test to verify it now passes** + +Run: `cd console && yarn vitest run src/platform/dataflows/DataflowDetailPage.test.tsx` +Expected: PASS, including the "shows a LIR group box when the toggle is on" test from Task 4. + +- [ ] **Step 5: Run the full dataflow test suite** + +Run: `cd console && yarn vitest run src/platform/dataflows src/api/materialize/dataflow` +Expected: PASS, all tests. + +- [ ] **Step 6: Typecheck and lint** + +Run: `cd console && yarn typecheck && yarn lint` +Expected: no errors. + +- [ ] **Step 7: Commit** + +```bash +git add console/src/platform/dataflows/dataflowGraph.ts console/src/platform/dataflows/DataflowToolbar.tsx console/src/platform/dataflows/DataflowDetailPage.tsx console/src/platform/dataflows/DataflowDetailPage.test.tsx +git commit -m "console: add Show LIR groups toolbar toggle, off by default" +``` + +--- + +### Task 6: Click a group's header to open its detail panel + +**Files:** +- Modify: `console/src/platform/dataflows/LirPanel.tsx` (export the existing summary card for reuse) +- Modify: `console/src/platform/dataflows/NodeDetailPanel.tsx` +- Modify: `console/src/platform/dataflows/DataflowDetailPage.tsx` +- Test: `console/src/platform/dataflows/DataflowDetailPage.test.tsx` + +**Interfaces:** +- Consumes: `DataflowGraphView`'s `onLirGroupClick` prop (Task 4); `lirIndex`, `lirSummary` (both already exist in `dataflowGraph.ts`, unchanged). +- Produces: `Selection` gains a `"lirGroup"` variant; no new exports beyond `LirSummaryCard`. + +- [ ] **Step 1: Write the failing test** + +Append to `console/src/platform/dataflows/DataflowDetailPage.test.tsx`, reusing dataflow 40's fixture added in Task 4, Step 3: + +```typescript + it("opens the detail panel with LIR summary info when a group header is clicked", async () => { + await renderComponent( + + } + /> + , + { + initialRouterEntries: ["/clusters/u5/test_cluster/dataflows/40"], + }, + ); + + await screen.findByTestId(`node-${nodeIdOf([40, 1])}`); + await userEvent.click(screen.getByLabelText("Show LIR groups")); + await userEvent.click(screen.getByTestId("node-u7/1")); + + // The title and LirSummaryCard both render "LIR 1: Join::Differential", + // so assert at least one match rather than a single getByText, which + // throws on more than one hit. + expect( + screen.getAllByText(/LIR 1: Join::Differential/).length, + ).toBeGreaterThan(0); + // Rows unique to LirSummaryCard, not NodeDetail's node-shaped rows. + expect(screen.getByText("Records")).toBeInTheDocument(); + expect(screen.getByText("Memory")).toBeInTheDocument(); + }); +``` + +(Adjust the LIR id/operator text to match whatever this file's dataflow fixture actually uses — see Task 4, Step 3's note about matching the fixture's LIR span.) + +- [ ] **Step 2: Run the test to verify it fails** + +Run: `cd console && yarn vitest run src/platform/dataflows/DataflowDetailPage.test.tsx -t "opens the detail panel with LIR summary"` +Expected: FAIL — clicking the group header currently calls `onLirGroupClick`, which `DataflowDetailPage` doesn't pass yet, so nothing opens. + +- [ ] **Step 3: Export `LirSummaryCard` for reuse** + +In `console/src/platform/dataflows/LirPanel.tsx`, change: +```typescript +const LirSummaryCard = ({ node }: { node: LirTreeNode }) => ( +``` +to: +```typescript +export const LirSummaryCard = ({ node }: { node: LirTreeNode }) => ( +``` + +- [ ] **Step 4: Add the `"lirGroup"` selection variant to `NodeDetailPanel.tsx`** + +Change the `Selection` type: +```typescript +export type Selection = + | { kind: "node"; node: VisibleNode; connectedEdges?: SelectedEdge[] } + | { kind: "edge"; edge: SelectedEdge } + | { kind: "lirGroup"; node: LirTreeNode }; +``` + +Add `LirTreeNode` to the existing `import { ... } from "./dataflowGraph";` line, and import `LirSummaryCard` from `./LirPanel`: +```typescript +import { + formatElapsedNs, + type LirTreeNode, + type PortPeer, + type VisibleNode, +} from "./dataflowGraph"; +import { LirSummaryCard } from "./LirPanel"; +``` + +Change the `title` computation in `NodeDetailPanel`: +```typescript + const title = + selection.kind === "node" + ? selection.node.label + : selection.kind === "edge" + ? `${selection.edge.sourceLabel} → ${selection.edge.targetLabel}` + : `LIR ${selection.node.info.lirId}: ${selection.node.info.operator}`; +``` + +Change the body's conditional rendering (the `{selection.kind === "node" ? (...) : (...)}` block) to a three-way branch: +```typescript + {selection.kind === "node" ? ( + + ) : selection.kind === "edge" ? ( + + ) : ( + + )} +``` + +- [ ] **Step 5: Wire `onLirGroupClick` in `DataflowDetailPage.tsx`** + +Add `lirIndex` and `lirSummary` to the existing `import { ... } from "./dataflowGraph";` block (they're already exported by `dataflowGraph.ts`, just not currently imported here). + +Add a new callback near `onTogglePinLir`: +```typescript + const onLirGroupClick = React.useCallback( + (group: { id: string }) => { + if (!data) return; + const entry = lirIndex(data.structure).get(group.id); + if (!entry) return; + setSelection({ + kind: "lirGroup", + node: { + key: group.id, + info: entry.info, + memberIds: entry.memberIds, + children: [], + summary: lirSummary(data.structure, entry.memberIds), + }, + }); + }, + [data], + ); +``` + +Add `onLirGroupClick={onLirGroupClick}` to the `` element's props, alongside the existing `onJumpToPeer={onJumpToPeer}` line. + +- [ ] **Step 6: Run the test to verify it passes** + +Run: `cd console && yarn vitest run src/platform/dataflows/DataflowDetailPage.test.tsx` +Expected: PASS, all tests including the new one. + +- [ ] **Step 7: Typecheck and lint** + +Run: `cd console && yarn typecheck && yarn lint` +Expected: no errors. + +- [ ] **Step 8: Full verification pass** + +Run: `cd console && yarn vitest run src/platform/dataflows src/api/materialize/dataflow && yarn typecheck && yarn lint` +Expected: everything PASSES. + +Manually confirm the feature end-to-end against a real `environmentd` (per `mz-run`): start it, run a query that produces nested LIR spans (e.g. a materialized view with a table function like `generate_series` nested inside another, matching the reference screenshot this plan was scoped from), open its dataflow in the visualizer, toggle "Show LIR groups" on, and confirm nested dashed boxes render without overlapping siblings, edges crossing a box boundary still draw sensibly, and clicking a box header opens the detail panel with the right LIR id/operator/summary. This is a manual step outside the automated test loop; note the outcome to the user rather than silently assuming it looks right. + +- [ ] **Step 9: Commit** + +```bash +git add console/src/platform/dataflows/LirPanel.tsx console/src/platform/dataflows/NodeDetailPanel.tsx console/src/platform/dataflows/DataflowDetailPage.tsx console/src/platform/dataflows/DataflowDetailPage.test.tsx +git commit -m "console: open LIR summary in the detail panel from a group header click" +``` From dc36de57ef7a89d64498b5344fcc7f7a8b61d158 Mon Sep 17 00:00:00 2001 From: Moritz Hoffmann Date: Wed, 8 Jul 2026 22:42:39 +0200 Subject: [PATCH 26/45] Add LIR grouping: groupByLir, elk compound nesting, LirGroupNode component Co-Authored-By: Claude Sonnet 5 --- .../platform/dataflows/dataflowGraph.test.ts | 115 ++++++++++++++++++ .../src/platform/dataflows/dataflowGraph.ts | 88 ++++++++++++++ .../src/platform/dataflows/elkGraph.test.ts | 114 +++++++++++++++++ console/src/platform/dataflows/elkGraph.ts | 82 ++++++++++--- .../src/platform/dataflows/nodeStyle.test.ts | 13 +- console/src/platform/dataflows/nodeStyle.ts | 30 ++++- console/src/platform/dataflows/nodes.tsx | 33 +++++ .../src/platform/dataflows/useElkLayout.ts | 12 +- 8 files changed, 465 insertions(+), 22 deletions(-) diff --git a/console/src/platform/dataflows/dataflowGraph.test.ts b/console/src/platform/dataflows/dataflowGraph.test.ts index 9caebf99e6257..db7345ab056e4 100644 --- a/console/src/platform/dataflows/dataflowGraph.test.ts +++ b/console/src/platform/dataflows/dataflowGraph.test.ts @@ -17,7 +17,9 @@ import { decorateGraph, DEFAULT_FILTERS, deriveVisibleGraph, + groupByLir, lirIndex, + type LirInfo, type LirSpanRow, lirTree, nodeIdOf, @@ -951,3 +953,116 @@ describe("rerouteHiddenNodes", () => { expect(r.edges).toEqual(g.edges); }); }); + +describe("groupByLir", () => { + const node = ( + id: string, + operatorId: number, + lir: LirInfo[] = [], + ): VisibleNode => ({ + id, + kind: "operator", + label: id, + stats: null, + transitive: null, + transitiveSkew: null, + childCount: 0, + lir, + address: null, + operatorId: BigInt(operatorId), + peers: [], + }); + const port = (id: string): VisibleNode => ({ + id, + kind: "port", + label: id, + stats: null, + transitive: null, + transitiveSkew: null, + childCount: 0, + lir: [], + address: null, + operatorId: null, + peers: [], + }); + const span = ( + exportId: string, + lirId: string, + nesting: number, + parentLirId: string | null, + ): LirInfo => ({ exportId, lirId, parentLirId, nesting, operator: lirId }); + + it("groups nothing when no node has lir data", () => { + const g = groupByLir([node("a", 1), node("b", 2)]); + expect(g.groups).toEqual([]); + expect(g.parentOf.size).toEqual(0); + }); + + it("groups a flat run under one lir id", () => { + const s = span("u1", "10", 0, null); + const g = groupByLir([ + node("a", 1, [s]), + node("b", 2, [s]), + node("c", 3, [s]), + ]); + expect(g.groups.map((x) => x.id)).toEqual(["u1/10"]); + expect(g.parentOf.get("a")).toEqual("u1/10"); + expect(g.parentOf.get("b")).toEqual("u1/10"); + expect(g.parentOf.get("c")).toEqual("u1/10"); + }); + + it("nests an inner span inside an outer span", () => { + const outer = span("u1", "10", 0, null); + const inner = span("u1", "11", 1, "10"); + const g = groupByLir([ + node("a", 1, [outer]), + node("b", 2, [outer, inner]), + node("c", 3, [outer, inner]), + node("d", 4, [outer]), + ]); + expect(g.groups.map((x) => x.id)).toEqual(["u1/10", "u1/11"]); + expect(g.parentOf.get("u1/11")).toEqual("u1/10"); + expect(g.parentOf.get("a")).toEqual("u1/10"); + expect(g.parentOf.get("b")).toEqual("u1/11"); + expect(g.parentOf.get("c")).toEqual("u1/11"); + expect(g.parentOf.get("d")).toEqual("u1/10"); + }); + + it("leaves ports and lir-less nodes ungrouped alongside a group", () => { + const s = span("u1", "10", 0, null); + const g = groupByLir([node("a", 1, [s]), port("p"), node("b", 2)]); + expect(g.groups.map((x) => x.id)).toEqual(["u1/10"]); + expect(g.parentOf.get("a")).toEqual("u1/10"); + expect(g.parentOf.has("p")).toEqual(false); + expect(g.parentOf.has("b")).toEqual(false); + }); + + it("picks the lowest exportId, leaving other-export-only nodes ungrouped", () => { + const sA = span("u1", "5", 0, null); + const sB = span("u2", "3", 0, null); + const g = groupByLir([ + node("a", 1, [sA]), + node("b", 2, [sB]), + node("c", 3, [sA, sB]), + ]); + expect(g.groups.map((x) => x.id)).toEqual(["u1/5"]); + expect(g.parentOf.get("a")).toEqual("u1/5"); + expect(g.parentOf.has("b")).toEqual(false); + expect(g.parentOf.get("c")).toEqual("u1/5"); + }); + + it("sorts input by operatorId regardless of array order", () => { + const outer = span("u1", "10", 0, null); + const inner = span("u1", "11", 1, "10"); + // Deliberately out of operatorId order. + const g = groupByLir([ + node("d", 4, [outer]), + node("b", 2, [outer, inner]), + node("a", 1, [outer]), + node("c", 3, [outer, inner]), + ]); + expect(g.groups.map((x) => x.id)).toEqual(["u1/10", "u1/11"]); + expect(g.parentOf.get("b")).toEqual("u1/11"); + expect(g.parentOf.get("c")).toEqual("u1/11"); + }); +}); diff --git a/console/src/platform/dataflows/dataflowGraph.ts b/console/src/platform/dataflows/dataflowGraph.ts index f7bc74dd4f1e9..a9c298686e6af 100644 --- a/console/src/platform/dataflows/dataflowGraph.ts +++ b/console/src/platform/dataflows/dataflowGraph.ts @@ -991,3 +991,91 @@ export function lirTree( } return roots; } + +export interface LirGroupNode { + id: NodeId; // `${exportId}/${lirId}`, matching lirIndex's key + exportId: string; + lirId: string; + operator: string; + nesting: number; +} + +// A grouping layer over an already-derived VisibleGraph, the same way +// decorateGraph layers dimming and color over it: nothing here replaces +// `nodes`, so search, dimming, and click handling keep working against the +// flat list untouched. +export interface LirGrouping { + groups: LirGroupNode[]; // outer groups appear before groups nested inside them + parentOf: Map; // a VisibleNode or LirGroupNode id -> its immediate enclosing group's id +} + +// Groups a scope's direct children into the LIR tree that encloses them, for +// drawing nested boxes on the canvas. LIR spans are contiguous ranges over +// operator ids and properly nested: a region is always fully enclosed by a +// single LIR node's range. A dataflow can back several exports, and a shared +// subplan can carry lir entries from more than one of them at once; a +// compound-node tree needs one parent chain per node, so this groups by the +// lowest exportId (string-compared) present among the given nodes only. A +// node whose only lir entries belong to a different export renders +// ungrouped, alongside nodes with no lir data at all (e.g. ports). +export function groupByLir(nodes: readonly VisibleNode[]): LirGrouping { + const groups: LirGroupNode[] = []; + const parentOf = new Map(); + const ordered = nodes + .filter((n) => n.operatorId !== null) + .slice() + .sort((a, b) => { + const x = a.operatorId!; + const y = b.operatorId!; + return x < y ? -1 : x > y ? 1 : 0; + }); + if (ordered.length === 0) return { groups, parentOf }; + + let chosenExport: string | null = null; + for (const n of ordered) { + for (const l of n.lir) { + if (chosenExport === null || l.exportId < chosenExport) { + chosenExport = l.exportId; + } + } + } + if (chosenExport === null) return { groups, parentOf }; + + const groupIndex = new Map(); + const stack: LirGroupNode[] = []; // open groups, outer to inner + for (const node of ordered) { + const stackKeys = node.lir + .filter((l) => l.exportId === chosenExport) + .sort((a, b) => a.nesting - b.nesting) + .map((l) => `${l.exportId}/${l.lirId}`); + let commonLen = 0; + while ( + commonLen < stack.length && + commonLen < stackKeys.length && + stack[commonLen].id === stackKeys[commonLen] + ) { + commonLen++; + } + stack.length = commonLen; // pop back to the shared prefix + for (let depth = commonLen; depth < stackKeys.length; depth++) { + const key = stackKeys[depth]; + let group = groupIndex.get(key); + if (!group) { + const info = node.lir.find((l) => `${l.exportId}/${l.lirId}` === key)!; + group = { + id: key, + exportId: info.exportId, + lirId: info.lirId, + operator: info.operator, + nesting: info.nesting, + }; + groupIndex.set(key, group); + groups.push(group); + } + if (depth > 0) parentOf.set(group.id, stack[depth - 1].id); + stack.push(group); + } + if (stack.length > 0) parentOf.set(node.id, stack[stack.length - 1].id); + } + return { groups, parentOf }; +} diff --git a/console/src/platform/dataflows/elkGraph.test.ts b/console/src/platform/dataflows/elkGraph.test.ts index 430874eeb68c0..a3b845e485e76 100644 --- a/console/src/platform/dataflows/elkGraph.test.ts +++ b/console/src/platform/dataflows/elkGraph.test.ts @@ -7,11 +7,13 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. +import ELK from "elkjs"; import { describe, expect, it } from "vitest"; import { buildDataflowStructure, deriveVisibleGraph, + groupByLir, nodeIdOf, } from "./dataflowGraph"; import { CHANNELS, LIR_SPANS, OPS } from "./dataflowGraph.test"; @@ -45,3 +47,115 @@ describe("toElkGraph", () => { expect(positions[elk.children![0].id]).toMatchObject({ x: 10, y: 20 }); }); }); + +describe("toElkGraph with a LIR grouping", () => { + const s = buildDataflowStructure(OPS, CHANNELS, LIR_SPANS); + + it("nests a group's members as elk children, sized only by content", () => { + const visible = deriveVisibleGraph(s, s.root); + const grouping = groupByLir(visible.nodes); + const elk = toElkGraph(visible, grouping); + const group = elk.children!.find((c) => c.id === grouping.groups[0].id)!; + expect(group.children!.map((c) => c.id)).toEqual( + expect.arrayContaining([nodeIdOf([5, 1])]), + ); + // Auto-sized by elk from its content, not one of NODE_DIMENSIONS's fixed sizes. + expect(group.width).toBeUndefined(); + expect(group.height).toBeUndefined(); + // Flat sibling stays a direct child of the root, not absorbed into the group. + expect(elk.children!.map((c) => c.id)).toEqual( + expect.arrayContaining([nodeIdOf([5, 2])]), + ); + }); + + it("keeps every edge at the root level regardless of nesting depth", () => { + const visible = deriveVisibleGraph(s, s.root); + const grouping = groupByLir(visible.nodes); + const elk = toElkGraph(visible, grouping); + expect(elk.edges!.map((e) => e.id)).toEqual(visible.edges.map((e) => e.id)); + for (const child of elk.children!) expect(child.edges).toBeUndefined(); + }); + + it("round-trips nested positions unchanged (parent-relative, no conversion)", () => { + const visible = deriveVisibleGraph(s, s.root); + const grouping = groupByLir(visible.nodes); + const elk = toElkGraph(visible, grouping); + const group = elk.children!.find((c) => c.id === grouping.groups[0].id)!; + group.x = 12; + group.y = 87; + group.width = 132; + group.height = 259; + const member = group.children!.find((c) => c.id === nodeIdOf([5, 1]))!; + member.x = 16; + member.y = 24; + const positions = extractPositions(elk); + // The member's extracted position is exactly its own x/y, not offset by + // the group's: elk already reports it relative to the group. + expect(positions[member.id]).toMatchObject({ x: 16, y: 24 }); + expect(positions[group.id]).toMatchObject({ + x: 12, + y: 87, + width: 132, + height: 259, + }); + }); + + it("real elk lays out nested groups with auto-sized bounds and positions relative to parent", async () => { + // Empirical check: elk auto-sizes a compound node from its children, + // reports child positions relative to their own parent (matching React + // Flow's parentId convention with no coordinate conversion needed), and + // processes edges declared at the root even with nested containers present, + // given hierarchyHandling: INCLUDE_CHILDREN. + const elk = new ELK(); + const graph = { + id: "root", + layoutOptions: { + "elk.algorithm": "layered", + "elk.direction": "DOWN", + "elk.hierarchyHandling": "INCLUDE_CHILDREN", + }, + children: [ + { id: "ungrouped", width: 100, height: 50 }, + { + id: "groupA", + layoutOptions: { "elk.padding": "[top=24,left=8,bottom=8,right=8]" }, + children: [ + { id: "a1", width: 100, height: 50 }, + { + id: "groupA_inner", + layoutOptions: { + "elk.padding": "[top=24,left=8,bottom=8,right=8]", + }, + children: [ + { id: "a2", width: 100, height: 50 }, + { id: "a3", width: 100, height: 50 }, + ], + }, + ], + }, + ], + edges: [ + { id: "e1", sources: ["ungrouped"], targets: ["a1"] }, + { id: "e2", sources: ["a1"], targets: ["a2"] }, + { id: "e3", sources: ["a2"], targets: ["a3"] }, + ], + }; + const result = await elk.layout(graph); + const groupA = result.children!.find((c) => c.id === "groupA")!; + const groupAInner = groupA.children!.find((c) => c.id === "groupA_inner")!; + // Auto-sized larger than any single member: real content-driven sizing. + expect(groupA.width!).toBeGreaterThan(100); + expect(groupA.height!).toBeGreaterThan(50); + // All edges remain at root even though nested containers are present; + // they layout correctly with hierarchyHandling: INCLUDE_CHILDREN. + expect((result.edges ?? []).map((e) => e.id)).toEqual( + expect.arrayContaining(["e1", "e2", "e3"]), + ); + // Nested child positions are relative to their immediate parent (groupA_inner). + // hierarchyHandling: INCLUDE_CHILDREN is the discriminating option that changes + // elk's nested layout: without it, a3 lands at x:128, y:24 instead. + // Pinning this exact position discriminates the option's presence in production. + const a3 = groupAInner.children!.find((c) => c.id === "a3")!; + expect(a3).toMatchObject({ x: 8, y: 94 }); + }); +}); diff --git a/console/src/platform/dataflows/elkGraph.ts b/console/src/platform/dataflows/elkGraph.ts index 41f86e80478a8..cf9d8acca7090 100644 --- a/console/src/platform/dataflows/elkGraph.ts +++ b/console/src/platform/dataflows/elkGraph.ts @@ -9,7 +9,7 @@ import type { ElkNode } from "elkjs/lib/elk-api"; -import type { VisibleGraph, VisibleNode } from "./dataflowGraph"; +import type { LirGrouping, VisibleGraph, VisibleNode } from "./dataflowGraph"; export interface NodePosition { x: number; @@ -35,18 +35,60 @@ const LAYOUT_OPTIONS: Record = { "elk.padding": "[top=48,left=16,bottom=16,right=16]", "elk.spacing.nodeNode": "24", "elk.layered.spacing.nodeNodeBetweenLayers": "48", + // Lets an edge crossing a LIR group's boundary be declared uniformly at + // the root: elk relocates it to the correct container itself. Harmless + // when there's no grouping at all (today's flat views). + "elk.hierarchyHandling": "INCLUDE_CHILDREN", }; -// A view is always one scope's direct children, so this is always a flat, -// single-level graph: nothing is ever nested inside anything else. -export function toElkGraph(graph: VisibleGraph): ElkNode { +// A LIR group is a label-only wrapper, sized purely from its content (no +// fixed width/height), with top padding reserved for the header strip +// LirGroupNode renders. +const GROUP_LAYOUT_OPTIONS: Record = { + "elk.algorithm": "layered", + "elk.direction": "DOWN", + "elk.padding": "[top=24,left=8,bottom=8,right=8]", + "elk.spacing.nodeNode": "24", + "elk.layered.spacing.nodeNodeBetweenLayers": "48", +}; + +// A view is always one scope's direct children, so without a grouping this +// is flat: nothing is nested inside anything else. With a grouping, a +// group's members (VisibleNodes or other groups) become its elk children; +// everything else stays a direct child of the root, exactly like today. +export function toElkGraph( + graph: VisibleGraph, + grouping?: LirGrouping, +): ElkNode { + const nodeById = new Map(graph.nodes.map((n) => [n.id, n])); + const groupById = new Map((grouping?.groups ?? []).map((g) => [g.id, g])); + const childrenOf = new Map(); // parent id ("" = root) -> ordered child ids + const addChild = (parentKey: string, id: string) => { + const list = childrenOf.get(parentKey); + if (list) list.push(id); + else childrenOf.set(parentKey, [id]); + }; + for (const n of graph.nodes) + addChild(grouping?.parentOf.get(n.id) ?? "", n.id); + for (const g of groupById.values()) + addChild(grouping?.parentOf.get(g.id) ?? "", g.id); + + const buildNode = (id: string): ElkNode => { + const group = groupById.get(id); + if (group) { + return { + id, + layoutOptions: GROUP_LAYOUT_OPTIONS, + children: (childrenOf.get(id) ?? []).map(buildNode), + }; + } + return { id, ...NODE_DIMENSIONS[nodeById.get(id)!.kind] }; + }; + return { id: "__root__", layoutOptions: LAYOUT_OPTIONS, - children: graph.nodes.map((n) => ({ - id: n.id, - ...NODE_DIMENSIONS[n.kind], - })), + children: (childrenOf.get("") ?? []).map(buildNode), edges: graph.edges.map((e) => ({ id: e.id, sources: [e.source], @@ -55,15 +97,23 @@ export function toElkGraph(graph: VisibleGraph): ElkNode { }; } +// Recurses into nested children: elk reports every node's x/y already +// relative to its own immediate parent, the same convention React Flow uses +// for a node with parentId set, so no coordinate conversion happens here — +// this only flattens the tree into one lookup map, keyed by id at any depth. export function extractPositions(layouted: ElkNode): Positions { const positions: Positions = {}; - for (const child of layouted.children ?? []) { - positions[child.id] = { - x: child.x ?? 0, - y: child.y ?? 0, - width: child.width ?? 0, - height: child.height ?? 0, - }; - } + const visit = (node: ElkNode) => { + for (const child of node.children ?? []) { + positions[child.id] = { + x: child.x ?? 0, + y: child.y ?? 0, + width: child.width ?? 0, + height: child.height ?? 0, + }; + visit(child); + } + }; + visit(layouted); return positions; } diff --git a/console/src/platform/dataflows/nodeStyle.test.ts b/console/src/platform/dataflows/nodeStyle.test.ts index 44c59605e03f9..4139ae3ebfbb8 100644 --- a/console/src/platform/dataflows/nodeStyle.test.ts +++ b/console/src/platform/dataflows/nodeStyle.test.ts @@ -9,7 +9,18 @@ import { describe, expect, it } from "vitest"; -import { prettyPrintChannelType } from "./nodeStyle"; +import { lirGroupColor, prettyPrintChannelType } from "./nodeStyle"; + +describe("lirGroupColor", () => { + it("is deterministic for the same lirId", () => { + expect(lirGroupColor("42")).toEqual(lirGroupColor("42")); + }); + + it("differs for different lirIds often enough to be useful", () => { + const colors = new Set(["1", "2", "3", "4", "5", "6"].map(lirGroupColor)); + expect(colors.size).toBeGreaterThan(1); + }); +}); describe("prettyPrintChannelType", () => { it("strips module paths, aliases Diff/Error, and brackets Vec", () => { diff --git a/console/src/platform/dataflows/nodeStyle.ts b/console/src/platform/dataflows/nodeStyle.ts index 6839ad6a1927a..f966c2f6dd6cc 100644 --- a/console/src/platform/dataflows/nodeStyle.ts +++ b/console/src/platform/dataflows/nodeStyle.ts @@ -7,7 +7,7 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -import type { VisibleNode } from "./dataflowGraph"; +import type { LirGroupNode, VisibleNode } from "./dataflowGraph"; export const COLORS = { noArrangementRegion: "#12b886", @@ -30,6 +30,28 @@ export const HIGHLIGHT_COLORS = { connected: "#8fd2f5", }; +// A small, visually distinct palette for LIR group borders/headers, cycled +// by a deterministic hash so the same lirId always gets the same color +// within one render (and across re-renders, since lirId is stable). +const GROUP_PALETTE = [ + "#e8590c", + "#5f3dc4", + "#0b7285", + "#2f9e44", + "#c2255c", + "#1971c2", + "#e67700", + "#7048e8", +]; + +export function lirGroupColor(lirId: string): string { + let hash = 0; + for (let i = 0; i < lirId.length; i++) { + hash = (hash * 31 + lirId.charCodeAt(i)) | 0; + } + return GROUP_PALETTE[Math.abs(hash) % GROUP_PALETTE.length]; +} + export function nodeFillColor(node: VisibleNode): string { const arranged = (node.transitive?.arrangementRecords ?? 0n) > 0n; const region = node.kind !== "operator" && node.kind !== "port"; @@ -83,3 +105,9 @@ export type FlowNodeData = { selected: boolean; activeMatch: boolean; }; + +export type FlowGroupData = { + group: LirGroupNode; + label: string; + color: string; +}; diff --git a/console/src/platform/dataflows/nodes.tsx b/console/src/platform/dataflows/nodes.tsx index 7a5c2ca0ffdea..dcce0dca8a31e 100644 --- a/console/src/platform/dataflows/nodes.tsx +++ b/console/src/platform/dataflows/nodes.tsx @@ -15,6 +15,7 @@ import { formatBytesShort } from "~/utils/format"; import type { VisibleNode } from "./dataflowGraph"; import { + type FlowGroupData, type FlowNodeData, formatElapsed, HIGHLIGHT_COLORS, @@ -124,3 +125,35 @@ export const PortNode = ({ data }: NodeProps & { data: FlowNodeData }) => ( ); + +// A label-only wrapper around its members, never itself a click target +// except its header (the body is pointerEvents="none" so clicks pass +// through to whatever member is underneath). Its width/height come from +// elk's auto-sized bounds, same as every other node in this file. +export const LirGroupNode = ({ data }: NodeProps & { data: FlowGroupData }) => ( + + + + + {data.label} + + + +); diff --git a/console/src/platform/dataflows/useElkLayout.ts b/console/src/platform/dataflows/useElkLayout.ts index d9b1a2a7103e2..f2530efcece86 100644 --- a/console/src/platform/dataflows/useElkLayout.ts +++ b/console/src/platform/dataflows/useElkLayout.ts @@ -17,10 +17,14 @@ import ELK, { type ElkNode } from "elkjs/lib/elk-api.js"; import ElkWorker from "elkjs/lib/elk-worker.min.js?worker"; import React from "react"; -import type { VisibleGraph } from "./dataflowGraph"; +import type { LirGrouping, VisibleGraph } from "./dataflowGraph"; import { extractPositions, type Positions, toElkGraph } from "./elkGraph"; -export function useElkLayout(graph: VisibleGraph | null, cacheKey: string) { +export function useElkLayout( + graph: VisibleGraph | null, + cacheKey: string, + grouping?: LirGrouping, +) { const elkRef = React.useRef | null>(null); const requestIdRef = React.useRef(0); const cacheRef = React.useRef(new Map()); @@ -53,7 +57,7 @@ export function useElkLayout(graph: VisibleGraph | null, cacheKey: string) { const elk = elkRef.current; if (!elk) return; const requestId = ++requestIdRef.current; - elk.layout(toElkGraph(graph)).then( + elk.layout(toElkGraph(graph, grouping)).then( (layouted: ElkNode) => { // Drop responses for superseded requests. if (requestId !== requestIdRef.current) return; @@ -66,7 +70,7 @@ export function useElkLayout(graph: VisibleGraph | null, cacheKey: string) { setState({ key: cacheKey, positions: null, error: String(error) }); }, ); - }, [graph, cacheKey, retryNonce]); + }, [graph, cacheKey, retryNonce, grouping]); return { positions: state.key === cacheKey ? state.positions : null, From 25553fe1c89df2611591af27b9ed2a65c5644325 Mon Sep 17 00:00:00 2001 From: Moritz Hoffmann Date: Wed, 8 Jul 2026 22:42:39 +0200 Subject: [PATCH 27/45] Wire LIR grouping into the view, add toolbar toggle and summary-click, fix viewport bug Co-Authored-By: Claude Sonnet 5 --- .../dataflows/DataflowDetailPage.test.tsx | 103 +++++++++++++++++- .../platform/dataflows/DataflowDetailPage.tsx | 23 ++++ .../platform/dataflows/DataflowGraphView.tsx | 88 ++++++++++++++- .../platform/dataflows/DataflowToolbar.tsx | 12 ++ console/src/platform/dataflows/LirPanel.tsx | 2 +- .../platform/dataflows/NodeDetailPanel.tsx | 13 ++- .../src/platform/dataflows/dataflowGraph.ts | 2 + .../src/platform/dataflows/elkGraph.test.ts | 76 ++++++++++++- console/src/platform/dataflows/elkGraph.ts | 34 ++++++ console/src/platform/dataflows/nodes.tsx | 11 +- .../2026-07-07-dataflow-lir-grouping-boxes.md | 11 +- 11 files changed, 356 insertions(+), 19 deletions(-) diff --git a/console/src/platform/dataflows/DataflowDetailPage.test.tsx b/console/src/platform/dataflows/DataflowDetailPage.test.tsx index b146eba0dd4ea..40b50213f08ef 100644 --- a/console/src/platform/dataflows/DataflowDetailPage.test.tsx +++ b/console/src/platform/dataflows/DataflowDetailPage.test.tsx @@ -7,7 +7,7 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -import { screen } from "@testing-library/react"; +import { screen, within } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import { http, HttpResponse } from "msw"; import React from "react"; @@ -41,7 +41,11 @@ vi.mock("@xyflow/react", () => ({ onNodeClick, onNodeDoubleClick, }: { - nodes: { id: string; data: { node: { label: string } } }[]; + nodes: { + id: string; + type?: string; + data: { node?: { label: string }; label?: string }; + }[]; children?: React.ReactNode; onNodeClick?: (e: unknown, node: unknown) => void; onNodeDoubleClick?: (e: unknown, node: unknown) => void; @@ -54,7 +58,7 @@ vi.mock("@xyflow/react", () => ({ onClick={() => onNodeClick?.(null, n)} onDoubleClick={() => onNodeDoubleClick?.(null, n)} > - {n.data.node.label} + {n.data.node ? n.data.node.label : n.data.label} ))} {children} @@ -188,6 +192,31 @@ const fanOutChannelsResult = { ], }; +// Dataflow 40: root [40], child [40,1] "Join" (operator id 41), covered by +// one LIR span, to exercise the "Show LIR groups" toggle in isolation from +// every other test's dataflow 7 fixture, which has no LIR spans at all. +const lirOperatorsResult = { + ...operatorsResult, + rows: [ + ["40", ["40"], "Dataflow", "0", "0", "0"], + ["41", ["40", "1"], "Join", "0", "0", "0"], + ], +}; +const lirSpansResult = { + desc: { + columns: [ + { name: "exportId" }, + { name: "lirId" }, + { name: "parentLirId" }, + { name: "nesting" }, + { name: "operator" }, + { name: "operatorIdStart" }, + { name: "operatorIdEnd" }, + ], + }, + rows: [["u7", "1", null, "0", "Join::Differential", "41", "42"]], +}; + beforeEach(() => { fitViewSpy.mockClear(); const store = getStore(); @@ -237,6 +266,11 @@ beforeEach(() => { ], }); } + if (body.includes("'40'")) { + return HttpResponse.json({ + results: [lirOperatorsResult, okResult, lirSpansResult, okResult], + }); + } return HttpResponse.json({ results: [operatorsResult, okResult, okResult, okResult], }); @@ -528,4 +562,67 @@ describe("DataflowDetailPage", () => { await screen.findByTestId(`node-${nodeIdOf([7, 1])}`), ).toHaveTextContent("Map"); }); + + it("shows a LIR group box when the toggle is on, and hides it when off", async () => { + await renderComponent( + + } + /> + , + { + initialRouterEntries: ["/clusters/u5/test_cluster/dataflows/40"], + }, + ); + + expect( + await screen.findByTestId(`node-${nodeIdOf([40, 1])}`), + ).toBeInTheDocument(); + expect( + within(screen.getByTestId("react-flow")).queryByText( + /Join::Differential/, + ), + ).not.toBeInTheDocument(); + + await userEvent.click(screen.getByLabelText("Show LIR groups")); + + const groupNode = screen.getByTestId("node-u7/1"); + expect(groupNode).toHaveTextContent("Join::Differential"); + + // Regression check for the onNodeDoubleClick guard added in Step 4: + // double-clicking a group must not throw (it read `.data.node.kind` + // unconditionally before the guard, which crashes on a group node) and + // must not navigate anywhere, since groups aren't scopes. + await userEvent.dblClick(groupNode); + expect(screen.getByTestId("node-u7/1")).toBeInTheDocument(); + }); + + it("opens the detail panel with LIR summary info when a group header is clicked", async () => { + await renderComponent( + + } + /> + , + { + initialRouterEntries: ["/clusters/u5/test_cluster/dataflows/40"], + }, + ); + + await screen.findByTestId(`node-${nodeIdOf([40, 1])}`); + await userEvent.click(screen.getByLabelText("Show LIR groups")); + await userEvent.click(screen.getByTestId("node-u7/1")); + + // The title and LirSummaryCard both render "LIR 1: Join::Differential", + // so assert at least one match rather than a single getByText, which + // throws on more than one hit. + expect( + screen.getAllByText(/LIR 1: Join::Differential/).length, + ).toBeGreaterThan(0); + // Rows unique to LirSummaryCard, not NodeDetail's node-shaped rows. + expect(screen.getByText("Records")).toBeInTheDocument(); + expect(screen.getByText("Memory")).toBeInTheDocument(); + }); }); diff --git a/console/src/platform/dataflows/DataflowDetailPage.tsx b/console/src/platform/dataflows/DataflowDetailPage.tsx index 0d0213a4ff6aa..b9f58f45634e0 100644 --- a/console/src/platform/dataflows/DataflowDetailPage.tsx +++ b/console/src/platform/dataflows/DataflowDetailPage.tsx @@ -36,6 +36,8 @@ import { DEFAULT_FILTERS, deriveVisibleGraph, type Filters, + lirIndex, + lirSummary, type NodeId, nodeIdOf, type PortPeer, @@ -254,6 +256,25 @@ const DataflowDetailPage = () => { [], ); + const onLirGroupClick = React.useCallback( + (group: { id: string }) => { + if (!data) return; + const entry = lirIndex(data.structure).get(group.id); + if (!entry) return; + setSelection({ + kind: "lirGroup", + node: { + key: group.id, + info: entry.info, + memberIds: entry.memberIds, + children: [], + summary: lirSummary(data.structure, entry.memberIds), + }, + }); + }, + [data], + ); + const allMatches = React.useMemo( () => (data ? allSearchMatches(data.structure, filters.search) : []), [data, filters.search], @@ -469,6 +490,7 @@ const DataflowDetailPage = () => { onNavigate={setFocusedScope} cacheKey={structureKey ?? ""} decorations={decorations} + showLirGroups={filters.showLirGroups} centerRef={centerRef} fitRef={fitRef} fitOnIds={pendingFitIds} @@ -496,6 +518,7 @@ const DataflowDetailPage = () => { onEdgeClick={(edge) => setSelection({ kind: "edge", edge })} onPaneClick={() => setSelection(null)} onJumpToPeer={onJumpToPeer} + onLirGroupClick={onLirGroupClick} /> {selection && ( void) | null>; fitOnIds?: string[] | null; onFit?: () => void; + // Toggled from the toolbar; grouping is computed here (from the same + // post-hideIdle, post-reroute node list elk actually lays out) rather than + // by the caller, so a hidden node can never end up as a group's only + // member. + showLirGroups?: boolean; + onLirGroupClick?: (group: LirGroupNodeData) => void; } // Exposes centering/fitting callbacks through refs once React Flow context @@ -202,6 +220,8 @@ export const DataflowGraphView = ({ fitRef, fitOnIds, onFit, + showLirGroups, + onLirGroupClick, }: DataflowGraphViewProps) => { const visible = React.useMemo(() => { // Hidden nodes get spliced out with connectivity preserved (a hidden idle @@ -220,7 +240,12 @@ export const DataflowGraphView = ({ }; }, [rawVisible, decorations?.hiddenNodeIds, decorations?.hiddenEdgeIds]); - const layoutKey = `${cacheKey}|${focusedScope}|${ + const grouping = React.useMemo( + () => (showLirGroups ? groupByLir(visible.nodes) : null), + [showLirGroups, visible.nodes], + ); + + const layoutKey = `${cacheKey}|${focusedScope}|${showLirGroups ?? false}|${ decorations?.hiddenNodeIds ? [...decorations.hiddenNodeIds].sort().join(",") : "" @@ -232,6 +257,18 @@ export const DataflowGraphView = ({ const { positions, layouting, error, retry } = useElkLayout( visible, layoutKey, + grouping ?? undefined, + ); + + // Only ViewportGuard needs this: everything else either reads positions + // relative to parentId (React Flow's own convention, matching elk's) or + // resolves absolute position through React Flow's internal APIs. + const absolutePositions = React.useMemo( + () => + positions + ? resolveAbsolutePositions(positions, grouping?.parentOf) + : null, + [positions, grouping], ); React.useEffect(() => { @@ -260,12 +297,43 @@ export const DataflowGraphView = ({ const nodes: Node[] = React.useMemo(() => { if (!positions) return []; - return visible.nodes.map((n) => { + const groupNodes: Node[] = (grouping?.groups ?? []).map((g) => { + const pos = positions[g.id] ?? { x: 0, y: 0, width: 0, height: 0 }; + return { + id: g.id, + type: "lirGroup", + position: { x: pos.x, y: pos.y }, + parentId: grouping!.parentOf.get(g.id), + width: pos.width, + height: pos.height, + // pointerEvents: "none" on React Flow's own node wrapper (not just + // inside LirGroupNode's own JSX) is the other half of the + // click-through contract LirGroupNode's comment documents: the + // wrapper defaults to pointer-events:all and LirGroupNode can't + // override an ancestor it doesn't render. With the wrapper opted + // out, LirGroupNode's header (pointerEvents:"auto") still receives + // clicks (a descendant can always re-enable itself), and a click on + // the group's empty body falls through to a member's own wrapper + // when one is there, or further to the pane when none is. + style: { width: pos.width, height: pos.height, pointerEvents: "none" }, + draggable: false, + connectable: false, + // Below its members, which draw over it; React Flow requires a + // group node to precede its children in this array, which the + // groupNodes-then-memberNodes concat below already guarantees, but + // an explicit zIndex also protects against member nodes elsewhere + // in the array being reordered by a future change. + zIndex: -1, + data: { group: g, label: g.operator, color: lirGroupColor(g.lirId) }, + }; + }); + const memberNodes: Node[] = visible.nodes.map((n) => { const pos = positions[n.id] ?? { x: 0, y: 0, ...NODE_DIMENSIONS[n.kind] }; return { id: n.id, type: n.kind, position: { x: pos.x, y: pos.y }, + parentId: grouping?.parentOf.get(n.id), // Match the Top/Bottom handles so bezier control points meet the // node edges. Without this the edge curves toward the default // Bottom/Top and appears detached across region boundaries. @@ -288,9 +356,11 @@ export const DataflowGraphView = ({ }, }; }); + return [...groupNodes, ...memberNodes]; }, [ visible, positions, + grouping, decorations?.dimmedNodeIds, decorations?.nodeColors, selectedId, @@ -360,6 +430,10 @@ export const DataflowGraphView = ({ // Double-click navigates into a region, so it must not also zoom. zoomOnDoubleClick={false} onNodeClick={(_, node) => { + if (node.type === "lirGroup") { + onLirGroupClick?.((node.data as { group: LirGroupNodeData }).group); + return; + } const visibleNode = (node.data as { node: VisibleNode }).node; const connectedEdges = visible.edges .filter( @@ -383,6 +457,8 @@ export const DataflowGraphView = ({ }} onPaneClick={onPaneClick} onNodeDoubleClick={(_, node) => { + // Groups aren't scopes: no drill-down, no jump, nothing happens. + if (node.type === "lirGroup") return; const visibleNode = (node.data as { node: VisibleNode }).node; if (visibleNode.kind === "region") { onNavigate(visibleNode.id); @@ -413,7 +489,7 @@ export const DataflowGraphView = ({ diff --git a/console/src/platform/dataflows/DataflowToolbar.tsx b/console/src/platform/dataflows/DataflowToolbar.tsx index d7314f8f6c2c3..78ff25e479f61 100644 --- a/console/src/platform/dataflows/DataflowToolbar.tsx +++ b/console/src/platform/dataflows/DataflowToolbar.tsx @@ -92,6 +92,18 @@ export const DataflowToolbar = ({ } /> + + + Show LIR groups + + + onFiltersChange({ ...filters, showLirGroups: e.target.checked }) + } + /> + ( value={formatBytesShort(node.summary.arrangementSize)} /> + ); @@ -162,8 +163,12 @@ const NodeDetail = ({ value={formatBytesShort(node.stats.arrangementSize)} /> + )} + {node.childCount > 0 && ( + + )} {node.transitive && node.childCount > 0 && ( <> + )} {node.transitiveSkew && ( @@ -187,6 +196,10 @@ const NodeDetail = ({ label="Memory skew" value={formatSkew(node.transitiveSkew.memorySkew)} /> + )} {node.lir.length > 0 && ( diff --git a/console/src/platform/dataflows/dataflowGraph.test.ts b/console/src/platform/dataflows/dataflowGraph.test.ts index ede4f6324c950..fe6e7c0ff217d 100644 --- a/console/src/platform/dataflows/dataflowGraph.test.ts +++ b/console/src/platform/dataflows/dataflowGraph.test.ts @@ -188,6 +188,89 @@ describe("buildDataflowStructure", () => { }); }); + it("computes own and transitive schedule counts alongside elapsed time", () => { + const s = buildDataflowStructure( + OPS.map((o) => + o.id === "12" + ? { ...o, scheduleCount: "100" } + : o.id === "13" + ? { ...o, scheduleCount: "50" } + : o, + ), + CHANNELS, + LIR_SPANS, + ); + const join = s.nodes.get(nodeIdOf([5, 1, 1]))!; + expect(join.own.scheduleCount).toEqual(100n); + const region = s.nodes.get(nodeIdOf([5, 1]))!; + // Region's own row sets no scheduleCount (defaults 0); transitive sums + // Join's and Map's. + expect(region.transitive.scheduleCount).toEqual(150n); + }); + + it("computes a region's overhead as its own elapsed time minus its direct children's own elapsed time", () => { + const overheadOps = OPS.map((o) => + o.id === "11" + ? { ...o, elapsedNs: "100" } + : o.id === "12" + ? { ...o, elapsedNs: "30" } + : o.id === "13" + ? { ...o, elapsedNs: "20" } + : o, + ); + const s = buildDataflowStructure(overheadOps, CHANNELS, LIR_SPANS); + const region = s.nodes.get(nodeIdOf([5, 1]))!; + // 100 - (30 + 20) = 50: time in the region's own scheduling loop, not + // attributable to either child. + expect(region.overheadNs).toEqual(50n); + // A leaf has no children to subtract, so this is just its own elapsed + // time -- not a meaningful "overhead" reading on its own. + const join = s.nodes.get(nodeIdOf([5, 1, 1]))!; + expect(join.overheadNs).toEqual(30n); + }); + + it("compares a region's own time against its direct children's own time, not their transitive rollup", () => { + // Timely's own-elapsed accounting is exclusive self time (verified + // against live introspection data: a real region's own.elapsedNs can + // sit well below its direct children's own.elapsedNs summed, which is + // only possible if "own" never included child execution to begin + // with). Join here has its own grandchild (a nested region inside it) + // whose time dwarfs Join's own row; overhead must compare Region + // against Join's own 5ns, not Join's transitive total. + const nested = buildDataflowStructure( + [ + { ...OPS[0], id: "20", address: ["6"], name: "Dataflow" }, + { + ...OPS[0], + id: "21", + address: ["6", "1"], + name: "Region", + elapsedNs: "10", + }, + { + ...OPS[0], + id: "22", + address: ["6", "1", "1"], + name: "Join", + elapsedNs: "5", + }, + { + ...OPS[0], + id: "23", + address: ["6", "1", "1", "1"], + name: "GrandchildLeaf", + elapsedNs: "1000", + }, + ], + [], + [], + ); + const region = nested.nodes.get(nodeIdOf([6, 1]))!; + // Join's transitive elapsed (5 + 1000 = 1005) would make this wildly + // negative if used instead of Join's own (5): 10 - 5 = 5. + expect(region.overheadNs).toEqual(5n); + }); + describe("skew", () => { // Join (id 12, [5,1,1]): worker 0 does 10ns/1000B, worker 1 does // 30ns/3000B -> 2x every direction. Map (id 13, [5,1,2]): only worker 0 @@ -210,7 +293,32 @@ describe("buildDataflowStructure", () => { expect(map.ownSkew.memorySkew).toEqual(0); // no memory data at all const sink = s.nodes.get(nodeIdOf([5, 2]))!; - expect(sink.ownSkew).toEqual({ cpuSkew: 0, memorySkew: 0 }); + expect(sink.ownSkew).toEqual({ + cpuSkew: 0, + memorySkew: 0, + scheduleSkew: 0, + }); + }); + + it("computes schedule-count skew the same way as CPU/memory skew", () => { + const s = buildDataflowStructure(OPS, CHANNELS, LIR_SPANS, [ + { + id: "12", + workerId: "0", + elapsedNs: null, + arrangementSize: null, + scheduleCount: "100", + }, + { + id: "12", + workerId: "1", + elapsedNs: null, + arrangementSize: null, + scheduleCount: "300", + }, + ]); + const join = s.nodes.get(nodeIdOf([5, 1, 1]))!; + expect(join.ownSkew.scheduleSkew).toBeCloseTo(1.5); // 300 / ((100+300)/2) }); it("computes transitive skew as the max of its own and its subtree's skew, matching EXPLAIN ANALYZE's rollup, not a merged-and-then-ratioed vector", () => { @@ -869,6 +977,37 @@ describe("decorateGraph", () => { expect(d.nodeColors.get(nodeIdOf([5, 2]))).toEqual("heat(0.00)"); }); + it("scheduleSkew heatmap colors by transitive schedule-count skew, same as cpu/memory", () => { + const skewed = buildDataflowStructure(OPS, CHANNELS, LIR_SPANS, [ + { + id: "12", + workerId: "0", + elapsedNs: null, + arrangementSize: null, + scheduleCount: "100", + }, + { + id: "12", + workerId: "1", + elapsedNs: null, + arrangementSize: null, + scheduleCount: "300", + }, + ]); + const skewedRoot = deriveVisibleGraph(skewed, skewed.root); + const d = decorateGraph( + skewedRoot, + { ...DEFAULT_FILTERS, heatmap: "scheduleSkew" }, + heat, + null, + 4, + ); + expect(d.nodeColors.get(regionId)).toEqual( + `heat(${(Math.log2(1.5) / Math.log2(4)).toFixed(2)})`, + ); + expect(d.nodeColors.get(nodeIdOf([5, 2]))).toEqual("heat(0.00)"); + }); + it("skew heatmap floor is exactly ratio 1, ceiling is exactly the worker count", () => { const perfectlyEven = buildDataflowStructure(OPS, CHANNELS, LIR_SPANS, [ { id: "12", workerId: "0", elapsedNs: "100", arrangementSize: "0" }, @@ -1079,11 +1218,13 @@ describe("lirTree", () => { arrangementRecords: 300n, arrangementSize: 0n, elapsedNs: 30n, + scheduleCount: 0n, }); expect(root.children[0].summary).toEqual({ arrangementRecords: 100n, arrangementSize: 0n, elapsedNs: 10n, + scheduleCount: 0n, }); }); }); @@ -1096,6 +1237,7 @@ describe("rerouteHiddenNodes", () => { stats: null, transitive: null, transitiveSkew: null, + overheadNs: null, childCount: 0, lir: [], address: null, @@ -1230,6 +1372,7 @@ describe("groupByLir", () => { stats: null, transitive: null, transitiveSkew: null, + overheadNs: null, childCount: 0, lir, address: null, @@ -1243,6 +1386,7 @@ describe("groupByLir", () => { stats: null, transitive: null, transitiveSkew: null, + overheadNs: null, childCount: 0, lir: [], address: null, diff --git a/console/src/platform/dataflows/dataflowGraph.ts b/console/src/platform/dataflows/dataflowGraph.ts index 7d2ad8b1249f8..5d751d4d4e26d 100644 --- a/console/src/platform/dataflows/dataflowGraph.ts +++ b/console/src/platform/dataflows/dataflowGraph.ts @@ -21,6 +21,12 @@ export interface NodeStats { arrangementRecords: bigint; arrangementSize: bigint; elapsedNs: bigint; + // Total times this operator was scheduled (summed over the scheduling + // duration histogram's buckets). A high count relative to elapsed time + // means the operator yields often -- each activation does little work, + // so per-activation overhead (waking the worker, progress tracking) + // dominates -- which raw elapsed time alone doesn't surface. + scheduleCount: bigint; } // How unevenly work for this node is spread across workers: worst worker's @@ -32,6 +38,7 @@ export interface NodeStats { export interface SkewStats { cpuSkew: number; memorySkew: number; + scheduleSkew: number; } export interface LirInfo { @@ -60,6 +67,19 @@ export interface DataflowNode { // which a merge would hide and a max surfaces. transitiveSkew: SkewStats; lir: LirInfo[]; // one entry per export span covering this operator id + // Timely's own-elapsed accounting is exclusive self time: an operator's + // own.elapsedNs already excludes time spent inside any nested/child + // operator's own activation, even though scheduling a region + // synchronously runs its children first (verified against live + // introspection data: a region's own.elapsedNs can come in well below + // its direct children's own.elapsedNs summed, which an inclusive + // wall-clock reading never could). Comparing self time to self time (own + // minus direct children's own, not their transitive) isolates the + // region's own dispatch overhead: pointstamp/progress tracking, + // activation logic, everything beyond simply handing off to each direct + // child once. Equals own.elapsedNs for a leaf (no children to subtract), + // which isn't a meaningful "overhead" reading on its own. + overheadNs: bigint; } export interface Channel { @@ -102,6 +122,9 @@ export interface VisibleNode { stats: NodeStats | null; transitive: NodeStats | null; transitiveSkew: SkewStats | null; + // null for synthetic port nodes, same as transitive/transitiveSkew (no + // operator row of their own to compute it from). + overheadNs: bigint | null; childCount: number; lir: LirInfo[]; address: Address | null; @@ -146,6 +169,11 @@ export interface OperatorRow { arrangementRecords: bigint | number | string | null; arrangementSize: bigint | number | string | null; elapsedNs: bigint | number | string | null; + // Optional (unlike the fields above) so fixtures unrelated to schedule + // counting don't all need updating: real query rows always provide it + // (see useDataflowGraphData.ts's COALESCE), omitting it just reads as 0 + // via toBigInt's undefined handling, same as an explicit null would. + scheduleCount?: bigint | number | string | null; } export interface ChannelRow { id: number | string; @@ -171,6 +199,8 @@ export interface PerWorkerStatRow { workerId: bigint | number | string; elapsedNs: bigint | number | string | null; arrangementSize: bigint | number | string | null; + // See OperatorRow.scheduleCount for why this is optional. + scheduleCount?: bigint | number | string | null; } const toBigInt = (v: bigint | number | string | null | undefined): bigint => @@ -238,15 +268,22 @@ export function buildDataflowStructure( name: "", parent: null, children: [], - own: { arrangementRecords: 0n, arrangementSize: 0n, elapsedNs: 0n }, + own: { + arrangementRecords: 0n, + arrangementSize: 0n, + elapsedNs: 0n, + scheduleCount: 0n, + }, transitive: { arrangementRecords: 0n, arrangementSize: 0n, elapsedNs: 0n, + scheduleCount: 0n, }, - ownSkew: { cpuSkew: 0, memorySkew: 0 }, - transitiveSkew: { cpuSkew: 0, memorySkew: 0 }, + ownSkew: { cpuSkew: 0, memorySkew: 0, scheduleSkew: 0 }, + transitiveSkew: { cpuSkew: 0, memorySkew: 0, scheduleSkew: 0 }, lir: [], + overheadNs: 0n, }, ], ]), @@ -273,6 +310,7 @@ export function buildDataflowStructure( arrangementRecords: toBigInt(row.arrangementRecords), arrangementSize: toBigInt(row.arrangementSize), elapsedNs: toBigInt(row.elapsedNs), + scheduleCount: toBigInt(row.scheduleCount), }; nodeIdByOperatorId.set(opId.toString(), id); nodes.set(id, { @@ -284,8 +322,8 @@ export function buildDataflowStructure( children: [], own, transitive: own, // replaced below - ownSkew: { cpuSkew: 0, memorySkew: 0 }, // replaced below - transitiveSkew: { cpuSkew: 0, memorySkew: 0 }, // replaced below + ownSkew: { cpuSkew: 0, memorySkew: 0, scheduleSkew: 0 }, // replaced below + transitiveSkew: { cpuSkew: 0, memorySkew: 0, scheduleSkew: 0 }, // replaced below lir: spans .filter((s) => s.start <= opId && opId < s.end) .map(({ exportId, lirId, parentLirId, nesting, operator }) => ({ @@ -295,6 +333,7 @@ export function buildDataflowStructure( nesting, operator, })), + overheadNs: 0n, // replaced below }); } const roots: NodeId[] = []; @@ -317,6 +356,7 @@ export function buildDataflowStructure( const ownCpuByNode = new Map(); const ownMemoryByNode = new Map(); + const ownScheduleByNode = new Map(); for (const row of perWorkerStats) { const id = nodeIdByOperatorId.get(toBigInt(row.id).toString()); if (!id) continue; // operator outside this dataflow's own address tree @@ -334,10 +374,19 @@ export function buildDataflowStructure( ); ownMemoryByNode.set(id, vec); } + if (row.scheduleCount != null) { + const vec = ownScheduleByNode.get(id) ?? new Map(); + vec.set( + workerId, + (vec.get(workerId) ?? 0n) + toBigInt(row.scheduleCount), + ); + ownScheduleByNode.set(id, vec); + } } let maxOwnElapsedNs = 0n; let maxOwnArrangementSize = 0n; + let maxOwnScheduleCount = 0n; for (const node of nodes.values()) { if (node.own.elapsedNs > maxOwnElapsedNs) { maxOwnElapsedNs = node.own.elapsedNs; @@ -345,12 +394,16 @@ export function buildDataflowStructure( if (node.own.arrangementSize > maxOwnArrangementSize) { maxOwnArrangementSize = node.own.arrangementSize; } + if (node.own.scheduleCount > maxOwnScheduleCount) { + maxOwnScheduleCount = node.own.scheduleCount; + } } const fillTransitive = (id: NodeId): void => { const node = nodes.get(id)!; const ownCpuVec = ownCpuByNode.get(id) ?? new Map(); const ownMemoryVec = ownMemoryByNode.get(id) ?? new Map(); + const ownScheduleVec = ownScheduleByNode.get(id) ?? new Map(); node.ownSkew = { cpuSkew: passesMagnitudeFloor(node.own.elapsedNs, maxOwnElapsedNs) ? skewRatio(ownCpuVec) @@ -361,21 +414,33 @@ export function buildDataflowStructure( ) ? skewRatio(ownMemoryVec) : 0, + scheduleSkew: passesMagnitudeFloor( + node.own.scheduleCount, + maxOwnScheduleCount, + ) + ? skewRatio(ownScheduleVec) + : 0, }; const t = { ...node.own }; let cpuSkew = node.ownSkew.cpuSkew; let memorySkew = node.ownSkew.memorySkew; + let scheduleSkew = node.ownSkew.scheduleSkew; + let childrenOwnElapsedNs = 0n; for (const c of node.children) { fillTransitive(c); const child = nodes.get(c)!; t.arrangementRecords += child.transitive.arrangementRecords; t.arrangementSize += child.transitive.arrangementSize; t.elapsedNs += child.transitive.elapsedNs; + t.scheduleCount += child.transitive.scheduleCount; + childrenOwnElapsedNs += child.own.elapsedNs; cpuSkew = Math.max(cpuSkew, child.transitiveSkew.cpuSkew); memorySkew = Math.max(memorySkew, child.transitiveSkew.memorySkew); + scheduleSkew = Math.max(scheduleSkew, child.transitiveSkew.scheduleSkew); } node.transitive = t; - node.transitiveSkew = { cpuSkew, memorySkew }; + node.transitiveSkew = { cpuSkew, memorySkew, scheduleSkew }; + node.overheadNs = node.own.elapsedNs - childrenOwnElapsedNs; }; fillTransitive(roots[0]); return { @@ -496,6 +561,7 @@ export function deriveVisibleGraph( stats: kind === "region" ? node.transitive : node.own, transitive: node.transitive, transitiveSkew: node.transitiveSkew, + overheadNs: node.overheadNs, childCount: node.children.length, lir: node.lir, address: node.address, @@ -658,6 +724,7 @@ export function deriveVisibleGraph( stats: null, transitive: null, transitiveSkew: null, + overheadNs: null, childCount: 0, lir: [], address: null, @@ -936,7 +1003,13 @@ export function rerouteHiddenNodes( export interface Filters { search: string; hideIdle: boolean; - heatmap: "off" | "elapsed" | "size" | "cpuSkew" | "memorySkew"; + heatmap: + | "off" + | "elapsed" + | "size" + | "cpuSkew" + | "memorySkew" + | "scheduleSkew"; heatmapThreshold: number; // 0..1 fraction of max showLirGroups: boolean; } @@ -974,7 +1047,7 @@ export function decorateGraph( // is mathematically bounded above by it (one worker doing 100% of a // fixed amount of work, the rest 0%, is the most skewed that work can // ever be split across this many workers). Used only by the cpuSkew/ - // memorySkew heatmap modes, as a fixed color-scale ceiling. + // memorySkew/scheduleSkew heatmap modes, as a fixed color-scale ceiling. workerCount: number, ): GraphDecorations { const d: GraphDecorations = { @@ -1014,9 +1087,14 @@ export function decorateGraph( return n.transitiveSkew?.cpuSkew ?? 0; case "memorySkew": return n.transitiveSkew?.memorySkew ?? 0; + case "scheduleSkew": + return n.transitiveSkew?.scheduleSkew ?? 0; } }; - const isSkew = heatmap === "cpuSkew" || heatmap === "memorySkew"; + const isSkew = + heatmap === "cpuSkew" || + heatmap === "memorySkew" || + heatmap === "scheduleSkew"; const candidates = graph.nodes.filter((n) => n.kind !== "port"); if (isSkew) { // A skew ratio is bounded above by workerCount (see decorateGraph's @@ -1147,6 +1225,7 @@ export function lirSummary( arrangementRecords: 0n, arrangementSize: 0n, elapsedNs: 0n, + scheduleCount: 0n, }; for (const id of memberIds) { const own = structure.nodes.get(id)?.own; @@ -1154,6 +1233,7 @@ export function lirSummary( summary.arrangementRecords += own.arrangementRecords; summary.arrangementSize += own.arrangementSize; summary.elapsedNs += own.elapsedNs; + summary.scheduleCount += own.scheduleCount; } return summary; } diff --git a/console/src/platform/dataflows/nodeStyle.test.ts b/console/src/platform/dataflows/nodeStyle.test.ts index f46051e775586..ff97b177c92f1 100644 --- a/console/src/platform/dataflows/nodeStyle.test.ts +++ b/console/src/platform/dataflows/nodeStyle.test.ts @@ -56,6 +56,7 @@ describe("nodeFillColor", () => { stats: null, transitive: null, transitiveSkew: null, + overheadNs: null, childCount: 0, lir: [], address: null, @@ -72,6 +73,7 @@ describe("nodeFillColor", () => { arrangementRecords: 100n, arrangementSize: 100n, elapsedNs: 0n, + scheduleCount: 0n, }, }); expect(unarranged).toEqual(operatorColor("Reduce")); @@ -87,6 +89,7 @@ describe("nodeFillColor", () => { arrangementRecords: 100n, arrangementSize: 100n, elapsedNs: 0n, + scheduleCount: 0n, }, }); expect(noArrangement).not.toEqual(arranged); From da45de97d4c94838339e5a81462344c1be3b06a9 Mon Sep 17 00:00:00 2001 From: Moritz Hoffmann Date: Wed, 8 Jul 2026 22:42:40 +0200 Subject: [PATCH 33/45] Add schedules heatmap, unfiltered own skew, and a region's own stats to the detail panel Co-Authored-By: Claude Sonnet 5 --- .../platform/dataflows/DataflowToolbar.tsx | 1 + .../platform/dataflows/NodeDetailPanel.tsx | 31 +++++++++++----- .../platform/dataflows/dataflowGraph.test.ts | 34 ++++++++++++++++++ .../src/platform/dataflows/dataflowGraph.ts | 36 +++++++++++++++++++ .../src/platform/dataflows/nodeStyle.test.ts | 2 ++ 5 files changed, 95 insertions(+), 9 deletions(-) diff --git a/console/src/platform/dataflows/DataflowToolbar.tsx b/console/src/platform/dataflows/DataflowToolbar.tsx index 17202ac581204..4f53a5ce2a5bb 100644 --- a/console/src/platform/dataflows/DataflowToolbar.tsx +++ b/console/src/platform/dataflows/DataflowToolbar.tsx @@ -118,6 +118,7 @@ export const DataflowToolbar = ({ + diff --git a/console/src/platform/dataflows/NodeDetailPanel.tsx b/console/src/platform/dataflows/NodeDetailPanel.tsx index 1a00aeeda309f..c25d4fc6c9ab5 100644 --- a/console/src/platform/dataflows/NodeDetailPanel.tsx +++ b/console/src/platform/dataflows/NodeDetailPanel.tsx @@ -152,18 +152,18 @@ const NodeDetail = ({ {node.childCount > 0 && ( )} - {node.stats && ( + {node.own && ( <> - - + + )} {node.childCount > 0 && ( @@ -189,15 +189,28 @@ const NodeDetail = ({ /> )} - {node.transitiveSkew && ( + {node.ownSkew && ( <> - + + + + )} + {node.transitiveSkew && node.childCount > 0 && ( + <> + + diff --git a/console/src/platform/dataflows/dataflowGraph.test.ts b/console/src/platform/dataflows/dataflowGraph.test.ts index fe6e7c0ff217d..e56921d63ae76 100644 --- a/console/src/platform/dataflows/dataflowGraph.test.ts +++ b/console/src/platform/dataflows/dataflowGraph.test.ts @@ -365,6 +365,27 @@ describe("buildDataflowStructure", () => { const region = s.nodes.get(nodeIdOf([5, 1]))!; expect(region.transitiveSkew.cpuSkew).toEqual(1); }); + + it("keeps the real own skew available unfiltered, for display, even when the magnitude floor suppresses it for rollup", () => { + const skewedOps = OPS.map((o) => + o.id === "12" + ? { ...o, elapsedNs: "1000000" } + : o.id === "13" + ? { ...o, elapsedNs: "1" } + : o, + ); + const s = buildDataflowStructure(skewedOps, CHANNELS, LIR_SPANS, [ + { id: "12", workerId: "0", elapsedNs: "500000", arrangementSize: "0" }, + { id: "12", workerId: "1", elapsedNs: "500000", arrangementSize: "0" }, + { id: "13", workerId: "0", elapsedNs: "1", arrangementSize: "0" }, + { id: "13", workerId: "1", elapsedNs: "0", arrangementSize: "0" }, + { id: "13", workerId: "2", elapsedNs: "0", arrangementSize: "0" }, + { id: "13", workerId: "3", elapsedNs: "0", arrangementSize: "0" }, + ]); + const map = s.nodes.get(nodeIdOf([5, 1, 2]))!; + expect(map.ownSkew.cpuSkew).toEqual(0); // suppressed, for rollup safety + expect(map.ownSkewRaw.cpuSkew).toBeCloseTo(4); // 1 / ((1+0+0+0)/4) + }); }); }); @@ -406,6 +427,13 @@ describe("deriveVisibleGraph", () => { const region = g.nodes[0]; expect(region.stats).toEqual(s.nodes.get(regionId)!.transitive); expect(region.childCount).toEqual(2); + // `stats` collapses to the subtree total for a quick canvas-level read, + // but the detail panel needs the region's own activity (5ns) kept + // distinct from that total (5+7+1=13ns), or a region's own dispatch + // cost becomes invisible next to its children's. + expect(region.own).toEqual(s.nodes.get(regionId)!.own); + expect(region.own!.elapsedNs).toEqual(5n); + expect(region.transitive!.elapsedNs).toEqual(13n); }); it("drilling into the region shows its children and an inbound port", () => { @@ -1236,6 +1264,8 @@ describe("rerouteHiddenNodes", () => { label: id, stats: null, transitive: null, + own: null, + ownSkew: null, transitiveSkew: null, overheadNs: null, childCount: 0, @@ -1371,6 +1401,8 @@ describe("groupByLir", () => { label: id, stats: null, transitive: null, + own: null, + ownSkew: null, transitiveSkew: null, overheadNs: null, childCount: 0, @@ -1385,6 +1417,8 @@ describe("groupByLir", () => { label: id, stats: null, transitive: null, + own: null, + ownSkew: null, transitiveSkew: null, overheadNs: null, childCount: 0, diff --git a/console/src/platform/dataflows/dataflowGraph.ts b/console/src/platform/dataflows/dataflowGraph.ts index 5d751d4d4e26d..db6d46ccfb7f7 100644 --- a/console/src/platform/dataflows/dataflowGraph.ts +++ b/console/src/platform/dataflows/dataflowGraph.ts @@ -59,6 +59,14 @@ export interface DataflowNode { own: NodeStats; transitive: NodeStats; // own + subtree, precomputed ownSkew: SkewStats; + // Same ratios as ownSkew, but never suppressed by the magnitude floor. + // ownSkew exists to seed transitiveSkew's MAX rollup (see its comment for + // why negligible nodes must not pollute an ancestor's reported skew); + // ownSkewRaw is this node's actual measured skew, for display when this + // exact node is selected, where a floor built for ancestor-safety would + // otherwise misreport a real (if individually low-stakes) imbalance as + // "no data". + ownSkewRaw: SkewStats; // The worst skew anywhere in the subtree (this node's own, or any // descendant's), matching EXPLAIN ANALYZE's own MAX-based rollup. Not a // merged-and-then-ratioed per-worker vector: two operators skewed @@ -119,8 +127,22 @@ export interface VisibleNode { id: string; kind: "operator" | "region" | "port"; label: string; + // For a region, the rolled-up subtree total (same as transitive): a + // quick canvas-level preview, not this node's own activity. See `own` + // for that. stats: NodeStats | null; transitive: NodeStats | null; + // This node's own stats, never rolled up: for a region, its own + // dispatch/activation cost distinct from anything its children did (see + // DataflowNode.overheadNs). `stats` collapses this into the subtree + // total for regions, which is the right quick read on the canvas but + // hides this node's own contribution entirely, which the detail panel + // needs to show on its own. + own: NodeStats | null; + // This node's own skew, unfiltered by the magnitude floor that guards + // transitiveSkew's ancestor rollup (see DataflowNode.ownSkewRaw) -- the + // real number for this exact node, not a heatmap-safe one. + ownSkew: SkewStats | null; transitiveSkew: SkewStats | null; // null for synthetic port nodes, same as transitive/transitiveSkew (no // operator row of their own to compute it from). @@ -281,6 +303,7 @@ export function buildDataflowStructure( scheduleCount: 0n, }, ownSkew: { cpuSkew: 0, memorySkew: 0, scheduleSkew: 0 }, + ownSkewRaw: { cpuSkew: 0, memorySkew: 0, scheduleSkew: 0 }, transitiveSkew: { cpuSkew: 0, memorySkew: 0, scheduleSkew: 0 }, lir: [], overheadNs: 0n, @@ -323,6 +346,7 @@ export function buildDataflowStructure( own, transitive: own, // replaced below ownSkew: { cpuSkew: 0, memorySkew: 0, scheduleSkew: 0 }, // replaced below + ownSkewRaw: { cpuSkew: 0, memorySkew: 0, scheduleSkew: 0 }, // replaced below transitiveSkew: { cpuSkew: 0, memorySkew: 0, scheduleSkew: 0 }, // replaced below lir: spans .filter((s) => s.start <= opId && opId < s.end) @@ -421,6 +445,11 @@ export function buildDataflowStructure( ? skewRatio(ownScheduleVec) : 0, }; + node.ownSkewRaw = { + cpuSkew: skewRatio(ownCpuVec), + memorySkew: skewRatio(ownMemoryVec), + scheduleSkew: skewRatio(ownScheduleVec), + }; const t = { ...node.own }; let cpuSkew = node.ownSkew.cpuSkew; let memorySkew = node.ownSkew.memorySkew; @@ -560,6 +589,8 @@ export function deriveVisibleGraph( label: node.name, stats: kind === "region" ? node.transitive : node.own, transitive: node.transitive, + own: node.own, + ownSkew: node.ownSkewRaw, transitiveSkew: node.transitiveSkew, overheadNs: node.overheadNs, childCount: node.children.length, @@ -723,6 +754,8 @@ export function deriveVisibleGraph( label: `${p.direction} ${p.port}`, stats: null, transitive: null, + own: null, + ownSkew: null, transitiveSkew: null, overheadNs: null, childCount: 0, @@ -1007,6 +1040,7 @@ export interface Filters { | "off" | "elapsed" | "size" + | "schedules" | "cpuSkew" | "memorySkew" | "scheduleSkew"; @@ -1083,6 +1117,8 @@ export function decorateGraph( return Number(n.transitive?.elapsedNs ?? 0n); case "size": return Number(n.transitive?.arrangementSize ?? 0n); + case "schedules": + return Number(n.transitive?.scheduleCount ?? 0n); case "cpuSkew": return n.transitiveSkew?.cpuSkew ?? 0; case "memorySkew": diff --git a/console/src/platform/dataflows/nodeStyle.test.ts b/console/src/platform/dataflows/nodeStyle.test.ts index ff97b177c92f1..da6f714b1180d 100644 --- a/console/src/platform/dataflows/nodeStyle.test.ts +++ b/console/src/platform/dataflows/nodeStyle.test.ts @@ -55,6 +55,8 @@ describe("nodeFillColor", () => { label: "Reduce", stats: null, transitive: null, + own: null, + ownSkew: null, transitiveSkew: null, overheadNs: null, childCount: 0, From 5f4fc14beedad28d3410281fa88c94bb429fca48 Mon Sep 17 00:00:00 2001 From: Moritz Hoffmann Date: Wed, 8 Jul 2026 22:28:11 +0200 Subject: [PATCH 34/45] Remove superseded LIR-grouping-boxes design doc and plan Grouping boxes shipped as part of the dataflow visualizer rebuild; these predate that work and no longer reflect the implementation. Co-Authored-By: Claude Sonnet 5 --- .../20260707_dataflow_lir_grouping_boxes.md | 102 -- .../2026-07-07-dataflow-lir-grouping-boxes.md | 1301 ----------------- 2 files changed, 1403 deletions(-) delete mode 100644 console/doc/design/20260707_dataflow_lir_grouping_boxes.md delete mode 100644 docs/superpowers/plans/2026-07-07-dataflow-lir-grouping-boxes.md diff --git a/console/doc/design/20260707_dataflow_lir_grouping_boxes.md b/console/doc/design/20260707_dataflow_lir_grouping_boxes.md deleted file mode 100644 index 02ef10ca6191c..0000000000000 --- a/console/doc/design/20260707_dataflow_lir_grouping_boxes.md +++ /dev/null @@ -1,102 +0,0 @@ -# Dataflow visualizer: LIR grouping boxes - -- Associated: [CNS-108](https://linear.app/materializeinc/issue/CNS-108), [CNS-109](https://linear.app/materializeinc/issue/CNS-109), builds on `20260706_dataflow_visualizer_rebuild.md` - -## The Problem - -The rebuilt dataflow visualizer (`20260706_dataflow_visualizer_rebuild.md`) shows LIR membership only as a badge, a tooltip on hover, and a side-panel tree. -Regions and LIR spans were deliberately kept as two different hierarchies: a region is a drill-down scope, a LIR span is an unrelated grouping over the same operators. -A user looking at a scope's operator graph cannot see at a glance which operators came from the same LIR node, which is exactly the grouping `EXPLAIN` output already draws (dashed boxes, one per LIR node, nested when LIR spans nest). -Matching that visual on the canvas would let a user correlate the rendered graph directly with the plan they already read. - -## Success Criteria - -* A user can toggle "Show LIR groups" in the toolbar and see, for the current scope, a dashed box around each LIR node's member operators, nested to match LIR span nesting. -* A box shows the LIR operator string as a header label. -* Clicking a box's header opens the same detail info the LIR side panel already shows for that LIR node (id, operator, member count, aggregated stats). -* Toggling off returns to exactly today's flat layout, byte-identical to the current behavior. -* Search, dimming, and heatmap decoration keep working unchanged on individual operators inside a box. - -## Out of Scope - -* Chains: gluing linear (in-degree 1, out-degree 1) operator runs into one visually merged box, a separate idea explored and shelved during this design's brainstorming as added complexity without a clean elk-native way to render it. -* Any change to how LIR data is queried or computed (`useDataflowGraphData.ts`, `mz_lir_mapping`) — grouping is a pure derivation over data already fetched. -* Heat-coloring or aggregate stats displayed directly on a group box; a box is a label-only wrapper, its members carry all stats as they do today. -* Drill-down or double-click behavior on a group box. LIR groups are not scopes and do not navigate. - -## Solution Proposal - -Add a pure grouping step between the existing decoration pass and the elk layout step, and teach the elk/React Flow rendering path to lay out and draw nested compound nodes, which it does not do today. - -### Data model - -A dataflow's direct children in a scope are already ordered by operator id, and each carries `lir: LirInfo[]`, an outer-to-inner nesting stack. -LIR spans are contiguous ranges over operator ids and properly nested: a region is always fully enclosed by a single LIR node's range, and a single region can contain operators from multiple LIR nodes, but always consecutive by operator id. -A new pure function in `dataflowGraph.ts`, `groupByLir(nodes: VisibleNode[]): LirGrouping`, exploits this with a single left-to-right scan and an explicit stack of open groups: for each node, pop groups whose `lirId` is no longer a prefix of the node's stack, push newly opened ones. -This is a bracket-matching walk, valid because of the contiguity and proper-nesting invariant above; it does not need to handle malformed input, since the invariant is guaranteed by how LIR spans are constructed. - -Rather than replacing the flat node list with a nested tree, grouping is a separate layer over it, the same way `decorateGraph` layers dimming and color over an unchanged `VisibleGraph`: every downstream consumer (search, dimming, click handling) keeps working against the flat list untouched, and only the layout/render step reads the grouping layer. -Some children stay ungrouped (empty `lir`, e.g. ports, which carry no LIR data and are never absorbed into a group); every other child, and every group itself, has an immediate parent recorded in `parentOf`: - -```typescript -interface LirGroupNode { - id: NodeId; // `${exportId}/${lirId}`, matching lirIndex's existing key - exportId: string; - lirId: string; - operator: string; - nesting: number; -} - -interface LirGrouping { - groups: LirGroupNode[]; // outer groups before the inner groups nested in them - parentOf: Map; // a VisibleNode or LirGroupNode id -> its immediate enclosing group's id -} -``` - -A dataflow can back several exports, and an operator's `lir: LirInfo[]` can hold entries from more than one of them at once, when a shared subplan feeds multiple exports. -A compound-node tree needs a single parent chain per node, so `groupByLir` groups by one export only: the lowest `exportId` (string-compared) present among the scope's direct children. -A child whose only `lir` entries belong to a different export renders ungrouped in that scope, alongside children with no LIR data at all. -This can under-group a multi-export scope, but never draws two overlapping box hierarchies over the same operators. - -### Rendering and layout - -`elkGraph.ts`'s `toElkGraph` is flat-only today, by its own comment: a view is always one scope's direct children, nothing nests. -LIR groups are the first use of elk's native compound-node support in this codebase. -`toElkGraph` grows a case for `LirGroupNode`: it becomes an elk parent node with its members as elk children (recursing for nested groups), no fixed width or height, since elk auto-sizes a parent from its children plus `elk.padding`, reserving a header strip for the label. -Ungrouped siblings and group roots sit together as top-level elk children, exactly like today's flat list when there are no groups. -Edges can still cross group boundaries (an edge's endpoints are unrelated to which LIR group either side belongs to), so the root layout options need `elk.hierarchyHandling: INCLUDE_CHILDREN`, harmless when there are no groups at all. - -React Flow's `parentId` mechanism (nested parent nodes, already used as one of the reasons the project chose React Flow) maps onto this directly, and needs no coordinate conversion: elk reports a child's `x`/`y` relative to its immediate parent's origin, the same convention React Flow uses for a node with `parentId` set. -A new `LirGroupNode` component renders the dashed border and a small header label (from `LirInfo.operator`, truncated with ellipsis like the existing LIR panel), colored by `lirId`, sized from elk's computed group bounds. -It carries no stats and is unaffected by heatmap coloring. - -Pipeline order: `deriveVisibleGraph` → `decorateGraph` → (if the toggle is on) `groupByLir` → `toElkGraph` → layout → render. -Grouping is the last, purely structural step over already-decorated nodes, so search highlighting, dimming, and heatmap coloring apply to members exactly as they do without groups; a group box just visually contains whichever of its members are dimmed or highlighted. -With the toggle off, `deriveVisibleGraph`'s output goes straight to `toElkGraph`, unrouted through `groupByLir`, identical to today's behavior. - -### Interactions - -Clicking a group's header selects that LIR node and opens `NodeDetailPanel` with the same info a `LirPanel` tree entry already produces (`lirIndex`/`lirSummary`): id, operator string, member count, aggregated stats. -No double-click or drill-down action on a group; LIR groups are not scopes. - -### Toolbar - -A new "Show LIR groups" checkbox in `DataflowToolbar`, off by default, alongside the existing search/hide-idle/heatmap controls. -Off, the view is exactly today's flat grid. - -## Minimal Viable Prototype - -The risk worth de-risking before full build is elk's compound-node layout quality and cost on a real, multi-level LIR-nested dataflow, since this codebase has never exercised that path. -A thin slice: run `groupByLir` and the elk-nesting change against one scope of the reference dataflow from the original design doc (or another with visibly nested LIR spans), confirm cross-group edge routing looks reasonable and layout time stays acceptable, before building out the toolbar toggle, `LirGroupNode` styling, and the click-to-detail wiring. - -## Alternatives - -* Render-layer-only grouping: keep elk flat, compute each group's bounding box in the render layer from its members' already-laid-out positions, and draw an absolutely-positioned dashed `` behind them without going through elk's compound-node support or React Flow's `parentId`. - Avoids touching `elkGraph.ts`, but duplicates layout knowledge elk already has (padding, header reservation, nested spacing) in a second place, and elk has no reason to leave room for a label strip or extra padding around a group it doesn't know exists, so boxes would collide with sibling groups or overlap edges routed through the gap. -* Always-on, no toggle. - Simpler UI, but permanently changes the default look of every scope that has LIR data and adds elk layout cost (nested compound-node layout is slower than flat) to every view, including ones where the operator/LIR correlation isn't what the user is looking for. - -## Open questions - -* None blocking. - Chains (gluing linear operator runs) were explored in the same brainstorming session and shelved; if revisited, see the discussion of why render-layer-only glue was rejected there, which mirrors the render-layer alternative rejected above. diff --git a/docs/superpowers/plans/2026-07-07-dataflow-lir-grouping-boxes.md b/docs/superpowers/plans/2026-07-07-dataflow-lir-grouping-boxes.md deleted file mode 100644 index 879096d515371..0000000000000 --- a/docs/superpowers/plans/2026-07-07-dataflow-lir-grouping-boxes.md +++ /dev/null @@ -1,1301 +0,0 @@ -# Dataflow LIR Grouping Boxes Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Draw nested dashed boxes around a scope's operators that share a LIR node, toggled from the dataflow visualizer toolbar, matching `EXPLAIN`'s own LIR grouping. - -**Architecture:** A new pure function `groupByLir` derives a grouping layer (which group each `VisibleNode` sits in, if any) over the already-flat `VisibleGraph`, the same way `decorateGraph` layers dimming/color over it without replacing it. `elkGraph.ts` turns that layer into elk's native compound-node nesting (the first use of it in this codebase); `DataflowGraphView.tsx` turns elk's result into React Flow `parentId` nesting and a new `LirGroupNode` render component. A toolbar switch and a `NodeDetailPanel` case round out the feature. - -**Tech Stack:** TypeScript, React, `@xyflow/react` (React Flow), `elkjs`, Chakra UI, Vitest. - -## Global Constraints - -* Design doc: `console/doc/design/20260707_dataflow_lir_grouping_boxes.md`. Every task below implements a specific section of it; read it first if anything here is ambiguous. -* Toggle defaults to off (`Filters.showLirGroups: false`); with it off, output must be byte-identical to today's flat layout. -* A node whose `lir` entries all belong to an export other than the scope's chosen export (lowest `exportId` present) renders ungrouped, never in two overlapping group hierarchies. -* No drill-down/double-click behavior on a group box; groups are label-only wrappers, never scopes. -* Run `cd console && yarn typecheck && yarn lint && yarn test ` before every commit in this plan (see `mz-run`/`mz-test` conventions); this plan's own commands below are the minimum, not a substitute. -* Copy comments verbatim where this plan quotes them from the design doc or existing code; don't invent new phrasing for an already-settled explanation. - ---- - -### Task 1: `groupByLir` pure function - -**Files:** -- Modify: `console/src/platform/dataflows/dataflowGraph.ts` -- Test: `console/src/platform/dataflows/dataflowGraph.test.ts` - -**Interfaces:** -- Consumes: `VisibleNode` (already defined in this file: `id`, `operatorId: bigint | null`, `lir: LirInfo[]`). -- Produces: - ```typescript - export interface LirGroupNode { - id: NodeId; // `${exportId}/${lirId}`, matching lirIndex's existing key - exportId: string; - lirId: string; - operator: string; - nesting: number; - } - export interface LirGrouping { - groups: LirGroupNode[]; // outer groups appear before groups nested inside them - parentOf: Map; // a VisibleNode or LirGroupNode id -> its immediate enclosing group's id - } - export function groupByLir(nodes: readonly VisibleNode[]): LirGrouping - ``` - Later tasks import `groupByLir`, `LirGrouping`, `LirGroupNode` from this file. - -- [ ] **Step 1: Write the failing tests** - -Add to `console/src/platform/dataflows/dataflowGraph.test.ts`, after the existing `import` block add `type LirInfo` and `groupByLir` to the named imports from `./dataflowGraph`, then append this new `describe` block at the end of the file: - -```typescript -describe("groupByLir", () => { - const node = ( - id: string, - operatorId: number, - lir: LirInfo[] = [], - ): VisibleNode => ({ - id, - kind: "operator", - label: id, - stats: null, - transitive: null, - transitiveSkew: null, - childCount: 0, - lir, - address: null, - operatorId: BigInt(operatorId), - peers: [], - }); - const port = (id: string): VisibleNode => ({ - id, - kind: "port", - label: id, - stats: null, - transitive: null, - transitiveSkew: null, - childCount: 0, - lir: [], - address: null, - operatorId: null, - peers: [], - }); - const span = ( - exportId: string, - lirId: string, - nesting: number, - parentLirId: string | null, - ): LirInfo => ({ exportId, lirId, parentLirId, nesting, operator: lirId }); - - it("groups nothing when no node has lir data", () => { - const g = groupByLir([node("a", 1), node("b", 2)]); - expect(g.groups).toEqual([]); - expect(g.parentOf.size).toEqual(0); - }); - - it("groups a flat run under one lir id", () => { - const s = span("u1", "10", 0, null); - const g = groupByLir([node("a", 1, [s]), node("b", 2, [s]), node("c", 3, [s])]); - expect(g.groups.map((x) => x.id)).toEqual(["u1/10"]); - expect(g.parentOf.get("a")).toEqual("u1/10"); - expect(g.parentOf.get("b")).toEqual("u1/10"); - expect(g.parentOf.get("c")).toEqual("u1/10"); - }); - - it("nests an inner span inside an outer span", () => { - const outer = span("u1", "10", 0, null); - const inner = span("u1", "11", 1, "10"); - const g = groupByLir([ - node("a", 1, [outer]), - node("b", 2, [outer, inner]), - node("c", 3, [outer, inner]), - node("d", 4, [outer]), - ]); - expect(g.groups.map((x) => x.id)).toEqual(["u1/10", "u1/11"]); - expect(g.parentOf.get("u1/11")).toEqual("u1/10"); - expect(g.parentOf.get("a")).toEqual("u1/10"); - expect(g.parentOf.get("b")).toEqual("u1/11"); - expect(g.parentOf.get("c")).toEqual("u1/11"); - expect(g.parentOf.get("d")).toEqual("u1/10"); - }); - - it("leaves ports and lir-less nodes ungrouped alongside a group", () => { - const s = span("u1", "10", 0, null); - const g = groupByLir([node("a", 1, [s]), port("p"), node("b", 2)]); - expect(g.groups.map((x) => x.id)).toEqual(["u1/10"]); - expect(g.parentOf.get("a")).toEqual("u1/10"); - expect(g.parentOf.has("p")).toEqual(false); - expect(g.parentOf.has("b")).toEqual(false); - }); - - it("picks the lowest exportId, leaving other-export-only nodes ungrouped", () => { - const sA = span("u1", "5", 0, null); - const sB = span("u2", "3", 0, null); - const g = groupByLir([node("a", 1, [sA]), node("b", 2, [sB]), node("c", 3, [sA, sB])]); - expect(g.groups.map((x) => x.id)).toEqual(["u1/5"]); - expect(g.parentOf.get("a")).toEqual("u1/5"); - expect(g.parentOf.has("b")).toEqual(false); - expect(g.parentOf.get("c")).toEqual("u1/5"); - }); - - it("sorts input by operatorId regardless of array order", () => { - const outer = span("u1", "10", 0, null); - const inner = span("u1", "11", 1, "10"); - // Deliberately out of operatorId order. - const g = groupByLir([ - node("d", 4, [outer]), - node("b", 2, [outer, inner]), - node("a", 1, [outer]), - node("c", 3, [outer, inner]), - ]); - expect(g.groups.map((x) => x.id)).toEqual(["u1/10", "u1/11"]); - expect(g.parentOf.get("b")).toEqual("u1/11"); - expect(g.parentOf.get("c")).toEqual("u1/11"); - }); -}); -``` - -- [ ] **Step 2: Run the tests to verify they fail** - -Run: `cd console && yarn vitest run src/platform/dataflows/dataflowGraph.test.ts` -Expected: FAIL with `groupByLir is not exported` / `groupByLir is not a function`. - -- [ ] **Step 3: Implement `groupByLir`** - -Add to `console/src/platform/dataflows/dataflowGraph.ts`, after `lirTree` (end of file): - -```typescript -export interface LirGroupNode { - id: NodeId; // `${exportId}/${lirId}`, matching lirIndex's key - exportId: string; - lirId: string; - operator: string; - nesting: number; -} - -// A grouping layer over an already-derived VisibleGraph, the same way -// decorateGraph layers dimming and color over it: nothing here replaces -// `nodes`, so search, dimming, and click handling keep working against the -// flat list untouched. -export interface LirGrouping { - groups: LirGroupNode[]; // outer groups appear before groups nested inside them - parentOf: Map; // a VisibleNode or LirGroupNode id -> its immediate enclosing group's id -} - -// Groups a scope's direct children into the LIR tree that encloses them, for -// drawing nested boxes on the canvas. LIR spans are contiguous ranges over -// operator ids and properly nested: a region is always fully enclosed by a -// single LIR node's range. A dataflow can back several exports, and a shared -// subplan can carry lir entries from more than one of them at once; a -// compound-node tree needs one parent chain per node, so this groups by the -// lowest exportId (string-compared) present among the given nodes only. A -// node whose only lir entries belong to a different export renders -// ungrouped, alongside nodes with no lir data at all (e.g. ports). -export function groupByLir(nodes: readonly VisibleNode[]): LirGrouping { - const groups: LirGroupNode[] = []; - const parentOf = new Map(); - const ordered = nodes - .filter((n) => n.operatorId !== null) - .slice() - .sort((a, b) => { - const x = a.operatorId!; - const y = b.operatorId!; - return x < y ? -1 : x > y ? 1 : 0; - }); - if (ordered.length === 0) return { groups, parentOf }; - - let chosenExport: string | null = null; - for (const n of ordered) { - for (const l of n.lir) { - if (chosenExport === null || l.exportId < chosenExport) { - chosenExport = l.exportId; - } - } - } - if (chosenExport === null) return { groups, parentOf }; - - const groupIndex = new Map(); - const stack: LirGroupNode[] = []; // open groups, outer to inner - for (const node of ordered) { - const stackKeys = node.lir - .filter((l) => l.exportId === chosenExport) - .sort((a, b) => a.nesting - b.nesting) - .map((l) => `${l.exportId}/${l.lirId}`); - let commonLen = 0; - while ( - commonLen < stack.length && - commonLen < stackKeys.length && - stack[commonLen].id === stackKeys[commonLen] - ) { - commonLen++; - } - stack.length = commonLen; // pop back to the shared prefix - for (let depth = commonLen; depth < stackKeys.length; depth++) { - const key = stackKeys[depth]; - let group = groupIndex.get(key); - if (!group) { - const info = node.lir.find((l) => `${l.exportId}/${l.lirId}` === key)!; - group = { - id: key, - exportId: info.exportId, - lirId: info.lirId, - operator: info.operator, - nesting: info.nesting, - }; - groupIndex.set(key, group); - groups.push(group); - } - if (depth > 0) parentOf.set(group.id, stack[depth - 1].id); - stack.push(group); - } - if (stack.length > 0) parentOf.set(node.id, stack[stack.length - 1].id); - } - return { groups, parentOf }; -} -``` - -- [ ] **Step 4: Run the tests to verify they pass** - -Run: `cd console && yarn vitest run src/platform/dataflows/dataflowGraph.test.ts` -Expected: PASS, all tests including the six new `groupByLir` cases. - -- [ ] **Step 5: Typecheck and lint** - -Run: `cd console && yarn typecheck && yarn lint` -Expected: no errors. - -- [ ] **Step 6: Commit** - -```bash -git add console/src/platform/dataflows/dataflowGraph.ts console/src/platform/dataflows/dataflowGraph.test.ts -git commit -m "$(cat <<'EOF' -console: add groupByLir for LIR-node grouping boxes - -Pure derivation over a scope's already-derived VisibleGraph, mirroring -decorateGraph's layering rather than replacing the flat node list, so -downstream search/dimming/click handling is untouched. -EOF -)" -``` - ---- - -### Task 2: elk nested compound-node layout - -**Files:** -- Modify: `console/src/platform/dataflows/elkGraph.ts` -- Modify: `console/src/platform/dataflows/useElkLayout.ts` -- Test: `console/src/platform/dataflows/elkGraph.test.ts` - -**Interfaces:** -- Consumes: `LirGrouping`, `LirGroupNode` from Task 1's `dataflowGraph.ts`. -- Produces: `toElkGraph(graph: VisibleGraph, grouping?: LirGrouping): ElkNode` (grouping now optional third-party consumers can omit), `extractPositions` unchanged in signature but now recurses into nested `children`. `useElkLayout(graph, cacheKey, grouping?)` gains an optional third parameter. - -- [ ] **Step 1: Write the failing tests** - -Add to `console/src/platform/dataflows/elkGraph.test.ts`. First, add `groupByLir` to the import from `./dataflowGraph`, and add this import at the top (a real, synchronous elk instance — not the worker-backed `elk-api` used at runtime — for the empirical round-trip test): - -```typescript -import ELK from "elkjs"; -``` - -Then append: - -```typescript -describe("toElkGraph with a LIR grouping", () => { - const s = buildDataflowStructure(OPS, CHANNELS, LIR_SPANS); - - it("nests a group's members as elk children, sized only by content", () => { - const visible = deriveVisibleGraph(s, s.root); - const grouping = groupByLir(visible.nodes); - const elk = toElkGraph(visible, grouping); - const group = elk.children!.find((c) => c.id === grouping.groups[0].id)!; - expect(group.children!.map((c) => c.id)).toEqual( - expect.arrayContaining([nodeIdOf([5, 1])]), - ); - // Auto-sized by elk from its content, not one of NODE_DIMENSIONS's fixed sizes. - expect(group.width).toBeUndefined(); - expect(group.height).toBeUndefined(); - // Flat sibling stays a direct child of the root, not absorbed into the group. - expect(elk.children!.map((c) => c.id)).toEqual( - expect.arrayContaining([nodeIdOf([5, 2])]), - ); - }); - - it("keeps every edge at the root level regardless of nesting depth", () => { - const visible = deriveVisibleGraph(s, s.root); - const grouping = groupByLir(visible.nodes); - const elk = toElkGraph(visible, grouping); - expect(elk.edges!.map((e) => e.id)).toEqual(visible.edges.map((e) => e.id)); - for (const child of elk.children!) expect(child.edges).toBeUndefined(); - }); - - it("round-trips nested positions unchanged (parent-relative, no conversion)", () => { - const visible = deriveVisibleGraph(s, s.root); - const grouping = groupByLir(visible.nodes); - const elk = toElkGraph(visible, grouping); - const group = elk.children!.find((c) => c.id === grouping.groups[0].id)!; - group.x = 12; - group.y = 87; - group.width = 132; - group.height = 259; - const member = group.children!.find((c) => c.id === nodeIdOf([5, 1]))!; - member.x = 16; - member.y = 24; - const positions = extractPositions(elk); - // The member's extracted position is exactly its own x/y, not offset by - // the group's: elk already reports it relative to the group. - expect(positions[member.id]).toMatchObject({ x: 16, y: 24 }); - expect(positions[group.id]).toMatchObject({ x: 12, y: 87, width: 132, height: 259 }); - }); - - it("real elk lays out nested groups with auto-sized bounds and cross-boundary edges", async () => { - // Empirical check of the assumption the design doc relies on: elk - // auto-sizes a compound node from its children, reports child positions - // relative to their own parent (matching React Flow's parentId - // convention with no coordinate conversion needed), and relocates edges - // that cross group boundaries to the right container even when every - // edge is declared at the root, given hierarchyHandling: INCLUDE_CHILDREN. - const elk = new ELK(); - const graph = { - id: "root", - layoutOptions: { - "elk.algorithm": "layered", - "elk.direction": "DOWN", - "elk.hierarchyHandling": "INCLUDE_CHILDREN", - }, - children: [ - { id: "ungrouped", width: 100, height: 50 }, - { - id: "groupA", - layoutOptions: { "elk.padding": "[top=24,left=8,bottom=8,right=8]" }, - children: [ - { id: "a1", width: 100, height: 50 }, - { - id: "groupA_inner", - layoutOptions: { "elk.padding": "[top=24,left=8,bottom=8,right=8]" }, - children: [ - { id: "a2", width: 100, height: 50 }, - { id: "a3", width: 100, height: 50 }, - ], - }, - ], - }, - ], - edges: [ - { id: "e1", sources: ["ungrouped"], targets: ["a1"] }, - { id: "e2", sources: ["a1"], targets: ["a2"] }, - { id: "e3", sources: ["a2"], targets: ["a3"] }, - ], - }; - const result = await elk.layout(graph); - const groupA = result.children!.find((c) => c.id === "groupA")!; - const groupAInner = groupA.children!.find((c) => c.id === "groupA_inner")!; - // Auto-sized larger than any single member: real content-driven sizing. - expect(groupA.width!).toBeGreaterThan(100); - expect(groupA.height!).toBeGreaterThan(50); - // e3 (both endpoints inside groupA_inner) is relocated there even though - // it was declared at the root. - expect((result.edges ?? []).some((e) => e.id === "e3")).toBe(false); - const findEdge = (node: typeof groupAInner, id: string) => - (node.edges ?? []).some((e) => e.id === id); - expect(findEdge(groupAInner, "e3")).toBe(true); - }); -}); -``` - -- [ ] **Step 2: Run the tests to verify they fail** - -Run: `cd console && yarn vitest run src/platform/dataflows/elkGraph.test.ts` -Expected: FAIL — `toElkGraph` doesn't accept a second argument yet, and the flat-only implementation never nests anything (`group.children` is `undefined`, and the "real elk" test itself should already pass in isolation since it doesn't touch this codebase's functions — confirm it passes standalone before moving on, since it is the risk-gate check, not a test of code you're about to write). - -- [ ] **Step 3: Implement nested `toElkGraph` and recursive `extractPositions`** - -Replace the full contents of `console/src/platform/dataflows/elkGraph.ts` from the `LAYOUT_OPTIONS` constant to the end of the file with: - -```typescript -const LAYOUT_OPTIONS: Record = { - "elk.algorithm": "layered", - "elk.direction": "DOWN", - "elk.padding": "[top=48,left=16,bottom=16,right=16]", - "elk.spacing.nodeNode": "24", - "elk.layered.spacing.nodeNodeBetweenLayers": "48", - // Lets an edge crossing a LIR group's boundary be declared uniformly at - // the root: elk relocates it to the correct container itself. Harmless - // when there's no grouping at all (today's flat views). - "elk.hierarchyHandling": "INCLUDE_CHILDREN", -}; - -// A LIR group is a label-only wrapper, sized purely from its content (no -// fixed width/height), with top padding reserved for the header strip -// LirGroupNode renders. -const GROUP_LAYOUT_OPTIONS: Record = { - "elk.algorithm": "layered", - "elk.direction": "DOWN", - "elk.padding": "[top=24,left=8,bottom=8,right=8]", - "elk.spacing.nodeNode": "24", - "elk.layered.spacing.nodeNodeBetweenLayers": "48", -}; - -// A view is always one scope's direct children, so without a grouping this -// is flat: nothing is nested inside anything else. With a grouping, a -// group's members (VisibleNodes or other groups) become its elk children; -// everything else stays a direct child of the root, exactly like today. -export function toElkGraph(graph: VisibleGraph, grouping?: LirGrouping): ElkNode { - const nodeById = new Map(graph.nodes.map((n) => [n.id, n])); - const groupById = new Map((grouping?.groups ?? []).map((g) => [g.id, g])); - const childrenOf = new Map(); // parent id ("" = root) -> ordered child ids - const addChild = (parentKey: string, id: string) => { - const list = childrenOf.get(parentKey); - if (list) list.push(id); - else childrenOf.set(parentKey, [id]); - }; - for (const n of graph.nodes) addChild(grouping?.parentOf.get(n.id) ?? "", n.id); - for (const g of groupById.values()) addChild(grouping?.parentOf.get(g.id) ?? "", g.id); - - const buildNode = (id: string): ElkNode => { - const group = groupById.get(id); - if (group) { - return { - id, - layoutOptions: GROUP_LAYOUT_OPTIONS, - children: (childrenOf.get(id) ?? []).map(buildNode), - }; - } - return { id, ...NODE_DIMENSIONS[nodeById.get(id)!.kind] }; - }; - - return { - id: "__root__", - layoutOptions: LAYOUT_OPTIONS, - children: (childrenOf.get("") ?? []).map(buildNode), - edges: graph.edges.map((e) => ({ - id: e.id, - sources: [e.source], - targets: [e.target], - })), - }; -} - -// Recurses into nested children: elk reports every node's x/y already -// relative to its own immediate parent, the same convention React Flow uses -// for a node with parentId set, so no coordinate conversion happens here — -// this only flattens the tree into one lookup map, keyed by id at any depth. -export function extractPositions(layouted: ElkNode): Positions { - const positions: Positions = {}; - const visit = (node: ElkNode) => { - for (const child of node.children ?? []) { - positions[child.id] = { - x: child.x ?? 0, - y: child.y ?? 0, - width: child.width ?? 0, - height: child.height ?? 0, - }; - visit(child); - } - }; - visit(layouted); - return positions; -} -``` - -Add `LirGrouping` to the existing `import type { VisibleGraph, VisibleNode } from "./dataflowGraph";` line at the top of the file (change to `import type { LirGrouping, VisibleGraph, VisibleNode } from "./dataflowGraph";`). - -- [ ] **Step 4: Run the tests to verify they pass** - -Run: `cd console && yarn vitest run src/platform/dataflows/elkGraph.test.ts` -Expected: PASS, all tests including the two pre-existing flat-graph tests (unaffected) and the four new ones. - -- [ ] **Step 5: Thread `grouping` through `useElkLayout`** - -In `console/src/platform/dataflows/useElkLayout.ts`: - -Change the import line to: -```typescript -import type { LirGrouping, VisibleGraph } from "./dataflowGraph"; -``` - -Change the function signature: -```typescript -export function useElkLayout( - graph: VisibleGraph | null, - cacheKey: string, - grouping?: LirGrouping, -) { -``` - -Change the layout call inside the second effect: -```typescript - elk.layout(toElkGraph(graph, grouping)).then( -``` - -Add `grouping` to that effect's dependency array: -```typescript - }, [graph, cacheKey, retryNonce, grouping]); -``` - -- [ ] **Step 6: Typecheck and lint** - -Run: `cd console && yarn typecheck && yarn lint` -Expected: no errors. (`useElkLayout.ts` has no dedicated test file today; its behavior is exercised through `DataflowGraphView`/`DataflowDetailPage` tests in Task 4.) - -- [ ] **Step 7: Commit** - -```bash -git add console/src/platform/dataflows/elkGraph.ts console/src/platform/dataflows/elkGraph.test.ts console/src/platform/dataflows/useElkLayout.ts -git commit -m "$(cat <<'EOF' -console: nest LIR groups as elk compound nodes - -First use of elk's native compound-node layout in this codebase. -Verified against the real elk engine (not the worker-backed elk-api -used at runtime): groups auto-size from content, child positions come -back already relative to their own parent, and hierarchyHandling: -INCLUDE_CHILDREN relocates a boundary-crossing edge to its correct -container even when every edge is declared uniformly at the root. -EOF -)" -``` - ---- - -### Task 3: `LirGroupNode` render component and color helper - -**Files:** -- Modify: `console/src/platform/dataflows/nodeStyle.ts` -- Modify: `console/src/platform/dataflows/nodes.tsx` -- Test: `console/src/platform/dataflows/nodeStyle.test.ts` - -**Interfaces:** -- Consumes: nothing new from earlier tasks (this is a leaf UI unit; Task 4 wires it in). -- Produces: `lirGroupColor(lirId: string): string`, `type FlowGroupData = { group: LirGroupNode; label: string; color: string }`, `LirGroupNode` (the React Flow component, named the same as the data-layer type from Task 1 — imported under an alias wherever both are needed, see Task 4). - -- [ ] **Step 1: Write the failing test** - -Add to `console/src/platform/dataflows/nodeStyle.test.ts`: - -```typescript -import { lirGroupColor } from "./nodeStyle"; - -describe("lirGroupColor", () => { - it("is deterministic for the same lirId", () => { - expect(lirGroupColor("42")).toEqual(lirGroupColor("42")); - }); - - it("differs for different lirIds often enough to be useful", () => { - const colors = new Set(["1", "2", "3", "4", "5", "6"].map(lirGroupColor)); - expect(colors.size).toBeGreaterThan(1); - }); -}); -``` - -- [ ] **Step 2: Run the test to verify it fails** - -Run: `cd console && yarn vitest run src/platform/dataflows/nodeStyle.test.ts` -Expected: FAIL with `lirGroupColor is not exported`. - -- [ ] **Step 3: Implement `lirGroupColor` and `FlowGroupData`** - -Add to `console/src/platform/dataflows/nodeStyle.ts`, after the `HIGHLIGHT_COLORS` constant: - -```typescript -// A small, visually distinct palette for LIR group borders/headers, cycled -// by a deterministic hash so the same lirId always gets the same color -// within one render (and across re-renders, since lirId is stable). -const GROUP_PALETTE = [ - "#e8590c", - "#5f3dc4", - "#0b7285", - "#2f9e44", - "#c2255c", - "#1971c2", - "#e67700", - "#7048e8", -]; - -export function lirGroupColor(lirId: string): string { - let hash = 0; - for (let i = 0; i < lirId.length; i++) { - hash = (hash * 31 + lirId.charCodeAt(i)) | 0; - } - return GROUP_PALETTE[Math.abs(hash) % GROUP_PALETTE.length]; -} -``` - -Add to the bottom of `console/src/platform/dataflows/nodeStyle.ts`, and add `LirGroupNode` to its existing `import type { VisibleNode } from "./dataflowGraph";` line (making it `import type { LirGroupNode, VisibleNode } from "./dataflowGraph";`): - -```typescript -export type FlowGroupData = { - group: LirGroupNode; - label: string; - color: string; -}; -``` - -- [ ] **Step 4: Run the test to verify it passes** - -Run: `cd console && yarn vitest run src/platform/dataflows/nodeStyle.test.ts` -Expected: PASS. - -- [ ] **Step 5: Implement the `LirGroupNode` component** - -Add to `console/src/platform/dataflows/nodes.tsx`, after the existing `PortNode` component, and add `FlowGroupData` to the existing `import { ... } from "./nodeStyle";` line: - -```typescript -// A label-only wrapper around its members, never itself a click target -// except its header (the body is pointerEvents="none" so clicks pass -// through to whatever member is underneath). Its width/height come from -// elk's auto-sized bounds, same as every other node in this file. -export const LirGroupNode = ({ - data, -}: NodeProps & { data: FlowGroupData }) => ( - - - - - {data.label} - - - -); -``` - -- [ ] **Step 6: Typecheck and lint** - -Run: `cd console && yarn typecheck && yarn lint` -Expected: no errors. (No isolated component test for this one, matching `OperatorNode`/`RegionNode`/`PortNode`, none of which have dedicated test files either; Task 4's integration test covers its rendering.) - -- [ ] **Step 7: Commit** - -```bash -git add console/src/platform/dataflows/nodeStyle.ts console/src/platform/dataflows/nodeStyle.test.ts console/src/platform/dataflows/nodes.tsx -git commit -m "console: add LirGroupNode component and lirGroupColor helper" -``` - ---- - -### Task 4: Wire grouping into `DataflowGraphView` - -**Files:** -- Modify: `console/src/platform/dataflows/DataflowGraphView.tsx` -- Modify: `console/src/platform/dataflows/DataflowDetailPage.test.tsx` (fix the inline React Flow mock) - -**Interfaces:** -- Consumes: `groupByLir`, `LirGrouping`, `LirGroupNode` (data type, Task 1); `LirGroupNode` (component, Task 3, imported as `LirGroupNodeComponent` to avoid the name clash); `lirGroupColor` (Task 3); `useElkLayout`'s new third parameter (Task 2). -- Produces: two new `DataflowGraphViewProps`: `showLirGroups?: boolean` and `onLirGroupClick?: (group: LirGroupNode) => void`, consumed by `DataflowDetailPage.tsx` in Tasks 5 and 6. - -- [ ] **Step 1: Fix the test mock's node-rendering assumption** - -The mock `ReactFlow` in `DataflowDetailPage.test.tsx` renders each node's label by reading `n.data.node.label`, which every operator/region/port node has but a group node (whose `data` is `{ group, label, color }`, no nested `.node`) does not. Before adding any group node to what gets rendered, fix the mock to handle both shapes. - -In `console/src/platform/dataflows/DataflowDetailPage.test.tsx`, change the `vi.mock("@xyflow/react", ...)` block's `ReactFlow` implementation: - -```typescript - ReactFlow: ({ - nodes, - children, - onNodeClick, - onNodeDoubleClick, - }: { - nodes: { - id: string; - type?: string; - data: { node?: { label: string }; label?: string }; - }[]; - children?: React.ReactNode; - onNodeClick?: (e: unknown, node: unknown) => void; - onNodeDoubleClick?: (e: unknown, node: unknown) => void; - }) => ( -
- {nodes.map((n) => ( -
onNodeClick?.(null, n)} - onDoubleClick={() => onNodeDoubleClick?.(null, n)} - > - {n.data.node ? n.data.node.label : n.data.label} -
- ))} - {children} -
- ), -``` - -- [ ] **Step 2: Run the full existing suite to confirm the mock fix alone changes nothing** - -Run: `cd console && yarn vitest run src/platform/dataflows/DataflowDetailPage.test.tsx` -Expected: PASS, same as before this step (this is a pure widening of the mock's type, not a behavior change). - -- [ ] **Step 3: Add a dedicated LIR fixture and the failing test for group rendering** - -The file's existing default dataflow ("7") fixture has zero LIR spans (its mocked SQL response sends `okResult`, an empty result, for the `lirSpans` query), and every other existing test relies on that staying empty. Rather than touching it, add a new dataflow id, following the same pattern already used for dataflow 8 (`portJumpOperatorsResult`) and 9 (`fanOutOperatorsResult`). - -Add these constants to `console/src/platform/dataflows/DataflowDetailPage.test.tsx`, near the other dataflow fixtures (after `fanOutChannelsResult`): - -```typescript -// Dataflow 40: root [40], child [40,1] "Join" (operator id 41), covered by -// one LIR span, to exercise the "Show LIR groups" toggle in isolation from -// every other test's dataflow 7 fixture, which has no LIR spans at all. -const lirOperatorsResult = { - ...operatorsResult, - rows: [ - ["40", ["40"], "Dataflow", "0", "0", "0"], - ["41", ["40", "1"], "Join", "0", "0", "0"], - ], -}; -const lirSpansResult = { - desc: { - columns: [ - { name: "exportId" }, - { name: "lirId" }, - { name: "parentLirId" }, - { name: "nesting" }, - { name: "operator" }, - { name: "operatorIdStart" }, - { name: "operatorIdEnd" }, - ], - }, - rows: [["u7", "1", null, "0", "Join::Differential", "41", "42"]], -}; -``` - -Add a new branch to the `http.post("*/api/sql", ...)` handler inside `beforeEach`, alongside the existing `'8'`/`'9'` branches (order among these branches doesn't matter, but put it before the final unconditional fallback): - -```typescript - if (body.includes("'40'")) { - return HttpResponse.json({ - results: [lirOperatorsResult, okResult, lirSpansResult, okResult], - }); - } -``` - -Then append this test to the `describe("DataflowDetailPage", ...)` block, following the same `renderComponent`/`initialRouterEntries` pattern the dataflow-8 tests already use (e.g. `"jumps to an output port's peer, same as an input port's peer"`): - -```typescript - it("shows a LIR group box when the toggle is on, and hides it when off", async () => { - await renderComponent( - - } - /> - , - { - initialRouterEntries: ["/clusters/u5/test_cluster/dataflows/40"], - }, - ); - - expect( - await screen.findByTestId(`node-${nodeIdOf([40, 1])}`), - ).toBeInTheDocument(); - expect(screen.queryByText(/Join::Differential/)).not.toBeInTheDocument(); - - await userEvent.click(screen.getByLabelText("Show LIR groups")); - - const groupNode = screen.getByTestId("node-u7/1"); - expect(groupNode).toHaveTextContent("Join::Differential"); - - // Regression check for the onNodeDoubleClick guard added in Step 4: - // double-clicking a group must not throw (it read `.data.node.kind` - // unconditionally before the guard, which crashes on a group node) and - // must not navigate anywhere, since groups aren't scopes. - await userEvent.dblClick(groupNode); - expect(screen.getByTestId("node-u7/1")).toBeInTheDocument(); - }); -``` - -This references `screen.getByLabelText("Show LIR groups")`, which doesn't exist until Task 5. Run it now to confirm it fails on the missing label (not a crash), then move on — it is expected to stay red until Task 5. - -- [ ] **Step 4: Implement grouping in `DataflowGraphView.tsx`** - -In `console/src/platform/dataflows/DataflowGraphView.tsx`: - -Change the import block to add the new pieces: -```typescript -import { ChannelEdge } from "./ChannelEdge"; -import { - type GraphDecorations, - groupByLir, - type LirGroupNode as LirGroupNodeData, - type NodeId, - type PortPeer, - rerouteHiddenNodes, - type VisibleEdge, - type VisibleGraph, - type VisibleNode, -} from "./dataflowGraph"; -import { NODE_DIMENSIONS, type Positions } from "./elkGraph"; -import { - LirGroupNode as LirGroupNodeComponent, - OperatorNode, - PortNode, - RegionNode, -} from "./nodes"; -import { lirGroupColor, nodeFillColor } from "./nodeStyle"; -import { useElkLayout } from "./useElkLayout"; -``` - -Add `lirGroup: LirGroupNodeComponent` to `nodeTypes`: -```typescript -const nodeTypes = { - operator: OperatorNode, - region: RegionNode, - port: PortNode, - lirGroup: LirGroupNodeComponent, -}; -``` - -Add two new props to `DataflowGraphViewProps` (near `onJumpToPeer`): -```typescript - // Toggled from the toolbar; grouping is computed here (from the same - // post-hideIdle, post-reroute node list elk actually lays out) rather than - // by the caller, so a hidden node can never end up as a group's only - // member. - showLirGroups?: boolean; - onLirGroupClick?: (group: LirGroupNodeData) => void; -``` - -Destructure them in the component signature (add `showLirGroups, onLirGroupClick,` to the existing destructured props list). - -After the existing `const visible = React.useMemo(...)` block (which computes the rerouted/filtered graph), add: -```typescript - const grouping = React.useMemo( - () => (showLirGroups ? groupByLir(visible.nodes) : null), - [showLirGroups, visible.nodes], - ); -``` - -Change `layoutKey` to include the toggle, so turning it on/off invalidates the layout cache: -```typescript - const layoutKey = `${cacheKey}|${focusedScope}|${showLirGroups ?? false}|${ - decorations?.hiddenNodeIds - ? [...decorations.hiddenNodeIds].sort().join(",") - : "" - }|${ - decorations?.hiddenEdgeIds - ? [...decorations.hiddenEdgeIds].sort().join(",") - : "" - }`; - const { positions, layouting, error, retry } = useElkLayout( - visible, - layoutKey, - grouping ?? undefined, - ); -``` - -Replace the `nodes: Node[] = React.useMemo(...)` block with one that emits group nodes before member nodes, and sets `parentId` on anything grouping places inside a group: -```typescript - const nodes: Node[] = React.useMemo(() => { - if (!positions) return []; - const groupNodes: Node[] = (grouping?.groups ?? []).map((g) => { - const pos = positions[g.id] ?? { x: 0, y: 0, width: 0, height: 0 }; - return { - id: g.id, - type: "lirGroup", - position: { x: pos.x, y: pos.y }, - parentId: grouping!.parentOf.get(g.id), - width: pos.width, - height: pos.height, - // pointerEvents: "none" on React Flow's own node wrapper (not just - // inside LirGroupNode's own JSX) is the other half of the - // click-through contract LirGroupNode's comment documents: the - // wrapper defaults to pointer-events:all and LirGroupNode can't - // override an ancestor it doesn't render. With the wrapper opted - // out, LirGroupNode's header (pointerEvents:"auto") still receives - // clicks (a descendant can always re-enable itself), and a click on - // the group's empty body falls through to a member's own wrapper - // when one is there, or further to the pane when none is. - style: { width: pos.width, height: pos.height, pointerEvents: "none" }, - draggable: false, - connectable: false, - // Below its members, which draw over it; React Flow requires a - // group node to precede its children in this array, which the - // groupNodes-then-memberNodes concat below already guarantees, but - // an explicit zIndex also protects against member nodes elsewhere - // in the array being reordered by a future change. - zIndex: -1, - data: { group: g, label: g.operator, color: lirGroupColor(g.lirId) }, - }; - }); - const memberNodes: Node[] = visible.nodes.map((n) => { - const pos = positions[n.id] ?? { x: 0, y: 0, ...NODE_DIMENSIONS[n.kind] }; - return { - id: n.id, - type: n.kind, - position: { x: pos.x, y: pos.y }, - parentId: grouping?.parentOf.get(n.id), - // Match the Top/Bottom handles so bezier control points meet the - // node edges. Without this the edge curves toward the default - // Bottom/Top and appears detached across region boundaries. - targetPosition: Position.Top, - sourcePosition: Position.Bottom, - // Set as explicit node fields, not just CSS style: the MiniMap and - // culled (onlyRenderVisibleElements) off-screen nodes both need a - // known size without waiting for DOM measurement. - width: pos.width, - height: pos.height, - style: { width: pos.width, height: pos.height }, - draggable: false, - connectable: false, - data: { - node: n, - dimmed: decorations?.dimmedNodeIds?.has(n.id) ?? false, - color: decorations?.nodeColors?.get(n.id) ?? nodeFillColor(n), - selected: n.id === selectedId, - activeMatch: n.id === activeMatchId, - }, - }; - }); - return [...groupNodes, ...memberNodes]; - }, [ - visible, - positions, - grouping, - decorations?.dimmedNodeIds, - decorations?.nodeColors, - selectedId, - activeMatchId, - ]); -``` - -Change the `onNodeClick` handler passed to `` to branch on group clicks first: -```typescript - onNodeClick={(_, node) => { - if (node.type === "lirGroup") { - onLirGroupClick?.((node.data as { group: LirGroupNodeData }).group); - return; - } - const visibleNode = (node.data as { node: VisibleNode }).node; - const connectedEdges = visible.edges - .filter( - (e) => e.source === visibleNode.id || e.target === visibleNode.id, - ) - .map((e) => ({ - ...e, - sourceLabel: labelById.get(e.source) ?? e.source, - targetLabel: labelById.get(e.target) ?? e.target, - })); - onNodeClick?.(visibleNode, connectedEdges); - }} -``` - -Also guard the existing `onNodeDoubleClick` handler, a few lines below `onNodeClick` in the same `` element: it currently reads `(node.data as { node: VisibleNode }).node.kind` unconditionally, which throws on a group node (`data` has no `.node`). Add the same type check at the top, before that line: -```typescript - onNodeDoubleClick={(_, node) => { - // Groups aren't scopes: no drill-down, no jump, nothing happens. - if (node.type === "lirGroup") return; - const visibleNode = (node.data as { node: VisibleNode }).node; - if (visibleNode.kind === "region") { - onNavigate(visibleNode.id); - } else if ( - visibleNode.kind === "port" && - visibleNode.peers.length === 1 - ) { - onJumpToPeer?.(visibleNode.peers[0]); - } - }} -``` - -- [ ] **Step 5: Run typecheck to confirm wiring compiles** - -Run: `cd console && yarn typecheck` -Expected: no errors. (The Task 3 test written in Step 3 is still expected to fail — it depends on Task 5's toolbar switch, which doesn't exist yet. Confirm it fails with "Unable to find a label with the text: Show LIR groups", not a crash, so you know the rest of the wiring is otherwise sound.) - -Run: `cd console && yarn vitest run src/platform/dataflows/DataflowDetailPage.test.tsx` -Expected: every test PASSES except the new one from Step 3, which fails on the missing label (not a crash/type error). - -- [ ] **Step 6: Lint** - -Run: `cd console && yarn lint` -Expected: no errors. - -- [ ] **Step 7: Commit** - -```bash -git add console/src/platform/dataflows/DataflowGraphView.tsx console/src/platform/dataflows/DataflowDetailPage.test.tsx -git commit -m "$(cat <<'EOF' -console: wire LIR grouping into DataflowGraphView - -Grouping is computed from the same post-hideIdle, post-reroute node -list elk actually lays out, so a hidden node never ends up as a -group's only member. Group nodes render behind their members via -React Flow's parentId nesting, one new nodeTypes entry, and an -onNodeClick branch for group-header clicks (wired to a real handler -in Task 6). - -This task's new toggle test doesn't pass yet — it depends on Task 5's -toolbar switch, which doesn't exist. -EOF -)" -``` - ---- - -### Task 5: Toolbar toggle - -**Files:** -- Modify: `console/src/platform/dataflows/dataflowGraph.ts` -- Modify: `console/src/platform/dataflows/DataflowToolbar.tsx` -- Modify: `console/src/platform/dataflows/DataflowDetailPage.tsx` -- Test: existing tests in `dataflowGraph.test.ts` and `DataflowDetailPage.test.tsx` (the one from Task 4, Step 3) - -**Interfaces:** -- Consumes: `DataflowGraphView`'s `showLirGroups` prop (Task 4). -- Produces: `Filters.showLirGroups: boolean`, `DEFAULT_FILTERS.showLirGroups: false`. - -- [ ] **Step 1: Update `DEFAULT_FILTERS` assertions if any exist, then add the field** - -Search for existing assertions against `DEFAULT_FILTERS`'s exact shape: - -Run: `cd console && grep -rn "DEFAULT_FILTERS" src/platform/dataflows/*.test.ts src/platform/dataflows/*.test.tsx` - -If any test asserts `DEFAULT_FILTERS` equals an object literal without `showLirGroups`, update it to include `showLirGroups: false`. (As of this plan being written, no such assertion exists — `DEFAULT_FILTERS` is only referenced by identity, e.g. `React.useState(DEFAULT_FILTERS)` — but confirm before proceeding, since the plan can't see future changes to this file.) - -In `console/src/platform/dataflows/dataflowGraph.ts`, change the `Filters` interface and `DEFAULT_FILTERS` constant: - -```typescript -export interface Filters { - search: string; - hideIdle: boolean; - heatmap: "off" | "elapsed" | "size" | "cpuSkew" | "memorySkew"; - heatmapThreshold: number; // 0..1 fraction of max - showLirGroups: boolean; -} - -export const DEFAULT_FILTERS: Filters = { - search: "", - hideIdle: false, - heatmap: "off", - heatmapThreshold: 0, - showLirGroups: false, -}; -``` - -- [ ] **Step 2: Add the toolbar switch** - -In `console/src/platform/dataflows/DataflowToolbar.tsx`, add a new `FormControl`/`Switch` pair after the existing "Hide idle" one: - -```typescript - - - Show LIR groups - - - onFiltersChange({ ...filters, showLirGroups: e.target.checked }) - } - /> - -``` - -The existing "Hide idle" switch has no `aria-label`, relying on its `FormLabel` for accessible naming via Chakra's `FormControl` association; do the same here (Chakra's `FormControl` context automatically associates the `FormLabel` with the `Switch`, so `getByLabelText("Show LIR groups")` in tests resolves without an explicit `aria-label`). Remove the `aria-label` line above if Chakra's automatic association already covers it — check by running the Task 4 test after this step; add `aria-label="Show LIR groups"` back only if the test can't find it by label text otherwise. - -- [ ] **Step 3: Wire the prop through `DataflowDetailPage.tsx`** - -In `console/src/platform/dataflows/DataflowDetailPage.tsx`, add `showLirGroups={filters.showLirGroups}` to the `` element's props (alongside the existing `decorations={decorations}` line). - -- [ ] **Step 4: Run the Task 4 toggle test to verify it now passes** - -Run: `cd console && yarn vitest run src/platform/dataflows/DataflowDetailPage.test.tsx` -Expected: PASS, including the "shows a LIR group box when the toggle is on" test from Task 4. - -- [ ] **Step 5: Run the full dataflow test suite** - -Run: `cd console && yarn vitest run src/platform/dataflows src/api/materialize/dataflow` -Expected: PASS, all tests. - -- [ ] **Step 6: Typecheck and lint** - -Run: `cd console && yarn typecheck && yarn lint` -Expected: no errors. - -- [ ] **Step 7: Commit** - -```bash -git add console/src/platform/dataflows/dataflowGraph.ts console/src/platform/dataflows/DataflowToolbar.tsx console/src/platform/dataflows/DataflowDetailPage.tsx console/src/platform/dataflows/DataflowDetailPage.test.tsx -git commit -m "console: add Show LIR groups toolbar toggle, off by default" -``` - ---- - -### Task 6: Click a group's header to open its detail panel - -**Files:** -- Modify: `console/src/platform/dataflows/LirPanel.tsx` (export the existing summary card for reuse) -- Modify: `console/src/platform/dataflows/NodeDetailPanel.tsx` -- Modify: `console/src/platform/dataflows/DataflowDetailPage.tsx` -- Test: `console/src/platform/dataflows/DataflowDetailPage.test.tsx` - -**Interfaces:** -- Consumes: `DataflowGraphView`'s `onLirGroupClick` prop (Task 4); `lirIndex`, `lirSummary` (both already exist in `dataflowGraph.ts`, unchanged). -- Produces: `Selection` gains a `"lirGroup"` variant; no new exports beyond `LirSummaryCard`. - -- [ ] **Step 1: Write the failing test** - -Append to `console/src/platform/dataflows/DataflowDetailPage.test.tsx`, reusing dataflow 40's fixture added in Task 4, Step 3: - -```typescript - it("opens the detail panel with LIR summary info when a group header is clicked", async () => { - await renderComponent( - - } - /> - , - { - initialRouterEntries: ["/clusters/u5/test_cluster/dataflows/40"], - }, - ); - - await screen.findByTestId(`node-${nodeIdOf([40, 1])}`); - await userEvent.click(screen.getByLabelText("Show LIR groups")); - await userEvent.click(screen.getByTestId("node-u7/1")); - - // The title and LirSummaryCard both render "LIR 1: Join::Differential", - // so assert at least one match rather than a single getByText, which - // throws on more than one hit. - expect( - screen.getAllByText(/LIR 1: Join::Differential/).length, - ).toBeGreaterThan(0); - // Rows unique to LirSummaryCard, not NodeDetail's node-shaped rows. - expect(screen.getByText("Records")).toBeInTheDocument(); - expect(screen.getByText("Memory")).toBeInTheDocument(); - }); -``` - -(Adjust the LIR id/operator text to match whatever this file's dataflow fixture actually uses — see Task 4, Step 3's note about matching the fixture's LIR span.) - -- [ ] **Step 2: Run the test to verify it fails** - -Run: `cd console && yarn vitest run src/platform/dataflows/DataflowDetailPage.test.tsx -t "opens the detail panel with LIR summary"` -Expected: FAIL — clicking the group header currently calls `onLirGroupClick`, which `DataflowDetailPage` doesn't pass yet, so nothing opens. - -- [ ] **Step 3: Export `LirSummaryCard` for reuse** - -In `console/src/platform/dataflows/LirPanel.tsx`, change: -```typescript -const LirSummaryCard = ({ node }: { node: LirTreeNode }) => ( -``` -to: -```typescript -export const LirSummaryCard = ({ node }: { node: LirTreeNode }) => ( -``` - -- [ ] **Step 4: Add the `"lirGroup"` selection variant to `NodeDetailPanel.tsx`** - -Change the `Selection` type: -```typescript -export type Selection = - | { kind: "node"; node: VisibleNode; connectedEdges?: SelectedEdge[] } - | { kind: "edge"; edge: SelectedEdge } - | { kind: "lirGroup"; node: LirTreeNode }; -``` - -Add `LirTreeNode` to the existing `import { ... } from "./dataflowGraph";` line, and import `LirSummaryCard` from `./LirPanel`: -```typescript -import { - formatElapsedNs, - type LirTreeNode, - type PortPeer, - type VisibleNode, -} from "./dataflowGraph"; -import { LirSummaryCard } from "./LirPanel"; -``` - -Change the `title` computation in `NodeDetailPanel`: -```typescript - const title = - selection.kind === "node" - ? selection.node.label - : selection.kind === "edge" - ? `${selection.edge.sourceLabel} → ${selection.edge.targetLabel}` - : `LIR ${selection.node.info.lirId}: ${selection.node.info.operator}`; -``` - -Change the body's conditional rendering (the `{selection.kind === "node" ? (...) : (...)}` block) to a three-way branch: -```typescript - {selection.kind === "node" ? ( - - ) : selection.kind === "edge" ? ( - - ) : ( - - )} -``` - -- [ ] **Step 5: Wire `onLirGroupClick` in `DataflowDetailPage.tsx`** - -Add `lirIndex` and `lirSummary` to the existing `import { ... } from "./dataflowGraph";` block (they're already exported by `dataflowGraph.ts`, just not currently imported here). - -Add a new callback near `onTogglePinLir`: -```typescript - const onLirGroupClick = React.useCallback( - (group: { id: string }) => { - if (!data) return; - const entry = lirIndex(data.structure).get(group.id); - if (!entry) return; - setSelection({ - kind: "lirGroup", - node: { - key: group.id, - info: entry.info, - memberIds: entry.memberIds, - children: [], - summary: lirSummary(data.structure, entry.memberIds), - }, - }); - }, - [data], - ); -``` - -Add `onLirGroupClick={onLirGroupClick}` to the `` element's props, alongside the existing `onJumpToPeer={onJumpToPeer}` line. - -- [ ] **Step 6: Run the test to verify it passes** - -Run: `cd console && yarn vitest run src/platform/dataflows/DataflowDetailPage.test.tsx` -Expected: PASS, all tests including the new one. - -- [ ] **Step 7: Typecheck and lint** - -Run: `cd console && yarn typecheck && yarn lint` -Expected: no errors. - -- [ ] **Step 8: Full verification pass** - -Run: `cd console && yarn vitest run src/platform/dataflows src/api/materialize/dataflow && yarn typecheck && yarn lint` -Expected: everything PASSES. - -Manually confirm the feature end-to-end against a real `environmentd` (per `mz-run`): start it, run a query that produces nested LIR spans (e.g. a materialized view with a table function like `generate_series` nested inside another, matching the reference screenshot this plan was scoped from), open its dataflow in the visualizer, toggle "Show LIR groups" on, and confirm nested dashed boxes render without overlapping siblings, edges crossing a box boundary still draw sensibly, and clicking a box header opens the detail panel with the right LIR id/operator/summary. This is a manual step outside the automated test loop; note the outcome to the user rather than silently assuming it looks right. - -- [ ] **Step 9: Commit** - -```bash -git add console/src/platform/dataflows/LirPanel.tsx console/src/platform/dataflows/NodeDetailPanel.tsx console/src/platform/dataflows/DataflowDetailPage.tsx console/src/platform/dataflows/DataflowDetailPage.test.tsx -git commit -m "console: open LIR summary in the detail panel from a group header click" -``` From f4f9fe1e67936813bd241f517e63f7b38e4b0206 Mon Sep 17 00:00:00 2001 From: Moritz Hoffmann Date: Wed, 8 Jul 2026 22:28:16 +0200 Subject: [PATCH 35/45] dataflowGraph: replace non-null assertions with mustGet, tidy doc comments Add mustGet(map, key) and use it in place of .get(x)! throughout dataflowGraph.ts and elkGraph.ts, so a missing key throws with the key in the message instead of a bare non-null-assertion crash. Convert the file's // doc-comment blocks to JSDoc and fix em-dash/semicolon comment style along the way. Also move formatElapsedNs to utils/format.ts: it's a generic duration formatter, not dataflow-specific, and utils/format.ts is where the rest of the codebase's formatters already live. Co-Authored-By: Claude Sonnet 5 --- .../src/platform/dataflows/dataflowGraph.ts | 291 +++++++++++------- console/src/platform/dataflows/elkGraph.ts | 6 +- console/src/utils/format.test.ts | 20 ++ console/src/utils/format.ts | 15 + 4 files changed, 221 insertions(+), 111 deletions(-) diff --git a/console/src/platform/dataflows/dataflowGraph.ts b/console/src/platform/dataflows/dataflowGraph.ts index db6d46ccfb7f7..8a65bda53ba54 100644 --- a/console/src/platform/dataflows/dataflowGraph.ts +++ b/console/src/platform/dataflows/dataflowGraph.ts @@ -13,8 +13,18 @@ export function nodeIdOf(address: Address): NodeId { return JSON.stringify(address); } -export function formatElapsedNs(ns: bigint): string { - return `${Math.round(Number(ns) / 1e9)}s`; +/** + * Looks up a key expected to be present by construction (e.g. a child id + * drawn from the same map it indexes into). Throws with the key in the + * message instead of a bare non-null assertion, so a broken invariant fails + * loudly at the lookup site rather than as a generic "undefined" downstream. + */ +export function mustGet(map: ReadonlyMap, key: K): V { + const value = map.get(key); + if (value === undefined) { + throw new Error(`missing expected map entry for key ${String(key)}`); + } + return value; } export interface NodeStats { @@ -23,18 +33,20 @@ export interface NodeStats { elapsedNs: bigint; // Total times this operator was scheduled (summed over the scheduling // duration histogram's buckets). A high count relative to elapsed time - // means the operator yields often -- each activation does little work, - // so per-activation overhead (waking the worker, progress tracking) - // dominates -- which raw elapsed time alone doesn't surface. + // means the operator yields often. Each activation does little work, so + // per-activation overhead (waking the worker, progress tracking) + // dominates, and raw elapsed time alone doesn't surface this. scheduleCount: bigint; } -// How unevenly work for this node is spread across workers: worst worker's -// value over the average, matching EXPLAIN ANALYZE ... WITH SKEW. Avg is -// over workers that show up at all for this node, not the full replica -// worker count. A worker with zero rows anywhere for a node is invisible to -// both, same convention. 1 means perfectly even, higher means more skewed, -// 0 means no data. +/** + * How unevenly work for this node is spread across workers: worst worker's + * value over the average, matching EXPLAIN ANALYZE ... WITH SKEW. Avg is + * over workers that show up at all for this node, not the full replica + * worker count. A worker with zero rows anywhere for a node is invisible to + * both, same convention. 1 means perfectly even, higher means more skewed, + * 0 means no data. + */ export interface SkewStats { cpuSkew: number; memorySkew: number; @@ -49,6 +61,12 @@ export interface LirInfo { operator: string; } +/** + * One node of a dataflow's internal address tree: pure introspection data + * (own/transitive stats and skew), independent of any particular scope's + * view. `deriveVisibleGraph` projects this into `VisibleNode`s for a given + * view; this type carries no view-specific computation itself. + */ export interface DataflowNode { id: NodeId; operatorId: bigint; @@ -61,7 +79,7 @@ export interface DataflowNode { ownSkew: SkewStats; // Same ratios as ownSkew, but never suppressed by the magnitude floor. // ownSkew exists to seed transitiveSkew's MAX rollup (see its comment for - // why negligible nodes must not pollute an ancestor's reported skew); + // why negligible nodes must not pollute an ancestor's reported skew). // ownSkewRaw is this node's actual measured skew, for display when this // exact node is selected, where a floor built for ancestor-safety would // otherwise misreport a real (if individually low-stakes) imbalance as @@ -107,9 +125,11 @@ export interface DataflowStructure { channels: Channel[]; } -// A boundary crossing this view can't show directly (the far side isn't part -// of it): kept so a port can still be jumped to, even fanning out to several -// peers (one output can feed multiple inputs). +/** + * A boundary crossing this view can't show directly (the far side isn't part + * of it): kept so a port can still be jumped to, even fanning out to several + * peers (one output can feed multiple inputs). + */ export interface PortPeer { address: Address; label: string; @@ -140,7 +160,7 @@ export interface VisibleNode { // needs to show on its own. own: NodeStats | null; // This node's own skew, unfiltered by the magnitude floor that guards - // transitiveSkew's ancestor rollup (see DataflowNode.ownSkewRaw) -- the + // transitiveSkew's ancestor rollup (see DataflowNode.ownSkewRaw). The // real number for this exact node, not a heatmap-safe one. ownSkew: SkewStats | null; transitiveSkew: SkewStats | null; @@ -157,6 +177,12 @@ export interface VisibleNode { peers: PortPeer[]; } +/** + * A rendered edge in one scope's view: a real `Channel`'s raw endpoints, + * resolved to whatever's actually visible at this scope (an exact node, or + * a collapsed region/port peeled back to it). Several real channels can + * merge onto one `VisibleEdge` when they share the same visible endpoints. + */ export interface VisibleEdge { id: string; source: string; @@ -183,7 +209,9 @@ export interface VisibleGraph { edges: VisibleEdge[]; } -// SQL row shapes (values arrive as strings or numbers, normalized here) +/** + * SQL row shapes (values arrive as strings or numbers, normalized here) + */ export interface OperatorRow { id: bigint | number | string; address: string[]; @@ -254,9 +282,9 @@ function skewRatio(vector: WorkerVector): number { // would otherwise become every ancestor's reported skew all the way to // the root. A node's own magnitude (elapsed for cpu, arrangement size for // memory) must be at least this fraction of the busiest single node's own -// magnitude anywhere in the dataflow to count; below it, ownSkew reports -// the same 0 ("no data") sentinel a node with no per-worker rows at all -// would. +// magnitude anywhere in the dataflow to count. Below that floor, ownSkew +// reports the same 0 ("no data") sentinel a node with no per-worker rows at +// all would. const SKEW_MAGNITUDE_FLOOR_FRACTION = 0.01; function passesMagnitudeFloor(own: bigint, maxOwn: bigint): boolean { @@ -266,6 +294,13 @@ function passesMagnitudeFloor(own: bigint, maxOwn: bigint): boolean { ); } +/** + * Builds the address tree and own/transitive stats from raw introspection + * rows. `operators` must contain at most one root (an address of length 1); + * more than one throws. An empty `operators` list (a dropped or transient + * dataflow) is valid input, not an error: it returns a single-node + * placeholder structure instead. + */ export function buildDataflowStructure( operators: OperatorRow[], channels: ChannelRow[], @@ -373,7 +408,7 @@ export function buildDataflowStructure( // Children in address order so layout and tests are deterministic. for (const node of nodes.values()) { node.children.sort((a, b) => { - const [x, y] = [nodes.get(a)!.address, nodes.get(b)!.address]; + const [x, y] = [mustGet(nodes, a).address, mustGet(nodes, b).address]; return x[x.length - 1] - y[y.length - 1]; }); } @@ -424,7 +459,7 @@ export function buildDataflowStructure( } const fillTransitive = (id: NodeId): void => { - const node = nodes.get(id)!; + const node = mustGet(nodes, id); const ownCpuVec = ownCpuByNode.get(id) ?? new Map(); const ownMemoryVec = ownMemoryByNode.get(id) ?? new Map(); const ownScheduleVec = ownScheduleByNode.get(id) ?? new Map(); @@ -457,7 +492,7 @@ export function buildDataflowStructure( let childrenOwnElapsedNs = 0n; for (const c of node.children) { fillTransitive(c); - const child = nodes.get(c)!; + const child = mustGet(nodes, c); t.arrangementRecords += child.transitive.arrangementRecords; t.arrangementSize += child.transitive.arrangementSize; t.elapsedNs += child.transitive.elapsedNs; @@ -488,12 +523,14 @@ export function buildDataflowStructure( }; } -// A view shows exactly one scope's direct children. Anything deeper is -// rolled up into its child's box, addressed at that box's depth (viewRoot's -// depth + 1). representativeInView is the general form of that projection, -// exported for translating arbitrary structure addresses (search matches, -// LIR members) into the box that would represent them in a given view, or -// null if the address isn't inside viewRootAddress's subtree at all. +/** + * A view shows exactly one scope's direct children. Anything deeper is + * rolled up into its child's box, addressed at that box's depth (viewRoot's + * depth + 1). representativeInView is the general form of that projection, + * exported for translating arbitrary structure addresses (search matches, + * LIR members) into the box that would represent them in a given view, or + * null if the address isn't inside viewRootAddress's subtree at all. + */ export function representativeInView( address: Address, viewRootAddress: Address, @@ -517,11 +554,11 @@ function representative(address: Address, viewRootAddress: Address): NodeId { // A scope boundary crossing is logged as two channels that share a port // number but use different addressing for each half: -// - the outer half (external <-> scope) addresses the scope by its own +// - The outer half (external <-> scope) addresses the scope by its own // operator address, e.g. an external producer's channel row targets // address [5,2] directly, the same address BuildRegion's own operator -// row uses; -// - the inner half (scope <-> child) addresses the scope's internal +// row uses. +// - The inner half (scope <-> child) addresses the scope's internal // fan-out point one level deeper, e.g. [5,2,0]. // Both halves are mapped to the same synthetic port id, keyed by (scope, // direction, port number), so they visually chain through one port node @@ -571,17 +608,19 @@ function endpointId( return { id: representative(address, viewRootAddress) }; } -// Renders exactly one scope's direct children, each either a leaf operator or -// a box summarizing a whole nested subtree (never expanded in place). -// Double-clicking a box navigates to a new view rooted there instead. +/** + * Renders exactly one scope's direct children, each either a leaf operator or + * a box summarizing a whole nested subtree (never expanded in place). + * Double-clicking a box navigates to a new view rooted there instead. + */ export function deriveVisibleGraph( structure: DataflowStructure, viewRoot: NodeId, ): VisibleGraph { - const viewRootNode = structure.nodes.get(viewRoot)!; + const viewRootNode = mustGet(structure.nodes, viewRoot); const viewRootAddress = viewRootNode.address; const nodes: VisibleNode[] = viewRootNode.children.map((id) => { - const node = structure.nodes.get(id)!; + const node = mustGet(structure.nodes, id); const kind = node.children.length === 0 ? "operator" : "region"; return { id, @@ -634,8 +673,8 @@ export function deriveVisibleGraph( // A scope's own boundary is logged as a pseudo-vertex [scope, 0], which // never gets an operator row of its own (only real children do). A // region nested two or more scopes deep has its outer half target that - // pseudo-vertex directly, one level further up than the region itself; - // peel back to the enclosing scope, which does have a row, so the jump + // pseudo-vertex directly, one level further up than the region itself. + // Peel back to the enclosing scope, which does have a row, so the jump // still lands somewhere instead of being dropped as if elided. let resolvedAddress = peerAddress; while ( @@ -647,7 +686,7 @@ export function deriveVisibleGraph( } const peerId = nodeIdOf(resolvedAddress); if (!structure.nodes.has(peerId)) return; // elided scope, nothing to jump to - const port = ports.get(portId(p))!; + const port = mustGet(ports, portId(p)); const existing = port.peers.find( (peer) => nodeIdOf(peer.address) === peerId, ); @@ -671,7 +710,7 @@ export function deriveVisibleGraph( ); port.peers.push({ address: resolvedAddress, - label: structure.nodes.get(peerId)!.name, + label: mustGet(structure.nodes, peerId).name, messagesSent, batchesSent, channelTypes: channelType ? [channelType] : [], @@ -681,7 +720,7 @@ export function deriveVisibleGraph( }; // A channel touching a collapsed region box always does so at exactly the // box's own address (Timely gates every scope crossing through a single - // boundary vertex, one hop at a time; a plain, non-port box reference is + // boundary vertex, one hop at a time. A plain, non-port box reference is // therefore never deeper than its own address, unlike addPeer's peers, // which land outside viewRoot's subtree entirely). What can still be // ambiguous is which of the box's own inner ports a given hop's port @@ -795,7 +834,7 @@ export function deriveVisibleGraph( // A non-port endpoint id must name one of this view's own boxes. Real // dataflows can log channels touching an address with no corresponding // operator row (e.g. an elided scope), and any address outside viewRoot's - // subtree resolves to an id this view never emits either; drop the + // subtree resolves to an id this view never emits either. Drop the // edge (but keep any port registered above) in both cases rather than // hand elk a dangling reference. if ( @@ -813,7 +852,7 @@ export function deriveVisibleGraph( // ancestor box, an address that isn't this box's boundary at all) has // no more precise a target than the box itself, already shown. if (!from.port) { - const box = structure.nodes.get(from.id)!; + const box = mustGet(structure.nodes, from.id); if (addressesEqual(ch.fromAddress, box.address)) { const inner = endpointId( structure, @@ -838,7 +877,7 @@ export function deriveVisibleGraph( } } if (!to.port) { - const box = structure.nodes.get(to.id)!; + const box = mustGet(structure.nodes, to.id); if (addressesEqual(ch.toAddress, box.address)) { const inner = endpointId( structure, @@ -888,12 +927,14 @@ export function deriveVisibleGraph( return { nodes: [...nodes, ...ports.values()], edges }; } -// The scope to navigate to so that every given address is directly visible, -// each as either itself (if it's already a direct child of the result) or -// the box that rolls it up. Backs off past any address that IS the raw -// common prefix (so every input lands strictly below the result, never -// equal to it) and past any prefix with no operator row of its own (an -// elided scope can't be navigated to, since it never appears in a view). +/** + * The scope to navigate to so that every given address is directly visible, + * each as either itself (if it's already a direct child of the result) or + * the box that rolls it up. Backs off past any address that IS the raw + * common prefix (so every input lands strictly below the result, never + * equal to it) and past any prefix with no operator row of its own (an + * elided scope can't be navigated to, since it never appears in a view). + */ export function commonAncestorScope( structure: DataflowStructure, addresses: readonly Address[], @@ -914,17 +955,19 @@ export function commonAncestorScope( return nodeIdOf(prefix); } -// Hiding a run of idle operators would otherwise sever every path through -// them, splitting the graph into disconnected islands even though a real -// (idle) path exists. Splices a pass-through edge across each hidden run, -// so hiding idle nodes never removes connectivity, only the boxes in -// between. The spliced edge sums messages/batches/types along the run, -// which is almost always 0/0 (that is why the run was hidden) and so -// renders with the same dashed "idle" styling as an ordinary quiet edge. -// -// Cycles (timely feedback loops) are real and must not hang this: `path` -// tracks nodes on the current walk, and re-entering one simply stops that -// branch rather than looping forever. +/** + * Hiding a run of idle operators would otherwise sever every path through + * them, splitting the graph into disconnected islands even though a real + * (idle) path exists. Splices a pass-through edge across each hidden run, + * so hiding idle nodes never removes connectivity, only the boxes in + * between. The spliced edge sums messages/batches/types along the run, + * which is almost always 0/0 (that is why the run was hidden) and so + * renders with the same dashed "idle" styling as an ordinary quiet edge. + * + * Cycles (timely feedback loops) are real and must not hang this: `path` + * tracks nodes on the current walk, and re-entering one simply stops that + * branch rather than looping forever. + */ export function rerouteHiddenNodes( graph: VisibleGraph, hiddenNodeIds: ReadonlySet, @@ -1056,7 +1099,9 @@ export const DEFAULT_FILTERS: Filters = { showLirGroups: true, }; -// decorateGraph returns every field set. +/** + * decorateGraph returns every field set. + */ export interface GraphDecorations { dimmedNodeIds: Set; hiddenNodeIds: Set; @@ -1065,13 +1110,15 @@ export interface GraphDecorations { searchMatches: string[]; // visible node ids, document order } -// Search matches anywhere in the whole dataflow, not just the current view -// (search is structure-wide so it stays useful once a graph is too big to -// show in one screen). searchInfo carries that lookup in, precomputed once -// per search string rather than re-walked per view. A box that doesn't -// itself match but contains a match deeper in its rolled-up subtree is left -// undimmed rather than excluded, so the match's path stays visible without -// forcing a navigation on every keystroke. +/** + * Search matches anywhere in the whole dataflow, not just the current view + * (search is structure-wide so it stays useful once a graph is too big to + * show in one screen). searchInfo carries that lookup in, precomputed once + * per search string rather than re-walked per view. A box that doesn't + * itself match but contains a match deeper in its rolled-up subtree is left + * undimmed rather than excluded, so the match's path stays visible without + * forcing a navigation on every keystroke. + */ export function decorateGraph( graph: VisibleGraph, filters: Filters, @@ -1172,9 +1219,11 @@ export interface SubtreeSearchMatches { containsMatch: Set; // matches, plus any ancestor of one } -// Computed once over the whole structure (not the current view), so a box is -// never dimmed just because the current scope hasn't drilled down to the -// match nested inside it yet. +/** + * Computed once over the whole structure (not the current view), so a box is + * never dimmed just because the current scope hasn't drilled down to the + * match nested inside it yet. + */ export function subtreeSearchMatches( structure: DataflowStructure, search: string, @@ -1184,7 +1233,7 @@ export function subtreeSearchMatches( const containsMatch = new Set(); if (!needle) return { matches, containsMatch }; const visit = (id: NodeId): boolean => { - const node = structure.nodes.get(id)!; + const node = mustGet(structure.nodes, id); let hit = node.name.toLowerCase().includes(needle); if (hit) matches.add(id); for (const c of node.children) { @@ -1193,7 +1242,9 @@ export function subtreeSearchMatches( if (hit) containsMatch.add(id); return hit; }; - for (const id of structure.nodes.get(structure.root)!.children) visit(id); + for (const id of mustGet(structure.nodes, structure.root).children) { + visit(id); + } return { matches, containsMatch }; } @@ -1204,9 +1255,11 @@ function addressCompare(a: Address, b: Address): number { return a.length - b.length; } -// The ordered, structure-wide match list prev/next cycles through: jumping -// to a match may navigate the view (unlike decorateGraph's searchMatches, -// which only ever lists what's already visible in the current scope). +/** + * The ordered, structure-wide match list prev/next cycles through: jumping + * to a match may navigate the view (unlike decorateGraph's searchMatches, + * which only ever lists what's already visible in the current scope). + */ export function allSearchMatches( structure: DataflowStructure, search: string, @@ -1221,9 +1274,11 @@ export function allSearchMatches( .sort((a, b) => addressCompare(a.address, b.address)); } -// Groups operator nodes by the LIR span covering them, keyed -// `${exportId}/${lirId}`. One dataflow can back several exports, so entries -// are not unique per lir id across exports. +/** + * Groups operator nodes by the LIR span covering them, keyed + * `${exportId}/${lirId}`. One dataflow can back several exports, so entries + * are not unique per lir id across exports. + */ export function lirIndex( structure: DataflowStructure, ): Map { @@ -1247,12 +1302,14 @@ export interface LirTreeNode { summary: NodeStats; } -// Sums each member's OWN stats (not transitive): every operator belonging to -// a LIR id appears in memberIds exactly once, including a parent LIR id's -// descendants (buildDataflowStructure assigns an operator to every ancestor -// span that contains it), so this matches EXPLAIN ANALYZE's own total_memory/ -// total_records/total_elapsed exactly without a separate query: the same -// operator rows are already fetched for the graph itself. +/** + * Sums each member's OWN stats (not transitive): every operator belonging to + * a LIR id appears in memberIds exactly once, including a parent LIR id's + * descendants (buildDataflowStructure assigns an operator to every ancestor + * span that contains it), so this matches EXPLAIN ANALYZE's own total_memory/ + * total_records/total_elapsed exactly without a separate query: the same + * operator rows are already fetched for the graph itself. + */ export function lirSummary( structure: DataflowStructure, memberIds: readonly NodeId[], @@ -1274,12 +1331,14 @@ export function lirSummary( return summary; } -// Arranges lirIndex's flat entries into a tree per export, following -// parentLirId. A lir id is assigned once its own build (and so every -// descendant's) completes, so within one export a higher lir id is never an -// ancestor of a lower one. Sorting each level descending by lir id therefore -// matches construction order without needing it as an explicit input, the -// same convention EXPLAIN ANALYZE's own tree rendering uses. +/** + * Arranges lirIndex's flat entries into a tree per export, following + * parentLirId. A lir id is assigned once its own build (and so every + * descendant's) completes, so within one export a higher lir id is never an + * ancestor of a lower one. Sorting each level descending by lir id therefore + * matches construction order without needing it as an explicit input, the + * same convention EXPLAIN ANALYZE's own tree rendering uses. + */ export function lirTree( structure: DataflowStructure, index: ReadonlyMap, @@ -1324,33 +1383,39 @@ export interface LirGroupNode { nesting: number; } -// A grouping layer over an already-derived VisibleGraph, the same way -// decorateGraph layers dimming and color over it: nothing here replaces -// `nodes`, so search, dimming, and click handling keep working against the -// flat list untouched. +/** + * A grouping layer over an already-derived VisibleGraph, the same way + * decorateGraph layers dimming and color over it: nothing here replaces + * `nodes`, so search, dimming, and click handling keep working against the + * flat list untouched. + */ export interface LirGrouping { groups: LirGroupNode[]; // outer groups appear before groups nested inside them parentOf: Map; // a VisibleNode or LirGroupNode id -> its immediate enclosing group's id } -// Groups a scope's direct children into the LIR tree that encloses them, for -// drawing nested boxes on the canvas. LIR spans are contiguous ranges over -// operator ids and properly nested: a region is always fully enclosed by a -// single LIR node's range. A dataflow can back several exports, and a shared -// subplan can carry lir entries from more than one of them at once; a -// compound-node tree needs one parent chain per node, so this groups by the -// lowest exportId (string-compared) present among the given nodes only. A -// node whose only lir entries belong to a different export renders -// ungrouped, alongside nodes with no lir data at all (e.g. ports). +/** + * Groups a scope's direct children into the LIR tree that encloses them, for + * drawing nested boxes on the canvas. LIR spans are contiguous ranges over + * operator ids and properly nested: a region is always fully enclosed by a + * single LIR node's range. A dataflow can back several exports, and a shared + * subplan can carry lir entries from more than one of them at once; a + * compound-node tree needs one parent chain per node, so this groups by the + * lowest exportId (string-compared) present among the given nodes only. A + * node whose only lir entries belong to a different export renders + * ungrouped, alongside nodes with no lir data at all (e.g. ports). + */ export function groupByLir(nodes: readonly VisibleNode[]): LirGrouping { const groups: LirGroupNode[] = []; const parentOf = new Map(); const ordered = nodes - .filter((n) => n.operatorId !== null) + .filter( + (n): n is VisibleNode & { operatorId: bigint } => n.operatorId !== null, + ) .slice() .sort((a, b) => { - const x = a.operatorId!; - const y = b.operatorId!; + const x = a.operatorId; + const y = b.operatorId; return x < y ? -1 : x > y ? 1 : 0; }); if (ordered.length === 0) return { groups, parentOf }; @@ -1385,7 +1450,13 @@ export function groupByLir(nodes: readonly VisibleNode[]): LirGrouping { const key = stackKeys[depth]; let group = groupIndex.get(key); if (!group) { - const info = node.lir.find((l) => `${l.exportId}/${l.lirId}` === key)!; + const info = node.lir.find((l) => `${l.exportId}/${l.lirId}` === key); + if (!info) { + // key was derived from this same node.lir list two lines up + // (filtered to chosenExport, mapped to "exportId/lirId"), so the + // matching entry always exists. This only guards the invariant. + throw new Error(`missing lir info for key ${key}`); + } group = { id: key, exportId: info.exportId, diff --git a/console/src/platform/dataflows/elkGraph.ts b/console/src/platform/dataflows/elkGraph.ts index 047c5bd298938..c16716ae59008 100644 --- a/console/src/platform/dataflows/elkGraph.ts +++ b/console/src/platform/dataflows/elkGraph.ts @@ -82,7 +82,11 @@ export function toElkGraph( children: (childrenOf.get(id) ?? []).map(buildNode), }; } - return { id, ...NODE_DIMENSIONS[nodeById.get(id)!.kind] }; + const node = nodeById.get(id); + if (!node) { + throw new Error(`missing visible node for id ${id} building elk graph`); + } + return { id, ...NODE_DIMENSIONS[node.kind] }; }; return { diff --git a/console/src/utils/format.test.ts b/console/src/utils/format.test.ts index b72a989ba5876..12cd9f52fcff5 100644 --- a/console/src/utils/format.test.ts +++ b/console/src/utils/format.test.ts @@ -10,12 +10,32 @@ import parse from "postgres-interval"; import { + formatElapsedNs, formatInterval, formatIntervalShort, fromSeconds, toSeconds, } from "./format"; +describe("formatElapsedNs", () => { + it("formats zero as a plain 0s", () => { + expect(formatElapsedNs(0n)).toEqual("0s"); + }); + + it("shows sub-second durations in milliseconds instead of collapsing to 0s", () => { + expect(formatElapsedNs(257_000_000n)).toEqual("257ms"); + }); + + it("shows single-digit-second durations with one decimal place", () => { + expect(formatElapsedNs(1_400_000_000n)).toEqual("1.4s"); + expect(formatElapsedNs(1_600_000_000n)).toEqual("1.6s"); + }); + + it("rounds durations of 10s or more to whole seconds", () => { + expect(formatElapsedNs(543_400_000_000n)).toEqual("543s"); + }); +}); + describe("formatIntervalShort", () => { it("only returns years if present", () => { const interval = parse("1 years 2 mons 3 days 04:05:06.789"); diff --git a/console/src/utils/format.ts b/console/src/utils/format.ts index a65076cc62944..2fea2d04a61af 100644 --- a/console/src/utils/format.ts +++ b/console/src/utils/format.ts @@ -60,6 +60,21 @@ export const formatBytesShort = (bytes: bigint) => { return `${numberFormatter.format(value)} ${unit}`; }; +/** + * Formats a nanosecond duration for display. Rounding straight to whole + * seconds collapses every sub-second duration to "0s" and can't + * distinguish e.g. 1.4s from 1.6s, which matters for a dataflow operator's + * own (as opposed to subtree) elapsed time -- often sub-second even when + * its ancestors run for minutes. + */ +export function formatElapsedNs(ns: bigint): string { + const seconds = Number(ns) / 1e9; + if (seconds === 0) return "0s"; + if (seconds < 1) return `${Math.round(seconds * 1000)}ms`; + if (seconds < 10) return `${seconds.toFixed(1)}s`; + return `${Math.round(seconds)}s`; +} + export function convertBytes(bytes: number, unit: ByteUnit) { switch (unit) { case "B": From a885f913eeaf6a51ce7a7c9dc6dd509779b5ee2e Mon Sep 17 00:00:00 2001 From: Moritz Hoffmann Date: Wed, 8 Jul 2026 22:28:21 +0200 Subject: [PATCH 36/45] clusters: narrow props.cluster once instead of asserting it at each use Destructure cluster near the top of each *Table component so the nested render() closures use the narrowed const instead of props.cluster!, which loses TypeScript's narrowing across the closure boundary. Co-Authored-By: Claude Sonnet 5 --- console/src/platform/clusters/IndexList.tsx | 5 +++-- console/src/platform/clusters/MaterializedViewsList.tsx | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/console/src/platform/clusters/IndexList.tsx b/console/src/platform/clusters/IndexList.tsx index 9e31accc1183a..aa89ca2b7b8ae 100644 --- a/console/src/platform/clusters/IndexList.tsx +++ b/console/src/platform/clusters/IndexList.tsx @@ -211,6 +211,7 @@ const IndexTable = (props: IndexTableProps) => { const regionSlug = useRegionSlug(); const dataflowVisualizerEnabled = flags["visualization-features"]; const { colors } = useTheme(); + const { cluster } = props; return ( <> @@ -282,7 +283,7 @@ const IndexTable = (props: IndexTableProps) => { )} {formattedLag} - {dataflowVisualizerEnabled && props.cluster && ( + {dataflowVisualizerEnabled && cluster && ( { { e.stopPropagation(); }} diff --git a/console/src/platform/clusters/MaterializedViewsList.tsx b/console/src/platform/clusters/MaterializedViewsList.tsx index 1dd0f1b5ae6cf..7afbb03a8a07c 100644 --- a/console/src/platform/clusters/MaterializedViewsList.tsx +++ b/console/src/platform/clusters/MaterializedViewsList.tsx @@ -230,6 +230,7 @@ const MaterializedViewTable = (props: MaterializedViewTableProps) => { const materializedViewPath = useBuildMaterializedViewPath(); const regionSlug = useRegionSlug(); const dataflowVisualizerEnabled = flags["visualization-features"]; + const { cluster } = props; const { colors } = useTheme(); @@ -286,7 +287,7 @@ const MaterializedViewTable = (props: MaterializedViewTableProps) => { )} {formattedLag} - {dataflowVisualizerEnabled && props.cluster && ( + {dataflowVisualizerEnabled && cluster && ( { { e.stopPropagation(); }} From 94c263a02a0b43709ba288f3a52c4b3beb81bf4a Mon Sep 17 00:00:00 2001 From: Moritz Hoffmann Date: Wed, 8 Jul 2026 22:28:31 +0200 Subject: [PATCH 37/45] dataflows: read colors through the theme instead of hardcoded values DataflowGraphView, nodes.tsx, ChannelEdge, and LirPanel now pull colors from useTheme() (or the raw colors module for palette/highlight constants that intentionally read the same in both modes) instead of hardcoded hex values and Chakra's light-only gray.* tokens. React Flow's Background/Controls/MiniMap are themed via their documented --xy-* CSS custom properties, since they render outside Chakra's component tree. lirGroupColor/operatorColor/nodeFillColor now take their palette/accent colors as parameters instead of module-level constants, so callers can pass the theme-resolved palette. Co-Authored-By: Claude Sonnet 5 --- .../platform/dataflows/ChannelEdge.test.tsx | 19 +-- .../src/platform/dataflows/ChannelEdge.tsx | 11 +- .../platform/dataflows/DataflowGraphView.tsx | 67 ++++++++--- console/src/platform/dataflows/LirPanel.tsx | 85 +++++++------- .../src/platform/dataflows/nodeStyle.test.ts | 109 ++++++++++++------ console/src/platform/dataflows/nodeStyle.ts | 101 +++++++--------- console/src/platform/dataflows/nodes.tsx | 34 +++--- 7 files changed, 249 insertions(+), 177 deletions(-) diff --git a/console/src/platform/dataflows/ChannelEdge.test.tsx b/console/src/platform/dataflows/ChannelEdge.test.tsx index e383c563c4aa9..e2d6779add733 100644 --- a/console/src/platform/dataflows/ChannelEdge.test.tsx +++ b/console/src/platform/dataflows/ChannelEdge.test.tsx @@ -7,10 +7,13 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. +import { ThemeProvider } from "@chakra-ui/react"; import { fireEvent, render, screen } from "@testing-library/react"; import { Position, ReactFlowProvider, useStoreApi } from "@xyflow/react"; import React from "react"; +import { lightTheme } from "~/theme"; + import { ChannelEdge, type ChannelEdgeData } from "./ChannelEdge"; import type { PortPeer } from "./dataflowGraph"; import { HIGHLIGHT_COLORS } from "./nodeStyle"; @@ -49,13 +52,15 @@ const DomNodeSetter = ({ children }: { children: React.ReactNode }) => { function renderEdge(data: ChannelEdgeData) { const { container } = render( - - - - - - - , + + + + + + + + + , ); return container.querySelector("path")!; } diff --git a/console/src/platform/dataflows/ChannelEdge.tsx b/console/src/platform/dataflows/ChannelEdge.tsx index f92a9ac69651d..bd6e952882ab0 100644 --- a/console/src/platform/dataflows/ChannelEdge.tsx +++ b/console/src/platform/dataflows/ChannelEdge.tsx @@ -7,7 +7,7 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -import { Button, HStack, Text, Tooltip } from "@chakra-ui/react"; +import { Button, HStack, Text, Tooltip, useTheme } from "@chakra-ui/react"; import { BaseEdge, EdgeLabelRenderer, @@ -16,6 +16,8 @@ import { } from "@xyflow/react"; import React from "react"; +import { MaterializeTheme } from "~/theme"; + import type { PortPeer } from "./dataflowGraph"; import { HIGHLIGHT_COLORS, prettyPrintChannelType } from "./nodeStyle"; @@ -47,6 +49,7 @@ const compactCount = Intl.NumberFormat("default", { const compact = (n: bigint) => compactCount.format(n); export const ChannelEdge = (props: EdgeProps & { data: ChannelEdgeData }) => { + const { colors } = useTheme(); const [path, labelX, labelY] = getBezierPath(props); const { messagesSent, @@ -103,7 +106,7 @@ export const ChannelEdge = (props: EdgeProps & { data: ChannelEdgeData }) => { position="absolute" transform={`translate(-50%, -50%) translate(${labelX}px, ${labelY}px)`} fontSize="2xs" - background="whiteAlpha.800" + background={colors.background.secondary} px={1} borderRadius="sm" opacity={dimmed ? 0.15 : 1} @@ -134,7 +137,7 @@ export const ChannelEdge = (props: EdgeProps & { data: ChannelEdgeData }) => { ); @@ -109,12 +116,12 @@ export const LirPanel = ({ onClick={onToggleCollapsed} aria-label="Collapse LIR panel" > - « + {[...tree.entries()].map(([exportId, roots]) => ( - + {exportId} {roots.map((root) => ( diff --git a/console/src/platform/dataflows/nodeStyle.test.ts b/console/src/platform/dataflows/nodeStyle.test.ts index da6f714b1180d..ab283a3e89344 100644 --- a/console/src/platform/dataflows/nodeStyle.test.ts +++ b/console/src/platform/dataflows/nodeStyle.test.ts @@ -11,41 +11,62 @@ import { describe, expect, it } from "vitest"; import { formatSkew, + hashString, lirGroupColor, nodeFillColor, operatorColor, prettyPrintChannelType, } from "./nodeStyle"; +const PALETTE = ["#111111", "#222222", "#333333", "#444444"]; +const ACCENT = { arranged: "#aaaaaa", notArranged: "#bbbbbb" }; + +describe("hashString", () => { + it("is deterministic for the same string", () => { + expect(hashString("abc")).toEqual(hashString("abc")); + }); + + it("differs for different strings often enough to be useful", () => { + const hashes = new Set(["1", "2", "3", "4", "5", "6"].map(hashString)); + expect(hashes.size).toBeGreaterThan(1); + }); +}); + describe("lirGroupColor", () => { it("is deterministic for the same lirId", () => { - expect(lirGroupColor("42")).toEqual(lirGroupColor("42")); + expect(lirGroupColor("42", PALETTE)).toEqual(lirGroupColor("42", PALETTE)); }); it("differs for different lirIds often enough to be useful", () => { - const colors = new Set(["1", "2", "3", "4", "5", "6"].map(lirGroupColor)); - expect(colors.size).toBeGreaterThan(1); + const picked = new Set( + ["1", "2", "3", "4", "5", "6"].map((id) => lirGroupColor(id, PALETTE)), + ); + expect(picked.size).toBeGreaterThan(1); + }); + + it("only ever picks from the given palette", () => { + expect(PALETTE).toContain(lirGroupColor("some-lir-id", PALETTE)); }); }); describe("operatorColor", () => { it("is deterministic for the same operator name", () => { - expect(operatorColor("Reduce")).toEqual(operatorColor("Reduce")); + expect(operatorColor("Reduce", PALETTE)).toEqual( + operatorColor("Reduce", PALETTE), + ); }); it("differs for different operator names often enough to be useful", () => { - const colors = new Set( - ["Map", "Filter", "Reduce", "Join", "FlatMap", "Arrange"].map( - operatorColor, + const picked = new Set( + ["Map", "Filter", "Reduce", "Join", "FlatMap", "Arrange"].map((name) => + operatorColor(name, PALETTE), ), ); - expect(colors.size).toBeGreaterThan(1); + expect(picked.size).toBeGreaterThan(1); }); - it("is never pure white, unlike the old fixed operator fill", () => { - for (const name of ["Map", "Reduce", "Join", ""]) { - expect(operatorColor(name)).not.toEqual("#ffffff"); - } + it("only ever picks from the given palette", () => { + expect(PALETTE).toContain(operatorColor("Reduce", PALETTE)); }); }); @@ -67,35 +88,51 @@ describe("nodeFillColor", () => { }; it("colors an operator by its name, not by whether it's arranged", () => { - const unarranged = nodeFillColor({ ...baseNode, kind: "operator" }); - const arranged = nodeFillColor({ - ...baseNode, - kind: "operator", - transitive: { - arrangementRecords: 100n, - arrangementSize: 100n, - elapsedNs: 0n, - scheduleCount: 0n, + const unarranged = nodeFillColor( + { ...baseNode, kind: "operator" }, + PALETTE, + ACCENT, + ); + const arranged = nodeFillColor( + { + ...baseNode, + kind: "operator", + transitive: { + arrangementRecords: 100n, + arrangementSize: 100n, + elapsedNs: 0n, + scheduleCount: 0n, + }, }, - }); - expect(unarranged).toEqual(operatorColor("Reduce")); - expect(arranged).toEqual(operatorColor("Reduce")); + PALETTE, + ACCENT, + ); + expect(unarranged).toEqual(operatorColor("Reduce", PALETTE)); + expect(arranged).toEqual(operatorColor("Reduce", PALETTE)); }); it("still colors a region by whether it's arranged, unaffected by name", () => { - const noArrangement = nodeFillColor({ ...baseNode, kind: "region" }); - const arranged = nodeFillColor({ - ...baseNode, - kind: "region", - transitive: { - arrangementRecords: 100n, - arrangementSize: 100n, - elapsedNs: 0n, - scheduleCount: 0n, + const noArrangement = nodeFillColor( + { ...baseNode, kind: "region" }, + PALETTE, + ACCENT, + ); + const arranged = nodeFillColor( + { + ...baseNode, + kind: "region", + transitive: { + arrangementRecords: 100n, + arrangementSize: 100n, + elapsedNs: 0n, + scheduleCount: 0n, + }, }, - }); - expect(noArrangement).not.toEqual(arranged); - expect(noArrangement).not.toEqual(operatorColor("Reduce")); + PALETTE, + ACCENT, + ); + expect(noArrangement).toEqual(ACCENT.notArranged); + expect(arranged).toEqual(ACCENT.arranged); }); }); diff --git a/console/src/platform/dataflows/nodeStyle.ts b/console/src/platform/dataflows/nodeStyle.ts index 16130bafabccd..f72a013331b5a 100644 --- a/console/src/platform/dataflows/nodeStyle.ts +++ b/console/src/platform/dataflows/nodeStyle.ts @@ -7,86 +7,65 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -import type { LirGroupNode, VisibleNode } from "./dataflowGraph"; +import colors from "~/theme/colors"; -export const COLORS = { - noArrangementRegion: "#12b886", - arrangementRegion: "#7950f2", -}; +import type { LirGroupNode, VisibleNode } from "./dataflowGraph"; -// Mirrors theme/colors.ts blue.500 and orange.400. This app doesn't emit -// Chakra tokens as CSS custom properties, so `var(--chakra-colors-*)` -// silently resolves to nothing inside a raw boxShadow/stroke value; these -// need the literal hex. +// Selection/match highlights read the same in light and dark mode (a bright +// ring reads fine against either canvas background), so these pin to the +// raw swatch module instead of a light/dark-aware theme token, only to keep +// them from drifting out of sync with the rest of the palette. export const HIGHLIGHT_COLORS = { - selected: "#0093e6", - activeMatch: "#fe581d", + selected: colors.blue[500], + activeMatch: colors.orange[400], // Same hue as `selected`, for an edge that merely touches the selected // node rather than being the clicked thing itself; ChannelEdge pairs this // with a thinner stroke than `selected` gets, so the two read as a clear - // emphasis/de-emphasis pair rather than two unrelated colors. + // emphasis/de-emphasis pair rather than two unrelated colors. No exact + // swatch matches this shade; chosen for the pairing, not a token. connected: "#8fd2f5", }; -// A small, visually distinct palette for LIR group borders/headers, cycled -// by a deterministic hash so the same lirId always gets the same color -// within one render (and across re-renders, since lirId is stable). -const GROUP_PALETTE = [ - "#e8590c", - "#5f3dc4", - "#0b7285", - "#2f9e44", - "#c2255c", - "#1971c2", - "#e67700", - "#7048e8", -]; - -export function lirGroupColor(lirId: string): string { - let hash = 0; - for (let i = 0; i < lirId.length; i++) { - hash = (hash * 31 + lirId.charCodeAt(i)) | 0; - } - return GROUP_PALETTE[Math.abs(hash) % GROUP_PALETTE.length]; +/** Deterministic 32-bit string hash, for cycling a fixed color palette. */ +export function hashString(s: string): number { + let h = 0; + for (let i = 0; i < s.length; i++) + h = (Math.imul(h, 31) + s.charCodeAt(i)) | 0; + return h; } -// A muted, light subset of the spectrum for operator fills: deliberately -// less saturated than GROUP_PALETTE so an operator's fill never reads as -// "this is a LIR group" at a glance, while still giving operators of the -// same kind (every "Reduce", every "Map") a consistent, recognizable color -// instead of the old fixed white/orange pair, which made an unarranged -// operator (the common case) invisible against a white canvas. -const OPERATOR_PALETTE = [ - "#a5d8ff", // blue - "#99e9f2", // cyan - "#96f2d7", // teal - "#b2f2bb", // green - "#d8f5a2", // lime - "#ffec99", // yellow - "#ffd8a8", // orange - "#fcc2d7", // pink - "#eebefa", // grape - "#d0bfff", // violet - "#bac8ff", // indigo -]; +// Cycled by a hash of lirId/operator name so the same id or name always +// gets the same color within one render (and across re-renders, since both +// are stable). Groups and operators share one palette: LirGroupNode's own +// border/label already marks a box as a group, so the fill color doesn't +// need to carry that distinction too. +export function lirGroupColor( + lirId: string, + palette: readonly string[], +): string { + return palette[Math.abs(hashString(lirId)) % palette.length]; +} -export function operatorColor(name: string): string { - let hash = 0; - for (let i = 0; i < name.length; i++) { - hash = (hash * 31 + name.charCodeAt(i)) | 0; - } - return OPERATOR_PALETTE[Math.abs(hash) % OPERATOR_PALETTE.length]; +export function operatorColor( + name: string, + palette: readonly string[], +): string { + return palette[Math.abs(hashString(name)) % palette.length]; } -export function nodeFillColor(node: VisibleNode): string { +export function nodeFillColor( + node: VisibleNode, + palette: readonly string[], + accent: { arranged: string; notArranged: string }, +): string { const region = node.kind !== "operator" && node.kind !== "port"; - if (!region) return operatorColor(node.label); + if (!region) return operatorColor(node.label, palette); const arranged = (node.transitive?.arrangementRecords ?? 0n) > 0n; - return arranged ? COLORS.arrangementRegion : COLORS.noArrangementRegion; + return arranged ? accent.arranged : accent.notArranged; } // skewRatio's 0 is a "no per-worker data at all" sentinel, not a real ratio -// (a real ratio is always >= 1); labeling it distinctly avoids it reading +// (a real ratio is always >= 1). Labeling it distinctly avoids it reading // as an implausibly perfect score. export function formatSkew(ratio: number): string { return ratio === 0 ? "no data" : ratio.toFixed(2); diff --git a/console/src/platform/dataflows/nodes.tsx b/console/src/platform/dataflows/nodes.tsx index 1878dfb9f7aed..f267ba152c40c 100644 --- a/console/src/platform/dataflows/nodes.tsx +++ b/console/src/platform/dataflows/nodes.tsx @@ -7,10 +7,11 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -import { Badge, Box, HStack, Text, Tooltip } from "@chakra-ui/react"; +import { Badge, Box, HStack, Text, Tooltip, useTheme } from "@chakra-ui/react"; import { Handle, type NodeProps, Position } from "@xyflow/react"; import React from "react"; +import { MaterializeTheme } from "~/theme"; import { formatBytesShort } from "~/utils/format"; import type { VisibleNode } from "./dataflowGraph"; @@ -111,20 +112,23 @@ export const RegionNode = ({ data }: NodeProps & { data: FlowNodeData }) => ( ); -export const PortNode = ({ data }: NodeProps & { data: FlowNodeData }) => ( - - {data.node.label} - - - -); +export const PortNode = ({ data }: NodeProps & { data: FlowNodeData }) => { + const { colors } = useTheme(); + return ( + + {data.node.label} + + + + ); +}; // A label-only wrapper around its members. Header is clickable // (pointerEvents="auto"), body is not (pointerEvents="none"). For clicks on From 4e2df5c17a697f2f63ca6b5a81a7be26431460de Mon Sep 17 00:00:00 2001 From: Moritz Hoffmann Date: Wed, 8 Jul 2026 22:28:36 +0200 Subject: [PATCH 38/45] dataflows: reuse TextLink and the shared Alert component DataflowBreadcrumbs' clickable crumbs now use TextLink instead of a manually-styled Text-as-button, and gain a title on the truncated final crumb (matching ObjectJourneyBreadcrumb's convention). Its structure.nodes.get(id)! lookup becomes an explicit throw with the missing id in the message. UsagePrivilegeAlert now renders through the shared Alert component instead of raw Chakra Alert/AlertIcon. Co-Authored-By: Claude Sonnet 5 --- .../dataflows/DataflowBreadcrumbs.tsx | 31 ++++++++++++------ .../dataflows/UsagePrivilegeAlert.tsx | 32 ++++++++++++------- 2 files changed, 41 insertions(+), 22 deletions(-) diff --git a/console/src/platform/dataflows/DataflowBreadcrumbs.tsx b/console/src/platform/dataflows/DataflowBreadcrumbs.tsx index a3e413075ebeb..3d658d59e8a5b 100644 --- a/console/src/platform/dataflows/DataflowBreadcrumbs.tsx +++ b/console/src/platform/dataflows/DataflowBreadcrumbs.tsx @@ -10,6 +10,7 @@ import { HStack, Text, useTheme } from "@chakra-ui/react"; import React from "react"; +import TextLink from "~/components/TextLink"; import { ChevronRightIcon } from "~/icons"; import { MaterializeTheme } from "~/theme"; @@ -19,19 +20,24 @@ import { type NodeId, } from "./dataflowGraph"; +export interface DataflowBreadcrumbsProps { + structure: DataflowStructure; + focusedScope: NodeId; + onNavigate: (scope: NodeId) => void; +} + export const DataflowBreadcrumbs = ({ structure, focusedScope, onNavigate, -}: { - structure: DataflowStructure; - focusedScope: NodeId; - onNavigate: (scope: NodeId) => void; -}) => { +}: DataflowBreadcrumbsProps) => { const { colors } = useTheme(); const path: DataflowNode[] = []; for (let id: NodeId | null = focusedScope; id; ) { - const node: DataflowNode = structure.nodes.get(id)!; + const node = structure.nodes.get(id); + if (!node) { + throw new Error(`missing dataflow node for id ${id} in breadcrumb path`); + } path.unshift(node); id = node.parent; } @@ -39,20 +45,25 @@ export const DataflowBreadcrumbs = ({ {path.map((node, i) => i === path.length - 1 ? ( - + {node.name} ) : ( - onNavigate(node.id)} > {node.name} - + ( - - - - You'll need{" "} - - USAGE - {" "} - privilege on this cluster to {action}. - - +import Alert from "~/components/Alert"; + +export interface UsagePrivilegeAlertProps { + action: string; +} + +export const UsagePrivilegeAlert = ({ action }: UsagePrivilegeAlertProps) => ( + + You'll need{" "} + + USAGE + {" "} + privilege on this cluster to {action}. + + } + /> ); From 984f5a4064d04e21bf301807659b71df8f0c5a73 Mon Sep 17 00:00:00 2001 From: Moritz Hoffmann Date: Wed, 8 Jul 2026 22:28:41 +0200 Subject: [PATCH 39/45] NodeDetailPanel: reuse TextLink/icons, extract named prop interfaces MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Swap the manually-styled LIR link for TextLink, and the «/»/→ glyphs for ChevronLeftIcon/ChevronRightIcon/ArrowRightIcon. Extract every inline prop object type to a named (and, for the panel's own props, exported) interface, matching the rest of the codebase's convention. Co-Authored-By: Claude Sonnet 5 --- .../platform/dataflows/NodeDetailPanel.tsx | 423 ++++++++++-------- 1 file changed, 237 insertions(+), 186 deletions(-) diff --git a/console/src/platform/dataflows/NodeDetailPanel.tsx b/console/src/platform/dataflows/NodeDetailPanel.tsx index c25d4fc6c9ab5..4121ccdd8d94d 100644 --- a/console/src/platform/dataflows/NodeDetailPanel.tsx +++ b/console/src/platform/dataflows/NodeDetailPanel.tsx @@ -7,13 +7,22 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -import { Box, Button, CloseButton, HStack, Text } from "@chakra-ui/react"; +import { + Box, + Button, + CloseButton, + HStack, + Text, + useTheme, +} from "@chakra-ui/react"; import React from "react"; -import { formatBytesShort } from "~/utils/format"; +import TextLink from "~/components/TextLink"; +import { ArrowRightIcon, ChevronLeftIcon, ChevronRightIcon } from "~/icons"; +import { MaterializeTheme } from "~/theme"; +import { formatBytesShort, formatElapsedNs } from "~/utils/format"; import { - formatElapsedNs, type LirTreeNode, type PortPeer, type VisibleNode, @@ -26,43 +35,58 @@ export type Selection = | { kind: "edge"; edge: SelectedEdge } | { kind: "lirGroup"; node: LirTreeNode }; -const Row = ({ label, value }: { label: string; value: string }) => ( - - - {label} - - - {value} - - -); +interface RowProps { + label: string; + value: string; +} + +const Row = ({ label, value }: RowProps) => { + const { colors } = useTheme(); + return ( + + + {label} + + + {value} + + + ); +}; + +interface TypeRowProps { + channelTypes: string[]; +} // Channel types are Rust container type signatures (pretty-printed, but // still nested-generic shaped, e.g. "[Rc>]") long enough // that they get their own full-width stacked row instead of Row's // space-between line. -const TypeRow = ({ channelTypes }: { channelTypes: string[] }) => ( - - - Type - - - {channelTypes.map(prettyPrintChannelType).join(", ") || "unknown"} - - -); +const TypeRow = ({ channelTypes }: TypeRowProps) => { + const { colors } = useTheme(); + return ( + + + Type + + + {channelTypes.map(prettyPrintChannelType).join(", ") || "unknown"} + + + ); +}; -// onJumpTo is omitted for the compact form used inside a node's "Connected -// edges" list (which can be long for a hub node): landing lists there would -// swamp the panel. The dedicated edge-selection view always passes it, to -// show where a merged edge's real channels land inside a collapsed region. -const EdgeRows = ({ - edge, - onJumpTo, -}: { +interface EdgeRowsProps { edge: SelectedEdge; + // Omitted for the compact form used inside a node's "Connected edges" + // list (which can be long for a hub node): landing lists there would + // swamp the panel. The dedicated edge-selection view always passes it, + // to show where a merged edge's real channels land inside a collapsed + // region. onJumpTo?: (peer: PortPeer) => void; -}) => ( +} + +const EdgeRows = ({ edge, onJumpTo }: EdgeRowsProps) => ( <> @@ -90,34 +114,47 @@ const EdgeRows = ({ ); +interface PeerRowProps { + peer: PortPeer; + onJumpTo: (peer: PortPeer) => void; +} + // A port's peer lives outside the current view (that's why it's a port and // not a drawn edge), so its only affordance is a jump: navigate to wherever // the peer actually is instead of tracing a line to it. -const PeerRow = ({ - peer, - onJumpTo, -}: { - peer: PortPeer; - onJumpTo: (peer: PortPeer) => void; -}) => ( - - - - → {peer.label} - - - - - - - -); +const PeerRow = ({ peer, onJumpTo }: PeerRowProps) => { + const { colors } = useTheme(); + return ( + + + + + + {peer.label} + + + + + + + + + ); +}; + +interface LirGroupDetailProps { + node: LirTreeNode; +} // Same Row-based shape as NodeDetail, rather than a visually distinct card, // so a LIR group's details read like every other selection in this panel. -const LirGroupDetail = ({ node }: { node: LirTreeNode }) => ( +const LirGroupDetail = ({ node }: LirGroupDetailProps) => ( <> @@ -132,135 +169,156 @@ const LirGroupDetail = ({ node }: { node: LirTreeNode }) => ( ); +interface NodeDetailProps { + node: VisibleNode; + connectedEdges?: SelectedEdge[]; + onJumpTo: (peer: PortPeer) => void; + onSelectLir: (exportId: string, lirId: string) => void; +} + const NodeDetail = ({ node, connectedEdges, onJumpTo, onSelectLir, -}: { - node: VisibleNode; - connectedEdges?: SelectedEdge[]; +}: NodeDetailProps) => { + const { colors } = useTheme(); + return ( + <> + + {node.address && } + {node.operatorId !== null && ( + + )} + {node.childCount > 0 && ( + + )} + {node.own && ( + <> + + + + + + )} + {node.childCount > 0 && ( + + )} + {node.transitive && node.childCount > 0 && ( + <> + + + + + + )} + {node.ownSkew && ( + <> + + + + + )} + {node.transitiveSkew && node.childCount > 0 && ( + <> + + + + + )} + {node.lir.length > 0 && ( + + + LIR + + {node.lir.map((l) => ( + onSelectLir(l.exportId, l.lirId)} + > + LIR {l.lirId} ({l.exportId}): {l.operator} + + ))} + + )} + {connectedEdges && connectedEdges.length > 0 && ( + + + Connected edges + + {connectedEdges.map((e) => ( + + + {e.sourceLabel} → {e.targetLabel} + + + + ))} + + )} + {node.peers.length > 0 && ( + + + Connects outside this view + + {node.peers.map((p) => ( + + ))} + + )} + + ); +}; + +export interface NodeDetailPanelProps { + selection: Selection; + collapsed: boolean; + onToggleCollapsed: () => void; + onClose: () => void; onJumpTo: (peer: PortPeer) => void; onSelectLir: (exportId: string, lirId: string) => void; -}) => ( - <> - - {node.address && } - {node.operatorId !== null && ( - - )} - {node.childCount > 0 && ( - - )} - {node.own && ( - <> - - - - - - )} - {node.childCount > 0 && ( - - )} - {node.transitive && node.childCount > 0 && ( - <> - - - - - - )} - {node.ownSkew && ( - <> - - - - - )} - {node.transitiveSkew && node.childCount > 0 && ( - <> - - - - - )} - {node.lir.length > 0 && ( - - - LIR - - {node.lir.map((l) => ( - onSelectLir(l.exportId, l.lirId)} - > - LIR {l.lirId} ({l.exportId}): {l.operator} - - ))} - - )} - {connectedEdges && connectedEdges.length > 0 && ( - - - Connected edges - - {connectedEdges.map((e) => ( - - - {e.sourceLabel} → {e.targetLabel} - - - - ))} - - )} - {node.peers.length > 0 && ( - - - Connects outside this view - - {node.peers.map((p) => ( - - ))} - - )} - -); +} export const NodeDetailPanel = ({ selection, @@ -269,14 +327,7 @@ export const NodeDetailPanel = ({ onClose, onJumpTo, onSelectLir, -}: { - selection: Selection; - collapsed: boolean; - onToggleCollapsed: () => void; - onClose: () => void; - onJumpTo: (peer: PortPeer) => void; - onSelectLir: (exportId: string, lirId: string) => void; -}) => { +}: NodeDetailPanelProps) => { if (collapsed) { return ( - « + ); @@ -323,7 +374,7 @@ export const NodeDetailPanel = ({ onClick={onToggleCollapsed} aria-label="Collapse details panel" > - » + From 2c204f7f6b33cca945ceedb2376f88cb2bbc8cb4 Mon Sep 17 00:00:00 2001 From: Moritz Hoffmann Date: Wed, 8 Jul 2026 22:28:47 +0200 Subject: [PATCH 40/45] Extract useLastGoodByKey, dedup the last-good-result pattern useDataflowGraphData and useDataflowList each hand-rolled the same retain-last-good-result-while-a-new-key-loads logic (with two different implementations), guarding against briefly showing stale data mislabeled under a new key during a loading transition. Extract one hook and use it in both places. useDataflowGraphData also stops throwing on a malformed dataflow id: dataflowIdLiteral now returns undefined instead of throwing, and the hook surfaces an `error` field instead of crashing to the error boundary on a bad URL. Co-Authored-By: Claude Sonnet 5 --- .../dataflow/useDataflowGraphData.test.ts | 28 ++-- .../dataflow/useDataflowGraphData.ts | 118 ++++++++-------- .../materialize/dataflow/useDataflowList.ts | 29 +--- .../api/materialize/useLastGoodByKey.test.ts | 130 ++++++++++++++++++ .../src/api/materialize/useLastGoodByKey.ts | 72 ++++++++++ 5 files changed, 278 insertions(+), 99 deletions(-) create mode 100644 console/src/api/materialize/useLastGoodByKey.test.ts create mode 100644 console/src/api/materialize/useLastGoodByKey.ts diff --git a/console/src/api/materialize/dataflow/useDataflowGraphData.test.ts b/console/src/api/materialize/dataflow/useDataflowGraphData.test.ts index 4b1bcad9c80e7..66a5198782726 100644 --- a/console/src/api/materialize/dataflow/useDataflowGraphData.test.ts +++ b/console/src/api/materialize/dataflow/useDataflowGraphData.test.ts @@ -155,18 +155,22 @@ describe("useDataflowGraphData", () => { expect(result.current.data?.workerCount).toEqual(8); }); - it("rejects a non-numeric dataflow id", async () => { + it("rejects a non-numeric dataflow id without ever compiling a query for it", async () => { const Wrapper = await createProviderWrapper(); - expect(() => - renderHook( - () => - useDataflowGraphData({ - clusterName: "c", - replicaName: "r1", - dataflowId: "7; DROP", - }), - { wrapper: Wrapper }, - ), - ).toThrow(); + // A malformed id is only reachable via a hand-edited URL, so this must + // report an error the page can render, not throw and crash to the error + // boundary over a URL typo. + const { result } = renderHook( + () => + useDataflowGraphData({ + clusterName: "c", + replicaName: "r1", + dataflowId: "7; DROP", + }), + { wrapper: Wrapper }, + ); + expect(result.current.error).toEqual("invalid dataflow id: 7; DROP"); + expect(result.current.loading).toEqual(false); + expect(result.current.data).toBeNull(); }); }); diff --git a/console/src/api/materialize/dataflow/useDataflowGraphData.ts b/console/src/api/materialize/dataflow/useDataflowGraphData.ts index 45a6440a5258b..0f3d55841a3f7 100644 --- a/console/src/api/materialize/dataflow/useDataflowGraphData.ts +++ b/console/src/api/materialize/dataflow/useDataflowGraphData.ts @@ -11,10 +11,10 @@ import { sql } from "kysely"; import React from "react"; import { escapedLiteral as lit, useSqlManyTyped } from "~/api/materialize"; +import { useLastGoodByKey } from "~/api/materialize/useLastGoodByKey"; import { buildDataflowStructure, type ChannelRow, - type DataflowStructure, type LirSpanRow, type OperatorRow, type PerWorkerStatRow, @@ -32,10 +32,13 @@ interface ReplicaWorkerCountRow { workerCount: bigint | number | string; } +// A malformed id can only reach this hook via a hand-edited URL (the route +// itself only ever supplies real dataflow ids), so this returns undefined +// for the caller to render as a not-found state, rather than throwing +// inside useMemo and crashing to the error boundary over a typo in the +// address bar. function dataflowIdLiteral(dataflowId: string) { - if (!/^\d+$/.test(dataflowId)) { - throw new Error(`invalid dataflow id: ${dataflowId}`); - } + if (!/^\d+$/.test(dataflowId)) return undefined; return sql`${lit(dataflowId)}::uint8`; } @@ -59,6 +62,7 @@ export function useDataflowGraphData(params?: DataflowGraphParams) { ) return null; const id = dataflowIdLiteral(dataflowId); + if (id === undefined) return null; return { operators: sql` SELECT @@ -182,15 +186,30 @@ export function useDataflowGraphData(params?: DataflowGraphParams) { }; }, [dataflowId, clusterName, replicaName]); - const { results, error, databaseError, loading, refetch } = useSqlManyTyped( - queries, - { - cluster: params?.clusterName, - replica: params?.replicaName, - // This query can be slow for large dataflows. - timeout: 30_000, - }, - ); + const { + results, + error: queryError, + databaseError, + loading: queryLoading, + refetch, + } = useSqlManyTyped(queries, { + cluster: params?.clusterName, + replica: params?.replicaName, + // This query can be slow for large dataflows. + timeout: 30_000, + }); + + // A malformed dataflow id (only reachable via a hand-edited URL) never + // compiles a query, so `loading` would otherwise stay false forever with + // no error to show. Surfacing it here, rather than throwing inside the + // queries useMemo above, lets the page render a real error message + // instead of crashing to the error boundary over a URL typo. + const invalidDataflowId = + dataflowId !== undefined && dataflowIdLiteral(dataflowId) === undefined; + const error = invalidDataflowId + ? `invalid dataflow id: ${dataflowId}` + : queryError; + const loading = !invalidDataflowId && queryLoading; // JSON tuple, not a delimiter-joined string. Cluster and replica names are // identifiers that can themselves contain any delimiter we might pick. @@ -201,57 +220,30 @@ export function useDataflowGraphData(params?: DataflowGraphParams) { params.dataflowId, ]) : null; - const [lastGood, setLastGood] = React.useState<{ - key: string; - data: { - structure: DataflowStructure; - workerCount: number; - fetchedAt: Date; - }; - } | null>(null); - // Whether a fetch has actually started under the current key. Guards against - // tagging the previous fetch's still-resident results with the new key in - // the render after the selection changes but before useSqlMany's runSql - // effect flips loading. Without it the old graph would show under the new - // selection until the new fetch resolves (CNS-109). - const sawLoadingRef = React.useRef(false); - - // Reset acceptance synchronously when the selection changes. Using the - // render-phase state-adjustment pattern so the reset lands before any effect - // runs, not one render later. - const [trackedKey, setTrackedKey] = React.useState(key); - if (key !== trackedKey) { - setTrackedKey(key); - sawLoadingRef.current = false; - } - - // Tag successful results with the key they were fetched for, but only once a - // fetch has started under that key. - React.useEffect(() => { - if (loading) sawLoadingRef.current = true; - if (key && results && !error && !loading && sawLoadingRef.current) { - setLastGood({ - key, - data: { - structure: buildDataflowStructure( - results.operators ?? [], - results.channels ?? [], - results.lirSpans ?? [], - results.perWorkerStats ?? [], - ), - // A replica always has at least 1 worker; falling back to 1 (a - // heatmap ceiling of log2(1) = 0, disabling skew coloring - // entirely) only matters if the query somehow returns no row. - workerCount: Number(results.replicaWorkers?.[0]?.workerCount ?? 1), - fetchedAt: new Date(), - }, - }); - } - }, [key, results, error, loading]); + const compute = React.useCallback( + (r: NonNullable) => ({ + structure: buildDataflowStructure( + r.operators ?? [], + r.channels ?? [], + r.lirSpans ?? [], + r.perWorkerStats ?? [], + ), + // A replica always has at least 1 worker; falling back to 1 (a heatmap + // ceiling of log2(1) = 0, disabling skew coloring entirely) only + // matters if the query somehow returns no row. + workerCount: Number(r.replicaWorkers?.[0]?.workerCount ?? 1), + fetchedAt: new Date(), + }), + [], + ); + const data = useLastGoodByKey({ + key, + results, + error, + loading, + compute, + }); - // Error always wins, and data for other params never renders. - const data = - !error && lastGood && lastGood.key === key ? lastGood.data : null; return { data, error, databaseError, loading, refetch }; } diff --git a/console/src/api/materialize/dataflow/useDataflowList.ts b/console/src/api/materialize/dataflow/useDataflowList.ts index 4feb55ff4c99f..168c3c5426f95 100644 --- a/console/src/api/materialize/dataflow/useDataflowList.ts +++ b/console/src/api/materialize/dataflow/useDataflowList.ts @@ -11,6 +11,7 @@ import { sql } from "kysely"; import React from "react"; import { useSqlManyTyped } from "~/api/materialize"; +import { useLastGoodByKey } from "~/api/materialize/useLastGoodByKey"; import { queryBuilder } from "../db"; @@ -88,32 +89,12 @@ export function useDataflowList(params?: DataflowListParams) { const key = params ? JSON.stringify([params.clusterName, params.replicaName]) : null; - const [lastGood, setLastGood] = React.useState<{ - key: string; - data: DataflowListEntry[]; - } | null>(null); - // The key for which a fetch has actually started. Without this, the - // previous replica's still-resident results get tagged with the new key in - // the render after the selection changes but before useSqlMany's effect - // flips loading, showing the old replica's dataflow ids under the new - // replica (they're per-replica, so navigating one is meaningless, and for - // a transient dataflow that's gone on the new replica, a crash). Since a - // key change alone can't equal a not-yet-updated sawLoadingForKey, this - // doubles as the reset: no separate render-phase state adjustment needed. - const [sawLoadingForKey, setSawLoadingForKey] = React.useState( - null, + const compute = React.useCallback( + (r: NonNullable) => normalize(r.list ?? []), + [], ); + const data = useLastGoodByKey({ key, results, error, loading, compute }); - React.useEffect(() => { - if (loading) setSawLoadingForKey(key); - if (key && results && !error && !loading && sawLoadingForKey === key) { - setLastGood({ key, data: normalize(results.list ?? []) }); - } - }, [key, results, error, loading, sawLoadingForKey]); - - // Error always wins, and data for other params never renders. - const data = - !error && lastGood && lastGood.key === key ? lastGood.data : null; return { data, error, databaseError, loading, refetch }; } diff --git a/console/src/api/materialize/useLastGoodByKey.test.ts b/console/src/api/materialize/useLastGoodByKey.test.ts new file mode 100644 index 0000000000000..6422150d02783 --- /dev/null +++ b/console/src/api/materialize/useLastGoodByKey.test.ts @@ -0,0 +1,130 @@ +// Copyright Materialize, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +import { renderHook } from "@testing-library/react"; + +import { + useLastGoodByKey, + type UseLastGoodByKeyParams, +} from "./useLastGoodByKey"; + +type Params = UseLastGoodByKeyParams; + +function setup(initial: Params) { + return renderHook((props: Params) => useLastGoodByKey(props), { + initialProps: initial, + }); +} + +const compute = (r: string) => `computed:${r}`; + +// A real query hook always starts loading on first mount for a fresh key, +// only later flipping to `loading: false` with a result. Mounting directly +// in the already-loaded state (skipping that transition) would never set +// sawLoadingRef, so this drives a hook through the realistic sequence +// before a test builds further scenarios on top of an accepted result. +function primeAccepted( + key: string, + results: string, + fn: (r: string) => string, +) { + const rendered = setup({ + key, + results: null, + error: null, + loading: true, + compute: fn, + }); + rendered.rerender({ key, results, error: null, loading: false, compute: fn }); + return rendered; +} + +describe("useLastGoodByKey", () => { + it("returns null until a result arrives", () => { + const { result } = setup({ + key: "a", + results: null, + error: null, + loading: true, + compute, + }); + expect(result.current).toBeNull(); + }); + + it("accepts a result once loading finishes for the same key", () => { + const { result, rerender } = setup({ + key: "a", + results: null, + error: null, + loading: true, + compute, + }); + rerender({ key: "a", results: "r1", error: null, loading: false, compute }); + expect(result.current).toEqual("computed:r1"); + }); + + it("keeps showing the previous key's data while a new key is loading, not stale data mislabeled under the new key", () => { + const { result, rerender } = primeAccepted("a", "r1", compute); + expect(result.current).toEqual("computed:r1"); + + // Key changes; a fetch hasn't started yet (loading still false from the + // previous render). Without CNS-109's guard, the still-resident "r1" + // result would get relabeled as belonging to key "b". + rerender({ key: "b", results: "r1", error: null, loading: false, compute }); + expect(result.current).toBeNull(); + + // The new key's fetch starts (loading flips true), still with no new + // result yet. + rerender({ key: "b", results: "r1", error: null, loading: true, compute }); + expect(result.current).toBeNull(); + + // The new key's own result arrives. + rerender({ key: "b", results: "r2", error: null, loading: false, compute }); + expect(result.current).toEqual("computed:r2"); + }); + + it("clears data immediately on error, even before a refetch resolves", () => { + const { result, rerender } = primeAccepted("a", "r1", compute); + expect(result.current).toEqual("computed:r1"); + + rerender({ + key: "a", + results: "r1", + error: "boom", + loading: false, + compute, + }); + expect(result.current).toBeNull(); + }); + + it("recomputes only when compute is referentially stable across renders with the same results", () => { + let calls = 0; + const countingCompute = (r: string) => { + calls++; + return `computed:${r}`; + }; + const { rerender } = primeAccepted("a", "r1", countingCompute); + expect(calls).toEqual(1); + rerender({ + key: "a", + results: "r1", + error: null, + loading: false, + compute: countingCompute, + }); + rerender({ + key: "a", + results: "r1", + error: null, + loading: false, + compute: countingCompute, + }); + expect(calls).toEqual(1); + }); +}); diff --git a/console/src/api/materialize/useLastGoodByKey.ts b/console/src/api/materialize/useLastGoodByKey.ts new file mode 100644 index 0000000000000..eed6c83ede00c --- /dev/null +++ b/console/src/api/materialize/useLastGoodByKey.ts @@ -0,0 +1,72 @@ +// Copyright Materialize, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +import React from "react"; + +export interface UseLastGoodByKeyParams { + // Null when there's nothing to fetch yet (e.g. required params missing). + // Identifies the request a result belongs to (e.g. a JSON tuple of the + // params that vary the query), not the query text itself. + key: string | null; + results: TResult | null | undefined; + error: unknown; + loading: boolean; + // Must be referentially stable (e.g. wrapped in useCallback with an empty + // or narrow dependency list) or this recomputes on every render even when + // `results` hasn't actually changed. + compute: (results: TResult) => TData; +} + +/** + * Retains the last successfully computed value for a given key, so a caller + * keeps showing prior data while a new fetch for a *different* key is in + * flight, without briefly flashing the previous key's data under the new + * one. A result only replaces `lastGood` once a fetch has actually started + * under the current key (see `sawLoadingRef` below): without that guard, a + * key change followed by data that hasn't refetched yet would tag the old + * fetch's still-resident result with the new key in the render before the + * underlying query's own effect flips `loading` (CNS-109). + */ +export function useLastGoodByKey({ + key, + results, + error, + loading, + compute, +}: UseLastGoodByKeyParams): TData | null { + const [lastGood, setLastGood] = React.useState<{ + key: string; + data: TData; + } | null>(null); + + const sawLoadingRef = React.useRef(false); + + // Reset acceptance synchronously when the key changes, using the + // render-phase state-adjustment pattern so the reset lands before any + // effect runs, not one render later. + const [trackedKey, setTrackedKey] = React.useState(key); + if (key !== trackedKey) { + setTrackedKey(key); + // Deliberate render-phase write, not a side effect: it must land before + // this render commits, so a stale sawLoadingRef can't be read by the + // effect below in the same commit (one render later would be too late). + // eslint-disable-next-line react-compiler/react-compiler + sawLoadingRef.current = false; + } + + React.useEffect(() => { + if (loading) sawLoadingRef.current = true; + if (key && results != null && !error && !loading && sawLoadingRef.current) { + setLastGood({ key, data: compute(results) }); + } + }, [key, results, error, loading, compute]); + + // Error always wins, and data for other keys never renders. + return !error && lastGood && lastGood.key === key ? lastGood.data : null; +} From 4492aa31cfe109af7cf5e28f60aede33ffb3112d Mon Sep 17 00:00:00 2001 From: Moritz Hoffmann Date: Wed, 8 Jul 2026 22:28:53 +0200 Subject: [PATCH 41/45] Disable skew heatmap options on single-worker replicas Skew is a worst-worker-over-average ratio: with one worker there's nothing to compare against, so every node's ratio is trivially 1 (or the no-data 0 sentinel), and the heatmap silently rendered no color at all, indistinguishable from being off. Disable the three skew options instead, with a tooltip explaining why. Also reuses this as the pass to swap the toolbar's raw Input/Select for the shared SearchInput/SimpleSelect components. Co-Authored-By: Claude Sonnet 5 --- .../dataflows/DataflowToolbar.test.tsx | 58 +++++++++++++++++++ .../platform/dataflows/DataflowToolbar.tsx | 44 +++++++++++--- 2 files changed, 93 insertions(+), 9 deletions(-) create mode 100644 console/src/platform/dataflows/DataflowToolbar.test.tsx diff --git a/console/src/platform/dataflows/DataflowToolbar.test.tsx b/console/src/platform/dataflows/DataflowToolbar.test.tsx new file mode 100644 index 0000000000000..6481c6921ad66 --- /dev/null +++ b/console/src/platform/dataflows/DataflowToolbar.test.tsx @@ -0,0 +1,58 @@ +// Copyright Materialize, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +import { ThemeProvider } from "@chakra-ui/react"; +import { render, screen } from "@testing-library/react"; +import React from "react"; + +import { lightTheme } from "~/theme"; + +import { DEFAULT_FILTERS } from "./dataflowGraph"; +import { DataflowToolbar } from "./DataflowToolbar"; + +function renderToolbar(workerCount: number) { + return render( + + {}} + matchCount={0} + matchIndex={0} + onJump={() => {}} + workerCount={workerCount} + /> + , + ); +} + +describe("DataflowToolbar", () => { + it("disables the skew heatmap options on a single-worker replica, where skew is undefined", () => { + renderToolbar(1); + for (const label of [ + "Heat: CPU skew", + "Heat: memory skew", + "Heat: schedule skew", + ]) { + expect(screen.getByRole("option", { name: label })).toBeDisabled(); + } + // Non-skew heatmap modes still work fine on a single worker. + expect(screen.getByRole("option", { name: "Heat: elapsed" })).toBeEnabled(); + }); + + it("enables the skew heatmap options once there's more than one worker to compare", () => { + renderToolbar(4); + for (const label of [ + "Heat: CPU skew", + "Heat: memory skew", + "Heat: schedule skew", + ]) { + expect(screen.getByRole("option", { name: label })).toBeEnabled(); + } + }); +}); diff --git a/console/src/platform/dataflows/DataflowToolbar.tsx b/console/src/platform/dataflows/DataflowToolbar.tsx index 4f53a5ce2a5bb..d5376ef7a8fd0 100644 --- a/console/src/platform/dataflows/DataflowToolbar.tsx +++ b/console/src/platform/dataflows/DataflowToolbar.tsx @@ -12,8 +12,6 @@ import { FormControl, FormLabel, HStack, - Input, - Select, Slider, SliderFilledTrack, SliderThumb, @@ -23,6 +21,9 @@ import { } from "@chakra-ui/react"; import React from "react"; +import SearchInput from "~/components/SearchInput"; +import SimpleSelect from "~/components/SimpleSelect"; + import { type Filters } from "./dataflowGraph"; export interface DataflowToolbarProps { @@ -31,6 +32,12 @@ export interface DataflowToolbarProps { matchCount: number; matchIndex: number; onJump: (delta: 1 | -1) => void; + // Skew is a worst-worker-over-average ratio: on a single-worker replica + // there's nothing to compare against, so every node's ratio is trivially + // 1 (or the "no data" 0 sentinel), and the heatmap would silently render + // no color at all, indistinguishable from being off. Disabling the option + // instead of shipping a heatmap mode that quietly does nothing. + workerCount: number; } export const DataflowToolbar = ({ @@ -39,7 +46,9 @@ export const DataflowToolbar = ({ matchCount, matchIndex, onJump, + workerCount, }: DataflowToolbarProps) => { + const skewDisabled = workerCount <= 1; const [search, setSearch] = React.useState(filters.search); // Debounce search input into the filters object. React.useEffect(() => { @@ -52,7 +61,7 @@ export const DataflowToolbar = ({ }, [search, filters, onFiltersChange]); return ( - - + + + + Date: Wed, 8 Jul 2026 22:29:00 +0200 Subject: [PATCH 42/45] Add isInsufficientPrivilegeError, use it for the dataflows permission check DatabaseError.ts's isPermissonError expects a nested result.error.code shape; the dataflow pages get an ExecuteSqlError directly, a different shape, so add a predicate for it rather than force-fitting the existing one. useDataflowIdForExport now also surfaces databaseError, so DataflowsPage can check both its own and the export-id query's error. Co-Authored-By: Claude Sonnet 5 --- .../materialize/dataflow/useDataflowIdForExport.ts | 4 ++-- console/src/api/materialize/executeSql.tsx | 11 +++++++++++ console/src/platform/dataflows/DataflowsPage.tsx | 11 +++++------ 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/console/src/api/materialize/dataflow/useDataflowIdForExport.ts b/console/src/api/materialize/dataflow/useDataflowIdForExport.ts index 9df8af53533b5..4bcd8929e9330 100644 --- a/console/src/api/materialize/dataflow/useDataflowIdForExport.ts +++ b/console/src/api/materialize/dataflow/useDataflowIdForExport.ts @@ -40,12 +40,12 @@ export function useDataflowIdForExport(params?: DataflowIdForExportParams) { }; }, [exportId]); - const { results, error, loading } = useSqlManyTyped(queries, { + const { results, error, databaseError, loading } = useSqlManyTyped(queries, { cluster: params?.clusterName, replica: params?.replicaName, }); const dataflowId = !error && results ? (results.row?.[0]?.dataflowId ?? null) : null; - return { dataflowId, loading, error }; + return { dataflowId, loading, error, databaseError }; } diff --git a/console/src/api/materialize/executeSql.tsx b/console/src/api/materialize/executeSql.tsx index 21d6588f98e6f..d2a8d49a433bc 100644 --- a/console/src/api/materialize/executeSql.tsx +++ b/console/src/api/materialize/executeSql.tsx @@ -62,6 +62,17 @@ export function isExecuteSqlError(error: unknown): error is ExecuteSqlError { return error != null && typeof error === "object" && "errorMessage" in error; } +/** True for a `databaseError` denied by a missing USAGE/SELECT grant. */ +export function isInsufficientPrivilegeError( + error: ExecuteSqlError | null | undefined, +): boolean { + return ( + !!error && + "code" in error && + error.code === ErrorCode.INSUFFICIENT_PRIVILEGE + ); +} + export function buildExecuteSqlUrl(environmentdHttpAddress: string) { return new URL( `${apiClient.mzHttpUrlScheme}://${environmentdHttpAddress}/api/sql`, diff --git a/console/src/platform/dataflows/DataflowsPage.tsx b/console/src/platform/dataflows/DataflowsPage.tsx index 4cdcb517a196e..4d872ea65d605 100644 --- a/console/src/platform/dataflows/DataflowsPage.tsx +++ b/console/src/platform/dataflows/DataflowsPage.tsx @@ -22,14 +22,13 @@ import { Link, Navigate, useParams, useSearchParams } from "react-router-dom"; import { useDataflowIdForExport } from "~/api/materialize/dataflow/useDataflowIdForExport"; import { useDataflowList } from "~/api/materialize/dataflow/useDataflowList"; -import { ErrorCode } from "~/api/materialize/types"; +import { isInsufficientPrivilegeError } from "~/api/materialize/executeSql"; import ErrorBox from "~/components/ErrorBox"; import LabeledSelect from "~/components/LabeledSelect"; import { MainContentContainer } from "~/layouts/BaseLayout"; import { useAllClusters } from "~/store/allClusters"; -import { formatBytesShort } from "~/utils/format"; +import { formatBytesShort, formatElapsedNs } from "~/utils/format"; -import { formatElapsedNs } from "./dataflowGraph"; import { UsagePrivilegeAlert } from "./UsagePrivilegeAlert"; const DataflowsPage = () => { @@ -63,6 +62,7 @@ const DataflowsPage = () => { dataflowId, loading: exportLoading, error: exportError, + databaseError: exportDatabaseError, } = useDataflowIdForExport(exportParams); if (!cluster) return null; @@ -74,9 +74,8 @@ const DataflowsPage = () => { } } const permissionError = - databaseError && - "code" in databaseError && - databaseError.code === ErrorCode.INSUFFICIENT_PRIVILEGE; + isInsufficientPrivilegeError(databaseError) || + isInsufficientPrivilegeError(exportDatabaseError); // The export resolved cleanly but no dataflow is running for it on this // replica. Surface that above the list, which still renders below. From 60029267a2cb64f88e9bd2f5f350adbbd1246b2d Mon Sep 17 00:00:00 2001 From: Moritz Hoffmann Date: Wed, 8 Jul 2026 22:29:10 +0200 Subject: [PATCH 43/45] DataflowDetailPage: wire up mustGet, isInsufficientPrivilegeError, workerCount Replace the remaining .get(scope)! lookups with mustGet, drop the locally-duplicated hashString in favor of nodeStyle's, simplify permissionError to isInsufficientPrivilegeError, and pass workerCount through to DataflowToolbar for the skew-heatmap disable. Co-Authored-By: Claude Sonnet 5 --- .../platform/dataflows/DataflowDetailPage.tsx | 32 +++++++------------ 1 file changed, 12 insertions(+), 20 deletions(-) diff --git a/console/src/platform/dataflows/DataflowDetailPage.tsx b/console/src/platform/dataflows/DataflowDetailPage.tsx index 3b744a17a7f60..656f2d1c5474f 100644 --- a/console/src/platform/dataflows/DataflowDetailPage.tsx +++ b/console/src/platform/dataflows/DataflowDetailPage.tsx @@ -20,7 +20,7 @@ import { import { useDataflowGraphData } from "~/api/materialize/dataflow/useDataflowGraphData"; import { useDataflowList } from "~/api/materialize/dataflow/useDataflowList"; -import { ErrorCode } from "~/api/materialize/types"; +import { isInsufficientPrivilegeError } from "~/api/materialize/executeSql"; import ErrorBox from "~/components/ErrorBox"; import LabeledSelect from "~/components/LabeledSelect"; import { MainContentContainer } from "~/layouts/BaseLayout"; @@ -39,6 +39,7 @@ import { type Filters, lirIndex, lirSummary, + mustGet, type NodeId, nodeIdOf, type PortPeer, @@ -49,21 +50,12 @@ import { DataflowGraphView } from "./DataflowGraphView"; import { DataflowToolbar } from "./DataflowToolbar"; import { LirPanel } from "./LirPanel"; import { NodeDetailPanel, type Selection } from "./NodeDetailPanel"; +import { hashString } from "./nodeStyle"; import { UsagePrivilegeAlert } from "./UsagePrivilegeAlert"; // Injected into decorateGraph so the pure graph module stays d3-free. const heatColor = (t: number) => interpolateYlOrRd(0.15 + 0.85 * t); -// Stable 32-bit hash so a pure stats refresh (identical node ids) keeps the -// same structure key and reuses the layout, while any structural change -// produces a new key and relayouts. -function hashString(s: string): number { - let h = 0; - for (let i = 0; i < s.length; i++) - h = (Math.imul(h, 31) + s.charCodeAt(i)) | 0; - return h; -} - const DataflowDetailPage = () => { const { clusterId, dataflowId } = useParams(); const navigate = useNavigate(); @@ -222,7 +214,7 @@ const DataflowDetailPage = () => { if (!data || addresses.length === 0) return; const scope = commonAncestorScope(data.structure, addresses); setFocusedScope(scope); - const scopeAddress = data.structure.nodes.get(scope)!.address; + const scopeAddress = mustGet(data.structure.nodes, scope).address; const targets = [ ...new Set( addresses @@ -289,7 +281,7 @@ const DataflowDetailPage = () => { return; } const scope = commonAncestorScope(data.structure, [peer.address]); - const scopeAddress = data.structure.nodes.get(scope)!.address; + const scopeAddress = mustGet(data.structure.nodes, scope).address; const targetId = representativeInView(peer.address, scopeAddress); if (targetId) focusOn(scope, targetId); }, @@ -340,7 +332,7 @@ const DataflowDetailPage = () => { // this same handler would have the second silently discard the // first (see updateSearchParams's doc comment above). const scope = commonAncestorScope(data.structure, addresses); - const scopeAddress = data.structure.nodes.get(scope)!.address; + const scopeAddress = mustGet(data.structure.nodes, scope).address; const targets = [ ...new Set( addresses @@ -481,8 +473,10 @@ const DataflowDetailPage = () => { // A member outside the current scope, or rolled up into one of its // boxes, highlights that box instead of being silently dropped. if (lirHighlight) { - const focusedScopeAddress = - data.structure.nodes.get(focusedScope)!.address; + const focusedScopeAddress = mustGet( + data.structure.nodes, + focusedScope, + ).address; const highlighted = new Set(); for (const id of lirHighlight) { const address = data.structure.nodes.get(id)?.address; @@ -529,10 +523,7 @@ const DataflowDetailPage = () => { ); } - const permissionError = - databaseError && - "code" in databaseError && - databaseError.code === ErrorCode.INSUFFICIENT_PRIVILEGE; + const permissionError = isInsufficientPrivilegeError(databaseError); return ( // minH=0: MainContentContainer is a flex column item of BaseLayout's @@ -621,6 +612,7 @@ const DataflowDetailPage = () => { matchCount={allMatches.length} matchIndex={activeMatchIndex} onJump={onJump} + workerCount={data.workerCount} /> {filters.heatmap !== "off" && ( From 10aa5c2cda39b2076b1d837a1811188398e0d530 Mon Sep 17 00:00:00 2001 From: Moritz Hoffmann Date: Wed, 8 Jul 2026 19:07:12 +0200 Subject: [PATCH 44/45] Align dataflow node/panel styling with the workflow graph's visual language Canvas nodes: 8px card radius, subtle resting shadow, and text-ui-med/ text-small type tokens for name and stat lines, matching the workflow dependency graph's GraphNode. Selection color, edge animation, and elk's compact layout spacing are unchanged (elk auto-sizes to content, unlike the workflow graph's fixed-size dagre cards, so no spacing changes apply). Detail panel: row labels use text-ui-med/foreground.tertiary (matching SidebarItemLabel), row values keep monospace for numeric column alignment, and stat/LIR/edge/peer groups get a border-top divider and text-small section label, matching SidebarSection without its collapse behavior. Co-Authored-By: Claude Sonnet 5 --- .../platform/dataflows/NodeDetailPanel.tsx | 77 ++++++++++--------- console/src/platform/dataflows/nodes.tsx | 17 ++-- 2 files changed, 53 insertions(+), 41 deletions(-) diff --git a/console/src/platform/dataflows/NodeDetailPanel.tsx b/console/src/platform/dataflows/NodeDetailPanel.tsx index 4121ccdd8d94d..df54baf2ae48e 100644 --- a/console/src/platform/dataflows/NodeDetailPanel.tsx +++ b/console/src/platform/dataflows/NodeDetailPanel.tsx @@ -44,7 +44,7 @@ const Row = ({ label, value }: RowProps) => { const { colors } = useTheme(); return ( - + {label} @@ -54,6 +54,23 @@ const Row = ({ label, value }: RowProps) => { ); }; +// A border-top divider between stat groups (own vs subtree vs skew, or a +// titled block like LIR/Connected edges), matching the workflow graph +// sidebar's SidebarSection dividers, without that component's collapse +// behavior, which this panel's shorter, denser content doesn't need. +const StatGroup = ({ children }: { children: React.ReactNode }) => { + const { colors } = useTheme(); + return ( + + {children} + + ); +}; + +const SectionLabel = (props: React.PropsWithChildren) => ( + +); + interface TypeRowProps { channelTypes: string[]; } @@ -66,7 +83,7 @@ const TypeRow = ({ channelTypes }: TypeRowProps) => { const { colors } = useTheme(); return ( - + Type @@ -92,24 +109,20 @@ const EdgeRows = ({ edge, onJumpTo }: EdgeRowsProps) => ( {onJumpTo && edge.sourceLandings.length > 0 && ( - - - Inside {edge.sourceLabel} - + + Inside {edge.sourceLabel} {edge.sourceLandings.map((p) => ( ))} - + )} {onJumpTo && edge.targetLandings.length > 0 && ( - - - Inside {edge.targetLabel} - + + Inside {edge.targetLabel} {edge.targetLandings.map((p) => ( ))} - + )} ); @@ -194,7 +207,7 @@ const NodeDetail = ({ )} {node.own && ( - <> + - + )} {node.childCount > 0 && ( )} {node.transitive && node.childCount > 0 && ( - <> + - + )} {node.ownSkew && ( - <> + - + )} {node.transitiveSkew && node.childCount > 0 && ( - <> + - + )} {node.lir.length > 0 && ( - - - LIR - + + LIR {node.lir.map((l) => ( ))} - + )} {connectedEdges && connectedEdges.length > 0 && ( - - - Connected edges - + + Connected edges {connectedEdges.map((e) => ( ))} - + )} {node.peers.length > 0 && ( - - - Connects outside this view - + + Connects outside this view {node.peers.map((p) => ( ))} - + )} ); diff --git a/console/src/platform/dataflows/nodes.tsx b/console/src/platform/dataflows/nodes.tsx index f267ba152c40c..9967ac2a6a5bf 100644 --- a/console/src/platform/dataflows/nodes.tsx +++ b/console/src/platform/dataflows/nodes.tsx @@ -45,6 +45,11 @@ const highlightShadow = (data: FlowNodeData): string | undefined => { return undefined; }; +// A subtle resting shadow (matching the app's other node-graph view) so +// cards read as raised surfaces even when unselected; highlightShadow +// overrides it with a colored ring for selected/matched nodes. +const RESTING_SHADOW = "0px 0.5px 2.5px 0 rgba(0, 0, 0, 0.08)"; + const CardShell = ({ data, children, @@ -54,7 +59,7 @@ const CardShell = ({ }) => ( {children} @@ -79,11 +84,11 @@ export const OperatorNode = ({ data }: NodeProps & { data: FlowNodeData }) => ( isDisabled={data.node.lir.length === 0} > - + {data.node.label} {statLines(data.node).map((line) => ( - + {line} ))} @@ -97,7 +102,7 @@ export const OperatorNode = ({ data }: NodeProps & { data: FlowNodeData }) => ( export const RegionNode = ({ data }: NodeProps & { data: FlowNodeData }) => ( - + {data.node.label} @@ -105,7 +110,7 @@ export const RegionNode = ({ data }: NodeProps & { data: FlowNodeData }) => ( {statLines(data.node).map((line) => ( - + {line} ))} From b26299d448d2084133af4277dd10fd1a0c2bddc9 Mon Sep 17 00:00:00 2001 From: Moritz Hoffmann Date: Wed, 8 Jul 2026 22:30:26 +0200 Subject: [PATCH 45/45] Fix unreadable node text on dark hashed palette colors CardShell and LirGroupNode never set an explicit text color, so they inherited the default (dark) body text regardless of the hashed palette color behind them. colors.lineGraph mixes dark and light swatches in both themes (it's indexed by hash, not by color mode), so a dark background with the default dark text was unreadable. Add textColorFor(background), picking black or white by comparing WCAG contrast ratios against the fill's actual luminance, and use it in both places. Co-Authored-By: Claude Sonnet 5 --- .../src/platform/dataflows/nodeStyle.test.ts | 13 +++++++++ console/src/platform/dataflows/nodeStyle.ts | 29 +++++++++++++++++++ console/src/platform/dataflows/nodes.tsx | 4 ++- 3 files changed, 45 insertions(+), 1 deletion(-) diff --git a/console/src/platform/dataflows/nodeStyle.test.ts b/console/src/platform/dataflows/nodeStyle.test.ts index ab283a3e89344..95356149acb99 100644 --- a/console/src/platform/dataflows/nodeStyle.test.ts +++ b/console/src/platform/dataflows/nodeStyle.test.ts @@ -16,6 +16,7 @@ import { nodeFillColor, operatorColor, prettyPrintChannelType, + textColorFor, } from "./nodeStyle"; const PALETTE = ["#111111", "#222222", "#333333", "#444444"]; @@ -147,6 +148,18 @@ describe("formatSkew", () => { }); }); +describe("textColorFor", () => { + it("picks light text on a dark background", () => { + // colors.lineGraph's purple[700], the color from the reported contrast bug + expect(textColorFor("#391D7E")).toEqual("#FFF"); + }); + + it("picks dark text on a light background", () => { + // colors.lineGraph's purple[200] + expect(textColorFor("#C8B5FF")).toEqual("#111"); + }); +}); + describe("prettyPrintChannelType", () => { it("strips module paths, aliases Diff/Error, and brackets Vec", () => { expect( diff --git a/console/src/platform/dataflows/nodeStyle.ts b/console/src/platform/dataflows/nodeStyle.ts index f72a013331b5a..a35f6e24d5fd6 100644 --- a/console/src/platform/dataflows/nodeStyle.ts +++ b/console/src/platform/dataflows/nodeStyle.ts @@ -53,6 +53,35 @@ export function operatorColor( return palette[Math.abs(hashString(name)) % palette.length]; } +// WCAG relative luminance of a "#RRGGBB" color, for picking readable text. +function relativeLuminance(hex: string): number { + const channels = [ + parseInt(hex.slice(1, 3), 16), + parseInt(hex.slice(3, 5), 16), + parseInt(hex.slice(5, 7), 16), + ].map((c) => { + const s = c / 255; + return s <= 0.03928 ? s / 12.92 : Math.pow((s + 0.055) / 1.055, 2.4); + }); + return 0.2126 * channels[0] + 0.7152 * channels[1] + 0.0722 * channels[2]; +} + +// colors.lineGraph mixes dark and light swatches in both themes (it's +// indexed by hash, not by theme), so a node's fill color carries no +// relationship to the current color mode. Text on top of it needs a +// contrast decision from the fill's actual luminance, not a theme token. +// Pins to raw black/white (not theme foreground tokens) since those exist +// specifically for text on the app's own background, not on an arbitrary +// hashed fill. +export function textColorFor(background: string): string { + const luminance = relativeLuminance(background); + // WCAG contrast ratio, (lighter + 0.05) / (darker + 0.05), against each + // candidate; picking the higher one is equivalent to maximizing contrast. + const contrastWithWhite = 1.05 / (luminance + 0.05); + const contrastWithBlack = (luminance + 0.05) / 0.05; + return contrastWithBlack > contrastWithWhite ? colors.black : colors.white; +} + export function nodeFillColor( node: VisibleNode, palette: readonly string[], diff --git a/console/src/platform/dataflows/nodes.tsx b/console/src/platform/dataflows/nodes.tsx index 9967ac2a6a5bf..36a5a8b76aa08 100644 --- a/console/src/platform/dataflows/nodes.tsx +++ b/console/src/platform/dataflows/nodes.tsx @@ -20,6 +20,7 @@ import { type FlowNodeData, formatElapsed, HIGHLIGHT_COLORS, + textColorFor, } from "./nodeStyle"; // Records and size share one line, not two: the fixed node height only has @@ -66,6 +67,7 @@ const CardShell = ({ height="100%" overflow="hidden" background={data.color} + color={textColorFor(data.color)} opacity={data.dimmed ? 0.25 : 1} boxShadow={highlightShadow(data) ?? RESTING_SHADOW} > @@ -163,7 +165,7 @@ export const LirGroupNode = ({ data }: NodeProps & { data: FlowGroupData }) => ( background={data.color} borderBottomRightRadius="md" > - + {data.label}