/** * One join, one rule: which compiled substep does a docflow binding belong to? * * The dashboard used to answer this six separate times with the same * expression — `binding.line >= substep.sourceRange.startLine && binding.line * <= substep.sourceRange.endLine`. That compares two DIFFERENT constructs by * containment and is wrong on every chained play: static analysis sets a * dataset stage's `sourceRange` to `ctx.dataset(key, rows)` alone (the head of * the chain), while the members that belong to it annotate `.withColumn(...)` * calls further down. On `find-ctos` the stage spans lines 54–55 and its four * members bind at 57, 79, 84 and 89 — the join returned nothing, so loop * members got no coverage chip and no provider logo. * * The docflow language already carries a structural key, and lint already * enforces it, so the join does not need a coordinate: * * - A **subgraph member**'s `out:` names per-row COLUMNS of the dataset it * annotates. `docflow_loop_incomplete` validates those names against the * dataset's real computed columns. * - Any **other** binding's `out:` names the ASSIGNED VARIABLE of the annotated * statement. `docflow_output_not_found` is a hard `plays check` error when it * does not. * * So: symbol first. A binding whose `out:` root names a column a dataset * computes belongs to that dataset, wherever either one sits in the file. That * key survives reformatting, an inserted comment, and a change in how the AST * walk shapes ranges — the failures that produced this bug. * * Position remains the fallback for exactly the case where the language gives * no symbol to join on: an assignment-named node (`out:"searchResult"` on a * `ctx.tools.execute` statement, `out:"companies"` on the dataset statement * itself), where the compiler's own name for the substep (`company_search`, * `cto_targets`) is by design NOT the author's variable. This mirrors ADR 0016 * rule 1, which resolves a binding by symbol and keeps position as the fallback * when the symbol abstains. * * The fallback compares like with like, which the old join did not: a * substep's extent is its own range UNIONED with every nested step's range — * the whole construct the substep owns, not its head call. Among containing * candidates the smallest extent wins, so a dataset nested inside another * dataset's per-row body resolves to the inner one. * * Failure is a state, not a default. When no substep owns a binding the * resolution says so (`reason`), and callers render nothing rather than * borrowing a neighbour's numbers or logo. The loud gate lives upstream where * it belongs: `plays check` already fails a binding whose `out:` names neither * a real column nor the assigned variable. */ import type { PlayDocflow, PlayDocflowBinding } from './docflow'; import type { PlayStaticSourceRange, PlayStaticSubstep, } from './static-pipeline'; export type DocflowDatasetSubstep = Extract< PlayStaticSubstep, { type: 'dataset' } >; export type DocflowOwnerSubstep = Extract< PlayStaticSubstep, { type: 'dataset' | 'csv' } >; export type DocflowToolSubstep = Extract; export type DocflowPlayCallSubstep = Extract< PlayStaticSubstep, { type: 'play_call' } >; export type DocflowStepSuiteSubstep = Extract< PlayStaticSubstep, { type: 'step_suite' } >; /** * Every substep in a pipeline tree, parents before children. Top-level substeps * are not enough for child plays: measured on the shipped prebuilts, the only * real `ctx.runPlay` call site (`people-search-to-email`) is a `play_call` * nested inside a dataset's per-row body, so a top-level-only scan finds none. */ export function flattenSubsteps( substeps: readonly PlayStaticSubstep[], ): PlayStaticSubstep[] { const flat: PlayStaticSubstep[] = []; const visit = (candidate: PlayStaticSubstep) => { flat.push(candidate); const nested = (candidate as { steps?: PlayStaticSubstep[] }).steps; if (Array.isArray(nested)) for (const step of nested) visit(step); }; for (const substep of substeps) visit(substep); return flat; } /** How a binding reached its substep — see the module header for the rules. */ export type DocflowJoinVia = 'symbol' | 'extent'; export type DocflowJoinFailure = /** The binding names no joinable symbol and no substep extent contains it. */ | 'no-owner' /** Two or more substeps claim the symbol and none contains the line. */ | 'ambiguous'; export type DocflowBindingJoin = | { substep: TSubstep; via: DocflowJoinVia; reason?: undefined } | { substep: null; via?: undefined; reason: DocflowJoinFailure }; export type DocflowLineExtent = { startLine: number; endLine: number }; /** * The line span of the whole construct a substep owns: its own range unioned * with every nested step's range. A `ctx.dataset(...)` chain's own range covers * only the head call, so a dataset's per-row body is only visible once the * `.withColumn(...)` steps are folded in. */ export function substepSourceExtent( substep: PlayStaticSubstep, ): DocflowLineExtent | null { const ranges: PlayStaticSourceRange[] = []; const collect = (candidate: PlayStaticSubstep) => { if (candidate.sourceRange) ranges.push(candidate.sourceRange); const nested = (candidate as { steps?: PlayStaticSubstep[] }).steps; if (Array.isArray(nested)) for (const step of nested) collect(step); }; collect(substep); if (ranges.length === 0) return null; return { startLine: Math.min(...ranges.map((range) => range.startLine)), endLine: Math.max(...ranges.map((range) => range.endLine)), }; } /** * The distinct first path segments of a binding's `out:` contract, dropping * `$output` (a return value names no substep symbol). */ export function docflowOutputRoots(binding: PlayDocflowBinding): string[] { return [ ...new Set( (binding.outputs ?? []) .map((path) => path.split('.')[0] ?? '') .filter((root) => root.length > 0 && root !== '$output'), ), ]; } /** * The names of the per-row columns a dataset computes — the exact set a loop * member's `out:` is validated against by `docflow_loop_incomplete`. Seed/input * columns are excluded on purpose: a member claims work the dataset DOES, not a * column it merely carries. */ export function datasetComputedColumnNames( substep: DocflowDatasetSubstep, ): Set { const names = new Set(); const add = (name: string | undefined | null) => { if (name && name !== 'row_number') names.add(name); }; for (const step of substep.steps ?? []) { if ('field' in step) add(step.field); } for (const column of substep.columns ?? []) { if (column.source !== 'datasetColumn') continue; add(column.sqlName ?? column.id); add(column.id); for (const producer of column.producers ?? []) { add(producer.field.split('.')[0]); } } for (const column of substep.sheetContract?.columns ?? []) { if (column.source !== 'datasetColumn') continue; add(column.field ?? column.id); add(column.sqlName); } return names; } /** * The extent a positional fallback measures a binding against. Defaults to the * whole construct a substep owns ({@link substepSourceExtent}); a caller whose * construct is a single CALL passes {@link substepOwnSourceExtent} instead. */ export type DocflowExtentResolver = ( substep: PlayStaticSubstep, ) => DocflowLineExtent | null; /** * A substep's OWN call range, ignoring nested steps. The right extent for a * construct whose annotation sits directly above one call — `ctx.runSteps(...)` * — where the union would instead span every leg the builder declares, and for * an imported builder those legs live in ANOTHER FILE, so the union is not even * a coherent interval. */ export function substepOwnSourceExtent( substep: PlayStaticSubstep, ): DocflowLineExtent | null { const range = substep.sourceRange; return range ? { startLine: range.startLine, endLine: range.endLine } : null; } function extentContainsLine( substep: PlayStaticSubstep, line: number, extentOf: DocflowExtentResolver, ): DocflowLineExtent | null { const extent = extentOf(substep); if (!extent) return null; return line >= extent.startLine && line <= extent.endLine ? extent : null; } /** Smallest containing extent wins, so a nested construct beats its parent. */ function smallestContaining( candidates: readonly TSubstep[], line: number, extentOf: DocflowExtentResolver = substepSourceExtent, ): TSubstep | null { let best: TSubstep | null = null; let bestSpan = Number.POSITIVE_INFINITY; for (const candidate of candidates) { const extent = extentContainsLine(candidate, line, extentOf); if (!extent) continue; const span = extent.endLine - extent.startLine; if (span < bestSpan) { best = candidate; bestSpan = span; } } return best; } /** * One call site, identified by what it IS. A pipeline exposes the same construct * through more than one tree — the stages tree and the flattened compiled * substeps each carry their own copy of a nested `play_call` — so a naive match * count sees N copies of one statement and calls it ambiguous. Ambiguity has to * mean "two DIFFERENT statements compute this symbol", which is the only case a * guess could get wrong. */ function substepIdentity(substep: PlayStaticSubstep): string { const range = substep.sourceRange; const position = range ? `${range.sourcePath ?? ''}:${range.startLine}:${range.startColumn ?? ''}-${range.endLine}:${range.endColumn ?? ''}` : 'no-range'; const name = (substep as { field?: string }).field ?? (substep as { alias?: string }).alias ?? ''; return `${substep.type}::${name}::${position}`; } function joinBySymbolThenExtent( binding: PlayDocflowBinding, candidates: readonly TSubstep[], symbolsOf: (substep: TSubstep) => Set, extentOf: DocflowExtentResolver = substepSourceExtent, ): DocflowBindingJoin { const roots = docflowOutputRoots(binding); if (roots.length > 0) { const byIdentity = new Map(); for (const candidate of candidates) { const symbols = symbolsOf(candidate); if (!roots.some((root) => symbols.has(root))) continue; const identity = substepIdentity(candidate); if (!byIdentity.has(identity)) byIdentity.set(identity, candidate); } const matches = [...byIdentity.values()]; if (matches.length === 1) return { substep: matches[0]!, via: 'symbol' }; if (matches.length > 1) { // Two constructs compute the same column name. The one whose body the // annotation actually sits in decides; otherwise this is drift, and a // guess would attribute a member's rows to the wrong table. const narrowed = smallestContaining(matches, binding.line, extentOf); if (narrowed) return { substep: narrowed, via: 'symbol' }; return { substep: null, reason: 'ambiguous' }; } } const positional = smallestContaining(candidates, binding.line, extentOf); if (positional) return { substep: positional, via: 'extent' }; return { substep: null, reason: 'no-owner' }; } /** * The dataset (or CSV) substep a docflow binding belongs to. Datasets carry * computed-column symbols; a CSV substep has rows but no per-row column work, * so it can only be reached by extent. */ export function resolveDocflowOwnerSubstep( binding: PlayDocflowBinding, substeps: readonly PlayStaticSubstep[], ): DocflowBindingJoin { const owners = substeps.filter( (substep): substep is DocflowOwnerSubstep => substep.type === 'dataset' || substep.type === 'csv', ); return joinBySymbolThenExtent(binding, owners, (substep) => substep.type === 'dataset' ? datasetComputedColumnNames(substep) : new Set(), ); } /** Same join, narrowed to dataset substeps (CSV has no per-row columns). */ export function resolveDocflowOwnerDataset( binding: PlayDocflowBinding, substeps: readonly PlayStaticSubstep[], ): DocflowBindingJoin { const resolution = resolveDocflowOwnerSubstep(binding, substeps); if (resolution.substep?.type === 'dataset') { return { substep: resolution.substep, via: resolution.via! }; } return { substep: null, reason: resolution.substep ? 'no-owner' : resolution.reason, }; } /** * The top-level tool substep a docflow binding names. A tool substep's `field` * is the column it produces, so a member's `out:"col"` joins by symbol; an * assignment-named action node (`out:"searchResult"`) falls back to extent. */ export function resolveDocflowBoundTool( binding: PlayDocflowBinding, substeps: readonly PlayStaticSubstep[], ): DocflowBindingJoin { const tools = substeps.filter( (substep): substep is DocflowToolSubstep => substep.type === 'tool', ); return joinBySymbolThenExtent( binding, tools, (substep) => new Set([substep.field]), ); } /** * The `ctx.runPlay` call a `type:"play"` docflow node names. Same join as every * other binding — symbol first, extent as the fallback — over the FLATTENED * substep tree, because a child call inside a `.withColumn(...)` body is nested * under its dataset. A `play_call`'s `field` is the column or variable the child * result lands in, which is exactly what the node's `out:` names. */ export function resolveDocflowBoundPlayCall( binding: PlayDocflowBinding, substeps: readonly PlayStaticSubstep[], ): DocflowBindingJoin { const playCalls = flattenSubsteps(substeps).filter( (substep): substep is DocflowPlayCallSubstep => substep.type === 'play_call', ); return joinBySymbolThenExtent( binding, playCalls, (substep) => new Set(substep.field.split('.')), ); } // ── waterfalls: the step suite and its legs ───────────────────────────────── // // `steps().step('hunter_email', …).step('leadmagic_email', …).return(…)` compiles // to ONE `step_suite` substep whose children are the legs. Six shipped prebuilts // run one of these as their whole scalar body, 4–16 legs deep, and until this // join existed an authored node over the waterfall resolved to nothing: the // suite is not a `tool`, so `resolveDocflowBoundTool` skipped it and the node // rendered with no provider, no legs, and no order — strictly less than the // undiagrammed `StepSuiteCard` beside it. // // A leg is NOT one substep. The extractor emits one substep per tool CALL inside // the leg's resolver, so a leg that finds a phone and then validates it emits // two, both carrying the leg's own `field`. `contact-to-phone-waterfall` compiles // 21 substeps for 11 legs that way. The leg is the `field`; its substeps are how // it works. /** The dotted tail of a leg's field relative to its suite (`steps.a` -> `a`). */ function legNameFromField(suiteField: string, legField: string): string { return legField.startsWith(`${suiteField}.`) ? legField.slice(suiteField.length + 1) : legField; } /** * One leg of a waterfall, as the source declares it. Ordered, deduplicated by * field, and carrying only decidable facts about the text — never a claim about * whether this run reached it. */ export type DocflowStepSuiteLeg = { /** The leg's durable step name — the string the author passed `.step(…)`. */ name: string; /** The suite-qualified field, i.e. the compiled substep key. */ field: string; /** The first provider call the leg makes, when it makes one. */ toolId: string | null; /** Every provider call the leg makes, in order. */ toolIds: string[]; /** The leg is `runIf`-guarded, so whether it runs depends on earlier legs. */ conditional: boolean; /** Every substep of the leg is statically off (`runIf: () => false`). */ disabled: boolean; /** The leg is itself a nested `steps()` program. */ nested: boolean; }; /** * The legs of a suite, in declaration order. Deduplicated by field so a * find-then-validate leg reads as one attempt; `conditional` is true when ANY of * a leg's substeps is guarded, `disabled` only when EVERY one is off. */ export function stepSuiteLegs( suite: DocflowStepSuiteSubstep, ): DocflowStepSuiteLeg[] { const byField = new Map(); for (const child of suite.steps ?? []) { const field = (child as { field?: string }).field ?? ''; if (!field) continue; const toolIds = flattenSubsteps([child]) .filter((step): step is DocflowToolSubstep => step.type === 'tool') .map((step) => step.toolId); const existing = byField.get(field); if (existing) { existing.toolIds.push(...toolIds); existing.toolId ??= toolIds[0] ?? null; existing.conditional ||= child.conditional === true; existing.disabled &&= child.disabled === true; existing.nested ||= child.type === 'step_suite'; continue; } byField.set(field, { name: legNameFromField(suite.field, field), field, toolId: toolIds[0] ?? null, toolIds: [...toolIds], conditional: child.conditional === true, disabled: child.disabled === true, nested: child.type === 'step_suite', }); } return [...byField.values()]; } /** * The `step_suite` a docflow node names — the whole waterfall as one step. * * Symbol first, like every other join. Inside a dataset the suite's `field` IS * the column it fills (`.withColumn('email_result', personToEmailSteps())` * compiles to `step_suite` field `email_result`), so `out:"email_result"` binds * by name. A scalar `ctx.runSteps(personalEmailSteps(), input)` has no such * name — the extractor calls it `steps` because the argument is a call, not a * named program — so `out:"result"` falls through to position. * * That fallback measures the suite's OWN call range, never the union with its * legs: an imported builder's legs live in another file, and unioning line * numbers across files produces an interval that means nothing. The annotation * sits directly above the `ctx.runSteps(...)` call, which is exactly what the * own range covers. */ export function resolveDocflowBoundStepSuite( binding: PlayDocflowBinding, substeps: readonly PlayStaticSubstep[], ): DocflowBindingJoin { const suites = flattenSubsteps(substeps).filter( (substep): substep is DocflowStepSuiteSubstep => substep.type === 'step_suite', ); return joinBySymbolThenExtent( binding, suites, (substep) => new Set([substep.field, substep.field.split('.').pop()!]), substepOwnSourceExtent, ); } /** One leg, with the suite it belongs to. */ export type DocflowBoundStepLeg = { suite: DocflowStepSuiteSubstep; leg: DocflowStepSuiteLeg; }; /** * The waterfall LEG a docflow node names, for a member drawn inside a waterfall * region. Symbol only — a leg name is always available (the author wrote it as a * string literal in `.step('', …)`), so there is nothing position could * add, and the legs of an imported builder have no line coordinate this file * could compare against anyway. * * Ambiguity is real and reported: two suites in one play may both declare a leg * called `validate`, and picking one would attribute a provider to the wrong * cascade. */ export function resolveDocflowBoundStepLeg( binding: PlayDocflowBinding, substeps: readonly PlayStaticSubstep[], ): DocflowBindingJoin { const roots = docflowOutputRoots(binding); if (roots.length === 0) return { substep: null, reason: 'no-owner' }; const seen = new Set(); const matches: DocflowBoundStepLeg[] = []; for (const suite of flattenSubsteps(substeps)) { if (suite.type !== 'step_suite') continue; for (const leg of stepSuiteLegs(suite)) { if (!roots.includes(leg.name) && !roots.includes(leg.field)) continue; // The same suite reaches this walk through both the stages tree and the // flattened compiled substeps, so identity is the leg's field, not the // object. if (seen.has(leg.field)) continue; seen.add(leg.field); matches.push({ suite, leg }); } } if (matches.length === 1) return { substep: matches[0]!, via: 'symbol' }; return { substep: null, reason: matches.length > 1 ? 'ambiguous' : 'no-owner', }; } // ── the waterfall region ──────────────────────────────────────────────────── export type DocflowWaterfallRegionMember = { nodeId: string; leg: DocflowStepSuiteLeg; /** 1-based position of the leg in the cascade the suite declares. */ position: number; }; /** * A `subgraph` whose members are the legs of one cascade. * * Ownership needs no edge and no coordinate: a member's `out:` names a leg, a leg * belongs to exactly one suite, so the members THEMSELVES say which cascade the * region is. That is strictly better than the dataset loop region, which has to * fall back to "which dataset does an edge touch" and then disambiguate by line. */ export type DocflowWaterfallRegion = { subgraphId: string; suite: DocflowStepSuiteSubstep; /** Drawn legs, in the order the SUITE declares them, not the order drawn. */ members: DocflowWaterfallRegionMember[]; /** * Members that carry an `out:` and still name no leg of this suite. A claim * the region cannot back — surfaced, never rendered as if fine. */ foreignMemberIds: string[]; }; export type DocflowWaterfallRegionIndex = { regions: DocflowWaterfallRegion[]; /** * Subgraphs whose members name legs of MORE than one cascade. One region * cannot be two waterfalls, and picking one would attribute providers to the * wrong cascade. */ splitRegions: Array<{ subgraphId: string; suiteFields: string[] }>; }; /** * Every waterfall region in a diagram. A subgraph with no leg-bound member is * not one (it is a dataset loop, or pure presentation) and is absent from both * lists, so this is safe to run over any diagram. */ export function resolveDocflowWaterfallRegions( docflow: PlayDocflow | null | undefined, substeps: readonly PlayStaticSubstep[], ): DocflowWaterfallRegionIndex { const regions: DocflowWaterfallRegion[] = []; const splitRegions: Array<{ subgraphId: string; suiteFields: string[] }> = []; if (!docflow?.subgraphs?.length) return { regions, splitRegions }; const bindingByNodeId = new Map( docflow.bindings.map((binding) => [binding.nodeId, binding]), ); for (const subgraph of docflow.subgraphs) { const resolved: Array<{ nodeId: string; suite: DocflowStepSuiteSubstep; leg: DocflowStepSuiteLeg; }> = []; const foreignMemberIds: string[] = []; for (const memberId of subgraph.memberIds) { const binding = bindingByNodeId.get(memberId); // An unbound member, or one that names nothing, is presentation — a note // drawn inside the region. Only a member making a claim can be wrong. if (!binding || docflowOutputRoots(binding).length === 0) continue; const join = resolveDocflowBoundStepLeg(binding, substeps); if (join.substep) { resolved.push({ nodeId: memberId, ...join.substep }); } else { foreignMemberIds.push(memberId); } } if (resolved.length === 0) continue; const suiteFields = [ ...new Set(resolved.map((entry) => entry.suite.field)), ]; if (suiteFields.length > 1) { splitRegions.push({ subgraphId: subgraph.id, suiteFields }); continue; } const suite = resolved[0]!.suite; const order = stepSuiteLegs(suite).map((leg) => leg.field); regions.push({ subgraphId: subgraph.id, suite, members: resolved .map((entry) => ({ nodeId: entry.nodeId, leg: entry.leg, position: order.indexOf(entry.leg.field) + 1, })) .sort((left, right) => left.position - right.position), // Only a member that claimed a leg of THIS suite and missed is foreign. // A member naming a leg of another suite already made the region split. foreignMemberIds, }); } return { regions, splitRegions }; } export type DocflowOwnerIndex = { /** Owning dataset/CSV substep per bound docflow node id. */ ownerByNodeId: Map; /** Bindings no substep owns, with why — a state to surface, not to default. */ unresolved: Array<{ nodeId: string; reason: DocflowJoinFailure }>; }; /** Resolves every binding in a docflow once, for callers that need the map. */ export function resolveDocflowOwnerIndex( docflow: PlayDocflow | null | undefined, substeps: readonly PlayStaticSubstep[], ): DocflowOwnerIndex { const ownerByNodeId = new Map(); const unresolved: Array<{ nodeId: string; reason: DocflowJoinFailure }> = []; for (const binding of docflow?.bindings ?? []) { const resolution = resolveDocflowOwnerSubstep(binding, substeps); if (resolution.substep) ownerByNodeId.set(binding.nodeId, resolution.substep); else unresolved.push({ nodeId: binding.nodeId, reason: resolution.reason }); } return { ownerByNodeId, unresolved }; }