/** * Copyright (c) 2025 Elara AI Pty Ltd * Licensed under BSL 1.1. See LICENSE for details. */ import { type TreePath } from '@elaraai/e3-types'; import type { TaskRunner } from './execution/interfaces.js'; import type { StorageBackend, LockHandle } from './storage/interfaces.js'; /** * Parse a keypath string (from pathToString) back to TreePath. * * The keypath format is: .field1.field2 (dot-separated field names) * Quoted identifiers use backticks: .field1.`complex/name` * * @param pathStr - The path string in keypath format * @returns TreePath array of path segments */ export declare function parsePathString(pathStr: string): TreePath; /** * Result of executing a single task in the dataflow. */ export interface TaskExecutionResult { /** Task name */ name: string; /** Whether the task was cached */ cached: boolean; /** Execution ID (UUIDv7) - present for executed or cached tasks */ executionId?: string; /** Final state */ state: 'success' | 'failed' | 'error' | 'skipped'; /** Error message if state is 'error' */ error?: string; /** Exit code if state is 'failed' */ exitCode?: number; /** Duration in milliseconds */ duration: number; } /** * Result of a dataflow execution. */ export interface DataflowResult { /** Overall success - true if all tasks completed successfully */ success: boolean; /** Dataflow run ID (UUIDv7) */ runId: string; /** Number of tasks executed (not from cache) */ executed: number; /** Number of tasks served from cache */ cached: number; /** Number of tasks that failed */ failed: number; /** Number of tasks skipped due to upstream failure */ skipped: number; /** Number of tasks re-executed due to input changes */ reexecuted: number; /** Per-task results */ tasks: TaskExecutionResult[]; /** Total duration in milliseconds */ duration: number; } /** * Options for dataflow execution. */ export interface DataflowOptions { /** Maximum concurrent task executions (default: 4) */ concurrency?: number; /** Force re-execution even if cached (default: false) */ force?: boolean; /** Filter to run only specific task(s) by exact name */ filter?: string; /** External workspace lock to use. */ lock?: LockHandle; /** AbortSignal for cancellation. */ signal?: AbortSignal; /** Task runner for executing individual tasks. */ runner?: TaskRunner; /** Callback when a task starts */ onTaskStart?: (name: string) => void; /** Callback when a task completes */ onTaskComplete?: (result: TaskExecutionResult) => void; /** Callback for task stdout */ onStdout?: (taskName: string, data: string) => void; /** Callback for task stderr */ onStderr?: (taskName: string, data: string) => void; /** Callback when an input dataset changes during execution (reactive dataflow) */ onInputChanged?: (path: string, previousHash: string, newHash: string) => void; /** Callback when a task is invalidated due to input change */ onTaskInvalidated?: (taskName: string, reason: string) => void; /** Callback when a task is deferred due to inconsistent input versions */ onTaskDeferred?: (taskName: string, conflictPath: string) => void; } /** * Execute all tasks in a workspace according to the dependency graph. * * Delegates to `LocalOrchestrator` which implements reactive fixpoint * execution using step functions. After each task completes, input changes * are detected and affected tasks are invalidated and re-executed. * * @param storage - Storage backend * @param repo - Repository identifier * @param ws - Workspace name * @param options - Execution options * @returns Result of the dataflow execution * * @throws {WorkspaceLockError} If workspace is locked by another process * @throws {WorkspaceNotFoundError} If workspace doesn't exist * @throws {WorkspaceNotDeployedError} If workspace has no package deployed * @throws {TaskNotFoundError} If filter specifies a task that doesn't exist * @throws {DataflowError} If execution fails for other reasons */ export declare function dataflowExecute(storage: StorageBackend, repo: string, ws: string, options?: DataflowOptions): Promise; /** * Execute dataflow with an externally-held lock. * The lock is released automatically when execution completes or fails. * * @param storage - Storage backend * @param repo - Repository identifier * @param ws - Workspace name * @param options - Execution options (lock must be provided) * @returns Promise that resolves when execution completes */ export declare function dataflowStart(storage: StorageBackend, repo: string, ws: string, options: DataflowOptions & { lock: LockHandle; }): Promise; /** * Get the dependency graph for a workspace (for visualization/debugging). * * @param storage - Storage backend * @param repo - Repository identifier * @param ws - Workspace name * @returns Graph information * * @throws {WorkspaceNotFoundError} If workspace doesn't exist * @throws {WorkspaceNotDeployedError} If workspace has no package deployed * @throws {DataflowError} If graph building fails for other reasons */ export declare function dataflowGetGraph(storage: StorageBackend, repo: string, ws: string): Promise<{ tasks: Array<{ name: string; hash: string; inputs: string[]; output: string; dependsOn: string[]; }>; }>; /** * Graph structure returned by dataflowGetGraph. */ export interface DataflowGraph { tasks: Array<{ name: string; hash: string; inputs: string[]; output: string; dependsOn: string[]; }>; } /** * Find all tasks affected by input changes (transitive dependents). * An affected task is one whose output could change due to the input change. * * @param graph - The dependency graph * @param changes - Array of changed input paths * @returns Array of affected task names */ export declare function findAffectedTasks(graph: DataflowGraph, changes: Array<{ path: string; }>): string[]; /** * Get tasks that are ready to execute given the set of completed tasks. * * A task is ready when all tasks it depends on have completed. * * @param graph - The dependency graph from dataflowGetGraph * @param completedTasks - Set of task names that have completed * @returns Array of task names that are ready to execute */ export declare function dataflowGetReadyTasks(graph: DataflowGraph, completedTasks: Set): string[]; /** * Get the dependency closure of a task: the task itself plus every task it * transitively depends on (its upstream producers). * * A `--filter ` run is scoped to this closure so the target's upstream * can run — resolving from cache — and make the target ready. Without it the * filtered target would wait forever on dependencies the filter excluded from * scheduling. * * @param graph - The dependency graph from dataflowGetGraph * @param target - Name of the task whose closure to compute * @returns Set of task names: the target plus all transitive dependencies */ export declare function dataflowGetDependencyClosure(graph: DataflowGraph, target: string): Set; /** * Check if a task execution is cached for the given inputs. * * @param storage - Storage backend * @param repo - Repository path * @param taskHash - Hash of the TaskObject * @param inputHashes - Array of input dataset hashes (in order) * @returns Output hash if cached, null if execution needed */ export declare function dataflowCheckCache(storage: StorageBackend, repo: string, taskHash: string, inputHashes: string[]): Promise; /** * Find tasks that should be skipped when a task fails. * * Returns all tasks that transitively depend on the failed task, * excluding already completed or already skipped tasks. * * @param graph - The dependency graph from dataflowGetGraph * @param failedTask - Name of the task that failed * @param completedTasks - Set of task names already completed * @param skippedTasks - Set of task names already skipped * @returns Array of task names that should be skipped */ export declare function dataflowGetDependentsToSkip(graph: DataflowGraph, failedTask: string, completedTasks: Set, skippedTasks: Set): string[]; /** * Resolve input hashes for a task from current workspace state. * * @param storage - Storage backend * @param repo - Repository path * @param ws - Workspace name * @param task - Task info from the graph * @returns Array of hashes (null if input is unassigned) */ export declare function dataflowResolveInputHashes(storage: StorageBackend, repo: string, ws: string, task: DataflowGraph['tasks'][0]): Promise>; //# sourceMappingURL=dataflow.d.ts.map