/** * AiAssistTool — interactive prompt-based segmentation (experimental). * * The clinician left-clicks foreground / background points (or drags a box / * scribbles) on the current slice; a backend model returns a region mask that * is written into an INDEPENDENT scratch MaskVolume (`aiAssistMaskVolume`, * registered as `maskData.volumes['aiScratch']`) — never the validated layers. * * Network lives in the app layer: this tool only (a) maps screen → voxel-slice * coordinates, (b) accumulates the prompt for the current slice, (c) fires * `onPrompt` so the app can call the backend, and (d) applies the returned mask * via `applyMask`. The app's `useAiAssist` composable wires (c)→(d). * * Sandbox: snapshot on enter (`snapshot`), then on exit either `discard` * (restore snapshot) or merge is handled by NrrdTools (clone + pushGroup). * * Coordinate mapping supports all three views (axial/sagittal/coronal); it * inverts the per-axis display transform of RenderingUtils.renderSliceToCanvas * (only coronal 'y' is vertically flipped). Merge captures painted voxels from * any view (the scratch is a full 3D volume). */ import { BaseTool } from "./BaseTool"; import type { ToolContext } from "./BaseTool"; import { MaskVolume } from "../core"; import type { SliceRenderHostDeps } from "./ToolHost"; /** Scratch volume key in maskData.volumes — kept OUT of image.layers so * compositeAllLayers ignores it (we render it ourselves in start()). */ export declare const AI_SCRATCH_LAYER = "aiScratch"; export type AiPromptTool = "point" | "box" | "scribble" | "lasso"; export interface AiPromptPoint { x: number; y: number; z?: number; label: number; } export interface AiPromptPayload { axis: "x" | "y" | "z"; sliceIndex: number; width: number; height: number; points: AiPromptPoint[]; box?: { x0: number; y0: number; x1: number; y1: number; label: number; }; scribble?: AiPromptPoint[]; scribbleRadius?: number; /** Closed-contour lasso: ordered contour points (implicitly closed — the * backend connects the last point back to the first). One contour per gesture. */ lasso?: AiPromptPoint[]; } export interface AiMaskResult { axis: "x" | "y" | "z"; sliceIndex: number; width: number; height: number; rle: number[]; /** 3D result (engine B / nnInteractive): inclusive [lo,hi] slice span + one RLE per slice. */ sliceRange?: [number, number]; slices?: number[][]; } type AiAssistHostDeps = Pick; export declare class AiAssistTool extends BaseTool { private host; /** Independent scratch volume (also registered at maskData.volumes['aiScratch']). */ private scratchVol; /** Snapshot of the scratch buffer taken on enter (for discard). */ private snapshotData; /** Frozen regions ("New region" commits here). Pixels recorded here survive a * later re-prediction — even on the SAME channel — so multiple separate regions * can be drawn without the newest prediction wiping the previous ones. Null = * nothing frozen yet (applySliceRle then behaves as a single live region). */ private committedVol; private promptTool; private polarity; /** The label value (1-255) the AI currently paints into = the active Segmentation. * Each Segmentation is a label value in the single-channel scratch volume; its * colour lives in the volume's colorMap (set via setSegmentColor). */ private activeLabel; /** Scribble brush radius (px), user-adjustable via slider. */ private scribbleSize; /** Accumulated prompts for the CURRENT slice; reset on slice/axis change. */ private points; private scribble; /** Lasso contour sent to the backend (densified along the curve at finish). */ private lasso; private box; private activeSlice; /** Placed lasso vertices (voxel + screen + polarity). Empty = not editing. */ private lassoVerts; /** Undo / redo snapshot stacks of the vertex list (for Ctrl+Z / Ctrl+Y). */ private lassoUndo; private lassoRedo; /** Index of the vertex currently hovered (shows a red ✕; click removes it). −1 = none. */ private lassoHoverIdx; /** Screen-pixel radius for hit-testing a hovered vertex. */ private static readonly LASSO_HIT_R; /** Double-click detection (a dbl-click finishes the lasso). */ private lastLassoDownTime; private lastLassoDownPos; private dragging; private dragStart; private dragScreenStart; private dragScreen; private screenScribble; private hoverScreen; /** Hide the point seed markers once a prediction has painted (only the mask * should remain). A new point click re-shows them until its mask returns. */ private hidePointMarkers; /** Fired when a prompt gesture completes — app calls backend, then applyMask(). */ onPrompt: ((payload: AiPromptPayload) => void) | null; /** Fired whenever the lasso vertex set changes (add/delete/undo/redo/finish/cancel), * so the panel can reactively show the Finish button + vertex count. */ onLassoChange: ((count: number, editing: boolean) => void) | null; constructor(ctx: ToolContext, host: AiAssistHostDeps); setPromptTool(tool: AiPromptTool): void; setPolarity(label: number): void; setScribbleSize(size: number): void; /** Select the active Segmentation by its label value (1-255). Switching starts a * fresh prompt set — otherwise accumulated prompts would be re-predicted and * repainted into the new label, recolouring the previous segmentation's region. */ setActiveSegment(label: number): void; getActiveLabel(): number; /** Set a Segmentation's colour (its label's entry in the scratch volume colorMap). * The 2D overlay repaints next frame (copper3d's render loop re-reads the map). */ setSegmentColor(label: number, color: { r: number; g: number; b: number; a: number; }): void; /** "New segmentation": freeze the current regions (so they persist) and switch to * a fresh label, exactly like the old "New region" but targeting a new label. */ newSegment(label: number): void; /** Delete a Segmentation: erase every voxel painted with its label from BOTH the * live scratch and the frozen (committed) volume, so the region disappears from * the view (otherwise orphaned label voxels would render with the wrong colour). * Goes via setRawData so the volume's version counter bumps (overlay repaints). */ clearSegment(label: number): void; getScratchVolume(): MaskVolume | null; /** Serialize the scratch volume as per-z-slice RLE (only NON-empty slices), * for persisting to ai_generated_nii_LPS on the backend. Binary (any channel → * 1); RLE is alternating run lengths starting with a 0-run — the exact format * the backend's rle_decode expects. Uses getSliceUint8 (the same path the * clinician layer's /api/mask/replace uses) so orientation matches. */ getScratchSlices(): { axis: "z"; width: number; height: number; slices: { sliceIndex: number; rle: number[]; }[]; } | null; /** Per-SEGMENTATION serialization for the 3D build: one binary RLE per non-empty * z-slice PER label (no binarize-merge), so the backend writes a multi-label * NIfTI and colours the GLB per segmentation. Includes both live and frozen * regions (both live in scratchVol). */ getScratchSegments(): { axis: "z"; width: number; height: number; segments: { label: number; slices: { sliceIndex: number; rle: number[]; }[]; }[]; } | null; /** Create (or reuse) the scratch volume sized to the layer grid and register * it under maskData.volumes['aiScratch']. Snapshots the empty buffer. */ enter(): void; /** Remove the scratch volume from the registry and drop references. */ exit(): void; /** Discard everything painted since enter() — restore the snapshot. */ discard(): void; /** "New region": freeze every voxel painted so far so it survives later * re-predictions (even on the same channel), then start a fresh prompt set. * This is what makes drawing multiple separate regions actually work — without * it the next prediction's same-channel cleanup erases the previous region. */ commitRegion(): void; /** Clear the in-progress prompt set (e.g. slice/axis change). Does NOT freeze — * use commitRegion() for the "New region" action. */ resetPrompts(): void; /** Drop the in-progress lasso vertices + undo/redo history (does not touch a * prediction already painted from a finished lasso). */ private clearLassoEditing; /** Tell the app layer the lasso vertex set changed (drives the Finish button). */ private notifyLasso; /** Voxel-slice (width,height) for the current axis. */ private sliceDims; /** Screen offset → voxel-slice (vx,vy) for the current axis. */ private toVoxel; /** Voxel-slice (vx,vy) → screen offset (px). Exact inverse of toVoxel, so * markers / lasso curves drawn from voxel coords track pan AND zoom each frame * (the freehand previews store screen px directly; persistent overlays don't). */ private voxelToScreen; onPointerDown(e: MouseEvent): void; /** Lasso click: double-click finishes; click on a vertex deletes it; otherwise * add a vertex. Nothing is sent to the backend until finishLasso(). */ private onLassoPointerDown; onPointerMove(e: MouseEvent): void; onPointerUp(_e: MouseEvent): void; /** Reset the accumulated prompt set when the user scrubs to a new slice. */ private syncSlice; private emit; /** Index of the lasso vertex under (sx,sy) within the hit radius, else −1. * Uses live voxel→screen so it tracks pan/zoom. Topmost (latest) wins. */ private lassoVertAt; /** Snapshot the vertex list for undo (clears redo). */ private pushLassoUndo; /** Public: undo the last add/delete of a lasso vertex (Ctrl+Z while editing). */ lassoUndoAction(): void; /** Public: redo (Ctrl+Y / Ctrl+Shift+Z while editing). */ lassoRedoAction(): void; /** Public: abandon the in-progress lasso (Esc). */ cancelLasso(): void; /** Public: true while the user is placing lasso vertices (≥1 vertex). */ isLassoEditing(): boolean; /** Public: number of placed lasso vertices (drives the Finish button label/enable). */ getLassoVertCount(): number; /** Vertices ordered by polar angle around their centroid, so the closed curve is * a SIMPLE (non-self-intersecting) loop regardless of click order — otherwise * connecting points in click order + closing easily produces a bow-tie crossing. * Trade-off: deep concavities collapse to their star-shaped hull, which is fine * for a "rough loop around a blob". */ private orderedLassoVerts; /** Public: close the curve, densify it, and send it to the backend as the lasso * contour. Needs ≥3 vertices (a real area). Then clears the editing state. */ finishLasso(): void; /** Sample a smooth CLOSED Catmull-Rom spline through `pts` → denser polyline. * `segs` samples per control-point span. <3 pts → returns a copy unchanged. */ private sampleClosedSpline; applyMask(result: AiMaskResult): void; /** Merge one RLE-encoded predicted slice into the scratch volume at sliceIndex. * Predicted pixels → current channel; stale current-channel pixels cleared * UNLESS they belong to a frozen (committed) region; OTHER channels preserved. */ private applySliceRle; renderOverlay(targetCtx: CanvasRenderingContext2D): void; /** A seed marker (small filled dot + ring) for point prompts. */ private drawSeedMarker; /** Render the lasso editing overlay (curve + fill + vertices + hover ✕). */ private renderLassoOverlay; } export {};