/** * text.d.ts - Main Text Inference Pipeline (Thin Orchestrator) * * This module orchestrates inference by delegating to specialized modules: * - state.js: Holds model configuration, weights, and runtime state * - generator.js: Handles token generation loops and decoding * - init.js: Initialization, weight loading, KV cache, RoPE * * The pipeline maintains state and coordinates the flow from input tokens to generated output. * * @module inference/pipelines/text */ import { PipelineState } from './text/state.js'; import { PipelineGenerator } from './text/generator.js'; import type { Manifest } from './text/config.js'; import type { WeightLoadResult, PipelineContexts } from './text/init.js'; import type { GenerateOptions, KVCacheSnapshot, LogitsStepResult, PrefillResult, PrefillEmbeddingResult, SequenceEncodeOptions, SequenceEncodeResult, AdvanceEmbeddingResult, LayerWeights, ExpertWeights, RouterWeights, GenerationResult, PipelineStats, BatchingStats, WorkloadPhaseTiming, NativeLoRAPrefillOptions, NativeLoRAPrefillResult } from './text/types.js'; import type { ChatMessage } from './text/chat-format.js'; import type { LoRAAdapter } from './text/lora.js'; import type { DiffusionPipeline } from './diffusion/pipeline.js'; import type { EnergyPipeline } from './energy/pipeline.js'; import type { StructuredJsonHeadPipeline } from './structured/json-head-pipeline.js'; import type { EnergyRowHeadPipeline } from './energy-head/row-head-pipeline.js'; import { getBufferPool as getGlobalBufferPool } from '../../memory/buffer-pool.js'; import type { EmulationStats } from '../../config/schema/index.js'; import type { DiffusionGemmaCanvasLogitsInput, DiffusionGemmaCanvasStepInput, DiffusionGemmaCanvasStepResult, } from './text/generator.js'; // Re-export types for external use export type { GenerateOptions, KVCacheSnapshot, LogitsStepResult, PrefillResult, PrefillEmbeddingResult, SequenceEncodeOptions, SequenceEncodeResult, AdvanceEmbeddingResult, LayerWeights, ExpertWeights, RouterWeights, GenerationResult, PipelineStats, BatchingStats, NativeLoRAPrefillOptions, NativeLoRAPrefillResult }; export type { PipelineContexts }; export interface ChatRequestInput { messages: ChatMessage[]; } export type PromptInput = string | ChatMessage[] | ChatRequestInput; export declare function buildConservativeMultimodalGenerationOptions( options?: GenerateOptions ): GenerateOptions; export declare class AbortError extends Error { code: 'ABORT_ERR'; constructor(message?: string); } export declare function isAbortError(err: unknown): boolean; // ============================================================================ // Main Inference Pipeline Class // ============================================================================ export declare class InferencePipeline extends PipelineState { private generator; // Progress callback private _onProgress; private _preloadedWeights; constructor(); // ========================================================================== // Initialization // ========================================================================== initialize(contexts?: PipelineContexts): Promise; loadModel(manifest: Manifest): Promise; private _loadWeights(): Promise; private _ensureVisionWeightsLoaded(): Promise; private _ensureAudioWeightsLoaded(): Promise; setPreloadedWeights(weights: WeightLoadResult): void; private _initRoPE(): Promise; private _resolveLayerPipeline(): void; // ========================================================================== // Generation Delegates // ========================================================================== generate(prompt: PromptInput, options?: GenerateOptions): AsyncGenerator; generateTokens(prompt: PromptInput, options?: GenerateOptions): AsyncGenerator; generateTokenIds( prompt: PromptInput, options?: GenerateOptions ): Promise<{ tokenIds: number[]; stats: PipelineStats }>; resetToSeqLen(seqLen: number): void; decodeStepLogits(currentIds: number[], options?: GenerateOptions): Promise; advanceWithToken(tokenId: number, options?: GenerateOptions): Promise; advanceWithTokenAndEmbedding(tokenId: number, options?: GenerateOptions): Promise; prefillKVOnly(prompt: PromptInput, options?: GenerateOptions): Promise; prefillForLoRATraining( inputIds: readonly number[] | Int32Array | Uint32Array, options: NativeLoRAPrefillOptions ): Promise; computeDiffusionGemmaCanvasLogits( args: DiffusionGemmaCanvasLogitsInput, options?: GenerateOptions & { __internalGenerate?: boolean } ): Promise; computeDiffusionGemmaCanvasStep( args: DiffusionGemmaCanvasStepInput, options?: GenerateOptions & { __internalGenerate?: boolean } ): Promise; prefillWithEmbedding(prompt: PromptInput, options?: GenerateOptions): Promise; embed(prompt: string, options?: GenerateOptions): Promise<{ embedding: Float32Array; tokens: number[]; seqLen: number; embeddingMode: string; phase?: WorkloadPhaseTiming | null; }>; embedBatch(prompts: string[], options?: GenerateOptions): Promise>; encodeSequence(sequence: string, options?: SequenceEncodeOptions): Promise; prefillWithLogits(prompt: PromptInput, options?: GenerateOptions): Promise; prefillWithTokenLogits(prompt: PromptInput, tokenIds: readonly number[], options?: GenerateOptions): Promise<{ seqLen: number; tokens: number[]; tokenIds: number[]; logits: Float32Array; logitsByTokenId: Record; phase?: WorkloadPhaseTiming | null; }>; prefillWithTokenLogitsFromKV(prefix: KVCacheSnapshot, prompt: PromptInput, tokenIds: readonly number[], options?: GenerateOptions): Promise<{ seqLen: number; prefixTokens: number[]; tokens: number[]; tokenIds: number[]; logits: Float32Array; logitsByTokenId: Record; phase?: WorkloadPhaseTiming | null; }>; applyKVCacheSnapshot(snapshot: KVCacheSnapshot): void; generateWithPrefixKV( prefix: KVCacheSnapshot, prompt: PromptInput, options?: GenerateOptions ): AsyncGenerator; // ========================================================================== // Utility Methods // ========================================================================== getStats(): PipelineStats; getBatchingStats(): BatchingStats; getMemoryStats(): { used: number; pool?: { currentBytesAllocated?: number; peakBytesAllocated?: number; activeBuffers?: number; pooledBuffers?: number }; kvCache?: { allocated?: number; used?: number; seqLen?: number; maxSeqLen?: number; layout?: string | null; kvDtype?: string | null; counters?: Record | null; }; emulation?: EmulationStats; }; getKVCacheStats(): { seqLen: number; maxSeqLen: number } | null; getBufferPool(): ReturnType | null; unload(): Promise; setLoRAAdapter(adapter: LoRAAdapter | null): void; getActiveLoRA(): LoRAAdapter | null; reset(): void; releaseGPUResources(): void; } // ============================================================================ // Factory Function // ============================================================================ export declare function createPipeline( manifest: Manifest, contexts?: PipelineContexts ): Promise< InferencePipeline | EmbeddingPipeline | DiffusionPipeline | EnergyPipeline | StructuredJsonHeadPipeline | EnergyRowHeadPipeline >; export declare class EmbeddingPipeline extends InferencePipeline { generate(prompt: PromptInput, options?: GenerateOptions): AsyncGenerator; } export { InferencePipeline as Pipeline };