/** * @public * Schema type for event fields and response definitions. * Maps to LLM tool parameter types (JSON Schema subset). */ export type SchemaType = { type: "string"; description?: string; } | { type: "number"; description?: string; } | { type: "boolean"; description?: string; } | { type: "enum"; values: string[]; description?: string; } | { type: "array"; items: SchemaType; description?: string; } | { type: "object"; properties: Record; description?: string; }; /** * @public * A single field definition for a event declaration. */ export interface EventField { name: string; schema: SchemaType; } /** * @public * An event that the evaluation LLM can raise. * The declaration (name, description, fields) is sent to the planning * LLM so it can design the spec around these events. At evaluation time, * events are emitted via onLLMResponse. */ export interface EventDeclaration { name: string; description?: string; fields: EventField[]; } /** * @public * Aggregation strategy for local analysis tools. */ export type AggregationStrategy = "raw" | "sampled" | "summarised" | "bucketed"; /** * @public * Audio energy tool configuration. */ export interface AudioEnergyTool { tool: "audio_energy"; windowSecs: number; aggregation?: AggregationStrategy; sampleN?: number; bucketSecs?: number; } /** * @public * Voice activity detection tool configuration. */ export interface VADTool { tool: "vad"; windowSecs: number; confidenceThreshold?: number; aggregation?: AggregationStrategy; sampleN?: number; bucketSecs?: number; } /** * @public * Object detection tool configuration. */ export interface ObjectDetectionTool { tool: "object_detection"; windowSecs: number; confidenceThreshold?: number; aggregation?: AggregationStrategy; sampleN?: number; bucketSecs?: number; } /** * @public * Scene change detection tool configuration. */ export interface SceneChangeTool { tool: "scene_change"; windowSecs: number; threshold?: number; downscaleWidth?: number; } /** @public */ export interface SceneSemanticsTool { tool: "scene_semantics"; windowSecs: number; } /** @public */ export interface OCRTool { tool: "ocr"; windowSecs: number; } /** @public */ export interface FacesTool { tool: "faces"; windowSecs: number; } /** * @public * Configuration for a single local analysis tool (discriminated union on `tool`). */ export type LocalAnalysisConfig = AudioEnergyTool | VADTool | ObjectDetectionTool | SceneChangeTool | SceneSemanticsTool | OCRTool | FacesTool; /** * @public * The specification produced by the planning phase. * - state: opaque JSON designed by the planning LLM for the evaluation LLM * to track temporal context. Norsk does not interpret this. * - prompt: the instruction sent to the evaluation LLM alongside each frame * and the current state. * - explanation: optional reasoning from the planning LLM about its design * choices (why it chose certain state fields, etc.) * - localAnalysis: optional list of local analysis tools to enable * - includeFrame: whether to include JPEG frames in LLM calls (default true) * - events: function declarations the evaluation LLM can call. Carried * through from the plan phase so the same events are registered at * evaluation time. * * Internally, the SDK implementation may carry additional handler state * but this is not part of the public interface. */ export interface ReasoningSpec { readonly state: object; readonly prompt: string; readonly explanation?: string; readonly localAnalysis?: LocalAnalysisConfig[]; readonly includeFrame?: boolean; readonly events?: EventDeclaration[]; } /** * @public * Gemini provider for reasoning */ export type ReasoningGeminiApiAuth = { authType: "geminiApi"; googleApiKey: string; }; /** * @public * Vertex AI provider for reasoning */ export type ReasoningVertexAuth = { authType: "vertex"; project: string; location: string; serviceAccountFile?: string; }; /** * @public * Gemini provider configuration for reasoning */ export type ReasoningGeminiProvider = { providerType: "gemini"; auth: ReasoningGeminiApiAuth | ReasoningVertexAuth; model: string; }; /** * @public * OpenAI provider configuration for reasoning */ export type ReasoningOpenAIProvider = { providerType: "openai"; apiKey: string; model: string; }; /** * @public * Claude provider configuration for reasoning */ export type ReasoningClaudeProvider = { providerType: "claude"; apiKey: string; model: string; maxTokens?: number; }; /** * @public * Local reasoning provider configuration */ export type ReasoningLocalProvider = { providerType: "local"; modelName: string; }; /** * @public * A scripted response the mock provider should emit on a given request. * * Test-only — used with {@link ReasoningMockProvider}. Should not be used * in production workflows. */ export type ReasoningMockResponse = { kind: "text"; text: string; } | { kind: "function_call"; name: string; argumentsJson: string; } | { kind: "error"; errorKind: string; message?: string; }; /** * @public * Test-only provider that returns deterministic, scripted responses * instead of calling a real LLM. Used by the ReasoningMock-tagged tests * in client/workspaces/media-examples/tests/. Should not be used in * production workflows. * * Modes: * mirror — emit a function_call for the first non-reserved tool * declared in the spec (i.e. not `_update_state`) * scripted — emit responses[i] for the i-th request, clamping at end * error_injection — fail the first failFirstN requests, then succeed * echo — emit a text response containing the prompt verbatim */ export type ReasoningMockProvider = { providerType: "mock"; mode: { kind: "mirror"; includeStateArgs?: boolean; } | { kind: "scripted"; responses: ReasoningMockResponse[]; echoState?: boolean; } | { kind: "error_injection"; failFirstN: number; errorKind: "MALFORMED_FUNCTION_CALL" | "RATE_LIMIT" | "TIMEOUT"; successResponse: ReasoningMockResponse; } | { kind: "echo"; }; }; /** * @public * Union of all reasoning provider configurations */ export type ReasoningProvider = ReasoningGeminiProvider | ReasoningOpenAIProvider | ReasoningClaudeProvider | ReasoningLocalProvider | ReasoningMockProvider; /** * @public * Settings for creating a reasoning plan session. * * Two modes of operation: * - Event mode: provide `events`. The planning LLM is told about * the available events and designs the spec around them. At evaluation time, * events are emitted via onLLMResponse. * - Text mode: omit `events`, provide `onResponse`. The evaluation LLM * returns free-form text/JSON, passed through to onResponse. * - Both can be provided: tool-calling mode with onResponse capturing any * text the LLM generates alongside tool calls. */ export interface ReasoningPlanSettings { query: string; provider: ReasoningProvider; events?: EventDeclaration[]; onResponse?: (response: unknown) => void; onProposedSpec?: (spec: ReasoningSpec) => void; onError?: (error: string) => void; averageResponseTimeMs?: number; /** Local analysis tools available for the planner to consider. * Each tool config describes the tool, its data window, and aggregation strategy. */ localAnalysisTools?: LocalAnalysisConfig[]; /** Reference documents (markdown) providing domain context for the planning LLM. * These are included in the system instruction and cached across frames. */ contextDocuments?: string[]; /** How often (in seconds) the evaluation runs. Informs the planner about the analysis cadence. */ analysisIntervalSecs?: number; } /** * @public * A session for interactively planning a ReasoningSpec via AI. * * This is NOT a media node. It wraps a bidirectional gRPC stream for the * Plan API, allowing you to submit a query, refine the proposed spec, and * accept the final result. */ export declare class ReasoningPlanSession { /** * Send feedback to refine the proposed spec */ refine(feedback: string): void; /** * Accept the current proposed spec, returning the final ReasoningSpec */ accept(): Promise; /** * Close the plan session */ close(): void; } //# sourceMappingURL=reasoningPlan.d.ts.map