// @ts-nocheck import { SpeechRecognizer } from "microsoft-cognitiveservices-speech-sdk"; import { EventEmitter } from "events"; import { ITranscriptionConfig } from "./ITranscriptionService"; import type { TelemetryIntegrationOptions, TelemetryIntegrationWarning, TelemetryRow, TranscriptionMetricsSnapshot, TelemetrySnapshot, } from "./services/transcription/metrics/types"; export * from "./services/transcription/metrics/types"; export * from "./services/transcription/auth"; import type { DirectModeAuth } from "./services/transcription/auth"; export interface ITranscriptionService extends EventEmitter { initialize(mediaStream: MediaStream): void; startTranscription(): void; stopTranscription(): Promise; reprocessAudio?(): Promise; getBatchReprocessStatus?(): BatchReprocessStatus; getBatchReprocessRemainingTime?(): number | null; pauseTranscription(): void; resumeTranscription(): void; getTelemetrySnapshot?(): TelemetrySnapshot | null; getTelemetryRows?(): TelemetryRow[]; clearTelemetryRows?(): void; resetTelemetry?(): void; getSttSessionId?(): string | null; getSttSessionIds?(): string[]; } export type SupportedLanguage = "pt-BR" | "en-US" | "es-ES"; export type Providers = | "sofya_compliance" | "oracle" | "sofya_as_service" | "sofya_whisper_flow" | "stt_wvad"; export interface ConnectionResilienceConfig { connectTimeoutMs?: number; healthCheckIntervalMs?: number; maxBufferedAmountBytes?: number; drainStallTimeoutMs?: number; preRecoveryReplayMs?: number; reconnectBaseDelayMs?: number; reconnectMaxDelayMs?: number; reconnectDelayGrowFactor?: number; minConnectionUptimeMs?: number; maxReconnectAttempts?: number; } export interface FinalUploadConfig { mode?: "stream" | "blob"; format?: "pcm" | "wav"; endpoint?: string; } /** * Runtime lifecycle for batch reprocessing of retained consultation audio. */ export type BatchReprocessState = | "unsupported" | "waiting_for_stop" | "unconfigured" | "ready" | "running" | "expired" | "error"; export type BatchReprocessErrorCode = | "UNSUPPORTED_PROVIDER" | "REPROCESS_IN_PROGRESS" | "SESSION_NOT_STOPPED" | "NOT_CONFIGURED" | "AUDIO_NOT_AVAILABLE" | "AUDIO_EXPIRED" | "HTTP_ERROR" | "INVALID_RESPONSE" | "CORS_BLOCKED"; /** * Stable SDK error for batch reprocess failures and availability checks. */ export class BatchReprocessError extends Error { readonly code: BatchReprocessErrorCode; readonly cause: unknown; readonly status: number | null; } /** * Snapshot of the batch reprocess capability for UI state and availability checks. */ export interface BatchReprocessStatus { state: BatchReprocessState; supported: boolean; configured: boolean; available: boolean; expiresAt: number | null; remainingMs: number | null; lastStartedAt: number | null; lastCompletedAt: number | null; lastError: BatchReprocessError | null; } /** * Optional overrides for the batch reprocess endpoint. * * The SDK inherits the realtime connection context by default: * - base host * - authorization token * - extra configured headers * - `x-external-id` * - `transcription_language` * - `translation_language` * * When `endpoint` is omitted, the SDK derives the batch URL as * `${realtimeBase}/api/transcriber`. */ export interface BatchReprocessConfig { endpoint?: string; parseResponse?: (response: Response) => Promise; } export interface BatchConfig { parseResponse?: (response: Response) => Promise; } export type SttAuditIngestionWarningCode = | "missing_api_key" | "fetch_unavailable" | "invalid_endpoint" | "http_error" | "network_error" | "dispatch_failed"; export interface SttAuditIngestionWarning { code: SttAuditIngestionWarningCode; message: string; status?: number; attempt?: number; endpoint?: string; detail?: Record | null; } export interface SttAuditIngestionConfig { endpoint?: string; threadId?: string; externalIdentifier?: string; headers?: Record; } export type ResilienceConnectionState = | "connecting" | "connected" | "reconnecting" | "disconnected" | "stopped"; export type ResilienceWebSocketState = | "connecting" | "open" | "closing" | "closed" | "unavailable"; export interface ConnectionResilienceStatus { connectionState: ResilienceConnectionState; websocketState: ResilienceWebSocketState; browserOnline: boolean; hasConnectedOnce: boolean; isRecovering: boolean; isBufferingAudio: boolean; isDrainingAudio: boolean; transportBufferedAmountBytes: number; pendingBufferedAudioBytes: number; pendingBufferedAudioChunks: number; persistedBufferedAudioBytes: number; persistedBufferedAudioSegments: number; totalBufferedAudioBytes: number; bufferingStartedAt: number | null; drainStartedAt: number | null; lastDrainProgressAt: number | null; drainCycles: number; drainExitReason: | "completed" | "socket_unavailable" | "send_failed" | "stalled" | "stopped" | null; reconnectAttempt: number | null; nextReconnectDelayMs: number | null; finishDeliveryState: "sent" | "skipped_socket_closed" | "timed_out" | null; maxReconnectAttempts: number; remainingReconnectAttempts: number | null; lastDisconnect: { code?: number; reason?: string; wasClean?: boolean; errorMessage?: string; } | null; } export interface DebugAuditConfig { enabled?: boolean; advancedMetrics?: boolean; autoDownload?: boolean; fileName?: string; label?: string; metadata?: Record; } export interface DebugAuditEvent { name: string; detail: unknown; timestamp: number; } export interface DebugTranscriptEntry { text: string; timestamp: number; } export type SessionReportSummaryOutcome = | "healthy" | "degraded_recovered" | "degraded_unrecovered" | "failed"; export type SessionReportHealthStatus = | "inactive" | "healthy" | "degraded" | "critical" | "unsupported"; export type SessionReportFindingSeverity = "info" | "warning" | "critical"; export type SessionReportPhaseType = | "created" | "ready" | "started" | "connected" | "reconnecting" | "reconnected" | "disconnected" | "stopped"; export type SessionReportIncidentType = | "offline" | "online" | "buffering_started" | "buffering_cleared" | "drain_started" | "drain_completed" | "drain_stalled" | "error" | "fatal_error" | "terminal_disconnect"; export type SessionReportTranscriptMilestoneType = | "first_partial" | "first_final" | "last_final" | "first_diarization"; export interface SessionReportSummary { outcome: SessionReportSummaryOutcome; sessionDurationMs: number | null; partialTranscriptCount: number; finalTranscriptCount: number; diarizationCount: number; avgFirstPartialLatencyMs: number | null; p95FirstPartialLatencyMs: number | null; avgFinalLatencyMs: number | null; p95FinalLatencyMs: number | null; avgStabilizationLatencyMs: number | null; p95StabilizationLatencyMs: number | null; reconnectCount: number | null; recoverySuccessCount: number | null; terminalDisconnectCount: number | null; avgRecoveryDurationMs: number | null; p95RecoveryDurationMs: number | null; peakBufferedAudioBytes: number | null; bufferingDurationMs: number | null; drainingDurationMs: number | null; bufferStallCount: number | null; drainExitReason: | "completed" | "socket_unavailable" | "send_failed" | "stalled" | "stopped" | null; finishDeliveryState: "sent" | "skipped_socket_closed" | "timed_out" | null; offlineDurationMs: number | null; flapCount: number | null; rttMs: number | null; downlinkMbps: number | null; clippingEventCount: number | null; silenceRatio: number | null; speechActivityRatio: number | null; audioCallbackGapCount: number | null; longestAudioCallbackGapMs: number | null; } export interface SessionReportHealthGroup { supported: boolean; status: SessionReportHealthStatus; reasons: string[]; } export interface SessionReportHealth { transcriptUi: SessionReportHealthGroup; session: SessionReportHealthGroup; connection: SessionReportHealthGroup; recovery: SessionReportHealthGroup; buffering: SessionReportHealthGroup; browserNetwork: SessionReportHealthGroup; audioCapture: SessionReportHealthGroup; } export interface SessionReportPhase { type: SessionReportPhaseType; timestamp: number; } export interface SessionReportIncident { type: SessionReportIncidentType; timestamp: number; endedAt: number | null; detail: unknown; } export interface SessionReportTranscriptMilestone { type: SessionReportTranscriptMilestoneType; timestamp: number; text: string | null; } export interface SessionReportTimeline { phases: SessionReportPhase[]; incidents: SessionReportIncident[]; transcriptMilestones: SessionReportTranscriptMilestone[]; } export interface SessionReportFinding { code: string; severity: SessionReportFindingSeverity; message: string; } export interface SessionReportDashboardCard { status: SessionReportHealthStatus; headline: string; metrics: Record; } export interface SessionReportDashboard { heroStats: { outcome: SessionReportSummaryOutcome; sessionDurationMs: number | null; finalTranscriptCount: number; reconnectCount: number | null; peakBufferedAudioBytes: number | null; offlineDurationMs: number | null; }; cards: { transcript: SessionReportDashboardCard; connection: SessionReportDashboardCard; recovery: SessionReportDashboardCard; buffering: SessionReportDashboardCard; network: SessionReportDashboardCard; audio: SessionReportDashboardCard; }; } export interface SessionReport { schemaVersion: 2; generatedAt: string; session: { sessionId: string; label: string | null; requestedProvider: Providers | "apiKey"; resolvedProvider: Providers | null; createdAt: string; startedAt: string | null; stoppedAt: string | null; durationMs: number | null; hasStarted: boolean; hasStopped: boolean; }; summary: SessionReportSummary; health: SessionReportHealth; timeline: SessionReportTimeline; findings: SessionReportFinding[]; dashboard: SessionReportDashboard; } export interface DebugAuditFile { schemaVersion: 2; kind: "sofya-transcriber-debug-audit"; sessionId: string; label: string | null; generatedAt: string; createdAt: string; startedAt: string | null; stoppedAt: string | null; sessionDurationMs: number | null; requestedProvider: Providers | "apiKey"; resolvedProvider: Providers | null; /** * Server-side session ids (`stt_session` event) received since the last * `startTranscription()`, in connection order. Empty for servers that do * not send `session_id` or for non-Whisper providers. */ sttSessionIds: string[]; metadata: Record | null; environment: { userAgent: string | null; locationHref: string | null; }; report: SessionReport; } export interface BaseConfig { language: SupportedLanguage; /** * Opaque client-side identifier sent as the `x-external-id` query parameter * on every realtime connection (including reconnections) and copied verbatim * into the recording metadata. Never use personal data (CPF, medical record, * name, e-mail). Use `[A-Za-z0-9._-]`, starting with a letter or digit, up * to 64 characters, so it is used as-is in the bucket path; other values * (up to 512 characters) are hashed in the path. */ external_id?: string; /** * Requests (`true`) or refuses (`false`) server-side recording of the * session. Sent as the `record` query parameter on every realtime connection. * When omitted, the parameter is not sent and the server environment default * applies. Recording never changes transcription behavior and is best-effort. */ record?: boolean; protocols?: string | string[]; resilience?: ConnectionResilienceConfig; debug?: boolean | DebugAuditConfig; telemetry?: TelemetryIntegrationOptions; } export interface SofyaComplianceConfig extends BaseConfig { token: string; compartmentId: string; region: string; } /** * Optional authentication for direct-endpoint mode * (`{ provider, endpoint, config }`). * * - `{ type: "none" }` (default when omitted) keeps the previous behavior. * - `{ type: "api_key", key, transport }` sends the STT API key. Browsers * cannot set headers on a WebSocket handshake, so the key travels either as * the extra subprotocol token `x-api-key.` (`transport: "subprotocol"`, * the default) or as the `x-api-key` query parameter (`transport: "query"`, * which leaks the key into proxy and browser logs). * * With `"subprotocol"` the key must be an RFC 6455 token (letters, digits and * `!#$%&'*+-.^_\`|~`); otherwise the SDK throws `SofyaAuthError` before * opening the connection. The key is also sent as the `x-api-key` header on * the HTTP calls the SDK makes (batch reprocess, audit ingestion) when that * header is not already configured. * * - `{ type: "jwt", token }` sends a JWT instead of the API key (the two are * exclusive). The JWT goes first in the WebSocket subprotocol list, so the * server echoes it, and as `Authorization: Bearer ` on the batch, * batch reprocess and audit ingestion calls. An `Authorization` header set * in `headers` still wins. Final upload never carries credentials. * * **Pass the raw JWT, without `Bearer `: the SDK adds the prefix.** A token * starting with `Bearer ` throws `SofyaAuthError("INVALID_JWT")`. In * realtime mode the JWT must be an RFC 6455 token (a standard base64url JWT * always is), otherwise `SofyaAuthError("INVALID_JWT_CHARACTERS")`. Do not * repeat the JWT in `protocols`, or it is sent twice. */ export interface DirectModeAuthConfig { auth?: DirectModeAuth; } export interface SofyaSpeechConfig extends BaseConfig, DirectModeAuthConfig { translation_lang?: SupportedLanguage; /** * @deprecated Use `auth: { type: "jwt", token }` (the raw JWT, without * `Bearer `). Still sent as `Authorization: Bearer `; removed in 1.0.0. */ token?: string; headers?: Record; finalUpload?: FinalUploadConfig; /** * Whisper-only batch reprocess configuration. * * The retained consultation audio is only available after `stopTranscription()` * and while the durable IndexedDB session remains within the current 30 minute TTL. */ batchReprocess?: BatchReprocessConfig; /** * Optional STT audit ingestion endpoint configuration. * * When configured, the SDK posts the final session audit automatically after * `stopTranscription()` resolves internal stop flow. */ auditIngestion?: SttAuditIngestionConfig; } export interface SofyaBatchConfig extends BaseConfig, DirectModeAuthConfig { translation_lang?: SupportedLanguage; /** * @deprecated Use `auth: { type: "jwt", token }` (the raw JWT, without * `Bearer `). Still sent as `Authorization: Bearer `; removed in 1.0.0. */ token?: string; headers?: Record; batch?: BatchConfig; auditIngestion?: SttAuditIngestionConfig; } /** * `apiKey` mode resolves its providers through the reasoner `/providers` * endpoint and authenticates with that key, so it does not take `auth`. */ export interface ApiKeyConfig extends Omit, "auth" | "token"> { token?: string; } export type WhisperProvider = | "sofya_as_service" | "sofya_whisper_flow" | "stt_wvad"; export type NonWhisperProvider = "sofya_compliance" | "oracle"; export type ApiKeyConnection = { apiKey: string; mode?: "realtime"; config?: ApiKeyConfig; }; export type RealtimeWhisperConnection = { provider: WhisperProvider; mode?: "realtime"; endpoint: string; config: SofyaSpeechConfig; }; export type BatchWhisperConnection = { provider: WhisperProvider; mode: "batch"; endpoint: string; config: SofyaBatchConfig; }; export type WhisperConnection = | RealtimeWhisperConnection | BatchWhisperConnection; export type NonWhisperConnection = { provider: NonWhisperProvider; mode?: "realtime"; endpoint: string; config: SofyaComplianceConfig; }; export type Connection = | ApiKeyConnection | WhisperConnection | NonWhisperConnection; /** * Payload of the `stt_session` event, emitted when the server session id first * appears in a transcript message or changes (reconnection). The id identifies * the STT connection; it does not state whether the session is being recorded. */ export interface SttSessionInfo { /** Server session id (UUID v4). One websocket connection = one session id. */ sttSessionId: string; /** The configured `external_id` sent as `x-external-id` on this connection, or `null`. */ externalId: string | null; /** 1-based connection counter since `startTranscription()`; reconnections increment it. */ connectionAttempt: number; } export interface CommonTranscriber extends EventEmitter { startTranscription(mediaStream: MediaStream): void; stopTranscription(): Promise; pauseTranscription(): void; resumeTranscription(): void; getResilienceStatus(): ConnectionResilienceStatus | null; getTelemetrySnapshot(): TelemetrySnapshot | null; getTelemetryRows(): TelemetryRow[]; clearTelemetryRows(): void; resetTelemetry(): void; getSttSessionId(): string | null; getSttSessionIds(): string[]; getDebugAudit(): Promise; downloadDebugAudit(fileName?: string): Promise; clearDebugAudit(): Promise; on(event: "recognizing", listener: (text: string) => void): this; on(event: "recognized", listener: (text: string) => void): this; on( event: "recognized_diarization", listener: ( diarization: Array<{ end: number; sentence: string; speaker: string; start: number; }> ) => void ): this; on(event: "nomatch", listener: () => void): this; on(event: "error", listener: (error: any) => void): this; on(event: "ready", listener: () => void): this; on(event: "stopped", listener: () => void): this; on(event: "connected", listener: () => void): this; on( event: "disconnected", listener: ( details: { code?: number; reason?: string; wasClean?: boolean; error?: Error; } ) => void ): this; on( event: "reconnecting", listener: (details: { attempt: number; delay: number }) => void ): this; on(event: "reconnected", listener: () => void): this; on( event: "resilience_status", listener: (status: ConnectionResilienceStatus) => void ): this; on( event: "telemetry", listener: (telemetry: TelemetrySnapshot) => void ): this; on( event: "telemetry_row", listener: (row: TelemetryRow) => void ): this; on( event: "telemetry_integration_warning", listener: (warning: TelemetryIntegrationWarning) => void ): this; on( event: "stt_audit_ingestion_warning", listener: (warning: SttAuditIngestionWarning) => void ): this; on(event: "stt_session", listener: (info: SttSessionInfo) => void): this; on(event: string, listener: (...args: any[]) => void): this; } export interface WhisperCapableTranscriber extends CommonTranscriber { /** * Reprocesses the retained consultation audio through the batch endpoint. * * This capability is available only after `stopTranscription()` and while the * retained durable audio is still available inside the post-stop TTL window. */ reprocessAudio(): Promise; /** * Returns the latest batch reprocess capability snapshot, including TTL metadata. */ getBatchReprocessStatus(): BatchReprocessStatus; /** * Returns the remaining retained-audio TTL in milliseconds, or `null` when the * batch capability is unavailable. */ getBatchReprocessRemainingTime(): number | null; on( event: "batch_reprocess_status", listener: (status: BatchReprocessStatus) => void ): this; } export interface BatchTranscriber extends Omit { stopTranscription(): Promise; getResilienceStatus(): null; } export class SofyaTranscriber extends EventEmitter implements ITranscriptionService { /** * Initializes a new instance of the SofyaTranscriber class. * @param connection The connection object required for authentication with the transcription service. */ constructor(connection: Connection); /** * Starts the transcription process. This method listens for recognizing, recognized, and other relevant events. * @param mediaStream The MediaStream to be used for transcription. * Throws an error if the recognizer has not been initialized. */ startTranscription(mediaStream: MediaStream): void; /** * Stops the ongoing transcription process. */ stopTranscription(): Promise; /** * Reprocesses the retained consultation audio through the configured batch endpoint. */ reprocessAudio(): Promise; /** * Returns the current batch reprocess capability snapshot, including TTL metadata. */ getBatchReprocessStatus(): BatchReprocessStatus; /** * Returns the remaining retained-audio TTL in milliseconds, or `null` when * batch reprocessing is unavailable. */ getBatchReprocessRemainingTime(): number | null; /** * Pauses the ongoing transcription process. */ pauseTranscription(): void; /** * Resumes the ongoing transcription process. */ resumeTranscription(): void; /** * Returns the latest resilience status snapshot for the active realtime transport. */ getResilienceStatus(): ConnectionResilienceStatus | null; /** * Returns the latest telemetry snapshot for the active realtime session. */ getTelemetrySnapshot(): TelemetrySnapshot | null; /** * Returns the server session id of the current realtime connection, or `null` * when the server has not sent one yet (older servers, non-Whisper providers, * or before the first transcript message of the run). */ getSttSessionId(): string | null; /** * Returns every server session id received since `startTranscription()`, in * connection order (one per connection, reconnections included). Cleared on * the next `startTranscription()`. */ getSttSessionIds(): string[]; /** * Returns telemetry rows retained in the in-memory ring buffer. */ getTelemetryRows(): TelemetryRow[]; /** * Clears retained telemetry rows. */ clearTelemetryRows(): void; /** * Resets telemetry for a fresh session. */ resetTelemetry(): void; /** * Builds the current exported audit snapshot asynchronously. * Essential audit data is always available; advanced metrics are controlled by debug config. */ getDebugAudit(): Promise; /** * Downloads the current audit snapshot as a JSON file. * Returns the file name used for the download, or null when no file was produced. * Successful SDK-managed downloads clear the persisted audit session. */ downloadDebugAudit(fileName?: string): Promise; /** * Clears the current debug audit session from memory and persistence. */ clearDebugAudit(): Promise; /** * Event emitted when speech is being recognized. * @event recognizing * @param text The text that is currently being recognized. */ on(event: "recognizing", listener: (text: string) => void): this; /** * Event emitted when speech is successfully recognized. * @event recognized * @param text The recognized text. */ on(event: "recognized", listener: (text: string) => void): this; /** * Event emitted when diarization is successfully recognized. * @event recognized_diarization * @param text The recognized diarization. */ on( event: "recognized_diarization", listener: ( diarization: Array<{ end: number; sentence: string; speaker: string; start: number; }> ) => void ): this; /** * Event emitted when no match is found for the recognized speech. * @event nomatch */ on(event: "nomatch", listener: () => void): this; /** * Event emitted when an error occurs during the transcription process. * @event error * @param error The error details. */ on(event: "error", listener: (error: any) => void): this; /** * Event emitted when an the transcription service is ready to start. * @event ready */ on(event: "ready", listener: () => void): this; /** * Event emitted when the transcription session stops. * @event stopped */ on(event: "stopped", listener: () => void): this; /** * Event emitted when the underlying realtime transport connects. * @event connected */ on(event: "connected", listener: () => void): this; /** * Event emitted when the underlying realtime transport is terminally disconnected after retries are exhausted. * @event disconnected */ on( event: "disconnected", listener: ( details: { code?: number; reason?: string; wasClean?: boolean; error?: Error; } ) => void ): this; /** * Event emitted when the underlying realtime transport is reconnecting and has not terminally closed. * @event reconnecting */ on( event: "reconnecting", listener: (details: { attempt: number; delay: number }) => void ): this; /** * Event emitted when the SDK recovers a realtime connection after a disconnect. * @event reconnected */ on(event: "reconnected", listener: () => void): this; /** * Event emitted whenever the resilience status snapshot changes. * @event resilience_status */ on( event: "resilience_status", listener: (status: ConnectionResilienceStatus) => void ): this; /** * Event emitted whenever the telemetry snapshot changes. * @event telemetry */ on( event: "telemetry", listener: (telemetry: TelemetrySnapshot) => void ): this; /** * Event emitted for each telemetry row written to the in-memory ring buffer. * @event telemetry_row */ on( event: "telemetry_row", listener: (row: TelemetryRow) => void ): this; /** * Event emitted whenever the batch reprocess capability snapshot changes. * @event batch_reprocess_status */ on( event: "batch_reprocess_status", listener: (status: BatchReprocessStatus) => void ): this; /** * Event emitted when a telemetry integration provider reports a non-fatal warning. * @event telemetry_integration_warning */ on( event: "telemetry_integration_warning", listener: (warning: TelemetryIntegrationWarning) => void ): this; /** * Event emitted when SDK-managed STT audit ingestion fails or is skipped. * @event stt_audit_ingestion_warning */ on( event: "stt_audit_ingestion_warning", listener: (warning: SttAuditIngestionWarning) => void ): this; /** * Event emitted when the server session id first appears in a transcript * message or changes (reconnection). Never emitted by * servers that do not send `session_id`. * @event stt_session */ on(event: "stt_session", listener: (info: SttSessionInfo) => void): this; on(event: string, listener: (...args: any[]) => void): this; } /** * Preferred transcriber factory with provider-aware return typing. */ export function createTranscriber( connection: RealtimeWhisperConnection ): WhisperCapableTranscriber; export function createTranscriber( connection: BatchWhisperConnection ): BatchTranscriber; export function createTranscriber( connection: ApiKeyConnection ): CommonTranscriber; export function createTranscriber( connection: NonWhisperConnection ): CommonTranscriber;