import { ExtractRequestBody, ExtractResponseBody, } from "@noya-app/noya-multiplayer-react"; import { Base64 } from "@noya-app/noya-utils"; import { Observable } from "@noya-app/observable"; import { AssetManager } from "./AssetManager"; import { TranscriptionRequest } from "./rpc/routes"; import { RPCManager } from "./rpcManager"; type BaseTranscribeOptions = { fileName?: string; contentType?: string; endpoint?: string; apiKey?: string; }; export type TranscribeOptions = BaseTranscribeOptions & ({ data: Uint8Array } | { assetId: string }); export type TranscriptionResult = ExtractResponseBody<"POST /api/transcriptions">; export type AssetTranscriptionResult = | ({ type: "success" } & Omit) | { type: "error"; error: Error } | { type: "processing" }; const emptyAssetTranscriptionResult: AssetTranscriptionResult = { type: "processing", }; export class TranscriptionManager { constructor( private rpcManager: RPCManager, private assetManager: AssetManager ) {} // Keyed by whatever the user provided as the assetId (which may be a stableId) assetTranscriptions$ = new Observable< Record >({}); getAssetTranscription$( assetId: string, options: BaseTranscribeOptions ): Observable { const value = this.assetTranscriptions$.get(assetId); if (!value) { this.transcribe({ assetId, ...options }); } return this.assetTranscriptions$.map( (assetTranscriptions) => assetTranscriptions[assetId] ?? emptyAssetTranscriptionResult ); } async transcribe(options: TranscribeOptions): Promise { let payload: ExtractRequestBody<"POST /api/transcriptions"> | undefined; if ("data" in options) { payload = { ...options, type: "data", data: Base64.encode(options.data) }; } else if ("assetId" in options) { const realAssetId = this.assetManager.getRealAssetId(options.assetId); if (!realAssetId) { throw new Error(`Asset ${options.assetId} not found`); } payload = { ...options, type: "assetId", assetId: realAssetId }; } const updateAssetTranscription = (result: AssetTranscriptionResult) => { if ("assetId" in options) { this.assetTranscriptions$.set(options.assetId, result); } }; try { if (!payload) { throw new Error("Missing transcription data or assetId"); } updateAssetTranscription({ type: "processing" }); const response = await this.rpcManager.requestRoute( "POST /api/transcriptions", { body: JSON.stringify(payload), headers: { "Content-Type": "application/json" }, } ); const result = this.rpcManager.getResponseBody(response) as | TranscriptionResult | undefined; if (!result) { const error = new Error("Transcription failed"); updateAssetTranscription({ type: "error", error }); throw error; } updateAssetTranscription({ type: "success", ...result }); return result; } catch (error) { updateAssetTranscription({ type: "error", error: error as Error }); throw error; } } } type AssetResolverResult = { data: ArrayBufferLike; contentType?: string; }; export async function requestTranscription( options: TranscriptionRequest & { endpoint: string }, { resolveAsset, }: { resolveAsset: (assetId: string) => Promise; } ): Promise { let resolvedFileName = options.fileName; let resolvedContentType = options.contentType; let arrayBuffer: ArrayBufferLike; if ("assetId" in options) { const asset = await resolveAsset(options.assetId); if (!asset) { throw new Error(`Asset ${options.assetId} not found`); } arrayBuffer = asset.data; resolvedContentType = resolvedContentType ?? asset.contentType; resolvedFileName = resolvedFileName ?? options.assetId; } else if ("data" in options) { arrayBuffer = Base64.decode(options.data).buffer; } else { throw new Error("Missing transcription payload"); } const blob = new Blob([arrayBuffer as ArrayBuffer], { type: resolvedContentType ?? "application/octet-stream", }); const formData = new FormData(); formData.append("file", blob, resolvedFileName ?? "upload"); const transcriptionResponse = await fetch(options.endpoint, { method: "POST", body: formData, headers: { ...(options.apiKey ? { Authorization: `Bearer ${options.apiKey}` } : {}), }, }); const payload = await validateTranscriptionResponse(transcriptionResponse); return payload; } async function validateTranscriptionResponse( response: Response ): Promise { const responseText = await response.text(); let parsed: any; try { parsed = JSON.parse(responseText); } catch { parsed = undefined; } if (!response.ok) { const errorMessage = (typeof parsed === "object" && parsed !== null && "error" in parsed ? String(parsed.error) : undefined) ?? responseText ?? "Transcription failed"; throw new Error(errorMessage); } if (!parsed || typeof parsed !== "object" || !("success" in parsed)) { throw new Error("Unexpected transcription response format"); } return normalizeTranscriptionResponse(parsed, responseText); } function normalizeTranscriptionResponse( parsed: unknown, fallbackText: string ): TranscriptionResult { const isObject = (value: unknown): value is Record => typeof value === "object" && value !== null; const transcriptionText = isObject(parsed) ? typeof parsed.transcription === "string" ? parsed.transcription : typeof parsed.text === "string" ? parsed.text : fallbackText : fallbackText; const metadata = isObject(parsed) && isObject(parsed.metadata) ? parsed.metadata : {}; const segments = isObject(parsed) && Array.isArray(parsed.segments) ? parsed.segments : []; const success = isObject(parsed) && "success" in parsed ? Boolean(parsed.success) : true; return { success, transcription: transcriptionText, segments, metadata, }; }