/** * pi-video-transcribe -- Video transcription with speaker diarization. * * Provides the `transcribe_video` tool for the LLM to call. * Uses AssemblyAI for transcription + diarization. * Chapters and summaries via LLM Gateway (auto_chapters/summarization are deprecated). * Requires ffmpeg for local video files (auto-detected with install instructions). */ import type { ExtensionAPI } from "@earendil-works/pi-coding-agent"; import { Type } from "typebox"; import { AssemblyAI } from "assemblyai"; import { execFile } from "node:child_process"; import { existsSync, mkdtempSync, rmSync } from "node:fs"; import { join, extname } from "node:path"; import { tmpdir } from "node:os"; import { promisify } from "node:util"; const execFileAsync = promisify(execFile); // --------------------------------------------------------------------------- // Constants // --------------------------------------------------------------------------- const LLM_GATEWAY_URL = process.env.ASSEMBLYAI_EU === "true" ? "https://llm-gateway.eu.assemblyai.com/v1/chat/completions" : "https://llm-gateway.assemblyai.com/v1/chat/completions"; const SPEECH_MODELS = ["universal-3-pro", "universal-2"] as const; // Video extensions ffmpeg can handle const VIDEO_EXTENSIONS = new Set([ ".mp4", ".mkv", ".avi", ".mov", ".webm", ".flv", ".wmv", ".m4v", ".mpg", ".mpeg", ".3gp", ".ogv", ".ts", ".mts", ]); // --------------------------------------------------------------------------- // Helpers // --------------------------------------------------------------------------- function formatTimestamp(ms: number): string { const totalSeconds = Math.floor(ms / 1000); const h = Math.floor(totalSeconds / 3600); const m = Math.floor((totalSeconds % 3600) / 60); const s = totalSeconds % 60; if (h > 0) { return `${h}:${m.toString().padStart(2, "0")}:${s.toString().padStart(2, "0")}`; } return `${m}:${s.toString().padStart(2, "0")}`; } function isUrl(source: string): boolean { return source.startsWith("http://") || source.startsWith("https://"); } function isVideoFile(source: string): boolean { if (isUrl(source)) return false; const ext = extname(source).toLowerCase(); return VIDEO_EXTENSIONS.has(ext); } async function checkFfmpeg(): Promise { try { await execFileAsync("ffmpeg", ["-version"], { timeout: 5000 }); return null; } catch { return getFfmpegInstallInstructions(); } } function getFfmpegInstallInstructions(): string { const platform = process.platform; let instructions: string; if (platform === "win32") { instructions = [ "## ffmpeg Installation (Windows)", "", "### Option A -- winget (recommended)", "```", "winget install Gyan.FFmpeg", "```", "### Option B -- Chocolatey", "```", "choco install ffmpeg", "```", "### Option C -- Scoop", "```", "scoop install ffmpeg", "```", "### Option D -- Manual", "1. Download from https://www.gyan.dev/ffmpeg/builds/ (release full)", "2. Extract to e.g. `C:\\ffmpeg`", "3. Add `C:\\ffmpeg\\bin` to your system PATH", "", "After installation, restart your terminal and run `ffmpeg -version` to verify.", ].join("\n"); } else if (platform === "darwin") { instructions = [ "## ffmpeg Installation (macOS)", "", "### Option A -- Homebrew (recommended)", "```", "brew install ffmpeg", "```", "### Option B -- MacPorts", "```", "sudo port install ffmpeg", "```", "After installation, run `ffmpeg -version` to verify.", ].join("\n"); } else { instructions = [ "## ffmpeg Installation (Linux)", "", "### Debian / Ubuntu", "```", "sudo apt update && sudo apt install ffmpeg", "```", "### Fedora", "```", "sudo dnf install ffmpeg", "```", "### Arch", "```", "sudo pacman -S ffmpeg", "```", "### Alpine", "```", "apk add ffmpeg", "```", "After installation, run `ffmpeg -version` to verify.", ].join("\n"); } return [ "WARNING: **ffmpeg is required** for local video files but was not found on your system.", "", instructions, ].join("\n"); } // --------------------------------------------------------------------------- // Audio extraction // --------------------------------------------------------------------------- async function extractAudio( videoPath: string, signal?: AbortSignal, ): Promise<{ audioPath: string; cleanup: () => void }> { const tempDir = mkdtempSync(join(tmpdir(), "pi-video-")); const audioPath = join(tempDir, "audio.wav"); await execFileAsync("ffmpeg", [ "-i", videoPath, "-vn", "-acodec", "pcm_s16le", "-ar", "16000", "-ac", "1", "-y", audioPath, ], { signal, timeout: 300_000 }); return { audioPath, cleanup: () => { try { rmSync(tempDir, { recursive: true, force: true }); } catch {} }, }; } // --------------------------------------------------------------------------- // LLM Gateway (replaces deprecated auto_chapters + summarization) // --------------------------------------------------------------------------- async function llmGatewayChat( apiKey: string, systemPrompt: string, userContent: string, signal?: AbortSignal, ): Promise { const res = await fetch(LLM_GATEWAY_URL, { method: "POST", headers: { "Authorization": apiKey, "Content-Type": "application/json", }, body: JSON.stringify({ model: "claude-sonnet-4-6", max_tokens: 2000, messages: [ { role: "system", content: systemPrompt }, { role: "user", content: userContent }, ], }), signal: signal ?? null, }); if (!res.ok) { const text = await res.text(); throw new Error(`LLM Gateway error ${res.status}: ${text}`); } const data = await res.json() as any; return data.choices?.[0]?.message?.content ?? ""; } // --------------------------------------------------------------------------- // Build Markdown result // --------------------------------------------------------------------------- interface Utterance { speaker: string; text: string; start: number; end: number; } function buildTranscriptMarkdown(utterances: Utterance[], durationSec: number, languageCode?: string, confidence?: number): string { const lines: string[] = []; lines.push("# Video Transcription"); lines.push(""); lines.push(`- **Duration:** ${formatTimestamp(durationSec * 1000)}`); lines.push(`- **Language:** ${languageCode || "auto-detected"}`); if (confidence != null) { lines.push(`- **Confidence:** ${(confidence * 100).toFixed(1)}%`); } lines.push(""); if (utterances.length > 0) { const speakerOrder: string[] = []; for (const u of utterances) { if (!speakerOrder.includes(u.speaker)) speakerOrder.push(u.speaker); } lines.push("## Speakers"); lines.push(""); speakerOrder.forEach((s, i) => lines.push(`${i + 1}. **${s}**`)); lines.push(""); lines.push("## Transcript"); lines.push(""); for (const u of utterances) { const start = formatTimestamp(u.start); const end = formatTimestamp(u.end); lines.push(`**${u.speaker}** [${start} -- ${end}]:`); lines.push(`> ${u.text}`); lines.push(""); } } else { lines.push("## Transcript"); lines.push(""); lines.push("(No utterance data available)"); lines.push(""); } return lines.join("\n"); } function buildStructured( utterances: Utterance[], fullText: string | null, summary: string | null, chapters: string | null, durationSec: number, languageCode?: string | null, confidence?: number | null, ) { return { utterances: utterances.map((u) => ({ speaker: u.speaker, text: u.text, start: u.start, end: u.end, })), fullText, summary, chapters, audioDuration: durationSec, languageCode: languageCode ?? null, confidence: confidence ?? null, }; } // --------------------------------------------------------------------------- // Extension entry point // --------------------------------------------------------------------------- export default function (pi: ExtensionAPI) { const apiKey = process.env.ASSEMBLYAI_API_KEY; if (!apiKey) { pi.on("session_start", async (_event, ctx) => { ctx.ui.notify( "pi-video-transcribe: Set ASSEMBLYAI_API_KEY to enable video transcription.", "warning", ); }); return; } const client = new AssemblyAI({ apiKey }); // ── Tool: transcribe_video ────────────────────────────────────────── pi.registerTool({ name: "transcribe_video", label: "Transcribe Video", description: "Transcribe a video or audio file with speaker diarization (who said what). " + "Returns per-speaker utterances with timestamps. " + "Optionally generates a summary and chapter breakdown via LLM Gateway. " + "Accepts local file paths (video or audio) and URLs (http/https). " + "For local video files, ffmpeg must be installed. " + "If ffmpeg is missing, the tool returns installation instructions.", promptSnippet: "Transcribe video/audio and identify speakers", promptGuidelines: [ "Use transcribe_video when the user wants to understand a video or audio recording with per-speaker transcription.", "transcribe_video handles mp4, mkv, avi, mov, webm, mp3, wav, m4a and many more formats.", "transcribe_video returns speaker-labeled utterances, optional summary and chapters via LLM Gateway.", "Present the transcription result to the user in a clear, readable format.", ], parameters: Type.Object({ source: Type.String({ description: "Path to local video/audio file or URL (http/https). Supports mp4, mkv, avi, mov, webm, mp3, wav, m4a, flac, ogg, and more.", }), speakers_expected: Type.Optional( Type.Number({ description: "Number of speakers expected. Improves diarization accuracy if known. Only set if you are certain.", }), ), speaker_min: Type.Optional( Type.Number({ description: "Minimum number of speakers expected. Use with speaker_max when you know a range.", }), ), speaker_max: Type.Optional( Type.Number({ description: "Maximum number of speakers expected. Set slightly higher than actual count for best results.", }), ), include_summary: Type.Optional( Type.Boolean({ description: "Generate a bullet-point summary via LLM Gateway (default: true).", }), ), include_chapters: Type.Optional( Type.Boolean({ description: "Generate chapter breakdown with headlines via LLM Gateway (default: true).", }), ), language_code: Type.Optional( Type.String({ description: "BCP-47 language code, e.g. 'de', 'en', 'fr', 'es'. Auto-detected if omitted.", }), ), }), async execute(toolCallId, params, signal, onUpdate, ctx) { const { source } = params; // ── Validate source ────────────────────────────────────── if (isUrl(source)) { onUpdate?.({ content: [{ type: "text", text: `Transcribing from URL: ${source}...` }], }); } else if (!existsSync(source)) { return { content: [{ type: "text", text: `File not found: ${source}` }], isError: true, }; } else if (isVideoFile(source)) { onUpdate?.({ content: [{ type: "text", text: `Checking ffmpeg...` }], }); const ffmpegError = await checkFfmpeg(); if (ffmpegError) { return { content: [{ type: "text", text: ffmpegError }], isError: true, }; } } // ── Extract audio if needed ────────────────────────────── let audioSource = source; let cleanup: (() => void) | null = null; if (!isUrl(source) && isVideoFile(source)) { try { onUpdate?.({ content: [{ type: "text", text: `Extracting audio from ${source}...` }], }); const result = await extractAudio(source, signal ?? undefined); audioSource = result.audioPath; cleanup = result.cleanup; onUpdate?.({ content: [{ type: "text", text: `Audio extracted. Transcribing via AssemblyAI (universal-3-pro)...` }], }); } catch (err: any) { if (err.name === "AbortError" || signal?.aborted) { return { content: [{ type: "text", text: "Transcription cancelled." }], isError: true, }; } return { content: [{ type: "text", text: `Failed to extract audio: ${err.message}\n\nMake sure ffmpeg is installed and the video file is valid.`, }], isError: true, }; } } else if (!isUrl(source)) { onUpdate?.({ content: [{ type: "text", text: `Transcribing audio file: ${source}...` }], }); } // ── Build AssemblyAI config ────────────────────────────── // NOTE: speech_models is REQUIRED. Do NOT use deprecated // auto_chapters / summarization / summary_type / summary_model. const config: Record = { audio: audioSource, speech_models: [...SPEECH_MODELS], speaker_labels: true, }; if (params.speakers_expected) { config.speakers_expected = params.speakers_expected; } if (params.speaker_min || params.speaker_max) { config.speaker_options = {}; if (params.speaker_min) config.speaker_options.min_speakers_expected = params.speaker_min; if (params.speaker_max) config.speaker_options.max_speakers_expected = params.speaker_max; } if (params.language_code) { config.language_code = params.language_code; } // ── Transcribe ─────────────────────────────────────────── let transcript: any; try { transcript = await client.transcripts.transcribe(config); if (transcript.status === "error") { return { content: [{ type: "text", text: `Transcription failed: ${transcript.error}`, }], isError: true, }; } } catch (err: any) { if (err.name === "AbortError" || signal?.aborted) { return { content: [{ type: "text", text: "Transcription cancelled." }], isError: true, }; } return { content: [{ type: "text", text: `AssemblyAI error: ${err.message}` }], isError: true, }; } finally { cleanup?.(); } // ── Parse utterances ───────────────────────────────────── const utterances: Utterance[] = (transcript.utterances ?? []).map((u: any) => ({ speaker: u.speaker, text: u.text, start: u.start, end: u.end, })); const durationSec = typeof transcript.audio_duration === "number" ? transcript.audio_duration : 0; const fullText = transcript.text ?? ""; const languageCode = transcript.language_code ?? undefined; const confidence = transcript.confidence ?? undefined; // ── Build base transcript markdown ─────────────────────── let markdown = buildTranscriptMarkdown(utterances, durationSec, languageCode, confidence); // ── Summary via LLM Gateway ────────────────────────────── let summary: string | null = null; if (params.include_summary !== false && fullText) { onUpdate?.({ content: [{ type: "text", text: "Generating summary via LLM Gateway..." }], }); try { summary = await llmGatewayChat( apiKey, "Produce a concise bullet-point summary of the following transcript. Use the same language as the transcript.", fullText, signal ?? undefined, ); if (summary) { markdown += "\n## Summary\n\n" + summary + "\n"; } } catch (err: any) { // Non-fatal -- transcript is still valid markdown += "\n## Summary\n\n(Summary generation failed: " + err.message + ")\n"; } } // ── Chapters via LLM Gateway ───────────────────────────── let chapters: string | null = null; if (params.include_chapters !== false && fullText) { onUpdate?.({ content: [{ type: "text", text: "Generating chapters via LLM Gateway..." }], }); try { chapters = await llmGatewayChat( apiKey, "Divide the following transcript into logical chapters. For each chapter, output a headline " + "and a 1-2 sentence summary. Format as markdown with ### headings. Use the same language as the transcript.", fullText, signal ?? undefined, ); if (chapters) { markdown += "\n## Chapters\n\n" + chapters + "\n"; } } catch (err: any) { markdown += "\n## Chapters\n\n(Chapter generation failed: " + err.message + ")\n"; } } return { content: [{ type: "text", text: markdown }], details: buildStructured(utterances, fullText, summary, chapters, durationSec, languageCode, confidence), }; }, }); }