import type { ArollSegment, FillerCandidate, PauseCandidate, PauseClassification, RepetitionCandidate, SentenceAnalysis, TalkingHeadPolicy, TranscriptAnalysis, TranscriptWord, WordTranscript, } from "./contracts.ts"; import { analyzeContent } from "./content-analysis.ts"; const HESITATION_FILLERS = new Set(["嗯", "呃", "额", "唔", "呣", "em", "um", "uh"]); const CONTEXTUAL_FILLERS = new Set(["啊", "呀", "那个", "这个", "就是", "然后"]); export const DEFAULT_POLICY: TalkingHeadPolicy = { cutThresholdMs: 500, headPaddingMs: 50, tailPaddingMs: 80, }; function finiteNonNegative(value: unknown): value is number { return typeof value === "number" && Number.isFinite(value) && value >= 0; } function classifyPause(durationMs: number): PauseClassification { if (durationMs >= 400) return "safe"; if (durationMs >= 150) return "review"; return "unsafe"; } function validatePolicy(policy: TalkingHeadPolicy): void { if (!Number.isFinite(policy.cutThresholdMs) || policy.cutThresholdMs < 150) { throw new Error("cutThresholdMs must be at least 150ms"); } for (const [name, value] of [["headPaddingMs", policy.headPaddingMs], ["tailPaddingMs", policy.tailPaddingMs]] as const) { if (!Number.isFinite(value) || value < 30 || value > 200) { throw new Error(`${name} must be within the 30-200ms working window`); } } } function flattenWords(transcript: WordTranscript): TranscriptWord[] { if (typeof transcript.text !== "string" || !Array.isArray(transcript.sentences)) { throw new Error("Transcript must contain text and sentences"); } const words = transcript.sentences.flatMap((sentence) => { if (!sentence || !Number.isInteger(sentence.id) || typeof sentence.text !== "string" || !finiteNonNegative(sentence.beginMs) || !finiteNonNegative(sentence.endMs) || sentence.endMs <= sentence.beginMs || !Array.isArray(sentence.words) || sentence.words.length === 0) { throw new Error("Transcript contains an invalid sentence timestamp or metadata"); } const first = sentence.words[0]!; const last = sentence.words.at(-1)!; if (!finiteNonNegative(first.beginMs) || !finiteNonNegative(last.endMs) || first.beginMs < sentence.beginMs || last.endMs > sentence.endMs) { throw new Error("Transcript sentence timestamp does not contain its word timestamps"); } return sentence.words; }); if (words.length === 0) throw new Error("Transcript contains no word timestamps"); let previousEnd = -1; for (const word of words) { if (typeof word.text !== "string" || !word.text.trim() || !finiteNonNegative(word.beginMs) || !finiteNonNegative(word.endMs) || word.endMs <= word.beginMs) { throw new Error("Transcript contains an invalid word timestamp"); } if (word.beginMs < previousEnd) throw new Error("Transcript word timestamps overlap or are out of order"); previousEnd = word.endMs; } return words.map((word) => ({ ...word, punctuation: word.punctuation ?? "" })); } function normalizedSpokenText(text: string): string { return text.trim().toLowerCase().replace(/^[\p{P}\p{S}\s]+|[\p{P}\p{S}\s]+$/gu, ""); } function fillerCandidates(transcript: WordTranscript, words: TranscriptWord[]): FillerCandidate[] { const candidates: FillerCandidate[] = []; let wordIndex = 0; for (let sentenceIndex = 0; sentenceIndex < transcript.sentences.length; sentenceIndex += 1) { const sentence = transcript.sentences[sentenceIndex]!; for (const _word of sentence.words) { const word = words[wordIndex]!; const normalized = normalizedSpokenText(word.text); const hesitation = HESITATION_FILLERS.has(normalized); const contextual = CONTEXTUAL_FILLERS.has(normalized); if (hesitation || contextual) { candidates.push({ id: `filler-${String(candidates.length + 1).padStart(3, "0")}`, wordIndex, sentenceIndex, text: word.text, startMs: word.beginMs, endMs: word.endMs, kind: hesitation ? "hesitation" : "discourse", matchConfidence: hesitation ? "exact" : "contextual", recommendation: "review", contextText: sentence.text, reasons: hesitation ? ["The token commonly marks hesitation, but may still carry delivery intent."] : ["The token can be either a filler or meaningful discourse, so context is required."], }); } wordIndex += 1; } } return candidates; } function sentenceAnalyses(transcript: WordTranscript, fillers: FillerCandidate[]): SentenceAnalysis[] { let wordStartIndex = 0; const fillersBySentence = new Map(); for (const filler of fillers) { const current = fillersBySentence.get(filler.sentenceIndex) ?? []; current.push(filler); fillersBySentence.set(filler.sentenceIndex, current); } return transcript.sentences.map((sentence, sentenceIndex) => { const wordEndIndex = wordStartIndex + sentence.words.length - 1; const sentenceFillers = fillersBySentence.get(sentenceIndex) ?? []; const deliveryCues: SentenceAnalysis["deliveryCues"] = []; const evidence: string[] = []; const hesitationCount = sentenceFillers.filter((candidate) => candidate.kind === "hesitation").length; if (hesitationCount > 0) { deliveryCues.push("hesitation"); evidence.push(`Contains ${hesitationCount} hesitation-lexicon token(s).`); } if (/[??]/u.test(sentence.text) || sentence.words.some((word) => /[??]/u.test(word.punctuation ?? ""))) { deliveryCues.push("question"); evidence.push("Question punctuation is present in the transcript."); } if (/[!!]/u.test(sentence.text) || sentence.words.some((word) => /[!!]/u.test(word.punctuation ?? ""))) { deliveryCues.push("emphasis"); evidence.push("Emphasis punctuation is present in the transcript."); } if (deliveryCues.length === 0) { deliveryCues.push("neutral"); evidence.push("No explicit delivery cue was found in transcript text."); } const analysis: SentenceAnalysis = { sentenceIndex, sentenceId: sentence.id, beginMs: sentence.beginMs, endMs: sentence.endMs, text: sentence.text, wordStartIndex, wordEndIndex, deliveryCues, confidence: "low", evidence, }; wordStartIndex = wordEndIndex + 1; return analysis; }); } function repetitionCandidates(transcript: WordTranscript, words: TranscriptWord[]): RepetitionCandidate[] { const candidates: RepetitionCandidate[] = []; let sentenceWordStart = 0; for (let sentenceIndex = 0; sentenceIndex < transcript.sentences.length; sentenceIndex += 1) { const sentence = transcript.sentences[sentenceIndex]!; for (let offset = 1; offset < sentence.words.length; offset += 1) { const firstWordIndex = sentenceWordStart + offset - 1; const secondWordIndex = sentenceWordStart + offset; const first = words[firstWordIndex]!; const second = words[secondWordIndex]!; const normalized = normalizedSpokenText(first.text); if (!normalized || normalized !== normalizedSpokenText(second.text)) continue; candidates.push({ id: `repetition-${String(candidates.length + 1).padStart(3, "0")}`, sentenceIndex, text: first.text, firstWordIndex, secondWordIndex, startMs: first.beginMs, endMs: second.endMs, recommendation: "review", contextText: sentence.text, reasons: ["Two adjacent normalized tokens are identical; review whether this is a false start or intentional emphasis."], }); } sentenceWordStart += sentence.words.length; } return candidates; } function wordSentenceIndexes(transcript: WordTranscript): number[] { return transcript.sentences.flatMap((sentence, sentenceIndex) => sentence.words.map(() => sentenceIndex)); } function candidatesFrom( words: TranscriptWord[], transcript: WordTranscript, fillers: FillerCandidate[], policy: TalkingHeadPolicy, ): PauseCandidate[] { const candidates: PauseCandidate[] = []; const sentenceIndexes = wordSentenceIndexes(transcript); const fillersByWordIndex = new Map(); for (const filler of fillers) { fillersByWordIndex.set(filler.wordIndex, [...(fillersByWordIndex.get(filler.wordIndex) ?? []), filler]); } for (let index = 1; index < words.length; index += 1) { const before = words[index - 1]; const after = words[index]; if (!before || !after) continue; const durationMs = after.beginMs - before.endMs; if (durationMs <= 0) continue; const beforeSentenceIndex = sentenceIndexes[index - 1]!; const afterSentenceIndex = sentenceIndexes[index]!; const adjacentFillers = [ ...(fillersByWordIndex.get(index - 1) ?? []), ...(fillersByWordIndex.get(index) ?? []), ]; const sentenceBoundary = beforeSentenceIndex !== afterSentenceIndex; const beforeSentence = transcript.sentences[beforeSentenceIndex]!; const expressiveBoundary = /[!!??…]/u.test(before.punctuation) || /[!!??…]\s*$/u.test(beforeSentence.text); const reasons: string[] = []; let recommendation: PauseCandidate["recommendation"]; if (durationMs < 150) { recommendation = "keep"; reasons.push("The gap is shorter than 150ms and removing it risks robotic cadence."); } else if (adjacentFillers.length > 0) { recommendation = "review"; reasons.push("The gap touches a possible filler whose meaning must be judged in sentence context."); } else if (expressiveBoundary) { recommendation = "review"; reasons.push("The pause follows expressive punctuation and may carry emphasis, emotion, or a question beat."); } else if (sentenceBoundary) { recommendation = "review"; reasons.push("The gap is between ASR segments. Review whether it marks a real sentence, paragraph, topic, or deliberate delivery boundary."); } else if (durationMs >= policy.cutThresholdMs) { recommendation = "cut"; reasons.push(`The unprotected word gap meets the ${policy.cutThresholdMs}ms automatic cut threshold.`); } else { recommendation = "review"; reasons.push("The gap is noticeable but does not meet the automatic cut threshold."); } candidates.push({ id: `pause-${String(candidates.length + 1).padStart(3, "0")}`, startMs: before.endMs, endMs: after.beginMs, durationMs, classification: classifyPause(durationMs), beforeText: `${before.text}${before.punctuation}`, afterText: after.text, boundary: beforeSentenceIndex === afterSentenceIndex ? "within-sentence" : "between-sentences", semanticBoundary: /[。.!!??…]/u.test(before.punctuation) || (sentenceBoundary && /[。.!!??…]\s*$/u.test(beforeSentence.text)) ? "punctuation" : "unknown", context: { before: transcript.sentences[beforeSentenceIndex]!.text, after: transcript.sentences[afterSentenceIndex]!.text, }, adjacentFillerIds: adjacentFillers.map((candidate) => candidate.id), recommendation, reasons, }); } return candidates; } function defaultSegments(words: TranscriptWord[], policy: TalkingHeadPolicy, candidates: PauseCandidate[]): ArollSegment[] { const segments: ArollSegment[] = []; const candidateByGap = new Map(candidates.map((candidate) => [`${candidate.startMs}:${candidate.endMs}`, candidate])); let segmentStart = Math.max(0, words[0]!.beginMs - policy.headPaddingMs); for (let index = 1; index < words.length; index += 1) { const before = words[index - 1]!; const after = words[index]!; const candidate = candidateByGap.get(`${before.endMs}:${after.beginMs}`); if (candidate?.recommendation !== "cut") continue; segments.push({ id: `a-${String(segments.length + 1).padStart(3, "0")}`, sourceStartMs: segmentStart, sourceEndMs: before.endMs + policy.tailPaddingMs, }); segmentStart = Math.max(0, after.beginMs - policy.headPaddingMs); } segments.push({ id: `a-${String(segments.length + 1).padStart(3, "0")}`, sourceStartMs: segmentStart, sourceEndMs: words.at(-1)!.endMs + policy.tailPaddingMs, }); return segments; } export function timelineDuration(segments: ArollSegment[]): number { return segments.reduce((total, segment) => total + segment.sourceEndMs - segment.sourceStartMs, 0); } export function analyzeTranscript( transcript: WordTranscript, overrides: Partial = {}, ): TranscriptAnalysis { const policy = { ...DEFAULT_POLICY, ...overrides }; validatePolicy(policy); const words = flattenWords(transcript); const fillers = fillerCandidates(transcript, words); const repetitions = repetitionCandidates(transcript, words); const candidates = candidatesFrom(words, transcript, fillers, policy); const segments = defaultSegments(words, policy, candidates); return { schemaVersion: 3, text: transcript.text, words, ...analyzeContent(transcript, words), sentences: sentenceAnalyses(transcript, fillers), fillers, repetitions, candidates, segments, outputDurationMs: timelineDuration(segments), }; }