import type { ContentCandidate, TranscriptDiagnostics, TranscriptWord, WordRange, WordTranscript } from './contracts.ts'; const normalize = (text: string) => text.normalize('NFKC').toLowerCase().replace(/[\p{P}\p{S}\s]+/gu, ''); const LIMIT = 200; /** Bounded retrieval hints, never a completeness claim or an automatic deletion rule. */ export function analyzeContent(transcript: WordTranscript, words: TranscriptWord[]): { contentCandidates: ContentCandidate[]; diagnostics: TranscriptDiagnostics; } { const contentCandidates: ContentCandidate[] = []; const sentenceAt = transcript.sentences.flatMap((sentence, index) => sentence.words.map(() => index)); const chars: string[] = []; const wordAt: number[] = []; words.forEach((word, index) => { for (const char of normalize(word.text)) { chars.push(char); wordAt.push(index); } }); let truncated = false; const add = (kind: ContentCandidate['kind'], ranges: WordRange[], reason: string) => { if (contentCandidates.some((candidate) => candidate.ranges.every((range, index) => { const other = ranges[index]!; return range.startWordIndex <= other.startWordIndex && range.endWordIndex >= other.endWordIndex; }))) return; if (contentCandidates.length >= LIMIT) { truncated = true; return; } contentCandidates.push({ id: `content-${String(contentCandidates.length + 1).padStart(3, '0')}`, kind, ranges, contextTexts: ranges.map((range) => transcript.sentences.slice(sentenceAt[range.startWordIndex]!, sentenceAt[range.endWordIndex]! + 1).map((s) => s.text).join('\n')), recommendation: 'review', confidence: 'low', reason, }); }; const seen = new Map(); for (let i = 0; i + 6 <= chars.length; i++) { const seed = chars.slice(i, i + 6).join(''); const previous = seen.get(seed) ?? []; for (const start of previous) { if (start + 6 > i) continue; let length = 6; while (length < 160 && start + length < i && i + length < chars.length && chars[start + length] === chars[i + length]) length++; const first = { startWordIndex: wordAt[start]!, endWordIndex: wordAt[start + length - 1]! }; const second = { startWordIndex: wordAt[i]!, endWordIndex: wordAt[i + length - 1]! }; if (first.endWordIndex >= second.startWordIndex) continue; add(sentenceAt[first.startWordIndex] === sentenceAt[second.startWordIndex] && i - start - length <= 20 ? 'restart' : 'repeated-passage', [first, second], 'A phrase occurs twice. Compare complete takes, false starts, intentional repetition, and information unique to each take before deciding.'); } if (previous.length < 8) seen.set(seed, [...previous, i]); else { truncated = true; seen.set(seed, [...previous.slice(1), i]); } } // Fuzzy sentence comparison complements exact phrases, including lightly paraphrased retakes. const grams = transcript.sentences.map((sentence) => { const text = Array.from(normalize(sentence.text)); return new Set(text.length < 12 ? [] : text.slice(0, -2).map((_c, i) => text.slice(i, i + 3).join(''))); }); let offset = 0; const ranges = transcript.sentences.map((sentence) => { const range = { startWordIndex: offset, endWordIndex: offset + sentence.words.length - 1 }; offset += sentence.words.length; return range; }); for (let i = 0; i < grams.length; i++) { for (let j = i + 1; j < Math.min(grams.length, i + 13); j++) { const a = grams[i]!, b = grams[j]!; if (!a.size || !b.size) continue; const common = [...a].filter((gram) => b.has(gram)).length; if (2 * common / (a.size + b.size) >= 0.45) add('similar-passage', [ranges[i]!, ranges[j]!], 'Nearby ASR segments share substantial wording. This is a retake hint, not proof of redundant meaning; preserve unique information.'); } } const zeroGapPairs = words.slice(1).filter((word, i) => word.beginMs === words[i]!.endMs).length; const adjacentWordPairs = Math.max(0, words.length - 1); const warnings = [ 'Candidates are incomplete retrieval hints. Review the entire transcript and source for retakes, false starts, and unique information.', 'ASR segment boundaries are not verified semantic boundaries; punctuation is only a low-confidence cue.', ]; if (adjacentWordPairs >= 10 && zeroGapPairs / adjacentWordPairs >= 0.8) warnings.push( 'Most word timestamps are contiguous. Zero timestamp gaps do not prove continuous speech; inspect original audio/waveforms for within-segment pauses.'); if (truncated) warnings.push('Content candidate retrieval was bounded/truncated; full-passage editorial review remains required.'); return { contentCandidates, diagnostics: { adjacentWordPairs, zeroGapPairs, positiveGapPairs: adjacentWordPairs - zeroGapPairs, contentCandidatesTruncated: truncated, warnings } }; }