/* eslint-disable no-restricted-syntax */ import { Sha256 } from '@aws-crypto/sha256-browser'; import { CreateSnippetParams } from '../services/SnippetService'; import { Document } from '../services/DocumentService'; import { SRTLine } from '../interfaces/SRTLine'; import SnippetEdgeUtil from './SnippetEdgeUtil'; /** * DO NOT CHANGE THIS VALUE * * It is used to compute the document segments in the database * and will break the embeddings if improperly changed. * * TODO: Figure out how to safely update the documents without having * TODO: to recompute the embeddings */ export const MIN_TOKENS_PER_SNIPPET = 80; export const MAX_TOKENS_PER_SNIPPET = 200; export type HashParams = { syncId: string; documentId: string; text: string; startSeconds: number; } const hash = async ({ syncId, documentId, text, startSeconds, }: HashParams): Promise => { const hash = new Sha256(); const data = `${syncId}:${documentId}:${text}:${startSeconds}`; hash.update(data); const result = await hash.digest(); return Buffer.from(result).toString('base64'); }; const tokensInSnippet = (snippet: CreateSnippetParams): number => { return snippet.text.split(' ').length; }; const createSnippetParamsFromLines = async ( lines: SRTLine[], document: Document, index: number, pageNumber: number | undefined, ): Promise => { const startTime = lines[0]?.startTime; const startSeconds = lines[0]?.startSeconds; const endTime = lines[lines.length - 1]?.endTime; const endSeconds = lines[lines.length - 1]?.endSeconds; const text = lines.filter((line) => Boolean(line.text)).map((line) => line.text).join(' '); const snippetHash = await hash({ syncId: document.syncId || '', documentId: document._id, text, startSeconds: startSeconds || 0, }); return { organizationId: document.organizationId, sourceId: document.sourceId, syncId: document.syncId || '', documentId: document._id, title: document.title, text, type: document.type, url: document.url, domain: document.domain, lang: document.lang, index, hash: snippetHash, startTime, startSeconds, endTime, endSeconds, releasedAt: document.releasedAt, views: document.views, pageNumber, }; }; const createSnippetParamsFromDocument = async ( document: Document, minTokensPerSnippet = MIN_TOKENS_PER_SNIPPET, maxTokensPerSnippet = MAX_TOKENS_PER_SNIPPET, ): Promise => { const { srt } = document; const createParams: CreateSnippetParams[] = []; const memory: SRTLine[] = []; let index = 0; for (const line of srt) { try { const isEndOfSentence = (() => { if (!line.text) return false; return line.text.endsWith('.') || line.text.endsWith('?') || line.text.endsWith('!'); })(); memory.push(line); const params = await createSnippetParamsFromLines( memory, document, index, memory?.[0]?.pageNumber ?? undefined, // Use page number of first line for snippet ); const isOverMax = tokensInSnippet(params) >= maxTokensPerSnippet; if (isEndOfSentence || isOverMax || line === srt[srt.length - 1]) { if (tokensInSnippet(params) >= minTokensPerSnippet) { createParams.push(params); memory.length = 0; index++; } } } catch (e) { console.log('Snippet Parsing Error Caught'); console.log(e); console.log(document); throw e; } } return createParams; }; const SnippetUtil = { hash, tokensInSnippet, createSnippetParamsFromLines, createSnippetParamsFromDocument, ...SnippetEdgeUtil, }; export default SnippetUtil;