import { readFile } from "node:fs/promises"; import { embedAndUpsert } from "@vellumai/plugin-api"; import { and, eq } from "drizzle-orm"; import { getDb } from "../../../../../persistence/db-connection.js"; import type { EmbeddingInput } from "../../../../../persistence/embeddings/embedding-types.js"; import { asString } from "../../../../../persistence/job-utils.js"; import type { MemoryJob } from "../../../../../persistence/jobs-store.js"; import { mediaAssets, memorySegments, memorySummaries, messages, } from "../../../../../persistence/schema/index.js"; import { memoryDbOrNull } from "../../memory-db.js"; import { extractMediaBlocks } from "../../message-media.js"; export async function embedSegmentJob(job: MemoryJob): Promise { const segmentId = asString(job.payload.segmentId); if (!segmentId) { return; } const mem = memoryDbOrNull("embedSegmentJob"); if (!mem) { return; } const segment = mem .select() .from(memorySegments) .where(eq(memorySegments.id, segmentId)) .get(); if (!segment) { return; } await embedAndUpsert("segment", segment.id, segment.text, { conversation_id: segment.conversationId, message_id: segment.messageId, created_at: segment.createdAt, }); } export async function embedSummaryJob(job: MemoryJob): Promise { const summaryId = asString(job.payload.summaryId); if (!summaryId) { return; } const db = memoryDbOrNull("embedSummaryJob"); if (!db) { return; } const summary = db .select() .from(memorySummaries) .where(eq(memorySummaries.id, summaryId)) .get(); if (!summary) { return; } await embedAndUpsert( "summary", summary.id, `[${summary.scope}] ${summary.summary}`, { kind: summary.scope, created_at: summary.startAt, last_seen_at: summary.endAt, }, ); } export async function embedMediaJob(job: MemoryJob): Promise { const assetId = asString(job.payload.assetId); if (!assetId) { return; } const db = getDb(); const asset = db .select() .from(mediaAssets) .where(eq(mediaAssets.id, assetId)) .get(); if (!asset || asset.status !== "indexed") { return; } // Read the media file from disk const fileData = await readFile(asset.filePath); // Determine modality from mediaType const input: EmbeddingInput = { type: asset.mediaType as "image" | "audio" | "video", data: fileData, mimeType: asset.mimeType, }; await embedAndUpsert("media", asset.id, input, { created_at: asset.createdAt, kind: asset.mediaType, subject: asset.title, }); } export async function embedAttachmentJob(job: MemoryJob): Promise { const messageId = asString(job.payload.messageId); const blockIndex = job.payload.blockIndex as number; if (!messageId || typeof blockIndex !== "number") { return; } const db = getDb(); const message = db .select() .from(messages) .where(and(eq(messages.id, messageId), eq(messages.finalized, 1))) .get(); if (!message) { return; } const mediaBlocks = extractMediaBlocks(message.content); const block = mediaBlocks.find((b) => b.index === blockIndex); if (!block) { return; } const input: EmbeddingInput = { type: block.type, data: block.data, mimeType: block.mimeType, }; // Use messageId + blockIndex as targetId for uniqueness const targetId = `${messageId}:${blockIndex}`; await embedAndUpsert("media", targetId, input, { created_at: message.createdAt, message_id: messageId, conversation_id: message.conversationId, }); }