import { eq } from "drizzle-orm"; import { z } from "zod"; import { getConfig } from "../../config/loader.js"; import { addMessage, createConversation, type MessageRole, } from "../../persistence/conversation-crud.js"; import { getConversationByKey, setConversationKey, } from "../../persistence/conversation-key-store.js"; import { getDb } from "../../persistence/db-connection.js"; import { enqueueLexicalIndexForMessage } from "../../persistence/job-handlers/message-lexical.js"; import { conversations as conversationsTable, messages as messagesTable, } from "../../persistence/schema/index.js"; import { indexMessageNow } from "../../plugins/defaults/memory/indexer.js"; import { getLogger } from "../../util/logger.js"; import { withSqliteRetry } from "../../util/sqlite-retry.js"; import { ACTOR_PRINCIPALS } from "../auth/route-policy.js"; import { BadRequestError } from "./errors.js"; import type { RouteDefinition, RouteHandlerArgs } from "./types.js"; const log = getLogger("conversations-import-routes"); // -- Types -- interface ImportMessage { role: MessageRole; content: string | Array<{ type: string; text: string }>; createdAt?: number; } interface ImportConversation { sourceKey?: string; title: string; createdAt?: number; updatedAt?: number; messages: ImportMessage[]; } interface ImportPayload { conversations: ImportConversation[]; } // -- Helpers (ported from CLI) -- function resolveTimestamps( conv: ImportConversation, messages: ImportMessage[], ): { convCreatedAt: number; convUpdatedAt: number; messageTimestamps: number[]; } { const now = Date.now(); const convCreatedAt = conv.createdAt ?? now; const convUpdatedAt = conv.updatedAt ?? conv.createdAt ?? now; const messageTimestamps = messages.map((msg, i) => { if (msg.createdAt != null) { return msg.createdAt; } return convCreatedAt + i; }); return { convCreatedAt, convUpdatedAt, messageTimestamps }; } /** * The messages store is UI-facing (`ConversationMessage`), so only renderable * turns are persisted. Non-renderable roles (e.g. agent-context `system` * rows an export might carry) are dropped here rather than imported — the * import still succeeds with the displayable turns. */ function isRenderableRole(role: string): role is "user" | "assistant" { return role === "user" || role === "assistant"; } /** * Provenance `source` for an imported conversation. Export tooling prefixes * `sourceKey` with the originating provider (e.g. `chatgpt:abc123`), so the * substring before the first colon is normalized (lowercased, non * `[a-z0-9-]` runs collapsed to `-`) into `import:`; only keys * with no usable prefix fall back to `import:unknown`. Distinguishes * imported rows from the schema default `"user"` (schema/conversations.ts). */ function deriveImportSource(sourceKey: string | undefined): string { const colonIdx = sourceKey?.indexOf(":") ?? -1; if (sourceKey === undefined || colonIdx <= 0) { return "import:unknown"; } const provider = sourceKey .slice(0, colonIdx) .toLowerCase() .replace(/[^a-z0-9-]+/g, "-") .replace(/^-+|-+$/g, ""); return provider ? `import:${provider}` : "import:unknown"; } // -- Handler -- async function handleConversationsImport({ body }: RouteHandlerArgs) { if ( !body || !Array.isArray((body as Record).conversations) ) { throw new BadRequestError("conversations array required"); } const payload = body as unknown as ImportPayload; const db = getDb(); const memoryConfig = getConfig().memory; let imported = 0; let skipped = 0; let totalMessages = 0; const errors: Array<{ index: number; sourceKey?: string; error: string }> = []; for (let idx = 0; idx < payload.conversations.length; idx++) { const conv = payload.conversations[idx]; if (!conv || typeof conv !== "object") { errors.push({ index: idx, error: "invalid conversation entry" }); continue; } try { // Dedup via sourceKey if (conv.sourceKey) { const existing = getConversationByKey(conv.sourceKey); if (existing) { skipped++; continue; } } const messages = conv.messages.filter((msg) => isRenderableRole(msg.role), ); const { convCreatedAt, convUpdatedAt, messageTimestamps } = resolveTimestamps(conv, messages); const source = deriveImportSource(conv.sourceKey); const conversation = await withSqliteRetry( () => createConversation({ title: conv.title, source }), { op: "conversationsImport.createConversation" }, ); for (const msg of messages) { const contentStr = typeof msg.content === "string" ? msg.content : JSON.stringify(msg.content); await addMessage(conversation.id, msg.role, contentStr, { skipIndexing: true, }); } // Override conversation timestamps db.update(conversationsTable) .set({ createdAt: convCreatedAt, updatedAt: convUpdatedAt, lastMessageAt: messageTimestamps[messageTimestamps.length - 1], }) .where(eq(conversationsTable.id, conversation.id)) .run(); // Override message timestamps const dbMessages = db .select({ id: messagesTable.id }) .from(messagesTable) .where(eq(messagesTable.conversationId, conversation.id)) .orderBy(messagesTable.createdAt) .all(); for ( let i = 0; i < dbMessages.length && i < messageTimestamps.length; i++ ) { db.update(messagesTable) .set({ createdAt: messageTimestamps[i] }) .where(eq(messagesTable.id, dbMessages[i].id)) .run(); } // Index messages for (let i = 0; i < dbMessages.length && i < messages.length; i++) { const msg = messages[i]; const contentStr = typeof msg.content === "string" ? msg.content : JSON.stringify(msg.content); try { await indexMessageNow( { messageId: dbMessages[i].id, conversationId: conversation.id, role: msg.role, content: contentStr, createdAt: messageTimestamps[i], }, memoryConfig, ); } catch (err) { log.warn( "Failed to index imported message %s in conversation %s: %s", dbMessages[i].id, conversation.id, err instanceof Error ? err.message : String(err), ); } // Dual-write the imported message into the lexical index. Import inserts // rows directly, bypassing the `addMessage` persist path, so enqueue // here to keep the lexical index in lockstep with the segment index. enqueueLexicalIndexForMessage(dbMessages[i].id); } if (conv.sourceKey) { setConversationKey(conv.sourceKey, conversation.id); } imported++; totalMessages += messages.length; } catch (err) { errors.push({ index: idx, sourceKey: conv.sourceKey, error: err instanceof Error ? err.message : String(err), }); } } return { ok: errors.length === 0, imported, skipped, messages: totalMessages, errors, }; } // -- Routes -- export const ROUTES: RouteDefinition[] = [ { operationId: "conversations_import", endpoint: "conversations/import", method: "POST", policy: { requiredScopes: ["chat.write"], allowedPrincipalTypes: ACTOR_PRINCIPALS, }, handler: handleConversationsImport, summary: "Import conversations", description: "Import conversations from a standard JSON payload. Created " + "conversations record a provenance source of `import:` " + "derived from the `sourceKey` prefix (e.g. `chatgpt:abc123` -> " + "`import:chatgpt`), or `import:unknown` when no prefix is present.", tags: ["conversations"], requestBody: z.object({ conversations: z.array( z.object({ sourceKey: z.string().optional(), title: z.string(), createdAt: z.number().optional(), updatedAt: z.number().optional(), messages: z.array( z.object({ role: z.string(), content: z.union([ z.string(), z.array(z.object({ type: z.string(), text: z.string() })), ]), createdAt: z.number().optional(), }), ), }), ), }), responseBody: z.object({ ok: z.boolean(), imported: z.number(), skipped: z.number(), messages: z.number(), errors: z.array( z.object({ index: z.number(), sourceKey: z.string().optional(), error: z.string(), }), ), }), }, ];