import * as plugins from '../../plugins.js'; import { DcRouterDb } from '../classes.dcrouter-db.js'; const TTL = plugins.smartdata.smartdataTtlValues; const DB_OPERATION_TIMEOUT_MS = 5_000; /** * Email status in the cache */ export type TCachedEmailStatus = 'pending' | 'processing' | 'queued' | 'delivered' | 'failed' | 'deferred' | 'stored' | 'accepted' | 'flagged' | 'rejected'; /** * Direction of an accepted email: 'inbound' = received from a remote peer, * 'outbound' = locally submitted for delivery. */ export type TCachedEmailDirection = 'inbound' | 'outbound'; export type TCachedEmailSecurityDisposition = 'accepted' | 'flagged' | 'rejected'; export interface ICachedEmailListFilter { direction?: TCachedEmailDirection; recipientDomain?: string; search?: string; from?: number; to?: number; } export interface ICachedEmailTrafficBucket { bucketStart: number; sent: number; received: number; failed: number; } export type TCachedEmailSmtpDeliveryPhase = | 'connect' | 'greeting' | 'ehlo' | 'starttls' | 'tls_handshake' | 'post_tls_ehlo' | 'auth' | 'mail_from' | 'rcpt_to' | 'data_command' | 'message_body' | 'final_response' | 'rset' | 'quit' | 'unknown'; export interface ICachedEmailSmtpTransaction { id: string; queueItemId: string; queueAttempt: number; targetHost: string; targetPort: number; recipientDomain?: string; recipients: string[]; startedAt: string; completedAt: string; durationMs: number; outcome: 'succeeded' | 'failed'; retryable: boolean; errorType?: string; smtpCode?: number; recipientResults?: Array<{ recipient: string; accepted: boolean; responseCode: number; }>; failurePhase?: TCachedEmailSmtpDeliveryPhase; error?: string; outboundAuthentication?: ICachedEmailOutboundAuthenticationCheck; transcript: Array<{ timestampMs: number; phase: TCachedEmailSmtpDeliveryPhase; direction: 'client' | 'server' | 'system'; text: string; responseCode?: number; }>; } export interface ICachedEmailOutboundAuthenticationCheck { state: 'pending' | 'completed' | 'indeterminate' | 'not-evaluated'; overall: 'pass' | 'fail' | 'indeterminate'; checkedAt: string; signedMessageSha256?: string; error?: string; identity?: { sourceIp: string; addressFamily: 'ipv4' | 'ipv6'; heloDomain: string; receiverHostname: string; envelopeFrom: string; dkimDomain?: string; dkimSelector?: string; }; dkim?: Array<{ is_valid: boolean; domain: string | null; selector: string | null; status: string; details: string | null; }>; spf?: { result: string; domain: string; ip: string; explanation: string | null } | null; dmarc?: { passed: boolean; policy: string; domain: string; dkim_result: string; spf_result: string; action: string; details: string | null; } | null; } /** * Helper to get the smartdata database instance */ const getDb = () => DcRouterDb.getInstance().getDb(); /** * CachedEmail - Stores email queue items in the cache * * Used for persistent email queue storage, tracking delivery status, * and maintaining email history for the configured TTL period. */ @plugins.smartdata.Collection(() => getDb()) export class CachedEmail extends plugins.smartdata.SmartdataCachedDocument { /** * Unique identifier for this email */ @plugins.smartdata.unI() @plugins.smartdata.svDb() public id!: string; /** * Email message ID (RFC 822 Message-ID header) */ @plugins.smartdata.svDb() public messageId!: string; /** * Sender email address (envelope from) */ @plugins.smartdata.svDb() public from!: string; /** * Recipient email addresses */ @plugins.smartdata.svDb() public to!: string[]; /** * CC recipients */ @plugins.smartdata.svDb() public cc!: string[]; /** * BCC recipients */ @plugins.smartdata.svDb() public bcc!: string[]; /** * Email subject */ @plugins.smartdata.svDb() public subject!: string; /** Legacy inline RFC822 content. New messages use rawContentObjectKey. */ @plugins.smartdata.svDb() public rawContent?: string; /** SmartBucket object key containing the raw RFC822 message. */ @plugins.smartdata.svDb() public rawContentObjectKey?: string; /** Raw RFC822 message size in bytes. */ @plugins.smartdata.svDb() public rawContentSize: number = 0; /** Resolved outbound identity id that scopes a service-mail replay key. */ @plugins.smartdata.svDb() public submissionCredentialId?: string; /** Caller-supplied replay key. It must remain absent for non-idempotent mail. */ @plugins.smartdata.svDb() public submissionIdempotencyKey?: string; /** Digest of the exact authenticated outbound submission semantics. */ @plugins.smartdata.svDb() public submissionDigest?: string; /** Bounded durable SMTP transaction history copied from SmartMTA. */ @plugins.smartdata.svDb() public smtpTransactions: ICachedEmailSmtpTransaction[] = []; /** * Current status of the email */ @plugins.smartdata.index() @plugins.smartdata.svDb() public status!: TCachedEmailStatus; /** Attachment count derived from the parsed message at acceptance. */ @plugins.smartdata.svDb() public attachmentCount?: number; /** JSON-serialized inbound SPF/DKIM/DMARC verdicts from the SMTP session. */ @plugins.smartdata.svDb() public inboundSecurityResults?: string; /** Authentication disposition, kept separate from delivery/lifecycle status. */ @plugins.smartdata.index() @plugins.smartdata.svDb() public inboundSecurityDisposition?: TCachedEmailSecurityDisposition; /** * Number of delivery attempts */ @plugins.smartdata.svDb() public attempts: number = 0; /** * Maximum number of delivery attempts */ @plugins.smartdata.svDb() public maxAttempts: number = 3; /** * Timestamp for next delivery attempt. * For SmartMTA-managed deliveries this is a RECOVERY LEASE (SmartMTA's retry * time + grace), used by the spool to detect a silently dead queue — it is * NOT the user-facing retry schedule; that lives in smartMtaNextAttempt. */ @plugins.smartdata.svDb() public nextAttempt!: Date; /** * SmartMTA's actual next scheduled retry (display value for the ops UI). * Only set while the delivery is deferred inside the SmartMTA queue. */ @plugins.smartdata.svDb() public smartMtaNextAttempt?: Date; /** * Last error message if delivery failed */ @plugins.smartdata.svDb() public lastError!: string; /** * Timestamp when the email was successfully delivered */ @plugins.smartdata.svDb() public deliveredAt!: Date; /** * Sender domain (for querying/filtering) */ @plugins.smartdata.svDb() public senderDomain!: string; /** * Direction of the accepted message (inbound receipt vs local submission) */ @plugins.smartdata.svDb() public direction: TCachedEmailDirection = 'outbound'; /** * Lowercased recipient domains (for querying/filtering) */ @plugins.smartdata.svDb() public recipientDomains: string[] = []; /** * Epoch ms when the message was accepted (for newest-first listings) */ @plugins.smartdata.index() @plugins.smartdata.svDb() public acceptedAt: number = 0; /** * Priority level (higher = more important) */ @plugins.smartdata.svDb() public priority: number = 0; /** * JSON-serialized route data */ @plugins.smartdata.svDb() public routeData!: string; /** * DKIM signature status */ @plugins.smartdata.svDb() public dkimSigned: boolean = false; constructor() { super(); this.setTTL(TTL.DAYS_30); // Default 30-day TTL this.status = 'pending'; this.to = []; this.cc = []; this.bcc = []; } public override async createSavableObject(): Promise { const savableObject = await super.createSavableObject(); const submissionFields = [ this.submissionCredentialId, this.submissionIdempotencyKey, this.submissionDigest, ]; if (submissionFields.every((valueArg) => valueArg === undefined)) { delete savableObject.submissionCredentialId; delete savableObject.submissionIdempotencyKey; delete savableObject.submissionDigest; } else if ( typeof this.submissionCredentialId !== 'string' || !/^[A-Za-z0-9][A-Za-z0-9._:-]{0,199}$/.test(this.submissionCredentialId) || typeof this.submissionIdempotencyKey !== 'string' || !/^[A-Za-z0-9][A-Za-z0-9:._/-]{0,199}$/.test(this.submissionIdempotencyKey) || typeof this.submissionDigest !== 'string' || !/^sha256:[a-f0-9]{64}$/.test(this.submissionDigest) ) { throw new Error('CachedEmail submission idempotency metadata is invalid'); } if (this.rawContentObjectKey) { delete savableObject.rawContent; } if (this.inboundSecurityDisposition === undefined) { delete savableObject.inboundSecurityDisposition; } return savableObject; } public static async findBySubmissionIdempotencyKey( credentialIdArg: string, idempotencyKeyArg: string, ): Promise { if ( typeof credentialIdArg !== 'string' || !/^[A-Za-z0-9][A-Za-z0-9._:-]{0,199}$/.test(credentialIdArg) || typeof idempotencyKeyArg !== 'string' || !/^[A-Za-z0-9][A-Za-z0-9:._/-]{0,199}$/.test(idempotencyKeyArg) ) throw new Error('CachedEmail submission idempotency selector is invalid'); return await CachedEmail.getInstance({ submissionCredentialId: credentialIdArg, submissionIdempotencyKey: idempotencyKeyArg, }); } /** * Create a new CachedEmail with a unique ID */ public static createNew(): CachedEmail { const email = new CachedEmail(); email.id = plugins.uuid.v4(); return email; } /** * Find an email by ID */ public static async findById(id: string): Promise { return await CachedEmail.getInstance({ id, }); } /** * Find all emails with a specific status */ public static async findByStatus(status: TCachedEmailStatus): Promise { return await CachedEmail.getInstances({ status, }); } /** * Find all emails pending delivery (status = pending and nextAttempt <= now) */ public static async findPendingForDelivery(limit = 25): Promise { const now = new Date(); return await CachedEmail.findLimited({ status: { $in: ['pending', 'deferred', 'processing'] }, nextAttempt: { $lte: now }, }, limit); } public static async findQueuedForRecovery(limit = 25): Promise { return await CachedEmail.findLimited({ status: 'queued', }, limit); } private static async findLimited( filter: Record, limit: number, sort?: Record, ): Promise { const safeLimit = Number.isFinite(limit) ? Math.max(0, Math.floor(limit)) : 25; if (safeLimit === 0) { return []; } const collection = (CachedEmail as typeof CachedEmail & { collection: plugins.smartdata.SmartdataCollection; }).collection; const cursor = await collection.getCursor( filter, CachedEmail as unknown as typeof plugins.smartdata.SmartDataDbDoc, ); if (sort) { cursor.mongodbCursor.sort(sort); } cursor.mongodbCursor.limit(safeLimit); try { return await cursor.toArray(); } finally { await cursor.close(); } } private static getNativeCollection() { return getDb().mongoDb.collection('CachedEmail'); } private static buildListQuery(filterArg: ICachedEmailListFilter): Record { const query: Record = {}; if (filterArg.direction) { query.direction = filterArg.direction; } if (filterArg.recipientDomain) { query.recipientDomains = filterArg.recipientDomain.toLowerCase(); } if (filterArg.search) { if (typeof filterArg.search !== 'string' || filterArg.search.length > 200) { throw new Error('Invalid CachedEmail search'); } const escapedSearch = filterArg.search.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); query.$or = [ { from: { $regex: escapedSearch, $options: 'i' } }, { to: { $regex: escapedSearch, $options: 'i' } }, { subject: { $regex: escapedSearch, $options: 'i' } }, { messageId: { $regex: escapedSearch, $options: 'i' } }, ]; } if (filterArg.from !== undefined || filterArg.to !== undefined) { const from = filterArg.from; const to = filterArg.to; if ( !Number.isSafeInteger(from) || !Number.isSafeInteger(to) || (from as number) < 0 || (to as number) < (from as number) ) { throw new Error('Invalid CachedEmail time range'); } query.acceptedAt = { $gte: from, $lte: to }; } return query; } /** * Find the most recently accepted emails, newest first. */ public static async findRecent( limit = 200, filter: ICachedEmailListFilter = {}, ): Promise { const query = CachedEmail.buildListQuery(filter); return await CachedEmail.findLimited(query, limit, { acceptedAt: -1 }); } public static async findOldestAcceptedAt( filterArg: ICachedEmailListFilter, cutoffArg: number, ): Promise { if (!Number.isSafeInteger(cutoffArg) || cutoffArg < 0) { throw new Error('CachedEmail.findOldestAcceptedAt requires a valid cutoff'); } const query = CachedEmail.buildListQuery(filterArg); query.acceptedAt = { $gte: cutoffArg }; const collection = await CachedEmail.getNativeCollection(); const row = await collection.findOne( query, { projection: { acceptedAt: 1 }, sort: { acceptedAt: 1 }, timeoutMS: DB_OPERATION_TIMEOUT_MS, }, ); const acceptedAt = Number(row?.acceptedAt); return Number.isSafeInteger(acceptedAt) && acceptedAt >= 0 ? acceptedAt : null; } public static async aggregateEmailLogTraffic( filterArg: ICachedEmailListFilter, windowStartArg: number, windowEndArg: number, bucketSizeMsArg: number, ): Promise { if ( !Number.isSafeInteger(windowStartArg) || !Number.isSafeInteger(windowEndArg) || !Number.isSafeInteger(bucketSizeMsArg) || windowStartArg < 0 || windowEndArg < windowStartArg || bucketSizeMsArg <= 0 ) { throw new Error('CachedEmail.aggregateEmailLogTraffic received an invalid window'); } const query = CachedEmail.buildListQuery(filterArg); query.acceptedAt = { $gte: windowStartArg, $lte: windowEndArg }; const collection = await CachedEmail.getNativeCollection(); const cursor = collection.find( query, { projection: { acceptedAt: 1, direction: 1, status: 1 }, batchSize: 1000, timeoutMS: DB_OPERATION_TIMEOUT_MS, }, ); try { const buckets = new Map(); while (await cursor.hasNext()) { const row = await cursor.next(); if (!row) { continue; } const acceptedAt = Number(row.acceptedAt); if (!Number.isSafeInteger(acceptedAt)) { continue; } const bucketStart = Math.floor(acceptedAt / bucketSizeMsArg) * bucketSizeMsArg; let bucket = buckets.get(bucketStart); if (!bucket) { bucket = { bucketStart, sent: 0, received: 0, failed: 0 }; buckets.set(bucketStart, bucket); } if (row.status === 'failed' || row.status === 'rejected') { bucket.failed++; } else if (row.direction === 'outbound') { bucket.sent++; } else if (row.direction === 'inbound') { bucket.received++; } } return [...buckets.values()].sort((firstArg, secondArg) => ( firstArg.bucketStart - secondArg.bucketStart )); } finally { await cursor.close({ timeoutMS: DB_OPERATION_TIMEOUT_MS }); } } public static async findRecentSecurityCandidates( limitArg: number, cutoffArg: number, ): Promise { const safeLimit = Math.min(Math.max(Math.floor(limitArg), 1), 1000); if (!Number.isSafeInteger(cutoffArg) || cutoffArg < 0) { throw new Error('CachedEmail.findRecentSecurityCandidates requires a valid cutoff'); } return await CachedEmail.findLimited({ direction: 'inbound', inboundSecurityDisposition: { $in: ['flagged', 'rejected'] }, acceptedAt: { $gte: cutoffArg }, }, safeLimit, { acceptedAt: -1 }); } /** * Find emails addressed to a recipient domain */ public static async findByRecipientDomain(domain: string): Promise { return await CachedEmail.getInstances({ recipientDomains: { $all: [domain.toLowerCase()] }, }); } /** * Find emails by sender domain */ public static async findBySenderDomain(domain: string): Promise { return await CachedEmail.getInstances({ senderDomain: domain, }); } /** * Mark as delivered */ public markDelivered(): void { this.status = 'delivered'; this.deliveredAt = new Date(); } /** * Mark as terminally stored (catch-all inbound): visible in the email log, * pruned after the retention window. */ public markStored(retentionMs: number): void { this.status = 'stored'; this.deliveredAt = new Date(); // Base-class TTL: the CacheCleaner deletes expired rows and their raw // payloads via beforeDeleteCachedEmail. this.setTTL(retentionMs); } /** * Mark as failed with error */ public markFailed(error: string): void { this.status = 'failed'; this.lastError = error; } public appendSmtpTransaction(transactionArg: ICachedEmailSmtpTransaction): void { const transactions = (this.smtpTransactions || []).filter( (transaction) => transaction.id !== transactionArg.id, ); transactions.push(structuredClone(transactionArg)); this.smtpTransactions = transactions.slice(-20); } /** * Increment attempt counter and schedule next attempt */ public scheduleRetry(delayMs: number = 5 * 60 * 1000): void { this.attempts++; this.status = 'deferred'; this.nextAttempt = new Date(Date.now() + delayMs); // If max attempts reached, mark as failed if (this.attempts >= this.maxAttempts) { this.status = 'failed'; this.lastError = `Max attempts (${this.maxAttempts}) reached`; } } /** * Extract sender domain from email address */ public updateSenderDomain(): void { if (this.from) { const match = this.from.match(/@([^>]+)>?$/); if (match) { this.senderDomain = match[1].toLowerCase(); } } } /** * Derive recipient domains from the recipient address list */ public updateRecipientDomains(): void { const domains = new Set(); for (const address of this.to || []) { const match = address.match(/@([^>]+)>?$/); if (match) { domains.add(match[1].toLowerCase()); } } this.recipientDomains = [...domains]; } }