/** * Encryption for backup artifacts — file in, file out, streamed. * * Deliberately NOT `encryptSecret`/`decryptSecret`. Those are string-in, * string-out and correct for what they were built for: short values headed * for a DB column. Backup artifacts are the opposite shape, and running them * through a string API cost three full in-memory copies with an expansion * factor at each step: * * read the tar 774 MB Buffer * .toString('base64') 1032 MB string * encrypt to hex 2064 MB string (hex is 2 chars per byte) * JSON.stringify 2064 MB string * * — about 6.9 GB live for one 774 MB module, which is what OOM-killed the * forgejo backup. Worse, it had a ceiling no amount of RAM could raise: * hex-of-base64 is 2.67 chars per input byte against a ~2^31 max string * length, so the old path simply could not represent a tar over ~805 MB. * * Here the plaintext never exists in memory at all. `createCipheriv` is a * Transform, so file → cipher → file runs in constant memory regardless of * artifact size, and both intermediate encodings disappear (the ciphertext * is written as raw bytes, so an artifact is now *smaller* than its tar * rather than 2.67x larger). * * On-disk format, all binary: * * magic 8 bytes "CELILOBK" * version 1 byte currently 1 * iv 16 bytes * ciphertext ... streamed * auth tag 16 bytes trailer — GCM only produces it after final() * * The tag has to be a trailer because it does not exist until the last byte * has been encrypted, and seeking back to patch a header would mean the * writer could no longer be a plain stream. Reading it costs one 16-byte * positional read before the stream starts. * * `decryptFileToFile` also reads the previous format (a JSON envelope of * base64-of-hex). The magic bytes are the discriminator: the old writer * always emitted JSON, so a file starting with `{` is legacy. The envelope's * own schemaVersion cannot serve — it lives *inside* the encrypted tar and * is unreadable until after decryption. */ import { createCipheriv, createDecipheriv, randomBytes } from 'node:crypto'; import { appendFileSync, closeSync, createReadStream, createWriteStream, openSync, readFileSync, readSync, statSync, writeFileSync, } from 'node:fs'; import { pipeline } from 'node:stream/promises'; import { decryptSecret } from '../secrets/encryption'; import { EncryptionEnvelopeSchema, parseJsonWithValidation } from '../validation/schemas'; const ALGORITHM = 'aes-256-gcm'; /** Identifies a streamed artifact. Legacy artifacts begin with `{`. */ export const ARTIFACT_MAGIC = Buffer.from('CELILOBK', 'ascii'); /** Bumped only for an incompatible layout change; readers reject unknown values. */ export const ARTIFACT_FORMAT_VERSION = 1; const IV_LENGTH = 16; const AUTH_TAG_LENGTH = 16; const HEADER_LENGTH = ARTIFACT_MAGIC.length + 1 + IV_LENGTH; /** Read `length` bytes at `offset` without opening a stream. */ function readBytesAt(path: string, offset: number, length: number): Buffer { const buffer = Buffer.alloc(length); const fd = openSync(path, 'r'); try { readSync(fd, buffer, 0, length, offset); } finally { closeSync(fd); } return buffer; } /** Whether this artifact uses the streamed format rather than the JSON envelope. */ export function isStreamedArtifact(path: string): boolean { if (statSync(path).size < ARTIFACT_MAGIC.length) return false; return readBytesAt(path, 0, ARTIFACT_MAGIC.length).equals(ARTIFACT_MAGIC); } /** * Encrypt `srcPath` to `destPath` in constant memory. */ export async function encryptFileToFile( srcPath: string, destPath: string, masterKey: Buffer, ): Promise { const iv = randomBytes(IV_LENGTH); const cipher = createCipheriv(ALGORITHM, masterKey, iv); const out = createWriteStream(destPath); out.write(Buffer.concat([ARTIFACT_MAGIC, Buffer.from([ARTIFACT_FORMAT_VERSION]), iv])); await pipeline(createReadStream(srcPath), cipher, out); // Available only once the stream has run final(), i.e. after the pipeline // resolves. Appending 16 bytes is O(1) and keeps the writer a plain stream. appendFileSync(destPath, cipher.getAuthTag()); } /** * Decrypt `srcPath` to `destPath`. Handles both the streamed format and the * legacy JSON envelope. * * Throws on a truncated artifact, an unknown format version, or a failed * authentication tag (wrong master key, or tampered/corrupted ciphertext). */ export async function decryptFileToFile( srcPath: string, destPath: string, masterKey: Buffer, ): Promise { if (!isStreamedArtifact(srcPath)) { decryptLegacyArtifact(srcPath, destPath, masterKey); return; } const size = statSync(srcPath).size; const overhead = HEADER_LENGTH + AUTH_TAG_LENGTH; if (size < overhead) { throw new Error( `Backup artifact is truncated: ${size} bytes, but the format needs at least ${overhead}.`, ); } if (size === overhead) { throw new Error('Backup artifact contains no data (header and auth tag only).'); } const header = readBytesAt(srcPath, 0, HEADER_LENGTH); const version = header[ARTIFACT_MAGIC.length]; if (version !== ARTIFACT_FORMAT_VERSION) { throw new Error( `Backup artifact uses format version ${version}, but this celilo understands ${ARTIFACT_FORMAT_VERSION}. Upgrade celilo to restore it.`, ); } const decipher = createDecipheriv( ALGORITHM, masterKey, header.subarray(ARTIFACT_MAGIC.length + 1), ); decipher.setAuthTag(readBytesAt(srcPath, size - AUTH_TAG_LENGTH, AUTH_TAG_LENGTH)); // `end` is inclusive, so the last ciphertext byte is the one before the tag. await pipeline( createReadStream(srcPath, { start: HEADER_LENGTH, end: size - AUTH_TAG_LENGTH - 1 }), decipher, createWriteStream(destPath), ); } /** * Read an artifact written before the streamed format. * * Reads the whole thing into memory, which is fine precisely because these * are the artifacts the old writer produced: it could not emit one much over * ~805 MB without dying, so the bound this function relies on is the same bug * that motivated the new format. New artifacts never take this path. */ function decryptLegacyArtifact(srcPath: string, destPath: string, masterKey: Buffer): void { const envelope = parseJsonWithValidation( readFileSync(srcPath, 'utf-8'), EncryptionEnvelopeSchema, 'backup artifact envelope', ); writeFileSync(destPath, Buffer.from(decryptSecret(envelope, masterKey), 'base64')); }