/** * Reclaiming disk from superseded build revisions. * * The registry had no delete path at all: `yank` sets a boolean in the index * and frees nothing, so every revision ever published was retained forever. * A `+N` build revision is auto-assigned on every publish, so the store grew * with every release until the disk filled and the release pipeline stopped. * * ── What the policy keeps, and why it cannot break a fetch ── * * A version is `X.Y.Z+N`: a released semver plus a build revision. The bloat * is the rebuilds, not the releases — a module embedding compiled binaries * republished twenty times carries twenty ~73 MB payloads of one release. * * So the sweep keeps the newest `keepBuildRevisions` of EVERY release and * drops the rest. No release ever disappears, which is what makes the safety * argument checkable rather than a judgement call: * * - The only version a celilo client can ask for is the one * `RegistryClient.latestVersion` picks — the last non-yanked line in the * index. There is no `--version` flag, no lockfile, and `IndexEntry.deps` * is always `[]`, so nothing pins a specific revision. * - {@link planModuleSweep} retains that line explicitly, on top of the * per-release rule, so the property holds even for an index whose lines * are out of publish order. * - Every release keeps a downloadable payload, so an operator rolling back * to an older release still can. * * ── Ordering, and why it is index-first ── * * The download route reads the payload file directly and never consults the * index. That asymmetry decides the order: * * - index line first, then payload — the window is "unlisted but still * served". Nothing 404s. A crash here leaves an orphan payload, which * wastes the disk we are reclaiming but is otherwise correct. * - payload first, then index — the window is "listed but 404s", an * actively wrong registry. A crash here leaves it that way. * * So: index first. The orphan a crash leaves behind is then reclaimed by the * NEXT run, which is what makes the sweep idempotent and safe to interrupt. * The orphan pass runs FIRST for a second reason: it is pure unlink and needs * no free space, so it can still make progress on a disk with nothing left to * write with — the state that produced this code. */ import type { IndexEntry, RegistryStorage } from './storage'; export interface SweepPolicy { /** * Build revisions to keep per released semver. Must be >= 1 — keeping zero * would delete a whole release, which this policy never does. */ keepBuildRevisions: number; } export const DEFAULT_KEEP_BUILD_REVISIONS = 1; export interface ModuleSweepPlan { name: string; /** Index entries to retain, in their original order. */ keep: IndexEntry[]; /** Versions whose index line and payload both go. */ remove: string[]; /** * Payloads on disk with no index line. Unreachable by search, and the * publisher's immutability check still sees them — `packageExists` reads the * FILESYSTEM, so an orphan makes its version unpublishable forever with no * repair path anywhere else in the product. Safe to delete, and deleting it * IS the repair. */ orphans: string[]; /** * Index lines whose payload is missing. REPORTED, NEVER REMOVED. * * This is the direction the sweep must never create, and the reason it takes * the index line before the payload. Removing the line would in fact repair * the module — a dangling entry already 404s on download, and dropping it * makes `latestVersion` resolve to a version that can actually be served. * It is left alone anyway, because a missing payload and an unreadable store * are indistinguishable from here: an unmounted volume or a wrong DATA_DIR * makes every payload look absent, and an auto-remove would then delete the * whole index. Reporting costs nothing and cannot destroy anything. */ dangling: string[]; /** * The module's index lists versions and NOT ONE has a payload on disk. That * is a broken or unmounted store rather than N independent losses, so the * sweep does nothing at all to this module — see {@link ModuleSweepPlan.dangling}. */ unreadable: boolean; } export interface SweepReport { modules: Array<{ name: string; removed: string[]; orphans: string[]; dangling: string[]; bytes: number; }>; removedCount: number; orphanCount: number; /** Index lines with no payload. Reported for a human; never acted on. */ danglingCount: number; /** Modules skipped entirely because their whole payload set is missing. */ unreadable: string[]; reclaimedBytes: number; dryRun: boolean; } /** * Split `X.Y.Z+N` (or `E:X.Y.Z+N`) into the release it belongs to and its * build revision. Returns null for anything that does not parse — the caller * treats that as "keep", because a version the sweep cannot reason about is * not one it should delete. */ export function parseRevision(vers: string): { release: string; revision: number } | null { const plus = vers.lastIndexOf('+'); if (plus <= 0) return null; const release = vers.slice(0, plus); const revision = Number(vers.slice(plus + 1)); if (!Number.isInteger(revision) || revision < 0) return null; return { release, revision }; } /** * Decide what one module keeps. Pure — no filesystem, no storage. * * `storedVersions` is what is on DISK, which is deliberately not the same set * as `entries`: a previous interrupted run can leave a payload whose index * line is already gone, and reclaiming it is the whole reason this takes both. */ export function planModuleSweep( name: string, entries: IndexEntry[], storedVersions: string[], policy: SweepPolicy, ): ModuleSweepPlan { const keepBuildRevisions = Math.max(1, Math.trunc(policy.keepBuildRevisions)); const stored = new Set(storedVersions); const dangling = entries.filter((e) => !stored.has(e.vers)).map((e) => e.vers); // Every listed version missing its payload is one broken store, not N // independent losses. Touch nothing. if (entries.length > 0 && dangling.length === entries.length) { return { name, keep: entries, remove: [], orphans: [], dangling, unreadable: true }; } /** * Retention is computed over the SERVABLE entries only — the ones whose * payload is actually there. * * Doing it over every index line is subtly wrong in a way that bites exactly * when the store is already damaged. If a module's NEWEST line is dangling, * that line satisfies both the per-release rule and the latest-line rule, so * every older version — the servable ones — becomes "superseded" and is * swept. The module is then listed, has one index line, and cannot be * downloaded at all. The sweep would not have created the dangling entry, * but it would have removed the last thing that still worked. */ const servable = entries.filter((e) => stored.has(e.vers)); const retained = new Set(); // The newest `keepBuildRevisions` of each release. const byRelease = new Map>(); for (const entry of servable) { const parsed = parseRevision(entry.vers); if (!parsed) { // Unparseable — never a publish this server accepted, so it is hand- // written state. Keep it and move on. retained.add(entry.vers); continue; } const group = byRelease.get(parsed.release) ?? []; group.push({ vers: entry.vers, revision: parsed.revision }); byRelease.set(parsed.release, group); } for (const group of byRelease.values()) { group.sort((a, b) => b.revision - a.revision); for (const { vers } of group.slice(0, keepBuildRevisions)) retained.add(vers); } // What every client actually resolves to, retained explicitly rather than // inferred from the rule above. The rule already covers it for an index in // publish order; stating it means a disordered or hand-edited index cannot // turn into an unfetchable module. const lastServable = servable.at(-1); if (lastServable) retained.add(lastServable.vers); const lastUnyanked = [...servable].reverse().find((e) => !e.yanked); if (lastUnyanked) retained.add(lastUnyanked.vers); // Only ever removes something it can see. A dangling line is left in the // index untouched and reported instead. const remove = servable.filter((e) => !retained.has(e.vers)).map((e) => e.vers); const removing = new Set(remove); const indexed = new Set(entries.map((e) => e.vers)); return { name, keep: entries.filter((e) => !removing.has(e.vers)), remove, orphans: storedVersions.filter((v) => !indexed.has(v)), dangling, unreadable: false, }; } /** * Apply one module's plan. Orphans first (needs no free space), then the * index rewrite, then the payloads the rewrite just unlisted. */ function applyModuleSweep( storage: RegistryStorage, plan: ModuleSweepPlan, dryRun: boolean, ): number { let bytes = 0; for (const version of plan.orphans) { bytes += storage.packageSize(plan.name, version); if (!dryRun) storage.removePackage(plan.name, version); } for (const version of plan.remove) { bytes += storage.packageSize(plan.name, version); } if (plan.remove.length > 0 && !dryRun) { storage.updateIndex(plan.name, plan.keep); for (const version of plan.remove) storage.removePackage(plan.name, version); } return bytes; } /** Sweep every module in the store. */ export function sweep(storage: RegistryStorage, policy: SweepPolicy, dryRun = false): SweepReport { const modules: SweepReport['modules'] = []; let removedCount = 0; let orphanCount = 0; let reclaimedBytes = 0; const unreadable: string[] = []; let danglingCount = 0; for (const name of storage.storedNames()) { const plan = planModuleSweep( name, storage.readIndex(name), storage.storedVersions(name), policy, ); if (plan.unreadable) { unreadable.push(name); continue; } danglingCount += plan.dangling.length; if (plan.remove.length === 0 && plan.orphans.length === 0 && plan.dangling.length === 0) { continue; } const bytes = applyModuleSweep(storage, plan, dryRun); modules.push({ name, removed: plan.remove, orphans: plan.orphans, dangling: plan.dangling, bytes, }); removedCount += plan.remove.length; orphanCount += plan.orphans.length; reclaimedBytes += bytes; } return { modules, removedCount, orphanCount, danglingCount, unreadable, reclaimedBytes, dryRun, }; }