import type { FilesPlugin } from "../index.js"; export interface DedupOptions { /** * Where the content-addressed blobs live, as a key prefix. Defaults to * `".dedup"`. The bytes of `photos/a.jpg` are stored once at * `".dedup/"`, and the logical key holds a small pointer to it. * Objects under this prefix are hidden from `list()` (unless you list within * it) and are never themselves de-duplicated. Writes into it through the * instance (`upload`, `signedUploadUrl`, or a `copy` / `move` destination) * and a `move` out of it are rejected, so a caller can't overwrite or * remove the content every pointer resolves to, and the `files-sdk/api` * gateway refuses client keys inside it. */ prefix?: string; } /** * Content-address object bodies so identical content is stored only once. On * `upload` the body is hashed (SHA-256); the bytes are written a single time to * a content-addressed blob under a store prefix (`.dedup/` by default), and the * logical key holds a tiny pointer (an empty object whose `metadata` records the * hash). Re-uploading content already in the store **skips the byte upload** and * just writes the pointer — and because the pointer is what `copy` / `move` * relocate, copying a de-duplicated file is near-free and shares the same blob. * * Reads are transparent: `download` follows the pointer to the blob (ranges * included — blobs are stored verbatim), and `head` / `list` report the logical * size with the internal fields stripped, all for `upload([...])` / * `download([...])` bulk calls too. `head` / `list` return metadata only, * read from the pointer without fetching the blob. The reported `etag` is the * content hash (the pointer's own ETag is the same for every key), so it * changes exactly when the content does — `sync()`'s default etag comparison * and `versioning()` ids stay correct. Objects without this plugin's marker * (pre-existing or written elsewhere) pass straight through, so it's safe to * enable on a mixed bucket. * * Provider-agnostic: it uses only the Web Crypto API (no native deps) and the * `metadata` the SDK already round-trips, so it works on any adapter that * supports metadata. De-duplication is most effective on the **outside** of the * array, before any body-transforming plugin — encrypted bytes don't de-dup * (a random per-object key makes identical inputs encrypt differently), so place * it first: `plugins: [dedup(), compression(), encryption(key)]`. * * Trade-offs, by design: * - **Buffers the whole body** to hash it, so it's unsuitable for very large or * unknown-length streams. `multipart` / `control` apply to the blob write * only (the pointer is always a single empty write); when the content is * already stored, no bytes are sent and a `control` is left undriven. * - **Reads cost a second fetch** — the pointer, then the blob (a ranged read * does a `head` first). `head` / `list` add nothing; they read the pointer. * On adapters whose `list()` returns no metadata (S3 and the * S3-compatibles), `list()` items can't be recognized as pointers, so they * report the pointer's own size (`0`) and ETag; a `download()` still follows * the pointer to the content. * - **`url()` / `signedUploadUrl()` throw** — a presigned GET would hand out the * empty pointer, and a presigned PUT would bypass content-addressing. Download * through the instance instead. `files.capabilities` reports * `signedUrl.supported` and `signedUpload.supported` as `false` to match, so * the `files-sdk/api` gateway proxies uploads and downloads through the * instance. * - **Blobs aren't garbage-collected.** `delete` (and overwrite) drop the * pointer but leave the content addressed, so it's reused if the content * reappears; reclaim unreferenced blobs with a storage lifecycle rule or a * periodic sweep. Deleting a blob that a pointer still references makes * that pointer's reads fail with `NotFound`. * - **The store is write-protected through the instance.** `upload`, * `signedUploadUrl`, and `copy` / `move` into the store prefix throw, as * does a `move` out of it, so no caller can overwrite or relocate * `.dedup/` and change what every pointer to it returns. The * `files-sdk/api` gateway refuses client keys and list prefixes inside the * store altogether. Blob reads don't re-verify the hash (that would buffer * every download), so anyone with raw provider write access to the store * can still substitute content — lock that down like the rest of the bucket. * - **Conditional operations throw** — a pointer's native ETag is identical * for every key, so no provider compare-and-set can guard its content. * * @param options optional `{ prefix }` — where blobs are stored. * @example * ```ts * import { createFiles } from "files-sdk"; * import { s3 } from "files-sdk/s3"; * import { dedup } from "files-sdk/dedup"; * * const files = createFiles({ * adapter: s3({ bucket: "uploads" }), * plugins: [dedup()], * }); * * await files.upload("a.png", bytes); * await files.upload("b.png", bytes); // same content — no second byte upload * await files.copy("a.png", "c.png"); // shares the one stored blob * ``` */ export declare const dedup: (options?: DedupOptions) => FilesPlugin; //# sourceMappingURL=index.d.ts.map