import { Color, Object3D } from "three";

import { computeElementId, occurrenceOfName, resolveInCollection, resolveInGraph, type AddressableElement, type GraphNode, type NeedleAddress } from "@needle-tools/addressing";

import { removeComponent } from "../engine_components.js";
import { editorGuidKeyName } from "../engine_constants.js";
import { destroy, instantiate, setActive } from "../engine_gameobject.js";
import { createComponentFromSerializedData } from "../engine_gltf_builtin_components.js";
import { originalComponentNameKey } from "../engine_instantiate_resolve.js";
import { deriveStableId } from "../engine_networking_instantiate.js";
import { parseSync } from "../engine_loaders.js";
import { loadPMREM } from "../engine_pmrem.js";
import { deserializeObject } from "../engine_serialization.js";
import { ImplementationInformation, type ISerializable, SerializationContext } from "../engine_serialization_core.js";
import { Context } from "../engine_setup.js";
import type { IComponent, Model, UIDProvider } from "../engine_types.js";
import { TypeStore } from "../engine_typestore.js";

/**
 * Needle project document — evaluator for the composition format
 * (needle-editor plans/project-format-v0.md).
 *
 * A project document is COMPOSITION ONLY: it references glTF assets, places them,
 * adds components (defs) and patches properties (overs) on the loaded content.
 * Content itself always lives in glTF. Evaluation is ONE-SHOT and happens before
 * the component lifecycle starts: loaded components exist dormant until the engine's
 * pre-setup phase, so `active`/`enabled` opinions apply before anything ever executes.
 */

// ---------------------------------------------------------------------------
// document types (the wire format — see plans/project-format-v0.md)
// ---------------------------------------------------------------------------

/** the version discriminator key every project document must carry */
export const PROJECT_DOCUMENT_KEY = "needle:project";

export type ProjectDocumentAsset = {
    /** where the asset lives — resolved against `EvaluateProjectDocumentOptions.baseUrl` */
    src: string;
    /**
     * Optional PROVIDER IDENTITY for assets whose resources (textures, buffers) live at
     * provider-specific urls that `src` alone can't reach (e.g. PolyHaven). The host recovers
     * the provider's resolver from this identity at load time (see
     * `EvaluateProjectDocumentOptions.resolveAssetUrl`). Self-contained assets omit it and
     * load by `src`. The engine ignores it; only the host (editor) reads it.
     */
    asset?: { id: string; provider?: string };
    /**
     * Optional SEMANTIC KIND, decided at USE time by the writer ("model", "material",
     * "hdr", "texture", "image", "video", "audio", "code", "text") — the same file can
     * be a texture on a material slot or a plain image in UI, and downstream
     * processing (compression at publish) follows the use, not the extension. The
     * engine ignores it; only hosts/tooling read it.
     */
    kind?: string;
};

export type ProjectDocumentReference = {
    /** project-local id of this reference — the resolution scope (or entity handle) for targets */
    id: string;
    /** key into the document's `assets` map */
    asset: string;
    /**
     * ELEMENT REFERENCE discriminator: when present, this reference instantiates the
     * addressed element INSIDE the asset (e.g. a material) into the composition's ENTITY
     * POOL instead of placing a scene subtree. Entity references are first-class composed
     * entities: overs can target them (`{ ref: <id>, path: "map/repeat" }`) and values can
     * point at them (`{ ref: <id> }` — e.g. a mesh's material assignment). N consumers of
     * the same entity share ONE runtime instance by construction (USD: a Material-prim
     * reference; assignment ≈ material:binding).
     */
    address?: NeedleAddress;
    /** mount the placed root under a node of another reference; scene references only.
     * NOTE: placement TRS is NOT a reference field — it is ordinary overs on the root
     * (`{ ref, path: "position" }`), like every other property (USD: xformOps on the
     * referencing prim are plain authored attributes). One mechanism, no precedence
     * ambiguity between a transform field and a position over. */
    parent?: { ref: string; address: NeedleAddress };
};

export type ProjectDocumentTarget = {
    /** which placed reference the address resolves WITHIN; omitted → the composition root */
    ref?: string;
    /** fat address of the element inside the reference (needle-id / anchor / namePath) */
    address?: NeedleAddress;
    /**
     * Property path within the resolved element, "/"-separated.
     *
     * A segment landing on a COLLECTION may be a `<field>:<value>` qualifier instead of an
     * index — `components/type:DragControls/enabled`, `components/guid:6f1f…/enabled` —
     * so a document never depends on array order (`userData.components` reorders whenever
     * a component is added or removed). A bare numeric segment on an array is still an
     * index, but it is now an explicit opt-in rather than the only option.
     * `components` is a synthesized segment on a node: the live layout
     * (`userData.components`) stays an implementation detail.
     *
     * Reserved LAST segment: `active` — with `value: false` it removes whatever the rest
     * of the path resolved to (the node itself, or a component) from the composed result.
     */
    path?: string;
};

export type ProjectDocumentDef = {
    target: ProjectDocumentTarget;
    /**
     * When present, the def CREATES a new node (USD-proper: def defines prims) under the
     * resolved target (composition root when the target is empty) and the components
     * apply to IT — e.g. the runtime start camera. Without `create`, components apply to
     * the resolved existing node.
     */
    create?: { name?: string; position?: number[]; quaternion?: number[]; scale?: number[] };
    /** VERBATIM NEEDLE_components wire entries ({ name, guid, ...fields }) */
    components: Array<{ name: string } & Record<string, unknown>>;
};

export type ProjectDocumentOver = {
    target: ProjectDocumentTarget;
    value: unknown;
};

export type ProjectDocument = {
    [PROJECT_DOCUMENT_KEY]: string;
    /** layer metadata (USD's "layer metadata" concept — e.g. `name`); free-form */
    metadata?: Record<string, unknown>;
    assets?: Record<string, ProjectDocumentAsset>;
    references?: ProjectDocumentReference[];
    defs?: ProjectDocumentDef[];
    overs?: ProjectDocumentOver[];
    /** unknown arcs are ignored by evaluation and must be preserved by writers */
    [arc: string]: unknown;
};

/** structural check for the version discriminator — the opener/sniffing entry point */
export function isProjectDocument(value: unknown): value is ProjectDocument {
    return !!value && typeof value === "object" && typeof (value as Record<string, unknown>)[PROJECT_DOCUMENT_KEY] === "string";
}

// ---------------------------------------------------------------------------
// evaluation
// ---------------------------------------------------------------------------

export type ProjectEvaluationIssue = {
    arc: "reference" | "def" | "over";
    message: string;
};

export type ProjectEvaluationResult = {
    /** the object all non-mounted references were added to */
    root: Object3D;
    /** placed reference roots by reference id.
     * POPULATED AT `completed`: references are INSTANTIATED from a parse-once template
     * in the engine's pre-setup phase (after the template's component deserialization),
     * so the map is empty until the context ticked once — await `completed` first. */
    references: Map<string, Object3D>;
    /** per-reference Model views by reference id (`scene` is that reference's placed
     * INSTANCE; parser etc. come from the shared template). Populated at `completed`,
     * like `references`. */
    models: Map<string, Model>;
    /**
     * misses and errors — an issue never aborts evaluation, the arc is skipped.
     * Populated up to `completed` (defs/overs apply in the engine's pre-setup phase).
     */
    issues: ProjectEvaluationIssue[];
    /**
     * Resolves once defs/overs have been applied. They run in the engine's pre-setup
     * phase (AFTER the assets' own component deserialization, BEFORE awake) — the
     * context must tick once (held or running) for this to happen.
     */
    completed: Promise<void>;
};

export type EvaluateProjectDocumentOptions = {
    /** where references are placed; defaults to `context.scene` */
    parent?: Object3D;
    /** resolves `assets.src` for the document — tests/tools provide in-memory data here */
    resolveAsset?: (assetId: string, src: string) => Promise<ArrayBuffer | string>;
    /** base for resolving relative `assets.src` urls with the default (fetch) resolver */
    baseUrl?: string;
    /**
     * Rewrites a resource url (texture / external buffer) encountered while parsing an
     * asset — the asset service's `resolveUrl` re-entering at load time. Needed for provider
     * assets (e.g. PolyHaven) whose textures live at urls that plain relative resolution
     * against `assets.src` cannot reach. Applied on an isolated per-parse manager (see
     * {@link parseSync}); the host (editor opener) supplies it from `IAssetsService` so this
     * evaluator carries NO provider knowledge.
     */
    resolveAssetUrl?: (assetId: string, url: string) => string;
    /**
     * AGGREGATE download progress across the document's assets: `loaded` bytes summed
     * over all asset downloads, `total` summed over the known Content-Lengths (0 while
     * nothing reported a length). Fired per received chunk — the loader chain adapts it
     * to the ProgressEvent shape the loading UI expects.
     */
    onProgress?: (loaded: number, total: number) => void;
};

const documentImplementationInformation = new ImplementationInformation();

/**
 * Evaluate a project document into the given context. One-shot; never throws for
 * content issues (they are collected in the result), only for a non-document input.
 *
 * Order (plans/project-format-v0.md): load + place references (document order) →
 * mounts → [pre-setup phase:] asset component deserialization (the engine's own
 * queue, FIFO) → defs → overs. Resolution is scoped to the target reference's
 * subtree; misses are reported and skipped (exact resolution only in v0).
 */
export async function evaluateProjectDocument(context: Context, document: ProjectDocument, options?: EvaluateProjectDocumentOptions): Promise<ProjectEvaluationResult> {
    if (!isProjectDocument(document)) {
        throw new TypeError(`evaluateProjectDocument: not a project document (missing "${PROJECT_DOCUMENT_KEY}")`);
    }

    const root = options?.parent ?? (context.scene as unknown as Object3D);
    const references = new Map<string, Object3D>();
    const models = new Map<string, Model>();
    const issues: ProjectEvaluationIssue[] = [];
    const issue = (arc: ProjectEvaluationIssue["arc"], message: string) => { issues.push({ arc, message }); };

    // Per-asset parse options: threads the host's resolveAssetUrl (asset-service resolveUrl)
    // to parseSync so an asset's resources resolve at load time. Undefined when no rewrite is
    // provided, so parseSync keeps its plain relative resolution.
    const parseOptionsFor = (assetId: string) =>
        options?.resolveAssetUrl ? { resolveUrl: (url: string) => options.resolveAssetUrl!(assetId, url) } : undefined;

    // ---- references: load assets (cached per asset id), place roots ----
    // aggregate download progress across all asset fetches: every chunk updates the
    // asset's {loaded, total} slot and reports the SUMS — one bar over the whole set
    const progressByAsset = new Map<string, { loaded: number; total: number }>();
    const reportProgress = () => {
        if (!options?.onProgress) return;
        let loaded = 0, total = 0;
        for (const entry of progressByAsset.values()) {
            loaded += entry.loaded;
            total += entry.total;
        }
        options.onProgress(loaded, total);
    };

    const assetData = new Map<string, Promise<ArrayBuffer | string | null>>();
    const loadAssetData = (assetId: string): Promise<ArrayBuffer | string | null> => {
        let pending = assetData.get(assetId);
        if (!pending) {
            const src = document.assets?.[assetId]?.src;
            if (typeof src !== "string") {
                pending = Promise.resolve(null);
            }
            else if (options?.resolveAsset) {
                pending = options.resolveAsset(assetId, src).catch(err => {
                    issue("reference", `asset "${assetId}" failed to resolve: ${err}`);
                    return null;
                });
            }
            else {
                const url = options?.baseUrl ? new URL(src, options.baseUrl).href : src;
                pending = fetch(url).then(res => {
                    if (!res.ok) throw new Error(`${res.status} ${res.statusText}`);
                    // SPA/dev servers answer MISSING files with 200 + their index.html —
                    // without this check that surfaces later as a cryptic gltf parse
                    // error ("Unexpected token '<' … is not valid JSON")
                    const contentType = res.headers.get("Content-Type") ?? "";
                    if (contentType.includes("text/html")) {
                        throw new Error(`the server answered with an HTML page (Content-Type: ${contentType}) — the file probably does not exist there. Documents record srcs AS SAVED: a dev-server url (e.g. localhost) does not exist on a deployment.`);
                    }
                    // stream so the download reports byte progress; fall back to a
                    // plain buffer read when the body is not streamable
                    if (!res.body || !options?.onProgress) return res.arrayBuffer();
                    const slot = { loaded: 0, total: Number(res.headers.get("Content-Length")) || 0 };
                    progressByAsset.set(assetId, slot);
                    reportProgress();
                    return readBodyWithProgress(res.body, received => {
                        slot.loaded = received;
                        // servers without Content-Length: keep total >= loaded so the
                        // bar never exceeds 100% (it jumps to done at the end)
                        if (slot.total < received) slot.total = received;
                        reportProgress();
                    });
                }).catch(err => {
                    issue("reference", `asset "${assetId}" failed to load from ${url}: ${err}`);
                    return null;
                });
            }
            assetData.set(assetId, pending);
        }
        return pending;
    };

    // parse-once cache: EVERY consumer — placed references, entity references, assets
    // loaded on demand by overs — shares one parsed TEMPLATE per asset id. Placed
    // references then INSTANTIATE per-reference copies from it (shared geometry/
    // materials/textures, per-copy stable guids).
    const parsedAssets = new Map<string, Promise<Model | null>>();
    const loadParsedAsset = (assetId: string, arc: ProjectEvaluationIssue["arc"] = "over"): Promise<Model | null> => {
        let pending = parsedAssets.get(assetId);
        if (!pending) {
            pending = (async () => {
                // three DISTINGUISHABLE failure modes — an opaque "failed to load" cost a
                // debugging session (missing assets-map entry read as a network problem)
                if (!document.assets?.[assetId]) {
                    issue(arc, `asset "${assetId}" has no entry in the document's assets map`);
                    return null;
                }
                const data = await loadAssetData(assetId);
                if (data === null) return null; // fetch failure already reported by loadAssetData
                const src = document.assets[assetId].src ?? assetId;
                const model = (await parseSync(context, data, src, null, parseOptionsFor(assetId))) ?? null;
                if (!model) issue(arc, `asset "${assetId}" data loaded but PARSING failed (see console for the parser error)`);
                if (model) {
                    stampParsedMaterials(model);
                    // Park the template: a DETACHED scene still counts as active hierarchy
                    // (engine-test-pinned), so its queued components would AWAKE on the next
                    // pump in a running context. Deactivate in the same synchronous turn as
                    // the parse resolution — before any frame can run them. Copies are
                    // instantiated INTO the parked template (no awake) and re-activated
                    // after reparenting; the template itself is destroyed once all
                    // references are instantiated.
                    setActive(model.scene as unknown as Object3D, false);
                }
                return model;
            })();
            parsedAssets.set(assetId, pending);
        }
        return pending;
    };

    // ---- ENTITY references (with an address): instantiate into the entity pool -----
    // Not placed in the graph. One promise per id — every consumer (assign values,
    // entity-targeted overs) chains on the same instantiation, so N users share ONE
    // runtime instance by construction and assign/patch ordering stops being temporal.
    const entities = new Map<string, Promise<object | null>>();
    for (const ref of document.references ?? []) {
        if (!ref || typeof ref.id !== "string" || typeof ref.asset !== "string" || !ref.address) continue;
        if (entities.has(ref.id)) {
            issue("reference", `duplicate reference id "${ref.id}" — entry skipped`);
            continue;
        }
        const address = ref.address;
        entities.set(ref.id, (async (): Promise<object | null> => {
            if (address.type === "material") {
                const model = await loadParsedAsset(ref.asset);
                if (!model) return null; // load/parse failure already reported
                const material = findMaterialInModel(model, address);
                if (!material) issue("reference", `entity "${ref.id}": material did not resolve in asset "${ref.asset}" (${JSON.stringify(address)})`);
                return material;
            }
            if (address.type === "hdr") {
                // WHOLE-ASSET entity (address is only a type): the asset itself,
                // instantiated as an environment texture (PMREM — hdr/exr/ktx2)
                const src = document.assets?.[ref.asset]?.src;
                if (typeof src !== "string") {
                    issue("reference", `entity "${ref.id}": unknown asset "${ref.asset}"`);
                    return null;
                }
                const texture = await loadPMREM(src, context.renderer);
                if (!texture) issue("reference", `entity "${ref.id}": environment texture failed to load from ${src}`);
                return texture;
            }
            issue("reference", `entity "${ref.id}": element type "${address.type ?? "?"}" is not instantiable yet`);
            return null;
        })());
    }

    // Placed references: parse ONCE per asset (template, via loadParsedAsset), then
    // INSTANTIATE per reference — deferred to the pre-setup phase so clones copy FULLY
    // DESERIALIZED template components (the loader's queued deserialization runs first,
    // FIFO). Each copy gets STABLE, ORDER-INDEPENDENT guids derived from the reference
    // id carried in the document: deriveStableId(ref.id, originalFileGuid) — same
    // guids on every client and load (engine-global lookups, networking ownership).
    const pendingPlacements: Array<{ ref: ProjectDocumentReference; model: Model }> = [];
    const pendingIds = new Set<string>();
    for (const ref of document.references ?? []) {
        if (!ref || typeof ref.id !== "string" || typeof ref.asset !== "string") {
            issue("reference", `invalid reference entry: ${JSON.stringify(ref)}`);
            continue;
        }
        if (ref.address) continue; // entity reference — pooled above, never placed
        if (pendingIds.has(ref.id)) {
            issue("reference", `duplicate reference id "${ref.id}" — entry skipped`);
            continue;
        }
        const model = await loadParsedAsset(ref.asset, "reference");
        if (!model) {
            // load/parse failure reported by loadParsedAsset (unknown asset, fetch, parser)
            continue;
        }
        if (!model.scene) {
            issue("reference", `reference "${ref.id}": asset "${ref.asset}" did not produce a scene`);
            continue;
        }
        pendingIds.add(ref.id);
        pendingPlacements.push({ ref, model });
    }

    // ---- instantiate + mounts + defs + overs: the engine's pre-setup phase ---
    // The templates' own components deserialize in pre-setup callbacks the loader
    // queued during parseSync (FIFO) — queuing AFTER the loads guarantees this
    // callback sees fully deserialized templates, and still runs strictly before awake.
    const completed = new Promise<void>((resolve) => {
        context.new_scripts_pre_setup_callbacks.push(() => {
            // instantiate ALL copies FIRST, attach + activate afterwards: each
            // instantiate() pumps processNewScripts, which awakens ACTIVE components in
            // a running context — an already-activated copy A would awake mid-callback
            // (before defs/overs) while copy B instantiates. All copies stay parked
            // (inherited from the deactivated template) until the last pump inside this
            // callback has happened; awake then follows at the engine's next natural
            // pump, strictly after the document opinions below.
            const placedCopies: Object3D[] = [];
            for (const { ref, model } of pendingPlacements) {
                const template = model.scene as unknown as Object3D;
                const provider: UIDProvider = {
                    seed: 0,
                    // elements without an original id fall back to a per-reference
                    // sequence (deterministic for an unchanged template traversal)
                    generateUUID: () => deriveStableId(ref.id, `seq:${provider.seed++}`),
                    remapUUID: (originalId: string) => deriveStableId(ref.id, originalId),
                };
                const placed = instantiate(template as never, { idProvider: provider, context: context as never } as never) as unknown as Object3D | null;
                if (!placed) {
                    issue("reference", `reference "${ref.id}": instantiating asset "${ref.asset}" failed`);
                    continue;
                }
                references.set(ref.id, placed);
                // per-reference Model view: parser/animations from the shared template,
                // scene = THIS reference's instance (what editor open-file tracking needs)
                models.set(ref.id, { ...model, scene: placed } as unknown as Model);
                // Keep parser.associations valid for the copy: the loader keyed the
                // TEMPLATE's objects, but tooling (editor source-asset attribution)
                // looks up the PLACED instances. Materials/meshes/textures are shared
                // with the template, so their keys stay valid — only Object3D keys
                // need mirroring. instantiate preserves child order, so a parallel
                // walk pairs template nodes with their copies.
                const associations = (model as unknown as { parser?: { associations?: Map<object, unknown> } }).parser?.associations;
                if (associations) {
                    const mirror = (a: Object3D, b: Object3D) => {
                        const record = associations.get(a);
                        if (record !== undefined) associations.set(b, record);
                        const count = Math.min(a.children.length, b.children.length);
                        for (let i = 0; i < count; i++) mirror(a.children[i], b.children[i]);
                    };
                    mirror(template, placed);
                }
                placedCopies.push(placed);
            }
            for (const placed of placedCopies) {
                root.add(placed);
                // undo only the park flag inherited from the template root — authored
                // visibility lives on deeper nodes and is preserved by the clone
                setActive(placed, true);
            }

            // mounts (after all placements so forward references work)
            // placement TRS is ordinary overs on the placed root — no transform pass here
            for (const ref of document.references ?? []) {
                const placed = ref?.id ? references.get(ref.id) : undefined;
                if (!placed) continue;
                if (ref.parent) {
                    const host = references.get(ref.parent.ref);
                    if (!host) {
                        issue("reference", `reference "${ref.id}": parent ref "${ref.parent.ref}" not placed — staying at the composition root`);
                    }
                    else {
                        const mount = resolveInGraph(ref.parent.address ?? {}, host as unknown as GraphNode) as unknown as Object3D | null;
                        if (!mount) {
                            issue("reference", `reference "${ref.id}": parent address did not resolve in "${ref.parent.ref}" — staying at the composition root`);
                        }
                        else {
                            mount.add(placed);
                        }
                    }
                }
            }

            // the placement templates served their purpose — destroy them so their
            // dormant registered components are cleanly unregistered. dispose=false:
            // geometries/materials/textures are SHARED with the instantiated copies.
            const destroyedTemplates = new Set<object>();
            for (const { model } of pendingPlacements) {
                if (model.scene && !destroyedTemplates.has(model.scene)) {
                    destroyedTemplates.add(model.scene);
                    destroy(model.scene as never, true, false);
                }
            }

            // sync opinions (defs, property/active overs) apply strictly before awake;
            // async ones (reference-valued material overs → load the material asset) run
            // after and settle `completed`. applyOvers does its sync pass before the
            // first await, so ordering for non-async overs is preserved.
            applyDefs(context, document, root, references, issue);
            applyOvers(context, document, root, references, entities, issue).finally(() => resolve());
        });
    });

    return { root, references, models, issues, completed };
}

// ---------------------------------------------------------------------------
// target resolution
// ---------------------------------------------------------------------------

function resolveTargetObject(target: ProjectDocumentTarget | undefined, root: Object3D, references: Map<string, Object3D>): Object3D | null {
    const scope = target?.ref !== undefined ? references.get(target.ref) : root;
    if (!scope) return null;
    if (!target?.address) return scope;
    return resolveInGraph(target.address, scope as unknown as GraphNode) as unknown as Object3D | null;
}

function getComponents(obj: Object3D): IComponent[] {
    const components = (obj as unknown as { userData?: { components?: IComponent[] } }).userData?.components;
    return Array.isArray(components) ? components : [];
}

// ---------------------------------------------------------------------------
// property paths
// ---------------------------------------------------------------------------

/** `<field>:<value>` — matches an element of a collection by one of its fields */
const QUALIFIER = /^([A-Za-z_][A-Za-z0-9_-]*):([\s\S]*)$/;

export function splitPath(path: string): string[] {
    // "~1" → "/" and "~0" → "~" (RFC 6901 escaping, unescaped in that order)
    return path.split("/").filter(s => s.length > 0).map(s => s.replace(/~1/g, "/").replace(/~0/g, "~"));
}

/**
 * Advance one path segment. Objects take the segment as a literal key; COLLECTIONS take
 * either a numeric index or a `<field>:<value>` qualifier. Qualifier parsing is confined
 * to collections, so a plain object property whose name contains ":" is never mistaken
 * for one.
 */
function stepPath(current: unknown, segment: string, refScope: string | undefined, report: (message: string) => void): unknown {
    // synthesized: a node's components, without exposing the userData layout
    if ((current as Object3D | undefined)?.isObject3D === true && segment === "components") {
        return getComponents(current as Object3D);
    }
    if (!Array.isArray(current)) {
        return (current as Record<string, unknown> | undefined)?.[segment];
    }
    if (/^\d+$/.test(segment)) return current[Number(segment)];

    const qualifier = QUALIFIER.exec(segment);
    if (!qualifier) {
        report(`"${segment}" is neither an index nor a <field>:<value> qualifier`);
        return undefined;
    }
    const [, field, value] = qualifier;

    // a component's readable identity is its TYPE, and runtime constructor names are
    // decorator-wrapped ("_DragControls") — match the original name too
    const matches = current.filter(element => {
        if (field === "type") {
            return element?.constructor?.name === value || (element as Record<symbol, unknown>)?.[originalComponentNameKey] === value;
        }
        if (field === "guid") {
            // a component's LIVE guid is regenerated per load (construction) and per
            // copy (instantiate remap) — the FILE's guid, which is what documents
            // record, survives on the instance under editorGuidKeyName (deserialization
            // stores it, instantiate copies it). Match both.
            const el = element as Record<string, unknown> | undefined;
            return el?.guid === value || el?.[editorGuidKeyName] === value;
        }
        return (element as Record<string, unknown> | undefined)?.[field] === value;
    });

    if (matches.length === 1) return matches[0];
    if (matches.length > 1) {
        report(`"${segment}" matched ${matches.length} elements — the document must name one unambiguously`);
        return undefined;
    }
    // instantiated copies carry DERIVED guids (deriveStableId(ref.id, fileGuid)), so a
    // guid authored against the FILE (older documents, cross-session captures) resolves
    // through the same derivation — the derivation function IS the old→new map.
    if (field === "guid" && refScope) {
        const derived = deriveStableId(refScope, value);
        return current.find(element => (element as { guid?: string } | undefined)?.guid === derived);
    }
    return undefined;
}

/** walk every segment but the last; returns the container holding the final key */
function walkPath(element: object, segments: string[], refScope: string | undefined, report: (message: string) => void): { container: object; chain: object[] } | null {
    const chain: object[] = [element];
    let current: unknown = element;
    for (let i = 0; i < segments.length - 1; i++) {
        const next = stepPath(current, segments[i], refScope, message => report(`at "${segments[i]}": ${message}`));
        if (!next || typeof next !== "object") {
            report(`does not resolve at "${segments[i]}"`);
            return null;
        }
        current = next;
        chain.push(next as object);
    }
    return { container: current as object, chain };
}

/** a live three material, seen through the addressing package's duck type */
type MaterialElement = AddressableElement & Record<string, unknown>;

/** every distinct material used anywhere under `root`, in traversal order */
function collectMaterials(root: Object3D): MaterialElement[] {
    const materials: MaterialElement[] = [];
    root.traverse(obj => {
        const material = (obj as unknown as { material?: unknown }).material;
        for (const m of Array.isArray(material) ? material : material ? [material] : []) {
            if (m && typeof m === "object" && !materials.includes(m as MaterialElement)) {
                materials.push(m as MaterialElement);
            }
        }
    });
    return materials;
}

/** materials are not scene-graph nodes — resolve them against the meshes of the scope
 *  (plus any subtrees deactivation detached from it: the material resource outlives
 *  its consumer's presence in the composed result) */
function resolveTargetMaterial(scope: Object3D, address: NeedleAddress, detachedRoots?: readonly Object3D[]): { material: Record<string, unknown> } | null {
    // ONE ladder for every element type (anchor → id → name), from the addressing
    // package — materials used to carry their own, which is why anchoring a material
    // had no effect and why the two material resolvers here disagreed about whether a
    // name match had to be unique.
    const materials = collectMaterials(scope);
    for (const detached of detachedRoots ?? []) {
        for (const material of collectMaterials(detached)) {
            if (!materials.includes(material)) materials.push(material);
        }
    }
    const material = resolveInCollection(address, materials);
    return material ? { material: material as Record<string, unknown> } : null;
}

// ---------------------------------------------------------------------------
// defs
// ---------------------------------------------------------------------------

function applyDefs(context: Context, document: ProjectDocument, root: Object3D, references: Map<string, Object3D>, issue: (arc: "def", message: string) => void) {
    const defs = document.defs ?? [];
    if (!defs.length) return;

    // one serialization context rooted at the COMPOSITION root: {guid} references in
    // def data resolve across the whole composed result (cross-reference wiring)
    const serializationContext = new SerializationContext(root);
    serializationContext.context = context;
    serializationContext.implementationInformation = documentImplementationInformation;

    for (const def of defs) {
        let obj = resolveTargetObject(def?.target, root, references);
        if (!obj) {
            issue("def", `def target did not resolve: ${JSON.stringify(def?.target)}`);
            continue;
        }
        if (def.create) {
            // the def DEFINES a node under the resolved parent (USD-proper def). When a
            // child with the same name ALREADY exists it is ADOPTED (transform applied,
            // components deserialize onto matching existing instances below) — a def
            // over an existing prim composes onto it. This is how the start-camera def
            // takes over the engine's own camera instead of creating a twin.
            const existing = typeof def.create.name === "string"
                ? (obj.children as Object3D[] | undefined)?.find(c => c?.name === def.create!.name)
                : undefined;
            const node = existing ?? new Object3D();
            if (!existing) {
                if (typeof def.create.name === "string") node.name = def.create.name;
                obj.add(node);
            }
            if (Array.isArray(def.create.position) && def.create.position.length >= 3) node.position.fromArray(def.create.position);
            if (Array.isArray(def.create.quaternion) && def.create.quaternion.length >= 4) node.quaternion.fromArray(def.create.quaternion);
            if (Array.isArray(def.create.scale) && def.create.scale.length >= 3) node.scale.fromArray(def.create.scale);
            obj = node;
        }
        for (const compData of def.components ?? []) {
            if (!compData || typeof compData.name !== "string") {
                issue("def", `invalid component entry: ${JSON.stringify(compData)}`);
                continue;
            }
            const type = TypeStore.get(compData.name);
            if (!type) {
                issue("def", `unknown component type "${compData.name}"`);
                continue;
            }
            // NODE-REFERENCE fields ({ref: <reference id>, address?}) — the document
            // wire form of an Object3D field. Split out BEFORE deserialization (the
            // ObjectSerializer knows nothing about them) and assigned directly after,
            // resolved against the composed result: the references map is complete
            // here, so cross-reference wiring works regardless of load order.
            const { data: cleanData, nodeRefs } = splitNodeReferenceFields(compData, references);
            let component: object;
            // ADOPT: when the (possibly adopted) node already carries a component of
            // this type, the def data deserializes ONTO it — no duplicate component
            const existingComponents = (obj.userData as { components?: IComponent[] } | undefined)?.components;
            const adopted = existingComponents?.find(c => c instanceof type);
            if (adopted) {
                if ("guid" in cleanData) (adopted as unknown as Record<string, unknown>)[editorGuidKeyName] = cleanData.guid;
                serializationContext.object = obj;
                serializationContext.target = adopted;
                deserializeObject(adopted as unknown as ISerializable, cleanData, serializationContext);
                component = adopted;
            }
            else {
                // the loader's shared creation core (construct → assign → context → guid →
                // add dormant + camera/physics eager hooks); we run inside pre-setup, so
                // deserialization happens immediately instead of being queued
                const instance = createComponentFromSerializedData(obj, type, cleanData as { name: string; guid?: string }, serializationContext);
                serializationContext.object = obj;
                serializationContext.target = instance;
                deserializeObject(instance as unknown as ISerializable, cleanData, serializationContext);
                component = instance;
            }
            for (const nodeRef of nodeRefs) {
                const resolved = resolveTargetObject(nodeRef.target, root, references);
                if (!resolved) {
                    issue("def", `component "${compData.name}" field "${nodeRef.field}" node reference did not resolve: ${JSON.stringify(nodeRef.target)}`);
                    continue;
                }
                (component as Record<string, unknown>)[nodeRef.field] = resolved;
            }
        }
    }
}

/**
 * Split NODE-REFERENCE-shaped fields (`{ref: <known reference id>, address?}`) out of a
 * def's component data. Returns the data WITHOUT those fields (deserialization would
 * drop or misread them) plus the extractions for direct post-deserialize assignment.
 * Only values whose ref is a KNOWN reference id qualify — anything else stays for the
 * regular serializers.
 */
function splitNodeReferenceFields(compData: Record<string, unknown>, references: Map<string, Object3D>): { data: Record<string, unknown>; nodeRefs: Array<{ field: string; target: ProjectDocumentTarget }> } {
    let data = compData;
    const nodeRefs: Array<{ field: string; target: ProjectDocumentTarget }> = [];
    for (const [key, value] of Object.entries(compData)) {
        if (key === "name" || key === "guid") continue;
        if (!value || typeof value !== "object" || Array.isArray(value)) continue;
        const candidate = value as { ref?: unknown; address?: unknown };
        if (typeof candidate.ref !== "string" || !references.has(candidate.ref)) continue;
        if (data === compData) data = { ...compData }; // copy-on-write — never mutate the document
        delete data[key];
        nodeRefs.push({ field: key, target: candidate as ProjectDocumentTarget });
    }
    return { data, nodeRefs };
}

// ---------------------------------------------------------------------------
// overs
// ---------------------------------------------------------------------------

async function applyOvers(context: Context, document: ProjectDocument, root: Object3D, references: Map<string, Object3D>, entities: Map<string, Promise<object | null>>, issue: (arc: "over", message: string) => void): Promise<void> {
    // async overs (entity-targeted overs and entity-ref values await instantiation);
    // collected and awaited AFTER the synchronous pass so non-async overs stay pre-awake.
    const pending: Promise<void>[] = [];
    // per-entity apply chain: overs touching the same entity keep document order among
    // themselves by chaining on the entity's tail promise
    const entityTails = new Map<string, Promise<void>>();
    const chainOnEntity = (refId: string, work: (entity: object | null) => void): void => {
        const instantiation = entities.get(refId)!;
        const prev = entityTails.get(refId) ?? Promise.resolve();
        const tail = Promise.all([instantiation, prev]).then(([entity]) => work(entity));
        entityTails.set(refId, tail);
        pending.push(tail);
    };

    // Subtrees REMOVED by deactivation ("active": false → removeFromParent), per scope
    // ref. Materials are RESOURCES, not graph nodes — a material used only by a
    // deactivated mesh must stay addressable (doc order legally deactivates first and
    // patches after; the material is shared and may gain active consumers later), so
    // material resolution searches these detached subtrees too.
    const deactivatedRoots = new Map<string | undefined, Object3D[]>();

    for (const over of document.overs ?? []) {
        const target = over?.target;
        if (!target || typeof target.path !== "string" || target.path.length === 0) {
            issue("over", `over without a target path: ${JSON.stringify(over)}`);
            continue;
        }

        // ---- ENTITY-targeted over: patch the pooled entity itself ----------------
        if (target.ref !== undefined && entities.has(target.ref)) {
            const refId = target.ref, path = target.path, value = over.value, tgt = target;
            chainOnEntity(refId, entity => {
                if (!entity) return; // instantiation failure already reported
                applyValueAtPath(entity, path, value, issue, tgt);
            });
            continue;
        }

        // materials resolve against the scope's meshes, everything else via the graph
        let element: object | null = null;
        let owner: Object3D | null = null;
        if (target.ref === "$scene") {
            // reserved ref: the CONTEXT SCENE — scene-level opinions (background,
            // environment, fog) apply to the scene itself, NOT the composition parent
            // (which is a plain container when the evaluator runs with a `parent`).
            // With an ADDRESS, the target is a node INSIDE the scene (e.g. the engine's
            // own camera — the start-camera pose overs). `$`-prefix is reserved.
            element = target.address
                ? resolveInGraph(target.address, context.scene as unknown as GraphNode) as unknown as object | null
                : context.scene as unknown as object;
        }
        else if (target.ref === "$renderer") {
            // reserved ref: the CONTEXT RENDERER — renderer-level opinions
            // (toneMapping, toneMappingExposure). Addresses make no sense here.
            if (target.address) {
                issue("over", `"$renderer" targets take no address: ${JSON.stringify(target)}`);
                continue;
            }
            element = context.renderer as unknown as object;
        }
        else if (target.address?.type === "material") {
            const scope = target.ref !== undefined ? references.get(target.ref) : root;
            element = scope ? resolveTargetMaterial(scope, target.address, deactivatedRoots.get(target.ref))?.material ?? null : null;
        }
        else {
            owner = resolveTargetObject(target, root, references);
            element = owner;
        }
        if (!element) {
            issue("over", `over target did not resolve: ${JSON.stringify(target)}`);
            continue;
        }

        // reserved LAST segment: "active" — deactivation (USD semantics: opinions on
        // content the document does not own; false removes it from the composed result).
        // Whatever the rest of the path resolved to is what gets deactivated, so a node
        // ("active") and a component ("components/guid:…/active") share one mechanism.
        const segments = splitPath(target.path);
        if (segments[segments.length - 1] === "active") {
            if (over.value !== false) continue;
            const holder = segments.length === 1
                ? element
                : walkPath(element, segments, target.ref, message => issue("over", `path "${target.path}" ${message}: ${JSON.stringify(target)}`))?.container;
            if (holder) {
                // remember detached subtrees so their material RESOURCES stay addressable
                if ((holder as Object3D).isObject3D) {
                    const detached = deactivatedRoots.get(target.ref) ?? [];
                    detached.push(holder as Object3D);
                    deactivatedRoots.set(target.ref, detached);
                }
                deactivate(holder, owner);
            }
            continue;
        }

        // ---- NODE-ref value ({ref: <reference id>, address?}): an Object3D field
        // assigned BY REFERENCE to a node of the composition (the wire form the
        // editor writes for object slots). Checked before the entity branch —
        // reference ids and entity ids are distinct pools.
        if (isEntityRefValue(over.value) && references.has(over.value.ref)) {
            const node = resolveTargetObject(over.value as ProjectDocumentTarget, root, references);
            if (!node) {
                issue("over", `node-reference value did not resolve: ${JSON.stringify(over.value)} for ${JSON.stringify(target)}`);
                continue;
            }
            applyValueAtPath(element, target.path, node, issue, target);
            continue;
        }

        // ---- ENTITY-ref value: assign the pooled entity (USD material:binding) ----
        // NEVER raw-assign a reference object (that breaks the property).
        if (isEntityRefValue(over.value)) {
            const refId = over.value.ref;
            const instantiation = entities.get(refId);
            if (!instantiation) {
                issue("over", `value references unknown entity "${refId}": ${JSON.stringify(target)}`);
                continue;
            }
            const el = element, path = target.path, tgt = target;
            chainOnEntity(refId, entity => {
                if (!entity) {
                    issue("over", `entity "${refId}" was not instantiated — assignment skipped: ${JSON.stringify(tgt)}`);
                    return;
                }
                applyValueAtPath(el, path, entity, issue, tgt);
            });
            continue;
        }
        if (isReferenceValued(over.value)) {
            issue("over", `reference-valued over not supported (skipped, not raw-assigned): ${JSON.stringify(target)}`);
            continue;
        }

        applyValueAtPath(element, target.path, over.value, issue, target);
    }

    await Promise.all(pending);
}

/** an over VALUE pointing at a pooled entity reference: `{ ref: "<reference id>" }` */
function isEntityRefValue(v: unknown): v is { ref: string } {
    return !!v && typeof v === "object" && typeof (v as { ref?: unknown }).ref === "string";
}

/** an over VALUE that is a reference to another element (never raw-assign these) */
function isReferenceValued(v: unknown): boolean {
    if (!v || typeof v !== "object") return false;
    const o = v as Record<string, unknown>;
    return typeof o.asset === "string" || o.node !== undefined || typeof o.guid === "string";
}

/**
 * Stamp DERIVED needle-ids onto a freshly parsed model's un-stamped materials
 * (deriving == stamping per the id contract — deterministic from the glTF material
 * name + occurrence, so the writer's capture-time derivation and this load-time
 * stamping always agree). Without this, an over addressing an UNNAMED material by
 * derived id could never resolve: id matching reads userData["needle-id"], and an
 * un-stamped asset has none ("over target did not resolve" on a color edit).
 * Materials stamped in the FILE (extras → userData) are never overwritten.
 */
function stampParsedMaterials(model: Model): void {
    const parser = (model as unknown as { parser?: { json?: { materials?: Array<{ name?: string }> }; associations?: Map<object, { materials?: number }> } }).parser;
    const jsonMaterials = parser?.json?.materials;
    const associations = parser?.associations;
    if (!jsonMaterials || !associations) return;
    for (const [element, association] of associations) {
        const index = association?.materials;
        if (typeof index !== "number") continue;
        const holder = element as { isMaterial?: boolean; userData?: Record<string, unknown> };
        if (holder.isMaterial !== true) continue;
        const userData = holder.userData ?? (holder.userData = {});
        if (typeof userData["needle-id"] !== "string") {
            userData["needle-id"] = computeElementId("material", jsonMaterials[index]?.name ?? "", occurrenceOfName(jsonMaterials, index));
        }
    }
}

/**
 * Resolve one of an asset's materials by address, through the shared ladder.
 *
 * The single-material fallback is CALLER policy, not part of the ladder, and it is a
 * HEURISTIC rather than a contract: library material assets happen to be glbs carrying
 * one material today, but nothing stops one gaining more, so this must never be the
 * primary path. It exists only so an entity reference still resolves when the recorded
 * address has drifted. When the asset does carry several materials the fallback simply
 * does not apply and the caller reports an unresolved entity — a visible miss, never a
 * silent guess at which material was meant.
 */
function findMaterialInModel(model: Model, address: NeedleAddress): object | null {
    const scene = (model as unknown as { scene?: Object3D }).scene;
    if (!scene) return null;
    const materials = collectMaterials(scene);
    return resolveInCollection(address, materials) ?? (materials.length === 1 ? materials[0] : null);
}

/** drain a response body into one buffer, reporting cumulative received bytes per chunk */
async function readBodyWithProgress(body: ReadableStream<Uint8Array>, onChunk: (receivedBytes: number) => void): Promise<ArrayBuffer> {
    const reader = body.getReader();
    const chunks: Uint8Array[] = [];
    let received = 0;
    for (; ;) {
        const { done, value } = await reader.read();
        if (done) break;
        if (value) {
            chunks.push(value);
            received += value.byteLength;
            onChunk(received);
        }
    }
    const result = new Uint8Array(received);
    let offset = 0;
    for (const chunk of chunks) {
        result.set(chunk, offset);
        offset += chunk.byteLength;
    }
    return result.buffer;
}

function deactivate(element: object, owner: Object3D | null) {
    if ((element as Object3D).isObject3D) {
        (element as Object3D).removeFromParent();
        return;
    }
    // component target: evaluation runs pre-awake, so removal means "never existed"
    const component = element as IComponent;
    const host = owner ?? (component.gameObject as unknown as Object3D | undefined);
    if (host) removeComponent(host, component);
}

function applyValueAtPath(element: object, path: string, value: unknown, issue: (arc: "over", message: string) => void, target: ProjectDocumentTarget) {
    const segments = splitPath(path);
    // the walked chain — materials/textures along it need `needsUpdate` after a set
    // (deep overs like "material/map/repeat" patch texture state that three only
    // uploads/recompiles when flagged)
    const walked = walkPath(element, segments, target.ref, message => issue("over", `path "${path}" ${message}: ${JSON.stringify(target)}`));
    if (!walked) return;
    const { container, chain } = walked;
    const current = container as Record<string, unknown>;
    const key = segments[segments.length - 1];
    const existing = current[key];
    // three.js value types (Vector3, Color, Quaternion, …) are patched in place from
    // arrays; primitives and everything else assign directly
    if (Array.isArray(value) && existing && typeof existing === "object" && typeof (existing as { fromArray?: unknown }).fromArray === "function") {
        (existing as { fromArray(v: unknown[]): void }).fromArray(value);
    }
    else if (Array.isArray(value) && value.length === 3 && key === "background" && (current as { isScene?: boolean }).isScene === true) {
        // scene.background is null OR a texture (environment) by default — a color over
        // must CONSTRUCT/REPLACE with a Color (raw-assigning the array breaks rendering;
        // the generic fromArray branch above already handled an existing Color)
        current[key] = new Color().fromArray(value as number[]);
    }
    else {
        current[key] = value;
    }
    for (const touched of chain) {
        const o = touched as { isMaterial?: boolean; isTexture?: boolean; needsUpdate?: boolean };
        if (o.isMaterial === true || o.isTexture === true) o.needsUpdate = true;
    }
}