import { describe, expect, it } from "vitest"; import type { ToolcraftEnvelopePerformanceConfig, ToolcraftRenderPass, ToolcraftRenderPassCost, ToolcraftRendererStrategy, } from "./performance"; import { assessToolcraftRenderPlan } from "./performance-render-plan-assessment"; import { validateToolcraftPerformanceCoverage } from "./performance-coverage-validator"; import { benchmarkCandidates, benchmarkWorkload, createBenchmarkDecision, createEnvelopeConfig, createPass, } from "./performance-render-plan-test-fixtures"; import { testSchema } from "./performance-test-fixtures"; describe("Toolcraft render plan assessment", () => { it("requires cost and lifecycle metadata for every envelope pass", () => { const pass = { ...createPass(), cost: undefined, lifecycle: undefined, } as unknown as ToolcraftRenderPass; expect( assessToolcraftRenderPlan(testSchema, createEnvelopeConfig({ pass })).errors, ).toEqual( expect.arrayContaining([ 'rendererPipeline pass "effect" must declare cost for envelope render-plan assessment.', 'rendererPipeline pass "effect" must declare lifecycle for envelope render-plan assessment.', ]), ); }); it("allows a constant pass with no dimensions", () => { const pass = createPass({ cost: { dimensions: [], frequency: "frame", relationship: "constant", }, }); expect( assessToolcraftRenderPlan(testSchema, createEnvelopeConfig({ pass })), ).toMatchObject({ errors: [], requiredBenchmarks: [], warnings: [] }); }); it("requires dimensions for every non-constant relationship", () => { const pass = createPass({ cost: { dimensions: [], frequency: "discrete", relationship: "linear", }, }); expect( assessToolcraftRenderPlan(testSchema, createEnvelopeConfig({ pass })).errors, ).toContain( 'rendererPipeline pass "effect" non-constant cost relationship "linear" requires at least one dimension.', ); }); it.each([ "quadratic", "product", "benchmark", ])( "requires measured renderer candidates for frame-frequency %s main-thread work", (relationship) => { const config = createEnvelopeConfig({ pass: createPass({ cost: { dimensions: ["output-units", "samples"], frequency: "frame", relationship, }, }), }); expect(assessToolcraftRenderPlan(testSchema, config)).toMatchObject({ errors: [], requiredBenchmarks: [ { candidates: benchmarkCandidates, passId: "effect", workload: benchmarkWorkload, }, ], }); }, ); it("requires a benchmark for interaction-frequency multiplicative work", () => { const pass = createPass({ cost: { dimensions: ["output-units", "samples"], frequency: "interaction", relationship: "product", }, }); expect( assessToolcraftRenderPlan(testSchema, createEnvelopeConfig({ pass })) .requiredBenchmarks, ).toEqual([ { candidates: benchmarkCandidates, passId: "effect", workload: benchmarkWorkload, }, ]); }); it("keeps non-accelerated pass candidates minimal", () => { const pass = createPass({ kind: "text-layout", runsOn: "worker" }); expect( assessToolcraftRenderPlan(testSchema, createEnvelopeConfig({ pass })) .requiredBenchmarks, ).toEqual([ { candidates: ["canvas-2d"], passId: "effect", workload: benchmarkWorkload, }, ]); }); it.each(["decode", "preprocess"] as const)( "rejects frame-frequency %s work", (kind) => { const pass = createPass({ cost: { dimensions: ["output-units"], frequency: "frame", relationship: "linear", }, id: `${kind}-source`, kind, output: kind === "decode" ? "source" : "intermediate", runsOn: "worker", }); expect( assessToolcraftRenderPlan(testSchema, createEnvelopeConfig({ pass })).errors, ).toContain( `rendererPipeline pass "${kind}-source" must not run ${kind} work at frame frequency. Decode and source preprocessing must be retained outside the frame loop.`, ); }, ); it.each(["decode", "preprocess"] as const)( "rejects interaction-frequency %s work", (kind) => { const pass = createPass({ cost: { dimensions: ["output-units"], frequency: "interaction", relationship: "linear", }, id: `${kind}-interaction`, kind, output: kind === "decode" ? "source" : "intermediate", runsOn: "worker", }); expect( assessToolcraftRenderPlan(testSchema, createEnvelopeConfig({ pass })).errors, ).toContain( `rendererPipeline pass "${kind}-interaction" must not run ${kind} work at interaction frequency. Decode and source preprocessing must be retained outside high-frequency paths.`, ); }, ); it("rejects retained resources scoped to a render call", () => { const pass = createPass({ lifecycle: { cache: "retained-resource", resourceScope: "call", }, }); expect( assessToolcraftRenderPlan(testSchema, createEnvelopeConfig({ pass })).errors, ).toContain( 'rendererPipeline pass "effect" retains a resource with call scope. Retained resources must outlive an individual render call.', ); }); it("rejects malformed cost, lifecycle, and dimension declarations at runtime", () => { const pass = createPass({ cost: { dimensions: ["missing-dimension", "missing-dimension"], frequency: "continuous", relationship: "fast", } as unknown as ToolcraftRenderPass["cost"], lifecycle: { cache: "persistent", resourceScope: "global", } as unknown as ToolcraftRenderPass["lifecycle"], }); const firstAssessment = assessToolcraftRenderPlan( testSchema, createEnvelopeConfig({ pass }), ); expect(firstAssessment).toEqual( assessToolcraftRenderPlan(testSchema, createEnvelopeConfig({ pass })), ); expect(firstAssessment.errors).toEqual( expect.arrayContaining([ 'rendererPipeline pass "effect" cost.frequency "continuous" is not supported.', 'rendererPipeline pass "effect" cost.relationship "fast" is not supported.', 'rendererPipeline pass "effect" cost dimension "missing-dimension" does not exist in workloadEnvelope.dimensions.', 'rendererPipeline pass "effect" cost dimensions must be unique; received duplicate "missing-dimension".', 'rendererPipeline pass "effect" lifecycle.cache "persistent" is not supported.', 'rendererPipeline pass "effect" lifecycle.resourceScope "global" is not supported.', ]), ); }); it("reports non-array cost dimensions without throwing", () => { const pass = createPass({ cost: { dimensions: null, frequency: "frame", relationship: "product", } as unknown as ToolcraftRenderPass["cost"], }); expect(() => assessToolcraftRenderPlan(testSchema, createEnvelopeConfig({ pass })), ).not.toThrow(); expect( assessToolcraftRenderPlan(testSchema, createEnvelopeConfig({ pass })).errors, ).toContain('rendererPipeline pass "effect" cost.dimensions must be an array.'); }); it("reports malformed pipelines and pass entries without throwing", () => { const malformedPipelines: readonly unknown[] = [ "bad", {}, { interactionInvalidation: [], passes: null }, { interactionInvalidation: [], passes: [null] }, ]; for (const rendererPipeline of malformedPipelines) { const config = { ...createEnvelopeConfig(), rendererPipeline, } as unknown as ToolcraftEnvelopePerformanceConfig; expect(() => assessToolcraftRenderPlan(testSchema, config)).not.toThrow(); expect(() => validateToolcraftPerformanceCoverage(testSchema, config), ).not.toThrow(); expect( assessToolcraftRenderPlan(testSchema, config).errors.length, ).toBeGreaterThan(0); } }); it("reports malformed nested pipeline fields without throwing", () => { const malformedPipelines: readonly unknown[] = [ { interactionInvalidation: [], passes: [{}], }, { interactionInvalidation: [], passes: [ { id: "effect", inputs: "source.input", invalidatedBy: [], kind: "pixel-transform", output: "preview", quality: "full", runsOn: "main", }, ], }, { interactionInvalidation: [ { interaction: "control-drag", invalidates: null, targets: [null], }, ], passes: [], }, ]; for (const rendererPipeline of malformedPipelines) { const config = { ...createEnvelopeConfig(), rendererPipeline, } as unknown as ToolcraftEnvelopePerformanceConfig; expect(() => assessToolcraftRenderPlan(testSchema, config)).not.toThrow(); expect(() => validateToolcraftPerformanceCoverage(testSchema, config), ).not.toThrow(); expect( validateToolcraftPerformanceCoverage(testSchema, config).some((error) => error.startsWith("rendererPipeline."), ), ).toBe(true); } }); it("requires interactiveMax for frame and interaction benchmark dimensions", () => { const pass = createPass({ cost: { dimensions: ["batch-only"], frequency: "frame", relationship: "benchmark", }, }); const config = { ...createEnvelopeConfig({ pass }), workloadEnvelope: { dimensions: [ { batchMax: 100, defaultValue: 1, id: "batch-only", mapping: "direct", source: { id: "batch-only", kind: "external-input" }, unit: "items", }, ], }, } satisfies ToolcraftEnvelopePerformanceConfig; const assessment = assessToolcraftRenderPlan(testSchema, config); expect(assessment.errors).toContain( 'rendererPipeline pass "effect" cost dimension "batch-only" must declare interactiveMax because frame and interaction work is assessed at the interactive ceiling.', ); expect(assessment.requiredBenchmarks).toEqual([]); }); it("rejects malformed pass and cost dimension ids without throwing", () => { const pass = createPass({ cost: { dimensions: ["", " samples "], frequency: "frame", relationship: "product", }, id: " effect ", }); const assessment = assessToolcraftRenderPlan( testSchema, createEnvelopeConfig({ pass }), ); expect(assessment.errors).toEqual( expect.arrayContaining([ "rendererPipeline pass at index 0 must have a non-empty trimmed id.", 'rendererPipeline pass " effect " cost dimension at index 0 must be a non-empty trimmed workload dimension id.', 'rendererPipeline pass " effect " cost dimension at index 1 must be a non-empty trimmed workload dimension id.', ]), ); }); it("rejects unsupported configured renderer strategies", () => { const config = createEnvelopeConfig({ rendererStrategy: "native" as ToolcraftRendererStrategy, }); const assessment = assessToolcraftRenderPlan(testSchema, config); expect(assessment.errors).toContain( 'Envelope rendererStrategy "native" is not supported.', ); expect(assessment.requiredBenchmarks).toEqual([]); }); it("does not force accelerated candidates for CPU-only pass kinds", () => { const pass = createPass({ id: "layout", kind: "text-layout" }); expect( assessToolcraftRenderPlan(testSchema, createEnvelopeConfig({ pass })) .requiredBenchmarks, ).toEqual([ { candidates: ["canvas-2d"], passId: "layout", workload: benchmarkWorkload, }, ]); }); it("benchmarks linear high-frequency accelerated passes", () => { const pass = createPass({ cost: { dimensions: ["output-units"], frequency: "frame", relationship: "linear", }, }); expect( assessToolcraftRenderPlan(testSchema, createEnvelopeConfig({ pass })) .requiredBenchmarks, ).toEqual([ { candidates: benchmarkCandidates, passId: "effect", workload: { "output-units": 12 }, }, ]); }); it("keeps a CPU baseline when the selected accelerated pass already runs on GPU", () => { const pass = createPass({ runsOn: "gpu" }); const config = createEnvelopeConfig({ pass, rendererStrategy: "webgl" }); expect(assessToolcraftRenderPlan(testSchema, config).requiredBenchmarks).toEqual([ { candidates: ["canvas-2d", "webgl"], passId: "effect", workload: benchmarkWorkload, }, ]); }); it("adds WebGPU only when it is the selected implementation", () => { const pass = createPass({ runsOn: "gpu" }); const config = createEnvelopeConfig({ pass, rendererStrategy: "webgpu" }); expect(assessToolcraftRenderPlan(testSchema, config).requiredBenchmarks).toEqual([ { candidates: ["canvas-2d", "webgl", "webgpu"], passId: "effect", workload: benchmarkWorkload, }, ]); }); it("orders workload dimensions by code unit for deterministic evidence", () => { const config = { ...createEnvelopeConfig(), rendererPipeline: { ...createEnvelopeConfig().rendererPipeline!, passes: [ createPass({ cost: { dimensions: ["ä", "z"], frequency: "frame", relationship: "product", }, }), ], }, workloadEnvelope: { dimensions: [ { defaultValue: 1, id: "ä", interactiveMax: 2, mapping: "direct", source: { id: "a-umlaut", kind: "external-input" }, unit: "units", }, { defaultValue: 1, id: "z", interactiveMax: 3, mapping: "direct", source: { id: "z", kind: "external-input" }, unit: "units", }, ], }, } satisfies ToolcraftEnvelopePerformanceConfig; expect( Object.keys( assessToolcraftRenderPlan(testSchema, config).requiredBenchmarks[0]! .workload, ), ).toEqual(["z", "ä"]); }); });