/** * Head-of-line repro for the per-lane scheduler in `runMemoryJobsOnce`. * * Before this fix, all non-embed jobs ran through a single bounded worker * pool, so a long-running `graph_consolidate` LLM call would pin every slot * and starve fast-lane jobs (e.g. `memory_v2_activation_recompute`) * for the duration of that call. * * The new scheduler runs slow / fast / embed lanes in parallel pools, each * with its own concurrency budget. This test enqueues a wave of slow LLM * jobs alongside fast jobs and asserts that every fast job completes before * any slow job's promise resolves — proving the lanes are truly independent. */ import { beforeAll, beforeEach, describe, expect, mock, setDefaultTimeout, test, } from "bun:test"; import { eq } from "drizzle-orm"; import { setConfig } from "./helpers/set-config.js"; // beforeAll runs initializeDb(), which can exceed bun's 5s default hook // timeout when the CI runner is saturated (one bun process per test file, // workers = CPU count). Raise the file-level default so DB setup isn't // killed mid-flight. Each test file runs in its own process, so this // doesn't leak. setDefaultTimeout(30_000); // ── Config seed (before the tested module reads config) ──────────── // Per-lane caps: 1 slow slot (so only 1 of the 5 enqueued slow jobs runs in // this tick) and a generous fast cap so every fast job both gets claimed // and gets a slot in the lane pool. The OLD shared-pool scheduler — which // claimed jobs without lane awareness and ran them through a single // workerConcurrency-sized pool — would pin its slots on the first claimed // slow jobs and force the fast jobs to queue behind a 200ms LLM call. // `memory.v2.enabled` is off so v1 is the active memory tier: the slow-lane // job here is `graph_consolidate`, whose handler completes v1 graph rows as // a no-op while concept-page memory is active — this test needs it to // genuinely run. // Everything else (including `memory.enabled: true`) is the schema default. setConfig("memory", { v2: { enabled: false }, jobs: { slowLlmConcurrency: 1, fastConcurrency: 5, embedConcurrency: 1, workerConcurrency: 2, }, }); // ── Track timestamps so we can assert ordering ───────────────────── type CompletionRecord = { type: string; conversationId: string; completedAt: number; }; const completions: CompletionRecord[] = []; // Slow-lane handler: blocks for SLOW_DELAY_MS. The test asserts every fast // job completes before any slow job — a single 200ms window is plenty. const SLOW_DELAY_MS = 200; mock.module("../plugins/defaults/memory/v1/graph/consolidation.js", () => ({ runConsolidation: async ( scopeId: string, ): Promise<{ totalUpdated: number; totalDeleted: number; totalMergeEdges: number; }> => { await new Promise((resolve) => setTimeout(resolve, SLOW_DELAY_MS)); completions.push({ type: "graph_consolidate", conversationId: scopeId, completedAt: Date.now(), }); return { totalUpdated: 0, totalDeleted: 0, totalMergeEdges: 0 }; }, })); // Fast-lane handler: resolves on the next microtask. The test fires this // many times in parallel; nothing should block. mock.module("../plugins/defaults/memory/v2/backfill-jobs.js", () => ({ memoryV2ActivationRecomputeJob: async (job: { payload: { scopeId?: string }; }): Promise => { completions.push({ type: "memory_v2_activation_recompute", conversationId: job.payload.scopeId ?? "", completedAt: Date.now(), }); return 0; }, memoryV2MigrateJob: async (): Promise => {}, })); mock.module("../plugins/defaults/memory/substrate/reembed-job.js", () => ({ memoryV2ReembedJob: async (): Promise => {}, })); // Stub remaining heavy boundaries that we never exercise but that get pulled // in transitively through jobs-worker's eager imports. These aren't strictly // required if the host machine can resolve them, but mocking them keeps the // test hermetic and fast under `bun test`. mock.module("../persistence/db-maintenance.js", () => ({ maybeRunDbMaintenance: () => {}, })); import { getMemoryDb } from "../persistence/db-connection.js"; import { initializeDb } from "../persistence/db-init.js"; import { _resetQdrantBreaker } from "../persistence/embeddings/qdrant-circuit-breaker.js"; import { enqueueMemoryJob } from "../persistence/jobs-store.js"; import { memoryJobs } from "../persistence/schema/index.js"; import { registerMemoryPluginJobHandlers } from "../plugins/defaults/memory/job-handler-registration.js"; import { runMemoryJobsOnce } from "../plugins/defaults/memory/jobs-worker.js"; describe("memory jobs worker lane scheduling", () => { beforeAll(async () => { registerMemoryPluginJobHandlers(); await initializeDb(); }); beforeEach(() => { const db = getMemoryDb()!; db.run("DELETE FROM memory_jobs"); completions.length = 0; _resetQdrantBreaker(); }); test("fast lane completes before slow lane releases its slot", async () => { // 5 slow `graph_consolidate` jobs across distinct scopes (so they would // serialize behind a single shared pool) plus 5 fast // `memory_v2_activation_recompute` jobs across distinct scopes. for (let i = 0; i < 5; i++) { enqueueMemoryJob("graph_consolidate", { scopeId: `slow-${i}` }); } for (let i = 0; i < 5; i++) { enqueueMemoryJob("memory_v2_activation_recompute", { scopeId: `fast-${i}`, }); } await runMemoryJobsOnce(); const fastDone = completions.filter( (c) => c.type === "memory_v2_activation_recompute", ); const slowDone = completions.filter((c) => c.type === "graph_consolidate"); // Slow lane is capped at 1 in this test, so only 1 slow job ran in this // tick. Fast lane has cap 2, but with 5 fast jobs it runs all 5 because // each handler resolves on the next microtask. expect(fastDone).toHaveLength(5); expect(slowDone).toHaveLength(1); // Head-of-line guarantee: every fast completion timestamp must precede // the (single) slow completion timestamp. Under the old shared pool with // workerConcurrency=2 and 1 slow slot occupied, this still held only if // a fast slot freed up first — but with 2 slow jobs in flight (the old // claim path would have claimed multiple slow jobs into the shared pool) // both slots would be pinned for SLOW_DELAY_MS and fast work would queue // behind. With the lane scheduler, fast work has its own pool. const earliestSlow = Math.min(...slowDone.map((c) => c.completedAt)); for (const fast of fastDone) { expect(fast.completedAt).toBeLessThan(earliestSlow); } // Sanity: exactly 1 of the 5 enqueued slow jobs reached `completed` (the // slow lane's per-tick budget). The other 4 are still pending and will be // picked up on subsequent ticks. (FIFO ordering inside the slow lane is // covered separately in jobs-store-qdrant-breaker.test.ts.) // // Note: a sixth `graph_consolidate` row may also appear here — the // `maybeEnqueueGraphMaintenanceJobs` tail of `runMemoryJobsOnce` enqueues // its own maintenance job whose checkpoint is missing in this fresh DB. // That row is irrelevant; we only care about the completed-vs-pending // counts of jobs we explicitly seeded. const completedSlow = countSlowByStatus("completed"); expect(completedSlow).toBe(1); }); }); function countSlowByStatus( status: "pending" | "running" | "completed", ): number { const db = getMemoryDb()!; return db .select() .from(memoryJobs) .where(eq(memoryJobs.type, "graph_consolidate")) .all() .filter((row) => row.status === status).length; }