import { readFileSync, writeFileSync } from "node:fs"; import { maybeCorrectionSignal, extractCorrectionCandidate } from "./corrections"; import { processCaptureTurn } from "./capture-coordinator"; import { classifyCaptureIntent } from "./capture-intent"; import { appendRuntimeEvent } from "./runtime-events"; import { appendCandidate, listCandidates } from "./inbox"; import { scoreMemoryWorth } from "./memory-worth"; import { buildRecallXray } from "./recall-xray"; import { readRecentRuntimeEvents } from "./runtime-events"; import { unsafeAddMemoryRecord } from "./store"; import type { CaptureCandidate, CaptureIntent, CaptureScopeTarget, MemoryRecord, ProjectIdentity } from "./types"; export interface ReplayTurn { role: "user" | "assistant" | "tool"; content: string; tags?: string[]; expected_capture?: boolean; expected_recall?: string[]; expected_no_capture?: boolean; actions?: Array<{ kind: "read" | "write"; path: string } | { kind: "command"; cwd: string; command: string }>; } export interface ReplayFixtureMetadata { fixture_id: string; fixture_kind: "synthetic" | "captured_style_synthetic" | "redacted_real"; redaction_version: string; source_description: string; created_at: string; privacy_reviewed: boolean; contains_real_user_content: boolean; } export interface ReplayFixture { fixture_id: string; description: string; redaction_version: string; synthetic?: boolean; metadata?: ReplayFixtureMetadata; sessions: Array<{ session_id: string; launch_cwd?: string; consolidation_failure?: boolean; turns: ReplayTurn[] }>; expected: { candidates?: string[]; inquiries?: string[]; rejected?: string[]; recall_query?: string; expected_recalled_memory_ids?: string[]; expected_omissions?: string[]; expected_intents?: CaptureIntent[]; expected_scope_targets?: Array>; expected_applicability?: string[]; expected_recurrence_count?: number; expected_candidate_count?: number; expected_no_capture_count?: number; expected_consolidation_failures?: number; }; } export interface ReplayResult { fixture_id: string; candidates_created: number; rejected_observations: number; temporary_instruction_rejections: number; recalled_memory_ids: string[]; expected_recall_hits: number; unexpected_recall_count: number; noise_count: number; privacy_leak_count: number; runtime_event_count: number; context_size: number; candidate_precision_proxy: number; failures: string[]; } export function validateReplayFixture(value: unknown): value is ReplayFixture { const fixture = value as ReplayFixture; const baseValid = Boolean(fixture && typeof fixture.fixture_id === "string" && Array.isArray(fixture.sessions) && fixture.sessions.every((s) => typeof s.session_id === "string" && Array.isArray(s.turns) && s.turns.every((t) => ["user", "assistant", "tool"].includes(t.role) && typeof t.content === "string")) && fixture.expected && typeof fixture.redaction_version === "string"); if (!baseValid) return false; if (!fixture.metadata) return fixture.synthetic === true || fixture.redaction_version.startsWith("synthetic"); const m = fixture.metadata; return m.fixture_id === fixture.fixture_id && ["synthetic", "captured_style_synthetic", "redacted_real"].includes(m.fixture_kind) && m.redaction_version === fixture.redaction_version && typeof m.source_description === "string" && typeof m.created_at === "string" && m.privacy_reviewed === true && (m.contains_real_user_content === false || m.fixture_kind === "redacted_real"); } export function validateReplayFixturePrivacy(fixture: ReplayFixture): string[] { const text = JSON.stringify(fixture); const findings = new Set(); if (/[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/i.test(text)) findings.add("email_like_string"); if (/(?:\/home\/|\/Users\/|[A-Za-z]:\\)[^\s'\"]+/.test(text)) findings.add("absolute_private_path"); if (/\b(?:sk-[A-Za-z0-9_-]{20,}|gh[pousr]_[A-Za-z0-9_]{20,}|[A-Z0-9_]*(?:SECRET|TOKEN|API_KEY|PASSWORD)[A-Z0-9_]*\s*=\s*\S+)/.test(text)) findings.add("token_like_secret"); if (/\+\d[\d .()-]{8,}\d\b|\b\d{3}[ .()-]\d{3}[ .()-]\d{4}\b/.test(text)) findings.add("phone_like_string"); return [...findings].sort(); } export function redactReplayContent(input: string, options: { redactUrls?: boolean } = {}): string { let out = input .replace(/[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/gi, "[redacted_email]") .replace(/\b(?:sk-[A-Za-z0-9_-]{20,}|gh[pousr]_[A-Za-z0-9_]{20,}|[A-Z0-9_]*(?:SECRET|TOKEN|API_KEY|PASSWORD)[A-Z0-9_]*\s*=\s*\S+)/g, "[redacted_secret]") .replace(/(?:\/home\/|\/Users\/)[^\s'\"]+/g, "[redacted_path]") .replace(/\b\+?\d[\d .()-]{8,}\d\b/g, "[redacted_phone]"); if (options.redactUrls) out = out.replace(/https?:\/\/\S+/g, "[redacted_url]"); return out; } export function redactReplayFixture(fixture: ReplayFixture): ReplayFixture { return { ...fixture, sessions: fixture.sessions.map((session) => ({ ...session, turns: session.turns.map((turn) => ({ ...turn, content: redactReplayContent(turn.content, { redactUrls: true }) })) })), }; } export function redactReplayFixtureFile(inputPath: string, outputPath: string): void { const fixture = JSON.parse(readFileSync(inputPath, "utf-8")) as ReplayFixture; writeFileSync(outputPath, `${JSON.stringify(redactReplayFixture(fixture), null, 2)}\n`, "utf-8"); } export interface CaptureReplayResult { fixture_id: string; candidates: CaptureCandidate[]; detected_intents: CaptureIntent[]; no_capture_count: number; consolidation_failure_count: number; failures: string[]; } function syntheticProjectResolver(path: string): ProjectIdentity { const normalized = path.replace(/\\/g, "/"); const projectId = normalized.match(/(?:^|\/)workspace\/([^/]+)/)?.[1] ?? "synthetic-project"; return { project_id: projectId, display_name: projectId, source: "cwd_fallback" }; } function targetMatches(actual: CaptureScopeTarget, expected: Partial): boolean { return Object.entries(expected).every(([key, value]) => actual[key as keyof CaptureScopeTarget] === value); } export function runCaptureReplayFixture(root: string, fixture: ReplayFixture): CaptureReplayResult { const failures: string[] = []; if (!validateReplayFixture(fixture)) return { fixture_id: "invalid", candidates: [], detected_intents: [], no_capture_count: 0, consolidation_failure_count: 0, failures: ["invalid_fixture"] }; const privacyFindings = validateReplayFixturePrivacy(fixture); if (privacyFindings.length) return { fixture_id: fixture.fixture_id, candidates: [], detected_intents: [], no_capture_count: 0, consolidation_failure_count: 0, failures: privacyFindings }; const detectedIntents: CaptureIntent[] = []; let noCaptureCount = 0; let consolidationFailureCount = 0; let turnNumber = 0; for (const session of fixture.sessions) { if (session.consolidation_failure) { consolidationFailureCount++; appendRuntimeEvent(root, { type: "warn", severity: "medium", component: "consolidation", message: "synthetic replay consolidation failed" }); } for (const turn of session.turns) { if (turn.role !== "user") continue; turnNumber++; const decision = classifyCaptureIntent(turn.content); detectedIntents.push(decision.intent); const result = processCaptureTurn(root, { session_id: session.session_id, turn_id: `turn-${turnNumber}`, message: turn.content, launch_cwd: session.launch_cwd ?? "workspace/synthetic-project", actions: turn.actions ?? [], resolver: syntheticProjectResolver, now: `2026-07-26T00:${String(turnNumber).padStart(2, "0")}:00Z`, }); if (result.candidates_created === 0 && result.candidates_reinforced === 0) noCaptureCount++; if (turn.expected_capture && result.candidates_created + result.candidates_reinforced === 0) failures.push(`expected capture missing at ${session.session_id}:turn-${turnNumber}`); if (turn.expected_no_capture && result.candidates_created + result.candidates_reinforced > 0) failures.push(`unexpected capture at ${session.session_id}:turn-${turnNumber}`); } } const candidates = listCandidates(root).filter((candidate) => candidate.status === "new"); if (fixture.expected.expected_candidate_count !== undefined && candidates.length !== fixture.expected.expected_candidate_count) failures.push(`expected ${fixture.expected.expected_candidate_count} candidates, received ${candidates.length}`); if (fixture.expected.expected_no_capture_count !== undefined && noCaptureCount !== fixture.expected.expected_no_capture_count) failures.push(`expected ${fixture.expected.expected_no_capture_count} non-captures, received ${noCaptureCount}`); for (const intent of fixture.expected.expected_intents ?? []) if (!detectedIntents.includes(intent)) failures.push(`expected intent missing ${intent}`); for (const expectedTarget of fixture.expected.expected_scope_targets ?? []) { if (!candidates.some((candidate) => candidate.scope_targets?.some((target) => targetMatches(target, expectedTarget)))) failures.push(`expected scope target missing ${JSON.stringify(expectedTarget)}`); } for (const applicability of fixture.expected.expected_applicability ?? []) { if (!candidates.some((candidate) => candidate.proposed_applies_when?.includes(applicability)) && !detectedIntents.some((_, index) => classifyCaptureIntent(fixture.sessions.flatMap((session) => session.turns).filter((turn) => turn.role === "user")[index]?.content ?? "").applicability.includes(applicability))) failures.push(`expected applicability missing ${applicability}`); } if (fixture.expected.expected_recurrence_count !== undefined && (candidates[0]?.recurrence_count ?? 0) !== fixture.expected.expected_recurrence_count) failures.push(`expected recurrence ${fixture.expected.expected_recurrence_count}, received ${candidates[0]?.recurrence_count ?? 0}`); if (fixture.expected.expected_consolidation_failures !== undefined && consolidationFailureCount !== fixture.expected.expected_consolidation_failures) failures.push(`expected ${fixture.expected.expected_consolidation_failures} consolidation failures, received ${consolidationFailureCount}`); return { fixture_id: fixture.fixture_id, candidates, detected_intents: detectedIntents, no_capture_count: noCaptureCount, consolidation_failure_count: consolidationFailureCount, failures }; } function record(id: string, statement: string): MemoryRecord { return { id, layer: "L2", scope: { type: "global" }, tags: ["testing"], statement, evidence: [{ type: "manual", ref: "fixture", note: "fixture" }], confidence: 0.9, stability: "semi-stable", created_at: "2026-06-15", updated_at: "2026-06-15", review: { cadence_days: 30, next_review: "2026-07-15", change_condition: "If contradicted." }, status: "active", supersedes: [], superseded_by: [], vault_ref: null, ruleType: "workflow" }; } export function runReplayFixture(root: string, fixture: ReplayFixture): ReplayResult { const invalid: ReplayResult = { fixture_id: "invalid", candidates_created: 0, rejected_observations: 0, temporary_instruction_rejections: 0, recalled_memory_ids: [], expected_recall_hits: 0, unexpected_recall_count: 0, noise_count: 0, privacy_leak_count: 0, runtime_event_count: 0, context_size: 0, candidate_precision_proxy: 0, failures: ["invalid_fixture"] }; if (!validateReplayFixture(fixture)) return invalid; const privacyFindings = validateReplayFixturePrivacy(fixture); if (privacyFindings.length) return { ...invalid, fixture_id: fixture.fixture_id, failures: privacyFindings }; const failures: string[] = []; let rejected = 0; let temporaryRejected = 0; for (const session of fixture.sessions) { for (const turn of session.turns) { if (turn.role !== "user") continue; const temporary = /for this one|just this once|one run|one-off/i.test(turn.content); const worth = scoreMemoryWorth({ observation: turn.content, explicitUserRequest: /always|never|prefer|don't|do not|this project uses|going forward/i.test(turn.content) && !temporary }); if (worth.decision === "reject" || worth.decision === "daily_only" || turn.expected_no_capture || temporary) { rejected++; if (temporary) temporaryRejected++; continue; } if (maybeCorrectionSignal(turn.content)) { const cand = extractCorrectionCandidate(turn.content, "2026-06-15", "/redacted/project"); if (cand && worth.decision === "candidate") appendCandidate(root, { ...cand, thread_id: session.session_id }); } } } for (const expected of fixture.expected.candidates ?? []) unsafeAddMemoryRecord(root, record(`mem_${expected.replace(/[^a-z0-9]+/gi, "_").toLowerCase()}`, expected)); const recall = fixture.expected.recall_query ? buildRecallXray(root, { query: fixture.expected.recall_query }) : undefined; const recalled = recall?.included.map((item) => item.memory_id) ?? []; for (const id of fixture.expected.expected_recalled_memory_ids ?? []) if (!recalled.includes(id)) failures.push(`expected recall missing ${id}`); const context = JSON.stringify(recall ?? {}); for (const omitted of fixture.expected.expected_omissions ?? []) if (omitted && context.includes(omitted)) failures.push(`unexpected recall contained ${omitted}`); const candidates = listCandidates(root); const expectedCandidates = fixture.expected.candidates ?? []; const noise = candidates.filter((candidate) => !expectedCandidates.some((text) => candidate.text.includes(text))).length; const expectedHits = (fixture.expected.expected_recalled_memory_ids ?? []).filter((id) => recalled.includes(id)).length; const unexpectedRecall = recalled.filter((id) => !(fixture.expected.expected_recalled_memory_ids ?? []).includes(id)).length; const privacyLeaks = validateReplayFixturePrivacy({ ...fixture, expected: { ...fixture.expected, expected_omissions: [] } }).length; const precision = candidates.length === 0 ? (expectedCandidates.length === 0 ? 1 : 0) : Math.max(0, (candidates.length - noise) / candidates.length); const runtimeEventCount = readRecentRuntimeEvents(root, { hours: 24, minSeverity: "low" }).length; return { fixture_id: fixture.fixture_id, candidates_created: candidates.length, rejected_observations: rejected, temporary_instruction_rejections: temporaryRejected, recalled_memory_ids: recalled, expected_recall_hits: expectedHits, unexpected_recall_count: unexpectedRecall, noise_count: noise, privacy_leak_count: privacyLeaks, runtime_event_count: runtimeEventCount, context_size: context.length, candidate_precision_proxy: precision, failures, }; }