/** * Shadow sampling — T0 preparation for a future TypeSafe (Jev) advisory experiment. * * When enabled, each settled loop round appends one JSONL record holding exactly * what an offline "plan freshness / actionability" evaluator would later need: * the objective, the plan ChatGPT returned, and the repo delta that resulted. * * Boundaries, kept verdict-consistent: * * - Observation-only (A15 posture): sampling never changes loop behavior. There * is no code path from a sample back into any decision, and a sampling failure * is swallowed rather than breaking a round. * - Redaction at collection time (A18): free-text fields pass through * redactForDiagnostics because this dataset is meant to leave the machine * (it will be sent to an external API in the T0 experiment). * - No authority: samples are input to a future *advisory* evaluation only. * The agreed rollout is Shadow → Advisory → Soft gate, never direct authority, * and never inside the three gates, capability, or delivery reconciliation. * - Records-class storage: the file follows the state root (paths.ts), so test * isolation contains it automatically and the watched repo stays clean. */ import { appendFileSync, mkdirSync } from "node:fs"; import { dirname } from "node:path"; import { redactForDiagnostics } from "./diagnostics.ts"; import { shadowSamplesPath } from "./paths.ts"; import type { Plan } from "../core/protocol.ts"; import type { RoundRecord } from "../core/cycle.ts"; export const SHADOW_SCHEMA_VERSION = 1 as const; export interface ShadowSample { schemaVersion: typeof SHADOW_SCHEMA_VERSION; /** Every record states what it is: observed data with zero authority. */ provenance: { source: "shadow-sampler"; authority: "none" }; recordedAt: string; round: { index: number; startedAt: number; settledAt: number; decision: string; taskId: string | null; planKey: string; ok: boolean; pushed: boolean | null; }; objective: string; rootRequest: string; /** The plan as parsed from the ChatGPT reply; `raw` included, redacted. */ plan: { decision: string; goal: string; steps: string[]; files: string[]; commands: string[]; verify: string[]; risks: string[]; notes: string; raw: string; }; /** What actually happened in the repo while the plan executed. */ delta: { changedFiles: string[] }; } export interface ShadowSampleInput { record: RoundRecord; objective: string; rootRequest: string; plan: Plan; } /** * Off by default. Enabled by config (`shadowSampling: true`) or, scoped to one * process, by WEBGPT_SHADOW_SAMPLES=1 — the Canary start path uses the env form * so nothing is persisted into the user's config file. */ export function shadowSamplingEnabled( config?: { shadowSampling?: boolean }, env: NodeJS.ProcessEnv = process.env, ): boolean { const flag = env.WEBGPT_SHADOW_SAMPLES?.trim().toLowerCase(); if (flag === "1" || flag === "true" || flag === "yes") return true; return config?.shadowSampling === true; } function redact(text: string): string { return redactForDiagnostics(text); } function redactAll(texts: string[]): string[] { return texts.map(redact); } /** Pure builder: redaction happens here, so no caller can forget it. */ export function buildShadowSample(input: ShadowSampleInput): ShadowSample { const now = Date.now(); return { schemaVersion: SHADOW_SCHEMA_VERSION, provenance: { source: "shadow-sampler", authority: "none" }, recordedAt: new Date(now).toISOString(), round: { index: input.record.index, startedAt: input.record.startedAt, settledAt: now, decision: input.record.decision, taskId: input.record.taskId, planKey: input.record.planKey, ok: input.record.ok, pushed: input.record.pushed, }, objective: redact(input.objective), rootRequest: redact(input.rootRequest), plan: { decision: input.plan.decision, goal: redact(input.plan.goal), steps: redactAll(input.plan.steps), files: redactAll(input.plan.files), commands: redactAll(input.plan.commands), verify: redactAll(input.plan.verify), risks: redactAll(input.plan.risks), notes: redact(input.plan.notes), raw: redact(input.plan.raw), }, delta: { changedFiles: [...input.record.changedFiles] }, }; } /** * Append one sample as a JSONL line. Returns false on any failure instead of * throwing: a sampler must never be a reason a round breaks. */ export function appendShadowSample(sample: ShadowSample, path: string = shadowSamplesPath()): boolean { try { mkdirSync(dirname(path), { recursive: true }); appendFileSync(path, JSON.stringify(sample) + "\n", "utf8"); return true; } catch { return false; } }