{"version":3,"sources":["../src/replay.ts"],"sourcesContent":["import { createHash } from \"node:crypto\";\nimport { mkdir, readFile, writeFile } from \"node:fs/promises\";\nimport { dirname, relative, resolve } from \"node:path\";\nimport type { JsonValue } from \"./harness\";\n\ntype MaybePromise<T> = T | Promise<T>;\n\nconst DEFAULT_REPLAY_DIR = \".vitest-evals/recordings\";\n\n/** Replay mode used for tool recording and playback. */\nexport type ReplayMode = \"off\" | \"auto\" | \"strict\" | \"record\";\n\n/** Metadata attached to tool calls or errors touched by replay. */\nexport type ReplayMetadata = {\n  status: \"recorded\" | \"replayed\";\n  recordingPath: string;\n  cacheKey: string;\n};\n\n/** JSON recording persisted for one replayable tool execution. */\nexport interface ToolRecording<\n  TArgs extends JsonValue = JsonValue,\n  TResult extends JsonValue = JsonValue,\n> {\n  writtenAt: string;\n  toolName: string;\n  input: TArgs;\n  output?: TResult;\n  error?: {\n    message: string;\n    type?: string;\n    [key: string]: JsonValue | undefined;\n  };\n  metadata?: Record<string, JsonValue | undefined>;\n}\n\n/** Per-tool replay configuration for keying and sanitizing recordings. */\nexport interface ToolReplayConfig<\n  TArgs extends JsonValue = JsonValue,\n  TResult extends JsonValue = JsonValue,\n  TContext = unknown,\n> {\n  key?: (args: TArgs, context: TContext) => MaybePromise<JsonValue>;\n  sanitize?: (\n    recording: ToolRecording<TArgs, TResult>,\n  ) => MaybePromise<ToolRecording<TArgs, TResult>>;\n  version?: string;\n}\n\n/** Executes a tool call with optional recording or replay behavior. */\nexport async function executeWithReplay<\n  TArgs extends JsonValue,\n  TResult extends JsonValue,\n  TContext,\n>({\n  toolName,\n  args,\n  context,\n  execute,\n  replay,\n}: {\n  toolName: string;\n  args: TArgs;\n  context: TContext;\n  execute: (args: TArgs, context: TContext) => MaybePromise<TResult>;\n  replay: boolean | ToolReplayConfig<TArgs, TResult, TContext> | undefined;\n}) {\n  const replayConfig = normalizeReplayConfig(replay);\n  const replayMode = resolveReplayMode();\n\n  if (!replayConfig || replayMode === \"off\") {\n    return {\n      result: await execute(args, context),\n    };\n  }\n\n  const cacheKeyInput = replayConfig.key\n    ? await replayConfig.key(args, context)\n    : args;\n  const cacheKey = createCacheKey(\n    toolName,\n    cacheKeyInput,\n    replayConfig.version,\n  );\n  const absoluteRecordingPath = resolve(\n    process.cwd(),\n    resolveReplayDirectory(),\n    toolName,\n    `${cacheKey}.json`,\n  );\n  const recordingPath = relative(process.cwd(), absoluteRecordingPath);\n\n  if (replayMode === \"auto\" || replayMode === \"strict\") {\n    const recording = await readRecording<TArgs, TResult>(\n      absoluteRecordingPath,\n    );\n    if (recording) {\n      const replayMetadata = {\n        status: \"replayed\",\n        recordingPath,\n        cacheKey,\n      } satisfies ReplayMetadata;\n\n      if (recording.error) {\n        throw attachReplayMetadata(\n          deserializeRecordedError(recording.error),\n          replayMetadata,\n        );\n      }\n\n      return {\n        result: recording.output as TResult,\n        replay: replayMetadata,\n      };\n    }\n\n    if (replayMode === \"strict\") {\n      throw new Error(\n        `Missing replay recording for ${toolName}: ${recordingPath}`,\n      );\n    }\n  }\n\n  try {\n    const result = await execute(args, context);\n    const replayMetadata = {\n      status: \"recorded\",\n      recordingPath,\n      cacheKey,\n    } satisfies ReplayMetadata;\n\n    await writeRecording(absoluteRecordingPath, replayConfig, {\n      writtenAt: new Date().toISOString(),\n      toolName,\n      input: args,\n      output: result,\n      metadata: {\n        cacheKey,\n        version: replayConfig.version,\n        mode: replayMode,\n      },\n    });\n\n    return {\n      result,\n      replay: replayMetadata,\n    };\n  } catch (error) {\n    const replayMetadata = {\n      status: \"recorded\",\n      recordingPath,\n      cacheKey,\n    } satisfies ReplayMetadata;\n\n    await writeRecording(absoluteRecordingPath, replayConfig, {\n      writtenAt: new Date().toISOString(),\n      toolName,\n      input: args,\n      error: serializeToolError(error),\n      metadata: {\n        cacheKey,\n        version: replayConfig.version,\n        mode: replayMode,\n      },\n    });\n\n    throw attachReplayMetadata(error, replayMetadata);\n  }\n}\n\n/** Reads replay metadata attached to a thrown tool error. */\nexport function getReplayMetadataFromError(error: unknown) {\n  if (\n    error &&\n    typeof error === \"object\" &&\n    \"vitestEvalsReplay\" in error &&\n    isReplayMetadata(\n      (error as { vitestEvalsReplay?: unknown }).vitestEvalsReplay,\n    )\n  ) {\n    return (error as { vitestEvalsReplay: ReplayMetadata }).vitestEvalsReplay;\n  }\n\n  return undefined;\n}\n\n/** Converts replay metadata into the JSON-safe shape stored on tool calls. */\nexport function normalizeReplayMetadata(replay: ReplayMetadata | undefined) {\n  if (!replay) {\n    return undefined;\n  }\n\n  return {\n    replay: {\n      status: replay.status,\n      recordingPath: replay.recordingPath,\n      cacheKey: replay.cacheKey,\n    },\n  } satisfies Record<string, JsonValue>;\n}\n\nfunction normalizeReplayConfig<\n  TArgs extends JsonValue,\n  TResult extends JsonValue,\n  TContext,\n>(replay: boolean | ToolReplayConfig<TArgs, TResult, TContext> | undefined) {\n  if (!replay) {\n    return null;\n  }\n\n  return replay === true ? {} : replay;\n}\n\nfunction resolveReplayMode(): ReplayMode {\n  const value = process.env.VITEST_EVALS_REPLAY_MODE;\n  if (\n    value === \"auto\" ||\n    value === \"strict\" ||\n    value === \"record\" ||\n    value === \"off\"\n  ) {\n    return value;\n  }\n\n  return \"auto\";\n}\n\nfunction resolveReplayDirectory() {\n  return process.env.VITEST_EVALS_REPLAY_DIR ?? DEFAULT_REPLAY_DIR;\n}\n\nfunction createCacheKey(\n  toolName: string,\n  input: JsonValue,\n  version: string | undefined,\n) {\n  return createHash(\"sha256\")\n    .update(\n      stableStringify({\n        toolName,\n        input,\n        version: version ?? null,\n      }),\n    )\n    .digest(\"hex\");\n}\n\nfunction stableStringify(value: JsonValue): string {\n  if (value === null || typeof value !== \"object\") {\n    return JSON.stringify(value);\n  }\n\n  if (Array.isArray(value)) {\n    return `[${value.map((item) => stableStringify(item)).join(\",\")}]`;\n  }\n\n  const keys = Object.keys(value).sort();\n  return `{${keys\n    .map((key) => `${JSON.stringify(key)}:${stableStringify(value[key])}`)\n    .join(\",\")}}`;\n}\n\nasync function readRecording<\n  TArgs extends JsonValue,\n  TResult extends JsonValue,\n>(recordingPath: string): Promise<ToolRecording<TArgs, TResult> | null> {\n  try {\n    const content = await readFile(recordingPath, \"utf8\");\n    return JSON.parse(content) as ToolRecording<TArgs, TResult>;\n  } catch {\n    return null;\n  }\n}\n\nasync function writeRecording<\n  TArgs extends JsonValue,\n  TResult extends JsonValue,\n  TContext,\n>(\n  recordingPath: string,\n  replay: ToolReplayConfig<TArgs, TResult, TContext>,\n  recording: ToolRecording<TArgs, TResult>,\n) {\n  const sanitized = replay.sanitize\n    ? await replay.sanitize(recording)\n    : recording;\n  await mkdir(dirname(recordingPath), { recursive: true });\n  await writeFile(recordingPath, JSON.stringify(sanitized, null, 2));\n}\n\nfunction serializeToolError(error: unknown) {\n  if (error instanceof Error) {\n    return {\n      message: error.message,\n      type: error.name,\n    };\n  }\n\n  return {\n    message: String(error),\n    type: \"Error\",\n  };\n}\n\nfunction deserializeRecordedError(error: {\n  message: string;\n  type?: string;\n}) {\n  const replayedError = new Error(error.message);\n  replayedError.name = error.type ?? \"Error\";\n  return replayedError;\n}\n\nfunction attachReplayMetadata(error: unknown, replay: ReplayMetadata) {\n  const baseError =\n    error instanceof Error\n      ? error\n      : new Error(String(error ?? \"Unknown error\"));\n  return Object.assign(baseError, {\n    vitestEvalsReplay: replay,\n  });\n}\n\nfunction isReplayMetadata(value: unknown): value is ReplayMetadata {\n  return (\n    Boolean(value) &&\n    typeof value === \"object\" &&\n    value !== null &&\n    \"status\" in value &&\n    \"recordingPath\" in value &&\n    \"cacheKey\" in value\n  );\n}\n"],"mappings":";AAAA,SAAS,kBAAkB;AAC3B,SAAS,OAAO,UAAU,iBAAiB;AAC3C,SAAS,SAAS,UAAU,eAAe;AAK3C,IAAM,qBAAqB;AA2C3B,eAAsB,kBAIpB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,GAMG;AACD,QAAM,eAAe,sBAAsB,MAAM;AACjD,QAAM,aAAa,kBAAkB;AAErC,MAAI,CAAC,gBAAgB,eAAe,OAAO;AACzC,WAAO;AAAA,MACL,QAAQ,MAAM,QAAQ,MAAM,OAAO;AAAA,IACrC;AAAA,EACF;AAEA,QAAM,gBAAgB,aAAa,MAC/B,MAAM,aAAa,IAAI,MAAM,OAAO,IACpC;AACJ,QAAM,WAAW;AAAA,IACf;AAAA,IACA;AAAA,IACA,aAAa;AAAA,EACf;AACA,QAAM,wBAAwB;AAAA,IAC5B,QAAQ,IAAI;AAAA,IACZ,uBAAuB;AAAA,IACvB;AAAA,IACA,GAAG,QAAQ;AAAA,EACb;AACA,QAAM,gBAAgB,SAAS,QAAQ,IAAI,GAAG,qBAAqB;AAEnE,MAAI,eAAe,UAAU,eAAe,UAAU;AACpD,UAAM,YAAY,MAAM;AAAA,MACtB;AAAA,IACF;AACA,QAAI,WAAW;AACb,YAAM,iBAAiB;AAAA,QACrB,QAAQ;AAAA,QACR;AAAA,QACA;AAAA,MACF;AAEA,UAAI,UAAU,OAAO;AACnB,cAAM;AAAA,UACJ,yBAAyB,UAAU,KAAK;AAAA,UACxC;AAAA,QACF;AAAA,MACF;AAEA,aAAO;AAAA,QACL,QAAQ,UAAU;AAAA,QAClB,QAAQ;AAAA,MACV;AAAA,IACF;AAEA,QAAI,eAAe,UAAU;AAC3B,YAAM,IAAI;AAAA,QACR,gCAAgC,QAAQ,KAAK,aAAa;AAAA,MAC5D;AAAA,IACF;AAAA,EACF;AAEA,MAAI;AACF,UAAM,SAAS,MAAM,QAAQ,MAAM,OAAO;AAC1C,UAAM,iBAAiB;AAAA,MACrB,QAAQ;AAAA,MACR;AAAA,MACA;AAAA,IACF;AAEA,UAAM,eAAe,uBAAuB,cAAc;AAAA,MACxD,YAAW,oBAAI,KAAK,GAAE,YAAY;AAAA,MAClC;AAAA,MACA,OAAO;AAAA,MACP,QAAQ;AAAA,MACR,UAAU;AAAA,QACR;AAAA,QACA,SAAS,aAAa;AAAA,QACtB,MAAM;AAAA,MACR;AAAA,IACF,CAAC;AAED,WAAO;AAAA,MACL;AAAA,MACA,QAAQ;AAAA,IACV;AAAA,EACF,SAAS,OAAO;AACd,UAAM,iBAAiB;AAAA,MACrB,QAAQ;AAAA,MACR;AAAA,MACA;AAAA,IACF;AAEA,UAAM,eAAe,uBAAuB,cAAc;AAAA,MACxD,YAAW,oBAAI,KAAK,GAAE,YAAY;AAAA,MAClC;AAAA,MACA,OAAO;AAAA,MACP,OAAO,mBAAmB,KAAK;AAAA,MAC/B,UAAU;AAAA,QACR;AAAA,QACA,SAAS,aAAa;AAAA,QACtB,MAAM;AAAA,MACR;AAAA,IACF,CAAC;AAED,UAAM,qBAAqB,OAAO,cAAc;AAAA,EAClD;AACF;AAGO,SAAS,2BAA2B,OAAgB;AACzD,MACE,SACA,OAAO,UAAU,YACjB,uBAAuB,SACvB;AAAA,IACG,MAA0C;AAAA,EAC7C,GACA;AACA,WAAQ,MAAgD;AAAA,EAC1D;AAEA,SAAO;AACT;AAGO,SAAS,wBAAwB,QAAoC;AAC1E,MAAI,CAAC,QAAQ;AACX,WAAO;AAAA,EACT;AAEA,SAAO;AAAA,IACL,QAAQ;AAAA,MACN,QAAQ,OAAO;AAAA,MACf,eAAe,OAAO;AAAA,MACtB,UAAU,OAAO;AAAA,IACnB;AAAA,EACF;AACF;AAEA,SAAS,sBAIP,QAA0E;AAC1E,MAAI,CAAC,QAAQ;AACX,WAAO;AAAA,EACT;AAEA,SAAO,WAAW,OAAO,CAAC,IAAI;AAChC;AAEA,SAAS,oBAAgC;AACvC,QAAM,QAAQ,QAAQ,IAAI;AAC1B,MACE,UAAU,UACV,UAAU,YACV,UAAU,YACV,UAAU,OACV;AACA,WAAO;AAAA,EACT;AAEA,SAAO;AACT;AAEA,SAAS,yBAAyB;AAChC,SAAO,QAAQ,IAAI,2BAA2B;AAChD;AAEA,SAAS,eACP,UACA,OACA,SACA;AACA,SAAO,WAAW,QAAQ,EACvB;AAAA,IACC,gBAAgB;AAAA,MACd;AAAA,MACA;AAAA,MACA,SAAS,WAAW;AAAA,IACtB,CAAC;AAAA,EACH,EACC,OAAO,KAAK;AACjB;AAEA,SAAS,gBAAgB,OAA0B;AACjD,MAAI,UAAU,QAAQ,OAAO,UAAU,UAAU;AAC/C,WAAO,KAAK,UAAU,KAAK;AAAA,EAC7B;AAEA,MAAI,MAAM,QAAQ,KAAK,GAAG;AACxB,WAAO,IAAI,MAAM,IAAI,CAAC,SAAS,gBAAgB,IAAI,CAAC,EAAE,KAAK,GAAG,CAAC;AAAA,EACjE;AAEA,QAAM,OAAO,OAAO,KAAK,KAAK,EAAE,KAAK;AACrC,SAAO,IAAI,KACR,IAAI,CAAC,QAAQ,GAAG,KAAK,UAAU,GAAG,CAAC,IAAI,gBAAgB,MAAM,GAAG,CAAC,CAAC,EAAE,EACpE,KAAK,GAAG,CAAC;AACd;AAEA,eAAe,cAGb,eAAsE;AACtE,MAAI;AACF,UAAM,UAAU,MAAM,SAAS,eAAe,MAAM;AACpD,WAAO,KAAK,MAAM,OAAO;AAAA,EAC3B,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAEA,eAAe,eAKb,eACA,QACA,WACA;AACA,QAAM,YAAY,OAAO,WACrB,MAAM,OAAO,SAAS,SAAS,IAC/B;AACJ,QAAM,MAAM,QAAQ,aAAa,GAAG,EAAE,WAAW,KAAK,CAAC;AACvD,QAAM,UAAU,eAAe,KAAK,UAAU,WAAW,MAAM,CAAC,CAAC;AACnE;AAEA,SAAS,mBAAmB,OAAgB;AAC1C,MAAI,iBAAiB,OAAO;AAC1B,WAAO;AAAA,MACL,SAAS,MAAM;AAAA,MACf,MAAM,MAAM;AAAA,IACd;AAAA,EACF;AAEA,SAAO;AAAA,IACL,SAAS,OAAO,KAAK;AAAA,IACrB,MAAM;AAAA,EACR;AACF;AAEA,SAAS,yBAAyB,OAG/B;AACD,QAAM,gBAAgB,IAAI,MAAM,MAAM,OAAO;AAC7C,gBAAc,OAAO,MAAM,QAAQ;AACnC,SAAO;AACT;AAEA,SAAS,qBAAqB,OAAgB,QAAwB;AACpE,QAAM,YACJ,iBAAiB,QACb,QACA,IAAI,MAAM,OAAO,SAAS,eAAe,CAAC;AAChD,SAAO,OAAO,OAAO,WAAW;AAAA,IAC9B,mBAAmB;AAAA,EACrB,CAAC;AACH;AAEA,SAAS,iBAAiB,OAAyC;AACjE,SACE,QAAQ,KAAK,KACb,OAAO,UAAU,YACjB,UAAU,QACV,YAAY,SACZ,mBAAmB,SACnB,cAAc;AAElB;","names":[]}