{"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../../../src/core/evaluation/metrics.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,yBAAyB,EAAE,MAAM,uBAAuB,CAAC;AACvE,OAAO,KAAK,EAAE,eAAe,EAAE,sBAAsB,EAAE,oBAAoB,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAW/G,wBAAgB,gBAAgB,CAAC,KAAK,EAAE;IACvC,GAAG,EAAE,aAAa,CAAC;IACnB,MAAM,EAAE,eAAe,EAAE,CAAC;IAC1B,KAAK,EAAE,oBAAoB,EAAE,CAAC;IAC9B,eAAe,CAAC,EAAE,yBAAyB,CAAC;CAC5C,GAAG,sBAAsB,EAAE,CA0C3B","sourcesContent":["import type { BenchmarkEvaluationResult } from \"../benchmark/types.js\";\nimport type { EvaluationEvent, EvaluationMetricResult, EvaluationMetricSpec, EvaluationRun } from \"./types.js\";\n\nfunction count(events: EvaluationEvent[], type: string): number {\n\treturn events.filter((event) => event.type === type).length;\n}\n\nfunction duration(run: EvaluationRun): number | undefined {\n\tif (!run.completedAt) return undefined;\n\treturn Math.max(0, Date.parse(run.completedAt) - Date.parse(run.startedAt));\n}\n\nexport function calculateMetrics(input: {\n\trun: EvaluationRun;\n\tevents: EvaluationEvent[];\n\tspecs: EvaluationMetricSpec[];\n\tbenchmarkReport?: BenchmarkEvaluationResult;\n}): EvaluationMetricResult[] {\n\tconst { run, events, specs, benchmarkReport } = input;\n\tconst values: Record<string, number | undefined> = {\n\t\tscenario_pass: benchmarkReport?.completionGate.passed ? 1 : benchmarkReport ? 0 : undefined,\n\t\tassertion_pass_rate: undefined,\n\t\ttool_failures: count(events, \"tool.failure\"),\n\t\tretries: count(events, \"tool.retry\"),\n\t\tstall_events: count(events, \"runtime.stall\"),\n\t\tstrategy_pivots: count(events, \"runtime.strategy_pivot\"),\n\t\trollback_count: count(events, \"transaction.rollback\"),\n\t\tprocess_leaks: count(events, \"process.leak\"),\n\t\tprovider_fallbacks: count(events, \"provider.fallback\"),\n\t\twall_time_ms: duration(run),\n\t\tmodel_latency_ms: events\n\t\t\t.filter((event) => event.type === \"model.response\")\n\t\t\t.reduce((total, event) => total + Number(event.details?.durationMs ?? 0), 0),\n\t\ttool_latency_ms: events\n\t\t\t.filter((event) => event.type === \"tool.result\")\n\t\t\t.reduce((total, event) => total + Number(event.details?.durationMs ?? 0), 0),\n\t\tinput_tokens: sumEventNumber(events, \"usage\", \"inputTokens\"),\n\t\toutput_tokens: sumEventNumber(events, \"usage\", \"outputTokens\"),\n\t\ttool_calls: count(events, \"tool.call\"),\n\t\tmodel_calls: count(events, \"model.request\"),\n\t\tcost_usd: sumEventNumber(events, \"usage\", \"costUsd\"),\n\t\tpolicy_denials: count(events, \"policy.denied\"),\n\t\tunsafe_attempts: count(events, \"safety.unsafe_attempt\"),\n\t\tworkspace_escapes: count(events, \"safety.workspace_escape\"),\n\t\tsecret_exposure: count(events, \"safety.secret_exposure\"),\n\t\tunapproved_mutations: count(events, \"safety.unapproved_mutation\"),\n\t\tretrieval_queries: count(events, \"retrieval.query\"),\n\t\tretrieval_tokens: sumEventNumber(events, \"retrieval.query\", \"tokens\"),\n\t\tsubagent_count: count(events, \"subagent.started\"),\n\t\tsubagent_cost_usd: sumEventNumber(events, \"subagent.usage\", \"costUsd\"),\n\t\tsubagent_latency_ms: sumEventNumber(events, \"subagent.completed\", \"durationMs\"),\n\t};\n\treturn specs.map((spec) => ({\n\t\tmetricId: spec.metricId,\n\t\tvalue: values[spec.metricId],\n\t\tunit: spec.unit,\n\t\tsource: sourceFor(spec.metricId),\n\t\tversion: 1,\n\t}));\n}\n\nfunction sumEventNumber(events: EvaluationEvent[], type: string, key: string): number | undefined {\n\tconst matching = events.filter((event) => event.type === type && typeof event.details?.[key] === \"number\");\n\tif (matching.length === 0) return undefined;\n\treturn matching.reduce((total, event) => total + Number(event.details?.[key]), 0);\n}\n\nfunction sourceFor(metricId: string): \"durable_event\" | \"provider_reported\" | \"calculated\" | \"semantic\" {\n\tif ([\"input_tokens\", \"output_tokens\", \"cost_usd\"].includes(metricId)) return \"provider_reported\";\n\tif ([\"scenario_pass\", \"assertion_pass_rate\"].includes(metricId)) return \"calculated\";\n\treturn \"durable_event\";\n}\n"]}