import type { EntityManager } from '@mikro-orm/postgresql' import { AgentRun, AgentCorrection, AgentProposal, AgentMetricRollup } from '../../data/entities' import type { AgentMetricRollupMetrics } from '../../data/validators' export type MetricScope = { tenantId: string; organizationId: string } /** The relative windows the /agents/:id/metrics endpoint can serve. */ export const ROLLUP_WINDOW_MS: Record = { '24h': 24 * 60 * 60 * 1000, '7d': 7 * 24 * 60 * 60 * 1000, '30d': 30 * 24 * 60 * 60 * 1000, '90d': 90 * 24 * 60 * 60 * 1000, } /** * Rollups are bucketed to this boundary so a re-run within the same bucket * recomputes the SAME `(windowStart, windowEnd)` row instead of inserting a new * one — that boundary is what makes the upsert idempotent. Defaults to the * scheduler's 300s interval; align them if the interval changes. */ export const ROLLUP_BUCKET_MS = 5 * 60 * 1000 /** Floor a timestamp to the rollup bucket boundary. */ function floorToBucket(ms: number): number { return Math.floor(ms / ROLLUP_BUCKET_MS) * ROLLUP_BUCKET_MS } /** * Compute the per-agent KPIs over `[since, until)` — lifted verbatim from the * live compute in api/agents/[id]/metrics/route.ts so the worker (writes * rollups) and the metrics route (live fallback) share one implementation. All * reads are non-encrypted aggregate columns, so plain em.find/count is correct. */ export async function computeAgentMetrics( em: EntityManager, scope: MetricScope, options: { agentId: string; since: Date; until?: Date }, ): Promise { const { agentId, since } = options const createdAt = options.until ? { $gte: since, $lt: options.until } : { $gte: since } // Eval replays write real AgentRun rows with real latency and cost. Counting // them here would let a 500-case suite skew the very dashboards used to judge // the agent, so production metrics measure production traffic only. const runs = await em.find( AgentRun, { ...scope, agentId, createdAt, deletedAt: null, source: 'runtime' }, { orderBy: { createdAt: 'desc' } }, ) const totalRuns = runs.length const evaluated = runs.filter((run) => run.evalPassed !== null && run.evalPassed !== undefined) const evalPassRate = evaluated.length ? evaluated.filter((run) => run.evalPassed === true).length / evaluated.length : null const runIds = runs.map((run) => run.id) const overrides = runIds.length ? await em.count(AgentCorrection, { ...scope, agentRunId: { $in: runIds } }) : 0 const overrideRate = totalRuns ? overrides / totalRuns : null const latencies = runs.map((run) => run.latencyMs).filter((n): n is number => typeof n === 'number') const avgLatencyMs = latencies.length ? latencies.reduce((sum, n) => sum + n, 0) / latencies.length : null const costMinorTotal = runs.reduce((sum, run) => sum + (typeof run.costMinor === 'number' ? run.costMinor : 0), 0) // Additive observability keys (data-honesty pass). Counts (errorRuns, // evalPassedRuns) are stored alongside the rates so org-level readers can // aggregate across agents — rates are not additive, counts are. const errorRuns = runs.filter((run) => run.status === 'error').length const errorRate = totalRuns ? errorRuns / totalRuns : null const evalPassedRuns = evaluated.filter((run) => run.evalPassed === true).length const p95LatencyMs = percentileOf(latencies, 0.95) // Floored to the SAME runtime runs the metrics above count. Without the // `runId` floor the proposal counts silently included every non-runtime run's // proposals — eval replays, and now workflow dry runs — while the run counts // beside them excluded those runs, so `approveUnchangedRate` was a ratio over // two different populations. `runIds` is already the runtime-only set. const proposalWindow = { ...scope, agentId, createdAt, runId: { $in: runIds } } const [unchanged, changed] = runIds.length ? await Promise.all([ em.count(AgentProposal, { ...proposalWindow, disposition: { $in: ['approved', 'auto_approved'] } }), em.count(AgentProposal, { ...proposalWindow, disposition: { $in: ['edited', 'rejected'] } }), ]) : [0, 0] const disposedProposals = unchanged + changed const approveUnchangedRate = disposedProposals ? unchanged / disposedProposals : null return { totalRuns, evaluatedRuns: evaluated.length, evalPassRate, overrides, overrideRate, avgLatencyMs, costMinorTotal, disposedProposals, approveUnchangedRate, errorRuns, errorRate, p95LatencyMs, evalPassedRuns, } } /** Nearest-rank percentile over an unsorted sample; null when empty. */ export function percentileOf(values: number[], fraction: number): number | null { if (values.length === 0) return null const sorted = [...values].sort((a, b) => a - b) const rank = Math.min(sorted.length - 1, Math.max(0, Math.ceil(fraction * sorted.length) - 1)) return sorted[rank] } /** * Recompute rollups for every agent with runs in the org, across each canonical * window the endpoint serves, and upsert the rows idempotently per * `(organizationId, agentId, windowStart)`. `windowEnd` is bucketed to * ROLLUP_BUCKET_MS so re-running the same interval re-stamps the SAME row rather * than duplicating it. */ export async function writeRollupsForOrg(em: EntityManager, scope: MetricScope): Promise<{ written: number }> { const windowEndMs = floorToBucket(Date.now()) const windowEnd = new Date(windowEndMs) const runs = await em.find( AgentRun, { ...scope, deletedAt: null, source: 'runtime' }, { fields: ['agentId'], orderBy: { agentId: 'asc' } }, ) const agentIds = Array.from(new Set(runs.map((run) => run.agentId))) let written = 0 for (const agentId of agentIds) { for (const windowKey of Object.keys(ROLLUP_WINDOW_MS)) { const windowStart = new Date(windowEndMs - ROLLUP_WINDOW_MS[windowKey]) const metrics = await computeAgentMetrics(em, scope, { agentId, since: windowStart, until: windowEnd }) const existing = await em.findOne(AgentMetricRollup, { organizationId: scope.organizationId, agentId, windowStart, }) if (existing) { existing.windowEnd = windowEnd existing.computedAt = new Date() existing.metrics = metrics em.persist(existing) } else { em.persist( em.create(AgentMetricRollup, { tenantId: scope.tenantId, organizationId: scope.organizationId, agentId, windowStart, windowEnd, computedAt: new Date(), metrics, }), ) } written += 1 } } await em.flush() return { written } }