import { monitorEventLoopDelay } from "node:perf_hooks"; import { collectDefaultMetrics, Counter, Gauge, Histogram, Registry, } from "prom-client"; import { guardedWith, type Metrics } from "../metrics.js"; export const createMetrics = (registry: Registry): Metrics.Instance => { if (!(registry instanceof Registry)) { throw new Error("metricsRegistry must be a prom-client Registry"); } collectDefaultMetrics({ prefix: "pinboard_", register: registry }); const httpRequestsTotal = new Counter({ name: "pinboard_http_requests_total", help: "Total pinboard HTTP requests.", labelNames: ["method", "route", "status_code"] as const, registers: [registry], }); const httpRequestDurationSeconds = new Histogram({ name: "pinboard_http_request_duration_seconds", help: "Pinboard HTTP request duration in seconds.", labelNames: ["method", "route", "status_code"] as const, buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10], registers: [registry], }); const internalErrorsTotal = new Counter({ name: "pinboard_internal_errors_total", help: "Errors caught in background loops, by source.", labelNames: ["source"] as const, registers: [registry], }); let domainSources: Metrics.DomainSources | null = null; new Gauge({ name: "pinboard_registered_workers", help: "Live registered workers.", registers: [registry], async collect() { if (domainSources !== null) this.set(await domainSources.workers()); }, }); new Gauge({ name: "pinboard_active_placements", help: "Active placements across namespaces.", registers: [registry], async collect() { if (domainSources !== null) this.set(await domainSources.placements()); }, }); new Gauge({ name: "pinboard_active_hostnames", help: "Distinct hostnames with live registered workers.", registers: [registry], async collect() { if (domainSources !== null) { this.set((await domainSources.worlds()).hostnames); } }, }); new Gauge({ name: "pinboard_active_envs", help: "Distinct envs with live registered workers.", registers: [registry], async collect() { if (domainSources !== null) this.set((await domainSources.worlds()).envs); }, }); new Counter({ name: "pinboard_placement_resolutions_total", help: "Placement resolutions by outcome.", labelNames: ["outcome"] as const, registers: [registry], collect() { if (domainSources === null) return; this.reset(); for (const [outcome, count] of Object.entries( domainSources.resolutions(), )) { if (count > 0) this.inc({ outcome }, count); } }, }); const eventLoopLagSeconds = new Histogram({ name: "pinboard_event_loop_lag_seconds", help: "Event loop lag sampled every 2 seconds (mean over the window).", buckets: [0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1], registers: [registry], }); const guarded = guardedWith((source) => internalErrorsTotal.inc({ source })); const eventLoopDelay = monitorEventLoopDelay({ resolution: 20 }); eventLoopDelay.enable(); const sampler = setInterval( guarded("metrics_sampler", () => { const lagSeconds = eventLoopDelay.mean / 1e9; if (Number.isFinite(lagSeconds)) eventLoopLagSeconds.observe(lagSeconds); eventLoopDelay.reset(); }), 2_000, ); sampler.unref(); return { forwardsTotal: new Counter({ name: "pinboard_forwards_total", help: "Total data-plane requests forwarded to workers.", labelNames: ["kind"] as const, registers: [registry], }), forwardFailuresTotal: new Counter({ name: "pinboard_forward_failures_total", help: "Data-plane forwards that did not complete, by outcome.", labelNames: ["outcome"] as const, registers: [registry], }), takeoversTotal: new Counter({ name: "pinboard_takeovers_total", help: "Cross-hostname takeover encounters, by strategy and outcome.", labelNames: ["strategy", "outcome"] as const, registers: [registry], }), degradedHeartbeatsTotal: new Counter({ name: "pinboard_degraded_heartbeats_total", help: "Heartbeats whose reverse challenge probe failed, by reason.", labelNames: ["reason"] as const, registers: [registry], }), registerRejectionsTotal: new Counter({ name: "pinboard_register_rejections_total", help: "Rejected worker registrations, by account, pool, and reason.", labelNames: ["account", "pool", "reason"] as const, registers: [registry], }), dataPlaneConnections: new Gauge({ name: "pinboard_data_plane_connections", help: "In-flight proxied data-plane connections on this node.", registers: [registry], }), internalError: (source) => internalErrorsTotal.inc({ source }), guarded, trackHttpRequest: (method) => { const start = process.hrtime.bigint(); const track: Metrics.RequestTrack = { route: "unknown", done: (statusCode) => { const labels = { method, route: track.route, status_code: String(statusCode), }; httpRequestsTotal.inc(labels); httpRequestDurationSeconds.observe( labels, Number(process.hrtime.bigint() - start) / 1e9, ); }, }; return track; }, setDomainSources: (sources) => { domainSources = sources; }, close: () => { clearInterval(sampler); eventLoopDelay.disable(); }, }; };