/** * @packageDocumentation * * Configure your libp2p node with Prometheus metrics: * * ```typescript * import { createLibp2p } from 'libp2p' * import { prometheusMetrics } from '@libp2p/prometheus-metrics' * * const node = await createLibp2p({ * metrics: prometheusMetrics() * }) * ``` * * Then use the `prom-client` module to supply metrics to the Prometheus/Graphana client using your http framework: * * ```JavaScript * import client from 'prom-client' * * async function handler (request, h) { * return h.response(await client.register.metrics()) * .type(client.register.contentType) * } * ``` * * All Prometheus metrics are global so there's no other work required to extract them. * * ## Queries * * Some useful queries are: * * ### Data sent/received * * ``` * rate(libp2p_data_transfer_bytes_total[30s]) * ``` * * ### CPU usage * * ``` * rate(process_cpu_user_seconds_total[30s]) * 100 * ``` * * ### Memory usage * * ``` * nodejs_memory_usage_bytes * ``` * * ### DHT query time * * ``` * libp2p_kad_dht_wan_query_time_seconds * ``` * * or * * ``` * libp2p_kad_dht_lan_query_time_seconds * ``` * * ### TCP transport dialer errors * * ``` * rate(libp2p_tcp_dialer_errors_total[30s]) * ``` */ import { statfs } from 'node:fs/promises' import { totalmem } from 'node:os' import { serviceCapabilities } from '@libp2p/interface' import { collectDefaultMetrics, register } from 'prom-client' import { PrometheusCounterGroup } from './counter-group.ts' import { PrometheusCounter } from './counter.ts' import { PrometheusHistogramGroup } from './histogram-group.ts' import { PrometheusHistogram } from './histogram.ts' import { PrometheusMetricGroup } from './metric-group.ts' import { PrometheusMetric } from './metric.ts' import { PrometheusSummaryGroup } from './summary-group.ts' import { PrometheusSummary } from './summary.ts' import type { ComponentLogger, Logger, MultiaddrConnection, Stream, CalculatedMetricOptions, Counter, CounterGroup, Metric, MetricGroup, MetricOptions, Metrics, CalculatedHistogramOptions, CalculatedSummaryOptions, HistogramOptions, Histogram, HistogramGroup, SummaryOptions, Summary, SummaryGroup, MessageStream } from '@libp2p/interface' import type { DefaultMetricsCollectorConfiguration, Registry, RegistryContentType } from 'prom-client' // export helper functions for creating buckets export { linearBuckets, exponentialBuckets } from 'prom-client' // prom-client metrics are global const metrics = new Map() export interface PrometheusMetricsInit { /** * Use a custom registry to register metrics. * By default, the global registry is used to register metrics. */ registry?: Registry /** * By default we collect default metrics - CPU, memory etc, to not do * this, pass true here */ collectDefaultMetrics?: boolean /** * prom-client options to pass to the `collectDefaultMetrics` function */ defaultMetrics?: DefaultMetricsCollectorConfiguration /** * All metrics in prometheus are global so to prevent clashes in naming * we reset the global metrics registry on creation - to not do this, * pass true here */ preserveExistingMetrics?: boolean /** * The current filesystem usage is reported as the metric * `nodejs_fs_usage_bytes` using the `statfs` function from `node:fs` - the * default location to stat is the current working directory, configured this * location here */ statfsLocation?: string } export interface PrometheusCalculatedMetricOptions extends CalculatedMetricOptions { registry?: Registry } export interface PrometheusCalculatedHistogramOptions extends CalculatedHistogramOptions { registry?: Registry } export interface PrometheusCalculatedSummaryOptions extends CalculatedSummaryOptions { registry?: Registry } export interface PrometheusMetricsComponents { logger: ComponentLogger } class PrometheusMetrics implements Metrics { private readonly log: Logger private transferStats: Map private readonly registry?: Registry constructor (components: PrometheusMetricsComponents, init?: Partial) { this.log = components.logger.forComponent('libp2p:prometheus-metrics') this.registry = init?.registry if (init?.preserveExistingMetrics !== true) { this.log('Clearing existing metrics') metrics.clear() register?.clear() } if (init?.collectDefaultMetrics !== false) { this.log('Collecting default metrics') collectDefaultMetrics({ ...init?.defaultMetrics, register: this.registry ?? init?.defaultMetrics?.register }) } // holds global and per-protocol sent/received stats this.transferStats = new Map() this.log('Collecting data transfer metrics') this.registerCounterGroup('libp2p_data_transfer_bytes_total', { label: 'protocol', calculate: () => { const output: Record = {} for (const [key, value] of this.transferStats.entries()) { output[key] = value } // reset counts for next time this.transferStats.clear() return output } }) this.log('Collecting memory metrics') this.registerMetricGroup('nodejs_memory_usage_bytes', { label: 'memory', calculate: () => { return { ...process.memoryUsage() } } }) const totalMemoryMetric = this.registerMetric('nodejs_memory_total_bytes') totalMemoryMetric.update(totalmem()) this.log('Collecting filesystem metrics') this.registerMetricGroup('nodejs_fs_usage_bytes', { label: 'filesystem', calculate: async () => { const stats = await statfs(init?.statfsLocation ?? process.cwd()) const total = stats.bsize * stats.blocks const available = stats.bsize * stats.bavail return { total, free: stats.bsize * stats.bfree, available, used: (available / total) * 100 } } }) } readonly [Symbol.toStringTag] = '@libp2p/metrics-prometheus' readonly [serviceCapabilities]: string[] = [ '@libp2p/metrics' ] start (): void { } stop (): void { this.transferStats.clear() } /** * Increment the transfer stat for the passed key, making sure * it exists first */ _incrementValue (key: string, value: number): void { const existing = this.transferStats.get(key) ?? 0 this.transferStats.set(key, existing + value) } /** * Override the sink/source of the stream to count the bytes * in and out */ _track (stream: MessageStream, name: string): void { // Do not attach a `message` listener here. // // AbstractMessageStream dispatches buffered inbound data only when there is // at least one `message` listener. Adding a metrics listener too early can // cause the read buffer to be consumed before protocol handlers attach their // own readers (e.g. identify), leading to lost first frames. // // Instead, wrap dispatchEvent to observe already-dispatched message events // without increasing listenerCount('message'). const dispatchEvent = stream.dispatchEvent.bind(stream) stream.dispatchEvent = (evt: Event): boolean => { if (evt.type === 'message') { this._incrementValue(`${name} received`, (evt as MessageEvent<{ byteLength: number }>).data.byteLength) } return dispatchEvent(evt) } const send = stream.send.bind(stream) stream.send = (buf) => { this._incrementValue(`${name} sent`, buf.byteLength) return send(buf) } } trackMultiaddrConnection (maConn: MultiaddrConnection): void { this._track(maConn, 'global') } trackProtocolStream (stream: Stream): void { if (stream.protocol == null) { // protocol not negotiated yet, should not happen as the upgrader // calls this handler after protocol negotiation return } this._track(stream, stream.protocol) } registerMetric (name: string, opts: PrometheusCalculatedMetricOptions): void registerMetric (name: string, opts?: MetricOptions): Metric registerMetric (name: string, opts: any = {}): any { if (name == null || name.trim() === '') { throw new Error('Metric name is required') } let metric = metrics.get(name) if (metric != null) { this.log('reuse existing metric', name) return metric } this.log('register metric', name) metric = new PrometheusMetric(name, { registry: this.registry, ...opts }) metrics.set(name, metric) if (opts.calculate == null) { return metric } } registerMetricGroup (name: string, opts: PrometheusCalculatedMetricOptions>): void registerMetricGroup (name: string, opts?: MetricOptions): MetricGroup registerMetricGroup (name: string, opts: any = {}): any { if (name == null || name.trim() === '') { throw new Error('Metric group name is required') } let metricGroup = metrics.get(name) if (metricGroup != null) { this.log('reuse existing metric', name) return metricGroup } this.log('register metric group', name) metricGroup = new PrometheusMetricGroup(name, { registry: this.registry, ...opts }) metrics.set(name, metricGroup) if (opts.calculate == null) { return metricGroup } } registerCounter (name: string, opts: PrometheusCalculatedMetricOptions): void registerCounter (name: string, opts?: MetricOptions): Counter registerCounter (name: string, opts: any = {}): any { if (name == null || name.trim() === '') { throw new Error('Counter name is required') } let counter = metrics.get(name) if (counter != null) { this.log('reuse existing counter', name) return counter } this.log('register counter', name) counter = new PrometheusCounter(name, { registry: this.registry, ...opts }) metrics.set(name, counter) if (opts.calculate == null) { return counter } } registerCounterGroup (name: string, opts: PrometheusCalculatedMetricOptions>): void registerCounterGroup (name: string, opts?: MetricOptions): CounterGroup registerCounterGroup (name: string, opts: any = {}): any { if (name == null || name.trim() === '') { throw new Error('Counter group name is required') } let counterGroup = metrics.get(name) if (counterGroup != null) { this.log('reuse existing counter group', name) return counterGroup } this.log('register counter group', name) counterGroup = new PrometheusCounterGroup(name, { registry: this.registry, ...opts }) metrics.set(name, counterGroup) if (opts.calculate == null) { return counterGroup } } registerHistogram (name: string, opts: PrometheusCalculatedHistogramOptions): void registerHistogram (name: string, opts?: HistogramOptions): Histogram registerHistogram (name: string, opts: any = {}): any { if (name == null || name.trim() === '') { throw new Error('Histogram name is required') } let metric = metrics.get(name) if (metric != null) { this.log('reuse existing histogram', name) return metric } this.log('register histogram', name) metric = new PrometheusHistogram(name, { registry: this.registry, ...opts }) metrics.set(name, metric) if (opts.calculate == null) { return metric } } registerHistogramGroup (name: string, opts: PrometheusCalculatedHistogramOptions>): void registerHistogramGroup (name: string, opts?: HistogramOptions): HistogramGroup registerHistogramGroup (name: string, opts: any = {}): any { if (name == null || name.trim() === '') { throw new Error('Histogram group name is required') } let metricGroup = metrics.get(name) if (metricGroup != null) { this.log('reuse existing histogram group', name) return metricGroup } this.log('register histogram group', name) metricGroup = new PrometheusHistogramGroup(name, { registry: this.registry, ...opts }) metrics.set(name, metricGroup) if (opts.calculate == null) { return metricGroup } } registerSummary (name: string, opts: PrometheusCalculatedSummaryOptions): void registerSummary (name: string, opts?: SummaryOptions): Summary registerSummary (name: string, opts: any = {}): any { if (name == null || name.trim() === '') { throw new Error('Summary name is required') } let metric = metrics.get(name) if (metric != null) { this.log('reuse existing summary', name) return metric } this.log('register summary', name) metric = new PrometheusSummary(name, { registry: this.registry, ...opts }) metrics.set(name, metric) if (opts.calculate == null) { return metric } } registerSummaryGroup (name: string, opts: PrometheusCalculatedSummaryOptions>): void registerSummaryGroup (name: string, opts?: SummaryOptions): SummaryGroup registerSummaryGroup (name: string, opts: any = {}): any { if (name == null || name.trim() === '') { throw new Error('Summary group name is required') } let metricGroup = metrics.get(name) if (metricGroup != null) { this.log('reuse existing summary group', name) return metricGroup } this.log('register summary group', name) metricGroup = new PrometheusSummaryGroup(name, { registry: this.registry, ...opts }) metrics.set(name, metricGroup) if (opts.calculate == null) { return metricGroup } } createTrace (): any { // no-op } traceFunction any> (name: string, fn: T): T { // no-op return fn } } export function prometheusMetrics (init?: Partial): (components: PrometheusMetricsComponents) => Metrics { return (components) => { return new PrometheusMetrics(components, init) } }