{ "__inputs": [ { "name": "DS_PROMETHEUS", "label": "Prometheus", "description": "", "type": "datasource", "pluginId": "prometheus", "pluginName": "Prometheus" } ], "__elements": [], "__requires": [ { "type": "grafana", "id": "grafana", "name": "Grafana", "version": "8.4.2" }, { "type": "panel", "id": "heatmap", "name": "Heatmap", "version": "" }, { "type": "datasource", "id": "prometheus", "name": "Prometheus", "version": "1.0.0" }, { "type": "panel", "id": "stat", "name": "Stat", "version": "" }, { "type": "panel", "id": "text", "name": "Text", "version": "" }, { "type": "panel", "id": "timeseries", "name": "Time series", "version": "" } ], "annotations": { "list": [ { "builtIn": 1, "datasource": "-- Grafana --", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "target": { "limit": 100, "matchAny": false, "tags": [], "type": "dashboard" }, "type": "dashboard" } ] }, "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, "id": null, "iteration": 1662553761426, "links": [ { "asDropdown": true, "icon": "external link", "includeVars": true, "keepTime": true, "tags": ["lodestar"], "targetBlank": false, "title": "Lodestar dashboards", "tooltip": "", "type": "dashboards", "url": "" } ], "liveNow": false, "panels": [ { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 10, "panels": [], "title": "Node summary", "type": "row" }, { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 30, "gradientMode": "opacity", "hideFrom": { "graph": false, "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "locale" }, "overrides": [ { "matcher": { "id": "byName", "options": "(time() - lodestar_genesis_time) / 12" }, "properties": [ { "id": "color", "value": { "fixedColor": "rgba(48, 208, 255, 0.51)", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "process restart" }, "properties": [ { "id": "color", "value": { "fixedColor": "dark-red", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "process restart" }, "properties": [ { "id": "decimals", "value": 0 }, { "id": "min", "value": 0.1 }, { "id": "max", "value": 1 }, { "id": "custom.axisPlacement", "value": "right" }, { "id": "unit", "value": "none" }, { "id": "mappings", "value": [ { "options": { "from": 0, "result": { "text": "" }, "to": 1 }, "type": "range" } ] } ] }, { "matcher": { "id": "byName", "options": "head slot" }, "properties": [ { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "clock slot true" }, "properties": [ { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "clock slot" }, "properties": [ { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 1 }, "id": 2, "options": { "graph": {}, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "7.4.5", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": false, "expr": "(time() - lodestar_genesis_time) / 12", "hide": false, "instant": false, "interval": "", "legendFormat": "clock slot", "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": false, "expr": "beacon_head_slot", "hide": false, "interval": "", "legendFormat": "head slot", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "changes(process_start_time_seconds[5m])", "hide": false, "interval": "", "legendFormat": "process restart", "refId": "B" } ], "title": "Slot", "type": "timeseries" }, { "fieldConfig": { "defaults": { "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] }, "gridPos": { "h": 3, "w": 2, "x": 12, "y": 1 }, "id": 18, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["last"], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "pluginVersion": "8.4.2", "targets": [ { "expr": "sum(lodestar_peers_by_direction_count)", "interval": "", "legendFormat": "", "refId": "A" } ], "title": "Peer Count", "transformations": [], "type": "stat" }, { "fieldConfig": { "defaults": { "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] }, "gridPos": { "h": 3, "w": 2, "x": 14, "y": 1 }, "id": 4, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "pluginVersion": "8.4.2", "targets": [ { "expr": "beacon_head_slot", "interval": "", "legendFormat": "", "refId": "A" } ], "title": "Head Slot", "type": "stat" }, { "fieldConfig": { "defaults": { "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] }, "gridPos": { "h": 3, "w": 2, "x": 16, "y": 1 }, "id": 20, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "pluginVersion": "8.4.2", "targets": [ { "expr": "beacon_finalized_epoch", "interval": "", "legendFormat": "", "refId": "A" } ], "title": "Finalized Epoch", "type": "stat" }, { "fieldConfig": { "defaults": { "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] }, "gridPos": { "h": 3, "w": 2, "x": 18, "y": 1 }, "id": 22, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "pluginVersion": "8.4.2", "targets": [ { "exemplar": false, "expr": "validator_monitor_validators", "interval": "", "legendFormat": "", "refId": "A" } ], "title": "Validators connected", "type": "stat" }, { "fieldConfig": { "defaults": { "mappings": [], "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 3, "w": 2, "x": 20, "y": 1 }, "id": 72, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "text": {}, "textMode": "value" }, "pluginVersion": "8.4.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": false, "expr": "rate(process_cpu_user_seconds_total{job=\"beacon\"} [1m])", "interval": "", "legendFormat": "", "refId": "A" } ], "title": "CPU %", "type": "stat" }, { "fieldConfig": { "defaults": { "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "decbytes" }, "overrides": [] }, "gridPos": { "h": 3, "w": 2, "x": 22, "y": 1 }, "id": 73, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "text": {}, "textMode": "value" }, "pluginVersion": "8.4.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": false, "expr": "process_heap_bytes{job=\"beacon\"}", "interval": "", "legendFormat": "", "refId": "A" } ], "title": "Heap", "type": "stat" }, { "fieldConfig": { "defaults": { "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "dateTimeFromNow" }, "overrides": [] }, "gridPos": { "h": 2, "w": 2, "x": 12, "y": 4 }, "id": 52, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "pluginVersion": "8.4.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": false, "expr": "process_start_time_seconds{job=\"beacon\"}*1000", "instant": false, "interval": "", "legendFormat": "", "refId": "A" } ], "title": "Uptime since", "type": "stat" }, { "fieldConfig": { "defaults": { "mappings": [ { "options": { "0": { "text": "Stalled" }, "1": { "text": "SyncingFinalized" }, "2": { "text": "SyncingHead" }, "3": { "text": "Synced" } }, "type": "value" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 2, "w": 2, "x": 14, "y": 4 }, "id": 101, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "text": {}, "textMode": "value" }, "pluginVersion": "8.4.2", "targets": [ { "expr": "lodestar_sync_status", "interval": "", "legendFormat": "", "refId": "A" } ], "title": "Sync status", "type": "stat" }, { "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 2, "w": 2, "x": 16, "y": 4 }, "id": 71, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "text": {}, "textMode": "name" }, "pluginVersion": "8.4.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": false, "expr": "lodestar_version{job=\"beacon\"}", "format": "time_series", "instant": true, "interval": "", "legendFormat": "{{network}}", "refId": "A" } ], "title": "Network", "type": "stat" }, { "fieldConfig": { "defaults": { "color": { "fixedColor": "green", "mode": "fixed" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [] } }, "overrides": [] }, "gridPos": { "h": 2, "w": 4, "x": 18, "y": 4 }, "id": 54, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "text": {}, "textMode": "name" }, "pluginVersion": "8.4.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": false, "expr": "lodestar_version{job=\"beacon\"}", "format": "time_series", "instant": true, "interval": "", "legendFormat": "{{version}}", "refId": "A" } ], "title": "Lodestar version", "transformations": [], "type": "stat" }, { "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 2, "w": 2, "x": 22, "y": 4 }, "id": 56, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "text": {}, "textMode": "name" }, "pluginVersion": "8.4.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": false, "expr": "nodejs_version_info{job=\"beacon\"}", "instant": true, "interval": "", "legendFormat": "{{version}}", "refId": "A" } ], "title": "NodeJS", "type": "stat" }, { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 6 }, "id": 21, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "8.4.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "lodestar_peers_by_direction_count", "hide": false, "interval": "", "legendFormat": "{{direction}}", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": false, "expr": "lodestar_gossip_peer_score_by_threshold_count{threshold=\"mesh\"}", "hide": false, "interval": "", "legendFormat": "with_gossip_mesh_score", "refId": "A" } ], "title": "Peer count", "type": "timeseries" }, { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 17, "gradientMode": "opacity", "hideFrom": { "graph": false, "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 4, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 9 }, "id": 196, "links": [], "options": { "graph": {}, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "single", "sort": "none" } }, "pluginVersion": "7.4.5", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": false, "expr": "sum(rate(validator_monitor_prev_epoch_on_chain_attester_miss_total[$rate_interval]))\n/\n(\n sum(rate(validator_monitor_prev_epoch_on_chain_attester_hit_total[$rate_interval]))\n +\n sum(rate(validator_monitor_prev_epoch_on_chain_attester_miss_total[$rate_interval]))\n) > 0", "interval": "", "legendFormat": "attester", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": false, "expr": "sum(rate(validator_monitor_prev_epoch_on_chain_target_attester_miss_total[$rate_interval]))\n/\n(\n sum(rate(validator_monitor_prev_epoch_on_chain_target_attester_hit_total[$rate_interval]))\n +\n sum(rate(validator_monitor_prev_epoch_on_chain_target_attester_miss_total[$rate_interval]))\n) > 0", "hide": false, "interval": "", "legendFormat": "target", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": false, "expr": "sum(rate(validator_monitor_prev_epoch_on_chain_head_attester_miss_total[$rate_interval]))\n/\n(\n sum(rate(validator_monitor_prev_epoch_on_chain_head_attester_hit_total[$rate_interval]))\n +\n sum(rate(validator_monitor_prev_epoch_on_chain_head_attester_miss_total[$rate_interval]))\n) > 0", "hide": false, "interval": "", "legendFormat": "head", "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": false, "expr": "1 - avg(validator_monitor_prev_epoch_on_chain_attester_correct_head_total)", "hide": false, "interval": "", "legendFormat": "wrong_head_ratio", "refId": "D" } ], "title": "Validators prev epoch miss ratio", "type": "timeseries" }, { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 6, "w": 12, "x": 12, "y": 13 }, "id": 528, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "8.4.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": false, "expr": "avg(lodestar_gossip_mesh_peers_by_type_count) by (instance)", "hide": false, "interval": "", "legendFormat": "core_topics_avg", "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": false, "expr": "avg(lodestar_gossip_mesh_peers_by_beacon_attestation_subnet_count > 0) by (instance)", "hide": false, "interval": "", "legendFormat": "attestation_subnets_avg", "refId": "A" } ], "title": "Mesh peers on core topic", "type": "timeseries" }, { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "GWei", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 5, "w": 12, "x": 0, "y": 17 }, "id": 534, "options": { "legend": { "calcs": [], "displayMode": "hidden", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "8.4.0-beta1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": false, "expr": "avg(\n delta(validator_monitor_prev_epoch_on_chain_balance_total[32m])\n)", "hide": false, "interval": "", "legendFormat": "balance_delta", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": false, "expr": "avg(\n delta(validator_monitor_prev_epoch_on_chain_balance[32m])\n)", "hide": false, "interval": "", "legendFormat": "balance_delta_new", "refId": "B" } ], "title": "balance delta prev epoch", "type": "timeseries" }, { "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 3, "w": 12, "x": 12, "y": 19 }, "id": 527, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "center", "orientation": "vertical", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "value_and_name" }, "pluginVersion": "8.4.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": false, "expr": "lodestar_peers_by_client_count", "hide": false, "instant": false, "interval": "", "legendFormat": "{{client}}", "refId": "B" } ], "type": "stat" }, { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "log": 2, "type": "log" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 22 }, "id": 530, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": false, "expr": "rate(beacon_fork_choice_find_head_seconds_sum[$rate_interval])", "interval": "", "legendFormat": "fork_choice_find_head", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": false, "expr": "rate(lodestar_block_processor_queue_job_time_seconds_sum[$rate_interval])", "hide": false, "interval": "", "legendFormat": "block_processor", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": false, "expr": "rate(lodestar_stfn_epoch_transition_seconds_sum[$rate_interval])", "hide": false, "interval": "", "legendFormat": "stfn_epoch", "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": false, "expr": "rate(lodestar_stfn_process_block_seconds_sum[$rate_interval])", "hide": false, "interval": "", "legendFormat": "stfn_block", "refId": "D" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": false, "expr": "sum(rate(nodejs_gc_duration_seconds_sum{job=\"beacon\"} [$rate_interval])) by (instance)", "hide": false, "interval": "", "legendFormat": "gc_duration_sum", "refId": "E" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": false, "expr": "sum(rate(nodejs_gc_pause_seconds_total{job=\"beacon\"} [$rate_interval])) by (instance)", "hide": false, "interval": "", "legendFormat": "gc_pause_sum", "refId": "G" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": false, "expr": "sum(rate(lodestar_bls_thread_pool_time_seconds_sum[$rate_interval])) by (instance)", "hide": false, "interval": "", "legendFormat": "bls_thread_pool_sum", "refId": "F" } ], "title": "Usage rate (time)", "type": "timeseries" }, { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 22 }, "id": 532, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": false, "expr": "rate(beacon_reqresp_outgoing_request_roundtrip_time_seconds_sum[$rate_interval])\n/\nrate(beacon_reqresp_outgoing_request_roundtrip_time_seconds_count[$rate_interval])", "interval": "", "legendFormat": "reqresp_outgoing_roundtrip", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": false, "expr": "rate(lodestar_eth1_http_client_request_time_seconds_sum{routeId=\"getBlockNumber\"}[$rate_interval])\n/\nrate(lodestar_eth1_http_client_request_time_seconds_count{routeId=\"getBlockNumber\"}[$rate_interval])", "hide": false, "interval": "", "legendFormat": "eth1_getBlockNumber_roundtrip", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": false, "expr": "avg_over_time( scrape_duration_seconds{job=\"beacon\"} [$rate_interval])", "hide": false, "interval": "", "legendFormat": "prometheus_beacon_scrape_roundtrip", "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": false, "expr": "avg_over_time( scrape_duration_seconds{job=\"validator\"} [$rate_interval])", "hide": false, "interval": "", "legendFormat": "prometheus_validator_scrape_roundtrip", "refId": "E" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": false, "expr": "rate(vc_rest_api_client_request_time_seconds_sum{routeId=\"produceAttestationData\"}[$rate_interval])\n/\nrate(vc_rest_api_client_request_time_seconds_count{routeId=\"produceAttestationData\"}[$rate_interval])", "hide": false, "interval": "", "legendFormat": "rest_produceAttestation_roundtrip", "refId": "D" } ], "title": "Roundtrip reqest times - I/O lag", "type": "timeseries" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 30 }, "id": 487, "panels": [ { "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 3, "w": 4, "x": 0, "y": 34 }, "id": 493, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "8.4.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": false, "expr": "beacon_head_slot", "interval": "", "legendFormat": "", "refId": "A" } ], "title": "head slot", "type": "stat" }, { "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 3, "w": 4, "x": 4, "y": 34 }, "id": 492, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "8.4.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": false, "expr": "beacon_finalized_epoch", "interval": "", "legendFormat": "", "refId": "A" } ], "title": "finalized epoch", "type": "stat" }, { "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 3, "w": 4, "x": 8, "y": 34 }, "id": 484, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "8.4.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": false, "expr": "beacon_previous_justified_epoch", "interval": "", "legendFormat": "", "refId": "A" } ], "title": "previous justified epoch", "type": "stat" }, { "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 3, "w": 4, "x": 12, "y": 34 }, "id": 483, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "8.4.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": false, "expr": "beacon_current_justified_epoch", "interval": "", "legendFormat": "", "refId": "A" } ], "title": "current justified epoch", "type": "stat" }, { "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 3, "w": 4, "x": 16, "y": 34 }, "id": 485, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "8.4.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": false, "expr": "beacon_current_validators{status=\"active\"}", "hide": false, "interval": "", "legendFormat": "", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": false, "expr": "beacon_current_active_validators", "hide": true, "interval": "", "legendFormat": "", "refId": "A" } ], "title": "current active validators", "type": "stat" }, { "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 3, "w": 4, "x": 20, "y": 34 }, "id": 488, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "8.4.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": false, "expr": "beacon_processed_deposits_total", "interval": "", "legendFormat": "", "refId": "A" } ], "title": "processed deposits", "type": "stat" }, { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 37 }, "id": 490, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": false, "expr": "delta(beacon_fork_choice_reorg_total[$rate_interval])", "interval": "", "legendFormat": "", "refId": "A" } ], "title": "Reorgs", "type": "timeseries" }, { "cards": {}, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateMagma", "exponent": 0.5, "mode": "spectrum" }, "dataFormat": "tsbuckets", "gridPos": { "h": 8, "w": 12, "x": 12, "y": 37 }, "heatmap": {}, "hideZeroBuckets": false, "highlightCards": true, "id": 491, "legend": { "show": false }, "reverseYBuckets": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": false, "expr": "rate(beacon_fork_choice_reorg_distance_bucket[$rate_interval])", "format": "heatmap", "interval": "", "intervalFactor": 10, "legendFormat": "{{le}}", "refId": "A" } ], "title": "Reorgs depth", "tooltip": { "show": true, "showHistogram": false }, "type": "heatmap", "xAxis": { "show": true }, "yAxis": { "format": "short", "logBase": 1, "show": true }, "yBucketBound": "auto" }, { "cards": {}, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateMagma", "exponent": 0.5, "mode": "spectrum" }, "dataFormat": "tsbuckets", "gridPos": { "h": 8, "w": 12, "x": 0, "y": 45 }, "heatmap": {}, "hideZeroBuckets": false, "highlightCards": true, "id": 495, "legend": { "show": false }, "reverseYBuckets": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": false, "expr": "12*rate(beacon_imported_block_parent_distance_bucket[$rate_interval])-1", "format": "heatmap", "interval": "", "intervalFactor": 10, "legendFormat": "{{le}}", "refId": "A" } ], "title": "Block parent distance (except 1)", "tooltip": { "show": true, "showHistogram": false }, "type": "heatmap", "xAxis": { "show": true }, "yAxis": { "format": "short", "logBase": 1, "show": true }, "yBucketBound": "auto" }, { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 45 }, "id": 497, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "single", "sort": "none" } }, "title": "Panel Title", "type": "timeseries" } ], "title": "Network status", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 31 }, "id": 104, "panels": [ { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "slots", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "opacity", "hideFrom": { "graph": false, "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 6, "w": 12, "x": 0, "y": 35 }, "id": 102, "options": { "graph": {}, "legend": { "calcs": [], "displayMode": "hidden", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "7.4.5", "targets": [ { "expr": "(time() - lodestar_genesis_time) / 12 - lodestar_clock_slot", "hide": false, "interval": "", "legendFormat": "clock drift", "refId": "B" } ], "title": "Clock drift", "type": "timeseries" }, { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "axisSoftMax": 3, "axisSoftMin": 0, "barAlignment": 0, "drawStyle": "line", "fillOpacity": 33, "gradientMode": "opacity", "hideFrom": { "graph": false, "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineStyle": { "fill": "solid" }, "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 0, "mappings": [ { "options": { "0": { "text": "Stalled" }, "1": { "text": "SyncingFinalized" }, "2": { "text": "SyncingHead" }, "3": { "text": "Synced" } }, "type": "value" } ], "max": 3, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 6, "w": 12, "x": 12, "y": 35 }, "id": 172, "options": { "graph": {}, "legend": { "calcs": [], "displayMode": "hidden", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "7.4.5", "targets": [ { "exemplar": false, "expr": "lodestar_sync_status", "format": "time_series", "instant": false, "interval": "", "legendFormat": "status", "refId": "A" } ], "title": "Sync status", "type": "timeseries" }, { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "slots", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "opacity", "hideFrom": { "graph": false, "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 6, "w": 12, "x": 0, "y": 41 }, "id": 171, "options": { "graph": {}, "legend": { "calcs": [], "displayMode": "hidden", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "7.4.5", "targets": [ { "exemplar": false, "expr": "lodestar_clock_slot - beacon_head_slot", "hide": false, "interval": "", "legendFormat": "clock drift", "refId": "B" } ], "title": "head drift", "type": "timeseries" }, { "gridPos": { "h": 6, "w": 12, "x": 12, "y": 41 }, "id": 99, "options": { "content": "", "mode": "markdown" }, "pluginVersion": "8.4.2", "targets": [ { "exemplar": false, "expr": "lodestar_sync_status", "format": "time_series", "instant": false, "interval": "", "legendFormat": "status", "refId": "A" } ], "title": "-", "type": "text" } ], "title": "Node status", "type": "row" } ], "refresh": "10s", "schemaVersion": 35, "style": "dark", "tags": ["lodestar"], "templating": { "list": [ { "current": { "selected": false, "text": "default", "value": "default" }, "hide": 0, "includeAll": false, "label": "datasource", "multi": false, "name": "DS_PROMETHEUS", "options": [], "query": "prometheus", "queryValue": "", "refresh": 1, "regex": "", "skipUrlSync": false, "type": "datasource" }, { "auto": true, "auto_count": 30, "auto_min": "10s", "current": { "selected": false, "text": "6h", "value": "6h" }, "hide": 0, "label": "rate() interval", "name": "rate_interval", "options": [ { "selected": false, "text": "auto", "value": "$__auto_interval_rate_interval" }, { "selected": false, "text": "1m", "value": "1m" }, { "selected": false, "text": "10m", "value": "10m" }, { "selected": false, "text": "30m", "value": "30m" }, { "selected": false, "text": "1h", "value": "1h" }, { "selected": true, "text": "6h", "value": "6h" }, { "selected": false, "text": "12h", "value": "12h" }, { "selected": false, "text": "1d", "value": "1d" }, { "selected": false, "text": "7d", "value": "7d" }, { "selected": false, "text": "14d", "value": "14d" }, { "selected": false, "text": "30d", "value": "30d" } ], "query": "1m,10m,30m,1h,6h,12h,1d,7d,14d,30d", "queryValue": "", "refresh": 2, "skipUrlSync": false, "type": "interval" }, { "datasource": { "type": "prometheus", "uid": "prometheus_local" }, "filters": [ { "condition": "", "key": "instance", "operator": "=", "value": "unstable-lg1k-hzax41" } ], "hide": 0, "name": "Filters", "skipUrlSync": false, "type": "adhoc" } ] }, "time": { "from": "now-24h", "to": "now" }, "timepicker": { "refresh_intervals": ["5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d"] }, "timezone": "", "title": "Lodestar", "uid": "lodestar", "version": 7, "weekStart": "" }