{ "__inputs": [ { "name": "DS_PROMETHEUS", "label": "Prometheus", "description": "", "type": "datasource", "pluginId": "prometheus", "pluginName": "Prometheus" } ], "__elements": [], "__requires": [ { "type": "grafana", "id": "grafana", "name": "Grafana", "version": "8.4.2" }, { "type": "datasource", "id": "prometheus", "name": "Prometheus", "version": "1.0.0" }, { "type": "panel", "id": "text", "name": "Text", "version": "" }, { "type": "panel", "id": "timeseries", "name": "Time series", "version": "" } ], "annotations": { "list": [ { "builtIn": 1, "datasource": "-- Grafana --", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "target": { "limit": 100, "matchAny": false, "tags": [], "type": "dashboard" }, "type": "dashboard" } ] }, "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, "id": null, "iteration": 1661342107287, "links": [ { "asDropdown": true, "icon": "external link", "includeVars": true, "keepTime": true, "tags": ["lodestar"], "targetBlank": false, "title": "Lodestar dashboards", "tooltip": "", "type": "dashboards", "url": "" } ], "liveNow": false, "panels": [ { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 92, "panels": [], "title": "BLS worker pool", "type": "row" }, { "gridPos": { "h": 3, "w": 24, "x": 0, "y": 1 }, "id": 154, "options": { "content": "Verifies signature sets in a thread pool of workers. Must ensure that signatures are verified fast and efficiently.", "mode": "markdown" }, "pluginVersion": "8.4.2", "targets": [ { "expr": "rate(lodestar_bls_thread_pool_time_seconds_sum[$rate_interval])", "interval": "", "legendFormat": "{{workerId}}", "refId": "A" } ], "title": "About", "type": "text" }, { "description": "Utilization rate = total CPU time per worker per second. Graph is stacked. This ratios should be high since BLS verification is the limiting factor in the node's throughput.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 4 }, "id": 94, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "8.4.0-beta1", "targets": [ { "expr": "rate(lodestar_bls_thread_pool_time_seconds_sum[$rate_interval])", "interval": "", "legendFormat": "{{workerId}}", "refId": "A" } ], "title": "BLS thread pool - utilization rate", "type": "timeseries" }, { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 4 }, "id": 519, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": false, "expr": "rate(lodestar_bls_aggregated_pubkeys_total[$rate_interval])", "interval": "", "legendFormat": "pubkeys / sec", "refId": "A" } ], "title": "Aggregated pubkeys / sec", "type": "timeseries" }, { "description": "Average sync time to validate a single signature set. Note that the set may have been verified in batch. In most normal hardware this value should be ~1-2ms", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 23, "gradientMode": "opacity", "hideFrom": { "graph": false, "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 12 }, "id": 151, "options": { "graph": {}, "legend": { "calcs": [], "displayMode": "hidden", "placement": "bottom" }, "tooltip": { "mode": "single", "sort": "none" }, "tooltipOptions": { "mode": "multi" } }, "pluginVersion": "7.4.5", "targets": [ { "expr": "sum(delta(lodestar_bls_thread_pool_time_seconds_sum[$rate_interval]))/sum(delta(lodestar_bls_thread_pool_success_jobs_signature_sets_count[$rate_interval]))", "interval": "", "legendFormat": "pool", "refId": "A" } ], "title": "BLS thread pool - sig set verification time / set", "type": "timeseries" }, { "description": "Raw throughput of the thread pool. How many individual signature sets are successfully validated per second", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 23, "gradientMode": "opacity", "hideFrom": { "graph": false, "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 12 }, "id": 96, "options": { "graph": {}, "legend": { "calcs": [], "displayMode": "hidden", "placement": "bottom" }, "tooltip": { "mode": "single", "sort": "none" }, "tooltipOptions": { "mode": "multi" } }, "pluginVersion": "7.4.5", "targets": [ { "expr": "rate(lodestar_bls_thread_pool_success_jobs_signature_sets_count[$rate_interval])", "interval": "", "legendFormat": "pool", "refId": "A" } ], "title": "BLS thread pool - sig sets / sec", "type": "timeseries" }, { "description": "Total length of the job queue. Note: this queue is not bounded", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "opacity", "hideFrom": { "graph": false, "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 5, "w": 12, "x": 0, "y": 20 }, "id": 150, "options": { "graph": {}, "legend": { "calcs": [], "displayMode": "hidden", "placement": "bottom" }, "tooltip": { "mode": "single", "sort": "none" }, "tooltipOptions": { "mode": "multi" } }, "pluginVersion": "7.4.5", "targets": [ { "expr": "lodestar_bls_thread_pool_queue_length", "interval": "", "legendFormat": "pool", "refId": "A" } ], "title": "BLS thread pool - queue length", "type": "timeseries" }, { "description": "How much async time job spent waiting in the job queue before being picked up. This number should be really low <100ms to ensure signatures are validated fast.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "opacity", "hideFrom": { "graph": false, "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 20 }, "id": 95, "options": { "graph": {}, "legend": { "calcs": [], "displayMode": "hidden", "placement": "bottom" }, "tooltip": { "mode": "single", "sort": "none" }, "tooltipOptions": { "mode": "multi" } }, "pluginVersion": "7.4.5", "targets": [ { "expr": "delta(lodestar_bls_thread_pool_queue_job_wait_time_seconds_sum[$rate_interval])/delta(lodestar_bls_thread_pool_queue_job_wait_time_seconds_count[$rate_interval])", "interval": "", "legendFormat": "pool", "refId": "A" } ], "title": "BLS thread pool - job wait time", "type": "timeseries" }, { "description": "Async time from sending a message to the worker and the worker receiving it.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 23, "gradientMode": "opacity", "hideFrom": { "graph": false, "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "log": 2, "type": "log" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 6, "w": 12, "x": 0, "y": 25 }, "id": 148, "options": { "graph": {}, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "7.4.5", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": false, "expr": "delta(lodestar_bls_thread_pool_latency_to_worker_sum[6m])/delta(lodestar_bls_thread_pool_latency_to_worker_count[6m])", "interval": "", "legendFormat": "to worker", "refId": "to worker" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": false, "expr": "delta(lodestar_bls_thread_pool_latency_from_worker_sum[6m])/delta(lodestar_bls_thread_pool_latency_from_worker_count[6m])", "hide": false, "interval": "", "legendFormat": "from worker", "refId": "from worker" } ], "title": "BLS worker pool - to from worker latency", "type": "timeseries" }, { "description": "What percentage of total signature sets were verified in batch, which is an optimization to reduce verification costs by x2. For a synced node this should be ~100%", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "opacity", "hideFrom": { "graph": false, "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 28 }, "id": 147, "options": { "graph": {}, "legend": { "calcs": [], "displayMode": "hidden", "placement": "bottom" }, "tooltip": { "mode": "single", "sort": "none" }, "tooltipOptions": { "mode": "multi" } }, "pluginVersion": "7.4.5", "targets": [ { "expr": "delta(lodestar_bls_thread_pool_batch_sigs_success_total[$rate_interval])/delta(lodestar_bls_thread_pool_success_jobs_signature_sets_count[$rate_interval])", "interval": "", "legendFormat": "pool", "refId": "A" } ], "title": "BLS thread pool - % of sigs verified in batch", "type": "timeseries" }, { "description": "Average signatures per set. This number is decided by the time of object submitted to the pool:\n- Sync blocks: 128\n- Aggregates: 3\n- Attestations: 1", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 19, "gradientMode": "opacity", "hideFrom": { "graph": false, "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 5, "w": 12, "x": 0, "y": 31 }, "id": 98, "options": { "graph": {}, "legend": { "calcs": [], "displayMode": "hidden", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "7.4.5", "targets": [ { "expr": "delta(lodestar_bls_thread_pool_sig_sets_started_total[$rate_interval])/(delta(lodestar_bls_thread_pool_jobs_started_total[$rate_interval])>0)", "interval": "", "legendFormat": "pool", "refId": "A" } ], "title": "BLS worker pool - sigs per set", "type": "timeseries" }, { "description": "How many individual signature sets are invalid vs (valid + invalid). We don't control this number since peers may send us invalid signatures. This number should be very low since we should ban bad peers. If it's too high the batch optimization may not be worth it.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [ { "matcher": { "id": "byName", "options": "pool" }, "properties": [ { "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 35 }, "id": 153, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "8.4.0-beta1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": false, "expr": "delta(lodestar_bls_thread_pool_error_jobs_signature_sets_count[$rate_interval])\n/\n(\n delta(lodestar_bls_thread_pool_success_jobs_signature_sets_count[$rate_interval])\n + delta(lodestar_bls_thread_pool_error_jobs_signature_sets_count[$rate_interval])\n)", "interval": "", "legendFormat": "sig_error_rate", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": false, "expr": "rate(lodestar_bls_thread_pool_batch_retries_total[$rate_interval])", "hide": false, "interval": "", "legendFormat": "batch_retries", "refId": "B" } ], "title": "BLS thread pool - Error rates", "type": "timeseries" }, { "description": "Average sets per job. A set may contain +1 signatures. This number should be higher than 1 to reduce communication costs", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 17, "gradientMode": "opacity", "hideFrom": { "graph": false, "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 6, "w": 12, "x": 0, "y": 36 }, "id": 97, "options": { "graph": {}, "legend": { "calcs": [], "displayMode": "hidden", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "7.4.5", "targets": [ { "expr": "delta(lodestar_bls_thread_pool_jobs_started_total[$rate_interval])/delta(lodestar_bls_thread_pool_job_groups_started_total[$rate_interval])", "interval": "", "legendFormat": "pool", "refId": "A" } ], "title": "BLS worker pool - sets per job", "type": "timeseries" } ], "refresh": "10s", "schemaVersion": 35, "style": "dark", "tags": ["lodestar"], "templating": { "list": [ { "current": { "selected": false, "text": "default", "value": "default" }, "hide": 0, "includeAll": false, "label": "datasource", "multi": false, "name": "DS_PROMETHEUS", "options": [], "query": "prometheus", "queryValue": "", "refresh": 1, "regex": "", "skipUrlSync": false, "type": "datasource" }, { "auto": true, "auto_count": 30, "auto_min": "10s", "current": { "selected": false, "text": "30m", "value": "30m" }, "hide": 0, "label": "rate() interval", "name": "rate_interval", "options": [ { "selected": false, "text": "auto", "value": "$__auto_interval_rate_interval" }, { "selected": false, "text": "1m", "value": "1m" }, { "selected": false, "text": "10m", "value": "10m" }, { "selected": true, "text": "30m", "value": "30m" }, { "selected": false, "text": "1h", "value": "1h" }, { "selected": false, "text": "6h", "value": "6h" }, { "selected": false, "text": "12h", "value": "12h" }, { "selected": false, "text": "1d", "value": "1d" }, { "selected": false, "text": "7d", "value": "7d" }, { "selected": false, "text": "14d", "value": "14d" }, { "selected": false, "text": "30d", "value": "30d" } ], "query": "1m,10m,30m,1h,6h,12h,1d,7d,14d,30d", "queryValue": "", "refresh": 2, "skipUrlSync": false, "type": "interval" }, { "datasource": { "type": "prometheus", "uid": "prometheus_local" }, "filters": [ { "condition": "", "key": "instance", "operator": "=", "value": "unstable-lg1k-hzax41" } ], "hide": 0, "name": "Filters", "skipUrlSync": false, "type": "adhoc" } ] }, "time": { "from": "now-24h", "to": "now" }, "timepicker": { "refresh_intervals": ["5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d"] }, "timezone": "", "title": "Lodestar - BLS thread pool", "uid": "lodestar_bls_thread_pool", "version": 2, "weekStart": "" }