{ "__requires": [ { "id": "grafana", "name": "Grafana", "type": "grafana", "version": "10.0.0" }, { "id": "prometheus", "name": "Prometheus", "type": "datasource", "version": "1.0.0" }, { "id": "loki", "name": "Loki", "type": "datasource", "version": "1.0.0" }, { "id": "row", "name": "Row", "type": "panel", "version": "" }, { "id": "timeseries", "name": "Time series", "type": "panel", "version": "" }, { "id": "stat", "name": "Stat", "type": "panel", "version": "" }, { "id": "bargauge", "name": "Bar gauge", "type": "panel", "version": "" }, { "id": "table", "name": "Table", "type": "panel", "version": "" }, { "id": "text", "name": "Text", "type": "panel", "version": "" } ], "annotations": { "list": [] }, "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, "id": null, "links": [ { "icon": "external link", "includeVars": true, "keepTime": true, "targetBlank": false, "title": "Overview", "tooltip": "Open the overview with the current time range.", "type": "link", "url": "/d/observme-overview/observme-overview?${__url_time_range}" }, { "icon": "external link", "includeVars": true, "keepTime": true, "targetBlank": false, "title": "SLO Health", "tooltip": "Open SLO scorecards, burn rates, and alert thresholds with the current time range.", "type": "link", "url": "/d/observme-slo-health/observme-slo-health?${__url_time_range}" }, { "icon": "external link", "includeVars": true, "keepTime": true, "targetBlank": false, "title": "Errors", "tooltip": "Open parsed failure logs and trace links.", "type": "link", "url": "/d/observme-errors/observme-errors?${__url_time_range}" }, { "icon": "external link", "includeVars": true, "keepTime": true, "targetBlank": false, "title": "Trace Journey", "tooltip": "Open trace/log drill-downs for propagation failures.", "type": "link", "url": "/d/observme-trace-journey/observme-trace-journey?${__url_time_range}" } ], "liveNow": false, "panels": [ { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 100, "panels": [], "title": "Health", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Recent ObservMe handler activity from observme_events_observed_total. A zero value means the selected range is idle or no matching events have been observed; use the adjacent failure chips to distinguish healthy idle from unhealthy export behavior.", "fieldConfig": { "defaults": { "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "blue", "value": null }, { "color": "green", "value": 1e-9 } ] }, "unit": "ops" }, "overrides": [] }, "gridPos": { "h": 4, "w": 4, "x": 0, "y": 1 }, "id": 10, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "(sum(rate(observme_events_observed_total[$__rate_interval])) or vector(0))", "legendFormat": "observed events", "range": true, "refId": "A" } ], "title": "Observed event liveness", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Composite runtime export state: 1 (green) means active telemetry with no local drops, export errors, or redaction failures; 0.5 (blue) means idle or unobserved telemetry with no failures; values below 1 indicate active failures, and 0 indicates failures without observed-event liveness. Use the adjacent liveness and failure chips to distinguish idle from missing instrumentation.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null }, { "color": "blue", "value": 0.5 }, { "color": "yellow", "value": 0.500001 }, { "color": "green", "value": 1 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 4, "w": 4, "x": 4, "y": 1 }, "id": 5, "links": [ { "targetBlank": false, "title": "Open SLO Health", "url": "/d/observme-slo-health/observme-slo-health?${__url_time_range}" } ], "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "clamp_max(((((sum(rate(observme_events_observed_total[$__rate_interval])) or vector(0)) > bool 0) * clamp_min(1 - (((sum(rate(observme_telemetry_dropped_total[$__rate_interval])) or vector(0)) + (sum(rate(observme_export_errors_total[$__rate_interval])) or vector(0)) + (sum(rate(observme_redaction_failures_total[$__rate_interval])) or vector(0))) / clamp_min((sum(rate(observme_events_observed_total[$__rate_interval])) or vector(0)), 1e-9)), 0)) + (((sum(rate(observme_events_observed_total[$__rate_interval])) or vector(0)) == bool 0) * (((sum(rate(observme_telemetry_dropped_total[$__rate_interval])) or vector(0)) + (sum(rate(observme_export_errors_total[$__rate_interval])) or vector(0)) + (sum(rate(observme_redaction_failures_total[$__rate_interval])) or vector(0))) == bool 0) * 0.5)), 1)", "legendFormat": "clean export score", "range": true, "refId": "A" } ], "title": "Collector/export health", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Local telemetry drop ratio for the selected window, bounded to 100%. Zero means no drops were observed; when event liveness is also zero, treat it as idle/unmeasured rather than proof of active health.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 1e-9 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 4, "w": 4, "x": 8, "y": 1 }, "id": 12, "links": [ { "targetBlank": false, "title": "Open drop logs", "url": "/d/observme-export-health/observme-export-health?${__url_time_range}&viewPanel=8" } ], "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "clamp_max((sum(rate(observme_telemetry_dropped_total[$__rate_interval])) or vector(0)) / clamp_min((sum(rate(observme_events_observed_total[$__rate_interval])) or vector(0)), 1e-9), 1)", "legendFormat": "drop ratio", "range": true, "refId": "A" } ], "title": "Telemetry drop ratio", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Exporter/backend error rate. Zero means no export errors were observed; when event liveness is also zero, treat the range as idle/unmeasured rather than proof of active health.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 1e-9 } ] }, "unit": "ops" }, "overrides": [] }, "gridPos": { "h": 4, "w": 4, "x": 12, "y": 1 }, "id": 13, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "(sum(rate(observme_export_errors_total[$__rate_interval])) or vector(0))", "legendFormat": "export errors", "range": true, "refId": "A" } ], "title": "Export error rate", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Redaction failure rate. Any non-zero value is critical when content capture is enabled. Zero means no failures were observed; when event liveness is also zero, treat the range as idle/unmeasured.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 1e-9 } ] }, "unit": "ops" }, "overrides": [] }, "gridPos": { "h": 4, "w": 4, "x": 16, "y": 1 }, "id": 14, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "(sum(rate(observme_redaction_failures_total[$__rate_interval])) or vector(0))", "legendFormat": "redaction failures", "range": true, "refId": "A" } ], "title": "Redaction failure rate", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Instrumentation overhead chip: p99 handler lifecycle latency should stay under the 10 ms SLO threshold. No histogram observations render as no data rather than a false healthy zero.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 10 } ] }, "unit": "ms" }, "overrides": [] }, "gridPos": { "h": 4, "w": 4, "x": 20, "y": 1 }, "id": 15, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "histogram_quantile(0.99, sum(rate(observme_handler_duration_ms_bucket[$__rate_interval])) by (le))", "legendFormat": "handler p99", "range": true, "refId": "A" } ], "title": "Handler p99 latency", "type": "stat" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 }, "id": 101, "panels": [], "title": "SLOs", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "30-day Observability Export SLO aligned with observme-slos.yaml: 99% of observed events are not locally dropped. The denominator guard makes no events and no drops evaluate to 1; treat that state as idle/unmeasured and confirm liveness instead of assuming a measured healthy workload.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null }, { "color": "yellow", "value": 0.98 }, { "color": "green", "value": 0.99 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 5, "w": 6, "x": 0, "y": 6 }, "id": 16, "links": [ { "targetBlank": false, "title": "Open SLO Health", "url": "/d/observme-slo-health/observme-slo-health?${__url_time_range}" } ], "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "clamp_max(clamp_min(1 - ((sum(rate(observme_telemetry_dropped_total[30d])) or vector(0)) / clamp_min((sum(rate(observme_events_observed_total[30d])) or vector(0)), 1e-9)), 0), 1)", "instant": true, "legendFormat": "export SLO", "range": false, "refId": "A" } ], "title": "Observability Export SLO (30d)", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Instrumentation Overhead SLO scorecard: p99 handler duration over 30 days should stay under 10 ms. No histogram observations render as no data rather than a false healthy zero.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 10 } ] }, "unit": "ms" }, "overrides": [] }, "gridPos": { "h": 5, "w": 6, "x": 6, "y": 6 }, "id": 17, "links": [ { "targetBlank": false, "title": "Open SLO Health", "url": "/d/observme-slo-health/observme-slo-health?${__url_time_range}" } ], "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "histogram_quantile(0.99, sum(rate(observme_handler_duration_ms_bucket[30d])) by (le))", "instant": true, "legendFormat": "handler p99", "range": false, "refId": "A" } ], "title": "Instrumentation Overhead SLO p99 (30d)", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Short-window burn rates normalized by a 1% error budget. Values above 1 consume error budget faster than target.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 1 }, { "color": "red", "value": 2 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 5, "w": 6, "x": 12, "y": 6 }, "id": 18, "options": { "displayMode": "gradient", "maxVizHeight": 300, "minVizHeight": 16, "minVizWidth": 8, "namePlacement": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showUnfilled": true, "sizing": "auto", "valueMode": "color" }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "((sum(rate(observme_telemetry_dropped_total[1h])) or vector(0)) / clamp_min((sum(rate(observme_events_observed_total[1h])) or vector(0)), 1e-9)) / 0.01", "instant": true, "legendFormat": "export drops", "range": false, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "clamp_max(clamp_min(((sum(rate(observme_handler_duration_ms_count[1h])) or vector(0)) - (sum(rate(observme_handler_duration_ms_bucket{le=\"10\"}[1h])) or vector(0))) / clamp_min((sum(rate(observme_handler_duration_ms_count[1h])) or vector(0)), 1e-9), 0), 1) / 0.01", "instant": true, "legendFormat": "handler >10ms", "range": false, "refId": "B" } ], "title": "Runtime SLO burn rate (1h)", "type": "bargauge" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Long-window burn rates normalized by a 1% error budget. Pair with the 1h view to identify sustained SLO risk.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 1 }, { "color": "red", "value": 2 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 5, "w": 6, "x": 18, "y": 6 }, "id": 19, "options": { "displayMode": "gradient", "maxVizHeight": 300, "minVizHeight": 16, "minVizWidth": 8, "namePlacement": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showUnfilled": true, "sizing": "auto", "valueMode": "color" }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "((sum(rate(observme_telemetry_dropped_total[30d])) or vector(0)) / clamp_min((sum(rate(observme_events_observed_total[30d])) or vector(0)), 1e-9)) / 0.01", "instant": true, "legendFormat": "export drops", "range": false, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "clamp_max(clamp_min(((sum(rate(observme_handler_duration_ms_count[30d])) or vector(0)) - (sum(rate(observme_handler_duration_ms_bucket{le=\"10\"}[30d])) or vector(0))) / clamp_min((sum(rate(observme_handler_duration_ms_count[30d])) or vector(0)), 1e-9), 0), 1) / 0.01", "instant": true, "legendFormat": "handler >10ms", "range": false, "refId": "B" } ], "title": "Runtime SLO burn rate (30d)", "type": "bargauge" }, { "description": "Alert thresholds mirrored from dashboards/observme-alerts.yaml for quick operational context.", "gridPos": { "h": 6, "w": 24, "x": 0, "y": 11 }, "id": 20, "options": { "code": { "language": "markdown", "showLineNumbers": false, "showMiniMap": false }, "content": "### Alert threshold reference\n\n- **ObservMeExportDropsDetected:** any local telemetry drops for 5m.\n- **ObservMeRedactionFailures:** any redaction failures for 5m.\n- **ObservMeTraceContextPropagationFailures:** any trace-context propagation failures for 10m.\n- **ObservMeActiveAgentsStuckHigh:** active agents > **100** for 30m.\n- Agent orchestration thresholds: p95 fan-out > **20** and p95 tree depth > **5** for 10m.", "mode": "markdown" }, "pluginVersion": "11.1.0", "title": "Alert threshold reference", "type": "text" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 17 }, "id": 102, "panels": [], "title": "Signals", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Sessions started and shut down in the selected time range. Zero means no matching session lifecycle samples exist for the range.", "fieldConfig": { "defaults": { "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 18 }, "id": 11, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "(sum(increase(observme_sessions_started_total[$__range])) or vector(0))", "instant": true, "legendFormat": "sessions started", "range": false, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "(sum(increase(observme_sessions_shutdown_total[$__range])) or vector(0))", "instant": true, "legendFormat": "sessions shutdown", "range": false, "refId": "B" } ], "title": "Session lifecycle", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Telemetry drop rate grouped by bounded drop reason. The zero fallback means no local drops were observed in the selected range.", "fieldConfig": { "defaults": { "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 1e-9 } ] }, "unit": "ops" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 18 }, "id": 1, "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "(sum by (reason) (rate(observme_telemetry_dropped_total[$__rate_interval])) or on() vector(0))", "legendFormat": "{{reason}}", "range": true, "refId": "A" } ], "title": "Telemetry drops", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Exporter/backend failure rate grouped by bounded reason. The zero fallback means no export failures were observed in the selected range.", "fieldConfig": { "defaults": { "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 1e-9 } ] }, "unit": "ops" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 18 }, "id": 3, "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "(sum by (reason) (rate(observme_export_errors_total[$__rate_interval])) or on() vector(0))", "legendFormat": "{{reason}}", "range": true, "refId": "A" } ], "title": "Export failures", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Redaction failure rate. The zero fallback means no redaction failures were observed in the selected range.", "fieldConfig": { "defaults": { "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 1e-9 } ] }, "unit": "ops" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 26 }, "id": 2, "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "(sum(rate(observme_redaction_failures_total[$__rate_interval])) or vector(0))", "legendFormat": "redaction failures", "range": true, "refId": "A" } ], "title": "Redaction failures", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Active spans by bounded operation as an SDK pressure signal. Zero means no active spans are exposed or the gauge series is absent; consult observed-event liveness before interpreting it as healthy idle.", "fieldConfig": { "defaults": { "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 26 }, "id": 4, "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "(sum by (operation) (observme_active_spans) or on() vector(0))", "legendFormat": "{{operation}}", "range": true, "refId": "A" } ], "title": "Active spans by operation", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Handler error rate grouped by operation and error class. The zero fallback means no handler errors were observed; if event liveness is also zero, the range is idle/unmeasured rather than confirmed healthy.", "fieldConfig": { "defaults": { "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 1e-9 } ] }, "unit": "ops" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 26 }, "id": 6, "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "(sum by (operation) (rate(observme_handler_errors_total[$__rate_interval])) or on() vector(0))", "legendFormat": "{{operation}}", "range": true, "refId": "A" } ], "title": "Handler error pressure", "type": "timeseries" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 34 }, "id": 103, "panels": [], "title": "Logs", "type": "row" }, { "datasource": { "type": "loki", "uid": "loki" }, "description": "Latest redaction.failed events first. Safe deterministic fields expose event, category, execution context, error type, trace, and span while raw lines are hidden. trace_id opens Tempo. Empty means no matching failure logs in the selected time range; this is healthy when no redaction failures occurred.", "fieldConfig": { "defaults": {}, "overrides": [ { "matcher": { "id": "byName", "options": "trace_id" }, "properties": [ { "id": "links", "value": [ { "targetBlank": false, "title": "Open Tempo trace", "url": "/explore?left=%7B%22datasource%22:%22tempo%22,%22queries%22:%5B%7B%22query%22:%22${__value.raw}%22%7D%5D,%22range%22:%7B%22from%22:%22${__from}%22,%22to%22:%22${__to}%22%7D%7D" } ] } ] } ] }, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 35 }, "id": 7, "links": [ { "targetBlank": false, "title": "Open Errors", "url": "/d/observme-errors/observme-errors?${__url_time_range}" } ], "options": { "cellHeight": "sm", "footer": { "countRows": false, "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true, "sortBy": [ { "desc": true, "displayName": "Time" } ] }, "pluginVersion": "11.1.0", "targets": [ { "datasource": { "type": "loki", "uid": "loki" }, "editorMode": "code", "expr": "{service_name=\"observme-pi-extension\", event_name=\"redaction.failed\"} | line_format `{\"event\":{{ printf \"%q\" .event_name }},\"category\":{{ printf \"%q\" .event_category }},\"session\":{{ printf \"%q\" .pi_session_id }},\"workflow\":{{ printf \"%q\" .pi_workflow_id }},\"agent\":{{ printf \"%q\" .pi_agent_id }},\"parent_agent\":{{ printf \"%q\" .pi_agent_parent_id }},\"agent_run\":{{ printf \"%q\" .pi_agent_run_id }},\"error_type\":{{ printf \"%q\" .error_type }},\"trace_id\":{{ printf \"%q\" .trace_id }},\"span_id\":{{ printf \"%q\" .span_id }}}`", "maxLines": 500, "queryType": "range", "range": true, "refId": "A" } ], "title": "Redaction failure logs", "transformations": [ { "id": "labelsToFields", "options": { "mode": "columns" } }, { "id": "extractFields", "options": { "format": "json", "keepTime": true, "replace": false, "source": "Line" } }, { "id": "organize", "options": { "excludeByName": { "Line": true, "id": true, "labels": true, "service_name": true, "telemetry_source": true, "tsNs": true }, "renameByName": { "event_category": "category", "event_name": "event", "pi_agent_id": "agent", "pi_agent_parent_id": "parent_agent", "pi_agent_run_id": "agent_run", "pi_session_id": "session", "pi_workflow_id": "workflow" } } } ], "type": "table" }, { "datasource": { "type": "loki", "uid": "loki" }, "description": "Latest telemetry.dropped events first. Safe deterministic fields expose event, category, execution context, error type, trace, and span while raw lines are hidden. trace_id opens Tempo. Empty means no matching failure logs in the selected time range; this is healthy when no local telemetry drops occurred.", "fieldConfig": { "defaults": {}, "overrides": [ { "matcher": { "id": "byName", "options": "trace_id" }, "properties": [ { "id": "links", "value": [ { "targetBlank": false, "title": "Open Tempo trace", "url": "/explore?left=%7B%22datasource%22:%22tempo%22,%22queries%22:%5B%7B%22query%22:%22${__value.raw}%22%7D%5D,%22range%22:%7B%22from%22:%22${__from}%22,%22to%22:%22${__to}%22%7D%7D" } ] } ] } ] }, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 35 }, "id": 8, "links": [ { "targetBlank": false, "title": "Open Errors", "url": "/d/observme-errors/observme-errors?${__url_time_range}" } ], "options": { "cellHeight": "sm", "footer": { "countRows": false, "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true, "sortBy": [ { "desc": true, "displayName": "Time" } ] }, "pluginVersion": "11.1.0", "targets": [ { "datasource": { "type": "loki", "uid": "loki" }, "editorMode": "code", "expr": "{service_name=\"observme-pi-extension\", event_name=\"telemetry.dropped\"} | line_format `{\"event\":{{ printf \"%q\" .event_name }},\"category\":{{ printf \"%q\" .event_category }},\"session\":{{ printf \"%q\" .pi_session_id }},\"workflow\":{{ printf \"%q\" .pi_workflow_id }},\"agent\":{{ printf \"%q\" .pi_agent_id }},\"parent_agent\":{{ printf \"%q\" .pi_agent_parent_id }},\"agent_run\":{{ printf \"%q\" .pi_agent_run_id }},\"error_type\":{{ printf \"%q\" .error_type }},\"trace_id\":{{ printf \"%q\" .trace_id }},\"span_id\":{{ printf \"%q\" .span_id }}}`", "maxLines": 500, "queryType": "range", "range": true, "refId": "A" } ], "title": "Telemetry drop logs", "transformations": [ { "id": "labelsToFields", "options": { "mode": "columns" } }, { "id": "extractFields", "options": { "format": "json", "keepTime": true, "replace": false, "source": "Line" } }, { "id": "organize", "options": { "excludeByName": { "Line": true, "id": true, "labels": true, "service_name": true, "telemetry_source": true, "tsNs": true }, "renameByName": { "event_category": "category", "event_name": "event", "pi_agent_id": "agent", "pi_agent_parent_id": "parent_agent", "pi_agent_run_id": "agent_run", "pi_session_id": "session", "pi_workflow_id": "workflow" } } } ], "type": "table" }, { "datasource": { "type": "loki", "uid": "loki" }, "description": "Latest export.failed events first. Safe deterministic fields expose event, category, execution context, error type, trace, and span while raw lines are hidden. trace_id opens Tempo. Empty means no matching failure logs in the selected time range; this is healthy when no exporter/backend failures occurred.", "fieldConfig": { "defaults": {}, "overrides": [ { "matcher": { "id": "byName", "options": "trace_id" }, "properties": [ { "id": "links", "value": [ { "targetBlank": false, "title": "Open Tempo trace", "url": "/explore?left=%7B%22datasource%22:%22tempo%22,%22queries%22:%5B%7B%22query%22:%22${__value.raw}%22%7D%5D,%22range%22:%7B%22from%22:%22${__from}%22,%22to%22:%22${__to}%22%7D%7D" } ] } ] } ] }, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 44 }, "id": 21, "links": [ { "targetBlank": false, "title": "Open Errors", "url": "/d/observme-errors/observme-errors?${__url_time_range}" } ], "options": { "cellHeight": "sm", "footer": { "countRows": false, "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true, "sortBy": [ { "desc": true, "displayName": "Time" } ] }, "pluginVersion": "11.1.0", "targets": [ { "datasource": { "type": "loki", "uid": "loki" }, "editorMode": "code", "expr": "{service_name=\"observme-pi-extension\", event_name=\"export.failed\"} | line_format `{\"event\":{{ printf \"%q\" .event_name }},\"category\":{{ printf \"%q\" .event_category }},\"session\":{{ printf \"%q\" .pi_session_id }},\"workflow\":{{ printf \"%q\" .pi_workflow_id }},\"agent\":{{ printf \"%q\" .pi_agent_id }},\"parent_agent\":{{ printf \"%q\" .pi_agent_parent_id }},\"agent_run\":{{ printf \"%q\" .pi_agent_run_id }},\"error_type\":{{ printf \"%q\" .error_type }},\"trace_id\":{{ printf \"%q\" .trace_id }},\"span_id\":{{ printf \"%q\" .span_id }}}`", "maxLines": 500, "queryType": "range", "range": true, "refId": "A" } ], "title": "Export failure logs", "transformations": [ { "id": "labelsToFields", "options": { "mode": "columns" } }, { "id": "extractFields", "options": { "format": "json", "keepTime": true, "replace": false, "source": "Line" } }, { "id": "organize", "options": { "excludeByName": { "Line": true, "id": true, "labels": true, "service_name": true, "telemetry_source": true, "tsNs": true }, "renameByName": { "event_category": "category", "event_name": "event", "pi_agent_id": "agent", "pi_agent_parent_id": "parent_agent", "pi_agent_run_id": "agent_run", "pi_session_id": "session", "pi_workflow_id": "workflow" } } } ], "type": "table" }, { "datasource": { "type": "loki", "uid": "loki" }, "description": "Latest trace_context.propagation_failed events first. Safe deterministic fields expose event, category, execution context, error type, trace, and span while raw lines are hidden. trace_id opens Tempo and the panel links to Trace Journey. Empty means no matching failure logs in the selected time range; this is healthy when trace context propagated successfully.", "fieldConfig": { "defaults": {}, "overrides": [ { "matcher": { "id": "byName", "options": "trace_id" }, "properties": [ { "id": "links", "value": [ { "targetBlank": false, "title": "Open Tempo trace", "url": "/explore?left=%7B%22datasource%22:%22tempo%22,%22queries%22:%5B%7B%22query%22:%22${__value.raw}%22%7D%5D,%22range%22:%7B%22from%22:%22${__from}%22,%22to%22:%22${__to}%22%7D%7D" } ] } ] } ] }, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 44 }, "id": 9, "links": [ { "targetBlank": false, "title": "Open Trace Journey", "url": "/d/observme-trace-journey/observme-trace-journey?${__url_time_range}" } ], "options": { "cellHeight": "sm", "footer": { "countRows": false, "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true, "sortBy": [ { "desc": true, "displayName": "Time" } ] }, "pluginVersion": "11.1.0", "targets": [ { "datasource": { "type": "loki", "uid": "loki" }, "editorMode": "code", "expr": "{service_name=\"observme-pi-extension\", event_name=\"trace_context.propagation_failed\"} | line_format `{\"event\":{{ printf \"%q\" .event_name }},\"category\":{{ printf \"%q\" .event_category }},\"session\":{{ printf \"%q\" .pi_session_id }},\"workflow\":{{ printf \"%q\" .pi_workflow_id }},\"agent\":{{ printf \"%q\" .pi_agent_id }},\"parent_agent\":{{ printf \"%q\" .pi_agent_parent_id }},\"agent_run\":{{ printf \"%q\" .pi_agent_run_id }},\"error_type\":{{ printf \"%q\" .error_type }},\"trace_id\":{{ printf \"%q\" .trace_id }},\"span_id\":{{ printf \"%q\" .span_id }}}`", "maxLines": 500, "queryType": "range", "range": true, "refId": "A" } ], "title": "Trace-context propagation failure logs", "transformations": [ { "id": "labelsToFields", "options": { "mode": "columns" } }, { "id": "extractFields", "options": { "format": "json", "keepTime": true, "replace": false, "source": "Line" } }, { "id": "organize", "options": { "excludeByName": { "Line": true, "id": true, "labels": true, "service_name": true, "telemetry_source": true, "tsNs": true }, "renameByName": { "event_category": "category", "event_name": "event", "pi_agent_id": "agent", "pi_agent_parent_id": "parent_agent", "pi_agent_run_id": "agent_run", "pi_session_id": "session", "pi_workflow_id": "workflow" } } } ], "type": "table" } ], "refresh": "30s", "schemaVersion": 39, "style": "dark", "tags": [ "observme", "export-health" ], "templating": { "list": [] }, "time": { "from": "now-6h", "to": "now" }, "timezone": "browser", "title": "ObservMe Export Health", "uid": "observme-export-health", "version": 4, "weekStart": "" }