{ "__requires": [ { "id": "grafana", "name": "Grafana", "type": "grafana", "version": "10.0.0" }, { "id": "prometheus", "name": "Prometheus", "type": "datasource", "version": "1.0.0" }, { "id": "loki", "name": "Loki", "type": "datasource", "version": "1.0.0" }, { "id": "row", "name": "Row", "type": "panel", "version": "" }, { "id": "timeseries", "name": "Time series", "type": "panel", "version": "" }, { "id": "stat", "name": "Stat", "type": "panel", "version": "" }, { "id": "table", "name": "Table", "type": "panel", "version": "" } ], "annotations": { "list": [] }, "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, "id": null, "links": [ { "targetBlank": false, "title": "Overview", "url": "/d/observme-overview/observme-overview?${__url_time_range}" }, { "targetBlank": false, "title": "Agent Node Graphs", "url": "/d/observme-agent-node-graphs/observme-agent-node-graphs?${__url_time_range}" }, { "targetBlank": false, "title": "Trace Journey", "url": "/d/observme-trace-journey/observme-trace-journey?${__url_time_range}" }, { "targetBlank": false, "title": "Errors", "url": "/d/observme-errors/observme-errors?${__url_time_range}" }, { "targetBlank": false, "title": "SLO Health", "url": "/d/observme-slo-health/observme-slo-health?${__url_time_range}" } ], "liveNow": false, "panels": [ { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 100, "panels": [], "title": "Health ratios", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Selected-range spawn failures divided by spawn attempts; observme_subagents_spawned_total increments when a spawn starts. Zero means no failures were observed or no spawn attempts occurred. Any non-zero result aligns with the warning-level ObservMeSubagentSpawnFailures alert.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 1e-12 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 5, "w": 6, "x": 0, "y": 1 }, "id": 14, "links": [ { "targetBlank": false, "title": "Open Trace Journey", "url": "/d/observme-trace-journey/observme-trace-journey?${__url_time_range}" } ], "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "11.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "(sum(increase(observme_subagent_spawn_failures_total[$__range])) or vector(0)) / clamp_min((sum(increase(observme_subagents_spawned_total[$__range])) or vector(0)), 1)", "refId": "A", "instant": true, "range": false } ], "title": "Spawn failure ratio", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Selected-range orphan events divided by observed orchestration opportunities (spawn starts plus agent runs). This is an aggregate pressure indicator, not a per-agent probability. Zero means no orphans were observed or no opportunities occurred; any non-zero orphan signal aligns with ObservMeOrphanAgentsDetected.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 1e-12 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 5, "w": 6, "x": 6, "y": 1 }, "id": 15, "links": [ { "targetBlank": false, "title": "Open Trace Journey", "url": "/d/observme-trace-journey/observme-trace-journey?${__url_time_range}" } ], "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "11.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "(sum(increase(observme_orphan_agents_total[$__range])) or vector(0)) / clamp_min((sum(increase(observme_subagents_spawned_total[$__range])) or vector(0)) + (sum(increase(observme_agent_runs_total[$__range])) or vector(0)), 1)", "refId": "A", "instant": true, "range": false } ], "title": "Orphan ratio vs spawns/runs", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Selected-range subagent trace-context propagation failures divided by known spawn starts. Root-depth failures are excluded so the numerator matches the spawn denominator. Zero means no matching failures were observed or no subagent spawns occurred; any non-zero result aligns with ObservMeTraceContextPropagationFailures.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 1e-12 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 5, "w": 6, "x": 12, "y": 1 }, "id": 16, "links": [ { "targetBlank": false, "title": "Open Trace Journey", "url": "/d/observme-trace-journey/observme-trace-journey?${__url_time_range}" } ], "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "11.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "(sum(increase(observme_trace_context_propagation_failures_total{subagent_depth=~\"[1-9][0-9]*\"}[$__range])) or vector(0)) / clamp_min((sum(increase(observme_subagents_spawned_total[$__range])) or vector(0)), 1)", "refId": "A", "instant": true, "range": false } ], "title": "Propagation failure ratio", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Selected-range parent recoveries divided by completed child-agent failures. Exact bounded state plus a fixed-size archive prevents retained or evicted child transitions from being recounted; archive collisions can only conservatively suppress novel transitions under accounting pressure. A range with no child failures renders 100% because no recovery was required; use the failure/recovery trend to distinguish healthy idle from active recovery. The 50% and 90% color bands are tunable guidance, not alert thresholds.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null }, { "color": "yellow", "value": 0.5 }, { "color": "green", "value": 0.9 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 5, "w": 6, "x": 18, "y": 1 }, "id": 17, "links": [ { "targetBlank": false, "title": "Open Trace Journey", "url": "/d/observme-trace-journey/observme-trace-journey?${__url_time_range}" } ], "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "11.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "clamp_max(((sum(increase(observme_parent_recovered_from_child_failure_total[$__range])) or vector(0)) / clamp_min((sum(increase(observme_child_agent_failures_total[$__range])) or vector(0)), 1)) + ((sum(increase(observme_child_agent_failures_total[$__range])) or vector(0)) == bool 0), 1)", "refId": "A", "instant": true, "range": false } ], "title": "Child recovery ratio", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Aggregate orphan-agent rate by the emitted bounded status and reason labels. The metric does not carry role, depth, or execution IDs; use the lineage table for per-execution context. No series can mean no orphan events in the range.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "ops" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 6 }, "id": 10, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "sum(rate(observme_orphan_agents_total[$__rate_interval])) by (status, reason)", "range": true, "refId": "A", "legendFormat": "{{status}} / {{reason}}" } ], "title": "Orphan agents", "type": "timeseries", "links": [ { "targetBlank": false, "title": "Open Trace Journey", "url": "/d/observme-trace-journey/observme-trace-journey?${__url_time_range}" }, { "targetBlank": false, "title": "Open Agent Node Graphs", "url": "/d/observme-agent-node-graphs/observme-agent-node-graphs?${__url_time_range}" } ] }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Trace-context propagation-failure rate grouped by safe role and depth labels.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "ops" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 6 }, "id": 11, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "sum(rate(observme_trace_context_propagation_failures_total[$__rate_interval])) by (agent_role, subagent_depth)", "range": true, "refId": "A", "legendFormat": "{{agent_role}} depth {{subagent_depth}}" } ], "title": "Trace-context propagation failures", "type": "timeseries", "links": [ { "targetBlank": false, "title": "Open Trace Journey", "url": "/d/observme-trace-journey/observme-trace-journey?${__url_time_range}" }, { "targetBlank": false, "title": "Open Agent Node Graphs", "url": "/d/observme-agent-node-graphs/observme-agent-node-graphs?${__url_time_range}" } ] }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Child-agent failure and parent-recovery rates grouped by safe role and depth labels. Exact bounded state plus a fixed-size archive prevents retained or evicted child transitions from being recounted; archive collisions can only conservatively suppress novel transitions under accounting pressure.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "ops" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 6 }, "id": 13, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "sum(rate(observme_child_agent_failures_total[$__rate_interval])) by (agent_role, subagent_depth)", "range": true, "refId": "A", "legendFormat": "child failures {{agent_role}} depth {{subagent_depth}}" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "sum(rate(observme_parent_recovered_from_child_failure_total[$__rate_interval])) by (agent_role, subagent_depth)", "range": true, "refId": "B", "legendFormat": "recovered {{agent_role}} depth {{subagent_depth}}" } ], "title": "Child-agent failures and recovered child failures", "type": "timeseries", "links": [ { "targetBlank": false, "title": "Open Trace Journey", "url": "/d/observme-trace-journey/observme-trace-journey?${__url_time_range}" }, { "targetBlank": false, "title": "Open Agent Node Graphs", "url": "/d/observme-agent-node-graphs/observme-agent-node-graphs?${__url_time_range}" } ] }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 14 }, "id": 101, "panels": [], "title": "Activity and latency", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Agent-run rate grouped only by the low-cardinality agent_role label. Role series retain emitted values exactly: v2 lead, helper, worker, and validator remain separate; root and the legacy subagent, orchestrator, reviewer, and unknown values remain visible during migration.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "ops" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 15 }, "id": 1, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "sum(rate(observme_agent_runs_total[$__rate_interval])) by (agent_role)", "range": true, "refId": "A", "legendFormat": "{{agent_role}}" } ], "title": "Agent runs by role", "type": "timeseries", "links": [ { "targetBlank": false, "title": "Open Trace Journey", "url": "/d/observme-trace-journey/observme-trace-journey?${__url_time_range}" }, { "targetBlank": false, "title": "Open Agent Node Graphs", "url": "/d/observme-agent-node-graphs/observme-agent-node-graphs?${__url_time_range}" } ] }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Subagent spawn rate grouped by low-cardinality depth, type, and reason labels.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "ops" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 15 }, "id": 2, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "sum(rate(observme_subagents_spawned_total[$__rate_interval])) by (subagent_depth, spawn_type, spawn_reason)", "range": true, "refId": "A", "legendFormat": "depth {{subagent_depth}} / {{spawn_type}} / {{spawn_reason}}" } ], "title": "Subagent spawns by depth and spawn type", "type": "timeseries", "links": [ { "targetBlank": false, "title": "Open Trace Journey", "url": "/d/observme-trace-journey/observme-trace-journey?${__url_time_range}" }, { "targetBlank": false, "title": "Open Agent Node Graphs", "url": "/d/observme-agent-node-graphs/observme-agent-node-graphs?${__url_time_range}" } ] }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Subagent spawn-failure rate grouped by the labels actually emitted on this counter: bounded spawn_type and error_class. Spawn reason and depth remain available on spawn-attempt metrics and in lineage logs.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "ops" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 15 }, "id": 3, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "sum(rate(observme_subagent_spawn_failures_total[$__rate_interval])) by (spawn_type, error_class)", "range": true, "refId": "A", "legendFormat": "{{spawn_type}} / {{error_class}}" } ], "title": "Subagent spawn failures", "type": "timeseries", "links": [ { "targetBlank": false, "title": "Open Trace Journey", "url": "/d/observme-trace-journey/observme-trace-journey?${__url_time_range}" }, { "targetBlank": false, "title": "Open Agent Node Graphs", "url": "/d/observme-agent-node-graphs/observme-agent-node-graphs?${__url_time_range}" } ] }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "p95 agent-run duration grouped by agent_role only.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "ms" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 23 }, "id": 4, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum(rate(observme_agent_run_duration_ms_bucket[$__rate_interval])) by (agent_role, le))", "range": true, "refId": "A", "legendFormat": "{{agent_role}}" } ], "title": "p95 agent-run duration", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "p95 launcher latency from spawn start through usable transport-handle acquisition or pre-handle launcher failure, grouped by bounded spawn labels. Delayed child execution, wait, join, and terminal result delivery are excluded.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "ms" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 23 }, "id": 5, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum(rate(observme_subagent_spawn_duration_ms_bucket[$__rate_interval])) by (agent_role, spawn_type, spawn_reason, le))", "range": true, "refId": "A", "legendFormat": "{{agent_role}} / {{spawn_type}} / {{spawn_reason}}" } ], "title": "p95 subagent-spawn duration", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "p95 parent wait and join latency grouped by agent_role only.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "ms" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 23 }, "id": 12, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum(rate(observme_agent_wait_duration_ms_bucket[$__rate_interval])) by (agent_role, le))", "range": true, "refId": "A", "legendFormat": "wait p95 {{agent_role}}" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum(rate(observme_agent_join_duration_ms_bucket[$__rate_interval])) by (agent_role, le))", "range": true, "refId": "B", "legendFormat": "join p95 {{agent_role}}" } ], "title": "Parent wait/join latency", "type": "timeseries" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 31 }, "id": 102, "panels": [], "title": "Tree shape", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Canonical leased-active agents grouped by emitted agent_role, plus the fleet total. Clean shutdown reaches zero after export/scrape propagation; after a crash, SIGKILL, cancelled job, or runner loss, the leased total converges to zero within the default 60-second lease plus one Prometheus scrape even when raw claims stay cached. Zero can also mean healthy idle or a failed-closed lease, so compare Export Health. /obs agents reports current-process child state, not this aggregate fleet view. The total=100 reference matches the deployment-tunable ObservMeActiveAgentsStuckHigh threshold.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 100 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 32 }, "id": 7, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "sum by (agent_role) (max by (observme_instance_id, agent_role) ((observme_active_agents > 0) and on (observme_instance_id) ((observme_agent_lease_expires_unixtime_seconds > time()) and (observme_agent_lease_expires_unixtime_seconds <= time() + 305)))) or on() vector(0)", "range": true, "refId": "A", "legendFormat": "{{agent_role}}" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "sum(max by (observme_instance_id) ((observme_active_agents > 0) and on (observme_instance_id) ((observme_agent_lease_expires_unixtime_seconds > time()) and (observme_agent_lease_expires_unixtime_seconds <= time() + 305)))) or vector(0)", "range": true, "refId": "B", "legendFormat": "total active agents" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "vector(100)", "range": true, "refId": "C", "legendFormat": "tunable alert threshold 100" } ], "title": "Active agents by role", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "p95 tree depth and width from bounded aggregate histogram labels; no execution IDs enter Prometheus. The depth=5 reference matches ObservMeExcessiveAgentTreeDepth and is tunable to the deployment delegation policy.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 5 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 32 }, "id": 8, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum(rate(observme_agent_tree_depth_bucket{subagent_depth!=\"\"}[$__rate_interval])) by (subagent_depth, le))", "range": true, "refId": "A", "legendFormat": "depth p95 {{subagent_depth}}" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum(rate(observme_agent_tree_width_bucket{subagent_depth!=\"\"}[$__rate_interval])) by (subagent_depth, le))", "range": true, "refId": "B", "legendFormat": "width p95 {{subagent_depth}}" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "vector(5)", "refId": "C", "legendFormat": "alert threshold depth 5", "range": true } ], "title": "Agent-tree depth and width", "type": "timeseries", "links": [ { "targetBlank": false, "title": "Open Trace Journey", "url": "/d/observme-trace-journey/observme-trace-journey?${__url_time_range}" }, { "targetBlank": false, "title": "Open Agent Node Graphs", "url": "/d/observme-agent-node-graphs/observme-agent-node-graphs?${__url_time_range}" } ] }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "p95 child fan-out grouped by bounded subagent depth. The fan-out=20 reference matches ObservMeRunawayAgentFanOut and is tunable to normal orchestrator workload.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 20 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 32 }, "id": 9, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum(rate(observme_agent_fanout_count_bucket{subagent_depth!=\"\"}[$__rate_interval])) by (subagent_depth, le))", "range": true, "refId": "A", "legendFormat": "depth {{subagent_depth}}" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "vector(20)", "refId": "B", "legendFormat": "alert threshold fan-out 20", "range": true } ], "title": "Fan-out per parent operation", "type": "timeseries", "links": [ { "targetBlank": false, "title": "Open Trace Journey", "url": "/d/observme-trace-journey/observme-trace-journey?${__url_time_range}" }, { "targetBlank": false, "title": "Open Agent Node Graphs", "url": "/d/observme-agent-node-graphs/observme-agent-node-graphs?${__url_time_range}" } ] }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 40 }, "id": 103, "panels": [], "title": "Top offenders", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Top agent roles by p95 run duration, paired with range run volume to avoid over-reading sparse percentile series.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 41 }, "id": 18, "links": [ { "targetBlank": false, "title": "Open Trace Journey", "url": "/d/observme-trace-journey/observme-trace-journey?${__url_time_range}" } ], "options": { "cellHeight": "sm", "footer": { "countRows": false, "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true }, "pluginVersion": "11.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "topk(10, histogram_quantile(0.95, sum(increase(observme_agent_run_duration_ms_bucket[$__range])) by (agent_role, le)))", "refId": "A", "legendFormat": "p95 {{agent_role}}", "format": "table", "instant": true, "range": false }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "topk(10, sum(increase(observme_agent_runs_total[$__range])) by (agent_role))", "refId": "B", "legendFormat": "runs {{agent_role}}", "format": "table", "instant": true, "range": false } ], "title": "Top slow agent roles", "type": "table" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Top selected-range spawn failures by emitted spawn_type and error_class, paired with spawn-attempt volume by spawn type/reason. The failure counter does not emit spawn_reason, so the volume frame is context rather than direct reason attribution; use lineage logs for an individual failure.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 41 }, "id": 19, "links": [ { "targetBlank": false, "title": "Open Trace Journey", "url": "/d/observme-trace-journey/observme-trace-journey?${__url_time_range}" } ], "options": { "cellHeight": "sm", "footer": { "countRows": false, "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true }, "pluginVersion": "11.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "topk(10, sum(increase(observme_subagent_spawn_failures_total[$__range])) by (spawn_type, error_class))", "refId": "A", "legendFormat": "failures {{spawn_type}} / {{error_class}}", "format": "table", "instant": true, "range": false }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "topk(10, sum(increase(observme_subagents_spawned_total[$__range])) by (spawn_type, spawn_reason))", "refId": "B", "legendFormat": "attempts {{spawn_type}} / {{spawn_reason}}", "format": "table", "instant": true, "range": false } ], "title": "Failing spawn reasons", "type": "table" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Top selected-range orphan counts grouped only by the metric's emitted bounded status and reason labels. Role, depth, and execution IDs are not available on this metric; use the lineage log rows for execution context.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 49 }, "id": 20, "links": [ { "targetBlank": false, "title": "Open Trace Journey", "url": "/d/observme-trace-journey/observme-trace-journey?${__url_time_range}" } ], "options": { "cellHeight": "sm", "footer": { "countRows": false, "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true }, "pluginVersion": "11.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "topk(10, sum(increase(observme_orphan_agents_total[$__range])) by (status, reason))", "refId": "A", "legendFormat": "orphans {{status}} / {{reason}}", "format": "table", "instant": true, "range": false } ], "title": "Orphans by status and reason", "type": "table" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Top selected-range p95 fan-out by emitted role/depth labels, paired with spawn-attempt volume for sparse-series context. The tunable fan-out=20 reference matches ObservMeRunawayAgentFanOut.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 49 }, "id": 21, "links": [ { "targetBlank": false, "title": "Open Trace Journey", "url": "/d/observme-trace-journey/observme-trace-journey?${__url_time_range}" } ], "options": { "cellHeight": "sm", "footer": { "countRows": false, "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true }, "pluginVersion": "11.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "topk(10, histogram_quantile(0.95, sum(increase(observme_agent_fanout_count_bucket{subagent_depth!=\"\"}[$__range])) by (agent_role, subagent_depth, le)))", "refId": "A", "legendFormat": "{{agent_role}} depth {{subagent_depth}}", "format": "table", "instant": true, "range": false }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "topk(10, sum(increase(observme_subagents_spawned_total[$__range])) by (agent_role, subagent_depth))", "refId": "B", "legendFormat": "spawn attempts {{agent_role}} depth {{subagent_depth}}", "format": "table", "instant": true, "range": false }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "vector(20)", "refId": "C", "legendFormat": "tunable alert threshold fan-out 20", "format": "table", "instant": true, "range": false } ], "title": "High fan-out roles", "type": "table" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 57 }, "id": 105, "panels": [], "title": "Lease health diagnostics", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Diagnostic positive lifecycle claims before lease qualification. These are not live-agent totals and may remain cached after a crash, SIGKILL, cancelled job, or runner loss; leased activity converges to zero within the default 60-second lease plus one Prometheus scrape. Clean shutdown exports zero immediately. Compare Expired active claims and Export Health. /obs agents reports only current-process child state, not fleet liveness.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 58 }, "id": 22, "links": [ { "targetBlank": false, "title": "Open Export Health", "url": "/d/observme-export-health/observme-export-health?${__url_time_range}" } ], "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "11.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "sum(max by (observme_instance_id) (observme_active_agents > 0)) or vector(0)", "instant": true, "range": false, "refId": "A" } ], "title": "Raw active claims (diagnostic)", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Positive raw claims whose lease has expired. They are stale diagnostics and never contribute to leased active totals; clean shutdown normally avoids them, while ungraceful exits appear after the default 60-second lease plus one Prometheus scrape and may remain until Collector cleanup. Yellow at 1 and red at 5 are deployment-tunable. Compare Raw active claims and Export Health. /obs agents reports only current-process child state, not fleet liveness.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 1 }, { "color": "red", "value": 5 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 58 }, "id": 23, "links": [ { "targetBlank": false, "title": "Open Export Health", "url": "/d/observme-export-health/observme-export-health?${__url_time_range}" } ], "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "11.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "sum(max by (observme_instance_id) ((observme_active_agents > 0) and on (observme_instance_id) (observme_agent_lease_expires_unixtime_seconds <= time()))) or vector(0)", "instant": true, "range": false, "refId": "A" } ], "title": "Expired active claims (diagnostic)", "type": "stat" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 66 }, "id": 104, "panels": [], "title": "Lineage logs", "type": "row" }, { "datasource": { "type": "loki", "uid": "loki" }, "description": "Chronological agent-tree handoffs and lineage failures. A deterministic safe JSON line exposes event, session, workflow, parent, child/agent, run, trace, and span fields; raw/content fields are hidden. trace_id opens Tempo and agent_run opens a filtered Trace Journey when values exist. child comes from pi.agent.child.id (promoted by the local Collector's Loki label hints); it is present on spawn/wait/join events and empty on events that carry no child correlation. Empty means no matching handoff or lineage-failure logs in the selected range.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short" }, "overrides": [ { "matcher": { "id": "byName", "options": "trace_id" }, "properties": [ { "id": "links", "value": [ { "targetBlank": false, "title": "Open Tempo trace", "url": "/explore?left=%7B%22datasource%22:%22tempo%22,%22queries%22:%5B%7B%22query%22:%22${__value.raw}%22%7D%5D,%22range%22:%7B%22from%22:%22${__from}%22,%22to%22:%22${__to}%22%7D%7D" } ] } ] }, { "matcher": { "id": "byName", "options": "agent_run" }, "properties": [ { "id": "links", "value": [ { "targetBlank": false, "title": "Open filtered Trace Journey", "url": "/d/observme-trace-journey/observme-trace-journey?${__url_time_range}&var-session_id=${__data.fields[\"session\"]}&var-workflow_id=${__data.fields[\"workflow\"]}&var-agent_id=${__data.fields[\"agent\"]}&var-agent_run_id=${__value.raw}" } ] } ] } ] }, "gridPos": { "h": 10, "w": 24, "x": 0, "y": 67 }, "id": 6, "options": { "cellHeight": "sm", "footer": { "countRows": false, "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true, "sortBy": [ { "desc": false, "displayName": "Time" } ] }, "targets": [ { "datasource": { "type": "loki", "uid": "loki" }, "editorMode": "code", "expr": "{service_name=\"observme-pi-extension\", event_category=\"agent-tree\", event_name=~\"agent[.]spawn[.](started|completed|failed)|agent[.](wait|join)[.](started|completed)|agent[.]orphaned|trace_context[.]propagation_failed\"} | line_format `{\"event\":{{ printf \"%q\" .event_name }},\"session\":{{ printf \"%q\" .pi_session_id }},\"workflow\":{{ printf \"%q\" .pi_workflow_id }},\"parent\":{{ printf \"%q\" .pi_agent_parent_id }},\"child\":{{ printf \"%q\" .pi_agent_child_id }},\"agent\":{{ printf \"%q\" .pi_agent_id }},\"agent_run\":{{ printf \"%q\" .pi_agent_run_id }},\"trace_id\":{{ printf \"%q\" .trace_id }},\"span_id\":{{ printf \"%q\" .span_id }}}`", "range": true, "refId": "A", "legendFormat": "{{event_name}}", "maxLines": 500, "queryType": "range" } ], "title": "Parent/child handoffs with trace links", "type": "table", "links": [ { "targetBlank": false, "title": "Open matching Trace Journey", "url": "/d/observme-trace-journey/observme-trace-journey?${__url_time_range}" } ], "transformations": [ { "id": "labelsToFields", "options": { "mode": "columns" } }, { "id": "extractFields", "options": { "format": "json", "keepTime": true, "replace": false, "source": "Line" } }, { "id": "organize", "options": { "excludeByName": { "Line": true, "id": true, "labels": true, "tsNs": true, "service_name": true, "telemetry_source": true }, "renameByName": { "event_name": "event", "pi_session_id": "session", "pi_workflow_id": "workflow", "pi_agent_parent_id": "parent", "pi_agent_child_id": "child", "pi_agent_id": "agent", "pi_agent_run_id": "agent_run", "trace_id": "trace_id", "span_id": "span_id" } } } ], "pluginVersion": "11.1.0" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 78 }, "id": 106, "panels": [], "title": "Agent identity", "type": "row" }, { "datasource": { "type": "loki", "uid": "loki" }, "description": "Loki-backed identity directory for running agents and child spawns. pi.agent.display_name and pi.agent.child.display_name are presentation text only and may be duplicated; pi.agent.id and pi.workflow.id remain the correlation keys used by links and joins. Capability and friendly names come from OTEL structured metadata rather than Prometheus labels. Empty means no matching session, agent-run, or spawn logs were observed in the selected range.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short" }, "overrides": [ { "matcher": { "id": "byName", "options": "trace_id" }, "properties": [ { "id": "links", "value": [ { "targetBlank": false, "title": "Open Tempo trace", "url": "/explore?left=%7B%22datasource%22:%22tempo%22,%22queries%22:%5B%7B%22query%22:%22${__value.raw}%22%7D%5D,%22range%22:%7B%22from%22:%22${__from}%22,%22to%22:%22${__to}%22%7D%7D" } ] } ] }, { "matcher": { "id": "byName", "options": "agent_id" }, "properties": [ { "id": "links", "value": [ { "targetBlank": false, "title": "Open agent in Trace Journey", "url": "/d/observme-trace-journey/observme-trace-journey?${__url_time_range}&var-workflow_id=${__data.fields[\"workflow_id\"]}&var-agent_id=${__value.raw}" } ] } ] }, { "matcher": { "id": "byName", "options": "workflow_id" }, "properties": [ { "id": "links", "value": [ { "targetBlank": false, "title": "Open workflow in Trace Journey", "url": "/d/observme-trace-journey/observme-trace-journey?${__url_time_range}&var-workflow_id=${__value.raw}&var-agent_id=${__data.fields[\"agent_id\"]}" } ] } ] } ] }, "gridPos": { "h": 11, "w": 24, "x": 0, "y": 79 }, "id": 24, "links": [ { "targetBlank": false, "title": "Open Trace Journey", "url": "/d/observme-trace-journey/observme-trace-journey?${__url_time_range}" } ], "options": { "cellHeight": "sm", "footer": { "countRows": false, "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true, "sortBy": [ { "desc": false, "displayName": "Time" } ] }, "pluginVersion": "11.1.0", "targets": [ { "datasource": { "type": "loki", "uid": "loki" }, "editorMode": "code", "expr": "{service_name=\"observme-pi-extension\", service_namespace=\"pi\"} | event_name=~\"session[.]started|agent[.]run[.]started|agent[.]spawn[.](started|completed|failed)\" | line_format `{\"event\":{{ printf \"%q\" .event_name }},\"agent_name\":{{ printf \"%q\" .pi_agent_display_name }},\"agent_role\":{{ printf \"%q\" .pi_agent_role }},\"agent_capability\":{{ printf \"%q\" .pi_agent_capability }},\"agent_id\":{{ printf \"%q\" .pi_agent_id }},\"parent_id\":{{ printf \"%q\" .pi_agent_parent_id }},\"workflow_id\":{{ printf \"%q\" .pi_workflow_id }},\"child_name\":{{ printf \"%q\" .pi_agent_child_display_name }},\"child_role\":{{ printf \"%q\" .pi_agent_child_role }},\"trace_id\":{{ printf \"%q\" .trace_id }}}`", "legendFormat": "{{event_name}}", "maxLines": 500, "queryType": "range", "range": true, "refId": "A" } ], "title": "Agent identity and spawned child names", "transformations": [ { "id": "labelsToFields", "options": { "mode": "columns" } }, { "id": "extractFields", "options": { "format": "json", "keepTime": true, "replace": false, "source": "Line" } }, { "id": "organize", "options": { "excludeByName": { "Line": true, "id": true, "labels": true, "tsNs": true, "service_name": true, "telemetry_source": true }, "indexByName": { "Time": 0, "event": 1, "agent_name": 2, "agent_role": 3, "agent_capability": 4, "agent_id": 5, "parent_id": 6, "workflow_id": 7, "child_name": 8, "child_role": 9, "trace_id": 10 }, "renameByName": { "event": "Event", "agent_name": "Running agent name", "agent_role": "Agent role", "agent_capability": "Capability", "agent_id": "Agent identity", "parent_id": "Parent", "workflow_id": "Workflow", "child_name": "Spawned child name", "child_role": "Spawned child role", "trace_id": "Trace ID" } } } ], "type": "table" } ], "refresh": "30s", "schemaVersion": 39, "tags": [ "observme", "agents", "subagents" ], "templating": { "list": [] }, "time": { "from": "now-6h", "to": "now" }, "timepicker": {}, "timezone": "browser", "title": "ObservMe Agents and Subagents", "uid": "observme-agents", "version": 3, "weekStart": "" }