{
  "__requires": [
    {
      "id": "grafana",
      "name": "Grafana",
      "type": "grafana",
      "version": "10.0.0"
    },
    {
      "id": "prometheus",
      "name": "Prometheus",
      "type": "datasource",
      "version": "1.0.0"
    },
    {
      "id": "nodeGraph",
      "name": "Node Graph",
      "type": "panel",
      "version": ""
    }
  ],
  "annotations": {
    "list": []
  },
  "editable": true,
  "fiscalYearStartMonth": 0,
  "graphTooltip": 0,
  "id": null,
  "links": [
    {
      "targetBlank": false,
      "title": "Agents dashboard",
      "url": "/d/observme-agents/observme-agents?${__url_time_range}"
    },
    {
      "targetBlank": false,
      "title": "Trace Journey",
      "url": "/d/observme-trace-journey/observme-trace-journey?${__url_time_range}"
    }
  ],
  "liveNow": false,
  "panels": [
    {
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "description": "Aggregate fleet topology built only from bounded Prometheus labels; it is not per-workflow truth or a single trace tree. Role nodes preserve exact v2 lead, helper, worker, and validator values, plus separate root and legacy subagent, orchestrator, reviewer, and unknown values. Role nodes combine canonical leased-current activity with selected-range run/spawn counts, so node and edge values must not be read as one execution. Clean shutdown removes leased activity after export/scrape propagation; after a crash, SIGKILL, cancelled job, or runner loss, leased activity converges to zero within the default 60-second lease plus one Prometheus scrape while raw claims may stay cached. Red nodes and edges identify spawn, lineage, or propagation failures. /obs agents is current-process child state; use Trace Journey for one execution.",
      "fieldConfig": {
        "defaults": {
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              }
            ]
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 14,
        "w": 24,
        "x": 0,
        "y": 0
      },
      "id": 1,
      "options": {
        "edges": {
          "mainStatUnit": "short",
          "secondaryStatUnit": "short"
        },
        "nodes": {
          "arcs": [],
          "mainStatUnit": "short",
          "secondaryStatUnit": "short"
        }
      },
      "pluginVersion": "11.1.0",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "label_replace(label_replace(label_replace(label_replace(label_replace(label_replace(vector(1), \"id\", \"pi-root\", \"__name__\", \".*\"), \"title\", \"Pi root\", \"id\", \".*\"), \"subTitle\", \"workflow entry\", \"id\", \".*\"), \"mainStat\", \"entry\", \"id\", \".*\"), \"secondaryStat\", \"aggregate\", \"id\", \".*\"), \"color\", \"blue\", \"id\", \".*\") or label_replace(label_replace(label_replace(label_replace(label_replace(label_replace((sum by (agent_role) (max by (observme_instance_id, agent_role) ((observme_active_agents > 0) and on (observme_instance_id) ((observme_agent_lease_expires_unixtime_seconds > time()) and (observme_agent_lease_expires_unixtime_seconds <= time() + 305)))) or sum(increase(observme_agent_runs_total[$__range])) by (agent_role) or sum(increase(observme_subagents_spawned_total[$__range])) by (agent_role)), \"id\", \"agent_$1\", \"agent_role\", \"(.+)\"), \"title\", \"Agent: $1\", \"agent_role\", \"(.+)\"), \"subTitle\", \"active or observed role\", \"agent_role\", \"(.+)\"), \"mainStat\", \"count\", \"agent_role\", \"(.+)\"), \"secondaryStat\", \"role\", \"agent_role\", \"(.+)\"), \"color\", \"green\", \"agent_role\", \"(.+)\") or label_replace(label_replace(label_replace(label_replace(label_replace(label_replace(sum(increase(observme_subagents_spawned_total[$__range])) by (subagent_depth), \"id\", \"subagents_depth_$1\", \"subagent_depth\", \"(.+)\"), \"title\", \"Subagents depth $1\", \"subagent_depth\", \"(.+)\"), \"subTitle\", \"spawned in range\", \"subagent_depth\", \"(.+)\"), \"mainStat\", \"spawns\", \"subagent_depth\", \"(.+)\"), \"secondaryStat\", \"depth\", \"subagent_depth\", \"(.+)\"), \"color\", \"orange\", \"subagent_depth\", \"(.+)\") or label_replace(label_replace(label_replace(label_replace(label_replace(label_replace((sum(increase(observme_subagent_spawn_failures_total[$__range])) by (error_class) > 0), \"id\", \"spawn_failure_$1\", \"error_class\", \"(.+)\"), \"title\", \"Spawn failure: $1\", \"error_class\", \"(.+)\"), \"subTitle\", \"failed spawns\", \"error_class\", \"(.+)\"), \"mainStat\", \"failures\", \"error_class\", \"(.+)\"), \"secondaryStat\", \"error\", \"error_class\", \"(.+)\"), \"color\", \"red\", \"error_class\", \"(.+)\") or label_replace(label_replace(label_replace(label_replace(label_replace(label_replace((sum(increase(observme_orphan_agents_total[$__range])) by (status, reason) > 0), \"id\", \"orphan_$1\", \"reason\", \"(.+)\"), \"title\", \"Orphan: $1\", \"reason\", \"(.+)\"), \"subTitle\", \"lineage missing\", \"reason\", \"(.+)\"), \"mainStat\", \"orphans\", \"reason\", \"(.+)\"), \"secondaryStat\", \"reason\", \"reason\", \"(.+)\"), \"color\", \"red\", \"reason\", \"(.+)\") or label_replace(label_replace(label_replace(label_replace(label_replace(label_replace((sum(increase(observme_trace_context_propagation_failures_total[$__range])) by (reason) > 0), \"id\", \"propagation_$1\", \"reason\", \"(.+)\"), \"title\", \"Propagation: $1\", \"reason\", \"(.+)\"), \"subTitle\", \"trace context failures\", \"reason\", \"(.+)\"), \"mainStat\", \"failures\", \"reason\", \"(.+)\"), \"secondaryStat\", \"reason\", \"reason\", \"(.+)\"), \"color\", \"red\", \"reason\", \"(.+)\")",
          "format": "table",
          "instant": true,
          "legendFormat": "nodes",
          "range": false,
          "refId": "nodes"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "label_join(label_replace(label_replace(label_replace(label_replace(label_replace((sum by (agent_role) (max by (observme_instance_id, agent_role) ((observme_active_agents > 0) and on (observme_instance_id) ((observme_agent_lease_expires_unixtime_seconds > time()) and (observme_agent_lease_expires_unixtime_seconds <= time() + 305)))) or sum(increase(observme_agent_runs_total[$__range])) by (agent_role)), \"source\", \"pi-root\", \"agent_role\", \"(.+)\"), \"target\", \"agent_$1\", \"agent_role\", \"(.+)\"), \"mainStat\", \"count\", \"agent_role\", \"(.+)\"), \"secondaryStat\", \"active/runs\", \"agent_role\", \"(.+)\"), \"color\", \"gray\", \"agent_role\", \"(.+)\"), \"id\", \":\", \"source\", \"target\") or label_join(label_replace(label_replace(label_replace(label_replace(label_replace(sum(increase(observme_subagents_spawned_total[$__range])) by (agent_role, subagent_depth, spawn_type, spawn_reason), \"source\", \"agent_$1\", \"agent_role\", \"(.+)\"), \"target\", \"subagents_depth_$1\", \"subagent_depth\", \"(.+)\"), \"mainStat\", \"spawns\", \"agent_role\", \"(.+)\"), \"secondaryStat\", \"spawn_type/reason\", \"agent_role\", \"(.+)\"), \"color\", \"gray\", \"agent_role\", \"(.+)\"), \"id\", \":\", \"source\", \"target\", \"spawn_type\", \"spawn_reason\") or label_join(label_replace(label_replace(label_replace(label_replace(label_replace((sum(increase(observme_subagent_spawn_failures_total[$__range])) by (error_class) > 0), \"source\", \"pi-root\", \"error_class\", \"(.+)\"), \"target\", \"spawn_failure_$1\", \"error_class\", \"(.+)\"), \"mainStat\", \"failures\", \"error_class\", \"(.+)\"), \"secondaryStat\", \"spawn\", \"error_class\", \"(.+)\"), \"color\", \"red\", \"error_class\", \"(.+)\"), \"id\", \":\", \"source\", \"target\") or label_join(label_replace(label_replace(label_replace(label_replace(label_replace((sum(increase(observme_orphan_agents_total[$__range])) by (status, reason) > 0), \"source\", \"pi-root\", \"reason\", \"(.+)\"), \"target\", \"orphan_$1\", \"reason\", \"(.+)\"), \"mainStat\", \"orphans\", \"reason\", \"(.+)\"), \"secondaryStat\", \"lineage\", \"reason\", \"(.+)\"), \"color\", \"red\", \"reason\", \"(.+)\"), \"id\", \":\", \"source\", \"target\") or label_join(label_replace(label_replace(label_replace(label_replace(label_replace((sum(increase(observme_trace_context_propagation_failures_total[$__range])) by (reason) > 0), \"source\", \"pi-root\", \"reason\", \"(.+)\"), \"target\", \"propagation_$1\", \"reason\", \"(.+)\"), \"mainStat\", \"failures\", \"reason\", \"(.+)\"), \"secondaryStat\", \"trace context\", \"reason\", \"(.+)\"), \"color\", \"red\", \"reason\", \"(.+)\"), \"id\", \":\", \"source\", \"target\")",
          "format": "table",
          "instant": true,
          "legendFormat": "edges",
          "range": false,
          "refId": "edges"
        }
      ],
      "transformations": [
        {
          "id": "labelsToFields",
          "options": {
            "mode": "columns"
          }
        },
        {
          "id": "organize",
          "options": {
            "excludeByName": {
              "Time": true
            },
            "renameByName": {
              "Value #nodes": "mainStat",
              "Value #edges": "mainStat",
              "mainStat": "detail__stat"
            }
          }
        }
      ],
      "title": "Pi root → agents → subagents",
      "type": "nodeGraph"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "description": "Aggregate fleet delegation topology by bounded role, spawn reason, and resulting depth; it is not per-workflow truth or a single trace tree. Role nodes may combine canonical leased-current activity with selected-range run/spawn counts, while other values are selected-range aggregates. Clean shutdown removes leased activity after export/scrape propagation; after an ungraceful exit, leased activity converges to zero within the default 60-second lease plus one Prometheus scrape while raw claims may stay cached. Red health nodes and edges summarize spawn failures. /obs agents is local current-process child state; use Trace Journey for one execution.",
      "fieldConfig": {
        "defaults": {
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              }
            ]
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 14,
        "w": 24,
        "x": 0,
        "y": 14
      },
      "id": 2,
      "options": {
        "edges": {
          "mainStatUnit": "short",
          "secondaryStatUnit": "short"
        },
        "nodes": {
          "arcs": [],
          "mainStatUnit": "short",
          "secondaryStatUnit": "short"
        }
      },
      "pluginVersion": "11.1.0",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "label_replace(label_replace(label_replace(label_replace(label_replace(label_replace((sum by (agent_role) (max by (observme_instance_id, agent_role) ((observme_active_agents > 0) and on (observme_instance_id) ((observme_agent_lease_expires_unixtime_seconds > time()) and (observme_agent_lease_expires_unixtime_seconds <= time() + 305)))) or sum(increase(observme_agent_runs_total[$__range])) by (agent_role) or sum(increase(observme_subagents_spawned_total[$__range])) by (agent_role)), \"id\", \"agent_$1\", \"agent_role\", \"(.+)\"), \"title\", \"Agent: $1\", \"agent_role\", \"(.+)\"), \"subTitle\", \"active or observed role\", \"agent_role\", \"(.+)\"), \"mainStat\", \"count\", \"agent_role\", \"(.+)\"), \"secondaryStat\", \"role\", \"agent_role\", \"(.+)\"), \"color\", \"green\", \"agent_role\", \"(.+)\") or label_replace(label_replace(label_replace(label_replace(label_replace(label_replace(sum(increase(observme_subagents_spawned_total[$__range])) by (spawn_reason), \"id\", \"spawn_reason_$1\", \"spawn_reason\", \"(.+)\"), \"title\", \"Spawn: $1\", \"spawn_reason\", \"(.+)\"), \"subTitle\", \"spawn reason\", \"spawn_reason\", \"(.+)\"), \"mainStat\", \"spawns\", \"spawn_reason\", \"(.+)\"), \"secondaryStat\", \"reason\", \"spawn_reason\", \"(.+)\"), \"color\", \"purple\", \"spawn_reason\", \"(.+)\") or label_replace(label_replace(label_replace(label_replace(label_replace(label_replace(sum(increase(observme_subagents_spawned_total[$__range])) by (subagent_depth), \"id\", \"subagents_depth_$1\", \"subagent_depth\", \"(.+)\"), \"title\", \"Subagents depth $1\", \"subagent_depth\", \"(.+)\"), \"subTitle\", \"spawned in range\", \"subagent_depth\", \"(.+)\"), \"mainStat\", \"spawns\", \"subagent_depth\", \"(.+)\"), \"secondaryStat\", \"depth\", \"subagent_depth\", \"(.+)\"), \"color\", \"orange\", \"subagent_depth\", \"(.+)\") or label_replace(label_replace(label_replace(label_replace(label_replace(label_replace((sum(increase(observme_subagent_spawn_failures_total[$__range])) > 0), \"id\", \"spawn-health\", \"__name__\", \".*\"), \"title\", \"Spawn failure health\", \"id\", \".*\"), \"subTitle\", \"failure-only aggregate\", \"id\", \".*\"), \"mainStat\", \"failures\", \"id\", \".*\"), \"secondaryStat\", \"health\", \"id\", \".*\"), \"color\", \"red\", \"id\", \".*\") or label_replace(label_replace(label_replace(label_replace(label_replace(label_replace((sum(increase(observme_subagent_spawn_failures_total[$__range])) by (error_class) > 0), \"id\", \"spawn_failure_$1\", \"error_class\", \"(.+)\"), \"title\", \"Spawn failure: $1\", \"error_class\", \"(.+)\"), \"subTitle\", \"failed spawns\", \"error_class\", \"(.+)\"), \"mainStat\", \"failures\", \"error_class\", \"(.+)\"), \"secondaryStat\", \"error class\", \"error_class\", \"(.+)\"), \"color\", \"red\", \"error_class\", \"(.+)\")",
          "format": "table",
          "instant": true,
          "legendFormat": "nodes",
          "range": false,
          "refId": "nodes"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "label_join(label_replace(label_replace(label_replace(label_replace(label_replace(sum(increase(observme_subagents_spawned_total[$__range])) by (agent_role, spawn_reason, spawn_type), \"source\", \"agent_$1\", \"agent_role\", \"(.+)\"), \"target\", \"spawn_reason_$1\", \"spawn_reason\", \"(.+)\"), \"mainStat\", \"spawns\", \"agent_role\", \"(.+)\"), \"secondaryStat\", \"spawn type\", \"agent_role\", \"(.+)\"), \"color\", \"gray\", \"agent_role\", \"(.+)\"), \"id\", \":\", \"source\", \"target\", \"spawn_type\") or label_join(label_replace(label_replace(label_replace(label_replace(label_replace(sum(increase(observme_subagents_spawned_total[$__range])) by (spawn_reason, subagent_depth), \"source\", \"spawn_reason_$1\", \"spawn_reason\", \"(.+)\"), \"target\", \"subagents_depth_$1\", \"subagent_depth\", \"(.+)\"), \"mainStat\", \"spawns\", \"spawn_reason\", \"(.+)\"), \"secondaryStat\", \"depth\", \"spawn_reason\", \"(.+)\"), \"color\", \"gray\", \"spawn_reason\", \"(.+)\"), \"id\", \":\", \"source\", \"target\") or label_join(label_replace(label_replace(label_replace(label_replace(label_replace((sum(increase(observme_subagent_spawn_failures_total[$__range])) by (error_class) > 0), \"source\", \"spawn-health\", \"error_class\", \"(.+)\"), \"target\", \"spawn_failure_$1\", \"error_class\", \"(.+)\"), \"mainStat\", \"failures\", \"error_class\", \"(.+)\"), \"secondaryStat\", \"error class\", \"error_class\", \"(.+)\"), \"color\", \"red\", \"error_class\", \"(.+)\"), \"id\", \":\", \"source\", \"target\")",
          "format": "table",
          "instant": true,
          "legendFormat": "edges",
          "range": false,
          "refId": "edges"
        }
      ],
      "transformations": [
        {
          "id": "labelsToFields",
          "options": {
            "mode": "columns"
          }
        },
        {
          "id": "organize",
          "options": {
            "excludeByName": {
              "Time": true
            },
            "renameByName": {
              "Value #nodes": "mainStat",
              "Value #edges": "mainStat",
              "mainStat": "detail__stat"
            }
          }
        }
      ],
      "title": "Agent role → spawn reason → subagent depth",
      "type": "nodeGraph"
    }
  ],
  "refresh": "30s",
  "schemaVersion": 39,
  "tags": [
    "observme",
    "pi",
    "agents",
    "subagents",
    "node-graph"
  ],
  "templating": {
    "list": []
  },
  "time": {
    "from": "now-6h",
    "to": "now"
  },
  "timepicker": {},
  "timezone": "browser",
  "title": "ObservMe Agent Node Graphs",
  "uid": "observme-agent-node-graphs",
  "version": 2,
  "weekStart": ""
}