{
  "__requires": [
    {
      "id": "grafana",
      "name": "Grafana",
      "type": "grafana",
      "version": "10.0.0"
    },
    {
      "id": "prometheus",
      "name": "Prometheus",
      "type": "datasource",
      "version": "1.0.0"
    },
    {
      "id": "loki",
      "name": "Loki",
      "type": "datasource",
      "version": "1.0.0"
    },
    {
      "id": "tempo",
      "name": "Tempo",
      "type": "datasource",
      "version": "1.0.0"
    },
    {
      "id": "row",
      "name": "Row",
      "type": "panel",
      "version": ""
    },
    {
      "id": "timeseries",
      "name": "Time series",
      "type": "panel",
      "version": ""
    },
    {
      "id": "stat",
      "name": "Stat",
      "type": "panel",
      "version": ""
    },
    {
      "id": "bargauge",
      "name": "Bar gauge",
      "type": "panel",
      "version": ""
    },
    {
      "id": "barchart",
      "name": "Bar chart",
      "type": "panel",
      "version": ""
    },
    {
      "id": "table",
      "name": "Table",
      "type": "panel",
      "version": ""
    },
    {
      "id": "logs",
      "name": "Logs",
      "type": "panel",
      "version": ""
    },
    {
      "id": "text",
      "name": "Text",
      "type": "panel",
      "version": ""
    }
  ],
  "annotations": {
    "list": []
  },
  "editable": true,
  "fiscalYearStartMonth": 0,
  "graphTooltip": 0,
  "id": null,
  "links": [
    {
      "targetBlank": false,
      "title": "Open matching LLM Conversations",
      "url": "/d/observme-llm-conversations/observme-llm-conversations?${__url_time_range}&${session_id:queryparam}&${workflow_id:queryparam}&${agent_id:queryparam}&${agent_run_id:queryparam}"
    },
    {
      "targetBlank": false,
      "title": "Agents",
      "url": "/d/observme-agents/observme-agents?${__url_time_range}"
    },
    {
      "targetBlank": false,
      "title": "Agent Node Graphs",
      "url": "/d/observme-agent-node-graphs/observme-agent-node-graphs?${__url_time_range}"
    }
  ],
  "liveNow": false,
  "panels": [
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 0
      },
      "id": 100,
      "panels": [],
      "title": "Summary",
      "type": "row"
    },
    {
      "datasource": null,
      "description": "A trace-first journey map for Pi sessions. Prometheus sections show bounded aggregate flow and latency; Loki and Tempo sections use the high-cardinality filters for one execution. Zero summary activity can be healthy idle, while missing telemetry should be checked in Export Health.",
      "fieldConfig": {
        "defaults": {},
        "overrides": []
      },
      "gridPos": {
        "h": 5,
        "w": 24,
        "x": 0,
        "y": 1
      },
      "id": 1,
      "options": {
        "code": {
          "language": "plaintext",
          "showLineNumbers": false,
          "showMiniMap": false
        },
        "content": "# ObservMe Trace Journey\n\nFollow how one Pi request travels through the system: `session → agent run → turn → LLM → tool/bash → subagent spawn → wait/join → branch/compaction`. Prometheus panels show aggregate flow and latency only; session, workflow, agent, and run filters apply to Loki/Tempo drill-downs. Start with summary and flow, inspect lineage logs for trace IDs, then open the Tempo waterfall. Empty journey rows can mean no matching work in the selected range; verify ingestion in Export Health when an active execution is unexpectedly absent.",
        "mode": "markdown"
      },
      "pluginVersion": "11.1.0",
      "title": "Trace journey map",
      "type": "text"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "description": "Canonical leased-active fleet total; Prometheus summary values are aggregate and are not filtered to the selected workflow. Clean shutdown reaches zero after export/scrape propagation; after a crash, SIGKILL, cancelled job, or runner loss, the leased total converges to zero within the default 60-second lease plus one Prometheus scrape even if raw claims stay cached. Zero can also mean healthy idle or a failed-closed lease, so check Export Health. /obs agents is current-process child state, not fleet liveness.",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              }
            ]
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 4,
        "w": 3,
        "x": 0,
        "y": 6
      },
      "id": 2,
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "auto",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "showPercentChange": false,
        "textMode": "auto",
        "wideLayout": true
      },
      "pluginVersion": "11.1.0",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "sum(max by (observme_instance_id) ((observme_active_agents > 0) and on (observme_instance_id) ((observme_agent_lease_expires_unixtime_seconds > time()) and (observme_agent_lease_expires_unixtime_seconds <= time() + 305)))) or vector(0)",
          "instant": true,
          "range": false,
          "refId": "A"
        }
      ],
      "title": "Active agents",
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "description": "Completed workflows during the selected dashboard range. Empty ranges render as zero.",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              }
            ]
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 4,
        "w": 3,
        "x": 3,
        "y": 6
      },
      "id": 3,
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "auto",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "showPercentChange": false,
        "textMode": "auto",
        "wideLayout": true
      },
      "pluginVersion": "11.1.0",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "sum(increase(observme_workflows_completed_total[$__range])) or vector(0)",
          "instant": true,
          "range": false,
          "refId": "A"
        }
      ],
      "title": "Completed workflows",
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "description": "Failed workflows during the selected dashboard range. Any non-zero value should be investigated with the ordered logs and Tempo traces below.",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 1
              }
            ]
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 4,
        "w": 3,
        "x": 6,
        "y": 6
      },
      "id": 16,
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "auto",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "showPercentChange": false,
        "textMode": "auto",
        "wideLayout": true
      },
      "pluginVersion": "11.1.0",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "sum(increase(observme_workflow_errors_total[$__range])) or vector(0)",
          "instant": true,
          "range": false,
          "refId": "A"
        }
      ],
      "title": "Failed workflows",
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "description": "Successful terminal workflows divided by completed plus failed workflows over the selected range. A zero can mean no terminal workflows or that every terminal workflow failed; interpret it beside both workflow count stats rather than as a standalone health signal.",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "red",
                "value": null
              },
              {
                "color": "yellow",
                "value": 0.95
              },
              {
                "color": "green",
                "value": 0.99
              }
            ]
          },
          "unit": "percentunit"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 4,
        "w": 3,
        "x": 9,
        "y": 6
      },
      "id": 17,
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "auto",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "showPercentChange": false,
        "textMode": "auto",
        "wideLayout": true
      },
      "pluginVersion": "11.1.0",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "(sum(increase(observme_workflows_completed_total[$__range])) or vector(0)) / clamp_min((sum(increase(observme_workflows_completed_total[$__range])) or vector(0)) + (sum(increase(observme_workflow_errors_total[$__range])) or vector(0)), 1)",
          "instant": true,
          "range": false,
          "refId": "A"
        }
      ],
      "title": "Workflow completion ratio",
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "description": "Subagent spawn operations in the selected range. Zero means no matching spawns were observed and can be healthy idle.",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              }
            ]
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 4,
        "w": 3,
        "x": 12,
        "y": 6
      },
      "id": 4,
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "auto",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "showPercentChange": false,
        "textMode": "auto",
        "wideLayout": true
      },
      "pluginVersion": "11.1.0",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "sum(increase(observme_subagents_spawned_total[$__range])) or vector(0)",
          "instant": true,
          "range": false,
          "refId": "A"
        }
      ],
      "title": "Subagent spawns",
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "description": "LLM requests in the selected range. Zero means no matching requests were observed and can be healthy idle.",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              }
            ]
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 4,
        "w": 3,
        "x": 15,
        "y": 6
      },
      "id": 5,
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "auto",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "showPercentChange": false,
        "textMode": "auto",
        "wideLayout": true
      },
      "pluginVersion": "11.1.0",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "sum(increase(observme_llm_requests_total[$__range])) or vector(0)",
          "instant": true,
          "range": false,
          "refId": "A"
        }
      ],
      "title": "LLM requests",
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "description": "Tool calls in the selected range. Zero means no matching calls were observed and can be healthy idle.",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              }
            ]
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 4,
        "w": 3,
        "x": 18,
        "y": 6
      },
      "id": 6,
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "auto",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "showPercentChange": false,
        "textMode": "auto",
        "wideLayout": true
      },
      "pluginVersion": "11.1.0",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "sum(increase(observme_tool_calls_total[$__range])) or vector(0)",
          "instant": true,
          "range": false,
          "refId": "A"
        }
      ],
      "title": "Tool calls",
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "description": "Trace-context propagation failures in the selected range. Zero means no matching failures were observed; investigate any non-zero value in the lineage logs and Tempo.",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 1
              }
            ]
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 4,
        "w": 3,
        "x": 21,
        "y": 6
      },
      "id": 7,
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "auto",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "showPercentChange": false,
        "textMode": "auto",
        "wideLayout": true
      },
      "pluginVersion": "11.1.0",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "sum(increase(observme_trace_context_propagation_failures_total[$__range])) or vector(0)",
          "instant": true,
          "range": false,
          "refId": "A"
        }
      ],
      "title": "Propagation failures",
      "type": "stat"
    },
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 10
      },
      "id": 101,
      "panels": [],
      "title": "Flow",
      "type": "row"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "description": "Per-minute aggregate work rate through major trace stages. Missing series indicate no matching metric samples; they are not synthesized into per-execution data.",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisBorderShow": false,
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "events/min",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 18,
            "gradientMode": "opacity",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "insertNulls": false,
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "never",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              }
            ]
          },
          "unit": "ops"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 24,
        "x": 0,
        "y": 11
      },
      "id": 8,
      "options": {
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true
        },
        "tooltip": {
          "mode": "multi",
          "sort": "none"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "60 * sum(rate(observme_turns_completed_total[$__rate_interval]))",
          "legendFormat": "turns",
          "range": true,
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "60 * sum(rate(observme_llm_requests_total[$__rate_interval]))",
          "legendFormat": "llm requests",
          "range": true,
          "refId": "B"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "60 * sum(rate(observme_tool_calls_total[$__rate_interval]))",
          "legendFormat": "tool calls",
          "range": true,
          "refId": "C"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "60 * sum(rate(observme_bash_executions_total[$__rate_interval]))",
          "legendFormat": "bash executions",
          "range": true,
          "refId": "D"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "60 * sum(rate(observme_subagents_spawned_total[$__rate_interval]))",
          "legendFormat": "subagent spawns",
          "range": true,
          "refId": "E"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "60 * sum(rate(observme_compactions_total[$__rate_interval]))",
          "legendFormat": "compactions",
          "range": true,
          "refId": "F"
        }
      ],
      "title": "Journey flow rate",
      "type": "timeseries"
    },
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 19
      },
      "id": 102,
      "panels": [],
      "title": "Latency",
      "type": "row"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "description": "Current p95 duration for major trace stages from bounded aggregate histograms. Sparse or idle windows can show no value; use Journey flow rate for volume context before choosing a representative Tempo trace.",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              }
            ]
          },
          "unit": "ms"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 24,
        "x": 0,
        "y": 20
      },
      "id": 9,
      "options": {
        "displayMode": "gradient",
        "maxVizHeight": 300,
        "minVizHeight": 10,
        "minVizWidth": 0,
        "namePlacement": "auto",
        "orientation": "horizontal",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "showUnfilled": true,
        "sizing": "auto"
      },
      "pluginVersion": "11.1.0",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "histogram_quantile(0.95, sum(rate(observme_agent_run_duration_ms_bucket[$__rate_interval])) by (le))",
          "instant": true,
          "legendFormat": "agent run",
          "range": false,
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "histogram_quantile(0.95, sum(rate(observme_turn_duration_ms_bucket[$__rate_interval])) by (le))",
          "instant": true,
          "legendFormat": "turn",
          "range": false,
          "refId": "B"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "histogram_quantile(0.95, sum(rate(observme_llm_request_duration_ms_bucket[$__rate_interval])) by (le))",
          "instant": true,
          "legendFormat": "llm",
          "range": false,
          "refId": "C"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "histogram_quantile(0.95, sum(rate(observme_tool_duration_ms_bucket[$__rate_interval])) by (le))",
          "instant": true,
          "legendFormat": "tool",
          "range": false,
          "refId": "D"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "histogram_quantile(0.95, sum(rate(observme_bash_duration_ms_bucket[$__rate_interval])) by (le))",
          "instant": true,
          "legendFormat": "bash",
          "range": false,
          "refId": "E"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "histogram_quantile(0.95, sum(rate(observme_agent_wait_duration_ms_bucket[$__rate_interval])) by (le))",
          "instant": true,
          "legendFormat": "wait",
          "range": false,
          "refId": "F"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "histogram_quantile(0.95, sum(rate(observme_agent_join_duration_ms_bucket[$__rate_interval])) by (le))",
          "instant": true,
          "legendFormat": "join",
          "range": false,
          "refId": "G"
        }
      ],
      "title": "p95 stage latency",
      "type": "bargauge"
    },
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 28
      },
      "id": 103,
      "panels": [],
      "title": "Agent lineage",
      "type": "row"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "description": "Aggregate fleet shape over time, not per-workflow truth: the leased-active role series is canonical current activity and preserves exact v2 lead, helper, worker, and validator values plus separate root and legacy subagent, orchestrator, reviewer, and unknown values. Tree histograms summarize all matching telemetry without workflow or agent IDs as metric labels. Clean shutdown removes leased activity after export/scrape propagation; after an ungraceful exit, leased activity converges to zero within the default 60-second lease plus one Prometheus scrape while raw claims may remain cached. /obs agents is local current-process child state. Constant lines mirror the depth=5 and fan-out=20 alert thresholds.",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisBorderShow": false,
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "count",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 12,
            "gradientMode": "opacity",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "insertNulls": false,
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "never",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              }
            ]
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 29
      },
      "id": 10,
      "options": {
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true
        },
        "tooltip": {
          "mode": "multi",
          "sort": "none"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "sum by (agent_role) (max by (observme_instance_id, agent_role) ((observme_active_agents > 0) and on (observme_instance_id) ((observme_agent_lease_expires_unixtime_seconds > time()) and (observme_agent_lease_expires_unixtime_seconds <= time() + 305)))) or on() vector(0)",
          "legendFormat": "active {{agent_role}}",
          "range": true,
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "histogram_quantile(0.95, sum(rate(observme_agent_tree_depth_bucket[$__rate_interval])) by (le))",
          "legendFormat": "tree depth p95",
          "range": true,
          "refId": "B"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "histogram_quantile(0.95, sum(rate(observme_agent_tree_width_bucket[$__rate_interval])) by (le))",
          "legendFormat": "tree width p95",
          "range": true,
          "refId": "C"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "histogram_quantile(0.95, sum(rate(observme_agent_fanout_count_bucket[$__rate_interval])) by (le))",
          "legendFormat": "fan-out p95",
          "range": true,
          "refId": "D"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "vector(5)",
          "refId": "E",
          "legendFormat": "depth alert threshold 5",
          "range": true
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "vector(20)",
          "refId": "F",
          "legendFormat": "fan-out alert threshold 20",
          "range": true
        }
      ],
      "title": "Agent tree shape",
      "type": "timeseries",
      "links": [
        {
          "targetBlank": false,
          "title": "Open Agents dashboard",
          "url": "/d/observme-agents/observme-agents?${__url_time_range}"
        },
        {
          "targetBlank": false,
          "title": "Open Agent Node Graphs",
          "url": "/d/observme-agent-node-graphs/observme-agent-node-graphs?${__url_time_range}"
        }
      ]
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "description": "Selected-range spawn attempts by emitted depth/reason labels, with failed spawn volume grouped separately by the failure counter’s emitted spawn_type/error_class labels. The two frames provide context and must not be read as direct label-matched attribution.",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisBorderShow": false,
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "spawns",
            "axisPlacement": "auto",
            "fillOpacity": 80,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineWidth": 1,
            "scaleDistribution": {
              "type": "linear"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              }
            ]
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 29
      },
      "id": 11,
      "options": {
        "barRadius": 0,
        "barWidth": 0.97,
        "fullHighlight": false,
        "groupWidth": 0.7,
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true
        },
        "orientation": "auto",
        "showValue": "auto",
        "stacking": "normal",
        "tooltip": {
          "mode": "multi",
          "sort": "none"
        },
        "xTickLabelRotation": 0,
        "xTickLabelSpacing": 0
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "sum(increase(observme_subagents_spawned_total[$__range])) by (subagent_depth, spawn_reason)",
          "instant": true,
          "legendFormat": "depth {{subagent_depth}} / {{spawn_reason}}",
          "range": false,
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "prometheus"
          },
          "editorMode": "code",
          "expr": "sum(increase(observme_subagent_spawn_failures_total[$__range])) by (spawn_type, error_class)",
          "refId": "B",
          "legendFormat": "failed {{spawn_type}} / {{error_class}}",
          "instant": true,
          "range": false
        }
      ],
      "title": "Subagent handoffs by depth and reason",
      "type": "barchart",
      "links": [
        {
          "targetBlank": false,
          "title": "Open Agents dashboard",
          "url": "/d/observme-agents/observme-agents?${__url_time_range}"
        }
      ]
    },
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 37
      },
      "id": 104,
      "panels": [],
      "title": "Logs",
      "type": "row"
    },
    {
      "datasource": {
        "type": "loki",
        "uid": "loki"
      },
      "description": "Chronological ObservMe journey events from Loki. Session, workflow, agent, and run filters narrow one execution; this remains a logs panel because Loki log streams do not provide state-timeline frames without extra transforms. Empty means no matching lifecycle logs were observed in the selected range.",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "fillOpacity": 70,
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineWidth": 0,
            "spanNulls": false
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              }
            ]
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 24,
        "x": 0,
        "y": 38
      },
      "id": 12,
      "options": {
        "dedupStrategy": "none",
        "enableLogDetails": true,
        "prettifyLogMessage": false,
        "showCommonLabels": false,
        "showLabels": true,
        "showTime": true,
        "sortOrder": "Ascending",
        "wrapLogMessage": true
      },
      "targets": [
        {
          "datasource": {
            "type": "loki",
            "uid": "loki"
          },
          "editorMode": "code",
          "expr": "{service_name=\"observme-pi-extension\", pi_session_id=~\"${session_id:regex}\", pi_workflow_id=~\"${workflow_id:regex}\", pi_agent_id=~\"${agent_id:regex}\", pi_agent_run_id=~\"${agent_run_id:regex}\", event_name=~\"session[.](started|shutdown)|workflow[.](started|completed|failed)|agent[.](run[.](started|completed|failed)|spawn[.](started|completed|failed)|wait[.](started|completed)|join[.](started|completed)|orphaned)|trace_context[.]propagation_failed|turn[.](started|completed)|llm[.]request[.](started|completed|failed)|tool[.]call[.](started|completed|failed)|bash[.]completed|branch[.]created|compaction[.]created\"}",
          "queryType": "range",
          "refId": "A",
          "maxLines": 1000,
          "range": true
        }
      ],
      "title": "Execution journey events",
      "type": "logs",
      "links": [
        {
          "targetBlank": false,
          "title": "Open matching LLM Conversations",
          "url": "/d/observme-llm-conversations/observme-llm-conversations?${__url_time_range}&${session_id:queryparam}&${workflow_id:queryparam}&${agent_id:queryparam}&${agent_run_id:queryparam}"
        }
      ]
    },
    {
      "datasource": {
        "type": "loki",
        "uid": "loki"
      },
      "description": "Chronological parent/child handoff and lineage-failure rows from Loki. The running agent's pi.agent.display_name appears beside pi.agent.id and pi.agent.role, while pi.agent.child.display_name and pi.agent.child.role identify presentation text for spawn rows. Names may be duplicated, so workflow, agent, child, trace, and span IDs remain the correlation and link keys. Empty means no matching handoff or lineage-failure logs were observed in the selected range.",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              }
            ]
          },
          "unit": "short"
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "trace_id"
            },
            "properties": [
              {
                "id": "links",
                "value": [
                  {
                    "targetBlank": false,
                    "title": "Open Tempo trace",
                    "url": "/explore?left=%7B%22datasource%22:%22tempo%22,%22queries%22:%5B%7B%22query%22:%22${__value.raw}%22%7D%5D,%22range%22:%7B%22from%22:%22${__from}%22,%22to%22:%22${__to}%22%7D%7D"
                  }
                ]
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "agent_run"
            },
            "properties": [
              {
                "id": "links",
                "value": [
                  {
                    "targetBlank": false,
                    "title": "Open filtered Trace Journey",
                    "url": "/d/observme-trace-journey/observme-trace-journey?${__url_time_range}&var-session_id=${__data.fields[\"session\"]}&var-workflow_id=${__data.fields[\"workflow\"]}&var-agent_id=${__data.fields[\"agent\"]}&var-agent_run_id=${__value.raw}"
                  }
                ]
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 10,
        "w": 24,
        "x": 0,
        "y": 46
      },
      "id": 13,
      "options": {
        "cellHeight": "sm",
        "footer": {
          "countRows": false,
          "fields": "",
          "reducer": [
            "sum"
          ],
          "show": false
        },
        "showHeader": true,
        "sortBy": [
          {
            "desc": false,
            "displayName": "Time"
          }
        ]
      },
      "targets": [
        {
          "datasource": {
            "type": "loki",
            "uid": "loki"
          },
          "editorMode": "code",
          "expr": "{service_name=\"observme-pi-extension\", service_namespace=\"pi\"} | pi_session_id=~\"${session_id:regex}\" | pi_workflow_id=~\"${workflow_id:regex}\" | pi_agent_id=~\"${agent_id:regex}\" | pi_agent_run_id=~\"${agent_run_id:regex}\" | event_category=\"agent-tree\" | event_name=~\"agent[.]spawn[.](started|completed|failed)|agent[.](wait|join)[.](started|completed)|agent[.]orphaned|trace_context[.]propagation_failed\" | line_format `{\"event\":{{ printf \"%q\" .event_name }},\"session\":{{ printf \"%q\" .pi_session_id }},\"workflow\":{{ printf \"%q\" .pi_workflow_id }},\"agent_name\":{{ printf \"%q\" .pi_agent_display_name }},\"agent_role\":{{ printf \"%q\" .pi_agent_role }},\"parent\":{{ printf \"%q\" .pi_agent_parent_id }},\"child\":{{ printf \"%q\" .pi_agent_child_id }},\"child_name\":{{ printf \"%q\" .pi_agent_child_display_name }},\"child_role\":{{ printf \"%q\" .pi_agent_child_role }},\"agent\":{{ printf \"%q\" .pi_agent_id }},\"agent_run\":{{ printf \"%q\" .pi_agent_run_id }},\"trace_id\":{{ printf \"%q\" .trace_id }},\"span_id\":{{ printf \"%q\" .span_id }}}`",
          "legendFormat": "{{event_name}}",
          "maxLines": 500,
          "queryType": "range",
          "range": true,
          "refId": "A"
        }
      ],
      "title": "Parent → child handoffs",
      "type": "table",
      "transformations": [
        {
          "id": "labelsToFields",
          "options": {
            "mode": "columns"
          }
        },
        {
          "id": "extractFields",
          "options": {
            "format": "json",
            "keepTime": true,
            "replace": false,
            "source": "Line"
          }
        },
        {
          "id": "organize",
          "options": {
            "excludeByName": {
              "Line": true,
              "id": true,
              "labels": true,
              "tsNs": true,
              "service_name": true,
              "telemetry_source": true
            },
            "renameByName": {
              "event": "Event",
              "agent_name": "Running agent name",
              "agent": "Agent ID",
              "agent_role": "Agent role",
              "parent": "Parent",
              "child_name": "Spawned child name",
              "child": "Child ID",
              "child_role": "Spawned child role",
              "workflow": "Workflow",
              "agent_run": "Agent run",
              "trace_id": "Trace ID",
              "span_id": "Span ID"
            },
            "indexByName": {
              "Time": 0,
              "event": 1,
              "agent_name": 2,
              "agent": 3,
              "agent_role": 4,
              "parent": 5,
              "child_name": 6,
              "child": 7,
              "child_role": 8,
              "workflow": 9,
              "agent_run": 10,
              "trace_id": 11,
              "span_id": 12
            }
          }
        }
      ],
      "pluginVersion": "11.1.0",
      "links": [
        {
          "targetBlank": false,
          "title": "Open Agents dashboard",
          "url": "/d/observme-agents/observme-agents?${__url_time_range}"
        },
        {
          "targetBlank": false,
          "title": "Open Agent Node Graphs",
          "url": "/d/observme-agent-node-graphs/observme-agent-node-graphs?${__url_time_range}"
        }
      ]
    },
    {
      "datasource": {
        "type": "loki",
        "uid": "loki"
      },
      "description": "Full chronological lifecycle log for the selected execution filters. Labels and log details remain available for correlation with Tempo; empty means no matching lifecycle logs were observed in the selected range.",
      "gridPos": {
        "h": 10,
        "w": 24,
        "x": 0,
        "y": 56
      },
      "id": 15,
      "options": {
        "dedupStrategy": "none",
        "enableLogDetails": true,
        "prettifyLogMessage": false,
        "showCommonLabels": false,
        "showLabels": true,
        "showTime": true,
        "sortOrder": "Ascending",
        "wrapLogMessage": true
      },
      "targets": [
        {
          "datasource": {
            "type": "loki",
            "uid": "loki"
          },
          "editorMode": "code",
          "expr": "{service_name=\"observme-pi-extension\", pi_session_id=~\"${session_id:regex}\", pi_workflow_id=~\"${workflow_id:regex}\", pi_agent_id=~\"${agent_id:regex}\", pi_agent_run_id=~\"${agent_run_id:regex}\", event_name=~\"session[.](started|shutdown)|workflow[.](started|completed|failed)|agent[.](run[.](started|completed|failed)|spawn[.](started|completed|failed)|wait[.](started|completed)|join[.](started|completed)|orphaned)|trace_context[.]propagation_failed|turn[.](started|completed)|llm[.]request[.](started|completed|failed)|tool[.]call[.](started|completed|failed)|bash[.]completed|branch[.]created|compaction[.]created\"}",
          "queryType": "range",
          "refId": "A",
          "maxLines": 1000,
          "range": true
        }
      ],
      "title": "Ordered journey log",
      "type": "logs",
      "links": [
        {
          "title": "Open matching Trace Journey filters",
          "url": "/d/observme-trace-journey/observme-trace-journey?${__url_time_range}&${session_id:queryparam}&${workflow_id:queryparam}&${agent_id:queryparam}&${agent_run_id:queryparam}",
          "targetBlank": false
        },
        {
          "title": "Open matching LLM Conversations",
          "url": "/d/observme-llm-conversations/observme-llm-conversations?${__url_time_range}&${session_id:queryparam}&${workflow_id:queryparam}&${agent_id:queryparam}&${agent_run_id:queryparam}",
          "targetBlank": false
        }
      ]
    },
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 66
      },
      "id": 105,
      "panels": [],
      "title": "Traces",
      "type": "row"
    },
    {
      "datasource": {
        "type": "tempo",
        "uid": "tempo"
      },
      "description": "Recent ObservMe traces returned by a valid service-scoped TraceQL search. Open a trace row for the waterfall, or use trace_id links in the handoff table for an exact execution. Dashboard links keep the current Loki execution filters beside the trace search.",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              }
            ]
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 9,
        "w": 24,
        "x": 0,
        "y": 67
      },
      "id": 14,
      "options": {
        "cellHeight": "sm",
        "footer": {
          "countRows": false,
          "fields": "",
          "reducer": [
            "sum"
          ],
          "show": false
        },
        "showHeader": true
      },
      "targets": [
        {
          "datasource": {
            "type": "tempo",
            "uid": "tempo"
          },
          "limit": 100,
          "query": "{ resource.service.name = \"observme-pi-extension\" }",
          "queryType": "traceql",
          "refId": "A",
          "tableType": "traces"
        }
      ],
      "title": "Recent Tempo traces",
      "type": "table",
      "links": [
        {
          "title": "Open matching journey logs",
          "url": "/d/observme-trace-journey/observme-trace-journey?${__url_time_range}&${session_id:queryparam}&${workflow_id:queryparam}&${agent_id:queryparam}&${agent_run_id:queryparam}",
          "targetBlank": false
        },
        {
          "title": "Open matching LLM Conversations",
          "url": "/d/observme-llm-conversations/observme-llm-conversations?${__url_time_range}&${session_id:queryparam}&${workflow_id:queryparam}&${agent_id:queryparam}&${agent_run_id:queryparam}",
          "targetBlank": false
        },
        {
          "title": "Open Agents dashboard",
          "url": "/d/observme-agents/observme-agents?${__url_time_range}",
          "targetBlank": false
        },
        {
          "title": "Open Agent Node Graphs",
          "url": "/d/observme-agent-node-graphs/observme-agent-node-graphs?${__url_time_range}",
          "targetBlank": false
        }
      ]
    }
  ],
  "refresh": "10s",
  "schemaVersion": 39,
  "style": "dark",
  "tags": [
    "observme",
    "pi",
    "trace",
    "journey",
    "tempo",
    "lineage"
  ],
  "templating": {
    "list": [
      {
        "allValue": ".*",
        "current": {
          "selected": true,
          "text": "All",
          "value": "$__all"
        },
        "datasource": {
          "type": "loki",
          "uid": "loki"
        },
        "definition": "label_values({service_name=\"observme-pi-extension\"}, pi_session_id)",
        "hide": 0,
        "includeAll": true,
        "label": "Session ID",
        "multi": true,
        "name": "session_id",
        "options": [],
        "query": "label_values({service_name=\"observme-pi-extension\"}, pi_session_id)",
        "refresh": 2,
        "regex": "",
        "skipUrlSync": false,
        "sort": 1,
        "type": "query"
      },
      {
        "allValue": ".*",
        "current": {
          "selected": true,
          "text": "All",
          "value": "$__all"
        },
        "datasource": {
          "type": "loki",
          "uid": "loki"
        },
        "definition": "label_values({service_name=\"observme-pi-extension\", pi_session_id=~\"${session_id:regex}\"}, pi_workflow_id)",
        "hide": 0,
        "includeAll": true,
        "label": "Workflow ID",
        "multi": true,
        "name": "workflow_id",
        "options": [],
        "query": "label_values({service_name=\"observme-pi-extension\", pi_session_id=~\"${session_id:regex}\"}, pi_workflow_id)",
        "refresh": 2,
        "regex": "",
        "skipUrlSync": false,
        "sort": 1,
        "type": "query"
      },
      {
        "allValue": ".*",
        "current": {
          "selected": true,
          "text": "All",
          "value": "$__all"
        },
        "datasource": {
          "type": "loki",
          "uid": "loki"
        },
        "definition": "label_values({service_name=\"observme-pi-extension\", pi_session_id=~\"${session_id:regex}\", pi_workflow_id=~\"${workflow_id:regex}\"}, pi_agent_id)",
        "hide": 0,
        "includeAll": true,
        "label": "Agent ID",
        "multi": true,
        "name": "agent_id",
        "options": [],
        "query": "label_values({service_name=\"observme-pi-extension\", pi_session_id=~\"${session_id:regex}\", pi_workflow_id=~\"${workflow_id:regex}\"}, pi_agent_id)",
        "refresh": 2,
        "regex": "",
        "skipUrlSync": false,
        "sort": 1,
        "type": "query"
      },
      {
        "allValue": ".*",
        "current": {
          "selected": true,
          "text": "All",
          "value": "$__all"
        },
        "datasource": {
          "type": "loki",
          "uid": "loki"
        },
        "definition": "label_values({service_name=\"observme-pi-extension\", pi_session_id=~\"${session_id:regex}\", pi_workflow_id=~\"${workflow_id:regex}\", pi_agent_id=~\"${agent_id:regex}\"}, pi_agent_run_id)",
        "hide": 0,
        "includeAll": true,
        "label": "Agent run ID",
        "multi": true,
        "name": "agent_run_id",
        "options": [],
        "query": "label_values({service_name=\"observme-pi-extension\", pi_session_id=~\"${session_id:regex}\", pi_workflow_id=~\"${workflow_id:regex}\", pi_agent_id=~\"${agent_id:regex}\"}, pi_agent_run_id)",
        "refresh": 2,
        "regex": "",
        "skipUrlSync": false,
        "sort": 1,
        "type": "query"
      }
    ]
  },
  "time": {
    "from": "now-6h",
    "to": "now"
  },
  "timepicker": {},
  "timezone": "browser",
  "title": "ObservMe Trace Journey",
  "uid": "observme-trace-journey",
  "version": 3,
  "weekStart": ""
}