{ "__requires": [ { "id": "grafana", "name": "Grafana", "type": "grafana", "version": "10.0.0" }, { "id": "prometheus", "name": "Prometheus", "type": "datasource", "version": "1.0.0" }, { "id": "timeseries", "name": "Time series", "type": "panel", "version": "" }, { "id": "stat", "name": "Stat", "type": "panel", "version": "" }, { "id": "bargauge", "name": "Bar gauge", "type": "panel", "version": "" }, { "id": "table", "name": "Table", "type": "panel", "version": "" } ], "annotations": { "list": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "enable": true, "hide": false, "iconColor": "rgba(31, 120, 193, 1)", "name": "Model changes", "expr": "sum(increase(observme_model_changes_total[$__interval])) by (provider, model) > 0", "titleFormat": "Model change", "textFormat": "{{provider}} / {{model}}" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "enable": true, "hide": false, "iconColor": "rgba(245, 183, 0, 1)", "name": "Thinking level changes", "expr": "sum(increase(observme_thinking_level_changes_total[$__interval])) by (environment) > 0", "titleFormat": "Thinking level change", "textFormat": "{{environment}}" } ] }, "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, "id": null, "links": [ { "icon": "external link", "includeVars": true, "keepTime": true, "targetBlank": false, "title": "Models", "tooltip": "Open provider/model traffic, reliability, and cost context.", "type": "link", "url": "/d/observme-models/observme-models?${__url_time_range}" }, { "icon": "external link", "includeVars": true, "keepTime": true, "targetBlank": false, "title": "Tools", "tooltip": "Open tool traffic, failures, sizes, and latency details.", "type": "link", "url": "/d/observme-tools/observme-tools?${__url_time_range}" }, { "icon": "external link", "includeVars": true, "keepTime": true, "targetBlank": false, "title": "Agents", "tooltip": "Open aggregate agent and subagent orchestration details.", "type": "link", "url": "/d/observme-agents/observme-agents?${__url_time_range}" }, { "icon": "external link", "includeVars": true, "keepTime": true, "targetBlank": false, "title": "Trace Journey", "tooltip": "Open trace/log drill-downs for latency spikes.", "type": "link", "url": "/d/observme-trace-journey/observme-trace-journey?${__url_time_range}" }, { "icon": "external link", "includeVars": true, "keepTime": true, "targetBlank": false, "title": "Errors", "tooltip": "Open failure logs and error ratios for correlated latency incidents.", "type": "link", "url": "/d/observme-errors/observme-errors?${__url_time_range}" } ], "liveNow": false, "panels": [ { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 100, "panels": [], "title": "Latency summary", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Rolling p95 duration for completed Pi turns. Compare Stage operation volume before acting on sparse samples. No series means no turn-duration bucket observations in the query window; latency is intentionally not synthesized as zero, so verify workload and Export Health before treating missing telemetry as healthy.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "ms" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 1 }, "id": 1, "links": [ { "targetBlank": false, "title": "Open Trace Journey", "url": "/d/observme-trace-journey/observme-trace-journey?${__url_time_range}" } ], "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum(rate(observme_turn_duration_ms_bucket[$__rate_interval])) by (le))", "legendFormat": "turn p95", "range": true, "refId": "A" } ], "title": "p95 turn latency", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Rolling p95 duration for LLM requests. Compare Stage operation volume and the bounded provider/model table before acting on sparse samples. No series means no LLM-duration bucket observations in the query window; latency is intentionally not synthesized as zero, so verify workload and Export Health before treating missing telemetry as healthy.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "ms" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 1 }, "id": 2, "links": [ { "targetBlank": false, "title": "Open Models", "url": "/d/observme-models/observme-models?${__url_time_range}" }, { "targetBlank": false, "title": "Open Trace Journey", "url": "/d/observme-trace-journey/observme-trace-journey?${__url_time_range}" } ], "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum(rate(observme_llm_request_duration_ms_bucket[$__rate_interval])) by (le))", "legendFormat": "llm p95", "range": true, "refId": "A" } ], "title": "p95 LLM latency", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Rolling p95 duration for tool calls. Compare Stage operation volume and the bounded tool table before acting on sparse samples. No series means no tool-duration bucket observations in the query window; latency is intentionally not synthesized as zero, so verify workload and Export Health before treating missing telemetry as healthy.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "ms" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 1 }, "id": 3, "links": [ { "targetBlank": false, "title": "Open Tools", "url": "/d/observme-tools/observme-tools?${__url_time_range}" }, { "targetBlank": false, "title": "Open Trace Journey", "url": "/d/observme-trace-journey/observme-trace-journey?${__url_time_range}" } ], "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum(rate(observme_tool_duration_ms_bucket[$__rate_interval])) by (le))", "legendFormat": "tool p95", "range": true, "refId": "A" } ], "title": "p95 tool latency", "type": "timeseries" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 9 }, "id": 101, "panels": [], "title": "Stage comparison", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Rolling p95 comparison across user turns, LLMs, tools, bash, agent runs, launcher spawns, waits, and joins for critical-path attribution. Spawn latency ends at usable transport-handle acquisition or pre-handle failure; child execution and result delivery remain in wait/join signals. Sparse stages can jump or disappear; compare selected-range operation volume. Missing duration series are left absent rather than shown as a healthy zero.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "ms" }, "overrides": [] }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 10 }, "id": 4, "links": [ { "targetBlank": false, "title": "Open Trace Journey", "url": "/d/observme-trace-journey/observme-trace-journey?${__url_time_range}" } ], "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum(rate(observme_turn_duration_ms_bucket[$__rate_interval])) by (le))", "legendFormat": "turn p95", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum(rate(observme_llm_request_duration_ms_bucket[$__rate_interval])) by (le))", "legendFormat": "llm p95", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum(rate(observme_tool_duration_ms_bucket[$__rate_interval])) by (le))", "legendFormat": "tool p95", "range": true, "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum(rate(observme_bash_duration_ms_bucket[$__rate_interval])) by (le))", "legendFormat": "bash p95", "range": true, "refId": "D" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum(rate(observme_agent_run_duration_ms_bucket[$__rate_interval])) by (le))", "legendFormat": "agent run p95", "range": true, "refId": "E" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum(rate(observme_subagent_spawn_duration_ms_bucket[$__rate_interval])) by (le))", "legendFormat": "spawn p95", "range": true, "refId": "F" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum(rate(observme_agent_wait_duration_ms_bucket[$__rate_interval])) by (le))", "legendFormat": "wait p95", "range": true, "refId": "G" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum(rate(observme_agent_join_duration_ms_bucket[$__rate_interval])) by (le))", "legendFormat": "join p95", "range": true, "refId": "H" } ], "title": "Latency comparison", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Selected-range p50/p95/p99 latency estimates for turns, LLMs, tools, bash, agent runs, subagent launcher spawn, parent wait, and parent join. Spawn latency ends at usable transport-handle acquisition or pre-handle failure and excludes delayed child execution/result delivery. Each instant query uses histogram bucket increases over $__range and retains le. Compare Stage operation volume: low counts make percentile ranks sparse, while no value means no matching duration observations or missing instrumentation and is not converted to zero.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "ms" }, "overrides": [] }, "gridPos": { "h": 10, "w": 24, "x": 0, "y": 18 }, "id": 5, "links": [ { "targetBlank": false, "title": "Open Trace Journey", "url": "/d/observme-trace-journey/observme-trace-journey?${__url_time_range}" } ], "options": { "cellHeight": "sm", "footer": { "countRows": false, "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "histogram_quantile(0.50, sum(increase(observme_turn_duration_ms_bucket[$__range])) by (le))", "legendFormat": "turn p50", "format": "table", "instant": true, "range": false, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum(increase(observme_turn_duration_ms_bucket[$__range])) by (le))", "legendFormat": "turn p95", "format": "table", "instant": true, "range": false, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "histogram_quantile(0.99, sum(increase(observme_turn_duration_ms_bucket[$__range])) by (le))", "legendFormat": "turn p99", "format": "table", "instant": true, "range": false, "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "histogram_quantile(0.50, sum(increase(observme_llm_request_duration_ms_bucket[$__range])) by (le))", "legendFormat": "llm p50", "format": "table", "instant": true, "range": false, "refId": "D" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum(increase(observme_llm_request_duration_ms_bucket[$__range])) by (le))", "legendFormat": "llm p95", "format": "table", "instant": true, "range": false, "refId": "E" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "histogram_quantile(0.99, sum(increase(observme_llm_request_duration_ms_bucket[$__range])) by (le))", "legendFormat": "llm p99", "format": "table", "instant": true, "range": false, "refId": "F" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "histogram_quantile(0.50, sum(increase(observme_tool_duration_ms_bucket[$__range])) by (le))", "legendFormat": "tool p50", "format": "table", "instant": true, "range": false, "refId": "G" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum(increase(observme_tool_duration_ms_bucket[$__range])) by (le))", "legendFormat": "tool p95", "format": "table", "instant": true, "range": false, "refId": "H" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "histogram_quantile(0.99, sum(increase(observme_tool_duration_ms_bucket[$__range])) by (le))", "legendFormat": "tool p99", "format": "table", "instant": true, "range": false, "refId": "I" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "histogram_quantile(0.50, sum(increase(observme_bash_duration_ms_bucket[$__range])) by (le))", "legendFormat": "bash p50", "format": "table", "instant": true, "range": false, "refId": "J" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum(increase(observme_bash_duration_ms_bucket[$__range])) by (le))", "legendFormat": "bash p95", "format": "table", "instant": true, "range": false, "refId": "K" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "histogram_quantile(0.99, sum(increase(observme_bash_duration_ms_bucket[$__range])) by (le))", "legendFormat": "bash p99", "format": "table", "instant": true, "range": false, "refId": "L" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "histogram_quantile(0.50, sum(increase(observme_agent_run_duration_ms_bucket[$__range])) by (le))", "legendFormat": "agent run p50", "format": "table", "instant": true, "range": false, "refId": "M" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum(increase(observme_agent_run_duration_ms_bucket[$__range])) by (le))", "legendFormat": "agent run p95", "format": "table", "instant": true, "range": false, "refId": "N" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "histogram_quantile(0.99, sum(increase(observme_agent_run_duration_ms_bucket[$__range])) by (le))", "legendFormat": "agent run p99", "format": "table", "instant": true, "range": false, "refId": "O" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "histogram_quantile(0.50, sum(increase(observme_subagent_spawn_duration_ms_bucket[$__range])) by (le))", "legendFormat": "spawn p50", "format": "table", "instant": true, "range": false, "refId": "P" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum(increase(observme_subagent_spawn_duration_ms_bucket[$__range])) by (le))", "legendFormat": "spawn p95", "format": "table", "instant": true, "range": false, "refId": "Q" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "histogram_quantile(0.99, sum(increase(observme_subagent_spawn_duration_ms_bucket[$__range])) by (le))", "legendFormat": "spawn p99", "format": "table", "instant": true, "range": false, "refId": "R" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "histogram_quantile(0.50, sum(increase(observme_agent_wait_duration_ms_bucket[$__range])) by (le))", "legendFormat": "wait p50", "format": "table", "instant": true, "range": false, "refId": "S" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum(increase(observme_agent_wait_duration_ms_bucket[$__range])) by (le))", "legendFormat": "wait p95", "format": "table", "instant": true, "range": false, "refId": "T" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "histogram_quantile(0.99, sum(increase(observme_agent_wait_duration_ms_bucket[$__range])) by (le))", "legendFormat": "wait p99", "format": "table", "instant": true, "range": false, "refId": "U" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "histogram_quantile(0.50, sum(increase(observme_agent_join_duration_ms_bucket[$__range])) by (le))", "legendFormat": "join p50", "format": "table", "instant": true, "range": false, "refId": "V" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum(increase(observme_agent_join_duration_ms_bucket[$__range])) by (le))", "legendFormat": "join p95", "format": "table", "instant": true, "range": false, "refId": "W" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "histogram_quantile(0.99, sum(increase(observme_agent_join_duration_ms_bucket[$__range])) by (le))", "legendFormat": "join p99", "format": "table", "instant": true, "range": false, "refId": "X" } ], "title": "Stage latency quantiles", "type": "table" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Selected-range instant operation counts for interpreting sparse percentile estimates. Zero means no matching counter increments were observed; it does not prove duration instrumentation is healthy. Wait/join have no dedicated completion counters, so agent-run volume is contextual rather than an exact wait/join sample count.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 28 }, "id": 6, "options": { "cellHeight": "sm", "footer": { "countRows": false, "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "sum(increase(observme_turns_completed_total[$__range])) or vector(0)", "legendFormat": "turn completions", "format": "table", "instant": true, "range": false, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "sum(increase(observme_llm_requests_total[$__range])) or vector(0)", "legendFormat": "llm requests", "format": "table", "instant": true, "range": false, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "sum(increase(observme_tool_calls_total[$__range])) or vector(0)", "legendFormat": "tool calls", "format": "table", "instant": true, "range": false, "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "sum(increase(observme_bash_executions_total[$__range])) or vector(0)", "legendFormat": "bash executions", "format": "table", "instant": true, "range": false, "refId": "D" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "sum(increase(observme_agent_runs_total[$__range])) or vector(0)", "legendFormat": "agent runs", "format": "table", "instant": true, "range": false, "refId": "E" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "sum(increase(observme_subagents_spawned_total[$__range])) or vector(0)", "legendFormat": "subagent spawns", "format": "table", "instant": true, "range": false, "refId": "F" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "sum(increase(observme_agent_runs_total[$__range])) by (agent_role)", "legendFormat": "agent runs by role (wait/join context) {{agent_role}}", "format": "table", "instant": true, "range": false, "refId": "G" } ], "title": "Stage operation volume", "type": "table" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 36 }, "id": 102, "panels": [], "title": "Top offenders", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Top 10 provider/model selected-range p95 LLM latency estimates with request counts. Histogram bucket increases retain provider, model, and le. Low request counts make percentile ranks sparse; an absent row means no matching observations and is not a zero-latency result.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "ms" }, "overrides": [ { "matcher": { "id": "byFrameRefID", "options": "B" }, "properties": [ { "id": "unit", "value": "short" } ] } ] }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 37 }, "id": 7, "links": [ { "targetBlank": false, "title": "Open Models", "url": "/d/observme-models/observme-models?${__url_time_range}" }, { "targetBlank": false, "title": "Open Trace Journey", "url": "/d/observme-trace-journey/observme-trace-journey?${__url_time_range}" } ], "options": { "cellHeight": "sm", "footer": { "countRows": false, "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "topk(10, histogram_quantile(0.95, sum(increase(observme_llm_request_duration_ms_bucket[$__range])) by (provider, model, le)))", "legendFormat": "p95 {{provider}} / {{model}}", "format": "table", "instant": true, "range": false, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "topk(10, sum(increase(observme_llm_requests_total[$__range])) by (provider, model))", "legendFormat": "requests {{provider}} / {{model}}", "format": "table", "instant": true, "range": false, "refId": "B" } ], "title": "Top slow provider/model with volume", "type": "table" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Top 10 tool selected-range p95 latency estimates with call counts. Histogram bucket increases retain tool_name and le. Low call counts make percentile ranks sparse; an absent row means no matching observations and is not a zero-latency result.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "ms" }, "overrides": [ { "matcher": { "id": "byFrameRefID", "options": "B" }, "properties": [ { "id": "unit", "value": "short" } ] } ] }, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 37 }, "id": 8, "links": [ { "targetBlank": false, "title": "Open Tools", "url": "/d/observme-tools/observme-tools?${__url_time_range}" }, { "targetBlank": false, "title": "Open Trace Journey", "url": "/d/observme-trace-journey/observme-trace-journey?${__url_time_range}" } ], "options": { "cellHeight": "sm", "footer": { "countRows": false, "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "topk(10, histogram_quantile(0.95, sum(increase(observme_tool_duration_ms_bucket[$__range])) by (tool_name, le)))", "legendFormat": "p95 {{tool_name}}", "format": "table", "instant": true, "range": false, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "topk(10, sum(increase(observme_tool_calls_total[$__range])) by (tool_name))", "legendFormat": "calls {{tool_name}}", "format": "table", "instant": true, "range": false, "refId": "B" } ], "title": "Top slow tools with volume", "type": "table" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Top 10 agent-role selected-range p95 run-latency estimates with run counts. Histogram bucket increases retain agent_role and le without aliases: v2 lead, helper, worker, and validator remain separate, as do root and legacy subagent, orchestrator, reviewer, and unknown values. Low run counts make percentile ranks sparse; an absent row means no matching observations and is not a zero-latency result.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "ms" }, "overrides": [ { "matcher": { "id": "byFrameRefID", "options": "B" }, "properties": [ { "id": "unit", "value": "short" } ] } ] }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 37 }, "id": 9, "links": [ { "targetBlank": false, "title": "Open Agents", "url": "/d/observme-agents/observme-agents?${__url_time_range}" }, { "targetBlank": false, "title": "Open Trace Journey", "url": "/d/observme-trace-journey/observme-trace-journey?${__url_time_range}" } ], "options": { "cellHeight": "sm", "footer": { "countRows": false, "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "topk(10, histogram_quantile(0.95, sum(increase(observme_agent_run_duration_ms_bucket[$__range])) by (agent_role, le)))", "legendFormat": "p95 {{agent_role}}", "format": "table", "instant": true, "range": false, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "topk(10, sum(increase(observme_agent_runs_total[$__range])) by (agent_role))", "legendFormat": "runs {{agent_role}}", "format": "table", "instant": true, "range": false, "refId": "B" } ], "title": "Top slow agent roles with volume", "type": "table" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 45 }, "id": 103, "panels": [], "title": "Detailed trends", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Top 10 rolling LLM request p95 latency trends grouped only by bounded provider and model labels. Compare the selected-range offender table for request volume. Sparse or absent buckets can make a series jump or disappear; missing duration telemetry is not converted to zero.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "ms" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 46 }, "id": 10, "links": [ { "targetBlank": false, "title": "Open Models", "url": "/d/observme-models/observme-models?${__url_time_range}" } ], "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "topk(10, histogram_quantile(0.95, sum(rate(observme_llm_request_duration_ms_bucket[$__rate_interval])) by (provider, model, le)))", "legendFormat": "{{provider}} / {{model}}", "range": true, "refId": "A" } ], "title": "LLM p95 latency by provider and model", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Top 10 rolling tool-call p95 latency trends grouped by the documented bounded tool_name label. Compare the selected-range offender table for call volume. Sparse or absent buckets can make a series jump or disappear; missing duration telemetry is not converted to zero.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "ms" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 46 }, "id": 11, "links": [ { "targetBlank": false, "title": "Open Tools", "url": "/d/observme-tools/observme-tools?${__url_time_range}" } ], "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "topk(10, histogram_quantile(0.95, sum(rate(observme_tool_duration_ms_bucket[$__rate_interval])) by (tool_name, le)))", "legendFormat": "{{tool_name}}", "range": true, "refId": "A" } ], "title": "Tool p95 latency by tool", "type": "timeseries" } ], "refresh": "30s", "schemaVersion": 39, "tags": [ "observme", "pi", "opentelemetry" ], "time": { "from": "now-6h", "to": "now" }, "timezone": "browser", "title": "ObservMe Latency", "uid": "observme-latency", "version": 2, "weekStart": "" }