{ "__requires": [ { "id": "grafana", "name": "Grafana", "type": "grafana", "version": "10.0.0" }, { "id": "prometheus", "name": "Prometheus", "type": "datasource", "version": "1.0.0" }, { "id": "row", "name": "Row", "type": "panel", "version": "" }, { "id": "timeseries", "name": "Time series", "type": "panel", "version": "" }, { "id": "stat", "name": "Stat", "type": "panel", "version": "" }, { "id": "bargauge", "name": "Bar gauge", "type": "panel", "version": "" }, { "id": "text", "name": "Text", "type": "panel", "version": "" } ], "annotations": { "list": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "enable": true, "hide": false, "iconColor": "rgba(31, 120, 193, 1)", "name": "Model changes", "expr": "sum(increase(observme_model_changes_total[$__interval])) by (provider, model) > 0", "titleFormat": "Model change", "textFormat": "{{provider}} / {{model}}" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "enable": true, "hide": false, "iconColor": "rgba(245, 183, 0, 1)", "name": "Thinking level changes", "expr": "sum(increase(observme_thinking_level_changes_total[$__interval])) by (environment) > 0", "titleFormat": "Thinking level change", "textFormat": "{{environment}}" } ] }, "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, "id": null, "links": [ { "icon": "external link", "includeVars": true, "keepTime": true, "targetBlank": false, "title": "Overview", "tooltip": "Open the overview with the current time range.", "type": "link", "url": "/d/observme-overview/observme-overview?${__url_time_range}" }, { "icon": "external link", "includeVars": true, "keepTime": true, "targetBlank": false, "title": "Models", "tooltip": "Open model traffic, latency, stop-reason, and cost-efficiency panels.", "type": "link", "url": "/d/observme-models/observme-models?${__url_time_range}" }, { "icon": "external link", "includeVars": true, "keepTime": true, "targetBlank": false, "title": "LLM lifecycle logs", "tooltip": "Open LLM token and non-content lifecycle logs.", "type": "link", "url": "/d/observme-logs-llm/observme-logs-llm?${__url_time_range}" }, { "icon": "external link", "includeVars": true, "keepTime": true, "targetBlank": false, "title": "LLM Conversations", "tooltip": "Open optional captured LLM content with privacy guidance.", "type": "link", "url": "/d/observme-llm-conversations/observme-llm-conversations?${__url_time_range}" }, { "icon": "external link", "includeVars": true, "keepTime": true, "targetBlank": false, "title": "Errors", "tooltip": "Open LLM failure ratios, failure logs, and trace links.", "type": "link", "url": "/d/observme-errors/observme-errors?${__url_time_range}" } ], "liveNow": false, "panels": [ { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 100, "panels": [], "title": "Spend overview", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Total provider-reported LLM spend in the selected dashboard range. Missing cost series render as zero. The $40/$50 colors are review references; the $50 alert threshold applies specifically to a one-hour window.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 40 }, { "color": "red", "value": 50 } ] }, "unit": "currencyUSD" }, "overrides": [] }, "gridPos": { "h": 5, "w": 6, "x": 0, "y": 1 }, "id": 1, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "11.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "sum(increase(observme_llm_cost_usd_total[$__range])) or vector(0)", "instant": true, "range": false, "refId": "A" } ], "title": "Total selected-range cost", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Average hourly LLM cost estimated from selected-range spend. Yellow/red align with 80%/100% of the default $50-in-1h cost-spike alert threshold; tune for your deployment.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 40 }, { "color": "red", "value": 50 } ] }, "unit": "currencyUSD" }, "overrides": [] }, "gridPos": { "h": 5, "w": 6, "x": 6, "y": 1 }, "id": 2, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "11.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "(sum(increase(observme_llm_cost_usd_total[$__range])) or vector(0)) / ($__range_s / 3600)", "instant": true, "range": false, "refId": "A" } ], "title": "Cost burn rate per hour", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Selected-range spend divided by the default $50/hour budget scaled to the dashboard range length. This is an average budget-envelope comparison, not the state of the rolling one-hour cost-spike alert.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 0.8 }, { "color": "red", "value": 1 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 5, "w": 6, "x": 12, "y": 1 }, "id": 3, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "11.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "(sum(increase(observme_llm_cost_usd_total[$__range])) or vector(0)) / (50 * ($__range_s / 3600))", "instant": true, "range": false, "refId": "A" } ], "title": "Selected-range budget usage", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Selected-range average burn extrapolated to 24 hours. Yellow/red align with 80%/100% of the default $1,200 daily equivalent; short or unrepresentative ranges can be noisy.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 960 }, { "color": "red", "value": 1200 } ] }, "unit": "currencyUSD" }, "overrides": [] }, "gridPos": { "h": 5, "w": 6, "x": 18, "y": 1 }, "id": 4, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "11.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "(sum(increase(observme_llm_cost_usd_total[$__range])) or vector(0)) * 86400 / $__range_s", "instant": true, "range": false, "refId": "A" } ], "title": "Projected 24h cost", "type": "stat" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 6 }, "id": 101, "panels": [], "title": "Cost attribution", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Top provider cost totals over the selected range. Missing cost series render as zero; open Models for traffic and cost-efficiency context.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "currencyUSD" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 7 }, "id": 5, "options": { "displayMode": "gradient", "maxVizHeight": 300, "minVizHeight": 16, "minVizWidth": 8, "namePlacement": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showUnfilled": true, "sizing": "auto", "valueMode": "color" }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "(topk(10, sum by (provider) (increase(observme_llm_cost_usd_total[$__range]))) or on() vector(0))", "instant": true, "legendFormat": "{{provider}}", "range": false, "refId": "A" } ], "title": "Cost by provider", "type": "bargauge" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Top provider/model cost totals over the selected range. Keeping provider in the grouping prevents identically named models from different providers being merged.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "currencyUSD" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 7 }, "id": 6, "options": { "displayMode": "gradient", "maxVizHeight": 300, "minVizHeight": 16, "minVizWidth": 8, "namePlacement": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showUnfilled": true, "sizing": "auto", "valueMode": "color" }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "(topk(10, sum by (provider, model) (increase(observme_llm_cost_usd_total[$__range]))) or on() vector(0))", "instant": true, "legendFormat": "{{provider}} / {{model}}", "range": false, "refId": "A" } ], "title": "Cost by provider/model", "type": "bargauge" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Selected-range provider-reported LLM cost grouped by deployment environment. Missing cost series render as zero.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "currencyUSD" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 7 }, "id": 7, "options": { "displayMode": "gradient", "maxVizHeight": 300, "minVizHeight": 16, "minVizWidth": 8, "namePlacement": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showUnfilled": true, "sizing": "auto", "valueMode": "color" }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "(sum by (environment) (increase(observme_llm_cost_usd_total[$__range])) or on() vector(0))", "instant": true, "legendFormat": "{{environment}}", "range": false, "refId": "A" } ], "title": "Cost by environment", "type": "bargauge" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Top provider/model shares of total selected-range LLM spend. This adds explicit percent-of-total context beside the absolute attribution panels; missing cost series render as zero.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "mappings": [], "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 15 }, "id": 14, "links": [ { "targetBlank": false, "title": "Open Models", "url": "/d/observme-models/observme-models?${__url_time_range}" } ], "options": { "displayMode": "gradient", "maxVizHeight": 300, "minVizHeight": 16, "minVizWidth": 8, "namePlacement": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showUnfilled": true, "sizing": "auto", "valueMode": "color" }, "pluginVersion": "11.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "(topk(10, sum by (provider, model) (increase(observme_llm_cost_usd_total[$__range])) / scalar(clamp_min((sum(increase(observme_llm_cost_usd_total[$__range])) or vector(0)), 1e-9))) or on() vector(0))", "instant": true, "legendFormat": "{{provider}} / {{model}}", "range": false, "refId": "A" } ], "title": "Cost share by provider/model", "type": "bargauge" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 22 }, "id": 102, "panels": [], "title": "Cost trend", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "LLM cost added across all providers and models for each Prometheus-safe rolling interval. $__rate_interval keeps the window large enough for counter samples; small costs display with six decimal places.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "decimals": 6, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "currencyUSD" }, "overrides": [] }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 23 }, "id": 8, "links": [ { "targetBlank": false, "title": "Open Models", "url": "/d/observme-models/observme-models?${__url_time_range}" }, { "targetBlank": false, "title": "Open LLM lifecycle logs", "url": "/d/observme-logs-llm/observme-logs-llm?${__url_time_range}" } ], "options": { "legend": { "calcs": [ "sum", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "11.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "sum(increase(observme_llm_cost_usd_total[$__rate_interval])) or vector(0)", "legendFormat": "all providers / models", "range": true, "refId": "A" } ], "title": "Cost over time", "type": "timeseries" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 31 }, "id": 103, "panels": [], "title": "Tokens and cache", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Highest cumulative token value observed per exporter series in the selected range, summed globally by type. This preserves short-lived Pi session exports that counter increases can miss. Provider/model detail stays in Models and LLM lifecycle logs.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 32 }, "id": 9, "options": { "displayMode": "gradient", "maxVizHeight": 300, "minVizHeight": 16, "minVizWidth": 8, "namePlacement": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showUnfilled": true, "sizing": "auto", "valueMode": "color" }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "sum(max_over_time(observme_llm_input_tokens_total[$__range])) or vector(0)", "instant": true, "legendFormat": "input", "range": false, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "sum(max_over_time(observme_llm_output_tokens_total[$__range])) or vector(0)", "instant": true, "legendFormat": "output", "range": false, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "sum(max_over_time(observme_llm_total_tokens_total[$__range])) or vector(0)", "instant": true, "legendFormat": "total", "range": false, "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "sum(max_over_time(observme_llm_reasoning_tokens_total[$__range])) or vector(0)", "instant": true, "legendFormat": "reasoning", "range": false, "refId": "D" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "sum(max_over_time(observme_llm_cache_read_tokens_total[$__range])) or vector(0)", "instant": true, "legendFormat": "cache read", "range": false, "refId": "E" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "sum(max_over_time(observme_llm_cache_write_tokens_total[$__range])) or vector(0)", "instant": true, "legendFormat": "cache write", "range": false, "refId": "F" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "sum(max_over_time(observme_llm_cache_write_1h_tokens_total[$__range])) or vector(0)", "instant": true, "legendFormat": "cache write 1h", "range": false, "refId": "G" } ], "title": "Token totals by type", "type": "bargauge" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Top provider/model cache-read, cache-write, and one-hour cache-write token totals over the selected range. Missing cache metrics render as zero.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 32 }, "id": 10, "options": { "displayMode": "gradient", "maxVizHeight": 300, "minVizHeight": 16, "minVizWidth": 8, "namePlacement": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showUnfilled": true, "sizing": "auto", "valueMode": "color" }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "(topk(10, sum by (provider, model) (increase(observme_llm_cache_read_tokens_total[$__range]))) or on() vector(0))", "instant": true, "legendFormat": "cache read {{provider}} / {{model}}", "range": false, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "(topk(10, sum by (provider, model) (increase(observme_llm_cache_write_tokens_total[$__range]))) or on() vector(0))", "instant": true, "legendFormat": "cache write {{provider}} / {{model}}", "range": false, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "(topk(10, sum by (provider, model) (increase(observme_llm_cache_write_1h_tokens_total[$__range]))) or on() vector(0))", "instant": true, "legendFormat": "cache write 1h {{provider}} / {{model}}", "range": false, "refId": "C" } ], "title": "Cache read/write token usage", "type": "bargauge" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Estimated cache-read share of reusable prompt tokens: cache reads divided by input plus cache-read tokens. This is a proxy, not a provider hit-count or health metric; zero can mean idle workload or unsupported cache accounting.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "mappings": [], "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "blue", "value": null } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 5, "w": 12, "x": 0, "y": 40 }, "id": 11, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "11.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "clamp_max((sum(increase(observme_llm_cache_read_tokens_total[$__range])) or vector(0)) / clamp_min(((sum(increase(observme_llm_cache_read_tokens_total[$__range])) or vector(0)) + (sum(increase(observme_llm_input_tokens_total[$__range])) or vector(0))), 1), 1)", "instant": true, "range": false, "refId": "A" } ], "title": "Cache read ratio", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "Provider-reported cache-read tokens over the selected range. Treat this as an estimated token saving only where cache-read accounting is supported; missing series render as zero.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "gray", "value": null }, { "color": "green", "value": 1 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 5, "w": 12, "x": 12, "y": 40 }, "id": 12, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "11.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "sum(increase(observme_llm_cache_read_tokens_total[$__range])) or vector(0)", "instant": true, "range": false, "refId": "A" } ], "title": "Estimated cache savings (tokens)", "type": "stat" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 45 }, "id": 104, "panels": [], "title": "Budget guidance", "type": "row" }, { "description": "Visible interpretation guidance for the dashboard's default budget references and forecasts.", "gridPos": { "h": 5, "w": 24, "x": 0, "y": 46 }, "id": 13, "options": { "code": { "language": "markdown", "showLineNumbers": false, "showMiniMap": false }, "content": "### Default budget references\n\n- **ObservMeCostSpike** alerts when LLM spend exceeds **$50 in 1 hour** for 15 minutes. Tune this budget-dependent threshold for your deployment.\n- **Selected-range budget usage** scales that $50/hour reference to the selected time range; 100% means spend matched the average envelope. It does not detect a short spike hidden inside a longer range.\n- **Projected 24h cost** extrapolates the average burn rate from the selected range. Very short, idle, or unrepresentative ranges can produce misleading forecasts.\n- Cost metrics use provider-reported usage. Missing cost series render as zero and do not prove that a provider supplied complete pricing data.", "mode": "markdown" }, "pluginVersion": "11.1.0", "title": "How to interpret budget and forecast panels", "type": "text" } ], "refresh": "30s", "schemaVersion": 39, "tags": [ "observme", "pi", "llm", "cost", "tokens" ], "time": { "from": "now-6h", "to": "now" }, "timezone": "browser", "title": "ObservMe Cost", "uid": "observme-cost", "version": 8, "weekStart": "" }