import type { ExtensionAPI } from "@earendil-works/pi-coding-agent"; import type { LoadSessionConfigOptions } from "../config/load-config.ts"; import type { ObservMeConfig } from "../config/schema.ts"; import type { PrometheusFetch, PrometheusMetricSeries, QueryResult } from "../query/prometheus.ts"; import { assertPrometheusVectorResult, createPrometheusQueryClient } from "../query/prometheus.ts"; import { boundObsCommandOutput, normalizeObsBackendLabel, selectObsCommandRows, } from "../safety/display-bounds.ts"; import { completeObsSubcommand, isExactObsSubcommandRequest } from "./obs-args.ts"; import { loadObsCommandConfig, notifyObsCommand } from "./obs-command-support.ts"; import { appendObsRecoveryHint, formatObsCommandDiagnostic, formatObsCommandFailure, sanitizeObsDiagnosticText, } from "./obs-diagnostics.ts"; export interface ObsToolsCommandContext { readonly cwd?: string; readonly ui: { notify: (message: string, type?: "info" | "warning" | "error") => Promise | void; }; readonly isProjectTrusted?: () => boolean | Promise; } export interface ObsToolCallRow { readonly toolName: string; readonly ratePerSecond: number; readonly timestampUnixSeconds?: string; } export interface ObsToolFailureRow { readonly toolName: string; readonly errorClass: string; readonly ratePerSecond: number; readonly timestampUnixSeconds?: string; } export type ObsToolsQuerySection = "calls" | "failures"; export interface ObsToolsQueryWarning { readonly section: ObsToolsQuerySection; readonly message: string; } export interface ObsToolsSnapshot { readonly window: "1h"; readonly callQuery: string; readonly failureQuery: string; readonly calls: readonly ObsToolCallRow[]; readonly failures: readonly ObsToolFailureRow[]; readonly queryWarnings?: readonly ObsToolsQueryWarning[]; } export type ObsToolsConfigLoader = (options: LoadSessionConfigOptions) => Promise; export type ObsToolsProvider = (ctx: ObsToolsCommandContext) => Promise | ObsToolsSnapshot; export interface ObsToolsSnapshotOptions { readonly loadConfig?: ObsToolsConfigLoader; readonly fetch?: PrometheusFetch; readonly env?: NodeJS.ProcessEnv; readonly configDirName?: string; } export interface RegisterObsToolsCommandOptions extends ObsToolsSnapshotOptions { readonly getTools?: ObsToolsProvider; } export const OBS_TOOLS_CALLS_PROMQL = "topk(10, sum(rate(observme_tool_calls_total[1h])) by (tool_name))"; export const OBS_TOOLS_FAILURES_PROMQL = "sum(rate(observme_tool_failures_total[1h])) by (tool_name, error_class)"; const OBS_COMMAND_NAME = "obs"; const OBS_TOOLS_SUBCOMMAND = "tools"; const OBS_TOOLS_WINDOW = "1h"; const OBS_TOOLS_USAGE = "Usage: /obs tools"; const OBS_TOOLS_ERROR_NEXT_ACTION = "run /obs health and verify query.grafana.url, Grafana credentials, and the Metrics datasource UID."; const OBS_TOOLS_CALLS_ERROR_NEXT_ACTION = "verify the Metrics datasource and observme_tool_calls_total with /obs health, then rerun /obs tools."; const OBS_TOOLS_FAILURES_ERROR_NEXT_ACTION = "verify the Metrics datasource and observme_tool_failures_total with /obs health, then rerun /obs tools."; const OBS_TOOLS_NO_CALLS_NEXT_ACTION = "run tool activity, then verify the Metrics datasource with /obs health."; const OBS_TOOLS_NO_FAILURES_NEXT_ACTION = "check after a failing tool call, then verify Metrics labels with /obs health."; type ObsToolsRequestStatus = "tools" | "usage"; interface ObsToolsQueryOutcome { readonly result: QueryResult; readonly warning?: ObsToolsQueryWarning; } interface ObsToolsQueryResults { readonly calls: QueryResult; readonly failures: QueryResult; readonly warnings: readonly ObsToolsQueryWarning[]; } export function registerObsToolsCommand(pi: ExtensionAPI, options: RegisterObsToolsCommandOptions = {}): void { const command = new ObsToolsCommand(options); pi.registerCommand(OBS_COMMAND_NAME, { description: "Show aggregate ObservMe tool call and failure rates. Usage: /obs tools", getArgumentCompletions: getObsToolsCommandArgumentCompletions, handler: command.handle.bind(command), }); } export async function handleObsToolsCommand( args: string, ctx: ObsToolsCommandContext, options: RegisterObsToolsCommandOptions = {}, ): Promise { if (parseObsToolsRequest(args) === "usage") { await notifyObsCommand(ctx, OBS_TOOLS_USAGE, "warning"); return; } try { const snapshot = await resolveObsToolsSnapshot(ctx, options); await notifyObsCommand(ctx, renderObsTools(snapshot), resolveObsToolsNotificationType(snapshot)); } catch (error) { await notifyObsCommand( ctx, formatObsCommandFailure("ObservMe tools unavailable", error, { subsystem: "Prometheus", nextAction: OBS_TOOLS_ERROR_NEXT_ACTION, }), "error", ); } } export function getObsToolsCommandArgumentCompletions(prefix: string): Array<{ value: string; label: string }> | null { return completeObsSubcommand(prefix, OBS_TOOLS_SUBCOMMAND); } export async function getObsToolsSnapshot( ctx: ObsToolsCommandContext, options: ObsToolsSnapshotOptions = {}, ): Promise { const config = await loadObsToolsConfig(ctx, options); const result = await queryObsTools(config, options); return { window: OBS_TOOLS_WINDOW, callQuery: OBS_TOOLS_CALLS_PROMQL, failureQuery: OBS_TOOLS_FAILURES_PROMQL, calls: result.calls.series.map(toObsToolCallRow).filter(isObsToolCallRow), failures: result.failures.series.map(toObsToolFailureRow).filter(isObsToolFailureRow), queryWarnings: result.warnings, }; } export function renderObsTools(snapshot: ObsToolsSnapshot): string { const calls = snapshot.calls.map(normalizeObsToolCallRow).filter(isObsToolCallRow); const failures = snapshot.failures.map(normalizeObsToolFailureRow).filter(isObsToolFailureRow); const callSelection = selectObsCommandRows(calls); const failureSelection = selectObsCommandRows(failures); const window = normalizeObsBackendLabel(snapshot.window) ?? OBS_TOOLS_WINDOW; const callsWarning = findObsToolsQueryWarning(snapshot, "calls"); const failuresWarning = findObsToolsQueryWarning(snapshot, "failures"); const lines = [`Tool calls by tool (last ${window})`]; if (callsWarning) { lines.push(renderObsToolsQueryWarning("Tool calls", callsWarning)); } else if (calls.length === 0) { lines.push(appendObsRecoveryHint("No tool call metrics found.", OBS_TOOLS_NO_CALLS_NEXT_ACTION)); } else { lines.push(...callSelection.rows.map(renderObsToolCallRow)); if (callSelection.omittedCount > 0) lines.push(`… ${callSelection.omittedCount} tool call row(s) omitted`); } lines.push(`Tool failures by tool/error (last ${window})`); if (failuresWarning) { lines.push(renderObsToolsQueryWarning("Tool failures", failuresWarning)); } else if (failures.length === 0) { lines.push(appendObsRecoveryHint("No tool failure metrics found.", OBS_TOOLS_NO_FAILURES_NEXT_ACTION)); } else { lines.push(...failureSelection.rows.map(renderObsToolFailureRow)); if (failureSelection.omittedCount > 0) { lines.push(`… ${failureSelection.omittedCount} tool failure row(s) omitted`); } } return boundObsCommandOutput(lines.join("\n")); } class ObsToolsCommand { readonly #options: RegisterObsToolsCommandOptions; constructor(options: RegisterObsToolsCommandOptions) { this.#options = options; } async handle(args: string, ctx: ObsToolsCommandContext): Promise { await handleObsToolsCommand(args, ctx, this.#options); } } async function resolveObsToolsSnapshot( ctx: ObsToolsCommandContext, options: RegisterObsToolsCommandOptions, ): Promise { if (options.getTools) return options.getTools(ctx); return getObsToolsSnapshot(ctx, options); } async function loadObsToolsConfig(ctx: ObsToolsCommandContext, options: ObsToolsSnapshotOptions): Promise { return loadObsCommandConfig(ctx, options); } async function queryObsTools(config: ObservMeConfig, options: ObsToolsSnapshotOptions): Promise { const client = createPrometheusQueryClient(config, { fetch: options.fetch }); const [callsResult, failuresResult] = await Promise.allSettled([ client.queryPrometheus(OBS_TOOLS_CALLS_PROMQL, undefined, { resultLimit: "metricSeries" }), client.queryPrometheus(OBS_TOOLS_FAILURES_PROMQL, undefined, { resultLimit: "metricSeries" }), ]); const calls = resolveObsToolsQueryOutcome(callsResult, "calls"); const failures = resolveObsToolsQueryOutcome(failuresResult, "failures"); const warnings = [calls.warning, failures.warning].filter(isObsToolsQueryWarning); return { calls: calls.result, failures: failures.result, warnings }; } function resolveObsToolsQueryOutcome( settled: PromiseSettledResult, section: ObsToolsQuerySection, ): ObsToolsQueryOutcome { if (settled.status === "rejected") { return createUnavailableObsToolsQueryOutcome(section, settled.reason); } try { assertPrometheusVectorResult(settled.value); return { result: settled.value }; } catch (error) { return createUnavailableObsToolsQueryOutcome(section, error); } } function createUnavailableObsToolsQueryOutcome( section: ObsToolsQuerySection, error: unknown, ): ObsToolsQueryOutcome { const nextAction = section === "calls" ? OBS_TOOLS_CALLS_ERROR_NEXT_ACTION : OBS_TOOLS_FAILURES_ERROR_NEXT_ACTION; return { result: createEmptyObsToolsQueryResult(), warning: { section, message: formatObsCommandDiagnostic(error, nextAction), }, }; } function createEmptyObsToolsQueryResult(): QueryResult { return { resultType: "vector", series: [] }; } function toObsToolCallRow(series: PrometheusMetricSeries): ObsToolCallRow | undefined { const ratePerSecond = parseRatePerSecond(series.value?.value); if (ratePerSecond === undefined) return undefined; return { toolName: normalizeMetricLabel(series.metric.tool_name), ratePerSecond, timestampUnixSeconds: series.value?.timestampUnixSeconds, }; } function toObsToolFailureRow(series: PrometheusMetricSeries): ObsToolFailureRow | undefined { const ratePerSecond = parseRatePerSecond(series.value?.value); if (ratePerSecond === undefined) return undefined; return { toolName: normalizeMetricLabel(series.metric.tool_name), errorClass: normalizeMetricLabel(series.metric.error_class), ratePerSecond, timestampUnixSeconds: series.value?.timestampUnixSeconds, }; } function normalizeObsToolCallRow(row: ObsToolCallRow): ObsToolCallRow | undefined { const ratePerSecond = parseRatePerSecond(row.ratePerSecond); if (ratePerSecond === undefined) return undefined; return { toolName: normalizeMetricLabel(row.toolName), ratePerSecond, timestampUnixSeconds: normalizeOptionalString(row.timestampUnixSeconds), }; } function normalizeObsToolFailureRow(row: ObsToolFailureRow): ObsToolFailureRow | undefined { const ratePerSecond = parseRatePerSecond(row.ratePerSecond); if (ratePerSecond === undefined) return undefined; return { toolName: normalizeMetricLabel(row.toolName), errorClass: normalizeMetricLabel(row.errorClass), ratePerSecond, timestampUnixSeconds: normalizeOptionalString(row.timestampUnixSeconds), }; } function parseRatePerSecond(value: string | number | undefined): number | undefined { if (value === undefined) return undefined; const ratePerSecond = Number(value); if (!Number.isFinite(ratePerSecond) || ratePerSecond < 0) return undefined; return ratePerSecond; } function normalizeMetricLabel(value: string | undefined): string { return normalizeObsBackendLabel(value) ?? "unknown"; } function normalizeOptionalString(value: string | undefined): string | undefined { return normalizeObsBackendLabel(value); } function resolveObsToolsNotificationType(snapshot: ObsToolsSnapshot): "info" | "warning" | "error" { const callsUnavailable = findObsToolsQueryWarning(snapshot, "calls") !== undefined; const failuresUnavailable = findObsToolsQueryWarning(snapshot, "failures") !== undefined; if (callsUnavailable && failuresUnavailable) return "error"; return callsUnavailable || failuresUnavailable ? "warning" : "info"; } function findObsToolsQueryWarning( snapshot: ObsToolsSnapshot, section: ObsToolsQuerySection, ): ObsToolsQueryWarning | undefined { return snapshot.queryWarnings?.find(warning => warning.section === section); } function renderObsToolsQueryWarning(label: string, warning: ObsToolsQueryWarning): string { return `${label} unavailable: ${sanitizeObsDiagnosticText(warning.message)}`; } function renderObsToolCallRow(row: ObsToolCallRow): string { return `${row.toolName}: ${formatRatePerSecond(row.ratePerSecond)}`; } function renderObsToolFailureRow(row: ObsToolFailureRow): string { return `${row.toolName} / ${row.errorClass}: ${formatRatePerSecond(row.ratePerSecond)}`; } function isObsToolCallRow(row: ObsToolCallRow | undefined): row is ObsToolCallRow { return row !== undefined; } function isObsToolFailureRow(row: ObsToolFailureRow | undefined): row is ObsToolFailureRow { return row !== undefined; } function isObsToolsQueryWarning(warning: ObsToolsQueryWarning | undefined): warning is ObsToolsQueryWarning { return warning !== undefined; } function parseObsToolsRequest(args: string): ObsToolsRequestStatus { return isExactObsSubcommandRequest(args, OBS_TOOLS_SUBCOMMAND) ? "tools" : "usage"; } function formatRatePerSecond(value: number): string { return `${trimTrailingFractionZeros(value.toFixed(4))}/s`; } function trimTrailingFractionZeros(value: string): string { if (!value.includes(".")) return value; let end = value.length; while (end > 0 && value[end - 1] === "0") end -= 1; const withoutTrailingZeros = value.slice(0, end); return withoutTrailingZeros.endsWith(".") ? withoutTrailingZeros.slice(0, -1) : withoutTrailingZeros; }