import { createHash, createHmac, timingSafeEqual } from "node:crypto"; import { existsSync } from "node:fs"; import { createServer, type IncomingMessage, type Server, type ServerResponse } from "node:http"; import { isIP } from "node:net"; import { dirname, join, resolve } from "node:path"; import { DEFAULT_LEASE_TTL_MS, DEFAULT_MAX_HOPS, DEFAULT_MESSAGE_RETENTION_MS, DEFAULT_MESSAGE_TTL_MS, DEFAULT_RATE_LIMIT_MAX, DEFAULT_RATE_LIMIT_WINDOW_MS, DEFAULT_STALE_AFTER_MS, IMPROVEMENT_AREAS, MAX_AGENT_HOST_CHARS, MAX_BODY_BYTES, MAX_CONTENT_CHARS, MAX_LEASE_RESOURCE_CHARS, MAX_LEASE_TTL_MS, MAX_MESSAGE_TTL_MS, MAX_SYNC_BATCH_EVENTS, MIN_LEASE_TTL_MS, MIN_MESSAGE_TTL_MS, MIN_MESSAGE_RETENTION_MS, ProtocolError, newId, nowIso, optionalString, parseBoundedInteger, parseDeliveryMode, requireString, toAgentRecord, type AgentRecord, type DeliveryMode, type HubEvent, type LeaseRecord, type MessageRecord, type RuntimePresenceRecord, type StoredSyncEvent, type WorkflowMessageContext, } from "./protocol.ts"; import { kxmCanonicalJson, syncEventSchemaErrors, type JsonValue } from "./project-config.ts"; import { kxmSyncEventHash } from "./sync-transform.ts"; import { workflowScopeExtras } from "./diagnostics.ts"; import { timingSafeStringCompare } from "./commands.ts"; import { arbitrate, explainContextItem, journalEntryToContextItem, memoryRecordToContextItem, rolePolicy } from "./arbiter.ts"; import { loadAuthoredMemory } from "./memory.ts"; import { contextItemAuditMetadata, CONTEXT_AUTHORITIES, CONTEXT_CONFIDENCES, type ContextAuthority, type ContextConfidence, type ContextItem } from "./context.ts"; import { rankRecall, relevanceTokens } from "./relevance.ts"; import { NativeStateProvider } from "./state.ts"; import { SkillLifecycle } from "./skills.ts"; import { compileKnowledgeWiki, lintKnowledgeWiki, type WikiSourcePool } from "./wiki.ts"; import { buildRetrospective, writeRetrospective } from "./retrospective.ts"; import { MeshStore, type LeaseRefusal, type StoredAgent } from "./store.ts"; import { canonicalWorkflowEvidenceKey, approveWorkflowDegradation, checkpointRun, improvementReport, applyJournalPromotion, JOURNAL_CATEGORIES, journalAttemptFor, journalEvidenceRequired, parseJournalCategory, rankImprovementSignals, renderWorkflowPrompt, resumeWorkflowFromSignal, valueAtPath, waitForWorkflowSignal, verifyWorkflowEvidenceReferences, workflowDefinitionHash, workflowEvidenceStrings, workflowWebhookSignature, WORKFLOW_WEBHOOK_MAX_SKEW_SECONDS, type ImprovementArea, type JournalCategory, type WebhookWorkflowDefinition, type WorkflowCheckpointStatus, type WorkflowEvidenceInput, type WorkflowEvidenceReferenceInput, type WorkflowJournalEntry, type WorkflowRun, type WorkflowSignalReceipt, type WorkflowStageState, type WorkflowVerifiedEvidence, type WorkflowWebhookScope, } from "./workflow.ts"; export interface RateLimitOptions { windowMs?: number; maxRequests?: number; } export interface MeshHubOptions { host?: string; port?: number; authToken?: string; projectTokens?: Record; dataPath?: string; assetsDir?: string; staleAfterMs?: number; messageTtlMs?: number; messageRetentionMs?: number; cleanupIntervalMs?: number; shutdownGraceMs?: number; rateLimit?: RateLimitOptions | false; webhookWorkflows?: WebhookWorkflowDefinition[]; logger?: (entry: Record) => void; skillsDir?: string; skillLifecycle?: SkillLifecycle; repoRoot?: string; /** The hub clock every lease decision reads. Injectable so a test can move a * lease past its deadline instead of sleeping through a real TTL; production * leaves it at `Date.now`. It is never a client-supplied time. */ now?: () => number; } export interface MeshHub { readonly server: Server; readonly state: { agents: Map; messages: Map; workflowRuns: Map; journal: Map; persistent: boolean; }; start(): Promise<{ host: string; port: number; url: string }>; close(): Promise; } type SseClient = { response: ServerResponse; heartbeat: NodeJS.Timeout; presenceOnly?: boolean }; type OpsSseClient = SseClient & { project: string }; type OpsTopic = "agents" | "messages" | "workflows"; type RateBucket = { startedAt: number; count: number }; const DEFAULT_WORKFLOW_WAIT_TIMEOUT_MS = 24 * 60 * 60_000; const MIN_WORKFLOW_WAIT_TIMEOUT_MS = 1_000; const MAX_WORKFLOW_WAIT_TIMEOUT_MS = 30 * 24 * 60 * 60_000; function isLoopback(host: string): boolean { if (host === "localhost" || host === "::1") return true; return isIP(host) === 4 && host.startsWith("127."); } /** Drop the agent key and attach hub-clocked presence. Every caller passes the * hub's own `staleAfterMs` and clock, so no reader ever sees a lease derived * from a client timestamp. */ function publicAgent(agent: StoredAgent, staleAfterMs: number, now = Date.now()): AgentRecord { const { key: _key, ...identity } = agent; return toAgentRecord(identity, staleAfterMs, now); } /** Runtime ids share the contract's opaque-id shape (`rtm_…`). */ const RUNTIME_ID_PATTERN = /^[a-z][a-z0-9]{1,15}_[A-Za-z0-9][A-Za-z0-9_-]{5,127}$/; function requireRuntimeId(value: unknown): string { const runtimeId = requireString(value, "runtimeId", { max: 144 }); if (!RUNTIME_ID_PATTERN.test(runtimeId)) { throw new ProtocolError(400, "runtimeId must be an opaque runtime id", "invalid_runtime_id"); } return runtimeId; } /** A Runtime's presence as readers see it. The lease is the same hub-clocked * window agents get: `heartbeatAt + staleAfterMs`. */ function runtimePresenceView(record: RuntimePresenceRecord, staleAfterMs: number, nowMs: number) { const heartbeatMs = Date.parse(record.heartbeatAt); const leaseExpiresMs = Number.isFinite(heartbeatMs) ? heartbeatMs + staleAfterMs : 0; return { runtimeId: record.runtimeId, ...(record.host ? { host: record.host } : {}), registeredAt: record.registeredAt, heartbeatAt: record.heartbeatAt, leaseExpiresAt: new Date(leaseExpiresMs).toISOString(), presence: leaseExpiresMs > nowMs ? "online" as const : "expired" as const, }; } function safeTokenEqual(actual: string | undefined, expected: string): boolean { return timingSafeStringCompare(actual, expected); } /** The wire answer for a lease the hub will not extend or release. Every one of * these means the caller's token no longer authorizes a shared write: the * holder must stop, not retry. The current lease rides along so the caller can * record who holds the resource now. */ function leaseRefusal(reason: LeaseRefusal, name: string, lease?: LeaseRecord): ProtocolError { if (reason === "missing") { return new ProtocolError(404, `no lease is held on ${name}`, "lease_not_found"); } if (reason === "expired") { return new ProtocolError( 409, `lease on ${name} expired at ${lease?.expiresAt ?? "its deadline"}; re-acquire to take it over`, "lease_expired", { lease }, ); } if (reason === "held") { return new ProtocolError( 409, `lease on ${name} is held by ${lease?.holderAgentName ?? "another agent"} until ${lease?.expiresAt ?? "its deadline"}`, "lease_held", { lease }, ); } return new ProtocolError( 409, `fencing token for ${name} was superseded; the hub is now at token ${lease?.fencingToken ?? "a newer value"}`, "lease_superseded", { lease }, ); } function bearerToken(request: IncomingMessage): string | undefined { const header = request.headers.authorization; return header?.startsWith("Bearer ") ? header.slice(7) : undefined; } function json(response: ServerResponse, statusCode: number, payload: unknown): void { const body = JSON.stringify(payload); response.writeHead(statusCode, { "content-type": "application/json; charset=utf-8", "content-length": Buffer.byteLength(body), "cache-control": "no-store", "x-content-type-options": "nosniff", }); response.end(body); } function text(response: ServerResponse, statusCode: number, body: string, contentType: string): void { response.writeHead(statusCode, { "content-type": contentType, "content-length": Buffer.byteLength(body), "cache-control": "no-store", "x-content-type-options": "nosniff", }); response.end(body); } async function readBody(request: IncomingMessage): Promise { const contentType = request.headers["content-type"] ?? ""; if (!contentType.toLowerCase().startsWith("application/json")) { throw new ProtocolError(415, "content-type must be application/json", "unsupported_media_type"); } const chunks: Buffer[] = []; let length = 0; for await (const chunk of request) { const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); length += buffer.length; if (length > MAX_BODY_BYTES) { throw new ProtocolError(413, "request body is too large", "payload_too_large"); } chunks.push(buffer); } return Buffer.concat(chunks); } function parseJsonBody(buffer: Buffer): Record { if (buffer.length === 0) return {}; try { const value = JSON.parse(buffer.toString("utf8")); if (value === null || Array.isArray(value) || typeof value !== "object") { throw new Error("body must be an object"); } return value as Record; } catch { throw new ProtocolError(400, "request body must be valid JSON object", "invalid_json"); } } async function readJson(request: IncomingMessage): Promise> { return parseJsonBody(await readBody(request)); } function sameWorkflowMessageContext( left: WorkflowMessageContext | undefined, right: WorkflowMessageContext | undefined, ): boolean { if (!left || !right) return left === right; return left.schema === right.schema && left.runId === right.runId && left.stageId === right.stageId && left.requirementKey === right.requirementKey && left.attempt === right.attempt; } function parseOptionalBoolean(value: unknown, field: string): boolean { if (value === undefined || value === null) return false; if (typeof value === "boolean") return value; throw new ProtocolError(400, `${field} must be a boolean`); } function sameIdempotentRequest( message: MessageRecord, target: string, content: string, delivery: DeliveryMode, correlationId: string | undefined, replyTo: string | undefined, hops: number, maxHops: number, ttlMs: number, workflowContext: WorkflowMessageContext | undefined, ): boolean { return (message.to === target || message.toName.toLowerCase() === target.toLowerCase()) && message.content === content && message.delivery === delivery && message.correlationId === correlationId && message.replyTo === replyTo && message.hops === hops && message.maxHops === maxHops && sameWorkflowMessageContext(message.workflowContext, workflowContext) && Date.parse(message.expiresAt) - Date.parse(message.createdAt) === ttlMs; } function parseContextAuthority(value: unknown): ContextAuthority { if (typeof value !== "string" || !CONTEXT_AUTHORITIES.includes(value as ContextAuthority)) { throw new ProtocolError(400, "authority must be one of policy, instruction, evidence, hypothesis", "invalid_context_request"); } return value as ContextAuthority; } function parseContextConfidence(value: unknown): ContextConfidence { if (typeof value !== "string" || !CONTEXT_CONFIDENCES.includes(value as ContextConfidence)) { throw new ProtocolError(400, "confidence must be one of verified, probable, uncertain", "invalid_context_request"); } return value as ContextConfidence; } function boundedStringList(value: unknown, field: string, maxItems = 32): string[] { if (value === undefined) return []; if (!Array.isArray(value) || value.length > maxItems) { throw new ProtocolError(400, `${field} must be an array with at most ${maxItems} items`); } return value.map((item, index) => requireString(item, `${field}[${index}]`, { max: 1_000 })); } function boundedWorkflowEvidence(value: unknown, field = "evidence", maxItems = 64): WorkflowEvidenceInput { if (value === undefined) return {}; if (!value || typeof value !== "object" || Array.isArray(value)) { throw new ProtocolError(400, `${field} must be an object keyed by required evidence identity`, "invalid_workflow_evidence"); } const entries = Object.entries(value as Record); if (entries.length > maxItems) { throw new ProtocolError(400, `${field} must contain at most ${maxItems} keyed items`, "invalid_workflow_evidence"); } const evidence = new Map(); for (const [rawRequirement, rawEvidence] of entries) { const requirement = canonicalWorkflowEvidenceKey( requireString(rawRequirement, `${field} requirement`, { max: 128 }), ); if (evidence.has(requirement)) { throw new ProtocolError( 400, `${field} contains duplicate normalized requirement identity: ${requirement}`, "invalid_workflow_evidence", ); } evidence.set(requirement, requireString(rawEvidence, `${field}.${requirement}`, { max: 1_000 })); } return Object.fromEntries(evidence); } function boundedWorkflowEvidenceReferences( value: unknown, field = "evidenceRefs", maxItems = 32, ): WorkflowEvidenceReferenceInput { if (value === undefined) return {}; if (!value || typeof value !== "object" || Array.isArray(value)) { throw new ProtocolError(400, `${field} must be an object keyed by required evidence identity`, "invalid_workflow_evidence_refs"); } const entries = Object.entries(value as Record); if (entries.length > maxItems) { throw new ProtocolError(400, `${field} must contain at most ${maxItems} requirements`, "invalid_workflow_evidence_refs"); } const references = new Map(); for (const [rawRequirement, rawReference] of entries) { const requirement = canonicalWorkflowEvidenceKey( requireString(rawRequirement, `${field} requirement`, { max: 128 }), ); if (references.has(requirement)) { throw new ProtocolError( 400, `${field} contains duplicate normalized requirement identity: ${requirement}`, "invalid_workflow_evidence_refs", ); } if (!rawReference || typeof rawReference !== "object" || Array.isArray(rawReference)) { throw new ProtocolError(400, `${field}.${requirement} must be an object`, "invalid_workflow_evidence_refs"); } const referenceObject = rawReference as Record; if (Object.keys(referenceObject).some((key) => key !== "messageIds")) { throw new ProtocolError( 400, `${field}.${requirement} may contain only messageIds; provenance is hub-derived`, "invalid_workflow_evidence_refs", ); } const rawIds = referenceObject.messageIds; if (!Array.isArray(rawIds) || rawIds.length < 1 || rawIds.length > 16) { throw new ProtocolError( 400, `${field}.${requirement}.messageIds must contain between 1 and 16 IDs`, "invalid_workflow_evidence_refs", ); } const messageIds = rawIds.map((candidate, index) => requireString(candidate, `${field}.${requirement}.messageIds[${index}]`, { max: 80 })); if (new Set(messageIds).size !== messageIds.length) { throw new ProtocolError(400, `${field}.${requirement}.messageIds must be unique`, "invalid_workflow_evidence_refs"); } references.set(requirement, { messageIds }); } return Object.fromEntries(references); } function requestedWorkflowMessageContext(value: unknown): Omit | undefined { if (value === undefined) return undefined; if (!value || typeof value !== "object" || Array.isArray(value)) { throw new ProtocolError(400, "workflowContext must be an object", "invalid_workflow_context"); } const context = value as Record; if (Object.keys(context).some((key) => !["runId", "stageId", "requirementKey", "attempt"].includes(key))) { throw new ProtocolError(400, "workflowContext contains unsupported fields", "invalid_workflow_context"); } if (context.attempt === undefined) { throw new ProtocolError(400, "workflowContext.attempt is required", "invalid_workflow_context"); } return { runId: requireString(context.runId, "workflowContext.runId", { max: 80 }), stageId: requireString(context.stageId, "workflowContext.stageId", { max: 64 }), requirementKey: canonicalWorkflowEvidenceKey( requireString(context.requirementKey, "workflowContext.requirementKey", { max: 128 }), ), attempt: parseBoundedInteger(context.attempt, "workflowContext.attempt", 1, 1, 20), }; } function validateWorkflowSignalContext( evidence: WorkflowEvidenceInput, runId: string, stageId: string, signalKey: string, ): void { const expectedContext: Record = { "workflow.run": runId, "workflow.stage": stageId, "workflow.signal": signalKey, }; for (const [contextKey, expectedValue] of Object.entries(expectedContext)) { const suppliedValue = evidence[contextKey]; if (suppliedValue === undefined || suppliedValue === expectedValue) continue; throw new ProtocolError( 409, `evidence ${contextKey} does not match the workflow signal route and active wait`, "workflow_signal_context_mismatch", { contextKey }, ); } } function workflowSignalKey(value: unknown): string { const key = requireString(value, "signalKey", { max: 128 }); if (!/^[A-Za-z0-9][A-Za-z0-9._:-]*$/.test(key)) { throw new ProtocolError(400, "signalKey may contain letters, numbers, dot, underscore, colon, and hyphen", "invalid_signal_key"); } return key; } export function createMeshHub(options: MeshHubOptions = {}): MeshHub { const host = options.host ?? "127.0.0.1"; const port = options.port ?? 7331; const authToken = options.authToken?.trim(); const projectTokens = Object.fromEntries( Object.entries(options.projectTokens ?? {}).map(([project, token]) => [project, token.trim()]), ); const staleAfterMs = options.staleAfterMs ?? DEFAULT_STALE_AFTER_MS; const defaultMessageTtlMs = options.messageTtlMs ?? DEFAULT_MESSAGE_TTL_MS; const messageRetentionMs = options.messageRetentionMs ?? DEFAULT_MESSAGE_RETENTION_MS; const cleanupIntervalMs = options.cleanupIntervalMs ?? Math.max(250, Math.floor(staleAfterMs / 3)); const shutdownGraceMs = options.shutdownGraceMs ?? 5_000; const rateLimit = options.rateLimit === false ? undefined : { windowMs: options.rateLimit?.windowMs ?? DEFAULT_RATE_LIMIT_WINDOW_MS, maxRequests: options.rateLimit?.maxRequests ?? DEFAULT_RATE_LIMIT_MAX, }; const webhookWorkflows = new Map((options.webhookWorkflows ?? []).map((workflow) => [workflow.id, workflow])); const logger = options.logger ?? (() => undefined); const hubNow = options.now ?? (() => Date.now()); const assetsDir = options.assetsDir; const hubRepoRoot = options.repoRoot ?? (options.dataPath && options.dataPath !== ":memory:" ? resolve(dirname(dirname(options.dataPath))) : process.cwd()); const store = new MeshStore(options.dataPath); const agents = store.agents; const messages = store.messages; function agentLogSide(id: string, fallbackName: string, side: "from" | "to") { const agent = agents.get(id); return { [side]: id, [`${side}Name`]: agent?.name ?? fallbackName, [`${side}Online`]: Boolean(agent?.online), ...(agent?.model ? { [`${side}Model`]: agent.model } : {}), }; } function messageLog(message: MessageRecord, fromId: string, fromName: string, toId: string, toName: string) { return { project: message.project, status: message.status, delivery: message.delivery, ...agentLogSide(fromId, fromName, "from"), ...agentLogSide(toId, toName, "to"), }; } const workflowRuns = store.workflowRuns; const journal = store.journal; const stateProvider = new NativeStateProvider(store); const skillLifecycle = options.skillLifecycle ?? (options.skillsDir || existsSync(join(process.cwd(), ".kxm", "skills")) ? new SkillLifecycle(options.skillsDir ?? join(process.cwd(), ".kxm", "skills")) : undefined); const streams = new Map>(); const opsStreams = new Set(); const rateBuckets = new Map(); const counters = { requests: 0, errors: 0, registrations: 0, messagesSent: 0, messagesReplied: 0, messagesCancelled: 0, messagesExpired: 0, messagesPurged: 0, webhooksAccepted: 0, workflowCheckpoints: 0, workflowWaits: 0, workflowSignals: 0, workflowWaitTimeouts: 0, workflowDegradations: 0, journalEntries: 0, contextRequests: 0, attemptLatencySecondsTotal: 0, meteredCostUsdTotal: 0, leasesGranted: 0, leasesRefused: 0, leasesReleased: 0, syncEventsAccepted: 0, syncEventsDuplicate: 0, syncEventsRefused: 0, syncConflicts: 0, runtimeHeartbeats: 0, }; let cleanupTimer: NodeJS.Timeout | undefined; let closed = false; function exportTerminalRetrospective(run: WorkflowRun): void { if (!assetsDir || (run.status !== "completed" && run.status !== "failed")) return; try { const entries = [...journal.values()].filter((entry) => entry.runId === run.id); const files = writeRetrospective(`${assetsDir}${process.platform === "win32" ? "\\" : "/"}retrospectives`, buildRetrospective(run, entries, run.updatedAt)); logger({ event: "workflow_retrospective_exported", workflowRunId: run.id, jsonPath: files.jsonPath, markdownPath: files.mdPath }); } catch (error) { logger({ event: "workflow_retrospective_export_failed", workflowRunId: run.id, error: error instanceof Error ? error.message : "retrospective_export_failed" }); } } if (!isLoopback(host) && !authToken) { store.close(); throw new Error("KXM_AUTH_TOKEN is required when binding beyond localhost"); } if ( staleAfterMs < 100 || defaultMessageTtlMs < MIN_MESSAGE_TTL_MS || messageRetentionMs < MIN_MESSAGE_RETENTION_MS || shutdownGraceMs < 0 ) { store.close(); throw new Error("stale and message TTL settings are below supported minimums"); } if (rateLimit && (rateLimit.windowMs < 100 || rateLimit.maxRequests < 1)) { store.close(); throw new Error("rate limit settings are invalid"); } if (webhookWorkflows.size !== (options.webhookWorkflows ?? []).length) { store.close(); throw new Error("webhook workflow IDs must be unique"); } for (const agent of agents.values()) { agent.online = false; store.saveAgent(agent); } function expectedProjectToken(project: string): string | undefined { return projectTokens[project] || authToken; } function requireProjectAuth(request: IncomingMessage, project: string): void { const expected = expectedProjectToken(project); if (!expected) return; if (!safeTokenEqual(bearerToken(request), expected)) { throw new ProtocolError(401, "invalid project authentication token", "invalid_auth", { operation: "other", nextAction: "check_project_token", }); } } /** Context callers authenticate either as a registered agent (project- * scoped to their own project) or with the administrative token (single * explicit project scope per request). Returns the validated project, a * stable caller identity for provenance, and which credential proved it. */ function contextCallerProject( request: IncomingMessage, requested: unknown, ): { project: string; caller: string; credential: "agent" | "admin" } { const project = requireString(requested, "project", { max: 200 }); const agentHeader = request.headers["x-kxm-agent-id"]; if (typeof agentHeader === "string" && agentHeader.trim()) { const agent = requireAgent(request); requireProjectAuth(request, agent.project); if (agent.project !== project) { throw new ProtocolError(403, "context requests are limited to the agent's project", "context_isolation_violation"); } return { project, caller: agent.id, credential: "agent" }; } requireAdminAuth(request); const callerHeader = request.headers["x-kxm-caller-id"]; const caller = typeof callerHeader === "string" && callerHeader.trim() ? callerHeader.trim() : "kxm-admin"; return { project, caller, credential: "admin" }; } function requireAdminAuth(request: IncomingMessage): void { if (!authToken && isLoopback(host)) return; if (!authToken || !safeTokenEqual(bearerToken(request), authToken)) { throw new ProtocolError(401, "invalid administrative authentication token", "invalid_auth", { operation: "other", nextAction: "check_project_token", }); } } function requireConfiguredAdminAuth(request: IncomingMessage, purpose = "workflow degradation approval"): void { if (!authToken) { throw new ProtocolError( 503, `KXM_AUTH_TOKEN must be configured for ${purpose}`, "admin_auth_not_configured", ); } requireAdminAuth(request); } function requireAgent(request: IncomingMessage, expectedId?: string): StoredAgent { const agentId = expectedId ?? String(request.headers["x-kxm-agent-id"] ?? ""); const agentKey = String(request.headers["x-kxm-agent-key"] ?? ""); const agent = agents.get(agentId); if (!agent || !agentKey || !safeTokenEqual(agentKey, agent.key)) { throw new ProtocolError(401, "invalid agent identity", "invalid_agent_identity", { operation: "other", nextAction: "reconnect_with_current_agent_key", }); } const wasOffline = !agent.online; agent.lastSeenAt = nowIso(); agent.online = true; store.saveAgent(agent); if (wasOffline) { broadcastPresence(agent); // An agent the stale sweep retired may still hold an open SSE stream (the sweep // marks offline without closing it). Without this flush, messages queued while it // read offline expire unread even though the stream could deliver them. if ((streams.get(agent.id)?.size ?? 0) > 0) flushPending(agent.id); } return agent; } function checkRateLimit(request: IncomingMessage, response: ServerResponse): void { if (!rateLimit) return; const key = String(request.headers["x-kxm-agent-id"] ?? request.socket.remoteAddress ?? "unknown"); const now = Date.now(); const current = rateBuckets.get(key); const bucket = !current || now - current.startedAt >= rateLimit.windowMs ? { startedAt: now, count: 0 } : current; bucket.count += 1; rateBuckets.set(key, bucket); const remaining = Math.max(0, rateLimit.maxRequests - bucket.count); response.setHeader("x-ratelimit-limit", String(rateLimit.maxRequests)); response.setHeader("x-ratelimit-remaining", String(remaining)); if (bucket.count > rateLimit.maxRequests) { const retryAfterSeconds = Math.max(1, Math.ceil((bucket.startedAt + rateLimit.windowMs - now) / 1_000)); response.setHeader("retry-after", String(retryAfterSeconds)); throw new ProtocolError(429, "request rate limit exceeded", "rate_limited"); } } function publish(agentId: string, event: HubEvent): boolean { const clients = streams.get(agentId); if (!clients || clients.size === 0) return false; let frame: string | undefined; let published = false; for (const client of clients) { if (client.presenceOnly && event.type !== "presence") continue; frame ??= `event: ${event.type}\ndata: ${JSON.stringify(event)}\n\n`; client.response.write(frame); published = true; } return published; } function publishOps(project: string, topic: OpsTopic): void { const frame = `event: ops\ndata: ${JSON.stringify({ type: "ops", project, topic, at: nowIso() })}\n\n`; for (const client of opsStreams) { if (client.project === project) client.response.write(frame); } } /** * Synchronized runs grouped by their home Runtime. Built from the gapless * prefix of each run only, so a missing sequence is shown as pending and * never papered over. `orphaned` is view state: the home Runtime's presence * lease has expired (or it never registered). Nothing is migrated. */ function homeRuntimesView(project: string, nowMs: number) { const presence = new Map(store.listRuntimePresence(project).map((record) => [record.runtimeId, record])); const runs = new Map(); for (const event of store.listSyncEvents(project)) { const key = `${event.projectId}\u0000${event.runId}`; const bucket = runs.get(key); if (bucket) bucket.push(event); else runs.set(key, [event]); } const homes = new Map>>(); for (const events of runs.values()) { const first = events[0]!; let cursor = 0; let status: string | undefined; let workflowId: string | undefined; let displayTitle: string | undefined; let updatedAt: string | undefined; for (const event of events) { if (event.sequence !== cursor + 1) break; cursor = event.sequence; const parsed = JSON.parse(event.bytes) as { occurredAt?: string; payload?: Record }; const payload = parsed.payload ?? {}; if (event.eventType.startsWith("run.") && typeof payload.status === "string") status = payload.status; if (typeof payload.workflowId === "string") workflowId = payload.workflowId; if (typeof payload.displayTitle === "string") displayTitle = payload.displayTitle; if (typeof parsed.occurredAt === "string") updatedAt = parsed.occurredAt; } const home = presence.get(first.homeRuntimeId); const orphaned = !home || runtimePresenceView(home, staleAfterMs, nowMs).presence === "expired"; const list = homes.get(first.homeRuntimeId) ?? []; list.push({ projectId: first.projectId, runId: first.runId, ...(workflowId ? { workflowId } : {}), ...(displayTitle ? { displayTitle } : {}), ...(status ? { status } : {}), lastSequence: cursor, pendingGap: events.length > cursor, ...(updatedAt ? { updatedAt } : {}), orphaned, }); homes.set(first.homeRuntimeId, list); } for (const runtimeId of presence.keys()) if (!homes.has(runtimeId)) homes.set(runtimeId, []); return [...homes.entries()] .sort(([left], [right]) => left.localeCompare(right)) .map(([runtimeId, homeRuns]) => { const record = presence.get(runtimeId); const view = record ? runtimePresenceView(record, staleAfterMs, nowMs) : undefined; return { runtimeId, ...(view ? { host: view.host, heartbeatAt: view.heartbeatAt, leaseExpiresAt: view.leaseExpiresAt } : {}), presence: view?.presence ?? "unknown", orphaned: !view || view.presence === "expired", runs: homeRuns .sort((left, right) => String(right.updatedAt ?? "").localeCompare(String(left.updatedAt ?? ""))) .slice(0, 16), runTotal: homeRuns.length, }; }); } function opsSnapshot(project: string) { const snapshotAt = Date.now(); const projectAgents = [...agents.values()] .filter((agent) => agent.project === project) .map((agent) => publicAgent(agent, staleAfterMs, snapshotAt)) .sort((left, right) => Number(right.online) - Number(left.online) || left.name.localeCompare(right.name)); const open = store.getOpenMessages(project); const projectRuns = [...workflowRuns.values()] .filter((run) => run.project === project) .sort((left, right) => Date.parse(right.updatedAt) - Date.parse(left.updatedAt)); return { project, fetchedAt: nowIso(), agents: projectAgents, openMessages: open.slice(0, 16).map((message) => ({ id: message.id, status: message.status, fromName: message.fromName, toName: message.toName, delivery: message.delivery, createdAt: message.createdAt, ...(message.correlationId ? { correlationId: message.correlationId } : {}), })), openMessageTotal: open.length, runs: projectRuns.slice(0, 8).map((run) => { const stages = (run.stages ?? []).slice(0, 16).map((stage) => ({ id: stage.id, label: stage.label, status: stage.status, ...(stage.attempts ? { attempts: stage.attempts } : {}), })); const done = stages.filter((stage) => stage.status === "passed" || stage.status === "failed" || stage.status === "warning").length; return { id: run.id, status: run.status, definitionId: run.definitionId, project: run.project, ...(run.currentStage ? { currentStage: run.currentStage } : {}), ...(run.targetAgentName ? { targetAgentName: run.targetAgentName } : {}), ...(run.updatedAt ? { updatedAt: run.updatedAt } : {}), ...(stages.length > 0 ? { progress: { done, total: stages.length }, stages } : {}), }; }), runTotal: projectRuns.length, homeRuntimes: homeRuntimesView(project, hubNow()), plans: [...journal.values()] .filter((entry) => entry.category === "plan" && projectRuns.some((run) => run.id === entry.runId)) .sort((left, right) => Date.parse(right.createdAt) - Date.parse(left.createdAt)) .slice(0, 16) .map((entry) => ({ id: entry.id, runId: entry.runId, summary: entry.summary.replace(/\s+/g, " ").trim().slice(0, 120), createdAt: entry.createdAt, ...(entry.stageId ? { stageId: entry.stageId } : {}), ...(entry.severity ? { severity: entry.severity } : {}), })), }; } function broadcastPresence(agent: StoredAgent): void { for (const candidate of agents.values()) { if (candidate.project === agent.project && candidate.id !== agent.id && candidate.online) { publish(candidate.id, { type: "presence", agent: publicAgent(agent, staleAfterMs) }); } } publishOps(agent.project, "agents"); } function findTarget(project: string, target: string, allowOffline = false): StoredAgent { const byId = agents.get(target); if (byId?.project === project && (allowOffline || byId.online)) return byId; const byName = [...agents.values()].find( (agent) => agent.project === project && (allowOffline || agent.online) && agent.name.toLowerCase() === target.toLowerCase(), ); if (!byName) throw new ProtocolError(404, `online target not found: ${target}`, "target_not_found"); return byName; } function findKnownTarget(project: string, target: string): StoredAgent { const byId = agents.get(target); if (byId?.project === project) return byId; const byName = [...agents.values()].find( (agent) => agent.project === project && agent.name.toLowerCase() === target.toLowerCase(), ); if (!byName) { throw new ProtocolError(409, `workflow target has not registered: ${target}`, "workflow_target_unavailable"); } return byName; } function resolveStageEvidencePolicies( stage: WebhookWorkflowDefinition["stages"][number], project: string, coordinator: StoredAgent, ): WorkflowStageState["resolvedEvidencePolicies"] { if (!stage.evidencePolicies) return undefined; const resolved = new Map[string]>(); for (const [requirementKey, policy] of Object.entries(stage.evidencePolicies)) { const eligible = new Map(); for (const selector of policy.eligibleAgents) { const agent = findKnownTarget(project, selector); if (agent.id === coordinator.id) { throw new ProtocolError( 409, `workflow evidence policy ${stage.id}/${requirementKey} cannot include the coordinator`, "workflow_evidence_policy_invalid", ); } eligible.set(agent.id, { id: agent.id, name: agent.name }); } if (eligible.size < policy.minProducers) { throw new ProtocolError( 409, `workflow evidence policy ${stage.id}/${requirementKey} resolves to ${eligible.size} unique producers but requires ${policy.minProducers}`, "workflow_evidence_policy_unresolvable", ); } resolved.set(requirementKey, { kind: "peer-reply", minProducers: policy.minProducers, eligibleProducers: [...eligible.values()], acceptedStatuses: ["replied"], ...(policy.degradation ? { degradation: { ...policy.degradation } } : {}), }); } return Object.fromEntries(resolved); } function authorizeWorkflowMessageContext( sender: StoredAgent, targetInput: string, requested: Omit, ): WorkflowMessageContext { const run = workflowRuns.get(requested.runId); if (!run || run.project !== sender.project || run.targetAgentId !== sender.id) { throw new ProtocolError(403, "workflow context is not assigned to this coordinator", "workflow_context_forbidden"); } const stage = run.stages.find((candidate) => candidate.id === requested.stageId); if (!stage || run.currentStage !== stage.id || run.status !== "running" || stage.status !== "in_progress") { throw new ProtocolError(409, "workflow context does not reference the active stage", "workflow_context_inactive"); } const requirementKey = canonicalWorkflowEvidenceKey(requested.requirementKey); const policy = stage.resolvedEvidencePolicies?.[requirementKey]; if (!policy) { throw new ProtocolError( 400, `workflow requirement ${requirementKey} does not accept peer evidence`, "workflow_evidence_policy_missing", ); } if (requested.attempt !== stage.attempts + 1) { throw new ProtocolError( 409, `workflow context attempt ${requested.attempt} does not match active attempt ${stage.attempts + 1}`, "workflow_context_attempt_mismatch", ); } const normalizedTarget = targetInput.toLowerCase(); const eligible = policy.eligibleProducers.find( (producer) => producer.id === targetInput || producer.name.toLowerCase() === normalizedTarget, ); if (!eligible) { throw new ProtocolError( 403, `target ${targetInput} is not eligible for workflow evidence ${requirementKey}`, "workflow_evidence_producer_forbidden", ); } return { schema: "pi-mesh.workflow-message-context.v1", runId: run.id, stageId: stage.id, requirementKey, attempt: requested.attempt, }; } function webhookEvent(request: IncomingMessage, payload: Record): string | undefined { const header = request.headers["x-github-event"]; if (typeof header === "string" && header.trim()) return header.trim(); const candidate = payload.webhookEvent ?? payload.event; return typeof candidate === "string" && candidate.trim() ? candidate.trim() : undefined; } function requireSha256Signature(signature: string): void { const separator = signature.indexOf("="); const algorithm = separator > 0 ? signature.slice(0, separator).toLowerCase() : ""; if (algorithm !== "sha256") { throw new ProtocolError(401, "webhook signature must use sha256", "webhook_signature_unsupported"); } } /** Jira and GitHub sign only the body, so a provider delivery ID is unsigned * routing data. Returns that provider's own delivery header. */ function verifyProviderWebhookSignature( request: IncomingMessage, body: Buffer, definition: WebhookWorkflowDefinition, ): string { if (definition.source === "generic") { throw new ProtocolError( 401, "generic workflow webhooks require x-kxm-signature over the timestamp, delivery ID, definition, and body", "webhook_signature_missing", ); } const signature = request.headers["x-hub-signature-256"] ?? request.headers["x-hub-signature"]; if (typeof signature !== "string") { throw new ProtocolError(401, "webhook signature is required", "webhook_signature_missing"); } requireSha256Signature(signature); const expected = `sha256=${createHmac("sha256", definition.secret).update(body).digest("hex")}`; if (!safeTokenEqual(signature, expected)) { throw new ProtocolError(401, "webhook signature is invalid", "webhook_signature_invalid"); } const deliveryHeader = definition.source === "jira" ? request.headers["x-atlassian-webhook-identifier"] : request.headers["x-github-delivery"]; return requireString(deliveryHeader, "webhook delivery identifier", { max: 128 }); } /** The KXM sender contract (see workflowWebhookSignedMaterial): the * signature binds the delivery ID, timestamp, and route scope. A valid * signature outside the skew window is refused as expired. */ function verifyKxmWebhookSignature( request: IncomingMessage, body: Buffer, secret: string, scope: WorkflowWebhookScope, ): string { const signature = request.headers["x-kxm-signature"]; if (typeof signature !== "string") { throw new ProtocolError( 401, "KXM webhooks require x-kxm-signature over the timestamp, delivery ID, route, and body", "webhook_signature_missing", ); } requireSha256Signature(signature); const timestamp = request.headers["x-kxm-timestamp"]; if (typeof timestamp !== "string" || !/^[0-9]{1,12}$/.test(timestamp)) { throw new ProtocolError(401, "x-kxm-timestamp must be Unix seconds", "webhook_timestamp_invalid"); } const deliveryId = requireString(request.headers["x-kxm-delivery-id"], "webhook delivery identifier", { max: 128 }); if (!safeTokenEqual(signature, workflowWebhookSignature(secret, scope, timestamp, deliveryId, body))) { throw new ProtocolError(401, "webhook signature is invalid", "webhook_signature_invalid"); } if (Math.abs(Date.now() / 1_000 - Number(timestamp)) > WORKFLOW_WEBHOOK_MAX_SKEW_SECONDS) { throw new ProtocolError( 401, `webhook timestamp is outside the ${WORKFLOW_WEBHOOK_MAX_SKEW_SECONDS}-second window; re-sign at send time`, "webhook_timestamp_expired", ); } return deliveryId; } function workflowPrompt( definition: WebhookWorkflowDefinition, runId: string, payload: Record, ): string { const rendered = renderWorkflowPrompt(definition.promptTemplate, payload); const stageList = definition.stages.map((stage, index) => { const peerPolicies = Object.entries(stage.evidencePolicies ?? {}).map(([requirement, policy]) => ` Peer evidence ${requirement}: ${policy.minProducers} unique replied producer(s) from ${policy.eligibleAgents.join(", ")}`); return [ `${index + 1}. ${stage.label} (stageId: ${stage.id}, maxAttempts: ${stage.maxAttempts})`, ` ${stage.instructions}`, ` Required evidence keys: ${stage.requiredEvidence.join(", ") || "none"}`, ...peerPolicies, ].join("\n"); }).join("\n"); return [ `Durable workflow run: ${runId}`, `Workflow: ${definition.id}`, "", rendered, "", "Execute these stages in order:", stageList, "", `At every stage, record material knowledge with kxm_workflow_record in one of these categories: ${JOURNAL_CATEGORIES.join(", ")}. Pass the stageId the entry belongs to; the hub binds the attempt and, when you omit area, uses the stage's declared area.`, "Keep reviewed configuration as YAML directly under .kxm/ (such as .kxm/project.yaml and .kxm/workflows/), logs in .kxm/logs, and durable workflow artifacts in .kxm/assets. Never create .kxm/config, which KXM refuses, and never commit runtime logs, .kxm/state, or secrets.", "Complete each stage with kxm_workflow_checkpoint. Supply evidence as an object whose keys exactly match the stage's required evidence keys. Unrelated keys never satisfy a requirement. A warning or failure must be corrected and checkpointed again until it passes or the attempt limit is reached.", "For a peer-evidence requirement, send or fan out with workflowContext containing this run ID, the exact stage ID, requirement key, and current 1-based attempt. At checkpoint, cite only the returned message IDs under evidenceRefs; the hub derives producer and reply provenance.", "When an external system must finish asynchronously, call kxm_workflow_wait with a stable signal key and any already-verified keyed evidence. That evidence is accumulated with the signed callback before the stage can pass; then settle the turn.", "For multi-agent planning, delegate to the requested peers, compare their proposals, record contradictions, and synthesize the strongest evidence-backed plan.", "Do not claim the workflow is complete until the checkpoint response reports completed=true.", ].join("\n"); } function createWorkflowResumeMessage( run: WorkflowRun, definition: WebhookWorkflowDefinition, deliveryId: string, signalKey: string, status: WorkflowCheckpointStatus, summary: string, evidence: WorkflowEvidenceInput, retry: boolean, ): MessageRecord { const createdAt = nowIso(); const ttlMs = parseBoundedInteger( definition.ttlMs, "workflow.ttlMs", defaultMessageTtlMs, MIN_MESSAGE_TTL_MS, MAX_MESSAGE_TTL_MS, ); const nextInstruction = retry ? `Correct the ${status} result, rerun the external check, then wait for a new signal or checkpoint the stage.` : `Continue with stage ${run.currentStage}.`; const content = requireString([ `Resume durable workflow run: ${run.id}`, `Workflow: ${definition.id}`, `External signal: ${signalKey}`, `Result: ${status}`, `Summary: ${summary}`, `Evidence: ${workflowEvidenceStrings(evidence).join(", ") || "none supplied"}`, "", nextInstruction, "Review the run with kxm_workflow_get and keep recording material learning with kxm_workflow_record (any of its ten categories; pass stageId for stage-bound entries).", "Do not claim the workflow is complete until the checkpoint response reports completed=true.", ].join("\n"), "workflow resume prompt", { max: MAX_CONTENT_CHARS }); const seq = store.nextAgentSequence(run.targetAgentId); const message: MessageRecord = { id: newId("msg"), project: run.project, from: `workflow:${run.id}`, fromName: `signal:${definition.id}`, to: run.targetAgentId, toName: run.targetAgentName, content, delivery: definition.delivery, hops: 0, maxHops: DEFAULT_MAX_HOPS, seq, correlationId: run.id, workflowRunId: run.id, idempotencyKey: `${definition.id}:signal:${deliveryId}`, createdAt, expiresAt: new Date(Date.parse(createdAt) + ttlMs).toISOString(), status: "queued", }; run.messageId = message.id; run.updatedAt = createdAt; return message; } function expireWorkflowWaits(): void { const timestamp = nowIso(); const now = Date.parse(timestamp); for (const run of workflowRuns.values()) { if (run.status !== "waiting" || !run.waiting || Date.parse(run.waiting.expiresAt) > now) continue; const transition = structuredClone(run); const waiting = transition.waiting!; const stage = transition.stages.find((candidate) => candidate.id === waiting.stageId); if (stage) { stage.status = "failed"; stage.summary = `Timed out waiting for external signal ${waiting.signalKey}`; stage.updatedAt = timestamp; } transition.status = "failed"; delete transition.currentStage; delete transition.waiting; transition.updatedAt = timestamp; const entry: WorkflowJournalEntry = { id: newId("journal"), runId: transition.id, agentId: transition.targetAgentId, category: "error", area: stage?.area ?? "harness", severity: "error", summary: `External workflow signal timed out: ${waiting.signalKey}`, evidence: [`wait-created:${waiting.createdAt}`, `wait-expired:${waiting.expiresAt}`], relatedEntryIds: [], createdAt: timestamp, ...(stage ? { stageId: stage.id, attempt: stage.attempts + 1 } : {}), }; const definition = webhookWorkflows.get(transition.definitionId); const ttlMs = parseBoundedInteger( definition?.ttlMs, "workflow.ttlMs", defaultMessageTtlMs, MIN_MESSAGE_TTL_MS, MAX_MESSAGE_TTL_MS, ); const seq = store.nextAgentSequence(transition.targetAgentId); const message: MessageRecord = { id: newId("msg"), project: transition.project, from: `workflow:${transition.id}`, fromName: `timeout:${transition.definitionId}`, to: transition.targetAgentId, toName: transition.targetAgentName, content: [ `Durable workflow run ${transition.id} failed while waiting for external signal ${waiting.signalKey}.`, `Stage: ${waiting.stageId}`, `Expected: ${waiting.summary}`, `Deadline: ${waiting.expiresAt}`, "Review whether the external action completed, record any follow-up outside this terminal run, and escalate or start a new retry-safe workflow only when appropriate.", ].join("\n"), delivery: definition?.delivery ?? "followUp", hops: 0, maxHops: DEFAULT_MAX_HOPS, seq, correlationId: transition.id, workflowRunId: transition.id, idempotencyKey: `${transition.definitionId}:timeout:${waiting.signalKey}:${waiting.expiresAt}`, createdAt: timestamp, expiresAt: new Date(Date.parse(timestamp) + ttlMs).toISOString(), status: "queued", }; transition.messageId = message.id; store.saveWorkflowTransition(transition, message, entry); publishOps(transition.project, "workflows"); publishOps(transition.project, "messages"); exportTerminalRetrospective(transition); if (agents.get(transition.targetAgentId)?.online) { publish(transition.targetAgentId, { type: "message", message }); } counters.workflowWaitTimeouts += 1; counters.journalEntries += 1; logger({ event: "workflow_wait_timed_out", workflowRunId: transition.id, stageId: waiting.stageId, signalKey: waiting.signalKey, messageId: message.id, }); } } function flushPending(agentId: string): void { expireMessages(); const cursor = store.getConsumerCursor(agentId); const pending = store.getPendingMessages(agentId, cursor); for (const message of pending) { publish(agentId, { type: "message", message }); } } function expireMessages(): void { const now = Date.now(); for (const message of store.getExpiringMessages(now)) { if ((message.status === "queued" || message.status === "delivered") && Date.parse(message.expiresAt) <= now) { message.status = "expired"; message.error = "message expired before a reply was received"; store.saveMessage(message); publishOps(message.project, "messages"); publish(message.from, { type: "expired", message }); publish(message.to, { type: "expired", message }); counters.messagesExpired += 1; logger({ event: "message_expired", messageId: message.id, ...messageLog(message, message.from, message.fromName, message.to, message.toName) }); const run = [...workflowRuns.values()].find( (candidate) => candidate.messageId === message.id && candidate.status === "running", ); if (run) { const expiredStage = run.stages.find((candidate) => candidate.id === run.currentStage); run.status = "failed"; delete run.currentStage; run.updatedAt = nowIso(); store.saveWorkflowRun(run); publishOps(run.project, "workflows"); const entry: WorkflowJournalEntry = { id: newId("journal"), runId: run.id, agentId: run.targetAgentId, category: "error", area: "harness", severity: "error", summary: "Workflow coordinator prompt expired before completion", evidence: [`message:${message.id}`], relatedEntryIds: [], createdAt: run.updatedAt, ...(expiredStage ? { stageId: expiredStage.id, attempt: expiredStage.attempts + 1 } : {}), }; store.saveJournalEntry(entry); counters.journalEntries += 1; exportTerminalRetrospective(run); } } } } function purgeTerminalMessages(): void { const swept = store.sweepRetention(messageRetentionMs); for (const message of swept.purgedMessages) { publishOps(message.project, "messages"); counters.messagesPurged += 1; logger({ event: "message_purged", messageId: message.id, ...messageLog(message, message.from, message.fromName, message.to, message.toName) }); } for (const runId of swept.purgedRuns) { logger({ event: "workflow_run_purged", runId }); } for (const lease of swept.purgedLeases) { logger({ event: "lease_purged", resource: lease.resource, project: lease.project, agentId: lease.holderAgentId, fencingToken: lease.fencingToken, expiresAt: lease.expiresAt, }); } } function metricsBody(): string { const onlineAgents = [...agents.values()].filter((agent) => agent.online).length; return [ "# HELP kxm_online_agents Current online agent count.", "# TYPE kxm_online_agents gauge", `kxm_online_agents ${onlineAgents}`, "# HELP kxm_messages Current retained message count.", "# TYPE kxm_messages gauge", `kxm_messages ${store.countMessages()}`, "# TYPE kxm_requests_total counter", `kxm_requests_total ${counters.requests}`, "# TYPE kxm_errors_total counter", `kxm_errors_total ${counters.errors}`, "# TYPE kxm_registrations_total counter", `kxm_registrations_total ${counters.registrations}`, "# TYPE kxm_messages_sent_total counter", `kxm_messages_sent_total ${counters.messagesSent}`, "# TYPE kxm_messages_replied_total counter", `kxm_messages_replied_total ${counters.messagesReplied}`, "# TYPE kxm_messages_cancelled_total counter", `kxm_messages_cancelled_total ${counters.messagesCancelled}`, "# TYPE kxm_messages_expired_total counter", `kxm_messages_expired_total ${counters.messagesExpired}`, "# TYPE kxm_messages_purged_total counter", `kxm_messages_purged_total ${counters.messagesPurged}`, "# TYPE kxm_webhooks_accepted_total counter", `kxm_webhooks_accepted_total ${counters.webhooksAccepted}`, "# TYPE kxm_workflow_checkpoints_total counter", `kxm_workflow_checkpoints_total ${counters.workflowCheckpoints}`, "# TYPE kxm_workflow_waits_total counter", `kxm_workflow_waits_total ${counters.workflowWaits}`, "# TYPE kxm_workflow_signals_total counter", `kxm_workflow_signals_total ${counters.workflowSignals}`, "# TYPE kxm_workflow_wait_timeouts_total counter", `kxm_workflow_wait_timeouts_total ${counters.workflowWaitTimeouts}`, "# TYPE kxm_workflow_degradations_total counter", `kxm_workflow_degradations_total ${counters.workflowDegradations}`, "# TYPE kxm_workflow_journal_entries_total counter", `kxm_workflow_journal_entries_total ${counters.journalEntries}`, "# TYPE kxm_context_requests_total counter", `kxm_context_requests_total ${counters.contextRequests}`, "# TYPE kxm_attempt_latency_seconds_total counter", `kxm_attempt_latency_seconds_total ${counters.attemptLatencySecondsTotal}`, "# TYPE kxm_metered_cost_usd_total counter", `kxm_metered_cost_usd_total ${counters.meteredCostUsdTotal}`, "# TYPE kxm_leases_granted_total counter", `kxm_leases_granted_total ${counters.leasesGranted}`, "# TYPE kxm_leases_refused_total counter", `kxm_leases_refused_total ${counters.leasesRefused}`, "# TYPE kxm_leases_released_total counter", `kxm_leases_released_total ${counters.leasesReleased}`, "# TYPE kxm_sync_events_accepted_total counter", `kxm_sync_events_accepted_total ${counters.syncEventsAccepted}`, "# TYPE kxm_sync_events_duplicate_total counter", `kxm_sync_events_duplicate_total ${counters.syncEventsDuplicate}`, "# TYPE kxm_sync_events_refused_total counter", `kxm_sync_events_refused_total ${counters.syncEventsRefused}`, "# TYPE kxm_sync_conflicts_total counter", `kxm_sync_conflicts_total ${counters.syncConflicts}`, "# TYPE kxm_runtime_heartbeats_total counter", `kxm_runtime_heartbeats_total ${counters.runtimeHeartbeats}`, "", ].join("\n"); } const server = createServer(async (request, response) => { const requestIdHeader = request.headers["x-request-id"]; const requestId = typeof requestIdHeader === "string" && /^[A-Za-z0-9._-]{1,80}$/.test(requestIdHeader) ? requestIdHeader : newId("req"); response.setHeader("x-request-id", requestId); response.setHeader("referrer-policy", "no-referrer"); counters.requests += 1; try { const url = new URL(request.url ?? "/", `http://${request.headers.host ?? "localhost"}`); const method = request.method ?? "GET"; if (method === "GET" && url.pathname === "/health") { json(response, 200, { ok: true, agents: [...agents.values()].filter((agent) => agent.online).length }); return; } if (method === "GET" && url.pathname === "/ready") { const ready = store.healthy(); json(response, ready ? 200 : 503, { ok: ready, storage: store.persistent ? "sqlite" : "memory" }); return; } checkRateLimit(request, response); const signalMatch = url.pathname.match(/^\/v1\/webhooks\/([^/]+)\/runs\/([^/]+)\/signals\/([^/]+)$/); if (method === "POST" && signalMatch) { const definitionId = decodeURIComponent(signalMatch[1]!); const runId = decodeURIComponent(signalMatch[2]!); const signalKey = workflowSignalKey(decodeURIComponent(signalMatch[3]!)); const definition = webhookWorkflows.get(definitionId); if (!definition) throw new ProtocolError(404, "webhook workflow not found", "webhook_not_found"); const rawBody = await readBody(request); const deliveryId = verifyKxmWebhookSignature( request, rawBody, definition.signalSecret ?? definition.secret, { definitionId: definition.id, runId, signalKey }, ); const body = parseJsonBody(rawBody); const run = workflowRuns.get(runId); if (!run || run.definitionId !== definition.id) { throw new ProtocolError(404, "workflow run not found", "workflow_not_found"); } const payloadHash = createHash("sha256").update(rawBody).digest("hex"); const existingReceipt = run.signalReceipts?.find((receipt) => receipt.deliveryId === deliveryId); if (existingReceipt) { if (existingReceipt.signalKey !== signalKey || existingReceipt.payloadHash !== payloadHash) { throw new ProtocolError( 409, "webhook delivery identifier was already used for a different signal", "workflow_signal_delivery_conflict", ); } json(response, 200, { duplicate: true, status: existingReceipt.status, stageId: existingReceipt.stageId, signalKey: existingReceipt.signalKey, receivedAt: existingReceipt.receivedAt, degraded: existingReceipt.degraded === true, degradedRequirements: existingReceipt.degradedRequirements ?? [], resumed: Boolean(existingReceipt.messageId), }); return; } expireWorkflowWaits(); const status = requireString(body.status, "status", { max: 16 }) as WorkflowCheckpointStatus; if (status !== "passed" && status !== "warning" && status !== "failed") { throw new ProtocolError(400, "status must be passed, warning, or failed", "invalid_checkpoint_status"); } const summary = requireString(body.summary, "summary", { max: 4_000 }); const evidence = boundedWorkflowEvidence(body.evidence); const receivedAt = nowIso(); const transition = structuredClone(workflowRuns.get(runId)!); if (transition.status === "waiting" && transition.waiting) { validateWorkflowSignalContext( evidence, runId, transition.waiting.stageId, signalKey, ); } const result = resumeWorkflowFromSignal(transition, signalKey, status, summary, evidence, receivedAt); const receipt: WorkflowSignalReceipt = { deliveryId, payloadHash, signalKey, stageId: result.stageId, status, receivedAt, }; if (result.degraded) { const degradedStage = transition.stages.find((candidate) => candidate.id === result.stageId)!; receipt.degraded = true; receipt.degradedRequirements = [...(degradedStage.degradedRequirements ?? [])]; } (transition.signalReceipts ??= []).push(receipt); let message: MessageRecord | undefined; let entry: WorkflowJournalEntry | undefined; if (status !== "passed") { const stage = transition.stages.find((candidate) => candidate.id === result.stageId)!; entry = { id: newId("journal"), runId: transition.id, agentId: transition.targetAgentId, category: "error", area: stage.area ?? "gates", severity: status === "failed" ? "error" : "warning", summary: `${stage.label}: ${summary}`, evidence: workflowEvidenceStrings(evidence), relatedEntryIds: [], createdAt: receivedAt, stageId: stage.id, attempt: stage.attempts, }; } else if (result.degraded) { const stage = transition.stages.find((candidate) => candidate.id === result.stageId)!; entry = { id: newId("journal"), runId: transition.id, agentId: transition.targetAgentId, category: "decision", area: stage.area ?? "security", severity: "warning", summary: `${stage.label} passed from a signed callback using a previously approved degraded peer quorum`, evidence: [ "class:workflow_quorum_degradation_used", ...(stage.degradedRequirements ?? []).map((requirement) => `requirement:${requirement}`), ], relatedEntryIds: [], createdAt: receivedAt, stageId: stage.id, attempt: stage.attempts, }; } if (transition.status === "running") { message = createWorkflowResumeMessage( transition, definition, deliveryId, signalKey, status, summary, evidence, result.retry, ); receipt.messageId = message.id; } store.saveWorkflowTransition(transition, message, entry); publishOps(transition.project, "workflows"); if (message) publishOps(transition.project, "messages"); exportTerminalRetrospective(transition); if (entry) counters.journalEntries += 1; if (message && agents.get(transition.targetAgentId)?.online) { publish(transition.targetAgentId, { type: "message", message }); } counters.workflowSignals += 1; counters.workflowCheckpoints += 1; logger({ event: "workflow_signal_received", workflowRunId: transition.id, deliveryId, signalKey, stageId: result.stageId, status, retry: result.retry, completed: result.completed, ...(message ? { messageId: message.id } : {}), }); json(response, message ? 202 : 200, { duplicate: false, status, stageId: result.stageId, signalKey, retry: result.retry, completed: result.completed, degraded: result.degraded === true, degradedRequirements: receipt.degradedRequirements ?? [], resumed: Boolean(message), }); return; } const webhookMatch = url.pathname.match(/^\/v1\/webhooks\/([^/]+)$/); if (method === "POST" && webhookMatch) { const definitionId = decodeURIComponent(webhookMatch[1]!); const definition = webhookWorkflows.get(definitionId); if (!definition) throw new ProtocolError(404, "webhook workflow not found", "webhook_not_found"); const rawBody = await readBody(request); // A KXM sender signs the delivery ID and timestamp. A Jira or GitHub // delivery signs only its body, so for those the body is the replay // identity: it may start one run, under one delivery ID. const bodyOnlySignature = request.headers["x-kxm-signature"] === undefined; const deliveryId = bodyOnlySignature ? verifyProviderWebhookSignature(request, rawBody, definition) : verifyKxmWebhookSignature(request, rawBody, definition.secret, { definitionId: definition.id }); const payload = parseJsonBody(rawBody); const event = webhookEvent(request, payload); if (definition.event && event !== definition.event) { response.writeHead(204, { "cache-control": "no-store" }).end(); return; } if (definition.filter && String(valueAtPath(payload, definition.filter.path) ?? "") !== definition.filter.equals) { response.writeHead(204, { "cache-control": "no-store" }).end(); return; } const payloadHash = createHash("sha256").update(rawBody).digest("hex"); const existing = [...workflowRuns.values()].find( (run) => run.definitionId === definition.id && run.deliveryId === deliveryId, ); if (existing) { if (existing.payloadHash !== payloadHash) { throw new ProtocolError( 409, "webhook delivery identifier was already used for a different body", "webhook_delivery_conflict", ); } json(response, 200, { duplicate: true, runId: existing.id, status: existing.status }); return; } if (bodyOnlySignature && [...workflowRuns.values()].some( (run) => run.definitionId === definition.id && run.payloadHash === payloadHash, )) { throw new ProtocolError( 409, "this signed body already started a run under another delivery identifier", "webhook_payload_replayed", ); } const target = findKnownTarget(definition.project, definition.target); const createdAt = nowIso(); const runId = newId("run"); const ttlMs = parseBoundedInteger( definition.ttlMs, "workflow.ttlMs", defaultMessageTtlMs, MIN_MESSAGE_TTL_MS, MAX_MESSAGE_TTL_MS, ); const content = requireString(workflowPrompt(definition, runId, payload), "workflow prompt", { max: MAX_CONTENT_CHARS, }); const seq = store.nextAgentSequence(target.id); const message: MessageRecord = { id: newId("msg"), project: definition.project, from: `workflow:${runId}`, fromName: `webhook:${definition.id}`, to: target.id, toName: target.name, content, delivery: definition.delivery, hops: 0, maxHops: DEFAULT_MAX_HOPS, seq, correlationId: runId, workflowRunId: runId, idempotencyKey: `${definition.id}:${deliveryId}`, createdAt, expiresAt: new Date(Date.parse(createdAt) + ttlMs).toISOString(), status: "queued", }; const stages = definition.stages.map((stage, index) => { const resolvedEvidencePolicies = resolveStageEvidencePolicies(stage, definition.project, target); return { ...stage, ...(resolvedEvidencePolicies ? { resolvedEvidencePolicies } : {}), status: index === 0 ? "in_progress" as const : "pending" as const, attempts: 0, evidence: {}, ...(index === 0 ? { startedAt: createdAt, updatedAt: createdAt } : {}), }; }); const run: WorkflowRun = { id: runId, definitionId: definition.id, source: definition.source, deliveryId, payloadHash, definitionHash: workflowDefinitionHash(definition), ...(event ? { event } : {}), project: definition.project, targetAgentId: target.id, targetAgentName: target.name, messageId: message.id, status: "running", currentStage: stages[0]!.id, stages, ...(definition.maxTransitions !== undefined ? { maxTransitions: definition.maxTransitions } : {}), ...(definition.reproOracle ? { reproOracle: definition.reproOracle } : {}), ...(definition.planHash ? { planHashConfig: definition.planHash } : {}), ...(definition.requirePlanHash ? { requirePlanHash: definition.requirePlanHash } : {}), createdAt, updatedAt: createdAt, }; store.saveMessage(message); store.saveWorkflowRun(run); publishOps(run.project, "messages"); publishOps(run.project, "workflows"); if (target.online) publish(target.id, { type: "message", message }); counters.webhooksAccepted += 1; logger({ event: "webhook_workflow_started", workflowId: definition.id, workflowRunId: run.id, deliveryId, project: definition.project, target: target.id, }); json(response, 202, { run, runId: run.id, duplicate: false }); return; } if (method === "GET" && url.pathname === "/metrics") { requireAdminAuth(request); text(response, 200, metricsBody(), "text/plain; version=0.0.4; charset=utf-8"); return; } if (method === "GET" && url.pathname === "/v1/ops/snapshot") { requireConfiguredAdminAuth(request, "operations metadata"); const project = requireString(url.searchParams.get("project"), "project", { max: 128 }); expireMessages(); expireWorkflowWaits(); purgeTerminalMessages(); json(response, 200, opsSnapshot(project)); return; } if (method === "GET" && url.pathname === "/v1/ops/events") { requireConfiguredAdminAuth(request, "operations metadata"); const project = requireString(url.searchParams.get("project"), "project", { max: 128 }); response.writeHead(200, { "content-type": "text/event-stream", "cache-control": "no-cache, no-transform", connection: "keep-alive", "x-accel-buffering": "no", "x-content-type-options": "nosniff", }); response.write(`event: ready\ndata: ${JSON.stringify({ type: "ops", project, topic: "agents", at: nowIso() })}\n\n`); const client: OpsSseClient = { project, response, heartbeat: setInterval(() => response.write(": heartbeat\n\n"), 15_000), }; client.heartbeat.unref(); opsStreams.add(client); request.on("close", () => { clearInterval(client.heartbeat); opsStreams.delete(client); }); return; } if (method === "GET" && url.pathname === "/v1/workflows") { expireWorkflowWaits(); const agent = requireAgent(request); requireProjectAuth(request, agent.project); const runs = [...workflowRuns.values()].filter( (run) => run.project === agent.project && run.targetAgentId === agent.id, ); json(response, 200, { runs }); return; } const workflowMatch = url.pathname.match(/^\/v1\/workflows\/([^/]+)$/); if (method === "GET" && workflowMatch) { expireWorkflowWaits(); const agent = requireAgent(request); requireProjectAuth(request, agent.project); const run = workflowRuns.get(decodeURIComponent(workflowMatch[1]!)); if (!run) throw new ProtocolError(404, "workflow run not found", "workflow_not_found"); if (run.project !== agent.project || run.targetAgentId !== agent.id) { throw new ProtocolError( 403, "workflow run is not visible to this agent", "workflow_forbidden", workflowScopeExtras("get", run.targetAgentName), ); } const entries = [...journal.values()].filter((entry) => entry.runId === run.id); json(response, 200, { run, journal: entries }); return; } const degradationMatch = url.pathname.match(/^\/v1\/workflows\/([^/]+)\/degradations$/); if (method === "POST" && degradationMatch) { requireConfiguredAdminAuth(request); const body = await readJson(request); // Resolve the mutable run only after the final await. Concurrent exact // retries then observe the first committed approval and remain // idempotent instead of cloning the same stale pre-approval state. const run = workflowRuns.get(decodeURIComponent(degradationMatch[1]!)); if (!run) throw new ProtocolError(404, "workflow run not found", "workflow_not_found"); const stageId = requireString(body.stageId, "stageId", { max: 64 }); const requirementKey = canonicalWorkflowEvidenceKey( requireString(body.requirementKey, "requirementKey", { max: 128 }), ); const reason = requireString(body.reason, "reason", { max: 1_000 }); const timestamp = nowIso(); const transition = structuredClone(run); const result = approveWorkflowDegradation( transition, stageId, requirementKey, reason, newId("approval"), timestamp, ); if (result.created) { const stage = transition.stages.find((candidate) => candidate.id === stageId)!; const entry: WorkflowJournalEntry = { id: newId("journal"), runId: transition.id, agentId: "kxm-admin", category: "decision", area: stage.area ?? "security", severity: "warning", summary: `Approved degraded peer quorum for ${stageId}/${requirementKey} attempt ${result.approval.attempt}`, details: reason, evidence: [ "class:workflow_quorum_degradation_approved", `approval:${result.approval.id}`, `policy-min:${result.approval.policyMinProducers}`, `approved-min:${result.approval.approvedMinProducers}`, ], relatedEntryIds: [], createdAt: timestamp, stageId, attempt: result.approval.attempt, }; store.saveWorkflowTransition(transition, undefined, entry); publishOps(transition.project, "workflows"); counters.workflowDegradations += 1; counters.journalEntries += 1; logger({ event: "workflow_degradation_approved", workflowRunId: transition.id, stageId, requirementKey, attempt: result.approval.attempt, approvalId: result.approval.id, }); } json(response, result.created ? 201 : 200, { run: result.created ? transition : run, approval: result.approval, duplicate: !result.created, }); return; } // ----- Context operating-system surface (v0.5, issue #34) ----- /** Pool for one project: stored context items plus journal evidence * mapped to context items. Agents never see provider internals. */ function projectContextPool(project: string, journalCategories?: JournalCategory[]): { pool: ContextItem[]; contradictionIds: string[] } { const pool = store.listContextItems(project); try { const authored = loadAuthoredMemory(hubRepoRoot); for (const rec of authored) { pool.push(memoryRecordToContextItem(rec, project)); } } catch { // ignore if .kxm/memory does not exist } const categoryFilter = journalCategories ? new Set(journalCategories) : undefined; const runIds = new Set( [...workflowRuns.values()].filter((run) => run.project === project).map((run) => run.id), ); const contradictionIds: string[] = []; for (const entry of journal.values()) { if (!runIds.has(entry.runId)) continue; if (categoryFilter && !categoryFilter.has(entry.category)) continue; pool.push(journalEntryToContextItem(entry, project)); if (entry.category === "contradiction") contradictionIds.push(`journal_${entry.id}`); } for (const contradiction of stateProvider.contradictions()) { if (contradiction.project !== project) continue; contradictionIds.push(...contradiction.competingCurrentIds, ...contradiction.competingProposalIds); } return { pool, contradictionIds }; } /** Full source pool for wiki compilation: every journal-derived item * (unfiltered) plus the authoritative state layer. */ function contextWikiPool(project: string, compiledAt: string): WikiSourcePool { const { pool, contradictionIds } = projectContextPool(project); const stateItems = store.listContextItems(project, ["state"]); const contradictions = stateProvider.contradictions().filter((entry) => entry.project === project); return { project, stateItems, contextItems: pool, contradictions, openContradictionItemIds: contradictionIds, compiledAt, }; } const contextGetMatch = url.pathname.match(/^\/v1\/context\/get$/); if (method === "POST" && contextGetMatch) { const body = await readJson(request); const { project: callerProject, caller: callerId } = contextCallerProject(request, body.project); const policy = rolePolicy(typeof body.role === "string" ? body.role : ""); const { pool, contradictionIds } = projectContextPool(callerProject, policy.journalCategories); const outcome = arbitrate(body, pool, { contradictionIds, ...(skillLifecycle ? { skillLifecycle } : {}), }); counters.contextRequests += 1; // Sizes, never the task text (Q-J). The caller still receives its own // audit, request included, in the response below. const assembledRequest = outcome.audit.request; logger({ event: "context_packet_assembled", project: assembledRequest.project, role: assembledRequest.role, ...(assembledRequest.workflowRunId !== undefined ? { workflowRunId: assembledRequest.workflowRunId } : {}), ...(assembledRequest.stageId !== undefined ? { stageId: assembledRequest.stageId } : {}), taskChars: assembledRequest.task.length, taskTokens: outcome.audit.relevance.taskTokens, matchedCandidates: outcome.audit.relevance.matchedCandidates, selectedIds: outcome.audit.selectedIds, provenanceSummary: outcome.audit.provenanceSummary, estimatedTokens: outcome.audit.estimatedTokens, budgetTokens: outcome.audit.budgetTokens, candidateCount: outcome.audit.candidateCount, excludedSuperseded: outcome.audit.excludedSuperseded, }); json(response, 200, { packet: outcome.packet, audit: outcome.audit }); return; } const contextRecallMatch = url.pathname.match(/^\/v1\/context\/recall$/); if (method === "POST" && contextRecallMatch) { const body = await readJson(request); const { project: callerProject, caller: callerId } = contextCallerProject(request, body.project); const query = requireString(body.query ?? "", "query", { max: 500, allowEmpty: true }); const kinds = Array.isArray(body.kinds) ? body.kinds.filter((kind: unknown): kind is string => typeof kind === "string") : undefined; const limit = parseBoundedInteger(body.limit, "limit", 25, 1, 100); const { pool } = projectContextPool(callerProject); const live = pool .filter((item) => item.status !== "superseded" && item.status !== "rejected") .filter((item) => kinds === undefined || kinds.includes(item.kind)); // Exact-phrase hits, then any-token BM25 hits, then id. const recalled = rankRecall(query, live, limit); counters.contextRequests += 1; logger({ event: "context_recall", project: callerProject, queryChars: query.length, queryTokens: new Set(relevanceTokens(query)).size, limit, results: recalled.length, }); json(response, 200, { items: recalled.map(({ item, relevance }) => ({ ...contextItemAuditMetadata(item), relevance })), unresolvedGaps: recalled.length === 0 ? ["no matching context records"] : [], }); return; } const contextStateMatch = url.pathname.match(/^\/v1\/context\/state$/); if (method === "POST" && contextStateMatch) { const body = await readJson(request); const { project: callerProject, caller: callerId } = contextCallerProject(request, body.project); const key = requireString(body.key, "key", { max: 200 }); const asOf = body.asOf === undefined || body.asOf === null ? undefined : requireString(body.asOf, "asOf", { max: 64 }); const current = await stateProvider.get(callerProject, key, asOf); counters.contextRequests += 1; json(response, 200, { state: current, key }); return; } const contextStateProposeMatch = url.pathname.match(/^\/v1\/context\/state\/propose$/); if (method === "POST" && contextStateProposeMatch) { const body = await readJson(request); const { project: callerProject, caller: callerId, credential } = contextCallerProject(request, body.project); // Origin follows the credential, never a name: an agent key proposes as // a peer, capped at evidence by the grant floor. Human origin can claim // policy, so it takes the configured administrative token with no // loopback bypass, as promotion does. if (credential === "admin") requireConfiguredAdminAuth(request, "human-origin state proposals"); if (body.proposedBy !== undefined && body.proposedBy !== callerId) { logger({ event: "security_alert", alert: "state_proposer_mismatch", project: callerProject, callerId, credential }); throw new ProtocolError( 403, "proposedBy must name the authenticated caller", "state_proposer_mismatch", ); } const origin = credential === "agent" ? "peer" : "human"; const proposalId = await stateProvider.propose({ schema: "kxm.state-change-proposal.v1", project: callerProject, key: requireString(body.key, "key", { max: 200 }), summary: requireString(body.summary, "summary", { max: 4_000 }), authority: parseContextAuthority(body.authority), confidence: parseContextConfidence(body.confidence), evidenceRefs: boundedStringList(body.evidenceRefs, "evidenceRefs", 32), proposedBy: callerId, origin, }); counters.contextRequests += 1; publishOps(callerProject, "workflows"); logger({ event: "context_state_proposed", project: callerProject, proposalId, proposedBy: callerId, origin }); json(response, 201, { proposalId }); return; } const contextStatePromoteMatch = url.pathname.match(/^\/v1\/context\/state\/promote$/); if (method === "POST" && contextStatePromoteMatch) { // Agents may propose state but never silently promote it: promotion is // an authorized control-plane decision with durable evidence. requireConfiguredAdminAuth(request, "state promotion"); const body = await readJson(request); const proposalId = requireString(body.proposalId, "proposalId", { max: 128 }); const project = requireString(body.project, "project", { max: 200 }); const evidence = boundedStringList(body.evidence, "evidence", 32); const promoter = (typeof body.promotedBy === "string" && body.promotedBy.trim()) ? body.promotedBy.trim() : (typeof body.caller === "string" && body.caller.trim()) ? body.caller.trim() : (typeof request.headers["x-kxm-caller-id"] === "string" && request.headers["x-kxm-caller-id"].trim()) ? request.headers["x-kxm-caller-id"].trim() : "kxm-admin"; const promoted = await stateProvider.promote(proposalId, evidence, promoter); counters.contextRequests += 1; publishOps(project, "workflows"); logger({ event: "context_state_promoted", project, proposalId, promotedId: promoted.id, stateKey: promoted.stateKey, promotedBy: promoter }); json(response, 200, { state: promoted }); return; } const contextEpisodeMatch = url.pathname.match(/^\/v1\/context\/episode$/); if (method === "POST" && contextEpisodeMatch) { const body = await readJson(request); const { project: callerProject, caller: callerId } = contextCallerProject(request, body.project); const runId = body.workflowRunId === undefined || body.workflowRunId === null ? undefined : requireString(body.workflowRunId, "workflowRunId", { max: 128 }); const runIds = new Set( [...workflowRuns.values()] .filter((run) => run.project === callerProject && (runId === undefined || run.id === runId)) .map((run) => run.id), ); const episodes = [...journal.values()] .filter((entry) => runIds.has(entry.runId)) .filter((entry) => entry.category === "error" || entry.category === "lesson" || entry.category === "observation" || entry.category === "experiment") .sort((left, right) => left.createdAt.localeCompare(right.createdAt) || left.id.localeCompare(right.id)) .map((entry) => journalEntryToContextItem(entry, callerProject)) .slice(0, 50); counters.contextRequests += 1; json(response, 200, { episodes }); return; } const contextExplainMatch = url.pathname.match(/^\/v1\/context\/explain$/); if (method === "POST" && contextExplainMatch) { const body = await readJson(request); const { project: callerProject, caller: callerId } = contextCallerProject(request, body.project); const id = requireString(body.id, "id", { max: 128 }); const { pool } = projectContextPool(callerProject); const explanation = explainContextItem(id, pool); counters.contextRequests += 1; json(response, 200, { found: explanation.item !== undefined, lineage: explanation.lineage, evidenceRefs: explanation.evidenceRefs, sources: explanation.sources, }); return; } const contextWikiCompileMatch = url.pathname.match(/^\/v1\/context\/wiki\/compile$/); if (method === "POST" && contextWikiCompileMatch) { const body = await readJson(request); const { project: callerProject } = contextCallerProject(request, body.project); const wikiPool = contextWikiPool(callerProject, nowIso()); const wiki = compileKnowledgeWiki(wikiPool); counters.contextRequests += 1; logger({ event: "context_wiki_compiled", project: callerProject, pages: wiki.audit.pages.length, contradictions: wiki.audit.contradictions, }); json(response, 200, { audit: wiki.audit, lint: lintKnowledgeWiki(wiki.pages, wikiPool), pages: [...wiki.pages].sort(([left], [right]) => left.localeCompare(right)).map(([path, content]) => ({ path, content })), }); return; } const journalPromotionMatch = url.pathname.match(/^\/v1\/journal\/([^/]+)\/promotion$/); if (method === "POST" && journalPromotionMatch) { // Governed promotion is a control-plane decision. Only hub admins // may decide it, and the deciding principal can never be the entry // author. Journal entries remain evidence: this endpoint changes a // learning lifecycle state, never gates, workflow policy, or // permissions. requireAdminAuth(request); const body = await readJson(request); const entryId = decodeURIComponent(journalPromotionMatch[1]!); const entry = journal.get(entryId); if (!entry) throw new ProtocolError(404, "journal entry not found", "journal_not_found"); const to = requireString(body.to, "to", { max: 24 }); if (to !== "approved" && to !== "rejected" && to !== "quarantined") { throw new ProtocolError( 400, "journal promotion target must be approved, rejected, or quarantined", "invalid_journal_promotion", ); } const evidenceRefs = boundedStringList(body.evidenceRefs, "evidenceRefs", 32); const updated = applyJournalPromotion( entry, { to, evidenceRefs, decidedBy: "kxm-admin", reason: requireString(body.reason, "reason", { max: 1_000 }), }, nowIso(), ); store.saveJournalEntry(updated); // Publish to the run's project (not its id) and refresh a terminal // run's exported retrospective with the new promotion state. const promotedRun = workflowRuns.get(entry.runId); if (promotedRun) { publishOps(promotedRun.project, "workflows"); exportTerminalRetrospective(promotedRun); } logger({ event: "journal_promotion_recorded", journalEntryId: entry.id, workflowRunId: entry.runId, category: entry.category, to, }); json(response, 200, { entry: updated }); return; } const waitMatch = url.pathname.match(/^\/v1\/workflows\/([^/]+)\/waits$/); if (method === "POST" && waitMatch) { expireWorkflowWaits(); const agent = requireAgent(request); requireProjectAuth(request, agent.project); const body = await readJson(request); // Do not retain a workflow object across body I/O; another request may // have advanced this run while the body was still arriving. const run = workflowRuns.get(decodeURIComponent(waitMatch[1]!)); if (!run) throw new ProtocolError(404, "workflow run not found", "workflow_not_found"); if (run.project !== agent.project || run.targetAgentId !== agent.id) { throw new ProtocolError( 403, "only the assigned coordinator can wait this workflow", "workflow_forbidden", workflowScopeExtras("wait", run.targetAgentName), ); } const stageId = requireString(body.stageId, "stageId", { max: 64 }); const signalKey = workflowSignalKey(body.signalKey); const summary = requireString(body.summary, "summary", { max: 4_000 }); const evidence = boundedWorkflowEvidence(body.evidence); const evidenceRefs = boundedWorkflowEvidenceReferences(body.evidenceRefs); const timeoutMs = parseBoundedInteger( body.timeoutMs, "timeoutMs", DEFAULT_WORKFLOW_WAIT_TIMEOUT_MS, MIN_WORKFLOW_WAIT_TIMEOUT_MS, MAX_WORKFLOW_WAIT_TIMEOUT_MS, ); const createdAt = nowIso(); const transition = structuredClone(run); const stage = transition.stages.find((candidate) => candidate.id === stageId); const verifiedEvidence: WorkflowVerifiedEvidence = stage && Object.keys(evidenceRefs).length ? verifyWorkflowEvidenceReferences( transition, stage, evidenceRefs, { getMessage: (messageId) => messages.get(messageId) }, createdAt, ) : {}; waitForWorkflowSignal( transition, stageId, signalKey, summary, createdAt, new Date(Date.parse(createdAt) + timeoutMs).toISOString(), evidence, verifiedEvidence, ); store.saveWorkflowTransition(transition); publishOps(transition.project, "workflows"); counters.workflowWaits += 1; logger({ event: "workflow_wait_started", workflowRunId: transition.id, stageId, signalKey, expiresAt: transition.waiting?.expiresAt, }); json(response, 202, { run: transition, instruction: "The agent may now settle this turn. A signed external signal will checkpoint the stage and resume the coordinator when more work is required.", }); return; } const checkpointMatch = url.pathname.match(/^\/v1\/workflows\/([^/]+)\/checkpoints$/); if (method === "POST" && checkpointMatch) { const agent = requireAgent(request); requireProjectAuth(request, agent.project); const body = await readJson(request); // Re-read authoritative state after body I/O so concurrent attempts // cannot checkpoint from the same stale run snapshot. const run = workflowRuns.get(decodeURIComponent(checkpointMatch[1]!)); if (!run) throw new ProtocolError(404, "workflow run not found", "workflow_not_found"); if (run.project !== agent.project || run.targetAgentId !== agent.id) { throw new ProtocolError( 403, "only the assigned coordinator can checkpoint this workflow", "workflow_forbidden", workflowScopeExtras("checkpoint", run.targetAgentName), ); } const stageId = requireString(body.stageId, "stageId", { max: 64 }); const status = requireString(body.status, "status", { max: 16 }) as WorkflowCheckpointStatus; if (status !== "passed" && status !== "warning" && status !== "failed") { throw new ProtocolError(400, "status must be passed, warning, or failed", "invalid_checkpoint_status"); } const summary = requireString(body.summary, "summary", { max: 4_000 }); const evidence = boundedWorkflowEvidence(body.evidence); const evidenceRefs = boundedWorkflowEvidenceReferences(body.evidenceRefs); if (status !== "passed" && Object.keys(evidenceRefs).length) { throw new ProtocolError( 400, "evidenceRefs are accepted only for a passing checkpoint or workflow wait", "invalid_workflow_evidence_refs", ); } const timestamp = nowIso(); const transition = structuredClone(run); const stage = transition.stages.find((candidate) => candidate.id === stageId); const verifiedEvidence: WorkflowVerifiedEvidence = stage && Object.keys(evidenceRefs).length ? verifyWorkflowEvidenceReferences( transition, stage, evidenceRefs, { getMessage: (messageId) => messages.get(messageId) }, timestamp, ) : {}; const result = checkpointRun( transition, stageId, status, summary, evidence, timestamp, verifiedEvidence, typeof body.outcome === "string" && body.outcome.trim() ? requireString(body.outcome, "outcome", { max: 64 }) : undefined, ); const checkpointStage = transition.stages.find((candidate) => candidate.id === stageId)!; // The attempt this checkpoint consumed. A self-edge transition re-enters // the stage and resets its counter, so prefer the transition record. const checkpointAttempt = result.transition?.attempt ?? checkpointStage.attempts; if (result.transition) { const transitionEntry: WorkflowJournalEntry = { id: newId("journal"), runId: transition.id, agentId: agent.id, category: "state-change", area: checkpointStage?.area ?? "workflow", severity: "info", summary: `typed transition ${result.transition.fromStage} -> ${result.transition.toStage} (${result.transition.outcome})`, evidence: result.transition.evidenceKeys.map((key) => `requirement:${key}`), relatedEntryIds: [], createdAt: timestamp, stageId, attempt: checkpointAttempt, }; store.saveWorkflowTransition(transition, undefined, transitionEntry); counters.journalEntries += 1; } if (result.exhausted) { const exhaustEntry: WorkflowJournalEntry = { id: newId("journal"), runId: transition.id, agentId: agent.id, category: "error", area: checkpointStage?.area ?? "workflow", severity: "error", summary: `transition budget exhausted at ${stageId} (outcome ${body.outcome ?? status}); run failed safely`, evidence: [`class:transition_budget_exhausted`, `stage:${stageId}`], relatedEntryIds: [], createdAt: timestamp, stageId, attempt: checkpointAttempt, }; store.saveWorkflowTransition(transition, undefined, exhaustEntry); counters.journalEntries += 1; } let entry: WorkflowJournalEntry | undefined; if (status !== "passed") { entry = { id: newId("journal"), runId: transition.id, agentId: agent.id, category: "error", area: checkpointStage.area ?? "workflow", severity: status === "failed" ? "error" : "warning", summary: `${checkpointStage.label}: ${summary}`, evidence: workflowEvidenceStrings(evidence), relatedEntryIds: [], createdAt: transition.updatedAt, stageId, attempt: checkpointAttempt, }; } else if (result.degraded) { entry = { id: newId("journal"), runId: transition.id, agentId: agent.id, category: "decision", area: checkpointStage.area ?? "security", severity: "warning", summary: `${checkpointStage.label} passed using an explicitly approved degraded peer quorum`, evidence: [ "class:workflow_quorum_degradation_used", ...(checkpointStage.degradedRequirements ?? []).map((requirement) => `requirement:${requirement}`), ], relatedEntryIds: [], createdAt: transition.updatedAt, stageId, attempt: checkpointAttempt, }; } store.saveWorkflowTransition(transition, undefined, entry); publishOps(transition.project, "workflows"); counters.workflowCheckpoints += 1; if (entry) counters.journalEntries += 1; exportTerminalRetrospective(transition); logger({ event: "workflow_checkpoint", workflowRunId: transition.id, stageId, status, attempt: transition.stages.find((candidate) => candidate.id === stageId)?.attempts, retry: result.retry, completed: result.completed, }); json(response, 200, { run: transition, retry: result.retry, completed: result.completed, instruction: result.retry ? "Correct the warning or failure, record what changed, rerun the relevant checks, and checkpoint this stage again." : result.completed ? "Workflow checkpoints are complete. Return the final outcome with links and evidence." : `Continue with stage ${transition.currentStage}.`, }); return; } const journalMatch = url.pathname.match(/^\/v1\/workflows\/([^/]+)\/journal$/); if (method === "POST" && journalMatch) { const agent = requireAgent(request); requireProjectAuth(request, agent.project); const body = await readJson(request); const run = workflowRuns.get(decodeURIComponent(journalMatch[1]!)); if (!run) throw new ProtocolError(404, "workflow run not found", "workflow_not_found"); if (run.project !== agent.project || run.targetAgentId !== agent.id) { throw new ProtocolError( 403, "only the assigned coordinator can journal this workflow", "workflow_forbidden", workflowScopeExtras("journal", run.targetAgentName), ); } const category = parseJournalCategory(body.category); // Stage provenance first: the stage supplies the default area, and // the hub (never the caller) derives the attempt from its state. let stage: WorkflowStageState | undefined; if (body.stageId !== undefined && body.stageId !== null) { const requestedStageId = requireString(body.stageId, "stageId", { max: 128 }); stage = run.stages.find((candidate) => candidate.id === requestedStageId); if (!stage) { throw new ProtocolError(400, `stageId ${requestedStageId} is not part of this workflow run`, "invalid_journal_relation"); } } const area = (body.area == null ? stage?.area : requireString(body.area, "area", { max: 24 })) as ImprovementArea | undefined; if (area === undefined || !IMPROVEMENT_AREAS.includes(area)) { throw new ProtocolError( 400, "area is required unless stageId names a stage that declares an area", "invalid_improvement_area", ); } const attempt = stage ? journalAttemptFor(stage) : undefined; const severity = requireString(body.severity ?? "info", "severity", { max: 16 }); if (severity !== "info" && severity !== "warning" && severity !== "error") { throw new ProtocolError(400, "severity must be info, warning, or error", "invalid_journal_severity"); } const relatedEntryIds = boundedStringList(body.relatedEntryIds, "relatedEntryIds", 16); if (relatedEntryIds.some((id) => journal.get(id)?.runId !== run.id)) { throw new ProtocolError( 400, "relatedEntryIds must reference journal entries in the same workflow run", "invalid_journal_relation", ); } const evidence = boundedStringList(body.evidence, "evidence"); if (journalEvidenceRequired(category) && evidence.length < 1) { throw new ProtocolError( 400, `journal category ${category} requires at least one durable evidence reference`, "journal_evidence_required", ); } const entry: WorkflowJournalEntry = { id: newId("journal"), runId: run.id, agentId: agent.id, category, area, severity, summary: requireString(body.summary, "summary", { max: 1_000 }), ...(body.details ? { details: requireString(body.details, "details", { max: 8_000 }) } : {}), evidence, relatedEntryIds, createdAt: nowIso(), ...(stage !== undefined ? { stageId: stage.id } : {}), ...(attempt !== undefined ? { attempt } : {}), }; store.saveJournalEntry(entry); counters.journalEntries += 1; // A late entry on a terminal run refreshes its exported retrospective. exportTerminalRetrospective(run); logger({ event: "workflow_journal_recorded", workflowRunId: run.id, journalEntryId: entry.id, category, area, severity, }); json(response, 201, { entry }); return; } if (method === "GET" && url.pathname === "/v1/improvements") { const agent = requireAgent(request); requireProjectAuth(request, agent.project); const projectRuns = new Map( [...workflowRuns.values()] .filter((run) => run.project === agent.project) .map((run) => [run.id, run] as const), ); const visibleRuns = new Set(projectRuns.keys()); const entries = [...journal.values()].filter((entry) => visibleRuns.has(entry.runId)); json(response, 200, { reports: improvementReport(entries), signals: rankImprovementSignals(entries, projectRuns), entries: entries.length, }); return; } if (method === "POST" && url.pathname === "/v1/agents/register") { const body = await readJson(request); const name = requireString(body.name, "name", { max: 64 }); const purpose = requireString(body.purpose ?? "General-purpose Pi agent", "purpose", { max: 256 }); const project = requireString(body.project, "project", { max: 128 }); requireProjectAuth(request, project); const model = optionalString(body.model, "model", 128); // A client declares which box it runs on. The label is a reading aid // only: the project token is still the whole of admission. const hostLabel = optionalString(body.host, "host", MAX_AGENT_HOST_CHARS); const existing = [...agents.values()].find( (agent) => agent.project === project && agent.name.toLowerCase() === name.toLowerCase(), ); if (existing?.online) { throw new ProtocolError(409, `agent name already active in project: ${name}`, "duplicate_agent_name"); } const timestamp = nowIso(); const agent: StoredAgent = existing ?? { id: newId("agt"), key: newId("key"), name, purpose, project, connectedAt: timestamp, lastSeenAt: timestamp, online: true, }; agent.key = newId("key"); agent.name = name; agent.purpose = purpose; agent.project = project; agent.connectedAt = timestamp; agent.lastSeenAt = timestamp; agent.online = true; if (model) agent.model = model; else delete agent.model; if (hostLabel) agent.host = hostLabel; else delete agent.host; store.saveAgent(agent); counters.registrations += 1; logger({ event: existing ? "agent_resumed" : "agent_registered", agentId: agent.id, name, project, ...(hostLabel ? { host: hostLabel } : {}), }); broadcastPresence(agent); json(response, existing ? 200 : 201, { agent: publicAgent(agent, staleAfterMs), agentKey: agent.key, resumed: Boolean(existing) }); return; } if (method === "GET" && url.pathname === "/v1/agents") { const current = requireAgent(request); requireProjectAuth(request, current.project); // Offline members are registered peers of the caller's own project // whose lease the sweep has already retired. They stay out of the // list unless a caller asks for them, so discovery keeps meaning // "who can answer right now". const includeOffline = url.searchParams.get("includeOffline") === "true"; const listedAt = Date.now(); const result = [...agents.values()] .filter((agent) => agent.project === current.project && (agent.online || includeOffline)) .map((agent) => publicAgent(agent, staleAfterMs, listedAt)); json(response, 200, { agents: result }); return; } const heartbeatMatch = url.pathname.match(/^\/v1\/agents\/([^/]+)\/heartbeat$/); if (method === "POST" && heartbeatMatch) { const current = requireAgent(request, decodeURIComponent(heartbeatMatch[1]!)); requireProjectAuth(request, current.project); await readJson(request); json(response, 200, { agent: publicAgent(current, staleAfterMs) }); return; } // An agent's open inbound requests, queued or delivered, in delivery order. A read // for a caller that holds no event-fed inbox of its own (a one-shot CLI call): it // acknowledges nothing and moves no cursor, so push delivery is unchanged. const inboxMatch = url.pathname.match(/^\/v1\/agents\/([^/]+)\/inbox$/); if (method === "GET" && inboxMatch) { const current = requireAgent(request, decodeURIComponent(inboxMatch[1]!)); requireProjectAuth(request, current.project); expireMessages(); json(response, 200, { messages: store.getPendingMessages(current.id) }); return; } const agentMatch = url.pathname.match(/^\/v1\/agents\/([^/]+)$/); if (method === "DELETE" && agentMatch) { const current = requireAgent(request, decodeURIComponent(agentMatch[1]!)); requireProjectAuth(request, current.project); current.online = false; current.lastSeenAt = nowIso(); store.saveAgent(current); broadcastPresence(current); logger({ event: "agent_unregistered", agentId: current.id, project: current.project }); response.writeHead(204, { "cache-control": "no-store" }).end(); return; } // Runtime presence (P5). A Runtime is a machine client, not an agent: it // is admitted by the project token and declares its id and host label. // The hub stamps the heartbeat with its own clock; that stamp plus // staleAfterMs is the presence lease the ops snapshot reads. if (method === "POST" && url.pathname === "/v1/runtime/presence") { const body = await readJson(request); const project = requireString(body.project, "project", { max: 128 }); requireProjectAuth(request, project); const runtimeId = requireRuntimeId(body.runtimeId); const hostLabel = optionalString(body.host, "host", MAX_AGENT_HOST_CHARS); const heartbeatAt = new Date(hubNow()).toISOString(); const existing = store.getRuntimePresence(project, runtimeId); const record: RuntimePresenceRecord = { runtimeId, project, ...(hostLabel ? { host: hostLabel } : {}), registeredAt: existing?.registeredAt ?? heartbeatAt, heartbeatAt, }; store.saveRuntimePresence(record); counters.runtimeHeartbeats += 1; if (!existing) logger({ event: "runtime_registered", project, runtimeId, ...(hostLabel ? { host: hostLabel } : {}) }); json(response, existing ? 200 : 201, { presence: runtimePresenceView(record, staleAfterMs, hubNow()) }); return; } // Run-fact sync (P5), not a peer transport. Each event is accepted once by // {projectId, runId, sequence}: identical bytes are an idempotent // duplicate, different bytes under a used sequence are refused and raised // as a security alert. Out-of-order events are held; the per-run cursor is // the gapless prefix, so a gap stays pending until it is filled. if (method === "POST" && url.pathname === "/v1/sync/events") { const body = await readJson(request); const project = requireString(body.project, "project", { max: 128 }); requireProjectAuth(request, project); const runtimeId = requireRuntimeId(body.runtimeId); if (!Array.isArray(body.events)) { throw new ProtocolError(400, "events must be an array", "invalid_sync_batch"); } if (body.events.length > MAX_SYNC_BATCH_EVENTS) { throw new ProtocolError(413, `a sync batch holds at most ${MAX_SYNC_BATCH_EVENTS} events`, "sync_batch_too_large"); } const receivedAt = new Date(hubNow()).toISOString(); const results: Array> = []; const touched = new Map(); for (const candidate of body.events as unknown[]) { const identity = candidate && typeof candidate === "object" && !Array.isArray(candidate) ? candidate as Record : {}; const echo = { ...(typeof identity.projectId === "string" ? { projectId: identity.projectId.slice(0, 144) } : {}), ...(typeof identity.runId === "string" ? { runId: identity.runId.slice(0, 144) } : {}), ...(Number.isInteger(identity.sequence) ? { sequence: identity.sequence } : {}), }; if (syncEventSchemaErrors(candidate) !== undefined) { counters.syncEventsRefused += 1; results.push({ ...echo, outcome: "rejected", code: "sync_event_invalid" }); continue; } const event = identity as { projectId: string; runId: string; sequence: number; homeRuntimeId: string; eventType: string }; if (event.homeRuntimeId !== runtimeId) { counters.syncEventsRefused += 1; logger({ event: "security_alert", alert: "sync_runtime_mismatch", project, runtimeId, ...echo, homeRuntimeId: event.homeRuntimeId }); results.push({ ...echo, outcome: "rejected", code: "sync_runtime_mismatch" }); continue; } const bytes = kxmCanonicalJson(candidate as JsonValue); const contentHash = kxmSyncEventHash(bytes); const outcome = store.ingestSyncEvent({ projectId: event.projectId, runId: event.runId, sequence: event.sequence, hubProject: project, homeRuntimeId: event.homeRuntimeId, eventType: event.eventType, contentHash, receivedAt, bytes, }); if (outcome.outcome === "conflict") { counters.syncConflicts += 1; counters.syncEventsRefused += 1; logger({ event: "security_alert", alert: "sync_sequence_conflict", reason: outcome.reason, project, runtimeId, ...echo, presentedHash: contentHash, ...(outcome.existingHash ? { existingHash: outcome.existingHash } : {}), }); results.push({ ...echo, outcome: "conflict", code: `sync_${outcome.reason}` }); continue; } if (outcome.outcome === "accepted") counters.syncEventsAccepted += 1; else counters.syncEventsDuplicate += 1; touched.set(`${event.projectId}\u0000${event.runId}`, { projectId: event.projectId, runId: event.runId }); results.push({ ...echo, outcome: outcome.outcome }); } const cursors = [...touched.values()].map((run) => ({ ...run, cursor: store.syncCursor(run.projectId, run.runId) })); if (results.some((result) => result.outcome === "accepted")) publishOps(project, "workflows"); json(response, 200, { results, cursors }); return; } // One writer, one clock. Every branch below decides inside a single store // transaction, so two agents racing for the same resource cannot both read it // free. The fencing token increments only on takeover, which is what lets a // holder that slept past its deadline be refused at commit instead of writing // behind whoever replaced it. const leaseMatch = url.pathname.match(/^\/v1\/leases\/([^/]+)\/(acquire|renew|release)$/); if (method === "POST" && leaseMatch) { const holder = requireAgent(request); requireProjectAuth(request, holder.project); const name = requireString(decodeURIComponent(leaseMatch[1]!), "resource", { max: MAX_LEASE_RESOURCE_CHARS }); const action = leaseMatch[2]!; const body = await readJson(request); // The caller never names the prefix, so the same branch in two projects is // two leases and neither project can reach the other's. const resource = `${holder.project}/${name}`; const nowMs = hubNow(); const leaseLog = { resource, project: holder.project, agentId: holder.id, agentName: holder.name }; if (action === "acquire") { const ttlMs = parseBoundedInteger(body.ttlMs, "ttlMs", DEFAULT_LEASE_TTL_MS, MIN_LEASE_TTL_MS, MAX_LEASE_TTL_MS); const outcome = store.acquireLease({ resource, project: holder.project, name, holderAgentId: holder.id, holderAgentName: holder.name, ttlMs, nowMs, }); if (!outcome.ok) { counters.leasesRefused += 1; logger({ event: "lease_denied", ...leaseLog, reason: outcome.reason, heldBy: outcome.lease?.holderAgentId }); throw leaseRefusal(outcome.reason, name, outcome.lease); } counters.leasesGranted += 1; logger({ event: outcome.renewed ? "lease_renewed" : "lease_acquired", ...leaseLog, fencingToken: outcome.lease.fencingToken, expiresAt: outcome.lease.expiresAt, }); json(response, 200, { lease: outcome.lease, renewed: outcome.renewed }); return; } if (body.fencingToken === undefined) { throw new ProtocolError(400, "fencingToken is required", "lease_token_required"); } const fencingToken = parseBoundedInteger(body.fencingToken, "fencingToken", 1, 1, Number.MAX_SAFE_INTEGER); if (action === "renew") { const ttlMs = parseBoundedInteger(body.ttlMs, "ttlMs", DEFAULT_LEASE_TTL_MS, MIN_LEASE_TTL_MS, MAX_LEASE_TTL_MS); const outcome = store.renewLease({ resource, holderAgentId: holder.id, fencingToken, ttlMs, nowMs }); if (!outcome.ok) { counters.leasesRefused += 1; logger({ event: "lease_denied", ...leaseLog, reason: outcome.reason, fencingToken }); throw leaseRefusal(outcome.reason, name, outcome.lease); } counters.leasesGranted += 1; logger({ event: "lease_renewed", ...leaseLog, fencingToken, expiresAt: outcome.lease.expiresAt }); json(response, 200, { lease: outcome.lease, renewed: true }); return; } const outcome = store.releaseLease({ resource, holderAgentId: holder.id, fencingToken }); if (!outcome.ok) { counters.leasesRefused += 1; logger({ event: "lease_denied", ...leaseLog, reason: outcome.reason, fencingToken }); throw leaseRefusal(outcome.reason, name, outcome.lease); } counters.leasesReleased += 1; logger({ event: "lease_released", ...leaseLog, fencingToken }); json(response, 200, { released: true, lease: outcome.lease }); return; } if (method === "GET" && url.pathname === "/v1/events") { const agentId = requireString(url.searchParams.get("agentId"), "agentId", { max: 80 }); const current = requireAgent(request, agentId); requireProjectAuth(request, current.project); const presenceOnly = url.searchParams.get("presenceOnly") === "true"; if (presenceOnly && current.model !== "tui") { throw new ProtocolError(403, "presence-only streams are reserved for metadata observers", "presence_stream_forbidden"); } response.writeHead(200, { "content-type": "text/event-stream", "cache-control": "no-cache, no-transform", connection: "keep-alive", "x-accel-buffering": "no", "x-content-type-options": "nosniff", ...(presenceOnly ? { "x-kxm-events-mode": "presence" } : {}), }); response.write(`event: ready\ndata: ${JSON.stringify({ agent: publicAgent(current, staleAfterMs) })}\n\n`); const client: SseClient = { response, heartbeat: setInterval(() => response.write(": heartbeat\n\n"), 15_000), ...(presenceOnly ? { presenceOnly: true } : {}), }; client.heartbeat.unref(); const clients = streams.get(agentId) ?? new Set(); clients.add(client); streams.set(agentId, clients); if (!presenceOnly) flushPending(agentId); request.on("close", () => { clearInterval(client.heartbeat); clients.delete(client); if (clients.size === 0) streams.delete(agentId); }); return; } if (method === "POST" && url.pathname === "/v1/messages") { const sender = requireAgent(request); requireProjectAuth(request, sender.project); const body = await readJson(request); const targetInput = requireString(body.target, "target", { max: 80 }); const content = requireString(body.content, "content", { max: MAX_CONTENT_CHARS }); const delivery = parseDeliveryMode(body.delivery); const requestedContext = requestedWorkflowMessageContext(body.workflowContext); const workflowContext = requestedContext ? authorizeWorkflowMessageContext(sender, targetInput, requestedContext) : undefined; const callerCorrelationId = optionalString(body.correlationId, "correlationId", 128); if (workflowContext && callerCorrelationId && callerCorrelationId !== workflowContext.runId) { throw new ProtocolError( 409, "correlationId must match workflowContext.runId", "workflow_context_correlation_mismatch", ); } const correlationId = workflowContext?.runId ?? callerCorrelationId; const replyTo = optionalString(body.replyTo, "replyTo", 80); const hops = parseBoundedInteger(body.hops, "hops", 0, 0, 100); const maxHops = parseBoundedInteger(body.maxHops, "maxHops", DEFAULT_MAX_HOPS, 1, 20); if (hops >= maxHops) { throw new ProtocolError( 400, `hop limit reached (${hops}/${maxHops}): this request would extend a chain of forwarded requests past its limit; answer the inbound request directly`, "hop_limit_reached", ); } const ttlMs = parseBoundedInteger( body.ttlMs, "ttlMs", defaultMessageTtlMs, MIN_MESSAGE_TTL_MS, MAX_MESSAGE_TTL_MS, ); const allowOffline = parseOptionalBoolean(body.allowOffline, "allowOffline"); const idempotencyKey = optionalString(body.idempotencyKey, "idempotencyKey", 128); if (idempotencyKey) { const existing = store.findMessageByIdempotency(sender.id, idempotencyKey); if (existing) { if (!sameIdempotentRequest( existing, targetInput, content, delivery, correlationId, replyTo, hops, maxHops, ttlMs, workflowContext, )) { throw new ProtocolError(409, "idempotency key was already used for another request", "idempotency_conflict"); } json(response, 200, { message: existing, idempotent: true }); return; } } const target = findTarget(sender.project, targetInput, allowOffline); if (target.id === sender.id) { throw new ProtocolError(400, "cannot send a request to yourself", "self_target"); } const createdAt = nowIso(); const seq = store.nextAgentSequence(target.id); const message: MessageRecord = { id: newId("msg"), project: sender.project, from: sender.id, fromName: sender.name, to: target.id, toName: target.name, content, delivery, hops, maxHops, seq, ...(correlationId ? { correlationId } : {}), ...(replyTo ? { replyTo } : {}), ...(idempotencyKey ? { idempotencyKey } : {}), ...(workflowContext ? { workflowRunId: workflowContext.runId, workflowContext } : {}), createdAt, expiresAt: new Date(Date.parse(createdAt) + ttlMs).toISOString(), status: "queued", }; store.saveMessage(message); publishOps(message.project, "messages"); if (target.online) publish(target.id, { type: "message", message }); counters.messagesSent += 1; logger({ event: "message_sent", messageId: message.id, hops, ...messageLog(message, sender.id, sender.name, target.id, target.name) }); json(response, 202, { message, idempotent: false }); return; } const ackMatch = url.pathname.match(/^\/v1\/messages\/([^/]+)\/ack$/); if (method === "POST" && ackMatch) { const receiver = requireAgent(request); requireProjectAuth(request, receiver.project); await readJson(request); expireMessages(); expireWorkflowWaits(); purgeTerminalMessages(); const message = messages.get(decodeURIComponent(ackMatch[1]!)); if (!message) throw new ProtocolError(404, "message not found", "message_not_found"); if (message.to !== receiver.id) { throw new ProtocolError(403, "only the recipient can acknowledge this message", "message_forbidden"); } if (message.status !== "queued" && message.status !== "delivered") { throw new ProtocolError(409, `cannot acknowledge a ${message.status} message`, "invalid_message_state"); } if (message.status === "queued") { message.status = "delivered"; message.deliveredAt = nowIso(); store.saveMessage(message); publishOps(message.project, "messages"); } if (typeof message.seq === "number") { store.advanceCursor(receiver.id, message.seq); } json(response, 200, { message }); return; } const replyMatch = url.pathname.match(/^\/v1\/messages\/([^/]+)\/reply$/); if (method === "POST" && replyMatch) { const receiver = requireAgent(request); requireProjectAuth(request, receiver.project); const body = await readJson(request); expireMessages(); const message = messages.get(decodeURIComponent(replyMatch[1]!)); if (!message) throw new ProtocolError(404, "message not found", "message_not_found"); if (message.to !== receiver.id) { throw new ProtocolError(403, "only the recipient can reply to this message", "message_forbidden"); } if (message.status === "replied") { throw new ProtocolError(409, "message already has a reply", "duplicate_reply"); } if (message.status === "cancelled" || message.status === "expired" || message.status === "error") { throw new ProtocolError(409, `cannot reply to a ${message.status} message`, "invalid_message_state"); } message.reply = { content: requireString(body.content, "content", { max: MAX_CONTENT_CHARS }), createdAt: nowIso(), }; message.repliedAt = message.reply.createdAt; message.status = "replied"; store.saveMessage(message); publishOps(message.project, "messages"); const workflowRun = [...workflowRuns.values()].find((run) => run.messageId === message.id); if (workflowRun?.status === "running") { const settledStage = workflowRun.stages.find((candidate) => candidate.id === workflowRun.currentStage); workflowRun.status = "failed"; delete workflowRun.currentStage; workflowRun.updatedAt = message.repliedAt; store.saveWorkflowRun(workflowRun); publishOps(workflowRun.project, "workflows"); const entry: WorkflowJournalEntry = { id: newId("journal"), runId: workflowRun.id, agentId: receiver.id, category: "error", area: "workflow", severity: "error", summary: "Coordinator settled before all required workflow checkpoints passed", evidence: [`message:${message.id}`], relatedEntryIds: [], createdAt: message.repliedAt, ...(settledStage ? { stageId: settledStage.id, attempt: settledStage.attempts + 1 } : {}), }; store.saveJournalEntry(entry); counters.journalEntries += 1; exportTerminalRetrospective(workflowRun); } publish(message.from, { type: "reply", message }); counters.messagesReplied += 1; logger({ event: "message_replied", messageId: message.id, ...messageLog(message, receiver.id, receiver.name, message.from, message.fromName) }); json(response, 200, { message }); return; } const messageMatch = url.pathname.match(/^\/v1\/messages\/([^/]+)$/); if (method === "DELETE" && messageMatch) { const current = requireAgent(request); requireProjectAuth(request, current.project); expireMessages(); const message = messages.get(decodeURIComponent(messageMatch[1]!)); if (!message) throw new ProtocolError(404, "message not found", "message_not_found"); if (message.from !== current.id) { throw new ProtocolError(403, "only the sender can cancel this message", "message_forbidden"); } if (message.status === "cancelled") { json(response, 200, { message }); return; } if (message.status !== "queued" && message.status !== "delivered") { throw new ProtocolError(409, `cannot cancel a ${message.status} message`, "invalid_message_state"); } message.status = "cancelled"; message.cancelledAt = nowIso(); message.error = "message cancelled by sender"; store.saveMessage(message); publishOps(message.project, "messages"); publish(message.to, { type: "cancelled", message }); counters.messagesCancelled += 1; logger({ event: "message_cancelled", messageId: message.id, ...messageLog(message, current.id, current.name, message.to, message.toName) }); json(response, 200, { message }); return; } if (method === "GET" && messageMatch) { const current = requireAgent(request); requireProjectAuth(request, current.project); expireMessages(); const message = messages.get(decodeURIComponent(messageMatch[1]!)); if (!message) throw new ProtocolError(404, "message not found", "message_not_found"); if (message.from !== current.id && message.to !== current.id) { throw new ProtocolError(403, "message is not visible to this agent", "message_forbidden"); } json(response, 200, { message }); return; } throw new ProtocolError(404, "route not found", "route_not_found"); } catch (error) { const statusCode = error instanceof ProtocolError ? error.statusCode : 500; const code = error instanceof ProtocolError ? error.code : "internal_error"; const internalMessage = error instanceof Error ? error.message : "unknown error"; const publicMessage = statusCode >= 500 ? "internal server error" : internalMessage; counters.errors += 1; logger({ event: "request_error", requestId, statusCode, code, ...(statusCode >= 500 ? { message: internalMessage } : {}) }); if (!response.headersSent) { json(response, statusCode, { error: publicMessage, code, requestId, ...(error instanceof ProtocolError && error.extras ? error.extras : {}), }); } else response.end(); } }); return { server, state: { agents, messages, workflowRuns, journal, persistent: store.persistent }, async start() { if (closed) throw new Error("hub is closed"); await new Promise((resolve, reject) => { server.once("error", reject); server.listen(port, host, () => { server.off("error", reject); resolve(); }); }); cleanupTimer = setInterval(() => { const cutoff = Date.now() - staleAfterMs; for (const agent of agents.values()) { if (agent.online && Date.parse(agent.lastSeenAt) < cutoff) { agent.online = false; store.saveAgent(agent); broadcastPresence(agent); logger({ event: "agent_stale", agentId: agent.id, project: agent.project }); } } expireMessages(); expireWorkflowWaits(); purgeTerminalMessages(); const rateCutoff = Date.now() - (rateLimit?.windowMs ?? 0); for (const [key, bucket] of rateBuckets) { if (bucket.startedAt < rateCutoff) rateBuckets.delete(key); } }, cleanupIntervalMs); cleanupTimer.unref(); const address = server.address(); if (!address || typeof address === "string") throw new Error("hub did not expose a TCP address"); return { host, port: address.port, url: `http://${host}:${address.port}` }; }, async close() { if (closed) return; closed = true; if (cleanupTimer) clearInterval(cleanupTimer); const serverClosed = server.listening ? new Promise((resolve, reject) => server.close((error) => (error ? reject(error) : resolve()))) : Promise.resolve(); const forceClose = setTimeout(() => server.closeAllConnections(), shutdownGraceMs); forceClose.unref(); for (const clients of streams.values()) { for (const client of clients) { clearInterval(client.heartbeat); client.response.end(); } } streams.clear(); for (const client of opsStreams) { clearInterval(client.heartbeat); client.response.end(); } opsStreams.clear(); server.closeIdleConnections(); try { await serverClosed; } finally { clearTimeout(forceClose); } store.close(); }, }; }