import { randomBytes } from "node:crypto"; import { resolve } from "node:path"; import { BoundedLogBuffer } from "./buffer.ts"; import { createStorageCatalog, type HarneryStorageCatalog } from "./catalog.ts"; import type { HarneryLoggerBinding, HarneryLogLevel, HarneryRegisteredStorageFamily, } from "./contract.ts"; import { encodeLogRecord, type HarneryLazyFields, type HarneryLogContext, type HarneryLogFields, type HarneryLogRecordV1, type HarneryLogScalar, sanitizeLogError, validateLogContext, validateLogEvent, validateLogFields, } from "./jsonl.ts"; import { LogMetricsAccumulator } from "./metrics.ts"; import { FileSegmentSink, familyLogDirectory } from "./segments.ts"; import { ProcessStormController } from "./storm-control.ts"; export interface HarneryLogger { isEnabled(level: HarneryLogLevel): boolean; child(context: HarneryLogContext): HarneryLogger; trace(event: string, fields?: HarneryLazyFields, error?: unknown): void; debug(event: string, fields?: HarneryLazyFields, error?: unknown): void; info(event: string, fields?: HarneryLazyFields, error?: unknown): void; warn(event: string, fields?: HarneryLazyFields, error?: unknown): void; error(event: string, fields?: HarneryLazyFields, error?: unknown): void; fatal(event: string, fields?: HarneryLazyFields, error?: unknown): void; flush(options?: { durability?: "memory" | "disk" }): Promise; close(): Promise; } export interface HarneryLoggerRuntime { logger(componentId: string, context?: HarneryLogContext): HarneryLogger; flush(options?: { durability?: "memory" | "disk" }): Promise; close(): Promise; } export interface HarneryLogLevelConfig { default?: HarneryLogLevel; families?: Readonly>; components?: Readonly>; } export type HarneryLogEnvironment = Readonly>; export class HarneryUnsupportedDurabilityError extends Error { constructor(readonly familyId: string) { super(`family ${familyId} does not support durable disk flush`); this.name = "HarneryUnsupportedDurabilityError"; } } interface QueuedRecord { record: HarneryLogRecordV1; encoded: Buffer; } interface FamilyPipeline { family: HarneryRegisteredStorageFamily; buffer: BoundedLogBuffer; metrics: LogMetricsAccumulator; storm: ProcessStormController; durableNextFlush: boolean; } const LEVELS: readonly HarneryLogLevel[] = ["trace", "debug", "info", "warn", "error", "fatal"]; export function createFileLoggerRuntime(options: { catalog: HarneryStorageCatalog; bindings?: readonly HarneryLoggerBinding[]; level_config?: HarneryLogLevelConfig; env?: HarneryLogEnvironment; strict_levels?: boolean; }): HarneryLoggerRuntime { return new LoggerRuntime( options.catalog, options.bindings ?? options.catalog.logger_bindings, options.level_config, options.env, options.strict_levels, ); } export function createInMemoryLoggerRuntime(options: { catalog: HarneryStorageCatalog; bindings: readonly HarneryLoggerBinding[]; records?: HarneryLogRecordV1[]; level_config?: HarneryLogLevelConfig; env?: HarneryLogEnvironment; }): HarneryLoggerRuntime & { records: HarneryLogRecordV1[] } { const records = options.records ?? []; const runtime = new LoggerRuntime( options.catalog, options.bindings, options.level_config, options.env ?? {}, false, async (items) => { records.push(...items.map((item) => item.record)); }, ); return Object.assign(runtime, { records }); } export function createNullLoggerRuntime(): HarneryLoggerRuntime { const logger: HarneryLogger = { isEnabled: () => false, child: () => logger, trace() {}, debug() {}, info() {}, warn() {}, error() {}, fatal() {}, async flush() {}, async close() {}, }; return { logger: () => logger, async flush() {}, async close() {} }; } class LoggerRuntime implements HarneryLoggerRuntime { readonly #catalog: HarneryStorageCatalog; readonly #bindings: Map; readonly #pipelines = new Map(); readonly #writerId = `${process.pid}-${Date.now()}-${randomBytes(8).toString("hex")}`; readonly #levels: HarneryLogLevelConfig; readonly #env: HarneryLogEnvironment; readonly #strict: boolean; readonly #drainOverride?: (items: readonly QueuedRecord[]) => Promise; #writerSeq = 0; #closed = false; constructor( catalog: HarneryStorageCatalog, bindings: readonly HarneryLoggerBinding[], levels?: HarneryLogLevelConfig, env: HarneryLogEnvironment = process.env, strict = true, drainOverride?: (items: readonly QueuedRecord[]) => Promise, ) { this.#catalog = catalog; this.#bindings = new Map(bindings.map((binding) => [binding.component_id, binding.family_id])); this.#levels = levels ?? {}; this.#env = env; this.#strict = strict; this.#drainOverride = drainOverride; for (const familyId of this.#bindings.values()) this.#pipeline(this.#catalog.require(familyId)); } logger(componentId: string, context: HarneryLogContext = {}): HarneryLogger { const familyId = this.#bindings.get(componentId); if (!familyId) throw new Error(`unknown logger binding: ${componentId}`); const family = this.#catalog.require(familyId); return new BoundLogger(this, componentId, family, validateLogContext(context)); } async flush(options: { durability?: "memory" | "disk" } = {}): Promise { const pipelines = [...this.#pipelines.values()]; if (options.durability === "disk") { for (const pipeline of pipelines) { if (pipeline.family.durability === "best-effort") throw new HarneryUnsupportedDurabilityError(pipeline.family.id); pipeline.durableNextFlush = true; } } for (const pipeline of pipelines) this.#emitStormSummaries(pipeline); await Promise.all(pipelines.map((pipeline) => pipeline.buffer.flush())); } async close(): Promise { if (this.#closed) return; this.#closed = true; for (const pipeline of this.#pipelines.values()) this.#emitStormSummaries(pipeline); await Promise.all([...this.#pipelines.values()].map((pipeline) => pipeline.buffer.close())); } enabled( componentId: string, family: HarneryRegisteredStorageFamily, level: HarneryLogLevel, ): boolean { if (family.policy.writes !== "active" || this.#closed) return false; const minimum = resolveLogLevel({ component_id: componentId, family, config: this.#levels, env: this.#env, strict: this.#strict, }); return LEVELS.indexOf(level) >= LEVELS.indexOf(minimum); } emit( componentId: string, family: HarneryRegisteredStorageFamily, context: HarneryLogContext, level: HarneryLogLevel, event: string, lazy: HarneryLazyFields = {}, error?: unknown, ): void { if (!this.enabled(componentId, family, level)) return; const pipeline = this.#pipeline(family); try { validateLogEvent(event); const fields = validateLogFields(typeof lazy === "function" ? lazy() : lazy, family.policy); const record: HarneryLogRecordV1 = { schema: "harnery.log-record/v1", kind: "record", emitted_at: new Date().toISOString(), family_id: family.id, policy_version: family.policy.policy_version, component_id: componentId, level, event, writer_id: this.#writerId, writer_seq: ++this.#writerSeq, context, fields, ...(error === undefined ? {} : { error: sanitizeLogError(error) }), }; const encoded = encodeLogRecord(record, family); pipeline.metrics.increment("accepted"); pipeline.metrics.increment("encoded"); pipeline.metrics.increment("bytes_accepted", encoded.byteLength); if (!pipeline.storm.admit(`${componentId}:${event}`, level, encoded.byteLength)) { pipeline.metrics.increment("sampled"); return; } pipeline.buffer.enqueue({ value: { record, encoded }, bytes: encoded.byteLength, high_priority: level === "warn" || level === "error" || level === "fatal", }); } catch { pipeline.metrics.failure("encoding"); } } #pipeline(family: HarneryRegisteredStorageFamily): FamilyPipeline { const current = this.#pipelines.get(family.id); if (current) return current; const metrics = new LogMetricsAccumulator(); const sink = this.#drainOverride ? undefined : new FileSegmentSink({ directory: familyLogDirectory(family), family }); const pipeline = {} as FamilyPipeline; pipeline.family = family; pipeline.metrics = metrics; pipeline.durableNextFlush = false; pipeline.storm = new ProcessStormController({ enabled: family.storm_policy?.enabled ?? true, window_ms: family.storm_policy?.window_ms ?? 1_000, max_exemplars: family.storm_policy?.max_exemplars ?? 3, }); const policy = family.buffer_policy ?? { max_bytes: 1024 * 1024, max_records: 2_000, flush_interval_ms: 250, high_severity_reserve_bytes: 128 * 1024, }; pipeline.buffer = new BoundedLogBuffer({ ...policy, minimum_high_priority_drain_interval_ms: 25, on_drop: (_reason, item) => { metrics.increment("dropped"); metrics.increment("bytes_dropped", item.encoded.byteLength); }, on_drain_error: () => metrics.failure("sink"), drain: async (items) => { const started = Date.now(); try { if (this.#drainOverride) await this.#drainOverride(items.map((item) => item.value)); else await sink!.append( items.map((item) => item.value.encoded), metrics.take(), pipeline.durableNextFlush, ); } finally { pipeline.durableNextFlush = false; metrics.observeLatency("flush_latency_ms", Date.now() - started); } }, }); this.#pipelines.set(family.id, pipeline); return pipeline; } #emitStormSummaries(pipeline: FamilyPipeline): void { for (const summary of pipeline.storm.drainSummaries(true)) { const record: HarneryLogRecordV1 = { schema: "harnery.log-record/v1", kind: "aggregate", emitted_at: summary.last_emitted_at, family_id: pipeline.family.id, policy_version: pipeline.family.policy.policy_version, component_id: "harnery.logger", level: "warn", event: "log.storm_aggregate", writer_id: this.#writerId, writer_seq: ++this.#writerSeq, context: {}, fields: { key: summary.key }, aggregate: { count: summary.count, represented_bytes: summary.represented_bytes, first_emitted_at: summary.first_emitted_at, last_emitted_at: summary.last_emitted_at, exemplar_digests: summary.exemplar_digests, }, }; const encoded = encodeLogRecord(record, pipeline.family); pipeline.buffer.enqueue({ value: { record, encoded }, bytes: encoded.byteLength, high_priority: true, }); pipeline.metrics.increment("coalesced", summary.count); } } } class BoundLogger implements HarneryLogger { constructor( readonly runtime: LoggerRuntime, readonly componentId: string, readonly family: HarneryRegisteredStorageFamily, readonly context: HarneryLogContext, ) {} isEnabled(level: HarneryLogLevel): boolean { return this.runtime.enabled(this.componentId, this.family, level); } child(context: HarneryLogContext): HarneryLogger { return new BoundLogger( this.runtime, this.componentId, this.family, validateLogContext({ ...this.context, ...context }), ); } trace(e: string, f?: HarneryLazyFields, x?: unknown): void { this.runtime.emit(this.componentId, this.family, this.context, "trace", e, f, x); } debug(e: string, f?: HarneryLazyFields, x?: unknown): void { this.runtime.emit(this.componentId, this.family, this.context, "debug", e, f, x); } info(e: string, f?: HarneryLazyFields, x?: unknown): void { this.runtime.emit(this.componentId, this.family, this.context, "info", e, f, x); } warn(e: string, f?: HarneryLazyFields, x?: unknown): void { this.runtime.emit(this.componentId, this.family, this.context, "warn", e, f, x); } error(e: string, f?: HarneryLazyFields, x?: unknown): void { this.runtime.emit(this.componentId, this.family, this.context, "error", e, f, x); } fatal(e: string, f?: HarneryLazyFields, x?: unknown): void { this.runtime.emit(this.componentId, this.family, this.context, "fatal", e, f, x); } flush(o?: { durability?: "memory" | "disk" }): Promise { return this.runtime.flush(o); } close(): Promise { return this.runtime.close(); } } export function resolveLogLevel(options: { component_id: string; family: HarneryRegisteredStorageFamily; config?: HarneryLogLevelConfig; env?: HarneryLogEnvironment; strict?: boolean; }): HarneryLogLevel { const env = options.env ?? process.env; const strict = options.strict ?? true; const scoped = parseScopedLevels(env.HARNERY_LOG_LEVELS, strict); const raw = scoped.components[options.component_id] ?? scoped.families[options.family.id] ?? options.config?.components?.[options.component_id] ?? options.config?.families?.[options.family.id] ?? env.HARNERY_LOG_LEVEL ?? options.config?.default ?? options.family.default_level ?? "info"; if (isLevel(raw)) return raw; if (strict) throw new Error(`invalid Harnery log level: ${raw}`); return options.family.default_level ?? "info"; } function parseScopedLevels( raw: string | undefined, strict: boolean, ): { components: Record; families: Record } { const result = { components: {} as Record, families: {} as Record, }; if (!raw) return result; for (const token of raw .split(",") .map((part) => part.trim()) .filter(Boolean)) { const [scope, value] = token.split("=", 2); const [kind, id] = (scope ?? "").split(":", 2); if (!id || !isLevel(value)) { if (strict) throw new Error(`invalid HARNERY_LOG_LEVELS entry: ${token}`); continue; } if (kind === "component") result.components[id] = value; else if (kind === "family") result.families[id] = value; else if (strict) throw new Error(`invalid HARNERY_LOG_LEVELS scope: ${token}`); } return result; } function isLevel(value: unknown): value is HarneryLogLevel { return typeof value === "string" && LEVELS.includes(value as HarneryLogLevel); } export function legacyLogFields(value: Record): HarneryLogFields { const out: HarneryLogFields = {}; for (const [key, candidate] of Object.entries(value)) { if (candidate === undefined) continue; if ( candidate === null || typeof candidate === "string" || typeof candidate === "number" || typeof candidate === "boolean" ) out[key] = candidate as HarneryLogScalar; else if ( Array.isArray(candidate) && candidate.every( (item) => item === null || ["string", "number", "boolean"].includes(typeof item), ) ) out[key] = candidate as HarneryLogScalar[]; else { const encoded = JSON.stringify(candidate); if (encoded !== undefined && Buffer.byteLength(encoded) <= 4_096) out[key] = encoded; } } return out; } export interface HarneryBoundedObjectArray { json: string; included: number; omitted: number; truncated: boolean; } /** Encode an allow-listed array of scalar records without slicing a JSON token. */ export function encodeBoundedLogObjectArray( values: readonly unknown[], options: { max_bytes: number; allowed_fields: readonly string[] }, ): HarneryBoundedObjectArray { if (!Number.isSafeInteger(options.max_bytes) || options.max_bytes < 2) { throw new Error("bounded log object array needs at least two bytes"); } const allowed = new Set(options.allowed_fields); if (allowed.size !== options.allowed_fields.length || allowed.size === 0) { throw new Error("bounded log object array needs unique allowed fields"); } const normalized = values.map((value, index) => { if (!value || typeof value !== "object" || Array.isArray(value)) { throw new Error(`bounded log object array item ${index} must be an object`); } const source = value as Record; for (const key of Object.keys(source)) { if (!allowed.has(key)) throw new Error(`bounded log object array rejects field: ${key}`); } const item: Record = {}; for (const key of options.allowed_fields) { const candidate = source[key]; if (candidate === undefined) continue; if ( candidate !== null && typeof candidate !== "string" && typeof candidate !== "boolean" && (typeof candidate !== "number" || !Number.isFinite(candidate)) ) { throw new Error(`bounded log object array field ${key} must be scalar`); } item[key] = candidate as HarneryLogScalar; } return item; }); const included: Record[] = []; let bytes = 2; for (const item of normalized) { const encoded = JSON.stringify(item); const nextBytes = Buffer.byteLength(encoded) + (included.length > 0 ? 1 : 0); if (bytes + nextBytes > options.max_bytes) break; included.push(item); bytes += nextBytes; } return { json: JSON.stringify(included), included: included.length, omitted: normalized.length - included.length, truncated: included.length !== normalized.length, }; } const PROCESS_RUNTIMES = new Map(); const PROCESS_LOGGER_BINDINGS = { "agent-hook": "agent-hook-debug-log", "agent-coord": "agent-coord-debug-log", "semantic-service": "semantic-service-log", "resource-observer": "resource-observer-log", supervisor: "supervisor-log", "governor-service": "governor-service-log", "presence-relay": "presence-relay-log", } as const; export type HarneryProcessLoggerComponent = keyof typeof PROCESS_LOGGER_BINDINGS; export function processLogger( coordRoot: string, componentId: HarneryProcessLoggerComponent, ): HarneryLogger { const root = resolve(coordRoot); let runtime = PROCESS_RUNTIMES.get(root); if (!runtime) { const bindings = Object.entries(PROCESS_LOGGER_BINDINGS).map(([component_id, family_id]) => ({ component_id, family_id, })); const catalog = createStorageCatalog({ coord_root: root }, { logger_bindings: bindings }); runtime = createFileLoggerRuntime({ catalog, bindings, strict_levels: false }); PROCESS_RUNTIMES.set(root, runtime); } return runtime.logger(componentId); } export async function closeProcessLoggers(): Promise { await Promise.all([...PROCESS_RUNTIMES.values()].map((runtime) => runtime.close())); PROCESS_RUNTIMES.clear(); }