/** Expand shorthand Accept names: json → application/json, xml → application/xml; else pass through. */
function expandAccept(accept: string): string {
	return accept === "json"
		? "application/json"
		: accept === "xml"
			? "application/xml"
			: accept;
}

/**
 * Built-in executor helpers — the three-helper v1 set.
 *
 * `restGet`  — single-request fetch with path templating, query assembly,
 *              Accept negotiation, and auth dispatch.
 * `paginate` — paginated fetch supporting offset-limit, nextLink, cursor,
 *              and page styles; optional `gatherAll` with ceiling.
 * `parseResponse` — XML→JSON conversion with charset correction.
 *
 * Every helper returns a structured `HelperResult` or throws `HelperError`.
 */

import { XMLParser } from "fast-xml-parser";
import { ssrfGuard } from "./ssrf-guard.js";
import { scrubSecretValues } from "./auth.js";
import { serverMessage, isPlanGated } from "./status-hint.js";
import {
	fetchUrl,
	redactSecretParams,
	redactSecretPathValues,
	type FetchOptions,
} from "./transport.js";
import {
	fillPathTemplate,
	joinUrl,
	tokenizeJsonPath,
	wireParamName,
} from "./path-template.js";
import type { TransformFn } from "./local-helpers.js"; // type-only — no runtime import (flat dependency direction)
import type {
	ApiGuide,
	DateParamFormat,
	ListStyle,
	Operation,
	QueryParamSpec,
	ResponseShape,
} from "./api-guide-types.js";

// ═══════════════════════════════════════════════════════════════════
// Error type
// ═══════════════════════════════════════════════════════════════════

export class HelperError extends Error {
	override name = "HelperError";

	constructor(
		/** Dotted field path, e.g. "params.date" or "auth.kind". */
		public readonly field: string,
		message: string,
		/** Human-readable description of what was expected. */
		public readonly expected?: string,
		/** What was actually found. */
		public readonly found?: string,
		/** Suggested fix, if the helper can offer one. */
		public readonly fix?: string,
		/** The full resolved URL that caused the error, if known. */
		public readonly url?: string,
	) {
		super(message);
	}
}

// ═══════════════════════════════════════════════════════════════════
// Result types
// ═══════════════════════════════════════════════════════════════════

export interface RestGetResult {
	/** Full parsed response body (JSON object/array or XML-converted). */
	data: unknown;
	/** Response headers. */
	headers: Record<string, string>;
	/** The full resolved URL that was fetched. */
	url: string;
	/** Effective query params actually sent (post-defaults, post-validation).
	 *  A `listStyle` param's multi-value array surfaces as a real `string[]`
	 *  keyed by the declared param name (the URL shows the true wire form). */
	params: Record<string, string | string[]>;
	/** Set when a post-response transform throws (raw `data` preserved). */
	transformWarning?: string;
	/** True when the transport served this response from cache (header hit
	 *  or 304 revalidation). Absent for live fetches. */
	cached?: boolean;
}

export interface PaginateResult {
	/** Accumulated items across all fetched pages. */
	items: unknown[];
	/** Total count of items fetched (may be less than server total due to ceiling). */
	totalFetched: number;
	/**
	 * Raw (untransformed) items whose post-response transform threw. Absent
	 * when no item fails — no item is ever dropped. Total raw items processed
	 * = `items.length + failedItems.length`.
	 */
	failedItems?: unknown[];
	/** Server-reported total count from the first page (when the guide
	 * declares `totalCountPath` and it resolves to a number/numeric string). */
	serverTotal?: number;
	/** True when `gatherAllMax` ceiling halted pagination. */
	ceilingHit: boolean;
	/** Every page URL fetched (bounded by page count). */
	urls: string[];
	/** Number of pages fetched (= urls.length). */
	pages: number;
	/** Effective query params actually sent (post-defaults, post-validation).
	 *  A `listStyle` param's multi-value array surfaces as a real `string[]`
	 *  keyed by the declared param name (the URL shows the true wire form). */
	params: Record<string, string | string[]>;
}

// ═══════════════════════════════════════════════════════════════════
// Internal helpers
// ═══════════════════════════════════════════════════════════════════

/**
 * Strict path templating for execution — a missing `{token}` throws.
 * (The shared `fillPathTemplate` keeps missing tokens literal for probing.)
 */
function fillPathStrict(path: string, params: Record<string, unknown>): string {
	return fillPathTemplate(path, params, (token) => {
		throw new HelperError(
			`params.${token}`,
			`Missing path parameter: ${token}`,
			`a value for "{${token}}" in path "${path}"`,
			"missing",
		);
	});
}

/**
 * Build query-string params: apply defaults for query params,
 * validate required params, and exclude path params.
 *
 * `secretParamNames`: the query-param names that are code-injected from
 * the secrets store. They are excluded from the returned agent-supplied map
 * AND skipped in the `passthrough` branch so an agent-supplied value can't
 * override or race the injection.
 */
function buildQueryParams(
	operation: Operation,
	params: Record<string, unknown>,
	secretParamNames?: Set<string>,
): Record<string, string | string[]> {
	const query: Record<string, string | string[]> = {};
	const pathParamSet = new Set(operation.pathParams);

	for (const [key, spec] of Object.entries(operation.params)) {
		if (pathParamSet.has(key)) continue; // already in path

		let val = params[key];

		// Apply default if not provided.
		if (val === undefined && spec.default !== undefined) {
			val = spec.default;
		}

		// Validate required.
		if (val === undefined && spec.required) {
			throw new HelperError(
				`params.${key}`,
				`Missing required query parameter: ${key}`,
				`a value for required param "${key}"`,
				"missing",
			);
		}

		if (val !== undefined) {
			// Date param normalization — convert to target format before
			// serialization. An array here is a loud error BEFORE normalization
			// (normalizeDateParam operates on scalars; an array would be
			// silently mangled on the wire as String(val) garbage).
			if (operation.dateParams && key in operation.dateParams) {
				if (Array.isArray(val)) {
					throw new HelperError(
						`params.${key}`,
						`Array value for date param "${key}" — date params are single-valued`,
						"a scalar date value",
						"an array",
						`Pass one date for params.${key} (remove it from dateParams if the API truly accepts multi-value dates — then declare listStyle:).`,
					);
				}
				val = normalizeDateParam(val, operation.dateParams[key]!);
			}
			// Scalars as today; nested objects as JSON (BOE's query_string DSL);
			// arrays only with a declared listStyle (multi-value). Shared with
			// the passthrough loop below — one validator, both sites.
			query[key] = serializeParamValue(key, val, spec.listStyle);
		}
	}

	// At-least-one-of constraint: at least one group member must be supplied.
	// Members can't be `required` or carry a `default` (parser-enforced), so
	// this is the only guard the group needs — same fail-closed seam as the
	// per-param `required` check above. Both api-fetch and /api verify route
	// through here (resolveOpForExecution → restGet/paginate), so one guard
	// covers both call sites.
	if (operation.requiresAnyOf && operation.requiresAnyOf.length > 0) {
		const satisfied = operation.requiresAnyOf.some(
			(name) => params[name] !== undefined,
		);
		if (!satisfied) {
			throw new HelperError(
				"params.requiresAnyOf",
				`Missing one of: ${operation.requiresAnyOf.join(", ")}`,
				`at least one of: ${operation.requiresAnyOf.join(", ")}`,
				"none supplied",
			);
		}
	}

	// `passthrough`: forward caller-supplied params not declared in the
	// recipe's `params` map onto the query string as-is. For APIs with an
	// open param surface (Infogami /query.json flat form, CKAN, OAI-PMH)
	// where the caller supplies type-specific keys at query time. Default
	// is a closed contract: extras are dropped so the agent gets a
	// predictable request, not a silent miss on an undeclared key.
	if (operation.passthrough) {
		for (const [key, val] of Object.entries(params)) {
			if (pathParamSet.has(key)) continue; // path params stay in path
			if (key in operation.params) continue; // already handled above
			// A secretQueryRefs param name is code-injected below the map —
			// drop any agent-supplied value so it can't override or race it.
			if (secretParamNames && secretParamNames.has(key)) continue;
			if (val === undefined) continue;
			let v: unknown = val;
			if (operation.dateParams && key in operation.dateParams) {
				if (Array.isArray(v)) {
					throw new HelperError(
						`params.${key}`,
						`Array value for date param "${key}" — date params are single-valued`,
						"a scalar date value",
						"an array",
					);
				}
				v = normalizeDateParam(v, operation.dateParams[key]!);
			}
			// Passthrough keys have no schema and can't declare a listStyle —
			// passing `undefined` makes any array a loud error. Escape valve:
			// declare the param (with a style) instead of passing it undeclared.
			query[key] = serializeParamValue(key, v, undefined);
		}
	}

	return query;
}

/**
 * Serialize one query-param value — the shared array-rules validator used by
 * BOTH serialization loops (declared params + passthrough) in
 * `buildQueryParams`. The parser's array-`default` validation mirrors these
 * rules at authoring time.
 *
 * - Scalars: exactly as today (`String(val)`).
 * - Nested objects: JSON (BOE's `query_string` DSL must not regress).
 * - Arrays: multi-value, only on a `listStyle`-declared param. Empty arrays
 *   and arrays of non-scalars are loud errors (an empty array would silently
 *   drop a param that already passed the `required`/`requiresAnyOf` guards;
 *   `[object Object]` on the wire is the silent-wrong-wire class this
 *   exists to kill). For `comma`, an element containing a `,` is also loud:
 *   the joined wire form is indistinguishable from a longer list.
 * - An array without a `listStyle` (including every passthrough array) is a
 *   loud error naming the param and the fix.
 *
 * Returns the stored value: a scalar string, or a real `string[]` for
 * multi-value (expanded to wire pairs in `buildUrl`, the one
 * Record→query-string boundary).
 */
function serializeParamValue(
	key: string,
	val: unknown,
	listStyle: ListStyle | undefined,
): string | string[] {
	if (Array.isArray(val)) {
		if (listStyle === undefined) {
			throw new HelperError(
				`params.${key}`,
				`Array value for query param "${key}" but the param declares no listStyle`,
				"a scalar, or an array on a listStyle-declared param",
				"an array",
				`Declare listStyle: comma | repeat | bracket on params.${key}, or pass a scalar`,
			);
		}
		if (val.length === 0) {
			throw new HelperError(
				`params.${key}`,
				`Empty array for multi-value query param "${key}" — the param would be silently dropped from the request`,
				"a non-empty array of scalar (string | number | boolean) elements",
				"[]",
			);
		}
		const out: string[] = [];
		for (const el of val) {
			if (
				el === null ||
				(typeof el !== "string" &&
					typeof el !== "number" &&
					typeof el !== "boolean")
			) {
				throw new HelperError(
					`params.${key}`,
					`Non-scalar element in multi-value query param "${key}"`,
					"an array of scalar (string | number | boolean) elements",
					typeof el === "object" && el !== null ? "an object" : String(el),
				);
			}
			if (listStyle === "comma" && typeof el === "string" && el.includes(",")) {
				throw new HelperError(
					`params.${key}`,
					`Element of "${key}" contains a comma — listStyle: comma joins with ',' and the joined wire form would be indistinguishable from a longer list`,
					"comma-free elements",
					`"${el}"`,
				);
			}
			out.push(String(el));
		}
		return out;
	}
	// Nested objects/arrays-of-last-resort serialize as JSON so structured
	// query params (e.g. BOE's ES query_string DSL) reach the wire as JSON,
	// not `[object Object]`. Scalars stay as String(val).
	return typeof val === "object" && val !== null
		? JSON.stringify(val)
		: String(val);
}

/**
 * Normalize a date string to a target format.
 * Accepts ISO 8601 (YYYY-MM-DD or with time tail), already-target format,
 * or non-date strings (passed through as-is).
 */
export function normalizeDateParam(
	val: unknown,
	format: DateParamFormat,
): string {
	const s = typeof val === "string" ? val : String(val ?? "");
	const m = s.match(/^(\d{4})-?(\d{1,2})-?(\d{1,2})(.*)$/);
	if (!m) return s;
	const [, y, mo, d] = m;
	const mm = mo!.padStart(2, "0");
	const dd = d!.padStart(2, "0");
	if (format === "yyyymmdd") return `${y}${mm}${dd}`;
	if (format === "yyyy-mm-dd") return `${y}-${mm}-${dd}`;
	return format === "iso8601" ? `${y}-${mm}-${dd}${m[4] ?? ""}` : s;
}

/**
 * Resolve a simple dot-delimited JSON path against an object.
 * Supports `data.items`, `resultados[0].campo`, negative array indexes
 * addressing from the end (`results[-1].id`), `$.items` prefix, and
 * quoted-bracket atomic keys for dot-containing names (`['@odata.nextLink']`).
 *
 * Returns `unknown` by design — the value at an arbitrary JSON path has no
 * named domain type until the caller knows the path they asked for.
 */
// pi-lens-ignore: ast-grep:no-unknown-returns
export function resolveJsonPath(obj: unknown, path: string): unknown {
	const parts = tokenizeJsonPath(path);
	if (parts === null) return undefined; // malformed path → miss, never a wrong match
	if (parts.length === 0) return obj;

	let current: unknown = obj;
	for (const part of parts) {
		if (current === null || typeof current !== "object") return undefined;
		if (Array.isArray(current)) {
			const idx = parseInt(part, 10);
			if (isNaN(idx)) return undefined;
			// Negative index addresses from the end; out of bounds → miss.
			const j = idx < 0 ? idx + current.length : idx;
			if (j < 0 || j >= current.length) return undefined;
			current = current[j];
		} else {
			current = (current as Record<string, unknown>)[part];
		}
	}
	return current;
}

// ═══════════════════════════════════════════════════════════════════
// parseResponse
// ═══════════════════════════════════════════════════════════════════

const xmlParser = new XMLParser({
	ignoreAttributes: false,
	attributeNamePrefix: "@_",
	textNodeName: "#text",
	parseAttributeValue: true,
	trimValues: true,
	// Strip element-name prefixes (message:GenericData → GenericData) so
	// itemsPath/totalCountPath use stable local names across XML providers.
	removeNSPrefix: true,
});

/**
 * Parse a response body according to the guide's response shape.
 *
 * The body is already decoded to a JavaScript string by the transport
 * layer, which uses the response's Content-Type charset with the recipe's
 * `shape.charset` as a fallback when the header omits one (an explicit
 * header charset wins). `parseResponse` itself does no decoding — the
 * `charset` field is consumed by the transport, not here.
 *
 * Returns `unknown` by design — an arbitrary API response body has no named
 * domain type; callers resolve JSON paths or run transforms against it.
 *
 * @param body    The response body string (correctly decoded).
 * @param shape   The response shape (format dictates XML vs JSON parsing).
 * @returns       Parsed JSON value or XML-converted object.
 */
// pi-lens-ignore: ast-grep:no-unknown-returns
export function parseResponse(body: string, shape: ResponseShape): unknown {
	if (shape.format === "text") {
		// Raw passthrough — returned as-is, no trim (matches xml/json branches).
		return body;
	}

	if (shape.format === "xml") {
		return xmlParser.parse(body);
	}

	// JSON
	try {
		return JSON.parse(body);
	} catch (e) {
		throw new HelperError(
			"response",
			`Invalid JSON response: ${e instanceof Error ? e.message : String(e)}`,
			"valid JSON",
			body.slice(0, 200),
		);
	}
}

// ═══════════════════════════════════════════════════════════════════
// URL construction & auth dispatch
// ═══════════════════════════════════════════════════════════════════

/**
 * Construct the full URL from apiHost + operation path + query params.
 *
 * Agent-supplied URLs are not SSRF-guarded here — the guard lives in
 * `paginate`'s nextLink branch (the one place the URL comes from the
 * remote server).
 */
function buildUrl(
	apiHost: string,
	resolvedPath: string,
	query: Record<string, string | string[]>,
	paramSpecs?: Record<string, QueryParamSpec>,
): string {
	// The one Record→query-string boundary. Scalars pass through as one
	// pair; a listStyle param's array expands here — `comma` joins (one
	// pair; elements were validated comma-free at build time), `repeat`
	// fans out one pair per element under the declared name, `bracket`
	// fans out with the wire key dressed +"[]". URLSearchParams encodes the
	// brackets (`id%5B%5D=`) and preserves insertion order (secret params
	// ride below the agent pairs — merge-order contract).
	const pairs: [string, string][] = [];
	for (const [key, val] of Object.entries(query)) {
		if (Array.isArray(val)) {
			const style = paramSpecs?.[key]?.listStyle;
			if (style === "comma") {
				pairs.push([key, val.join(",")]);
			} else if (style === "bracket") {
				const wireKey = `${key}[]`;
				for (const el of val) pairs.push([wireKey, el]);
			} else {
				for (const el of val) pairs.push([key, el]); // repeat
			}
		} else {
			pairs.push([key, val]);
		}
	}
	return joinUrl(apiHost, resolvedPath, new URLSearchParams(pairs).toString());
}

/**
 * Check auth.kind and branch accordingly.
 * v1 realizes `none` and `static-key` (header injection is resolved by
 * api-fetch and passed via opts.authHeaders — the store is not read here).
 *
 * This dispatch is a deliberate seam, not dead code: keeping the field +
 * dispatch now makes the future keyed-auth build additive (a new `kind`
 * behind this same `if`/`throw`) rather than a retrofit of every `restGet`/
 * `paginate` call site and the guide schema.
 */
function checkAuth(auth: ApiGuide["auth"]): void {
	switch (auth.kind) {
		case "none":
		case "static-key":
		case "oauth2":
			return;
		default: {
			// Exhaustiveness: a future fourth kind is a compile error here.
			const _exhaustive: never = auth;
			throw new HelperError(
				"auth.kind",
				`Auth kind "${_exhaustive}" is not supported in this version`,
				"one of: none | static-key | oauth2",
				String(_exhaustive),
			);
		}
	}
}

/**
 * Split the caller params around secret-owned path tokens: the
 * path fill sees the merged map (store value wins; any agent-supplied value
 * for a secret-owned token is dropped), while the query builder gets a
 * deletion-only copy. The merged map must never reach buildQueryParams — a
 * `passthrough: true` op without {token} in its path would otherwise leak
 * the plaintext token into the query string and result.params.
 */
export function splitPathSecretParams(
	params: Record<string, unknown>,
	pathSecrets: Record<string, string>,
): {
	fillParams: Record<string, unknown>;
	queryParamsForBuild: Record<string, unknown>;
} {
	if (Object.keys(pathSecrets).length === 0)
		return { fillParams: params, queryParamsForBuild: params };
	return {
		fillParams: { ...params, ...pathSecrets },
		queryParamsForBuild: Object.fromEntries(
			Object.entries(params).filter(([k]) => !(k in pathSecrets)),
		),
	};
}

/**
 * Compose the two URL redaction channels for a surfaced URL — query params
 * by name, path tokens by value. Both helpers early-return when their
 * input is empty, so callers may pass values/set unconditionally.
 */
function redactSurfacedUrl(
	url: string,
	secretParamNames: Set<string>,
	pathValues: string[],
): string {
	return redactSecretPathValues(
		redactSecretParams(url, secretParamNames),
		pathValues,
	);
}

/** Named args for fetchWithOpts — replaced an 11-param positional tail
 *  (review nit #4). Optional fields carry `| undefined` so callers can pass
 *  through possibly-undefined values without spreads; falsy booleans are
 *  no-ops (truthiness-checked below). */
interface FetchWithOptsArgs {
	/** Expanded Accept header value. */
	accept: string;
	/** Store-injected secret headers + literal auth.headers, merged. */
	extraHeaders?: Record<string, string> | undefined;
	/** Skip cache — force fresh fetch. */
	fresh?: boolean | undefined;
	/** Route through the transport's SSRF-guarded redirect path. */
	guardRedirects?: boolean | undefined;
	/** Charset fallback for servers that omit Content-Type charset. */
	fallbackCharset?: string | undefined;
	/** Lowercased injected header names — stripped on cross-domain redirects. */
	secretHeaderNames?: Set<string> | undefined;
	/** Suppress ETag cache when query secrets ride the URL. */
	hasQuerySecret?: boolean | undefined;
	/** Injected query-secret names — redacted from every surfaced URL. */
	secretQueryParamNames?: Set<string> | undefined;
	/** Path secrets present — cache-skip + guarded redirects. */
	hasPathSecret?: boolean | undefined;
	/** Redaction closure for cross-domain redirect hops. */
	redactPathSecret?: ((url: string) => string) | undefined;
}

function fetchWithOpts(
	url: string,
	args: FetchWithOptsArgs,
): ReturnType<typeof fetchUrl> {
	const opts: FetchOptions = {
		headers: { accept: args.accept, ...args.extraHeaders },
	};
	if (args.fresh !== undefined) opts.fresh = args.fresh;
	if (args.guardRedirects) opts.guardRedirects = true;
	if (args.fallbackCharset) opts.fallbackCharset = args.fallbackCharset;
	if (args.secretHeaderNames) opts.secretHeaderNames = args.secretHeaderNames;
	if (args.hasQuerySecret) opts.hasQuerySecret = true;
	if (args.secretQueryParamNames)
		opts.secretQueryParamNames = args.secretQueryParamNames;
	if (args.hasPathSecret) opts.hasPathSecret = true;
	if (args.redactPathSecret) opts.redactPathSecret = args.redactPathSecret;
	return fetchUrl(url, opts);
}

/**
 * Check fetch result status and throw a structured error on 4xx/5xx.
 *
 * Without this check, a non-2xx response is passed to `parseResponse`
 * which tries to JSON.parse an XML error body, producing a misleading
 * "Invalid JSON response" message.
 *
 * For XML error bodies (common in REST APIs), the `<text>` element is
 * extracted for a cleaner human message.
 */
function checkResponseStatus(
	result: {
		status: number;
		body: string;
		url?: string;
	},
	secretValues?: string[],
): void {
	if (result.status < 400) return;

	// Output-channel audit: scrub known store-injected secret values from the
	// error excerpt so a 401 body echoing an auth header can't leak the key
	// into agent context.
	const scrubbed = scrubSecretValues(result.body, secretValues);
	// Cap the raw body to avoid flooding the error output.
	let message = scrubbed.slice(0, 500);
	// Try to extract <text> from BOE-style XML error bodies.
	const textMatch = scrubbed.match(/<text>([\s\S]*?)<\/text>/i);
	if (textMatch) {
		message = textMatch[1]!.trim();
	}

	// 403 + structured JSON: surface the server's own reason and flag
	// plan-gating so a "plan doesn't support this endpoint" reads as a
	// key/subscription limitation, not a recipe bug. Same classifier as
	// api-probe — one implementation, two call sites. Parses the scrubbed
	// body (full, for the best chance the JSON is intact), never the raw one.
	// Unlike probe, fetch has no auth context, so it only adds the
	// plan-gating hint — no "auth configured correctly" claim.
	if (result.status === 403) {
		const reason = serverMessage(scrubbed);
		if (reason) {
			message = isPlanGated(scrubbed)
				? `${reason} (plan/subscription limitation on the key, not the recipe)`
				: reason;
		}
	}

	throw new HelperError(
		"response",
		`Unexpected HTTP ${result.status}: ${message}`,
		"HTTP 2xx",
		String(result.status),
		undefined,
		result.url,
	);
}

/**
 * Check a parsed 200 body for a declared present-only-on-error envelope
 * element (`Operation.errorPath`). Resolution of anything other than
 * `undefined` fails the call — presence is the signal, so `null`/""/`0`/
 * `false` all fire (an empty XML element parses to "" and must not read as
 * success); declared-absent is the API's not-an-error signal. Deliberately
 * the inverse of hasMorePath's truthiness (where a RESOLVED falsy value is
 * the stop signal).
 *
 * Generic capped stringify — no API-specific shape sniffing. Known secret
 * values are scrubbed and the (already-redacted) request URL rides the
 * error, same output-channel contract as checkResponseStatus.
 *
 * Placement (paginate loop): the very next statement after parseResponse —
 * BEFORE totalCountPath extraction and both exhaustion breaks. An error page
 * often misses itemsPath entirely (World Bank: error `[{"message":[…]}]` vs
 * success `[meta, records]`), so a check after the breaks never fires — the
 * walk exits with silent `items: []`.
 */
function checkErrorEnvelope(
	data: unknown,
	errorPath: string,
	url: string | undefined,
	secretValues?: string[],
): void {
	const resolved = resolveJsonPath(data, errorPath);
	if (resolved === undefined) return;
	const scrubbed = scrubSecretValues(JSON.stringify(resolved), secretValues);
	throw new HelperError(
		"response",
		`API error envelope at ${errorPath} (HTTP status was 200): ${scrubbed.slice(0, 500)}`,
		`nothing at ${errorPath} (declared-absent = not an error)`,
		undefined,
		undefined,
		url,
	);
}

/** Shared secret-handling prelude for both executors (restGet + paginate).
 *  Derives every store-secret-derived value the executors need — path-token
 *  fill, query assembly inputs, redaction inputs, and the merged header set —
 *  from the same inputs in the same way, so a hardening change to one
 *  executor can't silently miss the other. Security-critical: keep both
 *  executors on this one implementation. */
function buildExecutorEnv(
	guide: ApiGuide,
	params: Record<string, unknown>,
	opts?: SecretAuthOpts,
) {
	// Secret-owned path tokens (see splitPathSecretParams): the store fills
	// them BELOW the agent params map — agent-supplied values are dropped.
	const pathSecrets = opts?.secretPathParams ?? {};
	const pathValues = Object.values(pathSecrets);
	const hasPathSecrets = pathValues.length > 0;
	const { fillParams, queryParamsForBuild } = splitPathSecretParams(
		params,
		pathSecrets,
	);
	// Secret query params are injected BELOW the agent-supplied map — never
	// into it — so the returned `params` stays agent-supplied-only.
	const secretParamNames = opts?.secretQueryParamNames ?? new Set<string>();
	const secretParams = opts?.secretQueryParams ?? {};
	const hasQuerySecret = Object.keys(secretParams).length > 0;
	// Redaction closure for cross-domain redirect hops — built here where
	// the values are already in scope; the transport stays value-agnostic.
	const redactPathSecret = hasPathSecrets
		? (u: string) => redactSecretPathValues(u, pathValues)
		: undefined;
	// Merge store-injected secret headers with literal auth.headers. oauth2
	// carries no literal headers (the resolved Bearer token arrives via
	// opts.authHeaders).
	const literalHeaders =
		guide.auth.kind === "oauth2" ? undefined : guide.auth.headers;
	const extraHeaders = { ...literalHeaders, ...opts?.authHeaders };
	return {
		fillParams,
		queryParamsForBuild,
		pathValues,
		secretParamNames,
		secretParams,
		hasQuerySecret,
		hasPathSecrets,
		redactPathSecret,
		extraHeaders,
	};
}

// ═══════════════════════════════════════════════════════════════════
// restGet
// ═══════════════════════════════════════════════════════════════════

// ═══════════════════════════════════════════════════════════════════
// Store-injected secret auth opts (single source of truth)
// ═══════════════════════════════════════════════════════════════════

/** Store-injected auth opts, forwarded to the executor and reused by the
 *  caller for the output-channel audit (secret scrub). Declared once here;
 *  RestGetOptions / PaginateOptions / resolve-op's AuthOpts extend it so a
 *  producer-side field can't be silently dropped at the executor boundary. */
export interface SecretAuthOpts {
	/** Store-injected secret headers (kind: static-key). Merged with guide.auth.headers. */
	authHeaders?: Record<string, string>;
	/** Lowercased injected header names — stripped on cross-domain redirects. */
	secretHeaderNames?: Set<string>;
	/** Store-injected secret values — scrubbed from error bodies (output-channel audit). */
	secretValues?: string[];
	/** Store-injected secret query params — appended below the agent params map. */
	secretQueryParams?: Record<string, string>;
	/** The injected query-param names — redacted from every surfaced URL. */
	secretQueryParamNames?: Set<string>;
	/** Store-injected path tokens (kind: static-key secretPathRefs) — pathTokenName → resolved value. Fills {name} in the op path below the agent params map. */
	secretPathParams?: Record<string, string>;
}

export interface RestGetOptions extends SecretAuthOpts {
	/** Skip cache — force fresh fetch. */
	fresh?: boolean;
	/** Built-in post-response transform, applied when op.transform === true. */
	transformFn?: TransformFn | undefined;
	/** Guide directory name — transform context domain / helper routing. */
	dirName?: string | undefined;
}

/**
 * Execute a single `restGet` operation.
 *
 * 1. Replaces `{token}` path params from `params`.
 * 2. Assembles query params with defaults and required validation.
 * 3. Checks auth dispatch (`none` / `static-key` realized; `oauth2` rejected at parse).
 * 4. Constructs the full URL.
 * 5. Builds the Accept header.
 * 6. Fetches via the transport layer.
 * 7. Checks HTTP status — raises HelperError on 4xx/5xx.
 * 8. Parses the response based on the operation's `parse` override or
 *    the guide's `responseShape`.
 *
 * @returns The parsed response data and response headers.
 */
export async function restGet(
	apiHost: string,
	operation: Operation,
	params: Record<string, unknown>,
	guide: ApiGuide,
	opts?: RestGetOptions,
): Promise<RestGetResult> {
	const {
		fillParams,
		queryParamsForBuild,
		pathValues,
		secretParamNames,
		secretParams,
		hasQuerySecret,
		hasPathSecrets,
		redactPathSecret,
		extraHeaders,
	} = buildExecutorEnv(guide, params, opts);

	// Steps 1/2: fill path, build query (agent-supplied only — secret
	// param names are excluded, incl. from the passthrough branch).
	const resolvedPath = fillPathStrict(operation.path, fillParams);
	const query = buildQueryParams(
		operation,
		queryParamsForBuild,
		secretParamNames,
	);

	// 3. Auth dispatch.
	checkAuth(guide.auth);

	// 4. Build URL. The fetch uses the raw URL; every surfaced copy
	// (result.url, the URL stored on HelperError.url) is redacted. Path
	// secrets redact via redactSecretPathValues (value replace, raw +
	// both hex forms); they ride the path, not the query, so
	// redactSecretParams alone can't touch them.
	const fetchUrlRaw = buildUrl(
		apiHost,
		resolvedPath,
		{ ...query, ...secretParams },
		operation.params,
	);
	const url = redactSurfacedUrl(fetchUrlRaw, secretParamNames, pathValues);

	// 5. Build Accept header — json/xml shorthands expand; everything
	// else passes through as-is (e.g. application/atom+xml, */*).
	const accept = expandAccept(operation.accept);

	// 6. Fetch. Pass the effective shape's charset as a transport fallback
	//    for servers that omit a Content-Type charset (e.g. legacy Latin-1
	//    APIs); an explicit header charset always wins.
	const shape = operation.parse ?? guide.responseShape;
	const result = await fetchWithOpts(fetchUrlRaw, {
		accept,
		extraHeaders,
		fresh: opts?.fresh,
		fallbackCharset: shape.charset,
		secretHeaderNames: opts?.secretHeaderNames,
		hasQuerySecret,
		secretQueryParamNames: opts?.secretQueryParamNames,
		hasPathSecret: hasPathSecrets,
		redactPathSecret,
	});

	// 7. Check HTTP status before attempting to parse the body.
	// This turns "Invalid JSON response: <?xml..." into a clean
	// "Unexpected HTTP 400: El parámetro fecha..." message. Secret values are
	// scrubbed from the error excerpt so an auth-echo body can't leak a key.
	// The URL stored on the error object is the REDACTED one (computed
	// upstream of checkResponseStatus) so the secret never reaches the error.
	checkResponseStatus({ ...result, url }, opts?.secretValues);

	// 8. Parse response using the effective shape resolved above.
	let data = parseResponse(result.body, shape);

	// 8b. 200-with-error-envelope check (op.errorPath) — post-parse,
	// pre-transform: the error envelope is the raw parsed shape.
	if (operation.errorPath !== undefined) {
		checkErrorEnvelope(data, operation.errorPath, url, opts?.secretValues);
	}

	// 9. Post-response transform (optional). When api-fetch supplied a
	// `transformFn` (op.transform === true), apply it. A throwing transform is
	// caught: keep the raw `data` and carry a `transformWarning` so the caller
	// can surface "⚠ Transform failed:" — the op is NOT disabled, so a
	// subsequent call re-attempts the transform.
	let transformWarning: string | undefined;
	if (opts?.transformFn) {
		try {
			data = opts.transformFn(data, {
				operation: operation.name,
				domain: opts.dirName ?? "",
			});
		} catch (err) {
			transformWarning = err instanceof Error ? err.message : String(err);
		}
	}

	return {
		data,
		headers: result.headers,
		url,
		params: query,
		...(transformWarning === undefined ? {} : { transformWarning }),
		...(result.cached ? { cached: true } : {}),
	};
}

// ═══════════════════════════════════════════════════════════════════
// paginate
// ═══════════════════════════════════════════════════════════════════

export interface PaginateOptions extends SecretAuthOpts {
	/** When true, gather items up to `gatherAllMax`. Default false. */
	gatherAll?: boolean;
	/** Max items to gather when `gatherAll` is true. Overrides guide/operation default. */
	gatherAllMax?: number;
	/** Skip cache — force fresh fetch for each page. */
	fresh?: boolean;
	/** Bypass the nextLink SSRF guard (for testing against local servers). */
	skipSsrfGuard?: boolean;
	/** Built-in per-item transform, applied when op.transform === true. */
	transformFn?: TransformFn | undefined;
	/** Guide directory name — transform context domain / helper routing. */
	dirName?: string | undefined;
}

/**
 * Execute a paginated operation.
 *
 * Supports six pagination styles:
 *  - `offset-limit`: advance the `offset` param by the page size (row offset).
 *  - `nextLink`: follow the `next` URL from each page response.
 *  - `cursor`: pass the cursor from each page to the next.
 *  - `page`: incrementing page number + fixed page size.
 *  - `resumptionToken`: echo an opaque token from each page into a query param.
 *  - `tokenBag`: merge a set of continuation keys from each page into the next request.
 *
 * @returns Accumulated items across all pages.
 */
export async function paginate(
	apiHost: string,
	operation: Operation,
	params: Record<string, unknown>,
	guide: ApiGuide,
	opts?: PaginateOptions,
): Promise<PaginateResult> {
	const pagCfg = operation.pagination ?? guide.pagination;
	if (!pagCfg) {
		throw new HelperError(
			"pagination",
			"Cannot paginate — no pagination config in operation or guide",
			"a pagination block",
			"missing",
			"Add a pagination: block to the operation or the guide's top-level config",
		);
	}

	const gatherAll = opts?.gatherAll ?? false;
	const ceiling =
		opts?.gatherAllMax ?? operation.gatherAllMax ?? guide.gatherAllMax;
	const items: unknown[] = [];
	const failed: unknown[] = [];
	let ceilingHit = false;
	let serverTotal: number | undefined;

	// Resolve effective response shape (op-level parse or guide-level).
	const shape = operation.parse ?? guide.responseShape;
	const accept = expandAccept(operation.accept);

	// Auth dispatch — checked once up front (auth is constant per guide).
	checkAuth(guide.auth);

	const {
		fillParams,
		queryParamsForBuild,
		pathValues,
		secretParamNames,
		secretParams,
		hasQuerySecret,
		hasPathSecrets,
		redactPathSecret,
		extraHeaders,
	} = buildExecutorEnv(guide, params, opts);

	// State for the styles.
	let cursor: string | undefined;
	let nextUrl: string | undefined;
	let page: number | undefined;
	let token: string | undefined;
	let tokenBag: Record<string, string> | undefined;

	const style = pagCfg.style;

	// Seed offset/page params.
	if (style === "offset-limit" || style === "page") {
		// Caller value, else `pagination.base` (where this API's index
		// starts), else the recipe's declared default for this page param,
		// else the style's framework default: 0 for offset-limit (row
		// offsets start at 0) and 1 for page (nearly all page-indexed APIs are
		// 1-based; a rare 0-based page API overrides with
		// `base: 0` or `params: { page: { default: 0 } }`).
		const fallback = style === "page" ? 1 : 0;
		const rawPage =
			params[pagCfg.pageParam!] ??
			pagCfg.base ??
			operation.params[pagCfg.pageParam!]?.default ??
			fallback;
		page = typeof rawPage === "number" ? rawPage : parseInt(String(rawPage), 10);
		if (isNaN(page)) page = fallback;
	}

	// Compute effective params once — used for both per-page building and result transparency.
	// Agent-supplied only (secret param names excluded, incl. passthrough).
	const effectiveParams = buildQueryParams(
		operation,
		queryParamsForBuild,
		secretParamNames,
	);

	// Resolve the effective page size once for the seeding styles — the
	// caller's value, else the op's declared default (both already folded into
	// effectiveParams), else pagCfg.pageSize, else 50. Used for the per-page
	// pageSizeParam set AND the offset-limit advance, so a caller-supplied
	// size is honored end-to-end (the row-offset increment must match the size
	// the server honored, or pages overlap/skip).
	let effectivePageSize: number | undefined;
	if (style === "offset-limit" || style === "page") {
		const rawSize =
			pagCfg.pageSizeParam === undefined
				? undefined
				: (params[pagCfg.pageSizeParam] ?? effectiveParams[pagCfg.pageSizeParam]);
		effectivePageSize =
			rawSize === undefined ? (pagCfg.pageSize ?? 50) : Number(rawSize);
		if (isNaN(effectivePageSize)) effectivePageSize = 50;
	} else if (style === "cursor") {
		// Terminal fallback for cursor is OMIT (server default applies) — no
		// fabricated 50 like offset-limit (which needs a real number to advance
		// row offsets; cursor's position comes from the response cursor). Seed
		// effectiveParams (not pageParams — rebuilt per iteration) so
		// result.params stays honest about what was sent.
		if (
			pagCfg.pageSizeParam !== undefined &&
			pagCfg.pageSize !== undefined &&
			params[pagCfg.pageSizeParam] === undefined &&
			effectiveParams[pagCfg.pageSizeParam] === undefined
		) {
			effectiveParams[pagCfg.pageSizeParam] = String(pagCfg.pageSize);
		}
	}
	const urls: string[] = [];

	while (true) {
		// Build per-page params. Keyed supersession: pagination/tokenBag
		// writes below REPLACE same-named base values via plain overwrite —
		// only genuinely multi-value (listStyle) keys hold arrays, and the
		// parse-time collision guard guarantees those never collide with a
		// pagination/tokenBag wire name, so every write here is a scalar.
		const pageParams: Record<string, string | string[]> = {};
		for (const [key, val] of Object.entries(effectiveParams)) {
			pageParams[key] = val;
		}

		if (style === "offset-limit" || style === "page") {
			pageParams[pagCfg.pageParam!] = String(page);
			// effectivePageSize is always resolved (a number) in this branch.
			if (pagCfg.pageSizeParam) {
				pageParams[pagCfg.pageSizeParam] = String(effectivePageSize);
			}
		} else if (style === "cursor" && cursor !== undefined) {
			pageParams[pagCfg.cursorParam!] = cursor;
		} else if (style === "resumptionToken" && token !== undefined) {
			pageParams[pagCfg.tokenParam!] = token;
		} else if (style === "tokenBag" && tokenBag) {
			Object.assign(pageParams, tokenBag);
		}

		// Build URL.
		const resolvedPath = fillPathStrict(operation.path, fillParams);

		let url: string;
		if (style === "nextLink" && nextUrl) {
			// If nextUrl is relative, resolve against apiHost.
			url = nextUrl.startsWith("http")
				? nextUrl
				: new URL(nextUrl, apiHost).toString();
		} else {
			// Append secret query params below the agent-supplied page params.
			url = buildUrl(
				apiHost,
				resolvedPath,
				{ ...pageParams, ...secretParams },
				operation.params,
			);
		}

		// Every surfaced URL (incl. a server-supplied nextUrl that may
		// already carry the secret) is redacted at the capture point — query
		// params by name, path tokens by value.
		urls.push(redactSurfacedUrl(url, secretParamNames, pathValues));

		// NextLink guard — the URL comes from the remote server, so this is
		// the one place SSRF protection is load-bearing. `guardThisFetch`
		// both blocks the nextLink URL itself and turns on redirect-target
		// guarding in the transport layer (a malicious API can 302 a
		// nextLink to an internal host; when auth headers ship, the
		// Authorization header would attach to the redirect).
		const guardThisFetch =
			style === "nextLink" && !!nextUrl && !opts?.skipSsrfGuard;
		if (guardThisFetch) {
			const guard = ssrfGuard(url);
			if (!guard.ok) {
				// The SSRF-block error's errUrl is built separately from the other
				// surfaced URLs — redact it the same way (query by name, path by value).
				const errUrl = redactSurfacedUrl(url, secretParamNames, pathValues);
				throw new HelperError(
					"url",
					`URL blocked during pagination: ${guard.reason}`,
					"a safe, public URL",
					errUrl,
					undefined,
					errUrl,
				);
			}
		}

		// Fetch.
		const result = await fetchWithOpts(url, {
			accept,
			extraHeaders,
			fresh: opts?.fresh,
			guardRedirects: guardThisFetch,
			fallbackCharset: shape.charset,
			secretHeaderNames: opts?.secretHeaderNames,
			hasQuerySecret,
			secretQueryParamNames: opts?.secretQueryParamNames,
			hasPathSecret: hasPathSecrets,
			redactPathSecret,
		});

		// Check HTTP status before attempting to parse. Secret values scrubbed
		// from the error excerpt (output-channel audit). The URL stored on
		// the error object is redacted, computed upstream of checkResponseStatus.
		const redactedPageUrl = redactSurfacedUrl(url, secretParamNames, pathValues);
		checkResponseStatus({ ...result, url: redactedPageUrl }, opts?.secretValues);

		// Parse.
		const data = parseResponse(result.body, shape);

		// 200-with-error-envelope check (op.errorPath) — placement pin: the
		// very next statement after parseResponse, BEFORE the totalCountPath
		// extraction and both exhaustion breaks below. An error page often
		// misses itemsPath entirely, so a check after the breaks never fires
		// (the walk exits with silent items: []).
		if (operation.errorPath !== undefined) {
			checkErrorEnvelope(
				data,
				operation.errorPath,
				redactedPageUrl,
				opts?.secretValues,
			);
		}

		// Extract the server's reported total from the first page that resolves
		// one — before the empty-page break below, so a zero-result page that
		// still carries the count (e.g. `total_count: 0`) surfaces it rather
		// than losing it to the early `break`. Accepts a number directly or a
		// numeric string (coerced via Number()); a page that misses the path
		// leaves it undefined and later pages can still supply it.
		if (serverTotal === undefined && pagCfg.totalCountPath) {
			const raw = resolveJsonPath(data, pagCfg.totalCountPath);
			if (typeof raw === "number" && Number.isFinite(raw)) {
				serverTotal = raw;
			} else if (typeof raw === "string" && raw.trim() !== "") {
				const n = Number(raw);
				if (Number.isFinite(n)) serverTotal = n;
			}
		}

		// Extract items from this page.
		let pageItems = resolveJsonPath(data, pagCfg.itemsPath);
		// Normalize a single XML record (or scalar) into an array so the
		// declared list path always yields an array even with one element
		// (e.g. arXiv max_results=1, PubMed retmax=1 box a single record).
		if (pageItems != null && !Array.isArray(pageItems)) {
			pageItems = [pageItems];
		}
		if (Array.isArray(pageItems)) {
			if (pageItems.length === 0) break; // empty page → exhaustion
			// Apply the post-response transform per item. A throwing transform
			// keeps the raw item in `failed` — no item is dropped. The ceiling is
			// evaluated against total raw items processed (items + failed), so
			// partial failures cannot exceed the ceiling.
			const totalProcessed = items.length + failed.length;
			const remaining = ceiling - totalProcessed;
			const toProcess =
				gatherAll && pageItems.length > remaining
					? pageItems.slice(0, remaining)
					: pageItems;
			for (const item of toProcess) {
				if (opts?.transformFn) {
					try {
						items.push(
							opts.transformFn(item, {
								operation: operation.name,
								domain: opts.dirName ?? "",
							}),
						);
					} catch {
						failed.push(item);
					}
				} else {
					items.push(item);
				}
			}
			if (gatherAll && pageItems.length > remaining) {
				ceilingHit = true;
				break;
			}
			if (!gatherAll) break; // default = single page; gatherAll:true walks on
		} else {
			break; // items path didn't resolve to an array → stop
		}

		// Ceiling check.
		if (gatherAll && items.length + failed.length >= ceiling) {
			ceilingHit = true;
			break;
		}

		// hasMorePath done-flag (style-agnostic).
		// Sits after the ceiling checks so ceilingHit still wins when both fire
		// on the same page. The undefined carve-out is fail-open: a missing or
		// typo'd path never stops the walk (pre-existing exhaustion semantics
		// apply); a RESOLVED falsy value stops cleanly. Plain truthiness, no
		// coercion — the string "false" advances by design.
		if (pagCfg.hasMorePath !== undefined) {
			const v = resolveJsonPath(data, pagCfg.hasMorePath);
			if (v !== undefined && !v) break;
		}

		// Determine next page.
		const advanced = advancePagination(
			style,
			pagCfg,
			data,
			page,
			effectivePageSize,
		);

		if (!advanced) break;

		page = advanced.page;
		cursor = advanced.cursor;
		nextUrl = advanced.nextUrl;
		token = advanced.token;
		tokenBag = advanced.tokenBag;
	}

	return {
		items,
		totalFetched: items.length + failed.length,
		...(failed.length > 0 ? { failedItems: failed } : {}),
		...(serverTotal === undefined ? {} : { serverTotal }),
		ceilingHit,
		urls,
		pages: urls.length,
		params: effectiveParams,
	};
}

// ═══════════════════════════════════════════════════════════════════
// Pagination advance logic
// ═══════════════════════════════════════════════════════════════════

interface PaginationState {
	page?: number;
	cursor?: string;
	nextUrl?: string;
	token?: string;
	tokenBag?: Record<string, string>;
}

function advancePagination(
	style: string,
	cfg: NonNullable<Operation["pagination"]>,
	data: unknown,
	prevPage?: number,
	effectivePageSize?: number,
): PaginationState | null {
	if (style === "offset-limit") {
		// Row-offset semantics: the API skips `offset` items, so the next page
		// must advance by the effective page size (caller value → op default →
		// cfg.pageSize → 50) — the size actually sent — not by 1 (a +1 advance
		// re-reads the same rows and overlaps pages) and not by a stale
		// pageSize (overlaps/skips when the caller overrides the size). APIs
		// whose param is a true page index use style: page, which keeps the +1
		// advance.
		return {
			page: (prevPage ?? 0) + (effectivePageSize ?? cfg.pageSize ?? 50),
		};
	}

	if (style === "page") {
		return { page: (prevPage ?? 0) + 1 };
	}

	if (style === "nextLink") {
		const next = resolveJsonPath(data, cfg.nextLinkPath!);
		if (!next || typeof next !== "string") return null;
		return { nextUrl: next };
	}

	if (style === "cursor") {
		// Numeric cursors coerce to strings (tokenBag-style) BEFORE the
		// type/falsy check — a numeric 0 must advance like string "0", not die
		// at !next. nextLink stays string-strict (a numeric "next URL" is
		// garbage); booleans/objects don't coerce.
		const raw = resolveJsonPath(data, cfg.cursorPath!);
		const next = typeof raw === "number" ? String(raw) : raw;
		if (!next || typeof next !== "string") return null;
		return { cursor: next };
	}

	if (style === "resumptionToken") {
		// No !next guard here by design — "" and missing are exhaustion.
		const raw = resolveJsonPath(data, cfg.tokenPath!);
		const t = typeof raw === "number" ? String(raw) : raw; // coercion, before the check
		if (typeof t !== "string" || t === "") return null;
		return { token: t };
	}

	if (style === "tokenBag") {
		// ponytail: one resolveJsonPath call per continuation key — O(keys) per
		// page, keys ≈ 1–2 in practice. Fine. Revisit if an API emits a 50-key bag.
		// The path may be nested (e.g. "continue.rccontinue") but the param name
		// resent to the API is the last segment ("rccontinue") — matching the
		// key names inside the continuation object.
		const collected: Record<string, string> = {};
		for (const key of cfg.continuationParams ?? []) {
			const v = resolveJsonPath(data, key);
			if (v === undefined || v === null) continue;
			// Wire param = wireParamName (last dot segment, bracket dress
			// stripped — shared with the parser's listStyle collision guard so
			// the derivation exists once).
			collected[wireParamName(key)] = String(v);
		}
		return Object.keys(collected).length > 0 ? { tokenBag: collected } : null;
	}

	return null;
}