/*
RFC-1165 NLWeb /ask — framework-free core. Parses the NLWeb v0.55 request,
resolves prefer/accept-language, reuses the search retrieval seam (embedding +
Vectorize via injected ports), and assembles a deterministic extractive answer
(ADR-0097): verbatim chunk excerpts with [n] citation markers — no LLM text
generation at request time. schema_object per result is fetched from the
RFC-1164 .jsonld mirror via an injected fetchMirror port.
Determinism: the response is a pure function of (request, search manifest,
index state) — no timestamps, no randomness. Identical requests on a frozen
index produce byte-identical bodies (DNA-58 runtime analogue). query_id is a
sha256 of (site, lang, query text) — deterministic, never a UUID.
Honesty: unsupported prefer values are explicit failures (reason strings),
never silent degradation (RFC-1113 discipline); the no-answer path returns
total_results: 0 with a fixed localized sentence — never a synthesized answer.
- No astro/HTTP imports — this module is vitest-loadable pure logic;
the adapter in ./astro.ts wires env bindings, ASSETS fetch, and CORS.
- No index management or warmup — see agent-search-index.ts / agent-search-warm.ts.
- No streaming — NLWeb streaming: false only in v1 (RFC-1165).
- RFC-1165: created — parseNlwebRequest, resolveAcceptLanguage, nlwebQueryId,
runNlwebAsk (extractive assembly + mirror fetchMirror + no-answer table),
NLWEB_ constants, NlwebAskRequest/Response types.
*/
import {
SEARCH_DEFAULT_TOP_K,
SEARCH_EMBEDDING_MODEL,
type AgentSurfaceManifest,
type SearchChunk,
type SearchManifest,
} from "@warpgogol/werkstatt-shared/agent";
import type { AgentSearchPorts } from "./search-route.ts";
// ---------------------------------------------------------------------------
// Contracts
// ---------------------------------------------------------------------------
/** Pinned NLWeb specification version implemented here (external contract). */
export const NLWEB_VERSION = "0.55";
/** NLWeb modes/formats served by v1 — everything else is an honest failure. */
export const NLWEB_SUPPORTED = {
responseFormat: "conversational_search",
mode: "list",
} as const;
/** Minimum Vectorize score for a chunk to ground an answer (below → no-answer). */
export const NLWEB_MIN_SCORE = 0.5;
/** How many top chunks feed the extractive answer string. */
export const NLWEB_ANSWER_TOP_N = 3;
/** Per-chunk excerpt budget for the answer string and result description. */
export const NLWEB_EXCERPT_MAX = 280;
export interface NlwebAskRequest {
query: {
text: string;
/** Optional; MUST equal the served site's manifest `site` value. */
site?: string;
lang?: string;
topK?: number;
};
prefer?: {
streaming?: boolean;
response_format?: string;
mode?: string;
"accept-language"?: string;
};
meta?: Record;
}
/** One result item — NLWeb-conformant typed grounding entry. */
export interface NlwebResultItem {
"@type": string;
url: string;
name: string;
site: string;
score: number;
description: string;
schema_object?: { "@type": string; "@graph"?: unknown[] };
}
export interface NlwebAskResponse {
_meta: {
version: string;
query_id: string;
response_type: "answer" | "failure";
/** Echoed negotiated format — RFC-1165 _meta contract. */
response_format: "conversational_search";
site: string;
lang: string;
mode: string;
};
total_results: number;
results: NlwebResultItem[];
answer: string;
content?: NlwebResultItem[];
/** Present only on failure — machine-readable reason. */
reason?: string;
}
/** Result of a parsed request — either the request or an HTTP-mappable failure. */
export type NlwebParseResult =
{ ok: true; request: NlwebAskRequest } | { ok: false; status: number; reason: string };
/** Ports the adapter injects — the search route's seam plus the mirror fetch. */
export interface NlwebAskPorts extends AgentSearchPorts {
/**
* Fetch a page's `.jsonld` mirror (RFC-1164) by served path — the adapter
* resolves it via env.ASSETS.fetch; in tests a stub map suffices.
*/
fetchMirror: (mirrorPath: string) => Promise;
}
// ---------------------------------------------------------------------------
// Pure helpers
// ---------------------------------------------------------------------------
/**
* Deterministic query_id — sha256(site\nlang\ntext) truncated to 16 hex chars
* via Web Crypto (works in Workers and Node ≥19 without node:crypto).
*/
export async function nlwebQueryId(site: string, lang: string, text: string): Promise {
const bytes = new TextEncoder().encode(`${site}\n${lang}\n${text}`);
const digest = await globalThis.crypto.subtle.digest("SHA-256", bytes);
return [...new Uint8Array(digest)]
.map((b) => b.toString(16).padStart(2, "0"))
.join("")
.slice(0, 16);
}
/**
* RFC 9110 quality-list parsing — "uk-UA;q=0.9,de;q=0.8" → best supported
* language: q-sorted, exact match first, then base-subtag prefix match
* (uk-UA → uk). Absent/unparseable/no-match → the site default language.
*/
export function resolveAcceptLanguage(
header: string | undefined,
supported: string[],
defaultLang: string,
): string {
if (!header?.trim()) return defaultLang;
const tokens = header
.split(",")
.map((raw) => {
const [tag, ...params] = raw.trim().split(";");
const q = params.map((p) => p.trim()).find((p) => p.startsWith("q="));
const quality = q ? Number(q.slice(2)) : 1;
return { tag: tag.toLowerCase(), q: Number.isFinite(quality) ? quality : 1 };
})
.filter((t) => t.tag.length > 0)
.sort((a, b) => b.q - a.q || a.tag.localeCompare(b.tag));
for (const { tag } of tokens) {
const exact = supported.find((l) => l.toLowerCase() === tag);
if (exact) return exact;
const base = tag.split("-")[0];
const prefix = supported.find((l) => l.toLowerCase() === base);
if (prefix) return prefix;
}
return defaultLang;
}
/** Parse + validate the NLWeb request body — failures carry the HTTP status. */
export function parseNlwebRequest(body: unknown, manifestSite: string): NlwebParseResult {
if (!body || typeof body !== "object") {
return { ok: false, status: 400, reason: "invalid-request" };
}
const req = body as NlwebAskRequest;
const text = req.query?.text?.trim();
if (!text || text.length < 2 || text.length > 500) {
return { ok: false, status: 400, reason: "invalid-query-text" };
}
if (req.query.site !== undefined && req.query.site !== manifestSite) {
return { ok: false, status: 400, reason: "site-mismatch" };
}
const prefer = req.prefer ?? {};
if (prefer.streaming === true) {
return { ok: false, status: 400, reason: "unsupported-streaming" };
}
if (
prefer.response_format !== undefined &&
prefer.response_format !== NLWEB_SUPPORTED.responseFormat
) {
return { ok: false, status: 400, reason: "unsupported-response_format" };
}
if (prefer.mode !== undefined && prefer.mode !== NLWEB_SUPPORTED.mode) {
return { ok: false, status: 400, reason: "unsupported-mode" };
}
// accept-language tokens outside the supported set are ignored, not errors —
// they resolve to the default language in resolveAcceptLanguage (AC-9 fallback).
return { ok: true, request: { ...req, query: { ...req.query, text } } };
}
/** Deterministic localized no-answer sentences — fixed table, no lookup layers.
* Map (not an object literal): a lang token like "toString" would resolve an
* inherited Object.prototype method on a plain record — a non-string answer. */
const NO_ANSWER_EN = "No matching answer was found in the published content.";
const NO_ANSWER = new Map([
["de", "Es wurde keine passende Antwort im veröffentlichten Inhalt gefunden."],
["uk", "У опублікованому вмісті відповіді не знайдено."],
["en", NO_ANSWER_EN],
]);
export function noAnswerSentence(lang: string): string {
return NO_ANSWER.get(lang) ?? NO_ANSWER_EN;
}
/** Word-boundary excerpt — deterministic slice, no semantic rewriting. */
function excerptOf(text: string): string {
const trimmed = text.trim();
if (trimmed.length <= NLWEB_EXCERPT_MAX) return trimmed;
const slice = trimmed.slice(0, NLWEB_EXCERPT_MAX);
const lastSpace = slice.lastIndexOf(" ");
return `${slice.slice(0, lastSpace > 0 ? lastSpace : NLWEB_EXCERPT_MAX)}…`;
}
/** Derived mirror path for a route — mirrors index.html naming (RFC-1164).
* Canonical twin: mirrorUrlForRoute in
* packages/werkstatt-site/src/domain/share/astro/jsonld-mirror.ts — engine
* cannot import the site plugin, so the transform is duplicated here; keep
* both in lockstep if the .jsonld naming convention changes. */
export function nlwebMirrorPath(route: string): string {
const stripped = route.replace(/\/+$/, "");
return stripped === "" ? "index.jsonld" : `${stripped.replace(/^\//, "")}.jsonld`;
}
/** Collect the first @type of a mirror doc's @graph — for the result's @type. */
function mirrorPrimaryType(doc: unknown): string | undefined {
const graph = (doc as Record | null)?.["@graph"];
if (!Array.isArray(graph)) return undefined;
for (const node of graph) {
const t = (node as Record | null)?.["@type"];
if (typeof t === "string") return t;
if (Array.isArray(t) && typeof t[0] === "string") return t[0];
}
return undefined;
}
/** Extractive answer assembly — verbatim excerpts + [n] citation markers. */
function assembleAnswer(chunks: Array<{ chunk: SearchChunk; score: number }>): string {
return chunks.map(({ chunk }, i) => `[${i + 1}] ${excerptOf(chunk.text)}`).join("\n\n");
}
// ---------------------------------------------------------------------------
// Core — deterministic extractive /ask
// ---------------------------------------------------------------------------
/**
* Run one /ask request. Returns `{ status, body }` — the adapter maps status
* straight to the HTTP response. Every failure path is explicit (reason).
*/
export async function runNlwebAsk(
body: unknown,
manifest: AgentSurfaceManifest,
searchManifest: SearchManifest,
ports: NlwebAskPorts,
): Promise<{ status: number; body: NlwebAskResponse }> {
const site = manifest.site;
const defaultLang = manifest.languages.default;
const supportedLangs = manifest.languages.supported;
const parsed = parseNlwebRequest(body, site);
const meta = {
version: NLWEB_VERSION,
query_id: "",
response_format: NLWEB_SUPPORTED.responseFormat,
site,
lang: defaultLang,
mode: NLWEB_SUPPORTED.mode,
};
if (!parsed.ok) {
return {
status: parsed.status,
body: {
_meta: { ...meta, response_type: "failure" },
total_results: 0,
results: [],
answer: "",
reason: parsed.reason,
},
};
}
const req = parsed.request;
const lang = resolveAcceptLanguage(req.prefer?.["accept-language"], supportedLangs, defaultLang);
const query_id = await nlwebQueryId(site, lang, req.query.text);
const metaResolved = { ...meta, query_id, lang };
// Retrieval — same seam as the search route (embedding + Vectorize).
const embeddingResult = await ports.ai.run(SEARCH_EMBEDDING_MODEL, {
text: [req.query.text],
});
const queryVector = embeddingResult.data?.[0];
if (!queryVector) {
return {
status: 502,
body: {
_meta: { ...metaResolved, response_type: "failure" },
total_results: 0,
results: [],
answer: "",
reason: "embedding-empty",
},
};
}
const topK = req.query.topK ?? SEARCH_DEFAULT_TOP_K;
const vectorizeResult = await ports.searchIndex.query(queryVector, {
topK,
returnMetadata: "all",
});
const chunkById = new Map(searchManifest.chunks.map((c) => [c.id, c]));
// Language gate — resolved language first, default-language fallback (AC-9).
const langMatches = (c: SearchChunk) => c.lang === lang;
const candidates = (vectorizeResult.matches ?? [])
.map((m) => ({ chunk: chunkById.get(m.id), score: m.score }))
.filter((e): e is { chunk: SearchChunk; score: number } => Boolean(e.chunk));
const grounded = candidates.filter((e) => e.score >= NLWEB_MIN_SCORE);
const inLang = grounded.filter((e) => langMatches(e.chunk));
const selected = (inLang.length > 0 ? inLang : grounded).slice(0, NLWEB_ANSWER_TOP_N);
if (selected.length === 0) {
return {
status: 200,
body: {
_meta: { ...metaResolved, response_type: "answer" },
total_results: 0,
results: [],
answer: noAnswerSentence(lang),
},
};
}
// schema_object mirrors — dedupe by route, tolerate missing mirrors.
const mirrors = new Map();
for (const { chunk } of selected) {
const path = nlwebMirrorPath(chunk.url);
if (!mirrors.has(path)) mirrors.set(path, await ports.fetchMirror(path));
}
const results: NlwebResultItem[] = selected.map(({ chunk, score }) => {
const doc = mirrors.get(nlwebMirrorPath(chunk.url)) as
Record | null | undefined;
const primaryType = mirrorPrimaryType(doc) ?? "WebPage";
const graph = Array.isArray(doc?.["@graph"]) ? (doc["@graph"] as unknown[]) : undefined;
return {
"@type": primaryType,
url: chunk.canonicalUrl,
name: chunk.title ?? chunk.heading ?? chunk.canonicalUrl,
site,
score,
description: excerptOf(chunk.text),
...(doc
? { schema_object: { "@type": primaryType, ...(graph ? { "@graph": graph } : {}) } }
: {}),
};
});
return {
status: 200,
body: {
_meta: { ...metaResolved, response_type: "answer" },
total_results: results.length,
results,
answer: assembleAnswer(selected),
content: results,
},
};
}