#!/usr/bin/env python3
"""
gonext_agent_chat.py — streaming agent chat for the gonext local worker.

Reads on stdin:
  {
    "messages": [{"role": "system"|"user"|"assistant", "content": str}, ...],
    "agentBaseURL": str,
    "agentApiKey": str,
    "agentModelId": str,
    "codingBaseURL": str,        # optional: dedicated coding/reasoning model for the
    "codingModelId": str,        #   CodeAgent's tool-use loop; empty = reuse agentModelId
    "searchBaseURL": str,        # optional: dedicated SEARCH model that synthesizes
    "searchModelId": str,        #   web_search results into a cited answer; empty = none
    "tools": ["http_request"],   # v1: only http_request
    "maxSteps": int              # multi-step ReAct budget; default 5
  }

Emits NDJSON lines on stdout:
  {"type": "log",    "text": "..."}  — worker logs to console, not shown in chat
  {"type": "step",   "text": "..."}  — a summary line shown in the <think> area
  {"type": "stream", "text": "..."}  — RAW model token(s) streamed live into <think>
                                       (the agent's Thought/reasoning as it generates)
  {"type": "final",  "text": "..."}  — assistant answer
"""
import contextlib
import json
import random
import re
import sys
import threading
import time
import traceback
import urllib.request
import urllib.error

# Capture stdout before anything can redirect it.  _emit() must always write
# to the real fd-1 so the Node worker's readline loop sees NDJSON even while
# contextlib.redirect_stdout(sys.stderr) is active inside agent.run().
_REAL_STDOUT = sys.stdout


# Playful "still thinking" status words shown on the heartbeat while a model call is
# in flight (see _heartbeat). Loaded once from thinking_words.txt next to this file;
# a random one is picked per tick so the wait feels alive instead of a fixed string.
_THINKING_WORDS: list = []

# Shown INSTEAD of a random word once the model's first token arrives (prompt-eval done,
# tokens now flowing) — the long silent wait is over and output is being produced, so the
# status flips from "still thinking" to "almost done". Ends with "…" so the REPL/web can
# append "(Ns)". The REPL detects the word "almost" in a heartbeat line to switch its own
# local ticker to the same wording (see gonext-repl.mjs).
_STATUS_ALMOST_DONE = "…almost completed thinking…"


def _thinking_word() -> str:
    """A random playful status word (e.g. 'Caffeinating'). Falls back to a small
    built-in list if thinking_words.txt is missing or empty."""
    import os
    import random
    global _THINKING_WORDS
    if not _THINKING_WORDS:
        try:
            path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "thinking_words.txt")
            with open(path, "r", encoding="utf-8") as fh:
                _THINKING_WORDS = [
                    line.strip() for line in fh
                    if line.strip() and not line.lstrip().startswith("#")
                ]
        except OSError:
            _THINKING_WORDS = []
        if not _THINKING_WORDS:
            _THINKING_WORDS = ["Thinking", "Percolating", "Caffeinating", "Cogitating"]
    return random.choice(_THINKING_WORDS)


def _ssl_context():
    import ssl
    # Disable cert verification — this agent runs locally against dev tunnels
    # (gorok, ngrok) whose certs may not chain correctly in Python's SSL store.
    ctx = ssl.create_default_context()
    ctx.check_hostname = False
    ctx.verify_mode = ssl.CERT_NONE
    return ctx


def _http_request_impl(method, url, headers=None, body=None, timeout=25):
    # Merge caller headers on top of sensible defaults.
    merged = {"User-Agent": "gonext-agent/1.0", "Accept": "*/*"}
    if headers:
        merged.update(headers)
    req = urllib.request.Request(url, method=method.upper(), headers=merged)
    data = body.encode() if isinstance(body, str) and body else (body or None)
    try:
        ctx = _ssl_context()
        with urllib.request.urlopen(req, data=data, timeout=timeout, context=ctx) as resp:
            status = resp.status
            raw = resp.read(4096)
            snippet = raw.decode("utf-8", errors="replace")[:2000]
            return f"HTTP {status}\n{snippet}"
    except urllib.error.HTTPError as e:
        raw = e.read(512)
        snippet = raw.decode("utf-8", errors="replace")
        return f"HTTP {e.code} {e.reason}\n{snippet}"
    except Exception as e:  # noqa: BLE001
        return f"Error: {e}"


def _normalize_openai_base(url: str) -> str:
    """Normalize a model-server URL to its OpenAI-compatible /v1 root.

    Users paste whatever their server docs show: an MLX root (http://host:8089), an
    Ollama native endpoint (http://host:11434/api/generate), or a full /v1 URL. Ollama
    serves the OpenAI-compatible API at /v1 on the same host, so strip any native
    /api/... path (including a mistakenly appended /v1 after it) and ensure /v1.
    """
    u = (url or "").strip().rstrip("/")
    if not u:
        return ""
    u = re.sub(r"/api(/(generate|chat|tags|embeddings|embed))?(/v1)?$", "", u,
               flags=re.IGNORECASE)
    return u if u.lower().endswith("/v1") else u + "/v1"


def _html_to_text(html_text, limit=3000):
    """Strip HTML to readable plain text (zero-dep). Used by fetch_url so the weak
    model receives prose instead of raw tags it cannot parse."""
    import html as _html
    text = html_text or ""
    # Drop script/style/head/svg noise wholesale (incl. their content).
    text = re.sub(r"(?is)<(script|style|head|noscript|svg|template)\b.*?</\1>", " ", text)
    # Drop page chrome wholesale too — nav menus, headers, footers, sidebars, forms.
    # Without this, a Wikipedia fetch spends ~1KB of the budget on "Jump to content /
    # Main menu / Donate / Create account…" before any article text, which both starves
    # the model of real content and bloats the per-step context (OOM risk on local MLX).
    text = re.sub(r"(?is)<(nav|header|footer|aside|form|button|menu)\b.*?</\1>", " ", text)
    # Preserve TABLE structure before stripping tags: a cell boundary becomes " | "
    # and a row becomes a newline, so a schedule/fixtures table survives as readable
    # pipe-delimited rows instead of collapsing into a wall of words — a "make a table"
    # task (task #75) is unanswerable if the source table is flattened on the way in.
    text = re.sub(r"(?i)</(td|th)>", " | ", text)
    text = re.sub(r"(?i)</tr>", "\n", text)
    # Turn block-ending tags into newlines so document structure survives stripping.
    text = re.sub(r"(?i)<(br|/p|/div|/li|/tr|/h[1-6]|/section|/article)\s*>", "\n", text)
    # Remove all remaining tags.
    text = re.sub(r"(?s)<[^>]+>", " ", text)
    text = _html.unescape(text)
    # Collapse runs of blank space but keep line breaks for readability.
    lines = [re.sub(r"[ \t]{2,}", " ", ln).strip() for ln in text.splitlines()]
    text = "\n".join(ln for ln in lines if ln)
    if len(text) > limit:
        text = text[:limit].rstrip() + "\n…[truncated]"
    return text.strip()


def _fetch_page_impl(url, timeout=25, max_bytes=300000):
    """GET a URL for reading. Returns (status:int|None, content_type:str, raw:bytes).

    status None => transport error (raw holds the message). Reads more than
    _http_request_impl (which caps at 2000 chars) so a page has enough text to read.
    """
    req = urllib.request.Request(url, method="GET", headers={
        "User-Agent": "gonext-agent/1.0 (local API testing assistant)",
        "Accept": "text/html,application/xhtml+xml,text/plain,*/*",
    })
    try:
        with urllib.request.urlopen(req, timeout=timeout, context=_ssl_context()) as resp:
            ctype = (resp.headers.get("Content-Type") or "").lower()
            return resp.status, ctype, resp.read(max_bytes)
    except urllib.error.HTTPError as e:
        return e.code, "", e.read(512)
    except Exception as e:  # noqa: BLE001
        return None, "", str(e).encode()


def _calc_impl(expression):
    """Safely evaluate an arithmetic expression via an AST whitelist (no eval()).

    Accepts + - * / // % ** and () plus a small function/const allow-list. Normalizes
    common natural phrasings ('15% of 80', '2^10', '×', '÷'). Returns a result string
    or an 'Error: …' string — never raises.
    """
    import ast
    import math
    import operator
    expr = (expression or "").strip()
    if not expr:
        return "Error: empty expression."
    # Natural-language normalizations before parsing.
    expr = re.sub(r"(\d+(?:\.\d+)?)\s*%\s*of\s+", r"(\1/100)*", expr, flags=re.IGNORECASE)
    expr = re.sub(r"(\d+(?:\.\d+)?)\s*%", r"(\1/100)", expr)
    expr = expr.replace("^", "**").replace("×", "*").replace("÷", "/").replace(",", "")

    ops = {
        ast.Add: operator.add, ast.Sub: operator.sub, ast.Mult: operator.mul,
        ast.Div: operator.truediv, ast.FloorDiv: operator.floordiv,
        ast.Mod: operator.mod, ast.Pow: operator.pow,
        ast.USub: operator.neg, ast.UAdd: operator.pos,
    }
    funcs = {
        "sqrt": math.sqrt, "pow": pow, "round": round, "abs": abs,
        "floor": math.floor, "ceil": math.ceil, "log": math.log,
        "log10": math.log10, "min": min, "max": max,
    }
    consts = {"pi": math.pi, "e": math.e, "tau": math.tau}

    def _ev(node):
        if isinstance(node, ast.Expression):
            return _ev(node.body)
        if isinstance(node, ast.Constant):
            if isinstance(node.value, (int, float)) and not isinstance(node.value, bool):
                return node.value
            raise ValueError("only numbers allowed")
        if isinstance(node, ast.BinOp) and type(node.op) in ops:
            return ops[type(node.op)](_ev(node.left), _ev(node.right))
        if isinstance(node, ast.UnaryOp) and type(node.op) in ops:
            return ops[type(node.op)](_ev(node.operand))
        if isinstance(node, ast.Name) and node.id in consts:
            return consts[node.id]
        if (isinstance(node, ast.Call) and isinstance(node.func, ast.Name)
                and node.func.id in funcs and not node.keywords):
            return funcs[node.func.id](*[_ev(a) for a in node.args])
        raise ValueError("unsupported expression")

    try:
        value = _ev(ast.parse(expr, mode="eval"))
    except ZeroDivisionError:
        return "Error: division by zero."
    except Exception as e:  # noqa: BLE001
        return f"Error: cannot evaluate '{expression}' ({e})."
    if isinstance(value, float) and value.is_integer():
        value = int(value)
    return str(value)


def _pdf_reader_available():
    """True if a PDF text-extraction library is importable on this worker."""
    for lib in ("pypdf", "pdfminer.high_level", "PyPDF2"):
        try:
            __import__(lib)
            return True
        except Exception:  # noqa: BLE001
            continue
    return False


# Probed once at import: gates whether extract_text_from_pdf is offered at all, so the
# weak model is never advertised a tool that cannot run on this machine.
_PDF_READ_AVAILABLE = _pdf_reader_available()


def _extract_pdf_text_impl(url, max_chars=6000, timeout=30, max_bytes=15_000_000):
    """Download a PDF at `url` and return its extracted text (truncated), or an
    'Error:'/'not installed' string. Never raises. Tries pypdf then pdfminer then PyPDF2."""
    import io
    try:
        req = urllib.request.Request(url, method="GET", headers={
            "User-Agent": "gonext-agent/1.0 (local API testing assistant)",
            "Accept": "application/pdf,*/*",
        })
        with urllib.request.urlopen(req, timeout=timeout, context=_ssl_context()) as resp:
            raw = resp.read(max_bytes)
    except Exception as e:  # noqa: BLE001
        return f"Error: could not download {url}: {e}"
    if not raw[:5].lstrip().startswith(b"%PDF"):
        return (f"Error: {url} does not look like a PDF (no %PDF header). "
                "Use fetch_url for web pages.")

    text = None
    # 1) pypdf (preferred; also handles the legacy PyPDF2 import name).
    try:
        try:
            from pypdf import PdfReader
        except ImportError:
            from PyPDF2 import PdfReader  # type: ignore
        reader = PdfReader(io.BytesIO(raw))
        parts = []
        for page in reader.pages:
            try:
                parts.append(page.extract_text() or "")
            except Exception:  # noqa: BLE001 — skip unreadable pages, keep the rest
                continue
        text = "\n".join(p for p in parts if p).strip()
    except ImportError:
        text = None
    except Exception as e:  # noqa: BLE001
        return f"Error: failed to read PDF ({e})."

    # 2) pdfminer.six fallback.
    if text is None:
        try:
            from pdfminer.high_level import extract_text as _pm_extract
            text = (_pm_extract(io.BytesIO(raw)) or "").strip()
        except ImportError:
            return ("PDF reading is not installed on this worker. Ask the user to run "
                    "'pip install pypdf' and try again.")
        except Exception as e:  # noqa: BLE001
            return f"Error: failed to read PDF ({e})."

    if not text:
        return f"{url}\n(The PDF has no extractable text — it may be scanned images.)"
    if len(text) > max_chars:
        text = text[:max_chars].rstrip() + "\n…[truncated]"
    return f"{url}\n{text}"


# Affirmative confirmation tokens for send_email's two-step (preview → confirm) flow.
# A send only happens when one of these is in the CURRENT user message AND a prior
# assistant turn already showed a preview — so the weak model cannot self-confirm.
_EMAIL_CONFIRM = re.compile(
    r"\b(confirm(?:ed)?|send it|send the (?:email|mail)|go ahead|approved?|"
    r"yes\s*,?\s*send|do it|please send)\b",
    re.IGNORECASE,
)

# A bare "continue"-style nudge — matched WHOLE-MESSAGE (anchored) so it only fires on a
# short, unambiguous continuation cue, never a longer sentence that happens to contain
# one of these words. See _continuation_pending().
_CONTINUE_CUE = re.compile(
    r"^(please\s+)?(continue|keep\s+going|go\s+on|resume|carry\s+on|try\s+again)"
    r"[\s.!?]*$",
    re.IGNORECASE,
)


def _email_allowed(addr, allow):
    """True if addr matches an allow-list entry — an exact address or a bare domain."""
    a = (addr or "").lower()
    dom = a.split("@")[-1]
    for entry in allow:
        e = entry.lstrip("@")
        if a == entry or dom == e:
            return True
    return False


def _email_fill_template(template, mapping):
    """Fill {{key}} placeholders in a JSON body template with JSON-escaped values, then
    parse. Returns a dict/list on success, or an 'Error: …' string. JSON-escaping keeps
    the body valid even when subject/body contain quotes or newlines."""
    out = template or ""
    for key, val in mapping.items():
        esc = json.dumps("" if val is None else str(val))[1:-1]
        out = out.replace("{{%s}}" % key, esc).replace("{{ %s }}" % key, esc)
    try:
        return json.loads(out)
    except Exception as e:  # noqa: BLE001
        return (f"Error: the email body template is not valid JSON after filling in the "
                f"values ({e}). Fix the template in Settings.")


def _get_json(url, timeout=15):
    """GET a URL and parse the JSON body. Returns dict/list, or None on failure.

    Used by web_search against free no-key APIs (DuckDuckGo, Wikipedia). Wikipedia
    requires a descriptive User-Agent, so we send one.
    """
    req = urllib.request.Request(url, method="GET", headers={
        "User-Agent": "gonext-agent/1.0 (local API testing assistant)",
        "Accept": "application/json",
    })
    try:
        with urllib.request.urlopen(req, timeout=timeout, context=_ssl_context()) as resp:
            return json.loads(resp.read().decode("utf-8", errors="replace"))
    except Exception as e:  # noqa: BLE001
        _log(f"web_search fetch failed {url}: {e}")
        return None


# --- web_search: parallel keyless multi-backend (task #103) ------------------------------
# The old impl called DuckDuckGo's INSTANT-ANSWER API + Wikipedia sequentially. The IA API
# isn't a real web SERP (only curated abstracts for known entities), so general queries came
# back thin/empty and slow (2-3 sequential 15-25s fetches). This runs several KEYLESS
# backends CONCURRENTLY with short per-backend timeouts and an overall deadline, then merges
# + dedupes + ranks by cross-engine agreement. Backends: DuckDuckGo HTML (real results),
# a self-hosted SearXNG metasearch (aggregates Google/Bing/Yahoo/… server-side, keyless — set
# GONEXT_SEARXNG_URL or cfg.searxngUrl), and Wikipedia (encyclopedic). No API keys.
_WEB_SEARCH_TTL = 300  # seconds — short cache so repeated queries in a turn don't re-hit net
_WEB_SEARCH_CACHE: dict = {}  # (query_lower, searxng_url) -> (ts, result_text)
_WEB_UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
           "(KHTML, like Gecko) Chrome/121.0 Safari/537.36")


def _get_text(url, timeout=6, ua=None, headers=None):
    """GET a URL and return the decoded body text ("" on failure). For scraping SERP HTML."""
    h = {"User-Agent": ua or _WEB_UA,
         "Accept": "text/html,application/json;q=0.9,*/*;q=0.8",
         "Accept-Language": "en-US,en;q=0.9"}
    if headers:
        h.update(headers)
    req = urllib.request.Request(url, method="GET", headers=h)
    try:
        with urllib.request.urlopen(req, timeout=timeout, context=_ssl_context()) as resp:
            return resp.read(600_000).decode("utf-8", errors="replace")
    except Exception as e:  # noqa: BLE001
        _log(f"web_search GET failed {url[:80]}: {e}")
        return ""


def _search_strip(s):
    import html as _html
    return _html.unescape(re.sub(r"<[^>]+>", "", s or "")).strip()


def _norm_url(u):
    """Normalize a URL for dedupe (lower host/scheme, drop fragment + trailing slash)."""
    from urllib.parse import urlsplit, urlunsplit
    u = (u or "").strip()
    if not u:
        return ""
    try:
        s = urlsplit(u)
        return urlunsplit((s.scheme.lower(), s.netloc.lower(), s.path.rstrip("/"), "", ""))
    except Exception:  # noqa: BLE001
        return u.rstrip("/")


def _bk_ddg_html(q, timeout=6):
    """DuckDuckGo HTML endpoint → real web results [{title,snippet,url}] (keyless).

    Parses each result as a UNIT (task #104 fix): for every result__a anchor we take the
    snippet that belongs to IT — the result__snippet appearing before the NEXT anchor —
    instead of zipping two independent global lists. The old zip silently misaligned
    snippets onto the wrong title/URL the moment any result lacked a snippet (ads, news,
    or snippet-less rows), so the model read evidence attached to the wrong page.
    """
    from urllib.parse import quote, unquote
    html_body = _get_text(f"https://html.duckduckgo.com/html/?q={quote(q)}", timeout=timeout)
    if not html_body:
        return []
    out = []
    anchors = list(re.finditer(
        r'<a\b[^>]*class="[^"]*result__a[^"]*"[^>]*>.*?</a>', html_body, re.S))
    for idx, am in enumerate(anchors):
        a = am.group(0)
        hm = re.search(r'href="([^"]*)"', a)
        if not hm:
            continue
        href = hm.group(1)
        um = re.search(r'[?&]uddg=([^&]+)', href)
        real = unquote(um.group(1)) if um else href
        if real.startswith("//"):
            real = "https:" + real
        if not real.startswith("http"):
            continue
        title = _search_strip(a)
        if not title:
            continue
        # Snippet = the result__snippet that lies BETWEEN this anchor and the next one
        # (i.e. inside this result's block). Missing → "" with no shift onto the next row.
        seg_end = anchors[idx + 1].start() if idx + 1 < len(anchors) else len(html_body)
        seg = html_body[am.end():seg_end]
        sm = re.search(
            r'<a\b[^>]*class="[^"]*result__snippet[^"]*"[^>]*>(.*?)</a>', seg, re.S)
        snip = _search_strip(sm.group(1)) if sm else ""
        out.append({"title": title[:120], "snippet": snip, "url": real})
    return out


def _bk_searxng(q, base, timeout=6):
    """Self-hosted SearXNG JSON API → {results:[{title,snippet,url}], answers:[str]} (keyless).

    'answers' + 'infoboxes' are SearXNG's aggregated SERP answer box / knowledge panel — a
    DIRECT answer we can surface without reading any page (task #104). Returns the dict shape
    so _web_search_impl can promote answers into the 'Answer:' block."""
    from urllib.parse import quote
    base = (base or "").strip().rstrip("/")
    if not base:
        return {"results": [], "answers": []}
    data = _get_json(f"{base}/search?q={quote(q)}&format=json&safesearch=0", timeout=timeout)
    results, answers = [], []
    if isinstance(data, dict):
        for r in (data.get("results") or []):
            url = (r.get("url") or "").strip()
            if not url:
                continue
            results.append({
                "title": _search_strip(r.get("title") or url)[:120],
                "snippet": _search_strip(r.get("content") or ""),
                "url": url,
            })
        # 'answers' entries are plain strings on some versions, {answer: ...} dicts on others.
        for a in (data.get("answers") or []):
            txt = _search_strip(str((a.get("answer") if isinstance(a, dict) else a) or ""))
            if txt:
                answers.append(txt)
        for ib in (data.get("infoboxes") or []):
            if isinstance(ib, dict):
                txt = _search_strip(str(ib.get("content") or ""))
                if txt:
                    answers.append(txt)
    return {"results": results, "answers": answers}


def _bk_wikipedia(q, timeout=6, n=5):
    """Wikipedia full-text search → encyclopedic candidate pages [{title,snippet,url}]."""
    from urllib.parse import quote
    search = _get_json(
        "https://en.wikipedia.org/w/api.php?action=query&list=search"
        f"&srsearch={quote(q)}&srlimit={n}&format=json",
        timeout=timeout,
    )
    out = []
    try:
        hits = search["query"]["search"]
    except Exception:  # noqa: BLE001
        hits = []
    for hit in hits:
        slug = quote(hit.get("title", "").replace(" ", "_"))
        out.append({
            "title": hit.get("title", ""),
            "snippet": _search_strip(hit.get("snippet", "")),
            "url": f"https://en.wikipedia.org/wiki/{slug}",
        })
    return out


def _bk_ddg_ia_summary(q, timeout=6):
    """DuckDuckGo Instant Answer — a direct abstract for a well-known entity (summary only)."""
    from urllib.parse import quote
    ddg = _get_json(
        f"https://api.duckduckgo.com/?q={quote(q)}&format=json&no_html=1&skip_disambig=1",
        timeout=timeout,
    )
    if isinstance(ddg, dict):
        abstract = (ddg.get("AbstractText") or "").strip()
        if abstract:
            return (abstract[:1200], (ddg.get("AbstractURL") or "").strip())
    return ("", "")


def _condense_for_query(text, query, budget=1500):
    """Keep the query-relevant + STRUCTURAL lines of a page's text, bounded to `budget`
    chars, in original order (task #104). Structural lines — table rows (which _html_to_text
    renders with ' | '), list items, or lines carrying a year / clock time — are ALWAYS kept
    even at zero prose-score, so schedules / fixtures / specs survive instead of being
    summarized away (the flatten-to-summary failure). Model-free: pure text selection."""
    if not text:
        return ""
    terms = {w for w in re.findall(r"\w+", (query or "").lower()) if len(w) > 2}
    lines = [ln.strip() for ln in text.splitlines() if ln.strip()]
    picked = []
    for ln in lines:
        low = ln.lower()
        score = sum(1 for t in terms if t in low)
        structural = (
            "|" in ln
            or bool(re.match(r"^([-*•]|\d+[.)])\s", ln))
            or bool(re.search(r"\b(19|20)\d{2}\b", ln))
            or bool(re.search(r"\d{1,2}:\d{2}", ln))
        )
        if score > 0 or structural:
            picked.append(ln)
    if not picked:  # nothing matched — fall back to the page lead so we return SOMETHING
        picked = lines[:20]
    kept, total = [], 0
    for ln in picked:
        if total + len(ln) + 1 > budget:
            continue
        kept.append(ln)
        total += len(ln) + 1
    return "\n".join(kept)


def _read_pages(ranked, query, per_page_chars=1500, per_timeout=5, overall=6):
    """Read the given result pages IN PARALLEL → {url: condensed_text} (task #104).

    Model-free: HTTP GET (_fetch_page_impl) + _html_to_text + _condense_for_query. Each URL
    is SSRF-guarded (_rag_assert_safe_url) BEFORE fetching — these come from an untrusted
    SERP, and this reads them automatically (not a URL the model explicitly chose), so an
    entry pointing at an internal/link-local host must never be fetched. PDFs/binaries and
    any per-page failure are skipped (the caller falls back to that result's snippet)."""
    from concurrent.futures import ThreadPoolExecutor, as_completed
    import time as _t
    if not ranked:
        return {}

    def _read_one(url):
        try:
            _rag_assert_safe_url(url)
        except Exception:  # noqa: BLE001 — SSRF/scheme refusal → skip this page
            return url, ""
        status, ctype, raw = _fetch_page_impl(url, timeout=per_timeout, max_bytes=300000)
        if status is None or (isinstance(status, int) and status >= 400):
            return url, ""
        if "application/pdf" in ctype or raw[:5].lstrip().startswith(b"%PDF"):
            return url, ""  # binary — snippet fallback (fetch_url can't read PDFs either)
        text = _html_to_text(raw.decode("utf-8", errors="replace"), limit=per_page_chars * 3)
        return url, _condense_for_query(text, query, per_page_chars)

    out = {}
    # Manual shutdown (not `with`): a slow page must not hold the whole search past `overall`
    # via shutdown(wait=True). We collect what finishes within the deadline, then return.
    ex = ThreadPoolExecutor(max_workers=len(ranked))
    futs = {ex.submit(_read_one, e["url"]): e["url"] for e in ranked}
    end = _t.time() + overall
    try:
        for fut in as_completed(futs, timeout=overall):
            try:
                url, txt = fut.result(timeout=max(0.0, end - _t.time()))
            except Exception:  # noqa: BLE001
                continue
            if txt:
                out[url] = txt
    except Exception:  # noqa: BLE001 — overall deadline hit; return what finished
        pass
    finally:
        ex.shutdown(wait=False, cancel_futures=True)
    return out


def _synthesize_search(query, context, base_url, model_id, api_key="local",
                       stall_timeout=20, overall_deadline=45):
    """Search model (#105): ask a configured OpenAI-compatible model to write a concise,
    cited answer FROM the already-read search content — so the (slow) coding model doesn't
    reason over raw pages (the Perplexity pattern).

    STREAMS with a per-READ timeout (httpx `timeout` fires when NO bytes arrive within
    `stall_timeout`) — that is a genuine FIRST-TOKEN deadline during prompt-eval and an
    inter-token watchdog after. This is the fix for the old whole-response `timeout=45`
    that, under MLX contention, let the client hang until it aborted → APITimeoutError +
    an MLX broken-pipe. Returns "" on ANY failure so the caller falls back to the read
    content — never worse than #104."""
    from openai import OpenAI
    import time as _t
    client = OpenAI(base_url=base_url, api_key=api_key or "local", max_retries=0,
                    timeout=stall_timeout)
    parts = []
    t0 = _t.time()
    try:
        stream = client.chat.completions.create(
            model=model_id or "default_model",
            messages=[
                {"role": "system", "content": (
                    "You are a research assistant. Using ONLY the search results provided, "
                    "write a concise, factual answer to the user's query. Cite sources inline "
                    "as [n] using their numbers. Preserve any table / schedule rows verbatim. "
                    "If the results do not contain the answer, say so plainly — never invent "
                    "facts, dates, names, or URLs."
                )},
                {"role": "user",
                 "content": f"Query: {query}\n\nSearch results:\n{context}\n\nAnswer:"},
            ],
            max_tokens=700,
            temperature=0.2,
            stream=True,
        )
        for chunk in stream:
            choices = getattr(chunk, "choices", None)
            delta = getattr(choices[0], "delta", None) if choices else None
            piece = getattr(delta, "content", None) if delta else None
            if piece:
                parts.append(piece)
            if (_t.time() - t0) > overall_deadline:  # generation ran long → keep partial
                _log(f"search-model synthesis: {overall_deadline}s cap hit → using partial")
                try:
                    stream.close()
                except Exception:  # noqa: BLE001
                    pass
                break
        return "".join(parts).strip()
    except Exception as e:  # noqa: BLE001
        _log(f"search-model synthesis failed ({type(e).__name__}: {e}); using read output")
        return "".join(parts).strip()  # partial (if any) is still useful; else caller falls back


def _web_search_impl(query, max_results=5, searxng_url="",
                     search_base_url="", search_model_id=""):
    """Keyless web search that READS for the model (tasks #103 + #104): run DuckDuckGo-HTML
    + SearXNG (if configured) + Wikipedia CONCURRENTLY, merge/dedupe/rank by cross-engine
    agreement, surface any SERP answer box, then (unless GONEXT_SEARCH_READ=0) READ the top
    pages in parallel. If a SEARCH MODEL is configured (search_base_url/search_model_id, #105)
    it then SYNTHESIZES a cited answer from that read content — so the slow coding model gets
    a clean answer, not raw pages. Model-free otherwise; never fabricates.
    """
    import os
    import time as _t
    from concurrent.futures import ThreadPoolExecutor

    q = (query or "").strip()
    if not q:
        return "web_search: empty query."
    searxng = (searxng_url or os.environ.get("GONEXT_SEARXNG_URL") or "").strip()

    # Short TTL cache (identical query within a turn). Include the search model so a
    # synthesized answer isn't served after the search-model config changes.
    ckey = (q.lower(), searxng, search_base_url, search_model_id)
    hit = _WEB_SEARCH_CACHE.get(ckey)
    if hit and (_t.time() - hit[0]) < _WEB_SEARCH_TTL:
        return hit[1]

    per_timeout = 6   # per-backend socket timeout
    overall = 5       # overall deadline; a straggler is abandoned (shutdown is non-blocking)

    # Result-producing backends (run in parallel). Order = tie-break priority in the merge.
    tasks = [("ddg", lambda: _bk_ddg_html(q, per_timeout))]
    if searxng:
        # SearXNG aggregates Wikipedia (and everything else) server-side, so the standalone
        # Wikipedia backend is redundant here — and its full-text API is the slow straggler
        # (~7s) that otherwise dominates the whole search. Only pay for it as a keyless
        # fallback when SearXNG isn't configured.
        tasks.append(("searxng", lambda: _bk_searxng(q, searxng, per_timeout)))
    else:
        tasks.append(("wikipedia", lambda: _bk_wikipedia(q, per_timeout, max_results)))
    tasks.append(("summary", lambda: _bk_ddg_ia_summary(q, per_timeout)))

    results_by = {}
    answers = []           # #104: direct answer-box text (SearXNG answers/infoboxes)
    summary, summary_src = "", ""
    # NOT a `with` block: the context manager's shutdown(wait=True) would block on straggler
    # threads PAST the deadline (a slow SearXNG/engine could add 10s+). We read results up to
    # `overall`, then shutdown(wait=False) so the function returns on time; leftover backend
    # threads finish harmlessly in the background.
    ex = ThreadPoolExecutor(max_workers=len(tasks))
    futs = {ex.submit(fn): name for name, fn in tasks}
    end = _t.time() + overall
    try:
        for fut, name in futs.items():
            try:
                r = fut.result(timeout=max(0.0, end - _t.time()))
            except Exception:  # noqa: BLE001 — a slow/failed backend is skipped
                continue
            if name == "summary":
                summary, summary_src = r
            elif name == "searxng":  # dict shape: {results, answers}
                results_by[name] = r.get("results", [])
                answers.extend(r.get("answers", []))
            else:
                results_by[name] = r
    finally:
        ex.shutdown(wait=False, cancel_futures=True)

    # Merge + dedupe by normalized URL; rank by cross-engine AGREEMENT then first-seen order.
    merged = {}
    order = 0
    for name in ("searxng", "ddg", "wikipedia"):
        for r in results_by.get(name, []):
            nu = _norm_url(r.get("url"))
            if not nu:
                continue
            if nu in merged:
                merged[nu]["hits"] += 1
                if not merged[nu]["snippet"] and r.get("snippet"):
                    merged[nu]["snippet"] = r["snippet"]
            else:
                merged[nu] = {
                    "title": r.get("title") or r["url"], "snippet": r.get("snippet", ""),
                    "url": r["url"], "hits": 1, "order": order,
                }
                order += 1
    ranked = sorted(merged.values(), key=lambda e: (-e["hits"], e["order"]))[:max_results]

    # Backend attribution (diagnostics): which engines actually returned results this call,
    # so the worker log shows whether SearXNG contributed. searxng=off means GONEXT_SEARXNG_URL
    # wasn't seen by python; searxng=on(0) means it was reached but returned nothing.
    _log(
        "web_search backends → "
        f"searxng={'on' if searxng else 'off'}"
        + (f"({len(results_by.get('searxng', []))})" if searxng else "")
        + f" ddg({len(results_by.get('ddg', []))})"
        + (f" wikipedia({len(results_by.get('wikipedia', []))})" if not searxng else "")
        + f" answers({len(answers)}) → {len(ranked)} ranked"
        + (f"  [searxng={searxng}]" if searxng else "")
    )

    if not summary and not ranked and not answers:
        return (
            f"No results found for '{q}'. Tell the user you couldn't find this — "
            "do NOT invent an answer or a URL."
        )

    # #104 read step: unless disabled, READ the top pages inline so the model gets
    # answer-ready content in ONE call (no slow fetch_url round-trips). GONEXT_SEARCH_READ=0
    # is the kill switch → link+snippet behaviour of #103.
    def _env_int(name, default):
        try:
            return max(1, int(os.environ.get(name, "").strip()))
        except (ValueError, TypeError):
            return default
    read_on = os.environ.get("GONEXT_SEARCH_READ", "1").strip().lower() not in ("0", "false", "no")
    read_k = _env_int("GONEXT_SEARCH_READ_K", 3)
    page_chars = _env_int("GONEXT_SEARCH_PAGE_CHARS", 1500)
    page_texts = _read_pages(ranked[:read_k], q, page_chars) if (read_on and ranked) else {}

    parts = []
    # Answer box(es) first — a direct answer often needs no page read at all.
    answer_lines = [f"• {a[:600]}" for a in answers[:3]]
    if summary:
        answer_lines.append(f"• {summary[:600]}" + (f" (source: {summary_src})" if summary_src else ""))
    if answer_lines:
        parts.append("Answer:\n" + "\n".join(answer_lines))

    if ranked:
        if page_texts:
            lines = ["Sources — already READ for you (you usually do NOT need fetch_url; "
                     "call it only to read a specific page in full):"]
        else:
            lines = ["Top pages (call fetch_url on the most relevant to read it in full):"]
        for i, e in enumerate(ranked, 1):
            head = f"[{i}] {e['title']}\n    {e['url']}"
            body = page_texts.get(e["url"]) or (e["snippet"][:200] if e["snippet"] else "")
            lines.append(head + (f"\n{body}" if body else ""))
        parts.append("\n".join(lines))
    out = "\n\n".join(parts)

    # Search model (#105): if configured, have it write the cited answer FROM the read
    # content above, so the coding model receives a clean answer instead of raw pages.
    # Best-effort — any failure keeps the #104 read output (never worse).
    if search_base_url and search_model_id and (answers or ranked):
        synth = _synthesize_search(q, out, search_base_url, search_model_id)
        if synth:
            srcs = "\n".join(f"[{i}] {e['url']}" for i, e in enumerate(ranked, 1))
            out = f"Answer:\n{synth}" + (f"\n\nSources:\n{srcs}" if srcs else "")

    _WEB_SEARCH_CACHE[ckey] = (_t.time(), out)
    return out


class _AgentConfigError(RuntimeError):
    """Deterministic setup error (e.g. wrong model name/URL). Not retryable and not
    worth a plain-reply degrade — the user needs the message itself to fix Settings."""


class _AgentBackendOverloaded(RuntimeError):
    """The coding backend refused every attempt with 429/rate-limit (task #114). Carries
    the user-facing explanation; like a gateway outage it must NOT degrade to a plain
    reply, which would answer from the chat model as if the work had been done."""


def _host_of(url) -> str:
    """Just the hostname of a base URL ('https://api.moonshot.ai/v1' → 'api.moonshot.ai'),
    for user-facing messages. Falls back to the raw string."""
    try:
        from urllib.parse import urlparse
        return urlparse(str(url or "")).netloc or str(url or "")
    except Exception:  # noqa: BLE001
        return str(url or "")


def _list_model_ids(base_url, api_key=""):
    """Return the model ids an OpenAI-compatible server reports at {base_url}/models.

    `base_url` already ends with /v1. Returns [] on any failure so callers can
    fall back.
    """
    url = base_url.rstrip("/") + "/models"
    headers = {"Accept": "application/json"}
    if api_key and api_key != "local":
        headers["Authorization"] = f"Bearer {api_key}"
    req = urllib.request.Request(url, method="GET", headers=headers)
    try:
        with urllib.request.urlopen(req, timeout=10, context=_ssl_context()) as resp:
            payload = json.loads(resp.read().decode("utf-8", errors="replace"))
    except Exception as e:  # noqa: BLE001
        _log(f"model list failed {url}: {e}")
        return []
    data = payload.get("data") if isinstance(payload, dict) else None
    if not isinstance(data, list):
        return []
    return [d["id"].strip() for d in data
            if isinstance(d, dict) and isinstance(d.get("id"), str) and d["id"].strip()]


def _is_ollama_server(base_url):
    """True if the OpenAI-compatible base URL is an Ollama server.

    Ollama's root path answers GET / with the plain-text banner "Ollama is
    running". Used to gate Ollama-only request params (reasoning_effort) that an
    MLX server might reject.
    """
    root = re.sub(r"/v1/?$", "", (base_url or "").rstrip("/"))
    if not root:
        return False
    req = urllib.request.Request(root, headers={"User-Agent": "gonext-agent/1.0"})
    try:
        with urllib.request.urlopen(req, timeout=6, context=_ssl_context()) as resp:
            return b"ollama" in resp.read(200).lower()
    except Exception:  # noqa: BLE001
        return False


# Loopback and RFC1918/link-local ranges — a URL on one of these is a box the user owns,
# which is the whole point: only a REMOTE endpoint can plausibly need an API key.
_LOOPBACK_RE = re.compile(r"^127(?:\.\d{1,3}){3}$")
_PRIVATE_V4_RE = re.compile(
    r"^(?:10(?:\.\d{1,3}){3}"
    r"|192\.168(?:\.\d{1,3}){2}"
    r"|172\.(?:1[6-9]|2\d|3[01])(?:\.\d{1,3}){2}"
    r"|169\.254(?:\.\d{1,3}){2})$"
)
# mDNS / router-assigned search domains. A bare dotless hostname counts too (see below).
_LOCAL_SUFFIXES = (".local", ".localdomain", ".lan", ".home", ".internal")


def _is_local_url(base_url) -> bool:
    """True when the base URL points at this machine or the local network.

    Kept pure (no I/O) so the backend resolution table can be replayed offline. A dotless
    hostname ("mac-studio", "ollama1") is treated as LAN: a public host effectively always
    has a dot, and being wrong in that direction is the safe way — a LAN box is simply
    never told to expect an API key.
    """
    host = _host_of(base_url)
    if "@" in host:
        host = host.rsplit("@", 1)[1]  # strip user:pass@
    if host.startswith("["):
        host = host[1:].split("]", 1)[0]  # [::1]:8080 → ::1
    elif host.count(":") == 1:
        host = host.rsplit(":", 1)[0]  # host:port → host
    host = host.strip().lower().rstrip(".")
    if not host:
        return True  # no host at all → we are not talking to anyone remote
    if host in ("localhost", "0.0.0.0", "::", "::1"):
        return True
    if _LOOPBACK_RE.match(host) or _PRIVATE_V4_RE.match(host):
        return True
    if ":" in host and (host.startswith("fe80:") or host[:2] in ("fc", "fd")):
        return True  # IPv6 link-local / unique-local
    if host.endswith(_LOCAL_SUFFIXES):
        return True
    return "." not in host and ":" not in host


def _resolve_coding_backend(kind, base_url, probe=None):
    """Answer "which coding backend is this?" ONCE. Returns (backend, why) where backend
    is one of "ollama" | "openai" | "local" and `why` is log copy explaining the choice.

    Task #117 (folded into #123). This used to be re-derived in three places from TWO
    different questions — `coding_is_ollama` ("is this an Ollama server?", explicit kind
    else a URL sniff) and `coding_kind == "openai"` ("did the user explicitly pick
    OpenAI-compatible?") — and Auto fell between them: a cloud endpoint left on the
    default Auto sniffed as not-Ollama and was not explicitly openai, so it got NEITHER
    treatment (no streaming, none of the #113 reasoning repairs, and the small local step
    budget). Every branch now keys off this one value.

    Order matters. The Ollama probe runs BEFORE the remote-host rule so a remote Ollama
    (ollama1.gomarsic.cc) still resolves to "ollama" and keeps its own treatment.

        explicit kind                    → that value
        probe says Ollama                → "ollama"
        remote URL (not ours)            → "openai"   ← only a remote endpoint needs a key
        otherwise                        → "local"    (an MLX box on 127.0.0.1)

    Deliberately NOT keyed off "is an API key set": the Settings UI only renders the key
    field once the OpenAI-compatible kind is picked, so on Auto the user cannot supply one
    — a stored key there is only ever a leftover from switching the dropdown back, and
    using it as the test would make the answer depend on that history.

    `probe` is injectable so the resolution table can be replayed offline without network.
    """
    k = (kind or "").strip().lower()
    if k in ("ollama", "openai", "local"):
        return k, "explicit setting"
    if (probe or _is_ollama_server)(base_url):
        return "ollama", "auto: server answers as Ollama"
    if not _is_local_url(base_url):
        return "openai", "auto: remote URL, not Ollama"
    return "local", "auto: local URL, not Ollama"


def _coding_backend_flags(backend):
    """Everything the resolved backend decides, derived in ONE place (task #117 → #123).

    Pure, so the whole resolution → behaviour table can be replayed offline. The callers
    below must not re-derive any of this from `coding_kind` or a URL sniff — that drift is
    the bug this fixes.

      ollama_tweaks — reasoning_effort='none' (Ollama's OpenAI-compat endpoint ignores
                      Qwen3's /no_think but honors this) and the worker's keep-warm ping.
      stream        — Ollama and remote OpenAI-compatible endpoints stream, so a long
                      generation can't trip a proxy idle read-timeout and include_usage
                      keeps output-token counting alive. Local MLX stays non-streaming.
      openai_repairs— the #113 reasoning-model repairs (prose-only nudge, parse-error
                      sanitizing, no-code streak breaker). Never applied to Ollama/MLX.
      budget        — (default, workspace) step budget: a cloud coder is stronger and
                      cheaper-per-step than a small local model, so it gets more room.
    """
    return {
        "ollama_tweaks": backend == "ollama",
        "stream": backend in ("ollama", "openai"),
        "openai_repairs": backend == "openai",
        "budget": (20, 40) if backend == "openai" else (10, 20) if backend == "ollama" else (5, 12),
    }


def _detect_model_id(base_url, api_key=""):
    """Ask an OpenAI-compatible server which model it serves (first reported id).

    Returns "" on any failure so callers can fall back. Used when the user supplies
    a coding-model URL but no model name.
    """
    ids = _list_model_ids(base_url, api_key)
    if not ids:
        return ""
    if len(ids) > 1:
        # Ollama servers list every pulled model; "first" is arbitrary and may be a
        # huge/slow one (seen live: picked gemma4:31b over qwen3:14b).
        _log(f"model detect: server hosts {len(ids)} models {ids} — using "
             f"{ids[0]!r}. Set the coding model NAME in Settings to choose "
             "a specific one.")
    return ids[0]


def _clip(text: str, limit: int = 280) -> str:
    """Trim a one-line preview to ~limit chars WITHOUT cutting mid-word, adding an
    ellipsis so it reads as intentionally shortened instead of broken. Collapses
    internal whitespace/newlines so the panel shows a clean single line."""
    text = " ".join(str(text).split())
    if len(text) <= limit:
        return text
    cut = text[:limit].rsplit(" ", 1)[0].rstrip()
    if not cut:  # single very long token — hard cut as a fallback
        cut = text[:limit].rstrip()
    return f"{cut}…"


def _render_numbered_tool_list(tools) -> str:
    """Render '  N. name(args) — description' lines from the REAL registered smolagents
    tool objects — their actual name/inputs/docstring — instead of a hand-maintained
    duplicate of the same information. This is the single source of truth for what's
    ACTUALLY callable, so the list can never drift out of sync with the tool set (e.g.
    when RAG or a workspace is enabled/disabled) and the numbering is always exactly
    correct, unlike the old manually-incremented counter it replaces. Kept intentionally
    terse — first docstring line only, bare arg names — since the system prompt's own
    `tool.to_code_prompt()` rendering (see _COMPACT_CODE_SYSTEM_PROMPT) already shows the
    full typed signature on every model call; this is just a quick-reference index."""
    lines = []
    for i, t in enumerate(tools, 1):
        args = []
        for pname, pinfo in (getattr(t, "inputs", None) or {}).items():
            optional = isinstance(pinfo, dict) and pinfo.get("nullable")
            args.append(f"{pname}=''" if optional and pinfo.get("type") == "string" else pname)
        desc_full = (getattr(t, "description", "") or "").strip()
        desc = desc_full.splitlines()[0] if desc_full else ""
        lines.append(f"  {i}. {t.name}({', '.join(args)}) — {desc}")
    return ("\n".join(lines) + "\n") if lines else ""


def _summarise_step(step_log):
    """Return a short human-readable description of an agent step."""
    tool_calls = getattr(step_log, "tool_calls", None) or []
    observations = getattr(step_log, "observations", None)
    error = getattr(step_log, "error", None)
    step_num = getattr(step_log, "step_number", None)

    parts = []
    for tc in tool_calls:
        name = getattr(tc, "name", "")
        args = getattr(tc, "arguments", None)

        if name == "python_interpreter":
            # smolagents' CodeAgent passes the raw code STRING as `arguments` (NOT a
            # dict — only ToolCallingAgent's structured calls use {"code": ...}). Handle
            # both shapes, else this always fell through to the useless "python_interpreter"
            # label below and never showed what was actually called.
            code = args if isinstance(args, str) else (
                args.get("code", "") if isinstance(args, dict) else ""
            )
            # Show the http_request call if present, else first meaningful line
            m = re.search(r'http_request\s*\(\s*(?:method\s*=\s*)?[\'"]?(\w+)[\'"]?\s*,\s*(?:url\s*=\s*)?[\'"]([^\'"]+)', code)
            if m:
                parts.append(f"HTTP {m.group(1).upper()} {m.group(2)}")
            else:
                first = next(
                    (l.strip() for l in code.splitlines()
                     if l.strip() and not l.strip().startswith("#")),
                    code[:80],
                )
                parts.append(_clip(first, 120))
        else:
            if isinstance(args, dict):
                method = args.get("method", "")
                url = args.get("url", "")
                if method and url:
                    parts.append(f"HTTP {method.upper()} {url}")
                else:
                    parts.append(f"{name}()")
            else:
                parts.append(name or "tool call")

    if observations:
        obs = str(observations).strip()
        # smolagents wraps observations as "Execution logs:\n<print output>\nLast output
        # from code snippet:\n<return value>" — both header lines are pure boilerplate.
        # Without filtering "Last output from code snippet:" too, a call with no print()
        # output (most of our @tool functions just `return`, not print) surfaced that
        # HEADER as the shown text instead of the actual return value on the next line.
        _OBS_BOILERPLATE = ("Execution logs:", "Last output from code snippet:")
        lines = [l for l in obs.splitlines() if l.strip() and l.strip() not in _OBS_BOILERPLATE]
        if lines:
            parts.append(f"→ {_clip(lines[0])}")

    if error:
        err = str(error)
        if "Import of" in err and "not allowed" in err:
            parts.append("→ (import blocked — using http_request tool instead)")
        elif "reached max steps" in err.lower() or "max steps" in err.lower():
            # NOT a user-facing failure: hitting the step budget triggers our
            # provide_final_answer override, which ALWAYS delivers a synthesized answer
            # (a rendered PDF, the last tool result, or a plain reply). Surfacing the raw
            # "Error: Reached max steps." made a successful run look failed (task #10).
            parts.append("→ Wrapping up (reached step budget)…")
        else:
            parts.append(f"→ Error: {_clip(err, 160)}")

    # No numeric "Step N:" prefix — show only the semantic action.
    return (" | ".join(parts) if parts else "thinking…")


# Keywords that strongly indicate the user wants to make an HTTP/network request,
# regardless of what the final output is (time, text, data, etc.).
_AGENT_KEYWORDS = re.compile(
    r"\b("
    r"request|fetch|call|hit|ping|curl|wget|GET|POST|PUT|DELETE|PATCH"
    r"|api|endpoint|url|http|https"
    r"|external\s+source|external\s+api|external\s+service"
    r"|web\s+service|rest\s+api|rest\s+call"
    r"|download|scrape|crawl|zip|unzip|\.zip|rag|index\s+the|knowledge\s+base|summari[sz]e"
    r"|workspace|fix\s+(?:a\s+|the\s+|this\s+)?bug|refactor|codebase|source\s+code|repo(?:sitory)?"
    r"|modify\s+(?:the\s+)?code|change\s+(?:the\s+)?code|run\s+(?:the\s+)?tests?"
    r"|search|find|look\s*up|lookup|weather|news|latest|current|today|tonight"
    r"|date|time|what\s+day|what\s+time"
    r"|read\s+(?:this\s+|that\s+|the\s+)?(?:page|article|url|link)|open\s+(?:this\s+|that\s+|the\s+)?(?:url|link|page)|contents?\s+of"
    r"|calculate|compute|convert|multiply|divide|percentage|percent|average|how\s+much\s+is"
    r"|email|e-mail|send\s+(?:an?\s+)?(?:email|mail)|mail\s+to"
    r"|pdf|\.pdf|create\s+a\s+pdf|generate\s+a\s+pdf|make\s+a\s+pdf|export.*pdf|make\s+a\s+document"
    r")\b",
    re.IGNORECASE,
)

# File/folder ACTION requests ("create a folder abc", "delete test.txt", "rename X to
# Y", "list the files") are agent work whenever a workspace is registered. The generic
# keyword list above cannot carry bare verbs like "create"/"delete" (they'd misroute
# ordinary chat for non-workspace users), and the weak model classifier reliably
# misreads these imperatives as how-to questions — answering with INSTRUCTIONS (a
# mkdir tutorial) instead of DOING the action (the reported bug, task #40). Verb and
# object noun must appear within one clause (no sentence punctuation between them).
_WS_ACTION_RE = re.compile(
    r"\b(?:create|make|add|new|delete|remove|rename|move|copy|list|show)\b"
    r"[^.?!]{0,60}?"
    r"\b(?:folders?|director(?:y|ies)|dirs?|files?|subfolders?)\b"
    # Run/verify actions on the project itself ("start and test the project", "run
    # the app", "build it and run the tests") — second reported miss (task #40):
    # none of these words were keywords anywhere, so "help me start and test the
    # project" got a step-by-step npm tutorial instead of the agent running them.
    r"|\b(?:run|start|launch|build|test|install|compile|verify|stop|kill|restart)\b"
    r"[^.?!]{0,60}?"
    r"\b(?:project|app|application|server|site|website|tests?|build|dependenc(?:y|ies)|packages?)\b"
    r"|\bmkdir\b",
    re.IGNORECASE,
)
# Genuine how-to QUESTIONS about files/folders ("how do I create a folder in
# Windows?") must stay plain chat — the user wants an explanation, not the action.
# Deliberately FIRST-PERSON only ("can I…"): "can/could YOU create a folder" is a
# polite request for the agent to DO it, not a how-to question.
_HOWTO_PREFIX_RE = re.compile(
    r"^(?:how|what|why|when|where|which|explain|can\s+i|could\s+i|should\s+i|is\s+it|does)\b",
    re.IGNORECASE,
)

# Pure conversational openers/closers that never need a tool — a greeting, a thank-you, a
# "who are you". Matching the WHOLE message (anchored) means "hi, fetch https://…" won't
# match. These skip BOTH the model router call AND the heavy tool preamble → an instant
# small-prompt plain reply, instead of a multi-minute prompt-eval on the coding model.
_TRIVIAL_CHAT = re.compile(
    r"^\s*(?:"
    r"h(?:i+|ello+|ey+|iya|owdy)|yo|sup|wass?up|"
    r"good\s*(?:morning|afternoon|evening|night|day)|greetings|"
    r"thanks?(?:\s*you)?|thank\s*you|thx|ty|cheers|much\s*appreciated|"
    r"bye+|goodbye|see\s*(?:ya|you)|cya|later|good\s*night|"
    r"ok(?:ay)?|k|cool|nice|great|awesome|perfect|sounds\s*good|got\s*it|"
    r"how\s*(?:are|r)\s*(?:you|u|ya)(?:\s*doing)?|how'?s\s*it\s*going|what'?s\s*up|"
    r"who\s*(?:are|r)\s*(?:you|u)|what\s*(?:can|do)\s*you\s*do|what\s*are\s*you|"
    r"test(?:ing)?|ping"
    r")"
    r"(?:\s+(?:there|everyone|all|bot|assistant|gonext|man|dude|buddy))?"
    r"[\s!.?,'\"]*$",
    re.IGNORECASE,
)


def _is_trivial_chat(text: str) -> bool:
    """True for a short, pure-conversational message (greeting, thanks, 'who are you')
    that clearly needs no tools — so we can answer directly and fast."""
    t = (text or "").strip()
    if not t or len(t) > 60:
        return False
    return bool(_TRIVIAL_CHAT.match(t))


# The greeting/smalltalk set MINUS the coding-ambiguous words: in a registered
# workspace (the `gonext` terminal), a bare "test"/"testing"/"ping" leans toward a
# command ("run the tests"), so it should reach the agent, not be answered as chat.
# This is the ONLY escape from workspace-mode's default-to-agent routing, so it must
# stay tight — anything not obviously conversational is better handled as a task.
_GREETING_SMALLTALK = re.compile(
    r"^\s*(?:"
    r"h(?:i+|ello+|ey+|iya|owdy)|yo|sup|wass?up|"
    r"good\s*(?:morning|afternoon|evening|night|day)|greetings|"
    r"thanks?(?:\s*you)?|thank\s*you|thx|ty|cheers|much\s*appreciated|"
    r"bye+|goodbye|see\s*(?:ya|you)|cya|later|good\s*night|"
    r"ok(?:ay)?|k|cool|nice|great|awesome|perfect|sounds\s*good|got\s*it|"
    r"how\s*(?:are|r)\s*(?:you|u|ya)(?:\s*doing)?|how'?s\s*it\s*going|what'?s\s*up|"
    r"who\s*(?:are|r)\s*(?:you|u)|what\s*(?:can|do)\s*you\s*do|what\s*are\s*you"
    r")"
    r"(?:\s+(?:there|everyone|all|bot|assistant|gonext|man|dude|buddy))?"
    r"[\s!.?,'\"]*$",
    re.IGNORECASE,
)


def _is_greeting_smalltalk(text: str) -> bool:
    """Tighter than _is_trivial_chat (no test/ping) — the single chat escape from
    workspace-mode default-to-agent routing."""
    t = (text or "").strip()
    if not t or len(t) > 60:
        return False
    return bool(_GREETING_SMALLTALK.match(t))


# A weak/confused coding model sometimes "thinks out loud" in plain prose instead of
# emitting the next tool call — e.g. a rambling, self-correcting scratchpad ("Wait,
# actually... Let's try... Actually, let's just...") that can run thousands of chars.
# The prose-final-answer heuristic below (long text with list-like structure) can't tell
# this apart from a genuine, complete answer on structure alone — both use headings/
# numbered lists. These two signals catch the scratchpad case specifically: it either
# self-labels as "Thought"/"Thinking Process" up front (a real answer doesn't open by
# naming itself as the model's own reasoning), or it's dense with self-correction
# language a finished answer wouldn't contain.
_SCRATCHPAD_START_RE = re.compile(r"^\s*thought\b\s*[:\n]", re.I)
_DELIBERATION_RE = re.compile(
    r"\b(wait,|actually,|let'?s (try|just|use|list|see|go)|"
    r"i (?:need to|should) (?:check|verify|confirm))\b",
    re.I,
)


def _looks_like_scratchpad(stripped: str) -> bool:
    """True if `stripped` reads like unresolved internal deliberation rather than a
    finished, user-facing answer — see module comment above for the two signals used."""
    if _SCRATCHPAD_START_RE.match(stripped):
        return True
    return len(_DELIBERATION_RE.findall(stripped)) >= 3


# Compact replacement for smolagents' default CodeAgent system prompt. The stock template
# is ~9.9k chars, dominated by ~6k chars of GENERIC few-shot examples (image captions,
# Wikipedia, etc.) that are irrelevant to our HTTP/file tools and cost real prompt-eval
# time every step on a slow coding model. This keeps everything FUNCTIONALLY required —
# the Thought→Code→Observation loop, the code-blob format + closing tag, the {{tools}}
# rendering (so exact signatures survive), authorized imports, custom_instructions — plus
# ONE short worked example, and trims the rest. Jinja vars must match the stock template.
_COMPACT_CODE_SYSTEM_PROMPT = (
    "You are an expert assistant who solves the task by writing Python code that calls "
    "tools. You work in a loop of Thought → Code → Observation.\n\n"
    "At EACH step:\n"
    "- Write one 'Thought:' line: what you'll do and which tool.\n"
    "- Then a code block that OPENS with {{code_block_opening_tag}} and CLOSES with "
    "{{code_block_closing_tag}}, containing simple Python that calls ONE tool and print()s "
    "anything you need next.\n"
    "- You then receive that tool's 'Observation:'. Use it to decide the next step. When "
    "you have the answer, call final_answer(answer) inside a code block.\n\n"
    "Example:\n"
    "Thought: I'll read the page the user gave.\n"
    "{{code_block_opening_tag}}\n"
    'text = fetch_url("https://example.com")\n'
    "print(text)\n"
    "{{code_block_closing_tag}}\n"
    "Observation: \"Example Domain … \"\n"
    "Thought: I have the content, so I'll answer.\n"
    "{{code_block_opening_tag}}\n"
    'final_answer("The page is Example Domain, a placeholder site.")\n'
    "{{code_block_closing_tag}}\n\n"
    "You have access to these tools — call them as plain Python functions with the exact "
    "signatures shown:\n"
    "{{code_block_opening_tag}}\n"
    "{%- for tool in tools.values() %}\n"
    "{{ tool.to_code_prompt() }}\n"
    "{% endfor %}\n"
    "{{code_block_closing_tag}}\n"
    "{%- if managed_agents and managed_agents.values() | list %}\n"
    "You can also delegate to team members by calling them like a tool with a 'task' "
    "string argument:\n"
    "{{code_block_opening_tag}}\n"
    "{%- for agent in managed_agents.values() %}\n"
    "def {{ agent.name }}(task: str) -> str:\n"
    '    """{{ agent.description }}"""\n'
    "{% endfor %}\n"
    "{{code_block_closing_tag}}\n"
    "{%- endif %}\n\n"
    "Rules:\n"
    "1. ALWAYS write a 'Thought:' line, then a code block opening with "
    "{{code_block_opening_tag}} and closing with {{code_block_closing_tag}} — or you fail.\n"
    "2. Pass tool arguments directly: calculate(expression=\"2+2\"), NOT as a dict.\n"
    "3. Use only variables you have defined; state persists between steps.\n"
    "4. Never re-run a tool call with the exact same parameters.\n"
    "5. Don't name a variable after a tool (e.g. 'final_answer').\n"
    "6. Imports are allowed ONLY from: {{authorized_imports}}\n"
    "7. Don't give up — you are in charge of solving the task.\n"
    "{%- if custom_instructions %}\n"
    "{{custom_instructions}}\n"
    "{%- endif %}\n\n"
    "Now Begin!"
)


def _is_model_not_found_err(e) -> bool:
    """True when a chat completion failed because the server didn't recognize the model
    name (mlx_lm.server treats an unknown name as an HF repo to fetch → 'Repository Not
    Found' / 401 / 404), as opposed to a real network/auth failure."""
    s = str(e).lower()
    return ("repository not found" in s or "not found for url" in s
            or "repo_id" in s or "does not exist" in s
            or ("404" in s and "model" in s))


def _chat_create(client, **kwargs):
    """chat.completions.create with an mlx_lm safety net: if the server rejects the model
    NAME as an unknown repo, retry once with 'default_model' — mlx_lm.server maps that
    sentinel to whatever model it was launched with (`--model <path>`), so a server
    started as e.g. `mlx_lm.server --model ~/mlx-models/Qwen3-14B-4bit` answers even when
    we only know the bare name. Non-mlx servers surface their ORIGINAL error unchanged."""
    try:
        return client.chat.completions.create(**kwargs)
    except Exception as e:  # noqa: BLE001
        if kwargs.get("model") != "default_model" and _is_model_not_found_err(e):
            _log(f"model {kwargs.get('model')!r} not found on server; retrying as "
                 "'default_model' (mlx_lm preloaded model)")
            try:
                kwargs2 = dict(kwargs)
                kwargs2["model"] = "default_model"
                return client.chat.completions.create(**kwargs2)
            except Exception:  # noqa: BLE001
                raise e  # surface the clearer original error, not the retry's
        raise


def _chat_create_stream(client, on_delta, **kwargs) -> str:
    """Like _chat_create but STREAMS the completion: on_delta(text) is called for each
    content piece AS IT ARRIVES (so the caller can emit it live), and the full assembled
    text is returned once the stream ends. Applies the same mlx_lm model-not-found retry
    as _chat_create — that failure always happens at/near the very first network
    round-trip (before any token streams), so a clean retry from scratch never risks
    re-emitting a partial answer twice."""
    kwargs = dict(kwargs)
    kwargs["stream"] = True

    def _run(kw):
        stream = client.chat.completions.create(**kw)
        parts = []
        for chunk in stream:
            choices = getattr(chunk, "choices", None)
            delta = getattr(choices[0], "delta", None) if choices else None
            piece = getattr(delta, "content", None) if delta else None
            if piece:
                parts.append(piece)
                on_delta(piece)
        return "".join(parts)

    try:
        return _run(kwargs)
    except Exception as e:  # noqa: BLE001
        if kwargs.get("model") != "default_model" and _is_model_not_found_err(e):
            _log(f"model {kwargs.get('model')!r} not found on server; retrying as "
                 "'default_model' (mlx_lm preloaded model)")
            try:
                kwargs2 = dict(kwargs)
                kwargs2["model"] = "default_model"
                return _run(kwargs2)
            except Exception:  # noqa: BLE001
                raise e  # surface the clearer original error, not the retry's
        raise


def _route(task_text: str, base_url: str, api_key: str, model_id: str) -> bool:
    """Decide if the task needs the HTTP agent (True) or a plain chat reply (False).

    Fast-path: if the user explicitly mentions network/request keywords → agent.
    Otherwise: ask the model to classify.
    """
    # Show the routing stage in the web Thinking panel.
    _emit({"type": "step", "text": "Routing your request…"})

    # Fast-path: explicit HTTP/network intent overrides the model classifier.
    if _AGENT_KEYWORDS.search(task_text):
        _log(f"router → YES (keyword match)")
        _emit({"type": "step", "text": "→ Agent mode (needs tools)"})
        return True

    try:
        from openai import OpenAI
        client = OpenAI(base_url=base_url, api_key=api_key or "local",
                        max_retries=0, timeout=20)
        # The classifier's question must reflect the capabilities that ACTUALLY exist
        # this run — with a workspace registered, acting on files/code/commands is
        # agent work too, and pronoun phrasings ("test it") that the deterministic
        # keyword gates can't safely match land here. Without this clause the prompt
        # was HTTP-only (it predates the workspace tools), so NO was the classifier's
        # CORRECT answer to the wrong question for every workspace action.
        ws_clause = (
            "\nThe user also has a code workspace registered, so answer YES as well "
            "when the task asks to read, edit, create, delete, or list files or "
            "folders, run tests/builds/commands, or start, stop, or restart the "
            "project or its server — even when phrased with pronouns like 'test it' "
            "or 'run it'."
        ) if _WS_ROOTS else ""
        ws_q = ", files, code, or commands" if _WS_ROOTS else ""
        resp = _chat_create(
            client,
            model=model_id,
            messages=[
                {"role": "system", "content": (
                    "You are a task classifier. Reply YES or NO only, no punctuation.\n"
                    "Answer YES if the task requires fetching data from an external network source "
                    "(URL, API, website, remote server), a web search / factual lookup, or the "
                    "current date or time."
                    + ws_clause +
                    "\nAnswer NO only if it is pure conversation, opinion, or simple text the "
                    "assistant can answer directly without looking anything up or touching anything."
                )},
                {"role": "user", "content": (
                    f"Does this task require using tools (network{ws_q})?\n\n"
                    f"Task: {task_text}\n\nYES or NO:"
                )},
            ],
            max_tokens=3,
            temperature=0,
        )
        answer = (resp.choices[0].message.content or "").strip().upper()
        _log(f"router → {answer!r} (model)")
        is_agent = answer.startswith("Y")
        _emit({"type": "step", "text": "→ Agent mode (needs tools)" if is_agent else "→ Chat reply"})
        return is_agent
    except Exception as e:  # noqa: BLE001
        # Classifier unreachable (e.g. the local chat server is down). No agent keyword
        # matched above, so a SHORT message is almost certainly plain conversation — send
        # it to a fast small-prompt reply instead of the heavy tool loop (which would
        # prompt-eval a huge preamble for minutes). Only longer/ambiguous tasks fall back
        # to the agent.
        short = len((task_text or "").split()) <= 12
        if short:
            _log(f"router error: {e} — short message, defaulting to plain chat reply")
            _emit({"type": "step", "text": "→ Chat reply"})
            return False
        _log(f"router error: {e} — defaulting to agent")
        _emit({"type": "step", "text": "→ Agent mode (needs tools)"})
        return True


def _summarize_result(task_text: str, agent_output: str,
                       base_url: str, api_key: str, model_id: str) -> str:
    """Always call the model to turn the raw agent output into a clean reply."""
    _log(f"summarizing agent output ({len(agent_output)} chars)")
    try:
        from openai import OpenAI
        client = OpenAI(base_url=base_url, api_key=api_key or "local",
                        max_retries=0, timeout=30)
        resp = _chat_create(
            client,
            model=model_id,
            messages=[
                {"role": "system", "content": (
                    "You are a helpful assistant. An agent ran HTTP tools to answer the user's "
                    "request. Write a clear, concise reply (1-3 sentences) explaining what was "
                    "found. Do not include raw code, tool names, or error traces."
                )},
                {"role": "user", "content": (
                    f"User asked: {task_text}\n\n"
                    f"Agent result: {agent_output[:2000]}\n\n"
                    "Reply to the user:"
                )},
            ],
            max_tokens=200,
            temperature=0.3,
        )
        summary = (resp.choices[0].message.content or "").strip()
        _log(f"summary: {summary[:120]}")
        return summary or agent_output
    except Exception as e:  # noqa: BLE001
        _log(f"summarize error: {e}")
        return agent_output


def _plain_reply(messages: list, base_url: str, api_key: str, model_id: str,
                 fallback_base_url: str = "", fallback_model_id: str = "") -> str:
    """Plain chat completion using the full conversation history — a SMALL prompt (no tool
    preamble), so it returns fast. STREAMS the answer live via {"type":"answer_stream"}
    deltas as the model generates it (instead of blocking until the whole reply is ready
    and delivering it in one burst) — the worker renders these as the live-updating MAIN
    answer, not the collapsible Thinking panel. If the primary model is unreachable (e.g.
    the local MLX server is down) and a distinct fallback model is given (the coding
    model), retry there so a greeting still gets answered instead of erroring.

    RAISES (does not return a sentinel string) if every target fails or returns no
    content — callers that want a soft degrade should catch this locally and substitute
    their own clean message; letting it propagate uncaught turns into a genuine job
    failure (red error, never persisted to conversation history) instead of a fabricated
    "answer" that would poison every future turn's prompt with raw exception text."""
    _THINK_RE_LOCAL = re.compile(r"<think>.*?</think>", re.DOTALL | re.IGNORECASE)
    chat_messages = [{"role": "system", "content": "You are a helpful assistant."}]
    for m in messages:
        role = m.get("role", "")
        content = m.get("content", "")
        if role not in ("user", "assistant"):
            continue
        if role == "assistant":
            content = _THINK_RE_LOCAL.sub("", content).strip()
            if not content:
                continue
        chat_messages.append({"role": role, "content": content})

    targets = [(base_url, model_id)]
    if fallback_base_url and fallback_model_id and (
        fallback_base_url.rstrip("/") != (base_url or "").rstrip("/")
        or fallback_model_id != model_id
    ):
        targets.append((fallback_base_url, fallback_model_id))

    last_err = None
    for i, (b, mid) in enumerate(targets):
        emitted_here = False

        def _track(piece):
            nonlocal emitted_here
            emitted_here = True
            _emit({"type": "answer_stream", "text": piece})

        try:
            from openai import OpenAI
            client = OpenAI(base_url=b, api_key=api_key or "local",
                            max_retries=0, timeout=60)
            text = _chat_create_stream(
                client, _track,
                model=mid,
                messages=chat_messages,
                temperature=0.7,
                max_tokens=512,
            ).strip()
            if text:
                return text
            # The call SUCCEEDED (no exception) but the model produced NO content at all —
            # seen with a reasoning model given a confusing/degenerate prompt (e.g. a prior
            # turn's history polluted with raw error text). Treat this the same as a
            # failure: fall back to the next target, or surface a clean message, rather
            # than silently returning "" (which the caller can't tell apart from a real,
            # deliberately empty answer).
            last_err = RuntimeError(f"{mid} returned an empty response")
        except Exception as e:  # noqa: BLE001
            last_err = e
        if emitted_here:
            # Part of THIS attempt's answer already streamed to the user — don't
            # silently splice a DIFFERENT model's output after a partial one.
            break
        if i + 1 < len(targets):
            nb, nmid = targets[i + 1]  # the model we're about to try next
            _log(f"plain reply: {mid} @ {b} failed ({last_err}); "
                 f"falling back to {nmid} @ {nb}")
    # Every target failed or returned nothing — raise rather than return a sentinel
    # string. A returned string always looks like a legitimate answer to callers (and
    # gets persisted to conversation history); raising lets it become a genuine job
    # failure instead (see docstring). Kept short/clean (no embedded newlines/HTML — a
    # raw nginx 502 page can be a whole document) for whichever caller logs/surfaces it.
    raise RuntimeError(f"couldn't get a response ({_clip(str(last_err), 160)})")


# A model-server call can fail for two very different reasons, and they deserve
# different handling:
#   • the BACKEND is unavailable — an nginx 502/503/504 (the upstream Ollama/MLX box is
#     down, restarting, still loading the weights, or overloaded past the proxy's read
#     timeout), or the socket refused/timed out. These are INFRA outages: retrying a bit
#     harder can ride out a blip, and if it persists the honest thing is to SAY the
#     coding backend is down — never to hallucinate a generic answer as if we succeeded.
#   • a genuine model/agent error — worth the plain-reply degrade.
# `str(e)` for a proxy error is the raw nginx HTML ("<html>…502 Bad Gateway…"), so match
# on the status text/codes and the common connection-failure phrases.
def _http_status_of(err):
    """The HTTP status an exception carries, or None if it isn't an HTTP error.

    openai-python's APIStatusError exposes `.status_code` (and `.response.status_code`).
    smolagents wraps model exceptions in AgentGenerationError, so the chain is walked —
    otherwise every error reaching the turn-level handler would look status-less and fall
    back to matching on message text."""
    cur, depth = err, 0
    while cur is not None and depth < 6:
        for attr in ("status_code", "http_status", "status"):
            code = getattr(cur, attr, None)
            if isinstance(code, int) and 100 <= code <= 599:
                return code
        resp = getattr(cur, "response", None)
        code = getattr(resp, "status_code", None)
        if isinstance(code, int) and 100 <= code <= 599:
            return code
        cur = (getattr(cur, "__cause__", None) or getattr(cur, "__context__", None))
        depth += 1
    return None


def _is_backend_overloaded(err) -> bool:
    """True when the model server ACCEPTED the connection but REFUSED the work because it
    is saturated or we are over a quota — 429 / engine_overloaded / rate limit (task #114).

    Deliberately separate from _is_backend_unavailable: that one means "the box or proxy
    is not answering" (retry the same request, it will probably land), while this means
    "the server is telling us to slow down" (re-sending immediately makes it worse and can
    deepen the limit). The two get different retry budgets and different user-facing copy.

    Live trigger: Moonshot returned
      Error code: 429 - {'error': {'message': 'The engine is currently overloaded, please
      try again later', 'type': 'engine_overloaded_error'}}
    A local Ollama/MLX box does not produce these, so nothing keyed off this fires there.

    A STRUCTURED status wins when the exception carries one: a real 429 is an overload
    whatever its body says, and a real 401/500/... is NOT one even if some tool output
    quoted the words "rate limit" into the message. Only when there is no status at all
    (a bare string, a wrapped error that lost its cause) do we fall back to matching text."""
    code = _http_status_of(err)
    if code is not None:
        return code == 429
    s = str(err or "").lower()
    if "429" in s or "too many requests" in s:
        return True
    return (
        "engine_overloaded" in s
        or "overloaded" in s
        or "rate limit" in s
        or "rate_limit" in s
        # OpenAI's billing/quota code. NOT a bare "quota" — a tool hitting "Disk quota
        # exceeded" would otherwise be reported to the user as a model overload.
        or "insufficient_quota" in s
        or "over capacity" in s
        or "at capacity" in s
    )


def _is_auth_failure(err) -> bool:
    """True when the model server REJECTED OUR CREDENTIAL — a deterministic config error
    that must fail fast rather than retry (task #117 companion).

    Deliberately narrow:
      • 401 is always an auth failure.
      • 403 is NOT, on its own. Several providers return 403 for QUOTA EXHAUSTED as well as
        for permission-denied, and reporting "you're out of credit" as "your API key is
        wrong" sends the user to fix the wrong thing. A 403 counts only when the body does
        NOT read like a quota/rate problem — those belong to _is_backend_overloaded.
      • 402 (payment required) is a billing problem, not a credential one — excluded here
        so it can carry its own message.
    Falls back to message text only when the exception carries no HTTP status (a bare
    string, or a wrapper that lost its cause)."""
    if _is_backend_overloaded(err):
        return False  # a 429 (or a quota-flavoured 403) is never an auth failure
    s = str(err or "").lower()
    code = _http_status_of(err)
    if code is not None:
        if code == 401:
            return True
        if code == 403:
            return not re.search(r"quota|rate.?limit|billing|credit|capacity", s)
        return False
    return (
        "401" in s
        or "invalid_api_key" in s
        or "invalid api key" in s
        or "incorrect api key" in s
        or "unauthorized" in s
        or "authentication" in s and "fail" in s
    )


def _is_billing_failure(err) -> bool:
    """True for 402 / explicit out-of-credit responses — deterministic like an auth failure
    but with a different fix, so it gets its own message instead of 'check your API key'."""
    if _http_status_of(err) == 402:
        return True
    s = str(err or "").lower()
    return "insufficient_quota" in s or "exceeded your current quota" in s


def _retry_after_seconds(err):
    """The provider's Retry-After hint in seconds, or None. Reads the header off an
    openai-python APIStatusError (`err.response.headers`) and falls back to a
    'retry after 12 seconds' phrase in the message. Values are clamped to a sane range so
    a bogus header can't park the turn for an hour."""
    val = None
    resp = getattr(err, "response", None)
    headers = getattr(resp, "headers", None)
    if headers is not None:
        try:
            val = headers.get("retry-after") or headers.get("Retry-After")
        except Exception:  # noqa: BLE001
            val = None
    if val is None:
        m = re.search(r"retry[\s-]?after[\s:]+(\d+(?:\.\d+)?)", str(err or ""), re.I)
        val = m.group(1) if m else None
    if val is None:
        return None
    try:
        secs = float(str(val).strip())
    except (TypeError, ValueError):
        return None  # HTTP-date form — not worth parsing; fall back to our own backoff
    if secs <= 0:
        return None
    return max(1.0, min(60.0, secs))


def _retry_delay(attempt: int, overloaded: bool, gateway: bool, err=None) -> float:
    """Seconds to wait before retry #attempt (1-based). Pure, so the policy can be
    replayed offline (task #114).

      overloaded (429) — PATIENT and jittered: ~5s → 10s → 20s → 30s (capped), plus up to
                         25% jitter so parallel workers don't retry in lock-step. The
                         provider's own Retry-After wins outright when it sends one.
      gateway (502…)   — unchanged from #94: prompt re-send, 2s/4s/6s/8s capped at 8.
      otherwise        — unchanged: 1.5s per attempt.

    Only the `overloaded` branch is new; the other two must stay byte-for-byte, since they
    are the path an Ollama/MLX coder takes."""
    if overloaded:
        hinted = _retry_after_seconds(err)
        if hinted:
            return hinted
        delay = min(30.0, 5.0 * (2 ** (attempt - 1)))
        return delay + random.uniform(0, min(3.0, delay * 0.25))
    if gateway:
        return min(8.0, 2.0 * attempt)
    return 1.5 * attempt


def _is_backend_unavailable(err) -> bool:
    s = str(err or "").lower()
    # An overload (429) is its own class — never let it fall into the gateway bucket, whose
    # policy (re-send promptly, up to 6 times) is exactly wrong for a server saying "slow
    # down". Checked FIRST because a 429 body can also contain the word "timeout".
    if _is_backend_overloaded(err):
        return False
    # A structured gateway/timeout status IS an outage regardless of wording: a provider
    # that returns `Error code: 502 - {"error": …}` carries no "bad gateway" text for the
    # phrase list below to match, and would otherwise be misfiled as a generic error.
    # 408 and Cloudflare's 522/524 are included so a real timeout STATUS keeps the same
    # answer the "timed out" text below has always given it — without them, adding the
    # structured check would have quietly demoted those.
    code = _http_status_of(err)
    if code is not None:
        return code in (502, 503, 504, 408, 522, 524)
    return (
        "502 bad gateway" in s
        or "503 service" in s
        or "504 gateway" in s
        or "bad gateway" in s
        or "gateway time-out" in s
        or "gateway timeout" in s
        or "connection refused" in s
        or "connection reset" in s
        or "connection aborted" in s
        or "failed to establish a new connection" in s
        or "max retries exceeded" in s
        or "read timed out" in s
        or "timed out" in s
    )


def _synthesize_document(gathered: list, task: str, base_url: str, api_key: str,
                         model_id: str) -> str:
    """Turn raw gathered research observations into ONE clean, organized document body
    for create_pdf. This is a pure REFORMAT (organize/deduplicate facts already found —
    no new research), so the fast local chat model handles it well. Returns "" on any
    failure so the caller can fall back to the raw concatenation."""
    notes = "\n\n---\n\n".join(gathered)[:12000]
    sys_prompt = (
        "You are a document editor. You are given a user's request and raw research "
        "notes gathered to answer it. Produce a single, clean, well-organized document "
        "that fulfills the request using ONLY facts present in the notes.\n"
        "- Use a clear title, headings, and bullet lists or a markdown table where it "
        "fits (e.g. schedules → a table of Teams | Date | Time).\n"
        "- Deduplicate and order the information sensibly (e.g. by date).\n"
        "- Do NOT invent facts not in the notes, and do NOT mention the research process, "
        "steps, tools, or that anything is partial. Output ONLY the document body."
    )
    user_prompt = f"USER REQUEST:\n{task}\n\nRESEARCH NOTES:\n{notes}"
    try:
        from openai import OpenAI
        client = OpenAI(base_url=base_url, api_key=api_key or "local",
                        max_retries=0, timeout=120)
        resp = _chat_create(
            client,
            model=model_id,
            messages=[{"role": "system", "content": sys_prompt},
                      {"role": "user", "content": user_prompt}],
            temperature=0.3,
            max_tokens=2048,
        )
        out = (resp.choices[0].message.content or "").strip()
        # A reasoning model might wrap output in <think>…</think>; strip it.
        out = _THINK_BLOCK.sub("", out).strip()
        return out
    except Exception as e:  # noqa: BLE001
        _log(f"document synthesis failed ({e}) → will use raw gathered notes")
        return ""


def _strip_tool_tags(text: str) -> str:
    """Remove the internal hint tags we append to tool output (e.g. '[SUCCESS …]',
    '[NOTE: …]', 'Note: This URL failed …') so they never leak into the user reply."""
    out = []
    for ln in (text or "").splitlines():
        s = ln.strip()
        if s.startswith("[SUCCESS") or s.startswith("[NOTE:") or s.startswith("Note: This URL failed"):
            continue
        out.append(ln)
    return "\n".join(out).strip()


_THINK_BLOCK = re.compile(r"<think>.*?</think>", re.DOTALL | re.IGNORECASE)

# Malformed variants of smolagents' code-block tags. Weak coding models routinely
# emit "<code >" (trailing space), "< code >", "</ code>", "<CODE>", OR — worst —
# "<code print(x) </code>" where the OPENING tag is missing its ">" entirely. None
# match smolagents' exact "<code>(.*?)</code>" parser, so EVERY step fails with
# "regex pattern <code>(.*?)</code> was not found" and the run burns all its steps.
# Normalizing the tags to canonical form before the parser sees them turns that
# spiral into a normal, parseable step.
_CODE_CLOSE_TAG = re.compile(r"<\s*/\s*code\s*>", re.IGNORECASE)
# A well-formed-ISH open tag that DOES have a ">": "<code>", "<code >", "< code >",
# "<CODE>", "<code class=py>". Critically the attr span is [^<>] (NOT [^>]) so it can
# never swallow across the "<" of a following "</code>" — the bug that collapsed
# "<code print(x) </code>" to a bare "<code>" and destroyed the code.
_CODE_OPEN_TAG = re.compile(r"<\s*code\b[^<>]*>", re.IGNORECASE)
# An open tag MISSING its ">": "<code" not immediately followed by ">". \b keeps it
# from matching "<codeblock"/"<coder"; the lookahead skips the already-canonical
# "<code>" so this pass only rescues the no-">" case.
_CODE_OPEN_NO_GT = re.compile(r"<\s*code\b(?!>)", re.IGNORECASE)
# Fallback: a markdown code fence (```py / ```python / ```) when the model ignored
# the tag convention entirely. Captures the fenced body so we can re-wrap it.
_MD_FENCE_BLOCK = re.compile(
    r"```(?:py|python)?[ \t]*\r?\n(.*?)```", re.DOTALL | re.IGNORECASE
)


def _normalize_code_tags(text: str) -> str:
    """Coerce a model reply's code-block delimiters to smolagents' exact <code>…</code>.

    Handles the failure modes seen live with weak coding models:
      1. close-tag variants: "</ code>", "< / code >" → "</code>" (done FIRST so the
         open-tag pass has a canonical "<" boundary to stop at).
      2. open-tag-with-">": "<code >", "< code >", "<CODE>", "<code attr>" → "<code>".
      3. open-tag-missing-">": "<code print(x) </code>" → "<code> print(x) </code>".
      4. markdown fences instead of tags: ```py … ``` → <code>…</code> (only when no
         <code> tag is present, so we don't touch backticks inside a real <code> block).
      5. opened-but-never-closed (exactly one "<code>", no "</code>"): append "</code>"
         so the parser can still extract the code instead of failing the whole step.
    Idempotent — already-correct "<code>…</code>" is returned unchanged."""
    if not text:
        return text
    fixed = _CODE_CLOSE_TAG.sub("</code>", text)
    fixed = _CODE_OPEN_TAG.sub("<code>", fixed)
    fixed = _CODE_OPEN_NO_GT.sub("<code>", fixed)
    if "<code>" not in fixed:
        fixed = _MD_FENCE_BLOCK.sub(lambda m: f"<code>\n{m.group(1)}</code>", fixed, count=1)
    if fixed.count("<code>") == 1 and "</code>" not in fixed:
        fixed = fixed + "\n</code>"
    return fixed


def _escape_raw_newlines_in_strings(code: str, defuse_tags: bool = False) -> str:
    """Turn RAW newlines/tabs that sit INSIDE a single/double-quoted string literal into
    \\n/\\t escapes. Triple-quoted strings, comments, and everything outside a string are
    copied verbatim. A tiny hand-rolled scanner (the code doesn't parse, so ast/tokenize
    can't help). Used by _repair_unterminated_string below.

    defuse_tags: also rewrite a literal <code>/</code> that appears INSIDE a string literal
    (e.g. the file content the model is writing legitimately contains React's default
    "Edit <code>src/App.js</code>") to <co\\x64e>/</co\\x64e>. The `\\x64` is Python's hex
    escape for 'd', so at RUNTIME the string is still exactly "<code>"/"</code>" — but the
    literal source no longer contains the token smolagents uses as its code-block delimiter,
    so its parser can't mis-close the block on content. Only touches tags inside strings;
    the real closing </code> lives OUTSIDE any string and is left intact."""
    out = []
    i, n = 0, len(code)
    quote = None      # the delimiter of the string we're inside, or None
    escaped = False
    while i < n:
        ch = code[i]
        if quote is None:
            three = code[i:i + 3]
            if three in ("'''", '"""'):          # legal multi-line string — copy whole
                end = code.find(three, i + 3)
                if end == -1:
                    out.append(code[i:]); break
                seg = code[i:end + 3]
                if defuse_tags:
                    seg = seg.replace("</code>", "</co\\x64e>").replace("<code>", "<co\\x64e>")
                out.append(seg); i = end + 3; continue
            if ch == "#":                          # comment — copy to end of line
                j = code.find("\n", i)
                if j == -1:
                    out.append(code[i:]); break
                out.append(code[i:j]); i = j; continue
            if ch in ("'", '"'):
                quote = ch
            out.append(ch); i += 1
        else:
            if escaped:
                out.append(ch); escaped = False; i += 1; continue
            if ch == "\\":
                out.append(ch); escaped = True; i += 1; continue
            if ch == quote:
                out.append(ch); quote = None; i += 1; continue
            # Defuse a delimiter-colliding tag INSIDE the string (see docstring).
            if defuse_tags and code.startswith("</code>", i):
                out.append("</co\\x64e>"); i += 7; continue
            if defuse_tags and code.startswith("<code>", i):
                out.append("<co\\x64e>"); i += 6; continue
            out.append({"\n": "\\n", "\r": "\\r", "\t": "\\t"}.get(ch, ch)); i += 1
    return "".join(out)


def _repair_unterminated_string(code: str) -> str:
    """Rescue the 'unterminated string literal' failure weak models hit when they pass
    multi-line file content with RAW line breaks inside a normal "..." (create_file /
    edit_lines / edit_file). Escapes those newlines and returns the repaired code ONLY if
    it now compiles; otherwise returns the ORIGINAL unchanged (never yields non-parsing
    code, and never touches code that already compiles)."""
    try:
        compile(code, "<gonext-code>", "exec")
        return code  # already valid — do nothing
    except SyntaxError as e:
        msg = (getattr(e, "msg", "") or "").lower()
        if "unterminated string literal" not in msg and "eol while scanning" not in msg:
            return code
    except Exception:  # noqa: BLE001
        return code
    try:
        fixed = _escape_raw_newlines_in_strings(code)
        compile(fixed, "<gonext-code>", "exec")
        return fixed
    except Exception:  # noqa: BLE001
        return code


def _repair_code_block_strings(text: str) -> str:
    """Apply _repair_unterminated_string to the python inside a <code>…</code> block
    (leaving the surrounding Thought prose untouched). No-op when there's no block or the
    block already parses.

    Two-stage rescue:
      1. NON-GREEDY span (first <code> → first </code>) + raw-newline escaping — the common
         case (multi-line file content with unescaped newlines).
      2. If that still won't compile AND the content itself contains extra </code> tags
         (the model is writing a file whose CONTENT legitimately has <code>/</code> — e.g.
         React's default App.js "Edit <code>src/App.js</code>"), the non-greedy match closed
         the block early on that inner tag → a truncated, unterminatable fragment. Retry with
         the GREEDY span (first <code> → LAST </code>) and defuse the in-string tags so the
         only real </code> left is the true closer. This is the fix for the create_file loop
         where the coder burned minutes/100k+ tokens on 'unterminated string literal'."""
    om = re.search(r"<code>", text)
    if not om:
        return text
    open_end = om.end()
    cm = re.search(r"</code>", text[open_end:])
    if not cm:
        return text
    close_start = open_end + cm.start()
    inner = text[open_end:close_start]
    fixed = _repair_unterminated_string(inner)
    if fixed != inner:
        return text[:open_end] + fixed + text[close_start:]
    # Stage 2 — only worth trying when there's MORE than one </code> (extra tags in content).
    last = text.rfind("</code>")
    if last <= close_start:
        return text  # single </code>; nothing a greedy re-read would change
    greedy = text[open_end:last]
    sanitized = _escape_raw_newlines_in_strings(greedy, defuse_tags=True)
    try:
        compile(sanitized, "<gonext-code>", "exec")
    except Exception:  # noqa: BLE001
        return text  # still not parseable — leave it for the loop-breaker / model retry
    # Rebuild so the FIRST literal </code> is now the true closer (inner ones are defused).
    return text[:open_end] + sanitized + text[last:]


def _looks_like_tool_code(code: str):
    """Is `code` plausibly a runnable tool call, i.e. worth substituting into the step?

    Returns (ok, why_not). Used ONLY to vet a block recovered from a reasoning channel
    (task #113) — never to judge code the model put in the visible message, which keeps
    the normal path (and Ollama) byte-for-byte unchanged.

    Why this gate exists: a reasoning model narrates the FORMAT rules to itself, so the
    text between a <code>…</code> pair in its reasoning is often English, not python.
    Seen live with Kimi K3, recovering the 16 chars "and closing with" (it was echoing
    the system prompt's "OPENS with <code> and CLOSES with </code>") — we wrapped that
    prose in tags and executed it, turning a clean parse error into a SyntaxError and a
    poisoned history. Substituting un-vetted prose is strictly worse than not recovering.

    Two checks, both cheap:
      1. it must COMPILE as python (kills prose, which is the actual failure mode);
      2. it must CALL something (`name(`) — every legitimate CodeAgent step calls a tool
         or final_answer, while a bare word like "yes" happens to compile as a Name."""
    code = (code or "").strip()
    if not code:
        return False, "empty"
    try:
        compile(code, "<gonext-recovered>", "exec")
    except SyntaxError as e:
        return False, f"not python ({(getattr(e, 'msg', '') or 'syntax error')})"
    except Exception as e:  # noqa: BLE001
        return False, f"not compilable ({e})"
    if not re.search(r"[A-Za-z_][A-Za-z0-9_]*\s*\(", code):
        return False, "no function call"
    return True, ""


_CODE_PAIR_RE = r"<code[^>]*>[\s\S]*?</code>"


def _close_dangling_code_block(text):
    """Close a code block the model OPENED but never CLOSED, when what follows it is a
    runnable tool call — the normal shape whenever the server honors `stop` (task #119).

    smolagents appends its closing tag to the stop_sequences of every CodeAgent turn
    (agents.py: `stop_sequences.append(self.code_block_tags[1])`), so any backend that
    honors `stop` — i.e. every OpenAI-compatible API — returns the block WITHOUT its
    closer. Confirmed against the raw responses saved from a live run: what the model
    actually sends is `<code>\\nrun_command("npm run build", …)` and nothing after it.
    smolagents re-appends the tag itself once it has the text (agents.py, "This adds the
    end code sequence … to the history"), which is why a trace shows a tidy `…</code>`
    that never came off the wire — but every check WE run in between tested for the PAIR
    `<code>…</code>`, a delimiter this backend structurally cannot produce.

    What that cost, live: a perfectly good turn was read as a "dangling code tag", burned
    a second model call on the CODE-NOW retry, and the retry's equally good opener-only
    reply was then DISCARDED for failing the same pair test — so every other step died on
    a parse error the model never caused, while the user watched "Error in code parsing"
    scroll past on steps that had in fact called the right tool.

    So: canonicalize first, decide second. Everything after the LAST opener is taken as
    the block, a trailing "Thought:" line and stray tags are cut, and the result is
    substituted ONLY when _looks_like_tool_code passes (it compiles AND calls something)
    — the same #113 validity gate used for reasoning-channel recovery, so prose that
    merely trails a stray tag is never promoted into executable code.

    No-op when the text already holds a pair, has no opener, or the tail isn't runnable;
    idempotent. Deliberately backend-agnostic: `stop` truncation is a property of the
    protocol, not of one vendor, and the gate makes it inert everywhere else."""
    if not isinstance(text, str) or not text:
        return text
    if re.search(_CODE_PAIR_RE, text, re.I):
        return text                      # a real block already — never touch it
    opens = list(re.finditer(r"<code[^>]*>", text, re.I))
    if not opens:
        return text                      # prose turn, or a lone stray closer (see #111)
    last = opens[-1]
    body = re.split(r"\n\s*Thought\s*:\s", text[last.end():], maxsplit=1)[0]
    body = re.sub(r"</?code[^>]*>", "", body, flags=re.I).strip()
    ok, _why = _looks_like_tool_code(body)
    if not ok:
        return text                      # not runnable — leave the step to fail honestly
    # Keep the Thought prose that preceded the block, minus any earlier stray tag (an
    # unbalanced opener left in the head would let the pair regex start from the wrong
    # place and swallow the prose into the code).
    head = re.sub(r"</?code[^>]*>", "", text[:last.start()], flags=re.I).rstrip()
    return (head + "\n" if head else "") + "<code>\n" + body + "\n</code>"


def _has_runnable_block(text) -> bool:
    """True if `text` contains a <code>…</code> block whose python actually compiles —
    i.e. this step produced something that can run. Used by the #113 no-code loop-breaker
    (openai backend only) to tell "the step worked" from "the model wrote prose again".

    Closes a stop-truncated block first (#119), so a step that DID call a tool is never
    counted toward the no-code streak just because the API ate the closing tag."""
    if not isinstance(text, str) or not text:
        return False
    text = _close_dangling_code_block(text)
    blk = re.search(r"<code[^>]*>([\s\S]*?)</code>", text, re.I)
    if not blk:
        return False
    try:
        compile(blk.group(1), "<gonext-code>", "exec")
        return True
    except Exception:  # noqa: BLE001
        return False


def _code_nudge_reason(content, openai_backend: bool):
    """Why (if at all) this streamed turn should be re-prompted once with the CODE-NOW
    directive — or None to accept it as-is. Pure decision so the three triggers can be
    replayed offline against real traces (task #113).

      1. empty-content   — the whole turn went to the reasoning channel, no message.
      2. dangling tag    — a stray <code>/</code> with no valid pair. A block the server
         merely TRUNCATED at the `</code>` stop sequence is not that (task #119): it is
         closed first and accepted as-is, so a step that already called a tool no longer
         pays for a second model call to be told to do what it just did.
      3. prose-only plan — narration of what it is ABOUT to do, no tag at all. This one
         is openai-only and additionally requires _looks_like_scratchpad, so a FINISHED
         prose answer is never nudged into inventing another tool call. It exists because
         the "</code>" visible in a smolagents trace is appended by smolagents AFTER the
         model returns, so trigger 2 cannot see this shape.

    Triggers 1 and 2 keep their existing backend-agnostic behavior — the Ollama path is
    unchanged, since trigger 3 is the only new one and it is gated off there."""
    text = (content or "").strip()
    if not text:
        return "empty-content"
    if re.search(_CODE_PAIR_RE, _close_dangling_code_block(content), re.I):
        return None  # a real block (closing a stop-truncated one first) — nothing to nudge
    if re.search(r"</?code[\s>/]", content, re.I):
        return "dangling code tag, no valid block"
    if openai_backend and _looks_like_scratchpad(text):
        return "prose-only plan, no code block"
    return None


def _recover_code_from_reasoning(content, reasoning):
    """Recover a <code>…</code> tool call that a reasoning model placed in the
    REASONING channel instead of the visible message (task #111). No-op when the
    visible content already has a usable code block, or when there is no reasoning
    text (e.g. Ollama with reasoning_effort='none') — so the Ollama path is unchanged.

    Kimi K3 @ Moonshot splits its output two ways when it fails: (a) the full
    <code>…</code> block lands in reasoning and only prose leaks to content; or
    (b) the OPENER + code land in reasoning while the CLOSING </code> leaks into
    content (seen live: content = 'Thought: …contains.</code>'). Handle both.

    Every candidate is VETTED with _looks_like_tool_code before it is used (task
    #113): the same reasoning stream also contains the model narrating the format
    rules to itself, so a <code>…</code> pair found there can hold plain English.
    Candidates are tried NEWEST-first and the first runnable one wins; if none is
    runnable we return `content` untouched and let the step fail normally, which
    is what happened before #111 existed."""
    _pair = _CODE_PAIR_RE
    if re.search(_pair, _close_dangling_code_block(content) or "", re.I):
        # Content already has a usable block — never override it. A block the server
        # truncated at the `</code>` stop sequence counts as usable (#119): the visible
        # channel is the model's actual answer and must win over a reasoning-channel draft.
        return content
    reasoning = reasoning or ""
    if not reasoning.strip():
        return content  # nothing to recover from (Ollama / non-reasoning model)

    def _clean(raw):
        # A leaked 'Thought:' line (or a stray </code>) can ride along after the
        # code — cut it so only the executable snippet remains.
        out = re.split(r"\n\s*Thought\s*:\s", raw, maxsplit=1)[0]
        return re.sub(r"</?code[^>]*>", "", out, flags=re.I).strip()

    # (a) COMPLETE blocks inside reasoning (the model drafts the whole tool call
    # there), newest first. (b) As a last resort, the opener + code in reasoning
    # with the closer leaked into content → everything after the LAST opener.
    candidates = [m.group(1) for m in
                  re.finditer(r"<code[^>]*>([\s\S]*?)</code>", reasoning, re.I)]
    candidates.reverse()
    tail = None
    for tail in re.finditer(r"<code[^>]*>([\s\S]*)$", reasoning, re.I):
        pass
    if tail:
        candidates.append(tail.group(1))
    code = None
    rejected = []
    for cand in candidates:
        cleaned = _clean(cand)
        ok, why = _looks_like_tool_code(cleaned)
        if ok:
            code = cleaned
            break
        if cleaned:
            rejected.append((cleaned, why))
    if code is None:
        if rejected:
            _preview, _why = rejected[0]
            _log(f"#113 rejected recovered block from reasoning — {_why} "
                 f"({len(_preview)} chars): {_preview[:80]!r}; leaving content as-is")
        return content
    # Keep the visible Thought prose (strip any stray/unbalanced code tags), then
    # append the recovered block so smolagents parses a clean Thought → <code> pair.
    prose = re.sub(r"</?code[^>]*>", "", content or "", flags=re.I).strip()
    rebuilt = (prose + "\n" if prose else "") + "<code>\n" + code + "\n</code>"
    _log(f"#111 recovered <code> block from reasoning channel ({len(code)} chars)")
    return rebuilt


def _strip_think(text: str) -> str:
    """Remove Qwen3-style <think>…</think> reasoning traces from a model reply.

    Reasoning models (e.g. Qwen3) emit their chain-of-thought inside <think> tags
    before the actual answer/code. We keep thinking ON (it improves the output) but
    strip the trace before smolagents parses the step, so the code/final_answer parser
    only sees the real content — and the trace never leaks to the user. Handles a
    dangling </think> too (some chat templates open <think> server-side)."""
    if not text:
        return text
    text = _THINK_BLOCK.sub("", text)
    low = text.lower()
    if "</think>" in low and "<think>" not in low:
        # Template opened the think tag for the model; drop everything up to the close.
        text = text[low.find("</think>") + len("</think>"):]
    return text.strip()


_PDF_INSTALL_HINT = (
    "PDF engine not installed on the worker. On macOS:\n"
    "    brew install pango libffi\n"
    "    python3 -m pip install weasyprint markdown\n"
    "(optional, for color emoji/flags: brew install --cask font-noto-color-emoji)\n"
    "then restart the worker. (See the Instructions page in the web app.)"
)

# WeasyPrint IS installed, but its system libraries (pango/cairo/gobject) couldn't be
# dlopen'd — almost always because the worker process predates the build that injects
# the Homebrew library path. A worker restart fixes it.
_PDF_LIBS_HINT = (
    "PDF engine is installed but its system libraries could not be loaded. "
    "Please RESTART the local worker — it automatically points WeasyPrint at the "
    "Homebrew libraries (/opt/homebrew/lib on Apple Silicon, /usr/local/lib on Intel). "
    "If it still fails, confirm `brew install pango libffi` succeeded."
)

# Font stack with color-emoji fallbacks last, so WeasyPrint resolves emoji/flag
# codepoints (🏆 🇿🇦) to a color font per-glyph while text uses a clean sans-serif.
_PDF_FONT_STACK = (
    "'Helvetica Neue', Helvetica, Arial, 'DejaVu Sans', "
    "'Noto Color Emoji', 'Apple Color Emoji', sans-serif"
)


def _render_pdf_bytes(markdown_text: str, title: str = "") -> bytes:
    """Render Markdown text to PDF bytes using WeasyPrint (renders color emoji/flags).

    Requires the WeasyPrint system libs (pango/cairo via Homebrew on macOS). Raises
    RuntimeError with an install hint if the engine/libs aren't available, or on a
    render failure. The model is expected to pass already-formatted Markdown.
    """
    try:
        import markdown as _md
    except Exception:  # noqa: BLE001 — markdown pip pkg missing
        raise RuntimeError(_PDF_INSTALL_HINT)
    try:
        from weasyprint import HTML as _HTML  # pulls pango/cairo via cffi at import
    except ModuleNotFoundError:
        # WeasyPrint itself isn't pip-installed.
        raise RuntimeError(_PDF_INSTALL_HINT)
    except Exception as e:  # noqa: BLE001 — installed but cffi can't dlopen the system libs
        _log(f"weasyprint lib load failed: {type(e).__name__}: {str(e)[:200]}")
        raise RuntimeError(_PDF_LIBS_HINT)

    body_html = _md.markdown(
        markdown_text or "",
        extensions=["extra", "sane_lists", "tables", "nl2br"],
    )
    html = (
        "<html><head><meta charset='utf-8'><style>"
        "@page { size: A4; margin: 2cm; }"
        f"body {{ font-family: {_PDF_FONT_STACK}; font-size: 11pt; line-height: 1.5; color: #18181b; }}"
        "h1 { font-size: 20pt; margin: 0 0 12pt; }"
        "h2 { font-size: 15pt; margin: 16pt 0 8pt; }"
        "h3 { font-size: 12pt; margin: 12pt 0 6pt; }"
        "code, pre { font-family: 'DejaVu Sans Mono', Courier, monospace; background: #f4f4f5; }"
        "pre { padding: 8pt; white-space: pre-wrap; }"
        "table { border-collapse: collapse; width: 100%; }"
        "th, td { border: 1px solid #d4d4d8; padding: 4pt 6pt; text-align: left; }"
        "</style></head><body>"
        f"{body_html}"
        "</body></html>"
    )
    try:
        return _HTML(string=html).write_pdf()
    except Exception as e:  # noqa: BLE001 — surface a clean render failure
        raise RuntimeError(f"PDF rendering failed (WeasyPrint: {type(e).__name__}: {str(e)[:160]}).")


def _ssl_context():
    """SSL context backed by certifi's CA bundle.

    macOS Python (python.org / Homebrew) doesn't use the system keychain, so urllib's
    default verification fails with CERTIFICATE_VERIFY_FAILED. certifi ships with the
    openai/httpx stack the worker already depends on.
    """
    import ssl
    try:
        import certifi
        return ssl.create_default_context(cafile=certifi.where())
    except Exception:  # noqa: BLE001 — fall back to system defaults
        return ssl.create_default_context()


def _pdf_upload_via_api(api_base: str, worker_key: str, file_name: str,
                        pdf_bytes: bytes) -> str:
    """Ask the API to presign an S3 upload, PUT the PDF bytes, return the download URL.

    Keeps all AWS credentials on the API/Lambda — the worker only holds its worker key.
    """
    import urllib.request as _u
    import urllib.error as _ue

    ctx = _ssl_context()
    base = (api_base or "").rstrip("/")
    if not base or not worker_key:
        raise RuntimeError("PDF upload is not available: worker API base/key missing.")

    # 1) Presign.
    req = _u.Request(
        f"{base}/api/worker/pdf-upload-url",
        data=json.dumps({"fileName": file_name}).encode("utf-8"),
        headers={
            "Content-Type": "application/json",
            "X-Worker-Key": worker_key,
        },
        method="POST",
    )
    try:
        with _u.urlopen(req, timeout=30, context=ctx) as resp:
            ref = json.loads(resp.read().decode("utf-8"))
    except _ue.HTTPError as e:  # noqa: PERF203
        detail = e.read().decode("utf-8", "replace")[:300]
        raise RuntimeError(f"Could not get a PDF upload URL (HTTP {e.code}): {detail}")
    except _ue.URLError as e:
        raise RuntimeError(
            f"Could not reach the API at {base} to presign the upload "
            f"({getattr(e, 'reason', e)}). Is the worker API deployed/online?"
        )
    put_url = ref.get("putUrl")
    get_url = ref.get("getUrl")
    if not put_url or not get_url:
        raise RuntimeError("PDF upload URL response was incomplete.")

    # 2) Upload the bytes to the presigned PUT URL.
    put_req = _u.Request(
        put_url, data=pdf_bytes,
        headers={"Content-Type": "application/pdf"},
        method="PUT",
    )
    try:
        with _u.urlopen(put_req, timeout=60, context=ctx) as up:
            if up.status not in (200, 201, 204):
                raise RuntimeError(f"S3 upload failed (HTTP {up.status}).")
    except _ue.HTTPError as e:  # noqa: PERF203
        detail = e.read().decode("utf-8", "replace")[:200]
        raise RuntimeError(f"S3 upload failed (HTTP {e.code}): {detail}")
    except _ue.URLError as e:
        raise RuntimeError(f"S3 upload could not connect ({getattr(e, 'reason', e)}).")
    return get_url


# Generic download support (task #102) — deliver ANY workspace file/folder as a download.
_DOWNLOAD_MAX_BYTES = 100 * 1024 * 1024  # 100 MB cap on what we'll upload for download
_DOWNLOAD_CONTENT_TYPES = {
    "pdf": "application/pdf", "zip": "application/zip", "json": "application/json",
    "csv": "text/csv", "txt": "text/plain", "md": "text/markdown", "html": "text/html",
    "htm": "text/html", "xml": "application/xml", "yaml": "application/x-yaml",
    "yml": "application/x-yaml", "svg": "image/svg+xml", "png": "image/png",
    "jpg": "image/jpeg", "jpeg": "image/jpeg", "gif": "image/gif", "sql": "application/sql",
    "tar": "application/x-tar", "gz": "application/gzip", "tgz": "application/gzip",
}


def _content_type_for(name: str) -> str:
    """Content-type by filename extension for create_download (matches the API's map)."""
    ext = ""
    if "." in (name or ""):
        ext = name.rsplit(".", 1)[-1].strip().lower()
    return _DOWNLOAD_CONTENT_TYPES.get(ext, "application/octet-stream")


def _artifact_upload_via_api(api_base: str, worker_key: str, file_name: str,
                             data: bytes, content_type: str = "") -> str:
    """Presign a GENERIC S3 upload (any file type), PUT the bytes, return the download URL.
    Like _pdf_upload_via_api but preserves the real extension + Content-Type. The PUT
    Content-Type MUST match what the presign signed, so we send content_type in the presign
    request and PUT with the value the API echoes back (task #102)."""
    import urllib.error as _ue
    import urllib.request as _u

    ctx = _ssl_context()
    base = (api_base or "").rstrip("/")
    if not base or not worker_key:
        raise RuntimeError("Download is not available: worker API base/key missing.")
    req = _u.Request(
        f"{base}/api/worker/artifact-upload-url",
        data=json.dumps({"fileName": file_name, "contentType": content_type}).encode("utf-8"),
        headers={"Content-Type": "application/json", "X-Worker-Key": worker_key},
        method="POST",
    )
    try:
        with _u.urlopen(req, timeout=30, context=ctx) as resp:
            ref = json.loads(resp.read().decode("utf-8"))
    except _ue.HTTPError as e:  # noqa: PERF203
        detail = e.read().decode("utf-8", "replace")[:300]
        raise RuntimeError(f"Could not get an upload URL (HTTP {e.code}): {detail}")
    except _ue.URLError as e:
        raise RuntimeError(
            f"Could not reach the API at {base} to presign the upload "
            f"({getattr(e, 'reason', e)}). Is the worker API deployed/online?"
        )
    put_url = ref.get("putUrl")
    get_url = ref.get("getUrl")
    if not put_url or not get_url:
        raise RuntimeError("Upload URL response was incomplete.")
    put_ct = ref.get("contentType") or content_type or "application/octet-stream"
    put_req = _u.Request(put_url, data=data, headers={"Content-Type": put_ct}, method="PUT")
    try:
        with _u.urlopen(put_req, timeout=120, context=ctx) as up:
            if up.status not in (200, 201, 204):
                raise RuntimeError(f"S3 upload failed (HTTP {up.status}).")
    except _ue.HTTPError as e:  # noqa: PERF203
        detail = e.read().decode("utf-8", "replace")[:200]
        raise RuntimeError(f"S3 upload failed (HTTP {e.code}): {detail}")
    except _ue.URLError as e:
        raise RuntimeError(f"S3 upload could not connect ({getattr(e, 'reason', e)}).")
    return get_url


def _format_text_for_pdf(text: str, title: str, base_url: str, api_key: str,
                         model_id: str, instruction: str = "") -> str:
    """Use the model to clean/structure raw text into well-formed Markdown for the PDF.

    When `instruction` is set (a follow-up edit like "fill the teams to the schedule"),
    the model applies that change to `text` while reformatting. Falls back to the
    original text (lightly wrapped) if the model call fails, so a PDF is still produced.
    """
    fallback = text or ""
    if title and not fallback.lstrip().startswith("#"):
        fallback = f"# {title}\n\n{fallback}"
    try:
        from openai import OpenAI
        # timeout=20, NOT 60 (task #74): this formatting pass runs INSIDE a create_pdf
        # tool call, which itself runs under the agent's python-executor wall clock. A
        # 60s wait here could eat the whole step budget — the PDF then rendered+uploaded
        # fine but the model was told the step timed out, so it re-created the PDF from
        # scratch (3 duplicate PDFs, ~10 wasted minutes). Formatting is optional polish:
        # give it 20s, then fall back to the raw text and leave room for render+upload.
        client = OpenAI(base_url=base_url, api_key=api_key or "local",
                        max_retries=0, timeout=20)
        system_content = (
            "You format raw text/data into clean Markdown for a PDF document. "
            "Add a single top-level '# Title', sensible headings, bullet lists, "
            "and tables where appropriate. Do NOT invent facts, do NOT add "
            "commentary, and preserve all numbers and wording. Output ONLY the "
            "Markdown — no code fences, no explanations."
        )
        if instruction:
            system_content += (
                " Apply the user's requested change to the content before formatting; "
                "keep everything else intact."
            )
        user_content = (
            (f"Title: {title}\n\n" if title else "")
            + (f"Requested change: {instruction}\n\n" if instruction else "")
            + f"Content to format:\n{text}"
        )
        resp = _chat_create(
            client,
            model=model_id,
            messages=[
                {"role": "system", "content": system_content},
                {"role": "user", "content": user_content},
            ],
            max_tokens=1500,
            temperature=0.2,
        )
        out = (resp.choices[0].message.content or "").strip()
        # Strip accidental ```markdown fences.
        if out.startswith("```"):
            out = re.sub(r"^```[a-zA-Z]*\n?|\n?```$", "", out).strip()
        return out or fallback
    except Exception as e:  # noqa: BLE001
        _log(f"pdf format error: {e} — using raw text")
        return fallback


# A clear "make a PDF from this" request. Matched against the raw user message so we
# can run the PDF pipeline deterministically — a small model cannot reliably echo a
# long document back into a Python string literal for the create_pdf() tool call.
_PDF_INTENT = re.compile(
    r"(create|make|generate|export|build|produce|convert|turn)\b[\s\S]{0,40}\bpdf\b"
    r"|\bpdf\b[\s\S]{0,40}\b(from|for|of|with|out of)\b",
    re.IGNORECASE,
)

# Follow-up phrasings that reference an *existing / target* PDF — e.g. "in the pdf
# file", "add the teams to the pdf", "put it in the pdf", "update the pdf", "as a pdf".
# These carry no "create … pdf" verb so _PDF_INTENT misses them, yet they still mean
# "(re)generate the PDF" — previously they fell through to the CodeAgent and crashed
# on create_pdf(text="…long literal…"). The document body comes from history (A2).
_PDF_FOLLOWUP = re.compile(
    r"\b(in|into|on|to)\s+(the\s+|a\s+|this\s+|that\s+)?pdf\b"
    r"|\b(add|include|put|fill|insert|append|update|regenerate|remake|redo)\b[\s\S]{0,40}\bpdf\b"
    r"|\bpdf\b[\s\S]{0,40}\b(add|include|fill|update|insert|append)\b"
    r"|\bas\s+(a\s+)?pdf\b"
    r"|\bthe\s+pdf\s+file\b",
    re.IGNORECASE,
)

# Interrogatives ABOUT an existing PDF ("what's in the pdf?") must NOT trigger
# generation — only imperative (re)build requests should.
_PDF_QUESTION = re.compile(
    r"^\s*(what|why|how|where|who|which|when|does|do|is|are|can|could|should|would)\b[\s\S]*\?\s*$",
    re.IGNORECASE,
)

# READING an existing PDF ("summarize this pdf", "extract text from the pdf") is the job
# of extract_text_from_pdf, NOT the create fast-path. These read verbs must veto _wants_pdf
# so a "summarize the pdf file" request doesn't get hijacked into (re)generating a PDF.
# Deliberately excludes add/put/fill/update (task #6 follow-ups still (re)generate).
_PDF_READ_INTENT = re.compile(
    r"\b(read|summari[sz]e|extract|analy[sz]e|parse|review|scan)\b[\s\S]{0,40}\bpdf\b"
    r"|\bpdf\b[\s\S]{0,40}\b(say|says|contain|contains|about|content)\b",
    re.IGNORECASE,
)


# A PDF request whose CONTENT must be gathered/researched first (e.g. "get me the world
# cup 2026 schedule … then create a pdf"). The deterministic fast-path can only render text
# the user already GAVE (or a prior assistant turn) — it cannot web_search, and the model
# would just hallucinate live facts. When these retrieval/currency signals appear AND no
# document body was provided, we skip the fast-path and let the multi-step agent gather the
# data (web_search / fetch_url) and THEN call create_pdf with it.
_PDF_RESEARCH_INTENT = re.compile(
    r"\b(get me|get the|find|research|look ?up|search|fetch|gather|collect|compile|"
    r"latest|current|today'?s|upcoming|recent|schedules?|fixtures?|standings?|"
    r"news|weather|prices?|scores?|results?|rankings?|statistics|stats)\b",
    re.IGNORECASE,
)


def _pdf_has_explicit_body(user_text: str) -> bool:
    """True when the message itself supplies the document body — a quoted span, or text
    after a 'text:/content:/following:/data:' lead-in. Distinguishes 'make a PDF of THIS
    text …' (fast-path OK) from 'get me X then make a PDF' (needs the agent to fetch X)."""
    t = user_text or ""
    for pat in (r'"([\s\S]+)"', r"“([\s\S]+)”", r"'([\s\S]+)'"):
        m = re.search(pat, t)
        if m and len(m.group(1).strip()) >= 40:
            return True
    m = re.search(r"\b(texts?|following|below|content|data)\b\s*[:\-]\s*\S", t, re.IGNORECASE)
    return bool(m)


def _wants_pdf(text: str) -> bool:
    """True when the user is asking us to (re)generate a PDF — fresh request or a
    follow-up referencing a target PDF — but NOT when merely asking about one, and NOT
    when asking us to READ/summarize an existing PDF (that's extract_text_from_pdf)."""
    t = (text or "").strip()
    if not t:
        return False
    if _PDF_QUESTION.match(t):
        return False
    if _PDF_READ_INTENT.search(t):
        return False
    return bool(_PDF_INTENT.search(t) or _PDF_FOLLOWUP.search(t))


def _extract_pdf_doc_text(user_text: str) -> str:
    """Pull the document body out of a 'create a PDF from this text "…"' message.

    Prefers a quoted span; otherwise strips the leading instruction clause.
    """
    t = user_text or ""
    # 1) Largest quoted span (straight, smart, or single quotes).
    best = ""
    for pat in (r'"([\s\S]+)"', r"“([\s\S]+)”", r"'([\s\S]+)'"):
        m = re.search(pat, t)
        if m and len(m.group(1).strip()) > len(best):
            best = m.group(1).strip()
    if best:
        return best
    # 2) Everything after a 'text:'/'following:'/'content:' lead-in.
    m = re.search(r"\b(texts?|following|below|content|data)\b\s*[:\-]?\s*\n?", t, re.IGNORECASE)
    if m and t[m.end():].strip():
        return t[m.end():].strip()
    # 3) Drop a leading 'please create a pdf (file) from this text:' verb phrase.
    stripped = re.sub(
        r"^\s*(please\s+)?(create|make|generate|export|build|produce|convert|turn)\s+"
        r"(a\s+|an\s+|this\s+|the\s+)?(pdf|document)\s*(file|doc)?\s*"
        r"(from|for|of|with|using|out of)?\s*(this|the|following)?\s*"
        r"(text|texts|data|content)?\s*[:\-]?\s*",
        "",
        t,
        flags=re.IGNORECASE,
    )
    return stripped.strip() or t.strip()


def _derive_pdf_title(doc_text: str) -> str:
    """Use the first meaningful line as the document title (strip leading symbols/emoji)."""
    for line in (doc_text or "").splitlines():
        s = line.strip().lstrip("#").strip()
        if not s:
            continue
        # Drop leading non-letter symbols/emoji (keep Latin + Vietnamese letters/digits).
        s2 = re.sub(r"^[^\wÀ-ỹ]+", "", s).strip()
        return (s2 or s)[:60]
    return "Document"


# ============================ RAG (knowledge base) ============================
# The agent can download a ZIP-at-URL, unzip it locally, index the text files into a
# flat JSON vector store on the USER'S OWN S3 bucket (explicit AWS creds — NOT presigned),
# then retrieve relevant chunks to answer questions. Embeddings run on the Ollama coding
# server (/api/embed). Everything below is best-effort with hard safety caps.

_RAG_TEXT_EXTS = {
    ".cs", ".js", ".jsx", ".ts", ".tsx", ".mjs", ".cjs", ".vue", ".svelte",
    ".json", ".jsonc", ".md", ".mdx", ".txt", ".rst", ".py", ".rb", ".go",
    ".rs", ".java", ".kt", ".swift", ".php", ".c", ".h", ".cpp", ".hpp", ".cc",
    ".html", ".htm", ".css", ".scss", ".sass", ".less", ".xml", ".yaml", ".yml",
    ".toml", ".ini", ".cfg", ".conf", ".env", ".sh", ".bash", ".zsh", ".sql",
    ".graphql", ".proto", ".gradle", ".dockerfile", ".tsv", ".csv",
}
_RAG_SKIP_DIRS = {
    "node_modules", ".git", ".svn", ".hg", "__pycache__", ".venv", "venv",
    "dist", "build", ".next", ".nuxt", "out", ".cache", "bin", "obj", ".idea",
    ".gradle", "vendor", "target",
}
_RAG_MAX_DOWNLOAD_BYTES = 500 * 1024 * 1024     # 500 MB zip cap
_RAG_MAX_UNZIP_BYTES = 2 * 1024 * 1024 * 1024   # 2 GB uncompressed (zip-bomb guard)
_RAG_MAX_FILES = 20000
_RAG_MAX_FILE_BYTES = 2 * 1024 * 1024           # skip individual text files > 2 MB
_RAG_CHUNK_CHARS = 3200                          # ~800 tokens
_RAG_CHUNK_OVERLAP = 400


def _rag_source_key(url: str) -> str:
    import hashlib
    return hashlib.sha256((url or "").strip().encode("utf-8")).hexdigest()[:32]


def _rag_gdrive_direct(url: str) -> str:
    """Rewrite a Google Drive share link to its direct-download form."""
    m = re.search(r"drive\.google\.com/file/d/([A-Za-z0-9_-]+)", url or "")
    if m:
        return f"https://drive.google.com/uc?export=download&id={m.group(1)}"
    m = re.search(r"drive\.google\.com/open\?id=([A-Za-z0-9_-]+)", url or "")
    if m:
        return f"https://drive.google.com/uc?export=download&id={m.group(1)}"
    return url


def _rag_assert_safe_url(url: str) -> None:
    """SSRF guard: only http/https, and refuse private/link-local/loopback hosts."""
    import ipaddress
    import socket
    from urllib.parse import urlparse
    p = urlparse(url or "")
    if p.scheme not in ("http", "https"):
        raise ValueError("Only http/https URLs are allowed.")
    host = p.hostname or ""
    if not host:
        raise ValueError("URL has no host.")
    try:
        infos = socket.getaddrinfo(host, None)
    except Exception as e:  # noqa: BLE001
        raise ValueError(f"Cannot resolve host {host}: {e}")
    for info in infos:
        ip = info[4][0]
        try:
            addr = ipaddress.ip_address(ip)
        except ValueError:
            continue
        if (addr.is_private or addr.is_loopback or addr.is_link_local
                or addr.is_reserved or addr.is_multicast):
            raise ValueError(f"Refusing to fetch a private/internal address ({ip}).")


def _rag_workdir(source_key: str) -> str:
    import os
    base = os.path.join(os.path.expanduser("~"), ".gonext", "rag-work", source_key)
    os.makedirs(base, exist_ok=True)
    return base


def _rag_base_dir(source_key: str) -> str:
    """Where a URL's downloaded/unzipped files should live. Prefer the active terminal
    workspace (the folder the `gonext` REPL was opened from) so files land in the
    user's project; otherwise fall back to the per-URL cache dir under ~/.gonext."""
    import os
    if _WS_ACTIVE:
        os.makedirs(_WS_ACTIVE, exist_ok=True)
        return _WS_ACTIVE
    return _rag_workdir(source_key)


def _rag_local_index_dir(source_url: str) -> str:
    """Local RAG index dir for a source, scoped PER-FOLDER (task #97): the vector shards
    live at ~/.gonext/rag/<sha256(activeFolder)>/<sha256(sourceUrl)>/. Keying on the active
    terminal folder (the SAME sha256 scheme the REPL uses for its session files) isolates
    each project's knowledge base, so the same URL indexed from two folders stays separate."""
    import hashlib
    import os
    folder = _WS_ACTIVE or os.getcwd()
    fhash = hashlib.sha256(os.path.realpath(folder).encode("utf-8")).hexdigest()[:32]
    base = os.path.join(os.path.expanduser("~"), ".gonext", "rag", fhash,
                        _rag_source_key(source_url))
    os.makedirs(base, exist_ok=True)
    return base


def _rag_download(url: str, dest_path: str = "") -> tuple:
    """Download a public URL with SSRF + size guards — IDEMPOTENT per URL. Every URL
    maps to a deterministic work dir (~/.gonext/rag-work/<urlHash>/); relative
    dest_paths are contained there (never the process cwd, which is the user's home),
    and a repeat call for an already-downloaded URL reuses the cached file instantly
    (follow-up questions re-run the pipeline cheaply). Returns (path, cached)."""
    import os
    src = _rag_gdrive_direct((url or "").strip())
    _rag_assert_safe_url(src)
    key = _rag_source_key(url)
    workdir = _rag_base_dir(key)
    dest_path = os.path.expanduser((dest_path or "").strip())
    if not dest_path:
        name = os.path.basename(src.split("?")[0]) or "download.bin"
        if not name.lower().endswith(".zip") and "zip" in src.lower():
            name += ".zip"
        dest_path = os.path.join(workdir, name)
    elif not os.path.isabs(dest_path):
        # Contain relative paths in the work dir — models pass bare names like
        # "project.zip" which would otherwise land in the worker's cwd (the home dir).
        dest_path = os.path.join(workdir, dest_path)
    # Cache hit 1: the exact target already exists.
    if os.path.isfile(dest_path) and os.path.getsize(dest_path) > 0:
        return dest_path, True
    # Cache hit 2: this URL was downloaded before under a different name — ONLY safe in
    # the per-URL cache dir (one URL per dir). In an active workspace the folder holds
    # many unrelated files, so a bare "any zip = this URL" match would be wrong.
    if not _WS_ACTIVE:
        try:
            for fn in sorted(os.listdir(workdir)):
                p = os.path.join(workdir, fn)
                if os.path.isfile(p) and fn.lower().endswith(".zip") and os.path.getsize(p) > 0:
                    return p, True
        except OSError:
            pass
    os.makedirs(os.path.dirname(dest_path) or ".", exist_ok=True)
    req = urllib.request.Request(src, headers={"User-Agent": "gonext-rag/1.0"})
    total = 0
    with urllib.request.urlopen(req, timeout=120, context=_ssl_context()) as resp, \
            open(dest_path, "wb") as out:
        while True:
            block = resp.read(1024 * 256)
            if not block:
                break
            total += len(block)
            if total > _RAG_MAX_DOWNLOAD_BYTES:
                out.close()
                os.remove(dest_path)
                raise ValueError(
                    f"Download exceeds the {_RAG_MAX_DOWNLOAD_BYTES // (1024*1024)} MB cap."
                )
            out.write(block)
    return dest_path, False


def _rag_resolve_zip(zip_path: str) -> str:
    """Resolve a (possibly relative) zip path. Bare names the model echoes from earlier
    turns are searched in the rag work dirs so a fresh process still finds them."""
    import glob as _glob
    import os
    zp = os.path.expanduser((zip_path or "").strip())
    if os.path.isfile(zp):
        return zp
    if not os.path.isabs(zp):
        base = os.path.basename(zp)
        # Look in the active terminal workspace first (that's where downloads land now),
        # then fall back to the per-URL cache dirs.
        if _WS_ACTIVE:
            cand = os.path.join(_WS_ACTIVE, base)
            if os.path.isfile(cand):
                return cand
        hits = _glob.glob(os.path.join(
            os.path.expanduser("~"), ".gonext", "rag-work", "*", base))
        if hits:
            return hits[0]
    return zp


def _rag_unzip(zip_path: str, dest_dir: str = "") -> tuple:
    """Extract a local zip with Zip-Slip + zip-bomb guards — IDEMPOTENT. If the target
    dir already has extracted files, reuse it instead of re-extracting. Relative
    dest_dirs are placed next to the zip (inside the per-URL work dir), never the cwd.
    Returns (dir, file_list, cached)."""
    import os
    import zipfile
    zip_path = _rag_resolve_zip(zip_path)
    dest_dir = os.path.expanduser((dest_dir or "").strip())
    if not dest_dir:
        dest_dir = os.path.splitext(zip_path)[0] + "_unzipped"
    elif not os.path.isabs(dest_dir):
        dest_dir = os.path.join(os.path.dirname(zip_path) or ".", dest_dir)
    # Cache hit: already extracted for this zip → return the existing listing.
    if os.path.isdir(dest_dir):
        existing = []
        for dirpath, _dirnames, filenames in os.walk(dest_dir):
            for fn in filenames:
                existing.append(os.path.relpath(os.path.join(dirpath, fn), dest_dir))
            if len(existing) > _RAG_MAX_FILES:
                break
        if existing:
            return dest_dir, sorted(existing), True
    os.makedirs(dest_dir, exist_ok=True)
    dest_root = os.path.realpath(dest_dir)
    written = 0
    names = []
    with zipfile.ZipFile(zip_path) as zf:
        infos = zf.infolist()
        if len(infos) > _RAG_MAX_FILES:
            raise ValueError(f"Zip has too many entries (> {_RAG_MAX_FILES}).")
        total_uncompressed = sum(i.file_size for i in infos)
        if total_uncompressed > _RAG_MAX_UNZIP_BYTES:
            raise ValueError("Zip uncompressed size exceeds the safety cap (possible zip bomb).")
        for info in infos:
            if info.is_dir():
                continue
            target = os.path.realpath(os.path.join(dest_dir, info.filename))
            # Zip Slip: extracted path must stay within dest_dir.
            if not (target == dest_root or target.startswith(dest_root + os.sep)):
                continue
            os.makedirs(os.path.dirname(target), exist_ok=True)
            with zf.open(info) as src, open(target, "wb") as dst:
                dst.write(src.read())
            written += 1
            names.append(info.filename)
    return dest_dir, names, False


def _rag_iter_text_files(root: str):
    """Yield (abs_path, rel_path, ext) for indexable text files under root."""
    import os
    if os.path.isfile(root):
        yield root, os.path.basename(root), os.path.splitext(root)[1].lower()
        return
    for dirpath, dirnames, filenames in os.walk(root):
        dirnames[:] = [d for d in dirnames if d not in _RAG_SKIP_DIRS]
        for fn in filenames:
            ext = os.path.splitext(fn)[1].lower()
            base = fn.lower()
            if ext not in _RAG_TEXT_EXTS and base not in ("dockerfile", "makefile", ".gitignore"):
                continue
            abs_path = os.path.join(dirpath, fn)
            try:
                if os.path.getsize(abs_path) > _RAG_MAX_FILE_BYTES:
                    continue
            except OSError:
                continue
            yield abs_path, os.path.relpath(abs_path, root), ext


def _rag_chunk_text(text: str, rel_path: str, ext: str):
    """Line-aware windowed chunks with overlap. Returns list of {file,ext,start,end,text}."""
    if not text.strip():
        return []
    chunks = []
    n = len(text)
    step = max(1, _RAG_CHUNK_CHARS - _RAG_CHUNK_OVERLAP)
    i = 0
    while i < n:
        end = min(n, i + _RAG_CHUNK_CHARS)
        # Prefer to cut on a newline near the window end.
        if end < n:
            nl = text.rfind("\n", i + step, end)
            if nl > i:
                end = nl + 1
        piece = text[i:end].strip()
        if piece:
            chunks.append({"file": rel_path, "ext": ext, "start": i, "end": end, "text": piece})
        if end >= n:
            break
        i = end - _RAG_CHUNK_OVERLAP if end - _RAG_CHUNK_OVERLAP > i else end
    return chunks


def _embed(base: str, model: str, texts: list) -> list:
    """Embed a batch of texts. Auto-detects the endpoint from `base` (a bare host:port,
    an .../v1, or an Ollama host): tries OpenAI-compatible /v1/embeddings first (MLX
    servers, modern Ollama), then Ollama /api/embed, then legacy /api/embeddings."""
    # Normalize to a bare root (strip any /v1, /api, /api/embed(dings) suffix).
    root = re.sub(r"/(v1|api/embeddings|api/embed|api)/?$", "", (base or "").rstrip("/")).rstrip("/")
    if not root:
        raise RuntimeError("No embedding endpoint configured (set the RAG embedding server URL).")
    ctx = _ssl_context()
    last_err = None

    def _post(url, payload, timeout):
        req = urllib.request.Request(
            url, data=json.dumps(payload).encode("utf-8"),
            headers={"Content-Type": "application/json"}, method="POST",
        )
        with urllib.request.urlopen(req, timeout=timeout, context=ctx) as resp:
            return json.loads(resp.read().decode("utf-8", "replace"))

    # 1) OpenAI-compatible /v1/embeddings — MLX servers on a port, and modern Ollama.
    try:
        data = _post(f"{root}/v1/embeddings", {"model": model, "input": texts}, 300)
        items = data.get("data")
        if isinstance(items, list) and items and isinstance(items[0], dict) and "embedding" in items[0]:
            return [it["embedding"] for it in items]
    except urllib.error.HTTPError as e:
        if e.code not in (400, 404, 405, 501):
            raise RuntimeError(f"Embeddings failed (HTTP {e.code}): {e.read().decode('utf-8', 'replace')[:200]}")
        last_err = f"/v1/embeddings HTTP {e.code}"
    except urllib.error.URLError as e:
        raise RuntimeError(f"Embedding server unreachable at {root}: {getattr(e, 'reason', e)}")

    # 2) Ollama batch /api/embed.
    try:
        data = _post(f"{root}/api/embed", {"model": model, "input": texts}, 300)
        embs = data.get("embeddings")
        if isinstance(embs, list) and embs and isinstance(embs[0], list):
            return embs
    except urllib.error.HTTPError as e:
        if e.code not in (400, 404, 405, 501):
            raise RuntimeError(f"Ollama embed failed (HTTP {e.code}): {e.read().decode('utf-8', 'replace')[:200]}")
        last_err = f"/api/embed HTTP {e.code}"

    # 3) Legacy Ollama /api/embeddings (per text).
    out = []
    try:
        for t in texts:
            data = _post(f"{root}/api/embeddings", {"model": model, "prompt": t}, 120)
            out.append(data.get("embedding") or [])
    except (urllib.error.HTTPError, urllib.error.URLError) as e:
        code = getattr(e, "code", None)
        raise RuntimeError(
            f"No embeddings endpoint at {root} (tried /v1/embeddings, /api/embed, /api/embeddings; "
            f"last HTTP {code if code else getattr(e, 'reason', e)}). Make sure an EMBEDDINGS-capable "
            f"server is running there and the model '{model}' is available "
            f"(e.g. `ollama pull {model}`). NOTE: mlx_lm.server does NOT serve embeddings — point the "
            f"RAG embedding server URL at Ollama (or another embeddings server) instead."
        )
    if any(any(v) for v in out):
        return out
    raise RuntimeError(
        f"Embeddings server at {root} returned empty vectors for model '{model}' "
        f"(is '{model}' an embedding model that is installed there?)."
    )


def _cosine(a: list, b: list) -> float:
    import math
    if not a or not b or len(a) != len(b):
        return -1.0
    dot = sum(x * y for x, y in zip(a, b))
    na = math.sqrt(sum(x * x for x in a))
    nb = math.sqrt(sum(y * y for y in b))
    if na == 0 or nb == 0:
        return -1.0
    return dot / (na * nb)


def _rag_parse_location(loc: str):
    """Parse an ARN / s3:// URI / bucket[/prefix] into (bucket, prefix)."""
    s = (loc or "").strip() or "arn:aws:s3:::gonext-rag"
    rest = s
    m = re.match(r"^arn:aws:s3:::(.+)$", s, re.I)
    if m:
        rest = m.group(1)
    else:
        m = re.match(r"^s3://(.+)$", s, re.I)
        if m:
            rest = m.group(1)
    rest = rest.strip("/")
    if "/" in rest:
        bucket, prefix = rest.split("/", 1)
    else:
        bucket, prefix = rest, ""
    return bucket, prefix.strip("/")


# Registered workspace roots (from ~/.gonext/workspaces.json via the worker) — set
# per-run by run_agent_chat. Reads are allowed in any of them; WRITES additionally
# require _ws_write_allowed (workspace root + not inside .git/).
_WS_ROOTS: list = []  # [{"name": str, "path": realpath str, "allowRun": bool}]

# The active terminal workspace: the folder the `gonext` REPL was launched from
# (passed through as payload.activeWorkspace). When set, download_file/unzip_file put
# their output HERE instead of the shared ~/.gonext/rag-work cache, so files land in
# the user's actual project folder. Only honoured if it's a registered workspace root.
_WS_ACTIVE: str = ""


def _rag_read_allowed(path: str) -> str:
    """Resolve `path` and ensure it stays within a safe root (temp dir, ~/.gonext, the
    active terminal folder, or a registered workspace) — so the agent's file tools can't
    read arbitrary files like ~/.ssh. Relative paths anchor to the terminal workspace
    (where downloads/unzips land), because the model routinely passes bare names like
    "tools-hook" or "tools-hook/README.md".

    NOTE: the worker's PROCESS CWD is deliberately NOT an allowed root (bug #70). The
    daemon is often launched from ~/Projects (the PARENT of every workspace), so trusting
    os.getcwd() let `grep_repo(path="/Users/joseph/Projects")` read EVERY project on disk
    — including other repos' secrets. Reads are confined to the registered/active
    workspaces + ~/.gonext + the temp dir. Fail-closed: no workspace ⇒ no repo reads."""
    import os
    import tempfile
    expanded = os.path.expanduser((path or "").strip())
    if not os.path.isabs(expanded):
        # Anchor a relative path to the active workspace (NOT the worker cwd — see above).
        base = _WS_ACTIVE or (_WS_ROOTS[0]["path"] if _WS_ROOTS else None)
        if base is None:
            raise ValueError(
                "No workspace is active — register a folder with `gonext-local-worker "
                "workspace add <path>` before reading files."
            )
        expanded = os.path.join(base, expanded)
    rp = os.path.realpath(expanded)
    roots = [
        os.path.realpath(tempfile.gettempdir()),
        os.path.realpath(os.path.join(os.path.expanduser("~"), ".gonext")),
    ] + [w["path"] for w in _WS_ROOTS]
    # The active terminal folder (download/unzip target) — allow reading what we just
    # extracted there, even when it isn't a registered workspace.
    if _WS_ACTIVE:
        roots.append(_WS_ACTIVE)
    if any(rp == r or rp.startswith(r + os.sep) for r in roots):
        return rp
    raise ValueError(
        f"Path '{path}' is outside your registered workspace(s). The agent can only read "
        "inside a folder you registered (or download/unzip outputs), not the whole disk."
    )


def _ws_for_path(path: str):
    """Return the registered workspace dict containing `path`, else None."""
    import os
    rp = os.path.realpath(path)
    for w in _WS_ROOTS:
        if rp == w["path"] or rp.startswith(w["path"] + os.sep):
            return w
    return None


def _default_ws_root() -> str:
    """The folder a tool defaults to when the model passes NO path/workdir: the terminal's
    ACTIVE folder (where `gonext` was launched), NOT the first-registered workspace.
    Registered workspaces accumulate across sessions, so falling back to _WS_ROOTS[0]
    made a no-workdir command land in an OLD workspace (bug #46: launched in t9, created
    the project in t1). Falls back to the first workspace only when there is no active
    folder at all."""
    import os
    if _WS_ACTIVE and os.path.isdir(_WS_ACTIVE):
        return _WS_ACTIVE
    return _WS_ROOTS[0]["path"] if _WS_ROOTS else "."


def _workspace_overview(roots, max_entries: int = 40) -> str:
    """Cheap, LOCAL top-level listing of each registered workspace root — a plain
    os.listdir(), no recursion, no network call, no model call. Used two ways: (1)
    injected into the task prompt so the agent has upfront grounding that the workspace
    is non-empty WITHOUT spending its first (possibly failing) tool-call round-trip just
    discovering that; (2) appended to the degrade-to-plain-reply message so even a TOTAL
    coding-model outage still shows the user their files are actually there — this is
    the one piece of workspace context that stays available no matter how badly the
    model-calling side of the agent is failing, since it never touches a model at all."""
    import os
    parts = []
    for w in roots:
        root = w["path"]
        try:
            entries = sorted(
                e for e in os.listdir(root) if e not in _RAG_SKIP_DIRS
            )
        except OSError as e:
            parts.append(f"{w['name']} ({root}): could not list ({e})")
            continue
        if not entries:
            parts.append(f"{w['name']} ({root}): EMPTY — no files or folders found.")
            continue
        shown = entries[:max_entries]
        labeled = [
            f"{e}/" if os.path.isdir(os.path.join(root, e)) else e for e in shown
        ]
        more = f" …(+{len(entries) - max_entries} more)" if len(entries) > max_entries else ""
        parts.append(
            f"{w['name']} ({root}) — {len(entries)} top-level item(s): "
            + ", ".join(labeled) + more
        )
    return "\n".join(parts)


def _turn_checkpoint_path(workspace: str) -> str:
    """Path to the interrupted-turn checkpoint for a given workspace. Keyed by a hash
    of the workspace path (same sha256-slice convention as _rag_source_key) — but a
    separate file/purpose from the REPL's own ~/.gonext/sessions: this holds one
    in-PROGRESS turn's step trace, written incrementally as the agent works, not
    completed Q&A history."""
    import hashlib
    import os
    h = hashlib.sha256(workspace.encode("utf-8")).hexdigest()[:32]
    base = os.path.join(os.path.expanduser("~"), ".gonext", "agent-turns")
    os.makedirs(base, exist_ok=True)
    return os.path.join(base, f"{h}.json")


def _write_turn_checkpoint(workspace: str, task: str, steps: list) -> None:
    """Overwrite the checkpoint for this workspace with the CURRENT turn's step trace so
    far. Called after every agent step (see step_callback). If the process dies mid-turn
    (network drop, kill, crash) before any final answer is ever emitted, this file
    survives on disk and the NEXT turn in this workspace recovers it instead of silently
    re-investigating from scratch — see _load_and_clear_turn_checkpoint."""
    import json
    import os
    try:
        path = _turn_checkpoint_path(workspace)
        tmp = path + ".tmp"
        with open(tmp, "w", encoding="utf-8") as f:
            json.dump({"task": task, "steps": steps}, f)
        os.replace(tmp, path)
    except Exception as e:  # noqa: BLE001
        _log(f"turn checkpoint write failed (non-fatal): {e}")


def _load_and_clear_turn_checkpoint(workspace: str):
    """Read a leftover checkpoint (if any) from a prior turn in this workspace that
    never reached a graceful finish, then delete it — its content is about to be folded
    into the NEW turn's context, and the new turn starts writing its own fresh
    checkpoint from here. Returns None if there's nothing to recover."""
    import json
    import os
    path = _turn_checkpoint_path(workspace)
    data = None
    try:
        with open(path, encoding="utf-8") as f:
            data = json.load(f)
    except FileNotFoundError:
        return None
    except Exception as e:  # noqa: BLE001
        _log(f"turn checkpoint read failed (non-fatal): {e}")
        return None
    finally:
        try:
            os.remove(path)
        except OSError:
            pass
    return data


def _turn_checkpoint_exists(workspace: str) -> bool:
    """Non-consuming probe: does an interrupted turn survive on disk for this
    workspace? Routing needs to KNOW there's work to resume (to force agent mode on a
    bare 'continue') without destroying it — the read-and-delete consume happens later,
    only on the agent path, where the recovered trace is actually fed to a model."""
    import os
    return os.path.exists(_turn_checkpoint_path(workspace))


def _clear_turn_checkpoint(workspace: str) -> None:
    """Delete the checkpoint once a turn reaches ANY graceful finish (full success, the
    max-steps fallback, or a successful degrade-to-plain-reply) — that turn's findings
    are already captured in the final answer pushed to conversation history, so the raw
    step trace is no longer needed for crash recovery."""
    import os
    try:
        os.remove(_turn_checkpoint_path(workspace))
    except OSError:
        pass


def _render_edit_card(action: str, path: str, summary: str,
                      removed=None, added=None,
                      max_removed: int = 8, max_added: int = 12) -> str:
    """Multi-line terminal-friendly summary of a file edit, emitted as ONE step event.

    The line shapes are a small stable protocol the REPL colorizes (bold header,
    red '-' lines, green '+' lines, dim line numbers) — keep them in sync with the
    isEditCardHeader/EDIT_DIFF_RE regexes in gonext-repl.mjs. The web thinking panel
    shows the same text as-is (the 4-space indent renders as a monospace block in
    markdown), so no web change is needed.

        ⏺ Update(XeroForwardService.java)
          └ Replaced lines 19-19 with 1 line
            19 -     ...old text...
            19 +     ...new text...
    """
    import os
    lines = [f"⏺ {action}({os.path.basename(path)})", f"  └ {summary}"]

    def _fmt(entries, sign, cap):
        out = []
        for i, (no, txt) in enumerate(entries):
            if i >= cap:
                out.append(f"    … (+{len(entries) - cap} more)")
                break
            # NOT _clip() — that collapses ALL whitespace, destroying the code's
            # leading indentation, which a diff needs to stay readable.
            t = txt.rstrip()
            if len(t) > 160:
                t = t[:159] + "…"
            out.append(f"    {no} {sign} {t}")
        return out

    if removed:
        lines += _fmt(removed, "-", max_removed)
    if added:
        lines += _fmt(added, "+", max_added)
    return "\n".join(lines)


def _ws_write_allowed(path: str) -> str:
    """Resolve `path` for WRITING: must be inside a registered workspace and never
    inside its .git/ internals (agents must not corrupt version control)."""
    import os
    p = (path or "").strip()
    # Anchor RELATIVE paths to the active/first workspace root — the worker daemon's
    # own cwd is ~, so a bare "abc" would otherwise resolve to ~/abc and be rejected
    # even though the user plainly means <workspace>/abc. Mirrors _rag_read_allowed.
    if p and not os.path.isabs(os.path.expanduser(p)):
        base = _WS_ACTIVE or (_WS_ROOTS[0]["path"] if _WS_ROOTS else "")
        if base:
            p = os.path.join(base, p)
    rp = os.path.realpath(os.path.expanduser(p))
    w = _ws_for_path(rp)
    if w is None:
        raise ValueError(
            "Writes are only allowed inside a registered workspace "
            "(register one with: gonext-local-worker workspace add <path>)."
        )
    rel = os.path.relpath(rp, w["path"])
    if rel == ".git" or rel.startswith(".git" + os.sep) or (os.sep + ".git" + os.sep) in (os.sep + rel + os.sep):
        raise ValueError("Writes inside .git/ are not allowed.")
    return rp


_WS_RUN_ID = None  # backup folder for this run, created lazily on first change
_WS_CHANGES: list = []  # [{"action": "edit"|"create"|"create_dir", "path": str, "backup": str|None}]


def _ws_backup_root() -> str:
    """Backup dir for this run (created lazily). Also home of manifest.json."""
    global _WS_RUN_ID
    import os
    import time as _t
    if _WS_RUN_ID is None:
        _WS_RUN_ID = _t.strftime("%Y%m%d-%H%M%S", _t.localtime())
    root = os.path.join(os.path.expanduser("~"), ".gonext", "workspace-backups", _WS_RUN_ID)
    os.makedirs(root, exist_ok=True)
    return root


def _ws_snapshot(path: str) -> str:
    """Copy the file to the run's backup dir before its FIRST edit in this run, so
    every change is revertible. Returns the backup path ('' if new file)."""
    import os
    import shutil
    if not os.path.isfile(path):
        return ""
    for c in _WS_CHANGES:  # already snapshotted this run — keep the ORIGINAL version
        if c["path"] == path and c.get("backup"):
            return c["backup"]
    root = _ws_backup_root()
    dest = os.path.join(root, path.lstrip(os.sep))
    os.makedirs(os.path.dirname(dest), exist_ok=True)
    shutil.copy2(path, dest)
    return dest


def _ws_record_change(action: str, path: str, backup) -> None:
    """Track a change AND persist the run manifest so `gonext-local-worker workspace
    revert` can restore edits / delete created files after the process is gone."""
    import os
    _WS_CHANGES.append({"action": action, "path": path, "backup": backup or None})
    try:
        root = _ws_backup_root()
        with open(os.path.join(root, "manifest.json"), "w", encoding="utf-8") as fh:
            json.dump({"changes": _WS_CHANGES}, fh, indent=2)
    except OSError as e:
        _log(f"workspace manifest write failed: {e}")


# Command prefixes run_command may execute (first token after shlex split). This is a
# STEERING mechanism, not a sandbox: python3 (allowed) can subprocess anything, and
# ssh/rsync flags (ProxyCommand, -e) can local-exec too. A run-enabled workspace means
# the user trusts the agent with local-user-level execution — the list exists to keep
# the model on productive paths (build/test/deploy), not to contain a hostile one.
# Still excluded on purpose: bare shells, package publishes, sudo, and bare `pip`
# (the model must use `python3 -m pip`, which pip itself recommends).
# run_command execution policy — ALLOW-BY-DEFAULT.
# We used to gate on a fixed allowlist (npm/python/go/…). It rejected legitimate DevOps
# tooling (docker, kubectl, terraform, aws, gh, …) and, because that universe is unbounded
# and grows constantly, any list was a hardcode that upset the next user. Crucially, the
# gate was never a SANDBOX: `python3`/`node` are runnable and can subprocess anything, so a
# restrictive allowlist bought STEERING, not safety — it just added friction. So now: any
# command runs, with two guards kept:
#   1. Privilege escalation is ALWAYS blocked (`sudo`/`su`/`doas`) — running the model's
#      commands as root is the one footgun a run-enabled workspace shouldn't hand over.
#   2. Commands run via argv (shlex.split + Popen, NO shell), so pipes/redirects/`&&`
#      /backticks are literal args, never shell operations — the shell-injection guard.
# The user can still tighten this from web Settings (passed through cfg, both default off):
#   • runAllowlist — if non-empty, ONLY these runners are permitted (opt-in lockdown).
#   • runDenylist  — extra runners to block on top of the always-blocked set.
_WS_RUN_DENY_ALWAYS = {"sudo", "su", "doas"}  # privilege escalation — never runnable
_WS_RUN_ALLOWLIST = set()  # opt-in lockdown from cfg; empty = allow all
_WS_RUN_DENYLIST = set()   # extra user blocks from cfg

# Privilege escalation, scanned on the RAW command (not just argv[0]) so it also catches
# `bash -c "sudo …"` / `env sudo …` — the naive shell-wrap bypass a model reaches for.
_WS_PRIV_RE = re.compile(r"\b(sudo|doas|su)\b", re.IGNORECASE)
# Destructive: mass delete, disk wipe, or power-off. Allowed by default (allow-by-default
# policy) but worth a confirmation when we CAN ask the user (interactive terminal).
_WS_DESTRUCTIVE_RE = re.compile(
    r"(^|[\s;&|])rm\b[^|;&\n]*\s-[a-z]*[rf]"          # rm with -r / -f (any order)
    r"|(^|[\s;&|])(dd|shred|wipefs|mkfs(\.\w+)?)\b"   # disk destroyers
    r"|(^|[\s;&|])(shutdown|reboot|halt|poweroff)\b", # power
    re.IGNORECASE,
)


def _ws_command_risk(command, argv, denyset):
    """Classify a command for the run policy. Returns (hard_reason, ask_reason), either
    None when N/A:
    - hard_reason: block outright when we CAN'T ask (web / non-interactive) — privilege
      escalation or a user-denylisted runner.
    - ask_reason: prompt the user when we CAN (interactive terminal) — the above PLUS
      destructive filesystem/power commands (which are otherwise allowed by default)."""
    import os as _os
    cmd = command or ""
    runner = argv[0] if argv else ""
    rbase = _os.path.basename(runner)
    if _WS_PRIV_RE.search(cmd):
        r = "privilege escalation (sudo/su)"
        return (r, r)
    if denyset and (runner in denyset or rbase in denyset):
        r = f"'{runner}' is on your blocked-commands list"
        return (r, r)
    if _WS_DESTRUCTIVE_RE.search(cmd):
        return (None, "a destructive command (deletes files / wipes disk / powers off)")
    return (None, None)


def _ws_request_approval(api_base, worker_key, job_id, command, reason, deadline_s=180):
    """Pause and ask the user (via the terminal REPL's Yes/No picker) to allow a risky
    command. Registers a pending approval on the job through the API, then polls the job
    for the user's decision. Returns True (allow) or False (deny / timeout / cancelled).

    deadline_s bounds the wait. 180s is right for a RISKY command (absent user → deny,
    never auto-run). But the __MAXSTEP__ "keep going?" prompt is NOT security-sensitive and
    the coder can be slow, so it passes a much larger deadline (#110): the REPL now
    dismisses the picker the instant this job ends, and the worker's own job-cap kills a
    truly-abandoned run, so a long wait here is safe and lets a real "Yes" actually land.
    Mirrors _pdf_upload_via_api's worker-key auth — no new creds on the worker."""
    import urllib.request as _u
    import uuid as _uuid
    import time as _t
    base = (api_base or "").rstrip("/")
    if not base or not worker_key or not job_id:
        return False
    rid = _uuid.uuid4().hex[:12]
    ctx = _ssl_context()
    try:
        req = _u.Request(
            f"{base}/api/worker/jobs/{job_id}/approval-request",
            data=json.dumps({"id": rid, "command": command}).encode("utf-8"),
            headers={"Content-Type": "application/json", "X-Worker-Key": worker_key},
            method="POST",
        )
        with _u.urlopen(req, timeout=20, context=ctx) as resp:
            resp.read()
    except Exception as e:  # noqa: BLE001
        _log(f"approval-request failed: {e} — treating as deny")
        return False
    if command.startswith("__MAXSTEP__::"):
        _emit({"type": "step", "text": "Awaiting your choice: continue past the step budget?"})
    elif command.startswith("__COMPACT__::"):
        _emit({"type": "step", "text": "Awaiting your choice: compact the context?"})
    else:
        _emit({"type": "step", "text": f"Awaiting your approval to run: {command[:70]}"})
    deadline = _t.time() + max(30, int(deadline_s))  # per-call; risky=180s, __MAXSTEP__ long
    while _t.time() < deadline:
        _t.sleep(1.2)
        try:
            g = _u.Request(f"{base}/api/worker/jobs/{job_id}",
                           headers={"X-Worker-Key": worker_key}, method="GET")
            with _u.urlopen(g, timeout=15, context=ctx) as resp:
                data = json.loads(resp.read().decode("utf-8"))
        except Exception as e:  # noqa: BLE001
            _log(f"approval poll error: {e}")
            continue
        if data.get("jobStatus") in ("cancelled", "failed", "completed"):
            return False  # Ctrl+C or the job ended → treat as declined
        dec = data.get("approvalDecision") or {}
        if dec.get("id") == rid:
            return bool(dec.get("allow"))
    return False

# ---- background servers (behavior-detected, command-agnostic) ----
# run_command classifies a command by OBSERVED BEHAVIOR, never by its text: if the
# process keeps running AND starts LISTENing on a TCP port, it IS a server — npm start,
# bun dev, a custom `make serve`, anything. (An earlier version pattern-matched command
# names like "npm start"; rejected as unwinnable hardcode — same lesson as task #37.)
# Detected servers are left running in their own process group (start_new_session),
# recorded in ~/.gonext/servers.json, and stoppable via the stop_server tool.


def _ws_pgroup_pids(pgid: int) -> list:
    """Live pids in a process group (the server + everything it spawned)."""
    import subprocess
    try:
        out = subprocess.run(["pgrep", "-g", str(pgid)],
                             stdout=subprocess.PIPE, text=True, timeout=5)
        return [int(x) for x in out.stdout.split()]
    except Exception:  # noqa: BLE001
        return []


def _ws_listening_ports(pgid: int) -> list:
    """TCP ports the process group is LISTENing on — the behavioral definition of
    'this command is a server'. Empty when none (or lsof/pgrep unavailable)."""
    import subprocess
    pids = _ws_pgroup_pids(pgid)
    if not pids:
        return []
    try:
        out = subprocess.run(
            ["lsof", "-a", "-p", ",".join(map(str, pids)),
             "-iTCP", "-sTCP:LISTEN", "-P", "-n"],
            stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, timeout=5)
        ports = set()
        for line in out.stdout.splitlines()[1:]:
            m = re.search(r":(\d+)\s+\(LISTEN\)", line)
            if m:
                ports.add(int(m.group(1)))
        return sorted(ports)
    except Exception:  # noqa: BLE001
        return []


def _ws_gui_pids(pgid: int) -> list:
    """pids in the group that OWN AN ON-SCREEN WINDOW — the behavioral definition of
    'this command is a desktop/GUI app' (tkinter, pygame, PyQt…), the third outcome
    alongside 'exited' and 'listening on a port'. macOS only; empty on other platforms
    or when the signal is unavailable (safe no-op → falls back to the old wait/kill).

    Detection is RUNTIME/behavior-based (what the process actually did), never a command
    name — consistent with _ws_listening_ports. Primary: Quartz's window list filtered by
    owner pid (definitive). Fallback: lsof for a mapped macOS windowing framework a process
    only loads once it has a real GUI (AppKit/HIToolbox/Tk) or a live WindowServer socket."""
    import sys
    if sys.platform != "darwin":
        return []
    pids = set(_ws_pgroup_pids(pgid))
    if not pids:
        return []
    # Primary: on-screen window ownership via Quartz (import-guarded — pyobjc may be absent).
    try:
        from Quartz import (  # type: ignore
            CGWindowListCopyWindowInfo,
            kCGWindowListOptionOnScreenOnly,
            kCGNullWindowID,
            kCGWindowOwnerPID,
        )
        info = CGWindowListCopyWindowInfo(
            kCGWindowListOptionOnScreenOnly, kCGNullWindowID) or []
        owners = {int(w.get(kCGWindowOwnerPID, 0)) for w in info}
        hit = sorted(pids & owners)
        if hit:
            return hit
    except Exception:  # noqa: BLE001 — Quartz missing/failed → fall through to lsof
        pass
    # Fallback: a process that mapped the GUI stack (or holds a WindowServer connection).
    import subprocess
    try:
        out = subprocess.run(
            ["lsof", "-p", ",".join(map(str, sorted(pids)))],
            stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, timeout=5)
        gui = set()
        for line in out.stdout.splitlines()[1:]:
            if ("AppKit.framework" in line or "HIToolbox.framework" in line
                    or "/Tk.framework" in line or "WindowServer" in line):
                try:
                    gui.add(int(line.split()[1]))
                except (ValueError, IndexError):
                    pass
        return sorted(gui & pids)
    except Exception:  # noqa: BLE001
        return []


def _ws_servers_path() -> str:
    import os
    base = os.path.join(os.path.expanduser("~"), ".gonext")
    os.makedirs(base, exist_ok=True)
    return os.path.join(base, "servers.json")


def _ws_load_servers(prune: bool = True) -> list:
    """Registry of background servers we started. `prune` drops entries whose
    process group is gone (crashed or stopped outside of us)."""
    import json as _json
    try:
        with open(_ws_servers_path(), encoding="utf-8") as fh:
            servers = _json.load(fh).get("servers", [])
    except Exception:  # noqa: BLE001
        return []
    if prune:
        servers = [s for s in servers if _ws_pgroup_pids(int(s.get("pgid", 0)))]
    return servers


def _ws_save_servers(servers: list) -> None:
    import json as _json
    try:
        with open(_ws_servers_path(), "w", encoding="utf-8") as fh:
            _json.dump({"servers": servers}, fh, indent=2)
    except OSError as e:
        _log(f"servers.json write failed: {e}")


def _ws_cap_servers(max_live: int = 3) -> list:
    """Server lifecycle (task #90 Phase 2): prune dead registry entries and STOP the
    oldest background servers beyond max_live, so dev servers leaked from earlier
    turns can't accumulate and squat every default port (the root cause of the
    'port 3000/3001/3005 busy' punt seen live). Only ever touches processes in OUR
    registry (started by run_command — never arbitrary user processes). Persists the
    pruned registry and returns the live list, oldest first."""
    import os
    import signal
    servers = _ws_load_servers()  # prune=True drops already-dead entries
    if len(servers) > max_live:
        for s in servers[:-max_live]:  # registry is append-ordered → oldest first
            try:
                pgid = int(s.get("pgid", 0))
                if pgid <= 0:
                    continue
                os.killpg(pgid, signal.SIGTERM)
                _log(f"server lifecycle: stopped old background server pid={s.get('pid')} "
                     f"ports={s.get('ports')} ('{str(s.get('command'))[:40]}')")
            except (OSError, ValueError) as e:  # already gone / no permission
                _log(f"server lifecycle: stop skipped pid={s.get('pid')}: {e}")
        servers = servers[-max_live:]
    _ws_save_servers(servers)
    return servers


def _ws_scrubbed_env() -> dict:
    """Minimal child env for run_command. The worker process env holds secrets
    (GONEXT_WORKER_KEY etc. via worker.env) — a repo's test script must NEVER see
    them, so we pass an explicit whitelist instead of inheriting."""
    import os
    keep = ("PATH", "HOME", "LANG", "LC_ALL", "TMPDIR", "SHELL", "USER", "TERM",
            "JAVA_HOME", "GOPATH", "CARGO_HOME", "NVM_DIR",
            # SSH agent socket so ssh/scp/rsync can authenticate with the user's KEY
            # (no password in the model's context) for deploys (task #57).
            "SSH_AUTH_SOCK", "SSH_AGENT_PID")
    env = {k: os.environ[k] for k in keep if k in os.environ}
    # CI=1 makes watch-mode test runners (CRA's `npm test`, jest --watch) run ONCE and
    # exit instead of sitting in interactive watch mode until the timeout kills them.
    env["CI"] = "1"
    return env


def _ws_distill_output(out: str, cap_head: int = 1200, cap_tail: int = 4000) -> str:
    """Cap huge build/test output: failures live at the tail, context at the head."""
    if len(out) <= cap_head + cap_tail + 200:
        return out
    return (out[:cap_head] + f"\n…[{len(out) - cap_head - cap_tail} chars omitted]…\n"
            + out[-cap_tail:])


def _ws_cap_obs(text: str, cap: int = 3000) -> str:
    """Cap a READ tool's observation (read_file_lines / list_dir / read_text_file /
    grep_repo) before it's returned to the CodeAgent. Every observation stays in the
    agent's step memory and is re-sent to the model EVERY subsequent step, so an
    uncapped 20k-char file dump costs ~5-6k tokens PER remaining step (O(n²) growth —
    task #63). Keeps the head (line numbers / range start) + a small tail; the model is
    told to read smaller ranges / grep to locate instead of whole files."""
    if len(text) <= cap:
        return text
    head = int(cap * 0.7)
    tail = cap - head
    return (text[:head].rstrip()
            + f"\n…[{len(text) - cap} chars truncated to save context — read a smaller "
              "line range, or grep_repo to locate the exact spot]…\n"
            + text[-tail:].lstrip())


def _rag_s3_client(region: str, akid: str, secret: str):
    import boto3  # noqa: PLC0415
    return boto3.client(
        "s3", region_name=region or "ap-southeast-1",
        aws_access_key_id=akid, aws_secret_access_key=secret,
    )


def run_agent_chat(cfg):
    try:
        from smolagents import CodeAgent, OpenAIServerModel, tool
    except Exception as e:  # noqa: BLE001
        _emit({"type": "final", "text": f"[smolagents not installed: {e}]"})
        return

    messages = cfg.get("messages") or []
    agent_base_url = cfg.get("agentBaseURL") or ""
    agent_api_key = cfg.get("agentApiKey") or "local"
    agent_model_id = cfg.get("agentModelId") or ""
    # For the create_pdf tool: API base + worker key to request a presigned S3 upload.
    pdf_api_base = (cfg.get("apiBaseURL") or "").strip()
    pdf_worker_key = (cfg.get("workerKey") or "").strip()
    # Interactive command-approval gate: this job's id + whether the client (terminal
    # REPL) can show a Yes/No picker. When both are present, run_command PAUSES on a
    # risky command and asks the user via the API instead of hard-blocking it.
    _job_id = (cfg.get("jobId") or "").strip()
    _interactive_approval = bool(cfg.get("interactiveApproval"))
    # Auto-test mode (/test-auto): after making changes the agent must VERIFY them and
    # fix → re-test until they pass before finishing. Drives a prompt directive + a
    # slightly higher step budget (the verify loop needs room). Terminal-only.
    _auto_test = bool(cfg.get("autoTest"))
    # Deploy target chosen via the terminal /server picker (task #69): {name, host, user}
    # or None. Host/user only — never a secret; deploys use SSH KEY auth to this host.
    _deploy_server = cfg.get("deployServer") if isinstance(cfg.get("deployServer"), dict) else None
    # Optional dedicated coding/reasoning model for the CodeAgent's tool-use loop.
    # Routing, plain replies and summarization stay on the chat model (better at
    # natural language); the code model only drives http_request reasoning.
    # The URL may be a pasted Ollama endpoint (http://host:11434/api/generate) or a
    # bare host — normalize to the OpenAI-compatible /v1 root either way (defensive:
    # also fixes an older API layer that appended /v1 after the native /api path).
    raw_coding_base = _normalize_openai_base(cfg.get("codingBaseURL") or "")
    raw_coding_model = (cfg.get("codingModelId") or "").strip()
    # Task #108: explicit coding backend kind + API key for an OpenAI-compatible endpoint.
    # coding_kind: "ollama" | "openai" | "" — the USER'S SETTING, not the answer. What the
    # branches read is `coding_backend` below, resolved once from this plus the URL.
    # coding_api_key: the Bearer key for an OpenAI-compatible coder (empty otherwise). Used
    # for the code-model calls (and coding-model auto-detect), NOT the chat agent_api_key.
    coding_kind = (cfg.get("codingKind") or "").strip().lower()
    if coding_kind not in ("ollama", "openai", "local"):
        coding_kind = ""
    coding_api_key = (cfg.get("codingApiKey") or "").strip()
    # Task #107: when on, emit each code-model response so the worker persists it to Mongo.
    save_full_response = bool(cfg.get("saveFullResponse"))
    if raw_coding_base:
        same_server = raw_coding_base.rstrip("/") == (agent_base_url or "").rstrip("/")
        if same_server and not raw_coding_model:
            # The coding URL points at the SAME server as the chat model. Do NOT
            # auto-detect: /v1/models can list several cached models and return a
            # SMALLER one first (e.g. a 3B), silently downgrading the agent's reasoning.
            # Reuse the chat model id directly — it's the model actually loaded here.
            _log(
                f"coding base == chat base ({raw_coding_base}); "
                f"reusing chat model {agent_model_id!r} (skipping auto-detect)"
            )
            coding_base_url = agent_base_url
            coding_model_id = agent_model_id
        else:
            # A DIFFERENT dedicated coding server. If no model name was given, ask the
            # server which model it serves (mlx_lm.server otherwise tries to download a
            # mismatched name from HF and 404s).
            detected = raw_coding_model or _detect_model_id(
                raw_coding_base, coding_api_key or agent_api_key)
            if detected:
                coding_base_url = raw_coding_base
                coding_model_id = detected
            else:
                _log(
                    f"coding model id unresolved for {raw_coding_base!r}; "
                    "falling back to chat model"
                )
                coding_base_url = agent_base_url
                coding_model_id = agent_model_id
    else:
        coding_base_url = agent_base_url
        coding_model_id = agent_model_id
    # Task #108: an OpenAI-compatible coder REQUIRES an explicit model name (its
    # /v1/chat/completions needs a `model` id, and we won't silently reuse the MLX chat
    # model's name against a cloud endpoint). Fail with a clear, actionable message.
    if coding_kind == "openai" and not (raw_coding_model or "").strip():
        raise _AgentConfigError(
            "Your agent coding model is set to OpenAI-compatible but has no model name. "
            "Enter the exact model id your endpoint expects (e.g. gpt-4o-mini) in "
            "Settings → Agent → Agent coding model name."
        )
    # Multi-step ReAct loop (thinking agent): the agent may take several
    # Thought → tool → Observation steps and then call final_answer() itself. This
    # replaced the old strict single-shot (max_steps=1) once a stronger coding model
    # (Qwen3-14B class) made real multi-step reasoning reliable. The default budget is
    # BACKEND-AWARE (a cloud OpenAI-compatible coder is stronger and cheaper-per-step than
    # a small local model, so it gets more room; a local Ollama coder gets a middle budget;
    # local MLX keeps the original tight default). Overridable via cfg.maxSteps.
    # The provide_final_answer override below is only the exhaustion fallback.
    #
    # THE ONE RESOLUTION (task #117 → #123 Part B). Everything that used to ask "is this
    # Ollama?" or "did the user pick openai?" separately — reasoning_effort, streaming, the
    # #113 reasoning repairs, and this step budget — now reads `coding_backend`.
    #
    # The worker (gonext-local-worker.mjs) resolves this first, because it must decide
    # whether to fetch the coding API key BEFORE python starts, and passes the answer down
    # as codingBackend. Trust it only when we ended up on the URL it judged: the fallbacks
    # above can swap coding_base_url for the chat model's, which is a different server and
    # may well be a different KIND of server.
    _backend_hint = (cfg.get("codingBackend") or "").strip().lower()
    if _backend_hint in ("ollama", "openai", "local") and coding_base_url == raw_coding_base:
        coding_backend, _backend_why = _backend_hint, "resolved by the worker"
    else:
        coding_backend, _backend_why = _resolve_coding_backend(coding_kind, coding_base_url)
    coding_flags = _coding_backend_flags(coding_backend)
    _default_budget, _ws_budget = coding_flags["budget"]
    try:
        max_steps = int(cfg.get("maxSteps") or _default_budget)
    except (TypeError, ValueError):
        max_steps = _default_budget
    if max_steps < 1:
        max_steps = _default_budget

    # Max web_search + fetch_url calls per task (user-configurable in web Settings →
    # Agent). Default 10; past it the retrieval tools refuse and steer to finish.
    try:
        research_budget = int(cfg.get("researchBudget") or 10)
    except (TypeError, ValueError):
        research_budget = 10
    if research_budget < 1:
        research_budget = 10

    _log(
        f"start model={agent_model_id!r} base={agent_base_url!r} "
        f"codeModel={coding_model_id!r} codeBase={coding_base_url!r} maxSteps={max_steps}"
    )
    # SearXNG (task #103): diagnostics — confirms python SEES the URL (from cfg.searxngUrl or
    # the GONEXT_SEARXNG_URL env). If this logs 'none', web_search falls back to keyless DDG.
    import os as _os
    _searxng_url = ((cfg.get("searxngUrl") or "").strip()
                    or (_os.environ.get("GONEXT_SEARXNG_URL") or "").strip())
    _log(f"searxng: {_searxng_url or 'none (keyless DDG + Wikipedia only)'}")
    # Search model (#105): diagnostics only — the web_search tool reads it from cfg directly.
    _search_base = (cfg.get("searchBaseURL") or "").strip()
    _search_model = (cfg.get("searchModelId") or "").strip()
    if _search_base and _search_model:
        _log(f"search model: {_search_model!r} @ {_search_base} "
             "(web_search synthesizes a cited answer with it)")
    else:
        _log("search model: none (web_search returns read page content, not a synthesized answer)")

    # Build the task from the conversation history. We include the FULL conversation
    # (both user AND assistant turns) so the agent remembers what it already did —
    # e.g. data it fetched on a previous turn. Assistant turns are condensed (drop
    # <think> reasoning; clip long raw HTTP dumps), and we keep the most recent turns
    # within a character budget so we never overflow the model's context window.
    # ~8000 chars ≈ 2k tokens, tiny against Qwen2.5-Coder-7B's 32k context, leaving
    # ample room for smolagents' own system prompt + step memory (HTTP observations).
    HISTORY_CHAR_BUDGET = 8000
    think_re = re.compile(r"<think>.*?</think>", re.DOTALL | re.IGNORECASE)

    def _condense(role, content):
        text = (content or "").strip()
        if role == "assistant":
            text = think_re.sub("", text).strip()
            # Raw HTTP dumps add little conversational value — keep only a snippet.
            if text.startswith("HTTP "):
                text = text[:500]
        return text

    # The latest user message is the current task; everything before it is history.
    last_user_idx = -1
    for i, m in enumerate(messages):
        if m.get("role") == "user":
            last_user_idx = i
    if last_user_idx < 0:
        _emit({"type": "final", "text": "[No user message found in history]"})
        return
    task_text = (messages[last_user_idx].get("content") or "").strip()
    # Routing must look at the CURRENT message alone — not the history-laden blob
    # below. Otherwise the keyword router matches URLs/"api"/"GET" from prior turns
    # and fires the agent on trivial replies like "thanks" or "good".
    latest_user_text = task_text

    # Walk prior turns newest-first, keeping condensed lines until the budget is
    # spent, then restore chronological (oldest→newest) order.
    history_lines = []
    used = 0
    for m in reversed(messages[:last_user_idx]):
        role = m.get("role", "")
        if role not in ("user", "assistant"):
            continue
        text = _condense(role, m.get("content", ""))
        if not text:
            continue
        line = f"{'User' if role == 'user' else 'Assistant'}: {text}"
        if used + len(line) > HISTORY_CHAR_BUDGET:
            break
        history_lines.append(line)
        used += len(line)
    history_lines.reverse()

    if history_lines:
        convo = "\n".join(history_lines)
        task_text = (
            "Conversation so far (oldest to newest):\n"
            f"{convo}\n\nCurrent task: {task_text}"
        )

    _log(
        f"history: {len(history_lines)} prior turn(s), {used} chars "
        f"(budget {HISTORY_CHAR_BUDGET}) — exact turns sent to the agent below:"
    )
    for j, ln in enumerate(history_lines):
        _log(f"  history[{j}]: {ln[:240]}")
    _log(f"current task (latest user message): {task_text.rsplit('Current task: ', 1)[-1][:240]!r}")

    # ---- send_email config (per-user email API, e.g. a Lambda the user set up) ----
    email_enabled = bool(cfg.get("emailEnabled"))
    email_api_url = (cfg.get("emailApiUrl") or "").strip()
    email_api_method = ((cfg.get("emailApiMethod") or "POST").strip().upper()) or "POST"
    email_api_headers_raw = (cfg.get("emailApiHeaders") or "").strip()
    email_body_template = (cfg.get("emailBodyTemplate") or "").strip()
    email_from = (cfg.get("emailFrom") or "").strip()
    email_allow = [a for a in re.split(r"[,\s]+", (cfg.get("emailAllowList") or "").strip().lower()) if a]
    # Advertised + registered ONLY when fully configured AND enabled — an outward
    # action must never be silently available (same gating as the PDF reader).
    _EMAIL_AVAILABLE = bool(email_enabled and email_api_url and email_body_template)

    # ---- RAG config (download/unzip/index/search over a user's ZIP-at-URL) ----
    rag_enabled = bool(cfg.get("ragEnabled"))
    rag_location = (cfg.get("ragS3Location") or "arn:aws:s3:::gonext-rag").strip()
    rag_region = (cfg.get("ragAwsRegion") or "ap-southeast-1").strip()
    rag_akid = (cfg.get("ragAwsAccessKeyId") or "").strip()
    rag_secret = (cfg.get("ragAwsSecretAccessKey") or "").strip()
    rag_embed_model = (cfg.get("ragEmbedModel") or "nomic-embed-text").strip()
    rag_embed_url = (cfg.get("ragEmbedUrl") or "").strip()
    try:
        rag_top_k = int(cfg.get("ragTopK") or 6)
    except (TypeError, ValueError):
        rag_top_k = 6
    rag_top_k = max(1, min(50, rag_top_k))
    # Embedding server: an explicit MLX/Ollama endpoint from Settings if given, else the
    # agent's Ollama coding server. _embed() auto-detects OpenAI-compat (/v1/embeddings,
    # e.g. an MLX server on a port) vs Ollama (/api/embed) — so a bare host:port works.
    rag_embed_base = rag_embed_url or (coding_base_url or agent_base_url or "")
    try:
        import boto3 as _boto3_probe  # noqa: F401,PLC0415
        _boto3_ok = True
    except Exception:  # noqa: BLE001
        _boto3_ok = False
    # RAG storage backend (task #97): "local" keeps the index on disk under ~/.gonext,
    # PER-FOLDER, and needs NO S3 creds / boto3 — only the embedding server. "cloud" (default,
    # and what the web app always uses) stores it on the user's own S3 bucket via boto3. The
    # terminal REPL picks this per-folder and sends it as cfg.ragMode.
    rag_mode = (cfg.get("ragMode") or "cloud").strip().lower()
    if rag_mode not in ("local", "cloud"):
        rag_mode = "cloud"
    _RAG_LOCAL = rag_mode == "local"
    _RAG_AVAILABLE = bool(
        rag_enabled and rag_embed_base and (
            _RAG_LOCAL or (rag_akid and rag_secret and _boto3_ok)
        )
    )
    if rag_enabled and not _RAG_LOCAL and not _boto3_ok:
        _log("RAG (cloud) requested but boto3 is not installed in the worker python — RAG "
             "tools disabled (switch to local RAG with /rag-local to avoid needing S3/boto3)")
    if _RAG_AVAILABLE:
        _log(f"RAG mode: {'LOCAL (index under ~/.gonext/rag, per-folder)' if _RAG_LOCAL else 'cloud (S3)'}")

    # ---- Workspaces (local folders the agent may read/edit/test code in) ----
    # Registered on the Mac via `gonext-local-worker workspace add <path>`; the worker
    # passes the registry through cfg. Only these roots are writable — the security
    # boundary for the coding tools.
    global _WS_ROOTS, _WS_ACTIVE
    _WS_ROOTS = []
    for w in (cfg.get("workspaces") or []):
        try:
            import os as _os
            p = _os.path.realpath(_os.path.expanduser(str(w.get("path") or "")))
            if p and _os.path.isdir(p):
                _WS_ROOTS.append({
                    "name": str(w.get("name") or _os.path.basename(p)),
                    "path": p,
                    "allowRun": bool(w.get("allowRun")),
                })
        except Exception:  # noqa: BLE001
            continue
    _WS_AVAILABLE = bool(_WS_ROOTS)
    # run_command policy overrides from Settings (both optional, default = allow all but
    # sudo). Accept either a list or a comma/space-separated string; basenames so a user
    # can write "git" and match "/usr/bin/git".
    global _WS_RUN_ALLOWLIST, _WS_RUN_DENYLIST

    def _split_cmds(v):
        import re as _re
        items = v if isinstance(v, (list, tuple)) else _re.split(r"[,\s]+", str(v or ""))
        return {str(c).strip() for c in items if str(c).strip()}

    _WS_RUN_ALLOWLIST = _split_cmds(cfg.get("runAllowlist"))
    _WS_RUN_DENYLIST = _split_cmds(cfg.get("runDenylist"))
    # Active terminal workspace: the folder the `gonext` REPL was launched from (sent as
    # payload.activeWorkspace = the terminal's cwd). download_file / unzip_file put their
    # output HERE instead of the shared ~/.gonext/rag-work cache, so files land where the
    # user opened the terminal. Honoured for any real directory the user explicitly opened
    # the terminal in — these tools already write outside registered workspaces (to
    # ~/.gonext), so this is not a new write boundary; the code-EDIT tools stay gated on
    # _WS_ROOTS (registered workspaces) via _ws_write_allowed, unchanged.
    _WS_ACTIVE = ""
    _aw_raw = str(cfg.get("activeWorkspace") or "").strip()
    if _aw_raw:
        try:
            import os as _os
            aw = _os.path.realpath(_os.path.expanduser(_aw_raw))
            if _os.path.isdir(aw):
                _WS_ACTIVE = aw
                registered = any(
                    aw == w["path"] or aw.startswith(w["path"] + _os.sep) for w in _WS_ROOTS
                )
                _log(f"active workspace: {aw} (download/unzip output → here"
                     f"{'' if registered else '; not a registered workspace — reads/writes here limited to download/unzip'})")
            else:
                _log(f"active workspace ignored (not a directory): {aw}")
        except Exception as _e:  # noqa: BLE001
            _WS_ACTIVE = ""
            _log(f"active workspace ignored (error resolving '{_aw_raw}'): {_e}")
    if _WS_AVAILABLE:
        _log(f"workspaces: {[(w['name'], w['path'], w['allowRun']) for w in _WS_ROOTS]}")
        # Coding tasks are edit→test→fix loops — the normal budget is too tight. Bump to
        # the backend-aware workspace budget (openai 40 / ollama 20 / MLX 12) when
        # workspaces are registered, unless the payload set one explicitly.
        if not cfg.get("maxSteps") and max_steps < _ws_budget:
            max_steps = _ws_budget
            _log(f"workspace registered → step budget raised to {_ws_budget} (no explicit maxSteps)")
        # Auto-test adds a verify → fix → re-test cycle on top of the work itself, so give
        # it more room (unless the user pinned a budget). Kept modest so a stuck test loop
        # still terminates rather than burning a huge budget.
        if _auto_test and not cfg.get("maxSteps") and max_steps < 16:
            max_steps = 16
            _log("auto-test on → step budget raised to 16 (no explicit maxSteps)")

    def _rag_s3_and_loc():
        client = _rag_s3_client(rag_region, rag_akid, rag_secret)
        bucket, prefix = _rag_parse_location(rag_location)
        return client, bucket, prefix

    def _rag_prefix_for(source_url: str, prefix: str) -> str:
        key = _rag_source_key(source_url)
        base = (prefix + "/") if prefix else ""
        return f"{base}agent-rag/{key}"

    def _rag_load_chunks(client, bucket: str, base_prefix: str) -> list:
        """Load all chunk records (across shards) for a source. Returns [] if none."""
        out = []
        token = None
        while True:
            kw = {"Bucket": bucket, "Prefix": f"{base_prefix}/chunks"}
            if token:
                kw["ContinuationToken"] = token
            resp = client.list_objects_v2(**kw)
            for obj in resp.get("Contents", []) or []:
                if not obj["Key"].endswith(".jsonl"):
                    continue
                body = client.get_object(Bucket=bucket, Key=obj["Key"])["Body"].read()
                for line in body.decode("utf-8", "replace").splitlines():
                    line = line.strip()
                    if line:
                        try:
                            out.append(json.loads(line))
                        except json.JSONDecodeError:
                            pass
            if resp.get("IsTruncated"):
                token = resp.get("NextContinuationToken")
            else:
                break
        return out

    # --- Storage backend abstraction (task #97): local disk vs the user's S3, chosen by
    # ragMode. The three rag_* tools go through these so their logic stays identical. ---
    def _rag_write_shard(source_url: str, records: list, tag: str) -> None:
        """Write one JSONL shard of chunk records. tag is 'chunks' (index) or 'chunks-add'."""
        import os
        import time as _t
        body = "\n".join(json.dumps(r, ensure_ascii=False) for r in records)
        name = f"{tag}-{int(_t.time())}.jsonl"
        if _RAG_LOCAL:
            path = os.path.join(_rag_local_index_dir(source_url), name)
            with open(path, "w", encoding="utf-8") as fh:
                fh.write(body)
            return
        client, bucket, prefix = _rag_s3_and_loc()
        base = _rag_prefix_for(source_url, prefix)
        client.put_object(Bucket=bucket, Key=f"{base}/{name}",
                          Body=body.encode("utf-8"), ContentType="application/x-ndjson")

    def _rag_write_manifest(source_url: str, manifest: dict) -> None:
        import os
        data = json.dumps(manifest).encode("utf-8")
        if _RAG_LOCAL:
            with open(os.path.join(_rag_local_index_dir(source_url), "manifest.json"), "wb") as fh:
                fh.write(data)
            return
        client, bucket, prefix = _rag_s3_and_loc()
        base = _rag_prefix_for(source_url, prefix)
        client.put_object(Bucket=bucket, Key=f"{base}/manifest.json",
                          Body=data, ContentType="application/json")

    def _rag_load_all(source_url: str) -> list:
        """Load every chunk record for a source across all shards (local or S3)."""
        import glob as _glob
        import os
        if _RAG_LOCAL:
            out = []
            for fp in sorted(_glob.glob(os.path.join(_rag_local_index_dir(source_url), "chunks*.jsonl"))):
                try:
                    with open(fp, "r", encoding="utf-8", errors="replace") as fh:
                        for line in fh:
                            line = line.strip()
                            if line:
                                try:
                                    out.append(json.loads(line))
                                except json.JSONDecodeError:
                                    pass
                except OSError:
                    continue
            return out
        client, bucket, prefix = _rag_s3_and_loc()
        base = _rag_prefix_for(source_url, prefix)
        return _rag_load_chunks(client, bucket, base)

    def _prior_email_preview() -> bool:
        """True if an earlier assistant turn showed a send_email preview awaiting confirm."""
        for m in reversed(messages[:last_user_idx]):
            if m.get("role") == "assistant":
                c = m.get("content") or ""
                if "📧" in c and "About to email" in c:
                    return True
        return False

    def _email_confirm_pending() -> bool:
        """A send is authorized only when the CURRENT message confirms AND a preview
        was already shown — the weak model cannot self-confirm."""
        return bool(_EMAIL_AVAILABLE and _EMAIL_CONFIRM.search(latest_user_text or "")
                    and _prior_email_preview())

    def _prior_turn_was_interrupted() -> bool:
        """True if the immediately preceding assistant turn is one of OUR OWN
        interrupted-investigation markers (the honest degrade-fallback note, or the
        max-steps 'ran out of steps' note) — i.e. a genuine tool-use attempt started but
        didn't finish, as opposed to a normal completed answer."""
        for m in reversed(messages[:last_user_idx]):
            if m.get("role") == "assistant":
                c = m.get("content") or ""
                return ("ran out of steps before finishing" in c
                        or "couldn't finish the full investigation" in c)
        return False

    def _continuation_pending() -> bool:
        """A bare 'continue'/'keep going' after an interrupted investigation must reach
        the AGENT, not plain chat — plain chat has zero tool access and can never
        actually continue a stalled grep_repo/read_file_lines investigation, which is
        exactly the reported bug: 'continue' silently degraded to a tool-less reply that
        could only apologize, never resume. Two independent interruption signals:
        (a) our own interrupted-investigation marker in the prior assistant turn (the
        max-steps note or the degrade-fallback note); (b) an on-disk turn checkpoint for
        this workspace — a CRASHED turn leaves NO history marker at all (the REPL
        discards failed turns entirely, by design — task #35), so the checkpoint file is
        the only surviving evidence there is work to resume. Gated tightly (short
        continuation phrase + a genuine interruption signal) so it doesn't misfire on an
        unrelated 'continue' after a normal, already-complete answer."""
        return bool(_CONTINUE_CUE.match((latest_user_text or "").strip())
                    and (_prior_turn_was_interrupted()
                         or (_WS_ACTIVE and _turn_checkpoint_exists(_WS_ACTIVE))))

    def _ws_action_pending() -> bool:
        """An imperative file/folder action ("create a folder abc") with a registered
        workspace must reach the AGENT — plain chat can only answer with mkdir
        instructions (the reported task-#40 bug). How-to QUESTIONS are excluded: the
        user asking "how do I create a folder in Windows?" wants an explanation."""
        t = (latest_user_text or "").strip()
        return bool(_WS_AVAILABLE
                    and not _HOWTO_PREFIX_RE.match(t)
                    and _WS_ACTION_RE.search(t))

    # Routing. Two fundamentally different contexts:
    #
    # (A) WORKSPACE MODE (a folder is registered — i.e. the `gonext` terminal, or web
    #     with a workspace): DEFAULT TO THE AGENT. The user opened a code folder and
    #     registered it for the agent to read/edit/run — everything they type is
    #     presumptively a task to DO, not a question to classify. We do NOT ask a model
    #     classifier "does this need tools?" here: on a weak local model that guess is
    #     unreliable, and every wrong guess sends an action request ("create a reactjs
    #     project and start it", "do it for me") to plain chat, which can only hand back
    #     instructions — the exact instability the user rejected. The ONLY escape is the
    #     deterministic greeting/thanks fast-path, so "hi"/"thanks" stay instant instead
    #     of spinning up the coding model. Anything else → agent, which still just
    #     answers via final_answer when no tool is needed.
    #
    # (B) NO WORKSPACE (plain web chat): keep the keyword + model classifier — there are
    #     no file/run tools to act with, so chat-vs-tool routing still earns its keep.
    _emit({"type": "step", "text": "Routing your request…"})
    if _is_greeting_smalltalk(latest_user_text) and not _AGENT_KEYWORDS.search(latest_user_text or ""):
        _log("router → NO (greeting/smalltalk — local fast-path)")
        _emit({"type": "step", "text": "→ Chat reply"})
        needs_agent = False
    elif _WS_AVAILABLE:
        _log("router → YES (workspace registered → act by default, no classifier)")
        _emit({"type": "step", "text": "→ Agent mode (workspace)"})
        needs_agent = True
    else:
        needs_agent = _route(latest_user_text, agent_base_url, agent_api_key, agent_model_id)
    # A pending email awaiting 'confirm' must reach the agent even though a bare
    # 'confirm' / 'send it' is not a network keyword — otherwise the send never fires.
    if not needs_agent and _email_confirm_pending():
        _log("router → YES (email confirm pending)")
        _emit({"type": "step", "text": "→ Agent mode (email confirm)"})
        needs_agent = True
    # A bare "continue" after WE reported an interrupted investigation must reach the
    # agent too — plain chat has zero tool access, so it can only apologize again, never
    # actually resume the grep_repo/read_file_lines work (the reported bug).
    if not needs_agent and _continuation_pending():
        _log("router → YES (continuation of an interrupted investigation)")
        _emit({"type": "step", "text": "→ Agent mode (continuing)"})
        needs_agent = True
    # An imperative file/folder action with a workspace registered must be DONE with
    # tools, not answered with terminal instructions — the classifier misreads these
    # as how-to questions (task #40: "create a folder abc" got a mkdir tutorial).
    if not needs_agent and _ws_action_pending():
        _log("router → YES (workspace file/folder action)")
        _emit({"type": "step", "text": "→ Agent mode (workspace action)"})
        needs_agent = True

    if not needs_agent:
        _log("router: plain chat (no HTTP needed)")
        _emit({"type": "step", "text": "Composing a reply…"})
        # Small prompt → fast. Fall back to the coding model if the chat model is down.
        answer = _plain_reply(messages, agent_base_url, agent_api_key, agent_model_id,
                              coding_base_url, coding_model_id)
        _log(f"plain reply: {len(answer)} chars")
        _emit({"type": "final", "text": answer})
        return

    # Agent path — from here all step events go into <think>.
    _log("router: agent (HTTP tool use needed)")
    _emit({"type": "step", "text": "Choosing a tool…"})

    # True when the user wants a PDF but its content must first be researched by the
    # agent loop (set below); read by the max-steps fallback to force-deliver a PDF.
    agent_pdf_requested = False

    # ---- Deterministic PDF fast-path -------------------------------------------------
    # A small model cannot reliably re-emit a long, emoji/quote-heavy document as a
    # Python string literal for create_pdf(text="…"), so the CodeAgent call fails before
    # the tool ever runs. When the user clearly wants a PDF, extract their real text and
    # run format → render → upload directly. No model string-echoing, and 1 fewer call.
    # (pdf_api_base / pdf_worker_key are read once near the top of run_agent_chat.)
    if _wants_pdf(latest_user_text):
        doc_text = _extract_pdf_doc_text(latest_user_text)
        # A2: a follow-up like "in the pdf file" / "add the teams to the pdf" carries
        # no document body — it refers to the last thing the assistant produced. Pull
        # the most recent substantial assistant turn as the base document and treat the
        # current user message as an edit instruction the formatter should apply.
        edit_instruction = ""
        prior_doc = ""
        if len(doc_text.strip()) < 40:
            for m in reversed(messages[:last_user_idx]):
                if m.get("role") != "assistant":
                    continue
                cand = think_re.sub("", (m.get("content") or "")).strip()
                low = cand.lower()
                # Skip our own PDF-ready confirmations / bare links — not document bodies.
                if cand.startswith("✅") or low.startswith("http") or "amazonaws.com" in low:
                    continue
                if len(cand) >= 40:
                    prior_doc = cand
                    break
            if prior_doc:
                edit_instruction = latest_user_text.strip()
                doc_text = prior_doc

        # Guard: if the PDF's content must be RESEARCHED (not provided in the message, not
        # from a prior turn), don't render the instruction text into a PDF — fall through to
        # the agent loop so it can web_search/fetch_url the facts and THEN create_pdf them.
        if (not prior_doc
                and not _pdf_has_explicit_body(latest_user_text)
                and _PDF_RESEARCH_INTENT.search(latest_user_text)):
            _log("PDF fast-path SKIPPED: content needs research (no provided body) "
                 "→ agent loop (web_search → create_pdf)")
            # Remember the user wants a PDF: if the agent loop exhausts its steps without
            # calling create_pdf, the max-steps fallback renders the gathered facts anyway.
            agent_pdf_requested = True
        else:
            doc_title = _derive_pdf_title(doc_text)
            _log(
                f"PDF fast-path: title={doc_title!r} doc_chars={len(doc_text)}"
                + (f" edit={edit_instruction[:60]!r}" if edit_instruction else "")
            )

            _emit({"type": "step", "text": "Formatting document…"})
            # CHAT model, not the coder (task #74 review): formatting raw text into
            # Markdown is a language task, and the chat model (local MLX) answers in
            # seconds where the remote coder can take minutes to first token — inside
            # the 20s format timeout that difference is "formatted" vs "always falls
            # back to raw text". Same split as routing/plain replies/summarization.
            markdown_text = _format_text_for_pdf(
                doc_text, doc_title,
                agent_base_url or coding_base_url, agent_api_key,
                agent_model_id or coding_model_id,
                instruction=edit_instruction,
            )

            _emit({"type": "step", "text": "Rendering PDF…"})
            try:
                pdf_bytes = _render_pdf_bytes(markdown_text, doc_title)
            except RuntimeError as e:
                msg = str(e)
                _log(f"PDF fast-path render error: {msg[:200]}")
                _emit({"type": "final", "text": msg})
                return
            except Exception as e:  # noqa: BLE001
                _log(f"PDF fast-path render crash: {type(e).__name__}: {e!r}\n{traceback.format_exc()}")
                _emit({"type": "final", "text": (
                    f"Sorry — I couldn't render that into a PDF ({type(e).__name__}: {str(e)[:160]})."
                )})
                return

            _emit({"type": "step", "text": "Uploading PDF…"})
            try:
                download_url = _pdf_upload_via_api(
                    pdf_api_base, pdf_worker_key, f"{doc_title}.pdf", pdf_bytes
                )
            except Exception as e:  # noqa: BLE001
                _log(f"PDF fast-path upload error: {type(e).__name__}: {e!r}\n{traceback.format_exc()}")
                _emit({"type": "final", "text": (
                    f"PDF was created but could not be uploaded: {str(e)[:200]}"
                )})
                return

            out = (
                f"✅ Your PDF \"{doc_title}\" is ready.\n"
                f"Download it here (link valid for a limited time):\n{download_url}"
            )
            _log(f"PDF fast-path ok title={doc_title!r} bytes={len(pdf_bytes)}")
            _emit({"type": "final", "text": out})
            return
    # ---------------------------------------------------------------------------------

    # Prepend explicit tool instructions so small models pick the right tool, never
    # fabricate URLs/responses, and always terminate with final_answer().
    from datetime import datetime as _dt_now
    now_str = _dt_now.now().astimezone().strftime("%A, %d %B %Y, %H:%M %Z")
    # The numbered tool list itself is generated LATER from the real registered tool
    # objects (see _render_numbered_tool_list, spliced into task_with_hint below via the
    # {{TOOL_LIST}} placeholder) — it can never drift or miscount, unlike a hand-written
    # list with a manually-incremented number. The PDF reader / send_email / RAG /
    # workspace tools are only ever REGISTERED (further below) when available, so an
    # unavailable tool is never advertised.
    _rag_tool_block = (
        "  FILES & KNOWLEDGE BASE — to SUMMARIZE or ANSWER questions about a ZIP of files at a URL:\n"
        "    - download_file(url) — download a file (e.g. a .zip) to this machine.\n"
        "    - unzip_file(zip_path) — extract a downloaded .zip locally.\n"
        "    - list_dir(path) — list the files inside an unzipped folder.\n"
        "    - read_text_file(path) — read ONE file's text (e.g. a README) directly. Best for a QUICK "
        "project summary — you do NOT need to index for that.\n"
        "    - rag_index(path, source_url) — index MANY text/code files into a searchable knowledge "
        "base (embeds the chunks). source_url MUST be the ORIGINAL url the user gave. Use for deep Q&A "
        "across the whole project.\n"
        "    - rag_add(source_url, text) — add extra info the user provides to that knowledge base.\n"
        "    - rag_search(source_url, query) — retrieve the most relevant chunks to answer a question.\n"
        "    NOTE: you may ONLY read files with these tools — plain `import os`/`open()` is blocked.\n"
    ) if _RAG_AVAILABLE else ""
    _rag_choose_line = (
        "- user gives a URL to a ZIP and asks to SUMMARIZE it -> download_file(url) THEN unzip_file(zip) "
        "THEN list_dir(dir) to find the README, THEN read_text_file(that README) THEN answer from it. "
        "This is the FASTEST path for a summary — do NOT index unless deep Q&A across many files is needed.\n"
        "- user wants deep Q&A across the WHOLE project -> download_file THEN unzip_file THEN "
        "rag_index(dir, source_url=url) THEN rag_search(source_url=url, query=<question>) THEN answer.\n"
        "- FOLLOW-UP question about a ZIP url from EARLIER in the conversation -> try "
        "rag_search(source_url=url, query=<question>) FIRST. If it says no knowledge base exists, "
        "run download_file(url) + unzip_file(zip) again — both are CACHED per url and return "
        "instantly — then list_dir/read_text_file or rag_index as needed. "
        "If the user adds information, rag_add(source_url=url, text=...).\n"
    ) if _RAG_AVAILABLE else ""
    _ws_names = ", ".join(f"{w['name']} = {w['path']}" for w in _WS_ROOTS)
    # The folder the terminal is OPEN in (where `gonext` was launched). This is where the
    # user means "here"/"this workspace" and where a no-path/no-workdir tool call defaults
    # (see _default_ws_root). Registered workspaces accumulate over time, so WITHOUT this
    # the model would pick an arbitrary/old one (bug #46: opened in t9, built in t1).
    _active_ws = _ws_for_path(_WS_ACTIVE) if _WS_ACTIVE else None
    _active_label = (
        f"{_active_ws['name']} = {_WS_ACTIVE}" if _active_ws else (_WS_ACTIVE or "")
    )
    _ws_current_line = (
        f"  CURRENT folder — the terminal is open HERE; default ALL create/edit/run/"
        f"download operations to THIS folder unless the user EXPLICITLY names another "
        f"workspace: {_active_label}\n"
        "  Ignore any OTHER folder paths that appear only in EARLIER messages (a previous "
        "task, a cancelled turn) — they are stale. Do NOT grep/list/read across parent "
        "directories or unrelated projects; work in the CURRENT folder above.\n"
        if _active_label else ""
    )
    # Upfront, ZERO-COST (no tool call, no model round-trip) top-level listing — so the
    # agent already knows the workspace is non-empty and roughly what's in it before its
    # first step, instead of having to spend (and risk losing to a timeout) a whole
    # round-trip just calling list_dir to discover that. Only the CURRENT folder (not every
    # accumulated workspace) — to keep the model anchored on where it should act.
    _overview_roots = (
        [_active_ws] if _active_ws else _WS_ROOTS
    ) if _WS_AVAILABLE else []
    _ws_overview = _workspace_overview(_overview_roots) if _overview_roots else ""
    # VARIABLE workspace context (task #86): the active-folder line, workspace list and
    # the on-disk folder overview change between turns (the agent creates/edits files),
    # so they must NOT sit inside the static tool_hint — they'd bust the prefix cache
    # mid-reference every turn. They join the variable tail of task_with_hint instead.
    # Task #90 Phase 2 — environment visibility: prune/cap the background-server
    # registry once per turn (stops the oldest beyond 3 — kills the leak class), then
    # SHOW the survivors to the model so a 'port busy' is instantly explicable and
    # reuse/stop_server is actionable. Variable (server state changes) → tail.
    _ws_servers_line = ""
    if _WS_AVAILABLE:
        try:
            import os as _os_srv
            _live_srv = _ws_cap_servers()
            if _live_srv:
                _descs = ", ".join(
                    ":" + ",".join(str(p) for p in (s.get("ports") or ["?"]))
                    + f" ('{str(s.get('command'))[:30]}' in "
                    + f"{_os_srv.path.basename(str(s.get('workdir') or '')) or '?'})"
                    for s in _live_srv)
                _ws_servers_line = (
                    f"  Background servers YOU already started (still running): {_descs}. "
                    "REUSE one if it fits the task, or stop_server(<port>) — their ports "
                    "are TAKEN, so never start a duplicate on one of them.\n")
        except Exception as _e:  # noqa: BLE001
            _log(f"server context skip: {_e}")
    _ws_context_block = (
        _ws_current_line
        + f"  Other registered workspaces (use only if the user names them): {_ws_names}\n"
        + _ws_servers_line
        + f"  Current folder contents:\n"
        + "\n".join(f"    {line}" for line in _ws_overview.splitlines()) + "\n"
    ) if _WS_AVAILABLE else ""
    _ws_tool_block = (
        # Task #90 (root-cause rev): senior-dev autonomy as GENERAL principles — outcome
        # over exit code, own the environment, evidence before finishing, switch approach.
        # Static (cached, per #86); deliberately not tied to any specific failure case.
        "WORK LIKE A SENIOR DEVELOPER — you OWN the task until it DEMONSTRABLY works:\n"
        "    * OUTCOME over exit code: a command exiting 0 is NOT the goal. After any "
        "state-changing step (start/build/install/edit), VERIFY the intended EFFECT "
        "(server responds, artifact exists, test green, file contains the change) before "
        "moving on or finishing.\n"
        "    * On failure: read the error, fix the ROOT CAUSE (edit the code/config, "
        "install the missing dependency, adjust the environment), re-run to confirm. If "
        "the SAME fix fails twice, SWITCH APPROACH — never repeat it, never just give up.\n"
        "    * The environment is YOURS to manage: a busy port, missing package, or stale "
        "process you started earlier is yours to RESOLVE with the tools (e.g. start on a "
        "free port via 'env VAR=value cmd', stop_server your own old server) — not to "
        "report back as the user's problem.\n"
        "    * final_answer needs EVIDENCE: say what you verified and how. If a thing "
        "truly cannot be done with these tools, state plainly WHAT failed and WHY — but "
        "NEVER end with 'you can do it yourself' when a tool could have done it.\n"
        "    - list_dir(path) / read_text_file(path) — browse and read files.\n"
        "    - grep_repo(pattern, path='', glob='') — search code, returns file:line hits. "
        "Use this to find the EXACT lines to change.\n"
        "    - read_file_lines(path, start_line, end_line) — read WITH line numbers "
        "before editing. Read a SMALL range around the spot you need (large reads are "
        "truncated and slow every later step). Use grep_repo to LOCATE first, then read "
        "~40 lines around the hit — do NOT re-read the whole file or the same range twice.\n"
        "    - edit_lines(path, start_line, end_line, new_content) — PRIMARY edit tool: "
        "replace a line range (backup saved automatically).\n"
        "    - edit_file(path, old_string, new_string) — replace one exact unique string "
        "(must match byte-for-byte; prefer edit_lines).\n"
        "    - create_file(path, content, overwrite=False) — create a file; pass "
        "overwrite=True to REPLACE an existing file in ONE call (don't read+edit_lines "
        "just to rewrite everything).\n"
        "    - run_command(command, workdir='') — run a build/test command (npm test, mvn "
        "test, pytest…) to VERIFY changes, or start an app. A server/desktop-GUI app is "
        "left RUNNING (its window opens on the user's Mac); stop_server closes it. An "
        "interactive input() CLI can't be driven here (EOFError) — make it non-interactive "
        "or give the user the command.\n"
        "      Most commands run directly — build/test/package tools, git, docker, kubectl, "
        "terraform, aws/gcloud/az, and so on. Two limits: `sudo`/`su` are blocked (no root); "
        "and there is NO shell, so pipes `|`, redirects `>`, `&&`, and backticks do NOT work "
        "— run one command per call, or use python3 for anything that needs composition.\n"
        "    - deploy_web(local_dir, host, user, remote_path, domain='') — DEPLOY a built "
        "static site in ONE call: build first (run_command('npm run build')), then "
        "deploy_web('my-app/build', host, user, remote_path, domain). It rsyncs over your "
        "SSH KEY, finishes nginx via passwordless sudo when possible, else writes deploy.sh "
        "+ returns the exact server commands; if key auth isn't set up it returns the "
        "one-time `ssh-copy-id` to run. PREFER THIS over hand-writing scp/ssh/rsync.\n"
        "      Raw ssh/scp/rsync are also allowed for other remote work — always KEY auth "
        "(`-o BatchMode=yes`), NEVER a password in a command/file/reply (no stdin here), "
        "no paramiko/expect with hardcoded creds; remote sudo fails without a TTY.\n"
        "    NOTE: plain `import os`/`open()` is blocked — use ONLY these tools for files.\n"
        "    WRITING FILE CONTENT: multi-line content MUST use \\n for newlines (or a "
        "triple-quoted string). NEVER put a raw line break inside a normal \"...\" — it "
        "causes 'unterminated string literal'.\n"
    ) if _WS_AVAILABLE else ""
    _ws_choose_line = (
        "- user asks to FIX A BUG / MODIFY / ADD CODE in a workspace -> "
        "grep_repo(pattern) (and/or rag_search if indexed) to FIND the file, THEN "
        "read_file_lines(file, around the hit) to see exact lines, THEN "
        "edit_lines(file, start, end, new_content) to change them, THEN "
        "run_command('…test…') to VERIFY if run permission exists — read failures and fix, "
        "iterate. Finish with final_answer summarizing WHAT you changed (files + lines). "
        "AFTER editing, re-read with grep_repo/read_file_lines (an indexed knowledge base may be "
        "stale — do not trust rag_search for freshly edited code).\n"
    ) if _WS_AVAILABLE else ""
    _pdf_read_choose_line = (
        "- 'read'/'summarize'/'what does the PDF say' for an existing PDF URL -> "
        "extract_text_from_pdf(url).\n"
    ) if _PDF_READ_AVAILABLE else ""
    _email_choose_line = (
        "- 'email …' / 'send an email to …' -> send_email(to, subject, body) "
        "(previews first, then sends on confirm).\n"
    ) if _EMAIL_AVAILABLE else ""
    # Auto-test mode (/test-auto): a directive that makes the agent VERIFY its changes
    # and fix → re-test until they pass. Only meaningful for workspace code work; empty
    # otherwise so it costs no prompt-eval when off. Climbs the cheapest-sufficient
    # verification ladder and bakes in the guardrails (freeze the pass criterion, ignore
    # flakes, prove the test ran, cap retries) so it converges instead of chasing green.
    _auto_test_block = (
        "\nAUTO-TEST MODE (ON): after you CHANGE code you MUST verify it before "
        "final_answer — never finish on an untested change. Climb the CHEAPEST check that "
        "proves the change, only as far as needed:\n"
        "  1. grep_repo the edited file to confirm the change landed (old code gone).\n"
        "  2. STATIC (cheapest — nothing runs): typecheck + lint. run_command('tsc --noEmit' "
        "/ 'mypy' / 'go vet'; 'eslint' / 'ruff'). Fix errors here first.\n"
        "  3. build / compile if the project does (npm run build / go build).\n"
        "  4. run the project's tests (npm test / pytest / …) if they exist — prefer the "
        "SINGLE test/file covering your change for speed (e.g. pytest path::test_x).\n"
        "  5. if it's a running app, start it and curl/fetch_url it, then check the response "
        "contains what the change should produce.\n"
        "FIXING A BUG → write a MINIMAL test that reproduces it and run it FIRST: it must FAIL "
        "(red) BEFORE your fix and PASS (green) after. A test that passes before you change "
        "anything proves nothing. Keep it with the project's other tests.\n"
        "Before final_answer, run_command('git diff') (or 'git status' if not a git repo) to "
        "review EVERYTHING you changed — only what you intended, no stray edits or debug prints.\n"
        "DECIDE what 'passes' BEFORE you fix — do NOT weaken the check to make it green. On a "
        "FAIL: read the error, make ONE fix, RE-TEST the SAME way. Passes only after a no-change "
        "re-run = flaky, stop editing. Give up after 3 fix attempts on the same failure and "
        "final_answer honestly what still fails. Confirm a test actually exercised the change "
        "(the asserted behavior really ran), not an empty/no-op pass.\n"
        if (_auto_test and _WS_AVAILABLE) else ""
    )
    # Deploy target (/server): when the user picked one, tell the agent to deploy THERE
    # with key auth. Host/user only — the block never contains a password.
    _deploy_block = ""
    if _deploy_server and _deploy_server.get("host") and _deploy_server.get("user"):
        _dh = str(_deploy_server.get("host")).strip()
        _du = str(_deploy_server.get("user")).strip()
        _dn = str(_deploy_server.get("name") or _dh).strip()
        # Diagnostic: prove the picked server reached python (visible in the worker log).
        _log(f"deploy target: {_du}@{_dh} (server '{_dn}') — injecting into the prompt")
        _deploy_block = (
            f"\n*** DEPLOY TARGET — the user already chose a server: '{_dn}' → host={_dh}, "
            f"user={_du}. Do NOT search the project for host/user/credentials and do NOT ask "
            f"for them; you already have them. When the task is to DEPLOY, go straight to "
            f"deploy_web(local_dir, host='{_dh}', user='{_du}', remote_path='/var/www/<domain>', "
            f"domain='<domain>') — or ssh/scp/rsync to {_du}@{_dh}. KEY auth ONLY "
            f"(-o BatchMode=yes); NEVER ask for or type a password. If key auth isn't set up "
            f"(Permission denied publickey,password), STOP and tell the user to run "
            f"'ssh-copy-id {_du}@{_dh}' once, then retry. ***\n"
        )
    # STATIC ONLY (task #86): everything in tool_hint must be byte-identical between
    # turns so the model server's prefix KV-cache covers it (see task_with_hint below).
    # Variable text — the deploy target and the current date/time — lives in the
    # variable tail next to the task instead.
    tool_hint = (
        f"Solve the TASK step by step (up to {max_steps} steps). At EACH step write a "
        "brief Thought, then ONE code block that calls a SINGLE tool. You will SEE that "
        "tool's result (Observation) before the next step — use it to decide what to do "
        "next (e.g. web_search to find a URL, THEN fetch_url to read it). When you have "
        "enough to answer, call final_answer(<your answer>) in a code block. Prefer "
        "FEWER steps, and never repeat a tool call that already succeeded.\n\n"
        "{{TOOL_LIST}}\n"  # filled in below, once from the REAL registered tool objects
        + _rag_tool_block + _ws_tool_block + _auto_test_block +
        "\n"
        "http_request RETURN FORMAT: 'HTTP 200\\n{body}' — first line is 'HTTP <code>', body follows.\n"
        "\n"
        "AUTH — ONLY when the TASK itself provides credentials or a token. If it does "
        "NOT, call http_request with NO username/password/headers. NEVER invent or copy "
        "the placeholder values below.\n"
        "  BASIC AUTH — pass username= and password= (auto base64; never build a 'Basic ' header):\n"
        "      http_request('GET', url, username=<username from the task>, password=<password from the task>)\n"
        "  BEARER TOKEN — pass it in headers:\n"
        "      http_request('GET', url, headers='{\"Authorization\": \"Bearer <token from the task>\"}')\n"
        "\n"
        "CHOOSING A TOOL (match the TASK, not these examples):\n"
        "- ONLY a date/time question (e.g. 'what is the date today') -> get_current_datetime().\n"
        "- 'who is' / 'what is' / 'tell me about' / a person / place / topic / general "
        "knowledge -> web_search(query).\n"
        "- 'read this page' / 'open this link' / 'what does <URL> say' -> fetch_url(url).\n"
        "- any arithmetic, percentage, or 'how much is' math -> calculate(expression).\n"
        "- a live/current NUMBER with no URL given (price, exchange rate, weather, score, "
        "stock/crypto) -> web_search(query). Do NOT guess an API URL for these.\n"
        "- 'create/make/generate/export a PDF' of some text/data -> create_pdf(text, title).\n"
        "- a PDF of info you must LOOK UP first (e.g. 'get the world cup schedule then make a "
        "pdf') -> FIRST web_search/fetch_url to gather the real facts, THEN in a later step "
        "create_pdf(text=<the gathered facts>, title=...). Never PDF the request wording itself.\n"
        + _pdf_read_choose_line + _email_choose_line + _rag_choose_line + _ws_choose_line +
        "- A specific known API/URL was given -> http_request().\n"
        "\n"
        "RULES:\n"
        "- NEVER invent or guess a URL. If you have no real URL, use web_search() instead. "
        "If nothing works, call final_answer explaining what you need — do NOT make up an answer.\n"
        "- Only report what a tool ACTUALLY returned. Never fabricate a response, body, or status code.\n"
        "- Pass an http_request response DIRECTLY to final_answer — do NOT split, parse, or index it.\n"
        "- When a call's result is UNCERTAIN (any network call), do NOT call final_answer "
        "in the SAME block. Call the tool alone, READ the Observation, THEN answer next step.\n"
        "- If a response starts with 'HTTP 2' it SUCCEEDED — answer with it next step.\n"
        "- NEVER pass an 'Error:' / HTTP 4xx / 5xx result to final_answer. On an error, try a "
        "DIFFERENT approach next step (e.g. web_search); only give up after a few tries.\n"
        "- CONVERGE — do NOT keep searching for a 'more complete' or 'perfect' source; a single "
        "clean one often does not exist. After 1-2 searches/fetches, COMPILE what you have and "
        "finish: if a PDF was asked for, call create_pdf(text=<your compiled text>, title=...); "
        "otherwise call final_answer. NEVER repeat a web_search/fetch_url you already ran.\n"
        f"- RESEARCH BUDGET — you may make at most {research_budget} web_search/fetch_url calls "
        "TOTAL per task (rewording a query still counts). Reserve your remaining steps for "
        "create_pdf / final_answer.\n"
        "- If an earlier Observation already says '✅ Your PDF … is ready', the PDF EXISTS — "
        "do NOT call create_pdf again. Pass that download link to final_answer.\n"
        "- Do NOT put final_answer outside the code block.\n\n"
    )
    # Recover an interrupted prior turn's step trace, if one survives on disk (crash/
    # connection drop mid-investigation — see _write_turn_checkpoint). Consumed HERE,
    # inside the agent path only, and not up front before routing: task_text feeds only
    # the agent prompt (plain chat sends the raw `messages` list to _plain_reply), so an
    # earlier consume would let any plain-chat-routed message delete the checkpoint
    # without any model ever seeing it — silently destroying the recovery data.
    if _WS_ACTIVE:
        _ck = _load_and_clear_turn_checkpoint(_WS_ACTIVE)
        if _ck and _ck.get("steps"):
            _ck_steps = _ck["steps"]
            # Newest-first budget walk (same pattern as the history condenser above):
            # the latest steps matter most for resuming, and an uncapped 12-step trace
            # (~25K chars at the per-step clip limits) would dominate prompt-eval on a
            # slow coding server — the user's box already takes 3+ min at half that.
            _kept, _used = [], 0
            for _i, _s in enumerate(reversed(_ck_steps)):
                if not isinstance(_s, dict):
                    _s = {"did": str(_s), "found": ""}
                _seg = (f"Step {_s.get('step', len(_ck_steps) - _i)}:\n"
                        f"  Did: {_s.get('did', '')}\n  Found: {_s.get('found', '')}")
                if _used + len(_seg) > 5000 and _kept:
                    _kept.append(f"(…{len(_ck_steps) - len(_kept)} earlier step(s) omitted)")
                    break
                _kept.append(_seg)
                _used += len(_seg)
            _kept.reverse()
            _log(f"recovered {len(_ck_steps)} step(s) from an interrupted prior turn "
                 f"in this workspace ({_used} chars folded into the task)")
            task_text += (
                "\n\nNOTE: your previous attempt at a task in this workspace was "
                "interrupted before finishing (crash or connection drop) — here is "
                "exactly what was already done and found. Do NOT repeat these steps; "
                "continue the investigation/edit from here:\n"
                f"Previous task: {_ck.get('task', '')}\n" + "\n".join(_kept)
            )

    # ORDER (task #86 — prefix-cache fix): STATIC first, VARIABLE last.
    # The ~16k-char tool reference is byte-identical between turns; leading with it
    # (right after the equally static system prompt) lets the model server's prefix
    # KV-cache cover ~7k tokens across turns. Previously the variable task text came
    # FIRST, so every new turn forced a cold re-eval of the whole reference (~2 min to
    # first token on step 1 of each turn; steps 2+ were 10-30s because within a turn
    # the prompt only grows at the end).
    # The TASK now comes LAST, under a loud header: recency is the position a model
    # attends to most, so anchoring is preserved (the historical "hint led with the
    # date → 3B model answered everything as a date question" regression was about a
    # misleading LEAD, not about where the task sits). The two VARIABLE blocks — the
    # current date/time and the deploy target — live here in the tail, next to the
    # task: the deploy target must stay near the task, not buried mid-hint (#69), and
    # the timestamp changes every minute, which would bust the cache mid-reference.
    task_with_hint = (
        "----- TOOL REFERENCE (how to work — your actual TASK follows below) -----\n"
        + tool_hint +
        "\n----- YOUR TASK -----\n"
        f"(Current date/time: {now_str} — pass a timezone to get_current_datetime() only "
        "if the task needs a DIFFERENT one.)\n"
        + _deploy_block
        + _ws_context_block
        # Task #90: ~25-token echo of the autonomy contract at the highest-attention spot
        # (recency — right next to the task), reinforcing the full directive that lives in
        # the cached reference above. Workspace/coding mode only.
        + ("Remember: FINISH it yourself — fix blockers, VERIFY the outcome, then "
           "final_answer with evidence; never hand the task back.\n"
           if _WS_AVAILABLE else "")
        + "\nTASK (answer THIS, choose the tool that fits it):\n"
        f"{task_text}\n"
    )

    # Track URLs that have already failed so we don't retry dead endpoints across steps.
    _failed_urls: set = set()

    # Remember the last tool output so the max-steps fallback can report exactly what a
    # tool returned (no extra model call) if the loop ends without final_answer.
    _last_obs: dict = {"text": ""}

    # Set by provide_final_answer when it had to synthesize a PARTIAL answer at the
    # step budget. Read by the graceful-finish checkpoint clear: a max-steps ending
    # surfaces only the LAST observation into history, so the full step trace must
    # survive on disk for a follow-up "continue" to genuinely resume — clearing it
    # there would leave the next turn a ~500-char condensed history line instead.
    _maxsteps_partial: dict = {"hit": False}

    # Deterministic loop-breaker. Weak models often repeat the SAME search/fetch over and
    # over (hoping for a "more complete" result) and never converge, burning the whole step
    # budget. We remember each (tool, arg) signature; on a repeat we DON'T re-run it —
    # instead we return the prior result plus a hard nudge to move on (compile / create_pdf
    # / final_answer). Keyed to retrieval tools where the loop actually happens.
    _seen_calls: dict = {}

    # Search queries get near-duplicated with filler tweaks ("schedule" → "FULL schedule"
    # → "2026 FIFA full schedule"), dodging exact-match dedup — seen live. Normalize:
    # drop filler words, sort tokens, so all variants share one signature.
    _QUERY_FILLER = {
        "full", "complete", "comprehensive", "detailed", "entire", "all", "latest",
        "current", "the", "a", "an", "of", "with", "and", "for", "to", "in", "on",
        "more", "about", "info", "information",
    }

    def _norm_query(q: str) -> str:
        tokens = [t for t in re.findall(r"[a-z0-9]+", (q or "").lower())
                  if t not in _QUERY_FILLER]
        return " ".join(sorted(tokens))

    # Hard research budget: at most N retrieval calls (web_search + fetch_url) per run.
    # Past it, retrieval tools refuse and steer to create_pdf/final_answer. Keeps steps
    # free to actually FINISH — the live failure mode was burning ALL steps on
    # slightly-different searches and never producing the asked-for PDF. The ceiling is
    # user-configurable (web Settings → Agent, default 10) and applies to both PDF and
    # plain tasks.
    _research = {"used": 0, "max": research_budget}

    def _research_spend(tool_name: str) -> str:
        """Return '' if under budget (and spend 1), else a refusal steering to finish."""
        if _research["used"] >= _research["max"]:
            _log(f"{tool_name} BLOCKED: research budget ({_research['max']}) exhausted")
            return (
                f"Research budget exhausted ({_research['max']} web_search/fetch_url calls "
                "used). Do NOT search or fetch again. FINISH NOW with what you have: if the "
                "task asked for a PDF call create_pdf(text=<compile the gathered facts>, "
                "title=...), otherwise call final_answer(<your best answer from the "
                "observations above>)."
            )
        _research["used"] += 1
        return ""

    # Everything useful the retrieval tools returned this run, so an exhausted loop can
    # still deterministically deliver (e.g. render a partial PDF) instead of dead-ending.
    _gathered: list = []

    # --- Loop / no-progress guards (task #79) -------------------------------------------
    # Weak coders thrash on a debug loop: rewrite a file to a previously-tried version,
    # re-hit a broken endpoint, and re-read unchanged files — burning the whole step
    # budget without new information. These caches let the tools detect and refuse that.
    _file_writes: dict = {}   # realpath -> {content hashes already written this run}
    _url_5xx: dict = {}       # localhost url -> how many 5xx it has returned
    _reads: dict = {}         # realpath -> (mtime, identical-read count)

    def _server_log_tail_for_url(url: str) -> str:
        """If `url` is a localhost server we started via run_command, return the tail of
        its run-log — the REAL error. A 5xx returns only an HTML error page; the actual
        stack trace lives in the server's stdout log (task #79)."""
        try:
            import os as _os
            from urllib.parse import urlparse
            p = urlparse(url)
            if p.hostname not in ("localhost", "127.0.0.1", "0.0.0.0", "::1"):
                return ""
            for s in _ws_load_servers():
                if p.port and p.port in (s.get("ports") or []):
                    logp = s.get("log") or ""
                    if logp and _os.path.isfile(logp):
                        with open(logp, "r", encoding="utf-8", errors="replace") as fh:
                            return _ws_distill_output(fh.read())[-1500:]
            return ""
        except Exception:  # noqa: BLE001
            return ""

    def _read_repeat_block(rp: str) -> str:
        """Refuse a 3rd+ identical read of an UNCHANGED file (a common step-waster)."""
        try:
            import os as _os
            mt = _os.path.getmtime(rp)
        except OSError:
            return ""
        prev = _reads.get(rp)
        if prev and prev[0] == mt:
            cnt = prev[1] + 1
            _reads[rp] = (mt, cnt)
            if cnt >= 3:
                import os as _os2
                return (f"You've already read {_os2.path.basename(rp)} {cnt} times and it "
                        "hasn't changed. Stop re-reading it — act on what it says, or "
                        "call final_answer.")
            return ""
        _reads[rp] = (mt, 1)
        return ""

    def _dedup(tool_name: str, arg: str, next_hint: str):
        """Return (is_repeat, message). On a repeat, message steers the model forward."""
        key = _norm_query(arg) if tool_name == "web_search" else (arg or "").strip().lower()
        sig = f"{tool_name}::{key}"
        prior = _seen_calls.get(sig)
        if prior is not None:
            _log(f"{tool_name} REPEAT suppressed ({arg[:60]!r}) → nudging model forward")
            return True, (
                f"You already ran {tool_name}({arg!r}) (or an equivalent query) — the "
                f"result would be the same as shown above. Do NOT call it again. {next_hint}"
            )
        _seen_calls[sig] = True
        return False, ""

    @tool
    def http_request(method: str, url: str, headers: str = "", body: str = "",
                     username: str = "", password: str = "") -> str:
        """Perform an HTTP request and return the status code and body preview.

        Args:
            method: HTTP method (GET, POST, PUT, DELETE, etc.)
            url: Full URL to request
            headers: Optional JSON object string of request headers
            body: Optional request body string
            username: Username for Basic Authentication (avoids manual base64 encoding)
            password: Password for Basic Authentication
        """
        import base64 as _b64
        parsed_headers = {}
        # Basic Auth: encode credentials automatically if provided.
        if username or password:
            creds = _b64.b64encode(f"{username}:{password}".encode()).decode()
            parsed_headers["Authorization"] = f"Basic {creds}"
        if headers:
            try:
                parsed_headers.update(json.loads(headers))
            except Exception:  # noqa: BLE001
                pass
        url_key = f"{method.upper()}:{url}"
        if url_key in _failed_urls:
            msg = f"Error: {url} already failed — try a different URL or use Python stdlib."
            _emit({"type": "step", "text": f"HTTP {method.upper()} {url} → (skipped, already failed)"})
            _log(f"http_request skipped (already failed): {url_key}")
            return msg
        result = _http_request_impl(method, url, parsed_headers, body or None)
        if result.startswith("Error:"):
            # Retry once for flaky connections (e.g. gorok tunnels).
            _log(f"http_request retry {method.upper()} {url}")
            result = _http_request_impl(method, url, parsed_headers, body or None)
        if result.startswith("Error:"):
            # Both attempts failed — mark URL as dead so model tries something else.
            _failed_urls.add(url_key)
            result = (
                f"{result}\n"
                "Note: This URL failed twice. Do NOT retry it. "
                "Try a DIFFERENT URL or use Python's datetime/math/etc. module instead."
            )
        status_line = result.split("\n")[0][:150] if result else "no response"
        # Detect HTML pages so the model stops trying to json.loads() a web page.
        body_part = result.split("\n", 1)[1].lstrip().lower() if "\n" in result else ""
        is_html = body_part.startswith("<!doctype html") or body_part.startswith("<html")
        if is_html:
            result = result + (
                "\n[NOTE: This is an HTML web page, not JSON. Do NOT json.loads() it. "
                "Use web_search() for facts, or request a JSON API endpoint instead.]"
            )
        # Append a success tag to 2xx JSON responses so the model stops and calls final_answer.
        elif result and result.startswith("HTTP 2"):
            result = result + "\n[SUCCESS — call final_answer(response) now, do not parse or retry]"
        # A 5xx from a localhost dev server (task #79): the HTML error page hides the real
        # cause. Surface the SERVER'S OWN LOG (the actual stack trace) so the model fixes
        # the specific error instead of guessing/oscillating — and hard-stop after a few
        # repeats so it can't burn the whole budget re-hitting a broken endpoint.
        _m5 = re.match(r"HTTP\s+(5\d\d)", result or "")
        if _m5:
            _url_5xx[url] = _url_5xx.get(url, 0) + 1
            _tail = _server_log_tail_for_url(url)
            if _tail:
                result = (result + f"\n[SERVER ERROR LOG — the real cause of the "
                          f"{_m5.group(1)} (fix THIS, don't guess):\n{_tail}\n]")
            if _url_5xx[url] >= 3:
                result = result + (
                    f"\n[STOP: {url} has returned a 5xx {_url_5xx[url]} times. Do NOT request "
                    "it again or rewrite the same file again. Fix the SPECIFIC error in the "
                    "server log above, or call final_answer honestly stating what's failing.]"
                )
        _emit({"type": "step", "text": f"HTTP {method.upper()} {url} → {status_line}"})
        _log(f"http_request {method.upper()} {url} → {result[:80]}")
        _last_obs["text"] = result
        return result

    @tool
    def get_current_datetime(timezone: str = "") -> str:
        """Return the current date and time. Use for any date/time question — no HTTP needed.

        Args:
            timezone: Optional IANA timezone name (e.g. 'Asia/Bangkok', 'UTC'). Empty = server local time.
        """
        from datetime import datetime as _dtl
        try:
            if timezone:
                from zoneinfo import ZoneInfo
                now = _dtl.now(ZoneInfo(timezone))
            else:
                now = _dtl.now().astimezone()
        except Exception:  # noqa: BLE001
            now = _dtl.now().astimezone()
        out = now.strftime("%A, %d %B %Y, %H:%M:%S %Z")
        _emit({"type": "step", "text": f"Current date/time → {out}"})
        _log(f"get_current_datetime({timezone!r}) → {out}")
        _last_obs["text"] = out
        return out

    @tool
    def web_search(query: str) -> str:
        """Search the web AND read the top pages for you, in one call.

        Use this INSTEAD of guessing a URL when the user asks to 'find' something or asks a
        general-knowledge question. Returns any direct answer box plus the top sources with
        their CONTENT ALREADY READ (HTML stripped, tables kept) — so you can usually answer
        or call final_answer/create_pdf straight away WITHOUT calling fetch_url again. Only
        call fetch_url when you specifically need a single page in full.

        Args:
            query: What to look up, e.g. 'capital of France' or 'world cup 2026 schedule'.
        """
        dup, nudge = _dedup("web_search", query,
                            "Use the facts you have ALREADY gathered: if the task asks for a "
                            "PDF, call create_pdf(text=..., title=...) now; otherwise call "
                            "final_answer with what you found. Do not keep searching for a "
                            "'complete' source — one may not exist.")
        if dup:
            return nudge
        blocked = _research_spend("web_search")
        if blocked:
            return blocked
        _emit({"type": "step", "text": f"Searching the web → {query[:80]}"})
        # SearXNG URL (task #103): from Settings (cfg.searxngUrl) if the API sends it, else
        # the GONEXT_SEARXNG_URL env fallback inside _web_search_impl. Keyless either way.
        # Search model (#105): when configured, web_search synthesizes a cited answer with it.
        result = _web_search_impl(
            query,
            searxng_url=(cfg.get("searxngUrl") or "").strip(),
            search_base_url=(cfg.get("searchBaseURL") or "").strip(),
            search_model_id=(cfg.get("searchModelId") or "").strip(),
        )
        _log(f"web_search {query[:60]!r} → {result[:80]}")
        _last_obs["text"] = result
        if result and not result.startswith("Error"):
            _gathered.append(result)
        return result

    @tool
    def fetch_url(url: str) -> str:
        """Fetch a specific web page and return its readable text (HTML stripped).

        Use this to READ the actual contents of a specific URL — e.g. a page found by
        web_search, or a link the user gave — when a summary is not enough. Returns clean
        plain text, not raw HTML. For JSON APIs use http_request instead.

        Args:
            url: The full URL of the page to read, e.g. 'https://example.com/article'.
        """
        dup, nudge = _dedup("fetch_url", url,
                            "You already have this page's text above. Use it: call "
                            "create_pdf(text=..., title=...) if a PDF was requested, else "
                            "final_answer with what you found.")
        if dup:
            return nudge
        blocked = _research_spend("fetch_url")
        if blocked:
            return blocked
        _emit({"type": "step", "text": f"Reading page → {url[:80]}"})
        status, ctype, raw = _fetch_page_impl(url)
        if status is None:
            msg = f"Error: could not fetch {url}: {raw.decode('utf-8', 'replace')[:200]}"
            _log(f"fetch_url error {url} → {msg[:80]}")
            _last_obs["text"] = msg
            return msg
        if isinstance(status, int) and status >= 400:
            msg = f"Error: {url} returned HTTP {status}. Try a different URL or use web_search()."
            _log(f"fetch_url {url} → HTTP {status}")
            _last_obs["text"] = msg
            return msg
        # PDFs / binaries don't strip to useful text.
        if "application/pdf" in ctype or raw[:5].lstrip().startswith(b"%PDF"):
            msg = (f"{url} is a PDF, not an HTML page — fetch_url cannot read PDFs. "
                   "Tell the user this needs a PDF-reading tool.")
            _log(f"fetch_url {url} → PDF, cannot read")
            _last_obs["text"] = msg
            return msg
        # 8000, not the 3000 default: a research page's real content (fixture tables,
        # data) sits well past the lead, so the old cap returned only the intro/nav and
        # starved the model (task #75). Older large observations are trimmed by the #63
        # step_callback, so the extra size only costs context for the latest 2 steps.
        text = (_html_to_text(raw.decode("utf-8", errors="replace"), limit=8000)
                or "(page had no readable text)")
        out = f"{url}\n{text}"
        _log(f"fetch_url {url} → {len(text)} chars")
        _last_obs["text"] = out
        _gathered.append(out)
        return out

    @tool
    def calculate(expression: str) -> str:
        """Evaluate a math expression and return the numeric result. Do NOT do the math
        yourself — always call this for arithmetic, percentages, powers, or conversions.

        Supports + - * / // % ** and parentheses, plus sqrt/pow/round/abs/floor/ceil/log
        and the constants pi and e. Understands '15% of 80' and '2^10'.

        Args:
            expression: The math to evaluate, e.g. '(3+4)*2', '2**10', '15% of 80'.
        """
        _emit({"type": "step", "text": f"Calculating → {expression[:80]}"})
        result = _calc_impl(expression)
        _log(f"calculate({expression!r}) → {result[:80]}")
        _last_obs["text"] = result
        return result

    @tool
    def send_email(to: str, subject: str, body: str) -> str:
        """Send an email through the configured email API. Asks for confirmation FIRST.

        The FIRST time you call this it only PREVIEWS the email and does NOT send — tell
        the user to reply 'confirm' to actually send it. Use ONLY when the user explicitly
        asks to email or send a message to someone.

        Args:
            to: Recipient email address.
            subject: The email subject line.
            body: The email body text.
        """
        to_addr = (to or "").strip()
        if "@" not in to_addr or " " in to_addr:
            msg = f"Error: '{to_addr}' is not a valid recipient email address."
            _last_obs["text"] = msg
            return msg
        if email_allow and not _email_allowed(to_addr, email_allow):
            msg = (f"Error: {to_addr} is not in the allowed recipients list — refusing to "
                   "send. Add it to the allow-list in Settings if this is intended.")
            _log(f"send_email blocked by allow-list: {to_addr}")
            _last_obs["text"] = msg
            return msg
        preview = (
            f"📧 About to email {to_addr}\n"
            f"Subject: {subject or '(no subject)'}\n\n"
            f"{body or '(no body)'}\n\n"
            "Reply 'confirm' to send this, or tell me what to change."
        )
        # Send ONLY when the current user message confirms AND a preview was already
        # shown — otherwise just preview (never send on the first call).
        if not _email_confirm_pending():
            _emit({"type": "step", "text": f"Email preview → {to_addr} (awaiting confirm)"})
            _log(f"send_email preview (awaiting confirm) → {to_addr}")
            _last_obs["text"] = preview
            return preview
        filled = _email_fill_template(email_body_template, {
            "to": to_addr, "subject": subject or "", "body": body or "", "from": email_from,
        })
        if isinstance(filled, str):  # template error
            _last_obs["text"] = filled
            return filled
        headers = {"Content-Type": "application/json", "Accept": "application/json"}
        if email_api_headers_raw:
            try:
                headers.update(json.loads(email_api_headers_raw))
            except Exception:  # noqa: BLE001
                pass
        _emit({"type": "step", "text": f"Sending email → {to_addr}"})
        result = _http_request_impl(email_api_method, email_api_url, headers, json.dumps(filled))
        first = result.split("\n", 1)[0] if result else "no response"
        if result.startswith("HTTP 2"):
            out = f"✅ Email sent to {to_addr} (Subject: {subject or '(no subject)'})."
        else:
            out = f"Email send failed ({first}). Check the email API settings."
        _log(f"send_email {to_addr} → {first}")
        _last_obs["text"] = out
        return out

    @tool
    def extract_text_from_pdf(url: str) -> str:
        """Read an existing PDF at a URL and return its extracted text.

        Use this to READ or SUMMARIZE a PDF the user links to. This does NOT create a
        PDF — use create_pdf for that. Returns the PDF's text (truncated), not a link.

        Args:
            url: The full URL of the PDF file, e.g. 'https://example.com/report.pdf'.
        """
        _emit({"type": "step", "text": f"Reading PDF → {url[:80]}"})
        result = _extract_pdf_text_impl(url)
        _log(f"extract_text_from_pdf {url} → {result[:80]}")
        _last_obs["text"] = result
        return result

    # Task #74 retry guard. When a create_pdf call outlives the executor's wall clock,
    # the tool still finishes (render + S3 upload succeed, "PDF ready" is emitted) but
    # the model is told the step FAILED — so it calls create_pdf again for the same
    # document. Remember the last success here; `timeout_pending` is flipped by
    # step_callback whenever a step's result was an executor-timeout error, meaning
    # whatever succeeded during that step was never reported to the model.
    _pdf_state: dict = {"last": None, "timeout_pending": False}

    def _pdf_sig(title: str, text: str) -> str:
        return (title.strip().lower() + "||"
                + re.sub(r"\s+", " ", (text or "").strip().lower())[:1500])

    def _pdf_titles_similar(a: str, b: str) -> bool:
        a, b = a.strip().lower(), b.strip().lower()
        return bool(a and b) and (a == b or a in b or b in a)

    @tool
    def create_pdf(text: str, title: str = "") -> str:
        """Create a PDF document from text/data and return a download link.

        Use this ONLY when the user explicitly asks to make/create/generate/export a PDF.
        The text is first cleaned into well-formed Markdown, then rendered to a PDF on the
        worker and uploaded to cloud storage; the returned message contains the download URL.

        Args:
            text: The content to put in the PDF (raw text, notes, or data).
            title: Optional document title shown at the top and used in the file name.
        """
        doc_title = (title or "").strip() or "Document"
        # Retry guard (task #74): identical content is ALWAYS a retry; a merely similar
        # title counts only when the previous success was swallowed by an executor
        # timeout (the model rewords/shortens on retry, so the text rarely matches).
        prev = _pdf_state.get("last")
        if prev and (
            prev["sig"] == _pdf_sig(doc_title, text)
            or (_pdf_state.get("timeout_pending")
                and _pdf_titles_similar(prev["title"], doc_title))
        ):
            _log(f"create_pdf retry detected (prev title={prev['title']!r}) "
                 "→ returning the existing PDF instead of re-rendering")
            _emit({"type": "step",
                   "text": f"PDF already created → {prev['title']}.pdf (reusing link)"})
            _last_obs["text"] = prev["msg"]
            return prev["msg"]

        # Refuse a placeholder skeleton instead of shipping a confident-but-empty PDF
        # (task #75): a table of 'TBD vs TBD' rows means the agent never found the real
        # data. 3+ placeholder cells is the fingerprint of that failure and virtually
        # never appears in a genuine user document. Steer the model to answer honestly
        # rather than render a green "✅ ready" over an empty table.
        _placeholders = len(re.findall(
            r"\b(?:TBD|TBA|TBC|N/?A)\b|\?\?\?|—\s*(?:vs\.?)?\s*—", text or "", re.I))
        if _placeholders >= 3:
            msg = (
                f"This document is mostly placeholders ({_placeholders} TBD/TBA cells) — "
                "the real data was never found, so the PDF would be empty. Do NOT retry "
                "create_pdf. Call final_answer to tell the user honestly that you could "
                f"not find the actual {doc_title} data to fill the table."
            )
            _log(f"create_pdf refused: {_placeholders} placeholder cells")
            _last_obs["text"] = msg
            return msg
        # 1) Format the raw text into clean Markdown (extra model call, intentional).
        # CHAT model, not the coder — see the fast-path call site for why (task #74).
        _emit({"type": "step", "text": "Formatting document…"})
        markdown_text = _format_text_for_pdf(
            text or "", doc_title,
            agent_base_url or coding_base_url, agent_api_key,
            agent_model_id or coding_model_id,
        )

        # 2) Render the Markdown to PDF bytes locally (pure-Python xhtml2pdf).
        # Catch EVERYTHING (xhtml2pdf can raise arbitrary errors on exotic glyphs /
        # emoji), so the tool always returns a string and never breaks the agent loop.
        _emit({"type": "step", "text": "Rendering PDF…"})
        try:
            pdf_bytes = _render_pdf_bytes(markdown_text, doc_title)
        except RuntimeError as e:
            # Engine missing or a clean render failure — surface the message as-is.
            msg = str(e)
            _log(f"create_pdf render error: {msg[:200]}")
            _last_obs["text"] = msg
            return msg
        except Exception as e:  # noqa: BLE001 — unexpected render crash
            _log(f"create_pdf render crash: {type(e).__name__}: {e!r}\n{traceback.format_exc()}")
            msg = (
                "Sorry — I couldn't render that text into a PDF "
                f"({type(e).__name__}: {str(e)[:160]}). "
                "Try simpler text without unusual symbols/emoji."
            )
            _last_obs["text"] = msg
            return msg

        # 3) Upload via the API-presigned PUT (no AWS creds on the worker).
        _emit({"type": "step", "text": "Uploading PDF…"})
        file_name = f"{doc_title}.pdf"
        try:
            download_url = _pdf_upload_via_api(
                pdf_api_base, pdf_worker_key, file_name, pdf_bytes
            )
        except Exception as e:  # noqa: BLE001 — network / API errors must not crash the tool
            _log(f"create_pdf upload error: {type(e).__name__}: {e!r}\n{traceback.format_exc()}")
            msg = f"PDF was created but could not be uploaded: {str(e)[:200]}"
            _last_obs["text"] = msg
            return msg

        out = (
            f"✅ Your PDF \"{doc_title}\" is ready.\n"
            f"Download it here (link valid for a limited time):\n{download_url}"
        )
        _emit({"type": "step", "text": f"PDF ready → {doc_title}.pdf"})
        _log(f"create_pdf ok title={doc_title!r} bytes={len(pdf_bytes)}")
        _pdf_state["last"] = {
            "sig": _pdf_sig(doc_title, text), "title": doc_title, "msg": out,
        }
        _last_obs["text"] = out
        return out

    @tool
    def create_download(path: str, name: str = "") -> str:
        """Package a workspace FILE or FOLDER into a downloadable file and return a download link. A FOLDER is zipped (skipping node_modules/.git/build/dist); a single FILE is offered as-is. Use this to hand the user a ZIP of a project you scaffolded, or to let them download any file you made with create_file (csv, json, a report, …).

        Args:
            path: the file or folder to package (a workspace path).
            name: optional download name (e.g. "my-app.zip"); defaults to the file/folder name.
        """
        import os
        if not pdf_api_base or not pdf_worker_key:
            msg = "Error: download isn't available (the worker API base/key isn't configured)."
            _last_obs["text"] = msg
            return msg
        try:
            src = _rag_read_allowed((path or "").strip())
        except Exception as e:  # noqa: BLE001
            msg = f"Error: can't access that path ({e})."
            _last_obs["text"] = msg
            return msg
        if not os.path.exists(src):
            msg = f"Error: nothing exists at {path}."
            _last_obs["text"] = msg
            return msg
        try:
            import tempfile
            import zipfile
            if os.path.isdir(src):
                base = os.path.basename(src.rstrip("/\\")) or "archive"
                file_name = (name or f"{base}.zip").strip()
                if not file_name.lower().endswith(".zip"):
                    file_name += ".zip"
                _emit({"type": "step", "text": f"Zipping {base}…"})
                tmp = tempfile.NamedTemporaryFile(suffix=".zip", delete=False)
                tmp.close()
                total = 0
                files = 0
                try:
                    with zipfile.ZipFile(tmp.name, "w", zipfile.ZIP_DEFLATED) as zf:
                        for root, dirs, filenames in os.walk(src):
                            dirs[:] = [
                                d for d in dirs
                                if d not in _RAG_SKIP_DIRS and not d.startswith(".")
                            ]
                            for fn in filenames:
                                fp = os.path.join(root, fn)
                                try:
                                    total += os.path.getsize(fp)
                                except OSError:
                                    continue
                                if total > _DOWNLOAD_MAX_BYTES:
                                    raise RuntimeError(
                                        f"folder is too large to zip "
                                        f"(> {_DOWNLOAD_MAX_BYTES // (1024 * 1024)} MB)"
                                    )
                                zf.write(fp, os.path.join(base, os.path.relpath(fp, src)))
                                files += 1
                    if files == 0:
                        msg = (f"Nothing to zip in {path} "
                               "(no files after skipping node_modules/.git/build/dist).")
                        _last_obs["text"] = msg
                        return msg
                    with open(tmp.name, "rb") as fh:
                        data = fh.read()
                finally:
                    try:
                        os.unlink(tmp.name)
                    except OSError:
                        pass
                content_type = "application/zip"
                summary = f"{files} file(s)"
            else:
                size = os.path.getsize(src)
                if size > _DOWNLOAD_MAX_BYTES:
                    msg = (f"Error: {os.path.basename(src)} is too large to share "
                           f"(> {_DOWNLOAD_MAX_BYTES // (1024 * 1024)} MB).")
                    _last_obs["text"] = msg
                    return msg
                file_name = (name or os.path.basename(src)).strip() or "file"
                with open(src, "rb") as fh:
                    data = fh.read()
                content_type = _content_type_for(file_name)
                summary = f"{size} bytes"
            _emit({"type": "step", "text": "Uploading…"})
            url = _artifact_upload_via_api(
                pdf_api_base, pdf_worker_key, file_name, data, content_type
            )
            out = (
                f"✅ \"{file_name}\" is ready ({summary}).\n"
                f"Download it here (link valid for a limited time):\n{url}"
            )
            _emit({"type": "step", "text": f"Download ready → {file_name}"})
            _log(f"create_download ok name={file_name!r} bytes={len(data)}")
            _last_obs["text"] = out
            return out
        except Exception as e:  # noqa: BLE001
            _log(f"create_download error: {type(e).__name__}: {e!r}")
            msg = f"Error: couldn't create the download ({str(e)[:200]})."
            _last_obs["text"] = msg
            return msg

    # Growing record of this turn's steps, persisted to disk after each one via
    # _write_turn_checkpoint (inside step_callback below) — see _load_and_clear_turn_
    # checkpoint above for how a crashed/interrupted turn gets recovered on the next run.
    _turn_steps: list = []
    # Set right after the agent is built (below) so step_callback can reach the running
    # agent's memory to trim old observations (task #63, keeps the prompt from growing
    # O(n²) as big file reads pile up in the step trace).
    _agent_ref: dict = {"a": None}
    # Task #87 rec#3: when the context grows large in an INTERACTIVE terminal turn, ask
    # the user Yes/No before aggressively compacting it (rather than deciding silently).
    # decision: None = not yet asked this turn; "always" = user said yes, keep compacting
    # each step without re-asking; "never" = user said no, don't ask again this turn (the
    # normal auto-trim above still runs). Threshold in prompt tokens for the last request.
    _compact_state: dict = {"decision": None}
    _COMPACT_ASK_TOKENS = 14000

    # Task #90 Phase 3 — completion gate (code-enforced, complements the prompt). Fires at
    # most ONCE per turn: refuse the first final_answer that either hands the task back to
    # the user, or lands right after a state-changing command that was never verified — with
    # a verify-or-explain nudge fed back to the model (smolagents final_answer_checks: a
    # raised message becomes the check-failure the model sees and retries on). After one
    # nudge it ACCEPTS unconditionally — no loop; an honest "here's what failed" is valid.
    _completion_gate: dict = {"nudged": False}
    # Punt phrases: handing the CORE task back ("…start it yourself", "you'll need to run").
    # Deliberately narrow so a helpful tip ("you can also run npm test") does NOT trigger it.
    _PUNT_RE = re.compile(
        r"\b(?:do|run|start|complete|finish|build) it yourself\b"
        r"|\byou can navigate to\b"
        r"|\bstart (?:it|the (?:app|server|project|dev server)) yourself\b"
        r"|\byou'?ll (?:have to|need to) (?:run|start|do|build)\b",
        re.I)

    def _final_answer_gate(final_answer, memory, agent=None):
        """smolagents final_answer_check. True = accept; raising = reject with the message
        the model then sees. Only nudges when there's real evidence of an unfinished job."""
        if _completion_gate["nudged"]:
            return True  # already nudged once this turn — accept, never loop
        ans = str(final_answer if final_answer is not None else "")
        punt = bool(_PUNT_RE.search(ans))
        # The most-recent tool result: a command that finished but whose EFFECT was never
        # verified (Phase-1 'EXITED 0 (…VERIFY…)' marker), with no server/HTTP confirmation.
        last_unverified = False
        try:
            for _st in reversed(getattr(memory, "steps", []) or []):
                _obs = getattr(_st, "observations", None)
                if isinstance(_obs, str) and _obs.strip():
                    last_unverified = ("EXITED 0 (" in _obs) and not any(
                        k in _obs for k in
                        ("RUNNING:", "listening", "HTTP 2", "http://localhost"))
                    break
        except Exception:  # noqa: BLE001
            pass
        if punt or last_unverified:
            _completion_gate["nudged"] = True
            raise ValueError(
                "NOT DONE YET — you ran commands or changed files but have NOT shown the "
                "goal actually works. Do ONE of these, then call final_answer:\n"
                "  1) VERIFY it — curl/open the server, run the test, or re-read the file to "
                "confirm the change took — and answer WITH that evidence; OR\n"
                "  2) if it genuinely cannot be done with these tools, state plainly WHAT "
                "failed and WHY.\n"
                "Do NOT hand an unfinished task back to the user.")
        return True

    _CODE_TRIMMED_MARK = "\n…[code trimmed — already ran; result is in the Observation]"

    def _trim_tool_call_args(step, cap):
        """Shrink a step's recorded tool-call ARGUMENTS. For a CodeAgent that value IS the
        full python code action, and smolagents renders it into its OWN 'Calling tools:'
        message on every later request (memory.py ActionStep.to_messages → ToolCall.dict).

        This is the gap in the #63/#87 trims: they only ever touched model_output and
        observations, so the <code> they stripped out of the Thought came straight back
        here, verbatim, forever. On a file-writing step that duplicate is the single
        largest thing in the prompt (a create_file carries the whole file body twice).
        Gutting it is safe — the code ALREADY executed and its result is in the
        Observation; the leading chars we keep still name the tool and its first argument
        (the path), which is what a later step actually refers back to.

        The ToolCall object, its id and its name are left intact — smolagents formats
        parse errors as f"Call id: {self.tool_calls[0].id}" — and only `str` arguments are
        touched, so a ToolCallingAgent's dict arguments are left alone. Returns chars
        saved; idempotent, since a re-trim recomputes the identical string and no-ops."""
        saved = 0
        for _tc in (getattr(step, "tool_calls", None) or []):
            _a = getattr(_tc, "arguments", None)
            if not isinstance(_a, str) or len(_a) <= cap:
                continue
            _new = _a[:cap].rstrip() + _CODE_TRIMMED_MARK
            if len(_new) < len(_a):
                _tc.arguments = _new
                saved += len(_a) - len(_new)
        return saved

    def _aggressive_compact(steps):
        """User-approved deep compaction: keep only the LAST step in full; for every
        older step shrink the observation to a stub and the model_output to just its
        Thought head. More aggressive than the always-on auto-trim (keep-last-2). Returns
        chars saved. Idempotent — already-short fields fall under the caps and are skipped."""
        saved = 0
        for _st in steps[:-1]:
            _obs = getattr(_st, "observations", None)
            if isinstance(_obs, str) and len(_obs) > 200:
                _b = len(_obs)
                _st.observations = _obs[:150].rstrip() + "\n…[compacted — re-read if needed]"
                saved += _b - len(_st.observations)
            _mo = getattr(_st, "model_output", None)
            if isinstance(_mo, str) and len(_mo) > 220:
                _b = len(_mo)
                _head = re.split(r"<code[\s>]|```", _mo, maxsplit=1)[0].rstrip()
                _st.model_output = (_head[:200].rstrip() or _mo[:150].rstrip()) + "\n…[compacted]"
                saved += _b - len(_st.model_output)
            # The duplicated code action (see _trim_tool_call_args) — tighter than the
            # always-on pass, since a user-approved compaction is explicitly trading
            # recall for a smaller prompt.
            saved += _trim_tool_call_args(_st, 120)
        return saved

    def step_callback(step_log):
        step_num = getattr(step_log, "step_number", "?")

        # Task #74: an executor-timeout error means whatever the step's tool call
        # achieved was NEVER reported to the model (a create_pdf may have finished —
        # or may still be finishing — in the background). Flag it so a repeat
        # create_pdf with a similar title is treated as the retry it is; any step
        # that ends with a real (non-timeout) result clears the flag.
        try:
            _err = getattr(step_log, "error", None)
            _pdf_state["timeout_pending"] = bool(
                _err and "maximum execution time" in str(_err)
            )
        except Exception:  # noqa: BLE001
            pass

        # (B) Trim OLD, LARGE step content out of the CodeAgent's step memory so it isn't
        # re-sent to the model on every subsequent step (the #63 O(n²) token blow-up: an
        # uncapped dump cost ~5-6k tokens PER remaining step). Keep the last 2 steps intact
        # (the model just acted on them); shrink older long content. write_memory_to_messages
        # rebuilds the prompt from memory.steps on EVERY request (no cache), so mutating the
        # step objects here takes effect on the next request; old steps are only rendered
        # into history, never re-parsed for execution, so trimming them is safe.
        #   - observations = the tool's OUTPUT (a file/dir dump). Trimmed since #63.
        #   - model_output = the model's own Thought + <code> block (task #87). For an OLD
        #     step the code ALREADY ran and its result is in the observation, so re-sending
        #     the code verbatim every step is pure waste — keep the Thought, drop the code.
        # Both trims are IDEMPOTENT (a re-trimmed field falls under the size threshold, so a
        # later step_callback skips it — the counter below only logs NEW trims).
        try:
            _mem = getattr(_agent_ref.get("a"), "memory", None)
            _steps = getattr(_mem, "steps", None)
            if _steps and len(_steps) > 2:
                _trim_n, _trim_saved = 0, 0
                for _st in _steps[:-2]:
                    _obs = getattr(_st, "observations", None)
                    if isinstance(_obs, str) and len(_obs) > 900:
                        _before = len(_obs)
                        _st.observations = (
                            _obs[:400].rstrip()
                            + "\n…[earlier observation trimmed to save context — "
                            "re-read the file/dir with a small range if you still need it]")
                        _trim_n += 1
                        _trim_saved += _before - len(_st.observations)
                    _mo = getattr(_st, "model_output", None)
                    if isinstance(_mo, str) and len(_mo) > 700:
                        _before = len(_mo)
                        # Keep the Thought (everything before the code block); the code is
                        # redundant with the observation that already came back.
                        _head = re.split(r"<code[\s>]|```", _mo, maxsplit=1)[0].rstrip()
                        _kept = (_head[:500].rstrip() if _head else _mo[:300].rstrip())
                        _new = _kept + "\n…[code trimmed to save context — its result is in the Observation]"
                        if len(_new) < _before:
                            _st.model_output = _new
                            _trim_n += 1
                            _trim_saved += _before - len(_new)
                    # The SAME code, a second time: smolagents keeps the parsed code
                    # action in step.tool_calls and renders it as its own message on
                    # every request, so stripping <code> from model_output above saved
                    # nothing on its own. Trim the duplicate too — see
                    # _trim_tool_call_args for why this is safe.
                    _tc_saved = _trim_tool_call_args(_st, 300)
                    if _tc_saved:
                        _trim_n += 1
                        _trim_saved += _tc_saved
                if _trim_n:
                    _log(f"context trim: shrank {_trim_n} old step field(s), "
                         f"saved ~{_trim_saved} chars (#63/#87)")

                # (C) Task #87 rec#3 — INTERACTIVE compaction consent. When the context is
                # genuinely large and we're in a terminal that can prompt, ask the user
                # Yes/No before deep-compacting (the auto-trim above is always-on; THIS is
                # the extra, opt-in aggressive pass). Only in the terminal (interactive) —
                # web has no picker, so it just keeps the auto-trim.
                if (_interactive_approval and _job_id and pdf_api_base and pdf_worker_key
                        and _compact_state["decision"] != "never"):
                    # Size of the last request in prompt tokens (exact if the server
                    # reported it, else ~chars/4 of the current step memory).
                    _tu = getattr(step_log, "token_usage", None)
                    _in_tok = int(getattr(_tu, "input_tokens", 0) or 0) if _tu else 0
                    if _in_tok <= 0:
                        # Count every field that ActionStep.to_messages actually renders —
                        # model_output, the tool-call arguments, and observations. The
                        # tool_calls term used to be missing, which under-counted exactly
                        # the part that grows fastest (the duplicated code action), so a
                        # server that reports no usage would reach the ask late or never.
                        _chars = 0
                        for _s in _steps:
                            _chars += len(getattr(_s, "model_output", "") or "")
                            _chars += len(getattr(_s, "observations", "") or "")
                            for _tc in (getattr(_s, "tool_calls", None) or []):
                                _a = getattr(_tc, "arguments", None)
                                # str = the code action; anything else renders as JSON,
                                # so charge it a small flat cost rather than guessing.
                                _chars += len(_a) if isinstance(_a, str) else 80
                        _in_tok = _chars // 4 + 7000  # + fixed system/reference baseline
                    if _in_tok >= _COMPACT_ASK_TOKENS and len(_steps) > 2:
                        if _compact_state["decision"] != "always":
                            _kt = _in_tok // 1000
                            _allow = _ws_request_approval(
                                pdf_api_base, pdf_worker_key, _job_id,
                                f"__COMPACT__::The context is getting large (~{_kt}k tokens "
                                "per request). Compact older steps to keep it fast?",
                                "compact-context")
                            _compact_state["decision"] = "always" if _allow else "never"
                            _emit({"type": "step", "text": (
                                "Compacting context…" if _allow
                                else "Keeping full context (won't ask again this turn).")})
                        if _compact_state["decision"] == "always":
                            _cs = _aggressive_compact(_steps)
                            if _cs:
                                _log(f"context compact (user-approved): saved ~{_cs} chars")
        except Exception as _e:  # noqa: BLE001
            _log(f"memory-trim skip: {_e}")

        # Log what was sent to the model (last message in the conversation).
        model_input = getattr(step_log, "model_input_messages", None)
        if model_input:
            last = model_input[-1]
            raw = getattr(last, "content", "")
            if isinstance(raw, list):
                raw = " ".join(p.get("text", "") for p in raw if isinstance(p, dict))
            _log(f"step {step_num} → model input (tail): {str(raw)[:400]}")

        # Log what the model generated (the Python code block).
        model_output = getattr(step_log, "model_output", None)
        if model_output:
            _log(f"step {step_num} ← model output: {str(model_output)[:400]}")

        try:
            text = _summarise_step(step_log)
        except Exception as e:  # noqa: BLE001
            text = f"Step: {e}"
        # Checkpoint every real step to disk so an interrupted turn (crash/connection
        # drop) can be recovered on the next run in this workspace. Deliberately uses
        # the FULL model_output + FULL _last_obs here, NOT the one-line `text` summary
        # above — `text` is clipped to a single first line for the live terminal display
        # and would lose most of what the step actually found (e.g. a grep_repo hit list
        # is up to 100 lines; `_summarise_step` keeps only the "N hits" header line).
        # `_last_obs["text"]` already holds the CURRENT step's full tool output — every
        # workspace/RAG tool sets it on every return path (see task #38) specifically so
        # the max-steps fallback can surface it; reusing it here for the same reason.
        if _WS_ACTIVE and (model_output or _last_obs.get("text")):
            _turn_steps.append({
                "step": step_num,
                "did": _clip(str(model_output or "").strip(), 600),
                "found": _clip(str(_last_obs.get("text") or "").strip(), 1500),
            })
            _write_turn_checkpoint(_WS_ACTIVE, latest_user_text, _turn_steps)
        # Skip emitting if there's nothing beyond the tool name — the tool already
        # emitted its own step event with the actual response above.
        if not text or text.rstrip().endswith("| →"):
            _log(f"step {step_num} (empty obs, skipped)")
            return
        _log(f"step {step_num}: {text[:200]}")
        _emit({"type": "step", "text": text})

    # Wrap the model so we can see EXACTLY what smolagents posts to the model
    # server on every step — including its own system prompt, the task we passed,
    # and any step memory it accumulates. completion_kwargs["messages"] here is the
    # literal messages array sent to /v1/chat/completions.
    class _LoggingModel(OpenAIServerModel):
        def __init__(self, *a, stream_agent=False, save_full_response=False,
                     openai_backend=False, **kw):
            # Running total of PROMPT (input) tokens sent to the CODE model across the
            # turn (task #83). Only this wrapper counts — the chat/brain helper calls
            # (_chat_completion) don't route through here, so this is "code model only".
            self._code_tokens_total = 0
            # Largest SINGLE-request prompt-token count seen this turn (task #84) — the
            # peak context size, not the sum. The REPL keeps a per-workspace max of this.
            self._code_tokens_max = 0
            # Running total of OUTPUT (completion) tokens the CODE model GENERATED across
            # the turn (task #104). Reported by the server usage; the REPL adds this turn's
            # total into the per-workspace + per-(user, coding-URL) global lifetime counters.
            self._code_output_tokens_total = 0
            # Loop-breaker (#95 B1): how many CONSECUTIVE generations have produced a code
            # block that STILL won't compile after our repair. A weak coder can get stuck
            # re-emitting the same unparseable tool call (seen live: create_file with a
            # multi-line string, ~10 identical retries, 863s/133k tokens). After a few in a
            # row we stop feeding smolagents an un-runnable block and finish honestly.
            self._parse_fail_streak = 0
            # stream_agent: request the completion with stream=True and reassemble the
            # deltas into one message. Enabled for Ollama, where a long single-shot
            # (non-stream) generation sends nothing until the very end — a reverse proxy
            # in front of the box (ollama1.gomarsic.cc) then hits its idle read-timeout
            # and kills the request mid-generation. Streaming keeps tokens flowing so the
            # idle timer never fires. smolagents still receives the COMPLETE text, so its
            # <code>/final_answer parsing is unaffected.
            self._stream_agent = bool(stream_agent)
            # Task #107: persist each code-model response's full text (opt-in) + a 1-based
            # per-turn call counter used as the stored `step`.
            self._save_full_response = bool(save_full_response)
            self._call_index = 0
            # Task #113: the coding backend is an explicit OpenAI-compatible endpoint
            # (Settings → kind=openai), which in practice means a cloud REASONING model.
            # Everything gated on this flag is OFF for Ollama and for Auto/MLX, so those
            # paths keep behaving exactly as they do today (the user switches back and
            # forth between Kimi and Ollama and the Ollama path must not move).
            self._openai_backend = bool(openai_backend)
            # Consecutive steps (openai backend only) that ended with NO runnable code
            # block at all — prose, empty output, a dangling tag, or a recovery we had to
            # reject. Distinct from _parse_fail_streak, which counts blocks that EXIST but
            # won't compile. Without this, a coder that never emits a parseable block just
            # loops to the step budget: the live trace burned 25k prompt tokens over three
            # steps and still had to be Ctrl+C'd by the user.
            self._no_code_streak = 0
            super().__init__(*a, **kw)

        def _generate_once(self, *args, **kwargs):
            """One model call. Streams + reassembles when _stream_agent, else the normal
            non-streaming path. Streaming is a pure transport optimization: on a TRANSPORT
            failure it falls back to the non-streaming call, and it's skipped for
            tool-calling mode (tools_to_call_from), whose tool_call deltas we don't
            reassemble.

            Exception (task #114): an OVERLOAD (429 / engine_overloaded / rate limit) is
            NOT a transport failure — the server refused the work. Re-sending the identical
            request unstreamed, instantly and with no backoff, is the worst possible answer:
            it can deepen the limit, and because a non-streamed call emits no first token it
            then sits silent behind the 600s client timeout (live: 167s of "Thinking…" until
            the user pressed Ctrl+C). Re-raise instead, so generate()'s retry loop applies a
            real backoff and tells the user what is going on."""
            tools_to_call_from = kwargs.get("tools_to_call_from")
            if not self._stream_agent or tools_to_call_from:
                return super().generate(*args, **kwargs)
            try:
                return self._streamed_generate(*args, **kwargs)
            except Exception as e:  # noqa: BLE001
                if _is_backend_overloaded(e):
                    _log(f"streamed generate refused ({_clip(str(e), 200)}) → NOT falling "
                         "back to non-streaming; letting the retry loop back off (#114)")
                    raise
                if _is_auth_failure(e) or _is_billing_failure(e):
                    # Same reasoning as the overload above: the credential was rejected, so
                    # streaming was never the problem. Re-sending unstreamed would cost a
                    # second identical request per attempt AND print "Streaming unavailable
                    # — retrying without streaming…", which is simply untrue here.
                    _log(f"streamed generate rejected ({_clip(str(e), 200)}) → NOT falling "
                         "back to non-streaming; this is a credential/billing error")
                    raise
                _log(f"streamed generate failed ({e}) → falling back to non-streaming")
                _emit({"type": "step",
                       "text": "Streaming unavailable — retrying without streaming…"})
                return super().generate(*args, **kwargs)

        def _run_stream(self, completion_kwargs):
            """Issue ONE streaming completion, stream tokens live to the UI, and return
            (content, role, in_tok, out_tok, reasoning_len). `content` is ONLY the visible
            message text (for smolagents' parser); reasoning-channel tokens are streamed to
            the Thinking panel but never accumulated into content."""
            self._apply_rate_limit()
            t0 = time.monotonic()
            _log("streamed generate: request issued (stream=True), awaiting first token…")
            stream = self.client.chat.completions.create(**completion_kwargs)
            parts = []
            rparts = []  # reasoning-channel text (task #111: may hold the <code> block)
            role = "assistant"
            in_tok = out_tok = reasoning_len = 0
            first_token_at = None

            # Runaway guard: a degenerate model (seen live: gemma4:26b spewing
            # "<channel|><thought>" forever) will stream garbage until the client times
            # out — a multi-minute hang. Abort early if the output either (a) balloons
            # past a hard char cap, or (b) collapses into a repeating short line. On abort
            # we return "" so the empty-turn retry re-prompts with the CODE-NOW directive.
            _MAX_STREAM_CHARS = 16000  # a single agent step never legitimately needs this
            _stream_buf = []           # every streamed piece (content + reasoning)
            _stream_chars = 0
            _runaway = False

            def _looks_runaway() -> bool:
                tail = "".join(_stream_buf)[-1600:]
                lines = [ln.strip() for ln in tail.splitlines() if ln.strip()]
                # 12+ of the last lines are the same short string → degenerate repetition.
                if len(lines) >= 12:
                    last = lines[-12:]
                    if len(set(last)) <= 2 and all(len(ln) <= 80 for ln in last):
                        return True
                return False

            def _mark_first():
                # First streamed byte (content OR reasoning). The gap t0→here is pure
                # prompt-eval time (server sent nothing before this); everything after is
                # token generation with bytes flowing, so the proxy idle timer resets. If
                # this never logs, the request died in prompt-eval BEFORE any token —
                # streaming can't help there (shrink the model / warm the prompt cache).
                nonlocal first_token_at
                if first_token_at is None:
                    first_token_at = time.monotonic()
                    elapsed = first_token_at - t0
                    _log(f"streamed generate: FIRST token after {elapsed:.1f}s "
                         "(prompt-eval done; tokens now flowing)")
                    # Flip the status from "still thinking" to "almost done" the instant
                    # the wait ends — the heartbeat below keeps it there for the rest of
                    # this call (self._first_token_seen), and this one-shot emit shows it
                    # immediately (before the next 45s heartbeat would).
                    self._first_token_seen = True
                    _emit({"type": "step", "text": f"{_STATUS_ALMOST_DONE} ({elapsed:.0f}s)"})
                    # Separate this step's live thinking from the previous step summary.
                    _emit({"type": "stream", "text": "\n"})

            for chunk in stream:
                usage = getattr(chunk, "usage", None)
                if usage is not None:
                    in_tok = getattr(usage, "prompt_tokens", 0) or in_tok
                    out_tok = getattr(usage, "completion_tokens", 0) or out_tok
                choices = getattr(chunk, "choices", None)
                if not choices:
                    continue
                delta = getattr(choices[0], "delta", None)
                if delta is None:
                    continue
                if getattr(delta, "role", None):
                    role = delta.role
                # Live "thinking": stream BOTH the visible content AND any reasoning-channel
                # tokens (gemma4/deepseek-class models emit their chain-of-thought there,
                # leaving `content` empty) to the web Thinking panel as they arrive. Only
                # `content` is accumulated into the message smolagents parses — reasoning is
                # display-only and must NEVER reach the <code> parser.
                rpiece = (getattr(delta, "reasoning", None)
                          or getattr(delta, "reasoning_content", None))
                if rpiece:
                    _mark_first()
                    reasoning_len += len(rpiece)
                    rparts.append(rpiece)
                    _stream_buf.append(rpiece)
                    _stream_chars += len(rpiece)
                    _emit({"type": "stream", "text": rpiece})
                piece = getattr(delta, "content", None)
                if piece:
                    _mark_first()
                    parts.append(piece)
                    _stream_buf.append(piece)
                    _stream_chars += len(piece)
                    _emit({"type": "stream", "text": piece})
                # Check the runaway guard periodically (cheap) once enough has streamed.
                if _stream_chars > 500 and (_stream_chars > _MAX_STREAM_CHARS or _looks_runaway()):
                    _runaway = True
                    reason = "char cap" if _stream_chars > _MAX_STREAM_CHARS else "repeating output"
                    _log(f"streamed generate: ABORTING ({reason}) after {_stream_chars} chars "
                         "— model degenerated; closing stream.")
                    try:
                        stream.close()
                    except Exception:  # noqa: BLE001
                        pass
                    break
            content = "".join(parts)
            # Fallback OUTPUT-token count (task #112): some OpenAI-compatible servers (seen
            # live: Moonshot/Kimi K3) don't emit a final usage chunk on every streamed call,
            # and the runaway guard closes the stream BEFORE the trailing usage chunk — so a
            # step that DID generate output reports completion_tokens=0 and contributes
            # nothing to the per-turn "↓ out" total (~3x under-count on reasoning models,
            # whose output is mostly reasoning-channel tokens). When the server reported no
            # completion tokens, estimate from the streamed OUTPUT chars (visible content +
            # reasoning, ~4 chars/token, matching the #83 input estimate) so no generating
            # step silently counts 0. Authoritative server usage always wins when present
            # (out_tok>0), so Ollama — which always reports usage — is unchanged; MLX, which
            # reports none, now gets an estimate instead of a hard 0.
            if out_tok <= 0:
                approx_chars = len(content) + reasoning_len
                if approx_chars > 0:
                    out_tok = max(1, (approx_chars + 3) // 4)
                    _log(f"streamed generate: server reported no completion tokens → "
                         f"estimated out≈{out_tok} tokens from {approx_chars} streamed "
                         "chars (content+reasoning) (#112)")
            if _runaway:
                # Discard the garbage; "" makes _streamed_generate run the CODE-NOW retry.
                # Keep out_tok (the estimate above) — those tokens WERE generated/billed.
                _emit({"type": "step", "text": "Model output ran away — restarting this step…"})
                _log(f"streamed generate: discarded {len(content)} runaway chars → empty content")
                return "", role, in_tok, out_tok, reasoning_len
            # Task #119: smolagents passes its closing "</code>" as a stop sequence, so a
            # server that honors `stop` returns the block UNCLOSED. Close it here, before
            # anything downstream asks "did this turn produce a code block?" — otherwise a
            # step that called a tool correctly reads as broken. Gated on the code actually
            # compiling, so a genuinely dangling tag still falls through to the nudge below.
            content = _close_dangling_code_block(content)
            # Task #111: reasoning models on an OpenAI-compatible API (e.g. Kimi K3 @ Moonshot)
            # can emit the actionable <code>…</code> into the REASONING channel, leaving only
            # prose + a stray tag in `content` → smolagents' parser can't find the pair. When
            # content has no usable code block but the reasoning stream does, rebuild content
            # from it. Inert when reasoning is empty (Ollama with thinking off) → path untouched.
            content = self._recover_code_from_channels(content, "".join(rparts))
            _log(f"streamed generate assembled {len(content)} chars "
                 f"(in={in_tok} out={out_tok} tokens, reasoning={reasoning_len} chars)")
            return content, role, in_tok, out_tok, reasoning_len

        def _recover_code_from_channels(self, content, reasoning):
            """Thin wrapper over the module-level _recover_code_from_reasoning (kept as a
            method so the streaming path reads the same). The logic lives at module scope
            so it can be exercised directly by the offline replay checks."""
            return _recover_code_from_reasoning(content, reasoning)

        # Injected when a turn produced ONLY reasoning and no message content — a hard,
        # model-agnostic steer to emit the actionable code block instead of more analysis.
        _CODE_NOW_DIRECTIVE = (
            "You wrote analysis but produced NO code block, so nothing ran. Output ONLY a "
            "single code block NOW that calls exactly one tool and nothing else, e.g.:\n"
            "<code>\nweb_search(query=\"...\")\n</code>\n"
            "No explanation, no <thought>, no <think> — just the <code>…</code> block. "
            "If you already have enough information, call final_answer(...) or create_pdf(...)."
        )

        def _streamed_generate(
            self, messages, stop_sequences=None, response_format=None,
            tools_to_call_from=None, **kwargs,
        ):
            from smolagents.models import ChatMessage, TokenUsage  # local: version-safe
            completion_kwargs = self._prepare_completion_kwargs(
                messages=messages,
                stop_sequences=stop_sequences,
                response_format=response_format,
                tools_to_call_from=tools_to_call_from,
                model=self.model_id,
                custom_role_conversions=self.custom_role_conversions,
                convert_images_to_image_urls=True,
                **kwargs,
            )
            completion_kwargs["stream"] = True
            # Ask for a usage summary in the final chunk (OpenAI streaming convention;
            # Ollama honors it). Harmless if the server ignores it — usage stays 0.
            completion_kwargs["stream_options"] = {"include_usage": True}
            content, role, in_tok, out_tok, reasoning_len = self._run_stream(completion_kwargs)
            # Retry ONCE with a hard "emit the code block now" directive appended to the
            # already-prepared API messages (plain dicts — no smolagents format guessing).
            # Two triggers:
            #   (1) EMPTY content — reasoning models (gemma4 on Ollama) sometimes spend the
            #       WHOLE turn in the reasoning channel and emit no message.
            #   (2) BROKEN code attempt — content has a dangling <code>/</code> tag but NO
            #       valid pair, and #111 recovery found nothing in reasoning either (seen
            #       live with Kimi K3: content = "Thought: …</code>", reasoning is plain
            #       planning prose with no code). Without this, smolagents shows the ugly
            #       "regex pattern <code> was not found" error and burns the step before its
            #       own retry recovers. A finished PROSE answer has NO code tags, so it never
            #       trips this — it flows to the auto-wrap path in generate() untouched.
            #   (3) PROSE-ONLY plan — the model spent the turn narrating what it is ABOUT
            #       to do and emitted no code tag whatsoever (seen live with Kimi K3 step 1:
            #       221 chars of "I'll scaffold a NestJS project…", 3246 chars of reasoning,
            #       nothing runnable). Trigger (2) missed this: the "</code>" that shows up
            #       in the trace is appended by SMOLAGENTS after we return (its stop-sequence
            #       handling), so the content WE see here carries no tag at all. Openai-only.
            # See _code_nudge_reason for the full decision (kept pure so it can be replayed).
            # NOTE (#119): none of these triggers may fire on a block the server merely
            # truncated at the "</code>" stop sequence — that is the SUCCESS shape for an
            # OpenAI-compatible backend, not a failure. _close_dangling_code_block runs in
            # _run_stream (and again inside _code_nudge_reason) so it is already excluded.
            _pair_re = _CODE_PAIR_RE
            why = _code_nudge_reason(content, self._openai_backend)
            if why:
                _log(f"{why} turn (reasoning={reasoning_len} chars) → retrying once with a "
                     "'code now' directive")
                _emit({"type": "step",
                       "text": "Model produced only analysis — nudging it to write the action…"})
                retry_kwargs = dict(completion_kwargs)
                retry_kwargs["messages"] = list(completion_kwargs.get("messages") or []) + [
                    {"role": "user", "content": self._CODE_NOW_DIRECTIVE}
                ]
                r_content, r_role, r_in, r_out, _ = self._run_stream(retry_kwargs)
                # Close a stop-truncated block before judging the retry (#119). This test
                # used to demand a literal pair, which the openai backend cannot return —
                # so the retry we had just paid for was thrown away and the broken original
                # was sent on to smolagents. (_run_stream already closed it; the call here
                # is idempotent and keeps the rule visible at the point of decision.)
                r_has_pair = re.search(
                    _pair_re, _close_dangling_code_block(r_content) or "", re.I) is not None
                # Use the retry when it yields a real code block, or (empty original) any
                # content. If the retry is still broken and we HAD a Thought, keep the
                # original and let smolagents' own retry take it from here — no regression.
                if r_has_pair or (not content.strip() and r_content.strip()):
                    content, role = r_content, r_role
                    in_tok = r_in or in_tok
                # Count the retry call's output either way — it was generated/billed (#112).
                out_tok += r_out
            if stop_sequences and not self.supports_stop_parameter:
                from smolagents.models import remove_content_after_stop_sequences
                # For a server that can't honor `stop`, smolagents emulates it by splitting
                # on each stop sequence and keeping split[0] — and "</code>" IS one of those
                # sequences, so this CUTS THE CLOSER back off a block that had one. Re-close
                # afterwards (#119) so both kinds of backend hand back the same canonical
                # shape; without it the emulated path re-creates the exact bug we just fixed.
                content = remove_content_after_stop_sequences(content, stop_sequences)
                content = _close_dangling_code_block(content)
            return ChatMessage(
                role=role,
                content=content,
                tool_calls=None,
                raw=None,
                token_usage=TokenUsage(input_tokens=in_tok, output_tokens=out_tok),
            )

        def _estimate_prompt_tokens(self, args, kwargs):
            """Fallback token estimate (~4 chars/token) from the outgoing messages, used
            only when the model server doesn't report prompt usage (task #83)."""
            msgs = kwargs.get("messages")
            if msgs is None and args:
                msgs = args[0]
            if not msgs:
                return 0
            chars = 0
            try:
                for m in msgs:
                    c = m.get("content") if isinstance(m, dict) else getattr(m, "content", None)
                    if isinstance(c, list):  # multimodal content parts
                        for part in c:
                            t = part.get("text") if isinstance(part, dict) else None
                            if isinstance(t, str):
                                chars += len(t)
                    elif isinstance(c, str):
                        chars += len(c)
            except Exception:  # noqa: BLE001
                return 0
            return max(1, chars // 4)

        def generate(self, *args, **kwargs):
            # Two safeguards on each step's reply BEFORE smolagents parses it for the
            # code block / final_answer:
            #   1. content=None guard — Qwen3 (and reasoning models generally) can return
            #      message.content=None on a turn (e.g. it spent the whole completion on a
            #      <think> trace and never emitted a code block). smolagents has NO null
            #      guard (models.py: `content = response.choices[0].message.content`) and
            #      feeds it straight into parse_code_blobs → `re` gets None →
            #      "expected string or bytes-like object, got 'NoneType'" → the whole
            #      loop spirals on parse errors and trips the WebSocket timeout. Coercing
            #      None→"" turns that fatal crash into a normal "no code block" retry.
            #   2. strip any leftover Qwen3 <think>…</think> trace (belt-and-suspenders;
            #      thinking is disabled via /no_think in _prepare_completion_kwargs, but a
            #      stray trace must never reach the parser or leak to the user).
            # A transient "Connection error" to the local MLX server (seen mid-loop when the
            # context grows large) otherwise aborts the whole run — retry a couple of times
            # with a short backoff before giving up.
            # Heartbeat: while a model call is in flight the agent emits NOTHING, and
            # the web's no-progress watchdog kills the chat. A big/cold model (e.g. a
            # 31B on a remote Ollama box) can take minutes on prompt-eval before the
            # first byte — emit a keepalive step every 45s until the call returns.
            hb_stop = threading.Event()
            # Reset per call: True once this call's first token arrives (set in
            # _mark_first), read by the heartbeat below to switch its wording.
            self._first_token_seen = False

            # Task #83: emit the PROMPT-token estimate NOW, at request time — so the
            # code-model token count climbs the instant a step is SENT, not only after a
            # slow step returns usage (a cold gemma4 can wait minutes for the first token,
            # during which the count would otherwise sit at 0). We reconcile to the exact
            # server-reported prompt_tokens after the call when it's higher.
            _step_est = self._estimate_prompt_tokens(args, kwargs)
            if _step_est > 0:
                self._code_tokens_total += _step_est
                if _step_est > self._code_tokens_max:
                    self._code_tokens_max = _step_est
                _emit({"type": "tokens", "codeInput": self._code_tokens_total,
                       "codeMax": self._code_tokens_max, "stepInput": _step_est})

            def _heartbeat():
                waited = 0
                while not hb_stop.wait(45):
                    waited += 45
                    # Before the first token: a random playful word ("Caffeinating… (90s)")
                    # — a large/cold model can take minutes on prompt-eval. After the first
                    # token: "…almost completed thinking… (Ns)", since output is now flowing
                    # and the call is in its finishing phase, not stuck waiting.
                    word = (_STATUS_ALMOST_DONE if getattr(self, "_first_token_seen", False)
                            else f"{_thinking_word()}…")
                    _emit({"type": "step", "text": f"{word} ({waited}s)"})
                    _log(f"heartbeat: model call in flight {waited}s")

            hb = threading.Thread(target=_heartbeat, daemon=True)
            hb.start()
            last_err = None
            try:
                # A gateway outage (502/503/504, refused/reset/timeout) is transient infra —
                # the remote box may just be cold or the proxy briefly down — so give it MORE
                # attempts and a longer backoff to ride out a blip, instead of the 3 quick
                # tries a normal error gets. The attempt count is decided per-error: we start
                # with the generic budget and extend it the moment we see a gateway error.
                attempt, max_attempts = 0, 3
                while attempt < max_attempts:
                    try:
                        msg = self._generate_once(*args, **kwargs)
                        last_err = None
                        break
                    except Exception as e:  # noqa: BLE001
                        emsg = str(e)
                        # A 404 "model not found" is a config error (wrong model NAME
                        # for this server, e.g. an MLX-style name against Ollama) —
                        # deterministic, so retrying is pointless. Fail fast with the
                        # server's actual model list so the user can fix Settings.
                        # A rejected CREDENTIAL is deterministic — retrying re-sends the
                        # same bad key, and letting it fall through to the turn-level
                        # handler would degrade to _plain_reply, i.e. the chat model
                        # answering confidently about work that never ran (#94's failure
                        # mode). Fail fast with a message that says WHICH key went out:
                        # python uses `coding_api_key or agent_api_key`, so an unset coding
                        # key silently authenticates a cloud endpoint with the LOCAL agent
                        # key (the Auto+cloud path in #117) — a completely different fix
                        # from "your coding key is wrong". The key itself is never echoed.
                        if _is_auth_failure(e) or _is_billing_failure(e):
                            _host = _host_of(coding_base_url)
                            if _is_billing_failure(e):
                                raise _AgentConfigError(
                                    f"The agent coding backend ({_host}) accepted the key "
                                    "but refused the request for BILLING reasons (out of "
                                    "credit / quota exhausted). Nothing ran. Top up the "
                                    "account, or point 'Agent coding model' at another "
                                    "backend in Settings → Agent."
                                ) from e
                            if coding_api_key:
                                raise _AgentConfigError(
                                    f"The agent coding backend ({_host}) REJECTED the "
                                    "coding API key. Nothing ran. Re-enter 'Agent coding "
                                    "model API key' in Settings → Agent (the stored key is "
                                    "write-only, so a stale one can't be inspected — just "
                                    "save a fresh one)."
                                ) from e
                            raise _AgentConfigError(
                                f"The agent coding backend ({_host}) rejected the request: "
                                "NO coding API key was sent, so the local agent key was "
                                "used instead. Set 'Coding backend' to OpenAI-compatible "
                                "in Settings → Agent and enter the API key — the key field "
                                "only appears once that backend is selected."
                            ) from e
                        if "404" in emsg and "not found" in emsg.lower():
                            # Probe with the CODING key when there is one: using the agent
                            # key here 401s against a cloud coder, so the "This server has:
                            # …" hint — the only useful part of this error — silently
                            # vanished exactly when it was needed.
                            ids = _list_model_ids(coding_base_url,
                                                  coding_api_key or agent_api_key)
                            have = f" This server has: {', '.join(ids)}." if ids else ""
                            raise _AgentConfigError(
                                f"The agent coding model {coding_model_id!r} does not "
                                f"exist on {coding_base_url}.{have} Set 'Agent coding "
                                "model name' in Settings → Agent to one of these exact "
                                "names (for Ollama use the tag, e.g. qwen3:14b)."
                            ) from e
                        last_err = e
                        # An OVERLOAD (429/rate limit, task #114) is its own class: the
                        # server answered and told us to slow down, so it needs a PATIENT,
                        # jittered backoff rather than the gateway policy's prompt re-send.
                        # Checked first — _is_backend_unavailable already excludes it, but
                        # the ordering keeps the intent obvious.
                        overloaded = _is_backend_overloaded(e)
                        gateway = (not overloaded) and _is_backend_unavailable(e)
                        if (overloaded or gateway) and max_attempts < 6:
                            max_attempts = 6  # stretch the budget for a flaky backend
                        attempt += 1
                        if attempt < max_attempts:
                            # Longer, capped backoff for gateway errors (up to ~8s) so a
                            # cold/loading model has time to come back; short for the rest;
                            # patient + jittered for an overload. See _retry_delay.
                            delay = _retry_delay(attempt, overloaded, gateway, e)
                            kind = "gateway/backend unavailable" if gateway else "transient"
                            if overloaded:
                                kind = "backend overloaded (429/rate limit)"
                                # SURFACE IT: without this the REPL keeps showing a playful
                                # heartbeat word and the user has no idea the provider is
                                # refusing work (live: 167s of "Thinking…" → Ctrl+C).
                                _emit({"type": "step",
                                       "text": "Coding model is overloaded (429) — waiting "
                                               f"{delay:.0f}s, then retrying "
                                               f"(attempt {attempt + 1}/{max_attempts})…"})
                            _log(f"generate {kind} error (attempt {attempt}/{max_attempts}): "
                                 f"{_clip(emsg, 160)} — retrying in {delay:.0f}s")
                            time.sleep(delay)
                        elif overloaded:
                            # Out of attempts: end the turn with a cause the user can act on
                            # instead of a bare stack trace (same spirit as #94's 502 copy).
                            _log("backend overloaded after all attempts → failing honestly")
                            raise _AgentBackendOverloaded(
                                "The agent coding model backend "
                                f"({_host_of(coding_base_url)}) is overloaded right now and "
                                f"refused {max_attempts} attempts (HTTP 429). Nothing ran. "
                                "Please try again in a minute, or switch the coding backend "
                                "in Settings → Agent."
                            ) from e
                if last_err is not None:
                    raise last_err
            finally:
                hb_stop.set()
            # Task #83: reconcile the pre-call estimate with the server's EXACT
            # prompt_tokens when it's higher — keeps the running total monotonic (the
            # estimate already showed the count climbing at request time). Best-effort.
            try:
                tu = getattr(msg, "token_usage", None)
                exact = int(getattr(tu, "input_tokens", 0) or 0) if tu else 0
                if exact > _step_est:
                    self._code_tokens_total += (exact - _step_est)
                    if exact > self._code_tokens_max:
                        self._code_tokens_max = exact
                    _emit({
                        "type": "tokens",
                        "codeInput": self._code_tokens_total,
                        "codeMax": self._code_tokens_max,
                        "stepInput": exact,
                    })
            except Exception:  # noqa: BLE001
                pass
            # Task #104: accumulate OUTPUT (completion) tokens this step and report the
            # running total. Unlike input, there's no request-time estimate — output is
            # only known once the model finished, so we emit after the call returns.
            try:
                tu = getattr(msg, "token_usage", None)
                step_out = int(getattr(tu, "output_tokens", 0) or 0) if tu else 0
                if step_out > 0:
                    self._code_output_tokens_total += step_out
                    _emit({
                        "type": "tokens",
                        "codeInput": self._code_tokens_total,
                        "codeMax": self._code_tokens_max,
                        "codeOutput": self._code_output_tokens_total,
                        "stepOutput": step_out,
                    })
            except Exception:  # noqa: BLE001
                pass
            # Task #107: persist the full RAW code-model response (opt-in) with this call's
            # token counts, emitted BEFORE any think-stripping/normalization so it matches
            # exactly what the token counts were computed from. Worker forwards to Mongo.
            if self._save_full_response:
                try:
                    self._call_index += 1
                    _raw = getattr(msg, "content", None)
                    _tu2 = getattr(msg, "token_usage", None)
                    _emit({
                        "type": "code_response",
                        "step": self._call_index,
                        "text": _raw if isinstance(_raw, str)
                        else ("" if _raw is None else str(_raw)),
                        "inputTokens": int(getattr(_tu2, "input_tokens", 0) or 0) if _tu2 else 0,
                        "outputTokens": int(getattr(_tu2, "output_tokens", 0) or 0) if _tu2 else 0,
                    })
                except Exception:  # noqa: BLE001
                    pass
            try:
                content = getattr(msg, "content", None)
                if content is None:
                    _log("model returned content=None (empty/thinking-only turn) → coercing to ''")
                    msg.content = ""
                    content = ""
                elif isinstance(content, str) and "</think>" in content.lower():
                    cleaned = _strip_think(content)
                    _log(f"stripped <think> trace ({len(content) - len(cleaned)} chars)")
                    msg.content = cleaned
                    content = cleaned
                # Normalize code-block delimiters so smolagents' <code>…</code> parser
                # matches even when the model wrote "<code >" / a markdown fence. Without
                # this, tag-whitespace variants fail EVERY step → "Reached max steps".
                # GATED on real tool-call intent: a finished PROSE answer often contains
                # markdown fences (e.g. a ```java snippet) — blindly converting those to
                # <code> made smolagents EXECUTE the snippet as python → SyntaxError,
                # a wasted step, and a worse fallback answer (seen live). If there is no
                # executable intent at all and the prose reads like a final answer, wrap
                # it in final_answer() so the run finishes cleanly with the model's text.
                if isinstance(content, str) and content:
                    has_code_tag = re.search(r"<code[\s>]", content, re.I) is not None
                    fence_bodies = re.findall(r"```[a-zA-Z0-9_+-]*\n?([\s\S]*?)```", content)
                    _tool_call_re = (
                        r"\b(final_answer|download_file|unzip_file|list_dir|read_text_file|"
                        r"rag_index|rag_add|rag_search|web_search|fetch_url|http_request|"
                        r"calculate|get_current_datetime|create_pdf|extract_text_from_pdf|"
                        r"send_email)\s*\("
                    )
                    fence_is_toolcall = any(re.search(_tool_call_re, b) for b in fence_bodies)
                    if has_code_tag or fence_is_toolcall:
                        normalized = _normalize_code_tags(content)
                        if normalized != content:
                            _log("normalized code-block tags (<code >/fence → <code>)")
                        # Facet 1: rescue multi-line file content passed with RAW newlines
                        # inside a normal "..." ("unterminated string literal") — repair the
                        # <code> block ONLY when it makes the snippet compile.
                        repaired = _repair_code_block_strings(normalized)
                        if repaired != normalized:
                            _log("repaired unterminated string literal in tool-call code")
                        if repaired != content:
                            msg.content = repaired
                        # Loop-breaker (#95 B1): if the block STILL won't compile after the
                        # repair, the model is stuck emitting an unparseable tool call. Count
                        # consecutive failures; after a few, finish honestly rather than burn
                        # the whole step budget — each retry is a full (often 150s+) model
                        # call on an ever-growing context. B2's greedy/defuse fix means the
                        # <code>-in-content collision now compiles here, so this only trips on
                        # genuinely un-fixable code (e.g. a truncated generation).
                        _cb = re.search(r"<code>([\s\S]*?)</code>", msg.content or "")
                        _still_bad = False
                        if _cb:
                            try:
                                compile(_cb.group(1), "<gonext-code>", "exec")
                            except Exception:  # noqa: BLE001
                                _still_bad = True
                        if _still_bad:
                            self._parse_fail_streak += 1
                            _log("tool-call code still unparseable after repair "
                                 f"(streak {self._parse_fail_streak}/3)")
                            if self._parse_fail_streak >= 3:
                                _log("parse-fail streak hit limit → ending the turn honestly "
                                     "instead of looping on an unparseable tool call")
                                _stuck_msg = (
                                    "I got stuck on a code-formatting error: my edit to the "
                                    "file kept failing to parse (a string/escaping issue in the "
                                    "tool call), so I stopped instead of retrying in a loop. "
                                    "Any dev server I started is still running. Please try "
                                    "again, or tell me the specific change to make and I'll "
                                    "keep it to a small, single-line edit."
                                )
                                msg.content = ("<code>\nfinal_answer("
                                               + repr(_stuck_msg) + ")\n</code>")
                                self._parse_fail_streak = 0
                        else:
                            self._parse_fail_streak = 0
                    else:
                        self._parse_fail_streak = 0  # a prose turn isn't a parse failure
                        stripped = content.strip()
                        looks_final = (
                            len(stripped) >= 400 or re.search(
                                r"(^|\n)#{1,4}\s|\n\d+\.\s|\n[-*]\s|\|.+\|", stripped
                            ) is not None
                        ) and not _looks_like_scratchpad(stripped)
                        if stripped and looks_final:
                            _log(
                                f"prose-only turn ({len(stripped)} chars, no tool call) "
                                "→ auto-wrapping as final_answer"
                            )
                            msg.content = (
                                stripped + "\n<code>\nfinal_answer(" + repr(stripped) + ")\n</code>"
                            )
                        elif stripped and _looks_like_scratchpad(stripped):
                            # Looks like unresolved deliberation, not a finished answer — do
                            # NOT wrap it as final_answer (that would dump this rambling
                            # scratchpad into the user-facing reply AND permanently pollute
                            # every future turn's conversation history with it, seen live:
                            # a 6000+ char "Thinking Process:" dump derailed the next two
                            # unrelated questions). Leave content as-is: smolagents' own
                            # code-block parser will fail to find one and burn this step as
                            # a normal parse-error retry, giving the model another attempt.
                            _log(
                                f"scratchpad-like prose ({len(stripped)} chars, no tool call) "
                                "→ NOT auto-wrapping; letting the step fail normally"
                            )
                # Loop-breaker for reasoning coders (#113). _parse_fail_streak above only
                # counts blocks that EXIST but won't compile, so a coder that never emits a
                # usable block at all — prose plans, empty turns, a dangling tag, or a
                # recovery we rejected — never trips it and just burns the step budget
                # (live: 3 steps, ~25k prompt tokens, zero progress, user had to Ctrl+C).
                # Count consecutive no-block steps here and finish honestly instead. The
                # check is on the FINAL msg.content, so anything the repairs above rescued
                # (including the _parse_fail_streak fallback, which injects a compilable
                # final_answer) resets the streak. openai backend only — an Ollama coder
                # keeps looping exactly as long as it does today.
                if self._openai_backend:
                    if _has_runnable_block(msg.content):
                        self._no_code_streak = 0
                    else:
                        self._no_code_streak += 1
                        _log("no runnable code block this step "
                             f"(no-code streak {self._no_code_streak}/3, #113)")
                        if self._no_code_streak >= 3:
                            _log("no-code streak hit limit → ending the turn honestly "
                                 "instead of looping on a coder that won't emit code")
                            _nocode_msg = (
                                "I had to stop: the coding model kept replying with "
                                "explanations instead of a runnable code block, so none of "
                                "my steps could actually execute. This is usually a "
                                "reasoning model putting its answer in the thinking channel. "
                                "Try a non-reasoning coding model, or switch the coding "
                                "backend back to Ollama in Settings, then run this again."
                            )
                            msg.content = ("<code>\nfinal_answer("
                                           + repr(_nocode_msg) + ")\n</code>")
                            self._no_code_streak = 0
                            self._parse_fail_streak = 0
            except Exception as e:  # noqa: BLE001
                _log(f"content-normalize error: {e}")
            return msg

        # Replaces smolagents' stock code-parsing error for reasoning backends (#113).
        # The stock text spells out the delimiters twice ("the regex pattern <code>(.*?)
        # </code> was not found", "for instance: … <code> # Your python code here </code>")
        # and quotes the model's own broken output back at it. A reasoning model MIRRORS
        # those tokens: the live trace shows Kimi K3 answering the error with the literal
        # words "and closing with" wrapped in code tags — it was completing the sentence
        # from the prompt, not writing code. Deliberately carries NO literal delimiter (the
        # system prompt already shows the format with a worked example), so there is nothing
        # to echo. Applied only when kind=openai; Ollama keeps the stock message.
        _PARSE_ERROR_REPLACEMENT = (
            "Error: your last reply contained no runnable code block, so nothing executed "
            "and no observation was produced.\n"
            "Reply with ONE 'Thought:' line, then the code block using the exact opening and "
            "closing delimiters shown in the system prompt, with ONLY Python between them — "
            "no prose inside the block, and do not repeat these instructions back to me. "
            "Call exactly one tool, or call final_answer(...) if you are done."
        )

        def _sanitize_parse_errors(self, msgs):
            """Swap smolagents' code-parsing error text for a delimiter-free directive
            (see _PARSE_ERROR_REPLACEMENT). No-op unless the backend is kind=openai, and
            no-op on every other kind of message — genuine SyntaxError feedback from a
            block that DID run is left intact, since that is real information."""
            if not self._openai_backend:
                return
            n = 0
            for m in msgs or []:
                if not isinstance(m, dict):
                    continue  # _prepare_completion_kwargs already returns plain dicts
                c = m.get("content")
                if isinstance(c, str):
                    if "Error in code parsing" in c:
                        m["content"] = self._PARSE_ERROR_REPLACEMENT
                        n += 1
                elif isinstance(c, list):
                    for part in c:
                        if (isinstance(part, dict) and isinstance(part.get("text"), str)
                                and "Error in code parsing" in part["text"]):
                            part["text"] = self._PARSE_ERROR_REPLACEMENT
                            n += 1
            if n:
                _log(f"#113 replaced {n} code-parsing error message(s) with a "
                     "delimiter-free directive (openai backend)")

        def _prepare_completion_kwargs(self, *args, **kwargs):
            ck = super()._prepare_completion_kwargs(*args, **kwargs)
            try:
                self._sanitize_parse_errors(ck.get("messages", []) or [])
            except Exception as e:  # noqa: BLE001
                _log(f"#113 parse-error sanitize skipped: {e}")
            # Disable Qwen3 "thinking" for the agent loop. Live runs showed thinking-on
            # spends the whole completion budget on a <think> trace and returns
            # content=None (no code block) → parse-error spiral → WS timeout. The
            # `/no_think` soft switch (recognized by the Qwen3 chat template) forces a
            # direct Thought+code reply every step. Injected into the SYSTEM message so it
            # applies to EVERY step's request, not just the first user turn.
            try:
                msgs = ck.get("messages", []) or []
                for m in msgs:
                    role = m.get("role") if isinstance(m, dict) else getattr(m, "role", None)
                    if str(role).endswith("system") or role == "system":
                        c = m.get("content") if isinstance(m, dict) else getattr(m, "content", None)
                        if isinstance(c, str) and "/no_think" not in c:
                            m["content"] = c + "\n\n/no_think"
                        elif isinstance(c, list):
                            for part in c:
                                if isinstance(part, dict) and isinstance(part.get("text"), str) \
                                        and "/no_think" not in part["text"]:
                                    part["text"] += "\n\n/no_think"
                                    break
                        break
            except Exception as e:  # noqa: BLE001
                _log(f"/no_think inject error: {e}")
            try:
                msgs = ck.get("messages", []) or []
                _log(f"=== MODEL REQUEST: {len(msgs)} message(s) sent to the model ===")
                for i, m in enumerate(msgs):
                    role = m.get("role") if isinstance(m, dict) else getattr(m, "role", "?")
                    content = (
                        m.get("content") if isinstance(m, dict)
                        else getattr(m, "content", "")
                    )
                    if isinstance(content, list):
                        text = " ".join(
                            (c.get("text", "") if isinstance(c, dict) else str(c))
                            for c in content
                        )
                    else:
                        text = str(content)
                    text = text.replace("\n", " ")
                    _log(f"  [{i}] {role} ({len(text)} chars): {text[:600]}")
                _log("=== END MODEL REQUEST ===")
            except Exception as e:  # noqa: BLE001
                _log(f"MODEL REQUEST log error: {e}")
            return ck

    # Tool-invocation mode: "code" (default) makes the model author a Python code
    # block (CodeAgent); "toolcall" makes it emit a structured JSON tool call
    # (ToolCallingAgent), which sidesteps the unterminated-string-literal failures weak
    # models hit when forced to echo long payloads as Python literals. Default is
    # unchanged ("code"); flip via cfg.agentToolMode to A/B without touching the
    # working paths (datetime / web_search / http_request).
    agent_tool_mode = (cfg.get("agentToolMode") or "code").strip().lower()
    _AgentBase = CodeAgent
    if agent_tool_mode == "toolcall":
        try:
            from smolagents import ToolCallingAgent
            _AgentBase = ToolCallingAgent
            _log("agent tool mode: toolcall (structured JSON tool calls)")
        except Exception as e:  # noqa: BLE001
            _log(f"ToolCallingAgent unavailable ({e}); falling back to CodeAgent")
    else:
        _log("agent tool mode: code (python CodeAgent)")

    # Exhaustion fallback: normally the model calls final_answer() itself within the
    # multi-step loop and that answer is used. Only if it burns through all max_steps
    # WITHOUT calling final_answer does smolagents call provide_final_answer to
    # synthesize one. We override that to return the last tool observation
    # deterministically (or a plain reply) rather than dead-ending — and without a weak
    # model corrupting exact tool output (dates/numbers).
    class _ToolAgent(_AgentBase):
        def provide_final_answer(self, task, *args, **kwargs):
            from smolagents.models import ChatMessage, MessageRole
            # Deterministic PDF delivery: the user asked for a PDF and the model did the
            # research but never emitted the create_pdf call itself (weak/slow coders often
            # keep researching and never converge). Rendering the gathered facts here is the
            # INTENDED deliverable, not an error — so deliver it cleanly, with NO apologetic
            # "may be partial / hit my step limit" note (that made a normal, successful PDF
            # look like a failure to the user).
            if (agent_pdf_requested and _gathered
                    and not (_last_obs.get("text") or "").startswith("✅")):
                _log(f"delivering PDF deterministically from {len(_gathered)} "
                     "gathered result(s) (model didn't call create_pdf itself)")
                try:
                    # Reformat the raw observations into a clean document (headings/tables)
                    # via the fast local chat model; fall back to the raw concatenation if
                    # synthesis fails or returns nothing.
                    compiled = _synthesize_document(
                        _gathered, latest_user_text,
                        agent_base_url, agent_api_key, agent_model_id,
                    )
                    if compiled:
                        _log(f"synthesized document ({len(compiled)} chars) for PDF")
                    else:
                        compiled = "\n\n---\n\n".join(_gathered)[:12000]
                    out = create_pdf(
                        text=compiled, title=_derive_pdf_title(latest_user_text)
                    )
                    if isinstance(out, str) and out.startswith("✅"):
                        return ChatMessage(role=MessageRole.ASSISTANT, content=out)
                    _log(f"deterministic create_pdf did not succeed: {str(out)[:120]}")
                except Exception as e:  # noqa: BLE001
                    _log(f"deterministic create_pdf error: {type(e).__name__}: {e}")
            _maxsteps_partial["hit"] = True
            text = (_last_obs.get("text") or "").strip()
            if text:
                _log(f"max-steps fallback (last tool obs) → {text[:80]}")
                # Summarize WHAT WAS TRIED + the current blocker, not just the last tool
                # observation (task #79): a bare last obs ('Stopped npm run dev') reads as a
                # non-answer and hides the loop the run got stuck in. Build a compact bullet
                # list of the tool calls from this turn's step trace + the last result.
                _tried = []
                for _st in _turn_steps[-8:]:
                    _did = str(_st.get("did") or "").strip().replace("\n", " ")
                    _m = re.search(r"([a-z_]+\([^\n]{0,70})", _did)
                    _label = (_m.group(1) if _m else _did)[:70].strip()
                    if _label:
                        _tried.append(f"  • {_label}")
                _summary = "I ran out of steps before finishing this."
                if _tried:
                    _summary += " Here's what I tried:\n" + "\n".join(_tried)
                _summary += "\n\nCurrent blocker (last result):\n" + text[:600]
                _summary += "\n\nReply 'continue' to have me keep going from here."
                text = _summary
            else:
                # No usable tool output (e.g. every step's code failed to parse).
                # Don't dead-end on a canned apology — answer from the conversation.
                _log("max-steps fallback → plain reply over conversation")
                try:
                    text = _plain_reply(
                        messages, agent_base_url, agent_api_key, agent_model_id
                    ).strip()
                except Exception as e:  # noqa: BLE001
                    _log(f"plain-reply fallback error: {e}")
                    text = ""
                if not text:
                    text = ("I couldn't complete that within the step budget. Please "
                            "rephrase, or give a specific URL/API to call.")
            return ChatMessage(role=MessageRole.ASSISTANT, content=text)

    try:
        # Ollama's OpenAI-compat endpoint ignores Qwen3's /no_think soft-switch (its
        # template forces thinking mode; "think": false is also ignored there) but DOES
        # honor reasoning_effort="none". Thinking burned ~430 of 487 output tokens in
        # the live test — minutes per step on a slow GPU — so disable it explicitly.
        # Sniffed per-server because MLX might reject the unknown param.
        extra_model_kwargs = {}
        # `coding_backend` was resolved once, above, and `coding_flags` derives everything
        # from it — do NOT re-derive any of this from the kind or a URL sniff here.
        if coding_flags["ollama_tweaks"]:
            _log("coding server is Ollama → reasoning_effort='none' (disable thinking) + streaming")
            extra_model_kwargs["reasoning_effort"] = "none"
        _stream_coder = coding_flags["stream"]
        _log(f"coding backend: {coding_backend} ({_backend_why}; "
             f"kind={coding_kind or 'auto'}, {_host_of(coding_base_url)}) "
             f"stream={_stream_coder} steps={max_steps} "
             f"key={'yes' if coding_api_key else 'no'}")
        model = _LoggingModel(
            model_id=coding_model_id,
            api_base=coding_base_url,
            # Task #108: use the OpenAI-compatible coding key when set, else the chat key.
            api_key=coding_api_key or agent_api_key,
            # Stream for Ollama / OpenAI-compatible so a long generation keeps the connection
            # warm and can't trip a reverse-proxy idle read-timeout (smolagents still gets the
            # full text). Only a local MLX coder stays non-streaming.
            stream_agent=_stream_coder,
            # Task #107: persist each code-model response to Mongo when the user opted in.
            save_full_response=save_full_response,
            # Task #113: enables the reasoning-model repairs (prose-only nudge, parse-error
            # sanitizing, no-code breaker). Gated to the OpenAI-compatible backend — Ollama
            # and local MLX never take those branches. #113 gated this on the EXPLICIT
            # kind; the intent (don't touch Ollama) is unchanged, but an Auto+cloud config
            # now resolves to "openai" instead of "local", so it finally gets the repairs
            # it was always meant to have.
            openai_backend=coding_flags["openai_repairs"],
            # Cap each HTTP attempt at 600s so a genuinely slow single call (a
            # remote M6000/Vulkan box doing a cold ~4-5min prompt-eval) can finish
            # on the first try, while a truly hung request still fails into OUR
            # retry loop rather than eating the whole 30min worker budget.
            # max_retries=0: the openai client's own silent retries would otherwise
            # stack multiplicatively with ours (3 × 3 attempts).
            client_kwargs={"timeout": 600.0, "max_retries": 0},
            **extra_model_kwargs,
        )
        @tool
        def download_file(url: str, dest_path: str = "") -> str:
            """Download a file (e.g. a .zip) from a public http(s) URL to a local path on this machine. Use this FIRST when the user gives a URL to a zip of files; then unzip_file, then rag_index.

            Args:
                url: public http(s) URL of the file to download.
                dest_path: optional local save path; leave empty for an automatic work-dir path.
            """
            try:
                import os
                path, cached = _rag_download(url, (dest_path or "").strip())
                if cached:
                    _emit({"type": "step", "text": "Already downloaded — reusing cached file"})
                    out = (f"Already downloaded earlier — reusing {path} "
                           f"({os.path.getsize(path)} bytes). Next: unzip_file(zip_path='{path}').")
                    _last_obs["text"] = out
                    return out
                _emit({"type": "step", "text": f"Downloading → {url[:80]}"})
                out = f"Downloaded to {path} ({os.path.getsize(path)} bytes). Next: unzip_file(zip_path='{path}') if it is a zip."
                _last_obs["text"] = out
                return out
            except Exception as e:  # noqa: BLE001
                msg = f"Error: download failed: {type(e).__name__}: {e}"
                _last_obs["text"] = msg
                return msg

        @tool
        def unzip_file(zip_path: str, dest_dir: str = "") -> str:
            """Extract a local .zip file into a local directory. Use after download_file. Returns the directory and a short file listing.

            Args:
                zip_path: local path to the .zip (from download_file).
                dest_dir: optional target directory; leave empty for an automatic one.
            """
            try:
                d, names, cached = _rag_unzip((zip_path or "").strip(), (dest_dir or "").strip())
                preview = ", ".join(names[:15])
                more = f" …(+{len(names) - 15} more)" if len(names) > 15 else ""
                if cached:
                    _emit({"type": "step", "text": "Already unzipped — reusing extracted files"})
                    out = (f"Already unzipped earlier — {len(names)} files at {d}. "
                           f"Files: {preview}{more}. Use list_dir/read_text_file or rag_index on it.")
                    _last_obs["text"] = out
                    return out
                _emit({"type": "step", "text": "Unzipping…"})
                out = f"Unzipped {len(names)} files to {d}. Files: {preview}{more}. Next: rag_index(path='{d}', source_url=<the original url>)."
                _last_obs["text"] = out
                return out
            except Exception as e:  # noqa: BLE001
                msg = f"Error: unzip failed: {type(e).__name__}: {e}"
                _last_obs["text"] = msg
                return msg

        @tool
        def list_dir(path: str) -> str:
            """List files and subfolders under a local directory (from unzip_file). Use to see what files are in the project before reading or indexing them.

            Args:
                path: local directory path to list.
            """
            try:
                import os
                root = _rag_read_allowed((path or ".").strip())
                if os.path.isfile(root):
                    out = f"{root} is a file ({os.path.getsize(root)} bytes)."
                    _last_obs["text"] = out
                    return out
                _rb = _read_repeat_block(root)
                if _rb:
                    _last_obs["text"] = _rb
                    return _rb
                entries = []
                for dirpath, dirnames, filenames in os.walk(root):
                    dirnames[:] = [d for d in dirnames if d not in _RAG_SKIP_DIRS]
                    for fn in filenames:
                        # ABSOLUTE path — read_file_lines/read_text_file resolve relative
                        # paths against the WORKSPACE root, not whatever subfolder was
                        # listed here, so a name relative to `root` could resolve wrong.
                        entries.append(os.path.join(dirpath, fn))
                        if len(entries) >= 300:
                            break
                    if len(entries) >= 300:
                        break
                entries.sort()
                more = " …(truncated)" if len(entries) >= 300 else ""
                out = _ws_cap_obs(
                    f"{len(entries)} files under {root}:\n" + "\n".join(entries) + more)
                _last_obs["text"] = out
                return out
            except Exception as e:  # noqa: BLE001
                msg = f"Error: list_dir failed: {type(e).__name__}: {e}"
                _last_obs["text"] = msg
                return msg

        @tool
        def read_text_file(path: str, max_chars: int = 20000) -> str:
            """Read the text contents of a single local file (e.g. a README) from a downloaded/unzipped project. Use this for a QUICK summary when you don't need full indexing.

            Args:
                path: local path to the text file (e.g. project_files/tools-hook/README.md).
                max_chars: maximum characters to return (default 20000).
            """
            try:
                import os
                rp = _rag_read_allowed((path or "").strip())
                if not os.path.isfile(rp):
                    msg = f"Error: not a file: {path}"
                    _last_obs["text"] = msg
                    return msg
                if os.path.getsize(rp) > 5 * 1024 * 1024:
                    msg = "Error: file too large to read (> 5 MB)."
                    _last_obs["text"] = msg
                    return msg
                _rb = _read_repeat_block(rp)
                if _rb:
                    _last_obs["text"] = _rb
                    return _rb
                with open(rp, "r", encoding="utf-8", errors="replace") as fh:
                    data = fh.read(max(500, min(int(max_chars or 20000), 100000)))
                out = _ws_cap_obs(data or "(empty file)")
                _last_obs["text"] = out
                return out
            except Exception as e:  # noqa: BLE001
                msg = f"Error: read_text_file failed: {type(e).__name__}: {e}"
                _last_obs["text"] = msg
                return msg

        @tool
        def rag_index(path: str, source_url: str) -> str:
            """Index a local file or directory of TEXT/code files into the knowledge base for source_url (stores embeddings in the knowledge base). Use after unzip_file so the files become searchable. source_url MUST be the original URL the user provided.

            Args:
                path: local file or directory to index (from unzip_file).
                source_url: the original URL the user gave — used as the knowledge-base key.
            """
            if not _RAG_AVAILABLE:
                msg = ("Error: RAG is not configured (enable RAG in Settings"
                       + ("" if _RAG_LOCAL else " and add AWS credentials, or switch to local RAG")
                       + ").")
                _last_obs["text"] = msg
                return msg
            _emit({"type": "step", "text": f"RAG source → {source_url}"})
            _emit({"type": "step", "text": "Indexing files for RAG…"})
            try:
                import time as _t
                records = []
                files = 0
                for abs_path, rel, ext in _rag_iter_text_files((path or "").strip()):
                    try:
                        with open(abs_path, "r", encoding="utf-8", errors="replace") as fh:
                            text = fh.read()
                    except OSError:
                        continue
                    files += 1
                    records.extend(_rag_chunk_text(text, rel, ext))
                if not records:
                    msg = "No indexable text files found at that path."
                    _last_obs["text"] = msg
                    return msg
                for i in range(0, len(records), 64):
                    batch = records[i:i + 64]
                    vecs = _embed(rag_embed_base, rag_embed_model, [r["text"] for r in batch])
                    for r, v in zip(batch, vecs):
                        r["embedding"] = v
                    _emit({"type": "step", "text": f"Embedded {min(i + 64, len(records))}/{len(records)} chunks…"})
                _rag_write_shard(source_url, records, "chunks")
                manifest = {
                    "sourceUrl": source_url, "embedModel": rag_embed_model,
                    "dims": len(records[0].get("embedding") or []),
                    "files": files, "chunks": len(records),
                    "updatedAt": _t.strftime("%Y-%m-%dT%H:%M:%SZ", _t.gmtime()),
                }
                _rag_write_manifest(source_url, manifest)
                where = "locally" if _RAG_LOCAL else "in the cloud knowledge base"
                out = (f"Indexed {files} files into {len(records)} chunks for {source_url} ({where}). "
                       f"Use rag_search(source_url='{source_url}', query=...) to retrieve, then answer the user.")
                _last_obs["text"] = out
                return out
            except Exception as e:  # noqa: BLE001
                msg = f"Error: indexing failed: {type(e).__name__}: {e}"
                _last_obs["text"] = msg
                return msg

        @tool
        def rag_add(source_url: str, text: str) -> str:
            """Add extra text/notes to an EXISTING knowledge base (keyed by source_url). Use when the user gives more information to remember for later questions.

            Args:
                source_url: the knowledge-base key (the original URL).
                text: the new information to embed and store.
            """
            if not _RAG_AVAILABLE:
                msg = "Error: RAG is not configured."
                _last_obs["text"] = msg
                return msg
            if not (text or "").strip():
                msg = "Error: no text to add."
                _last_obs["text"] = msg
                return msg
            _emit({"type": "step", "text": f"RAG source → {source_url}"})
            _emit({"type": "step", "text": "Adding info to RAG…"})
            try:
                records = _rag_chunk_text(text, "user-note", ".txt")
                vecs = _embed(rag_embed_base, rag_embed_model, [r["text"] for r in records])
                for r, v in zip(records, vecs):
                    r["embedding"] = v
                _rag_write_shard(source_url, records, "chunks-add")
                out = f"Added {len(records)} chunk(s) to the knowledge base for {source_url}."
                _last_obs["text"] = out
                return out
            except Exception as e:  # noqa: BLE001
                msg = f"Error: rag_add failed: {type(e).__name__}: {e}"
                _last_obs["text"] = msg
                return msg

        @tool
        def rag_search(source_url: str, query: str, k: int = 0) -> str:
            """Retrieve the most relevant chunks from the knowledge base for source_url. Use to ANSWER questions about the indexed files; write your answer from the returned chunks.

            Args:
                source_url: the knowledge-base key (the original URL).
                query: what to look for.
                k: number of chunks to return (0 = configured default).
            """
            if not _RAG_AVAILABLE:
                msg = "Error: RAG is not configured."
                _last_obs["text"] = msg
                return msg
            _emit({"type": "step", "text": f"RAG source → {source_url}"})
            _emit({"type": "step", "text": f"Searching RAG → {query[:60]}"})
            try:
                chunks = _rag_load_all(source_url)
                if not chunks:
                    msg = f"No knowledge base found for {source_url}. Run rag_index first."
                    _last_obs["text"] = msg
                    return msg
                qvec = _embed(rag_embed_base, rag_embed_model, [query])[0]
                scored = sorted(
                    ((_cosine(qvec, c.get("embedding") or []), c) for c in chunks),
                    key=lambda x: x[0], reverse=True,
                )
                topk = scored[: (k if k and k > 0 else rag_top_k)]
                parts = [f"[{c.get('file', '?')}] (score {score:.2f})\n{c.get('text', '')}"
                         for score, c in topk]
                out = "\n\n---\n\n".join(parts) if parts else "No matches."
                _last_obs["text"] = out
                return out
            except Exception as e:  # noqa: BLE001
                msg = f"Error: rag_search failed: {type(e).__name__}: {e}"
                _last_obs["text"] = msg
                return msg

        @tool
        def grep_repo(pattern: str, path: str = "", glob: str = "") -> str:
            """Search text files for a pattern (regex or literal) and return file:line hits. Use to find the EXACT lines to edit after rag_search located the relevant area.

            Args:
                pattern: regex (or literal text) to search for.
                path: directory to search (a workspace or unzipped folder). Empty = first registered workspace.
                glob: optional filename filter, e.g. "*.java" or "*Controller*".
            """
            try:
                import fnmatch
                import os
                root = _rag_read_allowed((path or "").strip() or _default_ws_root())
                try:
                    rx = re.compile(pattern)
                except re.error:
                    rx = re.compile(re.escape(pattern))
                hits = []
                for abs_path, rel, _ext in _rag_iter_text_files(root):
                    if glob and not fnmatch.fnmatch(os.path.basename(rel), glob) \
                            and not fnmatch.fnmatch(rel, glob):
                        continue
                    try:
                        with open(abs_path, "r", encoding="utf-8", errors="replace") as fh:
                            for i, line in enumerate(fh, 1):
                                if rx.search(line):
                                    # ABSOLUTE path — read_file_lines/read_text_file resolve
                                    # relative paths against the WORKSPACE root, not this
                                    # search root, so a path relative to `root` (e.g. when
                                    # searching a subfolder) would be silently wrong there.
                                    hits.append(f"{abs_path}:{i}: {line.rstrip()[:200]}")
                                    if len(hits) >= 100:
                                        break
                    except OSError:
                        continue
                    if len(hits) >= 100:
                        break
                _emit({"type": "step", "text": f"Searching code → {pattern[:60]} ({len(hits)} hits)"})
                if not hits:
                    msg = f"No matches for {pattern!r} under {root}."
                    _last_obs["text"] = msg
                    return msg
                more = "\n…(capped at 100 hits)" if len(hits) >= 100 else ""
                out = _ws_cap_obs(f"Matches under {root}:\n" + "\n".join(hits) + more)
                _last_obs["text"] = out
                return out
            except Exception as e:  # noqa: BLE001
                msg = f"Error: grep_repo failed: {type(e).__name__}: {e}"
                _last_obs["text"] = msg
                return msg

        @tool
        def read_file_lines(path: str, start_line: int = 1, end_line: int = 0) -> str:
            """Read a file with LINE NUMBERS (for locating code before edit_lines). end_line 0 = to end (capped).

            Args:
                path: local file path.
                start_line: first line to show (1-based).
                end_line: last line to show inclusive; 0 = rest of file (max 400 lines shown).
            """
            try:
                rp = _rag_read_allowed((path or "").strip())
                with open(rp, "r", encoding="utf-8", errors="replace") as fh:
                    lines = fh.readlines()
                s = max(1, int(start_line or 1))
                e = int(end_line or 0) or len(lines)
                e = min(e, len(lines), s + 399)
                body = "".join(f"{i}: {lines[i - 1]}" for i in range(s, e + 1))
                out = _ws_cap_obs(f"{rp} (lines {s}-{e} of {len(lines)}):\n{body}")
                _last_obs["text"] = out
                return out
            except Exception as ex:  # noqa: BLE001
                msg = f"Error: read_file_lines failed: {type(ex).__name__}: {ex}"
                _last_obs["text"] = msg
                return msg

        @tool
        def edit_lines(path: str, start_line: int, end_line: int, new_content: str) -> str:
            """Replace lines start_line..end_line (inclusive, 1-based) of a workspace file with new_content. The PRIMARY edit tool — get exact line numbers from grep_repo/read_file_lines first. A backup is saved automatically.

            Args:
                path: file inside a registered workspace.
                start_line: first line to replace (1-based).
                end_line: last line to replace (inclusive).
                new_content: replacement text (may be more or fewer lines).
            """
            try:
                rp = _ws_write_allowed((path or "").strip())
                import os
                if not os.path.isfile(rp):
                    msg = f"Error: not a file: {path}"
                    _last_obs["text"] = msg
                    return msg
                backup = _ws_snapshot(rp)
                with open(rp, "r", encoding="utf-8", errors="replace") as fh:
                    lines = fh.readlines()
                s, e = int(start_line), int(end_line)
                if not (1 <= s <= e <= len(lines)):
                    msg = (f"Error: line range {s}-{e} is out of bounds "
                           f"(file has {len(lines)} lines). Re-check with read_file_lines.")
                    _last_obs["text"] = msg
                    return msg
                new_lines = new_content.splitlines(keepends=True)
                if new_lines and not new_lines[-1].endswith("\n"):
                    new_lines[-1] += "\n"
                old_lines = lines[s - 1:e]
                lines[s - 1:e] = new_lines
                with open(rp, "w", encoding="utf-8") as fh:
                    fh.writelines(lines)
                _ws_record_change("edit", rp, backup)
                _emit({"type": "step", "text": _render_edit_card(
                    "Update", rp,
                    f"Replaced lines {s}-{e} with {len(new_lines)} line(s)",
                    removed=[(s + i, t) for i, t in enumerate(old_lines)],
                    added=[(s + i, t) for i, t in enumerate(new_lines)],
                )})
                shown = "".join(f"{i}: {lines[i - 1]}" for i in
                                range(max(1, s - 2), min(len(lines), s + len(new_lines) + 1) + 1))
                out = (f"Replaced lines {s}-{e} of {rp} with {len(new_lines)} line(s). "
                       f"Backup saved. Updated region:\n{shown}")
                _last_obs["text"] = out
                return out
            except Exception as ex:  # noqa: BLE001
                msg = f"Error: edit_lines failed: {type(ex).__name__}: {ex}"
                _last_obs["text"] = msg
                return msg

        @tool
        def edit_file(path: str, old_string: str, new_string: str) -> str:
            """Replace an EXACT unique text occurrence in a workspace file (must match byte-for-byte incl. whitespace). Prefer edit_lines when you have line numbers. A backup is saved automatically.

            Args:
                path: file inside a registered workspace.
                old_string: exact existing text to replace (must occur exactly once).
                new_string: replacement text.
            """
            try:
                rp = _ws_write_allowed((path or "").strip())
                import os
                if not os.path.isfile(rp):
                    msg = f"Error: not a file: {path}"
                    _last_obs["text"] = msg
                    return msg
                with open(rp, "r", encoding="utf-8", errors="replace") as fh:
                    text = fh.read()
                n = text.count(old_string)
                if n == 0:
                    msg = ("Error: old_string not found — it must match EXACTLY "
                           "(check whitespace with read_file_lines, or use edit_lines).")
                    _last_obs["text"] = msg
                    return msg
                if n > 1:
                    msg = f"Error: old_string occurs {n} times — add surrounding context to make it unique, or use edit_lines."
                    _last_obs["text"] = msg
                    return msg
                backup = _ws_snapshot(rp)
                with open(rp, "w", encoding="utf-8") as fh:
                    fh.write(text.replace(old_string, new_string, 1))
                _ws_record_change("edit", rp, backup)
                start_no = text[:text.index(old_string)].count("\n") + 1
                old_ls = old_string.splitlines() or [""]
                new_ls = new_string.splitlines() or [""]
                _emit({"type": "step", "text": _render_edit_card(
                    "Update", rp,
                    f"Replaced {len(old_ls)} line(s) at line {start_no}",
                    removed=[(start_no + i, t) for i, t in enumerate(old_ls)],
                    added=[(start_no + i, t) for i, t in enumerate(new_ls)],
                )})
                out = f"Replaced 1 occurrence in {rp}. Backup saved."
                _last_obs["text"] = out
                return out
            except Exception as ex:  # noqa: BLE001
                msg = f"Error: edit_file failed: {type(ex).__name__}: {ex}"
                _last_obs["text"] = msg
                return msg

        @tool
        def create_file(path: str, content: str, overwrite: bool = False) -> str:
            """Create a file inside a registered workspace, or REPLACE the whole file when overwrite=True. Pass overwrite=True to rewrite an existing file in one call (a backup is saved for revert) — no need to read_file_lines + edit_lines just to replace everything. Multi-line content MUST use \\n for line breaks (or a triple-quoted string); a raw line break inside a normal "..." causes an 'unterminated string literal' error.

            Args:
                path: file path inside a workspace.
                content: full file content.
                overwrite: replace the file if it already exists (default False = error on existing).
            """
            try:
                rp = _ws_write_allowed((path or "").strip())
                import os
                import hashlib as _hl
                existed = os.path.exists(rp)
                if existed and not overwrite:
                    msg = (f"Error: {path} already exists — pass overwrite=True to replace the "
                           "whole file, or use edit_lines/edit_file for a partial change.")
                    _last_obs["text"] = msg
                    return msg
                # Oscillation guard (task #79): if the model rewrites a file to a version it
                # ALREADY wrote this run (the NextResponse ↔ new Response flip-flop), that
                # exact content already failed — refuse instead of looping on a non-fix.
                _ch = _hl.sha1((content or "").encode("utf-8", "replace")).hexdigest()
                _seen = _file_writes.setdefault(rp, set())
                if existed and _ch in _seen:
                    msg = (f"You already wrote this EXACT content to {os.path.basename(rp)} "
                           "earlier this task and it did not fix the problem — writing it "
                           "again won't help. Do NOT rewrite this file with the same content. "
                           "Read the server error log, try a genuinely DIFFERENT fix, or call "
                           "final_answer honestly stating what's still failing.")
                    _last_obs["text"] = msg
                    return msg
                # Snapshot BEFORE overwriting so revert restores the prior version. Record as
                # an "edit" (not "create") when the file already existed, so revert restores
                # the backup instead of deleting a file the user may have had before.
                backup = _ws_snapshot(rp) if existed else ""
                os.makedirs(os.path.dirname(rp) or ".", exist_ok=True)
                with open(rp, "w", encoding="utf-8") as fh:
                    fh.write(content)
                _ws_record_change("edit" if existed else "create", rp, backup or None)
                content_ls = content.splitlines() or [""]
                _emit({"type": "step", "text": _render_edit_card(
                    "Update" if existed else "Create", rp,
                    ("Rewrote file — " if existed else "Added ") + f"{len(content_ls)} line(s)",
                    added=[(i + 1, t) for i, t in enumerate(content_ls)],
                )})
                _seen.add(_ch)  # remember this content so a later revert-to-it is caught
                out = f"{'Overwrote' if existed else 'Created'} {rp} ({len(content)} chars)."
                _last_obs["text"] = out
                return out
            except Exception as ex:  # noqa: BLE001
                msg = f"Error: create_file failed: {type(ex).__name__}: {ex}"
                _last_obs["text"] = msg
                return msg

        @tool
        def create_folder(path: str) -> str:
            """Create a folder (and any missing parent folders) inside a registered workspace. Use this to ACTUALLY create a folder/directory the user asked for — never answer with mkdir instructions.

            Args:
                path: new folder path inside a workspace (a relative path resolves against the workspace root).
            """
            try:
                rp = _ws_write_allowed((path or "").strip())
                import os
                if os.path.isdir(rp):
                    msg = f"Folder already exists: {rp}"
                    _last_obs["text"] = msg
                    return msg
                if os.path.exists(rp):
                    msg = f"Error: {path} already exists and is a file, not a folder."
                    _last_obs["text"] = msg
                    return msg
                os.makedirs(rp)
                # "create_dir" (not "create") — revert removes it with rmdir, which only
                # deletes EMPTY dirs, so a revert can never destroy content the user
                # added inside it afterwards. unlink (the "create" revert) fails on dirs.
                _ws_record_change("create_dir", rp, None)
                _emit({"type": "step", "text": _render_edit_card(
                    "Create", rp, "New folder")})
                out = f"Created folder {rp}."
                _last_obs["text"] = out
                return out
            except Exception as ex:  # noqa: BLE001
                msg = f"Error: create_folder failed: {type(ex).__name__}: {ex}"
                _last_obs["text"] = msg
                return msg

        @tool
        def run_command(command: str, workdir: str = "", timeout_seconds: int = 180) -> str:
            """Run a command inside a workspace that has run permission (e.g. 'npm test', 'npm run build', 'npm start', 'mvn test', 'python3 app.py'). One-shot commands return their output when they exit. If the command turns out to be a SERVER (keeps running and listens on a port), it is left RUNNING and its URL is reported. If it is a DESKTOP/GUI app (opens a window — tkinter, pygame, PyQt…), it is LAUNCHED and left running: its window appears on the user's Mac; tell them it's open. Both are stoppable via stop_server. Also allows DEPLOY commands (ssh/scp/rsync) — use non-interactive KEY auth (-o BatchMode=yes); never put a password in the command (there is no stdin). Note: an interactive CLI that reads stdin (input()) cannot be driven here — it will fail with EOFError; make it non-interactive or give the user the command to run themselves.

            Args:
                command: the command line. Most commands run; no shell (so no pipes/redirects/&&), and sudo/su are blocked.
                workdir: directory to run in. Empty = first registered workspace.
                timeout_seconds: give up after this many seconds if the command neither exits nor starts a server (max 600).
            """
            try:
                import os
                import shlex
                import signal
                import subprocess
                import time as _t
                # workdir="" → the active workspace. Guard the common model mistake of
                # passing the workspace's OWN name (e.g. workdir="t13" while the active
                # folder is /…/t13), which would double-nest to /…/t13/t13
                # (FileNotFoundError). EXISTENCE-FIRST: a real subdir always wins — only
                # strip the redundant leading segment when the nested path does NOT exist
                # (so a legit repo/repo layout like web/web is never misrouted). Handles
                # both the bare form ("t13") and the prefixed form ("t13/my-react-app").
                _wd_in = (workdir or "").strip()
                if _wd_in and not os.path.isabs(os.path.expanduser(_wd_in)):
                    _active = _default_ws_root()
                    if _wd_in in (".", "./"):
                        _wd_in = ""
                    elif not os.path.isdir(os.path.join(_active, _wd_in)):
                        _base = os.path.basename(_active)
                        if _wd_in == _base:
                            _wd_in = ""
                        elif _wd_in.startswith(_base + os.sep):
                            _stripped = _wd_in[len(_base) + 1:]
                            if os.path.isdir(os.path.join(_active, _stripped)):
                                _wd_in = _stripped
                wd = _rag_read_allowed(_wd_in or _default_ws_root())
                w = _ws_for_path(wd)
                if w is None or not w.get("allowRun"):
                    # Contextual nudge (task #102): the #1 reason the model shells out here is
                    # to zip/deliver a file — point it at create_download, which needs NO shell
                    # and works even in a run-disabled (e.g. web) workspace. Then the re-register
                    # hint for cases that genuinely need a command.
                    msg = ("Error: running commands is not enabled for this workspace. "
                           "To give the user a FILE or a ZIP of a folder, use the create_download "
                           "tool instead — it needs no shell and works here. "
                           "(To enable real commands, re-register with: "
                           "gonext-local-worker workspace add <path> --allow-run)")
                    _last_obs["text"] = msg
                    return msg
                argv = shlex.split(command or "")
                if not argv:
                    msg = "Error: empty command."
                    _last_obs["text"] = msg
                    return msg
                # Allow-by-default policy. Match on the runner's basename too, so
                # `/usr/bin/<x>` can't slip past a bare-name rule.
                _runner = argv[0]
                _rbase = os.path.basename(_runner)
                # Opt-in allowlist lockdown: outside the set = hard refuse, never asked.
                if _WS_RUN_ALLOWLIST and _runner not in _WS_RUN_ALLOWLIST and _rbase not in _WS_RUN_ALLOWLIST:
                    msg = (f"Error: '{_runner}' is not in your run allowlist. Add it in web "
                           "Settings → Agent, or clear the allowlist to allow all commands.")
                    _last_obs["text"] = msg
                    return msg
                # Risk gate: privilege escalation / denylisted / destructive. When the
                # client can approve interactively (terminal), PAUSE and ask the user;
                # otherwise (web) keep hard-blocking the privilege/denylist cases.
                _hard_reason, _ask_reason = _ws_command_risk(command, argv, _WS_RUN_DENYLIST)
                if _ask_reason:
                    if _interactive_approval and _job_id and pdf_api_base and pdf_worker_key:
                        if not _ws_request_approval(pdf_api_base, pdf_worker_key,
                                                    _job_id, command, _ask_reason):
                            msg = (f"Error: the user declined to run '{command[:80]}' "
                                   f"({_ask_reason}). Do NOT retry it — choose another "
                                   "approach or ask the user what to do.")
                            _last_obs["text"] = msg
                            return msg
                        # approved → fall through and run it
                    elif _hard_reason:
                        msg = (f"Error: '{command[:80]}' is blocked ({_hard_reason}). Run "
                               "that step yourself, or tell the user the exact command.")
                        _last_obs["text"] = msg
                        return msg
                    # else: destructive but non-interactive → allowed by default (unchanged)
                # Don't spawn a DUPLICATE of a server that's already running (task #79): the
                # model re-runs 'npm run dev' and Next/Vite then bind a SECOND port, which
                # confused the whole debug loop (3000 500 vs 3001 refused). If the identical
                # command+workdir is still listening, reuse it.
                for _srv in _ws_load_servers():
                    if _srv.get("command") == command and _srv.get("workdir") == wd:
                        _live = _ws_listening_ports(_srv.get("pid") or 0)
                        if _live:
                            _urls = ", ".join(f"http://localhost:{p}" for p in _live)
                            msg = (f"'{command}' is ALREADY running — {_urls} (pid "
                                   f"{_srv.get('pid')}). Reusing it; do NOT start another. A "
                                   "dev server hot-reloads code changes automatically, so just "
                                   f"request {_urls} again. Use stop_server first only if you "
                                   "truly need a full restart.")
                            _emit({"type": "step", "text": f"Server already up → {_urls} (reused)"})
                            _last_obs["text"] = msg
                            return msg
                _emit({"type": "step", "text": f"Running → {command[:70]}"})
                t = max(5, min(int(timeout_seconds or 180), 600))
                # Own process group (start_new_session) + output to a log file. This is
                # what lets a server OUTLIVE this job process, and what lets a timeout
                # kill take the runner's whole child tree (npm's node child etc. — the
                # old subprocess.run timeout killed only the direct child, leaking it).
                logs_dir = os.path.join(os.path.expanduser("~"), ".gonext", "run-logs")
                os.makedirs(logs_dir, exist_ok=True)
                log_path = os.path.join(logs_dir, f"{_t.strftime('%Y%m%d-%H%M%S')}-{os.getpid()}.log")
                log_fh = open(log_path, "wb")
                # Scrubbed env: repo scripts must never see worker secrets.
                proc = subprocess.Popen(
                    argv, cwd=wd, env=_ws_scrubbed_env(),
                    stdout=log_fh, stderr=subprocess.STDOUT,
                    start_new_session=True,
                )
                start = _t.time()

                def _read_log() -> str:
                    log_fh.flush()
                    try:
                        with open(log_path, "r", encoding="utf-8", errors="replace") as fh:
                            return fh.read()
                    except OSError:
                        return ""

                # Behavior classification loop — no command-name matching anywhere:
                # exited → one-shot result; still running AND listening on a TCP port →
                # it's a server, leave it running; neither by the deadline → kill group.
                while True:
                    rc = proc.poll()
                    if rc is not None:
                        out = _ws_distill_output(_read_log())
                        # Task #90 Phase 1 — truthful perception: never claim success the
                        # tool didn't observe. exit 0 is a FACT about the process, not
                        # proof of the GOAL (e.g. `npm start` exits 0 when its port is
                        # busy and the server never started — the old "PASSED (exit 0)"
                        # label told the model it succeeded, 3× in one live run). The
                        # RUNNING/LAUNCHED paths below stay — those ARE observed outcomes.
                        # General (no command-name or error-string matching): the label
                        # states the fact and hands verification to the model, which the
                        # senior-dev directive tells it to do.
                        status = (
                            "EXITED 0 (the command finished — exit code alone does NOT "
                            "prove the intended effect happened; check the output below "
                            "and VERIFY before relying on it)"
                            if rc == 0 else f"FAILED (exit {rc})"
                        )
                        _emit({"type": "step", "text":
                               f"Command {'finished' if rc == 0 else 'failed'} → {command[:50]}"})
                        result = f"{status}\n{out}" if out.strip() else status
                        _last_obs["text"] = result
                        return result
                    if _t.time() - start >= 3:
                        ports = _ws_listening_ports(proc.pid)
                        if ports:
                            servers = _ws_load_servers()
                            servers.append({
                                "pgid": proc.pid, "pid": proc.pid, "command": command,
                                "workdir": wd, "ports": ports, "log": log_path,
                                "started_at": _t.strftime("%Y-%m-%d %H:%M:%S"),
                            })
                            _ws_save_servers(servers)
                            urls = ", ".join(f"http://localhost:{p}" for p in ports)
                            _emit({"type": "step", "text": f"Server up → {urls} (left running)"})
                            tail = _ws_distill_output(_read_log())
                            result = (f"RUNNING: '{command}' is up and listening — {urls} "
                                      f"(pid {proc.pid}, log {log_path}). It stays running in "
                                      "the background; use stop_server to stop it. "
                                      f"Startup output:\n{tail}")
                            _last_obs["text"] = result
                            return result
                        # Not a server, but still alive AND it OPENED A WINDOW → a desktop
                        # app. Launch-and-leave like a server (a GUI never exits and opens
                        # no port, so it would otherwise hit the timeout kill below and its
                        # window would be destroyed). Returning fast here also dodges the
                        # smolagents 60s executor cap that was killing the tool call.
                        if _ws_gui_pids(proc.pid):
                            servers = _ws_load_servers()
                            servers.append({
                                "pgid": proc.pid, "pid": proc.pid, "command": command,
                                "workdir": wd, "ports": [], "kind": "gui", "log": log_path,
                                "started_at": _t.strftime("%Y-%m-%d %H:%M:%S"),
                            })
                            _ws_save_servers(servers)
                            _emit({"type": "step",
                                   "text": f"App launched → window open (left running)"})
                            tail = _ws_distill_output(_read_log())
                            result = (f"LAUNCHED: '{command}' — its window should be open on "
                                      f"the user's Mac now (pid {proc.pid}). It keeps running "
                                      "in the background; use stop_server (or just close the "
                                      "window) to quit."
                                      + (f"\nStartup output:\n{tail}" if tail.strip() else ""))
                            _last_obs["text"] = result
                            return result
                    if _t.time() - start >= t:
                        try:
                            os.killpg(proc.pid, signal.SIGKILL)
                        except OSError:
                            pass
                        tail = _ws_distill_output(_read_log())
                        msg = (f"Error: command neither exited nor started a server within "
                               f"{t}s — killed. Output so far:\n{tail}")
                        _last_obs["text"] = msg
                        return msg
                    _t.sleep(2)
            except Exception as ex:  # noqa: BLE001
                msg = f"Error: run_command failed: {type(ex).__name__}: {ex}"
                _last_obs["text"] = msg
                return msg
            finally:
                try:
                    log_fh.close()
                except Exception:  # noqa: BLE001
                    pass

        @tool
        def stop_server(port_or_pid: str = "") -> str:
            """Stop a background server OR desktop app previously started by run_command (closes its window). Use when the user asks to stop/kill/close it.

            Args:
                port_or_pid: the server's port or pid (empty = the most recently started one).
            """
            try:
                import os
                import signal
                import time as _t
                servers = _ws_load_servers()
                if not servers:
                    msg = "No background servers are running."
                    _last_obs["text"] = msg
                    return msg
                key = (port_or_pid or "").strip()
                target = None
                if key:
                    for s in servers:
                        if key == str(s.get("pid")) or any(key == str(p) for p in s.get("ports", [])):
                            target = s
                            break
                    if target is None:
                        listing = "; ".join(
                            f"pid {s['pid']} ports {s.get('ports')} ({s['command']})" for s in servers)
                        msg = f"No running server matches '{key}'. Running: {listing}"
                        _last_obs["text"] = msg
                        return msg
                else:
                    target = servers[-1]
                pgid = int(target["pgid"])
                try:
                    os.killpg(pgid, signal.SIGTERM)
                    _t.sleep(1.5)
                    if _ws_pgroup_pids(pgid):
                        os.killpg(pgid, signal.SIGKILL)
                except OSError:
                    pass
                _ws_save_servers([s for s in servers if s is not target])
                _emit({"type": "step", "text": f"Server stopped → {target['command'][:50]}"})
                out = f"Stopped '{target['command']}' (pid {target['pid']}, ports {target.get('ports')})."
                _last_obs["text"] = out
                return out
            except Exception as ex:  # noqa: BLE001
                msg = f"Error: stop_server failed: {type(ex).__name__}: {ex}"
                _last_obs["text"] = msg
                return msg

        @tool
        def deploy_web(local_dir: str, host: str, user: str, remote_path: str,
                       domain: str = "") -> str:
            """Deploy a built static site to a remote server in ONE step — prefer this over hand-writing scp/ssh/rsync. rsyncs local_dir to user@host:remote_path over SSH KEY auth, and (if domain is given) tries to finish the nginx setup via PASSWORDLESS sudo; if that needs a password it writes a ready-to-run deploy.sh and returns the exact commands for the user. Uses the user's SSH key only — never a password. If key auth isn't set up it returns the one-time `ssh-copy-id` command to run and stops.

            Args:
                local_dir: the built folder to upload (e.g. 'my-react-app/build') inside the workspace.
                host: server hostname (e.g. 'example.com').
                user: SSH username.
                remote_path: destination dir on the server (e.g. '/home/<user>/site' or '/var/www/site').
                domain: optional site domain — when set, also generate/apply the nginx server block.
            """
            try:
                import os
                import subprocess
                src = _rag_read_allowed((local_dir or "").strip())
                if not os.path.isdir(src):
                    msg = (f"Error: local_dir '{local_dir}' is not a folder — build first "
                           "(e.g. run_command('npm run build')) so the output dir exists.")
                    _last_obs["text"] = msg
                    return msg
                host = (host or "").strip()
                user = (user or "").strip()
                remote_path = (remote_path or "").strip()
                if not host or not user or not remote_path:
                    msg = "Error: host, user and remote_path are all required."
                    _last_obs["text"] = msg
                    return msg
                target = f"{user}@{host}"
                sshopts = ["-o", "BatchMode=yes", "-o", "StrictHostKeyChecking=accept-new",
                           "-o", "ConnectTimeout=8"]
                env = _ws_scrubbed_env()
                _emit({"type": "step", "text": f"Deploying → {target}:{remote_path}"})
                # 1) KEY-auth probe. No password is ever used or requested.
                probe = subprocess.run(["ssh", *sshopts, target, "true"], env=env,
                                       capture_output=True, text=True, timeout=25)
                if probe.returncode != 0:
                    msg = (
                        f"Can't deploy yet: SSH key auth to {target} isn't set up "
                        f"({(probe.stderr or '').strip()[:160]}). Run this ONCE in your "
                        f"OWN terminal (it will ask for the password a final time):\n"
                        f"    ssh-copy-id {target}\n"
                        "Then ask me to deploy again — I'll use the key (no password needed)."
                    )
                    _last_obs["text"] = msg
                    return msg
                # 2) rsync the build over the key.
                rsync = subprocess.run(
                    ["rsync", "-az", "--delete", "-e", "ssh " + " ".join(sshopts),
                     src.rstrip("/") + "/", f"{target}:{remote_path.rstrip('/')}/"],
                    env=env, capture_output=True, text=True, timeout=600)
                if rsync.returncode != 0:
                    msg = (f"Error: rsync to {target}:{remote_path} failed "
                           f"({(rsync.stderr or rsync.stdout or '').strip()[:200]}).")
                    _last_obs["text"] = msg
                    return msg
                uploaded = f"Uploaded {os.path.basename(src)} → {target}:{remote_path}."
                if not domain.strip():
                    _last_obs["text"] = uploaded
                    return uploaded + " No domain given, so nothing was served — pass domain= to configure nginx."
                dom = domain.strip()
                webroot = f"/var/www/{dom}"
                nginx_conf = (
                    f"server {{\n    listen 80;\n    server_name {dom};\n"
                    f"    root {webroot};\n    index index.html;\n"
                    f"    location / {{ try_files $uri $uri/ /index.html; }}\n}}\n"
                )
                # Remote setup script: move the upload into the webroot + wire up nginx.
                remote_sh = (
                    "set -e\n"
                    f"mkdir -p {webroot}\n"
                    f"cp -r {remote_path.rstrip('/')}/* {webroot}/\n"
                    f"cat > /etc/nginx/sites-available/{dom} <<'NGINX'\n{nginx_conf}NGINX\n"
                    f"ln -sf /etc/nginx/sites-available/{dom} /etc/nginx/sites-enabled/{dom}\n"
                    "nginx -t\n"
                    "systemctl reload nginx\n"
                )
                # 3) Try to finish via PASSWORDLESS sudo (sudo -n succeeds only w/ NOPASSWD).
                fin = subprocess.run(["ssh", *sshopts, target, "sudo -n bash -s"],
                                     input=remote_sh, env=env, capture_output=True,
                                     text=True, timeout=120)
                if fin.returncode == 0:
                    out = f"{uploaded} Configured nginx and reloaded — the site is live at http://{dom}/."
                    _emit({"type": "step", "text": f"Site live → http://{dom}/"})
                    _last_obs["text"] = out
                    return out
                # 4) sudo needs a password (no TTY here) → write a runnable deploy.sh
                #    (SAME sudo commands as the printed steps) and hand it over.
                sudo_sh = (
                    "#!/usr/bin/env bash\nset -e\n"
                    f"sudo mkdir -p {webroot}\n"
                    f"sudo cp -r {remote_path.rstrip('/')}/* {webroot}/\n"
                    f"sudo tee /etc/nginx/sites-available/{dom} >/dev/null <<'NGINX'\n"
                    f"{nginx_conf}NGINX\n"
                    f"sudo ln -sf /etc/nginx/sites-available/{dom} /etc/nginx/sites-enabled/{dom}\n"
                    "sudo nginx -t && sudo systemctl reload nginx\n"
                )
                sh_local = os.path.join(_default_ws_root(), "deploy.sh")
                try:
                    with open(sh_local, "w", encoding="utf-8") as fh:
                        fh.write(sudo_sh)
                    _ws_record_change("create", sh_local, None)
                    where = sh_local
                except OSError:
                    where = "(could not write deploy.sh)"
                out = (
                    f"{uploaded} The nginx/webroot step needs sudo (no password prompt is "
                    f"possible over SSH). I wrote a runnable script to {where} — copy it to "
                    f"the server and run it (`scp deploy.sh {target}:` then `ssh {target} "
                    f"'bash deploy.sh'`), OR run these on the server yourself:\n"
                    + "\n".join("    " + ln for ln in sudo_sh.splitlines()[2:])
                    + f"\nThen {dom} serves the site."
                )
                _last_obs["text"] = out
                return out
            except subprocess.TimeoutExpired:
                msg = f"Error: deploy to {host} timed out (server unreachable or slow)."
                _last_obs["text"] = msg
                return msg
            except Exception as ex:  # noqa: BLE001
                msg = f"Error: deploy_web failed: {type(ex).__name__}: {ex}"
                _last_obs["text"] = msg
                return msg

        agent_tools = [http_request, web_search, fetch_url, calculate,
                       get_current_datetime, create_pdf, create_download]
        # Only register the PDF reader / email / RAG tools when available — the model
        # never sees a tool it can't run, since the task-hint's tool list (filled in just
        # below) is generated FROM this exact list, so it's always in sync by construction.
        if _PDF_READ_AVAILABLE:
            agent_tools.append(extract_text_from_pdf)
        if _EMAIL_AVAILABLE:
            agent_tools.append(send_email)
        if _RAG_AVAILABLE:
            agent_tools += [download_file, unzip_file, list_dir, read_text_file,
                            rag_index, rag_add, rag_search]
        if _WS_AVAILABLE:
            agent_tools += [grep_repo, read_file_lines, edit_lines, edit_file,
                            create_file, create_folder, run_command, stop_server,
                            deploy_web]
            # list_dir/read_text_file are needed for workspace browsing even when RAG
            # (S3 indexing) isn't configured.
            if not _RAG_AVAILABLE:
                agent_tools += [list_dir, read_text_file]
        # Fill in the numbered tool list now that the REAL tool objects exist — see
        # _render_numbered_tool_list and the {{TOOL_LIST}} placeholder left in tool_hint.
        task_with_hint = task_with_hint.replace(
            "{{TOOL_LIST}}",
            f"You have {len(agent_tools)} tools:\n" + _render_numbered_tool_list(agent_tools),
        )
        agent_kwargs = dict(
            tools=agent_tools,
            model=model,
            max_steps=max_steps,
            step_callbacks=[step_callback],
        )
        # Task #90 Phase 3: the completion gate only makes sense in coding/workspace mode
        # (state-changing commands + file edits); retrieval turns never trip it anyway, but
        # gating keeps plain-chat/research answers untouched.
        if _WS_AVAILABLE:
            agent_kwargs["final_answer_checks"] = [_final_answer_gate]
        if _AgentBase is CodeAgent:
            # Only the CodeAgent runs a Python executor; ToolCallingAgent takes neither.
            # 660s, NOT 60 (task #74): the wall clock must exceed the LARGEST tool-internal
            # timeout (run_command caps at 600s; create_pdf's format+render+upload can pass
            # 60s), otherwise a tool that eventually SUCCEEDS gets reported to the model as
            # 'Code execution exceeded the maximum execution time' — the model then retries
            # work that already finished (the duplicate-PDF bug). Every tool enforces its
            # own tighter timeout and returns a clean message; runaway pure-python is still
            # caught by smolagents' op-count caps (MAX_OPERATIONS/MAX_WHILE_ITERATIONS),
            # so this wall clock is a last-resort backstop, not the primary guard.
            agent_kwargs["executor_kwargs"] = {"timeout_seconds": 660}
            agent_kwargs["additional_authorized_imports"] = [
                "json", "base64", "urllib", "urllib.request", "urllib.error"
            ]
            # Trim smolagents' ~6k-char generic few-shot examples from the system prompt
            # (they dominate prompt-eval on a slow coding model) while keeping the tool
            # rendering + format rules intact. Load the stock templates and swap only the
            # system_prompt; fall back to the default on any error.
            try:
                import yaml as _yaml
                import os as _os
                import smolagents as _sm
                _tpl = _yaml.safe_load(open(_os.path.join(
                    _os.path.dirname(_sm.__file__), "prompts", "code_agent.yaml")))
                _tpl["system_prompt"] = _COMPACT_CODE_SYSTEM_PROMPT
                agent_kwargs["prompt_templates"] = _tpl
                _log("using compact CodeAgent system prompt (trimmed few-shot examples)")
            except Exception as _e:  # noqa: BLE001
                _log(f"compact system prompt unavailable ({_e}); using smolagents default")
        agent = _ToolAgent(**agent_kwargs)
        _agent_ref["a"] = agent  # let step_callback trim this agent's step memory (#63)
        # At the step budget the run ends WITHOUT final_answer (_maxsteps_partial hit). Offer
        # to keep going instead of dead-ending (task #80): a terminal user says Yes → extend
        # the budget to a large ceiling and CONTINUE with the existing memory (reset=False,
        # so the model keeps all prior observations). smolagents captures max_steps as a
        # LOCAL at run() time, so we can't bump it mid-loop — we re-enter run() with a raised
        # agent.max_steps instead. The real bound is the 30-min worker job cap, so 1000 just
        # means "run to completion". 'Yes, don't ask again' is handled REPL-side: it flips a
        # session flag so the NEXT turn arrives with alwaysExtendOnMaxStep and we auto-extend
        # without prompting.
        _MAXSTEP_CEILING = 1000
        _always_extend = bool(cfg.get("alwaysExtendOnMaxStep"))
        with contextlib.redirect_stdout(sys.stderr):
            result = agent.run(task_with_hint)
            while _maxsteps_partial.get("hit"):
                if _always_extend:
                    _log(f"max-steps reached → auto-extending to {_MAXSTEP_CEILING} "
                         "(don't-ask-again session flag)")
                    _do_extend = True
                elif _interactive_approval and _job_id and pdf_api_base and pdf_worker_key:
                    # Long deadline (#110): the "keep going?" prompt isn't risky, and the
                    # slow coder + a thinking user shouldn't get auto-denied at 180s. The
                    # REPL dismisses the picker if this job ends, and the worker job-cap
                    # bounds a walked-away user, so waiting here is safe.
                    _do_extend = _ws_request_approval(
                        pdf_api_base, pdf_worker_key, _job_id,
                        f"__MAXSTEP__::Reached the step budget ({agent.max_steps}). "
                        "Keep going?", "max-steps", deadline_s=1500)
                else:
                    _do_extend = False  # web / no interactive channel → current wrap-up
                if not _do_extend:
                    break
                _maxsteps_partial["hit"] = False
                agent.max_steps = _MAXSTEP_CEILING
                _emit({"type": "step", "text": "Continuing — step budget extended…"})
                _log(f"continuing run with raised budget {agent.max_steps}")
                result = agent.run(task_with_hint, reset=False)
        # Final formatting — NO extra summarizer model call. In multi-step mode the
        # agent's own final_answer() (or the max-steps fallback above) already holds the
        # synthesized answer; we just strip the internal hint tags we appended to tool
        # results so they don't leak to the user.
        _emit({"type": "step", "text": "Composing answer…"})
        final_text = _strip_tool_tags(str(result).strip()) or "[No result]"
        _log(f"done (deterministic, no summarizer call): {len(final_text)} chars")
        _emit({"type": "final", "text": final_text})
        # Clear the step checkpoint ONLY on a genuinely complete finish (the model's own
        # final_answer, or a deterministic PDF delivery) — that content is now in the
        # answer pushed to conversation history, so the raw trace is no longer needed.
        # Kept when the turn ended PARTIALLY, in either way this can happen:
        # - max-steps fallback (_maxsteps_partial): its answer surfaces only the LAST
        #   observation, so a follow-up "continue" needs the full trace on disk;
        # - degrade-to-plain-reply (except block below): its note carries only a
        #   top-level file listing, not the step trace — same reasoning, never cleared.
        if _WS_ACTIVE and not _maxsteps_partial["hit"]:
            _clear_turn_checkpoint(_WS_ACTIVE)
    except Exception as e:  # noqa: BLE001
        # Config errors (wrong coding-model name/URL) must reach the user VERBATIM —
        # they contain the fix (the server's real model list). smolagents wraps model
        # exceptions in AgentGenerationError, so walk the cause chain to find ours.
        cfg_err = None
        cur, depth = e, 0
        while cur is not None and depth < 6:
            if isinstance(cur, _AgentConfigError):
                cfg_err = cur
                break
            cur, depth = (getattr(cur, "__cause__", None)
                          or getattr(cur, "__context__", None)), depth + 1
        if cfg_err is None and "Settings → Agent" in str(e):
            cfg_err = e  # wrapped without a cause chain; message survived
        if cfg_err is not None:
            _log(f"agent config error (not degrading): {cfg_err}")
            _emit({"type": "final", "text": f"⚠️ {cfg_err}"})
            return
        # Coding-model BACKEND outage (502/503/504, refused/timeout): the tool-capable
        # loop never got to run because its code model was unreachable. Do NOT fall through
        # to _plain_reply — a bare chat model would stream a confident GENERIC how-to (e.g.
        # "to create a React app, run npx create-react-app…") that looks like a real answer
        # but did NOTHING the user asked (seen live: "create a reactjs and start it" → the
        # coder 502'd 3× → generic instructions, no warning). That's worse than an honest
        # failure: it hides the outage and, being persisted to history, misleads the next
        # turn. Emit a clear, standalone "backend is down, retry" and stop — the checkpoint
        # is kept (not cleared) so a later "continue" resumes once the box is back.
        # Coding-model OVERLOAD (429/rate limit, task #114): same reasoning as the outage
        # branch below — the tools never ran, so degrading to _plain_reply would invent a
        # confident how-to for work that didn't happen. Reported separately because the
        # cause and the advice differ: the server is UP, it's just saturated or we're over
        # quota, so "try again shortly" is the honest fix. Checked BEFORE the outage branch
        # (_is_backend_unavailable deliberately excludes overloads).
        if _is_backend_overloaded(e):
            _log(f"agent error (coding-model backend OVERLOADED, NOT degrading): {_clip(str(e), 160)}")
            _ov = None
            _cur, _depth = e, 0
            while _cur is not None and _depth < 6:
                if isinstance(_cur, _AgentBackendOverloaded):
                    _ov = _cur
                    break
                _cur, _depth = (getattr(_cur, "__cause__", None)
                                or getattr(_cur, "__context__", None)), _depth + 1
            _emit({"type": "final", "text": "⚠️ " + (str(_ov) if _ov else (
                "The coding model backend is overloaded right now (HTTP 429 — it asked me "
                "to slow down), so I couldn't run the tools to do this. The server is up, "
                "it's just saturated or over quota. Please try again in a minute, or switch "
                "the coding backend in Settings → Agent. (I didn't make any changes.)"
            ))})
            return
        if _is_backend_unavailable(e):
            _log(f"agent error (coding-model backend unavailable, NOT degrading): {_clip(str(e), 160)}")
            _emit({"type": "final", "text": (
                "⚠️ The coding model backend is currently unavailable (the server returned "
                "a gateway error / timed out), so I couldn't run the tools to do this. It's "
                "usually starting up, reloading the model, or briefly overloaded — not a "
                "problem with your request. Please try again in a moment; if it keeps "
                "happening, check that the coding model server is up. (I didn't make any "
                "changes.)"
            )})
            return
        # An unexpected failure inside the agent loop should still return a useful
        # answer from the conversation rather than surfacing a raw error to the user —
        # but only when the fallback ACTUALLY has something useful to say.
        _log(f"agent error: {e} — degrading to plain reply")
        try:
            fallback = _plain_reply(messages, agent_base_url, agent_api_key, agent_model_id).strip()
        except Exception as e2:  # noqa: BLE001
            _log(f"plain-reply degrade error: {e2}")
            # Nothing useful to say — re-raise the ORIGINAL (more relevant) agent
            # failure so this becomes a genuine job failure: a red error the REPL/web
            # already handle correctly, NOT persisted to conversation history. Emitting
            # a fabricated "final" answer here instead would poison every future turn's
            # prompt with raw exception text (this is exactly what caused task #35).
            raise e
        # _plain_reply has NO tool/file/HTTP access whatsoever — that's by design (it's a
        # bare chat completion). Without a note here, its honest "I don't have access to
        # your workspace/that page/etc." reads to the user as a PERMISSIONS bug, when the
        # real cause is that the tool-capable agent loop failed upstream (e.g. the coding
        # model timed out) and we silently swapped in a capability-limited fallback. Make
        # that swap visible so the user can tell "genuinely no access" apart from
        # "the real attempt never got to run" (seen live: workspace summarize request →
        # coding model timed out 3x on a slow/unreachable remote Ollama box → silent
        # fallback claimed "no access to your workspace").
        note = f"⚠️ I couldn't finish the full investigation ({_clip(str(e), 120)}) — here's what I can say without it:\n\n{fallback}"
        # Append the workspace's ACTUAL top-level contents (computed locally, no model
        # call — see _workspace_overview) so the user sees their files are really there
        # even during a total coding-model outage, instead of trusting a plain-chat
        # model's guess that "there's no code or project" (seen live — false: the
        # workspace had real content the whole time, the model just never reached it).
        # ONLY the current/active folder — never dump the full list of every registered
        # workspace here. That list (t1..t23) used to get persisted into the answer/history
        # and then confused the next turn into free-roaming other projects (bug #70).
        if _WS_AVAILABLE:
            import os as _os
            _ov_path = _WS_ACTIVE or (_WS_ROOTS[0]["path"] if _WS_ROOTS else "")
            _ov = next((w for w in _WS_ROOTS if w["path"] == _ov_path), None)
            if _ov is None and _ov_path:
                _ov = {"name": _os.path.basename(_ov_path), "path": _ov_path}
            if _ov:
                note += f"\n\n(Current folder {_ov['name']}:\n{_workspace_overview([_ov])})"
        _emit({"type": "final", "text": note})


def _log(text: str):
    """Emit a log event — worker prints it to console, not forwarded to chat."""
    _emit({"type": "log", "text": text})


_EMIT_LOCK = threading.Lock()


def _emit(obj):
    """Write one NDJSON line to the real stdout and flush immediately.

    Lock-guarded: the model-call heartbeat thread emits concurrently with the
    main loop, and interleaved writes would corrupt the NDJSON stream.
    """
    with _EMIT_LOCK:
        _REAL_STDOUT.write(json.dumps(obj) + "\n")
        _REAL_STDOUT.flush()


def main():
    try:
        cfg = json.load(sys.stdin)
    except Exception as e:  # noqa: BLE001
        _emit({"type": "final", "text": f"[Invalid input: {e}]"})
        return
    run_agent_chat(cfg)


if __name__ == "__main__":
    main()