#!/bin/bash
#
# fetch-document.sh
# Extract readable text from a local document or a document URL so an analysis
# run can cite a Word file, a PDF or a plain-text spec the way it cites a
# Confluence page. Before this existed nothing in the pipeline could read a
# .docx, which is the format most feature specs actually arrive in.
#
# Input forms:
#   ./fetch-document.sh /path/to/spec.docx
#   ./fetch-document.sh ~/notes/spec.md
#   ./fetch-document.sh https://example.com/spec.docx
#
# Optional env:
#   DOCUMENT_TIMEOUT_SECONDS   download bound, default 25
#   DOCUMENT_MAX_CHARS         truncate the extracted body, default 200000
#   DOCUMENT_PDFTOTEXT         pdf converter binary, default `pdftotext`. Some
#                              distributions ship it under another name or path,
#                              and the offline smoke points it at a binary that
#                              does not exist to exercise the degrade path.
#
# Output (stdout, single JSON object):
#   {
#     "fetchedAt": "<ISO8601>",
#     "source": { "url": "<url|null>", "path": "<local path>",
#                 "format": "docx|pdf|md|txt", "bytes": <n> },
#     "title": "<first heading or filename>",
#     "text": "<extracted plain text>",
#     "truncated": <bool>,
#     "degraded": <bool>, "degradeReason": "<reason|null>"
#   }
#
# Exit codes:
#   0  success (may be degraded-empty for a pdf with no converter)
#   3  download or read failure
#   4  bad usage / unsupported extension
#   6  converter not available (pdf only; the orchestrator treats this as
#      "referenced but not fetched" and continues)
#
# .docx needs no external tool: the file is a zip and `word/document.xml` is
# read with the python3 standard library, which the lib layer already depends
# on. Requiring pandoc or libreoffice would be the wrong dependency for a tool
# that must work on macOS, Linux and Windows alike. PDF has no stdlib path, so
# `pdftotext` is used when present and its absence is a soft skip rather than
# an error - on Windows that absence is the normal case.

set -uo pipefail

TIMEOUT="${DOCUMENT_TIMEOUT_SECONDS:-25}"
MAX_CHARS="${DOCUMENT_MAX_CHARS:-200000}"
SRC="${1:-}"

if [ -z "$SRC" ] || [ "$SRC" = "-h" ] || [ "$SRC" = "--help" ]; then
  echo "usage: $0 <path-or-url>   # .docx | .pdf | .md | .txt" >&2
  exit 4
fi

URL=""
LOCAL="$SRC"
TMPDL=""
cleanup() { [ -n "$TMPDL" ] && rm -f "$TMPDL"; }
trap cleanup EXIT

case "$SRC" in
  http://*|https://*)
    URL="$SRC"
    TMPDL=$(mktemp "${TMPDIR:-/tmp}/fetch-document.XXXXXX")
    if ! curl -sS -L --fail --max-time "$TIMEOUT" --connect-timeout 5 -o "$TMPDL" "$URL" 2>/dev/null; then
      printf '%s\n' '{"status":"failed","reason":"download-failed","service":"document"}' >&2
      exit 3
    fi
    # Keep the URL's extension so the format probe below still works.
    LOCAL="$TMPDL"
    EXT=$(printf '%s' "${URL%%\?*}" | tr 'A-Z' 'a-z' | sed -n 's/.*\.\([a-z0-9]\{1,5\}\)$/\1/p')
    ;;
  *)
    case "$LOCAL" in "~"/*) LOCAL="$HOME/${LOCAL#\~/}" ;; esac
    if [ ! -f "$LOCAL" ]; then
      printf '%s\n' '{"status":"failed","reason":"file-not-found","service":"document"}' >&2
      exit 3
    fi
    EXT=$(printf '%s' "$LOCAL" | tr 'A-Z' 'a-z' | sed -n 's/.*\.\([a-z0-9]\{1,5\}\)$/\1/p')
    ;;
esac

case "$EXT" in
  docx|pdf|md|markdown|txt) ;;
  *)
    printf '%s\n' '{"status":"failed","reason":"unsupported-format","service":"document"}' >&2
    exit 4 ;;
esac

DEGRADED=false
DEGRADE_REASON=""
PDFTOTEXT="${DOCUMENT_PDFTOTEXT:-pdftotext}"
if [ "$EXT" = "pdf" ] && ! command -v "$PDFTOTEXT" >/dev/null 2>&1; then
  DEGRADED=true
  DEGRADE_REASON="converter-not-available"
fi

SRC_URL="$URL" SRC_PATH="$SRC" LOCAL_IN="$LOCAL" EXT_IN="$EXT" \
MAX_IN="$MAX_CHARS" DEGRADED_IN="$DEGRADED" REASON_IN="$DEGRADE_REASON" \
PDFTOTEXT_IN="$PDFTOTEXT" \
python3 - <<'PY'
import datetime, json, os, re, subprocess, zipfile
import xml.etree.ElementTree as ET

path = os.environ["LOCAL_IN"]
ext = os.environ["EXT_IN"]
degraded = os.environ["DEGRADED_IN"] == "true"
reason = os.environ["REASON_IN"] or None
try:
    limit = int(os.environ["MAX_IN"])
except ValueError:
    limit = 200000

W = "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}"

def from_docx(p):
    # A .docx is a zip; the body is word/document.xml. <w:t> holds the runs,
    # <w:p> is the paragraph boundary. Standard library only, so the same code
    # path runs on every platform.
    with zipfile.ZipFile(p) as z:
        xml = z.read("word/document.xml")
    root = ET.fromstring(xml)
    out = []
    for para in root.iter(W + "p"):
        runs = [t.text or "" for t in para.iter(W + "t")]
        line = "".join(runs).strip()
        if line:
            out.append(line)
    return "\n\n".join(out)

def from_pdf(p):
    if degraded:
        return ""
    r = subprocess.run([os.environ.get("PDFTOTEXT_IN", "pdftotext"), "-layout", p, "-"],
                       capture_output=True, text=True)
    return r.stdout if r.returncode == 0 else ""

text = ""
err = None
try:
    if ext == "docx":
        text = from_docx(path)
    elif ext == "pdf":
        text = from_pdf(path)
    else:
        with open(path, "r", encoding="utf-8", errors="replace") as fh:
            text = fh.read()
except Exception as exc:                      # unreadable archive, bad xml, ...
    err = type(exc).__name__
    text = ""

if err and not degraded:
    degraded, reason = True, f"extract-failed:{err}"

truncated = len(text) > limit
if truncated:
    text = text[:limit]

title = ""
for line in text.split("\n"):
    s = line.strip().lstrip("#").strip()
    if s:
        title = s[:120]
        break
if not title:
    title = os.path.basename(os.environ["SRC_PATH"])

print(json.dumps({
    "fetchedAt": datetime.datetime.now(datetime.timezone.utc)
                 .isoformat().replace("+00:00", "Z"),
    "source": {
        "url": os.environ["SRC_URL"] or None,
        "path": os.environ["SRC_PATH"],
        "format": "md" if ext == "markdown" else ext,
        "bytes": os.path.getsize(path) if os.path.exists(path) else 0,
    },
    "title": title,
    "text": text,
    "truncated": truncated,
    "degraded": degraded,
    "degradeReason": reason,
}, ensure_ascii=False))
PY

if [ "$DEGRADED" = "true" ]; then
  exit 6
fi