#!/usr/bin/env python3
"""
crawlee-scraper.py — Self-contained web scraping script using Crawlee.

Supports two crawling strategies:
  - beautifulsoup (default): Fast HTTP-based scraping via httpx + BeautifulSoup.
    Best for static content, documentation pages, and API docs.
  - playwright: Headless browser scraping for JS-rendered pages, SPAs, and
    sites that require JavaScript execution to display content.

Usage:
  python3 crawlee-scraper.py --url URL [--strategy beautifulsoup|playwright]
    [--max-pages N] [--max-depth N] [--output json|text] [--extract links|text|all]

Auto-installs crawlee and playwright if not present.
"""

import argparse
import asyncio
import json
import subprocess
import sys
import importlib
import re
from datetime import timedelta
from urllib.parse import urljoin, urlparse
from typing import Any

# ---------------------------------------------------------------------------
# Auto-install crawlee if missing
# ---------------------------------------------------------------------------

def ensure_crawlee():
    """Ensure crawlee is installed, installing it if needed."""
    try:
        importlib.import_module("crawlee")
        return True
    except ImportError:
        pass

    print("[crawlee-scraper] Installing crawlee...", file=sys.stderr)
    try:
        subprocess.check_call(
            [sys.executable, "-m", "pip", "install", "crawlee[all]"],
            stdout=subprocess.DEVNULL,
            stderr=subprocess.PIPE,
        )
        return True
    except subprocess.CalledProcessError as e:
        # Try without [all] extras (minimal install — beautifulsoup only)
        try:
            subprocess.check_call(
                [sys.executable, "-m", "pip", "install", "crawlee[beautifulsoup]"],
                stdout=subprocess.DEVNULL,
                stderr=subprocess.PIPE,
            )
            return True
        except subprocess.CalledProcessError:
            print(f"[crawlee-scraper] Failed to install crawlee: {e}", file=sys.stderr)
            return False


def ensure_playwright():
    """Ensure playwright browsers are installed."""
    try:
        importlib.import_module("playwright")
        subprocess.check_call(
            [sys.executable, "-m", "playwright", "install", "chromium"],
            stdout=subprocess.DEVNULL,
            stderr=subprocess.PIPE,
        )
        return True
    except (ImportError, subprocess.CalledProcessError):
        try:
            subprocess.check_call(
                [sys.executable, "-m", "pip", "install", "playwright"],
                stdout=subprocess.DEVNULL,
                stderr=subprocess.PIPE,
            )
            subprocess.check_call(
                [sys.executable, "-m", "playwright", "install", "chromium"],
                stdout=subprocess.DEVNULL,
                stderr=subprocess.PIPE,
            )
            return True
        except subprocess.CalledProcessError as e:
            print(f"[crawlee-scraper] Failed to install playwright: {e}", file=sys.stderr)
            return False


# ---------------------------------------------------------------------------
# Complete page observations, with explicit resource failures (never prefixes)
# ---------------------------------------------------------------------------
MAX_PAGE_BYTES = 5 * 1024 * 1024
MAX_RESULT_BYTES = 8 * 1024 * 1024
DESTINATION_ATTRIBUTES = ("href", "src", "srcset", "action", "formaction", "poster", "data-src", "data-srcset", "data-original")


def extract_page(soup: Any, url: str) -> dict[str, Any]:
    """Retain the entire parser/DOM snapshot and clean complete body view.

    Snapshot provenance is deliberately not an HTTP-status or full-site claim.
    Script/style/template payloads stay in the retained snapshot; JSON-LD is
    additionally exposed without discarding malformed or duplicate entries.
    """
    from bs4 import Comment, NavigableString, Tag
    snapshot = str(soup)
    if len(snapshot.encode("utf-8")) > MAX_PAGE_BYTES:
        raise ValueError("Page snapshot exceeded the complete-page resource limit; no prefix was retained")
    body = soup.body or soup
    links = [{"url": urljoin(url, str(tag.get("href"))), "text": tag.get_text(" ", strip=True)}
             for tag in soup.find_all("a", href=True)]
    metadata = [{"name": str(tag.get("name") or tag.get("property") or tag.get("http-equiv") or ""),
                 "content": str(tag.get("content") or "")} for tag in soup.find_all("meta")]
    structured = []
    for tag in soup.find_all("script"):
        if "ld+json" in str(tag.get("type", "")).lower():
            payload = tag.string if tag.string is not None else tag.get_text()
            try:
                structured.append({"type": str(tag.get("type")), "value": json.loads(payload), "raw": payload})
            except (ValueError, TypeError):
                structured.append({"type": str(tag.get("type")), "raw": payload, "parse_error": True})

    parts: list[str] = []
    def visit(node: Any, preformatted: bool = False) -> None:
        if isinstance(node, Comment):
            return
        if isinstance(node, NavigableString):
            # Preserve code/pre whitespace exactly; normal HTML whitespace is
            # collapsed without removing navigation/footer or document tails.
            parts.append(str(node) if preformatted else re.sub(r"\s+", " ", str(node)))
            return
        if not isinstance(node, Tag):
            return
        for attribute in DESTINATION_ATTRIBUTES:
            value = node.get(attribute)
            if value is not None:
                parts.append("\n[" + attribute + "=" + str(value) + "]\n")
        if node.name == "img" and node.get("alt") is not None:
            parts.append(str(node.get("alt")))
        if node.name in ("script", "style", "template"):
            return
        block = node.name in ("address", "article", "aside", "blockquote", "br", "caption", "div", "footer", "form", "h1", "h2", "h3", "h4", "h5", "h6", "header", "hr", "li", "main", "nav", "p", "pre", "section", "table", "tr")
        if block:
            parts.append("\n")
        for child in node.children:
            visit(child, preformatted or node.name in ("pre", "code"))
            if node.name in ("td", "th"):
                parts.append("\t")
        if block:
            parts.append("\n")
    visit(body)
    title = soup.find("title")
    return {"url": url, "title": title.get_text(" ", strip=True) if title else "", "text": "".join(parts),
            "html_snapshot": snapshot, "links": links, "metadata": metadata, "structured_data": structured,
            "text_coverage": {"scope": "all_static_body_text", "rendered": False,
                              "excluded_payload_elements": ["script", "style", "template"],
                              "included_attributes": list(DESTINATION_ATTRIBUTES)}}


def checked_results(results: list[dict[str, Any]]) -> list[dict[str, Any]]:
    if len(json.dumps(results, ensure_ascii=False).encode("utf-8")) > MAX_RESULT_BYTES:
        raise ValueError("Crawl exceeded the complete-result resource limit; no partial success was retained")
    return results


async def crawl_pages(start_url: str, max_pages: int, max_depth: int, rendered: bool) -> list[dict[str, Any]]:
    from crawlee import ConcurrencySettings
    if rendered:
        from crawlee.playwright_crawler import PlaywrightCrawler as Crawler
    else:
        from crawlee.beautifulsoup_crawler import BeautifulSoupCrawler as Crawler
    results: list[dict[str, Any]] = []
    visited: set[str] = set()
    frontier = [start_url]
    domain = urlparse(start_url).netloc
    # Breadth-first explicit levels avoid relying on crawler-specific private
    # request depth fields. max_pages and max_depth define requested scope.
    for depth in range(max_depth + 1):
        urls = [url for url in frontier if url not in visited][:max_pages - len(results)]
        if not urls:
            break
        failures: list[str] = []
        crawler_options: dict[str, Any] = {
            "max_request_retries": 2,
            "request_handler_timeout": timedelta(seconds=60 if rendered else 30),
            "max_requests_per_crawl": len(urls),
            "concurrency_settings": ConcurrencySettings(max_concurrency=1),
        }
        if rendered:
            crawler_options.update({"headless": True, "browser_type": "chromium"})
        crawler = Crawler(**crawler_options)

        @crawler.router.default_handler
        async def handler(context: Any) -> None:
            url = context.request.url
            if rendered:
                from bs4 import BeautifulSoup
                await context.page.wait_for_load_state("domcontentloaded", timeout=15000)
                soup = BeautifulSoup(await context.page.content(), "html.parser")
            else:
                soup = context.soup
            page = extract_page(soup, url)
            page["depth"] = depth
            page["text_coverage"]["rendered"] = rendered
            # Record an observed status only when the actual response exposes it.
            response = getattr(context, "response", None) if rendered else getattr(context, "http_response", None)
            status = getattr(response, "status", None) if rendered else getattr(response, "status_code", None)
            if isinstance(status, int):
                if status >= 400:
                    raise RuntimeError(f"Page request failed with observed HTTP {status}")
                page["observed_status"] = status
            results.append(page)
            checked_results(results)

        @crawler.failed_request_handler
        async def failed(context: Any, error: Exception) -> None:
            failures.append(str(context.request.url))

        await crawler.run(urls)
        if failures or len(results) < len(visited) + len(urls):
            raise RuntimeError("Crawl did not retrieve all dispatched pages; no partial success was retained")
        visited.update(urls)
        if len(results) >= max_pages:
            break
        frontier = []
        seen = set(visited)
        for page in results:
            if page["depth"] != depth:
                continue
            for link in page["links"]:
                candidate = link["url"]
                parsed = urlparse(candidate)
                if parsed.scheme in ("http", "https") and parsed.netloc == domain and candidate not in seen:
                    frontier.append(candidate)
                    seen.add(candidate)
    return checked_results(results)


async def crawl_beautifulsoup(start_url: str, max_pages: int = 5, max_depth: int = 1, extract: str = "all") -> list[dict[str, Any]]:
    return await crawl_pages(start_url, max_pages, max_depth, False)


async def crawl_playwright(start_url: str, max_pages: int = 5, max_depth: int = 1, extract: str = "all") -> list[dict[str, Any]]:
    return await crawl_pages(start_url, max_pages, max_depth, True)


# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------

async def main():
    parser = argparse.ArgumentParser(description="Crawlee-based web scraper")
    parser.add_argument("--url", required=True, help="Starting URL to crawl")
    parser.add_argument(
        "--strategy",
        choices=["beautifulsoup", "playwright"],
        default="beautifulsoup",
        help="Crawling strategy (default: beautifulsoup)",
    )
    parser.add_argument(
        "--max-pages", type=int, default=5, help="Max pages to crawl (default: 5)"
    )
    parser.add_argument(
        "--max-depth", type=int, default=1, help="Max crawl depth (default: 1)"
    )
    parser.add_argument(
        "--output",
        choices=["json", "text"],
        default="json",
        help="Output format (default: json)",
    )
    parser.add_argument(
        "--extract",
        choices=["links", "text", "all"],
        default="all",
        help="What to extract (default: all)",
    )

    args = parser.parse_args()
    if not 1 <= args.max_pages <= 50 or not 0 <= args.max_depth <= 5:
        parser.error("max-pages must be 1..50 and max-depth must be 0..5")

    # Ensure crawlee is installed
    if not ensure_crawlee():
        result = {"error": "Failed to install crawlee. Install manually: pip install 'crawlee[all]'"}
        print(json.dumps(result))
        sys.exit(1)

    # For playwright strategy, ensure browsers are installed
    if args.strategy == "playwright":
        if not ensure_playwright():
            result = {"error": "Failed to install playwright browsers. Install manually: playwright install chromium"}
            print(json.dumps(result))
            sys.exit(1)

    # Run the crawler
    try:
        if args.strategy == "playwright":
            results = await crawl_playwright(
                args.url,
                max_pages=args.max_pages,
                max_depth=args.max_depth,
                extract=args.extract,
            )
        else:
            results = await crawl_beautifulsoup(
                args.url,
                max_pages=args.max_pages,
                max_depth=args.max_depth,
                extract=args.extract,
            )
    except Exception as e:
        result = {"error": f"Crawl failed: {str(e)}"}
        print(json.dumps(result))
        sys.exit(1)

    # Both display formats retain the complete structured source, including
    # metadata and JSON-LD. Refuse an oversized result instead of a prefix.
    output = {
        "success": True,
        "strategy": args.strategy,
        "content_complete": True,
        "requested_scope": {"max_pages": args.max_pages, "max_depth": args.max_depth},
        "pages_crawled": len(results),
        "results": results,
    }
    serialized = json.dumps(output, ensure_ascii=False, indent=2)
    if len(serialized.encode("utf-8")) > MAX_RESULT_BYTES:
        print(json.dumps({"error": "Crawl exceeded the complete-result resource limit; no partial success was retained"}))
        sys.exit(1)
    if args.output == "text":
        print("Complete structured crawl source:")
    print(serialized)


if __name__ == "__main__":
    asyncio.run(main())
