#!/usr/bin/env python3
"""
crawlee-scraper.py — Self-contained web scraping script using Crawlee.

Supports two crawling strategies:
  - beautifulsoup (default): Fast HTTP-based scraping via httpx + BeautifulSoup.
    Best for static content, documentation pages, and API docs.
  - playwright: Headless browser scraping for JS-rendered pages, SPAs, and
    sites that require JavaScript execution to display content.

Usage:
  python3 crawlee-scraper.py --url URL [--strategy beautifulsoup|playwright]
    [--max-pages N] [--max-depth N] [--output json|text] [--extract links|text|all]

Auto-installs crawlee and playwright if not present.
"""

import argparse
import asyncio
import json
import subprocess
import sys
import importlib
from typing import Any

# ---------------------------------------------------------------------------
# Auto-install crawlee if missing
# ---------------------------------------------------------------------------

def ensure_crawlee():
    """Ensure crawlee is installed, installing it if needed."""
    try:
        importlib.import_module("crawlee")
        return True
    except ImportError:
        pass

    print("[crawlee-scraper] Installing crawlee...", file=sys.stderr)
    try:
        subprocess.check_call(
            [sys.executable, "-m", "pip", "install", "crawlee[all]"],
            stdout=subprocess.DEVNULL,
            stderr=subprocess.PIPE,
        )
        return True
    except subprocess.CalledProcessError as e:
        # Try without [all] extras (minimal install — beautifulsoup only)
        try:
            subprocess.check_call(
                [sys.executable, "-m", "pip", "install", "crawlee[beautifulsoup]"],
                stdout=subprocess.DEVNULL,
                stderr=subprocess.PIPE,
            )
            return True
        except subprocess.CalledProcessError:
            print(f"[crawlee-scraper] Failed to install crawlee: {e}", file=sys.stderr)
            return False


def ensure_playwright():
    """Ensure playwright browsers are installed."""
    try:
        importlib.import_module("playwright")
        subprocess.check_call(
            [sys.executable, "-m", "playwright", "install", "chromium"],
            stdout=subprocess.DEVNULL,
            stderr=subprocess.PIPE,
        )
        return True
    except (ImportError, subprocess.CalledProcessError):
        try:
            subprocess.check_call(
                [sys.executable, "-m", "pip", "install", "playwright"],
                stdout=subprocess.DEVNULL,
                stderr=subprocess.PIPE,
            )
            subprocess.check_call(
                [sys.executable, "-m", "playwright", "install", "chromium"],
                stdout=subprocess.DEVNULL,
                stderr=subprocess.PIPE,
            )
            return True
        except subprocess.CalledProcessError as e:
            print(f"[crawlee-scraper] Failed to install playwright: {e}", file=sys.stderr)
            return False


# ---------------------------------------------------------------------------
# BeautifulSoup crawler
# ---------------------------------------------------------------------------

async def crawl_beautifulsoup(
    start_url: str,
    max_pages: int = 5,
    max_depth: int = 1,
    extract: str = "all",
) -> list[dict[str, Any]]:
    """Crawl using BeautifulSoup (HTTP-based, fast, no JS rendering)."""
    from crawlee.beautifulsoup_crawler import BeautifulSoupCrawler, BeautifulSoupCrawlingContext
    from crawlee import ConcurrencySettings

    results: list[dict[str, Any]] = []
    pages_crawled = 0

    crawler = BeautifulSoupCrawler(
        max_request_retries=2,
        request_handler_timeout=asyncio.timedelta(seconds=30),
        max_requests_per_crawl=max_pages,
        concurrency_settings=ConcurrencySettings(
            max_concurrency=3,
        ),
    )

    @crawler.router.default_handler
    async def handler(context: BeautifulSoupCrawlingContext) -> None:
        nonlocal pages_crawled
        if pages_crawled >= max_pages:
            return
        pages_crawled += 1

        soup = context.soup
        page_data: dict[str, Any] = {
            "url": context.request.url,
            "status": 200,
        }

        # Extract title
        title_tag = soup.find("title")
        page_data["title"] = title_tag.get_text(strip=True) if title_tag else ""

        # Extract text content
        if extract in ("text", "all"):
            # Remove script and style elements
            for tag in soup(["script", "style", "nav", "footer", "header"]):
                tag.decompose()
            text = soup.get_text(separator="\n", strip=True)
            # Collapse whitespace
            lines = [line.strip() for line in text.splitlines() if line.strip()]
            page_data["text"] = "\n".join(lines)

        # Extract links
        if extract in ("links", "all"):
            links = []
            for a_tag in soup.find_all("a", href=True):
                href = str(a_tag["href"])
                link_text = a_tag.get_text(strip=True)
                if href.startswith(("http://", "https://")):
                    links.append({"url": href, "text": link_text})
            page_data["links"] = links[:50]  # Cap at 50 links per page

        # Extract metadata
        meta_tags = {}
        for meta in soup.find_all("meta"):
            name = meta.get("name", meta.get("property", ""))
            content = meta.get("content", "")
            if name and content:
                meta_tags[str(name)] = str(content)
        if meta_tags:
            page_data["meta"] = meta_tags

        results.append(page_data)

        # Enqueue links for deeper crawling if depth allows
        if max_depth > 0:
            await context.enqueue_links(strategy="same-domain")

    await crawler.run([start_url])
    return results


# ---------------------------------------------------------------------------
# Playwright crawler
# ---------------------------------------------------------------------------

async def crawl_playwright(
    start_url: str,
    max_pages: int = 5,
    max_depth: int = 1,
    extract: str = "all",
) -> list[dict[str, Any]]:
    """Crawl using Playwright (headless browser, JS rendering)."""
    from crawlee.playwright_crawler import PlaywrightCrawler, PlaywrightCrawlingContext
    from crawlee import ConcurrencySettings

    results: list[dict[str, Any]] = []
    pages_crawled = 0

    crawler = PlaywrightCrawler(
        max_request_retries=2,
        request_handler_timeout=asyncio.timedelta(seconds=60),
        max_requests_per_crawl=max_pages,
        headless=True,
        browser_type="chromium",
        concurrency_settings=ConcurrencySettings(
            max_concurrency=2,
        ),
    )

    @crawler.router.default_handler
    async def handler(context: PlaywrightCrawlingContext) -> None:
        nonlocal pages_crawled
        if pages_crawled >= max_pages:
            return
        pages_crawled += 1

        page = context.page
        page_data: dict[str, Any] = {
            "url": context.request.url,
            "status": 200,
        }

        # Wait for content to load
        try:
            await page.wait_for_load_state("networkidle", timeout=15000)
        except Exception:
            await page.wait_for_load_state("domcontentloaded", timeout=10000)

        # Extract title
        page_data["title"] = await page.title()

        # Extract text content
        if extract in ("text", "all"):
            # Remove non-content elements via JS
            text = await page.evaluate("""() => {
                const remove = document.querySelectorAll('script, style, nav, footer, header, [role="navigation"]');
                remove.forEach(el => el.remove());
                return document.body ? document.body.innerText : '';
            }""")
            lines = [line.strip() for line in str(text).splitlines() if line.strip()]
            page_data["text"] = "\n".join(lines)

        # Extract links
        if extract in ("links", "all"):
            links = await page.evaluate("""() => {
                const anchors = document.querySelectorAll('a[href]');
                return Array.from(anchors).slice(0, 50).map(a => ({
                    url: a.href,
                    text: a.innerText.trim()
                })).filter(l => l.url.startsWith('http'));
            }""")
            page_data["links"] = links

        # Extract meta tags
        meta_tags = await page.evaluate("""() => {
            const metas = document.querySelectorAll('meta[name], meta[property]');
            const result = {};
            metas.forEach(m => {
                const key = m.getAttribute('name') || m.getAttribute('property');
                const val = m.getAttribute('content');
                if (key && val) result[key] = val;
            });
            return result;
        }""")
        if meta_tags:
            page_data["meta"] = meta_tags

        results.append(page_data)

        # Enqueue links for deeper crawling
        if max_depth > 0:
            await context.enqueue_links(strategy="same-domain")

    await crawler.run([start_url])
    return results


# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------

async def main():
    parser = argparse.ArgumentParser(description="Crawlee-based web scraper")
    parser.add_argument("--url", required=True, help="Starting URL to crawl")
    parser.add_argument(
        "--strategy",
        choices=["beautifulsoup", "playwright"],
        default="beautifulsoup",
        help="Crawling strategy (default: beautifulsoup)",
    )
    parser.add_argument(
        "--max-pages", type=int, default=5, help="Max pages to crawl (default: 5)"
    )
    parser.add_argument(
        "--max-depth", type=int, default=1, help="Max crawl depth (default: 1)"
    )
    parser.add_argument(
        "--output",
        choices=["json", "text"],
        default="json",
        help="Output format (default: json)",
    )
    parser.add_argument(
        "--extract",
        choices=["links", "text", "all"],
        default="all",
        help="What to extract (default: all)",
    )

    args = parser.parse_args()

    # Ensure crawlee is installed
    if not ensure_crawlee():
        result = {"error": "Failed to install crawlee. Install manually: pip install 'crawlee[all]'"}
        print(json.dumps(result))
        sys.exit(1)

    # For playwright strategy, ensure browsers are installed
    if args.strategy == "playwright":
        if not ensure_playwright():
            result = {"error": "Failed to install playwright browsers. Install manually: playwright install chromium"}
            print(json.dumps(result))
            sys.exit(1)

    # Run the crawler
    try:
        if args.strategy == "playwright":
            results = await crawl_playwright(
                args.url,
                max_pages=args.max_pages,
                max_depth=args.max_depth,
                extract=args.extract,
            )
        else:
            results = await crawl_beautifulsoup(
                args.url,
                max_pages=args.max_pages,
                max_depth=args.max_depth,
                extract=args.extract,
            )
    except Exception as e:
        result = {"error": f"Crawl failed: {str(e)}"}
        print(json.dumps(result))
        sys.exit(1)

    # Output results
    if args.output == "text":
        for page in results:
            print(f"=== {page.get('title', 'Untitled')} ===")
            print(f"URL: {page.get('url', '')}")
            if "text" in page:
                print(page["text"][:5000])
            if "links" in page:
                print(f"\nLinks ({len(page['links'])}):")
                for link in page["links"][:20]:
                    print(f"  - {link.get('text', '')}: {link.get('url', '')}")
            print()
    else:
        output = {
            "success": True,
            "strategy": args.strategy,
            "pages_crawled": len(results),
            "results": results,
        }
        print(json.dumps(output, ensure_ascii=False, indent=2))


if __name__ == "__main__":
    asyncio.run(main())