"""The logic behind the MCP server (#129) – no MCP SDK needed, so this runs on every Python version."""

import asyncio
import json
from datetime import datetime

import pytest

from proxyscraper import agent
from proxyscraper.checker import CheckResult

from .fakes import http_forward_proxy, serve, target_server


def row(n, ptype="http", country="DE", latency=100, **extra):
    base = {"url": f"{ptype}://1.1.1.{n}:80", "proxy": f"1.1.1.{n}:80", "ptype": ptype, "country": country,
            "latency": latency, "exit_ip": f"9.9.9.{n}", "https": True, "anonymity": "elite",
            "org": "Example GmbH", "asn": 64500, "hosting": False, "blocklisted": False, "streak": 1}
    base.update(extra)
    return base


ROWS = [row(1, latency=300), row(2, "socks5", latency=100), row(3, country="US", https=False),
        row(4, "socks4", anonymity="anonymous", hosting=True), row(5, blocklisted=True, streak=30)]
STATS = {"updated": "2026-09-26T12:17:00+00:00", "total": 5, "run_hours": 1}


# --------------------------------------------------------------------------- live list

def fake_fetch(calls, payloads=None, fail=False):
    payloads = payloads or {"proxies.json": ROWS, "stats.json": STATS}

    async def fetch(url, timeout=0, headers=None):
        calls.append(url)
        if fail:
            raise ConnectionError("HTTP 503")
        return json.dumps(payloads[url.rsplit("/", 1)[1]]).encode()
    return fetch


def test_live_list_is_cached_for_a_few_minutes():
    calls, now = [], [1000.0]
    published = datetime.fromisoformat(STATS["updated"]).timestamp()
    source = agent.LiveSource(fetch=fake_fetch(calls), clock=lambda: now[0], ttl=300,
                              wall=lambda: published + 3700)  # an hour old: reloaded after ttl, not stale

    async def go():
        first = await source.get()
        now[0] += 200
        await source.get()
        now[0] += 200  # 400 s after the first load
        return first, await source.get()

    first, _ = asyncio.run(go())
    assert len(first.rows) == 5 and first.updated == STATS["updated"] and first.run_hours == 1
    assert len(calls) == 4  # proxies.json + stats.json, twice – the middle call came from the cache


def test_an_old_website_copy_loses_to_a_newer_branch():
    """The website is uploaded after the branch is pushed, and that upload may fail: the newer list wins."""
    old = dict(STATS, updated="2026-09-26T09:17:00+00:00")
    payloads = {"https://site.example": {"proxies.json": ROWS[:1], "stats.json": old},
                "https://raw.example": {"proxies.json": ROWS, "stats.json": STATS}}

    async def fetch(url, timeout=0, headers=None):
        base, name = url.rsplit("/", 1)
        return json.dumps(payloads[base][name]).encode()

    now = datetime.fromisoformat(STATS["updated"]).timestamp() + 600
    source = agent.LiveSource(fetch=fetch, bases=tuple(payloads), wall=lambda: now)
    live = asyncio.run(source.get())
    assert live.updated == STATS["updated"] and len(live.rows) == 5


def test_a_failed_refresh_keeps_the_old_list():
    calls, now = [], [0.0]
    good = fake_fetch(calls)
    source = agent.LiveSource(fetch=good, clock=lambda: now[0], ttl=300)

    async def go():
        await source.get()
        source.fetch = fake_fetch(calls, fail=True)
        now[0] += 1000
        return await source.get()

    assert len(asyncio.run(go()).rows) == 5


def test_no_list_at_all_says_what_to_do():
    source = agent.LiveSource(fetch=fake_fetch([], fail=True))
    with pytest.raises(agent.AgentError, match="check_proxies"):
        asyncio.run(source.get())


# --------------------------------------------------------------------------- filters

def urls(rows):
    return [r["url"] for r in rows]


def test_select_filters_and_sorts_best_first():
    # the one listed for 30 runs first, then the fastest of the new ones
    assert urls(agent.select(ROWS))[:2] == ["http://1.1.1.5:80", "socks5://1.1.1.2:80"]
    assert urls(agent.select(ROWS, protocol="socks5")) == ["socks5://1.1.1.2:80"]
    assert set(urls(agent.select(ROWS, countries=["us"]))) == {"http://1.1.1.3:80"}  # case doesn't matter
    assert "http://1.1.1.3:80" not in urls(agent.select(ROWS, https_only=True))
    assert "socks4://1.1.1.4:80" not in urls(agent.select(ROWS, elite_only=True))
    assert "socks4://1.1.1.4:80" not in urls(agent.select(ROWS, exclude_datacenter=True))
    assert "http://1.1.1.5:80" not in urls(agent.select(ROWS, exclude_blocklisted=True))
    assert urls(agent.select(ROWS, max_latency_ms=150)) == ["http://1.1.1.5:80", "socks5://1.1.1.2:80",
                                                            "http://1.1.1.3:80", "socks4://1.1.1.4:80"]


def test_stable_means_listed_for_24_hours_whatever_the_interval():
    assert urls(agent.select(ROWS, stable_only=True, run_hours=1)) == ["http://1.1.1.5:80"]  # streak 30 >= 24
    assert len(agent.select(ROWS, stable_only=True, run_hours=6)) == 1                      # 30 runs * 6 h


def test_min_uptime_keeps_the_ones_listed_most_of_the_week():
    rows = [row(1, "http", uptime_7d=95), row(2, "http", uptime_7d=40), row(3, "http")]  # 3: list without uptime yet
    assert urls(agent.select(rows, min_uptime=90)) == ["http://1.1.1.1:80"]
    assert len(agent.select(rows)) == 3


def test_bad_filters_are_explained():
    with pytest.raises(agent.AgentError, match="protocol"):
        agent.select(ROWS, protocol="ftp")
    with pytest.raises(agent.AgentError, match="two-letter"):
        agent.select(ROWS, countries=["Germany"])


def test_proxy_as_an_agent_sees_it():
    shown = agent.describe(row(7, "socks5", streak=5, org="Hetzner Online GmbH", hosting=True), run_hours=1)
    assert shown == {"url": "socks5://1.1.1.7:80", "protocol": "socks5", "address": "1.1.1.7:80", "country": "DE",
                     "latency_ms": 100, "https": True, "anonymity": "elite", "provider": "Hetzner Online GmbH",
                     "datacenter": True, "blocklisted": False, "up_for_hours": 5, "uptime_7d_percent": None,
                     "works_on": [], "speed_kbps": None, "checked_at": None, "checks": {}}
    assert agent.describe(row(7, "socks5", uptime_7d=93))["uptime_7d_percent"] == 93


def test_describe_takes_check_results_too():
    result = CheckResult("http 2.2.2.2:8080", "http", "2.2.2.2:8080", 250, "9.9.9.9", https=False, country="NL")
    shown = agent.describe(result)
    assert shown["url"] == "http://2.2.2.2:8080" and shown["latency_ms"] == 250 and shown["up_for_hours"] is None


def test_the_note_helps_when_little_matches():
    note = agent.shortage_note(ROWS, wanted=10, matched=1, countries=["US"])
    assert "1 proxy" in note and "DE (4)" in note  # suggests where there are more


# --------------------------------------------------------------------------- fetching pages

def test_only_public_http_targets_are_fetched():
    for bad in ("file:///etc/passwd", "ftp://example.com/", "http://localhost/", "http://127.0.0.1:8080/",
                "http://10.0.0.5/", "http://[::1]/", "https://169.254.169.254/latest/meta-data/", "example.com"):
        with pytest.raises(agent.AgentError):
            agent.check_target(bad)
    assert agent.check_target("https://example.com/a?b=1") == "https://example.com/a?b=1"


def test_html_becomes_readable_text():
    html = b"""<html><head><title>T</title><style>p{color:red}</style><script>alert(1)</script></head>
    <body><h1>Hello &amp; welcome</h1><p>First   line</p><ul><li>one</li><li>two</li></ul>
    <noscript>enable js</noscript><p>Last</p></body></html>"""
    text = agent.html_to_text(html.decode())
    assert text.splitlines() == ["T", "Hello & welcome", "First line", "one", "two", "Last"]


def with_fetcher(client, rows_for=None, strategy="weighted"):
    """A real RotatingServer in front of real local proxies and a local target site."""
    async def go():
        target_srv, target_port = await serve(target_server)
        proxy_srv, proxy_port = await serve(http_forward_proxy)
        rows = (rows_for or (lambda port: [row(1, proxy=f"127.0.0.1:{port}", url=f"http://127.0.0.1:{port}")]))(
            proxy_port)
        live = agent.LiveList(rows=rows, stats=STATS, loaded_at=0.0)

        class Source:
            async def get(self):
                return live

        fetcher = agent.PageFetcher(Source(), allow_private=True, timeout=5, strategy=strategy)
        try:
            return await client(fetcher, target_port)
        finally:
            await fetcher.close()
            target_srv.close()
            proxy_srv.close()
    return asyncio.run(go())


def test_fetch_goes_through_a_proxy_and_says_which():
    async def client(fetcher, port):
        return await fetcher.fetch(f"http://127.0.0.1:{port}/ok")

    page = with_fetcher(client)
    assert page["status"] == 200 and page["text"] == "ok" and not page["truncated"]
    assert page["via"].startswith("http://127.0.0.1:")


def test_fetch_switches_to_the_next_proxy_when_one_is_dead():
    def rows(port):
        return [row(1, proxy="127.0.0.1:1", url="http://127.0.0.1:1", latency=10),  # nothing listens there
                row(2, proxy=f"127.0.0.1:{port}", url=f"http://127.0.0.1:{port}", latency=500)]

    async def client(fetcher, port):
        page = await fetcher.fetch(f"http://127.0.0.1:{port}/ok")
        return page, fetcher.server.stats.recent[-1].attempts

    page, attempts = with_fetcher(client, rows, strategy="fastest")  # the dead one is the fastest: tried first
    assert page["status"] == 200 and attempts == 2
    assert page["via"] != "http://127.0.0.1:1"


def test_fetch_reports_http_errors_instead_of_failing():
    async def client(fetcher, port):
        return await fetcher.fetch(f"http://127.0.0.1:{port}/forbidden")

    page = with_fetcher(client)
    assert page["status"] == 403


def test_fetch_cuts_long_pages():
    async def client(fetcher, port):
        return await fetcher.fetch(f"http://127.0.0.1:{port}/ok", max_chars=1)

    page = with_fetcher(client)
    assert page["text"] == "o" and page["truncated"] is True


def test_no_matching_proxy_is_explained():
    async def client(fetcher, port):
        return await fetcher.fetch(f"http://127.0.0.1:{port}/ok", country="JP")

    with pytest.raises(agent.AgentError, match="JP"):
        with_fetcher(client)


# --------------------------------------------------------------------------- fresh checks

def test_fresh_check_uses_the_live_list_and_reports_progress(monkeypatch):
    seen, ticks = {}, []

    async def fake_find(**kwargs):
        seen.update(kwargs)
        await asyncio.sleep(0.05)
        return [CheckResult("socks5 3.3.3.3:1080", "socks5", "3.3.3.3:1080", 90, "9.9.9.9", https=True,
                            country="DE", anonymity="elite")]

    monkeypatch.setattr(agent, "find_proxies_async", fake_find)

    async def progress(elapsed, message):
        ticks.append(message)

    found = asyncio.run(agent.check_fresh(want=5, protocol="socks5", countries=["de"], https_only=True,
                                          progress=progress, tick=0.01))
    assert [p["url"] for p in found] == ["socks5://3.3.3.3:1080"]
    assert seen["_recheck"] == "live" and seen["types"] == ["socks5"] and seen["countries"] == ["DE"]
    assert seen["want"] == 5 and seen["https"] is True and seen["verbose"] is False
    assert ticks  # the client heard from us while the check ran


def test_full_scan_mode_starts_from_all_sources(monkeypatch):
    seen = {}

    async def fake_find(**kwargs):
        seen.update(kwargs)
        return []

    monkeypatch.setattr(agent, "find_proxies_async", fake_find)
    asyncio.run(agent.check_fresh(want=3, mode="full"))
    assert seen["_recheck"] is None and seen["types"] == ["http", "socks4", "socks5"]


def test_closing_the_server_ends_open_tunnels():
    """A tunnel whose target never hangs up must not keep close() waiting or leave tasks behind."""
    from proxyscraper.server import ProxyPool, RotatingServer

    async def silent_target(reader, writer):
        await reader.read()  # holds the connection until the other side goes away

    async def go():
        target_srv, target_port = await serve(silent_target)
        proxy_srv, proxy_port = await serve(http_forward_proxy)
        result = CheckResult(f"http 127.0.0.1:{proxy_port}", "http", f"127.0.0.1:{proxy_port}", 100, "9.9.9.9",
                             https=True)
        rotating = RotatingServer(ProxyPool([result]), port=0, timeout=3)
        await rotating.start()
        _reader, writer = await asyncio.open_connection("127.0.0.1", rotating.port)
        writer.write(f"CONNECT 127.0.0.1:{target_port} HTTP/1.1\r\nHost: x\r\n\r\n".encode())
        writer.write(b"\x16\x03\x01")  # a first packet, so the tunnel is really in use
        await writer.drain()
        await asyncio.sleep(0.3)
        await asyncio.wait_for(rotating.close(), 3)
        left = [t for t in asyncio.all_tasks() if "_handle" in repr(t) and not t.done()]
        writer.close()
        target_srv.close()
        proxy_srv.close()
        return left

    assert asyncio.run(go()) == []


def dead_rows(port, n=3):
    """n dead proxies that are faster than the one working proxy on `port`."""
    dead = [row(10 + i, proxy=f"127.0.0.1:{i + 1}", url=f"http://127.0.0.1:{i + 1}", latency=10 + i) for i in range(n)]
    return [*dead, row(2, proxy=f"127.0.0.1:{port}", url=f"http://127.0.0.1:{port}", latency=900)]


def test_agents_get_more_attempts_than_the_command_line_server():
    async def client(fetcher, port):
        page = await fetcher.fetch(f"http://127.0.0.1:{port}/ok")
        return page, fetcher.server.stats.recent[-1].attempts

    page, attempts = with_fetcher(client, dead_rows, strategy="fastest")  # 3 dead ones first, then the working one
    assert page["status"] == 200 and attempts == 4


def test_no_proxy_getting_through_is_an_error_not_a_502_page():
    async def client(fetcher, port):
        return await fetcher.fetch(f"http://127.0.0.1:{port}/ok")

    def only_dead(port):
        return dead_rows(port)[:3]

    with pytest.raises(agent.AgentError, match="block"):
        with_fetcher(client, only_dead)


async def cut_off_target(reader, writer):
    """Promises 1000 bytes, sends 10 and hangs up – like a proxy that drops the connection mid-download."""
    await reader.readuntil(b"\r\n\r\n")
    writer.write(b"HTTP/1.1 200 OK\r\nContent-Length: 1000\r\n\r\n0123456789")
    await writer.drain()
    writer.close()


def test_a_download_that_breaks_off_is_explained_not_a_crash():
    async def go():
        target_srv, target_port = await serve(cut_off_target)
        proxy_srv, proxy_port = await serve(http_forward_proxy)
        live = agent.LiveList(rows=[row(1, proxy=f"127.0.0.1:{proxy_port}", url=f"http://127.0.0.1:{proxy_port}")],
                              stats=STATS, loaded_at=0.0)

        class Source:
            async def get(self):
                return live

        fetcher = agent.PageFetcher(Source(), allow_private=True, timeout=5)
        try:
            await fetcher.fetch(f"http://127.0.0.1:{target_port}/big")
        finally:
            await fetcher.close()
            target_srv.close()
            proxy_srv.close()

    with pytest.raises(agent.AgentError, match="dropped the page halfway"):
        asyncio.run(go())


# --------------------------------------------------------------------------- review of #130

def test_fetch_never_bypasses_the_proxy(monkeypatch):
    """no_proxy and macOS' proxy exceptions (*.local, 169.254/16) must not send a request out directly."""
    import urllib.request
    monkeypatch.setenv("no_proxy", "*")
    monkeypatch.setattr(urllib.request, "proxy_bypass", lambda host: True)

    async def client(fetcher, port):
        page = await fetcher.fetch(f"http://127.0.0.1:{port}/ok")
        return page, fetcher.server.stats.requests

    page, requests = with_fetcher(client)
    assert page["status"] == 200 and page["via"] and requests == 1


def test_redirects_are_followed_and_checked_like_the_first_url():
    async def followed(fetcher, port):
        return await fetcher.fetch(f"http://127.0.0.1:{port}/redirect")

    page = with_fetcher(followed)
    assert page["status"] == 200 and page["text"] == "ok" and page["final_url"].endswith("/ok")

    def refuse_ok(url):
        if url.endswith("/ok"):
            raise agent.AgentError("private")
        return url

    async def refused(fetcher, port):
        fetcher.check = refuse_ok  # like a public page redirecting to 10.0.0.1
        return await fetcher.fetch(f"http://127.0.0.1:{port}/redirect")

    with pytest.raises(agent.AgentError, match="private"):
        with_fetcher(refused)


def test_disguised_local_addresses_are_refused_too():
    for bad in ("http://localhost./", "http://127.1/", "http://2130706433/", "http://0x7f000001/",
                "http://printer.local/", "http://[::ffff:127.0.0.1]/"):
        with pytest.raises(agent.AgentError):
            agent.check_target(bad)


def test_private_ipv4_wrapped_in_ipv6_is_refused():
    """NAT64 (64:ff9b::/96), 6to4, Teredo and IPv4-compatible addresses carry an IPv4 – a private one inside
    is still the local network (the same rule as netpolicy.is_public_address)."""
    for bad in ("http://[64:ff9b::7f00:1]/", "http://[64:ff9b::a00:5]/", "http://[2002:7f00:1::]/",
                "http://[::7f00:1]/"):
        with pytest.raises(agent.AgentError, match="private or reserved"):
            agent.check_target(bad)
    assert agent.check_target("http://[64:ff9b::5db8:d822]/") == "http://[64:ff9b::5db8:d822]/"  # a public IPv4


def test_parallel_fetches_on_a_fresh_fetcher():
    async def client(fetcher, port):
        return await asyncio.gather(*(fetcher.fetch(f"http://127.0.0.1:{port}/ok") for _ in range(3)))

    assert [p["status"] for p in with_fetcher(client)] == [200, 200, 200]


async def slow_target(reader, writer):
    await reader.readuntil(b"\r\n\r\n")
    await asyncio.sleep(0.3)
    writer.write(b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\n\r\nslow")
    await writer.drain()
    writer.close()


def test_a_slow_fetch_reports_progress():
    ticks = []

    async def go():
        target_srv, target_port = await serve(slow_target)
        proxy_srv, proxy_port = await serve(http_forward_proxy)
        live = agent.LiveList(rows=[row(1, proxy=f"127.0.0.1:{proxy_port}", url=f"http://127.0.0.1:{proxy_port}")],
                              stats=STATS, loaded_at=0.0)

        class Source:
            async def get(self):
                return live

        async def progress(elapsed, message):
            ticks.append(message)

        fetcher = agent.PageFetcher(Source(), allow_private=True, timeout=5)
        try:
            return await fetcher.fetch(f"http://127.0.0.1:{target_port}/", progress=progress, tick=0.05)
        finally:
            await fetcher.close()
            target_srv.close()
            proxy_srv.close()

    assert asyncio.run(go())["text"] == "slow" and ticks


async def cutting_proxy(reader, writer):
    """Answers every request itself with a body that stops early."""
    await reader.readuntil(b"\r\n\r\n")
    writer.write(b"HTTP/1.1 200 OK\r\nContent-Length: 1000\r\n\r\n0123")
    await writer.drain()
    writer.close()


def test_a_proxy_that_broke_off_is_not_picked_again():
    async def go():
        target_srv, target_port = await serve(target_server)
        bad_srv, bad_port = await serve(cutting_proxy)
        good_srv, good_port = await serve(http_forward_proxy)
        live = agent.LiveList(rows=[row(1, proxy=f"127.0.0.1:{bad_port}", url=f"http://127.0.0.1:{bad_port}",
                                        latency=10),
                                    row(2, proxy=f"127.0.0.1:{good_port}", url=f"http://127.0.0.1:{good_port}",
                                        latency=900)], stats=STATS, loaded_at=0.0)

        class Source:
            async def get(self):
                return live

        fetcher = agent.PageFetcher(Source(), allow_private=True, timeout=5, strategy="fastest")
        try:
            page = await fetcher.fetch(f"http://127.0.0.1:{target_port}/ok")
            return page, good_port, [e.disabled for e in fetcher.server.pool.entries]
        finally:
            await fetcher.close()
            for s in (target_srv, bad_srv, good_srv):
                s.close()

    page, good_port, disabled = asyncio.run(go())
    assert page["text"] == "ok" and page["via"] == f"http://127.0.0.1:{good_port}"
    assert disabled == [True, False]  # proven bad: another proxy delivered the same page


def test_while_github_is_down_the_old_list_is_served_without_waiting():
    calls, now = [], [0.0]
    source = agent.LiveSource(fetch=fake_fetch(calls), clock=lambda: now[0], ttl=300)

    async def go():
        await source.get()
        source.fetch = fake_fetch(calls, fail=True)
        now[0] += 1000
        await source.get()                # tries once, fails, serves the old list
        tried = len(calls)
        now[0] += 10
        await source.get()                # shortly after: straight from the cache
        return tried, len(calls)

    tried, after = asyncio.run(go())
    assert after == tried


def test_an_unknown_charset_falls_back_to_utf8():
    assert agent.decode_body("grüße".encode(), "utf8mb4") == "grüße"
    assert agent.decode_body("grüße".encode("latin-1"), "latin-1") == "grüße"


# --------------------------------------------------------------------------- second review of #130

def test_a_retired_proxy_stays_out_while_the_list_is_the_same():
    """The list is re-downloaded every few minutes, but only a new hour's list may bring a proxy back."""
    async def go():
        target_srv, target_port = await serve(target_server)
        proxy_srv, proxy_port = await serve(http_forward_proxy)
        rows = [row(1, proxy=f"127.0.0.1:{proxy_port}", url=f"http://127.0.0.1:{proxy_port}")]
        state = {"live": agent.LiveList(rows=rows, stats=STATS, loaded_at=0.0)}

        class Source:
            async def get(self):
                return state["live"]

        fetcher = agent.PageFetcher(Source(), allow_private=True, timeout=5)
        try:
            await fetcher.fetch(f"http://127.0.0.1:{target_port}/ok")
            fetcher.server.pool.retire(fetcher.server.pool.entries[0])
            state["live"] = agent.LiveList(rows=rows, stats=STATS, loaded_at=300.0)  # same list, loaded again
            with pytest.raises(agent.AgentError):
                await fetcher.fetch(f"http://127.0.0.1:{target_port}/ok")
            return fetcher.server.pool.entries[0].disabled
        finally:
            await fetcher.close()
            target_srv.close()
            proxy_srv.close()

    assert asyncio.run(go()) is True


def test_names_that_resolve_to_private_addresses_are_refused(monkeypatch):
    import socket as socket_module

    def fake_getaddrinfo(host, *args, **kwargs):
        ip = {"127.0.0.1.nip.io": "127.0.0.1", "intranet.corp.example": "10.1.2.3"}.get(host, "93.184.216.34")
        return [(socket_module.AF_INET, socket_module.SOCK_STREAM, 6, "", (ip, 0))]

    monkeypatch.setattr(agent.socket, "getaddrinfo", fake_getaddrinfo)
    for bad in ("http://127.0.0.1.nip.io/", "https://intranet.corp.example/wiki"):
        with pytest.raises(agent.AgentError, match="resolves to"):
            agent.check_target(bad)
    assert agent.check_target("https://example.com/") == "https://example.com/"


def test_urls_are_normalized_or_explained():
    assert agent.check_scheme("http://bücher.de/x?y=1") == "http://xn--bcher-kva.de/x?y=1"
    for bad in ("https://example.com:99999/", "http://exa mple.com/"):
        with pytest.raises(agent.AgentError):
            agent.check_scheme(bad)


def test_a_redirect_from_https_to_http_is_not_followed():
    assert agent.next_hop("https://a.example/x", "/y", agent.check_scheme) == "https://a.example/y"
    with pytest.raises(agent.AgentError, match="plain http"):
        agent.next_hop("https://a.example/x", "http://a.example/y", agent.check_scheme)
    assert agent.next_hop("http://a.example/x", "https://a.example/y", agent.check_scheme) == "https://a.example/y"


async def endless_redirect_target(reader, writer):
    """A redirect with a body that never ends – reading it would hang."""
    await reader.readuntil(b"\r\n\r\n")
    writer.write(b"HTTP/1.1 302 Found\r\nLocation: /ok\r\nContent-Length: 100000000\r\n\r\n")
    try:
        while True:
            writer.write(b"x" * 1024)
            await writer.drain()
            await asyncio.sleep(0.01)
    except (ConnectionError, OSError):
        pass


def test_a_redirect_body_is_not_read():
    async def go():
        target_srv, target_port = await serve(endless_redirect_target)
        ok_srv, ok_port = await serve(target_server)
        proxy_srv, proxy_port = await serve(http_forward_proxy)
        live = agent.LiveList(rows=[row(1, proxy=f"127.0.0.1:{proxy_port}", url=f"http://127.0.0.1:{proxy_port}")],
                              stats=STATS, loaded_at=0.0)

        class Source:
            async def get(self):
                return live

        # the redirect points to /ok on the redirecting server itself, which answers every request the same –
        # the check swaps the port so the second hop lands on the normal target
        def check(url):
            return url.replace(f":{target_port}/ok", f":{ok_port}/ok")

        fetcher = agent.PageFetcher(Source(), timeout=5, check=check)
        try:
            return await asyncio.wait_for(fetcher.fetch(f"http://127.0.0.1:{target_port}/start"), 5)
        finally:
            await fetcher.close()
            for s in (target_srv, ok_srv, proxy_srv):
                s.close()

    assert asyncio.run(go())["text"] == "ok"


async def silent_target(reader, writer):
    """Takes the request and never answers."""
    await reader.read()


def test_a_cancelled_fetch_does_not_leave_its_download_hanging():
    """Cancelling the tool call must end the blocking download too, not leave it waiting for minutes."""
    async def go():
        target_srv, target_port = await serve(silent_target)
        proxy_srv, proxy_port = await serve(http_forward_proxy)
        live = agent.LiveList(rows=[row(1, proxy=f"127.0.0.1:{proxy_port}", url=f"http://127.0.0.1:{proxy_port}")],
                              stats=STATS, loaded_at=0.0)

        class Source:
            async def get(self):
                return live

        fetcher = agent.PageFetcher(Source(), allow_private=True, timeout=5)
        task = asyncio.ensure_future(fetcher.fetch(f"http://127.0.0.1:{target_port}/"))
        await asyncio.sleep(0.3)
        assert fetcher.downloads_running == 1
        task.cancel()
        with pytest.raises(asyncio.CancelledError):
            await task
        for _ in range(20):  # the thread notices within a moment
            if fetcher.downloads_running == 0:
                break
            await asyncio.sleep(0.05)
        running = fetcher.downloads_running
        await fetcher.close()
        target_srv.close()
        proxy_srv.close()
        return running

    assert asyncio.run(go()) == 0


def test_a_certificate_failure_is_named():
    """Certificate check fails through this proxy – like a proxy that intercepts HTTPS."""
    from .fakes import serve_tls

    async def tls_ok(reader, writer):
        await reader.readuntil(b"\r\n\r\n")
        writer.write(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok")
        await writer.drain()
        writer.close()

    async def go():
        tls_srv, tls_port = await serve_tls(tls_ok)  # a certificate certifi doesn't trust
        proxy_srv, proxy_port = await serve(http_forward_proxy)
        live = agent.LiveList(rows=[row(1, proxy=f"127.0.0.1:{proxy_port}", url=f"http://127.0.0.1:{proxy_port}")],
                              stats=STATS, loaded_at=0.0)

        class Source:
            async def get(self):
                return live

        fetcher = agent.PageFetcher(Source(), allow_private=True, timeout=5)
        try:
            with pytest.raises(agent.AgentError, match="certificate"):
                await fetcher.fetch(f"https://localhost:{tls_port}/")
            return fetcher.server.pool.entries[0].disabled is False  # one proxy, same result: the site's fault
        finally:
            await fetcher.close()
            tls_srv.close()
            proxy_srv.close()

    assert asyncio.run(go()) is True


def test_results_dir_is_read_when_used_not_when_imported(tmp_path, monkeypatch):
    from proxyscraper import output, paths
    monkeypatch.setattr(paths, "RESULTS_DIR", tmp_path / "moved")
    writer = output.ResultWriter()
    writer.close()
    assert writer.run_dir.parent == tmp_path / "moved"


def test_the_live_list_is_kept_until_the_next_hourly_run():
    calls = []
    now, wall = [0.0], [1000.0]
    stats = dict(STATS, updated="1970-01-01T00:06:40+00:00", run_hours=1)  # updated at wall 400 s

    source = agent.LiveSource(fetch=fake_fetch(calls, {"proxies.json": ROWS, "stats.json": stats}),
                              clock=lambda: now[0], ttl=300, wall=lambda: wall[0])

    async def go():
        await source.get()             # list from 400 s, next run expected at 400 + 3600
        now[0] += 900
        wall[0] += 900
        await source.get()             # 15 minutes later: still the same hour's list, no download
        first = len(calls)
        now[0] += 3000
        wall[0] += 3000                # past the next run: download again
        await source.get()
        return first, len(calls)

    first, after = asyncio.run(go())
    assert first == 2 and after == 4


# --------------------------------------------------------------------------- third review of #130

def test_non_ascii_paths_are_percent_encoded():
    assert agent.check_scheme("https://de.wikipedia.org/wiki/München?q=größe") == \
        "https://de.wikipedia.org/wiki/M%C3%BCnchen?q=gr%C3%B6%C3%9Fe"
    assert agent.check_scheme("https://a.example/already%20encoded?x=%C3%BC") == \
        "https://a.example/already%20encoded?x=%C3%BC"  # nothing encoded twice


def test_urls_with_a_login_are_not_sent_through_public_proxies():
    with pytest.raises(agent.AgentError, match="login"):
        agent.check_scheme("http://user:secret@example.com/")


def test_a_non_text_charset_falls_back_to_utf8():
    assert agent.decode_body("grüße".encode(), "zlib") == "grüße"


def test_a_site_with_a_broken_certificate_costs_no_proxies():
    """Every proxy sees the same bad certificate: that's the site, not the proxies – nobody gets retired."""
    from .fakes import serve_tls

    async def tls_ok(reader, writer):
        await reader.readuntil(b"\r\n\r\n")
        writer.write(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok")
        await writer.drain()
        writer.close()

    async def go():
        tls_srv, tls_port = await serve_tls(tls_ok)  # a certificate certifi doesn't trust
        a_srv, a_port = await serve(http_forward_proxy)
        b_srv, b_port = await serve(http_forward_proxy)
        live = agent.LiveList(rows=[row(1, proxy=f"127.0.0.1:{a_port}", url=f"http://127.0.0.1:{a_port}"),
                                    row(2, proxy=f"127.0.0.1:{b_port}", url=f"http://127.0.0.1:{b_port}")],
                              stats=STATS, loaded_at=0.0)

        class Source:
            async def get(self):
                return live

        fetcher = agent.PageFetcher(Source(), allow_private=True, timeout=5)
        try:
            with pytest.raises(agent.AgentError, match="certificate"):
                await fetcher.fetch(f"https://localhost:{tls_port}/")
            return [e.disabled for e in fetcher.server.pool.entries]
        finally:
            await fetcher.close()
            for s in (tls_srv, a_srv, b_srv):
                s.close()

    assert asyncio.run(go()) == [False, False]


def test_a_real_error_after_a_proxy_fault_is_not_hidden():
    """Proxy A drops the page, proxy B gets it – and it redirects somewhere refused. That refusal is the news."""
    def refuse_ok(url):
        if url.endswith("/ok"):
            raise agent.AgentError("refused: private")
        return url

    async def go():
        target_srv, target_port = await serve(target_server)
        bad_srv, bad_port = await serve(cutting_proxy)
        good_srv, good_port = await serve(http_forward_proxy)
        live = agent.LiveList(rows=[row(1, proxy=f"127.0.0.1:{bad_port}", url=f"http://127.0.0.1:{bad_port}",
                                        latency=10),
                                    row(2, proxy=f"127.0.0.1:{good_port}", url=f"http://127.0.0.1:{good_port}",
                                        latency=900)], stats=STATS, loaded_at=0.0)

        class Source:
            async def get(self):
                return live

        fetcher = agent.PageFetcher(Source(), timeout=5, strategy="fastest", check=refuse_ok)
        try:
            await fetcher.fetch(f"http://127.0.0.1:{target_port}/redirect")
        finally:
            await fetcher.close()
            for s in (target_srv, bad_srv, good_srv):
                s.close()

    with pytest.raises(agent.AgentError, match="refused: private"):
        asyncio.run(go())
