import json
import random
from collections import Counter

import pytest

from proxyscraper import output, pipeline
from proxyscraper import sources as srcs
from proxyscraper.checker import CheckResult
from proxyscraper.geo import flag
from proxyscraper.history import MAX_FAIL_STREAK, ProxyHistory
from proxyscraper.ui import bar, fmt, pct


def result(key="http 1.1.1.1:80", latency=500, https=True, anonymity="elite", country="DE"):
    ptype, proxy = key.split(" ")
    return CheckResult(key, ptype, proxy, latency, "9.9.9.9", https, anonymity, country)


def test_attribute_results_counts_per_source():
    res = pipeline.ScrapeResult(["a", "b"], {"http 1.1.1.1:80": [0, 1], "http 2.2.2.2:80": [1]})
    out = pipeline.attribute_results(res, ["http 1.1.1.1:80", "http 2.2.2.2:80"], {"http 1.1.1.1:80"})
    assert out == {"a": (1, 1), "b": (2, 1)}


def test_prioritize_history_first_then_good_sources(tmp_path):
    quality = srcs.SourceStats(tmp_path / "stats.json")
    quality.record_checks({"good": (100, 50), "bad": (100, 0)})
    history = ProxyHistory(tmp_path / "h.json")
    history.record_ok("socks5 7.7.7.7:1080", 100, "7.7.7.7")
    history.record_ok("socks4 8.8.8.8:1080", 100, "8.8.8.8")  # type not requested
    res = pipeline.ScrapeResult(["bad", "good"], {"http 1.1.1.1:80": [0], "http 2.2.2.2:80": [1]})
    order = pipeline.prioritize(res, quality, history, ["http", "socks5"])
    assert order == ["socks5 7.7.7.7:1080", "http 2.2.2.2:80", "http 1.1.1.1:80"]


def test_best_sources_needs_minimum_sample():
    per = {"a": (100, 10), "b": (10, 5), "c": (50, 0)}
    assert [u for *_, u in pipeline.best_sources(per)] == ["a"]


def test_history_reliability_and_pruning(tmp_path):
    h = ProxyHistory(tmp_path / "h.json")
    h.record_ok("http 1.1.1.1:80", 300, "1.1.1.1", now=1000.0, country="DE", https=True)
    h.record_ok("http 2.2.2.2:80", 300, "2.2.2.2", now=1000.0)
    for _ in range(MAX_FAIL_STREAK):
        h.record_fail("http 2.2.2.2:80")
    h.record_fail("http 3.3.3.3:80")  # unknown -> isn't created
    assert h.ranked_keys()[0] == "http 1.1.1.1:80"
    assert h.prune(now=1001.0) == 1
    h.save()
    again = ProxyHistory(tmp_path / "h.json")
    assert list(again.records) == ["http 1.1.1.1:80"]
    assert again.get("http 1.1.1.1:80").country == "DE"


def test_result_writer_creates_all_formats(tmp_path):
    w = output.ResultWriter(run_dir=tmp_path / "run1", extra_file=tmp_path / "extra.txt")
    rows = [result("socks5 2.2.2.2:1080", 900), result("http 1.1.1.1:80", 200)]
    w.add_live(rows[0])
    files = w.finalize(rows)
    run = tmp_path / "run1"
    assert (run / "all.txt").read_text() == "http://1.1.1.1:80\nsocks5://2.2.2.2:1080\n"
    assert (run / "socks5.txt").read_text() == "2.2.2.2:1080\n"
    assert json.loads((run / "proxies.json").read_text())[0]["url"] == "http://1.1.1.1:80"
    assert (run / "proxies.csv").read_text().splitlines()[0].startswith("ptype,proxy,latency")
    assert (tmp_path / "extra.txt").exists()
    assert len(files) == 6  # all, http, socks5, json, csv, -o
    assert output.latest_run_dir(tmp_path) == run
    assert output.latest_results(tmp_path) == ["http://1.1.1.1:80", "socks5://2.2.2.2:1080"]


def test_latest_pointer_works_without_symlinks(tmp_path, monkeypatch):
    def no_symlinks(*args, **kwargs):
        raise OSError("symbolic links not permitted")  # Windows without admin/developer rights

    monkeypatch.setattr(output.os, "symlink", no_symlinks)
    for name in ("run1", "run2"):
        w = output.ResultWriter(run_dir=tmp_path / name)
        w.finalize([result(f"http 1.1.1.{len(name)}:80")])
    assert not (tmp_path / "latest").exists()
    assert output.latest_run_dir(tmp_path) == tmp_path / "run2"


@pytest.mark.parametrize("content", ["", "\n", "..", "../elsewhere", "run1/../..", "does-not-exist"])
def test_invalid_latest_pointer_is_ignored(tmp_path, content):
    (tmp_path / "run1").mkdir()
    (tmp_path / output.LATEST_POINTER).write_text(content)
    assert output.latest_run_dir(tmp_path) is None


def test_unreadable_latest_pointer_falls_back_to_symlink(tmp_path):
    (tmp_path / "run1").mkdir()
    (tmp_path / output.LATEST_POINTER).write_bytes(b"\xff\xfe kaputt")
    assert output.latest_run_dir(tmp_path) is None
    try:
        (tmp_path / "latest").symlink_to("run1", target_is_directory=True)
    except OSError:
        pytest.skip("no symlinks allowed")
    assert output.latest_run_dir(tmp_path) == tmp_path / "latest"


def test_latest_results_empty_without_runs(tmp_path):
    assert output.latest_run_dir(tmp_path) is None
    assert output.latest_results(tmp_path) == []
    assert not output.has_latest_results(tmp_path)


def test_has_latest_results(tmp_path):
    w = output.ResultWriter(run_dir=tmp_path / "run1")
    w.finalize([])
    assert not output.has_latest_results(tmp_path)  # run without hits
    w = output.ResultWriter(run_dir=tmp_path / "run2")
    w.finalize([result()])
    assert output.has_latest_results(tmp_path)


def test_history_exists(tmp_path):
    path = tmp_path / "h.json"
    assert not ProxyHistory.exists(path)
    path.write_text("{}")
    assert not ProxyHistory.exists(path)
    h = ProxyHistory(path)
    h.record_ok("http 1.1.1.1:80", 100, "1.1.1.1")
    h.save()
    assert ProxyHistory.exists(path)


def test_ui_helpers():
    assert fmt(1234567) == "1,234,567"
    assert pct(1, 3) == "33.3%" and pct(1, 0) == "–"
    assert bar(5, 10, 4, "green").plain == "██··"
    assert flag("DE") == "🇩🇪" and flag("") == "  "


def test_console_is_swappable_everywhere(monkeypatch):
    """All output goes through widgets.console – no frozen copy through an import."""
    import io

    from rich.console import Console

    from proxyscraper import app, cli
    from proxyscraper import pipeline as pipeline_module
    from proxyscraper.ui import widgets

    recorder = Console(file=io.StringIO(), width=80)
    monkeypatch.setattr(widgets, "console", recorder)
    for module in (app, cli, pipeline_module):
        assert getattr(module, "console", None) is None, module.__name__
    widgets.info("Test", "running")
    assert "running" in recorder.file.getvalue()


def test_card_subtitle_stays_on_one_line():
    import io

    from rich.console import Console

    from proxyscraper.ui.widgets import card

    console = Console(width=24, file=io.StringIO(), color_system=None)
    console.print(card("Saved", "12", "HTTPS only · min. anonymous · ≤ 3000 ms"))
    assert len(console.file.getvalue().splitlines()) == 5  # border, 3 lines, border


def test_run_checks_drops_unconfirmed_proxies(tmp_path):
    import asyncio
    import contextlib

    from proxyscraper.geo import GeoResolver
    from proxyscraper.options import RunOptions
    from proxyscraper.ui import CheckDashboard, LiveStats

    class StubChecker:
        async def check(self, key):
            ptype, proxy = key.split(" ")
            return CheckResult(key, ptype, proxy, 100, "9.9.9.9")

        async def confirm(self, r):
            return r.proxy.startswith("1.")  # 2.x.x.x is a "honeypot"

        async def tampers(self, r):
            return False

        async def enrich(self, r):
            r.https = True

    async def go():
        jobs = ["http 1.1.1.1:80", "http 2.2.2.2:80"]
        stats = LiveStats({"http": 2})
        writer = output.ResultWriter(run_dir=tmp_path / "run")
        dashboard = CheckDashboard(stats, writer.live_path, 2, True)
        run = await pipeline.run_checks(jobs, StubChecker(), RunOptions(no_geo=True), dashboard, writer,
                                        GeoResolver(enabled=False), live_factory=lambda _: contextlib.nullcontext())
        return run, stats, writer

    run, stats, writer = asyncio.run(go())
    assert [r.key for r in run.results] == ["http 1.1.1.1:80"]
    assert run.working == {"http 1.1.1.1:80"}
    assert stats.fakes == 1 and stats.found == 1
    assert writer.live_path.read_text(encoding="utf-8") == "http://1.1.1.1:80\n"


def test_anonymity_is_counted_without_https_test():
    from proxyscraper.ui import LiveStats

    stats = LiveStats({"http": 1})
    stats.add_working(result(anonymity="elite"))  # comes from the confirmation, also with --fast
    assert stats.anonymity["elite"] == 1 and stats.https_ok == 0


def test_run_checks_skips_https_test_when_filters_already_fail(tmp_path):
    import asyncio
    import contextlib

    from proxyscraper.geo import GeoResolver
    from proxyscraper.options import Filters, RunOptions
    from proxyscraper.ui import CheckDashboard, LiveStats

    enriched = []

    class StubChecker:
        async def check(self, key):
            ptype, proxy = key.split(" ")
            return CheckResult(key, ptype, proxy, 100, "9.9.9.9")

        async def confirm(self, r):
            r.anonymity = "elite" if r.proxy.startswith("1.") else "anonymous"
            return True

        async def tampers(self, r):
            return False

        async def enrich(self, r):
            enriched.append(r.key)
            r.https = True

    async def go():
        stats = LiveStats({"http": 2})
        writer = output.ResultWriter(run_dir=tmp_path / "run")
        opts = RunOptions(no_geo=True, filters=Filters(min_anonymity="elite"))
        await pipeline.run_checks(["http 1.1.1.1:80", "http 2.2.2.2:80"], StubChecker(), opts,
                                  CheckDashboard(stats, writer.live_path, 2, True), writer,
                                  GeoResolver(enabled=False), live_factory=lambda _: contextlib.nullcontext())
        return stats

    stats = asyncio.run(go())
    assert enriched == ["http 1.1.1.1:80"]   # the proxy that is only "anonymous" needs no HTTPS test
    assert stats.details_saved == 1 and stats.found == 2


def test_country_lookup_in_flight_at_scan_end_is_awaited(tmp_path, monkeypatch):
    """The geo worker already took the batch off the queue when the last check ends: the scan must
    still wait for that request, otherwise the hit keeps country='' and --country DE drops it."""
    import asyncio
    import contextlib

    from proxyscraper import geo as geo_module
    from proxyscraper.geo import GeoResolver
    from proxyscraper.options import Filters, RunOptions
    from proxyscraper.ui import CheckDashboard, LiveStats

    monkeypatch.setattr(geo_module, "GEO_CACHE_FILE", tmp_path / "geo_cache.json")

    async def go():
        lookup_started = asyncio.Event()
        release_lookup = asyncio.Event()

        async def slow_batch(url, timeout, method, body, headers):
            lookup_started.set()
            await release_lookup.wait()
            ips = json.loads(body)
            return 200, {}, json.dumps([{"status": "success", "query": ip, "countryCode": "DE"} for ip in ips]).encode()

        monkeypatch.setattr(geo_module, "http_request", slow_batch)

        class StubChecker:
            async def check(self, key):
                ptype, proxy = key.split(" ")
                if proxy.startswith("2."):
                    # the last check only finishes once the country request is under way, then the
                    # request is answered a moment later – with nothing left in geo.pending
                    await lookup_started.wait()
                    asyncio.get_running_loop().call_later(0.05, release_lookup.set)
                    return None
                return CheckResult(key, ptype, proxy, 100, "9.9.9.9", anonymity="elite")

            async def confirm(self, r):
                return True

            async def tampers(self, r):
                return False

            async def enrich(self, r):
                r.https = True

        stats = LiveStats({"http": 2})
        writer = output.ResultWriter(run_dir=tmp_path / "run")
        opts = RunOptions(filters=Filters(countries=["DE"]), concurrency=2)
        run = await asyncio.wait_for(
            pipeline.run_checks(["http 1.1.1.1:80", "http 2.2.2.2:80"], StubChecker(), opts,
                                CheckDashboard(stats, writer.live_path, 2, True), writer, GeoResolver(),
                                live_factory=lambda _: contextlib.nullcontext()),
            10)
        return run, opts, writer

    run, opts, writer = asyncio.run(go())
    assert [r.country for r in run.results] == ["DE"]
    assert opts.filters.accepts(run.results[0])
    assert writer.live_path.read_text(encoding="utf-8") == "http://1.1.1.1:80\n"


def test_scan_end_does_not_wait_for_a_broken_country_service(tmp_path, monkeypatch):
    import asyncio
    import contextlib
    import time

    from proxyscraper import geo as geo_module
    from proxyscraper.geo import GeoResolver
    from proxyscraper.options import RunOptions
    from proxyscraper.ui import CheckDashboard, LiveStats

    monkeypatch.setattr(geo_module, "GEO_CACHE_FILE", tmp_path / "geo_cache.json")

    async def broken(*args, **kwargs):
        raise OSError("unreachable")

    monkeypatch.setattr(geo_module, "http_request", broken)

    class StubChecker:
        async def check(self, key):
            ptype, proxy = key.split(" ")
            await asyncio.sleep(0.7)  # the geo worker polls every 0.5 s – let it fail first
            return CheckResult(key, ptype, proxy, 100, "9.9.9.9")

        async def confirm(self, r):
            return True

        async def tampers(self, r):
            return False

        async def enrich(self, r):
            r.https = True

    async def go():
        stats = LiveStats({"http": 1})
        writer = output.ResultWriter(run_dir=tmp_path / "run")
        geo = GeoResolver()
        geo.request("8.8.8.8")  # something for the worker to fail on
        started = time.monotonic()
        run = await pipeline.run_checks(["http 1.1.1.1:80"], StubChecker(), RunOptions(),
                                        CheckDashboard(stats, writer.live_path, 1, True), writer, geo,
                                        live_factory=lambda _: contextlib.nullcontext())
        return run, geo, time.monotonic() - started

    run, geo, took = asyncio.run(go())
    assert geo.failed
    assert run.results[0].country == ""
    assert took < 3


def _explore_setup(tmp_path, n_tail=40):
    """One well-known good source with many proxies, plus n_tail never-checked sources with 3 proxies each."""
    quality = srcs.SourceStats(tmp_path / "stats.json")
    quality.record_checks({"good": (1000, 300)})
    urls = ["good"] + [f"tail{i}" for i in range(n_tail)]
    index = {f"http 10.0.{i // 250}.{i % 250}:80": [0] for i in range(500)}
    for s in range(1, n_tail + 1):
        for j in range(3):
            index[f"http 20.{s}.0.{j}:80"] = [s]
    return pipeline.ScrapeResult(urls, index), quality


def test_prioritize_explores_under_checked_sources(tmp_path):
    res, quality = _explore_setup(tmp_path)
    history = ProxyHistory(tmp_path / "h.json")
    history.record_ok("http 7.7.7.7:80", 100, "7.7.7.7")
    limit = 201
    order = pipeline.prioritize(res, quality, history, ["http"], limit=limit, explore=0.2,
                                rng=random.Random(1))
    assert order[0] == "http 7.7.7.7:80"  # history still first
    assert len(order) == len(set(order)) == len(res.index) + 1  # full order, no duplicates
    picked = order[:limit]
    tail = [k for k in picked if k.startswith("http 20.")]
    assert len(tail) == 40  # 20 % of the 200 slots after the history
    assert {k.split(".")[1] for k in tail} == {str(s) for s in range(1, 41)}  # one from every tail source
    assert picked[1:161] == [k for k in order if k.startswith("http 10.")][:160]  # exploitation before


def test_prioritize_without_exploration_never_reaches_the_tail(tmp_path):
    res, quality = _explore_setup(tmp_path)
    history = ProxyHistory(tmp_path / "h.json")
    order = pipeline.prioritize(res, quality, history, ["http"], limit=200, explore=0, rng=random.Random(1))
    assert not [k for k in order[:200] if k.startswith("http 20.")]


def test_prioritize_exploration_is_seeded_and_rotates(tmp_path):
    res, quality = _explore_setup(tmp_path, n_tail=100)
    history = ProxyHistory(tmp_path / "h.json")

    def tail(seed):
        order = pipeline.prioritize(res, quality, history, ["http"], limit=100, explore=0.2,
                                    rng=random.Random(seed))
        return [k for k in order[:100] if k.startswith("http 20.")]

    assert tail(5) == tail(5)
    assert len(tail(5)) == 20 and len({k.split(".")[1] for k in tail(5)}) == 20  # 20 different sources
    assert set(tail(5)) != set(tail(6))  # another run samples another part of the long tail


def test_prioritize_exploration_round_robin_takes_more_per_source(tmp_path):
    res, quality = _explore_setup(tmp_path, n_tail=10)
    history = ProxyHistory(tmp_path / "h.json")
    order = pipeline.prioritize(res, quality, history, ["http"], limit=100, explore=0.25, rng=random.Random(2))
    tail = [k for k in order[:100] if k.startswith("http 20.")]
    assert len(tail) == 25  # 30 tail proxies in 10 sources: two rounds for all, a third for five
    per = Counter(k.split(".")[1] for k in tail)
    assert sorted(per.values()) == [2] * 5 + [3] * 5


def test_prioritize_without_limit_or_with_room_keeps_the_plain_order(tmp_path):
    res, quality = _explore_setup(tmp_path)
    history = ProxyHistory(tmp_path / "h.json")
    plain = pipeline.prioritize(res, quality, history, ["http"], explore=0, rng=random.Random(3))
    assert pipeline.prioritize(res, quality, history, ["http"], limit=0, rng=random.Random(3)) == plain
    # fewer candidates than the limit: everything is checked anyway
    assert pipeline.prioritize(res, quality, history, ["http"], limit=10_000, rng=random.Random(3)) == plain


def test_prioritize_explores_nothing_when_all_sources_are_known(tmp_path):
    quality = srcs.SourceStats(tmp_path / "stats.json")
    quality.record_checks({"a": (500, 50), "b": (500, 5)})
    res = pipeline.ScrapeResult(["a", "b"], {f"http 1.1.1.{i}:80": [i % 2] for i in range(100)})
    history = ProxyHistory(tmp_path / "h.json")
    plain = pipeline.prioritize(res, quality, history, ["http"], explore=0, rng=random.Random(4))
    assert pipeline.prioritize(res, quality, history, ["http"], limit=50, rng=random.Random(4)) == plain


def test_prioritize_history_larger_than_limit(tmp_path):
    res, quality = _explore_setup(tmp_path)
    history = ProxyHistory(tmp_path / "h.json")
    for i in range(5):
        history.record_ok(f"http 7.7.7.{i}:80", 100, "7.7.7.7")
    order = pipeline.prioritize(res, quality, history, ["http"], limit=3, rng=random.Random(5))
    assert all(k.startswith("http 7.7.7.") for k in order[:5])


def _second_chance_run(tmp_path, monkeypatch, confirm_plan, content_plan):
    """run_checks with a checker whose confirm/content end as the plans say, one entry per try and proxy."""
    import asyncio
    import contextlib

    from proxyscraper.checkstate import CONFIRM, CONTENT, FAILED, Check
    from proxyscraper.geo import GeoResolver
    from proxyscraper.options import RunOptions
    from proxyscraper.ui import CheckDashboard, LiveStats

    monkeypatch.setattr(pipeline, "SECOND_CHANCE_DELAY", 0.0)
    calls = Counter()

    def outcome(plan, r, name):
        calls[(name, r.key)] += 1
        state, reason = plan[r.key][calls[(name, r.key)] - 1]
        r.checks[name] = Check(state, reason)
        return state

    class StubChecker:
        async def check(self, key):
            ptype, proxy = key.split(" ")
            return CheckResult(key, ptype, proxy, 100, "9.9.9.9")

        async def confirm(self, r):
            return outcome(confirm_plan, r, CONFIRM) != FAILED

        async def tampers(self, r):
            return outcome(content_plan, r, CONTENT) == FAILED

        async def enrich(self, r):
            r.https = True

    async def go():
        jobs = sorted(set(confirm_plan))
        stats = LiveStats({"http": len(jobs)})
        writer = output.ResultWriter(run_dir=tmp_path / "run")
        dashboard = CheckDashboard(stats, writer.live_path, len(jobs), True)
        run = await pipeline.run_checks(jobs, StubChecker(), RunOptions(no_geo=True), dashboard, writer,
                                        GeoResolver(enabled=False), live_factory=lambda _: contextlib.nullcontext())
        return run, stats

    run, stats = asyncio.run(go())
    return run, stats, calls


def test_follow_ups_that_broke_off_get_a_second_chance(tmp_path, monkeypatch):
    ok, drop, flaky, gone = "http 1.1.1.1:80", "http 2.2.2.2:80", "http 3.3.3.3:80", "http 4.4.4.4:80"
    passed = ("passed", "")
    confirm_plan = {
        ok: [passed],
        drop: [("failed", "exit_ip_mismatch")],                 # a real verdict: no second try
        flaky: [("failed", "connect_timeout"), passed],          # broke off, works a little later
        gone: [passed, passed],
    }
    content_plan = {ok: [passed], flaky: [passed], gone: [("unavailable", "timeout"), ("unavailable", "timeout")]}
    run, stats, calls = _second_chance_run(tmp_path, monkeypatch, confirm_plan, content_plan)

    assert sorted(r.key for r in run.results) == [ok, flaky, gone]
    assert calls[("confirm", drop)] == 1 and calls[("confirm", flaky)] == 2 and calls[("content", gone)] == 2
    assert calls[("confirm", ok)] == 1  # nothing to repeat
    # every proxy is counted once per check, as its last try ended
    assert run.states["confirm"] == Counter({"passed": 3, "failed:exit_ip_mismatch": 1})
    assert run.states["content"] == Counter({"passed": 2, "unavailable:timeout": 1})
    assert run.states[pipeline.SECOND_CHANCE] == Counter({"passed": 2})
    assert stats.fakes == 1 and stats.found == 3
    assert sorted(run.checked) == sorted([ok, drop, flaky, gone])


def test_second_chance_failure_counts_once(tmp_path, monkeypatch):
    key = "http 5.5.5.5:80"
    plan = {key: [("failed", "connect_timeout"), ("failed", "timeout")]}
    run, stats, calls = _second_chance_run(tmp_path, monkeypatch, plan, {})
    assert run.results == [] and calls[("confirm", key)] == 2
    assert run.states["confirm"] == Counter({"failed:timeout": 1})
    assert run.states[pipeline.SECOND_CHANCE] == Counter({"failed": 1})
    assert stats.fakes == 1 and run.checked == [key]


def test_goal_reached_during_second_chance_counts_every_proxy_once(tmp_path, monkeypatch):
    """--want reached in one second chance while another is in its HTTPS test: the cut-off one must not end
    as working and unconfirmed at once."""
    import asyncio
    import contextlib

    from proxyscraper.checkstate import CONFIRM, CONTENT, Check
    from proxyscraper.geo import GeoResolver
    from proxyscraper.options import RunOptions
    from proxyscraper.ui import CheckDashboard, LiveStats

    monkeypatch.setattr(pipeline, "SECOND_CHANCE_DELAY", 0.0)
    keys = ["http 1.1.1.1:80", "http 2.2.2.2:80"]
    calls = Counter()

    class StubChecker:
        async def check(self, key):
            ptype, proxy = key.split(" ")
            return CheckResult(key, ptype, proxy, 100, "9.9.9.9")

        async def confirm(self, r):
            calls[r.key] += 1
            first = calls[r.key] == 1
            r.checks[CONFIRM] = Check("failed", "connect_timeout") if first else Check("passed", "")
            return not first

        async def tampers(self, r):
            r.checks[CONTENT] = Check("passed", "")
            return False

        async def enrich(self, r):
            await asyncio.sleep(1 if r.key == keys[1] else 0.1)
            r.https = True

    async def go():
        stats = LiveStats({"http": 2})
        writer = output.ResultWriter(run_dir=tmp_path / "run")
        dashboard = CheckDashboard(stats, writer.live_path, 2, True)
        run = await pipeline.run_checks(keys, StubChecker(), RunOptions(no_geo=True, want=1), dashboard, writer,
                                        GeoResolver(enabled=False), live_factory=lambda _: contextlib.nullcontext())
        return run, stats

    run, stats = asyncio.run(go())
    assert run.reached_goal
    assert stats.found + stats.fakes == 2
    assert sorted(run.checked) == keys
