"""Flow: assemble sources, load & parse in parallel, prioritize, check."""

from __future__ import annotations

import asyncio
import contextlib
import os
import random
from collections import Counter
from concurrent.futures import ProcessPoolExecutor
from dataclasses import dataclass, field
from pathlib import Path
from typing import AbstractSet, Callable, Dict, Iterable, List, Optional, Set, Tuple
from urllib.parse import urlsplit
from urllib.request import url2pathname

from . import sources as srcs
from .asndb import ProviderLookup
from .blocklist import Blocklist
from .checker import RETRY_REASONS, Checker, CheckResult, other_protocols
from .checkstate import BASIC, CONFIRM, CONTENT, HTTPS, PASSED, SKIPPED, Check, now_iso
from .compat import on_interrupt
from .fetchcache import FetchCache
from .geo import GeoResolver
from .history import ProxyHistory
from .judges import JudgeWatch, switch_label
from .netio import http_get, http_request
from .netpolicy import TargetPolicy
from .options import RunOptions
from .output import ResultWriter
from .parsing import PROXY_TYPES, make_key, parse_blob, split_key
from .ui import ACCENT, CheckDashboard, CollectView, LiveStats, fmt, widgets

# run.json `checks` key for the basic checks of another protocol (see run_checks: protocol fallback)
FALLBACK = "fallback"
# A proxy whose confirm or content request broke off in transport (RETRY_REASONS) right after a passed basic check
# gets its follow-ups once more at the end of the run, at least this many seconds later: free proxies drop
# connections for a while and come back. Before, ~350 confirms and ~280 content checks of every hourly run ended
# that way and the proxies were lost, although they had just answered the basic check.
SECOND_CHANCE_DELAY = 20.0
# run.json `checks` key: how the repeated follow-ups ended ("passed" = the proxy made it into the results)
SECOND_CHANCE = "second_chance"
# a proxy waiting for its second chance: (when its first try ended, its result, the job key, listed under that key)
Deferred = Tuple[float, CheckResult, str, bool]
AUTO_DISCOVER_AFTER_DAYS = 1.0  # daily: found sources are kept, so every search adds to the pile
# parse small lists directly – the detour through another process costs more than it saves
INLINE_PARSE_BYTES = 256 * 1024
DOWNLOAD_CONCURRENCY = 64
# Share of a limited run's slots (after the history) that goes to sources with too few checks to judge yet.
# The rest follows the learned hit rates; without it the order only ever confirms what it already knows.
EXPLORE_SHARE = 0.15
# A source with fewer (decayed) checks than this hasn't been sampled yet. Kept low on purpose: one round of
# samples lifts a source past it, so the share isn't spent on the same weak lists run after run (in a
# simulation, a bound of 31 – the weight of the prior – cost ~2 % of the hits for that reason).
EXPLORE_BELOW_CHECKED = 10


# --------------------------------------------------------------------------- #
# Sources
# --------------------------------------------------------------------------- #

@dataclass
class SourcePlan:
    sources: Dict[str, str]
    skipped: Counter
    n_curated: int
    n_meta: int
    meta_ok: int
    meta_total: int
    n_discovered: int
    n_own: int = 0
    discovery_ran: bool = False
    discovery_token: bool = False
    # found by discovery or listed in a third-party meta list: their names are someone else's to point anywhere
    untrusted: Set[str] = field(default_factory=set)


async def collect_sources(opts: RunOptions, quality: srcs.SourceStats) -> SourcePlan:
    """Curated + meta + discovered sources, minus the ones learned to be bad, plus the own ones (--source)."""
    own = dict(srcs.parse_source_spec(spec) for spec in opts.sources)
    wanted = set(opts.types)
    own = {u: t for u, t in own.items() if t == "auto" or t in wanted}
    if opts.only_sources:
        return SourcePlan(own, Counter(), 0, 0, 0, 0, 0, n_own=len(own))
    sources, meta = srcs.load_source_file()
    n_curated = len(sources)
    curated = set(sources)

    age = srcs.discovered_age_days()
    due = not opts.no_discover and (age is None or age > AUTO_DISCOVER_AFTER_DAYS)
    # asking gh for a token takes up to 5 s – only when discovery can actually run, and off the event loop
    token = await asyncio.to_thread(srcs.github_token) if (opts.discover or due) else None
    run_discovery = opts.discover or (due and token is not None)
    if run_discovery:
        max_repos = opts.discover_repos if token else min(opts.discover_repos, 40)
        with widgets.console.status(f"[bold {ACCENT}]Searching GitHub for new proxy lists …", spinner="dots") as status:
            found = await srcs.discover_github(
                http_get, token, max_repos,
                on_progress=lambda msg: status.update(f"[bold {ACCENT}]GitHub-Discovery:[/] {msg}"),
                known=sources,  # curated ones needn't be tried; found ones are, so they stay fresh
            )
        if found:
            srcs.save_discovered(found)
    discovered = srcs.load_discovered()

    with widgets.console.status(f"[bold {ACCENT}]Loading meta sources …", spinner="dots"):
        meta_found, meta_ok = await srcs.resolve_meta(meta, http_get)
    for extra in (meta_found, discovered):
        for url, ptype in extra.items():
            sources.setdefault(url, ptype)

    sources = {u: t for u, t in sources.items() if t == "auto" or t in wanted}
    skipped: Counter = Counter()
    if not opts.all_sources:
        active = {}
        for url, ptype in sources.items():
            reason = quality.skip_now(url)
            if reason:
                skipped[reason] += 1
            else:
                active[url] = ptype
        sources = active
    sources.update(own)  # asked for by name – never skipped, and the given type wins
    untrusted = (set(meta_found) | set(discovered)) & set(sources) - curated - set(own)
    return SourcePlan(
        sources, skipped, n_curated, len(meta_found), meta_ok, len(meta), len(discovered), n_own=len(own),
        discovery_ran=run_discovery, discovery_token=token is not None, untrusted=untrusted,
    )


@dataclass
class ScrapeResult:
    urls: List[str]
    # proxy key -> indices of the sources that list it (for priority and statistics)
    index: Dict[str, List[int]] = field(default_factory=dict)
    ok_sources: int = 0


# Public addresses on every redirect hop, any port: a discovered list's host may resolve to this machine or
# the local network by now, and that must not turn a scan into requests against internal services.
UNTRUSTED_SOURCE_POLICY = TargetPolicy(ports=None)


async def scrape(sources: Dict[str, str], types, quality: srcs.SourceStats, view: CollectView,
                 cache: Optional[FetchCache] = None, untrusted: AbstractSet[str] = frozenset()) -> ScrapeResult:
    """Loads every source and parses large lists in parallel on all CPU cores.

    That keeps the event loop free for more downloads instead of working through regexes for minutes.
    With the cache, unchanged lists are skipped via ETag (see fetchcache.py).
    """
    res = ScrapeResult(list(sources))
    cache = cache or FetchCache(enabled=False)
    # always parse every type, filtering only happens when sorting them in: the cache stores all types, and
    # the statistics must be able to tell "no proxies at all" from "none of the requested types"
    parse_types = PROXY_TYPES
    prefixes = tuple(f"{t} " for t in types)
    loop = asyncio.get_running_loop()
    sem = asyncio.Semaphore(DOWNLOAD_CONCURRENCY)
    index = res.index

    with ProcessPoolExecutor(max_workers=max(1, (os.cpu_count() or 2) - 1)) as pool:

        async def download(url: str):
            """-> (data, headers, keys from the cache). Exactly one of data and keys is set."""
            if url.startswith("file://"):  # own list from --source
                return await asyncio.to_thread(Path(url2pathname(urlsplit(url).path)).read_bytes), {}, None
            conditional = cache.conditional_headers(url, sources[url])
            policy = UNTRUSTED_SOURCE_POLICY if url in untrusted else None
            async with sem:
                status, headers, body = await http_request(url, timeout=30, headers=conditional or None,
                                                           policy=policy)
            if status == 304 and conditional:
                cached = cache.load(url)
                if cached is not None:
                    return None, headers, cached
                async with sem:  # cache file broken -> just load it again
                    status, headers, body = await http_request(url, timeout=30, policy=policy)
            if status != 200:
                raise ConnectionError(f"HTTP {status}")
            return body, headers, None

        async def fetch(i: int, url: str) -> None:
            data: Optional[bytes] = None
            keys = ""
            unchanged = False
            try:
                data, headers, cached = await download(url)
                if cached is not None:
                    keys, unchanged = cached, True
                    view.cached += 1
                else:
                    view.bytes += len(data)
                    if len(data) <= INLINE_PARSE_BYTES:
                        keys = parse_blob(data, sources[url], parse_types)
                    else:
                        keys = await loop.run_in_executor(pool, parse_blob, data, sources[url], parse_types)
                    if not url.startswith("file://"):
                        cache.store(url, headers, keys, sources[url])
            except Exception:  # source unreachable/broken – counts as a failure in the statistics
                pass
            n = parsed = 0
            if keys:
                parsed = keys.count("\n") + 1  # every type, also the ones this run doesn't want
                for key in keys.split("\n"):
                    if not key.startswith(prefixes):
                        continue
                    owners = index.get(key)
                    if owners is None:
                        index[key] = [i]
                    else:
                        owners.append(i)
                    n += 1
            quality.record_fetch(url, data, n, unchanged=unchanged, parsed=parsed)
            if n:
                res.ok_sources += 1
                view.ok += 1
            else:
                view.failed += 1
            view.unique = len(index)
            view.advance()

        await asyncio.gather(*(fetch(i, u) for i, u in enumerate(res.urls)))
    return res


def prioritize(res: ScrapeResult, quality: srcs.SourceStats, history: ProxyHistory, types,
               listed_only: bool = False, limit: int = 0, explore: float = EXPLORE_SHARE,
               rng: Optional[random.Random] = None) -> List[str]:
    """Order of checking: known working proxies first, then by quality of the sources.

    Within the same source quality, proxies that appear in more lists win. listed_only (--only-sources):
    the history only moves proxies of these lists to the front, it doesn't add others.

    limit (the run checks only the first `limit`, 0 = all): a share `explore` of the slots after the history
    goes to under-explored sources (see explore_sample) – without it, a source whose proxies never make it
    under the limit is never checked and keeps its prior score forever. The full order is returned either way.
    """
    rng = rng or random
    wanted = set(types)
    known = [k for k in history.ranked_keys() if split_key(k)[0] in wanted and (not listed_only or k in res.index)]
    known_set = set(known)

    scores = [quality.score(u) for u in res.urls]
    keys = [k for k in res.index if k not in known_set]
    rng.shuffle(keys)  # shuffle real ties so every type makes progress
    index = res.index

    def priority(key: str) -> float:
        owners = index[key]
        if len(owners) == 1:
            return scores[owners[0]]
        return max(map(scores.__getitem__, owners)) + 0.002 * min(len(owners), 10)

    prio = list(map(priority, keys))
    order = sorted(range(len(keys)), key=prio.__getitem__, reverse=True)
    ranked = [keys[i] for i in order]

    budget = limit - len(known)
    n_explore = int(budget * explore) if limit and budget > 0 and explore > 0 else 0
    if not n_explore or len(ranked) <= budget:
        return known + ranked  # everything fits anyway, or no exploration asked for
    head = ranked[: budget - n_explore]
    explored = explore_sample(res, quality, ranked[len(head):], n_explore, rng)
    if not explored:
        return known + ranked
    taken = set(explored)
    return known + head + explored + [k for k in ranked[len(head):] if k not in taken]


def explore_sample(res: ScrapeResult, quality: srcs.SourceStats, pool: List[str], n: int,
                   rng) -> List[str]:
    """Up to n keys from `pool`, spread round-robin over the under-explored sources that list them.

    Under-explored: fewer (decayed) checks than EXPLORE_BELOW_CHECKED – its learned hit rate is still
    about the prior. Each round takes one random key from every such source, in a random source order, so every
    one of them gets a sample when the budget allows and a different part of the long tail otherwise.
    """
    under = [quality.get(u).checked < EXPLORE_BELOW_CHECKED for u in res.urls]
    if not any(under):
        return []
    per_source: Dict[int, List[str]] = {}
    index = res.index
    for key in pool:
        for i in index[key]:
            if under[i]:
                per_source.setdefault(i, []).append(key)
    picked: List[str] = []
    seen: Set[str] = set()
    active = list(per_source)
    while active and len(picked) < n:
        rng.shuffle(active)
        still = []
        for i in active:
            bucket = per_source[i]
            while bucket:  # random draw without replacement; a key another source already gave is skipped
                j = rng.randrange(len(bucket))
                bucket[j], bucket[-1] = bucket[-1], bucket[j]
                key = bucket.pop()
                if key not in seen:
                    seen.add(key)
                    picked.append(key)
                    break
            if bucket:
                still.append(i)
            if len(picked) >= n:
                break
        active = still
    return picked


def attribute_results(res: ScrapeResult, checked: List[str], working: Set[str]) -> Dict[str, Tuple[int, int]]:
    """Attribute results to the sources -> url -> (checked, working)."""
    n_checked = [0] * len(res.urls)
    n_working = [0] * len(res.urls)
    for key in checked:
        ok = key in working
        for i in res.index.get(key, ()):
            n_checked[i] += 1
            if ok:
                n_working[i] += 1
    return {u: (n_checked[i], n_working[i]) for i, u in enumerate(res.urls) if n_checked[i]}


def best_sources(per_source: Dict[str, Tuple[int, int]], limit: int = 10, min_checked: int = 30):
    return sorted(
        ((w / c, w, c, u) for u, (c, w) in per_source.items() if c >= min_checked and w),
        reverse=True,
    )[:limit]


# --------------------------------------------------------------------------- #
# Checking
# --------------------------------------------------------------------------- #

@dataclass
class CheckRun:
    results: List[CheckResult] = field(default_factory=list)
    checked: List[str] = field(default_factory=list)
    working: Set[str] = field(default_factory=set)
    interrupted: bool = False
    reached_goal: bool = False
    judge_switches: List[str] = field(default_factory=list)  # "old → new"
    rechecked: int = 0  # checks that were repeated because a check target went down
    unavailable: int = 0  # basic checks that couldn't run on this machine (out of sockets): not in `checked`
    changed_pages: Counter = field(default_factory=Counter)  # content mismatches: with / without a <script tag
    # check name -> "passed" / "failed:reason" / "unavailable:reason" / "skipped:reason" -> how many proxies
    states: Dict[str, Counter] = field(default_factory=dict)
    # basic checks of another protocol on an address whose listed one answered wrongly, and how many passed them;
    # how they ended is in states[FALLBACK], not in the basic checks
    fallback_tried: int = 0
    fallback_found: int = 0

    def count(self, r: CheckResult, name: str) -> None:
        check = r.checks.get(name)
        if check is not None:
            self.states.setdefault(name, Counter())[check.label()] += 1


def check_counts(run: CheckRun, stats: LiveStats, failures: Counter) -> Dict[str, Dict[str, int]]:
    """How each check of a run ended (run.json's `checks`). `failures` are the checker's basic checks that didn't
    pass, fallback checks included: those are counted under FALLBACK only, so `basic` is one per listed entry
    (plus repeats after a check target switch)."""
    fallback = run.states.get(FALLBACK, Counter())
    basic_failures = Counter(failures) - Counter({k: v for k, v in fallback.items() if k != PASSED})
    basic = {PASSED: stats.found + stats.fakes + stats.tampered - run.fallback_found}
    basic.update(sorted(basic_failures.items()))
    return {BASIC: basic, **{name: dict(sorted(c.items())) for name, c in run.states.items()}}


class JobQueue:
    """Iterator over the jobs that still accepts stragglers after running empty.

    A generator would be exhausted forever after the first StopIteration – a proxy that a still
    running worker puts back after a check-target switch would then never be checked again."""

    def __init__(self, jobs: Iterable[str]):
        self.jobs = iter(jobs)
        self.retry: List[str] = []

    def __iter__(self):
        return self

    def __next__(self) -> str:
        for key in self.jobs:
            return key
        if self.retry:
            return self.retry.pop()
        raise StopIteration


async def run_checks(
    jobs: List[str],
    checker: Checker,
    opts: RunOptions,
    dashboard: CheckDashboard,
    writer: ResultWriter,
    geo: GeoResolver,
    live_factory: Callable,
    watch: Optional[JudgeWatch] = None,
    providers: Optional[ProviderLookup] = None,
    blocklist: Optional[Blocklist] = None,
    quiet: bool = False,
) -> CheckRun:
    """Checks `jobs` with `opts.concurrency` parallel workers until everything is done, the goal is
    reached or Ctrl+C is pressed.

    With `watch` the check target is monitored: if it goes down, the checks since the last successful
    probe are repeated and don't count for the source statistics.

    quiet: runs in the background next to another live view (--serve-refill) – no spinner, and Ctrl+C
    belongs to whoever runs in the foreground."""
    loop = asyncio.get_running_loop()
    stats = dashboard.s
    filters, details, want = opts.filters, opts.details, opts.want
    run = CheckRun()
    written: Set[str] = set()
    enriched: Set[str] = set()  # hits with a finished detail check (HTTPS may stay open)
    by_exit_ip: Dict[str, List[CheckResult]] = {}
    pending = JobQueue(jobs)  # all workers pull from the same queue – safe in asyncio without a lock

    # Check target outages: every check remembers which target ("generation") it started with and its number.
    # On a switch the old generation becomes suspicious from the last good probe on – failures from it
    # are repeated, including those that only finish after the switch. Hits are never suspicious
    # (the proxy did work), so every check is counted exactly once.
    generation = 0
    # Order instead of clock time: every check gets a running number when it starts. The event loop clock
    # is only accurate to ~15 ms on Windows – two events could easily get the same timestamp there.
    started_count = 0
    last_ok = 0  # this many checks had started at the last good probe (0 = start of the run)
    suspect_since: Dict[int, int] = {}       # replaced generation -> checks from this number on are suspicious
    recent_failures: List[Tuple[int, int, str]] = []  # failures since the last good probe

    def is_suspect(gen: int, started: int) -> bool:
        return gen in suspect_since and started > suspect_since[gen]

    def requeue(keys: List[str]) -> None:
        pending.retry.extend(keys)
        run.rechecked += len(keys)
        dashboard.add_rechecks(keys)

    def judge_ok() -> None:
        nonlocal last_ok
        last_ok = started_count
        recent_failures.clear()

    def judge_switched(old, new) -> None:
        nonlocal generation, last_ok
        suspect_since[generation] = last_ok
        generation += 1
        last_ok = started_count  # the new target was just probed successfully – the baseline from here on
        failed = {key for gen, started, key in recent_failures if is_suspect(gen, started)}
        recent_failures.clear()
        if failed:
            run.checked[:] = [k for k in run.checked if k not in failed]
        requeue(sorted(failed))
        run.judge_switches.append(switch_label(old, new))
        dashboard.judge_changed(new.judge.host)

    if watch:
        watch.on_ok, watch.on_switch = judge_ok, judge_switched
    all_workers: Optional[asyncio.Future] = None
    no_verdict: Set[str] = getattr(checker, "no_verdict", set())  # keys whose basic check couldn't run at all
    # Protocol fallback: an address whose listed protocol got an answer of another kind is tried as the other
    # protocols (checker.other_protocols) – unless that key is a candidate of this run anyway or was tried already.
    # Only for checkers that record why a basic check failed.
    mismatch: Optional[Dict[str, str]] = None
    if hasattr(checker, "protocol_mismatch"):
        mismatch = checker.protocol_mismatch = {}
    candidates: Set[str] = set(jobs)
    wanted_types = set(opts.types)

    async def basic_check(key: str) -> Optional[CheckResult]:
        """The basic check of `key`; if it failed in a way another protocol could explain, the first other
        protocol that passes on the same address (its result carries its own key)."""
        r = await checker.check(key)
        if r is not None or mismatch is None:
            return r
        reason = mismatch.pop(key, None)
        if reason is None:
            return None
        ptype, proxy = split_key(key)
        for other in other_protocols(ptype, reason):
            alt = make_key(other, proxy)
            if other not in wanted_types or alt in candidates:
                continue
            candidates.add(alt)
            run.fallback_tried += 1
            found, check = await checker.check_detailed(alt)
            run.states.setdefault(FALLBACK, Counter())[check.label()] += 1
            mismatch.pop(alt, None)
            no_verdict.discard(alt)
            if found is not None:
                run.fallback_found += 1
                return found
        return None

    def consider(r: CheckResult) -> None:
        """Live file & goal counter, as soon as all the info the filters need is there."""
        if r.key in written or (details and r.key not in enriched):
            return
        if filters.countries and not r.country:
            return  # country still coming – on_country calls again
        if filters.accepts(r):
            written.add(r.key)
            stats.passing += 1
            writer.add_live(r)
            if want and stats.passing >= want and all_workers and not all_workers.done():
                run.reached_goal = True
                all_workers.cancel()

    def on_country(ip: str, cc: str) -> None:
        for r in by_exit_ip.get(ip, ()):
            if not r.country:
                r.country = cc
                stats.countries[cc] += 1
                consider(r)

    geo.on_resolved = on_country
    geo_task = asyncio.ensure_future(geo.run())
    watch_task = asyncio.ensure_future(watch.run()) if watch else None

    deferred: List[Deferred] = []  # proxies waiting for their second chance

    def broke_off(r: CheckResult, name: str) -> bool:
        check = r.checks.get(name)
        return check is not None and check.state != PASSED and check.reason in RETRY_REASONS

    async def follow_up(r: CheckResult, key: str, listed: bool, last_try: bool,
                        item: Optional[Deferred] = None) -> bool:
        """Confirm, content and details of a proxy that passed its basic check. False: a follow-up request broke
        off in transport and this wasn't the last try – nothing is recorded, the proxy waits for second_chance.
        item: its entry in `deferred` (on the second chance), taken out as soon as the verdict is recorded – a
        stop after that must not count the proxy again."""

        def recorded() -> None:
            if item is not None:
                deferred.remove(item)

        # second, independent request – a proxy that only answers the check request passes the first by chance.
        # False only when the request ran and didn't confirm it; an unreachable confirmation target leaves
        # the proxy in, marked "unavailable" (r.checks) – it's the target that failed, not the proxy
        confirmed = await checker.confirm(r)
        if not confirmed and not last_try and broke_off(r, CONFIRM):
            return False
        if not confirmed:
            recorded()
            run.count(r, CONFIRM)
            if listed:
                run.checked.append(key)
            stats.fakes += 1
            return True
        # third request: does a known page arrive unchanged?
        modified = await checker.tampers(r)
        if not modified and not last_try and broke_off(r, CONTENT):
            return False
        recorded()
        run.count(r, CONFIRM)
        run.count(r, CONTENT)
        if modified:
            if listed:
                run.checked.append(key)
            stats.tampered += 1
            # the proxy is dropped here, so what was measured about its page is kept as a count
            changed = r.checks[CONTENT].data if CONTENT in r.checks else {}
            run.changed_pages["with_script_tag" if changed.get("script_tags") else "without_script_tag"] += 1
            return True
        if CONFIRM in r.checks and not r.checks[CONFIRM].ok:
            stats.confirm_unavailable += 1
        if providers:
            providers.annotate(r)
        if blocklist:
            r.blocklisted = await blocklist.lookup(r.exit_ip)
        if listed:
            run.checked.append(key)
        run.results.append(r)
        run.working.add(r.key)
        by_exit_ip.setdefault(r.exit_ip, []).append(r)
        r.country = geo.request(r.exit_ip)
        if r.country:
            stats.countries[r.country] += 1
        stats.add_working(r)
        if details:
            # HTTPS test only if the proxy can still pass the filters at all
            if filters.may_pass(r):
                await checker.enrich(r)
                enriched.add(r.key)
                stats.add_details(r)
            else:
                stats.details_saved += 1
                r.checks[HTTPS] = Check(SKIPPED, "filtered_out", at=now_iso())
        else:
            r.checks[HTTPS] = Check(SKIPPED, "fast_mode", at=now_iso())
        run.count(r, HTTPS)
        consider(r)
        return True

    async def second_chance() -> None:
        """Both follow-ups once more for the deferred proxies, each at least SECOND_CHANCE_DELAY after its first
        try. This time they count whatever they end as."""
        gate = asyncio.Semaphore(max(1, opts.concurrency))

        async def one(item: Deferred) -> None:
            at, r, key, listed = item
            await asyncio.sleep(max(0.0, at + SECOND_CHANCE_DELAY - loop.time()))
            async with gate:
                await follow_up(r, key, listed, last_try=True, item=item)
            outcome = PASSED if r.key in run.working else "failed"
            run.states.setdefault(SECOND_CHANCE, Counter())[outcome] += 1

        tasks = [asyncio.ensure_future(one(item)) for item in list(deferred)]
        try:
            await asyncio.gather(*tasks)
        except asyncio.CancelledError:
            for task in tasks:
                task.cancel()
            raise

    async def worker() -> None:
        nonlocal started_count
        for key in pending:
            started_count += 1
            gen, started = generation, started_count
            r = await basic_check(key)
            listed_type = key.split(" ", 1)[0]
            # a proxy found under another protocol counts for the protocol it was found as (checked and working)
            stats.add_checked(r.ptype if r is not None else listed_type, listed=listed_type)
            dashboard.advance()
            if r is None and key in no_verdict:
                no_verdict.discard(key)
                run.unavailable += 1  # our own shortage (sockets): no verdict, so nothing to learn from it
                continue
            if r is None and is_suspect(gen, started):
                requeue([key])  # only finished after the switch – still a victim of the outage
                continue
            if r is None:
                run.checked.append(key)
                if watch:
                    recent_failures.append((gen, started, key))
                continue
            listed = r.key == key
            if not listed:
                # found under another protocol: the listed one doesn't work. The job's key counts as checked and
                # failed (that is what the source listed); the other key is learned from the results like any hit
                run.checked.append(key)
            # A hit only counts as checked once the verdict is in: a worker cancelled during the next requests
            # (--want reached, Ctrl+C) must not record a proxy that just worked as a failure.
            if not await follow_up(r, key, listed, last_try=False):
                deferred.append((loop.time(), r, key, listed))

    with live_factory(dashboard):
        workers = [asyncio.ensure_future(worker()) for _ in range(min(opts.concurrency, len(jobs)))]
        all_workers = asyncio.gather(*workers)
        # Ctrl+C stops cleanly so the results still get saved (on Windows too)
        with contextlib.nullcontext() if quiet else on_interrupt(loop, lambda: all_workers.cancel()):
            try:
                await all_workers
                # the second chances take 20 s and more: a check target switch in that time puts basic checks
                # back into the queue (requeue), and nobody would take them out – so: workers again until both
                # are empty
                while deferred or pending.retry:
                    if pending.retry:
                        all_workers = asyncio.gather(*(asyncio.ensure_future(worker())
                                                       for _ in range(min(opts.concurrency, len(pending.retry)))))
                    else:
                        all_workers = asyncio.ensure_future(second_chance())
                    await all_workers
            except asyncio.CancelledError:
                run.interrupted = not run.reached_goal
        # stopped before their second chance was recorded: they did fail a follow-up, so they end as unconfirmed –
        # the basic check counts (check_counts) stay complete
        for *_, key, listed in deferred:
            if listed:
                run.checked.append(key)
            stats.fakes += 1

    if watch_task:
        watch_task.cancel()
    if quiet and run.interrupted:
        # nobody pressed Ctrl+C here – the task running us was cancelled (the server stops), so pass it on
        geo_task.cancel()
        raise asyncio.CancelledError
    # wait for open country lookups (briefly at most)
    geo.stop()
    # `outstanding` includes a batch the worker already took off the queue – its request may still be running
    if geo.outstanding and not geo.failed:
        message = f"[bold {ACCENT}]Looking up countries for {fmt(geo.outstanding)} exit IPs …"
        spinner = contextlib.nullcontext() if quiet else widgets.console.status(message, spinner="dots")
        with spinner, contextlib.suppress(asyncio.TimeoutError):
            await asyncio.wait_for(asyncio.shield(geo_task), 30)
    geo_task.cancel()
    return run
