name: proxy list

on:
  schedule:
    - cron: "17 * * * *"     # every hour – the checks take about 5–6 minutes
  workflow_dispatch:

permissions:
  contents: read

concurrency:
  group: proxy-list
  cancel-in-progress: false

jobs:
  update:
    runs-on: ubuntu-latest
    timeout-minutes: 45
    permissions:
      contents: write        # to push to the proxy-list branch and upload snapshots to the release
    env:
      COLUMNS: "110"
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
      - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7
        with:
          python-version: "3.13"
          cache: pip
      - run: pip install --require-hashes -r .github/requirements/run.txt

      # learned source statistics and history from the last run – the tool keeps learning here too
      - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
        with:
          path: data
          key: proxy-data-${{ github.run_id }}
          restore-keys: proxy-data-

      - name: Collect and check proxies
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}   # for the GitHub discovery
        run: python proxy_scraper.py -y --limit 150000 --timeout 10 --connect-timeout 4

      # Google, Reddit, Amazon, Instagram, TikTok and Discord turn many free proxies away – note which got through (~2 min).
      # Optional: if it fails, the list is published without it.
      - name: Check which big sites let them through
        continue-on-error: true
        timeout-minutes: 8
        run: python -m proxyscraper.sites "results/$(cat results/latest.txt)"

      # 100 KB through every HTTPS-capable proxy, 12 s at most each (about a minute). Optional like the sites.
      - name: Measure the download speed
        continue-on-error: true
        timeout-minutes: 5
        run: python -m proxyscraper.speed "results/$(cat results/latest.txt)"

      - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
        if: always()
        with:
          path: data
          key: proxy-data-${{ github.run_id }}

      - name: Prepare the list
        id: prepare
        run: |
          # history of the last runs (chart), how long each proxy has been listed (stable proxies)
          # and which runs it was listed in (uptime).
          # 404 means: doesn't exist yet (first run). Any other error: better not publish this time
          # than lose the history.
          for f in history.json streaks.json seen.json report/feed.xml; do
            code=$(curl -sL --create-dirs --retry 3 --retry-all-errors -o "$f" -w "%{http_code}" \
              "https://raw.githubusercontent.com/${GITHUB_REPOSITORY}/proxy-list/$f" || echo 000)
            if [ "$code" = "404" ]; then rm -f "$f"; elif [ "$code" != "200" ]; then
              echo "::warning::$f could not be loaded (HTTP $code) – the old list stays online"
              echo "publish=false" >> "$GITHUB_OUTPUT"; exit 0
            fi
          done
          if python -m proxyscraper.publish "results/$(cat results/latest.txt)" public --min 20 \
              --history history.json --streaks streaks.json --seen seen.json --feed report/feed.xml \
              --sources data/source_stats.json; then
            echo "publish=true" >> "$GITHUB_OUTPUT"
          else
            rc=$?
            [ "$rc" -eq 78 ] || exit "$rc"   # 78 = too few hits, the old list stays
            echo "publish=false" >> "$GITHUB_OUTPUT"
          fi

      - name: Publish to the proxy-list branch
        if: steps.prepare.outputs.publish == 'true'
        working-directory: public
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          git init -q -b proxy-list
          git config user.name "github-actions[bot]"
          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
          git add -A
          git commit -q -m "Proxy list of $(date -u +'%Y-%m-%d %H:%M UTC')"
          # a single commit that gets replaced every time – the history doesn't grow
          git push -q -f "https://x-access-token:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" proxy-list

      # The server is reached through the own Headscale VPN, not public SSH. TS_AUTHKEY: reusable, ephemeral
      # pre-auth key with tag:ci, which may only reach port 22 on tag:server. Optional like the upload below:
      # if the VPN is down, the run goes on (mirror, snapshot) and the error shows up as an annotation.
      - name: Connect to the VPN
        id: vpn
        if: steps.prepare.outputs.publish == 'true'
        continue-on-error: true
        timeout-minutes: 5
        uses: tailscale/github-action@064b10558093f38325e838ddea35486bdc6b1d5c # v4.4.0
        with:
          authkey: ${{ secrets.TS_AUTHKEY }}
          args: --login-server=https://vpn.aquaris.dev --accept-dns=false
          hostname: gh-proxy-scraper-${{ github.run_id }}
          timeout: 1m
          retry: 3
          ping: 100.64.0.1

      # The website itself lives on the own server (proxy-scraper.de, Caddy). The key may only rsync into the
      # site directory (rrsync on the server); a pinned host key, like the bot deploy. Without the secrets this
      # does nothing, and a failed upload leaves the last site online – the branch above is already published.
      - name: Upload the website to the server
        if: steps.prepare.outputs.publish == 'true' && steps.vpn.outcome == 'success'
        continue-on-error: true
        env:
          SITE_HOST: 100.64.0.1   # aquaris in the tailnet
          SITE_SSH_KEY: ${{ secrets.SITE_SSH_KEY }}
          SITE_KNOWN_HOSTS: ${{ secrets.SITE_KNOWN_HOSTS }}
        run: |
          if [ -z "$SITE_HOST" ] || [ -z "$SITE_SSH_KEY" ] || [ -z "$SITE_KNOWN_HOSTS" ]; then
            echo "SITE_HOST, SITE_SSH_KEY or SITE_KNOWN_HOSTS isn't set – skipped"; exit 0
          fi
          install -m 700 -d ~/.ssh
          printf '%s\n' "$SITE_SSH_KEY" > ~/.ssh/site && chmod 600 ~/.ssh/site
          printf '%s\n' "$SITE_KNOWN_HOSTS" > ~/.ssh/site_known_hosts
          echo "$SITE_HOST ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIF47uy20XdCvmXD/IQ652p473NilBsfvrnZ0lztxb/Az" >> ~/.ssh/site_known_hosts
          # --delay-updates: the new files replace the old ones together at the end, so visitors see no half-updated list.
          # No -t: every file here was just written, so -t would stamp the new time on unchanged files too and give
          # each of them a new ETag/Last-Modified every hour; --checksum decides what changed, unchanged files keep theirs.
          rsync -rl --checksum --delete-after --delay-updates --exclude .git \
            -e "ssh -i $HOME/.ssh/site -o IdentitiesOnly=yes -o BatchMode=yes -o UserKnownHostsFile=$HOME/.ssh/site_known_hosts" \
            public/ "proxysite@${SITE_HOST}:/"
          echo "uploaded $(find public -path public/.git -prune -o -type f -print | wc -l) files to the server"

      # The lists alone in a repository of their own – easier to find and to bookmark than a branch (mirror.py).
      # One commit per run, squashed at the start of every month so clones stay small.
      # Needs the secret LIST_REPO_KEY, a deploy key with write access there; without it this does nothing.
      - name: Mirror to the list repository
        if: steps.prepare.outputs.publish == 'true'
        continue-on-error: true
        env:
          LIST_REPO: maximilianfeix/free-proxy-list
          LIST_REPO_KEY: ${{ secrets.LIST_REPO_KEY }}
        run: |
          if [ -z "$LIST_REPO_KEY" ]; then echo "LIST_REPO_KEY isn't set – skipped"; exit 0; fi
          install -m 700 -d ~/.ssh
          printf '%s\n' "$LIST_REPO_KEY" > ~/.ssh/list_repo && chmod 600 ~/.ssh/list_repo
          # GitHub's published host keys (docs.github.com, "GitHub's SSH key fingerprints") – not trusted on first use
          cat > ~/.ssh/github_known_hosts <<'KEYS'
          github.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl
          github.com ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEmKSENjQEezOmxkZMy7opKgwFB9nkt5YRrYMjNuG5N87uRgg6CLrbo5wAdT/y6v0mKV0U2w0WZ2YB/++Tpockg=
          github.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7ndNxQowgcQnjshcLrqPEiiphnt+VTTvDP6mHBL9j1aNUkY4Ue1gvwnGLVlOhGeYrnZaMgRK6+PKCUXaDbC7qtbW8gIkhL7aGCsOr/C56SJMy/BCZfxd1nWzAOxSDPgVsmerOBYfNqltV9/hWCqBywINIR+5dIg6JTJ72pcEpEjcYgXkE2YEFXV1JHnsKgbLWNlhScqb2UmyRkQyytRLtL+38TGxkxCflmO+5Z8CSSNY7GidjMIZ7Q4zMjA2n1nGrlTDkzwDCsw+wqFPGQA179cnfGWOWRVruj16z6XyvxvjJwbz0wQZ75XK5tKSb7FNyeIEs4TT4jk+S4dhPeAUC5y+bDYirYgM4GC7uEnztnZyaVWQ7B381AK4Qdrwt51ZqExKbQpTUNn+EjqoTwvqNj4kqx5QUCI0ThS/YkOxJCXmPUWZbhjpCg56i+2aB6CmK2JGhn57K5mj0MNdBXA4/WnwH6XoPWJzK5Nyu2zB3nAZp+S5hpQs+p1vN1/wsjk=
          KEYS
          export GIT_SSH_COMMAND="ssh -i $HOME/.ssh/list_repo -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile=$HOME/.ssh/github_known_hosts"
          # without file contents: only the commits are needed to see when the history started
          git clone -q --filter=blob:none "git@github.com:${LIST_REPO}.git" list-repo
          message=$(python -m proxyscraper.mirror public list-repo --repo "$LIST_REPO")
          cd list-repo
          git config user.name "Maxi"
          git config user.email "223365242+maximilianfeix@users.noreply.github.com"
          git add -A
          if git diff --cached --quiet; then echo "nothing changed"; exit 0; fi
          started=$(git log --reverse --format=%cd --date=format:%Y-%m | head -n 1)
          if [ -n "$started" ] && [ "$started" != "$(date -u +%Y-%m)" ]; then
            git checkout -q --orphan fresh
            git commit -q -m "$message"
            git branch -M main
            git push -q -f origin main
            echo "new month – history squashed"
          else
            git commit -q -m "$message"
            git push -q origin main
          fi
          echo "pushed: $message"

      # Once a day the list is kept for good: a gzipped proxies.json as an asset of that year's snapshot release
      # (a release holds at most 1,000 files), for anyone who wants to look at free proxies over time.
      - name: Daily snapshot
        id: snapshot
        if: steps.prepare.outputs.publish == 'true'
        continue-on-error: true
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          name="proxies-$(date -u +%F).json.gz"
          release="snapshots-$(date -u +%Y)"
          if ! gh release view "$release" > /dev/null 2>&1; then
            gh release create "$release" --prerelease --target "$GITHUB_SHA" --title "Daily snapshots $(date -u +%Y)" \
              --notes "One file a day: every proxy that passed all checks in the first run of that day (UTC), with all details – the same fields as proxies.json on the proxy-list branch. For looking at free proxies over time. Updated automatically."
          fi
          if gh release view "$release" --json assets --jq '.assets[].name' | grep -qx "$name"; then
            echo "$name is already there"
          else
            gzip -9 -c public/proxies.json > "$name"
            gh release upload "$release" "$name"
            echo "uploaded $name ($(wc -c < "$name") bytes)"
          fi

      # The owner creates the dataset once, then sets HF_DATASET_REPO as a repository variable and a
      # fine-grained write token as the HF_TOKEN secret. Parquet lets the Hub expose the snapshots directly.
      - name: Mirror the daily snapshot to Hugging Face
        if: steps.prepare.outputs.publish == 'true' && steps.snapshot.outcome == 'success'
        continue-on-error: true
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          HF_DATASET_REPO: ${{ vars.HF_DATASET_REPO }}
          HF_TOKEN: ${{ secrets.HF_TOKEN }}
        run: |
          if [ -z "$HF_DATASET_REPO" ] || [ -z "$HF_TOKEN" ]; then
            echo "::notice::Set HF_DATASET_REPO and HF_TOKEN to enable the Hugging Face snapshot mirror"
            exit 0
          fi

          day="$(date -u +%F)"
          year="${day%%-*}"
          release="snapshots-$year"
          snapshot="proxies-$day.json.gz"
          parquet="proxies-$day.parquet"
          target="data/$year/$parquet"

          python -m pip install --quiet --require-hashes -r .github/requirements/hf.txt
          # the dataset card is kept in the repo; an unchanged card makes no commit
          hf upload "$HF_DATASET_REPO" docs/huggingface/README.md README.md --repo-type dataset \
              --commit-message "Update dataset card" || echo "::warning::Could not update the dataset card"
          if python - "$target" <<'PY'
          import os
          import sys

          from huggingface_hub import HfApi

          exists = HfApi(token=os.environ["HF_TOKEN"]).file_exists(
              repo_id=os.environ["HF_DATASET_REPO"],
              filename=sys.argv[1],
              repo_type="dataset",
          )
          raise SystemExit(0 if exists else 1)
          PY
          then
            echo "$target is already on Hugging Face"
            exit 0
          fi

          gh release download "$release" --pattern "$snapshot" --output "$RUNNER_TEMP/$snapshot"
          # one fixed schema for every day, so the Hub can read all of them as one table
          python .github/scripts/hf_parquet.py "$RUNNER_TEMP/$snapshot" "$RUNNER_TEMP/$parquet" "$day"

          for attempt in 1 2 3; do
            if hf upload "$HF_DATASET_REPO" "$RUNNER_TEMP/$parquet" "$target" --repo-type dataset \
                --commit-message "Add snapshot $day"; then
              exit 0
            fi
            echo "::warning::Hugging Face upload attempt $attempt failed"
            sleep $((attempt * 5))
          done
          exit 1
