agento3/scripts/scan_proxies_cffi.py

#!/usr/bin/env python3
"""
Optimized proxy scanner — uses port-filtered Webshare proxies for 72.6% success rate.

Key findings from v2 diagnostic scan (5000 proxies tested):
- Optimal port range: 13500-14499 → 72.6% success rate
- Best fingerprints: chrome123, chrome124, safari17_0 (equivalent)
- 99.7% of failures = CF_BLOCK_403 (IP reputation, not TLS fingerprint)
- Fingerprint rotation does NOT improve rate
- chrome120 = 7.5%, edge120 = 0% (DO NOT USE)
"""
import concurrent.futures
import os
import sys
import time

from curl_cffi import requests as curl_requests

TEST_URL = "https://archiveofourown.org/robots.txt"
TIMEOUT = 12
MAX_WORKERS = 50
PROXY_FILE = "/home/ubuntu/proxy.txt"
OUTPUT_FILE = "/dev/shm/working_proxies.txt"

# Optimal port range for Webshare proxies (verified by full scan)
#   Ports 13500-13999: 74.4% | Ports 14000-14499: 70.8%
#   Ports 10000-13499: ~32%  | Ports 14500-14999: ~52%
OPTIMAL_MIN_PORT = 13500
OPTIMAL_MAX_PORT = 14499

# Use only proven fingerprints
BROWSER_FINGERPRINTS = ["chrome123", "chrome124"]


def test_proxy(host_port: str) -> tuple:
    """Test proxy with fallback through fingerprints."""
    start = time.time()
    for imp in BROWSER_FINGERPRINTS:
        try:
            session = curl_requests.Session()
            session.impersonate = imp
            session.timeout = TIMEOUT
            session.proxies = {
                "http": f"http://{host_port}",
                "https": f"http://{host_port}",
            }
            session.headers.update({
                "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36",
                "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
                "Accept-Language": "en-US,en;q=0.9,zh-CN;q=0.8",
            })
            resp = session.get(TEST_URL)
            elapsed = time.time() - start
            session.close()
            if resp.status_code == 200:
                return (host_port, True, round(elapsed, 2), f"OK({imp})")
            elif resp.status_code in (403, 503) and imp != BROWSER_FINGERPRINTS[-1]:
                continue  # Try next fingerprint
            else:
                return (host_port, False, round(elapsed, 2), f"CF_BLOCK({resp.status_code})")
        except Exception as e:
            if imp != BROWSER_FINGERPRINTS[-1]:
                continue
            return (host_port, False, round(time.time() - start, 2), str(e)[:60])
    return (host_port, False, round(time.time() - start, 2), "ALL_FAILED")


def main():
    print(f"[Port-Optimized Scanner] Loading proxies from {PROXY_FILE}")

    proxies = []
    with open(PROXY_FILE) as f:
        for line in f:
            line = line.strip()
            if not line or ":" not in line:
                continue
            if "|" in line:
                line = line.split("|")[-1].strip()
            proxies.append(line)

    # Filter to optimal port range
    before = len(proxies)
    proxies = [p for p in proxies if ":" in p and OPTIMAL_MIN_PORT <= int(p.split(":")[-1]) <= OPTIMAL_MAX_PORT]
    print(f"Loaded {before} proxies, filtered to ports {OPTIMAL_MIN_PORT}-{OPTIMAL_MAX_PORT}: {len(proxies)}")
    print(f"Starting scan with {MAX_WORKERS} workers (timeout={TIMEOUT}s)...")

    working = []
    checked = 0
    start_time = time.time()

    with concurrent.futures.ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
        futures = {executor.submit(test_proxy, p): p for p in proxies}

        for future in concurrent.futures.as_completed(futures):
            host_port, is_ok, elapsed, note = future.result()
            checked += 1

            if is_ok:
                working.append((host_port, elapsed))
                working.sort(key=lambda x: x[1])

            if checked % 100 == 0 or is_ok:
                elapsed_total = time.time() - start_time
                rate = checked / elapsed_total if elapsed_total > 0 else 0
                eta = (len(proxies) - checked) / rate if rate > 0 else 0
                status = "✅" if is_ok else "❌"
                print(f"  [{checked}/{len(proxies)}] {status} {host_port} ({elapsed:.1f}s) {note} | "
                      f"Working: {len(working)} ({len(working)/checked*100:.1f}%) | "
                      f"{elapsed_total:.0f}s | ETA: {eta:.0f}s")

    with open(OUTPUT_FILE, "w") as f:
        for host_port, elapsed in working:
            f.write(f"{host_port}\n")

    elapsed_total = time.time() - start_time
    rate = len(working) / checked * 100 if checked else 0
    print(f"\n[Scan complete] {len(working)}/{checked} = {rate:.1f}% working (target: 70%+)")
    print(f"  Baseline (unfiltered): 44.8%")
    print(f"  Improvement: +{rate - 44.8:.1f} percentage points")
    print(f"  Target met: {'YES ✅' if rate >= 70 else 'NO ❌'}")
    print(f"  Time: {elapsed_total:.0f}s")
    print(f"  Saved to: {OUTPUT_FILE}")

    if working:
        print(f"\n  Top 10 fastest:")
        for hp, el in working[:10]:
            print(f"    {hp} ({el:.2f}s)")


if __name__ == "__main__":
    main()