""" AO3 反代后端 v4 — Cookie 感知 + CF 挑战用户浏览器求解 v4 vs v3: - Challenge token 映射：用户浏览器求解 CF 挑战时，保证同一 proxy 亲和性 - 挑战页面透传：所有代理都遇到 CF 挑战时，把挑战页发给用户浏览器求解 - Cookie 流：成功响应自动保存 proxy cookie，后续请求自动携带 - 统计面板展示 cookie-aware 代理数 """ import hashlib import json import logging import os import secrets import sys import threading import time from urllib.parse import urlparse, urlunparse from fastapi import FastAPI, Request, Response from fastapi.responses import HTMLResponse, JSONResponse, PlainTextResponse, RedirectResponse import uvicorn sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) from proxy_pool import get_proxy_pool from ao3_fetcher import fetch_url, is_cf_challenge from url_rewriter import rewrite_body, rewrite_response_headers, needs_rewrite, MIRROR_DOMAIN from cache import get_cache, get_ttl_for_path from stats import get_stats_collector logging.basicConfig( level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s", handlers=[logging.StreamHandler()], ) logger = logging.getLogger("ao3-backend") AO3_BASE = "https://archiveofourown.org" MIRROR_HOST = MIRROR_DOMAIN LOCAL_PATHS = {"/stats", "/health", "/metrics", "/favicon.ico", "/robots.txt"} # ─── Challenge Token Map (v4) ────────────────────────────────────────────── # Maps challenge_token → (method, ao3_url, headers, body, cookies, proxy_host, expires) _challenge_map: dict[str, tuple] = {} _challenge_lock = threading.Lock() CHALLENGE_TOKEN_TTL = 120 # 2 minutes for the user's browser to solve the challenge def _make_challenge_token() -> str: return secrets.token_urlsafe(16) def _store_challenge(token: str, method: str, ao3_url: str, headers: dict, body: bytes, cookies: dict, proxy_host: str): with _challenge_lock: # Clean expired tokens now = time.time() expired = [k for k, v in _challenge_map.items() if v[6] < now] for k in expired: del _challenge_map[k] _challenge_map[token] = (method, ao3_url, headers, body, cookies, proxy_host, now + CHALLENGE_TOKEN_TTL) def _get_challenge(token: str) -> tuple | None: with _challenge_lock: entry = _challenge_map.get(token) if entry and entry[6] > time.time(): del _challenge_map[token] return entry if entry: del _challenge_map[token] # expired return None # ─── App ─────────────────────────────────────────────────────────────────── app = FastAPI( title="AO3 Mirror", description="AO3 reverse proxy mirror for Chinese users", version="4.0.0", docs_url=None, redoc_url=None, ) def get_client_ip(request: Request) -> str: cf_ip = request.headers.get("CF-Connecting-IP") if cf_ip: return cf_ip forwarded = request.headers.get("X-Forwarded-For") if forwarded: return forwarded.split(",")[0].strip() return request.client.host if request.client else "unknown" def build_ao3_url(path: str, query: str = "") -> str: url = f"{AO3_BASE}{path}" if query: url = f"{url}?{query}" return url # ─── Challenge page URL rewriting ────────────────────────────────────────── def _rewrite_challenge_page(body: bytes, proxy_host: str, token: str) -> bytes: """Rewrite CF challenge page so all URLs go through mirror, and tag with challenge token.""" import re # Basic AO3 domain rewrite body = body.replace(b"archiveofourown.org", MIRROR_DOMAIN.encode()) # Rewrite /cdn-cgi/ challenge endpoints to go through mirror # These are CF's internal challenge platform URLs body = body.replace( b"/cdn-cgi/challenge-platform", f"/cdn-cgi/challenge-platform".encode() ) # Inject a marker meta tag so we can detect challenge pages coming back marker = f''.encode() marker_cookie = ( f'' ).encode() body = body.replace(b"", b"" + marker + marker_cookie, 1) if b"" not in body: body = b"" + marker + marker_cookie + b"" + body return body # ─── Response header helpers ─────────────────────────────────────────────── def filter_response_headers(headers: dict) -> dict: blocked = { "transfer-encoding", "content-encoding", "alt-svc", "cf-ray", "cf-cache-status", "cf-request-id", "server", "x-powered-by", "cross-origin-resource-policy", "cross-origin-embedder-policy", "cross-origin-opener-policy", "cross-origin-window-policy", "accept-ch", "critical-ch", } result = {} for key, value in headers.items(): if key.lower() in blocked: continue if key.lower().startswith("cf-"): continue result[key] = value result["X-Proxy"] = "AO3-Mirror/4.0" result["X-Cache"] = "MISS" result["Cache-Control"] = "public, max-age=60, s-maxage=60" return result def add_cors(response: Response): response.headers["Access-Control-Allow-Origin"] = "*" response.headers["Access-Control-Allow-Methods"] = "GET, POST, HEAD, OPTIONS, PUT, DELETE, PATCH" response.headers["Access-Control-Allow-Headers"] = "Content-Type, Authorization, Cookie, X-CSRF-Token" response.headers["Access-Control-Max-Age"] = "86400" response.headers["Access-Control-Allow-Credentials"] = "true" # ─── Local Routes ────────────────────────────────────────────────────────── @app.get("/health") async def health(): pool = get_proxy_pool() stats = pool.get_stats() resp = JSONResponse({ "status": "ok", "timestamp": time.time(), "version": "4.0.0", "proxy_pool": stats, }) resp.headers["Cache-Control"] = "no-store, no-cache, must-revalidate" resp.headers["Pragma"] = "no-cache" return resp @app.get("/robots.txt") async def robots(): return PlainTextResponse( "User-agent: *\nDisallow: /stats\nDisallow: /health\n" ) # ─── Stats Dashboard ─────────────────────────────────────────────────────── @app.get("/stats", response_class=HTMLResponse) async def stats_page(): collector = get_stats_collector() pool = get_proxy_pool() cache = get_cache() stats = collector.get_overview() proxy_stats = pool.get_stats() cache_stats = cache.get_stats() hourly = stats.get("hourly", []) chart_labels = json.dumps([ time.strftime("%H:%M", time.localtime(h["hour"])) for h in hourly[-24:] ]) chart_requests = json.dumps([h["total"] for h in hourly[-24:]]) chart_success = json.dumps([h["successful"] for h in hourly[-24:]]) chart_elapsed = json.dumps([h["avg_elapsed"] * 1000 for h in hourly[-24:]]) return f""" AO3 Mirror - 统计面板 v4

🛡 AO3 Mirror 统计面板 v4

总请求数
{stats['total_requests']:,}
运行 {stats['uptime_human']}
最近 1 分钟
{stats['recent_1m']:,} req
最近 5 分钟: {stats['recent_5m']:,}
成功率
{stats['success_rate']}%
失败: {stats['failed']:,}
缓存命中率
{stats['cache_rate']}%
已缓存: {stats['cached']:,}
平均延迟
{stats['avg_elapsed_ms']} ms
P99: {stats['p99_elapsed_ms']} ms
代理池
{proxy_stats['alive']}/{proxy_stats['total']}
可用: {proxy_stats['available']} | 受损: {proxy_stats['dead']} | Banned: {proxy_stats['banned']}
Cookie 代理
{proxy_stats['with_cookies']}/{proxy_stats['alive']}
已持有 cf_clearance
本地缓存
{cache_stats['size']}/{cache_stats['capacity']}
已用: {cache_stats['usage_pct']}%

请求趋势 (最近 24 小时)

响应延迟 (最近 24 小时)

热门路径

{''.join(f'' for p in stats['top_paths'][:15])}
路径请求数成功平均延迟
{p["path"]}{p["requests"]:,}0.8 else "badge-red"}">{round(p["successful"]/max(p["requests"],1)*100)}%{p["avg_elapsed"]*1000:.0f}ms
""" # ─── Metrics ─────────────────────────────────────────────────────────────── @app.get("/metrics") async def metrics(): collector = get_stats_collector() pool = get_proxy_pool() cache = get_cache() stats = collector.get_overview() proxy_stats = pool.get_stats() cache_stats = cache.get_stats() lines = [ "# HELP ao3_mirror_requests_total Total proxy requests", "# TYPE ao3_mirror_requests_total counter", f'ao3_mirror_requests_total {stats["total_requests"]}', "# HELP ao3_mirror_successful_requests Successful requests", "# TYPE ao3_mirror_successful_requests counter", f'ao3_mirror_successful_requests {stats["successful"]}', "# HELP ao3_mirror_failed_requests Failed requests", "# TYPE ao3_mirror_failed_requests counter", f'ao3_mirror_failed_requests {stats["failed"]}', "# HELP ao3_mirror_cache_hits Cache hit count", "# TYPE ao3_mirror_cache_hits counter", f'ao3_mirror_cache_hits {stats["cached"]}', "# HELP ao3_mirror_avg_elapsed_ms Average response time", "# TYPE ao3_mirror_avg_elapsed_ms gauge", f'ao3_mirror_avg_elapsed_ms {stats["avg_elapsed_ms"]}', "# HELP ao3_mirror_proxy_pool Proxy pool status", "# TYPE ao3_mirror_proxy_pool gauge", f'ao3_mirror_proxy_alive {proxy_stats["alive"]}', f'ao3_mirror_proxy_dead {proxy_stats["dead"]}', f'ao3_mirror_proxy_banned {proxy_stats["banned"]}', f'ao3_mirror_proxy_available {proxy_stats["available"]}', f'ao3_mirror_proxy_with_cookies {proxy_stats["with_cookies"]}', "# HELP ao3_mirror_cache_size Current cache size", "# TYPE ao3_mirror_cache_size gauge", f'ao3_mirror_cache_size {cache_stats["size"]}', ] return PlainTextResponse("\n".join(lines)) # ─── Proxy Core (v4) ────────────────────────────────────────────────────── @app.api_route("/{path:path}", methods=["GET", "POST", "HEAD", "OPTIONS", "PUT", "DELETE", "PATCH"]) async def proxy_handler(request: Request, path: str): """Main proxy handler — with cookie-aware fetching and CF challenge solving.""" start_time = time.time() client_ip = get_client_ip(request) if path == "" or path == "/": path = "" full_path = f"/{path}" if path else "/" if full_path in LOCAL_PATHS or full_path.startswith("/stats") or full_path.startswith("/health"): return JSONResponse({"error": "Not found"}, status_code=404) if request.method == "OPTIONS": resp = Response() add_cors(resp) return resp # ── v4: Challenge token check ────────────────────────────────────────── # If this request has a _cf_token cookie, it's part of a challenge resolution flow cf_token = request.cookies.get("_cf_token") or request.query_params.get("_cf_token") preferred_proxy = None if cf_token: challenge_entry = _get_challenge(cf_token) if challenge_entry: orig_method, orig_url, orig_headers, orig_body, orig_cookies, proxy_host, _ = challenge_entry preferred_proxy = proxy_host logger.info(f"Challenge resolution: using proxy {proxy_host} for {orig_url[:60]}") # Re-fetch the original request with the challenge proxy result = await fetch_url( url=orig_url, method=orig_method, headers=dict(request.headers), body=await request.body() if orig_method in ("POST", "PUT", "PATCH") else None, cookies=orig_cookies, is_api="/api/" in orig_url, preferred_proxy=preferred_proxy, ) if result["success"]: # Challenge solved! Proxy now has cf_clearance. Return content. elapsed = time.time() - start_time rewritten_body = rewrite_body(result["body"], result["headers"].get("Content-Type", "")) rewritten_headers = rewrite_response_headers(result["headers"]) final_headers = filter_response_headers(rewritten_headers) final_headers["X-Cache"] = "MISS" final_headers["X-CF-Status"] = "solved" # Set cf_clearance as a cookie on the mirror domain too # so subsequent requests benefit resp = Response(content=rewritten_body, status_code=result["status"], headers=final_headers) # Forward any Set-Cookie from AO3 (includes cf_clearance) for k, v in result["headers"].items(): if k.lower() == "set-cookie": # Rewrite domain v_rewritten = v.replace("domain=archiveofourown.org", f"domain={MIRROR_DOMAIN}") v_rewritten = v_rewritten.replace("domain=.archiveofourown.org", f"domain=.{MIRROR_DOMAIN}") resp.headers.add("Set-Cookie", v_rewritten) # Clean up challenge token cookie resp.delete_cookie("_cf_token", path="/") collector = get_stats_collector() collector.log_request(method=orig_method, path=full_path, status=result["status"], elapsed=elapsed, cached=False, proxy_host=preferred_proxy, client_ip=client_ip) add_cors(resp) return resp # ── Normal request flow ──────────────────────────────────────────────── query_string = request.url.query ao3_url = build_ao3_url(f"/{path}" if path else "/", query_string) # Check cache cache = get_cache() cached = cache.get(ao3_url, dict(request.headers)) if cached: body, resp_headers, status = cached elapsed = time.time() - start_time collector = get_stats_collector() collector.log_request(method=request.method, path=full_path, status=status, elapsed=elapsed, cached=True, client_ip=client_ip) headers = filter_response_headers(resp_headers) headers["X-Cache"] = "HIT" return Response(content=body, status_code=status, headers=headers) # Prepare request method = request.method client_headers = dict(request.headers) req_body = None if method in ("POST", "PUT", "PATCH"): req_body = await request.body() cookies = {} cookie_header = request.headers.get("cookie", "") if cookie_header: for pair in cookie_header.split(";"): if "=" in pair: k, v = pair.split("=", 1) cookies[k.strip()] = v.strip() # Forward to AO3 result = await fetch_url( url=ao3_url, method=method, headers=client_headers, body=req_body, cookies=cookies, is_api="/api/" in full_path or full_path.startswith("/api/"), preferred_proxy=preferred_proxy, ) elapsed = time.time() - start_time # ── v4: CF Challenge handling ────────────────────────────────────────── if result.get("is_challenge") and result.get("challenge_body"): challenge_proxy = result.get("challenge_proxy", "unknown") challenge_body = result["challenge_body"] # Generate challenge token for proxy affinity token = _make_challenge_token() _store_challenge(token, method, ao3_url, client_headers, req_body or b"", cookies, challenge_proxy) # Rewrite challenge page for user's browser rewritten_challenge = _rewrite_challenge_page(challenge_body, challenge_proxy, token) logger.warning( f"CF Challenge detected via {challenge_proxy} for {ao3_url[:80]}. " f"Sending challenge to user browser (token={token[:8]}...)" ) collector = get_stats_collector() collector.log_request(method=method, path=full_path, status=503, elapsed=elapsed, cached=False, proxy_host=challenge_proxy, client_ip=client_ip) # Return challenge page with 503 status + token cookie resp = HTMLResponse( content=rewritten_challenge, status_code=503, headers={ "X-CF-Challenge": "true", "X-CF-Challenge-Proxy": challenge_proxy, "Retry-After": "5", }, ) resp.set_cookie( key="_cf_token", value=token, path="/", max_age=CHALLENGE_TOKEN_TTL, httponly=False, # JS needs to read it samesite="lax", ) resp.delete_cookie("cf_clearance", path="/") # Clear stale clearance add_cors(resp) return resp # ── Standard failure ─────────────────────────────────────────────────── if not result["success"]: logger.error(f"Failed to fetch {ao3_url[:80]}: {result.get('error', 'unknown')}") collector = get_stats_collector() collector.log_request(method=method, path=full_path, status=502, elapsed=elapsed, cached=False, client_ip=client_ip) error_html = f""" AO3 Mirror - 暂时不可用

🔄 正在尝试连接 AO3

镜像站正在尝试通过代理重新连接 AO3 服务器。

请稍后刷新页面重试。

刷新页面
""" return HTMLResponse(content=error_html, status_code=502) # ── Success ──────────────────────────────────────────────────────────── ao3_status = result["status"] ao3_headers = result.get("headers", {}) raw_body = result.get("body", b"") content_type = ao3_headers.get("Content-Type", "") rewritten_body = rewrite_body(raw_body, content_type) rewritten_headers = rewrite_response_headers(ao3_headers) final_headers = filter_response_headers(rewritten_headers) final_headers["X-Cache"] = "MISS" # Cache successful GET responses if method == "GET" and 200 <= ao3_status < 400: ttl = get_ttl_for_path(full_path) cache.set(ao3_url, rewritten_body, rewritten_headers, ao3_status, ttl=ttl) # Handle redirects if 300 <= ao3_status < 400 and "location" in rewritten_headers: redirect_url = rewritten_headers["location"] if AO3_BASE in redirect_url: redirect_url = redirect_url.replace(AO3_BASE, "").replace("http://", "https://") return RedirectResponse(url=redirect_url, status_code=ao3_status) collector = get_stats_collector() collector.log_request(method=method, path=full_path, status=ao3_status, elapsed=elapsed, cached=False, proxy_host=result.get("proxy_host"), client_ip=client_ip) resp = Response(content=rewritten_body, status_code=ao3_status, headers=final_headers) # Forward Set-Cookie from AO3 (user session cookies + cf_clearance) for k, v in ao3_headers.items(): if k.lower() == "set-cookie": v_rewritten = v.replace("domain=archiveofourown.org", f"domain={MIRROR_DOMAIN}") v_rewritten = v_rewritten.replace("domain=.archiveofourown.org", f"domain=.{MIRROR_DOMAIN}") resp.headers.add("Set-Cookie", v_rewritten) add_cors(resp) return resp # ─── Startup / Shutdown ──────────────────────────────────────────────────── @app.on_event("startup") async def startup(): logger.info("AO3 Mirror backend v4 starting up...") get_proxy_pool() get_cache() get_stats_collector() logger.info("AO3 Mirror backend v4 started (cookie-aware + CF challenge solving)") @app.on_event("shutdown") async def shutdown(): logger.info("AO3 Mirror backend shutting down...") pool = get_proxy_pool() await pool.close_all() logger.info("AO3 Mirror backend stopped") # ─── Entry point ─────────────────────────────────────────────────────────── if __name__ == "__main__": port = int(os.environ.get("PORT", "8080")) workers = int(os.environ.get("WORKERS", "4")) logger.info(f"Starting AO3 Mirror v4 on port {port} with {workers} workers") uvicorn.run( "app:app", host="127.0.0.1", port=port, workers=workers, log_level="info", timeout_keep_alive=30, )