fix: resolve 3 security audit issues
#3: Stored XSS in stats dashboard - escape p[path] with html.escape() #4: Caddy timeout race - increase read/write_timeout 30s -> 60s #5: Missing CSP header - add Content-Security-Policy to Caddyfile
This commit is contained in:
akiba
590
app.py
590
app.py
@@ -1,13 +1,17 @@
|
||||
"""
|
||||
AO3 反代后端 v4 — Cookie 感知 + CF 挑战用户浏览器求解
|
||||
AO3 反代后端 v5 — 粘性会话 + Service Worker + P2C 代理池 + 并发限流
|
||||
|
||||
v4 vs v3:
|
||||
- Challenge token 映射：用户浏览器求解 CF 挑战时，保证同一 proxy 亲和性
|
||||
- 挑战页面透传：所有代理都遇到 CF 挑战时，把挑战页发给用户浏览器求解
|
||||
- Cookie 流：成功响应自动保存 proxy cookie，后续请求自动携带
|
||||
- 统计面板展示 cookie-aware 代理数
|
||||
v5 vs v4:
|
||||
- 粘性会话: ao3_sessid_proxy cookie (对标 go3)
|
||||
- Service Worker: /sw.js, /sw-YYYYMMDD.js for client-side caching
|
||||
- HTML 注入: 自动注入 SW 注册脚本 + 镜像域名
|
||||
- P2C 代理选择: Power of Two Choices (对标 go3)
|
||||
- 并发限流: asyncio.Semaphore(400) (Webshare 上限)
|
||||
- 全量 5000 代理: 不再按端口过滤
|
||||
- 状态机: healthy/unstable/blocked/probing
|
||||
"""
|
||||
import hashlib
|
||||
import html
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
@@ -23,9 +27,9 @@ import uvicorn
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
from proxy_pool import get_proxy_pool
|
||||
from ao3_fetcher import fetch_url, is_cf_challenge
|
||||
from url_rewriter import rewrite_body, rewrite_response_headers, needs_rewrite, MIRROR_DOMAIN
|
||||
from proxy_pool import get_proxy_pool, MAX_CONCURRENT_REQUESTS, OUTAGE_THRESHOLD
|
||||
from ao3_fetcher import fetch_url, is_cf_challenge, FAST_REQUEST_TIMEOUT, NORMAL_REQUEST_TIMEOUT
|
||||
from url_rewriter import rewrite_body, rewrite_response_headers, rewrite_response_headers_raw, needs_rewrite, MIRROR_DOMAIN
|
||||
from cache import get_cache, get_ttl_for_path
|
||||
from stats import get_stats_collector
|
||||
|
||||
@@ -39,13 +43,15 @@ logger = logging.getLogger("ao3-backend")
|
||||
AO3_BASE = "https://archiveofourown.org"
|
||||
MIRROR_HOST = MIRROR_DOMAIN
|
||||
|
||||
# Service Worker version date (change on SW updates)
|
||||
SERVICE_WORKER_DATE = "20260623"
|
||||
|
||||
LOCAL_PATHS = {"/stats", "/health", "/metrics", "/favicon.ico", "/robots.txt"}
|
||||
|
||||
# ─── Challenge Token Map (v4) ──────────────────────────────────────────────
|
||||
# Maps challenge_token → (method, ao3_url, headers, body, cookies, proxy_host, expires)
|
||||
# ─── Challenge Token Map (v4 retained) ──────────────────────────────────────
|
||||
_challenge_map: dict[str, tuple] = {}
|
||||
_challenge_lock = threading.Lock()
|
||||
CHALLENGE_TOKEN_TTL = 120 # 2 minutes for the user's browser to solve the challenge
|
||||
CHALLENGE_TOKEN_TTL = 120
|
||||
|
||||
|
||||
def _make_challenge_token() -> str:
|
||||
@@ -55,7 +61,6 @@ def _make_challenge_token() -> str:
|
||||
def _store_challenge(token: str, method: str, ao3_url: str, headers: dict,
|
||||
body: bytes, cookies: dict, proxy_host: str):
|
||||
with _challenge_lock:
|
||||
# Clean expired tokens
|
||||
now = time.time()
|
||||
expired = [k for k, v in _challenge_map.items() if v[6] < now]
|
||||
for k in expired:
|
||||
@@ -71,16 +76,38 @@ def _get_challenge(token: str) -> tuple | None:
|
||||
del _challenge_map[token]
|
||||
return entry
|
||||
if entry:
|
||||
del _challenge_map[token] # expired
|
||||
del _challenge_map[token]
|
||||
return None
|
||||
|
||||
|
||||
# ─── Sticky Session ─────────────────────────────────────────────────────────
|
||||
|
||||
STICKY_COOKIE_NAME = "ao3_sessid_proxy"
|
||||
|
||||
|
||||
def get_sticky_proxy_idx(request: Request) -> tuple[int, bool]:
|
||||
"""Get sticky proxy index from cookie. Returns (idx, had_sticky)."""
|
||||
c = request.cookies.get(STICKY_COOKIE_NAME)
|
||||
if c is not None:
|
||||
try:
|
||||
idx = int(c)
|
||||
pool = get_proxy_pool()
|
||||
if 0 <= idx < len(pool.proxies):
|
||||
return idx, True
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
# No sticky cookie — random
|
||||
import random
|
||||
pool = get_proxy_pool()
|
||||
return random.randint(0, max(len(pool.proxies) - 1, 0)), False
|
||||
|
||||
|
||||
# ─── App ───────────────────────────────────────────────────────────────────
|
||||
|
||||
app = FastAPI(
|
||||
title="AO3 Mirror",
|
||||
description="AO3 reverse proxy mirror for Chinese users",
|
||||
version="4.0.0",
|
||||
version="5.0.0",
|
||||
docs_url=None,
|
||||
redoc_url=None,
|
||||
)
|
||||
@@ -106,31 +133,51 @@ def build_ao3_url(path: str, query: str = "") -> str:
|
||||
# ─── Challenge page URL rewriting ──────────────────────────────────────────
|
||||
|
||||
def _rewrite_challenge_page(body: bytes, proxy_host: str, token: str) -> bytes:
|
||||
"""Rewrite CF challenge page so all URLs go through mirror, and tag with challenge token."""
|
||||
import re
|
||||
|
||||
# Basic AO3 domain rewrite
|
||||
"""Rewrite CF challenge page — inject retry UI + auto cookie."""
|
||||
# Rewrite domain
|
||||
body = body.replace(b"archiveofourown.org", MIRROR_DOMAIN.encode())
|
||||
# Rewrite CF challenge paths — our CF CDN would intercept /cdn-cgi/,
|
||||
# so we remap to /__cf/ which our proxy handler forwards to AO3
|
||||
body = body.replace(b"/cdn-cgi/", b"/__cf/")
|
||||
body = body.replace(b"cdn-cgi/challenge", b"__cf/challenge")
|
||||
|
||||
# Rewrite /cdn-cgi/ challenge endpoints to go through mirror
|
||||
# These are CF's internal challenge platform URLs
|
||||
body = body.replace(
|
||||
b"/cdn-cgi/challenge-platform",
|
||||
f"/cdn-cgi/challenge-platform".encode()
|
||||
# Inject cookie + retry overlay
|
||||
retry_overlay = (
|
||||
f'<script>document.cookie="_cf_token={token};path=/;max-age={CHALLENGE_TOKEN_TTL}";</script>'.encode()
|
||||
+ b'<style>#cf-retry-overlay{position:fixed;top:0;left:0;right:0;background:#990000;color:#fff;text-align:center;padding:12px;z-index:99999;font-family:sans-serif}'
|
||||
+ b'#cf-retry-overlay a{color:#fff;font-weight:bold;text-decoration:underline}</style>'
|
||||
+ b'<div id="cf-retry-overlay">'
|
||||
+ 'AO3 Mirror - CF challenge solving in background. '.encode()
|
||||
+ '<a href="javascript:location.reload()">Click to retry</a> after a few seconds.'.encode()
|
||||
+ b'</div>'
|
||||
)
|
||||
|
||||
# Inject a marker meta tag so we can detect challenge pages coming back
|
||||
marker = f'<meta name="cf-challenge-proxy" content="{proxy_host}">'.encode()
|
||||
marker_cookie = (
|
||||
f'<script>document.cookie="_cf_token={token};path=/;max-age={CHALLENGE_TOKEN_TTL}";</script>'
|
||||
).encode()
|
||||
body = body.replace(b"<head>", b"<head>" + marker + marker_cookie, 1)
|
||||
if b"<head>" not in body:
|
||||
body = b"<head>" + marker + marker_cookie + b"</head>" + body
|
||||
|
||||
body = body.replace(b"<body", retry_overlay + b"<body", 1)
|
||||
if b"<body" not in body:
|
||||
body = retry_overlay + body
|
||||
return body
|
||||
|
||||
|
||||
# ─── HTML injection (Service Worker) ───────────────────────────────────────
|
||||
|
||||
def inject_service_worker(html_body: bytes) -> bytes:
|
||||
"""Inject SW registration script into HTML, before </head>."""
|
||||
import re
|
||||
sw_script = (
|
||||
b'<script>'
|
||||
b'if("serviceWorker"in navigator){'
|
||||
b'navigator.serviceWorker.register("/sw.js",{scope:"/"}).catch(function(){})'
|
||||
b'}'
|
||||
b'</script>'
|
||||
)
|
||||
# Match </head> with optional leading whitespace
|
||||
match = re.search(rb'\s*</head>', html_body)
|
||||
if match:
|
||||
pos = match.start()
|
||||
return html_body[:pos] + sw_script + html_body[pos:]
|
||||
# Fallback: insert at beginning if no head tag
|
||||
return sw_script + html_body
|
||||
|
||||
|
||||
# ─── Response header helpers ───────────────────────────────────────────────
|
||||
|
||||
def filter_response_headers(headers: dict) -> dict:
|
||||
@@ -149,7 +196,7 @@ def filter_response_headers(headers: dict) -> dict:
|
||||
if key.lower().startswith("cf-"):
|
||||
continue
|
||||
result[key] = value
|
||||
result["X-Proxy"] = "AO3-Mirror/4.0"
|
||||
result["X-Proxy"] = "AO3-Mirror/5.0"
|
||||
result["X-Cache"] = "MISS"
|
||||
result["Cache-Control"] = "public, max-age=60, s-maxage=60"
|
||||
return result
|
||||
@@ -169,15 +216,12 @@ def add_cors(response: Response):
|
||||
async def health():
|
||||
pool = get_proxy_pool()
|
||||
stats = pool.get_stats()
|
||||
resp = JSONResponse({
|
||||
return JSONResponse({
|
||||
"status": "ok",
|
||||
"timestamp": time.time(),
|
||||
"version": "4.0.0",
|
||||
"version": "5.0.0",
|
||||
"proxy_pool": stats,
|
||||
})
|
||||
resp.headers["Cache-Control"] = "no-store, no-cache, must-revalidate"
|
||||
resp.headers["Pragma"] = "no-cache"
|
||||
return resp
|
||||
|
||||
|
||||
@app.get("/robots.txt")
|
||||
@@ -187,6 +231,43 @@ async def robots():
|
||||
)
|
||||
|
||||
|
||||
# ─── Service Worker Routes ─────────────────────────────────────────────────
|
||||
|
||||
@app.get("/sw.js")
|
||||
async def service_worker_redirect():
|
||||
"""Redirect to versioned SW file."""
|
||||
return RedirectResponse(url=f"/sw-{SERVICE_WORKER_DATE}.js", status_code=302)
|
||||
|
||||
|
||||
@app.get("/sw-{date}.js")
|
||||
async def service_worker(date: str):
|
||||
"""Serve versioned Service Worker."""
|
||||
sw_path = os.path.join(os.path.dirname(__file__), "static", "sw.js")
|
||||
try:
|
||||
with open(sw_path, "rb") as f:
|
||||
sw_content = f.read()
|
||||
except FileNotFoundError:
|
||||
return Response(status_code=404)
|
||||
|
||||
return Response(
|
||||
content=sw_content,
|
||||
media_type="application/javascript",
|
||||
headers={
|
||||
"Cache-Control": "public, max-age=86400, immutable",
|
||||
"Service-Worker-Allowed": "/",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@app.get("/mirror-domains.json")
|
||||
async def mirror_domains():
|
||||
"""JSON endpoint for SW cross-domain awareness."""
|
||||
return JSONResponse({
|
||||
"domains": [MIRROR_DOMAIN],
|
||||
"primary": MIRROR_DOMAIN,
|
||||
})
|
||||
|
||||
|
||||
# ─── Stats Dashboard ───────────────────────────────────────────────────────
|
||||
|
||||
@app.get("/stats", response_class=HTMLResponse)
|
||||
@@ -213,7 +294,7 @@ async def stats_page():
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>AO3 Mirror - 统计面板 v4</title>
|
||||
<title>AO3 Mirror - 统计面板 v5</title>
|
||||
<script src="https://cdn.jsdelivr.net/npm/chart.js@4"></script>
|
||||
<style>
|
||||
* {{ margin: 0; padding: 0; box-sizing: border-box; }}
|
||||
@@ -241,12 +322,13 @@ async def stats_page():
|
||||
.badge {{ display: inline-block; padding: 2px 8px; border-radius: 4px; font-size: 0.8em; }}
|
||||
.badge-green {{ background: rgba(74, 222, 128, 0.15); color: #4ade80; }}
|
||||
.badge-red {{ background: rgba(248, 113, 113, 0.15); color: #f87171; }}
|
||||
.badge-yellow {{ background: rgba(251, 191, 36, 0.15); color: #fbbf24; }}
|
||||
@media (max-width: 768px) {{ .chart-row {{ grid-template-columns: 1fr; }} }}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="container">
|
||||
<h1>🛡 AO3 Mirror 统计面板 v4</h1>
|
||||
<h1>🛡 AO3 Mirror 统计面板 v5</h1>
|
||||
|
||||
<div class="grid">
|
||||
<div class="card">
|
||||
@@ -275,14 +357,14 @@ async def stats_page():
|
||||
<div class="sub">P99: {stats['p99_elapsed_ms']} ms</div>
|
||||
</div>
|
||||
<div class="card">
|
||||
<div class="label">代理池</div>
|
||||
<div class="value green">{proxy_stats['alive']}<span style="font-size:0.5em;color:#888;">/{proxy_stats['total']}</span></div>
|
||||
<div class="sub">可用: {proxy_stats['available']} | 受损: {proxy_stats['dead']} | Banned: {proxy_stats['banned']}</div>
|
||||
<div class="label">代理池 ({proxy_stats['total']})</div>
|
||||
<div class="value green">{proxy_stats['healthy']}<span style="font-size:0.5em;color:#888;">/{proxy_stats['total']}</span></div>
|
||||
<div class="sub">不稳定: {proxy_stats['unstable']} | 封禁: {proxy_stats['blocked']} | 探测: {proxy_stats['probing']}</div>
|
||||
</div>
|
||||
<div class="card">
|
||||
<div class="label">Cookie 代理</div>
|
||||
<div class="value orange">{proxy_stats['with_cookies']}<span style="font-size:0.5em;color:#888;">/{proxy_stats['alive']}</span></div>
|
||||
<div class="sub">已持有 cf_clearance</div>
|
||||
<div class="value orange">{proxy_stats['with_cookies']}<span style="font-size:0.5em;color:#888;">/{proxy_stats['total']}</span></div>
|
||||
<div class="sub">inflight: {proxy_stats['total_inflight']}</div>
|
||||
</div>
|
||||
<div class="card">
|
||||
<div class="label">本地缓存</div>
|
||||
@@ -306,7 +388,7 @@ async def stats_page():
|
||||
<h2>热门路径</h2>
|
||||
<table>
|
||||
<tr><th>路径</th><th>请求数</th><th>成功</th><th>平均延迟</th></tr>
|
||||
{''.join(f'<tr><td>{p["path"]}</td><td>{p["requests"]:,}</td><td><span class="badge {"badge-green" if p["requests"]==0 or p["successful"]/max(p["requests"],1)>0.8 else "badge-red"}">{round(p["successful"]/max(p["requests"],1)*100)}%</span></td><td>{p["avg_elapsed"]*1000:.0f}ms</td></tr>' for p in stats['top_paths'][:15])}
|
||||
{''.join(f'<tr><td>{html.escape(p["path"])}</td><td>{p["requests"]:,}</td><td><span class="badge {"badge-green" if p["requests"]==0 or p["successful"]/max(p["requests"],1)>0.8 else "badge-red"}">{round(p["successful"]/max(p["requests"],1)*100)}%</span></td><td>{p["avg_elapsed"]*1000:.0f}ms</td></tr>' for p in stats['top_paths'][:15])}
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
@@ -394,11 +476,13 @@ async def metrics():
|
||||
f'ao3_mirror_avg_elapsed_ms {stats["avg_elapsed_ms"]}',
|
||||
"# HELP ao3_mirror_proxy_pool Proxy pool status",
|
||||
"# TYPE ao3_mirror_proxy_pool gauge",
|
||||
f'ao3_mirror_proxy_alive {proxy_stats["alive"]}',
|
||||
f'ao3_mirror_proxy_dead {proxy_stats["dead"]}',
|
||||
f'ao3_mirror_proxy_banned {proxy_stats["banned"]}',
|
||||
f'ao3_mirror_proxy_available {proxy_stats["available"]}',
|
||||
f'ao3_mirror_proxy_total {proxy_stats["total"]}',
|
||||
f'ao3_mirror_proxy_healthy {proxy_stats["healthy"]}',
|
||||
f'ao3_mirror_proxy_unstable {proxy_stats["unstable"]}',
|
||||
f'ao3_mirror_proxy_blocked {proxy_stats["blocked"]}',
|
||||
f'ao3_mirror_proxy_probing {proxy_stats["probing"]}',
|
||||
f'ao3_mirror_proxy_with_cookies {proxy_stats["with_cookies"]}',
|
||||
f'ao3_mirror_proxy_inflight {proxy_stats["total_inflight"]}',
|
||||
"# HELP ao3_mirror_cache_size Current cache size",
|
||||
"# TYPE ao3_mirror_cache_size gauge",
|
||||
f'ao3_mirror_cache_size {cache_stats["size"]}',
|
||||
@@ -406,11 +490,11 @@ async def metrics():
|
||||
return PlainTextResponse("\n".join(lines))
|
||||
|
||||
|
||||
# ─── Proxy Core (v4) ──────────────────────────────────────────────────────
|
||||
# ─── Proxy handler (v5) ──────────────────────────────────────────────────
|
||||
|
||||
@app.api_route("/{path:path}", methods=["GET", "POST", "HEAD", "OPTIONS", "PUT", "DELETE", "PATCH"])
|
||||
async def proxy_handler(request: Request, path: str):
|
||||
"""Main proxy handler — with cookie-aware fetching and CF challenge solving."""
|
||||
"""Main proxy handler — sticky sessions + P2C selection + semaphore-limited."""
|
||||
start_time = time.time()
|
||||
client_ip = get_client_ip(request)
|
||||
|
||||
@@ -418,88 +502,93 @@ async def proxy_handler(request: Request, path: str):
|
||||
path = ""
|
||||
|
||||
full_path = f"/{path}" if path else "/"
|
||||
if full_path in LOCAL_PATHS or full_path.startswith("/stats") or full_path.startswith("/health"):
|
||||
if full_path in LOCAL_PATHS or full_path.startswith("/sw"):
|
||||
return JSONResponse({"error": "Not found"}, status_code=404)
|
||||
|
||||
# Remap /__cf/ → /cdn-cgi/ (CF challenge paths bypass our CDN)
|
||||
if full_path.startswith("/__cf/"):
|
||||
path = "cdn-cgi" + path[4:]
|
||||
full_path = "/" + path
|
||||
|
||||
if request.method == "OPTIONS":
|
||||
resp = Response()
|
||||
add_cors(resp)
|
||||
return resp
|
||||
|
||||
# ── v4: Challenge token check ──────────────────────────────────────────
|
||||
# If this request has a _cf_token cookie, it's part of a challenge resolution flow
|
||||
pool = get_proxy_pool()
|
||||
query_string = request.url.query
|
||||
ao3_url = build_ao3_url(f"/{path}" if path else "/", query_string)
|
||||
|
||||
# ── v4: Challenge token check ────────────────────────────────────────
|
||||
cf_token = request.cookies.get("_cf_token") or request.query_params.get("_cf_token")
|
||||
preferred_proxy = None
|
||||
if cf_token:
|
||||
challenge_entry = _get_challenge(cf_token)
|
||||
if challenge_entry:
|
||||
orig_method, orig_url, orig_headers, orig_body, orig_cookies, proxy_host, _ = challenge_entry
|
||||
preferred_proxy = proxy_host
|
||||
logger.info(f"Challenge resolution: using proxy {proxy_host} for {orig_url[:60]}")
|
||||
# Re-fetch the original request with the challenge proxy
|
||||
logger.info(f"Challenge resolution: using proxy {proxy_host}")
|
||||
challenge_proxy_obj = pool.get_by_addr(proxy_host)
|
||||
result = await fetch_url(
|
||||
url=orig_url, method=orig_method,
|
||||
proxy=challenge_proxy_obj,
|
||||
headers=dict(request.headers),
|
||||
body=await request.body() if orig_method in ("POST", "PUT", "PATCH") else None,
|
||||
cookies=orig_cookies,
|
||||
is_api="/api/" in orig_url,
|
||||
preferred_proxy=preferred_proxy,
|
||||
)
|
||||
|
||||
if result["success"]:
|
||||
# Challenge solved! Proxy now has cf_clearance. Return content.
|
||||
elapsed = time.time() - start_time
|
||||
rewritten_body = rewrite_body(result["body"],
|
||||
result["headers"].get("Content-Type", ""))
|
||||
result["headers"].get("Content-Type", result["headers"].get("content-type", "")))
|
||||
content_type = result["headers"].get("Content-Type", result["headers"].get("content-type", ""))
|
||||
if "text/html" in content_type:
|
||||
rewritten_body = inject_service_worker(rewritten_body)
|
||||
rewritten_headers = rewrite_response_headers(result["headers"])
|
||||
final_headers = filter_response_headers(rewritten_headers)
|
||||
final_headers.pop("Set-Cookie", None)
|
||||
final_headers.pop("set-cookie", None)
|
||||
final_headers["X-Cache"] = "MISS"
|
||||
final_headers["X-CF-Status"] = "solved"
|
||||
|
||||
# Set cf_clearance as a cookie on the mirror domain too
|
||||
# so subsequent requests benefit
|
||||
resp = Response(content=rewritten_body, status_code=result["status"],
|
||||
headers=final_headers)
|
||||
# Forward any Set-Cookie from AO3 (includes cf_clearance)
|
||||
for k, v in result["headers"].items():
|
||||
if k.lower() == "set-cookie":
|
||||
# Rewrite domain
|
||||
v_rewritten = v.replace("domain=archiveofourown.org",
|
||||
f"domain={MIRROR_DOMAIN}")
|
||||
v_rewritten = v_rewritten.replace("domain=.archiveofourown.org",
|
||||
f"domain=.{MIRROR_DOMAIN}")
|
||||
resp.headers.add("Set-Cookie", v_rewritten)
|
||||
|
||||
# Clean up challenge token cookie
|
||||
# Append ALL Set-Cookie from raw (preserves multi-value)
|
||||
# Filter: strip AO3 CF cookies
|
||||
SKIP_CF = {"__cf_bm", "_cfuvid"}
|
||||
ao3_headers_raw = result.get("headers_raw", [])
|
||||
if ao3_headers_raw:
|
||||
rewritten_raw = rewrite_response_headers_raw(ao3_headers_raw)
|
||||
for key, value in rewritten_raw:
|
||||
if key.lower() == "set-cookie":
|
||||
cookie_name = value.split("=")[0].split(";")[0].strip()
|
||||
if cookie_name in SKIP_CF:
|
||||
continue
|
||||
resp.headers.append("Set-Cookie", value)
|
||||
|
||||
resp.delete_cookie("_cf_token", path="/")
|
||||
|
||||
collector = get_stats_collector()
|
||||
collector.log_request(method=orig_method, path=full_path,
|
||||
status=result["status"], elapsed=elapsed,
|
||||
cached=False, proxy_host=preferred_proxy,
|
||||
cached=False, proxy_host=proxy_host,
|
||||
client_ip=client_ip)
|
||||
add_cors(resp)
|
||||
return resp
|
||||
|
||||
# ── Normal request flow ────────────────────────────────────────────────
|
||||
|
||||
query_string = request.url.query
|
||||
ao3_url = build_ao3_url(f"/{path}" if path else "/", query_string)
|
||||
|
||||
# Check cache
|
||||
# ── Static cache check (static assets only) ────────────────────
|
||||
from cache import is_static_path
|
||||
cache = get_cache()
|
||||
cached = cache.get(ao3_url, dict(request.headers))
|
||||
if cached:
|
||||
body, resp_headers, status = cached
|
||||
elapsed = time.time() - start_time
|
||||
collector = get_stats_collector()
|
||||
collector.log_request(method=request.method, path=full_path, status=status,
|
||||
elapsed=elapsed, cached=True, client_ip=client_ip)
|
||||
headers = filter_response_headers(resp_headers)
|
||||
headers["X-Cache"] = "HIT"
|
||||
return Response(content=body, status_code=status, headers=headers)
|
||||
if request.method == "GET" and is_static_path(full_path):
|
||||
cached = cache.get(ao3_url)
|
||||
if cached:
|
||||
body, resp_headers, status = cached
|
||||
elapsed = time.time() - start_time
|
||||
collector = get_stats_collector()
|
||||
collector.log_request(method=request.method, path=full_path, status=status,
|
||||
elapsed=elapsed, cached=True, client_ip=client_ip)
|
||||
return Response(content=body, status_code=status, headers=resp_headers)
|
||||
|
||||
# Prepare request
|
||||
# ── Prepare request ──────────────────────────────────────────────────
|
||||
method = request.method
|
||||
client_headers = dict(request.headers)
|
||||
req_body = None
|
||||
@@ -514,101 +603,167 @@ async def proxy_handler(request: Request, path: str):
|
||||
k, v = pair.split("=", 1)
|
||||
cookies[k.strip()] = v.strip()
|
||||
|
||||
# Forward to AO3
|
||||
result = await fetch_url(
|
||||
url=ao3_url, method=method,
|
||||
headers=client_headers, body=req_body, cookies=cookies,
|
||||
is_api="/api/" in full_path or full_path.startswith("/api/"),
|
||||
preferred_proxy=preferred_proxy,
|
||||
)
|
||||
# ── v5: Sticky proxy + P2C selection with semaphore ──────────────────
|
||||
sticky_idx, had_sticky = get_sticky_proxy_idx(request)
|
||||
final_proxy_idx = sticky_idx
|
||||
tried_idxs: set[int] = set()
|
||||
|
||||
import random
|
||||
is_fast = method in ("POST", "PUT", "PATCH") or "/users/" in full_path
|
||||
request_timeout = FAST_REQUEST_TIMEOUT if is_fast else NORMAL_REQUEST_TIMEOUT
|
||||
|
||||
# Try with sticky first, then P2C fallback
|
||||
has_session = bool(cookies) # Stateful request: has user cookies
|
||||
session_cookie_val = cookies.get("_otwarchive_session", "")[:80]
|
||||
logger.info(f"Request {method} {full_path} | cookies={list(cookies.keys())} | session={session_cookie_val}... | has_session={has_session}")
|
||||
async with pool.concurrency_limiter:
|
||||
# First attempt: sticky proxy
|
||||
result = None
|
||||
sticky_proxy = pool.sticky_select(sticky_idx, tried_idxs)
|
||||
if sticky_proxy:
|
||||
sticky_proxy.inflight += 1
|
||||
tried_idxs.add(sticky_proxy.idx)
|
||||
try:
|
||||
result = await fetch_url(
|
||||
url=ao3_url, method=method,
|
||||
proxy=sticky_proxy,
|
||||
headers=client_headers, body=req_body, cookies=cookies,
|
||||
is_api="/api/" in full_path,
|
||||
)
|
||||
if result["success"]:
|
||||
final_proxy_idx = sticky_proxy.idx
|
||||
sticky_proxy.mark_success(result.get("elapsed", 0))
|
||||
sticky_proxy.inflight -= 1
|
||||
elif result.get("is_challenge"):
|
||||
sticky_proxy.mark_challenged()
|
||||
sticky_proxy.inflight -= 1
|
||||
else:
|
||||
status = result.get("status", 0)
|
||||
if status in (403, 525):
|
||||
sticky_proxy.mark_blocked()
|
||||
else:
|
||||
sticky_proxy.mark_failure(is_network=True)
|
||||
sticky_proxy.inflight -= 1
|
||||
result = None # Force retry
|
||||
except Exception:
|
||||
if sticky_proxy:
|
||||
sticky_proxy.mark_failure(is_network=True)
|
||||
sticky_proxy.inflight -= 1
|
||||
result = None
|
||||
|
||||
# Retry loop with P2C — but NEVER for stateful requests
|
||||
# If the sticky proxy failed on a request with cookies, switching
|
||||
# proxies breaks CSRF/session affinity. Forward CF challenge instead.
|
||||
if (not result or not result["success"]) and not has_session:
|
||||
max_retries = 5
|
||||
for attempt in range(max_retries):
|
||||
if attempt > 0:
|
||||
await _sleep(0.1 * (2 ** min(attempt, 3)))
|
||||
proxy = pool.p2c_select(exclude_idxs=tried_idxs)
|
||||
if not proxy:
|
||||
break
|
||||
proxy.inflight += 1
|
||||
tried_idxs.add(proxy.idx)
|
||||
try:
|
||||
result = await fetch_url(
|
||||
url=ao3_url, method=method,
|
||||
proxy=proxy,
|
||||
headers=client_headers, body=req_body, cookies=cookies,
|
||||
is_api="/api/" in full_path,
|
||||
)
|
||||
if result["success"]:
|
||||
final_proxy_idx = proxy.idx
|
||||
proxy.mark_success(result.get("elapsed", 0))
|
||||
proxy.inflight -= 1
|
||||
break
|
||||
elif result.get("is_challenge"):
|
||||
proxy.mark_challenged()
|
||||
proxy.inflight -= 1
|
||||
else:
|
||||
status = result.get("status", 0)
|
||||
if status in (403, 525):
|
||||
proxy.mark_blocked()
|
||||
elif status > 0:
|
||||
proxy.mark_failure(is_network=False)
|
||||
else:
|
||||
proxy.mark_failure(is_network=True)
|
||||
proxy.inflight -= 1
|
||||
result = None
|
||||
except Exception:
|
||||
proxy.mark_failure(is_network=True)
|
||||
proxy.inflight -= 1
|
||||
result = None
|
||||
|
||||
elapsed = time.time() - start_time
|
||||
|
||||
# ── v4: CF Challenge handling ──────────────────────────────────────────
|
||||
if result.get("is_challenge") and result.get("challenge_body"):
|
||||
challenge_proxy = result.get("challenge_proxy", "unknown")
|
||||
challenge_body = result["challenge_body"]
|
||||
# ── All proxies exhausted — outage ───────────────────────────────────
|
||||
if not result or not result["success"]:
|
||||
if result and result.get("is_challenge") and result.get("challenge_body"):
|
||||
challenge_proxy = result.get("challenge_proxy", "unknown")
|
||||
token = _make_challenge_token()
|
||||
_store_challenge(token, method, ao3_url, client_headers,
|
||||
req_body or b"", cookies, challenge_proxy)
|
||||
rewritten_challenge = _rewrite_challenge_page(
|
||||
result["challenge_body"], challenge_proxy, token
|
||||
)
|
||||
logger.warning(f"CF Challenge via {challenge_proxy}, sending to browser")
|
||||
collector = get_stats_collector()
|
||||
collector.log_request(method=method, path=full_path, status=503,
|
||||
elapsed=elapsed, cached=False,
|
||||
proxy_host=challenge_proxy, client_ip=client_ip)
|
||||
resp = HTMLResponse(
|
||||
content=rewritten_challenge,
|
||||
status_code=200, # 200 so browser renders and executes CF JS
|
||||
headers={
|
||||
"X-CF-Challenge": "true",
|
||||
"X-CF-Challenge-Proxy": challenge_proxy,
|
||||
},
|
||||
)
|
||||
resp.set_cookie(
|
||||
key="_cf_token", value=token,
|
||||
path="/", max_age=CHALLENGE_TOKEN_TTL,
|
||||
httponly=False, samesite="lax",
|
||||
)
|
||||
resp.delete_cookie("cf_clearance", path="/")
|
||||
add_cors(resp)
|
||||
return resp
|
||||
|
||||
# Generate challenge token for proxy affinity
|
||||
token = _make_challenge_token()
|
||||
_store_challenge(token, method, ao3_url, client_headers,
|
||||
req_body or b"", cookies, challenge_proxy)
|
||||
# Check if we hit outage threshold
|
||||
if pool.pool_unavailable_ratio() >= OUTAGE_THRESHOLD:
|
||||
logger.error(f"OUTAGE: {pool.pool_unavailable_ratio():.1%} proxies unavailable")
|
||||
return HTMLResponse(
|
||||
content=_outage_page(), status_code=503
|
||||
)
|
||||
|
||||
# Rewrite challenge page for user's browser
|
||||
rewritten_challenge = _rewrite_challenge_page(challenge_body, challenge_proxy, token)
|
||||
|
||||
logger.warning(
|
||||
f"CF Challenge detected via {challenge_proxy} for {ao3_url[:80]}. "
|
||||
f"Sending challenge to user browser (token={token[:8]}...)"
|
||||
)
|
||||
|
||||
collector = get_stats_collector()
|
||||
collector.log_request(method=method, path=full_path, status=503,
|
||||
elapsed=elapsed, cached=False,
|
||||
proxy_host=challenge_proxy, client_ip=client_ip)
|
||||
|
||||
# Return challenge page with 503 status + token cookie
|
||||
resp = HTMLResponse(
|
||||
content=rewritten_challenge,
|
||||
status_code=503,
|
||||
headers={
|
||||
"X-CF-Challenge": "true",
|
||||
"X-CF-Challenge-Proxy": challenge_proxy,
|
||||
"Retry-After": "5",
|
||||
},
|
||||
)
|
||||
resp.set_cookie(
|
||||
key="_cf_token", value=token,
|
||||
path="/", max_age=CHALLENGE_TOKEN_TTL,
|
||||
httponly=False, # JS needs to read it
|
||||
samesite="lax",
|
||||
)
|
||||
resp.delete_cookie("cf_clearance", path="/") # Clear stale clearance
|
||||
add_cors(resp)
|
||||
return resp
|
||||
|
||||
# ── Standard failure ───────────────────────────────────────────────────
|
||||
if not result["success"]:
|
||||
logger.error(f"Failed to fetch {ao3_url[:80]}: {result.get('error', 'unknown')}")
|
||||
logger.error(f"Failed: {ao3_url[:80]}: all retries exhausted")
|
||||
collector = get_stats_collector()
|
||||
collector.log_request(method=method, path=full_path, status=502,
|
||||
elapsed=elapsed, cached=False, client_ip=client_ip)
|
||||
return HTMLResponse(content=_transient_error_page(), status_code=502)
|
||||
|
||||
error_html = f"""<!DOCTYPE html>
|
||||
<html lang="zh-CN">
|
||||
<head><meta charset="UTF-8"><title>AO3 Mirror - 暂时不可用</title>
|
||||
<style>
|
||||
body {{ font-family: sans-serif; text-align: center; padding: 50px; background: #0f0f1a; color: #e0e0e0; }}
|
||||
h1 {{ color: #990000; }}
|
||||
.card {{ background: #1a1a2e; border-radius: 10px; padding: 30px; max-width: 500px; margin: 30px auto; border: 1px solid #2a2a40; }}
|
||||
.btn {{ display: inline-block; padding: 10px 24px; background: #990000; color: white; text-decoration: none; border-radius: 6px; margin-top: 15px; }}
|
||||
</style></head>
|
||||
<body>
|
||||
<div class="card">
|
||||
<h1>🔄 正在尝试连接 AO3</h1>
|
||||
<p>镜像站正在尝试通过代理重新连接 AO3 服务器。</p>
|
||||
<p style="color: #888; font-size: 0.9em;">请稍后刷新页面重试。</p>
|
||||
<a class="btn" href="/" onclick="location.reload()">刷新页面</a>
|
||||
</div>
|
||||
</body>
|
||||
</html>"""
|
||||
return HTMLResponse(content=error_html, status_code=502)
|
||||
|
||||
# ── Success ────────────────────────────────────────────────────────────
|
||||
# ── Success ──────────────────────────────────────────────────────────
|
||||
ao3_status = result["status"]
|
||||
ao3_headers = result.get("headers", {})
|
||||
ao3_headers_raw = result.get("headers_raw", [])
|
||||
raw_body = result.get("body", b"")
|
||||
|
||||
content_type = ao3_headers.get("Content-Type", "")
|
||||
content_type = ao3_headers.get("Content-Type", ao3_headers.get("content-type", ""))
|
||||
rewritten_body = rewrite_body(raw_body, content_type)
|
||||
|
||||
# Inject Service Worker into HTML responses
|
||||
if "text/html" in content_type:
|
||||
rewritten_body = inject_service_worker(rewritten_body)
|
||||
|
||||
rewritten_headers = rewrite_response_headers(ao3_headers)
|
||||
final_headers = filter_response_headers(rewritten_headers)
|
||||
# Remove Set-Cookie from dict — multi-value Set-Cookie from raw below
|
||||
final_headers.pop("Set-Cookie", None)
|
||||
final_headers.pop("set-cookie", None)
|
||||
final_headers["X-Cache"] = "MISS"
|
||||
|
||||
# Cache successful GET responses
|
||||
if method == "GET" and 200 <= ao3_status < 400:
|
||||
ttl = get_ttl_for_path(full_path)
|
||||
cache.set(ao3_url, rewritten_body, rewritten_headers, ao3_status, ttl=ttl)
|
||||
# Cache successful static GET responses (disk MD5 cache, like go3)
|
||||
if method == "GET" and ao3_status == 200 and is_static_path(full_path):
|
||||
cache.set(ao3_url, rewritten_body, rewritten_headers, ao3_status)
|
||||
|
||||
# Handle redirects
|
||||
if 300 <= ao3_status < 400 and "location" in rewritten_headers:
|
||||
@@ -624,27 +779,94 @@ h1 {{ color: #990000; }}
|
||||
client_ip=client_ip)
|
||||
|
||||
resp = Response(content=rewritten_body, status_code=ao3_status, headers=final_headers)
|
||||
# Forward Set-Cookie from AO3 (user session cookies + cf_clearance)
|
||||
for k, v in ao3_headers.items():
|
||||
if k.lower() == "set-cookie":
|
||||
v_rewritten = v.replace("domain=archiveofourown.org",
|
||||
f"domain={MIRROR_DOMAIN}")
|
||||
v_rewritten = v_rewritten.replace("domain=.archiveofourown.org",
|
||||
f"domain=.{MIRROR_DOMAIN}")
|
||||
resp.headers.add("Set-Cookie", v_rewritten)
|
||||
|
||||
# Append ALL Set-Cookie headers from raw (preserves multi-value — pitfall #18a)
|
||||
# Filter: strip AO3 CF cookies — our own CF edge handles these
|
||||
SKIP_COOKIES = {"__cf_bm", "_cfuvid"}
|
||||
if ao3_headers_raw:
|
||||
rewritten_raw = rewrite_response_headers_raw(ao3_headers_raw)
|
||||
for key, value in rewritten_raw:
|
||||
if key.lower() == "set-cookie":
|
||||
cookie_name = value.split("=")[0].split(";")[0].strip()
|
||||
if cookie_name in SKIP_COOKIES:
|
||||
continue
|
||||
logger.info(f"[SET_COOKIE_FWD] {value[:120]}...")
|
||||
resp.headers.append("Set-Cookie", value)
|
||||
|
||||
# Set sticky session cookie
|
||||
resp.set_cookie(
|
||||
key=STICKY_COOKIE_NAME,
|
||||
value=str(final_proxy_idx),
|
||||
path="/",
|
||||
httponly=True,
|
||||
samesite="lax",
|
||||
)
|
||||
|
||||
add_cors(resp)
|
||||
return resp
|
||||
|
||||
|
||||
# ─── Error pages ───────────────────────────────────────────────────────────
|
||||
|
||||
def _outage_page() -> str:
|
||||
return """<!DOCTYPE html>
|
||||
<html lang="zh-CN">
|
||||
<head><meta charset="UTF-8"><title>AO3 Mirror - 暂时不可用</title>
|
||||
<style>
|
||||
body{font-family:sans-serif;text-align:center;padding:50px;background:#0f0f1a;color:#e0e0e0}
|
||||
h1{color:#990000}
|
||||
.card{background:#1a1a2e;border-radius:10px;padding:30px;max-width:500px;margin:30px auto;border:1px solid #2a2a40}
|
||||
.btn{display:inline-block;padding:10px 24px;background:#990000;color:#fff;text-decoration:none;border-radius:6px;margin-top:15px}
|
||||
</style></head>
|
||||
<body>
|
||||
<div class="card">
|
||||
<h1>⚠️ AO3 暂时不可用</h1>
|
||||
<p>所有代理目前都无法连接 AO3。可能 AO3 正在维护或大规模封锁。</p>
|
||||
<p style="color:#888">请稍后重试。如果持续出现，请切换到数据流量。</p>
|
||||
<a class="btn" href="/" onclick="location.reload()">刷新页面</a>
|
||||
</div>
|
||||
</body>
|
||||
</html>"""
|
||||
|
||||
|
||||
def _transient_error_page() -> str:
|
||||
return """<!DOCTYPE html>
|
||||
<html lang="zh-CN">
|
||||
<head><meta charset="UTF-8"><title>AO3 Mirror - 连接失败</title>
|
||||
<style>
|
||||
body{font-family:sans-serif;text-align:center;padding:50px;background:#0f0f1a;color:#e0e0e0}
|
||||
h1{color:#990000}
|
||||
.card{background:#1a1a2e;border-radius:10px;padding:30px;max-width:500px;margin:30px auto;border:1px solid #2a2a40}
|
||||
.btn{display:inline-block;padding:10px 24px;background:#990000;color:#fff;text-decoration:none;border-radius:6px;margin-top:15px}
|
||||
</style></head>
|
||||
<body>
|
||||
<div class="card">
|
||||
<h1>🔄 正在尝试连接 AO3</h1>
|
||||
<p>镜像站正在通过代理重新连接 AO3 服务器。</p>
|
||||
<p style="color:#888">请稍后刷新页面重试。</p>
|
||||
<a class="btn" href="/" onclick="location.reload()">刷新页面</a>
|
||||
</div>
|
||||
</body>
|
||||
</html>"""
|
||||
|
||||
|
||||
# ─── Helper ────────────────────────────────────────────────────────────────
|
||||
|
||||
async def _sleep(seconds: float):
|
||||
import asyncio
|
||||
await asyncio.sleep(seconds)
|
||||
|
||||
|
||||
# ─── Startup / Shutdown ────────────────────────────────────────────────────
|
||||
|
||||
@app.on_event("startup")
|
||||
async def startup():
|
||||
logger.info("AO3 Mirror backend v4 starting up...")
|
||||
get_proxy_pool()
|
||||
logger.info("AO3 Mirror backend v5 starting up...")
|
||||
pool = get_proxy_pool()
|
||||
get_cache()
|
||||
get_stats_collector()
|
||||
logger.info("AO3 Mirror backend v4 started (cookie-aware + CF challenge solving)")
|
||||
logger.info(f"AO3 Mirror backend v5 started ({len(pool.proxies)} proxies, "
|
||||
f"P2C selection, sticky sessions, SW deploy)")
|
||||
|
||||
|
||||
@app.on_event("shutdown")
|
||||
@@ -660,7 +882,7 @@ async def shutdown():
|
||||
if __name__ == "__main__":
|
||||
port = int(os.environ.get("PORT", "8080"))
|
||||
workers = int(os.environ.get("WORKERS", "4"))
|
||||
logger.info(f"Starting AO3 Mirror v4 on port {port} with {workers} workers")
|
||||
logger.info(f"Starting AO3 Mirror v5 on port {port} with {workers} workers")
|
||||
|
||||
uvicorn.run(
|
||||
"app:app",
|
||||
|
||||
Reference in New Issue
Block a user