085cc7a140
Cookie-aware proxy pool with CF challenge solving - Tiered proxy pool (fast 50 + main 676) - Per-proxy cf_clearance cookie persistence - CF challenge detection + user-browser solving - Safari + Chrome TLS fingerprint rotation - Async FastAPI backend with LRU cache - Passive daemon with systemd supervision - Stats dashboard + Prometheus metrics
673 lines
29 KiB
Python
673 lines
29 KiB
Python
"""
|
||
AO3 反代后端 v4 — Cookie 感知 + CF 挑战用户浏览器求解
|
||
|
||
v4 vs v3:
|
||
- Challenge token 映射：用户浏览器求解 CF 挑战时，保证同一 proxy 亲和性
|
||
- 挑战页面透传：所有代理都遇到 CF 挑战时，把挑战页发给用户浏览器求解
|
||
- Cookie 流：成功响应自动保存 proxy cookie，后续请求自动携带
|
||
- 统计面板展示 cookie-aware 代理数
|
||
"""
|
||
import hashlib
|
||
import json
|
||
import logging
|
||
import os
|
||
import secrets
|
||
import sys
|
||
import threading
|
||
import time
|
||
from urllib.parse import urlparse, urlunparse
|
||
|
||
from fastapi import FastAPI, Request, Response
|
||
from fastapi.responses import HTMLResponse, JSONResponse, PlainTextResponse, RedirectResponse
|
||
import uvicorn
|
||
|
||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||
|
||
from proxy_pool import get_proxy_pool
|
||
from ao3_fetcher import fetch_url, is_cf_challenge
|
||
from url_rewriter import rewrite_body, rewrite_response_headers, needs_rewrite, MIRROR_DOMAIN
|
||
from cache import get_cache, get_ttl_for_path
|
||
from stats import get_stats_collector
|
||
|
||
logging.basicConfig(
|
||
level=logging.INFO,
|
||
format="%(asctime)s [%(levelname)s] %(message)s",
|
||
handlers=[logging.StreamHandler()],
|
||
)
|
||
logger = logging.getLogger("ao3-backend")
|
||
|
||
AO3_BASE = "https://archiveofourown.org"
|
||
MIRROR_HOST = MIRROR_DOMAIN
|
||
|
||
LOCAL_PATHS = {"/stats", "/health", "/metrics", "/favicon.ico", "/robots.txt"}
|
||
|
||
# ─── Challenge Token Map (v4) ──────────────────────────────────────────────
|
||
# Maps challenge_token → (method, ao3_url, headers, body, cookies, proxy_host, expires)
|
||
_challenge_map: dict[str, tuple] = {}
|
||
_challenge_lock = threading.Lock()
|
||
CHALLENGE_TOKEN_TTL = 120 # 2 minutes for the user's browser to solve the challenge
|
||
|
||
|
||
def _make_challenge_token() -> str:
|
||
return secrets.token_urlsafe(16)
|
||
|
||
|
||
def _store_challenge(token: str, method: str, ao3_url: str, headers: dict,
|
||
body: bytes, cookies: dict, proxy_host: str):
|
||
with _challenge_lock:
|
||
# Clean expired tokens
|
||
now = time.time()
|
||
expired = [k for k, v in _challenge_map.items() if v[6] < now]
|
||
for k in expired:
|
||
del _challenge_map[k]
|
||
_challenge_map[token] = (method, ao3_url, headers, body, cookies,
|
||
proxy_host, now + CHALLENGE_TOKEN_TTL)
|
||
|
||
|
||
def _get_challenge(token: str) -> tuple | None:
|
||
with _challenge_lock:
|
||
entry = _challenge_map.get(token)
|
||
if entry and entry[6] > time.time():
|
||
del _challenge_map[token]
|
||
return entry
|
||
if entry:
|
||
del _challenge_map[token] # expired
|
||
return None
|
||
|
||
|
||
# ─── App ───────────────────────────────────────────────────────────────────
|
||
|
||
app = FastAPI(
|
||
title="AO3 Mirror",
|
||
description="AO3 reverse proxy mirror for Chinese users",
|
||
version="4.0.0",
|
||
docs_url=None,
|
||
redoc_url=None,
|
||
)
|
||
|
||
|
||
def get_client_ip(request: Request) -> str:
|
||
cf_ip = request.headers.get("CF-Connecting-IP")
|
||
if cf_ip:
|
||
return cf_ip
|
||
forwarded = request.headers.get("X-Forwarded-For")
|
||
if forwarded:
|
||
return forwarded.split(",")[0].strip()
|
||
return request.client.host if request.client else "unknown"
|
||
|
||
|
||
def build_ao3_url(path: str, query: str = "") -> str:
|
||
url = f"{AO3_BASE}{path}"
|
||
if query:
|
||
url = f"{url}?{query}"
|
||
return url
|
||
|
||
|
||
# ─── Challenge page URL rewriting ──────────────────────────────────────────
|
||
|
||
def _rewrite_challenge_page(body: bytes, proxy_host: str, token: str) -> bytes:
|
||
"""Rewrite CF challenge page so all URLs go through mirror, and tag with challenge token."""
|
||
import re
|
||
|
||
# Basic AO3 domain rewrite
|
||
body = body.replace(b"archiveofourown.org", MIRROR_DOMAIN.encode())
|
||
|
||
# Rewrite /cdn-cgi/ challenge endpoints to go through mirror
|
||
# These are CF's internal challenge platform URLs
|
||
body = body.replace(
|
||
b"/cdn-cgi/challenge-platform",
|
||
f"/cdn-cgi/challenge-platform".encode()
|
||
)
|
||
|
||
# Inject a marker meta tag so we can detect challenge pages coming back
|
||
marker = f'<meta name="cf-challenge-proxy" content="{proxy_host}">'.encode()
|
||
marker_cookie = (
|
||
f'<script>document.cookie="_cf_token={token};path=/;max-age={CHALLENGE_TOKEN_TTL}";</script>'
|
||
).encode()
|
||
body = body.replace(b"<head>", b"<head>" + marker + marker_cookie, 1)
|
||
if b"<head>" not in body:
|
||
body = b"<head>" + marker + marker_cookie + b"</head>" + body
|
||
|
||
return body
|
||
|
||
|
||
# ─── Response header helpers ───────────────────────────────────────────────
|
||
|
||
def filter_response_headers(headers: dict) -> dict:
|
||
blocked = {
|
||
"transfer-encoding", "content-encoding",
|
||
"alt-svc", "cf-ray", "cf-cache-status", "cf-request-id",
|
||
"server", "x-powered-by",
|
||
"cross-origin-resource-policy", "cross-origin-embedder-policy",
|
||
"cross-origin-opener-policy", "cross-origin-window-policy",
|
||
"accept-ch", "critical-ch",
|
||
}
|
||
result = {}
|
||
for key, value in headers.items():
|
||
if key.lower() in blocked:
|
||
continue
|
||
if key.lower().startswith("cf-"):
|
||
continue
|
||
result[key] = value
|
||
result["X-Proxy"] = "AO3-Mirror/4.0"
|
||
result["X-Cache"] = "MISS"
|
||
result["Cache-Control"] = "public, max-age=60, s-maxage=60"
|
||
return result
|
||
|
||
|
||
def add_cors(response: Response):
|
||
response.headers["Access-Control-Allow-Origin"] = "*"
|
||
response.headers["Access-Control-Allow-Methods"] = "GET, POST, HEAD, OPTIONS, PUT, DELETE, PATCH"
|
||
response.headers["Access-Control-Allow-Headers"] = "Content-Type, Authorization, Cookie, X-CSRF-Token"
|
||
response.headers["Access-Control-Max-Age"] = "86400"
|
||
response.headers["Access-Control-Allow-Credentials"] = "true"
|
||
|
||
|
||
# ─── Local Routes ──────────────────────────────────────────────────────────
|
||
|
||
@app.get("/health")
|
||
async def health():
|
||
pool = get_proxy_pool()
|
||
stats = pool.get_stats()
|
||
resp = JSONResponse({
|
||
"status": "ok",
|
||
"timestamp": time.time(),
|
||
"version": "4.0.0",
|
||
"proxy_pool": stats,
|
||
})
|
||
resp.headers["Cache-Control"] = "no-store, no-cache, must-revalidate"
|
||
resp.headers["Pragma"] = "no-cache"
|
||
return resp
|
||
|
||
|
||
@app.get("/robots.txt")
|
||
async def robots():
|
||
return PlainTextResponse(
|
||
"User-agent: *\nDisallow: /stats\nDisallow: /health\n"
|
||
)
|
||
|
||
|
||
# ─── Stats Dashboard ───────────────────────────────────────────────────────
|
||
|
||
@app.get("/stats", response_class=HTMLResponse)
|
||
async def stats_page():
|
||
collector = get_stats_collector()
|
||
pool = get_proxy_pool()
|
||
cache = get_cache()
|
||
|
||
stats = collector.get_overview()
|
||
proxy_stats = pool.get_stats()
|
||
cache_stats = cache.get_stats()
|
||
|
||
hourly = stats.get("hourly", [])
|
||
chart_labels = json.dumps([
|
||
time.strftime("%H:%M", time.localtime(h["hour"]))
|
||
for h in hourly[-24:]
|
||
])
|
||
chart_requests = json.dumps([h["total"] for h in hourly[-24:]])
|
||
chart_success = json.dumps([h["successful"] for h in hourly[-24:]])
|
||
chart_elapsed = json.dumps([h["avg_elapsed"] * 1000 for h in hourly[-24:]])
|
||
|
||
return f"""<!DOCTYPE html>
|
||
<html lang="zh-CN">
|
||
<head>
|
||
<meta charset="UTF-8">
|
||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||
<title>AO3 Mirror - 统计面板 v4</title>
|
||
<script src="https://cdn.jsdelivr.net/npm/chart.js@4"></script>
|
||
<style>
|
||
* {{ margin: 0; padding: 0; box-sizing: border-box; }}
|
||
body {{ font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
|
||
background: #0f0f1a; color: #e0e0e0; padding: 20px; }}
|
||
.container {{ max-width: 1200px; margin: 0 auto; }}
|
||
h1 {{ font-size: 1.8em; margin-bottom: 20px; color: #990000; }}
|
||
h2 {{ font-size: 1.2em; margin-bottom: 12px; color: #ccc; }}
|
||
.grid {{ display: grid; grid-template-columns: repeat(auto-fit, minmax(220px, 1fr)); gap: 15px; margin-bottom: 25px; }}
|
||
.card {{ background: #1a1a2e; border-radius: 10px; padding: 18px; border: 1px solid #2a2a40; }}
|
||
.card .label {{ font-size: 0.8em; color: #888; margin-bottom: 5px; text-transform: uppercase; }}
|
||
.card .value {{ font-size: 1.8em; font-weight: bold; }}
|
||
.card .sub {{ font-size: 0.85em; color: #999; margin-top: 4px; }}
|
||
.green {{ color: #4ade80; }}
|
||
.red {{ color: #f87171; }}
|
||
.yellow {{ color: #fbbf24; }}
|
||
.blue {{ color: #60a5fa; }}
|
||
.purple {{ color: #a78bfa; }}
|
||
.orange {{ color: #fb923c; }}
|
||
.chart-container {{ background: #1a1a2e; border-radius: 10px; padding: 18px; margin-bottom: 25px; border: 1px solid #2a2a40; }}
|
||
.chart-row {{ display: grid; grid-template-columns: 1fr 1fr; gap: 15px; }}
|
||
table {{ width: 100%; border-collapse: collapse; }}
|
||
th, td {{ padding: 8px 12px; text-align: left; border-bottom: 1px solid #2a2a40; font-size: 0.9em; }}
|
||
th {{ color: #888; text-transform: uppercase; font-size: 0.8em; }}
|
||
.badge {{ display: inline-block; padding: 2px 8px; border-radius: 4px; font-size: 0.8em; }}
|
||
.badge-green {{ background: rgba(74, 222, 128, 0.15); color: #4ade80; }}
|
||
.badge-red {{ background: rgba(248, 113, 113, 0.15); color: #f87171; }}
|
||
@media (max-width: 768px) {{ .chart-row {{ grid-template-columns: 1fr; }} }}
|
||
</style>
|
||
</head>
|
||
<body>
|
||
<div class="container">
|
||
<h1>🛡 AO3 Mirror 统计面板 v4</h1>
|
||
|
||
<div class="grid">
|
||
<div class="card">
|
||
<div class="label">总请求数</div>
|
||
<div class="value blue">{stats['total_requests']:,}</div>
|
||
<div class="sub">运行 {stats['uptime_human']}</div>
|
||
</div>
|
||
<div class="card">
|
||
<div class="label">最近 1 分钟</div>
|
||
<div class="value yellow">{stats['recent_1m']:,} req</div>
|
||
<div class="sub">最近 5 分钟: {stats['recent_5m']:,}</div>
|
||
</div>
|
||
<div class="card">
|
||
<div class="label">成功率</div>
|
||
<div class="value green">{stats['success_rate']}%</div>
|
||
<div class="sub">失败: {stats['failed']:,}</div>
|
||
</div>
|
||
<div class="card">
|
||
<div class="label">缓存命中率</div>
|
||
<div class="value purple">{stats['cache_rate']}%</div>
|
||
<div class="sub">已缓存: {stats['cached']:,}</div>
|
||
</div>
|
||
<div class="card">
|
||
<div class="label">平均延迟</div>
|
||
<div class="value">{stats['avg_elapsed_ms']} ms</div>
|
||
<div class="sub">P99: {stats['p99_elapsed_ms']} ms</div>
|
||
</div>
|
||
<div class="card">
|
||
<div class="label">代理池</div>
|
||
<div class="value green">{proxy_stats['alive']}<span style="font-size:0.5em;color:#888;">/{proxy_stats['total']}</span></div>
|
||
<div class="sub">可用: {proxy_stats['available']} | 受损: {proxy_stats['dead']} | Banned: {proxy_stats['banned']}</div>
|
||
</div>
|
||
<div class="card">
|
||
<div class="label">Cookie 代理</div>
|
||
<div class="value orange">{proxy_stats['with_cookies']}<span style="font-size:0.5em;color:#888;">/{proxy_stats['alive']}</span></div>
|
||
<div class="sub">已持有 cf_clearance</div>
|
||
</div>
|
||
<div class="card">
|
||
<div class="label">本地缓存</div>
|
||
<div class="value">{cache_stats['size']}<span style="font-size:0.5em;color:#888;">/{cache_stats['capacity']}</span></div>
|
||
<div class="sub">已用: {cache_stats['usage_pct']}%</div>
|
||
</div>
|
||
</div>
|
||
|
||
<div class="chart-row">
|
||
<div class="chart-container">
|
||
<h2>请求趋势 (最近 24 小时)</h2>
|
||
<canvas id="requestChart" height="150"></canvas>
|
||
</div>
|
||
<div class="chart-container">
|
||
<h2>响应延迟 (最近 24 小时)</h2>
|
||
<canvas id="latencyChart" height="150"></canvas>
|
||
</div>
|
||
</div>
|
||
|
||
<div class="chart-container">
|
||
<h2>热门路径</h2>
|
||
<table>
|
||
<tr><th>路径</th><th>请求数</th><th>成功</th><th>平均延迟</th></tr>
|
||
{''.join(f'<tr><td>{p["path"]}</td><td>{p["requests"]:,}</td><td><span class="badge {"badge-green" if p["requests"]==0 or p["successful"]/max(p["requests"],1)>0.8 else "badge-red"}">{round(p["successful"]/max(p["requests"],1)*100)}%</span></td><td>{p["avg_elapsed"]*1000:.0f}ms</td></tr>' for p in stats['top_paths'][:15])}
|
||
</table>
|
||
</div>
|
||
</div>
|
||
|
||
<script>
|
||
new Chart(document.getElementById('requestChart'), {{
|
||
type: 'line',
|
||
data: {{
|
||
labels: {chart_labels},
|
||
datasets: [{{
|
||
label: '总请求',
|
||
data: {chart_requests},
|
||
borderColor: '#60a5fa',
|
||
backgroundColor: 'rgba(96,165,250,0.1)',
|
||
fill: true, tension: 0.3, pointRadius: 1,
|
||
}}, {{
|
||
label: '成功',
|
||
data: {chart_success},
|
||
borderColor: '#4ade80',
|
||
backgroundColor: 'rgba(74,222,128,0.1)',
|
||
fill: true, tension: 0.3, pointRadius: 1,
|
||
}}]
|
||
}},
|
||
options: {{
|
||
responsive: true, maintainAspectRatio: false,
|
||
plugins: {{ legend: {{ labels: {{ color: '#ccc' }} }} }},
|
||
scales: {{
|
||
x: {{ ticks: {{ color: '#888', maxTicksLimit: 12 }}, grid: {{ color: '#2a2a40' }} }},
|
||
y: {{ beginAtZero: true, ticks: {{ color: '#888' }}, grid: {{ color: '#2a2a40' }} }}
|
||
}}
|
||
}}
|
||
}});
|
||
new Chart(document.getElementById('latencyChart'), {{
|
||
type: 'bar',
|
||
data: {{
|
||
labels: {chart_labels},
|
||
datasets: [{{
|
||
label: '平均延迟 (ms)',
|
||
data: {chart_elapsed},
|
||
backgroundColor: 'rgba(167,139,250,0.5)',
|
||
borderColor: '#a78bfa', borderWidth: 1, borderRadius: 3,
|
||
}}]
|
||
}},
|
||
options: {{
|
||
responsive: true, maintainAspectRatio: false,
|
||
plugins: {{ legend: {{ labels: {{ color: '#ccc' }} }} }},
|
||
scales: {{
|
||
x: {{ ticks: {{ color: '#888', maxTicksLimit: 12 }}, grid: {{ color: '#2a2a40' }} }},
|
||
y: {{ beginAtZero: true, ticks: {{ color: '#888' }}, grid: {{ color: '#2a2a40' }} }}
|
||
}}
|
||
}}
|
||
}});
|
||
</script>
|
||
</body>
|
||
</html>"""
|
||
|
||
|
||
# ─── Metrics ───────────────────────────────────────────────────────────────
|
||
|
||
@app.get("/metrics")
|
||
async def metrics():
|
||
collector = get_stats_collector()
|
||
pool = get_proxy_pool()
|
||
cache = get_cache()
|
||
|
||
stats = collector.get_overview()
|
||
proxy_stats = pool.get_stats()
|
||
cache_stats = cache.get_stats()
|
||
|
||
lines = [
|
||
"# HELP ao3_mirror_requests_total Total proxy requests",
|
||
"# TYPE ao3_mirror_requests_total counter",
|
||
f'ao3_mirror_requests_total {stats["total_requests"]}',
|
||
"# HELP ao3_mirror_successful_requests Successful requests",
|
||
"# TYPE ao3_mirror_successful_requests counter",
|
||
f'ao3_mirror_successful_requests {stats["successful"]}',
|
||
"# HELP ao3_mirror_failed_requests Failed requests",
|
||
"# TYPE ao3_mirror_failed_requests counter",
|
||
f'ao3_mirror_failed_requests {stats["failed"]}',
|
||
"# HELP ao3_mirror_cache_hits Cache hit count",
|
||
"# TYPE ao3_mirror_cache_hits counter",
|
||
f'ao3_mirror_cache_hits {stats["cached"]}',
|
||
"# HELP ao3_mirror_avg_elapsed_ms Average response time",
|
||
"# TYPE ao3_mirror_avg_elapsed_ms gauge",
|
||
f'ao3_mirror_avg_elapsed_ms {stats["avg_elapsed_ms"]}',
|
||
"# HELP ao3_mirror_proxy_pool Proxy pool status",
|
||
"# TYPE ao3_mirror_proxy_pool gauge",
|
||
f'ao3_mirror_proxy_alive {proxy_stats["alive"]}',
|
||
f'ao3_mirror_proxy_dead {proxy_stats["dead"]}',
|
||
f'ao3_mirror_proxy_banned {proxy_stats["banned"]}',
|
||
f'ao3_mirror_proxy_available {proxy_stats["available"]}',
|
||
f'ao3_mirror_proxy_with_cookies {proxy_stats["with_cookies"]}',
|
||
"# HELP ao3_mirror_cache_size Current cache size",
|
||
"# TYPE ao3_mirror_cache_size gauge",
|
||
f'ao3_mirror_cache_size {cache_stats["size"]}',
|
||
]
|
||
return PlainTextResponse("\n".join(lines))
|
||
|
||
|
||
# ─── Proxy Core (v4) ──────────────────────────────────────────────────────
|
||
|
||
@app.api_route("/{path:path}", methods=["GET", "POST", "HEAD", "OPTIONS", "PUT", "DELETE", "PATCH"])
|
||
async def proxy_handler(request: Request, path: str):
|
||
"""Main proxy handler — with cookie-aware fetching and CF challenge solving."""
|
||
start_time = time.time()
|
||
client_ip = get_client_ip(request)
|
||
|
||
if path == "" or path == "/":
|
||
path = ""
|
||
|
||
full_path = f"/{path}" if path else "/"
|
||
if full_path in LOCAL_PATHS or full_path.startswith("/stats") or full_path.startswith("/health"):
|
||
return JSONResponse({"error": "Not found"}, status_code=404)
|
||
|
||
if request.method == "OPTIONS":
|
||
resp = Response()
|
||
add_cors(resp)
|
||
return resp
|
||
|
||
# ── v4: Challenge token check ──────────────────────────────────────────
|
||
# If this request has a _cf_token cookie, it's part of a challenge resolution flow
|
||
cf_token = request.cookies.get("_cf_token") or request.query_params.get("_cf_token")
|
||
preferred_proxy = None
|
||
if cf_token:
|
||
challenge_entry = _get_challenge(cf_token)
|
||
if challenge_entry:
|
||
orig_method, orig_url, orig_headers, orig_body, orig_cookies, proxy_host, _ = challenge_entry
|
||
preferred_proxy = proxy_host
|
||
logger.info(f"Challenge resolution: using proxy {proxy_host} for {orig_url[:60]}")
|
||
# Re-fetch the original request with the challenge proxy
|
||
result = await fetch_url(
|
||
url=orig_url, method=orig_method,
|
||
headers=dict(request.headers),
|
||
body=await request.body() if orig_method in ("POST", "PUT", "PATCH") else None,
|
||
cookies=orig_cookies,
|
||
is_api="/api/" in orig_url,
|
||
preferred_proxy=preferred_proxy,
|
||
)
|
||
|
||
if result["success"]:
|
||
# Challenge solved! Proxy now has cf_clearance. Return content.
|
||
elapsed = time.time() - start_time
|
||
rewritten_body = rewrite_body(result["body"],
|
||
result["headers"].get("Content-Type", ""))
|
||
rewritten_headers = rewrite_response_headers(result["headers"])
|
||
final_headers = filter_response_headers(rewritten_headers)
|
||
final_headers["X-Cache"] = "MISS"
|
||
final_headers["X-CF-Status"] = "solved"
|
||
|
||
# Set cf_clearance as a cookie on the mirror domain too
|
||
# so subsequent requests benefit
|
||
resp = Response(content=rewritten_body, status_code=result["status"],
|
||
headers=final_headers)
|
||
# Forward any Set-Cookie from AO3 (includes cf_clearance)
|
||
for k, v in result["headers"].items():
|
||
if k.lower() == "set-cookie":
|
||
# Rewrite domain
|
||
v_rewritten = v.replace("domain=archiveofourown.org",
|
||
f"domain={MIRROR_DOMAIN}")
|
||
v_rewritten = v_rewritten.replace("domain=.archiveofourown.org",
|
||
f"domain=.{MIRROR_DOMAIN}")
|
||
resp.headers.add("Set-Cookie", v_rewritten)
|
||
|
||
# Clean up challenge token cookie
|
||
resp.delete_cookie("_cf_token", path="/")
|
||
|
||
collector = get_stats_collector()
|
||
collector.log_request(method=orig_method, path=full_path,
|
||
status=result["status"], elapsed=elapsed,
|
||
cached=False, proxy_host=preferred_proxy,
|
||
client_ip=client_ip)
|
||
add_cors(resp)
|
||
return resp
|
||
|
||
# ── Normal request flow ────────────────────────────────────────────────
|
||
|
||
query_string = request.url.query
|
||
ao3_url = build_ao3_url(f"/{path}" if path else "/", query_string)
|
||
|
||
# Check cache
|
||
cache = get_cache()
|
||
cached = cache.get(ao3_url, dict(request.headers))
|
||
if cached:
|
||
body, resp_headers, status = cached
|
||
elapsed = time.time() - start_time
|
||
collector = get_stats_collector()
|
||
collector.log_request(method=request.method, path=full_path, status=status,
|
||
elapsed=elapsed, cached=True, client_ip=client_ip)
|
||
headers = filter_response_headers(resp_headers)
|
||
headers["X-Cache"] = "HIT"
|
||
return Response(content=body, status_code=status, headers=headers)
|
||
|
||
# Prepare request
|
||
method = request.method
|
||
client_headers = dict(request.headers)
|
||
req_body = None
|
||
if method in ("POST", "PUT", "PATCH"):
|
||
req_body = await request.body()
|
||
|
||
cookies = {}
|
||
cookie_header = request.headers.get("cookie", "")
|
||
if cookie_header:
|
||
for pair in cookie_header.split(";"):
|
||
if "=" in pair:
|
||
k, v = pair.split("=", 1)
|
||
cookies[k.strip()] = v.strip()
|
||
|
||
# Forward to AO3
|
||
result = await fetch_url(
|
||
url=ao3_url, method=method,
|
||
headers=client_headers, body=req_body, cookies=cookies,
|
||
is_api="/api/" in full_path or full_path.startswith("/api/"),
|
||
preferred_proxy=preferred_proxy,
|
||
)
|
||
|
||
elapsed = time.time() - start_time
|
||
|
||
# ── v4: CF Challenge handling ──────────────────────────────────────────
|
||
if result.get("is_challenge") and result.get("challenge_body"):
|
||
challenge_proxy = result.get("challenge_proxy", "unknown")
|
||
challenge_body = result["challenge_body"]
|
||
|
||
# Generate challenge token for proxy affinity
|
||
token = _make_challenge_token()
|
||
_store_challenge(token, method, ao3_url, client_headers,
|
||
req_body or b"", cookies, challenge_proxy)
|
||
|
||
# Rewrite challenge page for user's browser
|
||
rewritten_challenge = _rewrite_challenge_page(challenge_body, challenge_proxy, token)
|
||
|
||
logger.warning(
|
||
f"CF Challenge detected via {challenge_proxy} for {ao3_url[:80]}. "
|
||
f"Sending challenge to user browser (token={token[:8]}...)"
|
||
)
|
||
|
||
collector = get_stats_collector()
|
||
collector.log_request(method=method, path=full_path, status=503,
|
||
elapsed=elapsed, cached=False,
|
||
proxy_host=challenge_proxy, client_ip=client_ip)
|
||
|
||
# Return challenge page with 503 status + token cookie
|
||
resp = HTMLResponse(
|
||
content=rewritten_challenge,
|
||
status_code=503,
|
||
headers={
|
||
"X-CF-Challenge": "true",
|
||
"X-CF-Challenge-Proxy": challenge_proxy,
|
||
"Retry-After": "5",
|
||
},
|
||
)
|
||
resp.set_cookie(
|
||
key="_cf_token", value=token,
|
||
path="/", max_age=CHALLENGE_TOKEN_TTL,
|
||
httponly=False, # JS needs to read it
|
||
samesite="lax",
|
||
)
|
||
resp.delete_cookie("cf_clearance", path="/") # Clear stale clearance
|
||
add_cors(resp)
|
||
return resp
|
||
|
||
# ── Standard failure ───────────────────────────────────────────────────
|
||
if not result["success"]:
|
||
logger.error(f"Failed to fetch {ao3_url[:80]}: {result.get('error', 'unknown')}")
|
||
collector = get_stats_collector()
|
||
collector.log_request(method=method, path=full_path, status=502,
|
||
elapsed=elapsed, cached=False, client_ip=client_ip)
|
||
|
||
error_html = f"""<!DOCTYPE html>
|
||
<html lang="zh-CN">
|
||
<head><meta charset="UTF-8"><title>AO3 Mirror - 暂时不可用</title>
|
||
<style>
|
||
body {{ font-family: sans-serif; text-align: center; padding: 50px; background: #0f0f1a; color: #e0e0e0; }}
|
||
h1 {{ color: #990000; }}
|
||
.card {{ background: #1a1a2e; border-radius: 10px; padding: 30px; max-width: 500px; margin: 30px auto; border: 1px solid #2a2a40; }}
|
||
.btn {{ display: inline-block; padding: 10px 24px; background: #990000; color: white; text-decoration: none; border-radius: 6px; margin-top: 15px; }}
|
||
</style></head>
|
||
<body>
|
||
<div class="card">
|
||
<h1>🔄 正在尝试连接 AO3</h1>
|
||
<p>镜像站正在尝试通过代理重新连接 AO3 服务器。</p>
|
||
<p style="color: #888; font-size: 0.9em;">请稍后刷新页面重试。</p>
|
||
<a class="btn" href="/" onclick="location.reload()">刷新页面</a>
|
||
</div>
|
||
</body>
|
||
</html>"""
|
||
return HTMLResponse(content=error_html, status_code=502)
|
||
|
||
# ── Success ────────────────────────────────────────────────────────────
|
||
ao3_status = result["status"]
|
||
ao3_headers = result.get("headers", {})
|
||
raw_body = result.get("body", b"")
|
||
|
||
content_type = ao3_headers.get("Content-Type", "")
|
||
rewritten_body = rewrite_body(raw_body, content_type)
|
||
rewritten_headers = rewrite_response_headers(ao3_headers)
|
||
final_headers = filter_response_headers(rewritten_headers)
|
||
final_headers["X-Cache"] = "MISS"
|
||
|
||
# Cache successful GET responses
|
||
if method == "GET" and 200 <= ao3_status < 400:
|
||
ttl = get_ttl_for_path(full_path)
|
||
cache.set(ao3_url, rewritten_body, rewritten_headers, ao3_status, ttl=ttl)
|
||
|
||
# Handle redirects
|
||
if 300 <= ao3_status < 400 and "location" in rewritten_headers:
|
||
redirect_url = rewritten_headers["location"]
|
||
if AO3_BASE in redirect_url:
|
||
redirect_url = redirect_url.replace(AO3_BASE, "").replace("http://", "https://")
|
||
return RedirectResponse(url=redirect_url, status_code=ao3_status)
|
||
|
||
collector = get_stats_collector()
|
||
collector.log_request(method=method, path=full_path, status=ao3_status,
|
||
elapsed=elapsed, cached=False,
|
||
proxy_host=result.get("proxy_host"),
|
||
client_ip=client_ip)
|
||
|
||
resp = Response(content=rewritten_body, status_code=ao3_status, headers=final_headers)
|
||
# Forward Set-Cookie from AO3 (user session cookies + cf_clearance)
|
||
for k, v in ao3_headers.items():
|
||
if k.lower() == "set-cookie":
|
||
v_rewritten = v.replace("domain=archiveofourown.org",
|
||
f"domain={MIRROR_DOMAIN}")
|
||
v_rewritten = v_rewritten.replace("domain=.archiveofourown.org",
|
||
f"domain=.{MIRROR_DOMAIN}")
|
||
resp.headers.add("Set-Cookie", v_rewritten)
|
||
add_cors(resp)
|
||
return resp
|
||
|
||
|
||
# ─── Startup / Shutdown ────────────────────────────────────────────────────
|
||
|
||
@app.on_event("startup")
|
||
async def startup():
|
||
logger.info("AO3 Mirror backend v4 starting up...")
|
||
get_proxy_pool()
|
||
get_cache()
|
||
get_stats_collector()
|
||
logger.info("AO3 Mirror backend v4 started (cookie-aware + CF challenge solving)")
|
||
|
||
|
||
@app.on_event("shutdown")
|
||
async def shutdown():
|
||
logger.info("AO3 Mirror backend shutting down...")
|
||
pool = get_proxy_pool()
|
||
await pool.close_all()
|
||
logger.info("AO3 Mirror backend stopped")
|
||
|
||
|
||
# ─── Entry point ───────────────────────────────────────────────────────────
|
||
|
||
if __name__ == "__main__":
|
||
port = int(os.environ.get("PORT", "8080"))
|
||
workers = int(os.environ.get("WORKERS", "4"))
|
||
logger.info(f"Starting AO3 Mirror v4 on port {port} with {workers} workers")
|
||
|
||
uvicorn.run(
|
||
"app:app",
|
||
host="127.0.0.1",
|
||
port=port,
|
||
workers=workers,
|
||
log_level="info",
|
||
timeout_keep_alive=30,
|
||
)
|