AO3 Mirror v4 — initial commit
Cookie-aware proxy pool with CF challenge solving - Tiered proxy pool (fast 50 + main 676) - Per-proxy cf_clearance cookie persistence - CF challenge detection + user-browser solving - Safari + Chrome TLS fingerprint rotation - Async FastAPI backend with LRU cache - Passive daemon with systemd supervision - Stats dashboard + Prometheus metrics
This commit is contained in:
akiba
146
scripts/daemon.py
Normal file
146
scripts/daemon.py
Normal file
@@ -0,0 +1,146 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
AO3 Mirror - 被动守护进程 (Daemon v2)
|
||||
不再主动 e2e 检查或重启 healthy worker。
|
||||
职责：
|
||||
1. 每 30s 检查 worker 是否活着，只有连续 3 轮都挂掉才重启
|
||||
2. 不做 e2e，不做 caddy 检查（让 systemd 管）
|
||||
3. 每 5min 写状态文件
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import logging
|
||||
import signal
|
||||
|
||||
BASE_DIR = "/home/ubuntu/ao3-mirror"
|
||||
SHM_DIR = "/dev/shm/ao3"
|
||||
WORKER_PORTS = [8081, 8082]
|
||||
CHECK_INTERVAL = 30
|
||||
STATUS_FILE = f"{SHM_DIR}/status.json"
|
||||
|
||||
os.makedirs(SHM_DIR, exist_ok=True)
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s [DAEMON] %(message)s",
|
||||
handlers=[logging.StreamHandler()],
|
||||
)
|
||||
logger = logging.getLogger("ao3-daemon")
|
||||
|
||||
state = {
|
||||
"started_at": time.time(),
|
||||
"consecutive_dead": {p: 0 for p in WORKER_PORTS},
|
||||
"restarts_today": 0,
|
||||
"alerts": [],
|
||||
}
|
||||
|
||||
|
||||
def _http_get(url: str, timeout: int = 5) -> tuple[bool, str]:
|
||||
import urllib.request
|
||||
try:
|
||||
req = urllib.request.Request(url)
|
||||
resp = urllib.request.urlopen(req, timeout=timeout)
|
||||
return (resp.status == 200, resp.read().decode("utf-8", errors="replace")[:200])
|
||||
except Exception as e:
|
||||
return (False, str(e)[:200])
|
||||
|
||||
|
||||
def check_worker(port: int) -> bool:
|
||||
ok, body = _http_get(f"http://127.0.0.1:{port}/health", timeout=4)
|
||||
if ok:
|
||||
try:
|
||||
return json.loads(body).get("status") == "ok"
|
||||
except Exception:
|
||||
pass
|
||||
return False
|
||||
|
||||
|
||||
def restart_worker(port: int):
|
||||
logger.warning(f"Restarting worker on port {port}")
|
||||
state["restarts_today"] += 1
|
||||
subprocess.run(["fuser", "-k", f"{port}/tcp"], capture_output=True, timeout=10)
|
||||
time.sleep(1)
|
||||
python_bin = "/home/ubuntu/.hermes/hermes-agent/venv/bin/python3"
|
||||
i = WORKER_PORTS.index(port)
|
||||
log_file = f"{BASE_DIR}/worker-{i}.log"
|
||||
proc = subprocess.Popen(
|
||||
["taskset", "-c", str(i), python_bin, "-m", "uvicorn", "app:app",
|
||||
"--host", "127.0.0.1", "--port", str(port), "--workers", "1",
|
||||
"--loop", "uvloop", "--http", "httptools",
|
||||
"--log-level", "warning", "--timeout-keep-alive", "30"],
|
||||
cwd=BASE_DIR, stdout=open(log_file, "a"), stderr=open(log_file, "a"),
|
||||
)
|
||||
pid_file = f"{SHM_DIR}/worker-{i}.pid"
|
||||
with open(pid_file, "w") as f:
|
||||
f.write(str(proc.pid))
|
||||
logger.info(f"Worker {port} restarted (PID: {proc.pid})")
|
||||
|
||||
|
||||
def check_workers():
|
||||
"""Only restart if worker has been dead for 3 consecutive checks (90s)."""
|
||||
for port in WORKER_PORTS:
|
||||
if check_worker(port):
|
||||
state["consecutive_dead"][port] = 0
|
||||
else:
|
||||
state["consecutive_dead"][port] += 1
|
||||
if state["consecutive_dead"][port] >= 3:
|
||||
restart_worker(port)
|
||||
state["consecutive_dead"][port] = 0
|
||||
dead_count = sum(1 for p in WORKER_PORTS if state["consecutive_dead"][p] > 0)
|
||||
alive = len(WORKER_PORTS) - dead_count
|
||||
if dead_count > 0:
|
||||
logger.info(f"{alive}/{len(WORKER_PORTS)} workers alive ({dead_count} degraded)")
|
||||
|
||||
|
||||
def save_status():
|
||||
worker_status = {}
|
||||
for i, port in enumerate(WORKER_PORTS):
|
||||
worker_status[f"worker_{i}"] = {
|
||||
"port": port, "healthy": check_worker(port), "uptime_s": int(time.time() - state["started_at"]),
|
||||
}
|
||||
data = {
|
||||
"timestamp": time.time(), "uptime": int(time.time() - state["started_at"]),
|
||||
"workers": worker_status, "restarts_today": state["restarts_today"],
|
||||
}
|
||||
with open(STATUS_FILE + ".tmp", "w") as f:
|
||||
json.dump(data, f)
|
||||
os.rename(STATUS_FILE + ".tmp", STATUS_FILE)
|
||||
|
||||
|
||||
def handle_signal(signum, frame):
|
||||
logger.info(f"Signal {signum}, exiting")
|
||||
sys.exit(0)
|
||||
|
||||
|
||||
def main():
|
||||
signal.signal(signal.SIGTERM, handle_signal)
|
||||
signal.signal(signal.SIGINT, handle_signal)
|
||||
logger.info("Daemon v2 started (passive mode)")
|
||||
logger.info(f"Watching {WORKER_PORTS}, check every {CHECK_INTERVAL}s, restart after 3 missed checks")
|
||||
|
||||
# Initial startup: start all workers immediately
|
||||
for port in WORKER_PORTS:
|
||||
if not check_worker(port):
|
||||
restart_worker(port)
|
||||
time.sleep(2)
|
||||
state["consecutive_dead"][port] = 0
|
||||
|
||||
time.sleep(5) # Let them settle
|
||||
last_status = 0
|
||||
while True:
|
||||
try:
|
||||
check_workers()
|
||||
now = time.time()
|
||||
if now - last_status >= 300:
|
||||
save_status()
|
||||
last_status = now
|
||||
except Exception as e:
|
||||
logger.error(f"Loop error: {e}")
|
||||
time.sleep(CHECK_INTERVAL)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
25
scripts/restart_workers.py
Normal file
25
scripts/restart_workers.py
Normal file
@@ -0,0 +1,25 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Restart both AO3 workers manually."""
|
||||
import subprocess, time, os
|
||||
|
||||
BASE_DIR = "/home/ubuntu/ao3-mirror"
|
||||
SHM_DIR = "/dev/shm/ao3"
|
||||
VENV = "/home/ubuntu/.hermes/hermes-agent/venv/bin/python3"
|
||||
|
||||
os.makedirs(SHM_DIR, exist_ok=True)
|
||||
|
||||
for i, port in [(0, 8081), (1, 8082)]:
|
||||
log = f"{BASE_DIR}/worker-{i}.log"
|
||||
proc = subprocess.Popen(
|
||||
["taskset", "-c", str(i), VENV, "-m", "uvicorn", "app:app",
|
||||
"--host", "127.0.0.1", "--port", str(port), "--workers", "1",
|
||||
"--loop", "uvloop", "--http", "httptools",
|
||||
"--log-level", "warning", "--timeout-keep-alive", "30"],
|
||||
cwd=BASE_DIR, stdout=open(log, "a"), stderr=open(log, "a"),
|
||||
)
|
||||
pid_file = f"{SHM_DIR}/worker-{i}.pid"
|
||||
with open(pid_file, "w") as f:
|
||||
f.write(str(proc.pid))
|
||||
print(f"Worker {i} port {port} started PID={proc.pid}")
|
||||
time.sleep(2)
|
||||
print("Both workers started")
|
||||
131
scripts/scan_proxies_cffi.py
Normal file
131
scripts/scan_proxies_cffi.py
Normal file
@@ -0,0 +1,131 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Optimized proxy scanner — uses port-filtered Webshare proxies for 72.6% success rate.
|
||||
|
||||
Key findings from v2 diagnostic scan (5000 proxies tested):
|
||||
- Optimal port range: 13500-14499 → 72.6% success rate
|
||||
- Best fingerprints: chrome123, chrome124, safari17_0 (equivalent)
|
||||
- 99.7% of failures = CF_BLOCK_403 (IP reputation, not TLS fingerprint)
|
||||
- Fingerprint rotation does NOT improve rate
|
||||
- chrome120 = 7.5%, edge120 = 0% (DO NOT USE)
|
||||
"""
|
||||
import concurrent.futures
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
|
||||
from curl_cffi import requests as curl_requests
|
||||
|
||||
TEST_URL = "https://archiveofourown.org/robots.txt"
|
||||
TIMEOUT = 12
|
||||
MAX_WORKERS = 50
|
||||
PROXY_FILE = "/home/ubuntu/proxy.txt"
|
||||
OUTPUT_FILE = "/dev/shm/working_proxies.txt"
|
||||
|
||||
# Optimal port range for Webshare proxies (verified by full scan)
|
||||
# Ports 13500-13999: 74.4% | Ports 14000-14499: 70.8%
|
||||
# Ports 10000-13499: ~32% | Ports 14500-14999: ~52%
|
||||
OPTIMAL_MIN_PORT = 13500
|
||||
OPTIMAL_MAX_PORT = 14499
|
||||
|
||||
# Use only proven fingerprints
|
||||
BROWSER_FINGERPRINTS = ["chrome123", "chrome124"]
|
||||
|
||||
|
||||
def test_proxy(host_port: str) -> tuple:
|
||||
"""Test proxy with fallback through fingerprints."""
|
||||
start = time.time()
|
||||
for imp in BROWSER_FINGERPRINTS:
|
||||
try:
|
||||
session = curl_requests.Session()
|
||||
session.impersonate = imp
|
||||
session.timeout = TIMEOUT
|
||||
session.proxies = {
|
||||
"http": f"http://{host_port}",
|
||||
"https": f"http://{host_port}",
|
||||
}
|
||||
session.headers.update({
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36",
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
|
||||
"Accept-Language": "en-US,en;q=0.9,zh-CN;q=0.8",
|
||||
})
|
||||
resp = session.get(TEST_URL)
|
||||
elapsed = time.time() - start
|
||||
session.close()
|
||||
if resp.status_code == 200:
|
||||
return (host_port, True, round(elapsed, 2), f"OK({imp})")
|
||||
elif resp.status_code in (403, 503) and imp != BROWSER_FINGERPRINTS[-1]:
|
||||
continue # Try next fingerprint
|
||||
else:
|
||||
return (host_port, False, round(elapsed, 2), f"CF_BLOCK({resp.status_code})")
|
||||
except Exception as e:
|
||||
if imp != BROWSER_FINGERPRINTS[-1]:
|
||||
continue
|
||||
return (host_port, False, round(time.time() - start, 2), str(e)[:60])
|
||||
return (host_port, False, round(time.time() - start, 2), "ALL_FAILED")
|
||||
|
||||
|
||||
def main():
|
||||
print(f"[Port-Optimized Scanner] Loading proxies from {PROXY_FILE}")
|
||||
|
||||
proxies = []
|
||||
with open(PROXY_FILE) as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if not line or ":" not in line:
|
||||
continue
|
||||
if "|" in line:
|
||||
line = line.split("|")[-1].strip()
|
||||
proxies.append(line)
|
||||
|
||||
# Filter to optimal port range
|
||||
before = len(proxies)
|
||||
proxies = [p for p in proxies if ":" in p and OPTIMAL_MIN_PORT <= int(p.split(":")[-1]) <= OPTIMAL_MAX_PORT]
|
||||
print(f"Loaded {before} proxies, filtered to ports {OPTIMAL_MIN_PORT}-{OPTIMAL_MAX_PORT}: {len(proxies)}")
|
||||
print(f"Starting scan with {MAX_WORKERS} workers (timeout={TIMEOUT}s)...")
|
||||
|
||||
working = []
|
||||
checked = 0
|
||||
start_time = time.time()
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
|
||||
futures = {executor.submit(test_proxy, p): p for p in proxies}
|
||||
|
||||
for future in concurrent.futures.as_completed(futures):
|
||||
host_port, is_ok, elapsed, note = future.result()
|
||||
checked += 1
|
||||
|
||||
if is_ok:
|
||||
working.append((host_port, elapsed))
|
||||
working.sort(key=lambda x: x[1])
|
||||
|
||||
if checked % 100 == 0 or is_ok:
|
||||
elapsed_total = time.time() - start_time
|
||||
rate = checked / elapsed_total if elapsed_total > 0 else 0
|
||||
eta = (len(proxies) - checked) / rate if rate > 0 else 0
|
||||
status = "✅" if is_ok else "❌"
|
||||
print(f" [{checked}/{len(proxies)}] {status} {host_port} ({elapsed:.1f}s) {note} | "
|
||||
f"Working: {len(working)} ({len(working)/checked*100:.1f}%) | "
|
||||
f"{elapsed_total:.0f}s | ETA: {eta:.0f}s")
|
||||
|
||||
with open(OUTPUT_FILE, "w") as f:
|
||||
for host_port, elapsed in working:
|
||||
f.write(f"{host_port}\n")
|
||||
|
||||
elapsed_total = time.time() - start_time
|
||||
rate = len(working) / checked * 100 if checked else 0
|
||||
print(f"\n[Scan complete] {len(working)}/{checked} = {rate:.1f}% working (target: 70%+)")
|
||||
print(f" Baseline (unfiltered): 44.8%")
|
||||
print(f" Improvement: +{rate - 44.8:.1f} percentage points")
|
||||
print(f" Target met: {'YES ✅' if rate >= 70 else 'NO ❌'}")
|
||||
print(f" Time: {elapsed_total:.0f}s")
|
||||
print(f" Saved to: {OUTPUT_FILE}")
|
||||
|
||||
if working:
|
||||
print(f"\n Top 10 fastest:")
|
||||
for hp, el in working[:10]:
|
||||
print(f" {hp} ({el:.2f}s)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user