Unify broker-mode binding changes in database transactions: advance device generation, invalidate pending authorizations/tasks and prior pairing-code epochs, and persist audit plus invalidation outbox. Add authenticated device unbind alongside robot/admin paths. Pairing codes are hashed, generation- fenced and consumed atomically with binding. Replay Redis invalidation effects without undoing committed unbind. Guard online cleanup and delayed kill events by generation. Introduce a separate router admission credential with no platform-event injection authority. Router verifies signed nonce-bound internal admission on handshake, task traffic and periodic checks; reject stale sessions and control-plane outages. Persist task owner/device/generation/session/worker and original message mapping. Sign routing metadata, verify late results, claim replies once and mark ambiguous/crashed tasks unknown instead of replaying writes. Fence notifications across same-user rebind. Release in-memory slots/waiters even when database or Redis cleanup fails. Validation: 213 passed, 3 skipped (real PostgreSQL not configured), 30 existing uvicorn/websockets deprecation warnings. New lifecycle coverage: 28 tests, including real loopback WebSocket isolation, reverse completion, missed kill, periodic unbind checks, outbox recovery, transaction rollback and cleanup failures. Syntax, docs and staged diff checks pass. Default legacy mode remains unchanged. Broker mode now requires coordinated router admission configuration and a v2 client handshake. No production ECS configuration, real credentials or client installer was changed. PostgreSQL, real Redis multi-node, ARM64 and actual Work/client rollout remain stage 5.
221 lines
9.8 KiB
Python
221 lines
9.8 KiB
Python
import asyncio
|
|
import contextlib
|
|
import json
|
|
import secrets
|
|
import time
|
|
import hashlib
|
|
import httpx
|
|
from contextlib import asynccontextmanager
|
|
|
|
from fastapi import FastAPI, WebSocket, WebSocketDisconnect
|
|
from redis.asyncio import Redis
|
|
from redis.exceptions import WatchError
|
|
|
|
from .config import Settings
|
|
from .crypto import canonical, fingerprint, load_public, raw_public, verified, verify_device
|
|
|
|
|
|
async def compare_delete(redis, key, expected):
|
|
async with redis.pipeline() as pipe:
|
|
try:
|
|
await pipe.watch(key)
|
|
if await pipe.get(key) != expected:
|
|
return False
|
|
pipe.multi()
|
|
pipe.delete(key)
|
|
await pipe.execute()
|
|
return True
|
|
except WatchError:
|
|
return False
|
|
|
|
|
|
async def compare_expire(redis, key, expected, ttl):
|
|
async with redis.pipeline() as pipe:
|
|
try:
|
|
await pipe.watch(key)
|
|
if await pipe.get(key) != expected:
|
|
return False
|
|
pipe.multi()
|
|
pipe.expire(key, ttl)
|
|
await pipe.execute()
|
|
return True
|
|
except WatchError:
|
|
return False
|
|
|
|
|
|
def create_app(settings=None, redis=None, admission_transport=None):
|
|
s = settings or Settings(role="router")
|
|
r = redis or Redis.from_url(s.redis_url, decode_responses=True)
|
|
ca = s.ca_cert.read_bytes()
|
|
active = set()
|
|
internal = (httpx.AsyncClient(base_url=s.connector_internal_url.rstrip('/'),timeout=3,
|
|
trust_env=False,follow_redirects=False,transport=admission_transport) if s.credential_mode == "broker" else None)
|
|
connector_public = load_public(s.connector_public_key) if internal else None
|
|
|
|
async def admission(host, owner, generation, *, turn=None, session=None, expected_fp=None):
|
|
nonce = secrets.token_hex(16)
|
|
response = await internal.post("/internal/devices/"+host+"/admission",
|
|
headers={"X-Router-Admission-Secret":s.router_admission_secret},
|
|
json={"nonce":nonce,"turn_id":turn,"session_id":session})
|
|
if response.status_code != 200:
|
|
raise ValueError("device admission rejected")
|
|
data = verified(response.json(),connector_public)
|
|
if (data["nonce"] != nonce or data["host_id"] != host or data["owner"] != owner
|
|
or data["binding_generation"] != generation or data["environment"] != s.environment
|
|
or data["app_id"] != s.feishu_app_id or data["protocol_version"] != 2
|
|
or data["turn_id"] != turn or data["session_id"] != session
|
|
or not time.time() <= data["expires_at"] <= time.time()+10
|
|
or data["fingerprint"] != expected_fp):
|
|
raise ValueError("stale device admission")
|
|
return data
|
|
|
|
@asynccontextmanager
|
|
async def lifespan(app):
|
|
await r.ping()
|
|
yield
|
|
if internal:
|
|
await internal.aclose()
|
|
if redis is None:
|
|
await r.aclose()
|
|
|
|
app = FastAPI(title="Hainuo B2 router", lifespan=lifespan, docs_url=None, redoc_url=None)
|
|
app.state.redis = r
|
|
|
|
@app.get("/health")
|
|
async def health():
|
|
return {"status": "alive", "role": "router"}
|
|
|
|
@app.get("/ready")
|
|
async def ready():
|
|
await r.ping()
|
|
return {"status": "ready", "connections": len(active)}
|
|
|
|
@app.websocket("/device/ws")
|
|
async def device(ws: WebSocket):
|
|
from .common import rate_limit
|
|
from fastapi import HTTPException
|
|
await ws.accept()
|
|
if len(active) >= s.max_connections:
|
|
await ws.close(1013)
|
|
return
|
|
ip = ws.client.host if ws.client else "unknown"
|
|
try:
|
|
await rate_limit(r, "ws:" + ip, 30)
|
|
except HTTPException:
|
|
await ws.close(1008)
|
|
return
|
|
challenge = secrets.token_urlsafe(32)
|
|
host = session = presence = None
|
|
tasks = []
|
|
pub = None
|
|
try:
|
|
await ws.send_json({"type": "challenge", "challenge": challenge})
|
|
hello = await asyncio.wait_for(ws.receive_json(), timeout=10)
|
|
if len(json.dumps(hello)) > 20000:
|
|
raise ValueError("oversized hello")
|
|
host = hello["host_id"]
|
|
if not isinstance(host, str) or len(host) != 32 or any(c not in "0123456789abcdef" for c in host):
|
|
raise ValueError("invalid host")
|
|
public = verify_device(hello["cert"], ca, host)
|
|
proof = verified(hello["proof"], public)
|
|
if proof["challenge"] != challenge or proof["host_id"] != host or abs(time.time() - proof["ts"]) > 90:
|
|
raise ValueError("invalid proof")
|
|
if len(proof["kx"]) > 64 or len(proof["session_id"]) > 64:
|
|
raise ValueError("invalid session")
|
|
fp = fingerprint(hello["cert"])
|
|
if await r.get("hn:device:" + host) != fp or await r.exists("hn:revoked:" + host):
|
|
raise ValueError("unregistered/revoked")
|
|
generation = owner = None
|
|
if internal:
|
|
generation, owner = proof["binding_generation"], proof["binding_owner"]
|
|
if (type(generation) is not int or generation < 0 or type(proof.get("protocol_version")) is not int
|
|
or proof.get("protocol_version") != 2
|
|
or proof.get("environment") != s.environment or proof.get("app_id") != s.feishu_app_id):
|
|
raise ValueError("invalid binding handshake")
|
|
await admission(host,owner,generation,expected_fp=fp)
|
|
session = proof["session_id"]
|
|
presence = json.dumps({"cert": hello["cert"], "proof": hello["proof"], "challenge": challenge})
|
|
if not await r.set("hn:online:" + host, presence, nx=True, ex=30):
|
|
raise ValueError("already connected")
|
|
active.add(session)
|
|
pub = r.pubsub()
|
|
await pub.subscribe("hn:down:" + host, "hn:kill:" + host)
|
|
if internal:
|
|
await admission(host,owner,generation,expected_fp=fp)
|
|
await ws.send_json({"type": "connected", "session_id": session})
|
|
|
|
async def downstream():
|
|
async for msg in pub.listen():
|
|
if msg["type"] != "message":
|
|
continue
|
|
if msg["channel"] == "hn:kill:" + host:
|
|
if internal:
|
|
event = json.loads(msg["data"])
|
|
if generation >= event["binding_generation"]:
|
|
continue
|
|
await ws.close(1008)
|
|
return
|
|
if internal:
|
|
packet = json.loads(msg["data"])
|
|
routing = verified(packet["routing"],connector_public)
|
|
if (routing["host_id"] != host or routing["session_id"] != session
|
|
or routing["environment"] != s.environment or routing["app_id"] != s.feishu_app_id
|
|
or routing["binding_generation"] != generation
|
|
or routing["envelope_hash"] != hashlib.sha256(canonical(packet["envelope"])).hexdigest()):
|
|
raise ValueError("invalid instruction routing")
|
|
await admission(host,owner,generation,turn=routing["turn_id"],session=session,expected_fp=fp)
|
|
await ws.send_text(msg["data"])
|
|
|
|
async def heartbeat():
|
|
while True:
|
|
await asyncio.sleep(8)
|
|
if internal:
|
|
await admission(host,owner,generation,expected_fp=fp)
|
|
if await r.exists("hn:revoked:" + host) or not await compare_expire(r, "hn:online:" + host, presence, 30):
|
|
await ws.close(1008)
|
|
return
|
|
|
|
async def upstream():
|
|
while True:
|
|
text = await ws.receive_text()
|
|
if len(text.encode()) > s.max_body:
|
|
raise ValueError("oversized frame")
|
|
data = json.loads(text)
|
|
if data.get("type") == "ping":
|
|
continue
|
|
await rate_limit(r, "ws:reply:" + host, 120)
|
|
turn = data.get("turn_id", "")
|
|
if data.get("type") != "reply" or await r.get(f"hn:inflight:{host}:{turn}") != session:
|
|
raise ValueError("unsolicited reply")
|
|
envelope = data["envelope"]
|
|
verified(envelope, public)
|
|
if envelope["host_id"] != host or envelope["session_id"] != session:
|
|
raise ValueError("wrong reply destination")
|
|
if internal:
|
|
await admission(host,owner,generation,turn=turn,session=session,expected_fp=fp)
|
|
await r.publish("hn:replies", json.dumps(data))
|
|
|
|
tasks = [asyncio.create_task(x()) for x in (downstream, heartbeat, upstream)]
|
|
done, _ = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED)
|
|
for task in done:
|
|
task.result()
|
|
except (WebSocketDisconnect, asyncio.TimeoutError, ValueError, KeyError, TypeError):
|
|
with contextlib.suppress(Exception):
|
|
await ws.close(1008)
|
|
except Exception:
|
|
with contextlib.suppress(Exception):
|
|
await ws.close(1011)
|
|
finally:
|
|
active.discard(session)
|
|
for task in tasks:
|
|
task.cancel()
|
|
await asyncio.gather(*tasks, return_exceptions=True)
|
|
if pub:
|
|
with contextlib.suppress(Exception):
|
|
await pub.aclose()
|
|
if presence:
|
|
with contextlib.suppress(Exception):
|
|
await compare_delete(r, "hn:online:" + host, presence)
|
|
|
|
return app
|